gren 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.ja.txt +8 -0
- data/History.txt +10 -0
- data/Manifest.txt +18 -4
- data/Rakefile +6 -1
- data/bin/grenweb +11 -0
- data/lib/common/util.rb +3 -3
- data/lib/findgrep/findgrep.rb +46 -25
- data/lib/gren.rb +1 -1
- data/lib/grenweb/cli.rb +52 -0
- data/lib/grenweb/database.rb +179 -0
- data/lib/grenweb/grenweb.ru +35 -0
- data/lib/grenweb/grep.rb +52 -0
- data/lib/grenweb/help.rb +37 -0
- data/lib/grenweb/home.rb +37 -0
- data/lib/grenweb/html_renderer.rb +231 -0
- data/lib/grenweb/public/css/gren.css +63 -0
- data/lib/grenweb/public/images/gren-icon-mini.png +0 -0
- data/lib/grenweb/public/images/gren-icon.png +0 -0
- data/lib/grenweb/query.rb +82 -0
- data/lib/grenweb/searcher.rb +127 -0
- data/lib/grenweb/viewer.rb +53 -0
- data/lib/mkgrendb/mkgrendb.rb +15 -8
- data/test/test_gren.rb +1 -2
- data/test/test_gren_util.rb +33 -32
- data/test/test_grenweb_cli.rb +10 -0
- data/test/test_grenweb_html_renderer.rb +40 -0
- data/test/test_grenweb_query.rb +54 -0
- data/test/test_grenweb_searcher.rb +35 -0
- data/test/test_string_snip.rb +3 -1
- metadata +59 -15
- data/lib/gren/findgrep.rb +0 -215
- data/test/test_gren_cli.rb +0 -14
- data/test/test_grendb_cli.rb +0 -14
- data/test/test_mkgrendb_cli.rb +0 -14
Binary file
|
Binary file
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
#
|
3
|
+
# @file
|
4
|
+
# @brief クエリーの解析
|
5
|
+
# @author ongaeshi
|
6
|
+
# @date 2010/10/21
|
7
|
+
|
8
|
+
require 'rubygems'
|
9
|
+
require 'rack'
|
10
|
+
|
11
|
+
module Grenweb
|
12
|
+
class Query
|
13
|
+
include Rack::Utils
|
14
|
+
attr_reader :query_string
|
15
|
+
|
16
|
+
OPTIONS = [
|
17
|
+
['package', 'p'],
|
18
|
+
['filepath', 'fpath', 'f'],
|
19
|
+
['suffix', 's'],
|
20
|
+
]
|
21
|
+
|
22
|
+
def initialize(request)
|
23
|
+
@query_string = unescape(request.path_info.gsub(/\A\/|\/\z/, ''))
|
24
|
+
init_hash
|
25
|
+
parse
|
26
|
+
end
|
27
|
+
|
28
|
+
def escape_html
|
29
|
+
Rack::Utils::escape_html(@query_string)
|
30
|
+
end
|
31
|
+
|
32
|
+
def empty?
|
33
|
+
keywords.size == 0 && packages.size == 0 && fpaths.size == 0 && suffixs.size == 0
|
34
|
+
end
|
35
|
+
|
36
|
+
def keywords
|
37
|
+
@hash['keywords']
|
38
|
+
end
|
39
|
+
|
40
|
+
def packages
|
41
|
+
calc_param(0)
|
42
|
+
end
|
43
|
+
|
44
|
+
def fpaths
|
45
|
+
calc_param(1)
|
46
|
+
end
|
47
|
+
|
48
|
+
def suffixs
|
49
|
+
calc_param(2)
|
50
|
+
end
|
51
|
+
|
52
|
+
private
|
53
|
+
|
54
|
+
def calc_param(index)
|
55
|
+
OPTIONS[index].inject([]){|result, item| result.concat @hash[item] }
|
56
|
+
end
|
57
|
+
|
58
|
+
def init_hash
|
59
|
+
@hash = {}
|
60
|
+
@hash['keywords'] = []
|
61
|
+
|
62
|
+
OPTIONS.flatten.each do |key|
|
63
|
+
@hash[key] = []
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def parse
|
68
|
+
kp = OPTIONS.flatten.join('|')
|
69
|
+
parts = @query_string.scan(/(?:(#{kp}):)?(?:"(.+)"|(\S+))/)
|
70
|
+
|
71
|
+
parts.each do |key, quoted_value, value|
|
72
|
+
text = quoted_value || value
|
73
|
+
unless (key)
|
74
|
+
@hash['keywords'] << text
|
75
|
+
else
|
76
|
+
@hash[key] << text
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
@@ -0,0 +1,127 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
#
|
3
|
+
# @file
|
4
|
+
# @brief 検索処理本体
|
5
|
+
# @author ongaeshi
|
6
|
+
# @date 2010/10/13
|
7
|
+
|
8
|
+
require 'rack'
|
9
|
+
require File.join(File.dirname(__FILE__), 'database')
|
10
|
+
require File.join(File.dirname(__FILE__), 'html_renderer')
|
11
|
+
require File.join(File.dirname(__FILE__), 'query')
|
12
|
+
|
13
|
+
module Grenweb
|
14
|
+
class Searcher
|
15
|
+
include Rack::Utils
|
16
|
+
|
17
|
+
def initialize
|
18
|
+
end
|
19
|
+
|
20
|
+
def call(env)
|
21
|
+
@request = Rack::Request.new(env)
|
22
|
+
@query = Query.new(@request)
|
23
|
+
|
24
|
+
@response = Rack::Response.new
|
25
|
+
@response["Content-Type"] = "text/html; charset=UTF-8"
|
26
|
+
|
27
|
+
@nth = 3 # マッチした行の前後何行を表示するか
|
28
|
+
|
29
|
+
if @request.post? or @request['query']
|
30
|
+
post_request
|
31
|
+
else
|
32
|
+
search
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def post_request
|
39
|
+
query = @request['query'] || ''
|
40
|
+
if query.empty?
|
41
|
+
@request.path_info = "/"
|
42
|
+
else
|
43
|
+
@request.path_info = "/#{escape(query)}/"
|
44
|
+
end
|
45
|
+
@response.redirect(@request.url.split(/\?/, 2)[0])
|
46
|
+
@response.to_a
|
47
|
+
end
|
48
|
+
|
49
|
+
def search
|
50
|
+
render_header
|
51
|
+
render_search_box
|
52
|
+
render_search_result
|
53
|
+
render_footer
|
54
|
+
@response.to_a
|
55
|
+
end
|
56
|
+
|
57
|
+
def render_header
|
58
|
+
@response.write HTMLRendeler.header("gren : #{@query.escape_html}", "gren")
|
59
|
+
end
|
60
|
+
|
61
|
+
def render_search_box
|
62
|
+
@response.write HTMLRendeler.search_box(@query.escape_html)
|
63
|
+
end
|
64
|
+
|
65
|
+
def render_search_result
|
66
|
+
if @query.empty?
|
67
|
+
@response.write HTMLRendeler.empty_summary
|
68
|
+
else
|
69
|
+
records, total_records, elapsed = Database.instance.search(@query.keywords, @query.packages, @query.fpaths, @query.suffixs, calcPage, calcLimit)
|
70
|
+
render_search_summary(records, total_records, elapsed)
|
71
|
+
records.each { |record| @response.write(HTMLRendeler.result_record(record, @query.keywords, @nth)) }
|
72
|
+
render_pagination(calcPage, total_records)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def render_search_summary(records, total_records, elapsed)
|
77
|
+
pageStart = calcPage * calcLimit
|
78
|
+
@response.write HTMLRendeler.search_summary(@query.query_string,
|
79
|
+
total_records,
|
80
|
+
(total_records.zero? ? 0 : pageStart + 1)..(pageStart + records.size),
|
81
|
+
elapsed)
|
82
|
+
end
|
83
|
+
|
84
|
+
def render_pagination(page, total_records)
|
85
|
+
return if @query.empty?
|
86
|
+
return if total_records < calcLimit
|
87
|
+
|
88
|
+
last_page = (total_records / calcLimit.to_f).ceil
|
89
|
+
@response.write("<div class='pagination'>\n")
|
90
|
+
if page > 0
|
91
|
+
@response.write(HTMLRendeler.pagination_link(page - 1, "<<"))
|
92
|
+
end
|
93
|
+
last_page.times do |i|
|
94
|
+
if i == page
|
95
|
+
@response.write(HTMLRendeler.pagination_span(i))
|
96
|
+
else
|
97
|
+
@response.write(HTMLRendeler.pagination_link(i, i))
|
98
|
+
end
|
99
|
+
end
|
100
|
+
if page < (last_page - 1)
|
101
|
+
@response.write(HTMLRendeler.pagination_link(page + 1, ">>"))
|
102
|
+
end
|
103
|
+
@response.write("</div>\n")
|
104
|
+
end
|
105
|
+
|
106
|
+
def render_footer
|
107
|
+
@response.write HTMLRendeler.footer
|
108
|
+
end
|
109
|
+
|
110
|
+
private
|
111
|
+
|
112
|
+
# 1ページに表示する最大レコードを計算
|
113
|
+
def calcLimit
|
114
|
+
if @query.keywords.size == 0
|
115
|
+
100
|
116
|
+
else
|
117
|
+
20
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
# 現在ページを計算
|
122
|
+
def calcPage
|
123
|
+
(@request['page'] || 0).to_i
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
@@ -0,0 +1,53 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
#
|
3
|
+
# @file
|
4
|
+
# @brief ソースコードを表示する
|
5
|
+
# @author ongaeshi
|
6
|
+
# @date 2010/10/13
|
7
|
+
|
8
|
+
require 'rack'
|
9
|
+
require File.join(File.dirname(__FILE__), 'database')
|
10
|
+
|
11
|
+
module Grenweb
|
12
|
+
class Viewer
|
13
|
+
include Rack::Utils
|
14
|
+
|
15
|
+
def initialize
|
16
|
+
end
|
17
|
+
|
18
|
+
def call(env)
|
19
|
+
@request = Rack::Request.new(env)
|
20
|
+
@response = Rack::Response.new
|
21
|
+
@response["Content-Type"] = "text/html; charset=UTF-8"
|
22
|
+
|
23
|
+
record, elapsed = Database.instance.record(req2query)
|
24
|
+
|
25
|
+
if (record)
|
26
|
+
@response.write HTMLRendeler.header("gren : #{record.shortpath}", "gren")
|
27
|
+
@response.write HTMLRendeler.search_box("")
|
28
|
+
@response.write HTMLRendeler.view_summary(record.shortpath, elapsed)
|
29
|
+
@response.write HTMLRendeler.record_content(record)
|
30
|
+
else
|
31
|
+
@response.write HTMLRendeler.header("gren : not found.", "gren")
|
32
|
+
@response.write HTMLRendeler.search_box("")
|
33
|
+
@response.write HTMLRendeler.empty_summary
|
34
|
+
end
|
35
|
+
@response.write HTMLRendeler.footer
|
36
|
+
|
37
|
+
@response.to_a
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def req2query
|
43
|
+
unescape(@request.path_info.gsub(/\A\/|\/z/, ''))
|
44
|
+
end
|
45
|
+
|
46
|
+
def req2path(component='')
|
47
|
+
path = []
|
48
|
+
path << ((@request.script_name == "") ? '/' : @request.script_name)
|
49
|
+
path << component if (component)
|
50
|
+
path.join('/')
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
data/lib/mkgrendb/mkgrendb.rb
CHANGED
@@ -17,6 +17,7 @@ module Mkgrendb
|
|
17
17
|
puts "input_yaml : #{@input_yaml} found."
|
18
18
|
@src = YAML.load(open(@input_yaml).read())
|
19
19
|
@file_count = 0
|
20
|
+
@add_count = 0
|
20
21
|
@update_count = 0
|
21
22
|
@start_time = Time.now
|
22
23
|
end
|
@@ -29,7 +30,7 @@ module Mkgrendb
|
|
29
30
|
if (FileTest.directory? dir)
|
30
31
|
db_add_dir(dir)
|
31
32
|
else
|
32
|
-
db_add_file(STDOUT, dir)
|
33
|
+
db_add_file(STDOUT, dir, File.basename(dir))
|
33
34
|
end
|
34
35
|
end
|
35
36
|
@end_time = Time.now
|
@@ -54,7 +55,8 @@ module Mkgrendb
|
|
54
55
|
puts "input_yaml : #{@input_yaml} (#{Util::time_s(time)})"
|
55
56
|
puts "output_db : #{@output_db}*"
|
56
57
|
puts "files : #{@file_count}"
|
57
|
-
puts "
|
58
|
+
puts "add : #{@add_count}"
|
59
|
+
puts "update : #{@update_count}"
|
58
60
|
end
|
59
61
|
|
60
62
|
def dump()
|
@@ -65,6 +67,7 @@ module Mkgrendb
|
|
65
67
|
records.each do |record|
|
66
68
|
p record
|
67
69
|
puts "path : #{record.path}"
|
70
|
+
puts "shortpath : #{record.shortpath}"
|
68
71
|
puts "suffix : #{record.suffix}"
|
69
72
|
puts "timestamp : #{record.timestamp.strftime('%Y/%m/%d %H:%M:%S')}"
|
70
73
|
puts "content :", record.content ? record.content[0..64] : nil
|
@@ -82,6 +85,7 @@ module Mkgrendb
|
|
82
85
|
Groonga::Schema.define do |schema|
|
83
86
|
schema.create_table("documents") do |table|
|
84
87
|
table.string("path")
|
88
|
+
table.string("shortpath")
|
85
89
|
table.text("content")
|
86
90
|
table.time("timestamp")
|
87
91
|
table.text("suffix")
|
@@ -92,6 +96,7 @@ module Mkgrendb
|
|
92
96
|
:key_normalize => true,
|
93
97
|
:default_tokenizer => "TokenBigram") do |table|
|
94
98
|
table.index("documents.path", :with_position => true)
|
99
|
+
table.index("documents.shortpath", :with_position => true)
|
95
100
|
table.index("documents.content", :with_position => true)
|
96
101
|
table.index("documents.suffix", :with_position => true)
|
97
102
|
end
|
@@ -125,14 +130,15 @@ module Mkgrendb
|
|
125
130
|
private :db_open
|
126
131
|
|
127
132
|
def db_add_dir(dirname)
|
128
|
-
searchDirectory(STDOUT, dirname, 0)
|
133
|
+
searchDirectory(STDOUT, dirname, File.basename(dirname), 0)
|
129
134
|
end
|
130
135
|
private :db_add_dir
|
131
136
|
|
132
|
-
def db_add_file(stdout, filename)
|
137
|
+
def db_add_file(stdout, filename, shortpath)
|
133
138
|
# 格納するデータ
|
134
139
|
values = {
|
135
140
|
:path => filename,
|
141
|
+
:shortpath => shortpath,
|
136
142
|
:content => nil,
|
137
143
|
:timestamp => File.mtime(filename),
|
138
144
|
:suffix => File::extname(filename),
|
@@ -164,6 +170,7 @@ module Mkgrendb
|
|
164
170
|
values.each do |key, value|
|
165
171
|
if (key == :path)
|
166
172
|
if (isNewFile)
|
173
|
+
@add_count += 1
|
167
174
|
puts "add_file : #{value}"
|
168
175
|
else
|
169
176
|
@update_count += 1
|
@@ -176,12 +183,12 @@ module Mkgrendb
|
|
176
183
|
|
177
184
|
end
|
178
185
|
|
179
|
-
def searchDirectory(stdout, dir, depth)
|
186
|
+
def searchDirectory(stdout, dir, shortdir, depth)
|
180
187
|
Dir.foreach(dir) do |name|
|
181
188
|
next if (name == '.' || name == '..')
|
182
189
|
|
183
190
|
fpath = File.join(dir,name)
|
184
|
-
|
191
|
+
shortpath = File.join(shortdir,name)
|
185
192
|
|
186
193
|
# 除外ディレクトリならばパス
|
187
194
|
next if ignoreDir?(fpath)
|
@@ -192,10 +199,10 @@ module Mkgrendb
|
|
192
199
|
# ファイルならば中身を探索、ディレクトリならば再帰
|
193
200
|
case File.ftype(fpath)
|
194
201
|
when "directory"
|
195
|
-
searchDirectory(stdout, fpath, depth + 1)
|
202
|
+
searchDirectory(stdout, fpath, shortpath, depth + 1)
|
196
203
|
when "file"
|
197
204
|
unless ignoreFile?(fpath)
|
198
|
-
db_add_file(stdout, fpath)
|
205
|
+
db_add_file(stdout, fpath, shortpath)
|
199
206
|
@file_count += 1
|
200
207
|
puts "file_count : #{@file_count}" if (@file_count % 100 == 0)
|
201
208
|
end
|
data/test/test_gren.rb
CHANGED
data/test/test_gren_util.rb
CHANGED
@@ -1,32 +1,33 @@
|
|
1
|
-
require File.dirname(__FILE__)
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
match_datas
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
match_datas
|
22
|
-
match_datas << str.match(/
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
match_datas
|
28
|
-
match_datas << str.match(/
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
end
|
1
|
+
require File.join(File.dirname(__FILE__), "test_helper.rb")
|
2
|
+
require File.join(File.dirname(__FILE__), "../lib/common/util.rb")
|
3
|
+
|
4
|
+
class TestGrenSnip < Test::Unit::TestCase
|
5
|
+
def setup
|
6
|
+
end
|
7
|
+
|
8
|
+
def test_snip
|
9
|
+
str = "abcdefghijkmlnopqrstuvwxyz"
|
10
|
+
assert_equal(str, GrenSnip::snip(str, nil))
|
11
|
+
|
12
|
+
str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|12345678901234567890123456789012345678901234567890123456"
|
13
|
+
assert_equal(str, GrenSnip::snip(str, nil))
|
14
|
+
|
15
|
+
str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234561"
|
16
|
+
match_datas = []
|
17
|
+
match_datas << str.match(/123456789\|/)
|
18
|
+
assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>67890123456789012345678901234561", GrenSnip::snip(str, match_datas))
|
19
|
+
|
20
|
+
str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234561"
|
21
|
+
match_datas = []
|
22
|
+
match_datas << str.match(/123456789\|/)
|
23
|
+
match_datas << str.match(/34567/)
|
24
|
+
assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>67890123456789012345678901234561", GrenSnip::snip(str, match_datas))
|
25
|
+
|
26
|
+
str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234561"
|
27
|
+
match_datas = []
|
28
|
+
match_datas << str.match(/123456789\|/)
|
29
|
+
match_datas << str.match(/01234561/)
|
30
|
+
assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>8901234567890123456789012345678901234561", GrenSnip::snip(str, match_datas))
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|