gren 0.2.4 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,82 @@
1
+ # -*- coding: utf-8 -*-
2
+ #
3
+ # @file
4
+ # @brief クエリーの解析
5
+ # @author ongaeshi
6
+ # @date 2010/10/21
7
+
8
+ require 'rubygems'
9
+ require 'rack'
10
+
11
+ module Grenweb
12
+ class Query
13
+ include Rack::Utils
14
+ attr_reader :query_string
15
+
16
+ OPTIONS = [
17
+ ['package', 'p'],
18
+ ['filepath', 'fpath', 'f'],
19
+ ['suffix', 's'],
20
+ ]
21
+
22
+ def initialize(request)
23
+ @query_string = unescape(request.path_info.gsub(/\A\/|\/\z/, ''))
24
+ init_hash
25
+ parse
26
+ end
27
+
28
+ def escape_html
29
+ Rack::Utils::escape_html(@query_string)
30
+ end
31
+
32
+ def empty?
33
+ keywords.size == 0 && packages.size == 0 && fpaths.size == 0 && suffixs.size == 0
34
+ end
35
+
36
+ def keywords
37
+ @hash['keywords']
38
+ end
39
+
40
+ def packages
41
+ calc_param(0)
42
+ end
43
+
44
+ def fpaths
45
+ calc_param(1)
46
+ end
47
+
48
+ def suffixs
49
+ calc_param(2)
50
+ end
51
+
52
+ private
53
+
54
+ def calc_param(index)
55
+ OPTIONS[index].inject([]){|result, item| result.concat @hash[item] }
56
+ end
57
+
58
+ def init_hash
59
+ @hash = {}
60
+ @hash['keywords'] = []
61
+
62
+ OPTIONS.flatten.each do |key|
63
+ @hash[key] = []
64
+ end
65
+ end
66
+
67
+ def parse
68
+ kp = OPTIONS.flatten.join('|')
69
+ parts = @query_string.scan(/(?:(#{kp}):)?(?:"(.+)"|(\S+))/)
70
+
71
+ parts.each do |key, quoted_value, value|
72
+ text = quoted_value || value
73
+ unless (key)
74
+ @hash['keywords'] << text
75
+ else
76
+ @hash[key] << text
77
+ end
78
+ end
79
+ end
80
+ end
81
+ end
82
+
@@ -0,0 +1,127 @@
1
+ # -*- coding: utf-8 -*-
2
+ #
3
+ # @file
4
+ # @brief 検索処理本体
5
+ # @author ongaeshi
6
+ # @date 2010/10/13
7
+
8
+ require 'rack'
9
+ require File.join(File.dirname(__FILE__), 'database')
10
+ require File.join(File.dirname(__FILE__), 'html_renderer')
11
+ require File.join(File.dirname(__FILE__), 'query')
12
+
13
+ module Grenweb
14
+ class Searcher
15
+ include Rack::Utils
16
+
17
+ def initialize
18
+ end
19
+
20
+ def call(env)
21
+ @request = Rack::Request.new(env)
22
+ @query = Query.new(@request)
23
+
24
+ @response = Rack::Response.new
25
+ @response["Content-Type"] = "text/html; charset=UTF-8"
26
+
27
+ @nth = 3 # マッチした行の前後何行を表示するか
28
+
29
+ if @request.post? or @request['query']
30
+ post_request
31
+ else
32
+ search
33
+ end
34
+ end
35
+
36
+ private
37
+
38
+ def post_request
39
+ query = @request['query'] || ''
40
+ if query.empty?
41
+ @request.path_info = "/"
42
+ else
43
+ @request.path_info = "/#{escape(query)}/"
44
+ end
45
+ @response.redirect(@request.url.split(/\?/, 2)[0])
46
+ @response.to_a
47
+ end
48
+
49
+ def search
50
+ render_header
51
+ render_search_box
52
+ render_search_result
53
+ render_footer
54
+ @response.to_a
55
+ end
56
+
57
+ def render_header
58
+ @response.write HTMLRendeler.header("gren : #{@query.escape_html}", "gren")
59
+ end
60
+
61
+ def render_search_box
62
+ @response.write HTMLRendeler.search_box(@query.escape_html)
63
+ end
64
+
65
+ def render_search_result
66
+ if @query.empty?
67
+ @response.write HTMLRendeler.empty_summary
68
+ else
69
+ records, total_records, elapsed = Database.instance.search(@query.keywords, @query.packages, @query.fpaths, @query.suffixs, calcPage, calcLimit)
70
+ render_search_summary(records, total_records, elapsed)
71
+ records.each { |record| @response.write(HTMLRendeler.result_record(record, @query.keywords, @nth)) }
72
+ render_pagination(calcPage, total_records)
73
+ end
74
+ end
75
+
76
+ def render_search_summary(records, total_records, elapsed)
77
+ pageStart = calcPage * calcLimit
78
+ @response.write HTMLRendeler.search_summary(@query.query_string,
79
+ total_records,
80
+ (total_records.zero? ? 0 : pageStart + 1)..(pageStart + records.size),
81
+ elapsed)
82
+ end
83
+
84
+ def render_pagination(page, total_records)
85
+ return if @query.empty?
86
+ return if total_records < calcLimit
87
+
88
+ last_page = (total_records / calcLimit.to_f).ceil
89
+ @response.write("<div class='pagination'>\n")
90
+ if page > 0
91
+ @response.write(HTMLRendeler.pagination_link(page - 1, "<<"))
92
+ end
93
+ last_page.times do |i|
94
+ if i == page
95
+ @response.write(HTMLRendeler.pagination_span(i))
96
+ else
97
+ @response.write(HTMLRendeler.pagination_link(i, i))
98
+ end
99
+ end
100
+ if page < (last_page - 1)
101
+ @response.write(HTMLRendeler.pagination_link(page + 1, ">>"))
102
+ end
103
+ @response.write("</div>\n")
104
+ end
105
+
106
+ def render_footer
107
+ @response.write HTMLRendeler.footer
108
+ end
109
+
110
+ private
111
+
112
+ # 1ページに表示する最大レコードを計算
113
+ def calcLimit
114
+ if @query.keywords.size == 0
115
+ 100
116
+ else
117
+ 20
118
+ end
119
+ end
120
+
121
+ # 現在ページを計算
122
+ def calcPage
123
+ (@request['page'] || 0).to_i
124
+ end
125
+ end
126
+ end
127
+
@@ -0,0 +1,53 @@
1
+ # -*- coding: utf-8 -*-
2
+ #
3
+ # @file
4
+ # @brief ソースコードを表示する
5
+ # @author ongaeshi
6
+ # @date 2010/10/13
7
+
8
+ require 'rack'
9
+ require File.join(File.dirname(__FILE__), 'database')
10
+
11
+ module Grenweb
12
+ class Viewer
13
+ include Rack::Utils
14
+
15
+ def initialize
16
+ end
17
+
18
+ def call(env)
19
+ @request = Rack::Request.new(env)
20
+ @response = Rack::Response.new
21
+ @response["Content-Type"] = "text/html; charset=UTF-8"
22
+
23
+ record, elapsed = Database.instance.record(req2query)
24
+
25
+ if (record)
26
+ @response.write HTMLRendeler.header("gren : #{record.shortpath}", "gren")
27
+ @response.write HTMLRendeler.search_box("")
28
+ @response.write HTMLRendeler.view_summary(record.shortpath, elapsed)
29
+ @response.write HTMLRendeler.record_content(record)
30
+ else
31
+ @response.write HTMLRendeler.header("gren : not found.", "gren")
32
+ @response.write HTMLRendeler.search_box("")
33
+ @response.write HTMLRendeler.empty_summary
34
+ end
35
+ @response.write HTMLRendeler.footer
36
+
37
+ @response.to_a
38
+ end
39
+
40
+ private
41
+
42
+ def req2query
43
+ unescape(@request.path_info.gsub(/\A\/|\/z/, ''))
44
+ end
45
+
46
+ def req2path(component='')
47
+ path = []
48
+ path << ((@request.script_name == "") ? '/' : @request.script_name)
49
+ path << component if (component)
50
+ path.join('/')
51
+ end
52
+ end
53
+ end
@@ -17,6 +17,7 @@ module Mkgrendb
17
17
  puts "input_yaml : #{@input_yaml} found."
18
18
  @src = YAML.load(open(@input_yaml).read())
19
19
  @file_count = 0
20
+ @add_count = 0
20
21
  @update_count = 0
21
22
  @start_time = Time.now
22
23
  end
@@ -29,7 +30,7 @@ module Mkgrendb
29
30
  if (FileTest.directory? dir)
30
31
  db_add_dir(dir)
31
32
  else
32
- db_add_file(STDOUT, dir)
33
+ db_add_file(STDOUT, dir, File.basename(dir))
33
34
  end
34
35
  end
35
36
  @end_time = Time.now
@@ -54,7 +55,8 @@ module Mkgrendb
54
55
  puts "input_yaml : #{@input_yaml} (#{Util::time_s(time)})"
55
56
  puts "output_db : #{@output_db}*"
56
57
  puts "files : #{@file_count}"
57
- puts "updates : #{@update_count}"
58
+ puts "add : #{@add_count}"
59
+ puts "update : #{@update_count}"
58
60
  end
59
61
 
60
62
  def dump()
@@ -65,6 +67,7 @@ module Mkgrendb
65
67
  records.each do |record|
66
68
  p record
67
69
  puts "path : #{record.path}"
70
+ puts "shortpath : #{record.shortpath}"
68
71
  puts "suffix : #{record.suffix}"
69
72
  puts "timestamp : #{record.timestamp.strftime('%Y/%m/%d %H:%M:%S')}"
70
73
  puts "content :", record.content ? record.content[0..64] : nil
@@ -82,6 +85,7 @@ module Mkgrendb
82
85
  Groonga::Schema.define do |schema|
83
86
  schema.create_table("documents") do |table|
84
87
  table.string("path")
88
+ table.string("shortpath")
85
89
  table.text("content")
86
90
  table.time("timestamp")
87
91
  table.text("suffix")
@@ -92,6 +96,7 @@ module Mkgrendb
92
96
  :key_normalize => true,
93
97
  :default_tokenizer => "TokenBigram") do |table|
94
98
  table.index("documents.path", :with_position => true)
99
+ table.index("documents.shortpath", :with_position => true)
95
100
  table.index("documents.content", :with_position => true)
96
101
  table.index("documents.suffix", :with_position => true)
97
102
  end
@@ -125,14 +130,15 @@ module Mkgrendb
125
130
  private :db_open
126
131
 
127
132
  def db_add_dir(dirname)
128
- searchDirectory(STDOUT, dirname, 0)
133
+ searchDirectory(STDOUT, dirname, File.basename(dirname), 0)
129
134
  end
130
135
  private :db_add_dir
131
136
 
132
- def db_add_file(stdout, filename)
137
+ def db_add_file(stdout, filename, shortpath)
133
138
  # 格納するデータ
134
139
  values = {
135
140
  :path => filename,
141
+ :shortpath => shortpath,
136
142
  :content => nil,
137
143
  :timestamp => File.mtime(filename),
138
144
  :suffix => File::extname(filename),
@@ -164,6 +170,7 @@ module Mkgrendb
164
170
  values.each do |key, value|
165
171
  if (key == :path)
166
172
  if (isNewFile)
173
+ @add_count += 1
167
174
  puts "add_file : #{value}"
168
175
  else
169
176
  @update_count += 1
@@ -176,12 +183,12 @@ module Mkgrendb
176
183
 
177
184
  end
178
185
 
179
- def searchDirectory(stdout, dir, depth)
186
+ def searchDirectory(stdout, dir, shortdir, depth)
180
187
  Dir.foreach(dir) do |name|
181
188
  next if (name == '.' || name == '..')
182
189
 
183
190
  fpath = File.join(dir,name)
184
- fpath_disp = fpath.gsub(/^.\//, "")
191
+ shortpath = File.join(shortdir,name)
185
192
 
186
193
  # 除外ディレクトリならばパス
187
194
  next if ignoreDir?(fpath)
@@ -192,10 +199,10 @@ module Mkgrendb
192
199
  # ファイルならば中身を探索、ディレクトリならば再帰
193
200
  case File.ftype(fpath)
194
201
  when "directory"
195
- searchDirectory(stdout, fpath, depth + 1)
202
+ searchDirectory(stdout, fpath, shortpath, depth + 1)
196
203
  when "file"
197
204
  unless ignoreFile?(fpath)
198
- db_add_file(stdout, fpath)
205
+ db_add_file(stdout, fpath, shortpath)
199
206
  @file_count += 1
200
207
  puts "file_count : #{@file_count}" if (@file_count % 100 == 0)
201
208
  end
data/test/test_gren.rb CHANGED
@@ -1,7 +1,6 @@
1
- require File.dirname(__FILE__) + '/test_helper.rb'
1
+ require File.join(File.dirname(__FILE__), "test_helper.rb")
2
2
 
3
3
  class TestGren < Test::Unit::TestCase
4
-
5
4
  def setup
6
5
  end
7
6
 
@@ -1,32 +1,33 @@
1
- require File.dirname(__FILE__) + '/test_helper.rb'
2
-
3
- class TestGrenSnip < Test::Unit::TestCase
4
- def setup
5
- end
6
-
7
- def test_snip
8
- str = "abcdefghijkmlnopqrstuvwxyz"
9
- assert_equal(str, GrenSnip::snip(str, nil))
10
-
11
- str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|12345678901234567890123456789012345678901234567890123456"
12
- assert_equal(str, GrenSnip::snip(str, nil))
13
-
14
- str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234561"
15
- match_datas = []
16
- match_datas << str.match(/123456789\|/)
17
- assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>67890123456789012345678901234561", GrenSnip::snip(str, match_datas))
18
-
19
- str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234561"
20
- match_datas = []
21
- match_datas << str.match(/123456789\|/)
22
- match_datas << str.match(/34567/)
23
- assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>67890123456789012345678901234561", GrenSnip::snip(str, match_datas))
24
-
25
- str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234561"
26
- match_datas = []
27
- match_datas << str.match(/123456789\|/)
28
- match_datas << str.match(/01234561/)
29
- assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>8901234567890123456789012345678901234561", GrenSnip::snip(str, match_datas))
30
-
31
- end
32
- end
1
+ require File.join(File.dirname(__FILE__), "test_helper.rb")
2
+ require File.join(File.dirname(__FILE__), "../lib/common/util.rb")
3
+
4
+ class TestGrenSnip < Test::Unit::TestCase
5
+ def setup
6
+ end
7
+
8
+ def test_snip
9
+ str = "abcdefghijkmlnopqrstuvwxyz"
10
+ assert_equal(str, GrenSnip::snip(str, nil))
11
+
12
+ str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|12345678901234567890123456789012345678901234567890123456"
13
+ assert_equal(str, GrenSnip::snip(str, nil))
14
+
15
+ str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234561"
16
+ match_datas = []
17
+ match_datas << str.match(/123456789\|/)
18
+ assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>67890123456789012345678901234561", GrenSnip::snip(str, match_datas))
19
+
20
+ str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234561"
21
+ match_datas = []
22
+ match_datas << str.match(/123456789\|/)
23
+ match_datas << str.match(/34567/)
24
+ assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>67890123456789012345678901234561", GrenSnip::snip(str, match_datas))
25
+
26
+ str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234561"
27
+ match_datas = []
28
+ match_datas << str.match(/123456789\|/)
29
+ match_datas << str.match(/01234561/)
30
+ assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>8901234567890123456789012345678901234561", GrenSnip::snip(str, match_datas))
31
+
32
+ end
33
+ end