gren 0.2.3 → 0.2.4

Sign up to get free protection for your applications and to get access to all the features.
data/History.ja.txt CHANGED
@@ -1,3 +1,21 @@
1
+ === 0.2.4 2010/10/09
2
+
3
+ * mkgrendb
4
+ * データベースの形式を変更
5
+ * 古いmkgrendbで作成したデータベースは、 'mkgrendb --full' で再構築をして下さい。
6
+ * yamlに、ディレクトリではなく単一ファイルの指定もOKに
7
+
8
+ * grendb
9
+ * マッチ対象を行では無くファイルとする --match-file モードを追加
10
+ * 今までのモードは --match-line
11
+ * デフォルトのモードを --match-file に変更
12
+ * 今までの行マッチモードを使う場合は、 -l or --ml or --match-line と指定する
13
+ * 拡張子をキーとする検索モードを追加
14
+ * -s, --suffix
15
+ * --groonga-only オプションを作成
16
+ * 実際のファイルを辿らず、データベースの情報のみで探索出来るように
17
+ * 検索結果を更新時刻順で表示するように
18
+
1
19
  === 0.2.3 2010/09/25
2
20
 
3
21
  * mkgrendb
data/Manifest.txt CHANGED
@@ -12,6 +12,7 @@ lib/common/grenfiletest.rb
12
12
  lib/common/grensnip.rb
13
13
  lib/common/platform.rb
14
14
  lib/common/string_snip.rb
15
+ lib/common/util.rb
15
16
  lib/findgrep/findgrep.rb
16
17
  lib/findgrep/result.rb
17
18
  lib/gren.rb
data/bin/grendb CHANGED
@@ -7,5 +7,5 @@ require 'rubygems'
7
7
  require File.expand_path(File.dirname(__FILE__) + "/../lib/gren")
8
8
  require "grendb/cli"
9
9
 
10
- Version = "0.2.0"
10
+ Version = "0.2.4"
11
11
  Grendb::CLI.execute(STDOUT, ARGV)
data/bin/mkgrendb CHANGED
@@ -7,5 +7,5 @@ require 'rubygems'
7
7
  require File.expand_path(File.dirname(__FILE__) + "/../lib/gren")
8
8
  require "mkgrendb/cli"
9
9
 
10
- Version = "0.2.3"
10
+ Version = "0.2.4"
11
11
  Mkgrendb::CLI.execute(STDOUT, ARGV)
@@ -44,5 +44,19 @@ module Gren
44
44
  end
45
45
  end
46
46
  module_function :size_s
47
+
48
+ def dump_methods(c)
49
+ unless c.is_a?(Class)
50
+ c = c.class
51
+ end
52
+
53
+ while (true)
54
+ p c
55
+ break if (c == Object)
56
+ puts "↓ " + c.public_instance_methods(false).inspect
57
+ c = c.superclass
58
+ end
59
+ end
60
+ module_function :dump_methods
47
61
  end
48
62
  end
@@ -0,0 +1,98 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ module Gren
4
+ module Util
5
+ def time_s(time)
6
+ t = time.truncate
7
+ h = t / 3600
8
+ t = t % 3600
9
+ m = t / 60
10
+ t = t % 60
11
+ t += round(time - time.prec_i, 2)
12
+
13
+ if (h > 0 && m > 0)
14
+ "#{h}h #{m}m #{t}s"
15
+ elsif (m > 0)
16
+ "#{m}m #{t}s"
17
+ else
18
+ "#{t}sec"
19
+ end
20
+ end
21
+ module_function :time_s
22
+
23
+ def round(n, d)
24
+ (n * 10 ** d).round / 10.0 ** d
25
+ end
26
+ module_function :round
27
+
28
+ def size_s(size)
29
+ tb = 1024 ** 4
30
+ gb = 1024 ** 3
31
+ mb = 1024 ** 2
32
+ kb = 1024
33
+
34
+ if (size >= tb)
35
+ round(size / tb.prec_f, 2).to_s + "TB"
36
+ elsif (size >= gb)
37
+ round(size / gb.prec_f, 2).to_s + "GB"
38
+ elsif (size >= mb)
39
+ round(size / mb.prec_f, 2).to_s + "MB"
40
+ elsif (size >= kb)
41
+ round(size / kb.prec_f, 2).to_s + "KB"
42
+ else
43
+ size.to_s + "Byte"
44
+ end
45
+ end
46
+ module_function :size_s
47
+
48
+ # アルファベットと演算子で表示する数を変える
49
+ ALPHABET_DISP_NUM = 5
50
+ OPERATOR_DISP_NUM = 10
51
+
52
+ def p_classtree(c)
53
+ unless c.is_a?(Class)
54
+ c = c.class
55
+ end
56
+
57
+ while (true)
58
+ puts c.name
59
+ break if (c == Object)
60
+ p_classtree_sub(c)
61
+ c = c.superclass
62
+ end
63
+ end
64
+ module_function :p_classtree
65
+
66
+ def p_classtree_sub(c)
67
+ # メソッドの一覧を得る
68
+ group = c.public_instance_methods(false).sort.partition { |m| m =~ /\w/ }
69
+ array = group.flatten
70
+ operator_start_index = group[0].size
71
+ limit = ALPHABET_DISP_NUM
72
+
73
+ print (array.size > limit) ? "| " : "↓ "
74
+
75
+ counter = 0
76
+ array.each_with_index do |v, index|
77
+ if (index == operator_start_index)
78
+ limit = OPERATOR_DISP_NUM
79
+ counter = 0
80
+ puts
81
+ print (array.size - index > limit) ? "| " : "↓ "
82
+ end
83
+
84
+ if (counter >= limit)
85
+ counter = 0
86
+ puts
87
+ print (array.size - index > limit) ? "| " : "↓ "
88
+ end
89
+
90
+ print v + ", "
91
+ counter += 1
92
+ end
93
+ puts
94
+ end
95
+ module_function :p_classtree_sub
96
+
97
+ end
98
+ end
@@ -8,6 +8,8 @@ require File.join(File.dirname(__FILE__), '../common/platform')
8
8
  require File.join(File.dirname(__FILE__), '../common/grenfiletest')
9
9
  require File.join(File.dirname(__FILE__), '../common/grensnip')
10
10
  require 'groonga'
11
+ require File.join(File.dirname(__FILE__), '../common/util')
12
+ include Gren
11
13
 
12
14
  module FindGrep
13
15
  class FindGrep
@@ -20,11 +22,14 @@ module FindGrep
20
22
  :isSilent,
21
23
  :debugMode,
22
24
  :filePatterns,
25
+ :suffixs,
23
26
  :ignoreFiles,
24
27
  :ignoreDirs,
25
28
  :kcode,
26
29
  :noSnip,
27
- :dbFile)
30
+ :dbFile,
31
+ :groongaOnly,
32
+ :isMatchFile)
28
33
 
29
34
  DEFAULT_OPTION = Option.new([],
30
35
  [],
@@ -37,9 +42,12 @@ module FindGrep
37
42
  [],
38
43
  [],
39
44
  [],
45
+ [],
40
46
  Platform.get_shell_kcode,
41
47
  false,
42
- nil)
48
+ nil,
49
+ false,
50
+ false)
43
51
 
44
52
  def initialize(patterns, option)
45
53
  @patterns = patterns
@@ -112,9 +120,9 @@ module FindGrep
112
120
 
113
121
  # ドキュメントを検索
114
122
  documents = Groonga::Context.default["documents"]
115
-
123
+
116
124
  # 全てのパターンを検索
117
- records = documents.select do |record|
125
+ table = documents.select do |record|
118
126
  expression = nil
119
127
 
120
128
  # キーワード
@@ -137,21 +145,62 @@ module FindGrep
137
145
  end
138
146
  end
139
147
 
140
- # 検索方法
148
+ # 拡張子(OR)
149
+ se = suffix_expression(record)
150
+ expression &= se if (se)
151
+
152
+ # 検索式
141
153
  expression
142
154
  end
143
155
 
156
+ # タイムスタンプでソート
157
+ records = table.sort([{:key => "timestamp", :order => "descending"}])
158
+
144
159
  # データベースにヒット
145
160
  stdout.puts "Found : #{records.size} records."
146
161
 
147
162
  # 検索にヒットしたファイルを実際に検索
148
163
  records.each do |record|
149
- if FileTest.exist? record.path
150
- searchFile(stdout, record.path, record.path)
164
+ if (@option.groongaOnly)
165
+ searchGroongaOnly(stdout, record)
166
+ else
167
+ searchFile(stdout, record.path, record.path) if FileTest.exist?(record.path)
151
168
  end
152
169
  end
153
170
  end
154
171
 
172
+ def and_expression(key, list)
173
+ sub = nil
174
+
175
+ list.each do |word|
176
+ e = key =~ word
177
+ if sub.nil?
178
+ sub = e
179
+ else
180
+ sub &= e
181
+ end
182
+ end
183
+
184
+ sub
185
+ end
186
+
187
+ def suffix_expression(record)
188
+ sub = nil
189
+
190
+ @option.suffixs.each do |word|
191
+ e = record.suffix =~ word
192
+ if sub.nil?
193
+ sub = e
194
+ else
195
+ sub |= e
196
+ end
197
+ end
198
+
199
+ sub
200
+ end
201
+ private :suffix_expression
202
+
203
+
155
204
  def searchFromDir(stdout, dir, depth)
156
205
  if (@option.depth != -1 && depth > @option.depth)
157
206
  return
@@ -237,37 +286,57 @@ module FindGrep
237
286
  @result.search_count += 1
238
287
  @result.search_size += FileTest.size(fpath)
239
288
 
240
- # 検索本体
241
289
  @result.search_files << fpath_disp if (@option.debugMode)
242
290
 
243
- open(fpath, "r") { |file|
244
- match_file = false
291
+ open(fpath, "r") do |file|
292
+ searchData(stdout, file2data(file), fpath_disp)
293
+ end
294
+ end
295
+ private :searchFile
296
+
297
+ def searchGroongaOnly(stdout, record)
298
+ file_size = record.content.size
299
+
300
+ @result.count += 1
301
+ @result.size += file_size
302
+
303
+ @result.search_count += 1
304
+ @result.search_size += file_size
305
+
306
+ @result.search_files << record.path if (@option.debugMode)
307
+
308
+ searchData(stdout, record.content, record.path)
309
+ end
310
+ private :searchGroongaOnly
245
311
 
246
- file2data(file).each_with_index { |line, index|
247
- result, match_datas = match?(line)
312
+ def searchData(stdout, data, path)
313
+ match_file = false
248
314
 
249
- if ( result )
250
- header = "#{fpath_disp}:#{index + 1}:"
251
- line = GrenSnip::snip(line, match_datas) unless (@option.noSnip)
315
+ data.each_with_index { |line, index|
316
+ result, match_datas = match?(line)
252
317
 
253
- unless (@option.colorHighlight)
254
- stdout.puts header + line
255
- else
256
- stdout.puts HighLine::BLUE + header + HighLine::CLEAR + GrenSnip::coloring(line, match_datas)
257
- end
318
+ if ( result )
319
+ header = "#{path}:#{index + 1}:"
320
+ line = GrenSnip::snip(line, match_datas) unless (@option.noSnip)
258
321
 
259
- unless match_file
260
- @result.match_file_count += 1
261
- @result.match_files << fpath_disp if (@option.debugMode)
262
- match_file = true
263
- end
322
+ unless (@option.colorHighlight)
323
+ stdout.puts header + line
324
+ else
325
+ stdout.puts HighLine::BLUE + header + HighLine::CLEAR + GrenSnip::coloring(line, match_datas)
326
+ end
264
327
 
265
- @result.match_count += 1
328
+ unless match_file
329
+ @result.match_file_count += 1
330
+ @result.match_files << path if (@option.debugMode)
331
+ match_file = true
332
+ break if (@option.isMatchFile)
266
333
  end
267
- }
334
+
335
+ @result.match_count += 1
336
+ end
268
337
  }
269
338
  end
270
- private :searchFile
339
+ private :searchData
271
340
 
272
341
  def file2data(file)
273
342
  data = file.read
@@ -294,13 +363,25 @@ module FindGrep
294
363
 
295
364
  or_matchs = []
296
365
  @orRegexps.each {|v| or_matchs << v.match(line)}
297
-
298
- result = match_datas.all? && !sub_matchs.any? && (or_matchs.empty? || or_matchs.any?)
366
+
367
+ unless (@option.isMatchFile)
368
+ result = match_datas.all? && !sub_matchs.any? && (or_matchs.empty? || or_matchs.any?)
369
+ else
370
+ result = first_condition(match_datas, sub_matchs, or_matchs)
371
+ end
299
372
  result_match = match_datas + or_matchs
300
373
  result_match.delete(nil)
301
374
 
302
375
  return result, result_match
303
376
  end
304
377
  private :match?
378
+
379
+ def first_condition(match_datas, sub_matchs, or_matchs)
380
+ unless match_datas.empty?
381
+ match_datas[0]
382
+ else
383
+ or_matchs[0]
384
+ end
385
+ end
305
386
  end
306
387
  end
@@ -1,6 +1,6 @@
1
1
  # -*- coding: utf-8 -*-
2
2
  require 'find'
3
- require File.join(File.dirname(__FILE__), '../common/display_util')
3
+ require File.join(File.dirname(__FILE__), '../common/util')
4
4
  include Gren
5
5
 
6
6
  module FindGrep
@@ -34,8 +34,8 @@ module FindGrep
34
34
  end
35
35
 
36
36
  def print(stdout)
37
- stdout.puts "dir : #{@start_dir} (#{DisplayUtil::time_s(time)})"
38
- stdout.puts "files : #{@search_count} in #{@count} (#{DisplayUtil::size_s(@search_size)} in #{DisplayUtil::size_s(@size)})"
37
+ stdout.puts "dir : #{@start_dir} (#{Util::time_s(time)})"
38
+ stdout.puts "files : #{@search_count} in #{@count} (#{Util::size_s(@search_size)} in #{Util::size_s(@size)})"
39
39
  stdout.puts "match : #{@match_file_count} files, #{match_count} hit"
40
40
  end
41
41
 
data/lib/gren.rb CHANGED
@@ -2,5 +2,5 @@ $:.unshift(File.dirname(__FILE__)) unless
2
2
  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
3
3
 
4
4
  module Gren
5
- VERSION = '0.2.3'
5
+ VERSION = '0.2.4'
6
6
  end
data/lib/grendb/cli.rb CHANGED
@@ -8,14 +8,21 @@ module Grendb
8
8
  # オプション
9
9
  option = FindGrep::FindGrep::DEFAULT_OPTION
10
10
  option.dbFile = ENV['GRENDB_DEFAULT_DB']
11
+
12
+ # デフォルトのマッチモードは'File'
13
+ option.isMatchFile = true
11
14
 
12
15
  # オプション解析
13
16
  opt = OptionParser.new("#{File.basename($0)} [option] keyword1 [keyword2 ...]")
14
17
  opt.on('--db [GREN_DB_FILE]', 'Search from the grendb database.') {|v| option.dbFile = v }
15
18
  opt.on('-f KEYWORD', '--file-keyword KEYWORD', 'Path keyword. (Enable multiple call)') {|v| option.filePatterns << v}
19
+ opt.on('-s SUFFIX', '--suffix SUFFIX', 'Search suffix.') {|v| option.suffixs << v }
16
20
  opt.on('-i', '--ignore', 'Ignore case.') {|v| option.ignoreCase = true}
17
21
  opt.on('-c', '--color', 'Color highlight.') {|v| option.colorHighlight = true}
18
22
  opt.on('--no-snip', 'There being a long line, it does not snip.') {|v| option.noSnip = true }
23
+ opt.on('-g', '--groonga-only', 'Search only groonga db.') {|v| option.groongaOnly = true }
24
+ opt.on('--mf', '--match-file', 'Match file. (Default)') {|v| option.isMatchFile = true }
25
+ opt.on('-l', '--ml', '--match-line', 'Match line, same mode as "gren".') {|v| option.isMatchFile = false }
19
26
 
20
27
  opt.parse!(arguments)
21
28
 
data/lib/mkgrendb/cli.rb CHANGED
@@ -9,12 +9,14 @@ module Mkgrendb
9
9
  isDump = false
10
10
  isFull = false
11
11
  isDelete = false
12
+ isReport = false
12
13
 
13
14
  opt = OptionParser.new "#{File.basename($0)} INPUT_YAML1 [INPUT_YAML2 ...]"
14
15
  opt.on('--ddb', "--default-db", "Create or Update default DB. (Plase set ENV['GRENDB_DEFAULT_DB'])") {|v| input_yamls << ENV['GRENDB_DEFAULT_DB']}
15
16
  opt.on('--full', "Full update DB. (Delete and create)") {|v| isFull = true }
16
17
  opt.on('--delete', "Delete DB. (Not delete yaml)") {|v| isDelete = true }
17
18
  opt.on('--dump', "Dump DB.") {|v| isDump = true }
19
+ opt.on('--report', "Database Report.") {|v| isReport = true }
18
20
  opt.parse!(arguments)
19
21
 
20
22
  input_yamls.concat arguments
@@ -31,6 +33,8 @@ module Mkgrendb
31
33
  stdout.puts
32
34
  elsif (isDump)
33
35
  obj.dump
36
+ elsif (isReport)
37
+ obj.report
34
38
  else
35
39
  obj.update
36
40
  stdout.puts
@@ -5,7 +5,7 @@ require 'pathname'
5
5
  require 'rubygems'
6
6
  require 'groonga'
7
7
  require File.join(File.dirname(__FILE__), '../common/grenfiletest')
8
- require File.join(File.dirname(__FILE__), '../common/display_util')
8
+ require File.join(File.dirname(__FILE__), '../common/util')
9
9
  include Gren
10
10
 
11
11
  module Mkgrendb
@@ -25,7 +25,12 @@ module Mkgrendb
25
25
  db_create(@output_db)
26
26
  db_open(@output_db)
27
27
  @src["directory"].each do |dir|
28
- db_add_dir(File.expand_path(dir))
28
+ dir = File.expand_path(dir)
29
+ if (FileTest.directory? dir)
30
+ db_add_dir(dir)
31
+ else
32
+ db_add_file(STDOUT, dir)
33
+ end
29
34
  end
30
35
  @end_time = Time.now
31
36
  print_result
@@ -46,7 +51,7 @@ module Mkgrendb
46
51
 
47
52
  def print_result
48
53
  puts
49
- puts "input_yaml : #{@input_yaml} (#{DisplayUtil::time_s(time)})"
54
+ puts "input_yaml : #{@input_yaml} (#{Util::time_s(time)})"
50
55
  puts "output_db : #{@output_db}*"
51
56
  puts "files : #{@file_count}"
52
57
  puts "updates : #{@update_count}"
@@ -58,9 +63,11 @@ module Mkgrendb
58
63
  documents = Groonga::Context.default["documents"]
59
64
  records = documents.select
60
65
  records.each do |record|
66
+ p record
61
67
  puts "path : #{record.path}"
68
+ puts "suffix : #{record.suffix}"
62
69
  puts "timestamp : #{record.timestamp.strftime('%Y/%m/%d %H:%M:%S')}"
63
- puts "content :", record.content[0..64]
70
+ puts "content :", record.content ? record.content[0..64] : nil
64
71
  puts
65
72
  end
66
73
  end
@@ -77,6 +84,7 @@ module Mkgrendb
77
84
  table.string("path")
78
85
  table.text("content")
79
86
  table.time("timestamp")
87
+ table.text("suffix")
80
88
  end
81
89
 
82
90
  schema.create_table("terms",
@@ -85,7 +93,7 @@ module Mkgrendb
85
93
  :default_tokenizer => "TokenBigram") do |table|
86
94
  table.index("documents.path", :with_position => true)
87
95
  table.index("documents.content", :with_position => true)
88
- table.index("documents.timestamp", :with_position => true)
96
+ table.index("documents.suffix", :with_position => true)
89
97
  end
90
98
  end
91
99
  puts "create : #{filename} created."
@@ -127,6 +135,7 @@ module Mkgrendb
127
135
  :path => filename,
128
136
  :content => nil,
129
137
  :timestamp => File.mtime(filename),
138
+ :suffix => File::extname(filename),
130
139
  }
131
140
 
132
141
  # 検索するデータベース
metadata CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
5
5
  segments:
6
6
  - 0
7
7
  - 2
8
- - 3
9
- version: 0.2.3
8
+ - 4
9
+ version: 0.2.4
10
10
  platform: ruby
11
11
  authors:
12
12
  - ongaeshi
@@ -14,7 +14,7 @@ autorequire:
14
14
  bindir: bin
15
15
  cert_chain: []
16
16
 
17
- date: 2010-09-26 00:00:00 +09:00
17
+ date: 2010-10-10 00:00:00 +09:00
18
18
  default_executable:
19
19
  dependencies:
20
20
  - !ruby/object:Gem::Dependency
@@ -103,6 +103,7 @@ files:
103
103
  - lib/common/grensnip.rb
104
104
  - lib/common/platform.rb
105
105
  - lib/common/string_snip.rb
106
+ - lib/common/util.rb
106
107
  - lib/findgrep/findgrep.rb
107
108
  - lib/findgrep/result.rb
108
109
  - lib/gren.rb