milkode 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. data/.document +5 -0
  2. data/Gemfile +12 -0
  3. data/Gemfile.lock +18 -0
  4. data/HISTORY.rdoc +5 -0
  5. data/LICENSE.txt +20 -0
  6. data/README.rdoc +19 -0
  7. data/Rakefile +78 -0
  8. data/VERSION +1 -0
  9. data/bin/cdv +6 -0
  10. data/bin/cdview +10 -0
  11. data/bin/milk +10 -0
  12. data/lib/milkode/cdstk/cdstk.rb +508 -0
  13. data/lib/milkode/cdstk/cdstk_yaml.rb +106 -0
  14. data/lib/milkode/cdstk/cli_cdstk.rb +103 -0
  15. data/lib/milkode/cdstk/cli_cdstksub.rb +102 -0
  16. data/lib/milkode/cdview/cli_cdview.rb +43 -0
  17. data/lib/milkode/cdweb/app.rb +136 -0
  18. data/lib/milkode/cdweb/cli_cdweb.rb +111 -0
  19. data/lib/milkode/cdweb/config.ru +3 -0
  20. data/lib/milkode/cdweb/lib/coderay_wrapper.rb +94 -0
  21. data/lib/milkode/cdweb/lib/command.rb +67 -0
  22. data/lib/milkode/cdweb/lib/database.rb +260 -0
  23. data/lib/milkode/cdweb/lib/grep.rb +68 -0
  24. data/lib/milkode/cdweb/lib/mkurl.rb +56 -0
  25. data/lib/milkode/cdweb/lib/query.rb +81 -0
  26. data/lib/milkode/cdweb/lib/search_contents.rb +117 -0
  27. data/lib/milkode/cdweb/lib/search_files.rb +88 -0
  28. data/lib/milkode/cdweb/public/css/coderay.css +131 -0
  29. data/lib/milkode/cdweb/public/css/gren.css +75 -0
  30. data/lib/milkode/cdweb/public/images/directory.png +0 -0
  31. data/lib/milkode/cdweb/public/images/file.png +0 -0
  32. data/lib/milkode/cdweb/public/images/gren-icon-mini.png +0 -0
  33. data/lib/milkode/cdweb/public/images/gren-icon.png +0 -0
  34. data/lib/milkode/cdweb/views/filelist.haml +14 -0
  35. data/lib/milkode/cdweb/views/help.haml +16 -0
  36. data/lib/milkode/cdweb/views/index.haml +17 -0
  37. data/lib/milkode/cdweb/views/layout.haml +33 -0
  38. data/lib/milkode/cdweb/views/search.haml +12 -0
  39. data/lib/milkode/cdweb/views/view.haml +16 -0
  40. data/lib/milkode/common/dbdir.rb +39 -0
  41. data/lib/milkode/common/display_util.rb +62 -0
  42. data/lib/milkode/common/grenfiletest.rb +19 -0
  43. data/lib/milkode/common/grensnip.rb +37 -0
  44. data/lib/milkode/common/platform.rb +17 -0
  45. data/lib/milkode/common/string_snip.rb +61 -0
  46. data/lib/milkode/common/util.rb +144 -0
  47. data/lib/milkode/findgrep/findgrep.rb +408 -0
  48. data/lib/milkode/findgrep/result.rb +43 -0
  49. data/milkode.gemspec +159 -0
  50. data/test/data/abc.zip +0 -0
  51. data/test/data/nodir_abc.zip +0 -0
  52. data/test/data/nodir_abc_xpi.xpi +0 -0
  53. data/test/file_assert.rb +41 -0
  54. data/test/file_test_utils.rb +59 -0
  55. data/test/rake_test_loader.rb +7 -0
  56. data/test/runner.rb +11 -0
  57. data/test/test_bin_exec.rb +30 -0
  58. data/test/test_cdstk.rb +90 -0
  59. data/test/test_cdstk_yaml.rb +157 -0
  60. data/test/test_coderay_wrapper.rb +22 -0
  61. data/test/test_coderay_wrapper_data.rb +91 -0
  62. data/test/test_database.rb +78 -0
  63. data/test/test_dbdir.rb +60 -0
  64. data/test/test_gren_util.rb +34 -0
  65. data/test/test_helper.rb +2 -0
  66. data/test/test_mkurl.rb +30 -0
  67. data/test/test_query.rb +54 -0
  68. data/test/test_string_snip.rb +31 -0
  69. data/test/test_util.rb +40 -0
  70. metadata +300 -0
@@ -0,0 +1,408 @@
1
+ # -*- coding: utf-8 -*-
2
+ require 'find'
3
+ require 'milkode/findgrep/result'
4
+ require 'rubygems'
5
+ require 'termcolor'
6
+ require 'kconv'
7
+ require 'milkode/common/platform'
8
+ require 'milkode/common/grenfiletest'
9
+ require 'milkode/common/grensnip'
10
+ require 'groonga'
11
+ require 'milkode/common/util'
12
+ include Gren
13
+ require 'cgi'
14
+
15
+ module FindGrep
16
+ class FindGrep
17
+ Option = Struct.new(:keywordsNot,
18
+ :keywordsOr,
19
+ :directory,
20
+ :depth,
21
+ :ignoreCase,
22
+ :colorHighlight,
23
+ :isSilent,
24
+ :debugMode,
25
+ :filePatterns,
26
+ :suffixs,
27
+ :ignoreFiles,
28
+ :ignoreDirs,
29
+ :kcode,
30
+ :noSnip,
31
+ :dbFile,
32
+ :groongaOnly,
33
+ :isMatchFile,
34
+ :dispHtml)
35
+
36
+ DEFAULT_OPTION = Option.new([],
37
+ [],
38
+ ".",
39
+ -1,
40
+ false,
41
+ false,
42
+ false,
43
+ false,
44
+ [],
45
+ [],
46
+ [],
47
+ [],
48
+ Platform.get_shell_kcode,
49
+ false,
50
+ nil,
51
+ false,
52
+ false,
53
+ false)
54
+
55
+ attr_reader :documents
56
+
57
+ def initialize(patterns, option)
58
+ @patterns = patterns
59
+ @option = option
60
+ @patternRegexps = strs2regs(patterns, @option.ignoreCase)
61
+ @subRegexps = strs2regs(option.keywordsNot, @option.ignoreCase)
62
+ @orRegexps = strs2regs(option.keywordsOr, @option.ignoreCase)
63
+ @filePatterns = (!@option.dbFile) ? strs2regs(option.filePatterns) : []
64
+ @ignoreFiles = strs2regs(option.ignoreFiles)
65
+ @ignoreDirs = strs2regs(option.ignoreDirs)
66
+ @result = Result.new(option.directory)
67
+ open_database if (@option.dbFile)
68
+ end
69
+
70
+ def open_database()
71
+ # データベース開く
72
+ dbfile = Pathname(File.expand_path(@option.dbFile))
73
+
74
+ if dbfile.exist?
75
+ Groonga::Database.open(dbfile.to_s)
76
+ puts "open : #{dbfile} open."
77
+ else
78
+ raise "error : #{dbfile.to_s} not found!!"
79
+ end
80
+
81
+ # ドキュメントを取
82
+ @documents = Groonga::Context.default["documents"]
83
+ end
84
+
85
+ def strs2regs(strs, ignore = false)
86
+ regs = []
87
+
88
+ strs.each do |v|
89
+ option = 0
90
+ option |= Regexp::IGNORECASE if (ignore)
91
+ regs << Regexp.new(v, option)
92
+ end
93
+
94
+ regs
95
+ end
96
+
97
+ def searchAndPrint(stdout)
98
+ unless (@option.dbFile)
99
+ searchFromDir(stdout, @option.directory, 0)
100
+ else
101
+ searchFromDB(stdout, @option.directory)
102
+ end
103
+
104
+ @result.time_stop
105
+
106
+ if (!@option.isSilent && !@option.dispHtml)
107
+ if (@option.debugMode)
108
+ stdout.puts
109
+ stdout.puts "--- search --------"
110
+ print_fpaths stdout, @result.search_files
111
+ stdout.puts "--- match --------"
112
+ print_fpaths stdout, @result.match_files
113
+ stdout.puts "--- ignore-file --------"
114
+ print_fpaths stdout, @result.ignore_files
115
+ stdout.puts "--- ignore-dir --------"
116
+ print_fpaths stdout, @result.prune_dirs
117
+ stdout.puts "--- unreadable --------"
118
+ print_fpaths stdout, @result.unreadable_files
119
+ end
120
+
121
+ unless (@option.colorHighlight)
122
+ stdout.puts
123
+ else
124
+ stdout.puts HighLine::REVERSE + "------------------------------------------------------------" + HighLine::CLEAR
125
+ end
126
+
127
+ @result.print(stdout)
128
+ end
129
+ end
130
+
131
+ def searchFromDB(stdout, dir)
132
+ # 全てのパターンを検索
133
+ table = @documents.select do |record|
134
+ expression = nil
135
+
136
+ # キーワード
137
+ @patterns.each do |word|
138
+ sub_expression = record.content =~ word
139
+ if expression.nil?
140
+ expression = sub_expression
141
+ else
142
+ expression &= sub_expression
143
+ end
144
+ end
145
+
146
+ # パス
147
+ @option.filePatterns.each do |word|
148
+ sub_expression = record.path =~ word
149
+ if expression.nil?
150
+ expression = sub_expression
151
+ else
152
+ expression &= sub_expression
153
+ end
154
+ end
155
+
156
+ # 拡張子(OR)
157
+ se = suffix_expression(record)
158
+ expression &= se if (se)
159
+
160
+ # 検索式
161
+ expression
162
+ end
163
+
164
+ # タイムスタンプでソート
165
+ records = table.sort([{:key => "_score", :order => "descending"},
166
+ {:key => "timestamp", :order => "descending"}])
167
+
168
+ # データベースにヒット
169
+ stdout.puts "Found : #{records.size} records." unless (@option.dispHtml)
170
+
171
+ # 検索にヒットしたファイルを実際に検索
172
+ records.each do |record|
173
+ if (@option.groongaOnly)
174
+ searchGroongaOnly(stdout, record)
175
+ else
176
+ searchFile(stdout, record.path, record.path) if FileTest.exist?(record.path)
177
+ end
178
+ end
179
+ end
180
+
181
+ def and_expression(key, list)
182
+ sub = nil
183
+
184
+ list.each do |word|
185
+ e = key =~ word
186
+ if sub.nil?
187
+ sub = e
188
+ else
189
+ sub &= e
190
+ end
191
+ end
192
+
193
+ sub
194
+ end
195
+
196
+ def suffix_expression(record)
197
+ sub = nil
198
+
199
+ @option.suffixs.each do |word|
200
+ e = record.suffix =~ word
201
+ if sub.nil?
202
+ sub = e
203
+ else
204
+ sub |= e
205
+ end
206
+ end
207
+
208
+ sub
209
+ end
210
+ private :suffix_expression
211
+
212
+
213
+ def searchFromDir(stdout, dir, depth)
214
+ if (@option.depth != -1 && depth > @option.depth)
215
+ return
216
+ end
217
+
218
+ Dir.foreach(dir) do |name|
219
+ next if (name == '.' || name == '..')
220
+
221
+ fpath = File.join(dir,name)
222
+ fpath_disp = fpath.gsub(/^.\//, "")
223
+
224
+ # 除外ディレクトリならばパス
225
+ if ignoreDir?(fpath)
226
+ @result.prune_dirs << fpath_disp if (@option.debugMode)
227
+ next;
228
+ end
229
+
230
+ # 読み込み不可ならばパス
231
+ unless FileTest.readable?(fpath)
232
+ @result.unreadable_files << fpath_disp if (@option.debugMode)
233
+ next
234
+ end
235
+
236
+ # ファイルならば中身を探索、ディレクトリならば再帰
237
+ case File.ftype(fpath)
238
+ when "directory"
239
+ searchFromDir(stdout, fpath, depth + 1)
240
+ when "file"
241
+ searchFile(stdout, fpath, fpath_disp)
242
+ end
243
+ end
244
+ end
245
+ private :searchFromDir
246
+
247
+ def print_fpaths(stdout, data)
248
+ stdout.print data.join("\n")
249
+ stdout.puts if data.count > 0
250
+ stdout.puts "total: #{data.count}"
251
+ stdout.puts
252
+ end
253
+ private :print_fpaths
254
+
255
+ def ignoreDir?(fpath)
256
+ FileTest.directory?(fpath) &&
257
+ (GrenFileTest::ignoreDir?(File.basename(fpath)) || ignoreDirUser?(fpath))
258
+ end
259
+ private :ignoreDir?
260
+
261
+ def ignoreDirUser?(fpath)
262
+ @ignoreDirs.any? {|v| v.match File.basename(fpath) }
263
+ end
264
+ private :ignoreDirUser?
265
+
266
+ def ignoreFile?(fpath)
267
+ !correctFileUser?(fpath) ||
268
+ GrenFileTest::ignoreFile?(fpath) ||
269
+ ignoreFileUser?(fpath) ||
270
+ GrenFileTest::binary?(fpath)
271
+ end
272
+ private :ignoreFile?
273
+
274
+ def correctFileUser?(fpath)
275
+ @filePatterns.empty? ||
276
+ @filePatterns.any? {|v| v.match File.basename(fpath) }
277
+ end
278
+ private :correctFileUser?
279
+
280
+ def ignoreFileUser?(fpath)
281
+ @ignoreFiles.any? {|v| v.match File.basename(fpath) }
282
+ end
283
+ private :ignoreFileUser?
284
+
285
+ def searchFile(stdout, fpath, fpath_disp)
286
+ @result.count += 1
287
+ @result.size += FileTest.size(fpath)
288
+
289
+ # 除外ファイル
290
+ if ignoreFile?(fpath)
291
+ @result.ignore_files << fpath_disp if (@option.debugMode)
292
+ return
293
+ end
294
+
295
+ @result.search_count += 1
296
+ @result.search_size += FileTest.size(fpath)
297
+
298
+ @result.search_files << fpath_disp if (@option.debugMode)
299
+
300
+ open(fpath, "r") do |file|
301
+ searchData(stdout, file2data(file), fpath_disp)
302
+ end
303
+ end
304
+ private :searchFile
305
+
306
+ def searchGroongaOnly(stdout, record)
307
+ file_size = record.content.size
308
+
309
+ @result.count += 1
310
+ @result.size += file_size
311
+
312
+ @result.search_count += 1
313
+ @result.search_size += file_size
314
+
315
+ @result.search_files << record.path if (@option.debugMode)
316
+
317
+ searchData(stdout, record.content, record.path)
318
+ end
319
+ private :searchGroongaOnly
320
+
321
+ def searchData(stdout, data, path)
322
+ match_file = false
323
+
324
+ data.each_with_index { |line, index|
325
+ result, match_datas = match?(line)
326
+
327
+ if ( result )
328
+ unless (@option.dispHtml)
329
+ header = "#{path}:#{index + 1}:"
330
+ line = GrenSnip::snip(line, match_datas) unless (@option.noSnip)
331
+
332
+ unless (@option.colorHighlight)
333
+ stdout.puts header + line
334
+ else
335
+ stdout.puts HighLine::BLUE + header + HighLine::CLEAR + GrenSnip::coloring(line, match_datas)
336
+ end
337
+ else
338
+ line_no = index + 1
339
+ line = GrenSnip::snip(line, match_datas) unless (@option.noSnip)
340
+
341
+ stdout.puts <<EOF
342
+ <h2><a href="../::view#{path}">#{path}</a></h2>
343
+ <pre>
344
+ #{line_no} : #{CGI.escapeHTML(line)}
345
+ </pre>
346
+ EOF
347
+ end
348
+
349
+ unless match_file
350
+ @result.match_file_count += 1
351
+ @result.match_files << path if (@option.debugMode)
352
+ match_file = true
353
+ break if (@option.isMatchFile)
354
+ end
355
+
356
+ @result.match_count += 1
357
+ end
358
+ }
359
+ end
360
+ private :searchData
361
+
362
+ def file2data(file)
363
+ data = file.read
364
+
365
+ if (@option.kcode != Kconv::NOCONV)
366
+ file_kcode = Kconv::guess(data)
367
+
368
+ if (file_kcode != @option.kcode)
369
+ # puts "encode!! #{fpath} : #{@option.kcode} <- #{file_kcode}"
370
+ data = data.kconv(@option.kcode, file_kcode)
371
+ end
372
+ end
373
+
374
+ data = data.split("\n");
375
+ end
376
+ private :file2data
377
+
378
+ def match?(line)
379
+ match_datas = []
380
+ @patternRegexps.each {|v| match_datas << v.match(line)}
381
+
382
+ sub_matchs = []
383
+ @subRegexps.each {|v| sub_matchs << v.match(line)}
384
+
385
+ or_matchs = []
386
+ @orRegexps.each {|v| or_matchs << v.match(line)}
387
+
388
+ unless (@option.isMatchFile)
389
+ result = match_datas.all? && !sub_matchs.any? && (or_matchs.empty? || or_matchs.any?)
390
+ else
391
+ result = first_condition(match_datas, sub_matchs, or_matchs)
392
+ end
393
+ result_match = match_datas + or_matchs
394
+ result_match.delete(nil)
395
+
396
+ return result, result_match
397
+ end
398
+ private :match?
399
+
400
+ def first_condition(match_datas, sub_matchs, or_matchs)
401
+ unless match_datas.empty?
402
+ match_datas[0]
403
+ else
404
+ or_matchs[0]
405
+ end
406
+ end
407
+ end
408
+ end
@@ -0,0 +1,43 @@
1
+ # -*- coding: utf-8 -*-
2
+ require 'find'
3
+ require 'milkode/common/util'
4
+ include Gren
5
+
6
+ module FindGrep
7
+ class Result
8
+ attr_accessor :count
9
+ attr_accessor :search_count
10
+ attr_accessor :match_file_count
11
+ attr_accessor :match_count
12
+ attr_accessor :size
13
+ attr_accessor :search_size
14
+
15
+ attr_accessor :search_files
16
+ attr_accessor :match_files
17
+ attr_accessor :unreadable_files
18
+ attr_accessor :prune_dirs
19
+ attr_accessor :ignore_files
20
+
21
+ def initialize(start_dir)
22
+ @start_dir = File.expand_path(start_dir)
23
+ @count, @search_count, @match_file_count, @match_count, @size, @search_size = 0, 0, 0, 0, 0, 0
24
+ @start_time = Time.now
25
+ @search_files, @match_files, @unreadable_files, @prune_dirs, @ignore_files = [], [], [], [], []
26
+ end
27
+
28
+ def time_stop
29
+ @end_time = Time.now
30
+ end
31
+
32
+ def time
33
+ @end_time - @start_time
34
+ end
35
+
36
+ def print(stdout)
37
+ stdout.puts "dir : #{@start_dir} (#{Util::time_s(time)})"
38
+ stdout.puts "files : #{@search_count} in #{@count} (#{Util::size_s(@search_size)} in #{Util::size_s(@size)})"
39
+ stdout.puts "match : #{@match_file_count} files, #{match_count} hit"
40
+ end
41
+
42
+ end
43
+ end