gren 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +9 -5
- data/Manifest.txt +13 -5
- data/Rakefile +1 -1
- data/bin/gren +1 -1
- data/bin/grendb +11 -0
- data/bin/mkgrendb +11 -0
- data/lib/common/grenfiletest.rb +19 -0
- data/lib/common/grensnip.rb +37 -0
- data/lib/{platform.rb → common/platform.rb} +0 -0
- data/lib/{string_snip.rb → common/string_snip.rb} +0 -0
- data/lib/{gren → findgrep}/findgrep.rb +77 -23
- data/lib/{gren → findgrep}/result.rb +1 -1
- data/lib/gren.rb +1 -1
- data/lib/gren/cli.rb +3 -3
- data/lib/grendb/cli.rb +34 -0
- data/lib/mkgrendb/cli.rb +20 -0
- data/lib/mkgrendb/mkgrendb.rb +142 -0
- data/test/test_gren_util.rb +6 -6
- data/test/test_grendb_cli.rb +14 -0
- data/test/test_mkgrendb_cli.rb +14 -0
- metadata +39 -13
- data/lib/gren/util.rb +0 -39
data/History.txt
CHANGED
@@ -1,10 +1,14 @@
|
|
1
|
-
=== 0.2.0 2010/
|
1
|
+
=== 0.2.0 2010/09/13
|
2
2
|
|
3
|
-
|
3
|
+
* grooongaを利用して超高速検索を実現した、grendbをリリース
|
4
|
+
* mkgrendbでデータベース生成
|
5
|
+
* grendbでデータベース検索
|
4
6
|
|
5
|
-
|
6
|
-
|
7
|
-
|
7
|
+
=== 0.1.4 2010/08/14
|
8
|
+
|
9
|
+
* snipモード
|
10
|
+
* 一行が長過ぎる場合は適当に省略する機能を追加
|
11
|
+
* 圧縮したJavascript等で効果を発揮
|
8
12
|
|
9
13
|
=== 0.1.3 2010/08/13
|
10
14
|
|
data/Manifest.txt
CHANGED
@@ -4,13 +4,19 @@ PostInstall.txt
|
|
4
4
|
README.rdoc
|
5
5
|
Rakefile
|
6
6
|
bin/gren
|
7
|
+
bin/grendb
|
8
|
+
bin/mkgrendb
|
9
|
+
lib/common/grenfiletest.rb
|
10
|
+
lib/common/grensnip.rb
|
11
|
+
lib/common/platform.rb
|
12
|
+
lib/common/string_snip.rb
|
13
|
+
lib/findgrep/findgrep.rb
|
14
|
+
lib/findgrep/result.rb
|
7
15
|
lib/gren.rb
|
8
16
|
lib/gren/cli.rb
|
9
|
-
lib/
|
10
|
-
lib/
|
11
|
-
lib/
|
12
|
-
lib/platform.rb
|
13
|
-
lib/string_snip.rb
|
17
|
+
lib/grendb/cli.rb
|
18
|
+
lib/mkgrendb/cli.rb
|
19
|
+
lib/mkgrendb/mkgrendb.rb
|
14
20
|
rake_rdoc_custom.rb
|
15
21
|
script/console
|
16
22
|
script/destroy
|
@@ -18,5 +24,7 @@ script/generate
|
|
18
24
|
test/test_gren.rb
|
19
25
|
test/test_gren_cli.rb
|
20
26
|
test/test_gren_util.rb
|
27
|
+
test/test_grendb_cli.rb
|
21
28
|
test/test_helper.rb
|
29
|
+
test/test_mkgrendb_cli.rb
|
22
30
|
test/test_string_snip.rb
|
data/Rakefile
CHANGED
@@ -20,7 +20,7 @@ $hoe = Hoe.spec 'gren' do
|
|
20
20
|
self.rubyforge_name = self.name # TODO this is default value
|
21
21
|
|
22
22
|
# 必要とするgem
|
23
|
-
self.extra_deps = [['termcolor','>= 1.2.0']]
|
23
|
+
self.extra_deps = [['termcolor','>= 1.2.0'], ['rroonga','>= 1.0.0']]
|
24
24
|
|
25
25
|
# 本来はnewgemの中で設定されるべき(後で報告した方がいいかも)
|
26
26
|
self.extra_rdoc_files << "README.rdoc"
|
data/bin/gren
CHANGED
data/bin/grendb
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# Created on 2010-8-25.
|
4
|
+
# Copyright (c) 2010. All rights reserved.
|
5
|
+
|
6
|
+
require 'rubygems'
|
7
|
+
require File.expand_path(File.dirname(__FILE__) + "/../lib/gren")
|
8
|
+
require "grendb/cli"
|
9
|
+
|
10
|
+
Version = "0.2.0"
|
11
|
+
Grendb::CLI.execute(STDOUT, ARGV)
|
data/bin/mkgrendb
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# Created on 2010-9-12.
|
4
|
+
# Copyright (c) 2010. All rights reserved.
|
5
|
+
|
6
|
+
require 'rubygems'
|
7
|
+
require File.expand_path(File.dirname(__FILE__) + "/../lib/gren")
|
8
|
+
require "mkgrendb/cli"
|
9
|
+
|
10
|
+
Version = "0.2.0"
|
11
|
+
Mkgrendb::CLI.execute(STDOUT, ARGV)
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
|
3
|
+
module GrenFileTest
|
4
|
+
IGNORE_FILE = /(\A#.*#\Z)|(~\Z)|(\A\.#)/
|
5
|
+
IGNORE_DIR = /(\A\.svn\Z)|(\A\.git\Z)|(\ACVS\Z)/
|
6
|
+
|
7
|
+
def self.ignoreDir?(fpath)
|
8
|
+
IGNORE_DIR.match(File.basename(fpath))
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.ignoreFile?(fpath)
|
12
|
+
IGNORE_FILE.match(File.basename(fpath))
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.binary?(fpath)
|
16
|
+
s = File.read(fpath, 1024) or return false
|
17
|
+
return s.index("\x00")
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
|
3
|
+
require File.join(File.dirname(__FILE__), 'string_snip')
|
4
|
+
|
5
|
+
class GrenSnip
|
6
|
+
MAX_LINE_SIZE = 256
|
7
|
+
HEADER_SIZE = 32
|
8
|
+
MARGIN_SIZE = 32
|
9
|
+
DELIMITER = '<<snip>>'
|
10
|
+
|
11
|
+
def self.snip(str, match_datas)
|
12
|
+
return str if (str.size <= MAX_LINE_SIZE)
|
13
|
+
|
14
|
+
ranges = []
|
15
|
+
ranges << (0..HEADER_SIZE-1)
|
16
|
+
ranges << (-HEADER_SIZE..-1)
|
17
|
+
|
18
|
+
match_datas.each do |m|
|
19
|
+
ranges << (m.begin(0)-MARGIN_SIZE..m.end(0)+MARGIN_SIZE)
|
20
|
+
end
|
21
|
+
|
22
|
+
snipper = StringSnip.new(MAX_LINE_SIZE, DELIMITER)
|
23
|
+
return snipper.snip(str, ranges)
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.coloring(line, match_datas)
|
27
|
+
match_datas.each do |m|
|
28
|
+
line = line.split(m[0]).join(HighLine::ON_GREEN + m[0] + HighLine::CLEAR)
|
29
|
+
end
|
30
|
+
|
31
|
+
line = line.split(DELIMITER).join(HighLine::ON_CYAN + DELIMITER + HighLine::CLEAR)
|
32
|
+
|
33
|
+
line
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
|
File without changes
|
File without changes
|
@@ -4,14 +4,13 @@ require File.join(File.dirname(__FILE__), 'result')
|
|
4
4
|
require 'rubygems'
|
5
5
|
require 'termcolor'
|
6
6
|
require 'kconv'
|
7
|
-
require File.join(File.dirname(__FILE__), '../platform')
|
8
|
-
require File.join(File.dirname(__FILE__), '
|
7
|
+
require File.join(File.dirname(__FILE__), '../common/platform')
|
8
|
+
require File.join(File.dirname(__FILE__), '../common/grenfiletest')
|
9
|
+
require File.join(File.dirname(__FILE__), '../common/grensnip')
|
10
|
+
require 'groonga'
|
9
11
|
|
10
|
-
module
|
12
|
+
module FindGrep
|
11
13
|
class FindGrep
|
12
|
-
IGNORE_FILE = /(\A#.*#\Z)|(~\Z)|(\A\.#)/
|
13
|
-
IGNORE_DIR = /(\A\.svn\Z)|(\A\.git\Z)|(\ACVS\Z)/
|
14
|
-
|
15
14
|
Option = Struct.new(:keywordsSub,
|
16
15
|
:keywordsOr,
|
17
16
|
:directory,
|
@@ -24,7 +23,8 @@ module Gren
|
|
24
23
|
:ignoreFiles,
|
25
24
|
:ignoreDirs,
|
26
25
|
:kcode,
|
27
|
-
:noSnip
|
26
|
+
:noSnip,
|
27
|
+
:dbFile)
|
28
28
|
|
29
29
|
DEFAULT_OPTION = Option.new([],
|
30
30
|
[],
|
@@ -38,14 +38,16 @@ module Gren
|
|
38
38
|
[],
|
39
39
|
[],
|
40
40
|
Platform.get_shell_kcode,
|
41
|
-
false
|
41
|
+
false,
|
42
|
+
nil)
|
42
43
|
|
43
44
|
def initialize(patterns, option)
|
45
|
+
@patterns = patterns
|
44
46
|
@option = option
|
45
47
|
@patternRegexps = strs2regs(patterns, @option.ignoreCase)
|
46
48
|
@subRegexps = strs2regs(option.keywordsSub, @option.ignoreCase)
|
47
49
|
@orRegexps = strs2regs(option.keywordsOr, @option.ignoreCase)
|
48
|
-
@filePatterns = strs2regs(option.filePatterns)
|
50
|
+
@filePatterns = (!@option.dbFile) ? strs2regs(option.filePatterns) : []
|
49
51
|
@ignoreFiles = strs2regs(option.ignoreFiles)
|
50
52
|
@ignoreDirs = strs2regs(option.ignoreDirs)
|
51
53
|
@result = Result.new(option.directory)
|
@@ -64,7 +66,11 @@ module Gren
|
|
64
66
|
end
|
65
67
|
|
66
68
|
def searchAndPrint(stdout)
|
67
|
-
|
69
|
+
unless (@option.dbFile)
|
70
|
+
searchFromDir(stdout, @option.directory, 0)
|
71
|
+
else
|
72
|
+
searchFromDB(stdout, @option.directory)
|
73
|
+
end
|
68
74
|
|
69
75
|
@result.time_stop
|
70
76
|
|
@@ -93,7 +99,60 @@ module Gren
|
|
93
99
|
end
|
94
100
|
end
|
95
101
|
|
96
|
-
def
|
102
|
+
def searchFromDB(stdout, dir)
|
103
|
+
# データベース開く
|
104
|
+
dbfile = Pathname(File.expand_path(@option.dbFile))
|
105
|
+
|
106
|
+
if dbfile.exist?
|
107
|
+
Groonga::Database.open(dbfile.to_s)
|
108
|
+
puts "open : #{dbfile} open."
|
109
|
+
else
|
110
|
+
raise "error : #{dbfile.to_s} not found!!"
|
111
|
+
end
|
112
|
+
|
113
|
+
# ドキュメントを検索
|
114
|
+
documents = Groonga::Context.default["documents"]
|
115
|
+
|
116
|
+
# 全てのパターンを検索
|
117
|
+
records = documents.select do |record|
|
118
|
+
expression = nil
|
119
|
+
|
120
|
+
# キーワード
|
121
|
+
@patterns.each do |word|
|
122
|
+
sub_expression = record.content =~ word
|
123
|
+
if expression.nil?
|
124
|
+
expression = sub_expression
|
125
|
+
else
|
126
|
+
expression &= sub_expression
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
# パス
|
131
|
+
@option.filePatterns.each do |word|
|
132
|
+
sub_expression = record.path =~ word
|
133
|
+
if expression.nil?
|
134
|
+
expression = sub_expression
|
135
|
+
else
|
136
|
+
expression &= sub_expression
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
# 検索方法
|
141
|
+
expression
|
142
|
+
end
|
143
|
+
|
144
|
+
# データベースにヒット
|
145
|
+
stdout.puts "Found : #{records.size} records."
|
146
|
+
|
147
|
+
# 検索にヒットしたファイルを実際に検索
|
148
|
+
records.each do |record|
|
149
|
+
if FileTest.exist? record.path
|
150
|
+
searchFile(stdout, record.path, record.path)
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
def searchFromDir(stdout, dir, depth)
|
97
156
|
if (@option.depth != -1 && depth > @option.depth)
|
98
157
|
return
|
99
158
|
end
|
@@ -119,13 +178,13 @@ module Gren
|
|
119
178
|
# ファイルならば中身を探索、ディレクトリならば再帰
|
120
179
|
case File.ftype(fpath)
|
121
180
|
when "directory"
|
122
|
-
|
181
|
+
searchFromDir(stdout, fpath, depth + 1)
|
123
182
|
when "file"
|
124
183
|
searchFile(stdout, fpath, fpath_disp)
|
125
184
|
end
|
126
185
|
end
|
127
186
|
end
|
128
|
-
private :
|
187
|
+
private :searchFromDir
|
129
188
|
|
130
189
|
def print_fpaths(stdout, data)
|
131
190
|
stdout.print data.join("\n")
|
@@ -137,7 +196,7 @@ module Gren
|
|
137
196
|
|
138
197
|
def ignoreDir?(fpath)
|
139
198
|
FileTest.directory?(fpath) &&
|
140
|
-
(
|
199
|
+
(GrenFileTest::ignoreDir?(File.basename(fpath)) || ignoreDirUser?(fpath))
|
141
200
|
end
|
142
201
|
private :ignoreDir?
|
143
202
|
|
@@ -148,9 +207,9 @@ module Gren
|
|
148
207
|
|
149
208
|
def ignoreFile?(fpath)
|
150
209
|
!correctFileUser?(fpath) ||
|
151
|
-
|
210
|
+
GrenFileTest::ignoreFile?(fpath) ||
|
152
211
|
ignoreFileUser?(fpath) ||
|
153
|
-
binary?(fpath)
|
212
|
+
GrenFileTest::binary?(fpath)
|
154
213
|
end
|
155
214
|
private :ignoreFile?
|
156
215
|
|
@@ -165,11 +224,6 @@ module Gren
|
|
165
224
|
end
|
166
225
|
private :ignoreFileUser?
|
167
226
|
|
168
|
-
def binary?(file)
|
169
|
-
s = File.read(file, 1024) or return false
|
170
|
-
return s.index("\x00")
|
171
|
-
end
|
172
|
-
|
173
227
|
def searchFile(stdout, fpath, fpath_disp)
|
174
228
|
@result.count += 1
|
175
229
|
@result.size += FileTest.size(fpath)
|
@@ -194,12 +248,12 @@ module Gren
|
|
194
248
|
|
195
249
|
if ( result )
|
196
250
|
header = "#{fpath_disp}:#{index + 1}:"
|
197
|
-
line =
|
251
|
+
line = GrenSnip::snip(line, match_datas) unless (@option.noSnip)
|
198
252
|
|
199
253
|
unless (@option.colorHighlight)
|
200
254
|
stdout.puts header + line
|
201
255
|
else
|
202
|
-
stdout.puts HighLine::BLUE + header + HighLine::CLEAR +
|
256
|
+
stdout.puts HighLine::BLUE + header + HighLine::CLEAR + GrenSnip::coloring(line, match_datas)
|
203
257
|
end
|
204
258
|
|
205
259
|
unless match_file
|
data/lib/gren.rb
CHANGED
data/lib/gren/cli.rb
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
2
|
require 'optparse'
|
3
|
-
require File.join(File.dirname(__FILE__), 'findgrep')
|
3
|
+
require File.join(File.dirname(__FILE__), '../findgrep/findgrep')
|
4
4
|
|
5
5
|
module Gren
|
6
6
|
class CLI
|
7
7
|
def self.execute(stdout, arguments=[])
|
8
8
|
# オプション
|
9
|
-
option = FindGrep::DEFAULT_OPTION
|
9
|
+
option = FindGrep::FindGrep::DEFAULT_OPTION
|
10
10
|
|
11
11
|
# オプション解析
|
12
12
|
opt = OptionParser.new("#{File.basename($0)} [option] pattern")
|
@@ -28,7 +28,7 @@ module Gren
|
|
28
28
|
|
29
29
|
# 検索オブジェクトの生成
|
30
30
|
if (arguments.size > 0 || option.keywordsOr.size > 0)
|
31
|
-
findGrep = FindGrep.new(arguments, option)
|
31
|
+
findGrep = FindGrep::FindGrep.new(arguments, option)
|
32
32
|
findGrep.searchAndPrint(stdout)
|
33
33
|
else
|
34
34
|
stdout.print opt.help
|
data/lib/grendb/cli.rb
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
require 'optparse'
|
3
|
+
require File.join(File.dirname(__FILE__), '../findgrep/findgrep')
|
4
|
+
|
5
|
+
module Grendb
|
6
|
+
class CLI
|
7
|
+
def self.execute(stdout, arguments=[])
|
8
|
+
# オプション
|
9
|
+
option = FindGrep::FindGrep::DEFAULT_OPTION
|
10
|
+
option.dbFile = ENV['GRENDB_DEFAULT_DB']
|
11
|
+
|
12
|
+
# オプション解析
|
13
|
+
opt = OptionParser.new("#{File.basename($0)} [option] keyword1 [keyword2 ...]")
|
14
|
+
opt.on('--db [GREN_DB_FILE]', 'Search from the grendb database.') {|v| option.dbFile = v }
|
15
|
+
opt.on('-f KEYWORD', '--file-keyword KEYWORD', 'Path keyword. (Enable multiple call)') {|v| option.filePatterns << v}
|
16
|
+
opt.on('-i', '--ignore', 'Ignore case.') {|v| option.ignoreCase = true}
|
17
|
+
opt.on('-c', '--color', 'Color highlight.') {|v| option.colorHighlight = true}
|
18
|
+
opt.on('--no-snip', 'There being a long line, it does not snip.') {|v| option.noSnip = true }
|
19
|
+
|
20
|
+
opt.parse!(arguments)
|
21
|
+
|
22
|
+
# 検索オブジェクトの生成
|
23
|
+
if (option.dbFile && (arguments.size > 0 || option.keywordsOr.size > 0))
|
24
|
+
findGrep = FindGrep::FindGrep.new(arguments, option)
|
25
|
+
findGrep.searchAndPrint(stdout)
|
26
|
+
else
|
27
|
+
stdout.print opt.help
|
28
|
+
stdout.puts
|
29
|
+
stdout.puts "please set GREN DATABSE FILE!! (--db option, or set ENV['GRENDB_DEFAULT_DB'].)" unless option.dbFile
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
data/lib/mkgrendb/cli.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
require 'optparse'
|
3
|
+
require File.join(File.dirname(__FILE__), 'mkgrendb')
|
4
|
+
|
5
|
+
module Mkgrendb
|
6
|
+
class CLI
|
7
|
+
def self.execute(stdout, arguments=[])
|
8
|
+
opt = OptionParser.new "#{File.basename($0)} INPUT_YAML1 [INPUT_YAML2 ...]"
|
9
|
+
opt.parse!(arguments)
|
10
|
+
|
11
|
+
if (arguments.size >= 1)
|
12
|
+
# @todo 複数個引数に対応
|
13
|
+
obj = Mkgrendb.new(arguments[0])
|
14
|
+
obj.update
|
15
|
+
else
|
16
|
+
stdout.puts opt.help
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,142 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
require 'pathname'
|
6
|
+
|
7
|
+
base_directory = Pathname(__FILE__).dirname + ".."
|
8
|
+
$LOAD_PATH.unshift((base_directory + "ext").to_s)
|
9
|
+
$LOAD_PATH.unshift((base_directory + "lib").to_s)
|
10
|
+
|
11
|
+
require 'rubygems'
|
12
|
+
require 'groonga'
|
13
|
+
|
14
|
+
require File.join(File.dirname(__FILE__), '../common/grenfiletest')
|
15
|
+
|
16
|
+
module Mkgrendb
|
17
|
+
class Mkgrendb
|
18
|
+
def initialize(input_yaml)
|
19
|
+
@output_db = input_yaml.sub(/\.yaml$/, ".db")
|
20
|
+
@src = YAML.load(open(input_yaml).read())
|
21
|
+
db_create(@output_db)
|
22
|
+
end
|
23
|
+
|
24
|
+
def update
|
25
|
+
db_open(@output_db)
|
26
|
+
|
27
|
+
@src["directory"].each do |dir|
|
28
|
+
db_add_dir(File.expand_path(dir))
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def db_create(filename)
|
33
|
+
dbfile = Pathname(File.expand_path(filename))
|
34
|
+
dbdir = dbfile.dirname
|
35
|
+
dbdir.mkpath unless dbdir.exist?
|
36
|
+
|
37
|
+
unless dbfile.exist?
|
38
|
+
Groonga::Database.create(:path => dbfile.to_s)
|
39
|
+
Groonga::Schema.define do |schema|
|
40
|
+
schema.create_table("documents") do |table|
|
41
|
+
table.string("path")
|
42
|
+
table.text("content")
|
43
|
+
end
|
44
|
+
|
45
|
+
schema.create_table("terms",
|
46
|
+
:type => :patricia_trie,
|
47
|
+
:key_normalize => true,
|
48
|
+
:default_tokenizer => "TokenBigram") do |table|
|
49
|
+
table.index("documents.path")
|
50
|
+
table.index("documents.content")
|
51
|
+
end
|
52
|
+
end
|
53
|
+
puts "create : #{dbfile.to_s} created."
|
54
|
+
else
|
55
|
+
puts "message : #{dbfile.to_s} already exist."
|
56
|
+
end
|
57
|
+
end
|
58
|
+
private :db_create
|
59
|
+
|
60
|
+
def db_open(filename)
|
61
|
+
dbfile = Pathname(File.expand_path(filename))
|
62
|
+
|
63
|
+
if dbfile.exist?
|
64
|
+
Groonga::Database.open(dbfile.to_s)
|
65
|
+
puts "open : #{dbfile} open."
|
66
|
+
else
|
67
|
+
raise "error : #{dbfile.to_s} not found!!"
|
68
|
+
end
|
69
|
+
end
|
70
|
+
private :db_open
|
71
|
+
|
72
|
+
def db_add_dir(dirname)
|
73
|
+
searchDirectory(STDOUT, dirname, 0)
|
74
|
+
end
|
75
|
+
private :db_add_dir
|
76
|
+
|
77
|
+
def db_add_file(stdout, filename)
|
78
|
+
# 格納するデータ
|
79
|
+
values = {
|
80
|
+
:path => filename,
|
81
|
+
:content => open(filename).read
|
82
|
+
}
|
83
|
+
|
84
|
+
# 既に登録されているファイルならばそれを上書き、そうでなければ新規レコードを作成
|
85
|
+
documents = Groonga::Context.default["documents"]
|
86
|
+
_documents = documents.select do |record|
|
87
|
+
record["path"] == values[:path]
|
88
|
+
end
|
89
|
+
|
90
|
+
if _documents.size.zero?
|
91
|
+
document = documents.add
|
92
|
+
else
|
93
|
+
document = _documents.to_a[0].key
|
94
|
+
end
|
95
|
+
|
96
|
+
# データベースに格納
|
97
|
+
values.each do |key, value|
|
98
|
+
puts value if (key == :path)
|
99
|
+
document[key] = value
|
100
|
+
end
|
101
|
+
|
102
|
+
end
|
103
|
+
|
104
|
+
def searchDirectory(stdout, dir, depth)
|
105
|
+
Dir.foreach(dir) do |name|
|
106
|
+
next if (name == '.' || name == '..')
|
107
|
+
|
108
|
+
fpath = File.join(dir,name)
|
109
|
+
fpath_disp = fpath.gsub(/^.\//, "")
|
110
|
+
|
111
|
+
# 除外ディレクトリならばパス
|
112
|
+
next if ignoreDir?(fpath)
|
113
|
+
|
114
|
+
# 読み込み不可ならばパス
|
115
|
+
next unless FileTest.readable?(fpath)
|
116
|
+
|
117
|
+
# ファイルならば中身を探索、ディレクトリならば再帰
|
118
|
+
case File.ftype(fpath)
|
119
|
+
when "directory"
|
120
|
+
searchDirectory(stdout, fpath, depth + 1)
|
121
|
+
when "file"
|
122
|
+
unless ignoreFile?(fpath)
|
123
|
+
db_add_file(stdout, fpath)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def ignoreDir?(fpath)
|
130
|
+
FileTest.directory?(fpath) &&
|
131
|
+
GrenFileTest::ignoreDir?(fpath)
|
132
|
+
end
|
133
|
+
private :ignoreDir?
|
134
|
+
|
135
|
+
def ignoreFile?(fpath)
|
136
|
+
GrenFileTest::ignoreFile?(fpath) ||
|
137
|
+
GrenFileTest::binary?(fpath)
|
138
|
+
end
|
139
|
+
private :ignoreFile?
|
140
|
+
|
141
|
+
end
|
142
|
+
end
|
data/test/test_gren_util.rb
CHANGED
@@ -1,32 +1,32 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/test_helper.rb'
|
2
2
|
|
3
|
-
class
|
3
|
+
class TestGrenSnip < Test::Unit::TestCase
|
4
4
|
def setup
|
5
5
|
end
|
6
6
|
|
7
7
|
def test_snip
|
8
8
|
str = "abcdefghijkmlnopqrstuvwxyz"
|
9
|
-
assert_equal(str,
|
9
|
+
assert_equal(str, GrenSnip::snip(str, nil))
|
10
10
|
|
11
11
|
str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|12345678901234567890123456789012345678901234567890123456"
|
12
|
-
assert_equal(str,
|
12
|
+
assert_equal(str, GrenSnip::snip(str, nil))
|
13
13
|
|
14
14
|
str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234561"
|
15
15
|
match_datas = []
|
16
16
|
match_datas << str.match(/123456789\|/)
|
17
|
-
assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>67890123456789012345678901234561",
|
17
|
+
assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>67890123456789012345678901234561", GrenSnip::snip(str, match_datas))
|
18
18
|
|
19
19
|
str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234561"
|
20
20
|
match_datas = []
|
21
21
|
match_datas << str.match(/123456789\|/)
|
22
22
|
match_datas << str.match(/34567/)
|
23
|
-
assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>67890123456789012345678901234561",
|
23
|
+
assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>67890123456789012345678901234561", GrenSnip::snip(str, match_datas))
|
24
24
|
|
25
25
|
str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234561"
|
26
26
|
match_datas = []
|
27
27
|
match_datas << str.match(/123456789\|/)
|
28
28
|
match_datas << str.match(/01234561/)
|
29
|
-
assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>8901234567890123456789012345678901234561",
|
29
|
+
assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>8901234567890123456789012345678901234561", GrenSnip::snip(str, match_datas))
|
30
30
|
|
31
31
|
end
|
32
32
|
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "test_helper.rb")
|
2
|
+
require 'grendb/cli'
|
3
|
+
|
4
|
+
class TestGrendbCli < Test::Unit::TestCase
|
5
|
+
def setup
|
6
|
+
Grendb::CLI.execute(@stdout_io = StringIO.new, [])
|
7
|
+
@stdout_io.rewind
|
8
|
+
@stdout = @stdout_io.read
|
9
|
+
end
|
10
|
+
|
11
|
+
def test_print_default_output
|
12
|
+
# assert_match(/To update this executable/, @stdout)
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), "test_helper.rb")
|
2
|
+
require 'mkgrendb/cli'
|
3
|
+
|
4
|
+
class TestMkgrendbCli < Test::Unit::TestCase
|
5
|
+
def setup
|
6
|
+
Mkgrendb::CLI.execute(@stdout_io = StringIO.new, [])
|
7
|
+
@stdout_io.rewind
|
8
|
+
@stdout = @stdout_io.read
|
9
|
+
end
|
10
|
+
|
11
|
+
def test_print_default_output
|
12
|
+
# assert_match(/To update this executable/, @stdout)
|
13
|
+
end
|
14
|
+
end
|
metadata
CHANGED
@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
|
|
4
4
|
prerelease: false
|
5
5
|
segments:
|
6
6
|
- 0
|
7
|
-
-
|
8
|
-
-
|
9
|
-
version: 0.
|
7
|
+
- 2
|
8
|
+
- 0
|
9
|
+
version: 0.2.0
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- ongaeshi
|
@@ -14,7 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2010-
|
17
|
+
date: 2010-09-13 00:00:00 +09:00
|
18
18
|
default_executable:
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
@@ -32,9 +32,23 @@ dependencies:
|
|
32
32
|
type: :runtime
|
33
33
|
version_requirements: *id001
|
34
34
|
- !ruby/object:Gem::Dependency
|
35
|
-
name:
|
35
|
+
name: rroonga
|
36
36
|
prerelease: false
|
37
37
|
requirement: &id002 !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
segments:
|
42
|
+
- 1
|
43
|
+
- 0
|
44
|
+
- 0
|
45
|
+
version: 1.0.0
|
46
|
+
type: :runtime
|
47
|
+
version_requirements: *id002
|
48
|
+
- !ruby/object:Gem::Dependency
|
49
|
+
name: rubyforge
|
50
|
+
prerelease: false
|
51
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
38
52
|
requirements:
|
39
53
|
- - ">="
|
40
54
|
- !ruby/object:Gem::Version
|
@@ -44,11 +58,11 @@ dependencies:
|
|
44
58
|
- 4
|
45
59
|
version: 2.0.4
|
46
60
|
type: :development
|
47
|
-
version_requirements: *
|
61
|
+
version_requirements: *id003
|
48
62
|
- !ruby/object:Gem::Dependency
|
49
63
|
name: hoe
|
50
64
|
prerelease: false
|
51
|
-
requirement: &
|
65
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
52
66
|
requirements:
|
53
67
|
- - ">="
|
54
68
|
- !ruby/object:Gem::Version
|
@@ -58,12 +72,14 @@ dependencies:
|
|
58
72
|
- 1
|
59
73
|
version: 2.6.1
|
60
74
|
type: :development
|
61
|
-
version_requirements: *
|
75
|
+
version_requirements: *id004
|
62
76
|
description: gren is a next grep tool.
|
63
77
|
email:
|
64
78
|
- ongaeshi0621@gmail.com
|
65
79
|
executables:
|
66
80
|
- gren
|
81
|
+
- grendb
|
82
|
+
- mkgrendb
|
67
83
|
extensions: []
|
68
84
|
|
69
85
|
extra_rdoc_files:
|
@@ -78,13 +94,19 @@ files:
|
|
78
94
|
- README.rdoc
|
79
95
|
- Rakefile
|
80
96
|
- bin/gren
|
97
|
+
- bin/grendb
|
98
|
+
- bin/mkgrendb
|
99
|
+
- lib/common/grenfiletest.rb
|
100
|
+
- lib/common/grensnip.rb
|
101
|
+
- lib/common/platform.rb
|
102
|
+
- lib/common/string_snip.rb
|
103
|
+
- lib/findgrep/findgrep.rb
|
104
|
+
- lib/findgrep/result.rb
|
81
105
|
- lib/gren.rb
|
82
106
|
- lib/gren/cli.rb
|
83
|
-
- lib/
|
84
|
-
- lib/
|
85
|
-
- lib/
|
86
|
-
- lib/platform.rb
|
87
|
-
- lib/string_snip.rb
|
107
|
+
- lib/grendb/cli.rb
|
108
|
+
- lib/mkgrendb/cli.rb
|
109
|
+
- lib/mkgrendb/mkgrendb.rb
|
88
110
|
- rake_rdoc_custom.rb
|
89
111
|
- script/console
|
90
112
|
- script/destroy
|
@@ -92,7 +114,9 @@ files:
|
|
92
114
|
- test/test_gren.rb
|
93
115
|
- test/test_gren_cli.rb
|
94
116
|
- test/test_gren_util.rb
|
117
|
+
- test/test_grendb_cli.rb
|
95
118
|
- test/test_helper.rb
|
119
|
+
- test/test_mkgrendb_cli.rb
|
96
120
|
- test/test_string_snip.rb
|
97
121
|
has_rdoc: true
|
98
122
|
homepage: http://ongaeshi.github.com/gren
|
@@ -129,5 +153,7 @@ test_files:
|
|
129
153
|
- test/test_gren.rb
|
130
154
|
- test/test_gren_cli.rb
|
131
155
|
- test/test_gren_util.rb
|
156
|
+
- test/test_grendb_cli.rb
|
132
157
|
- test/test_helper.rb
|
158
|
+
- test/test_mkgrendb_cli.rb
|
133
159
|
- test/test_string_snip.rb
|
data/lib/gren/util.rb
DELETED
@@ -1,39 +0,0 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
|
-
|
3
|
-
require File.join(File.dirname(__FILE__), '../string_snip')
|
4
|
-
|
5
|
-
module Gren
|
6
|
-
class Util
|
7
|
-
MAX_LINE_SIZE = 256
|
8
|
-
HEADER_SIZE = 32
|
9
|
-
MARGIN_SIZE = 32
|
10
|
-
DELIMITER = '<<snip>>'
|
11
|
-
|
12
|
-
def self.snip(str, match_datas)
|
13
|
-
return str if (str.size <= MAX_LINE_SIZE)
|
14
|
-
|
15
|
-
ranges = []
|
16
|
-
ranges << (0..HEADER_SIZE-1)
|
17
|
-
ranges << (-HEADER_SIZE..-1)
|
18
|
-
|
19
|
-
match_datas.each do |m|
|
20
|
-
ranges << (m.begin(0)-MARGIN_SIZE..m.end(0)+MARGIN_SIZE)
|
21
|
-
end
|
22
|
-
|
23
|
-
snipper = StringSnip.new(MAX_LINE_SIZE, DELIMITER)
|
24
|
-
return snipper.snip(str, ranges)
|
25
|
-
end
|
26
|
-
|
27
|
-
def self.coloring(line, match_datas)
|
28
|
-
match_datas.each do |m|
|
29
|
-
line = line.split(m[0]).join(HighLine::ON_GREEN + m[0] + HighLine::CLEAR)
|
30
|
-
end
|
31
|
-
|
32
|
-
line = line.split(DELIMITER).join(HighLine::ON_CYAN + DELIMITER + HighLine::CLEAR)
|
33
|
-
|
34
|
-
line
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
|