codestock 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. data/.document +5 -0
  2. data/Gemfile +12 -0
  3. data/Gemfile.lock +18 -0
  4. data/LICENSE.txt +20 -0
  5. data/README.rdoc +19 -0
  6. data/Rakefile +63 -0
  7. data/VERSION +1 -0
  8. data/bin/cdstk +12 -0
  9. data/bin/cdv +11 -0
  10. data/bin/cdview +11 -0
  11. data/bin/cdweb +11 -0
  12. data/codestock.gemspec +131 -0
  13. data/lib/codestock.rb +0 -0
  14. data/lib/common/dbdir.rb +37 -0
  15. data/lib/common/display_util.rb +62 -0
  16. data/lib/common/grenfiletest.rb +19 -0
  17. data/lib/common/grensnip.rb +37 -0
  18. data/lib/common/platform.rb +17 -0
  19. data/lib/common/string_snip.rb +61 -0
  20. data/lib/common/util.rb +98 -0
  21. data/lib/findgrep/findgrep.rb +408 -0
  22. data/lib/findgrep/result.rb +43 -0
  23. data/lib/gren/cli.rb +65 -0
  24. data/lib/gren.rb +6 -0
  25. data/lib/grendb/cli.rb +43 -0
  26. data/lib/grenweb/cli.rb +54 -0
  27. data/lib/grenweb/database.rb +182 -0
  28. data/lib/grenweb/grenweb.ru +35 -0
  29. data/lib/grenweb/grep.rb +52 -0
  30. data/lib/grenweb/help.rb +40 -0
  31. data/lib/grenweb/home.rb +40 -0
  32. data/lib/grenweb/html_renderer.rb +244 -0
  33. data/lib/grenweb/public/css/gren.css +63 -0
  34. data/lib/grenweb/public/images/gren-icon-mini.png +0 -0
  35. data/lib/grenweb/public/images/gren-icon.png +0 -0
  36. data/lib/grenweb/query.rb +82 -0
  37. data/lib/grenweb/searcher.rb +130 -0
  38. data/lib/grenweb/viewer.rb +52 -0
  39. data/lib/mkgrendb/cli.rb +89 -0
  40. data/lib/mkgrendb/cli_old.rb +49 -0
  41. data/lib/mkgrendb/grendbyaml.rb +76 -0
  42. data/lib/mkgrendb/mkgrendb.rb +296 -0
  43. data/test/file_test_utils.rb +59 -0
  44. data/test/runner.rb +11 -0
  45. data/test/test_dbdir.rb +59 -0
  46. data/test/test_gren.rb +10 -0
  47. data/test/test_gren_util.rb +34 -0
  48. data/test/test_grendbyaml.rb +109 -0
  49. data/test/test_grenweb_cli.rb +10 -0
  50. data/test/test_grenweb_database.rb +37 -0
  51. data/test/test_grenweb_html_renderer.rb +41 -0
  52. data/test/test_grenweb_query.rb +54 -0
  53. data/test/test_grenweb_searcher.rb +35 -0
  54. data/test/test_helper.rb +2 -0
  55. data/test/test_mkgrendb.rb +163 -0
  56. data/test/test_string_snip.rb +31 -0
  57. metadata +229 -0
@@ -0,0 +1,182 @@
1
+ # -*- coding: utf-8 -*-
2
+ #
3
+ # @file
4
+ # @brief Grenwebで使用するデータベース
5
+ # @author ongaeshi
6
+ # @date 2010/10/17
7
+
8
+ require 'rubygems'
9
+ require 'pathname'
10
+ require 'singleton'
11
+ require 'groonga'
12
+ require File.join(File.dirname(__FILE__), "../common/dbdir")
13
+
14
+ module Grenweb
15
+ class Database
16
+ include Singleton
17
+ include CodeStock
18
+
19
+ def self.setup(db_dir)
20
+ @@db_dir = db_dir
21
+ end
22
+
23
+ def initialize
24
+ open(@@db_dir)
25
+ end
26
+
27
+ def open(db_dir)
28
+ dbfile = db_expand_groonga_path(db_dir)
29
+
30
+ if File.exist? dbfile
31
+ Groonga::Database.open(dbfile)
32
+ else
33
+ raise "error : #{dbfile} not found!!"
34
+ end
35
+
36
+ @documents = Groonga::Context.default["documents"]
37
+ end
38
+
39
+ def record(shortpath)
40
+ before = Time.now
41
+ table = @documents.select { |record| record.shortpath == shortpath }
42
+ elapsed = Time.now - before
43
+ return table.records[0], elapsed
44
+ end
45
+
46
+ def fileNum
47
+ @documents.select.size
48
+ end
49
+
50
+ def search(patterns, packages, fpaths, suffixs, page = 0, limit = -1)
51
+ before = Time.now
52
+
53
+ # 全てのパターンを検索
54
+ if (fpaths.include?("*"))
55
+ records, total_records = selectAll(page, limit)
56
+ else
57
+ records, total_records = searchMain(patterns, packages, fpaths, suffixs, page, limit)
58
+ end
59
+
60
+ # 検索にかかった時間
61
+ elapsed = Time.now - before
62
+
63
+ # 結果
64
+ return records, total_records, elapsed
65
+ end
66
+
67
+ def selectAll(page, limit)
68
+ table = @documents.select
69
+
70
+ # マッチ数
71
+ total_records = table.size
72
+
73
+ # 2010/10/29 ongaeshi
74
+ # 本当はこのようにgroongaAPIでソートしたいのだが上手くいかなかった
75
+ # # ファイル名順にソート
76
+ # records = table.sort([{:key => "shortpath", :order => "descending"}],
77
+ # :offset => page * limit,
78
+ # :limit => limit)
79
+
80
+ # ソート
81
+ records = table.records.sort_by{|record| record.shortpath.downcase }[page * limit, limit]
82
+
83
+ return records, total_records
84
+ end
85
+
86
+ def searchMain(patterns, packages, fpaths, suffixs, page, limit)
87
+ table = @documents.select do |record|
88
+ expression = nil
89
+
90
+ # キーワード
91
+ patterns.each do |word|
92
+ sub_expression = record.content =~ word
93
+ if expression.nil?
94
+ expression = sub_expression
95
+ else
96
+ expression &= sub_expression
97
+ end
98
+ end
99
+
100
+ # パッケージ(OR)
101
+ pe = package_expression(record, packages)
102
+ if (pe)
103
+ if expression.nil?
104
+ expression = pe
105
+ else
106
+ expression &= pe
107
+ end
108
+ end
109
+
110
+ # ファイルパス
111
+ fpaths.each do |word|
112
+ sub_expression = record.path =~ word
113
+ if expression.nil?
114
+ expression = sub_expression
115
+ else
116
+ expression &= sub_expression
117
+ end
118
+ end
119
+
120
+ # 拡張子(OR)
121
+ se = suffix_expression(record, suffixs)
122
+ if (se)
123
+ if expression.nil?
124
+ expression = se
125
+ else
126
+ expression &= se
127
+ end
128
+ end
129
+
130
+ # 検索式
131
+ expression
132
+ end
133
+
134
+ # マッチ数
135
+ total_records = table.size
136
+
137
+ # スコアとタイムスタンプでソート
138
+ records = table.sort([{:key => "_score", :order => "descending"},
139
+ {:key => "timestamp", :order => "descending"}],
140
+ :offset => page * limit,
141
+ :limit => limit)
142
+
143
+ return records, total_records
144
+ end
145
+ private :searchMain
146
+
147
+ def package_expression(record, packages)
148
+ sub = nil
149
+
150
+ # @todo 専用カラム package が欲しいところ
151
+ # でも今でもpackageはORとして機能してるからいいっちゃいい
152
+ packages.each do |word|
153
+ e = record.path =~ word
154
+ if sub.nil?
155
+ sub = e
156
+ else
157
+ sub |= e
158
+ end
159
+ end
160
+
161
+ sub
162
+ end
163
+ private :package_expression
164
+
165
+ def suffix_expression(record, suffixs)
166
+ sub = nil
167
+
168
+ suffixs.each do |word|
169
+ e = record.suffix =~ word
170
+ if sub.nil?
171
+ sub = e
172
+ else
173
+ sub |= e
174
+ end
175
+ end
176
+
177
+ sub
178
+ end
179
+ private :suffix_expression
180
+
181
+ end
182
+ end
@@ -0,0 +1,35 @@
1
+ # -*- mode: ruby; coding: utf-8 -*-
2
+ #
3
+ # @file
4
+ # @brief gren web検索
5
+ # @author ongaeshi
6
+ # @date 2010/10/13
7
+
8
+ require 'rubygems'
9
+ require 'rack'
10
+ require File.join(File.dirname(__FILE__), 'home')
11
+ require File.join(File.dirname(__FILE__), 'searcher')
12
+ require File.join(File.dirname(__FILE__), 'viewer')
13
+ require File.join(File.dirname(__FILE__), 'help')
14
+
15
+ use Rack::CommonLogger
16
+ use Rack::Runtime
17
+ use Rack::Static, :urls => ["/css", "/images"], :root => "public"
18
+ use Rack::ContentLength
19
+
20
+ map '/' do
21
+ run Grenweb::Home.new
22
+ end
23
+
24
+ map '/::search' do
25
+ run Grenweb::Searcher.new
26
+ end
27
+
28
+ map '/::view' do
29
+ run Grenweb::Viewer.new
30
+ end
31
+
32
+ map '/::help' do
33
+ run Grenweb::Help.new
34
+ end
35
+
@@ -0,0 +1,52 @@
1
+ # -*- coding: utf-8 -*-
2
+ #
3
+ # @file
4
+ # @brief grenwebで使用する行指向の検索
5
+ # @author ongaeshi
6
+ # @date 2010/10/18
7
+
8
+ module Grenweb
9
+ class Grep
10
+ attr_reader :content
11
+
12
+ def initialize(content)
13
+ @content = content.split("\n")
14
+ end
15
+
16
+ MatchLineResult = Struct.new(:index, :match_datas)
17
+
18
+ def match_lines_or(patterns)
19
+ result = []
20
+ patternRegexps = strs2regs(patterns, true) # @todo ignoreオプションを付ける
21
+
22
+ @content.each_with_index do |line, index|
23
+ match_datas = []
24
+ patternRegexps.each {|v| match_datas << v.match(line)}
25
+
26
+ if (match_datas.any?)
27
+ result << MatchLineResult.new(index, match_datas)
28
+ end
29
+ end
30
+
31
+ result
32
+ end
33
+
34
+ def context(result, num)
35
+ end
36
+
37
+ private
38
+
39
+ def strs2regs(strs, ignore = false)
40
+ regs = []
41
+
42
+ strs.each do |v|
43
+ option = 0
44
+ option |= Regexp::IGNORECASE if (ignore)
45
+ regs << Regexp.new(v, option)
46
+ end
47
+
48
+ regs
49
+ end
50
+ end
51
+ end
52
+
@@ -0,0 +1,40 @@
1
+ # -*- coding: utf-8 -*-
2
+ #
3
+ # @file
4
+ # @brief ホーム画面
5
+ # @author ongaeshi
6
+ # @date 2010/10/13
7
+
8
+ require 'rubygems'
9
+ require 'rack'
10
+ require File.join(File.dirname(__FILE__), 'database')
11
+ require File.join(File.dirname(__FILE__), 'html_renderer')
12
+ require File.join(File.dirname(__FILE__), 'query')
13
+
14
+ module Grenweb
15
+ class Help
16
+ include Rack::Utils
17
+
18
+ def call(env)
19
+ @env = env
20
+ @request = Rack::Request.new(env)
21
+ @query = Query.new(@request)
22
+
23
+ @response = Rack::Response.new
24
+ @response["Content-Type"] = "text/html; charset=UTF-8"
25
+
26
+ render
27
+ end
28
+
29
+ private
30
+
31
+ def render
32
+ r = HTMLRendeler.new(@request.script_name + '/..')
33
+ @response.write r.header("gren - help", "gren - help")
34
+ @response.write r.sample_code
35
+ @response.write r.footer
36
+ @response.to_a
37
+ end
38
+ end
39
+ end
40
+
@@ -0,0 +1,40 @@
1
+ # -*- coding: utf-8 -*-
2
+ #
3
+ # @file
4
+ # @brief ホーム画面
5
+ # @author ongaeshi
6
+ # @date 2010/10/13
7
+
8
+ require 'rubygems'
9
+ require 'rack'
10
+ require File.join(File.dirname(__FILE__), 'database')
11
+ require File.join(File.dirname(__FILE__), 'html_renderer')
12
+ require File.join(File.dirname(__FILE__), 'query')
13
+
14
+ module Grenweb
15
+ class Home
16
+ include Rack::Utils
17
+
18
+ def call(env)
19
+ @env = env
20
+ @request = Rack::Request.new(env)
21
+ @query = Query.new(@request)
22
+
23
+ @response = Rack::Response.new
24
+ @response["Content-Type"] = "text/html; charset=UTF-8"
25
+
26
+ render
27
+ end
28
+
29
+ private
30
+
31
+ def render
32
+ r = HTMLRendeler.new(@request.script_name)
33
+ @response.write r.header_home("gren", "gren", Version)
34
+ @response.write r.search_box
35
+ @response.write r.footer_home("??", Database.instance.fileNum)
36
+ @response.to_a
37
+ end
38
+ end
39
+ end
40
+
@@ -0,0 +1,244 @@
1
+ # -*- coding: utf-8 -*-
2
+ #
3
+ # @file
4
+ # @brief HTMLの描画ルーチン
5
+ # @author ongaeshi
6
+ # @date 2010/10/17
7
+
8
+ require 'rubygems'
9
+ require 'pathname'
10
+ require 'rack'
11
+ require File.join(File.dirname(__FILE__), 'grep')
12
+
13
+ module Grenweb
14
+ class HTMLRendeler
15
+ include Rack::Utils
16
+
17
+ def initialize(script_name)
18
+ @script_name = Pathname(script_name)
19
+ end
20
+
21
+ def header(title, header1)
22
+ <<EOS
23
+ <?xml version="1.0" encoding="utf-8"?>
24
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
25
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
26
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="ja" lang="ja">
27
+ <head>
28
+ <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
29
+ <!-- <meta name="robot" content="noindex,nofollow" /> -->
30
+ <title>#{title}</title>
31
+ <link rel="stylesheet" href="#{fullpath('css/gren.css')}" type="text/css" media="all" />
32
+ </head>
33
+ <body>
34
+ <div class="header">
35
+ <h1>
36
+ <a href="#{fullpath('')}"><img src="#{fullpath('images/gren-icon-mini.png')}" alt="gren-icon" border="0"/></a>
37
+ #{header1}
38
+ </h1>
39
+ </div>
40
+
41
+ <div class="content">
42
+ EOS
43
+ end
44
+
45
+ def footer
46
+ <<EOS
47
+ </div>
48
+
49
+ <div class="footer">
50
+ </div>
51
+ </body>
52
+ </html>
53
+ EOS
54
+ end
55
+
56
+ def header_home(title, header1, version)
57
+ <<EOS
58
+ <?xml version="1.0" encoding="utf-8"?>
59
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
60
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
61
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="ja" lang="ja">
62
+ <head>
63
+ <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
64
+ <!-- <meta name="robot" content="noindex,nofollow" /> -->
65
+ <title>#{title}</title>
66
+ <link rel="stylesheet" href="#{fullpath('css/gren.css')}" type="text/css" media="all" />
67
+ </head>
68
+ <body>
69
+ <div align="center">
70
+ <div class="header_home">
71
+ <h1>
72
+ <a href="#{fullpath('')}"><img src="#{fullpath('images/gren-icon.png')}" alt="gren-icon" border="0" height="100px"/></a>
73
+ #{header1} <font class="version">#{version}</font>
74
+ </h1>
75
+ </div>
76
+
77
+ <div class="content">
78
+ EOS
79
+ end
80
+
81
+ def footer_home(package, files)
82
+ <<EOS
83
+ </div>
84
+
85
+ <div class="footer_home">
86
+ <!-- <a href="#{fullpath('::search/p:*')}">#{package}</a>のパッケージ , -->
87
+ <a href="#{fullpath('::search/f:*')}">#{files}</a>のファイル<br>
88
+ <a href="#{fullpath('::help')}">ヘルプ</a> ,
89
+ <a href="http://ongaeshi.github.com/gren">grenについて</a>
90
+ </div>
91
+ </div>
92
+ </body>
93
+ </html>
94
+ EOS
95
+ end
96
+
97
+ def result_record(record, patterns, nth=1)
98
+ if (patterns.size > 0)
99
+ <<EOS
100
+ <dt class='result-record'><a href='#{fullpath("::view/" + Rack::Utils::escape_html(record.shortpath))}'>#{record.shortpath}</a></dt>
101
+ <dd>
102
+ <pre class='lines'>
103
+ #{result_record_match_line(record, patterns, nth)}
104
+ </pre>
105
+ </dd>
106
+ EOS
107
+ else
108
+ <<EOS
109
+ <dt class='result-record'><a href='#{fullpath("::view/" + Rack::Utils::escape_html(record.shortpath))}'>#{record.shortpath}</a></dt>
110
+ EOS
111
+ end
112
+ end
113
+
114
+ def result_record_match_line(record, patterns, nth)
115
+ str = ""
116
+
117
+ grep = Grep.new(record.content)
118
+ lines = grep.match_lines_or(patterns)
119
+
120
+ unless (lines.empty?)
121
+ index = lines[0].index
122
+
123
+ (index - nth..index + nth).each do |i|
124
+ if (0 <= i && i < grep.content.size)
125
+ match_datas = (i == index) ? lines[0].match_datas : []
126
+ str << line(i + 1, grep.content[i], match_datas) + "\n"
127
+ end
128
+ end
129
+ end
130
+
131
+ str
132
+ end
133
+
134
+ def record_content(record)
135
+ <<EOS
136
+ <pre>
137
+ #{record_content_line(record)}
138
+ </pre>
139
+ EOS
140
+ end
141
+
142
+ def record_content_line(record)
143
+ str = ""
144
+
145
+ grep = Grep.new(record.content)
146
+ grep.content.each_with_index do |l, index|
147
+ str << line(index + 1, l, []) + "\n"
148
+ end
149
+
150
+ str
151
+ end
152
+
153
+ def line(lineno, line, match_datas)
154
+ sprintf("%5d: %s", lineno, match_strong(Rack::Utils::escape_html(line), match_datas))
155
+ end
156
+
157
+ def match_strong(line, match_datas)
158
+ match_datas.each do |m|
159
+ line = line.split(m[0]).join('<strong>' + m[0] + '</strong>') unless (m.nil?)
160
+ end
161
+
162
+ line
163
+ end
164
+
165
+ def pagination_link(page, label)
166
+ href = "?page=#{page}"
167
+ pagination_span("<a href='#{href}'>#{label}</a>")
168
+ end
169
+
170
+ def pagination_span(content)
171
+ "<span class='pagination-link'>#{content}</span>\n"
172
+ end
173
+
174
+ def empty_summary()
175
+ <<EOS
176
+ <div class='search-summary'>
177
+ <p>gren web検索</p>
178
+ </div>
179
+ EOS
180
+ end
181
+
182
+ def search_summary(keyword, total_records, range, elapsed)
183
+ <<EOS
184
+ <div class='search-summary'>
185
+ <span class="keyword">#{keyword}</span>の検索結果:
186
+ <span class="total-entries">#{total_records}</span>件中
187
+ <span class="display-range">#{range.first} - #{range.last}</span>件(#{elapsed}秒)
188
+ </div>
189
+ EOS
190
+ end
191
+
192
+ def view_summary(path, elapsed)
193
+ <<EOS
194
+ <div class='search-summary'>
195
+ <span class="keyword">#{path}</span>(#{elapsed}秒)
196
+ </div>
197
+ EOS
198
+ end
199
+
200
+ def search_box(text = "")
201
+ <<EOS
202
+ <form method="post" action="#{fullpath('::search')}">
203
+ <p>
204
+ <input name="query" type="text" size="60" value="#{text}" />
205
+ <input type="submit" value="検索" />
206
+ </p>
207
+ </form>
208
+ EOS
209
+ end
210
+
211
+ def sample_code
212
+ <<EOS
213
+ <div class='sample-code'>
214
+ <ol>
215
+ <li>キーワードで検索<br>
216
+ #{link('def open')}
217
+ <li>1フレーズとして検索<br>
218
+ #{link('"def open"')}
219
+ <li>パッケージ名で絞り込み<br>
220
+ #{link('def open p:gren')}
221
+ <li>ファイル名や拡張子で絞り込み<br>
222
+ #{link('def open f:test s:rb')}
223
+ <li>組み合わせ<br>
224
+ #{link('p:gren p:tidtools s:rb f:test assert f:cli')}
225
+ </ol>
226
+ </div>
227
+ EOS
228
+ end
229
+
230
+ def link(keyword)
231
+ "<a href='#{fullpath('::search' + '/' + Rack::Utils::escape_html(keyword))}'>#{keyword}</a>"
232
+ end
233
+
234
+ def fullpath(path)
235
+ unless (path == '')
236
+ (@script_name + path).to_s
237
+ else
238
+ @script_name.to_s
239
+ end
240
+ end
241
+ end
242
+ end
243
+
244
+
@@ -0,0 +1,63 @@
1
+ body {
2
+ }
3
+
4
+ div.header {
5
+ }
6
+
7
+ div.footer {
8
+ }
9
+
10
+ div.header_home {
11
+ font-size: 170%;
12
+ }
13
+
14
+ div.footer_home {
15
+ font-size: 85%;
16
+ margin-top: 3em;
17
+ }
18
+
19
+ div.search-summary {
20
+ border-color:#F4CA3F;
21
+ background-color:#FEFFF0;
22
+ border-top-style:solid;
23
+ border-top-width:1px;
24
+ border-bottom-style:solid;
25
+ border-bottom-width:1px;
26
+ margin-bottom:1em;
27
+ margin-top:1em;
28
+ padding-bottom:5px;
29
+ padding-top:5px;
30
+ padding-left:5px;
31
+ }
32
+
33
+ dt.result-record {
34
+ font-size: 105%;
35
+ }
36
+
37
+ pre.lines {
38
+ }
39
+
40
+ div.pagination {
41
+ background-color:#FEFFF0;
42
+ text-align: center;
43
+ margin-bottom: 20px;
44
+ }
45
+
46
+ pre {
47
+ border: 1px solid #dedede;
48
+ margin-bottom: 5px;
49
+ padding-left: 5px;
50
+ background-color: #fdfdfd;
51
+ font-weight: inherit;
52
+ margin-top: 5px;
53
+ padding-top: 5px;
54
+ padding-bottom: 5px;
55
+ }
56
+
57
+ span.keyword {
58
+ font-weight: bold;
59
+ }
60
+
61
+ font.version {
62
+ font-size: 35%;
63
+ }