milkode 0.2.4 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/HISTORY.ja.rdoc +16 -0
- data/HISTORY.rdoc +25 -0
- data/Rakefile +3 -2
- data/VERSION +1 -1
- data/bin/gmilk +10 -0
- data/bin/milk +1 -1
- data/lib/milkode/cdstk/cdstk.rb +121 -6
- data/lib/milkode/cdstk/cdstk_command.rb +28 -0
- data/lib/milkode/cdstk/cdstk_yaml.rb +30 -3
- data/lib/milkode/cdstk/cli_cdstk.rb +37 -14
- data/lib/milkode/cdstk/cli_cdstksub.rb +36 -1
- data/lib/milkode/cdweb/app.rb +1 -1
- data/lib/milkode/cdweb/lib/coderay_wrapper.rb +4 -5
- data/lib/milkode/cdweb/lib/database.rb +2 -0
- data/lib/milkode/cdweb/lib/my_nokogiri.rb +34 -0
- data/lib/milkode/common/dbdir.rb +30 -1
- data/lib/milkode/common/util.rb +15 -3
- data/lib/milkode/findgrep/findgrep.rb +86 -18
- data/lib/milkode/findgrep/result.rb +2 -2
- data/lib/milkode/grep/cli_grep.rb +113 -0
- data/milkode.gemspec +32 -9
- data/test/data/a_project/cdstk.rb +634 -0
- data/test/data/a_project/cdstk_yaml.rb +130 -0
- data/test/data/b_project/runner.rb +11 -0
- data/test/data/b_project/test_dir.rb +21 -0
- data/test/milkode_test_work.rb +78 -0
- data/test/test_cdstk.rb +12 -2
- data/test/test_cdstk_command.rb +50 -0
- data/test/test_cdstk_yaml.rb +20 -3
- data/test/test_cli_cdstk.rb +71 -0
- data/test/test_cli_grep.rb +79 -0
- data/test/test_database.rb +1 -1
- data/test/test_dbdir.rb +11 -0
- metadata +43 -8
@@ -0,0 +1,113 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
require 'milkode/findgrep/findgrep'
|
5
|
+
require 'milkode/common/dbdir'
|
6
|
+
require 'milkode/cdstk/cdstk_yaml'
|
7
|
+
require 'milkode/cdstk/cdstk'
|
8
|
+
|
9
|
+
module Milkode
|
10
|
+
class CLI_Grep
|
11
|
+
def self.execute(stdout, arguments=[])
|
12
|
+
option = FindGrep::FindGrep::DEFAULT_OPTION.dup
|
13
|
+
option.dbFile = Dbdir.groonga_path(Dbdir.default_dir)
|
14
|
+
option.isSilent = true
|
15
|
+
|
16
|
+
my_option = {}
|
17
|
+
my_option[:packages] = []
|
18
|
+
|
19
|
+
begin
|
20
|
+
current_dir = package_root_dir(File.expand_path("."))
|
21
|
+
rescue NotFoundPackage => e
|
22
|
+
current_dir = File.expand_path(".")
|
23
|
+
end
|
24
|
+
|
25
|
+
# opt = OptionParser.new "#{File.basename($0)} [option] pattern"
|
26
|
+
opt = OptionParser.new "gmilk [option] pattern" # @memo milk grep からも呼ばれるため
|
27
|
+
opt.on('-f KEYWORD', '--file-keyword KEYWORD', 'File path. (Enable multiple call)') {|v| option.filePatterns << v; my_option[:find_mode] = true }
|
28
|
+
opt.on('-d DIR', '--directory DIR', 'Start directory. (deafult:".")') {|v| current_dir = File.expand_path(v); my_option[:find_mode] = true}
|
29
|
+
opt.on('-s SUFFIX', '--suffix SUFFIX', 'Suffix.') {|v| option.suffixs << v; my_option[:find_mode] = true }
|
30
|
+
opt.on('-r', '--root', 'Search from package root.') {|v| current_dir = package_root_dir(File.expand_path(".")) }
|
31
|
+
opt.on('-p PACKAGE', '--package PACKAGE', 'Specify search package.') {|v| setup_package(option, my_option, v) }
|
32
|
+
opt.on('-a', '--all', 'Search all package.') {|v| my_option[:all] = true }
|
33
|
+
opt.on('-n NUM', 'Limits the number of match to show.') {|v| option.matchCountLimit = v.to_i }
|
34
|
+
opt.on('-i', '--ignore', 'Ignore case.') {|v| option.ignoreCase = true}
|
35
|
+
opt.on('--color', 'Color highlight.') {|v| option.colorHighlight = true}
|
36
|
+
opt.on('--no-snip', 'There being a long line, it does not snip.') {|v| option.noSnip = true }
|
37
|
+
opt.on('--cache', 'Search only db.') {|v| option.groongaOnly = true }
|
38
|
+
opt.on('--verbose', 'Set the verbose level of output.') {|v| option.isSilent = false }
|
39
|
+
opt.on('-u', '--update', 'With update db.') {|v| my_option[:update] = true }
|
40
|
+
begin
|
41
|
+
opt.parse!(arguments)
|
42
|
+
rescue NotFoundPackage => e
|
43
|
+
stdout.puts "fatal: Not found package '#{e}'."
|
44
|
+
return
|
45
|
+
end
|
46
|
+
|
47
|
+
if option.packages.empty? && !my_option[:all]
|
48
|
+
if (package_dir_in? current_dir)
|
49
|
+
option.filePatterns << current_dir
|
50
|
+
else
|
51
|
+
stdout.puts "fatal: Not package dir '#{current_dir}'."
|
52
|
+
return
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
if (arguments.size > 0 || my_option[:find_mode])
|
57
|
+
# update
|
58
|
+
if my_option[:update]
|
59
|
+
cdstk = Cdstk.new(stdout, Dbdir.select_dbdir)
|
60
|
+
|
61
|
+
if (my_option[:all])
|
62
|
+
cdstk.update_all
|
63
|
+
elsif (my_option[:packages].empty?)
|
64
|
+
cdstk.update_package(package_root_dir(File.expand_path(".")))
|
65
|
+
else
|
66
|
+
my_option[:packages].each do |v|
|
67
|
+
cdstk.update_package(v)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
stdout.puts
|
72
|
+
end
|
73
|
+
|
74
|
+
# findgrep
|
75
|
+
findGrep = FindGrep::FindGrep.new(arguments, option)
|
76
|
+
findGrep.searchAndPrint(stdout)
|
77
|
+
else
|
78
|
+
stdout.print opt.help
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
private
|
83
|
+
|
84
|
+
def self.setup_package(option, my_option, keyword)
|
85
|
+
packages = yaml_load.list( CdstkYaml::Query.new([keyword]) ).map{|v| v['directory']}
|
86
|
+
raise NotFoundPackage.new keyword if (packages.empty?)
|
87
|
+
option.packages += packages
|
88
|
+
my_option[:packages] += packages
|
89
|
+
end
|
90
|
+
|
91
|
+
def self.package_dir_in?(dir)
|
92
|
+
yaml_load.package_root_dir(dir)
|
93
|
+
end
|
94
|
+
|
95
|
+
def self.package_root_dir(dir)
|
96
|
+
package_root = yaml_load.package_root_dir(dir)
|
97
|
+
|
98
|
+
if (package_root)
|
99
|
+
package_root
|
100
|
+
else
|
101
|
+
raise NotFoundPackage.new dir
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def self.yaml_load
|
106
|
+
CdstkYaml.load(Dbdir.select_dbdir)
|
107
|
+
end
|
108
|
+
|
109
|
+
# --- error ---
|
110
|
+
class NotFoundPackage < RuntimeError ; end
|
111
|
+
|
112
|
+
end
|
113
|
+
end
|
data/milkode.gemspec
CHANGED
@@ -5,14 +5,14 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{milkode}
|
8
|
-
s.version = "0.2.
|
8
|
+
s.version = "0.2.9"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["ongaeshi"]
|
12
|
-
s.date = %q{2011-
|
12
|
+
s.date = %q{2011-12-24}
|
13
13
|
s.description = %q{Line based local source code search engine & web-app.}
|
14
14
|
s.email = %q{ongaeshi0621@gmail.com}
|
15
|
-
s.executables = ["cdv", "milk", "cdview"]
|
15
|
+
s.executables = ["cdv", "gmilk", "milk", "cdview"]
|
16
16
|
s.extra_rdoc_files = [
|
17
17
|
"LICENSE.txt",
|
18
18
|
"README.rdoc"
|
@@ -29,8 +29,10 @@ Gem::Specification.new do |s|
|
|
29
29
|
"VERSION",
|
30
30
|
"bin/cdv",
|
31
31
|
"bin/cdview",
|
32
|
+
"bin/gmilk",
|
32
33
|
"bin/milk",
|
33
34
|
"lib/milkode/cdstk/cdstk.rb",
|
35
|
+
"lib/milkode/cdstk/cdstk_command.rb",
|
34
36
|
"lib/milkode/cdstk/cdstk_yaml.rb",
|
35
37
|
"lib/milkode/cdstk/cli_cdstk.rb",
|
36
38
|
"lib/milkode/cdstk/cli_cdstksub.rb",
|
@@ -43,6 +45,7 @@ Gem::Specification.new do |s|
|
|
43
45
|
"lib/milkode/cdweb/lib/database.rb",
|
44
46
|
"lib/milkode/cdweb/lib/grep.rb",
|
45
47
|
"lib/milkode/cdweb/lib/mkurl.rb",
|
48
|
+
"lib/milkode/cdweb/lib/my_nokogiri.rb",
|
46
49
|
"lib/milkode/cdweb/lib/query.rb",
|
47
50
|
"lib/milkode/cdweb/lib/search_contents.rb",
|
48
51
|
"lib/milkode/cdweb/lib/search_files.rb",
|
@@ -72,17 +75,26 @@ Gem::Specification.new do |s|
|
|
72
75
|
"lib/milkode/common/util.rb",
|
73
76
|
"lib/milkode/findgrep/findgrep.rb",
|
74
77
|
"lib/milkode/findgrep/result.rb",
|
78
|
+
"lib/milkode/grep/cli_grep.rb",
|
75
79
|
"milkode.gemspec",
|
80
|
+
"test/data/a_project/cdstk.rb",
|
81
|
+
"test/data/a_project/cdstk_yaml.rb",
|
76
82
|
"test/data/abc.zip",
|
83
|
+
"test/data/b_project/runner.rb",
|
84
|
+
"test/data/b_project/test_dir.rb",
|
77
85
|
"test/data/nodir_abc.zip",
|
78
86
|
"test/data/nodir_abc_xpi.xpi",
|
79
87
|
"test/file_assert.rb",
|
80
88
|
"test/file_test_utils.rb",
|
89
|
+
"test/milkode_test_work.rb",
|
81
90
|
"test/rake_test_loader.rb",
|
82
91
|
"test/runner.rb",
|
83
92
|
"test/test_bin_exec.rb",
|
84
93
|
"test/test_cdstk.rb",
|
94
|
+
"test/test_cdstk_command.rb",
|
85
95
|
"test/test_cdstk_yaml.rb",
|
96
|
+
"test/test_cli_cdstk.rb",
|
97
|
+
"test/test_cli_grep.rb",
|
86
98
|
"test/test_coderay_wrapper.rb",
|
87
99
|
"test/test_coderay_wrapper_data.rb",
|
88
100
|
"test/test_database.rb",
|
@@ -101,13 +113,21 @@ Gem::Specification.new do |s|
|
|
101
113
|
s.rubygems_version = %q{1.3.6}
|
102
114
|
s.summary = %q{Line based local source code search engine & web-app.}
|
103
115
|
s.test_files = [
|
116
|
+
"test/data/a_project/cdstk.rb",
|
117
|
+
"test/data/a_project/cdstk_yaml.rb",
|
118
|
+
"test/data/b_project/runner.rb",
|
119
|
+
"test/data/b_project/test_dir.rb",
|
104
120
|
"test/file_assert.rb",
|
105
121
|
"test/file_test_utils.rb",
|
122
|
+
"test/milkode_test_work.rb",
|
106
123
|
"test/rake_test_loader.rb",
|
107
124
|
"test/runner.rb",
|
108
125
|
"test/test_bin_exec.rb",
|
109
126
|
"test/test_cdstk.rb",
|
127
|
+
"test/test_cdstk_command.rb",
|
110
128
|
"test/test_cdstk_yaml.rb",
|
129
|
+
"test/test_cli_cdstk.rb",
|
130
|
+
"test/test_cli_grep.rb",
|
111
131
|
"test/test_coderay_wrapper.rb",
|
112
132
|
"test/test_coderay_wrapper_data.rb",
|
113
133
|
"test/test_database.rb",
|
@@ -130,8 +150,8 @@ Gem::Specification.new do |s|
|
|
130
150
|
s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
|
131
151
|
s.add_development_dependency(%q<rcov>, [">= 0"])
|
132
152
|
s.add_runtime_dependency(%q<termcolor>, [">= 1.2.0"])
|
133
|
-
s.add_runtime_dependency(%q<rroonga>, [">= 1.
|
134
|
-
s.add_runtime_dependency(%q<rack>, [">= 1.
|
153
|
+
s.add_runtime_dependency(%q<rroonga>, [">= 1.1.0"])
|
154
|
+
s.add_runtime_dependency(%q<rack>, [">= 1.3.4"])
|
135
155
|
s.add_runtime_dependency(%q<sinatra>, [">= 1.2.6"])
|
136
156
|
s.add_runtime_dependency(%q<launchy>, [">= 0.3.7"])
|
137
157
|
s.add_runtime_dependency(%q<coderay>, ["= 0.9.8"])
|
@@ -140,13 +160,14 @@ Gem::Specification.new do |s|
|
|
140
160
|
s.add_runtime_dependency(%q<haml>, [">= 3.1.2"])
|
141
161
|
s.add_runtime_dependency(%q<sass>, [">= 3.1.3"])
|
142
162
|
s.add_runtime_dependency(%q<nokogiri>, [">= 1.5.0"])
|
163
|
+
s.add_runtime_dependency(%q<hpricot>, [">= 0.8.2"])
|
143
164
|
else
|
144
165
|
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
145
166
|
s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
|
146
167
|
s.add_dependency(%q<rcov>, [">= 0"])
|
147
168
|
s.add_dependency(%q<termcolor>, [">= 1.2.0"])
|
148
|
-
s.add_dependency(%q<rroonga>, [">= 1.
|
149
|
-
s.add_dependency(%q<rack>, [">= 1.
|
169
|
+
s.add_dependency(%q<rroonga>, [">= 1.1.0"])
|
170
|
+
s.add_dependency(%q<rack>, [">= 1.3.4"])
|
150
171
|
s.add_dependency(%q<sinatra>, [">= 1.2.6"])
|
151
172
|
s.add_dependency(%q<launchy>, [">= 0.3.7"])
|
152
173
|
s.add_dependency(%q<coderay>, ["= 0.9.8"])
|
@@ -155,14 +176,15 @@ Gem::Specification.new do |s|
|
|
155
176
|
s.add_dependency(%q<haml>, [">= 3.1.2"])
|
156
177
|
s.add_dependency(%q<sass>, [">= 3.1.3"])
|
157
178
|
s.add_dependency(%q<nokogiri>, [">= 1.5.0"])
|
179
|
+
s.add_dependency(%q<hpricot>, [">= 0.8.2"])
|
158
180
|
end
|
159
181
|
else
|
160
182
|
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
161
183
|
s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
|
162
184
|
s.add_dependency(%q<rcov>, [">= 0"])
|
163
185
|
s.add_dependency(%q<termcolor>, [">= 1.2.0"])
|
164
|
-
s.add_dependency(%q<rroonga>, [">= 1.
|
165
|
-
s.add_dependency(%q<rack>, [">= 1.
|
186
|
+
s.add_dependency(%q<rroonga>, [">= 1.1.0"])
|
187
|
+
s.add_dependency(%q<rack>, [">= 1.3.4"])
|
166
188
|
s.add_dependency(%q<sinatra>, [">= 1.2.6"])
|
167
189
|
s.add_dependency(%q<launchy>, [">= 0.3.7"])
|
168
190
|
s.add_dependency(%q<coderay>, ["= 0.9.8"])
|
@@ -171,6 +193,7 @@ Gem::Specification.new do |s|
|
|
171
193
|
s.add_dependency(%q<haml>, [">= 3.1.2"])
|
172
194
|
s.add_dependency(%q<sass>, [">= 3.1.3"])
|
173
195
|
s.add_dependency(%q<nokogiri>, [">= 1.5.0"])
|
196
|
+
s.add_dependency(%q<hpricot>, [">= 0.8.2"])
|
174
197
|
end
|
175
198
|
end
|
176
199
|
|
@@ -0,0 +1,634 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
|
3
|
+
require 'yaml'
|
4
|
+
require 'pathname'
|
5
|
+
require 'rubygems'
|
6
|
+
require 'groonga'
|
7
|
+
require 'fileutils'
|
8
|
+
require 'pathname'
|
9
|
+
require 'milkode/cdstk/cdstk_yaml'
|
10
|
+
require 'milkode/common/grenfiletest'
|
11
|
+
require 'milkode/common/util'
|
12
|
+
require 'milkode/common/dir'
|
13
|
+
include Milkode
|
14
|
+
require 'kconv'
|
15
|
+
begin
|
16
|
+
require 'readline'
|
17
|
+
rescue LoadError
|
18
|
+
$no_readline = true
|
19
|
+
end
|
20
|
+
require 'milkode/cdweb/lib/database'
|
21
|
+
require 'open-uri'
|
22
|
+
require 'milkode/cdstk/cdstk_command'
|
23
|
+
|
24
|
+
module Milkode
|
25
|
+
class Cdstk
|
26
|
+
# バイグラムでトークナイズする。連続する記号・アルファベット・数字は一語として扱う。
|
27
|
+
# DEFAULT_TOKENIZER = "TokenBigram"
|
28
|
+
|
29
|
+
# 記号・アルファベット・数字もバイグラムでトークナイズする。
|
30
|
+
DEFAULT_TOKENIZER = "TokenBigramSplitSymbolAlphaDigit"
|
31
|
+
|
32
|
+
class ConvetError < RuntimeError ; end
|
33
|
+
|
34
|
+
def initialize(io = $stdout, db_dir = ".")
|
35
|
+
@db_dir = db_dir
|
36
|
+
Database.setup(@db_dir)
|
37
|
+
@out = io
|
38
|
+
# @out = $stdout # 強制出力
|
39
|
+
clear_count
|
40
|
+
end
|
41
|
+
|
42
|
+
def clear_count
|
43
|
+
@package_count = 0
|
44
|
+
@file_count = 0
|
45
|
+
@add_count = 0
|
46
|
+
@update_count = 0
|
47
|
+
@start_time = Time.now
|
48
|
+
end
|
49
|
+
|
50
|
+
def init
|
51
|
+
if Dir.emptydir?(@db_dir)
|
52
|
+
CdstkYaml.create(@db_dir)
|
53
|
+
@out.puts "create : #{yaml_file}"
|
54
|
+
db_create(db_file)
|
55
|
+
else
|
56
|
+
@out.puts "Can't create milkode db (Because not empty in #{db_dir_expand})"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def compatible?
|
61
|
+
db_open(db_file)
|
62
|
+
end
|
63
|
+
|
64
|
+
def update_all
|
65
|
+
print_result do
|
66
|
+
yaml = yaml_load
|
67
|
+
|
68
|
+
db_open(db_file)
|
69
|
+
|
70
|
+
yaml.list.each do |content|
|
71
|
+
update_dir_in(content["directory"])
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def update(args, options)
|
77
|
+
if (options[:all])
|
78
|
+
update_all
|
79
|
+
else
|
80
|
+
if (args.empty?)
|
81
|
+
path = File.expand_path('.')
|
82
|
+
package = yaml_load.package_root( path )
|
83
|
+
|
84
|
+
if (package)
|
85
|
+
print_result do
|
86
|
+
db_open(db_file)
|
87
|
+
update_dir_in(package["directory"])
|
88
|
+
end
|
89
|
+
else
|
90
|
+
@out.puts "Not registered. If you want to add, 'milk add #{path}'."
|
91
|
+
end
|
92
|
+
else
|
93
|
+
print_result do
|
94
|
+
update_list = yaml_load.list CdstkYaml::Query.new(args)
|
95
|
+
db_open(db_file)
|
96
|
+
update_list.each do |content|
|
97
|
+
update_dir_in(content["directory"])
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def update_package(dir)
|
105
|
+
db_open(db_file)
|
106
|
+
update_dir(dir)
|
107
|
+
end
|
108
|
+
|
109
|
+
def update_dir(dir)
|
110
|
+
update_dir_in(dir)
|
111
|
+
end
|
112
|
+
|
113
|
+
def add(contents)
|
114
|
+
# YAMLを読み込み
|
115
|
+
yaml = yaml_load
|
116
|
+
|
117
|
+
# コンテンツを読み込める形に変換
|
118
|
+
begin
|
119
|
+
contents.map!{|v|convert_content(v)}
|
120
|
+
rescue ConvetError
|
121
|
+
return
|
122
|
+
end
|
123
|
+
|
124
|
+
# 存在しないコンテンツがあった場合はその場で終了
|
125
|
+
contents.each do |v|
|
126
|
+
shortname = File.basename v
|
127
|
+
|
128
|
+
if (yaml.cant_add_directory? v)
|
129
|
+
error_alert("already exist '#{shortname}'.")
|
130
|
+
return
|
131
|
+
end
|
132
|
+
|
133
|
+
unless (File.exist? v)
|
134
|
+
error_alert("not found '#{v}'.")
|
135
|
+
return
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
# YAML更新
|
140
|
+
yaml.add(contents)
|
141
|
+
yaml.save
|
142
|
+
|
143
|
+
# 部分アップデート
|
144
|
+
print_result do
|
145
|
+
db_open(db_file)
|
146
|
+
contents.each do |dir|
|
147
|
+
update_dir(dir)
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def convert_content(src)
|
153
|
+
# httpファイルならダウンロード
|
154
|
+
begin
|
155
|
+
src = download_file(src)
|
156
|
+
rescue => e
|
157
|
+
error_alert("download failure '#{src}'.")
|
158
|
+
raise e # そのまま上に持ち上げてスタックトレース表示
|
159
|
+
end
|
160
|
+
|
161
|
+
# アーカイブファイルなら展開
|
162
|
+
begin
|
163
|
+
src = extract_file(src)
|
164
|
+
rescue => e
|
165
|
+
error_alert("extract failure '#{src}'.")
|
166
|
+
raise e # そのまま上に持ち上げてスタックトレース表示
|
167
|
+
end
|
168
|
+
|
169
|
+
# 絶対パスに変換
|
170
|
+
File.expand_path(src)
|
171
|
+
end
|
172
|
+
|
173
|
+
def extract_file(src)
|
174
|
+
ext = File.extname(src);
|
175
|
+
|
176
|
+
case ext
|
177
|
+
when '.zip', '.xpi'
|
178
|
+
alert("extract", "#{src}")
|
179
|
+
zip_dir = File.join(@db_dir, "packages/#{ext.sub(".", "")}")
|
180
|
+
result = File.join(zip_dir, Util::zip_extract(src, zip_dir))
|
181
|
+
else
|
182
|
+
src
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
186
|
+
def download_file(src)
|
187
|
+
if (src =~ /^https?:/)
|
188
|
+
download_file_in(src)
|
189
|
+
else
|
190
|
+
src
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
def download_file_in(url)
|
195
|
+
alert("download", "#{url}")
|
196
|
+
|
197
|
+
dst_dir = File.join(@db_dir, "packages/http")
|
198
|
+
FileUtils.mkdir_p dst_dir
|
199
|
+
|
200
|
+
filename = File.join(dst_dir, File.basename(url))
|
201
|
+
|
202
|
+
open(url) do |src|
|
203
|
+
open(filename, "wb") do |dst|
|
204
|
+
dst.write(src.read)
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
filename
|
209
|
+
end
|
210
|
+
|
211
|
+
def remove(args, options)
|
212
|
+
print_result do
|
213
|
+
db_open(db_file)
|
214
|
+
|
215
|
+
yaml = yaml_load
|
216
|
+
query = CdstkYaml::Query.new(args)
|
217
|
+
|
218
|
+
remove_list = yaml_load.list(query)
|
219
|
+
return if remove_list.empty?
|
220
|
+
|
221
|
+
list(args, {:verbose => true})
|
222
|
+
|
223
|
+
if options[:force] or yes_or_no("Remove #{remove_list.size} contents? (yes/no)")
|
224
|
+
# yamlから削除
|
225
|
+
yaml.remove(query)
|
226
|
+
yaml.save
|
227
|
+
|
228
|
+
# データベースからも削除
|
229
|
+
packages = remove_list.map{|v| File.basename v['directory']}
|
230
|
+
|
231
|
+
# 本当はパッケージの配列をまとめて渡した方が効率が良いのだが、表示を綺麗にするため
|
232
|
+
packages.each do |package|
|
233
|
+
alert("rm_package", package)
|
234
|
+
@package_count += 1
|
235
|
+
|
236
|
+
Database.instance.remove([package]) do |record|
|
237
|
+
alert("rm_record", record.path)
|
238
|
+
@file_count += 1
|
239
|
+
end
|
240
|
+
end
|
241
|
+
end
|
242
|
+
end
|
243
|
+
end
|
244
|
+
|
245
|
+
def yes_or_no(msg)
|
246
|
+
if ($no_readline)
|
247
|
+
@out.puts "Pass Cdstk#yes_or_no, because fail \"require 'readline'\"."
|
248
|
+
return true
|
249
|
+
end
|
250
|
+
|
251
|
+
@out.puts msg
|
252
|
+
while buf = Readline.readline("> ")
|
253
|
+
case buf
|
254
|
+
when 'yes'
|
255
|
+
return true
|
256
|
+
when 'no'
|
257
|
+
break
|
258
|
+
end
|
259
|
+
end
|
260
|
+
return false
|
261
|
+
end
|
262
|
+
|
263
|
+
def list(args, options)
|
264
|
+
query = (args.empty?) ? nil : CdstkYaml::Query.new(args)
|
265
|
+
a = yaml_load.list(query).map {|v| [File.basename(v['directory']), v['directory']] }
|
266
|
+
max = a.map{|v|v[0].length}.max
|
267
|
+
str = a.sort_by {|v|
|
268
|
+
v[0]
|
269
|
+
}.map {|v|
|
270
|
+
h = File.exist?(v[1]) ? '' : '? '
|
271
|
+
if (options[:verbose])
|
272
|
+
"#{(h + v[0]).ljust(max+2)} #{v[1]}"
|
273
|
+
else
|
274
|
+
"#{h}#{v[0]}"
|
275
|
+
end
|
276
|
+
}.join("\n")
|
277
|
+
|
278
|
+
@out.puts str
|
279
|
+
end
|
280
|
+
|
281
|
+
def pwd(options)
|
282
|
+
dir = options[:default] ? Dbdir.default_dir : db_dir_expand
|
283
|
+
|
284
|
+
if File.exist? dir
|
285
|
+
@out.puts dir
|
286
|
+
else
|
287
|
+
@out.puts "Not found db in #{Dir.pwd}"
|
288
|
+
end
|
289
|
+
end
|
290
|
+
|
291
|
+
def cleanup(options)
|
292
|
+
# 互換性テスト
|
293
|
+
db_open(db_file)
|
294
|
+
|
295
|
+
# cleanup開始
|
296
|
+
if (options[:force] or yes_or_no("cleanup contents? (yes/no)"))
|
297
|
+
print_result do
|
298
|
+
# yamlファイルのクリーンアップ
|
299
|
+
yaml = yaml_load
|
300
|
+
|
301
|
+
yaml.cleanup do |v|
|
302
|
+
alert("rm_package", v['directory'])
|
303
|
+
@package_count += 1
|
304
|
+
end
|
305
|
+
|
306
|
+
yaml.save
|
307
|
+
|
308
|
+
# データベースのクリーンアップ
|
309
|
+
Database.instance.cleanup do |record|
|
310
|
+
alert("rm_record", record.path)
|
311
|
+
@file_count += 1
|
312
|
+
end
|
313
|
+
end
|
314
|
+
end
|
315
|
+
end
|
316
|
+
|
317
|
+
def rebuild
|
318
|
+
db_delete(db_file)
|
319
|
+
db_create(db_file)
|
320
|
+
update_all
|
321
|
+
end
|
322
|
+
|
323
|
+
def dump
|
324
|
+
db_open(db_file)
|
325
|
+
|
326
|
+
documents = Groonga::Context.default["documents"]
|
327
|
+
records = documents.select
|
328
|
+
records.each do |record|
|
329
|
+
@out.puts record.inspect
|
330
|
+
@out.puts "path : #{record.path}"
|
331
|
+
@out.puts "shortpath : #{record.shortpath}"
|
332
|
+
@out.puts "suffix : #{record.suffix}"
|
333
|
+
@out.puts "timestamp : #{record.timestamp.strftime('%Y/%m/%d %H:%M:%S')}"
|
334
|
+
@out.puts "content :", record.content ? record.content[0..64] : nil
|
335
|
+
@out.puts
|
336
|
+
end
|
337
|
+
@out.puts "total #{records.size} record."
|
338
|
+
end
|
339
|
+
|
340
|
+
def dir(args, options)
|
341
|
+
yaml = yaml_load
|
342
|
+
|
343
|
+
if args.empty?
|
344
|
+
path = File.expand_path('.')
|
345
|
+
package = yaml.package_root(path)
|
346
|
+
|
347
|
+
if (package)
|
348
|
+
@out.puts package['directory']
|
349
|
+
else
|
350
|
+
@out.puts "Not registered. If you want to add, 'milk add #{path}'."
|
351
|
+
end
|
352
|
+
else
|
353
|
+
@out.puts yaml.list(CdstkYaml::Query.new(args)).map{|v|v['directory']}
|
354
|
+
end
|
355
|
+
end
|
356
|
+
|
357
|
+
def setdb(args, options)
|
358
|
+
if (options[:reset])
|
359
|
+
CdstkCommand.setdb_reset
|
360
|
+
@out.puts "Reset default db\n remove: #{Dbdir.milkode_db_dir}\n default_db: #{Dbdir.default_dir}"
|
361
|
+
elsif (args.empty?)
|
362
|
+
@out.puts Dbdir.default_dir
|
363
|
+
else
|
364
|
+
path = File.expand_path(args[0])
|
365
|
+
begin
|
366
|
+
CdstkCommand.setdb_set path
|
367
|
+
@out.puts "Set default db #{path}."
|
368
|
+
rescue CdstkCommand::NotExistDatabase
|
369
|
+
@out.puts "fatal: '#{path}' is not database."
|
370
|
+
end
|
371
|
+
end
|
372
|
+
end
|
373
|
+
|
374
|
+
def mcd(args, options)
|
375
|
+
@out.print <<EOF
|
376
|
+
# Copy to '.bashrc'.
|
377
|
+
mcd() {
|
378
|
+
cd `milk dir $1 $2 $3 $4 $5 $6 $7 $8 $9`; pwd
|
379
|
+
}
|
380
|
+
EOF
|
381
|
+
end
|
382
|
+
|
383
|
+
private
|
384
|
+
|
385
|
+
def db_file
|
386
|
+
Dbdir.expand_groonga_path(@db_dir)
|
387
|
+
end
|
388
|
+
|
389
|
+
def db_file_expand
|
390
|
+
File.expand_path(db_file)
|
391
|
+
end
|
392
|
+
|
393
|
+
def db_dir_expand
|
394
|
+
File.expand_path(@db_dir)
|
395
|
+
end
|
396
|
+
|
397
|
+
def custom_db?
|
398
|
+
db_dir_expand != Dbdir.default_dir
|
399
|
+
end
|
400
|
+
|
401
|
+
def yaml_file
|
402
|
+
CdstkYaml.yaml_file @db_dir
|
403
|
+
end
|
404
|
+
|
405
|
+
def yaml_load
|
406
|
+
CdstkYaml.load(@db_dir)
|
407
|
+
end
|
408
|
+
|
409
|
+
def update_dir_in(dir)
|
410
|
+
alert("package", File.basename(dir) )
|
411
|
+
@package_count += 1
|
412
|
+
|
413
|
+
dir = File.expand_path(dir)
|
414
|
+
|
415
|
+
if (!FileTest.exist?(dir))
|
416
|
+
@out.puts "[WARNING] : #{dir} (Not found, skip)"
|
417
|
+
elsif (FileTest.directory? dir)
|
418
|
+
db_add_dir(dir)
|
419
|
+
else
|
420
|
+
db_add_file(STDOUT, dir, File.basename(dir))
|
421
|
+
end
|
422
|
+
end
|
423
|
+
|
424
|
+
def time
|
425
|
+
@end_time - @start_time
|
426
|
+
end
|
427
|
+
|
428
|
+
def print_result
|
429
|
+
clear_count
|
430
|
+
yield
|
431
|
+
@end_time = Time.now
|
432
|
+
|
433
|
+
result_info
|
434
|
+
milkode_info
|
435
|
+
end
|
436
|
+
|
437
|
+
def result_info
|
438
|
+
r = []
|
439
|
+
r << "#{@package_count} packages" if @package_count > 0
|
440
|
+
r << "#{@file_count} records" if @file_count > 0
|
441
|
+
r << "#{@add_count} add" if @add_count > 0
|
442
|
+
r << "#{@update_count} update" if @update_count > 0
|
443
|
+
r.join(', ')
|
444
|
+
alert('result', "#{r.join(', ')}. (#{Gren::Util::time_s(time)})")
|
445
|
+
end
|
446
|
+
|
447
|
+
def milkode_info
|
448
|
+
alert('*milkode*', "#{yaml_load.package_num} packages, #{Database.instance.totalRecords} records in #{db_file}.")
|
449
|
+
end
|
450
|
+
|
451
|
+
def db_create(filename)
|
452
|
+
dbfile = Pathname(File.expand_path(filename))
|
453
|
+
dbdir = dbfile.dirname
|
454
|
+
dbdir.mkpath unless dbdir.exist?
|
455
|
+
|
456
|
+
unless dbfile.exist?
|
457
|
+
Groonga::Database.create(:path => dbfile.to_s)
|
458
|
+
db_define
|
459
|
+
@out.puts "create : #{filename} created."
|
460
|
+
else
|
461
|
+
@out.puts "message : #{filename} already exist."
|
462
|
+
end
|
463
|
+
end
|
464
|
+
|
465
|
+
def db_define
|
466
|
+
Groonga::Schema.define do |schema|
|
467
|
+
schema.create_table("documents", :type => :hash) do |table|
|
468
|
+
table.string("path")
|
469
|
+
table.string("shortpath")
|
470
|
+
table.text("content")
|
471
|
+
table.time("timestamp")
|
472
|
+
table.text("suffix")
|
473
|
+
end
|
474
|
+
|
475
|
+
schema.create_table("terms",
|
476
|
+
:type => :patricia_trie,
|
477
|
+
:key_normalize => true,
|
478
|
+
:default_tokenizer => DEFAULT_TOKENIZER) do |table|
|
479
|
+
table.index("documents.path", :with_position => true)
|
480
|
+
table.index("documents.shortpath", :with_position => true)
|
481
|
+
table.index("documents.content", :with_position => true)
|
482
|
+
table.index("documents.suffix", :with_position => true)
|
483
|
+
end
|
484
|
+
end
|
485
|
+
end
|
486
|
+
|
487
|
+
def db_open(filename)
|
488
|
+
dbfile = Pathname(File.expand_path(filename))
|
489
|
+
|
490
|
+
if dbfile.exist?
|
491
|
+
# データベースを開く
|
492
|
+
Groonga::Database.open(dbfile.to_s)
|
493
|
+
|
494
|
+
# 互換性テスト
|
495
|
+
db_compatible?
|
496
|
+
else
|
497
|
+
raise "error : #{dbfile.to_s} not found!!"
|
498
|
+
end
|
499
|
+
end
|
500
|
+
|
501
|
+
def db_delete(filename)
|
502
|
+
raise "Illegal file name : #{filename}." unless filename =~ /\.db$/
|
503
|
+
Dir.glob("#{filename}*").each do |f|
|
504
|
+
@out.puts "delete : #{f}"
|
505
|
+
FileUtils.rm_r(f)
|
506
|
+
end
|
507
|
+
end
|
508
|
+
private :db_delete
|
509
|
+
|
510
|
+
def db_add_dir(dirname)
|
511
|
+
searchDirectory(STDOUT, dirname, File.basename(dirname), 0)
|
512
|
+
end
|
513
|
+
private :db_add_dir
|
514
|
+
|
515
|
+
def db_add_file(stdout, filename, shortpath)
|
516
|
+
# ファイル名を全てUTF-8に変換
|
517
|
+
filename_utf8 = Util::filename_to_utf8(filename)
|
518
|
+
shortpath_utf8 = Util::filename_to_utf8(shortpath)
|
519
|
+
suffix_utf8 = File::extname(filename_utf8)
|
520
|
+
|
521
|
+
# 格納するデータ
|
522
|
+
values = {
|
523
|
+
:path => filename_utf8,
|
524
|
+
:shortpath => shortpath_utf8,
|
525
|
+
:content => nil,
|
526
|
+
:timestamp => File.mtime(filename),
|
527
|
+
:suffix => suffix_utf8,
|
528
|
+
}
|
529
|
+
|
530
|
+
# 検索するデータベース
|
531
|
+
documents = Groonga::Context.default["documents"]
|
532
|
+
|
533
|
+
record = documents[ values[:path] ]
|
534
|
+
isNewFile = false
|
535
|
+
|
536
|
+
unless record
|
537
|
+
document = documents.add(values[:path])
|
538
|
+
isNewFile = true
|
539
|
+
else
|
540
|
+
document = record
|
541
|
+
end
|
542
|
+
|
543
|
+
# タイムスタンプが新しければデータベースに格納
|
544
|
+
if (document[:timestamp] < values[:timestamp])
|
545
|
+
# 実際に使うタイミングでファイルの内容を読み込み
|
546
|
+
# values[:content] = open(filename).read
|
547
|
+
# データベース内の文字コードは'utf-8'で統一
|
548
|
+
values[:content] = Kconv.kconv(File.read(filename), Kconv::UTF8)
|
549
|
+
|
550
|
+
# データベースに格納
|
551
|
+
values.each do |key, value|
|
552
|
+
if (key == :path)
|
553
|
+
if (isNewFile)
|
554
|
+
@add_count += 1
|
555
|
+
alert("add_record", value)
|
556
|
+
else
|
557
|
+
@update_count += 1
|
558
|
+
alert("update", value)
|
559
|
+
end
|
560
|
+
end
|
561
|
+
document[key] = value
|
562
|
+
end
|
563
|
+
end
|
564
|
+
|
565
|
+
end
|
566
|
+
|
567
|
+
def searchDirectory(stdout, dir, shortdir, depth)
|
568
|
+
Dir.foreach(dir) do |name|
|
569
|
+
next if (name == '.' || name == '..')
|
570
|
+
|
571
|
+
fpath = File.join(dir,name)
|
572
|
+
shortpath = File.join(shortdir,name)
|
573
|
+
|
574
|
+
# 除外ディレクトリならばパス
|
575
|
+
next if ignoreDir?(fpath)
|
576
|
+
|
577
|
+
# 読み込み不可ならばパス
|
578
|
+
next unless FileTest.readable?(fpath)
|
579
|
+
|
580
|
+
# ファイルならば中身を探索、ディレクトリならば再帰
|
581
|
+
case File.ftype(fpath)
|
582
|
+
when "directory"
|
583
|
+
searchDirectory(stdout, fpath, shortpath, depth + 1)
|
584
|
+
when "file"
|
585
|
+
unless ignoreFile?(fpath)
|
586
|
+
db_add_file(stdout, fpath, shortpath)
|
587
|
+
@file_count += 1
|
588
|
+
# @out.puts "file_count : #{@file_count}" if (@file_count % 100 == 0)
|
589
|
+
end
|
590
|
+
end
|
591
|
+
end
|
592
|
+
end
|
593
|
+
|
594
|
+
def ignoreDir?(fpath)
|
595
|
+
FileTest.directory?(fpath) &&
|
596
|
+
GrenFileTest::ignoreDir?(fpath)
|
597
|
+
end
|
598
|
+
private :ignoreDir?
|
599
|
+
|
600
|
+
def ignoreFile?(fpath)
|
601
|
+
GrenFileTest::ignoreFile?(fpath) ||
|
602
|
+
GrenFileTest::binary?(fpath)
|
603
|
+
end
|
604
|
+
private :ignoreFile?
|
605
|
+
|
606
|
+
def alert(title, msg)
|
607
|
+
if (Util::platform_win?)
|
608
|
+
@out.puts "#{title.ljust(10)} : #{Kconv.kconv(msg, Kconv::SJIS)}"
|
609
|
+
else
|
610
|
+
@out.puts "#{title.ljust(10)} : #{msg}"
|
611
|
+
end
|
612
|
+
end
|
613
|
+
|
614
|
+
def error_alert(msg)
|
615
|
+
@out.puts "[fatal] #{msg}"
|
616
|
+
end
|
617
|
+
|
618
|
+
def db_compatible?
|
619
|
+
begin
|
620
|
+
db_define
|
621
|
+
rescue Groonga::Schema::Error => e
|
622
|
+
puts <<EOF
|
623
|
+
Milkode repository is old -> #{db_dir_expand}.
|
624
|
+
Please rebuild repository,
|
625
|
+
|
626
|
+
milk rebuild
|
627
|
+
|
628
|
+
See 'milk --help' or http://milkode.ongaeshi.me .
|
629
|
+
EOF
|
630
|
+
exit -1
|
631
|
+
end
|
632
|
+
end
|
633
|
+
end
|
634
|
+
end
|