sekka 1.1.1.pre → 1.1.3.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +39 -19
- data/VERSION.yml +1 -1
- data/bin/sekka-jisyo +11 -3
- data/bin/sekka-server +51 -30
- data/emacs/sekka.el +1 -1
- data/lib/sekka/approximatesearch.rb +6 -6
- data/lib/sekka/convert-jisyo.nnd +69 -76
- data/lib/sekka/henkan.nnd +11 -11
- data/lib/sekka/jisyo-db.nnd +10 -19
- data/lib/sekka/roman-lib.nnd +83 -6
- data/lib/sekka/sekkaversion.rb +2 -2
- data/lib/sekka/util.nnd +9 -1
- data/lib/sekkaserver.rb +3 -3
- data/test/approximate-bench.nnd +1 -1
- data/test/azik-verification.nnd +1 -1
- data/test/henkan-main.nnd +30 -30
- data/test/memcache.nnd +10 -10
- data/test/redis.nnd +6 -6
- data/test/roman-lib.nnd +10 -10
- data/test/sekka-dump-out-1.txt +1338 -1338
- metadata +2 -2
data/Rakefile
CHANGED
@@ -70,7 +70,7 @@ end
|
|
70
70
|
|
71
71
|
task :compile do
|
72
72
|
# generate version.rb
|
73
|
-
dictVersion = "1.2.
|
73
|
+
dictVersion = "1.2.2"
|
74
74
|
vh = Jeweler::VersionHelper.new "."
|
75
75
|
open( "./lib/sekka/sekkaversion.rb", "w" ) {|f|
|
76
76
|
f.puts( "class SekkaVersion" )
|
@@ -154,33 +154,53 @@ task :bench do
|
|
154
154
|
sh "time ruby -I ./lib /usr/local/bin/nendo ./test/henkan-bench.nnd"
|
155
155
|
end
|
156
156
|
|
157
|
-
task :alljisyo
|
157
|
+
task :alljisyo => [ :alljisyoS, :alljisyoL ]
|
158
|
+
task :alljisyoS => [ :jisyoS, :loadS, :dumpS ]
|
159
|
+
task :alljisyoL => [ :jisyoL, :loadL, :dumpL ]
|
158
160
|
|
159
161
|
task :jisyoS do
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
162
|
+
[ "N", "A" ].each {|x|
|
163
|
+
sh "time ./bin/sekka-jisyo convert#{x} ./data/SKK-JISYO.L.201008 > ./data/SEKKA-JISYO.SMALL.#{x}"
|
164
|
+
sh "time ./bin/sekka-jisyo convert#{x} ./data/SKK-JISYO.L.hira-kata >> ./data/SEKKA-JISYO.SMALL.#{x}"
|
165
|
+
sh "time ./bin/sekka-jisyo convert#{x} ./data/SKK-JISYO.hiragana-phrase >> ./data/SEKKA-JISYO.SMALL.#{x}"
|
166
|
+
sh "time ./bin/sekka-jisyo convert#{x} ./data/SKK-JISYO.hiragana-phrase2 >> ./data/SEKKA-JISYO.SMALL.#{x}"
|
167
|
+
}
|
164
168
|
end
|
165
169
|
|
166
170
|
task :jisyoL do
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
171
|
+
[ "N", "A" ].each {|x|
|
172
|
+
sh "time ./bin/sekka-jisyo convert#{x} ./data/SKK-JISYO.L.201008 > ./data/SEKKA-JISYO.LARGE.#{x}"
|
173
|
+
sh "time ./bin/sekka-jisyo convert#{x} ./data/SKK-JISYO.L.hira-kata >> ./data/SEKKA-JISYO.LARGE.#{x}"
|
174
|
+
sh "time ./bin/sekka-jisyo convert#{x} ./data/SKK-JISYO.fullname >> ./data/SEKKA-JISYO.LARGE.#{x}"
|
175
|
+
sh "time ./bin/sekka-jisyo convert#{x} ./data/SKK-JISYO.jinmei >> ./data/SEKKA-JISYO.LARGE.#{x}"
|
176
|
+
sh "time ./bin/sekka-jisyo convert#{x} ./data/SKK-JISYO.station >> ./data/SEKKA-JISYO.LARGE.#{x}"
|
177
|
+
sh "time ./bin/sekka-jisyo convert#{x} ./data/SKK-JISYO.hiragana-phrase >> ./data/SEKKA-JISYO.LARGE.#{x}"
|
178
|
+
sh "time ./bin/sekka-jisyo convert#{x} ./data/SKK-JISYO.hiragana-phrase2 >> ./data/SEKKA-JISYO.LARGE.#{x}"
|
179
|
+
}
|
180
|
+
end
|
181
|
+
|
182
|
+
task :loadS do
|
183
|
+
[ "N", "A" ].each {|x|
|
184
|
+
sh "time ./bin/sekka-jisyo load ./data/SEKKA-JISYO.SMALL.#{x} ./data/SEKKA-JISYO.SMALL.#{x}.tch"
|
185
|
+
}
|
174
186
|
end
|
175
187
|
|
176
|
-
task :
|
177
|
-
|
178
|
-
|
188
|
+
task :loadL do
|
189
|
+
[ "N", "A" ].each {|x|
|
190
|
+
sh "time ./bin/sekka-jisyo load ./data/SEKKA-JISYO.LARGE.#{x} ./data/SEKKA-JISYO.LARGE.#{x}.tch"
|
191
|
+
}
|
179
192
|
end
|
180
193
|
|
181
|
-
task :
|
182
|
-
|
183
|
-
|
194
|
+
task :dumpS do
|
195
|
+
[ "N", "A" ].each {|x|
|
196
|
+
sh "time ./bin/sekka-jisyo dump ./data/SEKKA-JISYO.SMALL.#{x}.tch > ./data/SEKKA-JISYO.SMALL.#{x}.tsv"
|
197
|
+
}
|
198
|
+
end
|
199
|
+
|
200
|
+
task :dumpL do
|
201
|
+
[ "N", "A" ].each {|x|
|
202
|
+
sh "time ./bin/sekka-jisyo dump ./data/SEKKA-JISYO.LARGE.#{x}.tch > ./data/SEKKA-JISYO.LARGE.#{x}.tsv"
|
203
|
+
}
|
184
204
|
end
|
185
205
|
|
186
206
|
|
data/VERSION.yml
CHANGED
data/bin/sekka-jisyo
CHANGED
@@ -76,7 +76,8 @@ core.evalStr( <<";;END-OF-SCRIPT" )
|
|
76
76
|
|
77
77
|
(define (display-help)
|
78
78
|
(print "Usage : ")
|
79
|
-
(print " sekka-jisyo
|
79
|
+
(print " sekka-jisyo convertA SKK-JISYO.X > SEKKA-JISYO.X ... output SEKKA-JISYO to STDOUT(AZIK data included)")
|
80
|
+
(print " sekka-jisyo convertN SKK-JISYO.X > SEKKA-JISYO.X ... output SEKKA-JISYO to STDOUT(AZIK data excluded)")
|
80
81
|
(print " sekka-jisyo load SEKKA-JISYO.X SEKKA-JISYO.X.tch ... load SEKKA-JISYO to Tokyo Cabinet DB")
|
81
82
|
(print " sekka-jisyo load SEKKA-JISYO.X SEKKA-JISYO.X.db ... load SEKKA-JISYO to ndbm DB")
|
82
83
|
(print " sekka-jisyo load SEKKA-JISYO.X redis:[hostname] ... load SEKKA-JISYO to redis DB on [hostname]")
|
@@ -85,13 +86,14 @@ core.evalStr( <<";;END-OF-SCRIPT" )
|
|
85
86
|
(print " sekka-jisyo dump SEKKA-JISYO.X.db ... dump ndbm to DUMPDATA(STDOUT)")
|
86
87
|
(print " sekka-jisyo dump redis:[hostname] ... dump redis DB on [hostname] to DUMPDATA(STDOUT)")
|
87
88
|
(print " sekka-jisyo dump redis: ... dump redis DB on localhost to DUMPDATE(STDOUT)")
|
89
|
+
(print " sekka-jisyo restore SEKKA-JISYO.X.tsv SEKKA-JISYO.X.tch ... restore Tokyo Cabinet DB from tsv-file")
|
88
90
|
(print " sekka-jisyo restore SEKKA-JISYO.X.tsv redis:[hostname] ... restore redis DB on [hostname] from tsv-file")
|
89
91
|
(print " sekka-jisyo restore SEKKA-JISYO.X.tsv redis: ... restore redis DB on localhost from tsv-file"))
|
90
92
|
|
91
93
|
|
92
94
|
(define (analyze-kvs-type filename)
|
93
95
|
(cond
|
94
|
-
((rxmatch #/[.]tch
|
96
|
+
((rxmatch #/[.]tch/ filename)
|
95
97
|
(values 'tokyocabinet filename))
|
96
98
|
((rxmatch #/[.]db$/ filename)
|
97
99
|
(values 'dbm filename))
|
@@ -113,7 +115,13 @@ core.evalStr( <<";;END-OF-SCRIPT" )
|
|
113
115
|
(else
|
114
116
|
(let1 command (string->symbol (first argv))
|
115
117
|
(cond
|
116
|
-
((eq? '
|
118
|
+
((eq? 'convertN command)
|
119
|
+
(if (< (length argv) 2)
|
120
|
+
(display-help)
|
121
|
+
(begin
|
122
|
+
(set! global-use-azik #f)
|
123
|
+
(convert-skk-jisyo (second argv)))))
|
124
|
+
((eq? 'convertA command)
|
117
125
|
(if (< (length argv) 2)
|
118
126
|
(display-help)
|
119
127
|
(convert-skk-jisyo (second argv))))
|
data/bin/sekka-server
CHANGED
@@ -16,17 +16,23 @@ DICTURL = "https://raw.github.com/kiyoka/sekka/master/public_dict/" + SekkaVersi
|
|
16
16
|
|
17
17
|
PIDFILE = DICTDIR + "/pid"
|
18
18
|
|
19
|
+
DICTTYPE = if ENV.has_key?( 'SEKKA_AZIK' )
|
20
|
+
"A"
|
21
|
+
else
|
22
|
+
"N"
|
23
|
+
end
|
24
|
+
|
19
25
|
TC_OPTS = "#xmsiz=256m"
|
20
|
-
TC_FILE = DICTDIR + "/SEKKA-JISYO.SMALL.tch" + TC_OPTS
|
21
|
-
TSVFILE = DICTDIR + "/SEKKA-JISYO.SMALL.tsv"
|
22
|
-
SUMFILE = DICTDIR + "/SEKKA-JISYO.SMALL.md5"
|
26
|
+
TC_FILE = DICTDIR + "/SEKKA-JISYO.SMALL.#{DICTTYPE}.tch" + TC_OPTS
|
27
|
+
TSVFILE = DICTDIR + "/SEKKA-JISYO.SMALL.#{DICTTYPE}.tsv"
|
28
|
+
SUMFILE = DICTDIR + "/SEKKA-JISYO.SMALL.#{DICTTYPE}.md5"
|
23
29
|
|
24
|
-
URLURL = DICTURL + "/SEKKA-JISYO.SMALL.url"
|
25
|
-
SUMURL = DICTURL + "/SEKKA-JISYO.SMALL.md5"
|
30
|
+
URLURL = DICTURL + "/SEKKA-JISYO.SMALL.#{DICTTYPE}.url"
|
31
|
+
SUMURL = DICTURL + "/SEKKA-JISYO.SMALL.#{DICTTYPE}.md5"
|
26
32
|
|
27
33
|
TC_FILE_LIST = [ DICTDIR + "/SEKKA-JISYO.CUSTOM.tch" + TC_OPTS,
|
28
|
-
DICTDIR + "/SEKKA-JISYO.LARGE.tch" + TC_OPTS,
|
29
|
-
DICTDIR + "/SEKKA-JISYO.SMALL.tch" + TC_OPTS ]
|
34
|
+
DICTDIR + "/SEKKA-JISYO.LARGE.#{DICTTYPE}.tch" + TC_OPTS,
|
35
|
+
DICTDIR + "/SEKKA-JISYO.SMALL.#{DICTTYPE}.tch" + TC_OPTS ]
|
30
36
|
|
31
37
|
MEMCACHED = "localhost:11211" # memcahced
|
32
38
|
|
@@ -50,6 +56,35 @@ def getSekkaDbInfo( env )
|
|
50
56
|
end
|
51
57
|
end
|
52
58
|
|
59
|
+
def checkJisyoIsInstalled( dictType, dictSource )
|
60
|
+
key = "SEKKA::VERSION"
|
61
|
+
STDERR.printf( "Info: Checking SEKKA jisyo on #{dictType} server...\n" )
|
62
|
+
begin
|
63
|
+
result = case dictType
|
64
|
+
when :tokyocabinet
|
65
|
+
require 'tokyocabinet'
|
66
|
+
require 'sekka/kvs'
|
67
|
+
tc = Kvs.new( :tokyocabinet )
|
68
|
+
tc.open( dictSource )
|
69
|
+
ret = tc.get( key )
|
70
|
+
tc.close()
|
71
|
+
ret
|
72
|
+
when :redis
|
73
|
+
begin
|
74
|
+
require 'redis'
|
75
|
+
require 'sekka/kvs'
|
76
|
+
redis = Kvs.new( :redis )
|
77
|
+
redis.open( dictSource )
|
78
|
+
ret = redis.get( key )
|
79
|
+
redis.close()
|
80
|
+
ret
|
81
|
+
rescue => e
|
82
|
+
STDERR.printf( "Error: sekka-server can't connect to redis(port=%s)...\n", dictSource )
|
83
|
+
exit 1
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
53
88
|
|
54
89
|
def main
|
55
90
|
if not File.directory?( DICTDIR )
|
@@ -115,38 +150,24 @@ def main
|
|
115
150
|
# .tchファイルが存在するか調べる
|
116
151
|
list = TC_FILE_LIST.select { |name| File.exist?( name ) }
|
117
152
|
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
STDERR.printf( "
|
123
|
-
|
124
|
-
|
125
|
-
# 中身を確認する(tchmgrで読めるものができている)
|
126
|
-
STDERR.printf( "Info: inform *.tch\n" )
|
127
|
-
cmd = sprintf( "tchmgr inform %s", TC_FILE )
|
153
|
+
# redisサーバーに辞書が投入済みか確認する
|
154
|
+
ok = checkJisyoIsInstalled( dictType, TC_FILE )
|
155
|
+
if 0 == list.size and (not ok)
|
156
|
+
# tsvファイルをuploadする
|
157
|
+
STDERR.printf( "Info: Uploading...\n" )
|
158
|
+
cmd = sprintf( "sekka-jisyo restore %s %s", TSVFILE, TC_FILE )
|
128
159
|
STDERR.printf( "Command : %s\n", cmd )
|
129
160
|
system( cmd )
|
130
161
|
end
|
162
|
+
STDERR.printf( "Info: [OK]\n" )
|
131
163
|
|
132
164
|
# 存在する.tchファイルの中で一番最初のファイルを使う。
|
133
165
|
list = TC_FILE_LIST.select { |name| File.exist?( name ) }
|
134
166
|
dictSource = list[0]
|
135
167
|
when :redis
|
136
|
-
key = "SEKKA::VERSION"
|
137
168
|
# redisサーバーに辞書が投入済みか確認する
|
138
|
-
|
139
|
-
|
140
|
-
require 'sekka/kvs'
|
141
|
-
redis = Kvs.new( :redis )
|
142
|
-
begin
|
143
|
-
redis.open( dictSource )
|
144
|
-
result = redis.get( key )
|
145
|
-
rescue => e
|
146
|
-
STDERR.printf( "Error: sekka-server can't connect to redis(port=%s)...\n", dictSource )
|
147
|
-
exit 1
|
148
|
-
end
|
149
|
-
unless result
|
169
|
+
ok = checkJisyoIsInstalled( dictType, dictSource )
|
170
|
+
unless ok
|
150
171
|
# tsvファイルをuploadする。
|
151
172
|
STDERR.printf( "Info: Uploading...\n" )
|
152
173
|
cmd = sprintf( "sekka-jisyo restore %s %s:%s", TSVFILE, dictType, dictSource )
|
data/emacs/sekka.el
CHANGED
@@ -42,11 +42,11 @@ class ApproximateSearch
|
|
42
42
|
def search( userid, kvs, keyword, type )
|
43
43
|
arr = []
|
44
44
|
case userid
|
45
|
-
when "
|
46
|
-
arr = searchByUser( "
|
45
|
+
when "M"
|
46
|
+
arr = searchByUser( "M", kvs, keyword, type )
|
47
47
|
else
|
48
48
|
h = {}
|
49
|
-
searchByUser( "
|
49
|
+
searchByUser( "M", kvs, keyword, type ).each { |item| h[ item[1] ] = item[0] }
|
50
50
|
searchByUser( userid, kvs, keyword, type ).each { |item| h[ item[1] ] = item[0] }
|
51
51
|
h.keys.each { |k| arr << [ h[k], k ] }
|
52
52
|
end
|
@@ -56,11 +56,11 @@ class ApproximateSearch
|
|
56
56
|
def searchByUser( userid, kvs, keyword, type )
|
57
57
|
pair = case type
|
58
58
|
when 'k' # okuri nashi kanji entry
|
59
|
-
["Ik
|
59
|
+
["Ik:" + userid + ":", keyword.downcase]
|
60
60
|
when 'K' # okuri ari kanji entry
|
61
|
-
["IK
|
61
|
+
["IK:" + userid + ":", keyword]
|
62
62
|
when 'h' # hiragana phrase entry
|
63
|
-
["Ih
|
63
|
+
["Ih:" + userid + ":", keyword.downcase]
|
64
64
|
else
|
65
65
|
raise sprintf( "Error: ApproximateSearch#search unknown type %s ", type )
|
66
66
|
end
|
data/lib/sekka/convert-jisyo.nnd
CHANGED
@@ -36,13 +36,13 @@
|
|
36
36
|
(use srfi-1)
|
37
37
|
(use sekka.util)
|
38
38
|
(use sekka.roman-lib)
|
39
|
-
|
39
|
+
(require "progressbar")
|
40
40
|
|
41
41
|
(define (expand-hiragana-phrase-entry key)
|
42
42
|
(let1 roman-list (gen-hiragana->roman-list key)
|
43
43
|
(if (< 1000 (length roman-list))
|
44
44
|
(begin
|
45
|
-
|
45
|
+
(sprintf " Warning: ignored entry [%s] (hiragana phrase), because too many pattens.\n" key)
|
46
46
|
#f) ;; パターン数が爆発した単語は無視する
|
47
47
|
(map
|
48
48
|
(lambda (x)
|
@@ -53,7 +53,7 @@
|
|
53
53
|
(let1 roman-list (gen-hiragana->roman-list key)
|
54
54
|
(if (< 1000 (length roman-list))
|
55
55
|
(begin
|
56
|
-
|
56
|
+
(sprintf " Warning: ignored entry [%s %s] , because too many pattens.\n" key value)
|
57
57
|
#f) ;; パターン数が爆発した単語は無視する
|
58
58
|
(append
|
59
59
|
(map
|
@@ -85,81 +85,74 @@
|
|
85
85
|
|
86
86
|
|
87
87
|
(define (convert-skk-jisyo-f f)
|
88
|
-
(
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
(expand-okuri-nashi-entry (first fields) (second fields)))
|
123
|
-
((rxmatch #/^([>あ-んー]+)([a-z])$/ (first fields))
|
124
|
-
=> (lambda (m)
|
125
|
-
;; 送り仮名ありデータ
|
126
|
-
(expand-okuri-ari-entry (rxmatch-substring m 1)
|
127
|
-
(rxmatch-substring m 2)
|
88
|
+
(let1 progress (ProgressBar.new "convert " (file-length f) STDERR)
|
89
|
+
|
90
|
+
(define (gen-sekka-entries line)
|
91
|
+
(progress.inc)
|
92
|
+
|
93
|
+
(let* ((line (line.sub #/\/$/ ""))
|
94
|
+
(fields (split-dict-line line)))
|
95
|
+
(cond
|
96
|
+
((rxmatch #/^\;/ line)
|
97
|
+
;; コメント行
|
98
|
+
#f)
|
99
|
+
((not fields)
|
100
|
+
;; フォーマットエラー
|
101
|
+
#f)
|
102
|
+
((rxmatch #/\;\;/ line)
|
103
|
+
;; フォーマットエラー
|
104
|
+
#f)
|
105
|
+
((and (is-hiragana (first fields))
|
106
|
+
(= 1 (length fields)))
|
107
|
+
;; 平仮名フレーズ
|
108
|
+
(expand-hiragana-phrase-entry (first fields)))
|
109
|
+
((and (is-hiragana (first fields))
|
110
|
+
(rxmatch #/^\/$/ (second fields)))
|
111
|
+
;; 平仮名フレーズ
|
112
|
+
(expand-hiragana-phrase-entry (first fields)))
|
113
|
+
((or (is-hiragana (first fields))
|
114
|
+
(rxmatch #/^([>#あ-んー]+)$/ (first fields)))
|
115
|
+
;; 送り仮名なしデータ
|
116
|
+
(expand-okuri-nashi-entry (first fields) (second fields)))
|
117
|
+
((rxmatch #/^([>あ-んー]+)([a-z])$/ (first fields))
|
118
|
+
=> (lambda (m)
|
119
|
+
;; 送り仮名ありデータ
|
120
|
+
(expand-okuri-ari-entry (rxmatch-substring m 1)
|
121
|
+
(rxmatch-substring m 2)
|
128
122
|
(second fields))))
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
123
|
+
((rxmatch #/[亜-瑤]+/ (first fields))
|
124
|
+
;; 漢字が1文字でも含まれている
|
125
|
+
#f)
|
126
|
+
((rxmatch #/[#]+/ (first fields))
|
127
|
+
;; # が1文字でも含まれている
|
128
|
+
#f)
|
129
|
+
((rxmatch #/^[ ]*$/ (first fields) )
|
130
|
+
;; 空文字列
|
131
|
+
#f)
|
132
|
+
((= 1 (length fields))
|
133
|
+
;; フィールドが1件しかない。
|
134
|
+
#f)
|
135
|
+
(else
|
136
|
+
(list (cons (first fields) (second fields)))))))
|
137
|
+
|
138
|
+
(let* ([lines
|
139
|
+
(map
|
140
|
+
(lambda (line)
|
141
|
+
(line.chomp))
|
142
|
+
(f.readlines))]
|
143
|
+
[result '#()])
|
144
|
+
(for-each
|
145
|
+
(lambda (x)
|
146
|
+
(let1 entries (gen-sekka-entries x)
|
147
|
+
(when entries
|
148
|
+
(for-each
|
149
|
+
(lambda (entry)
|
150
|
+
(result.push (sprintf "%s %s" (car entry) (cdr entry))))
|
151
|
+
entries))))
|
152
|
+
lines)
|
153
|
+
|
154
|
+
(to-list result))))
|
160
155
|
|
161
|
-
(to-list result)))
|
162
|
-
|
163
156
|
|
164
157
|
|
165
158
|
|
data/lib/sekka/henkan.nnd
CHANGED
@@ -377,7 +377,7 @@
|
|
377
377
|
(define (sekkaHenkan userid kvs cachesv keyword limit roman-method)
|
378
378
|
(define cache-exp-second (* 60 60))
|
379
379
|
(let* ((keyword (keyword.strip))
|
380
|
-
(sekka-keyword (+ "sekka
|
380
|
+
(sekka-keyword (+ "sekka:" roman-method ":" (limit.to_s) ":" keyword)))
|
381
381
|
(if-let1 fetched (and cachesv
|
382
382
|
(cachesv.get sekka-keyword #f))
|
383
383
|
(begin
|
@@ -389,13 +389,13 @@
|
|
389
389
|
(sekka-henkan userid kvs keyword limit (make-keyword roman-method)))
|
390
390
|
(and cachesv
|
391
391
|
(not (null? henkan-result))
|
392
|
-
(let1 fetched2 (cachesv.get "sekka
|
392
|
+
(let1 fetched2 (cachesv.get "sekka:(keys)" #f)
|
393
393
|
(cachesv.put! sekka-keyword (write-to-string henkan-result) cache-exp-second)
|
394
|
-
(cachesv.put! "sekka
|
394
|
+
(cachesv.put! "sekka:(keys)" (if fetched2
|
395
395
|
(+ fetched2 " " sekka-keyword)
|
396
396
|
sekka-keyword)
|
397
397
|
cache-exp-second)
|
398
|
-
(if-let1 v (cachesv.get "sekka
|
398
|
+
(if-let1 v (cachesv.get "sekka:(keys)" #f)
|
399
399
|
(begin
|
400
400
|
#?=(v.size)
|
401
401
|
#?=v)
|
@@ -408,13 +408,13 @@
|
|
408
408
|
;; Flush henkan-result cache data on cachesv(memcached)
|
409
409
|
(define (flushCacheServer cachesv)
|
410
410
|
#?="--- do (flushCacheServer) ---"
|
411
|
-
(if-let1 fetched (cachesv.get "sekka
|
411
|
+
(if-let1 fetched (cachesv.get "sekka:(keys)" #f)
|
412
412
|
(begin
|
413
413
|
(for-each
|
414
414
|
(lambda (x)
|
415
415
|
(cachesv.delete #?=x))
|
416
416
|
(to-list (fetched.split #/[ ]+/)))
|
417
|
-
(cachesv.delete "sekka
|
417
|
+
(cachesv.delete "sekka:(keys)"))
|
418
418
|
#f))
|
419
419
|
;; Export to Ruby world
|
420
420
|
(export-to-ruby flushCacheServer)
|
@@ -429,7 +429,7 @@
|
|
429
429
|
(dict-get userid kvs key #f))
|
430
430
|
|
431
431
|
(define (write-user-entry userid kvs key value)
|
432
|
-
(kvs.put! (+ userid "
|
432
|
+
(kvs.put! (+ userid ":" key) value))
|
433
433
|
|
434
434
|
(define (join-henkan-kouho lst)
|
435
435
|
(+ "/"
|
@@ -474,7 +474,7 @@
|
|
474
474
|
;; ユーザー辞書の全削除 (useridで指定したユーザーの辞書)
|
475
475
|
;; 削除した件数を返す。
|
476
476
|
(define (flushUserJisyo userid kvs)
|
477
|
-
(let1 lst (to-list (kvs.forward_match_keys (+ userid "
|
477
|
+
(let1 lst (to-list (kvs.forward_match_keys (+ userid ":")))
|
478
478
|
(for-each
|
479
479
|
(lambda (k)
|
480
480
|
(kvs.delete k))
|
@@ -499,13 +499,13 @@
|
|
499
499
|
|
500
500
|
;; "ユーザー語彙を"(stored)"にpush!する"
|
501
501
|
(define (kvs-push! userid kvs entry-str)
|
502
|
-
(let* ((orig (kvs.get (+ userid "
|
502
|
+
(let* ((orig (kvs.get (+ userid ":(stored)") "()"))
|
503
503
|
(orig (read-from-string orig)))
|
504
|
-
(kvs.put! (+ userid "
|
504
|
+
(kvs.put! (+ userid ":(stored)") (write-to-string (append orig (list entry-str))))
|
505
505
|
entry-str))
|
506
506
|
|
507
507
|
(cond
|
508
|
-
((memv dict-line (read-from-string (kvs.get (+ userid "
|
508
|
+
((memv dict-line (read-from-string (kvs.get (+ userid ":(stored)") "()")))
|
509
509
|
#?=(+ "already stored userid=[" userid "] tango=[" dict-line "]")
|
510
510
|
#f)
|
511
511
|
(else
|
data/lib/sekka/jisyo-db.nnd
CHANGED
@@ -40,8 +40,8 @@
|
|
40
40
|
(require "distributedtrie")
|
41
41
|
(use sekka.util)
|
42
42
|
|
43
|
-
(define master-prefix "
|
44
|
-
(define version-prefix "SEKKA
|
43
|
+
(define master-prefix "M")
|
44
|
+
(define version-prefix "SEKKA:VERSION")
|
45
45
|
(define okuri-ari-prefix "IK")
|
46
46
|
(define okuri-nashi-prefix "Ik")
|
47
47
|
(define hiragana-phrase-prefix "Ih")
|
@@ -56,12 +56,12 @@
|
|
56
56
|
|
57
57
|
|
58
58
|
;; dict-get search sequence
|
59
|
-
;; (1) try "userid
|
60
|
-
;; (2) try "
|
59
|
+
;; (1) try "userid:keyword" key
|
60
|
+
;; (2) try "M:keyword" key
|
61
61
|
(define (dict-get userid kvs key . fallback)
|
62
62
|
(if-let1 value
|
63
|
-
(or (kvs.get (+ userid "
|
64
|
-
(kvs.get (+ master-prefix "
|
63
|
+
(or (kvs.get (+ userid ":" key) #f)
|
64
|
+
(kvs.get (+ master-prefix ":" key) #f))
|
65
65
|
value
|
66
66
|
(let1 opt (get-optional fallback #f)
|
67
67
|
opt)))
|
@@ -119,18 +119,18 @@
|
|
119
119
|
(length okuri-nashi-list)
|
120
120
|
(length hiragana-phrase-list))))))
|
121
121
|
|
122
|
-
(let1 trie (DistributedTrie::Trie.new kvs (+ okuri-ari-prefix "
|
122
|
+
(let1 trie (DistributedTrie::Trie.new kvs (+ okuri-ari-prefix ":" userid ":"))
|
123
123
|
(trie-add-autocommit trie okuri-ari-list progress))
|
124
124
|
|
125
|
-
(let1 trie (DistributedTrie::Trie.new kvs (+ okuri-nashi-prefix "
|
125
|
+
(let1 trie (DistributedTrie::Trie.new kvs (+ okuri-nashi-prefix ":" userid ":"))
|
126
126
|
(trie-add-autocommit trie okuri-nashi-list progress))
|
127
127
|
|
128
|
-
(let1 trie (DistributedTrie::Trie.new kvs (+ hiragana-phrase-prefix "
|
128
|
+
(let1 trie (DistributedTrie::Trie.new kvs (+ hiragana-phrase-prefix ":" userid ":"))
|
129
129
|
(trie-add-autocommit trie hiragana-phrase-list progress)))))
|
130
130
|
|
131
131
|
|
132
132
|
(define (append-entry userid kvs _key value)
|
133
|
-
(let1 key (+ userid "
|
133
|
+
(let1 key (+ userid ":" _key)
|
134
134
|
(cond
|
135
135
|
((rxmatch #/^=/ _key)
|
136
136
|
;; hiragana-phrase entry
|
@@ -161,15 +161,6 @@
|
|
161
161
|
(kvs.put! key value))))))
|
162
162
|
|
163
163
|
|
164
|
-
(define (file-length f)
|
165
|
-
(let1 total 0
|
166
|
-
(for-each
|
167
|
-
(lambda (x)
|
168
|
-
(set! total (+ 1 total)))
|
169
|
-
f)
|
170
|
-
(f.rewind)
|
171
|
-
total))
|
172
|
-
|
173
164
|
(define (load-sekka-jisyo-f f filename)
|
174
165
|
(define keylist '())
|
175
166
|
(define (create-keylist kvs f)
|