sekka 1.0.0 → 1.1.0.pre

data/README.md

@@ -0,0 +1,8 @@
+# Sekka (石火) SKK like Japanese input method
+
+ [![Build Status](https://secure.travis-ci.org/kiyoka/sekka.png)](http://travis-ci.org/kiyoka/sekka)
+
+##  Sekka(石火)はSKKライクな日本語入力メソッドです。
+ <http://oldtype.sumibi.org/show-page/Sekka>
+
+ ![Logo]( http://pix.am/u2Hv.png )

data/Rakefile

@@ -20,8 +20,10 @@
 
 require 'rake'
 begin
-  require 'jeweler'
+  require 'jeweler2'
   Jeweler::Tasks.new do |gemspec|
+    vh = Jeweler::VersionHelper.new "."
+    gemspec.version = vh.to_s + ".pre"
     gemspec.name = "sekka"
     gemspec.summary = "Sekka is a SKK like input method."
     gemspec.description = "Sekka is a SKK like input method. Sekka server provides REST Based API. If you are SKK user, let's try it."
@@ -52,14 +54,14 @@ begin
                            "sekka-path"]
     gemspec.required_ruby_version = '>= 1.9.1'
     gemspec.add_dependency( "eventmachine" )
-    gemspec.add_dependency( "fuzzy-string-match", ">= 0.9.2" )
     gemspec.add_dependency( "memcache-client" )
     gemspec.add_dependency( "nendo", "= 0.6.4" )
+    gemspec.add_dependency( "distributed-trie" )
     gemspec.add_dependency( "rack" )
     gemspec.add_dependency( "ruby-progressbar" )
   end
 rescue LoadError
-  puts "Jeweler not available. Install it with: sudo gem install jeweler"
+  puts "Jeweler2 not available. Install it with: sudo gem install jeweler2"
 end
 
 
@@ -68,7 +70,7 @@ end
 
 task :compile do
   # generate version.rb
-  dictVersion = "0.9.2"
+  dictVersion = "1.2.0"
   vh = Jeweler::VersionHelper.new "."
   open( "./lib/sekka/sekkaversion.rb", "w" ) {|f|
     f.puts(   "class SekkaVersion" )

data/VERSION.yml

@@ -1,4 +1,4 @@
 ---
 :major: 1
-:minor: 0
+:minor: 1
 :patch: 0

data/bin/sekka-server

@@ -16,16 +16,17 @@ DICTURL = "https://raw.github.com/kiyoka/sekka/master/public_dict/" + SekkaVersi
 
 PIDFILE = DICTDIR + "/pid"
 
-TC_FILE = DICTDIR + "/SEKKA-JISYO.SMALL.tch"
+TC_OPTS = "#xmsiz=256m"
+TC_FILE = DICTDIR + "/SEKKA-JISYO.SMALL.tch" + TC_OPTS
 TSVFILE = DICTDIR + "/SEKKA-JISYO.SMALL.tsv"
 SUMFILE = DICTDIR + "/SEKKA-JISYO.SMALL.md5"
 
 URLURL  = DICTURL + "/SEKKA-JISYO.SMALL.url"
 SUMURL  = DICTURL + "/SEKKA-JISYO.SMALL.md5"
 
-TC_FILE_LIST = [ DICTDIR + "/SEKKA-JISYO.CUSTOM.tch",
-                 DICTDIR + "/SEKKA-JISYO.LARGE.tch",
-                 DICTDIR + "/SEKKA-JISYO.SMALL.tch" ]
+TC_FILE_LIST = [ DICTDIR + "/SEKKA-JISYO.CUSTOM.tch" + TC_OPTS,
+                 DICTDIR + "/SEKKA-JISYO.LARGE.tch"  + TC_OPTS,
+                 DICTDIR + "/SEKKA-JISYO.SMALL.tch"  + TC_OPTS ]
 
 MEMCACHED = "localhost:11211"                # memcahced
 
@@ -111,7 +112,7 @@ def main
 
   case dictType
   when :tokyocabinet
-    # .tchファイルが存在すうるか調べる
+    # .tchファイルが存在するか調べる
     list = TC_FILE_LIST.select { |name| File.exist?( name ) }
 
     if 0 == list.size

data/emacs/sekka.el

@@ -1620,7 +1620,7 @@ point から行頭方向に同種の文字列が続く間を漢字変換しま
 (setq default-input-method "japanese-sekka")
 
 (defconst sekka-version
-  "1.0.0" ;;SEKKA-VERSION
+  "1.1.0" ;;SEKKA-VERSION
   )
 (defun sekka-version (&optional arg)
   "入力モード変更"

data/lib/sekka/approximatesearch.rb

@@ -31,46 +31,42 @@
 #  
 #  $Id: 
 #
-require 'fuzzystringmatch'
+require 'distributedtrie'
 require 'sekka/kvs'
 
 class ApproximateSearch
   def initialize( jarow_shikii )
     @jarow_shikii = jarow_shikii
-    @jarow        = FuzzyStringMatch::JaroWinkler.create( :native )
-  end
-
-  def filtering( keyword, arr )
-    keyword = keyword.downcase
-    arr.map { |str|
-      val = @jarow.getDistance( keyword, str.downcase )
-      #printf( "   [%s] vs [%s] => %f\n", keyword, str.downcase, val )
-      (val > @jarow_shikii) ? [ val, str ] : false
-    }.select { |v| v }.sort_by {|item| 1.0 - item[0]}
   end
 
   def search( userid, kvs, keyword, type )
-    readymade_key = case type
-                    when 'k' # okuri nashi kanji entry
-                      "(" + keyword.slice( 0, 2 ).downcase + ")"
-                    when 'K' # okuri ari   kanji entry
-                      "(" + keyword.slice( 0, 2 ).upcase   + ")"
-                    when 'h' # hiragana phrase entry
-                      "{" + keyword.slice( 1, 2 ).downcase + "}"
-                    else
-                      raise sprintf( "Error: ApproximateSearch#search unknown type %s ", type )
-                    end
-
-    str = kvs.get( userid + "::" + readymade_key, false )
-    if not str
-      str = kvs.get( "MASTER::" + readymade_key )
-    end
-
-    #printf( "#readymade_key %s : %s\n", readymade_key, str )
-    if str
-      filtering( keyword, str.split( /[ ]+/ ))
+    arr = []
+    case userid
+    when "MASTER"
+      arr  = searchByUser( "MASTER", kvs, keyword, type )
     else
-      [ ]
+      h = {}
+      searchByUser( "MASTER", kvs, keyword, type ).each { |item| h[ item[1] ] = item[0] }
+      searchByUser(  userid,  kvs, keyword, type ).each { |item| h[ item[1] ] = item[0] }
+      h.keys.each { |k|  arr << [ h[k], k ] }
     end
+    arr.sort_by {|item| [1.0 - item[0], item[1]]}
+  end
+
+  def searchByUser( userid, kvs, keyword, type )
+    pair = case type
+           when 'k' # okuri nashi kanji entry
+             ["Ik::" + userid + "::", keyword.downcase]
+           when 'K' # okuri ari   kanji entry
+             ["IK::" + userid + "::", keyword]
+           when 'h' # hiragana phrase entry
+             ["Ih::" + userid + "::", keyword.downcase]
+           else
+             raise sprintf( "Error: ApproximateSearch#search unknown type %s ", type )
+           end
+    prefix   = pair[0]
+    _keyword = pair[1]
+    trie = DistributedTrie::Trie.new( kvs, prefix )
+    trie.fuzzySearch( _keyword, @jarow_shikii )
   end
 end

data/lib/sekka/henkan.nnd

@@ -496,45 +496,6 @@
          (set! user-keylist (cons k user-keylist))
          (append-entry userid kvs k v)))
      sekka-jisyo-data))
-    
-  (define (_create-ready-made-keylist keylist)
-    (receive (okuri-ari-hash
-              okuri-nashi-hash
-              hiragana-phrase-hash)
-        (create-2char-hash keylist)
-      
-      ;; OKURI-ARI
-      (for-each
-       (lambda (key)
-         (let1 fetched (kvs.get  (+ userid "::" "(" (sekka-upcase key) ")")
-                                 (kvs.get  (+ masterid "::" "(" (sekka-upcase key) ")") ""))
-           (kvs.put! (+ userid "::" "(" (sekka-upcase key) ")")
-                     (string-join (uniq (sort (append (to-list (fetched.split #/[ ]+/))
-                                                      (hash-table-get okuri-ari-hash key))))
-                                  " "))))
-       (hash-table-keys okuri-ari-hash))
-      
-      ;; OKURI-NASHI
-      (for-each
-       (lambda (key)
-         (let1 fetched (kvs.get  (+ userid "::" "(" (sekka-downcase key) ")")
-                                 (kvs.get  (+ masterid "::" "(" (sekka-downcase key) ")") ""))
-           (kvs.put! (+ userid "::" "(" (sekka-downcase key) ")")
-                     (string-join (uniq (sort (append (to-list (fetched.split #/[ ]+/))
-                                                      (hash-table-get okuri-nashi-hash key))))
-                                  " "))))
-       (hash-table-keys okuri-nashi-hash))
-
-      ;; HIRAGANA-PHRASE
-      (for-each
-       (lambda (key)
-         (let1 fetched (kvs.get  (+ userid "::" "{" (sekka-downcase key) "}")
-                                 (kvs.get  (+ masterid "::" "{" (sekka-downcase key) "}") ""))
-           (kvs.put! (+ userid "::" "{" (sekka-downcase key) "}")
-                     (string-join (uniq (sort (append (to-list (fetched.split #/[ ]+/))
-                                                      (hash-table-get hiragana-phrase-hash key))))
-                                  " "))))
-       (hash-table-keys hiragana-phrase-hash))))
 
   ;; "ユーザー語彙を"(stored)"にpush!する"
   (define (kvs-push! userid kvs entry-str)
@@ -557,7 +518,7 @@
               #f)
             (begin
               (insert-to-db lst)
-              (_create-ready-made-keylist user-keylist)
+              (setup-trie-index userid kvs user-keylist #f)
               (kvs-push! userid kvs dict-line)
               #?=(+ "user dict stored userid=[" userid "] tango=[" dict-line "]")
               #t))))

data/lib/sekka/jisyo-db.nnd

@@ -37,22 +37,14 @@
 (require "progressbar")
 (require "sekka/kvs")
 (require "sekka/sekkaversion")
+(require "distributedtrie")
 (use sekka.util)
 
-(define masterid  "MASTER")
-(define workid    "WORK")
-(define versionid "SEKKA::VERSION")
-
-(define alphabet-string "abcdefghijklmnopqrstuvwxyz>@;#")
-(define alphabet-lower-list (to-list (alphabet-string.split "")))
-(define alphabet-upper-list (to-list (. (sekka-upcase alphabet-string) split "")))
-(define alphabet-pairs (append-map
-                        (lambda (a)
-                          (map
-                           (lambda (b)
-                             (+ a b))
-                           (append alphabet-upper-list alphabet-lower-list)))
-                        alphabet-lower-list))
+(define master-prefix  "MASTER")
+(define version-prefix "SEKKA::VERSION")
+(define okuri-ari-prefix       "IK")
+(define okuri-nashi-prefix     "Ik")
+(define hiragana-phrase-prefix "Ih")
 
 ;; KVS type setting
 (define *kvs-type* 'tokyocabinet)  ;; default
@@ -69,69 +61,72 @@
 (define (dict-get userid kvs key . fallback)
   (if-let1 value
       (or (kvs.get (+ userid "::" key) #f)
-          (kvs.get (+ masterid "::" key) #f))
+          (kvs.get (+ master-prefix "::" key) #f))
     value
     (let1 opt (get-optional fallback #f)
       opt)))
 
-(define (create-2char-hash keylist)
-  (define okuri-ari-hash         (make-hash-table))
-  (define okuri-nashi-hash       (make-hash-table))
-  (define hiragana-phrase-hash   (make-hash-table))
-  (define (create-hash keylist)
-    (for-each
-     (lambda (k)
-       (when (rxmatch #/^[=a-zA-Z#^>-@`\;+:'\-]+$/ k)
-         (if (rxmatch #/^=/ k)
-             ;; HIRAGANA-PHRASE
-             (let1 sliced (sekka-downcase (k.slice 1 2))
-               (when (= 2 sliced.size)
-                 (hash-table-push! hiragana-phrase-hash   sliced k)))
-             ;; OKURI-ARI and OKURI-NASHI
-             (let1 sliced (sekka-downcase (k.slice 0 2))
-               (when (= 2 sliced.size)
-                 (if (rxmatch #/[A-Z`+]$/ k)
-                     (hash-table-push! okuri-ari-hash   sliced k)
-                     (hash-table-push! okuri-nashi-hash sliced k)))))))
-     keylist))
-  (create-hash keylist)
-  (values okuri-ari-hash
-          okuri-nashi-hash
-          hiragana-phrase-hash))
-
-(define (setup-ready-made-keylist kvs keylist)
-  (for-each
-   (lambda (key)
-     (let1 key (+ masterid "::" key)
-       (unless (kvs.get key #f)
-         (kvs.put! key ""))))
-   alphabet-pairs)
-  
-  (receive (okuri-ari-hash
-            okuri-nashi-hash
-            hiragana-phrase-hash)
-      (create-2char-hash keylist)
-
-    ;; OKURI-ARI
-    (for-each
-     (lambda (key)
-       (kvs.put! (+ masterid "::" "(" (sekka-upcase key) ")")
-                 (string-join (uniq (sort (hash-table-get okuri-ari-hash key))) " ")))
-     (hash-table-keys okuri-ari-hash))
 
-    ;; OKURI-NASHI
-    (for-each
-     (lambda (key)
-       (kvs.put! (+ masterid "::" "(" (sekka-downcase key) ")")
-                 (string-join (uniq (sort (hash-table-get okuri-nashi-hash key))) " ")))
-     (hash-table-keys okuri-nashi-hash))
+(define (setup-trie-index userid kvs keylist displayProgress)
 
-    ;; HIRAGANA-PHRASE
-    (for-each
-     (lambda (key)
-       (kvs.put! (+ masterid "::" "{" (sekka-downcase key) "}")
-                 (string-join (uniq (sort (hash-table-get hiragana-phrase-hash key))) " ")))
-     (hash-table-keys hiragana-phrase-hash))))
+  (define (trie-add-autocommit trie keylist progress)
+    (let ([i 0])
+      (for-each
+       (lambda (k)
+         (trie.addKey! k)
+         (set! i (+ i 1))
+         (when (= 0 (% i 100))
+           (trie.commit!))
+         (when progress
+           (progress.inc)))
+       keylist))
+    trie.commit!)
+
+  (define (grouping-keylist keylist progress)
+    (let ([okuri-ari-list         '()]
+          [okuri-nashi-list       '()]
+          [hiragana-phrase-list   '()])
+      (for-each
+       (lambda (k)
+         (if (rxmatch #/^[=a-zA-Z#^>-@`\;+:'\-]+$/ k)
+             (if (rxmatch #/^=/ k)
+                 ;; HIRAGANA-PHRASE
+                 (push! hiragana-phrase-list k)
+                 ;; OKURI-ARI and OKURI-NASHI
+                 (if (rxmatch #/[A-Z`+]$/ k)
+                     (push! okuri-ari-list   k)
+                     (push! okuri-nashi-list k))))
+         (when progress
+           (progress.inc)))
+       keylist)
+      (values okuri-ari-list
+              okuri-nashi-list
+              hiragana-phrase-list)))
+
+  (let1 progress (if displayProgress
+                     (ProgressBar.new "trie-tree  "  (* 2 (length keylist)) STDERR)
+                     #f)
+    (receive (okuri-ari-list
+              okuri-nashi-list
+              hiragana-phrase-list)
+        (grouping-keylist keylist progress)
+
+      (when progress 
+        (progress.set (+ (length keylist)
+                         (- (length keylist)
+                            (+
+                             (length okuri-ari-list)
+                             (length okuri-nashi-list)
+                             (length hiragana-phrase-list))))))
+
+      (let1 trie (DistributedTrie::Trie.new kvs (+ okuri-ari-prefix "::" userid "::"))
+        (trie-add-autocommit trie   okuri-ari-list       progress))
+      
+      (let1 trie (DistributedTrie::Trie.new kvs (+ okuri-nashi-prefix "::" userid "::"))
+        (trie-add-autocommit trie   okuri-nashi-list     progress))
+      
+      (let1 trie (DistributedTrie::Trie.new kvs (+ hiragana-phrase-prefix "::" userid "::"))
+        (trie-add-autocommit trie   hiragana-phrase-list progress)))))
 
 
 (define (append-entry userid kvs _key value)
@@ -166,21 +161,34 @@
         (kvs.put! key value))))))
 
 
+(define (file-length f)
+  (let1 total 0
+    (for-each
+     (lambda (x)
+       (set! total (+ 1 total)))
+     f)
+    (f.rewind)
+    total))
+
 (define (load-sekka-jisyo-f f filename)
   (define keylist '())
   (define (create-keylist kvs f)
-    (for-each
-     (lambda (line)
-       (let1 fields (split-dict-line line)
-         (set! keylist (cons (first fields) keylist))
-         (append-entry masterid kvs (first fields) (second fields))))
-     (f.readlines)))
+    (let1 progress (ProgressBar.new "data-body  " (file-length f) STDERR)
+      (for-each
+       (lambda (line)
+         (let* (
+                [line (line.chomp)]
+                [fields (split-dict-line line)])
+           (push! keylist (first fields))
+           (append-entry master-prefix kvs (first fields) (second fields)))
+         (progress.inc))
+       f)))
   (let1 kvs (Kvs.new (get-kvs-type))
     (kvs.open filename)
     (kvs.clear)
     (create-keylist kvs f)
-    (setup-ready-made-keylist kvs keylist)
-    (kvs.put! versionid SekkaVersion.version)
+    (setup-trie-index master-prefix kvs keylist #t)
+    (kvs.put! version-prefix SekkaVersion.version)
     (kvs.close)))
 
 
@@ -196,15 +204,6 @@
 
 
 (define (restore-sekka-jisyo-f f filename-or-hostname)
-  (define (file-length f)
-    (let1 total 0
-      (for-each
-       (lambda (x)
-         (set! total (+ 1 total)))
-       f)
-      (f.rewind)
-      total))
-    
   (define (restore f kvs progress)
     (let loop ([line (f.readline.chomp)])
       (let* (

data/lib/sekka/sekkaversion.rb

@@ -1,4 +1,4 @@
 class SekkaVersion
-  def  self.version()     "1.0.0"  end
-  def  self.dictVersion() "0.9.2"  end
+  def  self.version()     "1.1.0"  end
+  def  self.dictVersion() "1.2.0"  end
 end

data/lib/sekkaserver.rb

@@ -57,12 +57,20 @@ module SekkaServer
       (@kvs,@cachesv) = @core.openSekkaJisyo( SekkaServer::Config.dictType,
                                               SekkaServer::Config.dictSource,
                                               SekkaServer::Config.cacheSource )
+
+      version = @kvs.get( "SEKKA::VERSION" )
+      if not SekkaVersion.dictVersion == version
+        STDERR.printf(   "Sekka Error: require dict version [%s] but got [%s].\n", SekkaVersion.dictVersion, version )
+        exit( 1 )
+      end
+
       @queue = EM::Queue.new
       @mutex = Mutex.new
 
       STDERR.puts(   "----- Sekka Server Started -----" )
       STDERR.printf( "  Sekka version  : %s\n", SekkaVersion.version            )
       STDERR.printf( "  Nendo version  : %s\n", Nendo::Core.version             )
+      STDERR.printf( "  dict  version  : %s\n", SekkaVersion.dictVersion        )
       STDERR.printf( "  dict-type      : %s\n", SekkaServer::Config.dictType    )
       STDERR.printf( "  dict-db        : %s\n", SekkaServer::Config.dictSource  )
       STDERR.printf( "  memcached      : %s\n", SekkaServer::Config.cacheSource )