sekka 0.9.6 → 0.9.7

data/.gemtest

@@ -0,0 +1 @@
+

data/Rakefile

@@ -0,0 +1,217 @@
+# -*- mode: ruby; -*-
+#                                                  Rakefile for Sekka
+#
+# Release Engineering:
+#   1. edit the VERSION.yml file
+#   2. rake compile  &&   rake test   &&   rake gemspec   &&   rake build
+#      to generate sekka-x.x.x.gem
+#   3. install sekka-x.x.x.gem to clean environment and test
+#   4. rake release
+#   5. gem push pkg/sekka-x.x.x.gem   ( need gem version 1.3.6 or higer. Please "gem update --system" to update )
+#
+# Enviroment Variables:
+#   Please select from
+#     DB=dbm
+#     DB=tokyocabinet
+#     DB=redis
+#     DB=                (default)
+#     DB=all             dbm/tokyocabinet/redis
+#
+
+require 'rake'
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "sekka"
+    gemspec.summary = "Sekka is a SKK like input method."
+    gemspec.description = "Sekka is a SKK like input method. Sekka server provides REST Based API. If you are SKK user, let's try it."
+    gemspec.email = "kiyoka@sumibi.org"
+    gemspec.homepage = "http://github.com/kiyoka/sekka"
+    gemspec.authors = ["Kiyoka Nishiyama"]
+    gemspec.files = FileList['Rakefile',
+                             '.gemtest',
+                             'VERSION.yml',
+                             'README',
+                             'COPYING',
+                             'lib/*.rb',
+                             'lib/*.ru',
+                             'lib/sekka/*.rb',
+                             'lib/sekka/*.nnd',
+                             'bin/sekka-jisyo',
+                             'bin/sekka-server',
+                             'bin/sekka-benchmark',
+                             'bin/sekka-path',
+                             'test/*.nnd',
+                             'test/*.rb',
+                             'test/*.txt',
+                             'script/sekkaserver.*',
+                             'emacs/*.el'].to_a
+    gemspec.executables = ["sekka-jisyo",
+                           "sekka-server",
+                           "sekka-benchmark",
+                           "sekka-path"]
+    gemspec.required_ruby_version = '>= 1.9.1'
+    gemspec.add_dependency( "eventmachine" )
+    gemspec.add_dependency( "fuzzy-string-match", ">= 0.9.2" )
+    gemspec.add_dependency( "memcache-client" )
+    gemspec.add_dependency( "nendo", "= 0.6.4" )
+    gemspec.add_dependency( "rack" )
+    gemspec.add_dependency( "ruby-progressbar" )
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install jeweler"
+end
+
+
+task :default => [:test] do
+end
+
+task :compile do
+  # generate version.rb
+  dictVersion = "0.9.2"
+  vh = Jeweler::VersionHelper.new "."
+  open( "./lib/sekka/sekkaversion.rb", "w" ) {|f|
+    f.puts(   "class SekkaVersion" )
+    f.printf( "  def  self.version()     \"%s\"  end\n", vh )
+    f.printf( "  def  self.dictVersion() \"%s\"  end\n", dictVersion )
+    f.puts(   "end" )
+  }
+
+  # Replace Version Number
+  targetFile = "./emacs/sekka.el"
+  vh = Jeweler::VersionHelper.new "."
+  (original, modified) = open( targetFile, "r:utf-8" ) {|f|
+    lines = f.readlines
+    [ lines,
+      lines.map {|line|
+        if line.match( /;;SEKKA-VERSION/ )
+          sprintf( '  "%s" ;;SEKKA-VERSION', vh.to_s ) + "\n"
+        else
+          line
+        end
+      } ]
+  }
+  if original.join != modified.join
+    puts "Info: " + targetFile + " was updated."
+    open( targetFile, "w" ) {|f|
+      f.write( modified.join )
+    }
+  end
+end
+
+task :test do
+  sh "ruby -I ./lib ./bin/sekka-path > /tmp/path1"
+  path1 = open( "/tmp/path1" ) {|f|
+    f.readline.chomp
+  }
+  path2 = File.dirname( __FILE__ )
+  unless path1 == path2
+    puts STDERR.printf( "Error: on <sekka-path> requires [%s] but got [%s].", path2, path1 )
+    exit 1
+  end
+
+  sh "/bin/rm -f test.record test.tch"
+  files = []
+  files << "./test/memcache.nnd"
+  files << "./test/util.nnd"
+  files << "./test/alphabet-lib.nnd"
+  files << "./test/sharp-number.nnd"
+  files << "./test/roman-lib.nnd"
+  files << "./test/azik-verification.nnd"
+  files << "./test/jisyo.nnd"
+  files << "./test/google-ime.nnd"
+  STDERR.printf( "Info:  env DB=%s\n", ENV['DB'] )
+  case ENV['DB']
+  when 'dbm'
+    files << "./test/henkan-main.nnd  dbm"
+  when 'tokyocabinet'
+    files << "./test/henkan-main.nnd  tokyocabinet"
+  when 'redis'
+    files << "./test/redis.nnd"
+    files << "./test/henkan-main.nnd  redis"
+  when 'pure'
+    files << "./test/henkan-main.nnd  pure"
+  when 'all'
+    files << "./test/henkan-main.nnd  dbm"
+    files << "./test/henkan-main.nnd  tokyocabinet"
+    files << "./test/henkan-main.nnd  redis"
+    files << "./test/henkan-main.nnd  pure"
+  else # default
+    files << "./test/henkan-main.nnd  tokyocabinet"
+    files << "./test/henkan-main.nnd  pure"
+  end
+  files.each {|filename|
+    nendopath = `which nendo`.chomp
+    sh  sprintf( "time ruby -I ./lib %s %s", nendopath, filename )
+  }
+  sh "cat test.record"
+end
+
+task :bench do
+  sh "time ruby -I ./lib /usr/local/bin/nendo ./test/approximate-bench.nnd"
+  sh "time ruby -I ./lib /usr/local/bin/nendo ./test/henkan-bench.nnd"
+end
+
+task :alljisyo => [ :jisyoS, :jisyoL, :load, :dump  ]
+
+task :jisyoS do
+  sh "time ./bin/sekka-jisyo convert ./data/SKK-JISYO.L.201008           >  ./data/SEKKA-JISYO.SMALL"
+  sh "time ./bin/sekka-jisyo convert ./data/SKK-JISYO.L.hira-kata        >> ./data/SEKKA-JISYO.SMALL"
+  sh "time ./bin/sekka-jisyo convert ./data/SKK-JISYO.hiragana-phrase    >> ./data/SEKKA-JISYO.SMALL"
+  sh "time ./bin/sekka-jisyo convert ./data/SKK-JISYO.hiragana-phrase2   >> ./data/SEKKA-JISYO.SMALL"
+end
+
+task :jisyoL do
+  sh "time ./bin/sekka-jisyo convert ./data/SKK-JISYO.L.201008           >  ./data/SEKKA-JISYO.LARGE"
+  sh "time ./bin/sekka-jisyo convert ./data/SKK-JISYO.L.hira-kata        >> ./data/SEKKA-JISYO.LARGE"
+  sh "time ./bin/sekka-jisyo convert ./data/SKK-JISYO.fullname           >> ./data/SEKKA-JISYO.LARGE"
+  sh "time ./bin/sekka-jisyo convert ./data/SKK-JISYO.jinmei             >> ./data/SEKKA-JISYO.LARGE"
+  sh "time ./bin/sekka-jisyo convert ./data/SKK-JISYO.station            >> ./data/SEKKA-JISYO.LARGE"
+  sh "time ./bin/sekka-jisyo convert ./data/SKK-JISYO.hiragana-phrase    >> ./data/SEKKA-JISYO.LARGE"
+  sh "time ./bin/sekka-jisyo convert ./data/SKK-JISYO.hiragana-phrase2   >> ./data/SEKKA-JISYO.LARGE"
+end
+
+task :load do
+  sh "time ./bin/sekka-jisyo load    ./data/SEKKA-JISYO.SMALL  ./data/SEKKA-JISYO.SMALL.tch"
+  sh "time ./bin/sekka-jisyo load    ./data/SEKKA-JISYO.LARGE  ./data/SEKKA-JISYO.LARGE.tch"
+end
+
+task :dump do
+  sh "time ./bin/sekka-jisyo dump    ./data/SEKKA-JISYO.SMALL.tch > ./data/SEKKA-JISYO.SMALL.tsv"
+  sh "time ./bin/sekka-jisyo dump    ./data/SEKKA-JISYO.LARGE.tch > ./data/SEKKA-JISYO.LARGE.tsv"
+end
+
+
+# Fetched data from
+#   http://s-yata.jp/corpus/nwc2010/ngrams/
+task :phrase => [ "./data/6gm-0000.txt" ]  do
+  sh "time ruby -I ./lib /usr/local/bin/nendo ./data/hiragana_phrase_in_webcorpus.nnd           ./data/6gm-0000.txt | sort | uniq > /tmp/tmp.txt"
+  sh "time ruby -I ./lib /usr/local/bin/nendo ./data/writing_phrase_filter.nnd /tmp/tmp.txt  | sort | uniq     > ./data/SKK-JISYO.hiragana-phrase"
+end
+
+file "./data/6gm-0000.txt"  do
+  sh "wget http://dist.s-yata.jp/corpus/nwc2010/ngrams/word/over999/6gms/6gm-0000.xz -O /tmp/6gm-0000.xz"
+  sh "xz -cd /tmp/6gm-0000.xz > ./data/6gm-0000.txt"
+end
+
+task :phrase2 => [ "./data/ipadic.all.utf8.txt" ] do
+  sh "time ruby -I ./lib /usr/local/bin/nendo ./data/hiragana_phrase_in_ipadic.nnd             ./data/ipadic.all.utf8.txt | sort | uniq > ./data/SKK-JISYO.hiragana-phrase2"
+end
+
+file "./data/ipadic.all.utf8.txt" do
+  sh "wget http://chasen.aist-nara.ac.jp/stable/ipadic/ipadic-2.7.0.tar.gz -O /tmp/ipadic-2.7.0.tar.gz"
+  sh "tar zxfC /tmp/ipadic-2.7.0.tar.gz /tmp"
+  sh "iconv -f euc-jp -t utf-8 /tmp/ipadic-2.7.0/*.dic > ./data/ipadic.all.utf8.txt"
+end
+
+
+task :rackup do
+  # how to install mongrel is "gem install mongrel --pre"
+  sh "ruby -I ./lib ./bin/sekka-server"
+end
+
+task :katakanago do
+  sh "nkf --euc ./data/SKK-JISYO.L.201008 > tmpfile.euc"
+  sh "/usr/share/skktools/filters/abbrev-convert.rb -k tmpfile.euc | skkdic-expr2 | iconv -f=EUC-JP -t=UTF-8 > ./data/SKK-JISYO.L.hira-kata"
+  sh "/bin/rm -f tmpfile.euc"
+end

data/VERSION.yml

@@ -0,0 +1,4 @@
+---
+:major: 0
+:minor: 9
+:patch: 7

data/bin/sekka-server

@@ -5,8 +5,8 @@ require 'digest/md5'
 require 'fileutils'
 require 'rack'
 require 'uri'
-require File.expand_path(File.dirname(__FILE__) + "/../lib/sekkaconfig")
-require File.expand_path(File.dirname(__FILE__) + "/../lib/sekka/sekkaversion")
+require 'sekkaconfig'
+require 'sekka/sekkaversion'
 
 
 
@@ -160,8 +160,14 @@ def main
   # 設定項目をConfigオブジェクトに代入
   SekkaServer::Config.setup( dictType, dictSource, MEMCACHED, 12929, proxyHost, proxyPort )
 
-  # サーバースクリプトのrootディレクトリへ移動
-  FileUtils.cd(File.dirname(__FILE__) + "/../")
+  # rackに渡すための sekka.ru のインストールパスを求める。
+  vendordir = File.expand_path(File.dirname(__FILE__) + "/../lib")
+  if RbConfig::CONFIG[ 'vendordir' ]
+    if File.exists? RbConfig::CONFIG[ 'vendordir' ] + "/sekka.ru"
+      vendordir = RbConfig::CONFIG[ 'vendordir' ]
+    end
+  end
+  # print "vendordir = " + vendordir + "\n"
 
   # サーバー起動
   Rack::Server.start(
@@ -170,7 +176,7 @@ def main
                      :Port        => SekkaServer::Config.listenPort,
                      :Host        => "0.0.0.0",
                      :AccessLog   => [],
-                     :config      => "./lib/sekka.ru"
+                     :config      => vendordir + "/sekka.ru"
                      )
 end
 

data/emacs/sekka.el

@@ -1620,7 +1620,7 @@ point から行頭方向に同種の文字列が続く間を漢字変換しま
 (setq default-input-method "japanese-sekka")
 
 (defconst sekka-version
-  "0.9.6" ;;SEKKA-VERSION
+  "0.9.7" ;;SEKKA-VERSION
   )
 (defun sekka-version (&optional arg)
   "入力モード変更"

data/lib/sekka/approximatesearch.rb

@@ -37,7 +37,7 @@ require 'sekka/kvs'
 class ApproximateSearch
   def initialize( jarow_shikii )
     @jarow_shikii = jarow_shikii
-    @jarow        = FuzzyStringMatch::JaroWinkler.new.create( :native )
+    @jarow        = FuzzyStringMatch::JaroWinkler.create( :native )
   end
 
   def filtering( keyword, arr )

data/lib/sekka/convert-jisyo.nnd

@@ -42,7 +42,7 @@
   (let1 roman-list (gen-hiragana->roman-list key)
     (if (< 1000 (length roman-list))
         (begin
-          (STDERR.printf " Warning: ignored entry [%s] (hiragana phrase), because too many pattens.\n" key)
+          #?=(sprintf " Warning: ignored entry [%s] (hiragana phrase), because too many pattens.\n" key)
           #f) ;; パターン数が爆発した単語は無視する
         (map
          (lambda (x)
@@ -53,7 +53,7 @@
   (let1 roman-list (gen-hiragana->roman-list key)
     (if (< 1000 (length roman-list))
         (begin
-          (STDERR.printf " Warning: ignored entry [%s %s] , because too many pattens.\n" key value)
+          #?=(sprintf " Warning: ignored entry [%s %s] , because too many pattens.\n" key value)
           #f) ;; パターン数が爆発した単語は無視する
         (append
          (map

data/lib/sekka/jisyo-db.nnd

@@ -242,12 +242,12 @@
       (let ([d (. (Date.new 0) to_s)])
         (if (and (kvs.pure_put! "key_for_ping" d)
                  (string=? (kvs.get  "key_for_ping") d))
-            (STDERR.puts "Info: database file is clean")
+            #?="Info: database file is clean"
             ;; 正常に読み書きできないようであれば、データベースを修復する
             (begin
-              (STDERR.puts "Info: database file is NOT clean. try to fix...")
+              #?="Info: database file is NOT clean. try to fix..."
               (kvs.fixdb)
-              (STDERR.puts "Info: done."))))
+              #?="Info: done.")))
       (to-arr (list kvs cachesv)))))
 ;; Export to Ruby world
 (export-to-ruby openSekkaJisyo)

data/lib/sekka/sekkaversion.rb

@@ -1,4 +1,4 @@
 class SekkaVersion
-  def  self.version()     "0.9.6"  end
+  def  self.version()     "0.9.7"  end
   def  self.dictVersion() "0.9.2"  end
 end

data/lib/sekka.ru

@@ -32,5 +32,5 @@
 #  
 #  $Id: 
 #
-require './lib/sekkaserver'
+require 'sekkaserver'
 run SekkaServer::Server.new

data/lib/sekkaserver.rb

@@ -40,8 +40,8 @@ require 'eventmachine'
 require 'syslog'
 require 'uri'
 require 'date'
-require './lib/sekkaconfig'
-require './lib/sekka/sekkaversion'
+require 'sekkaconfig'
+require 'sekka/sekkaversion'
 require 'memcache'
 
 module SekkaServer

data/test/henkan-main.nnd

@@ -35,6 +35,7 @@
 ;;;
 (require "sekka/kvs")
 (use nendo.test)
+(use debug.null)
 (use sekka.convert-jisyo)
 (use sekka.henkan)
 (load "./test/common.nnd")
@@ -90,7 +91,7 @@
 ;;-------------------------------------------------------------------
 (with-open "./test/sekka-dump-out-1.txt"
            (lambda (f)
-             (let ((_input (f.read))
+             (let ((_answer (Digest::MD5.hexdigest (f.read)))
                    (_output1 (StringIO.new))
                    (_output2 (StringIO.new)))
                (_output1.set_encoding "utf-8")
@@ -99,30 +100,33 @@
                          (eq? dbtype 'redis)
                          (eq? dbtype 'dbm)
                          (eq? dbtype 'pure))
-                 (test-section "dump db")
-
-                 (test* "dump DB to SEKKA jisyo file. (1)"
-                        _input
-                        (begin
-                          (dump-sekka-jisyo-f _output1 target)
-                          (with-open "./test/sekka-dump-out-1.tmp"
-                                     (lambda (f) (f.write (_output1.string)))
-                                     "w")
-                          (_output1.string)))
-                 
-                 (test* "dump/restore DB to SEKKA jisyo file. (1)"
-                        _input
-                        (begin
-                          ;; ---- RESTORE ----
-                          (f.rewind)
-                          (restore-sekka-jisyo-f f target)
-                          
-                          ;; ---- DUMP    ----
-                          (dump-sekka-jisyo-f _output2 target)
-                          (with-open "./test/sekka-dump-out-2.tmp"
-                                     (lambda (f) (f.write (_output2.string)))
-                                     "w")
-                          (_output2.string)))))))
+                 (begin
+                   (test-section "dump db")
+                   
+                   (test* "dump DB to SEKKA jisyo file. (1)"
+                          _answer
+                          (begin
+                            (dump-sekka-jisyo-f _output1 target)
+                            (with-open "./test/sekka-dump-out-1.tmp"
+                                       (lambda (f) (f.write (_output1.string)))
+                                       "w")
+                            (Digest::MD5.hexdigest
+                             (_output1.string))))
+                   
+                   (test* "dump/restore DB to SEKKA jisyo file. (1)"
+                          _answer
+                          (begin
+                            ;; ---- RESTORE ----
+                            (f.rewind)
+                            (restore-sekka-jisyo-f f target)
+                            
+                            ;; ---- DUMP    ----
+                            (dump-sekka-jisyo-f _output2 target)
+                            (with-open "./test/sekka-dump-out-2.tmp"
+                                       (lambda (f) (f.write (_output2.string)))
+                                       "w")
+                            (Digest::MD5.hexdigest
+                             (_output2.string)))))))))
                  
   
 
@@ -1252,6 +1256,6 @@
 ;; ---後処理---
 (kvs.close)
 ;; 最終的な辞書の状態を目視するためのダンプ
-(dump-sekka-jisyo-f STDOUT target)
+;;(dump-sekka-jisyo-f STDOUT target)
 
 (test-end)

data/test/jisyo.nnd

@@ -34,6 +34,7 @@
 ;;;  $Id: 
 ;;;
 (use nendo.test)
+(use debug.null)
 (use sekka.convert-jisyo)
 
 (define debug-print-length 10000)