sekka 1.2.1 → 1.2.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7f2b64092011062870613c3906c9855418ca0e68
+  data.tar.gz: f2b131d6ff3d7b7f181a078fac541e03ac6d5b4c
+SHA512:
+  metadata.gz: b5091ad9ac65a240b45208acff8299269cfcf9210c7b95ccb92d61af5c52bc1cb1836201a9b7049c5c044011fcbae4840b5b20c5b140570456ee958de7ab1bc6
+  data.tar.gz: a782b597e4c870ca2cd3cf7deb32063b6183fe78d8cf0bc01ad54ceb5f2a2cb7905a9bc86ddbfefca0143944c3738a949ef5a89db46fe82583c873e7f251d5a9

data/Rakefile

@@ -52,7 +52,7 @@ begin
                            "sekka-server",
                            "sekka-benchmark",
                            "sekka-path"]
-    gemspec.required_ruby_version = '>= 1.9.1'
+    gemspec.required_ruby_version = '>= 1.9.2'
     gemspec.add_dependency( "eventmachine" )
     gemspec.add_dependency( "memcache-client" )
     gemspec.add_dependency( "nendo", "= 0.6.4" )
@@ -70,7 +70,7 @@ end
 
 task :compile do
   # generate version.rb
-  dictVersion = "1.2.2"
+  dictVersion = "1.3.0"
   vh = Jeweler::VersionHelper.new "."
   open( "./lib/sekka/sekkaversion.rb", "w" ) {|f|
     f.puts(   "class SekkaVersion" )
@@ -144,7 +144,7 @@ task :test do
   end
   files.each {|filename|
     nendopath = `which nendo`.chomp
-    sh  sprintf( "time ruby -I ./lib %s %s", nendopath, filename )
+    sh  sprintf( "ruby -I ./lib %s %s", nendopath, filename )
   }
   sh "cat test.record"
 end
@@ -204,18 +204,27 @@ task :dumpL do
 end
 
 
-# Fetched data from
-#   http://s-yata.jp/corpus/nwc2010/ngrams/
-task :phrase => [ "./data/6gm-0000.txt" ]  do
-  sh "time ruby -I ./lib /usr/local/bin/nendo ./data/hiragana_phrase_in_webcorpus.nnd           ./data/6gm-0000.txt | sort | uniq > /tmp/tmp.txt"
-  sh "time ruby -I ./lib /usr/local/bin/nendo ./data/writing_phrase_filter.nnd /tmp/tmp.txt  | sort | uniq     > ./data/SKK-JISYO.hiragana-phrase"
+task :phrase => [ "/tmp/jawiki.txt.gz", "./data/wikipedia/jawiki.hiragana.txt" ] do
+  sh "sort ./data/wikipedia/jawiki.hiragana.txt | uniq -c | sort > ./data/wikipedia/ranking.txt"
+  sh "ruby -I ./lib /usr/local/bin/nendo ./data/hiragana_phrase_in_wikipedia2.nnd ./data/wikipedia/ranking.txt > ./data/SKK-JISYO.hiragana-phrase"
 end
 
-file "./data/6gm-0000.txt"  do
-  sh "wget http://dist.s-yata.jp/corpus/nwc2010/ngrams/word/over999/6gms/6gm-0000.xz -O /tmp/6gm-0000.xz"
-  sh "xz -cd /tmp/6gm-0000.xz > ./data/6gm-0000.txt"
+file "./data/wikipedia/jawiki.hiragana.txt" do
+  sh "zcat /tmp/jawiki.txt.gz | mecab --input-buffer-size=65536 -O wakati --output=/tmp/jawiki.wakati.txt"
+  sh "ruby -I ./lib /usr/local/bin/nendo ./data/hiragana_phrase_in_wikipedia.nnd /tmp/jawiki.wakati.txt > ./data/wikipedia/jawiki.hiragana.txt"
+  sh "rm -f /tmp/jawiki.wakati.txt"
 end
 
+file "/tmp/jawiki.txt.gz" do
+  sh "mkdir -p ./data/wikipedia/txt"
+  sh "wget http://dumps.wikimedia.org/jawiki/latest/jawiki-latest-pages-articles.xml.bz2 -O /tmp/jawiki-latest-pages-articles.xml.bz2"
+  sh "wp2txt --input-file /tmp/jawiki-latest-pages-articles.xml.bz2 --output-dir ./data/wikipedia/txt"
+  sh "cat ./data/wikipedia/txt/*.txt | gzip -c > /tmp/jawiki.txt.gz"
+  sh "rm -f ./data/wikipedia/txt/*.txt"
+  sh "rm -f /tmp/jawiki-latest-pages-articles.xml.bz2"
+end
+
+
 task :phrase2 => [ "./data/ipadic.all.utf8.txt" ] do
   sh "time ruby -I ./lib /usr/local/bin/nendo ./data/hiragana_phrase_in_ipadic.nnd             ./data/ipadic.all.utf8.txt | sort | uniq > ./data/SKK-JISYO.hiragana-phrase2"
 end

data/VERSION.yml

@@ -1,4 +1,4 @@
 ---
 :major: 1
 :minor: 2
-:patch: 1
+:patch: 2

data/bin/sekka-server

@@ -22,17 +22,19 @@ DICTTYPE = if ENV.has_key?( 'SEKKA_AZIK' )
              "N"
            end
 
-TC_OPTS = "#xmsiz=256m"
-TC_FILE = DICTDIR + "/SEKKA-JISYO.SMALL.#{DICTTYPE}.tch" + TC_OPTS
-TSVFILE = DICTDIR + "/SEKKA-JISYO.SMALL.#{DICTTYPE}.tsv"
-SUMFILE = DICTDIR + "/SEKKA-JISYO.SMALL.#{DICTTYPE}.md5"
+DICTSIZE = if ENV.has_key?( 'SEKKA_LARGE' )
+             "LARGE"
+           else
+             "SMALL"
+           end
 
-URLURL  = DICTURL + "/SEKKA-JISYO.SMALL.#{DICTTYPE}.url"
-SUMURL  = DICTURL + "/SEKKA-JISYO.SMALL.#{DICTTYPE}.md5"
+TC_OPTS = "#xmsiz=256m"
+TC_FILE = DICTDIR + "/SEKKA-JISYO.#{DICTSIZE}.#{DICTTYPE}.tch" + TC_OPTS
+TSVFILE = DICTDIR + "/SEKKA-JISYO.#{DICTSIZE}.#{DICTTYPE}.tsv"
+SUMFILE = DICTDIR + "/SEKKA-JISYO.#{DICTSIZE}.#{DICTTYPE}.md5"
 
-TC_FILE_LIST = [ DICTDIR + "/SEKKA-JISYO.CUSTOM.tch" + TC_OPTS,
-                 DICTDIR + "/SEKKA-JISYO.LARGE.#{DICTTYPE}.tch"  + TC_OPTS,
-                 DICTDIR + "/SEKKA-JISYO.SMALL.#{DICTTYPE}.tch"  + TC_OPTS ]
+URLURL  = DICTURL + "/SEKKA-JISYO.#{DICTSIZE}.#{DICTTYPE}.url"
+SUMURL  = DICTURL + "/SEKKA-JISYO.#{DICTSIZE}.#{DICTTYPE}.md5"
 
 MEMCACHED = "localhost:11211"                # memcahced
 
@@ -147,12 +149,9 @@ def main
 
   case dictType
   when :tokyocabinet
-    # .tchファイルが存在するか調べる
-    list = TC_FILE_LIST.select { |name| File.exist?( name ) }
-
-    # redisサーバーに辞書が投入済みか確認する
+    # TokyoCabinetに辞書が投入済みか確認する
     ok = checkJisyoIsInstalled( dictType, TC_FILE )
-    if 0 == list.size and (not ok)
+    unless ok
       # tsvファイルをuploadする
       STDERR.printf( "Info: Uploading...\n" )
       cmd = sprintf( "sekka-jisyo restore %s %s", TSVFILE, TC_FILE )
@@ -160,10 +159,8 @@ def main
       system( cmd )
     end
     STDERR.printf( "Info: [OK]\n" )
+    dictSource = TC_FILE
 
-    # 存在する.tchファイルの中で一番最初のファイルを使う。
-    list = TC_FILE_LIST.select { |name| File.exist?( name ) }
-    dictSource = list[0]
   when :redis
     # redisサーバーに辞書が投入済みか確認する
     ok = checkJisyoIsInstalled( dictType, dictSource )

data/emacs/sekka.el

@@ -1620,7 +1620,7 @@ point から行頭方向に同種の文字列が続く間を漢字変換しま
 (setq default-input-method "japanese-sekka")
 
 (defconst sekka-version
-  "1.2.1" ;;SEKKA-VERSION
+  "1.2.2" ;;SEKKA-VERSION
   )
 (defun sekka-version (&optional arg)
   "入力モード変更"

data/lib/sekka/convert-jisyo.nnd

@@ -86,11 +86,13 @@
 
 
 (define (convert-skk-jisyo-f f)
+  (define progressbar-format     "%E |%B| %p%% %t")
   (let1 progress (ProgressBar.create (alist->hash-table
                                       `(
                                         [title  . "convert  "]
                                         [total  . ,(file-length f)]
-                                        [output . ,STDERR])))
+                                        [output . ,STDERR]
+                                        [format . ,progressbar-format])))
     (define (gen-sekka-entries line)
       (progress.increment)
       

data/lib/sekka/sekkaversion.rb

@@ -1,4 +1,4 @@
 class SekkaVersion
-  def  self.version()     "1.2.1"  end
-  def  self.dictVersion() "1.2.2"  end
+  def  self.version()     "1.2.2"  end
+  def  self.dictVersion() "1.3.0"  end
 end

data/test/sekka-dump-out-1.txt

@@ -1335,4 +1335,4 @@ M:みr	/見/観;(字義:みわたす)/視;(字義:じっとみる)/診;(diagnose
 M:わけ	/訳/和気;地名/理由;当て字/分け/分/譯;「訳」の旧字/戯奴
 M:わたしh	/私
 M:を	/小/緒/雄/男/牡/尾/悪/汚/惡;「悪」の旧字(人名用漢字)
-SEKKA:VERSION	1.2.2
+SEKKA:VERSION	1.3.0

metadata

@@ -1,52 +1,46 @@
 --- !ruby/object:Gem::Specification
 name: sekka
 version: !ruby/object:Gem::Version
-  version: 1.2.1
-  prerelease: 
+  version: 1.2.2
 platform: ruby
 authors:
 - Kiyoka Nishiyama
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-13 00:00:00.000000000 Z
+date: 2013-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: memcache-client
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: nendo
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -62,87 +55,76 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: distributed-trie
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: eventmachine
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: memcache-client
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: nendo
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -150,7 +132,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - '='
       - !ruby/object:Gem::Version
@@ -158,49 +139,43 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: distributed-trie
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: Sekka is a SKK like input method. Sekka server provides REST Based API.
@@ -263,26 +238,25 @@ files:
 - README.md
 homepage: http://github.com/kiyoka/sekka
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
-      version: 1.9.1
+      version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Sekka is a SKK like input method.
 test_files: []