sekka 1.7.0 → 1.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f14974e9acd87ad10f08d88a02162ef21a8c7fff
-  data.tar.gz: c9d7db3ca95a63b316d8089c947ab23424cde502
+  metadata.gz: 68e39f2abd18a04dded3aa414d166935798ea4b7
+  data.tar.gz: dd3c4f863160bc9d7d7310f5783f1ffbfcfdeba3
 SHA512:
-  metadata.gz: 97f809c4c8f84c6a5123a7b71ac6f0698c46d4104b1661eb1428ea557cf9c650f47f324d7b2e2c9dfe16e1e30ad3d4b531fc6dcd1595ac5ee34535b392b6b84e
-  data.tar.gz: 68af3d5572e414b082afb7ec85f2024912387a2df90b43de0f68c098854e1698f6def4c01190141e77099a121e749ea74ba54d246b9c5c23c8f9bd2347c8e91d
+  metadata.gz: a476ff53c1f26fe54f027916db088c7acbf4bad6a26874869f33e3b57045de9ff280c3364ade55af903846e68ef966faf9e9b032e31da3134dc8e10662d4ff0b
+  data.tar.gz: e74a861ddaaf1177b97f75f58a4236946c694375da1aee3c050965927d5751f218ecfdb6fcea5a17a885fa229c85788594c440caeb443bf50a31dbd3a61e16b4

data/.gitignore

@@ -26,4 +26,8 @@
 /test.log
 /test.record
 /test.tch
+/test1.mapdb
+/test2.mapdb
+/test_mapdb
+/test_mapdb.p
 /tmpfile.euc

data/.travis.yml

@@ -6,6 +6,7 @@ rvm:
  - 2.2.0
  - 2.3.0
  - 2.4.0
+ - jruby-19mode
 
 env:
  - DB=pure
@@ -13,3 +14,10 @@ env:
 
 gemfile:
  - gemfiles/Gemfile
+
+matrix:
+  allow_failures:
+    - rvm: jruby-19mode
+
+jdk:
+  - oraclejdk8

data/COPYING

@@ -54,4 +54,20 @@ Copyright of test/skk-azik-table.nnd ----------------------------------------
   This file is covered by GPL version 2
   ported from DDSKK-14.1.90
 
+Copy right of lib/sekka/jar/lz4-1.3.0.jar   ---------------------------------
+Copy right of lib/sekka/jar/elsa-3.0.0-M7.jar
+Copy right of lib/sekka/jar/kotlin-compiler-1.1.2-3.jar
+Copy right of lib/sekka/jar/kotlin-stdlib-jre8-1.1.2-3.jar
+Copy right of lib/sekka/jar/guava-19.0.jar
+Copy right of lib/sekka/jar/google-collections-1.0.jar
+Copy right of lib/sekka/jar/mapdb-3.1.0-SNAPSHOT.jar
+
+  This file is covered by Apache License Version 2.0
+
+Copy right of lib/sekka/jar/eclipse-collections-8.2.0.jar -------------------
+Copy right of lib/sekka/jar/eclipse-collections-api-8.2.0.jar
+
+  Eclipse Collections is open sourced under the Eclipse Public License v1.0
+  and the Eclipse Distribution License v1.0.
+
 -----------------------------------------------------------------------------

data/README.md

@@ -1,6 +1,6 @@
 # Sekka (石火) SKK like Japanese input method
 
- ![Logo]( https://dl.dropboxusercontent.com/u/3870066/blog/iStock_000016378483XTiny.jpg )　[![Build Status](https://travis-ci.org/kiyoka/sekka.svg?branch=master)](https://travis-ci.org/kiyoka/sekka)
+![Logo](https://www.dropbox.com/s/eabcg33iqx5h7nw/iStock_000016378483XTiny.jpg?raw=1)  [![Build Status](https://travis-ci.org/kiyoka/sekka.svg?branch=master)](https://travis-ci.org/kiyoka/sekka)
 
 ----
 

data/Rakefile

@@ -15,6 +15,8 @@
 #     DB=gdbm
 #     DB=tokyocabinet
 #     DB=redis
+#     DB=pure            pure Ruby(for travis-ci)
+#     DB=mapdb           for Java platform
 #     DB=                (default)
 #     DB=all             gdbm/tokyocabinet/redis
 #
@@ -76,13 +78,17 @@ task :test do
     exit 1
   end
 
-  ["test.record", "test.tch", "test.db" ].each {|name|
+  ["test.record", "test.tch", "test.db", "test1.mapdb", "test2.mapdb"].each {|name|
     File.unlink( name ) if File.exist?( name )
   }
   ["test.ldb"].each {|name|
     FileUtils.rm_rf( name ) if File.exist?( name )
   }
   files = []
+  if RUBY_PLATFORM == 'java'
+    files << "./test/jruby_mapdb.nnd"
+  end
+  files << "./test/downloader.nnd"
   files << "./test/memcache.nnd"
   files << "./test/util.nnd"
   files << "./test/alphabet-lib.nnd"
@@ -102,6 +108,8 @@ task :test do
   when 'redis'
     files << "./test/redis.nnd"
     files << "./test/henkan-main.nnd  redis"
+  when 'mapdb'
+    files << "./test/henkan-main.nnd  mapdb"
   when 'pure'
     files << "./test/henkan-main.nnd  pure"
   when 'all'
@@ -111,9 +119,14 @@ task :test do
     files << "./test/henkan-main.nnd  pure"
     files << "./test/henkan-main.nnd  leveldb"
   else # default
-    files << "./test/henkan-main.nnd  tokyocabinet"
-    files << "./test/henkan-main.nnd  pure"
-    files << "./test/henkan-main.nnd  leveldb"
+    if RUBY_PLATFORM == 'java'
+      files << "./test/henkan-main.nnd  mapdb"
+      files << "./test/henkan-main.nnd  pure"
+    else
+      files << "./test/henkan-main.nnd  tokyocabinet"
+      files << "./test/henkan-main.nnd  pure"
+      files << "./test/henkan-main.nnd  leveldb"
+    end
   end
   files.each {|filename|
     sh  sprintf( "export RUBY_THREAD_VM_STACK_SIZE=100000 ; ruby -I ./lib -S nendo -I ./lib -d %s", filename )
@@ -149,11 +162,11 @@ task :load do
 end
 
 task :dump do
-  sh sprintf( "ruby ./bin/sekka-jisyo dump    ./data/SEKKA-JISYO.N.tch#xmsiz=1024m > ./data/SEKKA-JISYO-%s.N.tsv", dictVersion )
+  sh sprintf( "ruby ./bin/sekka-jisyo dump    ./data/SEKKA-JISYO-%s.N.mapdb > ./data/SEKKA-JISYO-%s.N.tsv", dictVersion, dictVersion )
 end
 
 task :restore do
-  sh sprintf("export RUBY_THREAD_VM_STACK_SIZE=100000 ; ruby -I ./lib ./bin/sekka-jisyo restore  ./data/SEKKA-JISYO-%s.N.tsv jisyo.N.ldb", dictVersion )
+  sh sprintf("export RUBY_THREAD_VM_STACK_SIZE=100000 ; ruby -I ./lib ./bin/sekka-jisyo restore  ./data/SEKKA-JISYO-%s.N.tsv ./data/SEKKA-JISYO-%s.N.mapdb", dictVersion, dictVersion )
 end
 
 task :load_leveldb do

data/VERSION.yml

@@ -1,4 +1,4 @@
 ---
 :major: 1
 :minor: 7
-:patch: 0
+:patch: 1

data/bin/sekka-jisyo

@@ -85,14 +85,17 @@ core.evalStr( <<";;END-OF-SCRIPT" )
   (print "  sekka-jisyo load     SEKKA-JISYO.X      SEKKA-JISYO.X.tch      ... load     SEKKA-JISYO to Tokyo Cabinet DB")
   (print "  sekka-jisyo load     SEKKA-JISYO.X      SEKKA-JISYO.X.db       ... load     SEKKA-JISYO to gdbm          DB")
   (print "  sekka-jisyo load     SEKKA-JISYO.X      SEKKA-JISYO.X.ldb      ... load     SEKKA-JISYO to LevelDB")
+  (print "  sekka-jisyo load     SEKKA-JISYO.X      SEKKA-JISYO.X.mapdb    ... load     SEKKA-JISYO to MapDB")
   (print "  sekka-jisyo load     SEKKA-JISYO.X      redis:[hostname]       ... load     SEKKA-JISYO to redis DB on [hostname]")
   (print "  sekka-jisyo load     SEKKA-JISYO.X      redis:                 ... load     SEKKA-JISYO to redis DB on localhost")
   (print "  sekka-jisyo dump     SEKKA-JISYO.X.tch                         ... dump     Tokyo Cabinet DB         to   DUMPDATA(STDOUT)")
   (print "  sekka-jisyo dump     SEKKA-JISYO.X.db                          ... dump     gdbm                     to   DUMPDATA(STDOUT)")
   (print "  sekka-jisyo dump     SEKKA-JISYO.X.ldb                         ... dump     LevelDB                  to   DUMPDATA(STDOUT)")
+  (print "  sekka-jisyo dump     SEKKA-JISYO.X.mapdb                       ... dump     MapDB                    to   DUMPDATA(STDOUT)")
   (print "  sekka-jisyo dump     redis:[hostname]                          ... dump     redis DB on [hostname]   to   DUMPDATA(STDOUT)")
   (print "  sekka-jisyo dump     redis:                                    ... dump     redis DB on localhost    to   DUMPDATE(STDOUT)")
   (print "  sekka-jisyo restore  SEKKA-JISYO.X.tsv  SEKKA-JISYO.X.tch      ... restore  Tokyo Cabinet DB from tsv-file")
+  (print "  sekka-jisyo restore  SEKKA-JISYO.X.tsv  SEKKA-JISYO.X.mapdb    ... restore  MapDB            from tsv-file")
   (print "  sekka-jisyo restore  SEKKA-JISYO.X.tsv  redis:[hostname]       ... restore  redis DB on [hostname]   from tsv-file")
   (print "  sekka-jisyo restore  SEKKA-JISYO.X.tsv  redis:                 ... restore  redis DB on localhost    from tsv-file"))
 
@@ -105,6 +108,8 @@ core.evalStr( <<";;END-OF-SCRIPT" )
     (values 'gdbm filename))
    ((rxmatch #/[.]ldb$/ filename)
     (values 'leveldb filename))
+   ((rxmatch #/[.]mapdb$/ filename)
+    (values 'mapdb filename))
    ((rxmatch #/^redis:(.*)$/ filename)
     => (lambda (m)
          (let1 str (rxmatch-substring m 1)

data/bin/sekka-server

@@ -7,6 +7,7 @@ require 'rack'
 require 'uri'
 require 'sekkaconfig'
 require 'sekka/sekkaversion'
+require 'sekka/downloader'
 
 
 
@@ -21,14 +22,18 @@ TSV_FILE      = sprintf( "%s/SEKKA-JISYO-%s.N.tsv",            DICTDIR, SekkaVer
 TSV_SUMFILE   = sprintf( "%s/SEKKA-JISYO-%s.N.md5",            DICTDIR, SekkaVersion.dictVersion )
 LDB_FILE      = sprintf( "%s/SEKKA-JISYO-%s.N.ldb.tar.gz",     DICTDIR, SekkaVersion.dictVersion )
 LDB_SUMFILE   = sprintf( "%s/SEKKA-JISYO-%s.N.ldb.tar.gz.md5", DICTDIR, SekkaVersion.dictVersion )
+MAPDB_FILE    = sprintf( "%s/SEKKA-JISYO-%s.N.mapdb",          DICTDIR, SekkaVersion.dictVersion )
+MAPDB_SUMFILE = sprintf( "%s/SEKKA-JISYO-%s.N.mapdb.md5",      DICTDIR, SekkaVersion.dictVersion )
 
-GDBM_FILE     = sprintf( "%s/SEKKA-JISYO-%s.N.db",          DICTDIR, SekkaVersion.dictVersion )
-LEVELDB_FILE  = sprintf( "%s/SEKKA-JISYO-%s.N.ldb",         DICTDIR, SekkaVersion.dictVersion )
+GDBM_FILE     = sprintf( "%s/SEKKA-JISYO-%s.N.db",             DICTDIR, SekkaVersion.dictVersion )
+LEVELDB_FILE  = sprintf( "%s/SEKKA-JISYO-%s.N.ldb",            DICTDIR, SekkaVersion.dictVersion )
 
 TSV_URLURL    = sprintf( "%s/SEKKA-JISYO-%s.N.url",              DICTURL, SekkaVersion.dictVersion )
 TSV_SUMURL    = sprintf( "%s/SEKKA-JISYO-%s.N.md5",              DICTURL, SekkaVersion.dictVersion )
 LDB_URLURL    = sprintf( "%s/SEKKA-JISYO-%s.N.ldb.tar.gz.url",   DICTURL, SekkaVersion.dictVersion )
 LDB_SUMURL    = sprintf( "%s/SEKKA-JISYO-%s.N.ldb.tar.gz.md5",   DICTURL, SekkaVersion.dictVersion )
+MAPDB_URLURL  = sprintf( "%s/SEKKA-JISYO-%s.N.mapdb.url",        DICTURL, SekkaVersion.dictVersion )
+MAPDB_SUMURL  = sprintf( "%s/SEKKA-JISYO-%s.N.mapdb.md5",        DICTURL, SekkaVersion.dictVersion )
 
 MEMCACHED = "localhost:11211"                # memcahced
 
@@ -48,11 +53,17 @@ def getSekkaDbInfo( env )
       [ :gdbm, nil ]
     elsif m = ENV['SEKKA_DB'].match( /^level[a-z]+$/i )
       [ :leveldb, nil ]
+    elsif m = ENV['SEKKA_DB'].match( /^map[a-z]+$/i )
+      [ :mapdb, nil ]
     else
-      raise RuntimeError, "Error: env var SEKKA_DB requires [redis:hostname] or [redis:] or [tokyocabinet] or [leveldb]"
+      raise RuntimeError, "Error: env var SEKKA_DB requires [redis:hostname] or [redis:] or [tokyocabinet] or [leveldb] or [mapdb]"
     end
   else
-    [ :tokyocabinet, nil ]
+    if RUBY_PLATFORM == 'java'
+      [ :mapdb, nil ]
+    else
+      [ :tokyocabinet, nil ]
+    end
   end
 end
 
@@ -83,6 +94,13 @@ def checkJisyoIsInstalled( dictType, dictSource )
                ret = kvs.get( key )
                kvs.close()
                ret
+             when :mapdb
+               require 'sekka/kvs'
+               kvs = Kvs.new( :mapdb )
+               kvs.open( dictSource )
+               ret = kvs.get( key )
+               kvs.close()
+               ret
              when :redis
                begin
                  require 'redis'
@@ -103,38 +121,38 @@ end
 def downloadFile( targetfile, sumfile, urlurl, sumurl )
   if not File.exist?( targetfile )
     STDERR.printf( "Info: Downloading SEKKA-JISYO\n" )
-    # 辞書をダウンロードする
-    cmd = sprintf( "curl %s",          urlurl )
-    STDERR.printf( "Command : %s\n", cmd )
-    targeturl = open( "|" + cmd ) { |f|
-      f.read
-    }
-    STDERR.printf( "   download   URL of target file : %s\n", targeturl );
-    cmd = sprintf( "curl -o %s %s",    targetfile, targeturl )
-    STDERR.printf( "Command : %s\n", cmd )
-    system( cmd )
-    cmd = sprintf( "curl -o %s %s",    sumfile, sumurl )
-    STDERR.printf( "Command : %s\n", cmd )
-    system( cmd )
+    # 辞書のURLをダウンロードする
+    dl = Downloader.new( urlurl )
+    dl.download()
+    targeturl = dl.getBody().chomp
+    dl.clearBody()
+
+    # md5記載ファイルをダウンロード
+    dl = Downloader.new( sumurl )
+    dl.download()
+    md5body = dl.getBody()
+    correctSum = md5body.chomp.split[0]
+    dl.clearBody()
+    
+    # 辞書ファイル本体をダウンロード
+    STDERR.printf( "   downloading URL : %s ...\n", targeturl );
+    dl = Downloader.new( targeturl )
+    dl.downloadToFile( targetfile )
+    dl.clearBody()
 
     # チェックサムを確認する
-    downloadSum = ""
-    open( targetfile ) { |f|
-      dataBody = f.read
-      downloadSum = Digest::MD5.hexdigest( dataBody )
-    }
-    open( sumfile ) { |f|
-      correctSum = f.readline.chomp.split[0]
-      STDERR.printf( "   downloaded file's MD5 : %s\n", downloadSum )
-      STDERR.printf( "             correct MD5 : %s\n", correctSum   )
-      if downloadSum == correctSum
-        STDERR.printf( "Info:  downloaded file [%s] verify OK.\n", targetfile )
-      else
-        STDERR.printf( "Error: downloaded file [%s] verify NG.\n", targetfile )
-        File.unlink( targetfile )
-        exit( 1 )
-      end
-    }
+    md5 = Digest::MD5.file( targetfile )
+    downloadSum = md5.hexdigest
+
+    STDERR.printf( "   downloaded file's MD5 : %s\n", downloadSum )
+    STDERR.printf( "             correct MD5 : %s\n", correctSum   )
+    if downloadSum == correctSum
+      STDERR.printf( "Info:  downloaded file [%s] verify OK.\n", targetfile )
+    else
+      STDERR.printf( "Error: downloaded file [%s] verify NG.\n", targetfile )
+      File.unlink( targetfile )
+      exit( 1 )
+    end
   end
 end
   
@@ -165,6 +183,8 @@ def main
   case dictType
   when :leveldb
     downloadFile( LDB_FILE, LDB_SUMFILE, LDB_URLURL, LDB_SUMURL )
+  when :mapdb
+    downloadFile( MAPDB_FILE, MAPDB_SUMFILE, MAPDB_URLURL, MAPDB_SUMURL )
   else
     downloadFile( TSV_FILE, TSV_SUMFILE, TSV_URLURL, TSV_SUMURL )
   end
@@ -209,6 +229,17 @@ def main
     STDERR.printf( "Info: [OK]\n" )
     dictSource = TC_FILE
 
+  when :mapdb
+    # MapDBに辞書が投入済みか確認する
+    ok = checkJisyoIsInstalled( dictType, MAPDB_FILE )
+    unless ok
+      # tsvファイルをuploadする
+      STDERR.printf( "Error: mapdb file is not installed.\n" )
+      exit(1)
+    end
+    STDERR.printf( "Info: [OK]\n" )
+    dictSource = MAPDB_FILE
+
   when :redis
     # redisサーバーに辞書が投入済みか確認する
     ok = checkJisyoIsInstalled( dictType, dictSource )

data/data/.gitignore

@@ -20,6 +20,7 @@
 /SEKKA-JISYO-1.6.2.N.ldb
 /SEKKA-JISYO-1.6.2.N.ldb.tar.gz
 /SEKKA-JISYO-1.6.2.N.ldb.tar.gz.md5
+/SEKKA-JISYO-1.6.2.N.mapdb
 /SEKKA-JISYO-1.6.2.N.md5
 /SEKKA-JISYO-1.6.2.N.tsv
 /SEKKA-JISYO.CUSTOM

data/emacs/sekka.el

@@ -3,7 +3,7 @@
 ;; Copyright (C) 2010-2014 Kiyoka Nishiyama
 ;;
 ;; Author: Kiyoka Nishiyama <kiyoka@sumibi.org>
-;; Version: 1.7.0          ;;SEKKA-VERSION
+;; Version: 1.7.1          ;;SEKKA-VERSION
 ;; Keywords: ime, skk, japanese
 ;; Package-Requires: ((cl-lib "0.3") (concurrent "0.3.1") (popup "0.5.2"))
 ;; URL: https://github.com/kiyoka/sekka
@@ -1756,7 +1756,7 @@ point から行頭方向に同種の文字列が続く間を漢字変換しま
 (setq default-input-method "japanese-sekka")
 
 (defconst sekka-version
-  "1.7.0" ;;SEKKA-VERSION
+  "1.7.1" ;;SEKKA-VERSION
   )
 (defun sekka-version (&optional arg)
   "入力モード変更"

data/lib/sekka/downloader.rb

@@ -0,0 +1,102 @@
+# downloader.rb  -  "downloader for SEKKA-JISYO data"
+#
+#   Copyright (c) 2017  Kiyoka Nishiyama  <kiyoka@sumibi.org>
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#   1. Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+#
+#   3. Neither the name of the authors nor the names of its contributors
+#      may be used to endorse or promote products derived from this
+#      software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+#   TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+#   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+#   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+#   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+#   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+#  $Id:
+#
+
+require 'net/http'
+require 'uri'
+require 'tmpdir'
+require 'digest'
+
+class Downloader
+  def initialize( url, savepath = nil )
+    @url_str = url
+    @body = nil
+  end
+
+  def download()
+    url = URI.parse(@url_str)
+    if(url)
+      req = Net::HTTP::Get.new(url.path)
+      http = Net::HTTP.new(url.host,url.port)
+      if url.scheme == 'https'
+        http.use_ssl = true
+      end
+      res = http.request(req)
+      @body = res.body
+    end
+    return @body
+  end
+
+  def downloadToFile(path)
+    url = URI.parse(@url_str)
+    req = Net::HTTP::Get.new url.path
+    http = Net::HTTP.new(url.host, url.port)
+    if url.scheme == 'https'
+      http.use_ssl = true
+    end
+    http.request req do |response|
+      open path, 'wb' do |io|
+        response.read_body do |chunk|
+          io.write chunk
+        end
+      end
+    end
+  end
+
+  def getBodySize()
+    size = 0
+    if @body
+      size = @body.size
+    end
+    return size
+  end
+
+  def getBody()
+    return @body
+  end
+
+  def saveAs(path)
+    open(path,"w") {|f|
+      f.write(@body)
+    }
+  end
+
+  def clearBody()
+    @body = nil
+  end
+
+  def calcMD5()
+    Digest::MD5.hexdigest @body
+  end
+  
+end