sekka 1.7.0 → 1.7.1

data/lib/sekka/jisyo-db.nnd

@@ -107,11 +107,11 @@
               okuri-nashi-list
               hiragana-phrase-list)))
 
-  (let1 progress (if displayProgress
+  (let1 progress1 (if displayProgress
                      (ProgressBar.create (alist->hash-table
                                           `(
-                                            [title  . "trie-tree  "]
-                                            [total  . ,(* 2 (length keylist))]
+                                            [title  . "trie-tree(1)"]
+                                            [total  . ,(length keylist)]
                                             [output . ,STDERR]
                                             [format . ,progressbar-format]
                                             )))
@@ -119,7 +119,7 @@
     (receive (okuri-ari-list
               okuri-nashi-list
               hiragana-phrase-list)
-        (grouping-keylist keylist progress)
+        (grouping-keylist keylist progress1)
 
       (let1 okuri-nashi-reverse-list
           (filter-map
@@ -129,32 +129,38 @@
                  (str.reverse)))
            okuri-nashi-list)
 
-        (when progress 
-          (progress.progress= (+ (length keylist)
-                                 (- (length keylist)
-                                    (+
-                                     (length okuri-ari-list)
-                                     (length okuri-nashi-list)
-                                     (length okuri-nashi-reverse-list)
-                                     (length hiragana-phrase-list))))))
-        (set! keylist '()) ;; free memory
+        (let1 progress2
+            (if displayProgress
+                (ProgressBar.create (alist->hash-table
+                                     `(
+                                       [title  . "trie-tree(2)"]
+                                       [total  . ,(+
+                                                   (length okuri-ari-list)
+                                                   (length okuri-nashi-list)
+                                                   (length okuri-nashi-reverse-list)
+                                                   (length hiragana-phrase-list))]
+                                       [output . ,STDERR]
+                                       [format . ,progressbar-format]
+                                       )))
+                #f)
+          (set! keylist '()) ;; free memory
 
-        (let1 trie (DistributedTrie::Trie.new kvs (+ okuri-ari-prefix ":" userid ":"))
-          (trie-add-autocommit trie   okuri-ari-list       progress))
-        (set! okuri-ari-list '()) ;; free memory
-      
-        (let1 trie (DistributedTrie::Trie.new kvs (+ okuri-nashi-prefix ":" userid ":"))
-          (trie-add-autocommit trie   okuri-nashi-list     progress))
-        (set! okuri-nashi-list '()) ;; free memory
-
-        (let1 trie (DistributedTrie::Trie.new kvs (+ okuri-nashi-reverse-prefix ":" userid ":"))
-          (trie-add-autocommit trie   okuri-nashi-reverse-list     progress))
-        (set! okuri-nashi-reverse-list '()) ;; free memory
-
-        (let1 trie (DistributedTrie::Trie.new kvs (+ hiragana-phrase-prefix ":" userid ":"))
-          (trie-add-autocommit trie   hiragana-phrase-list progress))
-        (set! hiragana-phrase-list '()) ;; free memory
-        ))))
+          (let1 trie (DistributedTrie::Trie.new kvs (+ okuri-ari-prefix ":" userid ":"))
+            (trie-add-autocommit trie   okuri-ari-list       progress2))
+          (set! okuri-ari-list '()) ;; free memory
+          
+          (let1 trie (DistributedTrie::Trie.new kvs (+ okuri-nashi-prefix ":" userid ":"))
+            (trie-add-autocommit trie   okuri-nashi-list     progress2))
+          (set! okuri-nashi-list '()) ;; free memory
+          
+          (let1 trie (DistributedTrie::Trie.new kvs (+ okuri-nashi-reverse-prefix ":" userid ":"))
+            (trie-add-autocommit trie   okuri-nashi-reverse-list     progress2))
+          (set! okuri-nashi-reverse-list '()) ;; free memory
+          
+          (let1 trie (DistributedTrie::Trie.new kvs (+ hiragana-phrase-prefix ":" userid ":"))
+            (trie-add-autocommit trie   hiragana-phrase-list progress2))
+          (set! hiragana-phrase-list '()) ;; free memory
+          )))))
 
 (define (append-entry userid kvs _key value)
   (let1 key (+ userid ":" _key)
@@ -193,7 +199,7 @@
   (define (create-keylist kvs f)
     (let1 progress (ProgressBar.create (alist->hash-table
                                         `(
-                                          [title  . "data-body  "]
+                                          [title  . "data-body   "]
                                           [total  . ,(file-length f)]
                                           [output . ,STDERR]
                                           [format . ,progressbar-format]

data/lib/sekka/jruby_mapdb.rb

@@ -0,0 +1,113 @@
+# jruby_mapdb.rb  -  "wrapper library for MapDB(java)"
+#
+#   Copyright (c) 2017  Kiyoka Nishiyama  <kiyoka@sumibi.org>
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#   1. Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+#
+#   3. Neither the name of the authors nor the names of its contributors
+#      may be used to endorse or promote products derived from this
+#      software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+#   TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+#   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+#   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+#   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+#   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+#
+jarpath = File.expand_path('../jar', __FILE__) + '/'
+
+require jarpath + 'guava-19.0.jar'
+require jarpath + 'elsa-3.0.0-M7.jar'
+require jarpath + 'kotlin-stdlib-jre8-1.1.2-3.jar'
+require jarpath + 'kotlin-compiler-1.1.2-3.jar'
+require jarpath + 'google-collections-1.0.jar'
+require jarpath + 'eclipse-collections-8.2.0.jar'
+require jarpath + 'eclipse-collections-api-8.2.0.jar'
+require jarpath + 'lz4-1.3.0.jar'
+require jarpath + 'mapdb-3.1.0-SNAPSHOT.jar'
+
+module OrgMapdb
+  include_package "org.mapdb"
+end
+
+module MapDB
+  class Tree
+    def initialize(treeobj)
+      @treeobj = treeobj
+    end
+    def keys
+      @treeobj.key_set.to_a
+    end
+    def put(key,value)
+      @treeobj.put(key,value)
+    end
+    def set(key,value)
+      put(key,value)
+    end
+    def get(key)
+      @treeobj.get(key)
+    end
+    def delete(key)
+      @treeobj.remove(key)
+    end
+    def clear
+      @treeobj.clear
+    end
+    def close
+      #nothing to do
+    end
+  end
+
+  class DB
+    attr_reader :mapdb, :type
+    def initialize(dbname=nil,treename=nil)
+      @tree = nil
+      if dbname.nil?
+        @type = :MemoryDB
+        @mapdb = OrgMapdb::DBMaker.
+                   memoryDB().
+                   closeOnJvmShutdown().
+                   make()
+      else
+        @type = :FileDB
+        @mapdb = OrgMapdb::DBMaker.
+                   fileDB(Java::JavaIo::File.new("#{dbname}")).
+                   fileMmapEnableIfSupported().
+                   closeOnJvmShutdown().
+                   make()
+      end
+      if treename.nil?
+        raise ArgumentError("require treename.")
+      end
+      tree = @mapdb.treeMap("#{treename}").
+               keySerializer(OrgMapdb::Serializer.STRING).
+               valueSerializer(OrgMapdb::Serializer.STRING).
+               createOrOpen()
+      @tree = MapDB::Tree.new(tree)
+    end
+
+    def getTree()
+      return @tree
+    end
+
+    def close
+      @mapdb.close
+    end
+  end
+end

data/lib/sekka/kvs.rb

@@ -63,6 +63,13 @@ class Kvs
       @leveldbFlag = false
     end
 
+    @mapdbFlag = true
+    begin
+      require 'sekka/jruby_mapdb'
+    rescue LoadError
+      @mapdbFlag = false
+    end
+
     @dbtype = dbtype
     case dbtype
     when :tokyocabinet
@@ -94,6 +101,13 @@ class Kvs
         raise RuntimeError, "Kvs.new() missed require( 'leveldb' )."
       end
 
+    when :mapdb
+      if @mapdbFlag
+        # do nothing
+      else
+        raise RuntimeError, "Kvs.new() missed require( 'sekka/jruby_mapdb' )."
+      end        
+
     when :pure
       # do nothing
     else
@@ -121,6 +135,9 @@ class Kvs
         name = name + ".ldb"
       end
       @db = LevelDB::DB.new name
+    when :mapdb
+      @basedb = MapDB::DB.new(name,"sekka")
+      @db = @basedb.getTree
     when :pure
       @name = name
       if File.exist?( @name )
@@ -160,6 +177,8 @@ class Kvs
         @db[ key.force_encoding("ASCII-8BIT") ] = value.force_encoding("ASCII-8BIT")
       when :memcache
         @db.set( key.force_encoding("ASCII-8BIT"), value.force_encoding("ASCII-8BIT"), timeout )
+      when :mapdb
+        @db.set( key, value )
       when :pure
         @db[ key ] = value
       else
@@ -177,6 +196,8 @@ class Kvs
       case @dbtype
       when :memcache
         val = @db.get(key.force_encoding("ASCII-8BIT"))
+      when :mapdb
+        val = @db.get(key)
       else
         val = @db[ key ]
       end
@@ -200,7 +221,7 @@ class Kvs
 
   def clear()
     case @dbtype
-    when :tokyocabinet, :gdbm, :pure
+    when :tokyocabinet, :gdbm, :pure, :mapdb
       @db.clear
     when :leveldb
       @db.clear!
@@ -216,7 +237,7 @@ class Kvs
   # return array of key string
   def keys()
     case @dbtype
-    when :tokyocabinet, :gdbm, :redis
+    when :tokyocabinet, :gdbm, :redis, :mapdb
       @db.keys.map { |k|
         k.force_encoding("UTF-8")
       }
@@ -241,7 +262,7 @@ class Kvs
       }
     when :memcache
       raise RuntimeError, "Kvs#forward_match_keys method was not implemented for memcache."
-    when :gdbm, :leveldb, :pure
+    when :gdbm, :leveldb, :pure, :mapdb
       self.keys( ).select {|key|
         key.match( "^" + prefix )
       }
@@ -256,6 +277,8 @@ class Kvs
       @db.close
     when :memcache, :redis
       # do nothing
+    when :mapdb
+      @basedb.close
     when :pure
       File.open( @name, "w" ) { |f|
         f.print( @db )

data/lib/sekka/sekkaversion.rb

@@ -1,4 +1,4 @@
 class SekkaVersion
-  def  self.version()     "1.7.0"  end
+  def  self.version()     "1.7.1"  end
   def  self.dictVersion() "1.6.2"  end
 end

data/public_dict/1.6.2/SEKKA-JISYO-1.6.2.N.mapdb.md5

@@ -0,0 +1 @@
+5c84a1697656a8e05153c87dcb7c0371  SEKKA-JISYO-1.6.2.N.mapdb

data/public_dict/1.6.2/SEKKA-JISYO-1.6.2.N.mapdb.url

@@ -0,0 +1 @@
+https://s3-ap-northeast-1.amazonaws.com/sekkadict/1.6.2/SEKKA-JISYO-1.6.2.N.mapdb

data/test/downloader.nnd

@@ -0,0 +1,82 @@
+;;-*- mode: nendo; syntax: scheme -*-
+;;;
+;;; downloader.nnd - downloaderライブラリのテスト
+;;;  
+;;;   Copyright (c) 2017  Kiyoka Nishiyama  <kiyoka@sumibi.org>
+;;;   
+;;;   Redistribution and use in source and binary forms, with or without
+;;;   modification, are permitted provided that the following conditions
+;;;   are met:
+;;;   
+;;;   1. Redistributions of source code must retain the above copyright
+;;;      notice, this list of conditions and the following disclaimer.
+;;;  
+;;;   2. Redistributions in binary form must reproduce the above copyright
+;;;      notice, this list of conditions and the following disclaimer in the
+;;;      documentation and/or other materials provided with the distribution.
+;;;  
+;;;   3. Neither the name of the authors nor the names of its contributors
+;;;      may be used to endorse or promote products derived from this
+;;;      software without specific prior written permission.
+;;;  
+;;;   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+;;;   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+;;;   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+;;;   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+;;;   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+;;;   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+;;;   TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+;;;   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+;;;   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+;;;   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+;;;   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;  
+;;;  $Id: 
+;;;
+(require "sekka/downloader")
+(use nendo.test)
+(load "./test/common.nnd")
+
+(test-start "Downloader")
+;;===================================================================
+
+;;-------------------------------------------------------------------
+(test-section "download textfile")
+
+(test* "download md5"
+       65
+       (let1 dl (Downloader.new "https://raw.githubusercontent.com/kiyoka/sekka/master/public_dict/1.6.2/SEKKA-JISYO-1.6.2.N.md5")
+         (dl.download)
+         (dl.getBodySize)))
+
+(test* "download md5"
+       "./data/SEKKA-JISYO-1.6.2.N.tsv"
+       (let1 dl (Downloader.new "https://raw.githubusercontent.com/kiyoka/sekka/master/public_dict/1.6.2/SEKKA-JISYO-1.6.2.N.md5")
+         (dl.download)
+         (cadr (to-list (. (dl.getBody) split #/[ \n]+/)))))
+
+(when #f
+  (test* "download tsv"
+         196089257
+         (let1 dl (Downloader.new "https://s3-ap-northeast-1.amazonaws.com/sekkadict/1.6.2/SEKKA-JISYO-1.6.2.N.tsv" "/tmp/")
+           (dl.download)
+           (dl.saveAs "/tmp/t.tsv")
+           (dl.getBodySize))))
+
+(test* "calc md5"
+       "611ed4eed5d5b4ed23cc06ca18944d74"
+       (let1 dl (Downloader.new "https://raw.githubusercontent.com/kiyoka/sekka/master/public_dict/1.6.2/SEKKA-JISYO-1.6.2.N.md5")
+         (dl.download)
+         (dl.calcMD5)))
+
+(test* "download to file"
+       "c93df55bb4034c353d6aa8b40734329f  ./data/SEKKA-JISYO-1.6.2.N.tsv"
+       (let1 dl (Downloader.new "https://raw.githubusercontent.com/kiyoka/sekka/master/public_dict/1.6.2/SEKKA-JISYO-1.6.2.N.md5")
+         (dl.downloadToFile "/tmp/md5.txt")
+         (with-open "/tmp/md5.txt"
+                    (lambda (f)
+                      (f.read.chomp)))))
+
+;;===================================================================
+
+(test-end)

data/test/henkan-main.nnd

@@ -60,6 +60,8 @@
   (set! target "./test.dump"))
  ((eq? dbtype 'leveldb)
   (set! target "./test.ldb"))
+ ((eq? dbtype 'mapdb)
+  (set! target "./test2.mapdb"))
  ((eq? dbtype 'memcache)
   (error "memcached interface is obsolute."))
  (else
@@ -152,6 +154,8 @@
   (kvs.open target))
  ((eq? dbtype 'leveldb)
   (kvs.open target))
+ ((eq? dbtype 'mapdb)
+  (kvs.open target))
  ((eq? dbtype 'memcache)
   (error "memcached interface is obsolute."))
  (else

data/test/jruby_mapdb.nnd

@@ -0,0 +1,105 @@
+;;-*- mode: nendo; syntax: scheme -*-
+;;;
+;;; jruby_mapdb.nnd - jruby_mapdbライブラリのテスト
+;;;  
+;;;   Copyright (c) 2017  Kiyoka Nishiyama  <kiyoka@sumibi.org>
+;;;   
+;;;   Redistribution and use in source and binary forms, with or without
+;;;   modification, are permitted provided that the following conditions
+;;;   are met:
+;;;   
+;;;   1. Redistributions of source code must retain the above copyright
+;;;      notice, this list of conditions and the following disclaimer.
+;;;  
+;;;   2. Redistributions in binary form must reproduce the above copyright
+;;;      notice, this list of conditions and the following disclaimer in the
+;;;      documentation and/or other materials provided with the distribution.
+;;;  
+;;;   3. Neither the name of the authors nor the names of its contributors
+;;;      may be used to endorse or promote products derived from this
+;;;      software without specific prior written permission.
+;;;  
+;;;   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+;;;   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+;;;   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+;;;   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+;;;   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+;;;   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+;;;   TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+;;;   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+;;;   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+;;;   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+;;;   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;;  
+;;;  $Id: 
+;;;
+(require "sekka/jruby_mapdb")
+(use nendo.test)
+(load "./test/common.nnd")
+
+(test-start "MapDB (only Java)")
+;;===================================================================
+
+;;-------------------------------------------------------------------
+(test-section "DB access")
+
+(define db #f)
+(define tree #f)
+
+(set! db (MapDB::DB.new "test1.mapdb" "sekka1"))
+(test* "db open"
+       "val1"
+       (let1 tree (db.getTree)
+         (tree.put "key1" "val1")
+         (tree.get "key1")))
+(db.close)
+
+(set! db (MapDB::DB.new "test1.mapdb" "sekka2"))
+(test* "db put & get"
+       "val1val2val3"
+       (begin
+         (let1 tree (db.getTree)
+           (tree.put "key1" "val1")
+           (tree.put "key2" "val2")
+           (tree.put "key3" "val3")
+           (+ (tree.get "key1")
+              (tree.get "key2")
+              (tree.get "key3")))))
+(db.close)
+
+(set! db (MapDB::DB.new "test1.mapdb" "sekka3"))
+(test* "db keys"
+       '("key1" "key2" "key3" "key4")
+       (begin
+         (let1 tree (db.getTree)
+           (tree.put "key1" "1")
+           (tree.put "key2" "2")
+           (tree.put "key3" "3")
+           (tree.put "key4" "4")
+           (to-list (tree.keys)))))
+
+(test* "db delete"
+       '("key1" "key2" "key4")
+       (begin
+         (let1 tree (db.getTree)
+           (tree.delete "key3")
+           (to-list (tree.keys)))))
+
+(test* "db put kanji"
+       "まじぱねぇ /マジパネェ/"
+       (begin
+         (let1 tree (db.getTree)
+           (tree.put "M:i;chi" "Cいっち")
+           (tree.get "M:i;chi")
+           (tree.put "M:かに" "/蟹/可児/蠏")
+           (tree.get "M:かに")
+           (tree.put "dummy:(stored)" "まじぱねぇ /マジパネェ/")
+           (tree.get "dummy:(stored)")
+           )))
+       
+(db.close)
+
+;;===================================================================
+
+
+(test-end)