distributed-trie 0.8.0

data/.gemtest

@@ -0,0 +1 @@
+

data/COPYING

@@ -0,0 +1,33 @@
+Copyright and condition of use of main portion of the source:
+-----------------------------------------------------------------------------
+
+  Copyright (c) 2012 Kiyoka Nishiyama  <kiyoka@sumibi.org>
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+
+   1. Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+
+   3. Neither the name of the authors nor the names of its contributors
+      may be used to endorse or promote products derived from this
+      software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+  TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+-----------------------------------------------------------------------------

data/README.md

@@ -0,0 +1,59 @@
+# What is distributed-trie
+
+* distributed-trie is a trie library on key-value store.
+* It is scalable.
+* It supports Tokyo Cabinet / memcached / gdbm / pure hash / Redis / DynamoDB / SimpleDB
+
+## The reason why i developed 
+* I need a trie library for Sekka ( japanese input method ).
+* I need a trie library which written in pure Ruby.
+* I need a trie library which can scale out.
+
+## Installing 
+
+    gem install distributed-trie
+
+## Features
+* Add    keyword to trie.
+* Delete keyword to trie.         ( not implemented... )
+* commonPrefixSearch by keyword.
+* fuzzySearch by jaro winker edit distance.
+* search with user-defined-function.
+
+## Architecture
+* distributed-trie gem only manage trie data structure.
+* You should manage your application data which corresponds to trie key.
+
+![Figure]( http://pix.am/kYLz.png )
+
+
+## Sample code
+
+    require 'distributedtrie'
+    require 'distributedtrie/kvs/tokyocabinet'
+    kvsTc = DistributedTrie::KvsTc.new( '/tmp/distributed-trie.tch' )
+    trie = DistributedTrie::Trie.new( kvsTc, "Sample::" )
+    trie.addKey!( "apple"       )
+    trie.addKey!( "application" )
+    trie.addKey!( "orange"      )
+    trie.commit!
+    result = trie.commonPrefixSearch( "app" )
+    print result
+    # =>  [ "apple", "application" ]
+    result = trie.fuzzySearch( "app", 0.80 )
+    print result
+    # =>  [[0.9066666666666667, "apple"], [0.8236914600550963, "application"]]
+
+## Requires
+ - Ruby  1.9.1 or higher
+ - JRuby 1.6.6 or higher
+ - fuzzy-string-match gem
+
+## Author
+ - Copyright (C) Kiyoka Nishiyama <kiyoka@sumibi.org>
+
+## See also
+ - <http://github.com/kiyoka/distributed-trie>
+
+## License
+ - BSD License

data/Rakefile

@@ -0,0 +1,72 @@
+# -*- mode: ruby; -*-
+#                                           Rakefile for Distributed-Trie
+# Release Engineering
+#   1. edit the VERSION.yml file
+#   2. rake
+#   3. rake gemspec  &&   rake build
+#      to generate distributed-trie-x.x.x.gem
+#   4. install distributed-trie-x.x.x.gem to clean environment and test
+#   5. rake release
+#   6. gem push pkg/distributed-trie-x.x.x.gem   ( need gem version 1.3.6 or higer. Please "gem update --system" to update )
+
+require 'rake'
+begin
+  require 'jeweler2'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "distributed-trie"
+    gemspec.summary = "distributed-trie is a trie library on key-value store."
+    gemspec.description = "distributed-trie is a trie library on key-value store."
+    gemspec.email = "kiyoka@sumibi.org"
+    gemspec.homepage = "http://github.com/kiyoka/distributed-trie"
+    gemspec.authors = ["Kiyoka Nishiyama"]
+    gemspec.files = FileList[
+                    'Rakefile',
+                    '.gemtest',
+                    'VERSION.yml',
+                    'README.md',
+                    'COPYING',
+                    'lib/**/*.rb',
+                    'lib/*.png',
+                    'test/*'
+                      ].to_a
+    gemspec.add_development_dependency "rake"
+    gemspec.add_development_dependency "rspec"
+    gemspec.add_dependency( "fuzzy-string-match", ">= 0.9.3" )
+  end
+rescue LoadError
+  puts 'Jeweler2 not available. If you want to build a gemfile, please install with "sudo gem install jeweler2"'
+end
+
+task :default => [:test] do
+end
+
+task :test do
+  sh "time ruby -I ./lib `which rspec` -b   ./test/internal_spec.rb     -r ./test/rspec_formatter_for_emacs.rb -f CustomFormatter"
+  sh "time ruby -I ./lib `which rspec` -b   ./test/usecase_spec.rb      -r ./test/rspec_formatter_for_emacs.rb -f CustomFormatter"
+  sh "time ruby -I ./lib `which rspec` -b   ./test/bigdata_spec.rb      -r ./test/rspec_formatter_for_emacs.rb -f CustomFormatter"
+end
+
+DATA="aspell.dump.txt"
+#DATA="wlist_match1.txt"
+
+task :bench_setup do
+  sh "ruby -I ./lib ./benchmark/bench.rb  setup  ./data/#{DATA}"
+end
+
+task :bench do
+  # URL http://www.keithv.com/software/wlist/wlist_match1.zip
+  sh "ruby -I ./lib ./benchmark/bench.rb  main   ./data/#{DATA}"
+end
+
+task :bench_random do
+  sh "ruby -I ./lib ./benchmark/bench.rb random  ./data/#{DATA}"
+end
+
+task :dumptc do
+  sh "bash -c 'tchmgr list -pv /tmp/distributed-trie.tch > /tmp/distributed-trie.txt'"
+end
+
+task :data do
+  sh "aspell -l en dump master > ./data/aspell.dump.txt"
+end
+

data/VERSION.yml

@@ -0,0 +1,4 @@
+---
+:major: 0
+:minor: 8
+:patch: 0

data/lib/distributedtrie.rb

@@ -0,0 +1,38 @@
+#
+#                            Distributed Trie
+#
+#
+#   Copyright (c) 2012  Kiyoka Nishiyama  <kiyoka@sumibi.org>
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#   1. Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+#
+#   3. Neither the name of the authors nor the names of its contributors
+#      may be used to endorse or promote products derived from this
+#      software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+#   TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+#   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+#   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+#   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+#   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+#
+require 'distributedtrie/kvsif'
+require 'distributedtrie/trie'
+
+

data/lib/distributedtrie/kvs/base.rb

@@ -0,0 +1,55 @@
+#
+#                        Distributed Trie / KvsIFs
+#
+#
+#   Copyright (c) 2012  Kiyoka Nishiyama  <kiyoka@sumibi.org>
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#   1. Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+#
+#   3. Neither the name of the authors nor the names of its contributors
+#      may be used to endorse or promote products derived from this
+#      software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+#   TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+#   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+#   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+#   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+#   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+#
+require 'distributedtrie/kvsif'
+module DistributedTrie
+
+  # pure hash implementation
+  class KvsBase < KvsIf
+    def put!( key, value, timeout = 0 )
+      @db[ key.force_encoding("ASCII-8BIT") ] = value.force_encoding("ASCII-8BIT")
+    end
+
+    def get( key, fallback = false )
+      val = @db[ key ]
+      if val
+        val.force_encoding("UTF-8")
+      else
+        fallback
+      end
+    end
+
+    def enabled?()  true  end
+  end
+end

data/lib/distributedtrie/kvs/dbm.rb

@@ -0,0 +1,45 @@
+#
+#                        Distributed Trie / KvsDbm
+#
+#
+#   Copyright (c) 2012  Kiyoka Nishiyama  <kiyoka@sumibi.org>
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#   1. Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+#
+#   3. Neither the name of the authors nor the names of its contributors
+#      may be used to endorse or promote products derived from this
+#      software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+#   TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+#   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+#   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+#   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+#   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+#
+require 'distributedtrie/kvs/base'
+require 'dbm'
+module DistributedTrie
+
+  # dbm implementation
+  class KvsDbm < KvsBase
+    def initialize( dbFilename )
+      @db = DBM.new( dbFilename )
+    end
+  end
+end

data/lib/distributedtrie/kvs/dynamodb.rb

@@ -0,0 +1,81 @@
+#
+#                        Distributed Trie / KvsDydb
+#
+#
+#   Copyright (c) 2012  Kiyoka Nishiyama  <kiyoka@sumibi.org>
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#   1. Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+#
+#   3. Neither the name of the authors nor the names of its contributors
+#      may be used to endorse or promote products derived from this
+#      software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+#   TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+#   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+#   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+#   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+#   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+#
+require 'distributedtrie/kvs/base'
+module DistributedTrie
+
+  # AWS DynamoDB implementation
+  begin
+    require 'aws-sdk'
+    class KvsDydb < KvsBase
+      def initialize( tableName )
+        printf( "Amazon DynamoDB access_key_id:     %s\n", ENV['AMAZON_ACCESS_KEY_ID'])
+        printf( "Amazon DynamoDB secret_access_key: %s\n", ENV['AMAZON_SECRET_ACCESS_KEY'])
+        @tableName = tableName
+        @db = AWS::DynamoDB.new(
+                            :access_key_id      => ENV['AMAZON_ACCESS_KEY_ID'],
+                            :secret_access_key  => ENV['AMAZON_SECRET_ACCESS_KEY'],
+                            :dynamo_db_endpoint => 'dynamodb.ap-northeast-1.amazonaws.com',
+                            :use_ssl            => false )
+        @table = @db.tables[ @tableName ]
+        @table.hash_key = [ :key, :string ]
+      end
+      def put!( key, value, timeout = 0 )
+        item = @table.items.create( 'key' => key, 'val' => value.force_encoding("ASCII-8BIT"))
+        puts "DynamoDB put: " + key + " , " + value
+      end
+      def get( key, fallback = false )
+        item = @table.items[key]
+        val = nil
+        item.attributes.each { |name,value |
+          val = value      if name == "val"
+        }
+        if val
+          puts "DynamoDB get: " + key + "," + val
+          val.force_encoding("UTF-8")
+        else
+          fallback
+        end
+      end
+      def enabled?()   true   end
+
+      attr_reader :db
+    end
+  rescue LoadError
+    class KvsDydb < KvsBase
+      def initialize( tableName ) end
+      def enabled?()   false  end
+    end
+  end
+end

data/lib/distributedtrie/kvs/memcache.rb

@@ -0,0 +1,52 @@
+#
+#                        Distributed Trie / KvsMemcache
+#
+#
+#   Copyright (c) 2012  Kiyoka Nishiyama  <kiyoka@sumibi.org>
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#   1. Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#
+#   2. Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+#
+#   3. Neither the name of the authors nor the names of its contributors
+#      may be used to endorse or promote products derived from this
+#      software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+#   TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+#   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+#   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+#   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+#   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+#
+require 'distributedtrie/kvs/base'
+require 'memcache'
+module DistributedTrie
+
+  # Memcache client implementation
+  class KvsMemcache < KvsBase
+    def initialize( host = "localhost", port = "1978" )
+      @db = MemCache.new(
+                         host + ":" + port,
+                         :connect_timeout => 1000.0,
+                         :timeout => 1000.0 )
+    end
+
+    def put!( key, value, timeout = 0 )
+      @db.set( key.force_encoding("ASCII-8BIT"), value.force_encoding("ASCII-8BIT"), timeout )
+    end
+  end
+end