rufus-tokyo 0.1.0

data/CHANGELOG.txt

@@ -0,0 +1,8 @@
+
+= rufus-tokyo CHANGELOG.txt
+
+
+== rufus-tokyo - 0.1.0    released 2009/01/23
+
+- initial release
+

data/CREDITS.txt

@@ -0,0 +1,8 @@
+
+= CREDITS.txt rufus-tokyo gem
+
+many thanks to the authors of Tokyo Cabinet and ruby-ffi
+
+  http://tokyocabinet.sourceforge.net
+  http://kenai.com/projects/ruby-ffi
+

data/README.txt

@@ -0,0 +1,118 @@
+
+= rufus-tokyo
+
+ruby-ffi based interface to Tokyo Cabinet.
+
+It focuses on the abstract API for now (http://tokyocabinet.sourceforge.net/spex-en.html#tcadbapi)
+
+
+== installation
+
+  sudo gem install ffi rufus-tokyo
+
+  (see after 'usage' for how to install Tokyo Cabinet if required)
+
+
+== usage
+
+to create a hash (file named 'data.tch')
+
+  require 'rubygems'
+  require 'rufus/tokyo'
+
+  db = Rufus::Tokyo::Cabinet.new('data.tch')
+
+  db['nada'] = 'surf'
+
+  p db['nada'] # => 'surf'
+  p db['lost'] # => nil
+
+  5000.times { |i| db[i.to_s] = "x" }
+
+  p db.inject { |r, (k, v)| k } # => 4999
+
+  db.close
+
+
+more in the rdoc
+
+  http://rufus.rubyforge.org/rufus-tokyo/
+  http://rufus.rubyforge.org/rufus-tokyo/classes/Rufus/Tokyo/Cabinet.html
+
+or directly in the source
+
+  http://github.com/jmettraux/rufus-tokyo/blob/master/lib/rufus/tokyo.rb 
+
+
+== Tokyo Cabinet install
+
+On a Mac, you would do 
+
+  sudo port install tokyocabinet
+
+
+If you don't have Tokyo Cabinet on your system, you can get it and compile it :
+
+  git clone git://github.com/etrepum/tokyo-cabinet.git
+  cd tokyo-cabinet
+  git checkout 1.4.1
+  ./configure
+  make
+
+eventually
+
+  sudo make install
+
+
+== dependencies
+
+  the ruby gem ffi
+
+
+== mailing list
+
+On the rufus-ruby list[http://groups.google.com/group/rufus-ruby] :
+
+    http://groups.google.com/group/rufus-ruby
+
+
+== issue tracker
+
+http://rubyforge.org/tracker/?atid=18584&group_id=4812&func=browse
+
+
+== irc
+
+irc.freenode.net #ruote
+
+
+== source
+
+http://github.com/jmettraux/rufus-tokyo
+
+    git clone git://github.com/jmettraux/rufus-tokyo.git
+
+
+== credits
+
+many thanks to the authors of Tokyo Cabinet and to the authors of ruby-ffi
+
+  http://tokyocabinet.sourceforge.net
+  http://kenai.com/projects/ruby-ffi
+
+
+== author
+
+John Mettraux, jmettraux@gmail.com
+http://jmettraux.wordpress.com
+
+
+== the rest of Rufus
+
+http://rufus.rubyforge.org
+
+
+== license
+
+MIT
+

data/lib/rufus-tokyo.rb

@@ -0,0 +1,3 @@
+
+require 'rufus/tokyo'
+

data/lib/rufus/tokyo.rb

@@ -0,0 +1,243 @@
+#
+#--
+# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#++
+#
+
+#
+# "made in Japan"
+#
+# jmettraux@gmail.com
+#
+
+require 'rubygems'
+require 'ffi'
+
+module Rufus
+module Tokyo
+
+  VERSION = '0.1.0'
+
+  module Func #:nodoc#
+    extend FFI::Library
+
+    #
+    # find Tokyo Cabinet lib
+
+    paths = Array(ENV['TOKYO_CABINET_LIB'] || %w{
+      /opt/local/lib/libtokyocabinet.dylib
+      /usr/local/lib/libtokyocabinet.dylib
+      /usr/local/lib/libtokyocabinet.so
+    })
+
+    paths.each do |path|
+      if File.exist?(path)
+        ffi_lib(path)
+        @lib = path
+        break
+      end
+    end
+
+    attach_function :tcadbnew, [], :pointer
+
+    attach_function :tcadbopen, [ :pointer, :string ], :int
+    attach_function :tcadbclose, [ :pointer ], :int
+
+    attach_function :tcadbdel, [ :pointer ], :void
+
+    attach_function :tcadbrnum, [ :pointer ], :uint64
+    attach_function :tcadbsize, [ :pointer ], :uint64
+
+    attach_function :tcadbput2, [ :pointer, :string, :string ], :int
+    attach_function :tcadbget2, [ :pointer, :string ], :string
+    attach_function :tcadbout2, [ :pointer, :string ], :int
+
+    attach_function :tcadbiterinit, [ :pointer ], :int
+    attach_function :tcadbiternext2, [ :pointer ], :string
+
+    #def self.method_missing (m, *args)
+    #  mm = "tcadb#{m}"
+    #  self.respond_to?(mm) ? self.send(mm, *args) : super
+    #end
+      #
+      # makes JRuby unhappy, forget it.
+  end
+
+  #
+  # Returns the path to the Tokyo Cabinet dynamic library currently in use
+  #
+  def self.lib
+    Rufus::Tokyo::Func.instance_variable_get(:@lib)
+  end
+
+  #
+  # A 'cabinet', ie a Tokyo Cabinet database.
+  #
+  # Follows the abstract API described at :
+  #
+  #   http://tokyocabinet.sourceforge.net/spex-en.html#tcadbapi
+  #
+  # An usage example :
+  #
+  #   db = Rufus::Tokyo::Cabinet.new('test_data.tch')
+  #   db['pillow'] = 'Shonagon'
+  #
+  #   db.size # => 1
+  #   db['pillow'] # => 'Shonagon'
+  #
+  #   db.delete('pillow') # => 'Shonagon'
+  #   db.size # => 0
+  #
+  #   db.close
+  #
+  class Cabinet
+    include Enumerable
+
+    #
+    # Creates/opens the cabinet, raises an exception in case of
+    # creation/opening failure.
+    #
+    # This method accepts a 'name' parameter and an optional 'params' hash
+    # parameter.
+    #
+    # 'name' follows the syntax described at
+    #
+    #   http://tokyocabinet.sourceforge.net/spex-en.html#tcadbapi
+    #
+    # under tcadbopen(). For example :
+    #
+    #   db = Rufus::Tokyo::Cabinet.new('casket.tch#bnum=100000#opts=ld')
+    #
+    # will open (eventually create) a hash database backed in the file
+    # 'casket.tch' with a bucket number of 100000 and the 'large' and
+    # 'deflate' options (opts) turned on.
+    #
+    # == :hash or :tree
+    #
+    # Setting the name to :hash or :tree simply will create a in-memory hash
+    # or tree respectively (see #new_tree and #new_hash).
+    #
+    # == tuning parameters
+    #
+    # It's ok to use the optional params hash to pass tuning parameters and
+    # options, thus
+    #
+    #   db = Rufus::Tokyo::Cabinet.new('casket.tch#bnum=100000#opts=ld')
+    #
+    # and
+    #
+    #   db = Rufus::Tokyo::Cabinet.new(
+    #     'casket.tch', :bnum => 100000, :opts => 'ld')
+    #
+    # are equivalent.
+    #
+    # == mode
+    #
+    # To open a db in read-only mode :
+    #
+    #   db = Rufus::Tokyo::Cabinet.new('casket.tch#mode=r')
+    #   db = Rufus::Tokyo::Cabinet.new('casket.tch', :mode => 'r')
+    #
+    def initialize (name, params={})
+
+      @db = Rufus::Tokyo::Func.tcadbnew
+
+      name = '*' if name == :hash # in memory hash database
+      name = '+' if name == :tree # in memory B+ tree database
+
+      name = name + params.collect { |k, v| "##{k}=#{v}" }.join('')
+
+      (Rufus::Tokyo::Func.tcadbopen(@db, name) == 1) ||
+        raise("failed to open/create db '#{name}'")
+    end
+
+    #
+    # Returns a new in-memory hash. Accepts the same optional params hash
+    # as new().
+    #
+    def self.new_hash (params={})
+      self.new(:hash, params)
+    end
+
+    #
+    # Returns a new in-memory B+ tree. Accepts the same optional params hash
+    # as new().
+    #
+    def self.new_tree (params={})
+      self.new(:tree, params)
+    end
+
+    def []= (k, v)
+      Rufus::Tokyo::Func.tcadbput2(@db, k, v)
+    end
+
+    def [] (k)
+      Rufus::Tokyo::Func.tcadbget2(@db, k) rescue nil
+    end
+
+    #
+    # Removes a record from the cabinet, returns the value if successful
+    # else nil.
+    #
+    def delete (k)
+      v = self[k]
+      (Rufus::Tokyo::Func.tcadbout2(@db, k) == 1) ? v : nil
+    end
+
+    #
+    # Returns the number of records in the 'cabinet'
+    #
+    def size
+      Rufus::Tokyo::Func.tcadbrnum(@db)
+    end
+
+    #
+    # Returns the 'weight' of the db (in bytes)
+    #
+    def weight
+      Rufus::Tokyo::Func.tcadbsize(@db)
+    end
+
+    #
+    # Closes the cabinet (and frees the datastructure allocated for it),
+    # returns true in case of success.
+    #
+    def close
+      result = Rufus::Tokyo::Func.tcadbclose(@db)
+      Rufus::Tokyo::Func.tcadbdel(@db)
+      (result == 1)
+    end
+
+    #
+    # The classical Ruby each (unlocks the power of the Enumerable mixin)
+    #
+    def each
+
+      Rufus::Tokyo::Func.tcadbiterinit(@db) # concurrent access ??
+
+      while (k = (Rufus::Tokyo::Func.tcadbiternext2(@db) rescue nil))
+        yield(k, self[k])
+      end
+    end
+  end
+end
+end
+

data/test/cabinet_0_test.rb

@@ -0,0 +1,44 @@
+
+#
+# Testing rufus-tokyo
+#
+# Fri Jan 23 08:31:01 JST 2009
+#
+
+require File.dirname(__FILE__) + '/test_base'
+
+require 'test/unit'
+
+require 'rufus/tokyo'
+
+
+class CabinetZero < Test::Unit::TestCase
+
+  #def setup
+  #end
+  #def teardown
+  #end
+
+  def test_lib
+
+    assert_not_nil Rufus::Tokyo.lib
+  end
+
+  def test_basic_workflow
+
+    db = Rufus::Tokyo::Cabinet.new('test_data.tch')
+    db['pillow'] = 'Shonagon'
+
+    assert_equal 1, db.size
+    assert_equal 'Shonagon', db['pillow']
+
+    assert_equal 'Shonagon', db.delete('pillow')
+    assert_equal 0, db.size
+
+    assert db.close
+  end
+
+  # TODO : memory tests
+
+end
+

data/test/mem.rb

@@ -0,0 +1,49 @@
+
+%w{ lib test }.each do |path|
+  path = File.expand_path(File.dirname(__FILE__) + '/../' + path)
+  $: << path unless $:.include?(path)
+end
+
+require 'fileutils'
+
+
+def self_ps
+  ps = `ps -v #{$$}`.split("\n").last.split(' ')
+  %w{
+    pid stat time sl re pagein vsz rss lim tsiz pcpu pmem command
+  }.inject({}) { |h, k|
+    h[k.intern] = ps.shift; h
+  }
+end
+
+def pmem (msg)
+  p [ msg, "#{self_ps[:vsz].to_i / 1024}k" ]
+end
+
+pmem 'starting'
+
+require 'rubygems'
+require 'rufus/tokyo'
+
+FileUtils.rm('test_mem.tch')
+
+db = Rufus::Tokyo::Cabinet.new('test_mem.tch')
+
+pmem 'wired to db'
+
+500_000.times { |i| db[i.to_s] = "value#{i}" }
+
+pmem 'loaded 500_000 records'
+
+db.each { |k, v| k }
+
+pmem 'iterated 500_000 records'
+
+a = db.collect { |k, v| k + v }
+
+pmem 'collected 500_000 records'
+
+db.close
+
+pmem 'closed db'
+

data/test/test.rb

@@ -0,0 +1,5 @@
+
+require File.dirname(__FILE__) + '/test_base'
+
+require 'cabinet_0_test'
+

data/test/test_base.rb

@@ -0,0 +1,6 @@
+
+%w{ lib test }.each do |path|
+  path = File.expand_path(File.dirname(__FILE__) + '/../' + path)
+  $: << path unless $:.include?(path)
+end
+

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification 
+name: rufus-tokyo
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- John Mettraux
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-01-23 00:00:00 +09:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: ruby-ffi
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: 
+email: jmettraux@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.txt
+- CHANGELOG.txt
+- CREDITS.txt
+files: 
+- lib/rufus
+- lib/rufus/tokyo.rb
+- lib/rufus-tokyo.rb
+- test/cabinet_0_test.rb
+- test/mem.rb
+- test/test.rb
+- test/test_base.rb
+- README.txt
+- CHANGELOG.txt
+- CREDITS.txt
+has_rdoc: true
+homepage: http://rufus.rubyforge.org/
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: 
+- ruby-ffi
+rubyforge_project: rufus
+rubygems_version: 1.3.1
+signing_key: 
+specification_version: 2
+summary: ruby-ffi based lib to access Tokyo Cabinet
+test_files: 
+- test/test.rb