rufus-tokyo 0.1.6 → 0.1.7

data/CHANGELOG.txt

@@ -2,6 +2,17 @@
 = rufus-tokyo CHANGELOG.txt
 
 
+== rufus-tokyo - 0.1.7    released 2009/02/19
+
+- todo  : Rufus::Tokyo::Cabinet.new('filename', :type => :hash) now OK (Zev)
+- todo  : more documentation for Rufus::Tokyo::Cabinet#new
+- added : Rufus::Tokyo::Cabinet has a open method that takes a block like
+          File.open (Zev)
+- todo  : aligned Rufus::Tokyo::Table#new on ::Cabinet#new
+- todo  : added opts (:prefix, :limit, :native) to Rufus::Tokyo::Cabinet#keys,
+          Rufus::Tokyo::Table#keys and the corresponding Tyrant classes
+
+
 == rufus-tokyo - 0.1.6    released 2009/02/16
 
 - todo : Tyrant and TyrantTable now complain when used in lieu of each other

data/CREDITS.txt

@@ -4,13 +4,15 @@
 
 == authors
 
-John Mettraux http://jmettraux.wordpress.com
-Justin Reagor http://blog.kineticweb.com/
+John Mettraux  http://jmettraux.wordpress.com
+Justin Reagor  http://blog.kineticweb.com/
+Zev Blut  http://www.iknow.co.jp/users/zev
 
 
 == finally
 
-many thanks to the author of Tokyo Cabinet (Mikio Hirabayashi) and of ruby-ffi
+many thanks to the author of Tokyo Cabinet (Mikio Hirabayashi) and 
+the authors of ruby-ffi
 
   http://tokyocabinet.sourceforge.net
   http://kenai.com/projects/ruby-ffi

data/README.txt

@@ -240,6 +240,7 @@ many thanks to the author of Tokyo Cabinet, Mikio Hirabayashi, and to the author
 
 John Mettraux, jmettraux@gmail.com, http://jmettraux.wordpress.com
 Justin Reagor, http://blog.kineticweb.com/
+Zev Blut, http://www.iknow.co.jp/users/zev
 
 
 == the rest of Rufus

data/lib/rufus/tokyo.rb

@@ -22,13 +22,19 @@
 #++
 #
 
+#
+# "made in Japan"
+#
+# jmettraux@gmail.com
+#
+
 require 'ffi' # sudo gem install ffi
 
 
 module Rufus
   module Tokyo
 
-    VERSION = '0.1.6'
+    VERSION = '0.1.7'
 
     #
     # A common error class
@@ -40,7 +46,3 @@ end
 
 require 'rufus/tokyo/cabinet'
 
-# TODO : do not include me
-#require 'rufus/tokyo/tyrant'
-#require 'rufus/tokyo/dystopia'
-

data/lib/rufus/tokyo/cabinet/abstract.rb

@@ -74,48 +74,140 @@ module Rufus
       # 'casket.tch' with a bucket number of 100000 and the 'large' and
       # 'deflate' options (opts) turned on.
       #
-      # == :hash or :tree
+      # Note that there is an #open method similar to File#open for openening
+      # a db and closing it when it's no longer needed :
       #
-      # Setting the name to :hash or :tree simply will create a in-memory hash
-      # or tree respectively (see #new_tree and #new_hash).
+      #   Rufus::Tokyo::Cabinet.new('data.tch') do |db|
+      #     db['key'] = value
+      #   end
       #
-      # == tuning parameters
+      # == database name
       #
-      # It's ok to use the optional params hash to pass tuning parameters and
-      # options, thus
+      # From http://tokyocabinet.sourceforge.net/spex-en.html#tcadbapi :
       #
-      #   db = Rufus::Tokyo::Cabinet.new('casket.tch#bnum=100000#opts=ld')
+      # 'If it is "*", the database will be an on-memory hash database. If it is
+      #  "+", the database will be an on-memory tree database. If its suffix is
+      #  ".tch", the database will be a hash database. If its suffix is ".tcb",
+      #  the database will be a B+ tree database. If its suffix is ".tcf", the
+      #  database will be a fixed-length database. If its suffix is ".tct", the
+      #  database will be a table database.'
+      #
+      # You're supposed to give a path to the database file you want to use and
+      # Cabinet expects you to give the proper prefix.
+      #
+      #   db = Rufus::Tokyo::Cabinet.new('data.tch') # hash database
+      #   db = Rufus::Tokyo::Cabinet.new('data.tcb') # B+ tree db
+      #   db = Rufus::Tokyo::Cabinet.new('data.tcf') # fixed-length db
+      #
+      # will result with the same file names :
+      #
+      #   db = Rufus::Tokyo::Cabinet.new('data', :type => :hash) # hash database
+      #   db = Rufus::Tokyo::Cabinet.new('data', :type => :btree) # B+ tree db
+      #   db = Rufus::Tokyo::Cabinet.new('data', :type => :fixed) # fixed-length db
+      #
+      # You can open an in-memory hash and an in-memory B+ tree with :
+      #
+      #   h = Rufus::Tokyo::Cabinet.new(:mem_hash) # or
+      #   h = Rufus::Tokyo::Cabinet.new('*')
+      #
+      #   t = Rufus::Tokyo::Cabinet.new(:mem_tree) # or
+      #   t = Rufus::Tokyo::Cabinet.new('+')
+      #
+      # == parameters
       #
-      # and
+      # There are two ways to pass parameters at the opening of a db :
+      #
+      #   db = Rufus::Tokyo::Cabinet.new('data.tch#opts=ld#mode=w') # or
+      #   db = Rufus::Tokyo::Cabinet.new('data.tch', :opts => 'ld', :mode => 'w')
+      #
+      # most verbose :
       #
       #   db = Rufus::Tokyo::Cabinet.new(
-      #     'casket.tch', :bnum => 100000, :opts => 'ld')
+      #     'data', :type => :hash, :opts => 'ld', :mode => 'w')
+      #
+      # === mode
+      #
+      #   * :mode    a set of chars ('r'ead, 'w'rite, 'c'reate, 't'runcate,
+      #              'e' non locking, 'f' non blocking lock), default is 'wc'
+      #
+      # === other parameters
+      #
+      # 'On-memory hash database supports "bnum", "capnum", and "capsiz".
+      #  On-memory tree database supports "capnum" and "capsiz".
+      #  Hash database supports "mode", "bnum", "apow", "fpow", "opts",
+      #  "rcnum", and "xmsiz".
+      #  B+ tree database supports "mode", "lmemb", "nmemb", "bnum", "apow",
+      #  "fpow", "opts", "lcnum", "ncnum", and "xmsiz".
+      #  Fixed-length database supports "mode", "width", and "limsiz"'
+      #
+      #   * :opts    a set of chars ('l'arge, 'd'eflate, 'b'zip2, 't'cbs)
+      #              (usually empty or something like 'ld' or 'lb')
+      #
+      #   * :bnum    number of elements of the bucket array
+      #   * :apow    size of record alignment by power of 2 (defaults to 4)
+      #   * :fpow    maximum number of elements of the free block pool by
+      #              power of 2 (defaults to 10)
+      #   * :mutex   when set to true, makes sure only 1 thread at a time
+      #              accesses the table (well, Ruby, global thread lock, ...)
       #
-      # are equivalent.
+      #   * :rcnum   specifies the maximum number of records to be cached.
+      #              If it is not more than 0, the record cache is disabled.
+      #              It is disabled by default.
+      #   * :lcnum   specifies the maximum number of leaf nodes to be cached.
+      #              If it is not more than 0, the default value is specified.
+      #              The default value is 2048.
+      #   * :ncnum   specifies the maximum number of non-leaf nodes to be
+      #              cached. If it is not more than 0, the default value is
+      #              specified. The default value is 512.
       #
-      # == mode
+      #   * :xmsiz   specifies the size of the extra mapped memory. If it is
+      #              not more than 0, the extra mapped memory is disabled.
+      #              The default size is 67108864.
       #
-      # To open a db in read-only mode :
+      #   * :capnum  specifies the capacity number of records.
+      #   * :capsiz  specifies the capacity size of using memory.
       #
-      #   db = Rufus::Tokyo::Cabinet.new('casket.tch#mode=r')
-      #   db = Rufus::Tokyo::Cabinet.new('casket.tch', :mode => 'r')
+      #
+      # = NOTE :
+      #
+      # On reopening a file, Cabinet will tend to stick to the parameters as
+      # set when the file was opened. To change that, have a look at the
+      # man pages of the various command line tools coming with Tokyo Cabinet.
       #
       def initialize (name, params={})
 
         @db = lib.tcadbnew
 
-        name = '*' if name == :hash # in memory hash database
-        name = '+' if name == :tree # in memory B+ tree database
+        name = '*' if name == :mem_hash # in memory hash database
+        name = '+' if name == :mem_tree # in memory B+ tree database
+
+        if type = params.delete(:type)
+          name += { :hash => '.tch', :btree => '.tcb', :fixed => '.tcf' }[type]
+        end
 
         name = name + params.collect { |k, v| "##{k}=#{v}" }.join('')
 
         (lib.tcadbopen(@db, name) == 1) ||
-          raise("failed to open/create db '#{name}'")
+          raise("failed to open/create db '#{name}'  #{params.inspect}")
 
         self.default = params[:default]
         @default_proc ||= params[:default_proc]
       end
 
+      # Same args as initialize, but can take a block form that will
+      # close the db when done.  Similar to File.open
+      def self.open (name, params={})
+        db = self.new(name, params)
+        if block_given?
+          yield db
+          nil
+        else
+          db
+        end
+      ensure
+        db.close if block_given? && db
+      end
+
       #
       # Returns a new in-memory hash. Accepts the same optional params hash
       # as new().
@@ -229,11 +321,41 @@ module Rufus
       #
       # Returns an array with all the keys in the databse
       #
-      def keys
-        a = []
-        lib.abs_iterinit(@db)
-        while (k = (lib.abs_iternext2(@db) rescue nil)); a << k; end
-        a
+      # With no options given, this method will return all the keys (strings)
+      # in a Ruby array.
+      #
+      #   :prefix --> returns only the keys who match a given string prefix
+      #
+      #   :limit --> returns a limited number of keys
+      #
+      #   :native --> returns an instance of Rufus::Tokyo::List instead of
+      #     a Ruby Hash, you have to call #free on that List when done with it !
+      #     Else you're exposing yourself to a memory leak.
+      #
+      def keys (options={})
+
+        if pref = options[:prefix]
+
+          l = lib.abs_fwmkeys2(@db, pref, options[:limit] || -1)
+          l = Rufus::Tokyo::List.new(l)
+          options[:native] ? l : l.release
+
+        else
+
+          limit = options[:limit] || -1
+          limit = nil if limit < 1
+
+          l = options[:native] ? Rufus::Tokyo::List.new : []
+
+          lib.abs_iterinit(@db)
+
+          while (k = (lib.abs_iternext2(@db) rescue nil))
+            break if limit and l.size >= limit
+            l << k
+          end
+
+          l
+        end
       end
     end
 

data/lib/rufus/tokyo/cabinet/lib.rb

@@ -97,12 +97,18 @@ module Rufus
       attfunc :abs_sync, :tcadbsync, [ :pointer ], :int
       attfunc :abs_copy, :tcadbcopy, [ :pointer, :string ], :int
 
+      attfunc :abs_fwmkeys2, :tcadbfwmkeys2, [ :pointer, :string, :int ], :pointer
+
       #
       # tctdb functions
       #
       # http://tokyocabinet.sourceforge.net/spex-en.html#tctdbapi
 
       attfunc :tctdbnew, [], :pointer
+      attfunc :tctdbsetmutex, [ :pointer ], :int
+      attfunc :tctdbtune, [ :pointer, :uint64, :uint8, :uint8, :uint8 ], :int
+      attfunc :tctdbsetcache, [ :pointer, :uint32, :uint32, :uint32 ], :int
+      attfunc :tctdbsetxmsiz, [ :pointer, :uint64 ], :int
 
       attfunc :tctdbopen, [ :pointer, :string, :int ], :int
 
@@ -137,6 +143,8 @@ module Rufus
       attfunc :tctdbtrancommit, [ :pointer ], :int
       attfunc :tctdbtranabort, [ :pointer ], :int
 
+      attfunc :tab_fwmkeys2, :tctdbfwmkeys2, [ :pointer, :string, :int ], :pointer
+
       #
       # tctdbqry functions
       #

data/lib/rufus/tokyo/cabinet/table.rb

@@ -31,61 +31,94 @@
 module Rufus
   module Tokyo
 
-    module OpenModes
+    #
+    # Methods for setting up / tuning a Cabinet.
+    #
+    # Gets included into Table.
+    #
+    module CabinetConfig
+
+      protected
 
       #
-      # some Tokyo constants
+      # Given a path, a hash of parameters and a suffix,
+      #
+      # a) makes sure that the path has the given suffix or raises an exception
+      # b) gathers params found in the path (#) or in params
+      # c) determines the config as set by the parameters
+      #
+      def determine_conf (path, params, suffix)
 
-      OREADER = 1 << 0 # open as a reader
-      OWRITER = 1 << 1 # open as a writer
-      OCREAT = 1 << 2 # writer creating
-      OTRUNC = 1 << 3 # writer truncating
-      ONOLCK = 1 << 4 # open without locking
-      OLCKNB = 1 << 5 # lock without blocking
+        if path.index('#')
 
-      OTSYNC = 1 << 6 # synchronize every transaction (tctdb.h)
+          ss = path.split('#')
+          path = ss.shift
 
-      #
-      # Makes sure that a set of parameters is a hash (will transform an
-      # array into a hash if necessary)
-      #
-      def params_to_h (params)
+          ss.each { |p| pp = p.split('='); params[pp[0]] = pp[1] }
+        end
+
+        raise "path '#{path}' must be suffixed with #{suffix}" \
+          unless path[-suffix.length..-1] == suffix
+
+        params = params.inject({}) { |h, (k, v)| h[k.to_sym] = v; h }
 
-        params.is_a?(Hash) ?
-          params :
-          Array(params).inject({}) { |h, e| h[e] = true; h }
+        conf = {
+          :params => params,
+          :path => path,
+          :mode => determine_open_mode(params),
+          :mutex => (params[:mutex].to_s == 'true'),
+          #:indexes => params[:idx] || params[:indexes],
+          :xmsiz => (params[:xmsiz] || 67108864).to_i
+        }
+        conf.merge!(determine_tuning_values(params))
+        conf.merge(determine_cache_values(params))
       end
 
-      #
-      # Given params (array or hash), computes the open mode (an int)
-      # for the Tokyo Cabinet object.
-      #
-      def compute_open_mode (params)
+      def determine_open_mode (params) #:nodoc#
 
-        params = params_to_h(params)
+        mode = params[:mode].to_s
+        mode = 'wc' if mode.size < 1
 
-        i = {
-          :read => OREADER,
-          :reader => OREADER,
-          :write => OWRITER,
-          :writer => OWRITER,
-          :create => OCREAT,
-          :truncate => OTRUNC,
-          :no_lock => ONOLCK,
-          :lock_no_block => OLCKNB,
-          :sync_every => OTSYNC
+        {
+          'r' => (1 << 0), # open as a reader
+          'w' => (1 << 1), # open as a writer
+          'c' => (1 << 2), # writer creating
+          't' => (1 << 3), # writer truncating
+          'e' => (1 << 4), # open without locking
+          'f' => (1 << 5), # lock without blocking
+          's' => (1 << 6), # synchronize every transaction (tctdb.h)
 
-        }.inject(0) { |r, (k, v)|
+        }.inject(0) { |r, (c, v)|
 
-          r = r | v if params[k]; r
+          r = r | v if mode.index(c); r
         }
+      end
 
-        unless params[:read_only] || params[:readonly]
-          i = i | OCREAT
-          i = i | OWRITER
-        end
+      def determine_tuning_values (params) #:nodoc#
+
+        bnum = (params[:bnum] || 131071).to_i
+        apow = (params[:apow] || 4).to_i
+        fpow = (params[:fpow] || 10).to_i
 
-        i
+        o = params[:opts] || ''
+        o = {
+          'l' => 1 << 0, # large
+          'd' => 1 << 1, # deflate
+          'b' => 1 << 2, # bzip2
+          't' => 1 << 3, # tcbs
+          'x' => 1 << 4
+        }.inject(0) { |i, (k, v)| i = i | v if o.index(k); i }
+
+        { :bnum => bnum, :apow => apow, :fpow => fpow, :opts => o }
+      end
+
+      def determine_cache_values (params) #:nodoc#
+
+        {
+          :rcnum => params[:rcnum].to_i,
+          :lcnum => (params[:lcnum] || 2048).to_i,
+          :ncnum => (params[:ncnum] || 512).to_i
+        }
       end
     end
 
@@ -122,31 +155,82 @@ module Rufus
     class Table
 
       include HashMethods
-      include OpenModes
+      include CabinetConfig
 
       #
       # Creates a Table instance (creates or opens it depending on the args)
       #
       # For example,
       #
-      #   t = Rufus::Tokyo::Table.new('table.tdb', :create, :write)
+      #   t = Rufus::Tokyo::Table.new('table.tdb')
       #     # '.tdb' suffix is a must
       #
       # will create the table.tdb (or simply open it if already present)
       # and make sure we have write access to it.
-      # Note that the suffix (.tdc) is relevant to Tokyo Cabinet, using another
-      # will result in a Tokyo Cabinet error.
       #
-      def initialize (*args)
-
-        path = args.first # car
-        params = args[1..-1] # cdr
+      # == parameters
+      #
+      # Parameters can be set in the path or via the optional params hash (like
+      # in Rufus::Tokyo::Cabinet)
+      #
+      #   * :mode    a set of chars ('r'ead, 'w'rite, 'c'reate, 't'runcate,
+      #              'e' non locking, 'f' non blocking lock), default is 'wc'
+      #   * :opts    a set of chars ('l'arge, 'd'eflate, 'b'zip2, 't'cbs)
+      #              (usually empty or something like 'ld' or 'lb')
+      #
+      #   * :bnum    number of elements of the bucket array
+      #   * :apow    size of record alignment by power of 2 (defaults to 4)
+      #   * :fpow    maximum number of elements of the free block pool by
+      #              power of 2 (defaults to 10)
+      #   * :mutex   when set to true, makes sure only 1 thread at a time
+      #              accesses the table (well, Ruby, global thread lock, ...)
+      #
+      #   * :rcnum   specifies the maximum number of records to be cached.
+      #              If it is not more than 0, the record cache is disabled.
+      #              It is disabled by default.
+      #   * :lcnum   specifies the maximum number of leaf nodes to be cached.
+      #              If it is not more than 0, the default value is specified.
+      #              The default value is 2048.
+      #   * :ncnum   specifies the maximum number of non-leaf nodes to be
+      #              cached. If it is not more than 0, the default value is
+      #              specified. The default value is 512.
+      #
+      #   * :xmsiz   specifies the size of the extra mapped memory. If it is
+      #              not more than 0, the extra mapped memory is disabled.
+      #              The default size is 67108864.
+      #
+      # Some examples :
+      #
+      #   t = Rufus::Tokyo::Table.new('table.tdb')
+      #   t = Rufus::Tokyo::Table.new('table.tdb#mode=r')
+      #   t = Rufus::Tokyo::Table.new('table.tdb', :mode => 'r')
+      #   t = Rufus::Tokyo::Table.new('table.tdb#opts=ld#mode=r')
+      #   t = Rufus::Tokyo::Table.new('table.tdb', :opts => 'ld', :mode => 'r')
+      #
+      def initialize (path, params={})
 
-        mode = compute_open_mode(params)
+        conf = determine_conf(path, params, '.tdb')
 
         @db = lib.tctdbnew
 
-        libcall(:tctdbopen, path, mode)
+        #
+        # tune table
+
+        libcall(:tctdbsetmutex) if conf[:mutex]
+
+        libcall(:tctdbtune, conf[:bnum], conf[:apow], conf[:fpow], conf[:opts])
+
+        # TODO : set indexes here... well, there is already #set_index
+        #conf[:indexes]...
+
+        libcall(:tctdbsetcache, conf[:rcnum], conf[:lcnum], conf[:ncnum])
+
+        libcall(:tctdbsetxmsiz, conf[:xmsiz])
+
+        #
+        # open table
+
+        libcall(:tctdbopen, conf[:path], conf[:mode])
       end
 
       #
@@ -245,25 +329,43 @@ module Rufus
       end
 
       #
-      # Returns the value (as a Ruby Hash) else nil
+      # Returns an array of all the primary keys in the table
       #
-      # (the actual #[] method is provided by HashMethods)
+      # With no options given, this method will return all the keys (strings)
+      # in a Ruby array.
       #
-      def get (k)
-        m = lib.tab_get(@db, k, CabinetLib.strlen(k))
-        return nil if m.address == 0 # :( too bad, but it works
-        Map.to_h(m) # which frees the map
-      end
-      protected :get
-
+      #   :prefix --> returns only the keys who match a given string prefix
       #
-      # Returns an array of all the primary keys in the table
+      #   :limit --> returns a limited number of keys
       #
-      def keys
-        a = []
-        lib.tab_iterinit(@db)
-        while (k = (lib.tab_iternext2(@db) rescue nil)); a << k; end
-        a
+      #   :native --> returns an instance of Rufus::Tokyo::List instead of
+      #     a Ruby Hash, you have to call #free on that List when done with it !
+      #     Else you're exposing yourself to a memory leak.
+      #
+      def keys (options={})
+
+        if pref = options[:prefix]
+
+          l = lib.tab_fwmkeys2(@db, pref, options[:limit] || -1)
+          l = Rufus::Tokyo::List.new(l)
+          options[:native] ? l : l.release
+
+        else
+
+          limit = options[:limit] || -1
+          limit = nil if limit < 1
+
+          l = options[:native] ? Rufus::Tokyo::List.new : []
+
+          lib.tab_iterinit(@db)
+
+          while (k = (lib.tab_iternext2(@db) rescue nil))
+            break if limit and l.size >= limit
+            l << k
+          end
+
+          l
+        end
       end
 
       #
@@ -377,6 +479,17 @@ module Rufus
 
       protected
 
+      #
+      # Returns the value (as a Ruby Hash) else nil
+      #
+      # (the actual #[] method is provided by HashMethods)
+      #
+      def get (k)
+        m = lib.tab_get(@db, k, CabinetLib.strlen(k))
+        return nil if m.address == 0 # :( too bad, but it works
+        Map.to_h(m) # which frees the map
+      end
+
       def libcall (lib_method, *args)
 
         #(lib.send(lib_method, @db, *args) == 1) or raise_error

data/lib/rufus/tokyo/cabinet/util.rb

@@ -179,7 +179,7 @@ module Rufus
       # (by passing a list pointer, one can wrap an existing list pointer
       # into a handy instance of this class)
       #
-      def initialize (list_pointer = nil)
+      def initialize (list_pointer=nil)
         @list = list_pointer || clib.tclistnew
       end
 
@@ -333,6 +333,17 @@ module Rufus
       alias :free :close
       alias :destroy :close
 
+      #
+      # Closes (frees memory from it) this list and returns the ruby version
+      # of it
+      #
+      def release
+
+        a = self.to_a
+        self.close
+        a
+      end
+
       protected
 
       #

data/lib/rufus/tokyo/tyrant/lib.rb

@@ -84,6 +84,8 @@ module Rufus
       attfunc :abs_sync, :tcrdbsync, [ :pointer ], :int
       attfunc :abs_copy, :tcrdbcopy, [ :pointer, :string ], :int
 
+      attfunc :abs_fwmkeys2, :tcrdbfwmkeys2, [ :pointer, :string, :int ], :pointer
+
       #
       # table functions
 
@@ -111,6 +113,8 @@ module Rufus
 
       attfunc :tab_setindex, :tcrdbtblsetindex, [ :pointer, :string, :int ], :int
 
+      attfunc :tab_fwmkeys2, :tcrdbfwmkeys2, [ :pointer, :string, :int ], :pointer
+
       #
       # qry functions
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: rufus-tokyo
 version: !ruby/object:Gem::Version 
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors: 
 - John Mettraux
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-02-16 00:00:00 +09:00
+date: 2009-02-19 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency