rufus-tokyo 0.1.14 → 1.0.0

data/lib/rufus/tokyo/dystopia.rb

@@ -27,17 +27,20 @@ require 'rufus/tokyo'
 
 
 module Rufus::Tokyo
-
-  #
-  # An error for Dystopia
-  #
-  class DystopianError < RuntimeError
-    def new (error_code)
-      super("tokyo dystopia error #{error_code}")
+  module Dystopia
+    #
+    # An error for Dystopia
+    #
+    class Error < RuntimeError
+      def new (error_code)
+        super("tokyo dystopia error #{error_code}")
+      end
     end
   end
 end
 
 require 'rufus/tokyo/dystopia/lib'
+
+require 'rufus/tokyo/dystopia/core'
 require 'rufus/tokyo/dystopia/words'
 

data/lib/rufus/tokyo/dystopia/core.rb

@@ -0,0 +1,198 @@
+#--
+# Copyright (c) 2009, Jeremy Hinegardner, jeremy@copiousfreetime.org
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# Developed with and sponsored by Seven Scale <http://sevenscale.com/>,
+# creators of Open Syslog.
+#
+#++
+
+module Rufus::Tokyo::Dystopia
+  #
+  # Tokyo Dystopia Indexed Database
+  #
+  # http://tokyocabinet.sourceforge.net/dystopiadoc/#dystopiaapi
+  #
+  class Core
+    class Error < Rufus::Tokyo::Dystopia::Error; end
+
+    def self.lib
+      ::Rufus::Tokyo::DystopiaLib
+    end
+
+    def self.mode_to_bits( mode )
+      @modes_to_bits ||= {
+        "r"  => lib::READER,
+        "r+" => lib::READER | lib::WRITER,
+        "w"  => lib::WRITER | lib::CREAT | lib::TRUNC,
+        "w+" => lib::READER | lib::WRITER | lib::CREAT | lib::TRUNC,
+        "a"  => lib::WRITER | lib::CREAT,
+        "a+" => lib::READER | lib::WRITER | lib::CREAT ,
+      }
+
+      return @modes_to_bits[mode]
+    end
+
+    def self.locking_to_bits( locking )
+      @locking_to_bits ||= {
+        true  => 0,
+        false => lib::NOLCK,
+        :nonblocking => lib::LCKNB
+      }
+      return @locking_to_bits[locking]
+    end
+
+    def lib
+      Core.lib
+    end
+
+    #
+    # Opens/creates a new Tokyo dystopia database
+    #
+    # The modes are equivelent to those when opening a file:
+    #
+    # 'r'  : readonly
+    # 'r+' : read/write does not create or truncate
+    # 'w'  : write only, create and truncate
+    # 'w+' : read/write, create and truncate
+    # 'a'  : write only, create if db does not exist
+    # 'a+' : read/write, create if db does not exist
+    #
+    # The third parameter 'locking' can be one of 'true', 'false' or
+    # :nonblocking
+    #
+    def initialize( path, mode = "a+", locking = true )
+      mode_bits = Core.mode_to_bits( mode )
+      raise Error.new( "Invalid mode '#{mode}'" ) unless mode_bits
+      lock_bits = Core.locking_to_bits( locking )
+      raise Error.new( "Invalid Locking mode #{locking}" ) unless lock_bits
+
+      @db = lib.tcidbnew()
+
+      rc = lib.tcidbopen( @db, path, mode_bits | lock_bits )
+      raise_error unless rc == 1
+    end
+
+    #
+    # Close and detach from the database.  This instance can not be used anymore
+    #
+    def close
+      rc = lib.tcidbclose( @db )
+      raise_error unless rc == 1
+
+      lib.tcidbdel( @db )
+      raise_error unless rc == 1
+
+      @db = nil
+    end
+
+    #
+    # Add a new document to the database
+    #
+    def store( id, text )
+      rc = lib.tcidbput( @db, id, text )
+      raise_error unless rc == 1
+    end
+
+    #
+    # Remove the given document from the index
+    #
+    def delete( id )
+      rc = lib.tcidbout( @db, id )
+      raise_error unless rc == 1
+    end
+
+    #
+    # Return the document at the specified index
+    #
+    def fetch( id )
+      r = nil
+      begin
+        r = lib.tcidbget( @db, id )
+      rescue => e
+        # if we have 'no record found' then return nil
+        if lib.tcidbecode( @db ) == 22 then
+          return nil
+        else
+          raise_error
+        end
+      end
+      return r
+    end
+
+    #
+    # Return the document ids of the documents that matche the search expression
+    #
+    # http://tokyocabinet.sourceforge.net/dystopiadoc/#dystopiaapi and scroll
+    # down to 'Compound Expression of Search'
+    #
+    def search( expression )
+      out_count = ::FFI::MemoryPointer.new :pointer
+      out_list  = ::FFI::MemoryPointer.new :pointer
+      out_list  = lib.tcidbsearch2( @db, expression, out_count )
+
+      count = out_count.read_int
+      results = out_list.get_array_of_uint64(0, count )
+      return results
+    end
+
+    #
+    # Remove all records from the db
+    #
+    def clear
+      lib.tcidbvanish( @db )
+    end
+
+    #
+    # Report the path of the database
+    #
+    def path
+      s = lib.tcidbpath( @db )
+      return File.expand_path( s ) if s
+    end
+
+    #
+    # Return the number of records in the database
+    #
+    def count
+      lib.tcidbrnum( @db )
+    end
+    alias :rnum :count
+
+    #
+    # return the disk space used by the index
+    #
+    def fsize
+      lib.tcidbfsiz( @db )
+    end
+
+    protected
+
+    #
+    # Raises a dystopian error (asks the db which one)
+    #
+    def raise_error
+      code = lib.tcidbecode( @db )
+      msg  = lib.tcidberrmsg( code )
+      raise Error.new("[ERROR #{code}] : #{msg}")
+    end
+
+  end
+end

data/lib/rufus/tokyo/dystopia/lib.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009, John Mettraux, jmettraux@gmail.com
+# Copyright (c) 2009, Jeremy Hinegardner, John Mettraux
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -36,19 +36,81 @@ module Rufus::Tokyo
     # find Tokyo Dystopia lib
 
     paths = Array(ENV['TOKYO_DYSTOPIA_LIB'] || %w{
+      /usr/lib/libtokyodystopia.so
+      /usr/lib64/libtokyodystopia.so
       /opt/local/lib/libtokyodystopia.dylib
+      /opt/local/lib/libtokyodystopia.so
       /usr/local/lib/libtokyodystopia.dylib
       /usr/local/lib/libtokyodystopia.so
     })
 
-    if path = paths.find { |path| File.exist?(path) }
+    path = paths.find { |path| File.exist?(path) }
 
     raise "did not find Tokyo Dystopia libs on your system" unless path
 
     ffi_lib(path)
 
     #
-    # tcwdb functions
+    # tcidb functions - The Core API
+    #
+    # http://tokyocabinet.sourceforge.net/dystopiadoc/#dystopiaapi
+
+    # tuning options <dystopia.h>
+    LARGE   = 1 << 0
+    DEFLATE = 1 << 1
+    BZIP    = 1 << 2
+    TCBS    = 1 << 3
+
+    # open modes <dystopia.h>
+    READER  = 1 << 0
+    WRITER  = 1 << 1
+    CREAT   = 1 << 2
+    TRUNC   = 1 << 3
+    NOLCK   = 1 << 4
+    LCKNB   = 1 << 5
+
+    # for get modes
+    SUBSTR = 0
+    PREFIX = 1
+    SUFFIX = 2
+    FULL   = 3
+    TOKEN  = 4
+    TOKPRE = 5
+    TOKSUF = 6
+
+
+    attach_function :tcidberrmsg,    [ :int ], :string
+    attach_function :tcidbecode,     [ :pointer ], :int
+
+    attach_function :tcidbnew,       [], :pointer
+    attach_function :tcidbopen,      [ :pointer, :pointer, :int ], :int
+    attach_function :tcidbclose,     [ :pointer ], :int
+    attach_function :tcidbdel,       [ :pointer ], :void
+    attach_function :tcidbvanish,    [ :pointer ], :int
+
+    # TODO
+    #attach_function :tcidbtune,      [ :pointer, :int64, :int64, :int64, :int8 ], :int
+    #attach_function :tcidbsetcache,  [ :pointer, :int64, :int32 ], :int
+    #attach_function :tcidbsetfwmmax, [ :pointer, :uint32 ], :int
+    #attach_function :tcidbsync,      [ :pointer ], :int
+    #attach_function :tcidboptimize,  [ :pointer ], :int
+    #attach_function :tcidbcopy,      [ :pointer, :pointer ], :int
+
+    attach_function :tcidbpath,      [ :pointer ], :string
+    attach_function :tcidbrnum,      [ :pointer ], :uint64
+    attach_function :tcidbfsiz,      [ :pointer ], :uint64
+
+    attach_function :tcidbput,       [ :pointer, :int64, :string ], :int
+    attach_function :tcidbout,       [ :pointer, :int64 ], :int
+    attach_function :tcidbget,       [ :pointer, :int64 ], :string
+    #attach_function :tcidbsearch,    [ :pointer, :pointer, :int, :pointer ], :pointer
+    attach_function :tcidbsearch2,   [ :pointer, :string, :pointer], :pointer
+    #attach_function :tcidbiterinit,  [ :pointer ], :int
+    #attach_function :tcidbiternext,  [ :pointer ], :uint64
+
+
+    #
+    # tcwdb functions - The Word Database API
     #
     # http://tokyocabinet.sourceforge.net/dystopiadoc/#tcwdbapi
 

data/lib/rufus/tokyo/dystopia/words.rb

@@ -23,7 +23,7 @@
 #++
 
 
-module Rufus::Tokyo
+module Rufus::Tokyo::Dystopia
 
   #
   # Tokyo Dystopia words database.

data/lib/rufus/tokyo/outlen.rb

@@ -0,0 +1,46 @@
+#--
+# Copyright (c) 2009, Adam Keys, John Mettraux, jmettraux@gmail.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# Made in Japan.
+#++
+
+module Rufus::Tokyo
+  module Outlen
+
+    # A wrapper for library returning a string (binary data potentially)
+    #
+    def outlen_op (method, *args)
+
+      args.unshift(@db)
+
+      outlen = FFI::MemoryPointer.new(:int)
+      args << outlen
+
+      out = lib.send(method, *args)
+
+      return nil if out.address == 0
+      return out.get_bytes(0, outlen.get_int(0))
+    ensure
+      outlen.free
+    end
+  end
+end
+

data/lib/rufus/tokyo/query.rb

@@ -81,7 +81,15 @@ module Tokyo
       :bt => 13,
       :between => 13,
 
-      :numoreq => 14 # number which is equal to at least one token
+      :numoreq => 14, # number which is equal to at least one token
+
+      # full-text search
+      :ftsph => 15,
+      :ftsphrase => 15,
+      :phrase => 15,
+      :ftsand => 16,
+      :ftsor => 17,
+      :ftsex => 18
     }
 
     TDBQCNEGATE = 1 << 24

data/lib/rufus/tokyo/tyrant/abstract.rb

@@ -22,9 +22,8 @@
 # Made in Japan.
 #++
 
-
 require 'rufus/tokyo/ttcommons'
-
+require 'rufus/tokyo/tyrant/ext'
 
 module Rufus::Tokyo
 
@@ -39,6 +38,7 @@ module Rufus::Tokyo
   class Tyrant < Cabinet
 
     include TyrantCommons
+    include Ext
 
     attr_reader :host, :port
 
@@ -70,8 +70,8 @@ module Rufus::Tokyo
       @host = host
       @port = port
 
-      (lib.tcrdbopen(@db, host, port) == 1) ||
-        raise("couldn't connect to tyrant at #{host}:#{port}")
+      (lib.tcrdbopen(@db, host, port) == 1) || raise(
+        TokyoError.new("couldn't connect to tyrant at #{host}:#{port}"))
 
       if self.stat['type'] == 'table'
 
@@ -100,28 +100,6 @@ module Rufus::Tokyo
       raise 'not allowed to create files on the server'
     end
 
-    # Calls a lua embedded function
-    # (http://tokyocabinet.sourceforge.net/tyrantdoc/#luaext)
-    #
-    # Options are :global_locking and :record_locking
-    #
-    # Returns the return value of the called function.
-    #
-    # Nil is returned in case of failure.
-    #
-    def ext (func_name, key, value, opts={})
-
-      k = key.to_s
-      v = value.to_s
-
-      outlen_op(
-        :tcrdbext,
-        func_name.to_s,
-        compute_ext_opts(opts),
-        k, Rufus::Tokyo.blen(k),
-        v, Rufus::Tokyo.blen(v))
-    end
-
     # Tyrant databases DO NOT support the 'defrag' call. Calling this method
     # will raise an exception.
     #