bdb 0.1.0

data/ext/bdb.h

@@ -0,0 +1,104 @@
+
+#ifndef BDB2_H
+#define BDB2_H
+
+#include <ruby.h>
+
+#ifdef stat
+  #undef stat
+#endif
+
+#ifdef close
+  #undef close
+#endif
+
+#ifdef rename
+  #undef rename
+#endif
+
+#include <version.h>
+#include <db.h>
+
+#define NOTXN NULL
+
+#ifdef OPEN_MAX
+#define LMAXFD OPEN_MAX
+#else
+#ifdef FOPEN_MAX
+#define LMAXFD FOPEN_MAX
+#endif
+#endif
+#ifndef LMAXFD
+#error "No max fd define available."
+#endif
+
+#define FNLEN 40
+
+#define filename_copy(fp,fv) \
+  strncpy(fp,RSTRING_PTR(fv),FNLEN);
+
+#define filename_dup(fpd,fps) \
+  strncpy(fpd,fps,FNLEN);
+
+typedef struct s_envh {
+  VALUE self;
+  DB_ENV *env;
+  VALUE adb; /* Ruby array holding opened databases */
+  VALUE atxn; /* Ruby array holding open transactions */
+} t_envh;
+
+typedef struct s_dbh {
+  VALUE self;
+  DB *db;
+	int db_opened;
+  VALUE aproc;
+  VALUE sproc; /* key sorting callback */
+  
+  t_envh *env; /* Parent environment, NULL if not opened from one */
+  VALUE adbc; /* Ruby array holding opened cursor */
+  char filename[FNLEN+1];
+} t_dbh;
+
+typedef struct s_dbch {
+  VALUE self;
+  DBC *dbc;
+  t_dbh *db;
+  char filename[FNLEN+1];
+} t_dbch;
+
+typedef struct s_txnh {
+  VALUE self;
+  DB_TXN *txn;
+  t_envh *env;
+} t_txnh;
+
+#define cu(b,m) \
+  rb_define_const(b,#m,UINT2NUM(m))
+
+#define ci(b,m) \
+  rb_define_const(b,#m,INT2NUM(m))
+
+#define cs(b,m) \
+  rb_define_const(b,#m,rb_str_new2(m))
+
+#define simple_set(fname) \
+VALUE db_ ## fname ## _eq(VALUE obj, VALUE v) \
+{ \
+  rb_ivar_set(obj,fv_ ## fname,v); \
+  return obj; \
+} 
+
+#define attr_writer(fname) \
+  VALUE fname ## _writer(VALUE obj, VALUE v) \
+  {					     \
+    rb_ivar_set(obj,fv_ ## fname,v); \
+    return obj; \
+  }
+
+#define attr_reader(fname) \
+  VALUE fname ## _reader(VALUE obj) \
+  { \
+    return rb_ivar_get(obj,fv_ ## fname);  \
+  }
+
+#endif

data/ext/extconf.rb

@@ -0,0 +1,91 @@
+#!/usr/bin/env ruby
+require 'mkmf'
+
+inc, lib = dir_config('db')
+
+# OS X compatibility
+if(PLATFORM =~ /darwin/) then
+	# test if Bdb is probably universal
+	
+	filetype = (IO.popen("file #{inc}/../db_dump").readline.chomp rescue nil)
+	# if it's not universal, ARCHFLAGS should be set
+	if((filetype !~ /universal binary/) && ENV['ARCHFLAGS'].nil?) then
+		arch = (IO.popen("uname -m").readline.chomp rescue nil)
+		$stderr.write %{
+===========   WARNING   ===========
+
+You are building this extension on OS X without setting the 
+ARCHFLAGS environment variable, and BerkeleyDB does not appear 
+to have been built as a universal binary. If you are seeing this 
+message, that means that the build will probably fail.
+
+Try setting the environment variable ARCHFLAGS 
+to '-arch #{arch}' before building.
+
+For example:
+(in bash) $ export ARCHFLAGS='-arch #{arch}'
+(in tcsh) % setenv ARCHFLAGS '-arch #{arch}'
+
+Then try building again.
+
+===================================
+
+}
+		# We don't exit here. Who knows? It might build.
+	end
+end
+
+versions=%w(db-4.8 db-4.7 db-4.6 db-4.5 db-4.4 db-4.3 db-4.2)
+until versions.empty?
+  (lib_ok = have_library(versions.shift,'db_version', 'db.h')) && break
+end
+
+def create_header
+  if File.exist?("bdb_aux._c")
+    message("Not writing bdb_aux._c (defines), already exists\n")
+    return
+  end
+  
+  message("Writing bdb_aux._c (defines), this takes a while\n")
+  db_header = $CPPFLAGS.split.select { |f| f =~ /^-I/ }.map { |e| 
+    f = File.join(e[2..-1], 'db.h')
+    File.exists?(f) ? f : nil
+  }.select { |e| e }.first
+  
+  n=0
+  defines=[]
+  File.open(db_header) {|fd|
+    File.open("bdb_aux._c","w") {|hd|
+      hd.puts("/* This file automatically generated by extconf.rb */\n")
+      fd.each_line {|l|
+        if l =~ %r{^#define\s+(DBC?_\w*)\s+([^\/]*)\s*(.*?)(\/\*.*)?$}
+          name = $1
+          value = $2
+          if macro_defined?(name,"#include <db.h>")
+            case value
+            when /^"/
+              hd.print(%Q{cs(mBdb,%s);\n}%[name])
+            when /^\(?(0x|\d)/
+              hd.print(%Q{cu(mBdb,%s);\n}%[name])
+            when /^\(?-/
+              hd.print(%Q{ci(mBdb,%s);\n}%[name])
+            else
+              $stderr.puts "don't know how to handle #{name} #{value.strip}, guessing UINT"
+              hd.print(%Q{cu(mBdb,%s);\n}%[name])
+            end
+            n+=1
+          end
+        end
+      }
+    }
+    message("\nwrote #{n} defines\n")
+  }
+end
+
+if lib_ok
+  create_header
+  create_makefile('bdb')
+else
+  $stderr.puts("cannot create Makefile")
+  exit 1
+end

data/lib/bdb/base.rb

@@ -0,0 +1,60 @@
+require 'bdb'
+require 'tuple'
+require 'bdb/environment'
+require 'bdb/result_set'
+
+class Bdb::Base
+  def initialize(opts)
+    @config  = Bdb::Environment.config.merge(opts)
+    @indexes = {}
+  end
+  attr_reader :indexes
+  
+  def config(config = {})
+    @config.merge!(config)
+  end
+    
+  def index_by(field, opts = {})
+    raise "index on #{field} already exists" if indexes[field]
+    indexes[field] = opts
+  end
+  
+  def environment
+    @environment ||= Bdb::Environment.new(config[:path], self)
+  end
+  
+  def transaction(nested = true, &block)
+    environment.transaction(nested, &block)
+  end
+  
+  def synchronize(&block)
+    environment.synchronize(&block)
+  end
+  
+  def checkpoint(opts = {})
+    environment.synchronize(opts)
+  end
+  
+private
+
+  def get_field(field, value)
+    value.kind_of?(Hash) ? value[field] : value.send(field)
+  end
+end
+
+class Object
+  attr_accessor :bdb_locator_key
+end
+
+# Array comparison should try Tuple comparison first.
+class Array
+  cmp = instance_method(:<=>)
+
+  define_method(:<=>) do |other|
+    begin
+      Tuple.dump(self) <=> Tuple.dump(other)
+    rescue TypeError => e
+      cmp.bind(self).call(other)
+    end
+  end
+end

data/lib/bdb/database.rb

@@ -0,0 +1,184 @@
+require 'bdb/base'
+
+class Bdb::Database < Bdb::Base
+  def initialize(name, opts = {})
+    @name = name
+    super(opts)
+  end
+  attr_reader :name
+
+  def db(index = nil)
+    if @db.nil?
+      @db = {}
+      transaction(false) do
+        primary_db = environment.env.db
+        primary_db.pagesize = config[:page_size] if config[:page_size]
+        primary_db.open(transaction, name, nil, Bdb::Db::BTREE, Bdb::DB_CREATE, 0)
+        @db[:primary_key] = primary_db
+
+        indexes.each do |field, opts|
+          index_callback = lambda do |db, key, data|
+            value = Marshal.load(data)
+            index_key = value.kind_of?(Hash) ? value[:field] : value.send(field)
+            if opts[:multi_key] and index_key.kind_of?(Array)
+              # Index multiple keys. If the key is an array, you must wrap it with an outer array.
+              index_key.collect {|k| Tuple.dump(k)}
+            elsif index_key
+              # Index a single key.
+              Tuple.dump(index_key)
+            end
+          end
+          index_db = environment.env.db
+          index_db.flags = Bdb::DB_DUPSORT unless opts[:unique]
+          index_db.pagesize = config[:page_size] if config[:page_size]
+          index_db.open(transaction, "#{name}_by_#{field}", nil, Bdb::Db::BTREE, Bdb::DB_CREATE, 0)
+          primary_db.associate(transaction, index_db, Bdb::DB_CREATE, index_callback)
+          @db[field] = index_db
+        end
+      end
+    end
+    @db[index || :primary_key]
+  end
+
+  def close
+    return unless @db
+    synchronize do
+      @db.each {|field, db| db.close(0)}
+      @db = nil
+    end
+  end  
+
+  def count(field, key)
+    with_cursor(db(field)) do |cursor|
+      k, v = cursor.get(Tuple.dump(key), nil, Bdb::DB_SET)
+      k ? cursor.count : 0
+    end
+  end
+
+  def get(*keys, &block)
+    opts  = keys.last.kind_of?(Hash) ? keys.pop : {}
+    db    = db(opts[:field])
+    set   = Bdb::ResultSet.new(opts, &block)
+    flags = opts[:modify] ? Bdb::DB_RMW : 0
+    flags = 0 if environment.disable_transactions?
+
+    keys.each do |key|
+      key = get_key(key, opts)
+      if key == :all
+        with_cursor(db) do |cursor|          
+          if opts[:reverse]
+            k,v  = cursor.get(nil, nil, Bdb::DB_LAST | flags)          # Start at the last item.
+            iter = lambda {cursor.get(nil, nil, Bdb::DB_PREV | flags)} # Move backward.
+          else
+            k,v  = cursor.get(nil, nil, Bdb::DB_FIRST | flags)         # Start at the first item.
+            iter = lambda {cursor.get(nil, nil, Bdb::DB_NEXT | flags)} # Move forward.
+          end
+
+          while k
+            set << unmarshal(v, :tuple => k)
+            k,v = iter.call
+          end
+        end
+      elsif key.kind_of?(Range)
+        # Fetch a range of keys.
+        with_cursor(db) do |cursor|
+          first = Tuple.dump(key.first)
+          last  = Tuple.dump(key.last)
+
+          # Return false once we pass the end of the range.
+          cond = key.exclude_end? ? lambda {|k| k < last} : lambda {|k| k <= last}
+          if opts[:reverse]
+            iter = lambda {cursor.get(nil, nil, Bdb::DB_PREV | flags)} # Move backward.
+            
+            # Position the cursor at the end of the range.
+            k,v = cursor.get(last, nil, Bdb::DB_SET_RANGE | flags) || cursor.get(nil, nil, Bdb::DB_LAST | flags)
+            while k and not cond.call(k)
+              k,v = iter.call
+            end
+            
+            cond = lambda {|k| k >= first} # Change the condition to stop when we move past the start.
+          else
+            k,v  = cursor.get(first, nil, Bdb::DB_SET_RANGE | flags)   # Start at the beginning of the range.
+            iter = lambda {cursor.get(nil, nil, Bdb::DB_NEXT | flags)} # Move forward.
+          end
+          
+          while k and cond.call(k)
+            set << unmarshal(v, :tuple => k)
+            k,v = iter.call
+          end
+        end
+      else
+        if (db.flags & Bdb::DB_DUPSORT) == 0
+          synchronize do
+            # There can only be one item for each key.
+            data = db.get(transaction, Tuple.dump(key), nil, flags)
+            set << unmarshal(data, :key => key) if data
+          end
+        else
+          # Have to use a cursor because there may be multiple items with each key.
+          with_cursor(db) do |cursor|
+            k,v = cursor.get(Tuple.dump(key), nil, Bdb::DB_SET | flags)
+            while k
+              set << unmarshal(v, :tuple => k)
+              k,v = cursor.get(nil, nil, Bdb::DB_NEXT_DUP | flags)
+            end
+          end
+        end
+      end
+    end
+    set.results
+  rescue Bdb::ResultSet::LimitReached
+    set.results
+  end
+
+  def set(key, value, opts = {})
+    synchronize do
+      key   = Tuple.dump(key)
+      value = Marshal.dump(value)
+      flags = opts[:create] ? Bdb::DB_NOOVERWRITE : 0
+      db.put(transaction, key, value, flags)
+    end
+  end
+
+  def delete(key)
+    synchronize do
+      key = Tuple.dump(key)
+      db.del(transaction, key, 0)
+    end
+  end
+
+  # Deletes all records in the database. Beware!
+  def truncate!
+    synchronize do
+      db.truncate(transaction)
+    end
+  end
+
+private
+
+  def get_key(key, opts)
+    if opts[:partial] and not key.kind_of?(Range) and not key == :all
+      first = [*key]
+      last  = first + [true]
+      key   = first..last
+    end
+    key
+  end
+
+  def unmarshal(value, opts = {})
+    value = Marshal.load(value)
+    value.bdb_locator_key = opts[:tuple] ? Tuple.load(opts[:tuple]) : [*opts[:key]]
+    value
+  end
+
+  def with_cursor(db)
+    synchronize do
+      begin
+        cursor = db.cursor(transaction, 0)
+        yield(cursor)
+      ensure
+        cursor.close if cursor
+      end
+    end
+  end
+end

data/lib/bdb/environment.rb

@@ -0,0 +1,119 @@
+class Bdb::Environment
+  @@env = {}
+  def self.new(path, database = nil)
+    path = File.expand_path(path)
+    @@env[path] ||= super(path)
+    @@env[path].databases << database if database
+    @@env[path]
+  end
+
+  def self.config(config = {})
+    @config ||= {
+      :max_locks    => 5000,
+      :lock_timeout => 30 * 1000 * 1000,
+      :txn_timeout  => 30 * 1000 * 1000,
+      :cache_size   => 1  * 1024 * 1024,
+    }
+    @config.merge!(config)
+  end
+
+  def config(config = {})
+    @config ||= self.class.config
+    @config.merge!(config)
+  end
+
+  def initialize(path)
+    @path = path
+  end
+  attr_reader :path
+
+  def databases
+    @databases ||= []
+  end
+
+  def env
+    if @env.nil?
+      synchronize do
+        @env = Bdb::Env.new(0)
+        if disable_transactions?
+          env_flags = Bdb::DB_CREATE | Bdb::DB_INIT_MPOOL
+        else
+          env_flags = Bdb::DB_CREATE | Bdb::DB_INIT_TXN | Bdb::DB_INIT_LOCK |
+                      Bdb::DB_REGISTER | Bdb::DB_RECOVER | Bdb::DB_INIT_MPOOL | Bdb::DB_THREAD
+        end
+
+        @env.cachesize = config[:cache_size] if config[:cache_size]
+        @env.set_timeout(config[:txn_timeout],  Bdb::DB_SET_TXN_TIMEOUT)  if config[:txn_timeout]
+        @env.set_timeout(config[:lock_timeout], Bdb::DB_SET_LOCK_TIMEOUT) if config[:lock_timeout]
+        @env.set_lk_max_locks(config[:max_locks]) if config[:max_locks]
+        @env.set_lk_detect(Bdb::DB_LOCK_RANDOM)
+        @env.flags_on = Bdb::DB_TXN_WRITE_NOSYNC | Bdb::DB_TIME_NOTGRANTED
+        @env.open(path, env_flags, 0)
+
+        @exit_handler ||= at_exit { close }
+      end
+    end
+    @env
+  end
+
+  def close
+    return unless @env
+    synchronize do
+      databases.each {|database| database.close}
+      @env.close
+      @env = nil
+    end
+  end
+
+  def transaction(nested = true)
+    return @transaction unless block_given?
+    return yield if disable_transactions?
+
+    synchronize do
+      parent = @transaction
+      begin
+        @transaction = env.txn_begin(nested ? parent : nil, 0)
+        value = yield
+        @transaction.commit(0)
+        @transaction = nil
+        value
+      ensure
+        @transaction.abort if @transaction
+        @transaction = parent
+      end
+    end
+  end
+
+  def checkpoint(opts = {})
+    return if disable_transactions?
+    env.txn_checkpoint(opts[:kbyte] || 0, opts[:min] || 0, opts[:force] ? Bdb::DB_FORCE : 0)
+  end
+
+  def disable_transactions?
+    config[:disable_transactions]
+  end
+
+  def synchronize
+    @mutex ||= Mutex.new
+    if @thread_id == thread_id
+      yield
+    else
+      @mutex.synchronize do
+        begin
+          @thread_id = thread_id
+          Thread.exclusive { yield }
+        ensure
+          @thread_id = nil
+        end
+      end
+    end
+  rescue Bdb::DbError => e
+    exit!(9) if e.code == Bdb::DB_RUNRECOVERY    
+    retry if transaction.nil? and e.code == Bdb::DB_LOCK_DEADLOCK
+    raise e
+  end
+
+  def thread_id
+    Thread.current.object_id
+  end
+end