tdb 0.1.0 → 0.2.0

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 GVF=GIT-VERSION-FILE
-DEF_VER=v0.0.0.GIT
+DEF_VER=v0.2.0.GIT
 
 LF='
 '

data/README

@@ -8,17 +8,40 @@ write to the same databases used by Samba!
 == Features
 
 * Concurrent reader and writer processes may safely operate on the
-  same file.
+  same file.  This is great for MRI 1.8 and 1.9 where multi-core
+  performance is easiest to achieve with processes and not threads.
 
-* Releases the GVL for slow disk operations under Ruby 1.9
+* Fork-safe, you may fork and share the same TDB object in your parent
+  and child processes.
+
+* Releases the GVL for slow disk operations under Ruby 1.9 so
+  other threads can run (but not other TDB operations on the same file)
 
 * Includes several {hash functions}[link:Hash_Functions.html]
   not included by upstream TDB.
 
+== Caveats
+
+These caveats will be addressed upstream in
+{TDB2}[http://mid.gmane.org/201008021002.47351.rusty@rustcorp.com.au]
+
+* NOT native thread-safe by default, you MUST initialize your TDB
+  objects with <code>:threadsafe => true</code> or call
+  TDB#threadsafe! on each TDB object if you run with threads
+  under Ruby 1.9 (but not 1.8).
+
+* Database size is limited to 4G, even on 64-bit systems.
+
+* TDB should be created with an appropriate :hash_size for large databases
+  or performance will suffer.
+
 == Install
 
 The original tdb library from the {main site}[http://tdb.samba.org/] is
 required.  Debian users can just <code>apt-get install tdb-dev</code>.
+Non-Debian users: building against upstream tdb 1.2.2 and 1.2.7 are
+known to be broken, so installing tdb from the latest git is
+recommended.
 
 The library consists of a C extension so you'll need a C compiler
 and Ruby development libraries/headers.
@@ -33,6 +56,15 @@ You may also install it via RubyGems on RubyGems.org:
 
   gem install tdb
 
+If you have a tdb installation in a non-standard prefix, you
+will have to use:
+
+  gem install tdb -- --with-tdb-dir=$PFX
+
+Or if you have a non-standard prefix that linkers normally do not search:
+
+  gem install tdb -- --with-tdb-dir=$PFX --with-dldflags=-Wl,-rpath=$PFX/lib
+
 You can get the latest source via git from the following locations
 (these versions may not be stable):
 

data/ext/tdb/extconf.rb

@@ -7,6 +7,7 @@ dir_config('tdb')
 have_header('tdb.h') or abort 'tdb.h missing'
 have_library('tdb') or abort 'libtdb missing'
 have_func('tdb_jenkins_hash')
+have_func('tdb_repack')
 have_const('TDB_ERR_NESTING', 'tdb.h')
 
 create_makefile('tdb_ext')

data/ext/tdb/lookup3.c

@@ -1,4 +1,5 @@
 #include "rbtdb.h"
+#include <stdint.h>
 
 /*
  * lookup3 implementation copied from tdb.git

data/ext/tdb/tdb.c

@@ -158,7 +158,7 @@ static VALUE nogvl_open(void *ptr)
 	return (VALUE)tdb;
 }
 
-static void set_args(struct open_args *o, VALUE opts)
+static void set_args(VALUE self, struct open_args *o, VALUE opts)
 {
 	VALUE tmp;
 
@@ -203,8 +203,40 @@ static void set_args(struct open_args *o, VALUE opts)
 
 		o->hash_fn = (tdb_hash_func)NUM2ULONG(num);
 	}
+
+	tmp = rb_hash_aref(opts, ID2SYM(rb_intern("threadsafe")));
+	if (RTEST(tmp))
+		rb_funcall(self, rb_intern("threadsafe!"), 0);
 }
 
+/*
+ * :call-seq:
+ *
+ *	TDB.new("/path/to/file") -> TDB
+ *	TDB.new("/path/to/file", :hash_size => 666) -> TDB
+ *	TDB.new("/path/to/file", :hash => :murmur2) -> TDB
+ *	TDB.new("/path/to/file", :open_flags => IO::RDONLY) -> TDB
+ *	TDB.new("/path/to/file", :tdb_flags => TDB::NOSYNC) -> TDB
+ *
+ * Initializes a TDB context.  It takes several options.
+ *
+ * :hash_size - the number of buckets, this is the most important tuning
+ * parameter when creating large databases.  This parameter only affects
+ * the creation of new databases.
+ *
+ * :open_flags - a bit mask of IO flags passed directly to open(2),
+ * File.open-compatible flags are accepted.
+ *
+ * :hash - any of the hashes described in Hash_Functions.
+ * This must remain the same for all clients.
+ *
+ * :tdb_flags - a bitmask of any combination of TDB::CLEAR_IF_FIRST,
+ * TDB::INTERNAL, TDB::NOLOCK, TDB::NOMMAP, TDB::CONVERT,
+ * TDB::BIGENDIAN, TDB::NOSYNC, TDB::SEQNUM, TDB::VOLATILE,
+ * TDB::ALLOW_NESTING, TDB::DISALLOW_NESTING, TDB::INCOMPATIBLE_HASH
+ *
+ * :mode - octal mode mask passed to open(2)
+ */
 static VALUE init(int argc, VALUE *argv, VALUE self)
 {
 	struct tdb_context *tdb = db(self, 0);
@@ -214,7 +246,7 @@ static VALUE init(int argc, VALUE *argv, VALUE self)
 	if (tdb)
 		rb_raise(rb_eRuntimeError, "TDB already initialized");
 	rb_scan_args(argc, argv, "11", &path, &opts);
-	set_args(&o, opts);
+	set_args(self, &o, opts);
 
 	if (NIL_P(path))
 		o.tdb_flags |= TDB_INTERNAL;
@@ -587,12 +619,39 @@ static VALUE lockall_unmark(VALUE self)
 	return Qtrue;
 }
 
+/*
+ * clears out the database
+ */
+static VALUE clear(VALUE self)
+{
+	struct tdb_context *tdb = db(self, 1);
+	if ((int)my_tbr((rb_blocking_function_t *)tdb_wipe_all, tdb))
+		my_raise(tdb);
+	return self;
+}
+
+#ifdef HAVE_TDB_REPACK
+/* repacks a database to reduce fragmentation, available with tdb 1.2.x+ */
+static VALUE repack(VALUE self)
+{
+	struct tdb_context *tdb = db(self, 1);
+	if ((int)my_tbr((rb_blocking_function_t *)tdb_repack, tdb))
+		my_raise(tdb);
+	return self;
+}
+#endif /* HAVE_TDB_REPACK */
+
 void Init_tdb_ext(void)
 {
 	cTDB = rb_define_class("TDB", rb_cObject);
 
 	hashes = rb_hash_new();
-        rb_define_const(cTDB, "HASHES", hashes);
+
+	/*
+	 * Available hash functions, the key is the name of the hash
+	 * and the value is a pointer for internal for usage.
+	 */
+	rb_define_const(cTDB, "HASHES", hashes);
 
 	rb_define_alloc_func(cTDB, alloc);
 	rb_include_module(cTDB, rb_mEnumerable);
@@ -626,55 +685,57 @@ void Init_tdb_ext(void)
 	rb_define_method(cTDB, "unlockall_read", unlockall_read, 0);
 	rb_define_method(cTDB, "lockall_mark", lockall_mark, 0);
 	rb_define_method(cTDB, "lockall_unmark", lockall_unmark, 0);
+	rb_define_method(cTDB, "clear", clear, 0);
+#ifdef HAVE_TDB_REPACK
+	rb_define_method(cTDB, "repack", repack, 0);
+#endif /* HAVE_TDB_REPACK */
 
 	init_errors();
 	init_hashes();
 
-#define tdb_CONST(x) rb_define_const(cTDB, #x, UINT2NUM(TDB_##x))
-
 	/* just a readability place holder */
-	tdb_CONST(DEFAULT);
+	rb_define_const(cTDB, "DEFAULT", UINT2NUM(TDB_DEFAULT));
 
 	/* clear database if we are the only one with it open */
-	tdb_CONST(CLEAR_IF_FIRST);
+	rb_define_const(cTDB, "CLEAR_IF_FIRST", UINT2NUM(TDB_CLEAR_IF_FIRST));
 
 	/* don't store on disk, use in-memory database */
-	tdb_CONST(INTERNAL);
+	rb_define_const(cTDB, "INTERNAL", UINT2NUM(TDB_INTERNAL));
 
 	/* don't do any locking */
-	tdb_CONST(NOLOCK);
+	rb_define_const(cTDB, "NOLOCK", UINT2NUM(TDB_NOLOCK));
 
 	/* don't use mmap */
-	tdb_CONST(NOMMAP);
+	rb_define_const(cTDB, "NOMMAP", UINT2NUM(TDB_NOMMAP));
 
 	/* convert endian (internal use) */
-	tdb_CONST(CONVERT);
+	rb_define_const(cTDB, "CONVERT", UINT2NUM(TDB_CONVERT));
 
 	/* header is big-endian (internal use) */
-	tdb_CONST(BIGENDIAN);
+	rb_define_const(cTDB, "BIGENDIAN", UINT2NUM(TDB_BIGENDIAN));
 
 	/* don't use synchronous transactions */
-	tdb_CONST(NOSYNC);
+	rb_define_const(cTDB, "NOSYNC", UINT2NUM(TDB_NOSYNC));
 
 	/* maintain a sequence number */
-	tdb_CONST(SEQNUM);
+	rb_define_const(cTDB, "SEQNUM", UINT2NUM(TDB_SEQNUM));
 
 	/* Activate the per-hashchain freelist, default 5 */
-	tdb_CONST(VOLATILE);
+	rb_define_const(cTDB, "VOLATILE", UINT2NUM(TDB_VOLATILE));
 
 #ifdef TDB_ALLOW_NESTING
 	/* Allow transactions to nest */
-	tdb_CONST(ALLOW_NESTING);
+	rb_define_const(cTDB, "ALLOW_NESTING", UINT2NUM(TDB_ALLOW_NESTING));
 #endif
 
 #ifdef TDB_DISALLOW_NESTING
 	/* Disallow transactions to nest */
-	tdb_CONST(DISALLOW_NESTING);
+	rb_define_const(cTDB, "DISALLOW_NESTING", UINT2NUM(TDB_DISALLOW_NESTING));
 #endif
 
 #ifdef TDB_INCOMPATIBLE_HASH
-	/* Better hashing: can't be opened by tdb < 1.2.6. */
-	tdb_CONST(INCOMPATIBLE_HASH);
+	/* Better hashing, but can't be opened by tdb < 1.2.6. */
+	rb_define_const(cTDB, "INCOMPATIBLE_HASH", UINT2NUM(TDB_INCOMPATIBLE_HASH));
 #endif
 }
 

data/lib/tdb.rb

@@ -1,2 +1,16 @@
 # -*- encoding: binary -*-
 require 'tdb_ext'
+class TDB
+  autoload :MT, 'tdb/mt'
+
+  # makes the current TDB object thread-safe
+  def threadsafe!
+    extend MT
+  end
+
+  # will return true when TDB::MT is included in TDB or the TDB
+  # object is extended by TDB
+  def threadsafe?
+    false
+  end
+end

data/lib/tdb/mt.rb

@@ -0,0 +1,41 @@
+# -*- encoding: binary -*-
+module TDB::MT
+  def initialize
+    super
+    @lock = Mutex.new
+  end
+
+  wrap_methods = %w(
+    close closed? fetch [] store []= insert! modify! insert modify
+    key? has_key? include? member?
+    nuke! delete
+    lockall trylockall unlockall
+    lockall_read trylockall_read unlockall_read
+    lockall_mark lockall_unmark
+    clear
+  )
+  wrap_methods << :repack if TDB.method_defined?(:repack)
+  wrap_methods.each do |meth|
+    eval "def #{meth}(*args); @lock.synchronize { super }; end"
+  end
+
+  def each(&block)
+    @lock.synchronize do
+      super { |k,v| @lock.exclusive_unlock { yield(k,v) } }
+    end
+  end
+
+  def threadsafe?
+    true
+  end
+
+  def self.extended(obj)
+    obj.instance_eval { @lock = Mutex.new unless defined?(@lock) }
+  end
+
+  def self.included(klass)
+    ObjectSpace.each_object(klass) { |obj|
+      obj.instance_eval { @lock = Mutex.new unless defined?(@lock) }
+    }
+  end
+end

data/tdb.gemspec

@@ -5,7 +5,7 @@ description = File.read("README").split(/\n\n/)[1].strip
 
 Gem::Specification.new do |s|
   s.name = %q{tdb}
-  s.version = ENV["VERSION"]
+  s.version = ENV["VERSION"].dup
 
   s.homepage = 'http://bogomips.org/ruby-tdb/'
   s.authors = ["Ruby tdb hackers"]

data/test/test_tdb.rb

@@ -257,4 +257,28 @@ class TestTdb < Test::Unit::TestCase
     assert Process.waitpid2(pid)[1].success?
     assert_equal true, @tdb.trylockall
   end
+
+  def test_check_constant_typos
+    names = {}
+    TDB.constants.each { |const| names[const] = TDB.const_get(const) }
+    assert_equal TDB.constants.size, names.size
+
+    values = {}
+    TDB.constants.each { |const| values[TDB.const_get(const)] = const }
+    assert_equal TDB.constants.size, values.size
+  end
+
+  def test_clear
+    @tdb = TDB.new(nil)
+    @tdb["hello"] = "world"
+    assert_equal @tdb, @tdb.clear
+    assert ! @tdb.include?("hello")
+  end
+
+  def test_repack
+    @tdb = TDB.new(nil)
+    @tdb["hello"] = "world"
+    assert_equal @tdb, @tdb.repack
+    assert_equal "world", @tdb["hello"]
+  end if TDB.method_defined?(:repack)
 end

data/test/test_tdb_mt.rb

@@ -0,0 +1,85 @@
+# -*- encoding: binary -*-
+$stdout.sync = $stderr.sync = true
+require 'test/unit'
+require 'tempfile'
+$-w = true
+require 'tdb'
+
+class Test_TDB_MT < Test::Unit::TestCase
+  def setup
+    @tdb = @tmp = nil
+    @start_pid = $$
+  end
+
+  def teardown
+    return if @start_pid != $$
+    @tmp.close! if @tmp.respond_to?(:close!)
+    @tdb.close if @tdb && ! @tdb.closed?
+  end
+
+  def test_make_threadsafe
+    @tdb = TDB.new(nil)
+    assert_kind_of TDB, @tdb
+    assert ! @tdb.threadsafe?
+    assert_nothing_raised { @tdb.threadsafe! }
+    assert @tdb.threadsafe?
+    @tdb.each { |k,v| assert_equal v, @tdb[k] }
+  end
+
+  def test_init_threadsafe
+    @tdb = TDB.new(nil, :threadsafe => true)
+    assert @tdb.threadsafe?
+    @tdb.close
+    @tdb = TDB.new(nil, :threadsafe => false)
+    assert ! @tdb.threadsafe?
+    @tdb.close
+    @tdb = TDB.new(nil)
+    assert ! @tdb.threadsafe?
+    @tdb.close
+  end
+
+  def test_thread_safe_torture_test
+    @tdb = TDB.new(nil)
+    assert_nothing_raised { @tdb.threadsafe! }
+    pid = fork do
+      Thread.abort_on_exception = true
+      threads = []
+      blob = 'foo' * 1000
+      nr = 10000
+      t = Thread.new do
+        while true
+          Thread.pass
+          @tdb.to_a
+        end
+      end
+      threads << Thread.new { nr.times { |i| @tdb[i.to_s] = blob } }
+      threads << Thread.new { nr.times { |i| @tdb[i.to_s] = blob } }
+      threads << Thread.new { nr.times { |i| @tdb[i.to_s] = blob } }
+      threads << Thread.new { nr.times { |i| @tdb[i.to_s] = blob } }
+      threads << t
+
+      t.kill
+      threads.each { |t| t.join }
+    end
+    _, status = Process.waitpid2(pid)
+    assert status.success?, status.inspect
+  end
+
+  def test_check_methods
+    m = TDB.instance_methods.sort
+    m -= Object.instance_methods
+    m -= Enumerable.instance_methods
+    m.map! { |x| x.to_sym }
+    mt = TDB::MT.instance_methods.sort
+    m -= [ :threadsafe! ]
+    m += [ :include?, :member? ]
+    m.sort!
+    unwrapped = ( (m - mt) | (mt - m)).uniq
+    assert unwrapped.empty?, "unwrapped methods: #{unwrapped.inspect}"
+    @tdb = TDB.new(nil)
+    respond_to?(:refute_match) and
+      m.each { |meth| refute_match(/\bTDB::MT\b/, @tdb.method(meth).to_s) }
+    @tdb.threadsafe!
+    m.each { |meth| assert_match(/\bTDB::MT\b/, @tdb.method(meth).to_s) }
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: tdb
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 23
   prerelease: false
   segments: 
   - 0
-  - 1
+  - 2
   - 0
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Ruby tdb hackers
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-01 00:00:00 +00:00
+date: 2010-12-08 00:00:00 +00:00
 default_executable: 
 dependencies: []
 
@@ -37,6 +37,7 @@ extra_rdoc_files:
 - NEWS
 - ChangeLog
 - lib/tdb.rb
+- lib/tdb/mt.rb
 - ext/tdb/tdb.c
 files: 
 - .document
@@ -62,9 +63,11 @@ files:
 - ext/tdb/rbtdb.h
 - ext/tdb/tdb.c
 - lib/tdb.rb
+- lib/tdb/mt.rb
 - setup.rb
 - tdb.gemspec
 - test/test_tdb.rb
+- test/test_tdb_mt.rb
 has_rdoc: true
 homepage: http://bogomips.org/ruby-tdb/
 licenses: []
@@ -102,4 +105,5 @@ signing_key:
 specification_version: 3
 summary: Trivial Database bindings for Ruby
 test_files: 
+- test/test_tdb_mt.rb
 - test/test_tdb.rb