tdb 0.5.0 → 0.6.1

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 GVF=GIT-VERSION-FILE
-DEF_VER=v0.5.0.GIT
+DEF_VER=v0.6.0
 
 LF='
 '

data/GNUmakefile

@@ -1,155 +1,6 @@
-# use GNU Make to run tests in parallel, and without depending on RubyGems
 all::
-RUBY = ruby
-RAKE = rake
-RSYNC = rsync
-
-GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE
-	@./GIT-VERSION-GEN
--include GIT-VERSION-FILE
--include local.mk
-ifeq ($(DLEXT),) # "so" for Linux
-  DLEXT := $(shell $(RUBY) -rrbconfig -e 'puts Config::CONFIG["DLEXT"]')
-endif
-ifeq ($(RUBY_VERSION),)
-  RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION')
-endif
-
-install:
-	$(prep_setup_rb)
-	$(RM) -r .install-tmp
-	mkdir .install-tmp
-	$(RUBY) setup.rb all
-	$(RM) $^
-	$(RM) -r .install-tmp
-	$(prep_setup_rb)
-
-setup_rb_files := .config InstalledFiles
-prep_setup_rb := @-$(RM) $(setup_rb_files);$(MAKE) -C $(ext) clean
-
-extdir := ext/tdb
-clean:
-	-$(MAKE) -C $(extdir) clean
-	$(RM) $(setup_rb_files) $(extdir)/Makefile
-
-pkg_extra := GIT-VERSION-FILE NEWS ChangeLog LATEST
-ChangeLog: GIT-VERSION-FILE .wrongdoc.yml
-	wrongdoc prepare
-
-.manifest: ChangeLog
-	(git ls-files && for i in $@ $(pkg_extra); do echo $$i; done) | \
-		LC_ALL=C sort > $@+
-	cmp $@+ $@ || mv $@+ $@
-	$(RM) $@+
-
-doc: .document .wrongdoc.yml
-	find lib ext -type f -name '*.rbc' -exec rm -f '{}' ';'
-	$(RM) -r doc
-	wrongdoc all
-	install -m644 COPYING doc/COPYING
-	install -m644 $(shell grep '^[A-Z]' .document) doc/
-
-ifneq ($(VERSION),)
+RSYNC_DEST := bogomips.org:/srv/bogomips/ruby-tdb
 rfproject := qrp
 rfpackage := tdb
-pkggem := pkg/$(rfpackage)-$(VERSION).gem
-pkgtgz := pkg/$(rfpackage)-$(VERSION).tgz
-release_notes := release_notes-$(VERSION)
-release_changes := release_changes-$(VERSION)
-
-release-notes: $(release_notes)
-release-changes: $(release_changes)
-$(release_changes):
-	wrongdoc release_changes > $@+
-	$(VISUAL) $@+ && test -s $@+ && mv $@+ $@
-$(release_notes):
-	wrongdoc release_notes > $@+
-	$(VISUAL) $@+ && test -s $@+ && mv $@+ $@
-
-# ensures we're actually on the tagged $(VERSION), only used for release
-verify:
-	test x"$(shell umask)" = x0022
-	git rev-parse --verify refs/tags/v$(VERSION)^{}
-	git diff-index --quiet HEAD^0
-	test `git rev-parse --verify HEAD^0` = \
-	     `git rev-parse --verify refs/tags/v$(VERSION)^{}`
-
-fix-perms:
-	-git ls-tree -r HEAD | awk '/^100644 / {print $$NF}' | xargs chmod 644
-	-git ls-tree -r HEAD | awk '/^100755 / {print $$NF}' | xargs chmod 755
-
-gem: $(pkggem)
-
-install-gem: $(pkggem)
-	gem install $(CURDIR)/$<
-
-$(pkggem): .manifest fix-perms
-	gem build $(rfpackage).gemspec
-	mkdir -p pkg
-	mv $(@F) $@
-
-$(pkgtgz): distdir = $(basename $@)
-$(pkgtgz): HEAD = v$(VERSION)
-$(pkgtgz): .manifest fix-perms
-	@test -n "$(distdir)"
-	$(RM) -r $(distdir)
-	mkdir -p $(distdir)
-	tar cf - `cat .manifest` | (cd $(distdir) && tar xf -)
-	cd pkg && tar cf - $(basename $(@F)) | gzip -9 > $(@F)+
-	mv $@+ $@
-
-package: $(pkgtgz) $(pkggem)
-
-test-release: verify package $(release_notes) $(release_changes)
-release: verify package $(release_notes) $(release_changes)
-	# make tgz release on RubyForge
-	rubyforge add_release -f -n $(release_notes) -a $(release_changes) \
-	  $(rfproject) $(rfpackage) $(VERSION) $(pkgtgz)
-	# push gem to RubyGems.org
-	gem push $(pkggem)
-	# in case of gem downloads from RubyForge releases page
-	-rubyforge add_file \
-	  $(rfproject) $(rfpackage) $(VERSION) $(pkggem)
-	$(RAKE) raa_update VERSION=$(VERSION)
-	$(RAKE) publish_news VERSION=$(VERSION)
-else
-gem install-gem: GIT-VERSION-FILE
-	$(MAKE) $@ VERSION=$(GIT_VERSION)
-endif
-
-ext := $(extdir)/tdb_ext.$(DLEXT)
-$(extdir)/Makefile: $(extdir)/extconf.rb
-	cd $(@D) && $(RUBY) extconf.rb
-
-c_files := $(wildcard $(extdir)/*.[ch] $(extdir)/*/*.h)
-$(ext): $(c_files) $(extdir)/Makefile
-	$(MAKE) -C $(@D)
-
-all:: test
-
-build: $(ext)
-test_units := $(wildcard test/test_*.rb)
-test: test-unit
-test-unit: $(test_units)
-$(test_units): build
-	$(RUBY) -I lib:$(extdir) $@
-
-# this requires GNU coreutils variants
-publish_doc:
-	-git set-file-times
-	$(MAKE) doc
-	find doc/images -type f | \
-		TZ=UTC xargs touch -d '1970-01-01 00:00:06' doc/rdoc.css
-	$(MAKE) doc_gz
-	chmod 644 $$(find doc -type f)
-	$(RSYNC) -av doc/ bogomips.org:/srv/bogomips/ruby-tdb/
-	git ls-files | xargs touch
-
-# Create gzip variants of the same timestamp as the original so nginx
-# "gzip_static on" can serve the gzipped versions directly.
-doc_gz: docs = $(shell find doc -type f ! -regex '^.*\.\(gif\|jpg\|png\|gz\)$$')
-doc_gz:
-	for i in $(docs); do \
-	  gzip --rsyncable -9 < $$i > $$i.gz; touch -r $$i $$i.gz; done
 
-.PHONY: .FORCE-GIT-VERSION-FILE doc test $(test_units)
+include pkg.mk

data/HACKING

@@ -2,7 +2,7 @@
 
 === Code Compatibility
 
-We target Ruby 1.9.2 and eventually Rubinius 1.1+ and their respective C
+We target C Ruby 1.9.3+ and (time permitting) Rubinius and their respective C
 APIs.
 
 All of our C code should be compatible with all reasonably modern Unices

data/Hash_Functions

@@ -19,15 +19,27 @@ corruption, so don't do it.
   {lookup3 hash}[http://www.burtleburtle.net/bob/c/lookup3.c].
   <code>:hash => :jenkins_lookup3</code>
 
+== SipHash
+
+https://en.wikipedia.org/wiki/Sip_Hash
+https://131002.net/siphash/
+
+Currently (as of Ruby 2.1) the favored hash in Ruby for hash-flooding
+protection.
+
+* :siphash24 - the reference implementation
+
 == Murmur family
 
 The {Murmur}[https://sites.google.com/site/murmurhash/] family of hashes
-are supported by Ruby TDB.  MurmurHash3 will be supported as soon as it
-becomes finalized.  Most of these are not endian-neutral so databases
-are no compatible between machines of different endianness and were
-designed with x86 and x86_64 in mind (they may crash or not work on
+are supported by Ruby TDB.  Most of these are not endian-neutral so
+databases are no compatible between machines of different endianness and
+were designed with x86 and x86_64 in mind (they may crash or not work on
 other architectures).
 
+* :murmur3a - The latest 32-bit version optimized for x86
+   https://code.google.com/p/smhasher/wiki/MurmurHash3
+
 * :murmur2 - the simple and fast implementation
 
 * :murmur2a - words of the author:
@@ -61,7 +73,6 @@ other architectures).
 
 == Bernstein hashes
 
-* :djb3 - The hash currently favored by Bernstein.
-  See http://www.cse.yorku.ca/~oz/hash.html
+* :djb3 - See http://www.cse.yorku.ca/~oz/hash.html
 
 * :djb2 - See http://www.cse.yorku.ca/~oz/hash.html

data/LICENSE

@@ -4,7 +4,7 @@ revision control for names and email addresses of all of them.
 This library is free software; you can redistribute it and/or modify it
 under the terms of the GNU Lesser General Public License as published by
 the Free Software Foundation; either version
-{3}[http://www.gnu.org/licenses/lgpl-3.0.txt] of the License, or (at
+{2.1}[https://www.gnu.org/licenses/lgpl-2.1.txt] of the License, or (at
 your option) any later version.
 
 Ruby tdb is distributed in the hope that it will be useful, but WITHOUT
@@ -13,5 +13,4 @@ FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
 License for more details.
 
 You should have received a copy of the GNU Lesser General Public License
-along with Ruby tdb; if not, write to the Free Software
-Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301
+along with Ruby tdb; if not, see [https://www.gnu.org/licenses/].

data/README

@@ -8,8 +8,8 @@ write to the same databases used by Samba!
 == Features
 
 * Concurrent reader and writer processes may safely operate on the
-  same file.  This is great for MRI 1.8 and 1.9 where multi-core
-  performance is easiest to achieve with processes and not threads.
+  same file.  This is great for MRI where multi-core performance
+  is easiest to achieve with processes and not threads.
 
 * Fork-safe, you may fork and share the same TDB object in your parent
   and child processes.
@@ -70,13 +70,13 @@ Or if you have a non-standard prefix that linkers normally do not search:
 You can get the latest source via git from the following locations
 (these versions may not be stable):
 
-  git://git.bogomips.org/ruby-tdb.git
+  git://bogomips.org/ruby-tdb.git
   git://repo.or.cz/ruby-tdb.git (mirror)
 
 You may browse the code from the web and download the latest snapshot
 tarballs here:
 
-* http://git.bogomips.org/cgit/ruby-tdb.git (cgit)
+* http://bogomips.org/ruby-tdb.git (cgit)
 * http://repo.or.cz/w/ruby-tdb.git (gitweb)
 
 See the HACKING guide on how to contribute and build prerelease gems

data/Rakefile

@@ -1,37 +1 @@
 # -*- encoding: binary -*-
-desc "post to RAA"
-task :raa_update do
-  require 'net/http'
-  require 'net/netrc'
-  rc = Net::Netrc.locate('tdb-raa') or abort "~/.netrc not found"
-  password = rc.password
-
-  s = Gem::Specification.load('tdb.gemspec')
-  desc = [ s.description.strip ]
-  desc << ""
-  desc << "* #{s.email}"
-  desc << "* #{git_url}"
-  desc << "* #{cgit_url}"
-  desc = desc.join("\n")
-  uri = URI.parse('http://raa.ruby-lang.org/regist.rhtml')
-  form = {
-    :name => s.name,
-    :short_description => s.summary,
-    :version => s.version.to_s,
-    :status => 'experimental',
-    :owner => s.authors.first,
-    :email => s.email,
-    :category_major => 'Library',
-    :category_minor => 'Database',
-    :url => s.homepage,
-    :download => 'http://bogomips.org/ruby-tdb/files/',
-    :license => "LGPL",
-    :description_style => 'Plain',
-    :description => desc,
-    :pass => password,
-    :submit => 'Update',
-  }
-  res = Net::HTTP.post_form(uri, form)
-  p res
-  puts res.body
-end

data/TODO

@@ -1,5 +1,3 @@
-* port the final Murmur3 hash implementation
-
 * support more TDB-specific features
 
 * make it reasonably API-compatible with other DBM bindings

data/ext/tdb/extconf.rb

@@ -2,6 +2,7 @@ require 'mkmf'
 
 have_func('rb_thread_blocking_region')
 have_func('rb_thread_call_with_gvl')
+have_func('rb_thread_call_without_gvl', 'ruby/thread.h')
 
 dir_config('tdb')
 have_header('tdb.h') or abort 'tdb.h missing'

data/ext/tdb/hash_functions.c

@@ -10,12 +10,15 @@ static VALUE fn(VALUE self,VALUE str) \
 	return UINT2NUM(rbtdb_##fn(&data)); \
 }
 
+HASH_FN(siphash24)
 HASH_FN(murmur1)
 HASH_FN(murmur1_aligned)
 HASH_FN(murmur2)
 HASH_FN(murmur2a)
 HASH_FN(murmur2_neutral)
 HASH_FN(murmur2_aligned)
+HASH_FN(murmur3a)
+HASH_FN(murmur3f)
 HASH_FN(fnv1a)
 HASH_FN(djb2)
 HASH_FN(djb3)
@@ -27,12 +30,15 @@ void rbtdb_init_tdb_hash_functions(void)
 	VALUE cTDB = rb_const_get(rb_cObject, rb_intern("TDB"));
 	VALUE mHashFunctions = rb_define_module_under(cTDB, "HashFunctions");
 
+	HASH_M(siphash24);
 	HASH_M(murmur1);
 	HASH_M(murmur1_aligned);
 	HASH_M(murmur2);
 	HASH_M(murmur2a);
 	HASH_M(murmur2_neutral);
 	HASH_M(murmur2_aligned);
+	HASH_M(murmur3a);
+	HASH_M(murmur3f);
 	HASH_M(fnv1a);
 	HASH_M(djb2);
 	HASH_M(djb3);

data/ext/tdb/murmur3.c

@@ -0,0 +1,230 @@
+#include "rbtdb.h"
+/*
+ * https://sites.google.com/site/murmurhash/
+ *
+ * MurmurHash3 was written by Austin Appleby, and is placed in the public
+ * domain. The author hereby disclaims copyright to this source code.
+ *
+ * Eric Wong trivially ported this to C for Ruby tdb (32-bit versions only)
+ */
+
+#include <stdint.h>
+
+static inline uint32_t rotl32(uint32_t x, int8_t r)
+{
+	return (x << r) | (x >> (32 - r));
+}
+
+#define	ROTL32(x,y)	rotl32(x,y)
+
+#define BIG_CONSTANT(x) (x##LLU)
+
+/* ----------------------------------------------------------------------------
+ * Block read - if your platform needs to do endian-swapping or can only
+ * handle aligned reads, do the conversion here
+ */
+
+static inline uint32_t getblock(const uint32_t * p, int i)
+{
+	return p[i];
+}
+
+/* ----------------------------------------------------------------------------
+ * Finalization mix - force all bits of a hash block to avalanche
+ */
+
+static inline uint32_t fmix(uint32_t h)
+{
+	h ^= h >> 16;
+	h *= 0x85ebca6b;
+	h ^= h >> 13;
+	h *= 0xc2b2ae35;
+	h ^= h >> 16;
+
+	return h;
+}
+
+unsigned int rbtdb_murmur3a(TDB_DATA * key)
+{
+	const uint8_t *data = key->dptr;
+	int len = (int)key->dsize;
+	const int nblocks = len / 4;
+	static const uint32_t seed;
+	uint32_t h1 = seed;
+	int i;
+
+	static const uint32_t c1 = 0xcc9e2d51;
+	static const uint32_t c2 = 0x1b873593;
+
+	/* body */
+	const uint32_t *blocks = (const uint32_t *)(data + nblocks * 4);
+
+	for (i = -nblocks; i; i++) {
+		uint32_t k1 = getblock(blocks, i);
+
+		k1 *= c1;
+		k1 = ROTL32(k1, 15);
+		k1 *= c2;
+
+		h1 ^= k1;
+		h1 = ROTL32(h1, 13);
+		h1 = h1 * 5 + 0xe6546b64;
+	}
+
+	/* tail */
+	{
+		const uint8_t *tail = (const uint8_t *)(data + nblocks * 4);
+		uint32_t k1 = 0;
+
+		switch (len & 3) {
+		case 3:
+			k1 ^= tail[2] << 16;
+		case 2:
+			k1 ^= tail[1] << 8;
+		case 1:
+			k1 ^= tail[0];
+			k1 *= c1;
+			k1 = ROTL32(k1, 15);
+			k1 *= c2;
+			h1 ^= k1;
+		};
+	}
+
+	/* finalization */
+
+	h1 ^= len;
+
+	return fmix(h1);
+}
+
+static inline uint64_t rotl64(uint64_t x, int8_t r)
+{
+	return (x << r) | (x >> (64 - r));
+}
+
+#define ROTL64(x,y)	rotl64(x,y)
+
+static inline uint64_t getblock64(const uint64_t * p, int i)
+{
+	return p[i];
+}
+
+static inline uint64_t fmix64(uint64_t k)
+{
+	k ^= k >> 33;
+	k *= BIG_CONSTANT(0xff51afd7ed558ccd);
+	k ^= k >> 33;
+	k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53);
+	k ^= k >> 33;
+
+	return k;
+}
+
+/* this was a 128-bit hash for x86_64, but we only want 32-bits */
+unsigned int rbtdb_murmur3f(TDB_DATA * key)
+{
+	const uint8_t *data = key->dptr;
+	int len = (int)key->dsize;
+	const int nblocks = len / 16;
+	static const uint32_t seed;
+	uint64_t h1 = seed;
+	uint64_t h2 = seed;
+	uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5);
+	uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f);
+	int i;
+
+	/* body */
+	const uint64_t *blocks = (const uint64_t *)(data);
+
+	for (i = 0; i < nblocks; i++) {
+		uint64_t k1 = getblock64(blocks, i * 2 + 0);
+		uint64_t k2 = getblock64(blocks, i * 2 + 1);
+
+		k1 *= c1;
+		k1 = ROTL64(k1, 31);
+		k1 *= c2;
+		h1 ^= k1;
+
+		h1 = ROTL64(h1, 27);
+		h1 += h2;
+		h1 = h1 * 5 + 0x52dce729;
+
+		k2 *= c2;
+		k2 = ROTL64(k2, 33);
+		k2 *= c1;
+		h2 ^= k2;
+
+		h2 = ROTL64(h2, 31);
+		h2 += h1;
+		h2 = h2 * 5 + 0x38495ab5;
+	}
+
+	/* tail */
+	{
+		const uint8_t *tail = (const uint8_t *)(data + nblocks * 16);
+
+		uint64_t k1 = 0;
+		uint64_t k2 = 0;
+		#define CAST64(x) ((uint64_t)(x))
+
+		switch (len & 15) {
+		case 15:
+			k2 ^= CAST64(tail[14]) << 48;
+		case 14:
+			k2 ^= CAST64(tail[13]) << 40;
+		case 13:
+			k2 ^= CAST64(tail[12]) << 32;
+		case 12:
+			k2 ^= CAST64(tail[11]) << 24;
+		case 11:
+			k2 ^= CAST64(tail[10]) << 16;
+		case 10:
+			k2 ^= CAST64(tail[9]) << 8;
+		case 9:
+			k2 ^= CAST64(tail[8]) << 0;
+			k2 *= c2;
+			k2 = ROTL64(k2, 33);
+			k2 *= c1;
+			h2 ^= k2;
+
+		case 8:
+			k1 ^= CAST64(tail[7]) << 56;
+		case 7:
+			k1 ^= CAST64(tail[6]) << 48;
+		case 6:
+			k1 ^= CAST64(tail[5]) << 40;
+		case 5:
+			k1 ^= CAST64(tail[4]) << 32;
+		case 4:
+			k1 ^= CAST64(tail[3]) << 24;
+		case 3:
+			k1 ^= CAST64(tail[2]) << 16;
+		case 2:
+			k1 ^= CAST64(tail[1]) << 8;
+		case 1:
+			k1 ^= CAST64(tail[0]) << 0;
+			k1 *= c1;
+			k1 = ROTL64(k1, 31);
+			k1 *= c2;
+			h1 ^= k1;
+		};
+	}
+
+	/* finalization */
+
+	h1 ^= len;
+	h2 ^= len;
+
+	h1 += h2;
+	h2 += h1;
+
+	h1 = fmix64(h1);
+	h2 = fmix64(h2);
+
+	h1 += h2;
+
+	/* not needed for 32-bit hash */
+	/* h2 += h1; */
+
+	return (unsigned int)h1; /* truncate to 32-bits */
+}