tyler-bloom_filter 0.1.1

data/README.textile

@@ -0,0 +1,23 @@
+h1. Bloom Filter
+
+This is a Bloom filter for Ruby written in C.
+
+
+h2. What is a Bloom filter?
+
+I suck at explaining things.  Wikipedia doesn't.  http://wikipedia.org/wiki/Bloom_filter.
+
+But in short a Bloom filter is a probabilistic data structure which is used to test for membership in a set.  False positives can happen, but false negatives can not.  The exact error rate of a Bloom filter can be tuned by changing the size of it.
+
+
+h2. Why would I use a Bloom filter?
+
+They're extremely compact and quite fast.  The size of a Bloom filter is dependent upon the number of keys you'll be inserting into it and the acceptable error rate.  The smaller the size of the bloom filter, the higher the error rate.
+
+So, one idea is that you could use a Bloom filter as a pre-filter for database calls.  Before requesting a particular id from the database, you could query the Bloom filter.  If it returns true for the id, you can query the database for the data.  If it returns false, you don't.  Since false positives cannot happen, you'll never miss querying the database when you should.  However, you may (at a particular error rate) query the database when you don't necessarily need to.  But, as before, this can be tuned according to your needs.
+
+
+h2. Todo
+
+* Add benchmarks
+* Add graphs of size vs. error rate

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+patch: 1
+major: 0
+minor: 1

data/ext/bloom_filter/Makefile

@@ -0,0 +1,149 @@
+
+SHELL = /bin/sh
+
+#### Start of system configuration section. ####
+
+srcdir = .
+topdir = /System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/1.8/universal-darwin9.0
+hdrdir = $(topdir)
+VPATH = $(srcdir):$(topdir):$(hdrdir)
+prefix = $(DESTDIR)/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr
+exec_prefix = $(prefix)
+sitedir = $(DESTDIR)/Library/Ruby/Site
+rubylibdir = $(libdir)/ruby/$(ruby_version)
+docdir = $(datarootdir)/doc/$(PACKAGE)
+dvidir = $(docdir)
+datarootdir = $(prefix)/share
+archdir = $(rubylibdir)/$(arch)
+sbindir = $(exec_prefix)/sbin
+psdir = $(docdir)
+localedir = $(datarootdir)/locale
+htmldir = $(docdir)
+datadir = $(datarootdir)
+includedir = $(prefix)/include
+infodir = $(DESTDIR)/usr/share/info
+sysconfdir = $(prefix)/etc
+mandir = $(DESTDIR)/usr/share/man
+libdir = $(exec_prefix)/lib
+sharedstatedir = $(prefix)/com
+oldincludedir = $(DESTDIR)/usr/include
+pdfdir = $(docdir)
+sitearchdir = $(sitelibdir)/$(sitearch)
+bindir = $(exec_prefix)/bin
+localstatedir = $(prefix)/var
+sitelibdir = $(sitedir)/$(ruby_version)
+libexecdir = $(exec_prefix)/libexec
+
+CC = gcc
+LIBRUBY = $(LIBRUBY_SO)
+LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a
+LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME)
+LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)
+
+RUBY_EXTCONF_H = 
+CFLAGS   =  -fno-common -arch ppc -arch i386 -Os -pipe -fno-common 
+INCFLAGS = -I. -I$(topdir) -I$(hdrdir) -I$(srcdir)
+CPPFLAGS =  
+CXXFLAGS = $(CFLAGS) 
+DLDFLAGS = -L. -arch ppc -arch i386  
+LDSHARED = cc -arch ppc -arch i386 -pipe -bundle -undefined dynamic_lookup
+AR = ar
+EXEEXT = 
+
+RUBY_INSTALL_NAME = ruby
+RUBY_SO_NAME = ruby
+arch = universal-darwin9.0
+sitearch = universal-darwin9.0
+ruby_version = 1.8
+ruby = /System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/bin/ruby
+RUBY = $(ruby)
+RM = rm -f
+MAKEDIRS = mkdir -p
+INSTALL = /usr/bin/install -c
+INSTALL_PROG = $(INSTALL) -m 0755
+INSTALL_DATA = $(INSTALL) -m 644
+COPY = cp
+
+#### End of system configuration section. ####
+
+preload = 
+
+libpath = . $(libdir)
+LIBPATH =  -L"." -L"$(libdir)"
+DEFFILE = 
+
+CLEANFILES = mkmf.log
+DISTCLEANFILES = 
+
+extout = 
+extout_prefix = 
+target_prefix = 
+LOCAL_LIBS = 
+LIBS = $(LIBRUBYARG_SHARED)  -lpthread -ldl -lm  
+SRCS = bloom.c bloom_filter.c
+OBJS = bloom.o bloom_filter.o
+TARGET = bloom_filter
+DLLIB = $(TARGET).bundle
+EXTSTATIC = 
+STATIC_LIB = 
+
+RUBYCOMMONDIR = $(sitedir)$(target_prefix)
+RUBYLIBDIR    = $(sitelibdir)$(target_prefix)
+RUBYARCHDIR   = $(sitearchdir)$(target_prefix)
+
+TARGET_SO     = $(DLLIB)
+CLEANLIBS     = $(TARGET).bundle $(TARGET).il? $(TARGET).tds $(TARGET).map
+CLEANOBJS     = *.o *.a *.s[ol] *.pdb *.exp *.bak
+
+all:		$(DLLIB)
+static:		$(STATIC_LIB)
+
+clean:
+		@-$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES)
+
+distclean:	clean
+		@-$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
+		@-$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES)
+
+realclean:	distclean
+install: install-so install-rb
+
+install-so: $(RUBYARCHDIR)
+install-so: $(RUBYARCHDIR)/$(DLLIB)
+$(RUBYARCHDIR)/$(DLLIB): $(DLLIB)
+	$(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR)
+install-rb: pre-install-rb install-rb-default
+install-rb-default: pre-install-rb-default
+pre-install-rb: Makefile
+pre-install-rb-default: Makefile
+$(RUBYARCHDIR):
+	$(MAKEDIRS) $@
+
+site-install: site-install-so site-install-rb
+site-install-so: install-so
+site-install-rb: install-rb
+
+.SUFFIXES: .c .m .cc .cxx .cpp .C .o
+
+.cc.o:
+	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
+
+.cxx.o:
+	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
+
+.cpp.o:
+	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
+
+.C.o:
+	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
+
+.c.o:
+	$(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) -c $<
+
+$(DLLIB): $(OBJS)
+	@-$(RM) $@
+	$(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS)
+
+
+
+$(OBJS): ruby.h defines.h

data/ext/bloom_filter/bloom.c

@@ -0,0 +1,140 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <math.h>
+#include "bloom.h"
+
+#define TRUE 1
+#define FALSE 0
+#define jen_mix(a,b,c) { \
+  a -= b; a -= c; a ^= ( c >> 13 ); \
+  b -= c; b -= a; b ^= ( a << 8 ); \
+  c -= a; c -= b; c ^= ( b >> 13 ); \
+  a -= b; a -= c; a ^= ( c >> 12 ); \
+  b -= c; b -= a; b ^= ( a << 16 ); \
+  c -= a; c -= b; c ^= ( b >> 5 ); \
+  a -= b; a -= c; a ^= ( c >> 3 ); \
+  b -= c; b -= a; b ^= ( a << 10 ); \
+  c -= a; c -= b; c ^= ( b >> 15 ); \
+}
+
+#define bloom_index_jen(filter,key,len,salt_idx) \
+	bloom_hash_jen((void*)key, len, *(filter->salts + salt_idx)) % filter->bitset_size;
+
+
+Bloom *bloom_alloc() {
+	return (Bloom*)(malloc(sizeof(Bloom)));
+}
+
+void bloom_set_hashes(Bloom* filter, int hash_count) {
+	filter->hash_count = hash_count;
+	filter->salts = (unsigned int*)(malloc(sizeof(int) * hash_count));
+	int i;
+	for(i = 0; i < hash_count; i++)
+		*(filter->salts + i) = (unsigned int)rand();
+}
+
+void bloom_set_bitset(Bloom* filter, int bitset_size) {
+	filter->bitset_size = bitset_size;
+	filter->bitset = (bitstr_t*)(malloc(bitstr_size(bitset_size)));
+}
+
+Bloom *bloom_new(int bitset_size, int hash_count) {
+	Bloom *filter = bloom_alloc();
+	bloom_set_bitset(filter, bitset_size);
+	bloom_set_hashes(filter, hash_count);
+	return filter;
+}
+
+int bloom_size_for_error(double error, int key_count) {
+	return ceil((key_count * log(error)) / log(1.0 / (pow(2.0, log(2.0)))));
+}
+
+int bloom_ideal_hash_count(int size, int key_count) {
+	return round(log(2.0) * size / key_count);
+}
+
+Bloom *bloom_for_error_and_keys(double error, int key_count) {
+	int size = bloom_size_for_error(error, key_count);
+	int hashes = bloom_ideal_hash_count(size, key_count);
+	return bloom_alloc(size, hashes);
+}
+
+void bloom_free(Bloom *filter) {
+	free(filter->bitset);
+	free(filter);
+}
+
+
+void bloom_add(Bloom *filter, void *key, unsigned int key_size) {
+	int i;
+	for(i = 0; i < filter->hash_count; i++) {
+		int bit = bloom_index_jen(filter, key, key_size, i);
+		bit_set(filter->bitset, bit);
+	}
+}
+
+int bloom_get(Bloom *filter, void *key, unsigned int key_size) {
+	int i;
+	for(i = 0; i < filter->hash_count; i++) {
+		int bit = bloom_index_jen(filter, key, key_size, i);
+		if(bit_test(filter->bitset, bit) == 0)
+			return FALSE;
+	}
+
+	return TRUE;
+}
+
+unsigned int bloom_hash_jen (void *key, unsigned int length, unsigned int salt) {
+	unsigned char *k = (unsigned char*) key;
+	unsigned a, b;
+	unsigned c = salt;
+	unsigned int len = length;
+
+	a = b = 0x9e3779b9;
+
+	while ( len >= 12 ) {
+		a += ( k[0] + ( (unsigned)k[1] << 8 ) 
+			   + ( (unsigned)k[2] << 16 )
+			   + ( (unsigned)k[3] << 24 ) );
+		b += ( k[4] + ( (unsigned)k[5] << 8 ) 
+			   + ( (unsigned)k[6] << 16 )
+			   + ( (unsigned)k[7] << 24 ) );
+		c += ( k[8] + ( (unsigned)k[9] << 8 ) 
+			   + ( (unsigned)k[10] << 16 )
+			   + ( (unsigned)k[11] << 24 ) );
+
+		jen_mix ( a, b, c );
+
+		k += 12;
+		len -= 12;
+	}
+
+	c += length;
+
+	switch ( len ) {
+	case 11: c += ( (unsigned)k[10] << 24 );
+	case 10: c += ( (unsigned)k[9] << 16 );
+	case 9 : c += ( (unsigned)k[8] << 8 );
+		/* First byte of c reserved for length */
+	case 8 : b += ( (unsigned)k[7] << 24 );
+	case 7 : b += ( (unsigned)k[6] << 16 );
+	case 6 : b += ( (unsigned)k[5] << 8 );
+	case 5 : b += k[4];
+	case 4 : a += ( (unsigned)k[3] << 24 );
+	case 3 : a += ( (unsigned)k[2] << 16 );
+	case 2 : a += ( (unsigned)k[1] << 8 );
+	case 1 : a += k[0];
+	}
+
+	jen_mix ( a, b, c );
+
+	return c;
+}
+
+void bloom_status(Bloom *filter) {
+	printf("Bitset Size: %d\n", filter->bitset_size);
+	printf("Hash Count: %d\n", filter->hash_count);
+}
+
+

data/ext/bloom_filter/bloom.h

@@ -0,0 +1,20 @@
+#include <bitstring.h>
+
+typedef struct {
+	bitstr_t *bitset;
+	int bitset_size;
+	int hash_count;
+	unsigned int *salts;
+} Bloom;
+
+Bloom *bloom_alloc();
+Bloom *bloom_new(int bitset_size, int hash_count);
+void bloom_set_hashes(Bloom* filter, int hash_count);
+void bloom_set_bitset(Bloom* filter, int hash_count);
+Bloom *bloom_for_error_and_keys(double error, int key_count);
+void bloom_free(Bloom *filter);
+void bloom_add(Bloom *filter, void *key, unsigned int key_size);
+int bloom_get(Bloom *filter, void *key, unsigned int key_size);
+unsigned int bloom_hash_jen(void *key, unsigned int length, unsigned int salt);
+int bloom_size_for_error(double error, int key_count);
+int bloom_ideal_hash_count(int size, int key_count);

data/ext/bloom_filter/bloom_filter.c

@@ -0,0 +1,74 @@
+#include "bloom.h"
+#include "ruby.h"
+#include <stdlib.h>
+
+VALUE cBloomFilter;
+
+void rb_bloom_free(void *bloom) {
+	bloom_free(bloom);
+}
+
+static VALUE rb_bloom_alloc(VALUE klass) {
+	VALUE obj = Data_Wrap_Struct(klass, 0, rb_bloom_free, bloom_alloc());
+	return obj;
+}
+
+Bloom *get_bloom(VALUE self) {
+	Bloom *filter;
+	Data_Get_Struct(self, Bloom, filter);
+	return filter;
+}
+
+static VALUE rb_bloom_initialize(VALUE self, VALUE bitset_size, VALUE hash_count) {
+	Bloom *filter = get_bloom(self);
+	bloom_set_bitset(filter, NUM2INT(bitset_size));
+	bloom_set_hashes(filter, NUM2INT(hash_count));
+	return self;
+}
+
+static VALUE rb_bloom_set(VALUE self, VALUE key) {
+	Bloom *filter = get_bloom(self);
+	int hash_key = NUM2INT(rb_hash(key));
+	bloom_add(filter, &hash_key, sizeof(int));
+	return Qtrue;
+}
+
+static VALUE rb_bloom_get(VALUE self, VALUE key) {
+	Bloom *filter = get_bloom(self);
+	int hash_key = NUM2INT(rb_hash(key));
+	return bloom_get(filter, &hash_key, sizeof(int)) ? Qtrue : Qnil;
+}
+
+static VALUE rb_bloom_filter_size(VALUE self) {
+	Bloom *filter = get_bloom(self);
+	return INT2FIX(filter->bitset_size);
+}
+
+static VALUE rb_bloom_hash_count(VALUE self) {
+	Bloom *filter = get_bloom(self);
+	return INT2FIX(filter->hash_count);
+}
+
+static VALUE rb_bloom_for_error_rate(VALUE self, VALUE error_rate, VALUE key_count) {
+	VALUE rb_filter = rb_bloom_alloc(cBloomFilter);
+	Bloom *filter = get_bloom(rb_filter);
+
+	int size = bloom_size_for_error(NUM2DBL(error_rate), NUM2INT(key_count));
+	int hash_count = bloom_ideal_hash_count(size, NUM2INT(key_count));
+
+	bloom_set_bitset(filter, size);
+	bloom_set_hashes(filter, hash_count);
+
+	return rb_filter;
+}
+
+void Init_bloom_filter() {
+    cBloomFilter = rb_define_class("BloomFilter", rb_cObject);
+    rb_define_alloc_func(cBloomFilter, rb_bloom_alloc);
+	rb_define_method(cBloomFilter, "initialize", rb_bloom_initialize, 2);
+    rb_define_method(cBloomFilter, "get", rb_bloom_get, 1);
+    rb_define_method(cBloomFilter, "set", rb_bloom_set, 1);
+    rb_define_method(cBloomFilter, "filter_size", rb_bloom_filter_size, 0);
+    rb_define_method(cBloomFilter, "hash_count", rb_bloom_hash_count, 0);
+	rb_define_singleton_method(cBloomFilter, "for_error_rate", rb_bloom_for_error_rate, 2);
+}

data/ext/bloom_filter/extconf.rb

@@ -0,0 +1,3 @@
+require 'mkmf'
+create_makefile 'bloom_filter'
+

data/lib/bloom_filter.rb

@@ -0,0 +1 @@
+require File.dirname(__FILE__) + '/../ext/bloom_filter'

data/spec/bloom_filter_spec.rb

@@ -0,0 +1,86 @@
+require File.dirname(__FILE__) + '/../ext/bloom_filter/bloom_filter'
+
+describe BloomFilter do
+  describe :new do
+    it 'returns an instance of BloomFilter' do
+      BloomFilter.new(1024, 3).should be_an_instance_of(BloomFilter)
+    end
+  end
+
+  describe :set do
+    it 'accepts a key of any type' do
+      filter = BloomFilter.new(1024,3)
+      lambda { filter.set(123) }.should_not raise_error
+      lambda { filter.set('abc') }.should_not raise_error
+      lambda { filter.set(:foo) }.should_not raise_error
+      lambda { filter.set([1,2,3]) }.should_not raise_error
+      lambda { filter.set(:a => 1) }.should_not raise_error
+      lambda { filter.set(nil) }.should_not raise_error
+    end
+
+    it 'returns true' do
+      filter = BloomFilter.new(1024,3)
+      filter.set(123).should be_true
+    end
+  end
+
+  describe :get do
+    before :each do
+      @filter = BloomFilter.new(1024,3)
+      @filter.set(123)
+      @filter.set('abc')
+      @filter.set(:foo)
+      @filter.set([1,2,3])
+      @hash = { :a => 1 }
+      @filter.set(@hash)
+      @filter.set(nil)
+    end
+
+    it 'returns nil when the key does not exist' do
+      @filter.get('not in the filter').should be_nil
+    end
+
+    it 'returns true when the key exists' do
+      @filter.get(123).should be_true
+      @filter.get('abc').should be_true
+      @filter.get(:foo).should be_true
+      @filter.get([1,2,3]).should be_true
+      @filter.get(@hash).should be_true
+      @filter.get(nil).should be_true
+    end
+  end
+
+  describe :filter_size do
+    it 'returns the size of the bitset used in the filter' do
+      f = BloomFilter.new(1024,3)
+      f.filter_size.should == 1024
+
+      f = BloomFilter.new(10,3)
+      f.filter_size.should == 10
+    end
+  end
+
+  describe :hash_count do
+    it 'returns the number of hash functions used in the filter' do
+      f = BloomFilter.new(1024,3)
+      f.hash_count.should == 3
+
+      f = BloomFilter.new(1024,10)
+      f.hash_count.should == 10
+    end
+  end
+
+  describe :for_error_rate do
+    it 'returns a filter of the proper size' do
+      BloomFilter.for_error_rate(0.05,100).filter_size.should == 624
+      BloomFilter.for_error_rate(0.10,100).filter_size.should == 480
+      BloomFilter.for_error_rate(0.10,50).filter_size.should == 240
+    end
+
+    it 'returns a filter with the correct hash count' do
+      BloomFilter.for_error_rate(0.1,1000).hash_count.should == 3
+      BloomFilter.for_error_rate(0.01,1000).hash_count.should == 7
+      BloomFilter.for_error_rate(0.001,1000).hash_count.should == 10
+    end
+  end
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification 
+name: tyler-bloom_filter
+version: !ruby/object:Gem::Version 
+  version: 0.1.1
+platform: ruby
+authors: 
+- Tyler McMullen
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-05-11 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: Fast Bloom filter in C and Ruby.
+email: tyler@scribd.com
+executables: []
+
+extensions: 
+- ext/bloom_filter/extconf.rb
+extra_rdoc_files: []
+
+files: 
+- README.textile
+- VERSION.yml
+- lib/bloom_filter.rb
+- spec/bloom_filter_spec.rb
+- ext/bloom_filter
+- ext/bloom_filter/bloom.c
+- ext/bloom_filter/bloom.h
+- ext/bloom_filter/bloom_filter.c
+- ext/bloom_filter/extconf.rb
+- ext/bloom_filter/Makefile
+has_rdoc: false
+homepage: http://github.com/tyler/bloom_filter
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- ext
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Fast Bloom filter in C and Ruby.
+test_files: []
+