bloom-filter 0.1.0 → 0.1.1

data/README.md

@@ -20,7 +20,7 @@ BloomFilter is a ruby library that implements an in-memory [Bloom Filter](http:/
     .load
     #dump
     #insert
-    #exists?
+    #include?
 
 ```
 
@@ -30,14 +30,19 @@ BloomFilter is a ruby library that implements an in-memory [Bloom Filter](http:/
   require 'bloom-filter'
 
   filter = BloomFilter.new
-  filter = BloomFilter.new 100_000
+
+  # auto-calculate optimum bitmap size based on maximum number of items stored and desired max error rate.
+  filter = BloomFilter.new size: 100_000, error_rate: 0.01
+
+  # specify bitmap size & number of hash functions explicitly.
+  filter = BloomFilter.new bits: 100_000_0, hashes: 4
 
   filter.insert "foo"
-  filter.exists? "foo" #=> true
-  filter.exists? "bar" #=> false
+  filter.include? "foo" #=> true
+  filter.include? "bar" #=> false
 
   filter.dump "/tmp/random.bloom"
-  filter = BloomFilter.load "/tmp/random.bloom", 100_000
+  filter = BloomFilter.load "/tmp/random.bloom"
 ```
 
 ## See Also

data/ext/bloom_filter.c

@@ -5,6 +5,7 @@
 #include <errno.h>
 #include <string.h>
 #include <stdlib.h>
+#include <math.h>
 
 #include "ruby/ruby.h"
 #include "bloom-filter.h"
@@ -14,6 +15,11 @@
 #define TO_S(v)    rb_funcall(v, rb_intern("to_s"), 0)
 #define CSTRING(v) RSTRING_PTR(TO_S(v))
 
+typedef struct FileHeader {
+    uint64_t table_size;
+    uint64_t num_functions;
+} FileHeader;
+
 static void bloom_free(BloomFilter *filter) {
     if (filter)
         bloom_filter_free(filter);
@@ -33,17 +39,41 @@ BloomFilter* bloom_handle(VALUE self) {
 }
 
 VALUE bloom_initialize(int argc, VALUE *argv, VALUE self) {
-    size_t size;
-    VALUE table_size;
+    double error;
+    size_t nbits, nhash, nmax;
+
+    VALUE max_size, error_rate, bitmap_size, hash_count, options;
     BloomFilter *filter = 0;
 
-    rb_scan_args(argc, argv, "01", &table_size);
-    if (NIL_P(table_size))
-        size = 1000000;
-    else
-        size = atol(CSTRING(table_size));
+    rb_scan_args(argc, argv, "01", &options);
+    if (!NIL_P(options) && TYPE(options) != T_HASH)
+        rb_raise(rb_eArgError, "invalid options, expect hash");
 
-    filter = bloom_filter_new(size, string_nocase_hash, 4);
+    if (NIL_P(options)) {
+        nbits = 1000000;
+        nhash = 4;
+    }
+    else {
+        max_size    = rb_hash_aref(options, ID2SYM(rb_intern("size")));
+        error_rate  = rb_hash_aref(options, ID2SYM(rb_intern("error_rate")));
+        bitmap_size = rb_hash_aref(options, ID2SYM(rb_intern("bits")));
+        hash_count  = rb_hash_aref(options, ID2SYM(rb_intern("hashes")));
+
+        nhash = NIL_P(hash_count) ? 4 : NUM2ULONG(hash_count);
+
+        if (!NIL_P(bitmap_size))
+            nbits = NUM2ULONG(bitmap_size);
+        else if (!NIL_P(max_size)) {
+            nmax  = NUM2ULONG(max_size);
+            error = NIL_P(error_rate) ? 0.01 : NUM2DBL(error_rate);
+            nbits = ceil(fabs(log(error) * (double)nmax / pow(log(2), 2)));
+            nhash = ceil(0.7 * (double)nbits / (double)nmax);
+        }
+        else
+            rb_raise(rb_eArgError, "requires either size & error_rate or bits & hashes");
+    }
+
+    filter = bloom_filter_new(nbits, string_nocase_hash, nhash);
 
     if (!filter)
         rb_raise(rb_eNoMemError, "unable to allocate memory for BloomFilter");
@@ -58,17 +88,19 @@ VALUE bloom_insert(VALUE klass, VALUE string) {
     return Qtrue;
 }
 
-VALUE bloom_exists(VALUE klass, VALUE string) {
+VALUE bloom_include(VALUE klass, VALUE string) {
     BloomFilter *filter = bloom_handle(klass);
     return bloom_filter_query(filter, (BloomFilterValue)CSTRING(string)) ? Qtrue : Qfalse;
 }
 
 VALUE bloom_dump(VALUE klass, VALUE file) {
     int fd;
+    uint64_t nbits;
+    FileHeader header;
     BloomFilter *filter = bloom_handle(klass);
 
-    size_t size  = (filter->table_size + 7) / 8;
-    void *buffer = malloc(size);
+    nbits = (filter->table_size + 7) / 8;
+    void *buffer = malloc(nbits);
 
     if (!buffer)
         rb_raise(rb_eNoMemError, "out of memory dumping BloomFilter");
@@ -81,7 +113,16 @@ VALUE bloom_dump(VALUE klass, VALUE file) {
         rb_raise(rb_eIOError, "unable to open file. %s", strerror(errno));
     }
 
-    if (write(fd, buffer, size) != -1) {
+    header.table_size    = filter->table_size;
+    header.num_functions = filter->num_functions;
+
+    if (write(fd, &header, sizeof(header)) == -1) {
+        free(buffer);
+        close(fd);
+        rb_raise(rb_eIOError, "error dumping BloomFilter: %s\n", strerror(errno));
+    }
+
+    if (write(fd, buffer, nbits) != -1) {
         free(buffer);
         close(fd);
         return Qtrue;
@@ -90,47 +131,46 @@ VALUE bloom_dump(VALUE klass, VALUE file) {
         free(buffer);
         close(fd);
         rb_raise(rb_eIOError, "error dumping BloomFilter: %s\n", strerror(errno));
-        return Qfalse; // not reachable
     }
+
+    return Qfalse; // not reachable
 }
 
-VALUE bloom_load(int argc, VALUE *argv, VALUE klass) {
+VALUE bloom_load(VALUE klass, VALUE file) {
     int fd;
     void *buffer;
-    size_t size, bytes;
+    size_t nbits, bytes;
+    FileHeader header;
     BloomFilter *filter;
-    VALUE file, table_size, instance;
-
-    rb_scan_args(argc, argv, "11", &file, &table_size);
-    if (NIL_P(table_size)) {
-        size = 1000000;
-        table_size = INT2NUM(size);
-    }
-    else
-        size = atol(CSTRING(table_size));
+    VALUE instance;
 
     fd = open(CSTRING(file), O_RDONLY);
     if (fd == -1)
         rb_raise(rb_eIOError, "unable to open file: %s", strerror(errno));
 
-    size = (size + 7) / 8;
-    buffer = malloc(size);
+    if (read(fd, &header, sizeof(header)) != sizeof(header)) {
+        close(fd);
+        rb_raise(rb_eIOError, "unable to read file, header corrupted\n");
+    }
+
+    nbits  = (header.table_size + 7) / 8;
+    buffer = malloc(nbits);
     if (!buffer) {
         close(fd);
         rb_raise(rb_eNoMemError, "out of memory dumping BloomFilter");
     }
 
-    bytes = read(fd, buffer, size);
-    if (bytes != size) {
+    bytes = read(fd, buffer, nbits);
+    if (bytes != nbits) {
         free(buffer);
         close(fd);
-        rb_raise(rb_eStandardError, "unable to load BloomFilter, expected %ld but got %ld bytes", size, bytes);
+        rb_raise(rb_eStandardError, "unable to load BloomFilter, expected %ld but got %ld bytes", nbits, bytes);
     }
 
-    instance = bloom_allocate(klass);
-    bloom_initialize(1, &table_size, instance);
+    filter = bloom_filter_new(header.table_size, string_nocase_hash, header.num_functions);
+    bloom_filter_load(filter, buffer);
+    instance = Data_Wrap_Struct(klass, 0, bloom_free, filter);
 
-    bloom_filter_load(bloom_handle(instance), buffer);
     free(buffer);
     close(fd);
     return instance;
@@ -142,9 +182,9 @@ Init_bloom_filter() {
     rb_define_method(cBloom, "initialize", RUBY_METHOD_FUNC(bloom_initialize),  -1);
     rb_define_method(cBloom, "dump",       RUBY_METHOD_FUNC(bloom_dump),         1);
     rb_define_method(cBloom, "insert",     RUBY_METHOD_FUNC(bloom_insert),       1);
-    rb_define_method(cBloom, "exists?",    RUBY_METHOD_FUNC(bloom_exists),       1);
+    rb_define_method(cBloom, "include?",   RUBY_METHOD_FUNC(bloom_include),      1);
 
     rb_define_alloc_func(cBloom, bloom_allocate);
-    rb_define_singleton_method(cBloom, "load", RUBY_METHOD_FUNC(bloom_load), -1);
+    rb_define_singleton_method(cBloom, "load", RUBY_METHOD_FUNC(bloom_load), 1);
     rb_define_const(cBloom, "VERSION", rb_str_new2(RUBY_BLOOM_FILTER_VERSION));
 }

data/ext/version.h

@@ -1 +1 @@
-#define RUBY_BLOOM_FILTER_VERSION "0.1.0"
+#define RUBY_BLOOM_FILTER_VERSION "0.1.1"

data/test/test_basic.rb

@@ -6,18 +6,18 @@ describe 'BloomFilter primitives' do
   end
 
   it 'should create one with given size' do
-    assert BloomFilter.new(100)
+    assert BloomFilter.new(size: 100)
   end
 
   it 'should insert' do
-    assert filter = BloomFilter.new(100)
+    assert filter = BloomFilter.new(size: 100)
     assert filter.insert("foo")
   end
 
   it 'should allow membership checks' do
-    assert filter = BloomFilter.new(100)
+    assert filter = BloomFilter.new(size: 100)
     assert filter.insert("foo")
-    assert filter.exists?("foo")
-    assert !filter.exists?("bar")
+    assert filter.include?("foo")
+    assert !filter.include?("bar")
   end
 end

data/test/test_io.rb

@@ -5,11 +5,11 @@ describe 'BloomFilter load & dump' do
   it 'should dump and load a filter' do
     file = Tempfile.new("bloom-filter-test")
 
-    assert filter = BloomFilter.new(100)
+    assert filter = BloomFilter.new(size: 100)
     assert filter.insert("foo")
     assert filter.dump(file.path), "dump filter"
-    assert filter = BloomFilter.load(file.path, 100)
-    assert filter.exists?("foo")
-    assert !filter.exists?("bar")
+    assert filter = BloomFilter.load(file.path)
+    assert filter.include?("foo")
+    assert !filter.include?("bar")
   end
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 0
-  version: 0.1.0
+  - 1
+  version: 0.1.1
 platform: ruby
 authors: 
 - Bharanee Rathna
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-01-25 00:00:00 +11:00
+date: 2012-01-27 00:00:00 +11:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -66,7 +66,6 @@ files:
 - README.md
 - CHANGELOG
 - lib/bloom-filter.rb
-- lib/bloom_filter.so
 has_rdoc: true
 homepage: http://github.com/deepfryed/bloom-filter
 licenses: []