bloom-ruby 0.2.1

data/CHANGELOG

@@ -0,0 +1,15 @@
+=== 0.2.1 (2013-01-21)
+
+* Clean up compiler warnings.
+
+=== 0.2.0 (2012-04-27)
+
+* Added BloomFilter#bits
+
+=== 0.1.1 (2012-01-26)
+
+* Store table size in dumps.
+
+=== 0.1.0 (2012-01-25)
+
+* Initial version.

data/README.md

@@ -0,0 +1,67 @@
+# BloomFilter
+
+BloomFilter is a ruby library that implements an in-memory [Bloom Filter](http://en.wikipedia.org/wiki/Bloom_filter)
+
+## Dependencies
+
+  * ruby 1.9.1 or later
+
+## Installation
+
+```
+  gem install bloom-filter
+```
+
+## API
+
+```
+  BloomFilter
+    .new
+    .load
+    #dump
+    #insert
+    #include?
+    #bits
+    #binary
+
+```
+
+## Usage
+
+```ruby
+  require 'bloom-filter'
+
+  filter = BloomFilter.new
+
+  # auto-calculate optimum bitmap size based on maximum number of items stored and desired max error rate.
+  filter = BloomFilter.new size: 100_000, error_rate: 0.01
+
+  # specify bitmap size & number of hash functions explicitly.
+  filter = BloomFilter.new bits: 100_000_0, hashes: 4
+
+  filter.insert "foo"
+  filter.include? "foo" #=> true
+  filter.include? "bar" #=> false
+
+  filter.dump "/tmp/random.bloom"
+  filter = BloomFilter.load "/tmp/random.bloom"
+
+  bits   = filter.bits   #=> "10010100100111..."
+  binary = filter.binary #=> "\x83Ö\xAC\xEA\u00..."
+
+  filter2 = BloomFilter.new bits: 100_000_0, hashes: 4
+  filter2.binary = binary
+  filter2.include? "foo" #=> true
+```
+
+## See Also
+
+[https://github.com/igrigorik/bloomfilter-rb](https://github.com/igrigorik/bloomfilter-rb)
+
+## Home Page
+
+[https://github.com/deepfryed/bloom-filter](https://github.com/deepfryed/bloom-filter)
+
+## License
+
+MIT

data/ext/bloom-filter.c

@@ -0,0 +1,265 @@
+/*
+
+Copyright (c) 2005-2008, Simon Howard
+
+Permission to use, copy, modify, and/or distribute this software
+for any purpose with or without fee is hereby granted, provided
+that the above copyright notice and this permission notice appear
+in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
+WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
+AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR
+CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "bloom-filter.h"
+
+/* malloc() / free() testing */
+
+#ifdef ALLOC_TESTING
+#include "alloc-testing.h"
+#endif
+
+/* Salt values.  These salts are XORed with the output of the hash
+ * function to give multiple unique hashes. */
+
+static const unsigned int salts[] = {
+    0x5cee4612, 0xb5587b1c, 0xa250f2b0, 0xa3bf6d2a,
+    0x7a81bd1a, 0x92888d7f, 0x1dc977c7, 0xedc96624,
+    0x920c85d9, 0xf16066b3, 0xc6f0d4b3, 0x2b76eb86,
+    0xcacb3893, 0x493d81c5, 0xf5a133ac, 0x039740bf,
+    0x162b8224, 0xf841de90, 0xc3e5090d, 0x3bce93a7,
+    0xf1860334, 0xe832b5f1, 0xf5b6535b, 0xe4cf4fa6,
+    0x8357b769, 0x1442b07a, 0x21c5863d, 0xabc0d846,
+    0x6dc0d77a, 0x23a3992c, 0xe12179ba, 0xd81d1e23,
+    0xcff4727b, 0xe957ecfb, 0xee8f391a, 0x426efa23,
+    0x3a34ff2c, 0x8b875d94, 0x34fd0f63, 0xf159daae,
+    0xaabab8b3, 0xa83a07ba, 0x4e54fb33, 0xfb82fab8,
+    0x2ae2888f, 0xd1a307a8, 0xbe33322d, 0x87c73f86,
+    0x7270fa7e, 0x68673c55, 0x2c8026d0, 0xead8e422,
+    0xa3ee5132, 0xecb67767, 0x1c3b1ae5, 0x47adf5b6,
+    0xf4518d30, 0x46e62797, 0x9889aa76, 0x1405aadf,
+    0xf62f9124, 0x5c435ac5, 0x35b8dfe3, 0x651c08c5,
+};
+
+BloomFilter *bloom_filter_new (unsigned int table_size, BloomFilterHashFunc hash_func, unsigned int num_functions) {
+    BloomFilter *filter;
+
+    /* There is a limit on the number of functions which can be
+     * applied, due to the table size */
+
+    if (num_functions > sizeof (salts) / sizeof (*salts)) {
+        return NULL;
+    }
+
+    /* Allocate bloom filter structure */
+
+    filter = malloc (sizeof (BloomFilter));
+
+    if (filter == NULL) {
+        return NULL;
+    }
+
+    /* Allocate table, each entry is one bit; these are packed into
+     * bytes.  When allocating we must round the length up to the nearest
+     * byte. */
+
+    filter->table = calloc ((table_size + 7) / 8, 1);
+
+    if (filter->table == NULL) {
+        free (filter);
+        return NULL;
+    }
+
+    filter->hash_func = hash_func;
+    filter->num_functions = num_functions;
+    filter->table_size = table_size;
+
+    return filter;
+}
+
+void bloom_filter_free (BloomFilter * bloomfilter) {
+    free (bloomfilter->table);
+    free (bloomfilter);
+}
+
+void bloom_filter_insert (BloomFilter * bloomfilter, BloomFilterValue value) {
+    unsigned long hash;
+    unsigned long subhash;
+    unsigned int index;
+    unsigned int i;
+
+    /* Generate hash of the value to insert */
+
+    hash = bloomfilter->hash_func (value);
+
+    /* Generate multiple unique hashes by XORing with values in the
+     * salt table. */
+
+    for (i = 0; i < bloomfilter->num_functions; ++i) {
+
+        /* Generate a unique hash */
+
+        subhash = hash ^ salts[i];
+
+        /* Find the index into the table */
+
+        index = subhash % bloomfilter->table_size;
+
+        /* Insert into the table.
+         * index / 8 finds the byte index of the table,
+         * index % 8 gives the bit index within that byte to set. */
+
+        bloomfilter->table[index / 8] |= 1 << (index % 8);
+    }
+}
+
+int bloom_filter_query (BloomFilter * bloomfilter, BloomFilterValue value) {
+    unsigned long hash;
+    unsigned long subhash;
+    unsigned int index;
+    unsigned int i;
+    unsigned char b;
+    int bit;
+
+    /* Generate hash of the value to lookup */
+
+    hash = bloomfilter->hash_func (value);
+
+    /* Generate multiple unique hashes by XORing with values in the
+     * salt table. */
+
+    for (i = 0; i < bloomfilter->num_functions; ++i) {
+
+        /* Generate a unique hash */
+
+        subhash = hash ^ salts[i];
+
+        /* Find the index into the table to test */
+
+        index = subhash % bloomfilter->table_size;
+
+        /* The byte at index / 8 holds the value to test */
+
+        b = bloomfilter->table[index / 8];
+        bit = 1 << (index % 8);
+
+        /* Test if the particular bit is set; if it is not set,
+         * this value can not have been inserted. */
+
+        if ((b & bit) == 0) {
+            return 0;
+        }
+    }
+
+    /* All necessary bits were set.  This may indicate that the value
+     * was inserted, or the values could have been set through other
+     * insertions. */
+
+    return 1;
+}
+
+void bloom_filter_read (BloomFilter * bloomfilter, unsigned char *array) {
+    unsigned int array_size;
+
+    /* The table is an array of bits, packed into bytes.  Round up
+     * to the nearest byte. */
+
+    array_size = (bloomfilter->table_size + 7) / 8;
+
+    /* Copy into the buffer of the calling routine. */
+
+    memcpy (array, bloomfilter->table, array_size);
+}
+
+void bloom_filter_load (BloomFilter * bloomfilter, unsigned char *array) {
+    unsigned int array_size;
+
+    /* The table is an array of bits, packed into bytes.  Round up
+     * to the nearest byte. */
+
+    array_size = (bloomfilter->table_size + 7) / 8;
+
+    /* Copy from the buffer of the calling routine. */
+
+    memcpy (bloomfilter->table, array, array_size);
+}
+
+BloomFilter *bloom_filter_union (BloomFilter * filter1, BloomFilter * filter2) {
+    BloomFilter *result;
+    unsigned int i;
+    unsigned int array_size;
+
+    /* To perform this operation, both filters must be created with
+     * the same values. */
+
+    if (filter1->table_size != filter2->table_size
+        || filter1->num_functions != filter2->num_functions || filter1->hash_func != filter2->hash_func) {
+        return NULL;
+    }
+
+    /* Create a new bloom filter for the result */
+
+    result = bloom_filter_new (filter1->table_size, filter1->hash_func, filter1->num_functions);
+
+    if (result == NULL) {
+        return NULL;
+    }
+
+    /* The table is an array of bits, packed into bytes.  Round up
+     * to the nearest byte. */
+
+    array_size = (filter1->table_size + 7) / 8;
+
+    /* Populate the table of the new filter */
+
+    for (i = 0; i < array_size; ++i) {
+        result->table[i] = filter1->table[i] | filter2->table[i];
+    }
+
+    return result;
+}
+
+BloomFilter *bloom_filter_intersection (BloomFilter * filter1, BloomFilter * filter2) {
+    BloomFilter *result;
+    unsigned int i;
+    unsigned int array_size;
+
+    /* To perform this operation, both filters must be created with
+     * the same values. */
+
+    if (filter1->table_size != filter2->table_size
+        || filter1->num_functions != filter2->num_functions || filter1->hash_func != filter2->hash_func) {
+        return NULL;
+    }
+
+    /* Create a new bloom filter for the result */
+
+    result = bloom_filter_new (filter1->table_size, filter1->hash_func, filter1->num_functions);
+
+    if (result == NULL) {
+        return NULL;
+    }
+
+    /* The table is an array of bits, packed into bytes.  Round up
+     * to the nearest byte. */
+
+    array_size = (filter1->table_size + 7) / 8;
+
+    /* Populate the table of the new filter */
+
+    for (i = 0; i < array_size; ++i) {
+        result->table[i] = filter1->table[i] & filter2->table[i];
+    }
+
+    return result;
+}

data/ext/bloom-filter.h

@@ -0,0 +1,193 @@
+/*
+
+Copyright (c) 2005-2008, Simon Howard
+
+Permission to use, copy, modify, and/or distribute this software
+for any purpose with or without fee is hereby granted, provided
+that the above copyright notice and this permission notice appear
+in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
+WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
+AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR
+CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+ */
+
+/**
+ * @file bloom-filter.h
+ *
+ * @brief Bloom filter
+ *
+ * A bloom filter is a space efficient data structure that can be
+ * used to test whether a given element is part of a set.  Lookups
+ * will occasionally generate false positives, but never false
+ * negatives.
+ *
+ * To create a bloom filter, use @ref bloom_filter_new.  To destroy a
+ * bloom filter, use @ref bloom_filter_free.
+ *
+ * To insert a value into a bloom filter, use @ref bloom_filter_insert.
+ *
+ * To query whether a value is part of the set, use
+ * @ref bloom_filter_query.
+ */
+
+#ifndef ALGORITHM_BLOOM_FILTER_H
+#define ALGORITHM_BLOOM_FILTER_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * A value stored in a @ref BloomFilter.
+ */
+
+    typedef void *BloomFilterValue;
+
+/**
+ * Hash function used to generate hash values for values inserted into a
+ * bloom filter.
+ *
+ * @param data   The value to generate a hash value for.
+ * @return       The hash value.
+ */
+
+    typedef unsigned long (*BloomFilterHashFunc) (BloomFilterValue data);
+
+/**
+ * A bloom filter structure.
+ */
+
+typedef struct BloomFilter {
+    BloomFilterHashFunc hash_func;
+    unsigned char *table;
+    unsigned int table_size;
+    unsigned int num_functions;
+} BloomFilter;
+
+
+/**
+ * Create a new bloom filter.
+ *
+ * @param table_size       The size of the bloom filter.  The greater
+ *                         the table size, the more elements can be
+ *                         stored, and the lesser the chance of false
+ *                         positives.
+ * @param hash_func        Hash function to use on values stored in the
+ *                         filter.
+ * @param num_functions    Number of hash functions to apply to each
+ *                         element on insertion.  This running time for
+ *                         insertion and queries is proportional to this
+ *                         value.  The more functions applied, the lesser
+ *                         the chance of false positives.  The maximum
+ *                         number of functions is 64.
+ * @return                 A new hash table structure, or NULL if it
+ *                         was not possible to allocate the new bloom
+ *                         filter.
+ */
+
+    BloomFilter *bloom_filter_new (unsigned int table_size, BloomFilterHashFunc hash_func, unsigned int num_functions);
+
+/**
+ * Destroy a bloom filter.
+ *
+ * @param bloomfilter      The bloom filter to destroy.
+ */
+
+    void bloom_filter_free (BloomFilter * bloomfilter);
+
+/**
+ * Insert a value into a bloom filter.
+ *
+ * @param bloomfilter          The bloom filter.
+ * @param value                The value to insert.
+ */
+
+    void bloom_filter_insert (BloomFilter * bloomfilter, BloomFilterValue value);
+
+/**
+ * Query a bloom filter for a particular value.
+ *
+ * @param bloomfilter          The bloom filter.
+ * @param value                The value to look up.
+ * @return                     Zero if the value was definitely not
+ *                             inserted into the filter.  Non-zero
+ *                             indicates that it either may or may not
+ *                             have been inserted.
+ */
+
+    int bloom_filter_query (BloomFilter * bloomfilter, BloomFilterValue value);
+
+/**
+ * Read the contents of a bloom filter into an array.
+ *
+ * @param bloomfilter          The bloom filter.
+ * @param array                Pointer to the array to read into.  This
+ *                             should be (table_size + 7) / 8 bytes in
+ *                             length.
+ */
+
+    void bloom_filter_read (BloomFilter * bloomfilter, unsigned char *array);
+
+/**
+ * Load the contents of a bloom filter from an array.
+ * The data loaded should be the output read from @ref bloom_filter_read,
+ * from a bloom filter created using the same arguments used to create
+ * the original filter.
+ *
+ * @param bloomfilter          The bloom filter.
+ * @param array                Pointer to the array to load from.  This
+ *                             should be (table_size + 7) / 8 bytes in
+ *                             length.
+ */
+
+    void bloom_filter_load (BloomFilter * bloomfilter, unsigned char *array);
+
+/**
+ * Find the union of two bloom filters.  Values are present in the
+ * resulting filter if they are present in either of the original
+ * filters.
+ *
+ * Both of the original filters must have been created using the
+ * same parameters to @ref bloom_filter_new.
+ *
+ * @param filter1              The first filter.
+ * @param filter2              The second filter.
+ * @return                     A new filter which is an intersection of the
+ *                             two filters, or NULL if it was not possible
+ *                             to allocate memory for the new filter, or
+ *                             if the two filters specified were created
+ *                             with different parameters.
+ */
+
+    BloomFilter *bloom_filter_union (BloomFilter * filter1, BloomFilter * filter2);
+
+/**
+ * Find the intersection of two bloom filters.  Values are only ever
+ * present in the resulting filter if they are present in both of the
+ * original filters.
+ *
+ * Both of the original filters must have been created using the
+ * same parameters to @ref bloom_filter_new.
+ *
+ * @param filter1              The first filter.
+ * @param filter2              The second filter.
+ * @return                     A new filter which is an intersection of the
+ *                             two filters, or NULL if it was not possible
+ *                             to allocate memory for the new filter, or
+ *                             if the two filters specified were created
+ *                             with different parameters.
+ */
+
+    BloomFilter *bloom_filter_intersection (BloomFilter * filter1, BloomFilter * filter2);
+
+#ifdef __cplusplus
+}
+#endif
+#endif                          /* #ifndef ALGORITHM_BLOOM_FILTER_H */

data/ext/bloom_filter.c

@@ -0,0 +1,248 @@
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+#include <math.h>
+
+#include "ruby/ruby.h"
+#include "bloom-filter.h"
+#include "hash-string.h"
+#include "version.h"
+
+#define TO_S(v)    rb_funcall(v, rb_intern("to_s"), 0)
+#define CSTRING(v) RSTRING_PTR(TO_S(v))
+
+typedef struct FileHeader {
+    uint64_t table_size;
+    uint64_t num_functions;
+} FileHeader;
+
+static void bloom_free(BloomFilter *filter) {
+    if (filter)
+        bloom_filter_free(filter);
+}
+
+VALUE bloom_allocate(VALUE klass) {
+    BloomFilter *filter = 0;
+    return Data_Wrap_Struct(klass, 0, bloom_free, filter);
+}
+
+BloomFilter* bloom_handle(VALUE self) {
+    BloomFilter *filter = 0;
+    Data_Get_Struct(self, BloomFilter, filter);
+    if (!filter)
+        rb_raise(rb_eArgError, "invalid BloomFilter instance");
+    return filter;
+}
+
+VALUE bloom_initialize(int argc, VALUE *argv, VALUE self) {
+    double error;
+    size_t nbits, nhash, nmax;
+
+    VALUE max_size, error_rate, bitmap_size, hash_count, options;
+    BloomFilter *filter = 0;
+
+    rb_scan_args(argc, argv, "01", &options);
+    if (!NIL_P(options) && TYPE(options) != T_HASH)
+        rb_raise(rb_eArgError, "invalid options, expect hash");
+
+    if (NIL_P(options)) {
+        nbits = 1000000;
+        nhash = 4;
+    }
+    else {
+        max_size    = rb_hash_aref(options, ID2SYM(rb_intern("size")));
+        error_rate  = rb_hash_aref(options, ID2SYM(rb_intern("error_rate")));
+        bitmap_size = rb_hash_aref(options, ID2SYM(rb_intern("bits")));
+        hash_count  = rb_hash_aref(options, ID2SYM(rb_intern("hashes")));
+
+        nhash = NIL_P(hash_count) ? 4 : NUM2ULONG(hash_count);
+
+        if (!NIL_P(bitmap_size))
+            nbits = NUM2ULONG(bitmap_size);
+        else if (!NIL_P(max_size)) {
+            nmax  = NUM2ULONG(max_size);
+            error = NIL_P(error_rate) ? 0.01 : NUM2DBL(error_rate);
+            nbits = ceil(fabs(log(error) * (double)nmax / pow(log(2), 2)));
+            nhash = ceil(0.7 * (double)nbits / (double)nmax);
+        }
+        else
+            rb_raise(rb_eArgError, "requires either size & error_rate or bits & hashes");
+    }
+
+    filter = bloom_filter_new(nbits, string_nocase_hash, nhash);
+
+    if (!filter)
+        rb_raise(rb_eNoMemError, "unable to allocate memory for BloomFilter");
+
+    DATA_PTR(self) = filter;
+    return self;
+}
+
+
+VALUE bloom_insert(VALUE klass, VALUE string) {
+    BloomFilter *filter = bloom_handle(klass);
+    bloom_filter_insert(filter, (BloomFilterValue)CSTRING(string));
+    return Qtrue;
+}
+
+VALUE bloom_include(VALUE klass, VALUE string) {
+    BloomFilter *filter = bloom_handle(klass);
+    return bloom_filter_query(filter, (BloomFilterValue)CSTRING(string)) ? Qtrue : Qfalse;
+}
+
+VALUE bloom_dump(VALUE klass, VALUE file) {
+    int fd;
+    void *buffer;
+    uint64_t nbits;
+    FileHeader header;
+    BloomFilter *filter = bloom_handle(klass);
+
+    nbits  = (filter->table_size + 7) / 8;
+    buffer = malloc(nbits);
+
+    if (!buffer)
+        rb_raise(rb_eNoMemError, "out of memory dumping BloomFilter");
+
+    bloom_filter_read(filter, buffer);
+
+    fd = open(CSTRING(file), O_WRONLY|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR);
+    if (fd == -1) {
+        free(buffer);
+        rb_raise(rb_eIOError, "unable to open file. %s", strerror(errno));
+    }
+
+    header.table_size    = filter->table_size;
+    header.num_functions = filter->num_functions;
+
+    if (write(fd, &header, sizeof(header)) == -1) {
+        free(buffer);
+        close(fd);
+        rb_raise(rb_eIOError, "error dumping BloomFilter: %s\n", strerror(errno));
+    }
+
+    if (write(fd, buffer, nbits) != -1) {
+        free(buffer);
+        close(fd);
+        return Qtrue;
+    }
+    else {
+        free(buffer);
+        close(fd);
+        rb_raise(rb_eIOError, "error dumping BloomFilter: %s\n", strerror(errno));
+    }
+
+    return Qfalse; // not reachable
+}
+
+VALUE bloom_bits(VALUE klass) {
+    BloomFilter *filter = bloom_handle(klass);
+
+    VALUE bitmap;
+    char *buffer;
+    unsigned char b;
+    int i = 0, bit, nbits = filter->table_size;
+
+    buffer = (char *)malloc(nbits);
+    if (!buffer)
+        rb_raise(rb_eNoMemError, "out of memory dumping BloomFilter");
+
+    for (i = 0; i < nbits; i++) {
+        b = filter->table[i / 8];
+        bit = 1 << (i % 8);
+
+        if ((b & bit) == 0)
+            buffer[i] = '0';
+	    else
+	        buffer[i] = '1';
+    }
+
+    bitmap = rb_str_new(buffer, nbits);
+    free(buffer);
+    return bitmap;
+}
+
+VALUE bloom_binary(VALUE klass) {
+    BloomFilter *filter = bloom_handle(klass);
+
+    VALUE bitmap;
+    char *buffer;
+
+    int nbytes = (filter->table_size + 7) / 8;
+    buffer = (char *)malloc(nbytes);
+
+    if (!buffer)
+        rb_raise(rb_eNoMemError, "out of memory dumping BloomFilter");
+
+    bloom_filter_read(filter, buffer);
+
+    bitmap = rb_str_new(buffer, nbytes);
+    free(buffer);
+    return bitmap;
+}
+
+VALUE bloom_binary_set(VALUE klass, VALUE buffer) {
+    BloomFilter *filter = bloom_handle(klass);
+    char* ptr = (char *) RSTRING_PTR(buffer);
+    bloom_filter_load(filter, ptr);
+    return Qtrue;
+}
+
+VALUE bloom_load(VALUE klass, VALUE file) {
+    int fd;
+    void *buffer;
+    size_t nbits, bytes;
+    FileHeader header;
+    BloomFilter *filter;
+    VALUE instance;
+
+    fd = open(CSTRING(file), O_RDONLY);
+    if (fd == -1)
+        rb_raise(rb_eIOError, "unable to open file: %s", strerror(errno));
+
+    if (read(fd, &header, sizeof(header)) != sizeof(header)) {
+        close(fd);
+        rb_raise(rb_eIOError, "unable to read file, header corrupted\n");
+    }
+
+    nbits  = (header.table_size + 7) / 8;
+    buffer = malloc(nbits);
+    if (!buffer) {
+        close(fd);
+        rb_raise(rb_eNoMemError, "out of memory dumping BloomFilter");
+    }
+
+    bytes = read(fd, buffer, nbits);
+    if (bytes != nbits) {
+        free(buffer);
+        close(fd);
+        rb_raise(rb_eStandardError, "unable to load BloomFilter, expected %ld but got %ld bytes", nbits, bytes);
+    }
+
+    filter = bloom_filter_new(header.table_size, string_nocase_hash, header.num_functions);
+    bloom_filter_load(filter, buffer);
+    instance = Data_Wrap_Struct(klass, 0, bloom_free, filter);
+
+    free(buffer);
+    close(fd);
+    return instance;
+}
+
+void Init_bloom_filter() {
+    VALUE cBloom = rb_define_class("BloomFilter", rb_cObject);
+
+    rb_define_method(cBloom, "initialize", RUBY_METHOD_FUNC(bloom_initialize),  -1);
+    rb_define_method(cBloom, "dump",       RUBY_METHOD_FUNC(bloom_dump),         1);
+    rb_define_method(cBloom, "bits",       RUBY_METHOD_FUNC(bloom_bits),         0);
+    rb_define_method(cBloom, "binary",     RUBY_METHOD_FUNC(bloom_binary),       0);
+    rb_define_method(cBloom, "binary=",    RUBY_METHOD_FUNC(bloom_binary_set),   1);
+    rb_define_method(cBloom, "insert",     RUBY_METHOD_FUNC(bloom_insert),       1);
+    rb_define_method(cBloom, "include?",   RUBY_METHOD_FUNC(bloom_include),      1);
+
+    rb_define_alloc_func(cBloom, bloom_allocate);
+    rb_define_singleton_method(cBloom, "load", RUBY_METHOD_FUNC(bloom_load), 1);
+    rb_define_const(cBloom, "VERSION", rb_str_new2(RUBY_BLOOM_FILTER_VERSION));
+}

data/ext/extconf.rb

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'mkmf'
+create_makefile 'bloom_filter'

data/ext/hash-string.c

@@ -0,0 +1,58 @@
+/*
+
+Copyright (c) 2005-2008, Simon Howard
+
+Permission to use, copy, modify, and/or distribute this software
+for any purpose with or without fee is hereby granted, provided
+that the above copyright notice and this permission notice appear
+in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
+WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
+AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR
+CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+ */
+
+#include <ctype.h>
+
+#include "hash-string.h"
+
+/* String hash function */
+
+unsigned long string_hash (void *string) {
+    /* This is the djb2 string hash function */
+
+    unsigned long result = 5381;
+    unsigned char *p;
+
+    p = (unsigned char *) string;
+
+    while (*p != '\0') {
+        result = ((result << 5) ^ result) ^ (*p);
+        ++p;
+    }
+
+    return result;
+}
+
+/* The same function, with a tolower on every character so that
+ * case is ignored.  This code is duplicated for performance. */
+
+unsigned long string_nocase_hash (void *string) {
+    unsigned long result = 5381;
+    unsigned char *p;
+
+    p = (unsigned char *) string;
+
+    while (*p != '\0') {
+        result = ((result << 5) ^ result) ^ tolower (*p);
+        ++p;
+    }
+
+    return result;
+}

data/ext/hash-string.h

@@ -0,0 +1,56 @@
+/*
+
+Copyright (c) 2005-2008, Simon Howard
+
+Permission to use, copy, modify, and/or distribute this software
+for any purpose with or without fee is hereby granted, provided
+that the above copyright notice and this permission notice appear
+in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
+WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
+AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR
+CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+ */
+
+/**
+ * @file hash-string.h
+ *
+ * Hash functions for text strings.  For more information
+ * see @ref string_hash or @ref string_nocase_hash.
+ */
+
+#ifndef ALGORITHM_HASH_STRING_H
+#define ALGORITHM_HASH_STRING_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Generate a hash key from a string.
+ *
+ * @param string           The string.
+ * @return                 A hash key for the string.
+ */
+
+    unsigned long string_hash (void *string);
+
+/**
+ * Generate a hash key from a string, ignoring the case of letters.
+ *
+ * @param string           The string.
+ * @return                 A hash key for the string.
+ */
+
+    unsigned long string_nocase_hash (void *string);
+
+#ifdef __cplusplus
+}
+#endif
+#endif                          /* #ifndef ALGORITHM_HASH_STRING_H */

data/ext/version.h

@@ -0,0 +1 @@
+#define RUBY_BLOOM_FILTER_VERSION "0.2.1"

data/lib/bloom-filter.rb

@@ -0,0 +1 @@
+require 'bloom_filter'

data/test/helper.rb

@@ -0,0 +1,4 @@
+require 'bloom-filter'
+require 'minitest/spec'
+
+MiniTest::Unit.autorun

data/test/test_basic.rb

@@ -0,0 +1,29 @@
+require 'helper'
+
+describe 'BloomFilter primitives' do
+  it 'should create one with default size' do
+    assert BloomFilter.new
+  end
+
+  it 'should create one with given size' do
+    assert BloomFilter.new(size: 100)
+  end
+
+  it 'should insert' do
+    assert filter = BloomFilter.new(size: 100)
+    assert filter.insert("foo")
+  end
+
+  it 'should allow membership checks' do
+    assert filter = BloomFilter.new(size: 100)
+    assert filter.insert("foo")
+    assert filter.include?("foo")
+    assert !filter.include?("bar")
+  end
+
+  it 'should return bits as a string' do
+    assert filter = BloomFilter.new(bits: 5, hashes: 2)
+    assert filter.insert("a")
+    assert_equal "10010", filter.bits
+  end
+end

data/test/test_io.rb

@@ -0,0 +1,27 @@
+require 'helper'
+require 'tempfile'
+
+describe 'BloomFilter load & dump' do
+  it 'should dump and load a filter' do
+    file = Tempfile.new("bloom-filter-test")
+
+    assert filter = BloomFilter.new(size: 100)
+    assert filter.insert("foo")
+    assert filter.dump(file.path), "dump filter"
+    assert filter = BloomFilter.load(file.path)
+    assert filter.include?("foo")
+    assert !filter.include?("bar")
+  end
+
+  it 'should accept assigning the bits directly' do
+    assert filter = BloomFilter.new(bits: 80)
+    assert filter.insert("foo")
+    puts filter.binary.inspect
+    assert filter2 = BloomFilter.new(bits: 80)
+    assert filter2.binary = filter.binary
+    puts
+    puts filter2.binary.inspect
+    assert filter2.include?("foo")
+    assert !filter2.include?("bar")
+  end
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: bloom-ruby
+version: !ruby/object:Gem::Version
+  version: 0.2.1
+  prerelease: 
+platform: ruby
+authors:
+- Bharanee Rathna
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-01-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A fast Bloom Filter library for Ruby for unices.
+email:
+- deepfryed@gmail.com
+executables: []
+extensions:
+- ext/extconf.rb
+extra_rdoc_files: []
+files:
+- ext/bloom-filter.c
+- ext/hash-string.c
+- ext/bloom_filter.c
+- ext/bloom-filter.h
+- ext/hash-string.h
+- ext/version.h
+- ext/extconf.rb
+- test/helper.rb
+- test/test_io.rb
+- test/test_basic.rb
+- README.md
+- CHANGELOG
+- lib/bloom-filter.rb
+homepage: http://github.com/deepfryed/bloom-filter
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: A fast Bloom Filter library for Ruby.
+test_files: []