bloom_fit 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 11be2e4492a3e06ff7c401ebe04e8f5b462a48d8399dea08ecdc33726cbec31f
+  data.tar.gz: 0642065d4004d002fc51cbe07f77a92daf5b84b117abc6684c45f7ca82b7d757
+SHA512:
+  metadata.gz: 840f467007a4efc4bcf55c4e29c320d28b16daa819b1f54a1884c43bb5092add5c4522a2e6e524a5516031e48bc937eb4bfddf9e20767d85b18590b1245c1d23
+  data.tar.gz: 58b45319a8cc83342ab224f0430bd8849b4655b2be0b1758d788dbc68df7ecca9cc5ab2deb1c33419e07dc6d0f6c8c22a4c158e7001fc696ee695273bb9ad196

data/README.md

@@ -0,0 +1,78 @@
+# BloomFit makes Bloom Filter tuning easy
+
+BloomFit provides a MRI/C-based non-counting bloom filter for use in your Ruby projects. It is heavily based on [bloomfilter-rb]'s native implementation, but provides a better hashing distribution by using DJB2 over CRC32, avoids the need to supply a seed, removes counting abilities, improves performance for very large datasets, and will automatically calculate the bit size (m) and the number of hashes (k) when given a capacity and false-positive-rate.
+
+A [Bloom filter](http://en.wikipedia.org/wiki/Bloom_filter) is a space-efficient probabilistic data structure that is used to test whether an element is a member of a set. False positives are possible, but false negatives are not. Instead of using k different hash functions, this implementation a DJB2 hash with k seeds from the CRC table.
+
+Performance of the Bloom filter depends on the following:
+
+- size of the bit array
+- number of hash functions
+
+BloomFit is a fork of [bloomfilter-rb].
+
+## Resources
+
+- Background: [Bloom filter](http://en.wikipedia.org/wiki/Bloom_filter)
+- Determining parameters: [Scalable Datasets: Bloom Filters in Ruby](http://www.igvita.com/2008/12/27/scalable-datasets-bloom-filters-in-ruby/)
+- Applications & reasons behind bloom filter: [Flow analysis: Time based bloom filter](http://www.igvita.com/2010/01/06/flow-analysis-time-based-bloom-filters/)
+
+## Examples
+
+MRI/C implementation which creates an in-memory filter which can be saved and reloaded from disk.
+
+(COMING SOON) If you'd like to specify an expected item count and a false-positive rate that you can tolerate:
+
+```ruby
+require "bloom_fit"
+
+bf = BloomFit.new(capacity: 250, false_positive_rate: 0.001)
+bf.add("cat")
+bf.include?("cat")     # => true
+bf.include?("dog")     # => false
+
+# Hash syntax with a bloom filter!
+bf["bird"] = "bar"
+bf["bird"]             # => true
+bf["mouse"]            # => false
+
+bf.stats
+# => Number of filter bits (m): 3600
+# => Number of set bits (n): 20
+# => Number of filter hashes (k) : 10
+# => Predicted false positive rate = 0.00%
+```
+
+If you'd like more control over the traditional inputs like bit size and the number of hashes:
+
+```ruby
+require "bloom_fit"
+
+bf = BloomFit.new(size: 100, hashes: 2)
+bf.add("cat")
+bf.include?("cat")     # => true
+bf.include?("dog")     # => false
+
+# Hash syntax with a bloom filter!
+bf["bird"] = "bar"
+bf["bird"]             # => true
+bf["mouse"]            # => false
+
+bf.stats
+# => Number of filter bits (m): 100
+# => Number of set bits (n): 4
+# => Number of filter hashes (k) : 2
+# => Predicted false positive rate = 10.87%
+```
+
+## Credits
+
+- Tatsuya Mori <valdzone@gmail.com> (Original C implementation)
+- Ilya Grigorik [@igrigorik](https://github.com/igrigorik) ([bloomfilter-rb] gem)
+- Bharanee Rathna [@deepfryed](https://github.com/deepfryed) ([bloom-filter](https://github.com/deepfryed/bloom-filter) gem)
+
+## License
+
+[MIT License](https://rmm5t.mit-license.org/)
+
+[bloomfilter-rb]: https://github.com/igrigorik/bloomfilter-rb

data/Rakefile

@@ -0,0 +1,14 @@
+require 'bundler/gem_tasks'
+require 'rake'
+require 'rspec'
+require 'rspec/core/rake_task'
+require 'rake/extensiontask'
+
+Bundler::GemHelper.install_tasks
+Rake::ExtensionTask.new('cbloomfilter')
+RSpec::Core::RakeTask.new(:spec)
+Rake::Task[:spec].prerequisites << :clean
+Rake::Task[:spec].prerequisites << :compile
+
+desc "Default: run unit tests."
+task default: :spec

data/ext/cbloomfilter/cbloomfilter.c

@@ -0,0 +1,397 @@
+/*
+ *   cbloomfilter.c - simple Bloom Filter
+ *   (c) Tatsuya Mori <valdzone@gmail.com>
+ */
+
+#include "ruby.h"
+#include "crc32.h"
+
+#if !defined(RSTRING_LEN)
+# define RSTRING_LEN(x) (RSTRING(x)->len)
+# define RSTRING_PTR(x) (RSTRING(x)->ptr)
+#endif
+
+/* Reuse the standard CRC table for consistent seeds */
+static unsigned int *seeds = crc_table;
+
+static VALUE cBloomFilter;
+
+struct BloomFilter {
+    int m; /* # of buckets in a bloom filter */
+    int b; /* # of bits in a bloom filter bucket */
+    int k; /* # of hash functions */
+    int r; /* # raise on bucket overflow? */
+    unsigned char *ptr; /* bits data */
+    int bytes; /* size of byte data */
+};
+
+unsigned long djb2(unsigned char *str, int len) {
+    unsigned long hash = 5381;
+    unsigned char *c;
+    c = (unsigned char *) str;
+    while (len > 0) {
+        hash = ((hash << 5) ^ hash) ^ (*c);
+        --len;
+        ++c;
+    }
+    return hash;
+}
+
+void bits_free(struct BloomFilter *bf) {
+    ruby_xfree(bf->ptr);
+}
+
+void bucket_unset(struct BloomFilter *bf, int index) {
+    int byte_offset = (index * bf->b) / 8;
+    int bit_offset = (index * bf->b) % 8;
+    unsigned int c = bf->ptr[byte_offset];
+    c += bf->ptr[byte_offset + 1] << 8;
+    unsigned int mask = ((1 << bf->b) - 1) << bit_offset;
+    if ((c & mask) == 0) {
+      // do nothing
+    } else {
+        // reduce the counter: 11 00 => 10 00 (suppose bf->b is 2)
+        c -= (1 << bit_offset) & ((1 << 8) -1);
+        // shift the bitmap right by 1 bit: 10 00 => 01 00
+        c = (~mask & c) | ((c & mask) >> (bit_offset + 1) << bit_offset);
+
+        bf->ptr[byte_offset] = c & ((1 << 8) - 1);
+        bf->ptr[byte_offset + 1] = (c & ((1 << 16) - 1)) >> 8;
+    }
+}
+
+void bucket_set(struct BloomFilter *bf, int index) {
+    int byte_offset = (index * bf->b) / 8;
+    int bit_offset = (index * bf->b) % 8;
+    unsigned int c = bf->ptr[byte_offset];
+    c += bf->ptr[byte_offset + 1] << 8;
+    unsigned int mask = ((1 << bf->b) - 1) << bit_offset;
+    if ((c & mask) == mask) {
+        if (bf->r == 1) rb_raise(rb_eRuntimeError, "bucket got filled up");
+    } else {
+        c = c + ((1 << bit_offset) & ((1 << 8) -1)) | c;
+        bf->ptr[byte_offset] = c & ((1 << 8) - 1);
+        bf->ptr[byte_offset + 1] = (c & ((1 << 16) - 1)) >> 8;
+    }
+}
+
+int bucket_check(struct BloomFilter *bf, int index) {
+    int byte_offset = (index * bf->b) / 8;
+    int bit_offset = (index * bf->b) % 8;
+    unsigned int c = bf->ptr[byte_offset];
+    c += bf->ptr[byte_offset + 1] << 8;
+
+    unsigned int mask = ((1 << bf->b) - 1) << bit_offset;
+    return (c & mask) >> bit_offset;
+}
+
+int bucket_get(struct BloomFilter *bf, int index) {
+    int byte_offset = (index * bf->b) / 8;
+    int bit_offset = (index * bf->b) % 8;
+    unsigned int c = bf->ptr[byte_offset];
+    c += bf->ptr[byte_offset + 1] << 8;
+
+    unsigned int mask = ((1 << bf->b) - 1) << bit_offset;
+    return (c & mask) >> bit_offset;
+}
+
+static VALUE bf_s_new(int argc, VALUE *argv, VALUE self) {
+    struct BloomFilter *bf;
+    VALUE arg1, arg2, arg3, arg4, obj;
+    int m, k, b, r;
+
+    obj = Data_Make_Struct(self, struct BloomFilter, NULL, bits_free, bf);
+
+    /* default = Fugou approach :-) */
+    arg1 = INT2FIX(100000000);
+    arg2 = INT2FIX(4);
+    arg3 = INT2FIX(1);
+    arg4 = INT2FIX(0);
+
+    switch (argc) {
+        case 4:
+      if (argv[3] == Qtrue) {
+        arg4 = INT2FIX(1);
+      }
+        case 3:
+      arg3 = argv[2];
+        case 2:
+      arg2 = argv[1];
+        case 1:
+      arg1 = argv[0];
+      break;
+    }
+
+    m = FIX2INT(arg1);
+    k = FIX2INT(arg2);
+    b = FIX2INT(arg3);
+    r = FIX2INT(arg4);
+
+    if (b < 1 || b > 8)
+        rb_raise(rb_eArgError, "bucket size");
+    if (m < 1)
+        rb_raise(rb_eArgError, "array size");
+    if (k < 1)
+        rb_raise(rb_eArgError, "hash length");
+
+    bf->b = b;
+    bf->m = m;
+    bf->k = k;
+    bf->r = r;
+
+    bf->bytes = ((m * b) + 15) / 8;
+    bf->ptr = ALLOC_N(unsigned char, bf->bytes);
+
+    /* initialize the bits with zeros */
+    memset(bf->ptr, 0, bf->bytes);
+    rb_iv_set(obj, "@hash_value", rb_hash_new());
+
+    return obj;
+}
+
+static VALUE bf_clear(VALUE self) {
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    memset(bf->ptr, 0, bf->bytes);
+    return Qtrue;
+}
+
+static VALUE bf_m(VALUE self) {
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    return INT2FIX(bf->m);
+}
+
+static VALUE bf_k(VALUE self) {
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    return INT2FIX(bf->k);
+}
+
+static VALUE bf_b(VALUE self) {
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    return INT2FIX(bf->b);
+}
+
+static VALUE bf_r(VALUE self) {
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    return bf->r == 0 ? Qfalse : Qtrue;
+}
+
+static VALUE bf_set_bits(VALUE self){
+    struct BloomFilter *bf;
+    int i,j,count = 0;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    for (i = 0; i < bf->bytes; i++) {
+        for (j = 0; j < 8; j++) {
+            count += (bf->ptr[i] >> j) & 1;
+        }
+    }
+    return INT2FIX(count);
+}
+
+static VALUE bf_insert(VALUE self, VALUE key) {
+    VALUE skey;
+    unsigned long hash, index;
+    int i, len, m, k;
+    char *ckey;
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+
+    skey = rb_obj_as_string(key);
+    ckey = StringValuePtr(skey);
+    len = (int) (RSTRING_LEN(skey)); /* length of the string in bytes */
+
+    m = bf->m;
+    k = bf->k;
+
+    hash = (unsigned long) djb2(ckey, len);
+    for (i = 0; i <= k - 1; i++) {
+        index = (unsigned long) (hash ^ seeds[i]) % (unsigned int) (m);
+
+        /*  set a bit at the index */
+        bucket_set(bf, index);
+    }
+
+    return Qnil;
+}
+
+static VALUE bf_merge(VALUE self, VALUE other) {
+    struct BloomFilter *bf, *target;
+    int i;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    Data_Get_Struct(other, struct BloomFilter, target);
+    for (i = 0; i < bf->bytes; i++) {
+        bf->ptr[i] |= target->ptr[i];
+    }
+    return Qnil;
+}
+
+static VALUE bf_and(VALUE self, VALUE other) {
+    struct BloomFilter *bf, *bf_other, *target;
+    VALUE klass, obj, args[5];
+    int i;
+
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    Data_Get_Struct(other, struct BloomFilter, bf_other);
+    args[0] = INT2FIX(bf->m);
+    args[1] = INT2FIX(bf->k);
+    args[2] = INT2FIX(bf->b);
+    args[3] = INT2FIX(bf->r);
+    klass = rb_funcall(self,rb_intern("class"),0);
+    obj = bf_s_new(4,args,klass);
+    Data_Get_Struct(obj, struct BloomFilter, target);
+    for (i = 0; i < bf->bytes; i++){
+        target->ptr[i] = bf->ptr[i] & bf_other->ptr[i];
+    }
+
+    return obj;
+}
+
+static VALUE bf_or(VALUE self, VALUE other) {
+    struct BloomFilter *bf, *bf_other, *target;
+    VALUE klass, obj, args[5];
+    int i;
+
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    Data_Get_Struct(other, struct BloomFilter, bf_other);
+    args[0] = INT2FIX(bf->m);
+    args[1] = INT2FIX(bf->k);
+    args[2] = INT2FIX(bf->b);
+    args[3] = INT2FIX(bf->r);
+    klass = rb_funcall(self,rb_intern("class"),0);
+    obj = bf_s_new(4,args,klass);
+    Data_Get_Struct(obj, struct BloomFilter, target);
+    for (i = 0; i < bf->bytes; i++){
+        target->ptr[i] = bf->ptr[i] | bf_other->ptr[i];
+    }
+
+    return obj;
+}
+
+static VALUE bf_delete(VALUE self, VALUE key) {
+    unsigned long hash, index;
+    int i, len, m, k;
+    char *ckey;
+    VALUE skey;
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+
+    skey = rb_obj_as_string(key);
+    ckey = StringValuePtr(skey);
+    len = (int) (RSTRING_LEN(skey)); /* length of the string in bytes */
+
+    m = bf->m;
+    k = bf->k;
+
+    hash = (unsigned long) djb2(ckey, len);
+    for (i = 0; i <= k - 1; i++) {
+        index = (unsigned long) (hash ^ seeds[i]) % (unsigned int) (m);
+
+        /*  set a bit at the index */
+        bucket_unset(bf, index);
+    }
+
+    return Qnil;
+}
+
+
+static VALUE bf_include(int argc, VALUE* argv, VALUE self) {
+    unsigned long hash, index;
+    int i, len, m, k, tests_idx, vlen;
+    char *ckey;
+    VALUE tests, key, skey;
+    struct BloomFilter *bf;
+
+    rb_scan_args(argc, argv, "*", &tests);
+
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    vlen = RARRAY_LEN(tests);
+    for(tests_idx = 0; tests_idx < vlen; tests_idx++) {
+      key = rb_ary_entry(tests, tests_idx);
+      skey = rb_obj_as_string(key);
+      ckey = StringValuePtr(skey);
+      len = (int) (RSTRING_LEN(skey)); /* length of the string in bytes */
+
+      m = bf->m;
+      k = bf->k;
+
+      hash = (unsigned long) djb2(ckey, len);
+      for (i = 0; i <= k - 1; i++) {
+          index = (unsigned long) (hash ^ seeds[i]) % (unsigned int) (m);
+
+          /* check the bit at the index */
+          if (!bucket_check(bf, index)) {
+              return Qfalse; /* i.e., it is a new entry ; escape the loop */
+          }
+      }
+
+      return Qtrue;
+    }
+}
+
+static VALUE bf_to_s(VALUE self) {
+    struct BloomFilter *bf;
+    unsigned char *ptr;
+    int i;
+    VALUE str;
+
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    str = rb_str_new(0, bf->m);
+
+    ptr = (unsigned char *) RSTRING_PTR(str);
+    for (i = 0; i < bf->m; i++)
+        *ptr++ = bucket_get(bf, i) ? '1' : '0';
+
+    return str;
+}
+
+static VALUE bf_bitmap(VALUE self) {
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+
+    VALUE str = rb_str_new(0, bf->bytes);
+    unsigned char* ptr = (unsigned char *) RSTRING_PTR(str);
+
+    memcpy(ptr, bf->ptr, bf->bytes);
+
+    return str;
+}
+
+static VALUE bf_load(VALUE self, VALUE bitmap) {
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    unsigned char* ptr = (unsigned char *) RSTRING_PTR(bitmap);
+
+    memcpy(bf->ptr, ptr, bf->bytes);
+
+    return Qnil;
+}
+
+void Init_cbloomfilter(void) {
+    cBloomFilter = rb_define_class("CBloomFilter", rb_cObject);
+    rb_define_singleton_method(cBloomFilter, "new", bf_s_new, -1);
+    rb_define_method(cBloomFilter, "m", bf_m, 0);
+    rb_define_method(cBloomFilter, "k", bf_k, 0);
+    rb_define_method(cBloomFilter, "b", bf_b, 0);
+    rb_define_method(cBloomFilter, "r", bf_r, 0);
+    rb_define_method(cBloomFilter, "set_bits", bf_set_bits, 0);
+    /* rb_define_method(cBloomFilter, "s", bf_s, 0); */
+    rb_define_method(cBloomFilter, "insert", bf_insert, 1);
+    rb_define_method(cBloomFilter, "delete", bf_delete, 1);
+    rb_define_method(cBloomFilter, "include?", bf_include, -1);
+    rb_define_method(cBloomFilter, "clear", bf_clear, 0);
+    rb_define_method(cBloomFilter, "merge!", bf_merge, 1);
+    rb_define_method(cBloomFilter, "&", bf_and, 1);
+    rb_define_method(cBloomFilter, "|", bf_or, 1);
+
+    rb_define_method(cBloomFilter, "to_s", bf_to_s, 0);
+    rb_define_method(cBloomFilter, "bitmap", bf_bitmap, 0);
+    rb_define_method(cBloomFilter, "load", bf_load, 1);
+
+    /* functions that have not been implemented, yet */
+
+    //  rb_define_method(cBloomFilter, "<=>", bf_cmp, 1);
+}

data/ext/cbloomfilter/crc32.h

@@ -0,0 +1,76 @@
+/* simple CRC32 code */
+/*
+ * Copyright 2005 Aris Adamantiadis
+ *
+ * This file is part of the SSH Library
+ *
+ * The SSH Library is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at your
+ * option) any later version.
+ *
+ *
+ * The SSH Library is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the SSH Library; see the file COPYING.  If not, write to
+ * the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
+ * MA 02111-1307, USA. */
+
+static unsigned int crc_table[] = {
+    0x00000000UL, 0x77073096UL, 0xee0e612cUL, 0x990951baUL, 0x076dc419UL,
+    0x706af48fUL, 0xe963a535UL, 0x9e6495a3UL, 0x0edb8832UL, 0x79dcb8a4UL,
+    0xe0d5e91eUL, 0x97d2d988UL, 0x09b64c2bUL, 0x7eb17cbdUL, 0xe7b82d07UL,
+    0x90bf1d91UL, 0x1db71064UL, 0x6ab020f2UL, 0xf3b97148UL, 0x84be41deUL,
+    0x1adad47dUL, 0x6ddde4ebUL, 0xf4d4b551UL, 0x83d385c7UL, 0x136c9856UL,
+    0x646ba8c0UL, 0xfd62f97aUL, 0x8a65c9ecUL, 0x14015c4fUL, 0x63066cd9UL,
+    0xfa0f3d63UL, 0x8d080df5UL, 0x3b6e20c8UL, 0x4c69105eUL, 0xd56041e4UL,
+    0xa2677172UL, 0x3c03e4d1UL, 0x4b04d447UL, 0xd20d85fdUL, 0xa50ab56bUL,
+    0x35b5a8faUL, 0x42b2986cUL, 0xdbbbc9d6UL, 0xacbcf940UL, 0x32d86ce3UL,
+    0x45df5c75UL, 0xdcd60dcfUL, 0xabd13d59UL, 0x26d930acUL, 0x51de003aUL,
+    0xc8d75180UL, 0xbfd06116UL, 0x21b4f4b5UL, 0x56b3c423UL, 0xcfba9599UL,
+    0xb8bda50fUL, 0x2802b89eUL, 0x5f058808UL, 0xc60cd9b2UL, 0xb10be924UL,
+    0x2f6f7c87UL, 0x58684c11UL, 0xc1611dabUL, 0xb6662d3dUL, 0x76dc4190UL,
+    0x01db7106UL, 0x98d220bcUL, 0xefd5102aUL, 0x71b18589UL, 0x06b6b51fUL,
+    0x9fbfe4a5UL, 0xe8b8d433UL, 0x7807c9a2UL, 0x0f00f934UL, 0x9609a88eUL,
+    0xe10e9818UL, 0x7f6a0dbbUL, 0x086d3d2dUL, 0x91646c97UL, 0xe6635c01UL,
+    0x6b6b51f4UL, 0x1c6c6162UL, 0x856530d8UL, 0xf262004eUL, 0x6c0695edUL,
+    0x1b01a57bUL, 0x8208f4c1UL, 0xf50fc457UL, 0x65b0d9c6UL, 0x12b7e950UL,
+    0x8bbeb8eaUL, 0xfcb9887cUL, 0x62dd1ddfUL, 0x15da2d49UL, 0x8cd37cf3UL,
+    0xfbd44c65UL, 0x4db26158UL, 0x3ab551ceUL, 0xa3bc0074UL, 0xd4bb30e2UL,
+    0x4adfa541UL, 0x3dd895d7UL, 0xa4d1c46dUL, 0xd3d6f4fbUL, 0x4369e96aUL,
+    0x346ed9fcUL, 0xad678846UL, 0xda60b8d0UL, 0x44042d73UL, 0x33031de5UL,
+    0xaa0a4c5fUL, 0xdd0d7cc9UL, 0x5005713cUL, 0x270241aaUL, 0xbe0b1010UL,
+    0xc90c2086UL, 0x5768b525UL, 0x206f85b3UL, 0xb966d409UL, 0xce61e49fUL,
+    0x5edef90eUL, 0x29d9c998UL, 0xb0d09822UL, 0xc7d7a8b4UL, 0x59b33d17UL,
+    0x2eb40d81UL, 0xb7bd5c3bUL, 0xc0ba6cadUL, 0xedb88320UL, 0x9abfb3b6UL,
+    0x03b6e20cUL, 0x74b1d29aUL, 0xead54739UL, 0x9dd277afUL, 0x04db2615UL,
+    0x73dc1683UL, 0xe3630b12UL, 0x94643b84UL, 0x0d6d6a3eUL, 0x7a6a5aa8UL,
+    0xe40ecf0bUL, 0x9309ff9dUL, 0x0a00ae27UL, 0x7d079eb1UL, 0xf00f9344UL,
+    0x8708a3d2UL, 0x1e01f268UL, 0x6906c2feUL, 0xf762575dUL, 0x806567cbUL,
+    0x196c3671UL, 0x6e6b06e7UL, 0xfed41b76UL, 0x89d32be0UL, 0x10da7a5aUL,
+    0x67dd4accUL, 0xf9b9df6fUL, 0x8ebeeff9UL, 0x17b7be43UL, 0x60b08ed5UL,
+    0xd6d6a3e8UL, 0xa1d1937eUL, 0x38d8c2c4UL, 0x4fdff252UL, 0xd1bb67f1UL,
+    0xa6bc5767UL, 0x3fb506ddUL, 0x48b2364bUL, 0xd80d2bdaUL, 0xaf0a1b4cUL,
+    0x36034af6UL, 0x41047a60UL, 0xdf60efc3UL, 0xa867df55UL, 0x316e8eefUL,
+    0x4669be79UL, 0xcb61b38cUL, 0xbc66831aUL, 0x256fd2a0UL, 0x5268e236UL,
+    0xcc0c7795UL, 0xbb0b4703UL, 0x220216b9UL, 0x5505262fUL, 0xc5ba3bbeUL,
+    0xb2bd0b28UL, 0x2bb45a92UL, 0x5cb36a04UL, 0xc2d7ffa7UL, 0xb5d0cf31UL,
+    0x2cd99e8bUL, 0x5bdeae1dUL, 0x9b64c2b0UL, 0xec63f226UL, 0x756aa39cUL,
+    0x026d930aUL, 0x9c0906a9UL, 0xeb0e363fUL, 0x72076785UL, 0x05005713UL,
+    0x95bf4a82UL, 0xe2b87a14UL, 0x7bb12baeUL, 0x0cb61b38UL, 0x92d28e9bUL,
+    0xe5d5be0dUL, 0x7cdcefb7UL, 0x0bdbdf21UL, 0x86d3d2d4UL, 0xf1d4e242UL,
+    0x68ddb3f8UL, 0x1fda836eUL, 0x81be16cdUL, 0xf6b9265bUL, 0x6fb077e1UL,
+    0x18b74777UL, 0x88085ae6UL, 0xff0f6a70UL, 0x66063bcaUL, 0x11010b5cUL,
+    0x8f659effUL, 0xf862ae69UL, 0x616bffd3UL, 0x166ccf45UL, 0xa00ae278UL,
+    0xd70dd2eeUL, 0x4e048354UL, 0x3903b3c2UL, 0xa7672661UL, 0xd06016f7UL,
+    0x4969474dUL, 0x3e6e77dbUL, 0xaed16a4aUL, 0xd9d65adcUL, 0x40df0b66UL,
+    0x37d83bf0UL, 0xa9bcae53UL, 0xdebb9ec5UL, 0x47b2cf7fUL, 0x30b5ffe9UL,
+    0xbdbdf21cUL, 0xcabac28aUL, 0x53b39330UL, 0x24b4a3a6UL, 0xbad03605UL,
+    0xcdd70693UL, 0x54de5729UL, 0x23d967bfUL, 0xb3667a2eUL, 0xc4614ab8UL,
+    0x5d681b02UL, 0x2a6f2b94UL, 0xb40bbe37UL, 0xc30c8ea1UL, 0x5a05df1bUL,
+    0x2d02ef8dUL
+};

data/ext/cbloomfilter/extconf.rb

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby 
+require "mkmf"
+
+create_makefile("cbloomfilter")

data/lib/bloom_fit/version.rb

@@ -0,0 +1,3 @@
+class BloomFit
+  VERSION = "0.1.0".freeze
+end

data/lib/bloom_fit.rb

@@ -0,0 +1,107 @@
+require 'cbloomfilter'
+require 'bloom_fit/version'
+
+class BloomFit
+  BloomFit::ConfigurationMismatch = Class.new(ArgumentError)
+
+  attr_reader :bf
+
+  def initialize(opts = {})
+    @opts = {
+      :size    => 100,
+      :hashes  => 4,
+      :bucket  => 1,
+      :raise   => false
+    }.merge(opts)
+
+    # arg 1: m => size : number of buckets in a bloom filter
+    # arg 2: k => hashes : number of hash functions
+    # arg 3: b => bucket : number of bits per bucket
+    # arg 4: r => raise : whether to raise on bucket overflow
+
+    @bf = CBloomFilter.new(@opts[:size], @opts[:hashes], @opts[:bucket], @opts[:raise])
+  end
+
+  def insert(key)
+    @bf.insert(key)
+  end
+  alias :[]= :insert
+
+  def include?(*keys)
+    @bf.include?(*keys)
+  end
+  alias :key? :include?
+  alias :[] :include?
+
+  def delete(key); @bf.delete(key); end
+  def clear; @bf.clear; end
+  def size; @bf.set_bits; end
+  def merge!(o); @bf.merge!(o.bf); end
+
+  # Returns the number of bits that are set to 1 in the filter.
+  def set_bits
+    @bf.set_bits
+  end
+
+  # Computes the intersection of two Bloom filters.
+  # It assumes that both filters have the same size -
+  # if this is not true +BloomFit::ConfigurationMismatch+ is raised.
+  def &(o)
+    raise BloomFit::ConfigurationMismatch.new unless same_parameters?(o)
+    result = self.class.new
+    result.instance_variable_set(:@bf,@bf.&(o.bf))
+    result
+  end
+
+  # Computes the union of two Bloom filters.
+  # It assumes that both filters have the same size -
+  # if this is not true +BloomFit::ConfigurationMismatch+ is raised.
+  def |(o)
+    raise BloomFit::ConfigurationMismatch.new unless same_parameters?(o)
+    result = self.class.new
+    result.instance_variable_set(:@bf,@bf.|(o.bf))
+    result
+  end
+
+  def bitmap
+    @bf.bitmap
+  end
+
+  def marshal_load(ary)
+    opts, bitmap = *ary
+
+    initialize(opts)
+    @bf.load(bitmap) if !bitmap.nil?
+  end
+
+  def marshal_dump
+    [@opts, @bf.bitmap]
+  end
+
+  def self.load(filename)
+    Marshal.load(File.open(filename, 'r'))
+  end
+
+  def save(filename)
+    File.open(filename, 'w') do |f|
+      f << Marshal.dump(self)
+    end
+  end
+
+  def stats
+    fp = ((1.0 - Math.exp(-(@opts[:hashes] * size).to_f / @opts[:size])) ** @opts[:hashes]) * 100
+    printf "Number of filter buckets (m): %d\n", @opts[:size]
+    printf "Number of bits per buckets (b): %d\n", @opts[:bucket]
+    printf "Number of set bits (n): %d\n", set_bits
+    printf "Number of filter hashes (k) : %d\n", @opts[:hashes]
+    printf "Predicted false positive rate = %.2f%%\n", fp
+  end
+
+  protected
+
+  # Returns true if parameters of the +o+ther filter are
+  # the same.
+  def same_parameters?(o)
+    @bf.m == o.bf.m && @bf.k == o.bf.k && @bf.b == o.bf.b
+  end
+end

data/spec/bloom_fit_spec.rb

@@ -0,0 +1,152 @@
+require "helper"
+
+describe BloomFit do
+  it "should clear" do
+    bf = BloomFit.new(:size => 100, :hashes => 2, :bucket => 3, :raise => false)
+    bf.insert("test")
+    expect(bf.include?("test")).to be true
+    bf.clear
+    expect(bf.include?("test")).to be false
+  end
+
+  it "should merge" do
+    bf1 = BloomFit.new(:size => 100, :hashes => 2, :bucket => 3, :raise => false)
+    bf2 = BloomFit.new(:size => 100, :hashes => 2, :bucket => 3, :raise => false)
+    bf2.insert("test")
+    expect(bf1.include?("test")).to be false
+    bf1.merge!(bf2)
+    expect(bf1.include?("test")).to be true
+    expect(bf2.include?("test")).to be true
+  end
+
+  context "behave like a bloom filter" do
+    it "should test set membership" do
+      bf = BloomFit.new(:size => 100, :hashes => 2, :bucket => 3, :raise => false)
+      bf.insert("test")
+      bf.insert("test1")
+
+      expect(bf.include?("test")).to be true
+      expect(bf.include?("abcd")).to be false
+      expect(bf.include?("test", "test1")).to be true
+    end
+
+    it "should work with any object's to_s" do
+      subject.insert(:test)
+      subject.insert(:test1)
+      subject.insert(12345)
+
+      expect(subject.include?("test")).to be true
+      expect(subject.include?("abcd")).to be false
+      expect(subject.include?("test", "test1", '12345')).to be true
+    end
+
+    it "should return the number of bits set to 1" do
+      bf = BloomFit.new(:hashes => 4)
+      bf.insert("test")
+      expect(bf.set_bits).to be == 4
+      bf.delete("test")
+      expect(bf.set_bits).to be == 0
+
+      bf = BloomFit.new(:hashes => 1)
+      bf.insert("test")
+      expect(bf.set_bits).to be == 1
+    end
+
+    it "should return intersection with other filter" do
+      bf1 = BloomFit.new
+      bf1.insert("test")
+      bf1.insert("test1")
+
+      bf2 = BloomFit.new
+      bf2.insert("test")
+      bf2.insert("test2")
+
+      bf3 = bf1 & bf2
+      expect(bf3.include?("test")).to be true
+      expect(bf3.include?("test1")).to be false
+      expect(bf3.include?("test2")).to be false
+    end
+
+    it "should raise an exception when intersection is to be computed for incompatible filters" do
+      bf1 = BloomFit.new(:size => 10)
+      bf1.insert("test")
+
+      bf2 = BloomFit.new(:size => 20)
+      bf2.insert("test")
+
+      expect { bf1 & bf2 }.to raise_error(BloomFit::ConfigurationMismatch)
+    end
+
+    it "should return union with other filter" do
+      bf1 = BloomFit.new
+      bf1.insert("test")
+      bf1.insert("test1")
+
+      bf2 = BloomFit.new
+      bf2.insert("test")
+      bf2.insert("test2")
+
+      bf3 = bf1 | bf2
+      expect(bf3.include?("test")).to be true
+      expect(bf3.include?("test1")).to be true
+      expect(bf3.include?("test2")).to be true
+    end
+
+    it "should raise an exception when union is to be computed for incompatible filters" do
+      bf1 = BloomFit.new(:size => 10)
+      bf1.insert("test")
+
+      bf2 = BloomFit.new(:size => 20)
+      bf2.insert("test")
+
+      expect {bf1 | bf2}.to raise_error(BloomFit::ConfigurationMismatch)
+    end
+
+    it "should output current stats" do
+      subject.insert('test')
+      expect { subject.stats }.not_to raise_error
+    end
+  end
+
+  context "behave like counting bloom filter" do
+    it "should delete / decrement keys" do
+      subject.insert("test")
+      expect(subject.include?("test")).to be true
+
+      subject.delete("test")
+      expect(subject.include?("test")).to be false
+    end
+  end
+
+  context "serialize" do
+    after(:each) { File.unlink('bf.out') }
+
+    it "should marshall" do
+      bf = BloomFit.new
+      expect { bf.save('bf.out') }.not_to raise_error
+    end
+
+    it "should load from marshalled" do
+      subject.insert('foo')
+      subject.insert('bar')
+      subject.save('bf.out')
+
+      bf2 = BloomFit.load('bf.out')
+      expect(bf2.include?('foo')).to be true
+      expect(bf2.include?('bar')).to be true
+      expect(bf2.include?('baz')).to be false
+
+      expect(subject.send(:same_parameters?, bf2)).to be true
+    end
+
+    it "should serialize to a file size proporational its bucket size" do
+      fs_size = 0
+      8.times do |i|
+        bf = BloomFit.new(size: 10_000, bucket: i+1)
+        bf.save('bf.out')
+        prev_size, fs_size = fs_size, File.size('bf.out')
+        expect(prev_size).to be < fs_size
+      end
+    end
+  end
+end

data/spec/helper.rb

@@ -0,0 +1,2 @@
+require "bundler/setup"
+require "bloom_fit"

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: bloom_fit
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ilya Grigorik
+- Tatsuya Mori
+- Ryan McGeary
+- Beshad Talayeminaei
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: irb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3'
+email:
+- ilya@grigorik.com
+- valdzone@gmail.com
+- ryan@mcgeary.org
+- 'btalayeminaei@gmail.com '
+executables: []
+extensions:
+- ext/cbloomfilter/extconf.rb
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- ext/cbloomfilter/cbloomfilter.c
+- ext/cbloomfilter/crc32.h
+- ext/cbloomfilter/extconf.rb
+- lib/bloom_fit.rb
+- lib/bloom_fit/version.rb
+- spec/bloom_fit_spec.rb
+- spec/helper.rb
+homepage: https://github.com/rmm5t/bloom_fit
+licenses: []
+metadata:
+  homepage_uri: https://github.com/rmm5t/bloom_fit
+  bug_tracker_uri: https://github.com/rmm5t/bloom_fit/issues
+  changelog_uri: https://github.com/rmm5t/bloom_fit/blob/main/CHANGELOG.md
+  source_code_uri: https://github.com/rmm5t/bloom_fit
+  funding_uri: https://github.com/sponsors/rmm5t
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.9
+specification_version: 4
+summary: BloomFit helps you build correctly sized Bloom filters from expected set
+  size and target false positive rate.
+test_files:
+- spec/bloom_fit_spec.rb
+- spec/helper.rb