bloomfilter-rb 2.0.0

data/.gitignore

@@ -0,0 +1,4 @@
+*.o
+*.bundle
+*.swp
+ext/Makefile

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,29 @@
+PATH
+  remote: .
+  specs:
+    bloomfilter-rb (2.0.0)
+      redis (>= 2.1.1)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.2)
+    rake (0.8.7)
+    redis (2.1.1)
+    rspec (2.3.0)
+      rspec-core (~> 2.3.0)
+      rspec-expectations (~> 2.3.0)
+      rspec-mocks (~> 2.3.0)
+    rspec-core (2.3.1)
+    rspec-expectations (2.3.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bloomfilter-rb!
+  rake
+  redis (>= 2.1.1)
+  rspec

data/README.md

@@ -0,0 +1,87 @@
+# BloomFilter(s) in Ruby
+
+- Native (MRI/C) counting bloom filter
+- Redis-backed getbit/setbit non-counting bloom filter
+- Redis-backed set-based counting (+TTL) bloom filter
+
+Bloom filter is a space-efficient probabilistic data structure that is used to test whether an element is a member of a set. False positives are possible, but false negatives are not. For more detail, check the [wikipedia article](http://en.wikipedia.org/wiki/Bloom_filter). Instead of using k different hash functions, this implementation seeds the CRC32 hash with k different initial values (0, 1, ..., k-1). This may or may not give you a good distribution, it all depends on the data.
+
+Performance of the Bloom filter depends on a number of variables:
+
+- size of the bit array
+- size of the counter bucket
+- number of hash functions
+
+## Resources
+
+- Determining parameters: [Scalable Datasets: Bloom Filters in Ruby](http://www.igvita.com/2008/12/27/scalable-datasets-bloom-filters-in-ruby/)
+- Applications & reasons behind bloom filter: [Flow analysis: Time based bloom filter](http://www.igvita.com/2010/01/06/flow-analysis-time-based-bloom-filters/)
+
+***
+
+## MRI/C API Example
+
+MRI/C implementation which creates an in-memory filter which can be saved and reloaded from disk.
+
+    require 'bloomfilter'
+
+    bf = BloomFilter::Native.new(:size => 100, :hashes => 2, :seed => 1, :bucket => 3, :raise => false)
+    bf.insert("test")
+    bf.include?("test")     # => true
+    bf.include?("blah")     # => false
+
+    bf.delete("test")
+    bf.include?("test")     # => false
+
+    # Hash with a bloom filter!
+    bf["test2"] = "bar"
+    bf["test2"]             # => true
+    bf["test3"]             # => false
+
+    bf.stats
+    Number of filter bits (m): 10
+    Number of filter elements (n): 2
+    Number of filter hashes (k) : 2
+    Predicted false positive rate = 10.87%
+
+***
+
+## Redis-backed setbit/getbit bloom filter
+
+Uses [getbit](http://redis.io/commands/getbit)/[setbit](http://redis.io/commands/setbit) on Redis strings - efficient, fast, can be shared by multiple/concurrent processes.
+
+    bf = BloomFilter::Redis.new
+
+    bf.insert('test')
+    bf.include?('test')     # => true
+    bf.include?('blah')     # => false
+
+    bf.delete('test')
+    bf.include?('test')     # => false
+
+### Memory footprint
+
+- 1.0% error rate for 1M items, 10 bits/item: *2.5 mb*
+- 1.0% error rate for 150M items, 10 bits per item: *358.52 mb*
+- 0.1% error rate for 150M items, 15 bits per item: *537.33 mb*
+
+***
+
+## Redis-backed counting bloom filter with TTL's
+Uses regular Redis get/set counters to implement a counting filter with optional TTL expiry. Because each "bit" requires its own key in Redis, you do incur a much larger memory overhead.
+
+    bf = BloomFilter::CountingRedis.new(:ttl => 2)
+
+    bf.insert('test')
+    bf.include?('test')     # => true
+
+    sleep(2)
+    bf.include?('test')     # => false
+
+## Credits
+
+Tatsuya Mori <valdzone@gmail.com> (Original C implementation: http://vald.x0.com/sb/)
+
+## License
+
+(MIT License) - Copyright (c) 2011 Ilya Grigorik

data/Rakefile

@@ -0,0 +1,9 @@
+require 'rake'
+require 'rspec'
+require 'rspec/core/rake_task'
+require 'rake/extensiontask'
+require 'bundler'
+
+Bundler::GemHelper.install_tasks
+RSpec::Core::RakeTask.new(:spec)
+Rake::ExtensionTask.new('cbloomfilter')

data/bloomfilter-rb.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "bloomfilter/version"
+
+Gem::Specification.new do |s|
+  s.name        = "bloomfilter-rb"
+  s.version     = BloomFilter::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Ilya Grigorik", "Tatsuya Mori"]
+  s.email       = ["ilya@igvita.com"]
+  s.homepage    = "http://github.com/igrigorik/bloomfilter"
+  s.summary     = "Counting Bloom Filter implemented in Ruby"
+  s.description = s.summary
+  s.rubyforge_project = "bloomfilter-rb"
+
+  s.add_dependency "redis", ">= 2.1.1"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "rake"
+
+  s.extensions = ["ext/cbloomfilter/extconf.rb"]
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/examples/counting-redis.rb

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+require 'bloomfilter'
+
+bf = BloomFilter::CountingRedis.new(:ttl => 2, :server => {:host => 'localhost'})
+
+bf.insert('test')
+puts bf.include?('test')
+
+sleep(3)
+puts bf.include?('test')
+
+puts bf.stats

data/examples/pure-ruby-bf.rb

@@ -0,0 +1,52 @@
+#
+# Pure ruby implementation of a Bloom filter, just for kicks
+#
+
+require 'bitset'
+require 'zlib'
+
+class BloomFilter
+
+  def initialize(max_entries, num_hashes, seed)
+    @num_hashes = num_hashes
+    @size = max_entries.to_i
+    @bitmap = BitSet.new(@size)
+    @__mask = BitSet.new(@size)
+    @seed = seed
+  end
+
+  def insert(key)
+    mask = make_mask(key)
+    @bitmap |= mask
+  end
+
+  def new?(key)
+    mask = make_mask(key)
+    return ((@bitmap & mask) != mask);
+  end
+
+  def make_mask(key)
+    @__mask.clear
+    0.upto(@num_hashes.to_i - 1) do |i|
+      hash = Zlib.crc32(key, i + @seed)
+      @__mask.set(hash % @size, 1)
+    end
+    return @__mask
+  end
+end
+
+def main
+  bf = BloomFilter.new(1000000, 4, 0)
+  num = 0
+  while line = ARGF.gets
+    data = line.chop
+
+    if bf.new_entry?(data)
+      num += 1
+      bf.insert(data)
+    end
+  end
+  print "#element = #{num}\n"
+end
+
+main

data/examples/simple-native.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+require 'bloomfilter-rb'
+
+WORDS = %w(duck penguin bear panda)
+TEST = %w(penguin moose racooon)
+
+bf = BloomFilter::Native.new(:size => 100, :hashes => 2, :seed => 1, :bucket => 3, :raise => false)
+
+WORDS.each { |w| bf.insert(w) }
+TEST.each do |w|
+  puts "#{w}: #{bf.include?(w)}"
+end
+
+bf.stats
+
+#  penguin: true
+#  moose: false
+#  racooon: false
+#
+#  Number of filter buckets (m): 100
+#  Number of bits per buckets (b): 1
+#  Number of filter elements (n): 4
+#  Number of filter hashes (k) : 4
+#  Raise on overflow? (r) : false
+#  Predicted false positive rate = 0.05%

data/examples/simple-redis.rb

@@ -0,0 +1,31 @@
+require 'set'
+require 'lib/bloomfilter-rb'
+
+items = 1_000_00
+bits = 1
+
+# p BloomFilter::Redis.new(:size => items*bits, :hashes => 7) # 2.5 mb
+# p BloomFilter::Redis.new(:size => items*bits*5, :hashes => 7) # 13 mb
+# p BloomFilter::Redis.new(:size => items*bits*30, :hashes => 7) # 73 mb
+
+# 1%   error rate for 5M items/day, 10 bits per item, for 30 days of data: 358.52 mb
+# 0.1% error rate for 5M items/day, 15 bits per item, for 30 days of data: 537.33 mb
+
+bf = BloomFilter::Redis.new(:size => items*bits, :hashes => 7) # 2.5 mb
+
+seen = Set.new
+err = 0
+num = 100000
+
+num.times do
+  item = rand(items)
+
+  if bf.include?(item) != seen.include?(item)
+    err += 1
+  end
+
+  seen << item
+  bf.insert(item)
+end
+
+p [:error_rate, (err.to_f / num) * 100]

data/ext/cbloomfilter/cbloomfilter.c

@@ -0,0 +1,359 @@
+/*
+ *   cbloomfilter.c - simple Bloom Filter
+ *   (c) Tatsuya Mori <valdzone@gmail.com>
+ */
+
+#include "ruby.h"
+#include "crc32.h"
+
+#if !defined(RSTRING_LEN)
+# define RSTRING_LEN(x) (RSTRING(x)->len)
+# define RSTRING_PTR(x) (RSTRING(x)->ptr)
+#endif
+
+static VALUE cBloomFilter;
+
+struct BloomFilter {
+    int m; /* # of buckets in a bloom filter */
+    int b; /* # of bits in a bloom filter bucket */
+    int k; /* # of hash functions */
+    int s; /* # seed of hash functions */
+    int r; /* # raise on bucket overflow? */
+    int num_set; /* # of set bits */
+    unsigned char *ptr; /* bits data */
+    int bytes; /* size of byte data */
+};
+
+void bits_free(struct BloomFilter *bf) {
+    ruby_xfree(bf->ptr);
+}
+
+void bucket_unset(struct BloomFilter *bf, int index) {
+    int byte_offset = (index * bf->b) / 8;
+    int bit_offset = (index * bf->b) % 8;
+    unsigned int c = bf->ptr[byte_offset];
+    c += bf->ptr[byte_offset + 1] << 8;
+    unsigned int mask = ((1 << bf->b) - 1) << bit_offset;
+    if ((c & mask) == 0) {
+      // do nothing
+    } else {
+        bf->ptr[byte_offset] -= (1 << bit_offset) & ((1 << 8) - 1);
+        bf->ptr[byte_offset + 1] -= ((1 << bit_offset) & ((1 << 16) - 1)) >> 8;
+    }
+
+}
+
+void bucket_set(struct BloomFilter *bf, int index) {
+    int byte_offset = (index * bf->b) / 8;
+    int bit_offset = (index * bf->b) % 8;
+    unsigned int c = bf->ptr[byte_offset];
+    c += bf->ptr[byte_offset + 1] << 8;
+    unsigned int mask = ((1 << bf->b) - 1) << bit_offset;
+    if ((c & mask) == mask) {
+        if (bf->r == 1) rb_raise(rb_eRuntimeError, "bucket got filled up");
+    } else {
+        bf->ptr[byte_offset] += (1 << bit_offset) & ((1 << 8) - 1);
+        bf->ptr[byte_offset + 1] += ((1 << bit_offset) & ((1 << 16) - 1)) >> 8;
+    }
+}
+
+int bucket_check(struct BloomFilter *bf, int index) {
+    int byte_offset = (index * bf->b) / 8;
+    int bit_offset = (index * bf->b) % 8;
+    unsigned int c = bf->ptr[byte_offset];
+    c += bf->ptr[byte_offset + 1] << 8;
+
+    unsigned int mask = ((1 << bf->b) - 1) << bit_offset;
+    return (c & mask) >> bit_offset;
+}
+
+int bucket_get(struct BloomFilter *bf, int index) {
+    int byte_offset = (index * bf->b) / 8;
+    int bit_offset = (index * bf->b) % 8;
+    unsigned int c = bf->ptr[byte_offset];
+    c += bf->ptr[byte_offset + 1] << 8;
+
+    unsigned int mask = ((1 << bf->b) - 1) << bit_offset;
+    return (c & mask) >> bit_offset;
+}
+
+static VALUE bf_s_new(int argc, VALUE *argv, VALUE self) {
+    struct BloomFilter *bf;
+    VALUE arg1, arg2, arg3, arg4, arg5, obj;
+    int m, k, s, b, r, bytes;
+
+    obj = Data_Make_Struct(self, struct BloomFilter, NULL, bits_free, bf);
+
+    /* default = Fugou approach :-) */
+    arg1 = INT2FIX(100000000);
+    arg2 = INT2FIX(4);
+    arg3 = INT2FIX(0);
+    arg4 = INT2FIX(1);
+    arg5 = INT2FIX(0);
+
+    switch (argc) {
+        case 5:
+      if (argv[4] == Qtrue) {
+        arg5 = INT2FIX(1);
+      }
+        case 4:
+      arg4 = argv[3];
+        case 3:
+      arg3 = argv[2];
+        case 2:
+      arg2 = argv[1];
+        case 1:
+      arg1 = argv[0];
+      break;
+    }
+
+    m = FIX2INT(arg1);
+    k = FIX2INT(arg2);
+    s = FIX2INT(arg3);
+    b = FIX2INT(arg4);
+    r = FIX2INT(arg5);
+
+    if (b < 1 || b > 8)
+        rb_raise(rb_eArgError, "bucket size");
+    if (m < 1)
+        rb_raise(rb_eArgError, "array size");
+    if (k < 1)
+        rb_raise(rb_eArgError, "hash length");
+    if (s < 0)
+        rb_raise(rb_eArgError, "random seed");
+
+    bf->b = b;
+    bf->m = m;
+    bf->k = k;
+    bf->s = s;
+    bf->r = r;
+    bf->num_set = 0;
+
+    bf->bytes = ((m * b) + 15) / 8;
+    bf->ptr = ALLOC_N(unsigned char, bf->bytes);
+
+    /* initialize the bits with zeros */
+    memset(bf->ptr, 0, bf->bytes);
+    rb_iv_set(obj, "@hash_value", rb_hash_new());
+
+    return obj;
+}
+
+static VALUE bf_clear(VALUE self) {
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    memset(bf->ptr, 0, bf->bytes);
+    return Qtrue;
+}
+
+static VALUE bf_m(VALUE self) {
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    return INT2FIX(bf->m);
+}
+
+static VALUE bf_k(VALUE self) {
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    return INT2FIX(bf->k);
+}
+
+static VALUE bf_b(VALUE self) {
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    return INT2FIX(bf->b);
+}
+
+static VALUE bf_r(VALUE self) {
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    return bf->r == 0 ? Qfalse : Qtrue;
+}
+
+static VALUE bf_num_set(VALUE self) {
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    return INT2FIX(bf->num_set);
+}
+
+static VALUE bf_insert(VALUE self, VALUE key) {
+    VALUE skey;
+    int index, seed;
+    int i, len, m, k, s;
+    char *ckey;
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+
+    skey = rb_obj_as_string(key);
+    ckey = StringValuePtr(skey);
+    len = (int) (RSTRING_LEN(skey)); /* length of the string in bytes */
+
+    m = bf->m;
+    k = bf->k;
+    s = bf->s;
+
+    for (i = 0; i <= k - 1; i++) {
+        /* seeds for hash functions */
+        seed = i + s;
+
+        /* hash */
+        index = (int) (crc32((unsigned int) (seed), ckey, len) % (unsigned int) (m));
+
+        /*  set a bit at the index */
+        bucket_set(bf, index);
+    }
+
+    bf->num_set += 1;
+    return Qnil;
+}
+
+static VALUE bf_merge(VALUE self, VALUE other) {
+    struct BloomFilter *bf, *target;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    Data_Get_Struct(other, struct BloomFilter, target);
+    int i;
+    for (i = 0; i < bf->bytes; i++) {
+        bf->ptr[i] |= target->ptr[i];
+    }
+    return Qnil;
+}
+
+static VALUE bf_delete(VALUE self, VALUE key) {
+    int index, seed;
+    int i, len, m, k, s;
+    char *ckey;
+    VALUE skey;
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+
+    skey = rb_obj_as_string(key);
+    ckey = StringValuePtr(skey);
+    len = (int) (RSTRING_LEN(skey)); /* length of the string in bytes */
+
+    m = bf->m;
+    k = bf->k;
+    s = bf->s;
+
+    for (i = 0; i <= k - 1; i++) {
+        /* seeds for hash functions */
+        seed = i + s;
+
+        /* hash */
+        index = (int) (crc32((unsigned int) (seed), ckey, len) % (unsigned int) (m));
+
+        /*  set a bit at the index */
+        bucket_unset(bf, index);
+    }
+
+    bf->num_set += 1;
+    return Qnil;
+}
+
+
+static VALUE bf_include(int argc, VALUE* argv, VALUE self) {
+    int index, seed;
+    int i, len, m, k, s, tests_idx, vlen;
+    char *ckey;
+    VALUE tests, key, skey;
+    struct BloomFilter *bf;
+
+    rb_scan_args(argc, argv, "*", &tests);
+
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    vlen = RARRAY_LEN(tests);
+    for(tests_idx = 0; tests_idx < vlen; tests_idx++) {
+      key = rb_ary_entry(tests, tests_idx);
+      skey = rb_obj_as_string(key);
+      ckey = StringValuePtr(skey);
+      len = (int) (RSTRING_LEN(skey)); /* length of the string in bytes */
+
+      m = bf->m;
+      k = bf->k;
+      s = bf->s;
+
+      for (i = 0; i <= k - 1; i++) {
+          /* seeds for hash functions */
+          seed = i + s;
+
+          /* hash */
+          index = (int) (crc32((unsigned int) (seed), ckey, len) % (unsigned int) (m));
+
+          /* check the bit at the index */
+          if (!bucket_check(bf, index)) {
+              return Qfalse; /* i.e., it is a new entry ; escape the loop */
+          }
+      }
+
+      return Qtrue;
+    }
+
+}
+
+static VALUE bf_to_s(VALUE self) {
+    struct BloomFilter *bf;
+    unsigned char *ptr;
+    int i;
+    VALUE str;
+
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    str = rb_str_new(0, bf->m);
+
+    ptr = (unsigned char *) RSTRING_PTR(str);
+    for (i = 0; i < bf->m; i++)
+        *ptr++ = bucket_get(bf, i) ? '1' : '0';
+
+    return str;
+}
+
+static VALUE bf_bitmap(VALUE self) {
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+
+    VALUE str = rb_str_new(0, bf->m);
+    unsigned char* ptr = (unsigned char *) RSTRING_PTR(str);
+
+    int i;
+    for (i = 0; i < bf->m; i++)
+        *ptr++ = bucket_get(bf, i);
+
+    return str;
+}
+
+static VALUE bf_load(VALUE self, VALUE bitmap) {
+    struct BloomFilter *bf;
+    Data_Get_Struct(self, struct BloomFilter, bf);
+    unsigned char* ptr = (unsigned char *) RSTRING_PTR(bitmap);
+
+    int i;
+    for (i = 0; i < bf->m; i++) {
+      if (*ptr++)
+        bucket_set(bf, i);
+    }
+
+    return Qnil;
+}
+
+void Init_cbloomfilter(void) {
+    cBloomFilter = rb_define_class("CBloomFilter", rb_cObject);
+    rb_define_singleton_method(cBloomFilter, "new", bf_s_new, -1);
+    rb_define_method(cBloomFilter, "m", bf_m, 0);
+    rb_define_method(cBloomFilter, "k", bf_k, 0);
+    rb_define_method(cBloomFilter, "b", bf_b, 0);
+    rb_define_method(cBloomFilter, "r", bf_r, 0);
+    rb_define_method(cBloomFilter, "num_set", bf_num_set, 0);
+    rb_define_method(cBloomFilter, "insert", bf_insert, 1);
+    rb_define_method(cBloomFilter, "delete", bf_delete, 1);
+    rb_define_method(cBloomFilter, "include?", bf_include, -1);
+    rb_define_method(cBloomFilter, "clear", bf_clear, 0);
+    rb_define_method(cBloomFilter, "merge!", bf_merge, 1);
+
+    rb_define_method(cBloomFilter, "to_s", bf_to_s, 0);
+    rb_define_method(cBloomFilter, "bitmap", bf_bitmap, 0);
+    rb_define_method(cBloomFilter, "load", bf_load, 1);
+
+    /* functions that have not been implemented, yet */
+
+    //  rb_define_method(cBloomFilter, "&", bf_and, 1);
+    //  rb_define_method(cBloomFilter, "|", bf_or, 1);
+    //  rb_define_method(cBloomFilter, "<=>", bf_cmp, 1);
+}

data/ext/cbloomfilter/crc32.c

@@ -0,0 +1,32 @@
+/* simple CRC32 code */
+/*
+ * Copyright 2005 Aris Adamantiadis
+ *
+ * This file is part of the SSH Library
+ *
+ * The SSH Library is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at your
+ * option) any later version.
+ *
+ *
+ * The SSH Library is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the SSH Library; see the file COPYING.  If not, write to
+ * the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
+ * MA 02111-1307, USA. */
+
+#include "crc32.h"
+
+unsigned int crc32(unsigned int crc, char *buf, int len) {
+    while (len > 0) {
+        crc = crc_table[(crc ^ *buf) & 0xff] ^ (crc >> 8);
+        --len;
+        ++buf;
+    }
+    return crc;
+}

data/ext/cbloomfilter/crc32.h

@@ -0,0 +1,78 @@
+/* simple CRC32 code */
+/*
+ * Copyright 2005 Aris Adamantiadis
+ *
+ * This file is part of the SSH Library
+ *
+ * The SSH Library is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at your
+ * option) any later version.
+ *
+ *
+ * The SSH Library is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the SSH Library; see the file COPYING.  If not, write to
+ * the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
+ * MA 02111-1307, USA. */
+
+static unsigned int crc_table[] = {
+    0x00000000UL, 0x77073096UL, 0xee0e612cUL, 0x990951baUL, 0x076dc419UL,
+    0x706af48fUL, 0xe963a535UL, 0x9e6495a3UL, 0x0edb8832UL, 0x79dcb8a4UL,
+    0xe0d5e91eUL, 0x97d2d988UL, 0x09b64c2bUL, 0x7eb17cbdUL, 0xe7b82d07UL,
+    0x90bf1d91UL, 0x1db71064UL, 0x6ab020f2UL, 0xf3b97148UL, 0x84be41deUL,
+    0x1adad47dUL, 0x6ddde4ebUL, 0xf4d4b551UL, 0x83d385c7UL, 0x136c9856UL,
+    0x646ba8c0UL, 0xfd62f97aUL, 0x8a65c9ecUL, 0x14015c4fUL, 0x63066cd9UL,
+    0xfa0f3d63UL, 0x8d080df5UL, 0x3b6e20c8UL, 0x4c69105eUL, 0xd56041e4UL,
+    0xa2677172UL, 0x3c03e4d1UL, 0x4b04d447UL, 0xd20d85fdUL, 0xa50ab56bUL,
+    0x35b5a8faUL, 0x42b2986cUL, 0xdbbbc9d6UL, 0xacbcf940UL, 0x32d86ce3UL,
+    0x45df5c75UL, 0xdcd60dcfUL, 0xabd13d59UL, 0x26d930acUL, 0x51de003aUL,
+    0xc8d75180UL, 0xbfd06116UL, 0x21b4f4b5UL, 0x56b3c423UL, 0xcfba9599UL,
+    0xb8bda50fUL, 0x2802b89eUL, 0x5f058808UL, 0xc60cd9b2UL, 0xb10be924UL,
+    0x2f6f7c87UL, 0x58684c11UL, 0xc1611dabUL, 0xb6662d3dUL, 0x76dc4190UL,
+    0x01db7106UL, 0x98d220bcUL, 0xefd5102aUL, 0x71b18589UL, 0x06b6b51fUL,
+    0x9fbfe4a5UL, 0xe8b8d433UL, 0x7807c9a2UL, 0x0f00f934UL, 0x9609a88eUL,
+    0xe10e9818UL, 0x7f6a0dbbUL, 0x086d3d2dUL, 0x91646c97UL, 0xe6635c01UL,
+    0x6b6b51f4UL, 0x1c6c6162UL, 0x856530d8UL, 0xf262004eUL, 0x6c0695edUL,
+    0x1b01a57bUL, 0x8208f4c1UL, 0xf50fc457UL, 0x65b0d9c6UL, 0x12b7e950UL,
+    0x8bbeb8eaUL, 0xfcb9887cUL, 0x62dd1ddfUL, 0x15da2d49UL, 0x8cd37cf3UL,
+    0xfbd44c65UL, 0x4db26158UL, 0x3ab551ceUL, 0xa3bc0074UL, 0xd4bb30e2UL,
+    0x4adfa541UL, 0x3dd895d7UL, 0xa4d1c46dUL, 0xd3d6f4fbUL, 0x4369e96aUL,
+    0x346ed9fcUL, 0xad678846UL, 0xda60b8d0UL, 0x44042d73UL, 0x33031de5UL,
+    0xaa0a4c5fUL, 0xdd0d7cc9UL, 0x5005713cUL, 0x270241aaUL, 0xbe0b1010UL,
+    0xc90c2086UL, 0x5768b525UL, 0x206f85b3UL, 0xb966d409UL, 0xce61e49fUL,
+    0x5edef90eUL, 0x29d9c998UL, 0xb0d09822UL, 0xc7d7a8b4UL, 0x59b33d17UL,
+    0x2eb40d81UL, 0xb7bd5c3bUL, 0xc0ba6cadUL, 0xedb88320UL, 0x9abfb3b6UL,
+    0x03b6e20cUL, 0x74b1d29aUL, 0xead54739UL, 0x9dd277afUL, 0x04db2615UL,
+    0x73dc1683UL, 0xe3630b12UL, 0x94643b84UL, 0x0d6d6a3eUL, 0x7a6a5aa8UL,
+    0xe40ecf0bUL, 0x9309ff9dUL, 0x0a00ae27UL, 0x7d079eb1UL, 0xf00f9344UL,
+    0x8708a3d2UL, 0x1e01f268UL, 0x6906c2feUL, 0xf762575dUL, 0x806567cbUL,
+    0x196c3671UL, 0x6e6b06e7UL, 0xfed41b76UL, 0x89d32be0UL, 0x10da7a5aUL,
+    0x67dd4accUL, 0xf9b9df6fUL, 0x8ebeeff9UL, 0x17b7be43UL, 0x60b08ed5UL,
+    0xd6d6a3e8UL, 0xa1d1937eUL, 0x38d8c2c4UL, 0x4fdff252UL, 0xd1bb67f1UL,
+    0xa6bc5767UL, 0x3fb506ddUL, 0x48b2364bUL, 0xd80d2bdaUL, 0xaf0a1b4cUL,
+    0x36034af6UL, 0x41047a60UL, 0xdf60efc3UL, 0xa867df55UL, 0x316e8eefUL,
+    0x4669be79UL, 0xcb61b38cUL, 0xbc66831aUL, 0x256fd2a0UL, 0x5268e236UL,
+    0xcc0c7795UL, 0xbb0b4703UL, 0x220216b9UL, 0x5505262fUL, 0xc5ba3bbeUL,
+    0xb2bd0b28UL, 0x2bb45a92UL, 0x5cb36a04UL, 0xc2d7ffa7UL, 0xb5d0cf31UL,
+    0x2cd99e8bUL, 0x5bdeae1dUL, 0x9b64c2b0UL, 0xec63f226UL, 0x756aa39cUL,
+    0x026d930aUL, 0x9c0906a9UL, 0xeb0e363fUL, 0x72076785UL, 0x05005713UL,
+    0x95bf4a82UL, 0xe2b87a14UL, 0x7bb12baeUL, 0x0cb61b38UL, 0x92d28e9bUL,
+    0xe5d5be0dUL, 0x7cdcefb7UL, 0x0bdbdf21UL, 0x86d3d2d4UL, 0xf1d4e242UL,
+    0x68ddb3f8UL, 0x1fda836eUL, 0x81be16cdUL, 0xf6b9265bUL, 0x6fb077e1UL,
+    0x18b74777UL, 0x88085ae6UL, 0xff0f6a70UL, 0x66063bcaUL, 0x11010b5cUL,
+    0x8f659effUL, 0xf862ae69UL, 0x616bffd3UL, 0x166ccf45UL, 0xa00ae278UL,
+    0xd70dd2eeUL, 0x4e048354UL, 0x3903b3c2UL, 0xa7672661UL, 0xd06016f7UL,
+    0x4969474dUL, 0x3e6e77dbUL, 0xaed16a4aUL, 0xd9d65adcUL, 0x40df0b66UL,
+    0x37d83bf0UL, 0xa9bcae53UL, 0xdebb9ec5UL, 0x47b2cf7fUL, 0x30b5ffe9UL,
+    0xbdbdf21cUL, 0xcabac28aUL, 0x53b39330UL, 0x24b4a3a6UL, 0xbad03605UL,
+    0xcdd70693UL, 0x54de5729UL, 0x23d967bfUL, 0xb3667a2eUL, 0xc4614ab8UL,
+    0x5d681b02UL, 0x2a6f2b94UL, 0xb40bbe37UL, 0xc30c8ea1UL, 0x5a05df1bUL,
+    0x2d02ef8dUL
+};
+
+unsigned int crc32(unsigned int crc, char *buf, int len);

data/ext/cbloomfilter/extconf.rb

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby 
+require "mkmf"
+
+create_makefile("cbloomfilter")

data/lib/bloomfilter-rb.rb

@@ -0,0 +1,9 @@
+require 'redis'
+require 'zlib'
+
+require 'cbloomfilter'
+require 'bloomfilter/filter'
+require 'bloomfilter/native'
+require 'bloomfilter/counting_redis'
+require 'bloomfilter/redis'
+require 'bloomfilter/version'

data/lib/bloomfilter/counting_redis.rb

@@ -0,0 +1,61 @@
+module BloomFilter
+  class CountingRedis < Filter
+
+    def initialize(opts = {})
+      @opts = {
+        :size    => 100,
+        :hashes  => 4,
+        :seed    => Time.now.to_i,
+        :bucket  => 3,
+        :ttl => false,
+        :server => {}
+      }.merge opts
+      @db = ::Redis.new(@opts[:server])
+    end
+
+    def insert(key, ttl=nil)
+      ttl = @opts[:ttl] if ttl.nil?
+
+      indexes_for(key).each do |idx|
+        @db.incr idx
+        @db.expire(idx, ttl) if ttl
+      end
+    end
+    alias :[]= :insert
+
+    def delete(key)
+      indexes_for(key).each do |idx|
+        if @db.decr(idx).to_i <= 0
+          @db.del(idx)
+        end
+      end
+    end
+
+    def include?(*keys)
+      indexes = keys.collect { |key| indexes_for(key) }
+      not @db.mget(*indexes.flatten).include? nil
+    end
+    alias :key? :include?
+
+    def num_set
+      @db.keys("rbloom:*").size
+    end
+    alias :size :num_set
+
+    def clear
+      @db.flushdb
+    end
+
+    private
+
+      # compute index offsets for provided key
+      def indexes_for(key)
+        indexes = []
+        @opts[:hashes].times do |i|
+          indexes.push "rbloom:" + (Zlib.crc32("#{key}:#{i+@opts[:seed]}") % @opts[:size]).to_s
+        end
+
+        indexes
+      end
+  end
+end

data/lib/bloomfilter/filter.rb

@@ -0,0 +1,13 @@
+module BloomFilter
+  class Filter
+    def stats
+      fp = ((1.0 - Math.exp(-(@opts[:hashes] * size).to_f / @opts[:size])) ** @opts[:hashes]) * 100
+      printf "Number of filter buckets (m): %d\n" % @opts[:size]
+      printf "Number of bits per buckets (b): %d\n" % @opts[:bucket]
+      printf "Number of filter elements (n): %d\n" % size
+      printf "Number of filter hashes (k) : %d\n" % @opts[:hashes]
+      printf "Raise on overflow? (r) : %s\n" % @opts[:raise].to_s
+      printf "Predicted false positive rate = %.2f%\n" % fp
+    end
+  end
+end

data/lib/bloomfilter/native.rb

@@ -0,0 +1,65 @@
+module BloomFilter
+  class Native < Filter
+    attr_reader :bf
+
+    def initialize(opts = {})
+      @opts = {
+        :size    => 100,
+        :hashes  => 4,
+        :seed    => Time.now.to_i,
+        :bucket  => 3,
+        :raise   => false
+      }.merge(opts)
+
+      # arg 1: m => size : number of buckets in a bloom filter
+      # arg 2: k => hashes : number of hash functions
+      # arg 3: s => seed : seed of hash functions
+      # arg 4: b => bucket : number of bits in a bloom filter bucket
+      # arg 5: r => raise : raise on bucket overflow?
+
+      @bf = CBloomFilter.new(@opts[:size], @opts[:hashes], @opts[:seed], @opts[:bucket], @opts[:raise])
+    end
+
+    def insert(key)
+      @bf.insert(key)
+    end
+    alias :[]= :insert
+
+    def include?(*keys)
+      @bf.include?(*keys)
+    end
+    alias :key? :include?
+    alias :[] :include?
+
+    def delete(key); @bf.delete(key); end
+    def clear; @bf.clear; end
+    def size; @bf.num_set; end
+    def merge!(o); @bf.merge!(o.bf); end
+
+    def bitmap
+      @bf.bitmap
+    end
+
+    def marshal_load(ary)
+      opts, bitmap = *ary
+
+      @bf = Native.new(opts)
+      @bf.bf.load(bitmap) if !bitmap.nil?
+    end
+
+    def marshal_dump
+      [@opts, @bf.bitmap]
+    end
+
+    def self.load(filename)
+      Marshal.load(File.open(filename, 'r'))
+    end
+
+    def save(filename)
+      File.open(filename, 'w') do |f|
+        f << Marshal.dump(self)
+      end
+    end
+
+  end
+end

data/lib/bloomfilter/redis.rb

@@ -0,0 +1,69 @@
+module BloomFilter
+  class Redis < Filter
+
+    def initialize(opts = {})
+      @opts = {
+        :size    => 100,
+        :hashes  => 4,
+        :seed    => Time.now.to_i,
+        :namespace => 'redis',
+        :eager  => true,
+        :server => {}
+      }.merge opts
+      @db = ::Redis.new(@opts[:server])
+
+      if @opts[:eager]
+        # allocate the memory immediately
+        @db.setbit @opts[:namespace], @opts[:size], 1
+        @db.setbit @opts[:namespace], @opts[:size], 0
+      end
+    end
+
+    def insert(key, ttl=nil)
+      indexes_for(key) { |idx| @db.setbit @opts[:namespace], idx, 1 }
+    end
+    alias :[]= :insert
+
+    def include?(*keys)
+      keys.each do |key|
+        indexes_for(key) do |idx|
+          return false if @db.getbit(@opts[:namespace], idx).zero?
+        end
+      end
+
+      true
+    end
+    alias :key? :include?
+
+    def delete(key)
+      indexes_for(key) do |idx|
+        @db.setbit @opts[:namespace], idx, 0
+      end
+    end
+
+    def clear
+      @db.set @opts[:namespace], 0
+    end
+
+    def num_set
+      @db.strlen @opts[:namespace]
+    end
+    alias :size :num_set
+
+    def stats
+      printf "Number of filter buckets (m): %d\n" % @opts[:size]
+      printf "Number of filter hashes (k) : %d\n" % @opts[:hashes]
+    end
+
+    private
+
+      # compute index offsets for provided key
+      def indexes_for(key)
+        indexes = []
+        @opts[:hashes].times do |i|
+          yield Zlib.crc32("#{key}:#{i+@opts[:seed]}") % @opts[:size]
+        end
+      end
+
+  end
+end

data/lib/bloomfilter/version.rb

@@ -0,0 +1,3 @@
+module BloomFilter
+  VERSION = "2.0.0"
+end

data/spec/counting_redis_spec.rb

@@ -0,0 +1,52 @@
+require 'helper'
+
+describe BloomFilter::CountingRedis do
+  include BloomFilter
+
+  context "use Redis for storage" do
+    it "should store data in Redis" do
+      bf = CountingRedis.new
+
+      bf.insert(:abcd)
+      bf.insert('test')
+      bf.include?('test').should be_true
+      bf.key?('test').should be_true
+
+      bf.include?('test', 'test2').should be_false
+      bf.include?('test', 'abcd').should be_true
+    end
+
+    it "should accept a TTL value for a key" do
+      bf = CountingRedis.new(:ttl => 1)
+
+      bf.insert('test')
+      bf.include?('test').should be_true
+
+      sleep(2)
+      bf.include?('test').should be_false
+    end
+
+    it "should delete keys from Redis" do
+      bf = CountingRedis.new
+
+      bf.insert('test')
+      bf.include?('test').should be_true
+
+      bf.delete('test')
+      bf.include?('test').should be_false
+    end
+
+    it "should output current stats" do
+      bf = CountingRedis.new
+      bf.clear
+
+      bf.insert('test')
+      bf.size.should == 4
+      lambda { bf.stats }.should_not raise_error
+    end
+
+    it "should connect to remote redis server" do
+      lambda { CountingRedis.new }.should_not raise_error
+    end
+  end
+end

data/spec/helper.rb

@@ -0,0 +1,2 @@
+require 'bundler/setup'
+require 'bloomfilter-rb'

data/spec/native_spec.rb

@@ -0,0 +1,79 @@
+require 'helper'
+
+describe BloomFilter::Native do
+  include BloomFilter
+
+  it "should clear" do
+    bf = Native.new(:size => 100, :hashes => 2, :seed => 1, :bucket => 3, :raise => false)
+    bf.insert("test")
+    bf.include?("test").should be_true
+    bf.clear
+    bf.include?("test").should be_false
+  end
+
+  it "should merge" do
+    bf1 = Native.new(:size => 100, :hashes => 2, :seed => 1, :bucket => 3, :raise => false)
+    bf2 = Native.new(:size => 100, :hashes => 2, :seed => 1, :bucket => 3, :raise => false)
+    bf2.insert("test")
+    bf1.include?("test").should be_false
+    bf1.merge!(bf2)
+    bf1.include?("test").should be_true
+    bf2.include?("test").should be_true
+  end
+
+  context "behave like a bloomfilter" do
+    it "should test set memerbship" do
+      bf = Native.new(:size => 100, :hashes => 2, :seed => 1, :bucket => 3, :raise => false)
+      bf.insert("test")
+      bf.insert("test1")
+
+      bf.include?("test").should be_true
+      bf.include?("abcd").should be_false
+      bf.include?("test", "test1").should be_true
+    end
+
+    it "should work with any object's to_s" do
+      bf = Native.new
+      bf.insert(:test)
+      bf.insert(:test1)
+      bf.insert(12345)
+
+      bf.include?("test").should be_true
+      bf.include?("abcd").should be_false
+      bf.include?("test", "test1", '12345').should be_true
+    end
+  end
+
+  context "behave like counting bloom filter" do
+    it "should delete / decrement keys" do
+      bf = Native.new
+
+      bf.insert("test")
+      bf.include?("test").should be_true
+
+      bf.delete("test")
+      bf.include?("test").should be_false
+    end
+  end
+
+  context "serialize" do
+    after(:each) { File.unlink('bf.out') }
+
+    it "should marshall the bloomfilter" do
+      bf = Native.new
+      lambda { bf.save('bf.out') }.should_not raise_error
+    end
+
+    it "should load marshalled bloomfilter" do
+      bf = Native.new
+      bf.insert('foo')
+      bf.insert('bar')
+      bf.save('bf.out')
+
+      bf = Native.load('bf.out')
+      bf.include?('foo').should be_true
+      bf.include?('bar').should be_true
+      bf.include?('baz').should be_false
+    end
+  end
+end

data/spec/redis_spec.rb

@@ -0,0 +1,54 @@
+require 'helper'
+
+describe BloomFilter::Redis do
+  include BloomFilter
+
+  context "use Redis bitstring for storage" do
+    let(:bf) { Redis.new }
+
+    it "should store data in Redis" do
+      bf.insert(:abcd)
+      bf.insert('test')
+      bf.include?('test').should be_true
+      bf.key?('test').should be_true
+
+      bf.include?('test', 'test2').should be_false
+      bf.include?('test', 'abcd').should be_true
+    end
+
+    it "should delete keys from Redis" do
+      bf.insert('test')
+      bf.include?('test').should be_true
+
+      bf.delete('test')
+      bf.include?('test').should be_false
+    end
+
+    it "should clear Redis filter" do
+      bf.insert('test')
+      bf.include?('test').should be_true
+
+      bf.clear
+      bf.include?('test').should be_false
+    end
+
+    it "should output current stats" do
+      bf.clear
+      bf.insert('test')
+      lambda { bf.stats }.should_not raise_error
+    end
+
+    it "should connect to remote redis server" do
+      lambda { Redis.new }.should_not raise_error
+    end
+
+    it "should allow namespaced BloomFilters" do
+      bf1 = Redis.new(:namespace => :a)
+      bf2 = Redis.new(:namespace => :b)
+
+      bf1.insert('test')
+      bf1.include?('test').should be_true
+      bf2.include?('test').should be_false
+    end
+  end
+end

metadata

@@ -0,0 +1,133 @@
+--- !ruby/object:Gem::Specification 
+name: bloomfilter-rb
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 2
+  - 0
+  - 0
+  version: 2.0.0
+platform: ruby
+authors: 
+- Ilya Grigorik
+- Tatsuya Mori
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-01-05 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: redis
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 1
+        - 1
+        version: 2.1.1
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+description: Counting Bloom Filter implemented in Ruby
+email: 
+- ilya@igvita.com
+executables: []
+
+extensions: 
+- ext/cbloomfilter/extconf.rb
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- .rspec
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bloomfilter-rb.gemspec
+- examples/counting-redis.rb
+- examples/pure-ruby-bf.rb
+- examples/simple-native.rb
+- examples/simple-redis.rb
+- ext/cbloomfilter/cbloomfilter.c
+- ext/cbloomfilter/crc32.c
+- ext/cbloomfilter/crc32.h
+- ext/cbloomfilter/extconf.rb
+- lib/bloomfilter-rb.rb
+- lib/bloomfilter/counting_redis.rb
+- lib/bloomfilter/filter.rb
+- lib/bloomfilter/native.rb
+- lib/bloomfilter/redis.rb
+- lib/bloomfilter/version.rb
+- spec/counting_redis_spec.rb
+- spec/helper.rb
+- spec/native_spec.rb
+- spec/redis_spec.rb
+has_rdoc: true
+homepage: http://github.com/igrigorik/bloomfilter
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: bloomfilter-rb
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Counting Bloom Filter implemented in Ruby
+test_files: 
+- spec/counting_redis_spec.rb
+- spec/helper.rb
+- spec/native_spec.rb
+- spec/redis_spec.rb