bloombroom 1.0.0 → 1.2.0

data/benchmark/memory.rb

@@ -0,0 +1,8 @@
+module Bloombroom
+  class Process
+
+    def self.rss
+      `ps -o rss= -p #{::Process.pid}`.to_i
+    end
+  end
+end

data/bloombroom.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "bloombroom/version"
+
+Gem::Specification.new do |s|
+  s.name        = "bloombroom"
+  s.version     = Bloombroom::VERSION
+  s.authors     = ["Colin Surprenant"]
+  s.email       = ["colin.surprenant@gmail.com"]
+  s.homepage    = "https://github.com/colinsurprenant/bloombroom"
+  s.summary     = "bloom filters for bounded and unbounded (streaming) data, fast C/FFI FNV hashing and bit fields"
+  s.description = "bloombroom has two bloom filter implementations, a standard filter for bounded key space \
+                   and a continuous filter for unbounded keys (stream). also contains fast C/FFI FNV hashing and fast bit field and \
+                   bit bucket field (multi bits)."
+
+  s.files         = `git ls-files`.split($/)
+  s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ["lib"]
+  s.extensions    = ["ffi/bloombroom/hash/Rakefile"]
+
+  s.has_rdoc = false
+  s.license = 'Apache 2.0'
+  
+  s.add_dependency "ffi", ">= 1.0.0"
+  s.add_dependency "ffi-compiler"
+  s.add_development_dependency "rspec", ">= 2.8.0"
+end

data/ffi/bloombroom/hash/Rakefile

@@ -0,0 +1,5 @@
+require 'ffi-compiler/compile_task'
+
+FFI::Compiler::CompileTask.new('ffi_fnv') do |c|
+  # nothing yet
+end

data/lib/bloombroom.rb

@@ -1,13 +1,17 @@
+require 'ffi'
+require 'ffi-compiler/loader'
+
+module Bloombroom
+  class FNVFFI
+    extend FFI::Library
+    ffi_lib FFI::Compiler::Loader.find('ffi_fnv')
+  end
+end
+
 require "bloombroom/version"
 require "bloombroom/bits/bit_field"
 require "bloombroom/bits/bit_bucket_field"
 require "bloombroom/filter/bloom_helper"
 require "bloombroom/filter/bloom_filter"
 require "bloombroom/filter/continuous_bloom_filter"
-require "bloombroom/hash/fnv_a"
-require "bloombroom/hash/fnv_b"
-require "bloombroom/hash/cext_fnv"
 require "bloombroom/hash/ffi_fnv"
-
-module Bloombroom
-end

data/lib/bloombroom/hash/ffi_fnv.rb

@@ -2,14 +2,10 @@ require 'ffi'
 
 module Bloombroom
   class FNVFFI
-    extend FFI::Library
-
-    ffi_lib File.dirname(__FILE__) + "/" + (FFI::Platform.mac? ? "ffi_fnv.bundle" : FFI.map_library_name("ffi_fnv"))
-
-    attach_function :c_fnv1_32, :fnv1_32, [:string, :uint32], :uint32
-    attach_function :c_fnv1a_32, :fnv1a_32, [:string, :uint32], :uint32
-    attach_function :c_fnv1_64, :fnv1_64, [:string, :uint32], :uint64
-    attach_function :c_fnv1a_64, :fnv1a_64, [:string, :uint32], :uint64
+    attach_function :c_fnv1_32, :fnv1_32, [:buffer_in, :uint32], :uint32, :save_errno => false
+    attach_function :c_fnv1a_32, :fnv1a_32, [:buffer_in, :uint32], :uint32, :save_errno => false
+    attach_function :c_fnv1_64, :fnv1_64, [:buffer_in, :uint32], :uint64, :save_errno => false
+    attach_function :c_fnv1a_64, :fnv1a_64, [:buffer_in, :uint32], :uint64, :save_errno => false
 
     def self.fnv1_32(data)
       c_fnv1_32(data, data.size)

data/lib/bloombroom/version.rb

@@ -1,3 +1,3 @@
 module Bloombroom
-  VERSION = "1.0.0"
+  VERSION = "1.2.0"
 end

data/spec/bloombroom/bits/bit_bucket_field_spec.rb

@@ -0,0 +1,142 @@
+require 'spec_helper'
+# require 'bloombroom/bits/bit_bucket_field'
+
+describe Bloombroom::BitBucketField do
+
+  it "should be all unset at initialization" do
+    bf = Bloombroom::BitBucketField.new(8, 100)
+    100.times.each do |i|
+      bf[i].should == 0
+    end
+    bf.total_set.should == 0
+  end
+
+  it "should set value" do
+    bf = Bloombroom::BitBucketField.new(4, 16)
+
+    16.times.each do |i| 
+      bf[0] = i
+      bf[2] = i
+      bf[4] = i
+      bf[6] = i
+      bf[8] = i
+      bf[0].should == i
+      bf[1].should == 0
+      bf[2].should == i
+      bf[3].should == 0
+      bf[4].should == i
+      bf[5].should == 0
+      bf[6].should == i
+      bf[7].should == 0
+      bf[8].should == i
+    end
+
+    16.times.each do |i| 
+      bf[0] = 0
+      bf[1] = i
+      bf[2] = 0
+      bf[3] = i
+      bf[4] = 0
+      bf[5] = i
+      bf[6] = 0
+      bf[7] = i
+      bf[8] = 0
+      bf[0].should == 0
+      bf[1].should == i
+      bf[2].should == 0
+      bf[3].should == i
+      bf[4].should == 0
+      bf[5].should == i
+      bf[6].should == 0
+      bf[7].should == i
+      bf[8].should == 0
+    end
+
+    16.times do |value|
+      16.times.each do |i|
+        bf[i] = value
+      end
+      16.times.each do |i|
+        bf[i].should == value
+      end
+    end
+  end
+
+  it "should randomly set" do
+    bf = Bloombroom::BitBucketField.new(4, 1000)
+    random_buckets = Array.new(500) {rand(1000)}.uniq
+    random_values = Array.new(random_buckets.size) {rand(16)} # values between 0 and 15
+    bucket_value = random_buckets.zip(random_values)
+    other_buckets = Array.new(1000) {|i| i} - random_buckets
+
+    bucket_value.each{|b, v| bf[b] = v}
+    other_buckets.each{|i| bf[i].should == 0}
+    bucket_value.each{|b, v| bf[b].should == v}
+
+    other_buckets.each{|i| bf[i] = 0}
+    other_buckets.each{|i| bf[i].should == 0}
+    bucket_value.each{|b, v| bf[b].should == v}
+
+    random_buckets.each{|i| bf[i] = 0}
+    other_buckets.each{|i| bf[i].should == 0}
+    random_buckets.each{|i| bf[i].should == 0}
+  end
+
+  it "should report size" do
+    bf = Bloombroom::BitBucketField.new(4, 56)
+    bf.size.should == 56
+  end
+
+  it "should report total_set" do
+    bf = Bloombroom::BitBucketField.new(16, 100)
+    bf[0] = (2 ** 16) - 1
+    bf[1] = (2 ** 16) - 1
+    bf[2] = (2 ** 16) - 1
+    bf[4] = (2 ** 16) - 1
+    bf[50] = 1
+    bf.total_set.should == 5
+
+    bf = Bloombroom::BitBucketField.new(4, 1000)
+    random_buckets = Array.new(500) {rand(1000)}.uniq
+    random_values = Array.new(random_buckets.size) {rand(14) + 1} # values between 1 and 15
+    bucket_value = random_buckets.zip(random_values)
+    other_buckets = Array.new(1000) {|i| i} - random_buckets
+
+    bucket_value.each{|b, v| bf[b] = v}
+    bf.total_set.should == random_buckets.size
+
+    other_buckets.each{|i| bf[i] = 0}
+    other_buckets.each{|i| bf[i].should == 0}
+    bf.total_set.should == bucket_value.size
+
+    random_buckets.each{|i| bf[i] = 0}
+    bf.total_set.should == 0
+  end
+
+  it "should produce bit string using to_s" do
+    bf = Bloombroom::BitBucketField.new(4, 1)
+    bf[0] = 1
+    bf.to_s.should == "0001"
+    bf.to_s(10).should == "1"
+    bf[0] = 15
+    bf.to_s.should == "1111"
+    bf.to_s(10).should == "15"
+
+    bf = Bloombroom::BitBucketField.new(4, 2)
+    bf[0] = 3
+    bf[1] = 8
+    bf.to_s.should == "0011 1000"
+    bf.to_s(10).should == "3 8"
+
+    bf = Bloombroom::BitBucketField.new(4, 8)
+    bf[0] = 1
+    bf[2] = 2
+    bf[4] = 3
+    bf[6] = 4
+    bf.to_s.should == "0001 0000 0010 0000 0011 0000 0100 0000"
+    bf.to_s(10).should == "1 0 2 0 3 0 4 0"
+
+    lambda{bf.to_s(16)}.should raise_error
+  end
+
+end

data/spec/bloombroom/bits/bit_field_spec.rb

@@ -0,0 +1,108 @@
+require 'spec_helper'
+# require 'bloombroom/bits/bit_field'
+
+describe Bloombroom::BitField do
+
+  it "should be all unset at initialization" do
+    bf = Bloombroom::BitField.new(100)
+    (0..99).each do |i|
+      bf[i].should == 0
+      bf.include?(i).should be_false
+    end
+  end
+
+  it "should set and unset" do
+    bf = Bloombroom::BitField.new(1000)
+    bf[100].should == 0
+    bf.include?(100).should be_false
+    bf[101].should == 0
+    bf.include?(101).should be_false
+
+    bf[100] = 1
+    bf[100].should == 1
+    bf.include?(100).should be_true
+    bf[100] = 0
+    bf[100].should == 0
+    bf.include?(100).should be_false
+
+    bf.set(101)
+    bf[101].should == 1
+    bf.include?(101).should be_true
+    bf.unset(101)
+    bf[101].should == 0
+    bf.include?(101).should be_false
+  end
+
+  it "should unset" do
+    bf = Bloombroom::BitField.new(100)
+    (0..99).each{|i| bf.include?(i).should be_false}
+    (0..31).each{|i| bf.unset(i)}
+    (0..99).each{|i| bf.include?(i).should be_false}
+    
+    (0..31).each{|i| bf.set(i)}
+    (0..31).each{|i| bf.include?(i).should be_true}
+    (32..99).each{|i| bf.include?(i).should be_false}
+
+    unsetbits = [0, 5, 6, 10, 16, 23, 31]
+    unsetbits.each{|i| bf.unset(i)}
+    ((0..31).map{|i| i} - unsetbits).each{|i| bf.include?(i).should be_true}
+    unsetbits.each{|i| bf.include?(i).should be_false}
+    (32..99).each{|i| bf.include?(i).should be_false}
+  end
+
+  it "should randomly set and unset" do
+    bf = Bloombroom::BitField.new(1000)
+    random_bits = (0..250).map{|i| rand(1000)}
+    other_bits = (0..999).map{|i| i} - random_bits
+
+    random_bits.each{|i| bf.set(i)}
+    other_bits.each{|i| bf.include?(i).should be_false}
+    random_bits.each{|i| bf.include?(i).should be_true}
+
+    other_bits.each{|i| bf.unset(i)}
+    other_bits.each{|i| bf.include?(i).should be_false}
+    random_bits.each{|i| bf.include?(i).should be_true}
+
+    random_bits.each{|i| bf.unset(i)}
+    other_bits.each{|i| bf.include?(i).should be_false}
+    random_bits.each{|i| bf.include?(i).should be_false}
+  end
+
+  it "should randomly set and unset and support zero?" do
+    bf = Bloombroom::BitField.new(1000)
+    random_bits = (0..250).map{|i| rand(1000)}
+    other_bits = (0..999).map{|i| i} - random_bits
+
+    random_bits.each{|i| bf.set(i)}
+    other_bits.each{|i| bf.zero?(i).should be_true}
+    random_bits.each{|i| bf.zero?(i).should be_false}
+
+    other_bits.each{|i| bf.unset(i)}
+    other_bits.each{|i| bf.zero?(i).should be_true}
+    random_bits.each{|i| bf.zero?(i).should be_false}
+
+    random_bits.each{|i| bf.unset(i)}
+    other_bits.each{|i| bf.zero?(i).should be_true}
+    random_bits.each{|i| bf.zero?(i).should be_true}
+  end
+
+  it "should report size" do
+    bf = Bloombroom::BitField.new(456)
+    bf.size.should == 456
+  end
+
+  it "should produce bit string using to_s" do
+    bf = Bloombroom::BitField.new(10)
+    bf[1] = 1
+    bf[5] = 1
+    bf.to_s.should == "0100010000"
+  end
+
+  it "should report total_set" do
+    bf = Bloombroom::BitField.new(10)
+    bf[1] = 1
+    bf[5] = 1
+    bf.total_set.should == 2
+  end
+
+end

data/spec/bloombroom/filter/bloom_filter_spec.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+# require 'bloombroom'
+
+describe Bloombroom::BloomFilter do
+
+  it "should add" do
+    bf = Bloombroom::BloomFilter.new(1000, 5)
+    bf.include?("abc1").should be_false
+    bf.include?("abc2").should be_false
+    bf.include?("abc3").should be_false
+
+    bf.add("abc1")
+    bf.include?("abc1").should be_true
+    bf.include?("abc2").should be_false
+    bf.include?("abc3").should be_false
+
+    bf.add("abc2")
+    bf.include?("abc1").should be_true
+    bf.include?("abc2").should be_true
+    bf.include?("abc3").should be_false
+
+    bf.add("abc3")
+    bf.include?("abc1").should be_true
+    bf.include?("abc2").should be_true
+    bf.include?("abc3").should be_true
+  end
+
+  it "should keep track of size" do
+    bf = Bloombroom::BloomFilter.new(1000, 5)
+    bf.size.should == 0
+    bf.add("abc1")
+    bf.size.should == 1
+    bf.add("abc2")
+    bf.size.should == 2
+  end
+
+  it "should find m and k" do
+    bf = Bloombroom::BloomFilter.new(*Bloombroom::BloomHelper.find_m_k(10000, 0.001))
+    bf.m.should == 143776
+    bf.k.should == 10
+  end
+
+end

data/spec/bloombroom/filter/bloom_helper_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+# require 'bloombroom/filter/bloom_helper'
+
+describe Bloombroom::BloomHelper do
+
+  it "should multi_hash" do
+    h = Bloombroom::BloomHelper.multi_hash("feedfacedeadbeef", 5)
+    h.size.should == 5
+    # test vector for fnv1a_64 for "feedfacedeadbeef" -> 0xcac54572bb1a6fc8
+    h.should == Array.new(5) {|i| ((0xcac54572bb1a6fc8 & 0xFFFFFFFF00000000) >> 32) + (0xcac54572bb1a6fc8 & 0xFFFFFFFF) * (i + 1)}
+  end
+
+  it "should find m and k" do
+    Bloombroom::BloomHelper.find_m_k(10000, 0.01).should == [95851, 7]
+    Bloombroom::BloomHelper.find_m_k(10000, 0.001).should == [143776, 10]
+  end
+
+end

data/spec/bloombroom/filter/continuous_bloom_filter_spec.rb

@@ -0,0 +1,107 @@
+require 'spec_helper'
+# require 'bloombroom/filter/continuous_bloom_filter'
+# require 'bloombroom/filter/bloom_helper'
+
+describe Bloombroom::ContinuousBloomFilter do
+
+  it "should add" do
+    bf = Bloombroom::ContinuousBloomFilter.new(*Bloombroom::BloomHelper.find_m_k(10, 0.001), 0)
+    bf.include?("abc1").should be_false
+    bf.include?("abc2").should be_false
+    bf.include?("abc3").should be_false
+
+    bf.add("abc1")
+    bf.include?("abc1").should be_true
+    bf.include?("abc2").should be_false
+    bf.include?("abc3").should be_false
+
+    bf.add("abc2")
+    bf.include?("abc1").should be_true
+    bf.include?("abc2").should be_true
+    bf.include?("abc3").should be_false
+
+    bf.add("abc3")
+    bf.include?("abc1").should be_true
+    bf.include?("abc2").should be_true
+    bf.include?("abc3").should be_true
+  end
+
+  it "should find m and k" do
+    bf = Bloombroom::ContinuousBloomFilter.new(*Bloombroom::BloomHelper.find_m_k(10000, 0.001), 0)
+    bf.m.should == 143776
+    bf.k.should == 10
+  end
+
+  it "should expire" do
+    bf = Bloombroom::ContinuousBloomFilter.new(*Bloombroom::BloomHelper.find_m_k(100, 0.001), 0)
+    bf.add("abc1")
+    bf.include?("abc1").should be_true
+
+    bf.inc_time_slot
+    bf.add("abc2")
+    bf.include?("abc1").should be_true
+    bf.include?("abc2").should be_true
+    
+    bf.inc_time_slot
+    bf.add("abc3")
+    bf.include?("abc1").should be_true
+    bf.include?("abc2").should be_true
+    bf.include?("abc3").should be_true
+
+    bf.inc_time_slot
+    bf.add("abc4")
+    bf.include?("abc1").should be_false
+    bf.include?("abc2").should be_true
+    bf.include?("abc3").should be_true
+    bf.include?("abc4").should be_true
+
+    bf.inc_time_slot
+    bf.include?("abc1").should be_false
+    bf.include?("abc2").should be_false
+    bf.include?("abc3").should be_true
+    bf.include?("abc4").should be_true
+
+    bf.inc_time_slot
+    bf.include?("abc1").should be_false
+    bf.include?("abc2").should be_false
+    bf.include?("abc3").should be_false
+    bf.include?("abc4").should be_true
+
+    bf.inc_time_slot
+    bf.include?("abc1").should be_false
+    bf.include?("abc2").should be_false
+    bf.include?("abc3").should be_false
+    bf.include?("abc4").should be_false
+
+    bf = Bloombroom::ContinuousBloomFilter.new(*Bloombroom::BloomHelper.find_m_k(100, 0.1), 0)
+    keys = []
+    1.upto(100) do |i|
+      keys << "#{i}test#{i}"
+      bf.add(keys.last)
+      alive = keys[[keys.size - 3, 0].max, 3]
+      expired = keys - alive
+
+      alive.each{|key| bf.include?(key).should be_true}
+      expired.each{|key| bf.include?(key).should be_false}
+
+      bf.inc_time_slot
+    end
+  end
+
+  it "should compute elapse" do
+    bf = Bloombroom::ContinuousBloomFilter.new(*Bloombroom::BloomHelper.find_m_k(100, 0.1), 0)
+    bf.send(:elapsed, 1, 1).should == 0
+    bf.send(:elapsed, 1, 2).should == 1
+    bf.send(:elapsed, 1, 3).should == 2
+
+    bf.send(:elapsed, 2, 14).should == 12
+    bf.send(:elapsed, 2, 15).should == 13
+    bf.send(:elapsed, 2, 1).should == 14
+    bf.send(:elapsed, 3, 1).should == 13
+    bf.send(:elapsed, 15, 1).should == 1
+    bf.send(:elapsed, 15, 2).should == 2
+    bf.send(:elapsed, 15, 14).should == 14
+  end
+
+
+end