bloomer 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,6 @@
+source "http://rubygems.org"
+gemspec
+
+gem "rake"
+gem "yard"
+gem "rspec", '~> 2.7.0'

data/MIT-LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2012 Matthew McEachen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,41 @@
+# Bloomer: a pure-ruby bloom filter with no extra fluff
+
+
+[Bloom filters](http://en.wikipedia.org/wiki/Bloom_filter) are great for quickly checking to see if
+a given string has been seen before--in constant time, and using a fixed amount of RAM.
+
+This implementation is the Nth bloom filter gem written in ruby -- but, at the time of conception, the only one that
+
+* uses a robust set of hashing functions
+* can marshal state quickly
+* does not require EM or Redis or something else unrelated to simply implementing a bloom filter
+
+## Usage
+
+```ruby
+expected_size = 10_000
+false_positive_probability = 0.01
+b = Bloomer.new(expected_size, false_positive_probability)
+b.add "cat"
+b.include? "cat"
+#=> true
+bf.include? "dog"
+#=> false
+```
+
+Serialization is through [Marshal](http://ruby-doc.org/core-1.8.7/Marshal.html):
+
+```ruby
+b = Bloomer.new(10)
+b.add("a")
+s = Marshal.dump(b)
+new_b = Marshal.load(s)
+new_b.include? "a"
+#=> true
+```
+
+## History
+
+* 0.0.1 Bloom, there it is.
+
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "yard"
+YARD::Rake::YardocTask.new do |t|
+  t.files = ['lib/**/*.rb', 'README.md']
+end
+
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bloomer.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+require "bloomer"
+
+Gem::Specification.new do |s|
+  s.name        = "bloomer"
+  s.version     = Bloomer::VERSION
+  s.authors     = ["Matthew McEachen"]
+  s.email       = ["matthew+github@mceachen.org"]
+  s.homepage    = ""
+  s.summary     = %q{Pure-ruby bloom filter with minimal dependencies}
+  s.description = %q{Pure-ruby bloom filter with minimal dependencies}
+
+  s.rubyforge_project = "bloomer"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "bitarray"
+end

data/lib/bloomer.rb

@@ -0,0 +1,112 @@
+require 'bitarray'
+
+class Bloomer
+  VERSION = "0.0.1"
+
+  def initialize(expected_size, false_positive_probability = 0.001, opts = {})
+    @ba = opts[:ba] || begin
+      # m is the required number of bits in the array
+      m = -(expected_size * Math.log(false_positive_probability)) / (Math.log(2) ** 2)
+      BitArray.new(m.round)
+    end
+
+    # k is the number of hash functions that minimizes the probability of false positives
+    k = opts[:k] || Math.log(2) * (@ba.size / expected_size)
+    @hashes = Hashes.build(k.round)
+  end
+
+  def add string
+    indicies(string).each { |ea| @ba[ea] = 1 }
+  end
+
+  def include? string
+    !indicies(string).any? { |ea| @ba[ea] == 0 }
+  end
+
+  def _dump(depth)
+    [@hashes.size, Marshal.dump(@ba)].join("\n")
+  end
+
+  def self._load(data)
+    k, ba = data.split("\n", 2)
+    new(nil, nil, :k => k.to_i, :ba => Marshal.load(ba))
+  end
+
+  private
+
+  def indicies string
+    @hashes.collect do |h|
+      h.call(string) % @ba.size
+    end
+  end
+
+  class CircularQueue < Array
+    def rot!
+      first = self.shift
+      self.push(first)
+      first
+    end
+  end
+
+  class Hashes
+    PRIMES = [3571, 4219, 4447, 5167, 5419, 6211, 7057, 7351, 8269, 9241, 10267, 11719, 12097, 13267, 13669, 16651, 19441, 19927, 22447, 23497, 24571, 25117, 26227, 27361, 33391, 35317]
+
+    def self.build(number_of_hashes)
+      hashes = [djb_hash, js_hash, rs_hash, knr_hash, ruby_hash]
+      primes = CircularQueue.new PRIMES
+      while (number_of_hashes > hashes.size)
+        hashes += [:djb_hash, :js_hash, :rs_hash, :knr_hash, :ruby_hash].collect do |ea|
+          send(ea, primes.rot!, primes.rot!)
+        end
+      end
+      return hashes.first(number_of_hashes)
+    end
+
+    MAX = 2**31 - 1
+
+    # written by Professor Daniel J. Bernstein from comp.lang.c
+    def self.djb_hash(a = 5381, b = nil)
+      lambda do |data|
+        data.each_byte.inject(a) do |hash, ea|
+          ((hash << 5) + hash + ea) % MAX
+        end
+      end
+    end
+
+    # bitwise hash function written by Justin Sobel
+    def self.js_hash(a = 1315423911, b = nil)
+      lambda do |data|
+        data.each_byte.inject(a) do |hash, ea|
+          (hash ^ ((hash << 5) + ea + (hash >> 2))) % MAX
+        end
+      end
+    end
+
+    # simple hash function from Robert Sedgwicks Algorithms in C book
+    def self.rs_hash(a = 63689, b = 378551)
+      lambda do |data|
+        i, j = a, b
+        data.each_byte.inject(0) do |hash, ea|
+          i = (i * j) % MAX
+          (hash * i + ea) % MAX
+        end
+      end
+    end
+
+    # From Kernigham and Ritchie's "The C Programming Language"
+    def self.knr_hash(a = 1619, b = 911)
+      lambda do |data|
+        data.each_byte.inject(a) do |hash, ea|
+          ((hash * b) + ea) % MAX
+        end
+      end
+    end
+
+    # default hash
+    def self.ruby_hash(a = 1, b = 1)
+      lambda do |data|
+        (data.hash * a) % MAX
+      end
+    end
+  end
+end

data/spec/bloomer_spec.rb

@@ -0,0 +1,42 @@
+require "spec_helper"
+
+def rand_alpha(size)
+  chars = ('a'..'z').to_a + ('A'..'Z').to_a
+  (0...size).collect { chars[Kernel.rand(chars.length)] }.join
+end
+
+describe Bloomer do
+
+  it "should work trivially" do
+    b = Bloomer.new(10, 0.001)
+    b.add("a")
+    b.should include("a")
+    b.should_not include("")
+    b.should_not include("b")
+    b.add("b")
+    b.should include("b")
+    b.should_not include("")
+    b.add("")
+    b.should include("")
+  end
+
+  it "should find random strings" do
+    b = Bloomer.new(5_000, 0.001)
+    inputs = 1000.times.collect { rand_alpha(Kernel.rand(50)) }
+    inputs.each { |ea| b.add(ea) }
+    inputs.each { |ea| b.include?(ea).should be_true }
+    5000.times.each do
+      s = rand_alpha(Kernel.rand(50))
+      b.include?(s).should == inputs.include?(s)
+    end
+  end
+
+  it "should marshal state correctly" do
+    b = Bloomer.new(10, 0.001)
+    inputs = %q(a b c d)
+    inputs.each{|ea|b.add(ea)}
+    s = Marshal.dump(b)
+    new_b = Marshal.load(s)
+    inputs.each{|ea|new_b.should include(ea)}
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,6 @@
+require 'rspec'
+
+RSpec.configure do |config|
+  config.color_enabled = true
+  config.formatter = 'documentation'
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification 
+name: bloomer
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Matthew McEachen
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-01-18 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  name: bitarray
+  version_requirements: *id001
+description: Pure-ruby bloom filter with minimal dependencies
+email: 
+- matthew+github@mceachen.org
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- MIT-LICENSE
+- README.md
+- Rakefile
+- bloomer.gemspec
+- lib/bloomer.rb
+- spec/bloomer_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: bloomer
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Pure-ruby bloom filter with minimal dependencies
+test_files: 
+- spec/bloomer_spec.rb
+- spec/spec_helper.rb