bloom_filter 0.5.0 → 0.6.0

data/README.md

@@ -0,0 +1,54 @@
+# bloom_filter
+
+A simple BloomFilter implementation, usable in-process or as an EventMachine daemon.
+
+If you don't know what a bloom filter is, you should read up on it: [http://en.wikipedia.org/wiki/Bloom_filter](http://en.wikipedia.org/wiki/Bloom_filter)
+
+
+## Usage
+You can use it as an in-process data structure:
+    
+    bloom_filter = BloomFilter.new(100, 3) # 100 = bits, 3 = hash functions
+    bloom_filter.add("hello")
+    bloom_filter.include?("hello") #=> true
+
+or you can use it as a service:
+
+    bloom_filter = BloomFilter::Client.new("localhost", 4111)
+    bloom_filter.add("hello") # weeeee bits flying over network IO
+    bloom_filter.include?("hello") #=> true
+    
+To run it as a service, run:
+
+    bloom_filter_server -i localhost:4111 -n 1000000 -p 0.05
+    # -i is interface
+    # -n is estimated number of elements
+    # -p is desired false positive probability
+
+You can also make your in-process bloom filter by specifying your estimated number of elements and false positive probability
+
+    BloomFilter.new(*BloomFilter.optimal_values(1000000, 0.05))
+    
+### Saving your bloom filter
+You can dump/load your bloom filter:
+
+In process:
+
+    dumped = bloom_filter.dump
+    new_bloom_filter = BloomFilter.load(dumped)
+    
+Remote service:
+
+    bloom_filter.dump(path_to_file)
+    bloom_filter.load(path_to_file)
+    
+
+### TODO
+* Better documentation
+* CLI errors
+* CLI help command
+* Improve load/dump workflow
+  
+  
+  
+bloom filters are awesome btw.

data/Rakefile

@@ -5,12 +5,13 @@ begin
   require 'jeweler'
   Jeweler::Tasks.new do |gem|
     gem.name = "bloom_filter"
-    gem.summary = %Q{A simple Ruby BloomFilter implementation, usable in-process or in a client-server model.}
+    gem.summary = %Q{A simple BloomFilter implementation, usable in-process or as an EventMachine daemon.}
     gem.description = %Q{}
     gem.email = "misterfunnyarsal@gmail.com"
     gem.homepage = "http://github.com/arya/bloom_filter"
     gem.authors = ["Arya Asemanfar"]
     gem.add_development_dependency "thoughtbot-shoulda"
+    gem.add_development_dependency "mocha"
     gem.add_dependency "eventmachine", ">=0.12.8"
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end

data/VERSION

@@ -1 +1 @@
-0.5.0
+0.6.0

data/bloom_filter.gemspec

@@ -0,0 +1,69 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{bloom_filter}
+  s.version = "0.6.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Arya Asemanfar"]
+  s.date = %q{2010-01-23}
+  s.default_executable = %q{bloom_filter_server}
+  s.description = %q{}
+  s.email = %q{misterfunnyarsal@gmail.com}
+  s.executables = ["bloom_filter_server"]
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.md"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.md",
+     "Rakefile",
+     "VERSION",
+     "bin/bloom_filter_server",
+     "bloom_filter.gemspec",
+     "lib/bloom_filter.rb",
+     "lib/bloom_filter/client.rb",
+     "lib/bloom_filter/protocol.rb",
+     "lib/bloom_filter/server.rb",
+     "test/bloom_filter_client_test.rb",
+     "test/bloom_filter_server_test.rb",
+     "test/bloom_filter_test.rb",
+     "test/test_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/arya/bloom_filter}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{A simple BloomFilter implementation, usable in-process or as an EventMachine daemon.}
+  s.test_files = [
+    "test/bloom_filter_client_test.rb",
+     "test/bloom_filter_server_test.rb",
+     "test/bloom_filter_test.rb",
+     "test/test_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+      s.add_development_dependency(%q<mocha>, [">= 0"])
+      s.add_runtime_dependency(%q<eventmachine>, [">= 0.12.8"])
+    else
+      s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+      s.add_dependency(%q<mocha>, [">= 0"])
+      s.add_dependency(%q<eventmachine>, [">= 0.12.8"])
+    end
+  else
+    s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
+    s.add_dependency(%q<mocha>, [">= 0"])
+    s.add_dependency(%q<eventmachine>, [">= 0.12.8"])
+  end
+end

data/lib/bloom_filter/server.rb

@@ -75,7 +75,9 @@ class BloomFilter
         end
       when Protocol::LOAD
         begin
-          self.class.filter = BloomFilter.load(File.read(request_body))
+          File.open(request_body, "r") do |file|
+            self.class.filter.replace(file)
+          end
           write_response(true)
         rescue Exception => e
           write_response(false)

data/lib/bloom_filter.rb

@@ -10,6 +10,18 @@ class BloomFilter
     [m.round, k.round]
   end
   
+  def self.read(file)
+    m = file.gets(DUMP_SEPARATOR).to_i
+    k = file.gets(DUMP_SEPARATOR).to_i
+    bits = Array.new((m.to_f / BITS_PER_FIXNUM).ceil, 0)
+    index = 0
+    while line = file.gets(DUMP_SEPARATOR)
+      bits[index] = line.to_i
+      index += 1
+    end
+    [m, k, bits]
+  end
+  
   def self.load(dumped)
     m, k, *bits = dumped.split(DUMP_SEPARATOR).collect { |v| v.to_i }
     new(m, k, bits)
@@ -25,8 +37,11 @@ class BloomFilter
     @k.times do |i|
       self.set_bit(Zlib.crc32("#{i}#{el}") % @m)
     end
+    self
   end
   
+  alias_method :<<, :add
+  
   def include?(el)
     @k.times do |i|
       return false if !bit_set?(Zlib.crc32("#{i}#{el}") % @m)
@@ -34,10 +49,20 @@ class BloomFilter
     true
   end
   
+  def &(els)
+    els.select { |el| self.include?(el) }
+  end
+  
   def dump
     [@m, @k, *@bits].join(DUMP_SEPARATOR)
   end
   
+  def replace(file)
+    m, k, bits = self.class.read(file)
+    @m, @k = m, k
+    @bits.replace(bits)
+  end
+  
   protected
   def set_bit(n)
     index, offset = n / BITS_PER_FIXNUM, n % BITS_PER_FIXNUM

data/test/bloom_filter_client_test.rb

@@ -0,0 +1,61 @@
+require 'test_helper'
+require 'bloom_filter/client'
+
+class BloomFilterClientTest < Test::Unit::TestCase
+  context "with a client object" do
+    setup do
+      @socket = mock()
+      TCPSocket.expects(:new).with("localhost", 4111).returns(@socket)
+      IO.stubs(:select).returns(@socket)
+      @client = BloomFilter::Client.new("localhost", 4111, :timeout => 1)
+    end
+    
+    should "add elements" do
+      @socket.expects(:write).with("#{[6].pack("N")}#{BloomFilter::Protocol::ADD}hello")
+      @socket.expects(:read).with(4).returns([1].pack("N"))
+      @socket.expects(:read).with(1).returns("1")
+      assert @client.add("hello")
+    end
+    
+    should "check for elements" do
+      @socket.expects(:write).with("#{[6].pack("N")}#{BloomFilter::Protocol::INCLUDE}hello")
+      @socket.expects(:read).with(4).returns([1].pack("N"))
+      @socket.expects(:read).with(1).returns("1")
+      assert @client.include?("hello")
+    end
+    
+    should "return false for elements that arent there" do
+      @socket.expects(:write).with("#{[4].pack("N")}#{BloomFilter::Protocol::INCLUDE}bye")
+      @socket.expects(:read).with(4).returns([1].pack("N"))
+      @socket.expects(:read).with(1).returns("0")
+      assert !@client.include?("bye")
+    end
+    
+    should "check for multiple elements" do
+      @socket.expects(:write).with("#{[10].pack("N")}#{BloomFilter::Protocol::INCLUDE_MANY}hello,bye")
+      @socket.expects(:read).with(4).returns([2].pack("N"))
+      @socket.expects(:read).with(2).returns("10")
+      assert_equal ["hello"], @client & ["hello", "bye"]
+    end
+    
+    should "send dump" do
+      @socket.expects(:write).with("#{[7].pack("N")}#{BloomFilter::Protocol::DUMP}/tmp/f")
+      @socket.expects(:read).with(4).returns([1].pack("N"))
+      @socket.expects(:read).with(1).returns("1")
+      assert @client.dump("/tmp/f")
+    end
+    
+    should "send load" do
+      @socket.expects(:write).with("#{[7].pack("N")}#{BloomFilter::Protocol::LOAD}/tmp/f")
+      @socket.expects(:read).with(4).returns([1].pack("N"))
+      @socket.expects(:read).with(1).returns("1")
+      assert @client.load("/tmp/f")
+    end
+    
+    should "timeout" do
+      @socket.expects(:write).with("#{[6].pack("N")}#{BloomFilter::Protocol::ADD}hello")
+      IO.expects(:select).with([@socket], nil, nil, 1)
+      @client.add("hello")
+    end
+  end
+end

data/test/bloom_filter_server_test.rb

@@ -0,0 +1,81 @@
+require 'test_helper'
+require 'bloom_filter/server'
+
+class BloomFilterServerTest < Test::Unit::TestCase
+  def expects_send_data(str)
+    @em.expects(:send_data).with('sig', str, str.size)
+  end
+  
+  context "with a server" do
+    setup do
+      @server = BloomFilter.new_server(100, 0.1).new("sig")
+      @em = EventMachine
+      @filter = @server.class.filter
+    end
+    
+    should "add elements" do
+      @filter.expects(:add).with("hello")
+      expects_send_data("#{[1].pack('N')}1")
+      
+      @server.receive_data("#{[6].pack("N")}#{BloomFilter::Protocol::ADD}hello")
+    end
+  
+    should "check for elements" do
+      @filter.expects(:include?).with("hello").returns(true)
+      expects_send_data("#{[1].pack("N")}1")
+      @server.receive_data("#{[6].pack("N")}#{BloomFilter::Protocol::INCLUDE}hello")
+    end
+    
+    should "check for multiple elements" do
+      @filter.expects(:include?).with("hello").returns(true)
+      @filter.expects(:include?).with("bye").returns(false)
+      expects_send_data("#{[2].pack("N")}10")
+      @server.receive_data("#{[10].pack("N")}#{BloomFilter::Protocol::INCLUDE_MANY}hello,bye")
+    end
+    
+    
+    should "send dump" do
+      path = "/tmp/f"
+      @filter.expects(:dump).returns("dumped_data")
+      
+      file = mock()
+      File.expects(:open).with(path, 'w').yields(file)
+      file.expects(:write).with("dumped_data")
+      
+      expects_send_data("#{[1].pack("N")}1")
+      
+      
+      @server.receive_data("#{[path.size + 1].pack("N")}#{BloomFilter::Protocol::DUMP}#{path}")
+    end
+    
+    should "send load" do
+      path = "/tmp/f"
+      file = mock()
+      @filter.expects(:replace).with(file)
+      File.expects(:open).with(path, 'r').yields(file)
+      
+      expects_send_data("#{[1].pack("N")}1")
+      
+      @server.receive_data("#{[path.size + 1].pack("N")}#{BloomFilter::Protocol::LOAD}#{path}")
+    end
+    
+    
+    should "properly buffer partial messages" do
+      @filter.expects(:add).with("hello")
+      expects_send_data("#{[1].pack('N')}1")
+      
+      @server.receive_data("#{[6].pack("N")}#{BloomFilter::Protocol::ADD}hel")
+      @server.receive_data("lo")
+    end
+    
+    
+    should "handle two messages sent at once" do
+      @filter.expects(:add).with("hello")
+      expects_send_data("#{[1].pack('N')}1").twice()
+      @filter.expects(:add).with("bye")
+
+      @server.receive_data("#{[6].pack("N")}#{BloomFilter::Protocol::ADD}hello#{[4].pack("N")}#{BloomFilter::Protocol::ADD}bye")
+    end
+
+  end
+end

data/test/bloom_filter_test.rb

@@ -1,7 +1,54 @@
 require 'test_helper'
 
 class BloomFilterTest < Test::Unit::TestCase
-  should "probably rename this file and start testing for real" do
-    flunk "hey buddy, you should probably rename this file and start testing for real"
+  context "with a bloom filter" do
+    setup do
+      @bloom_filter = BloomFilter.new(1000, 3)
+    end
+    
+    should "not claim anything is in the set when its empty" do
+      assert !@bloom_filter.include?("hello world")
+    end
+    
+    should "claim an item we just added is in the set" do
+      @bloom_filter.add("hello world")
+      assert @bloom_filter.include?("hello world")
+    end
+    
+    should "dump into a string" do
+      assert @bloom_filter.dump.is_a?(String)
+    end
+    
+    should "a loaded set should claim the element we just added is in the set" do
+      @bloom_filter.add("hello world")
+      loaded = BloomFilter.load(@bloom_filter.dump)
+      assert loaded.include?("hello world")
+    end
+    
+    should "replace itself with another filter" do
+      @bloom_filter.add("hello world")
+      replacement = BloomFilter.new(1000, 3)
+      io = StringIO.new(replacement.dump)
+      @bloom_filter.replace(io)
+      assert !@bloom_filter.include?("hello world")
+    end
+    
+    should "allow chaining of adding" do
+      @bloom_filter << "hello" << "world"
+      assert @bloom_filter.include?("hello")
+      assert @bloom_filter.include?("world")
+    end
+    
+    should "intersect with an array" do
+      @bloom_filter << "hello" << "world"
+      assert_equal ["hello", "world"], @bloom_filter & ["hello", "world", "bye", "moon"]
+    end
+  end
+  
+  should "create a new filter from the dumped file of another" do
+    bloom_filter = BloomFilter.new(1000, 3)
+    bloom_filter.add("hello world")
+    bloom_filter_2 = BloomFilter.load(bloom_filter.dump)
+    assert bloom_filter_2.include?("hello world")
   end
 end

data/test/test_helper.rb

@@ -1,6 +1,7 @@
 require 'rubygems'
 require 'test/unit'
 require 'shoulda'
+require 'mocha'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: bloom_filter
 version: !ruby/object:Gem::Version 
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors: 
 - Arya Asemanfar
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-22 00:00:00 -08:00
+date: 2010-01-23 00:00:00 -08:00
 default_executable: bloom_filter_server
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -22,6 +22,16 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: "0"
     version: 
+- !ruby/object:Gem::Dependency 
+  name: mocha
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
 - !ruby/object:Gem::Dependency 
   name: eventmachine
   type: :runtime
@@ -40,19 +50,22 @@ extensions: []
 
 extra_rdoc_files: 
 - LICENSE
-- README.rdoc
+- README.md
 files: 
 - .document
 - .gitignore
 - LICENSE
-- README.rdoc
+- README.md
 - Rakefile
 - VERSION
 - bin/bloom_filter_server
+- bloom_filter.gemspec
 - lib/bloom_filter.rb
 - lib/bloom_filter/client.rb
 - lib/bloom_filter/protocol.rb
 - lib/bloom_filter/server.rb
+- test/bloom_filter_client_test.rb
+- test/bloom_filter_server_test.rb
 - test/bloom_filter_test.rb
 - test/test_helper.rb
 has_rdoc: true
@@ -82,7 +95,9 @@ rubyforge_project:
 rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
-summary: A simple Ruby BloomFilter implementation, usable in-process or in a client-server model.
+summary: A simple BloomFilter implementation, usable in-process or as an EventMachine daemon.
 test_files: 
+- test/bloom_filter_client_test.rb
+- test/bloom_filter_server_test.rb
 - test/bloom_filter_test.rb
 - test/test_helper.rb

data/README.rdoc

@@ -1,18 +0,0 @@
-= bloom_filter
-
-Description goes here.
-
-== Note on Patches/Pull Requests
- 
-* Fork the project.
-* Make your feature addition or bug fix.
-* Add tests for it. This is important so I don't break it in a
-  future version unintentionally.
-* Commit, do not mess with rakefile, version, or history.
-  (if you want to have your own version, that is fine but
-   bump version in a commit by itself I can ignore when I pull)
-* Send me a pull request. Bonus points for topic branches.
-
-== Copyright
-
-Copyright (c) 2010 Arya Asemanfar. See LICENSE for details.