bloom_fit 0.1.1 → 0.3.0

data/lib/bloom_fit.rb

@@ -1,107 +1,138 @@
-require 'cbloomfilter'
-require 'bloom_fit/version'
+require "forwardable"
+
+require "cbloomfilter"
+require "bloom_fit/configuration_mismatch"
+require "bloom_fit/version"
 
 class BloomFit
-  BloomFit::ConfigurationMismatch = Class.new(ArgumentError)
+  extend Forwardable
 
   attr_reader :bf
 
-  def initialize(opts = {})
-    @opts = {
-      :size    => 100,
-      :hashes  => 4,
-      :bucket  => 1,
-      :raise   => false
-    }.merge(opts)
+  # @param size [Integer] number of buckets in a bloom filter
+  # @param hashes [Integer] number of hash functions
+  def initialize(size: 1_000, hashes: 4)
+    @bf = CBloomFilter.new(size, hashes)
+  end
+
+  def_delegators :@bf, :m, :k, :bitmap, :include?, :clear, :set_bits
+
+  alias size m
+  alias hashes k
+  alias key? include?
+  alias [] include?
+  alias n set_bits
 
-    # arg 1: m => size : number of buckets in a bloom filter
-    # arg 2: k => hashes : number of hash functions
-    # arg 3: b => bucket : number of bits per bucket
-    # arg 4: r => raise : whether to raise on bucket overflow
+  def empty?
+    set_bits.zero?
+  end
 
-    @bf = CBloomFilter.new(@opts[:size], @opts[:hashes], @opts[:bucket], @opts[:raise])
+  # Adds the given key to the set and returns +self+.  Mimics the behavior of
+  # +Set#add+
+  def add(key)
+    @bf.add(key)
+    self
   end
+  alias << add
 
-  def insert(key)
-    @bf.insert(key)
+  # Adds the given key to the set if the value is truthy.  Mimics the behavior of
+  # +Hash#[]=+
+  def []=(key, value)
+    @bf.add(key) if value
   end
-  alias :[]= :insert
 
-  def include?(*keys)
-    @bf.include?(*keys)
+  # Adds the given key to the set and returns +self+. If the key is already
+  # the in set, returns +nil+. Mimics the behavior of +Set#add?+
+  def add?(key)
+    return nil if include?(key) # rubocop:disable Style/ReturnNilInPredicateMethodDefinition
+    add(key)
   end
-  alias :key? :include?
-  alias :[] :include?
 
-  def delete(key); @bf.delete(key); end
-  def clear; @bf.clear; end
-  def size; @bf.set_bits; end
-  def merge!(o); @bf.merge!(o.bf); end
+  # Returns a string of the set bits in hex format
+  def to_hex
+    length = ((size / 8.0).ceil * 8 / 4)
+    bitmap.unpack1("H*")[0...length]
+  end
 
-  # Returns the number of bits that are set to 1 in the filter.
-  def set_bits
-    @bf.set_bits
+  # Returns a string of the set bits in binary format
+  def to_binary
+    bitmap.unpack1("B*")[0...size]
   end
 
-  # Computes the intersection of two Bloom filters.
-  # It assumes that both filters have the same size -
-  # if this is not true +BloomFit::ConfigurationMismatch+ is raised.
-  def &(o)
-    raise BloomFit::ConfigurationMismatch.new unless same_parameters?(o)
-    result = self.class.new
-    result.instance_variable_set(:@bf,@bf.&(o.bf))
-    result
+  # Adds the set from another BloomFit filter or adds all the elements from an
+  # enumerable.  Mimics the behavior of +Set#merge+
+  def merge(other)
+    if other.is_a?(BloomFit)
+      raise BloomFit::ConfigurationMismatch unless same_parameters?(other)
+      @bf.merge(other.bf)
+    elsif other.respond_to?(:each_key)
+      other.each { |k, v| add(k) if v }
+    elsif other.is_a?(Enumerable)
+      other.each { |k| add(k) }
+    else
+      raise ArgumentError, "value must be enumerable or another BloomFit filter"
+    end
   end
 
-  # Computes the union of two Bloom filters.
-  # It assumes that both filters have the same size -
-  # if this is not true +BloomFit::ConfigurationMismatch+ is raised.
-  def |(o)
-    raise BloomFit::ConfigurationMismatch.new unless same_parameters?(o)
-    result = self.class.new
-    result.instance_variable_set(:@bf,@bf.|(o.bf))
-    result
+  # Computes the intersection of two Bloom filters. It requires that both
+  # filters have the same size; otherwise, +BloomFit::ConfigurationMismatch+
+  # is raised.
+  def &(other)
+    raise BloomFit::ConfigurationMismatch unless same_parameters?(other)
+    self.class.new(size:, hashes:).tap do |result|
+      result.instance_variable_set(:@bf, @bf.&(other.bf))
+    end
   end
+  alias intersection &
+
+  # Computes the union of two Bloom filters. It requires that both filters
+  # have the same size; otherwise, +BloomFit::ConfigurationMismatch+ is
+  # raised.
+  def |(other)
+    raise BloomFit::ConfigurationMismatch unless same_parameters?(other)
+    self.class.new(size:, hashes:).tap do |result|
+      result.instance_variable_set(:@bf, @bf.|(other.bf))
+    end
+  end
+  alias union |
+
+  def stats
+    fpr = ((1.0 - Math.exp(-(k * n).to_f / m))**k) * 100
 
-  def bitmap
-    @bf.bitmap
+    (+"").tap do |s|
+      s << format("Number of filter buckets (m):  %d\n",     m)
+      s << format("Number of set bits (n):        %d\n",     n)
+      s << format("Number of filter hashes (k):   %d\n",     k)
+      s << format("Predicted false positive rate: %.2f%%\n", fpr)
+    end
   end
 
   def marshal_load(ary)
-    opts, bitmap = *ary
+    size, hashes, bitmap = *ary
 
-    initialize(opts)
-    @bf.load(bitmap) if !bitmap.nil?
+    initialize(size:, hashes:)
+    @bf.load(bitmap) if bitmap
   end
 
   def marshal_dump
-    [@opts, @bf.bitmap]
+    [size, hashes, bitmap]
   end
 
   def self.load(filename)
-    Marshal.load(File.open(filename, 'r'))
+    Marshal.load(File.open(filename, "r")) # rubocop:disable Security/MarshalLoad
   end
 
   def save(filename)
-    File.open(filename, 'w') do |f|
+    File.open(filename, "w") do |f|
       f << Marshal.dump(self)
     end
   end
 
-  def stats
-    fp = ((1.0 - Math.exp(-(@opts[:hashes] * size).to_f / @opts[:size])) ** @opts[:hashes]) * 100
-    printf "Number of filter buckets (m): %d\n", @opts[:size]
-    printf "Number of bits per buckets (b): %d\n", @opts[:bucket]
-    printf "Number of set bits (n): %d\n", set_bits
-    printf "Number of filter hashes (k) : %d\n", @opts[:hashes]
-    printf "Predicted false positive rate = %.2f%%\n", fp
-  end
-
   protected
 
-  # Returns true if parameters of the +o+ther filter are
+  # Returns true if parameters of the +other+ filter are
   # the same.
-  def same_parameters?(o)
-    @bf.m == o.bf.m && @bf.k == o.bf.k && @bf.b == o.bf.b
+  def same_parameters?(other)
+    bf.m == other.bf.m && bf.k == other.bf.k
   end
 end

data/test/bloom_fit_test.rb

@@ -0,0 +1,344 @@
+require "test_helper"
+
+class BloomFitTest < Minitest::Spec
+  subject { BloomFit.new(size: 100, hashes: 4) }
+
+  describe "#empty?" do
+    it "returns true when nothing set" do
+      assert_equal true, subject.empty? # rubocop:disable Minitest/AssertTruthy
+      assert_empty subject
+    end
+
+    it "returns false when something set" do
+      subject << "key"
+      assert_equal false, subject.empty? # rubocop:disable Minitest/RefuteFalse
+      refute_empty subject
+    end
+  end
+
+  describe "#add" do
+    it "adds the key and returns self" do
+      assert_equal subject, subject.add("test1")
+      assert_equal subject, subject.add("test2")
+      assert_includes subject, "test1"
+      assert_includes subject, "test2"
+    end
+
+    it "is aliased as #<<" do
+      subject << "test1" << "test2"
+      assert_includes subject, "test1"
+      assert_includes subject, "test2"
+    end
+
+    it "is aliased as #[]=, and handles truthy/falsey values" do
+      subject["dog"] = :bar
+      subject["cat"] = :foo
+      assert_includes subject, "dog"
+      assert_includes subject, "cat"
+
+      subject["bat"] = nil
+      subject["pig"] = false
+      refute_includes subject, "bat"
+      refute_includes subject, "pig"
+    end
+
+    it "casts using #to_s as necessary" do
+      subject << :symbol << true << 12_345
+
+      assert_includes subject, "symbol"
+      assert_includes subject, :symbol
+      assert_includes subject, "true"
+      assert_includes subject, "12345"
+      assert_includes subject, 12_345
+    end
+  end
+
+  describe "#add?" do
+    it "adds new key and returns self" do
+      assert_equal subject, subject.add("test1")
+      assert_equal subject, subject.add("test2")
+      assert_includes subject, "test1"
+      assert_includes subject, "test2"
+    end
+
+    it "return nil if the key already exists" do
+      subject << "test1"
+      subject << "test2"
+      assert_includes subject, "test1"
+      assert_includes subject, "test2"
+      assert_nil subject.add?("test1")
+      assert_nil subject.add?("test2")
+    end
+  end
+
+  describe "#include?" do
+    it "returns true when a key is in the set" do
+      subject << "test1"
+      subject << "test2"
+      assert_equal true, subject.include?("test1") # rubocop:disable Minitest/AssertTruthy
+      assert_equal true, subject.include?("test2") # rubocop:disable Minitest/AssertTruthy
+    end
+
+    it "returns false when a key is not in the set" do
+      assert_equal false, subject.include?("test") # rubocop:disable Minitest/RefuteFalse
+      assert_equal false, subject.include?("nada") # rubocop:disable Minitest/RefuteFalse
+    end
+
+    it "is aliased as #key?" do
+      subject << "test1"
+      subject << "test2"
+      assert subject.key?("test1")
+      assert subject.key?("test2")
+      refute subject.key?("test3")
+    end
+
+    it "is aliased as #[]" do
+      subject << "test1"
+      subject << "test2"
+      assert subject["test1"]
+      assert subject["test2"]
+      refute subject["test3"]
+    end
+  end
+
+  describe "#clear" do
+    it "zeroes the bits" do
+      subject.add("test")
+      assert_includes subject, "test"
+      assert_includes subject.to_binary, "1"
+      subject.clear
+      refute_includes subject, "test"
+      refute_includes subject.to_binary, "1"
+    end
+  end
+
+  describe "#set_bits" do
+    it "returns the number of bits set to 1" do
+      bf = BloomFit.new(size: 100, hashes: 4)
+      bf.add("bits")
+      assert_equal 4, bf.set_bits
+
+      bf = BloomFit.new(size: 100, hashes: 1)
+      bf.add("bits")
+      assert_equal 1, bf.set_bits
+    end
+  end
+
+  describe "#bitmap" do
+    it "returns a binary bitmap of all zeros when empty (including a terminating byte)" do
+      bf = BloomFit.new(size: 16)
+      assert_equal "\x00\x00\x00".b, bf.bitmap
+    end
+
+    it "returns a binary bitmap representing the set" do
+      bf = BloomFit.new(size: 16, hashes: 4)
+      bf.add("something")
+      assert_equal "(\x82\x00".b, bf.bitmap
+    end
+
+    it "returns a binary bitmap representing the set even if not a multiple of 8 bits" do
+      bf = BloomFit.new(size: 20, hashes: 4)
+      bf.add("wow")
+      assert_equal "\x04\x14\x00\x00".b, bf.bitmap
+    end
+  end
+
+  describe "#to_hex" do
+    it "returns a hex bitmap of all zeros when empty" do
+      bf = BloomFit.new(size: 16)
+      assert_equal "0000", bf.to_hex
+    end
+
+    it "returns a hex bitmap of all zeros when empty if not a multiple of 8 bits" do
+      bf = BloomFit.new(size: 18)
+      assert_equal "000000", bf.to_hex
+    end
+
+    it "returns a hex bitmap representing the set" do
+      bf = BloomFit.new(size: 16, hashes: 4)
+      bf.add("cool")
+      assert_equal "1441", bf.to_hex
+    end
+  end
+
+  describe "#to_binary" do
+    it "returns a binary bitmap of all zeros when empty" do
+      bf = BloomFit.new(size: 16)
+      assert_equal "0000000000000000", bf.to_binary
+    end
+
+    it "returns a binary bitmap of all zeros when empty if not a multiple of 8 bits" do
+      bf = BloomFit.new(size: 19)
+      assert_equal "0000000000000000000", bf.to_binary
+    end
+
+    it "returns a binary bitmap representing the set" do
+      bf = BloomFit.new(size: 16, hashes: 4)
+      bf << "cool" << "cat"
+      assert_equal "1001011001101001", bf.to_binary
+    end
+  end
+
+  describe "#merge" do
+    it "merges another BloomFit filter" do
+      bf1 = BloomFit.new(size: 100, hashes: 2)
+      bf2 = BloomFit.new(size: 100, hashes: 2)
+      bf1 << "mouse"
+      bf2 << "cat" << "dog"
+      refute_includes bf1, "cat"
+      refute_includes bf1, "dog"
+      bf1.merge(bf2)
+      assert_includes bf1, "mouse"
+      assert_includes bf1, "cat"
+      assert_includes bf1, "dog"
+      refute_includes bf2, "mouse"
+      assert_includes bf2, "cat"
+      assert_includes bf2, "dog"
+    end
+
+    it "merges an array" do
+      subject << "mouse"
+      subject.merge %i[cat dog]
+      assert_includes subject, "mouse"
+      assert_includes subject, "cat"
+      assert_includes subject, "dog"
+    end
+
+    it "merges a set" do
+      subject << "mouse"
+      subject.merge Set.new(%w[cat dog])
+      assert_includes subject, "mouse"
+      assert_includes subject, "cat"
+      assert_includes subject, "dog"
+    end
+
+    it "merges a hash ignoring falsey values" do
+      subject << "mouse"
+      subject.merge({ cat: 1, dog: 2, ant: false, bug: nil })
+      assert_includes subject, "mouse"
+      assert_includes subject, "cat"
+      assert_includes subject, "dog"
+      refute_includes subject, "ant"
+      refute_includes subject, "bug"
+    end
+
+    it "raises when merge is between incompatible filters" do
+      bf1 = BloomFit.new(size: 10)
+      bf2 = BloomFit.new(size: 20)
+      assert_raises(BloomFit::ConfigurationMismatch) { bf1.merge(bf2) }
+    end
+  end
+
+  describe "#&" do
+    it "returns intersection of both filters" do
+      bf1 = BloomFit.new(size: 35, hashes: 4)
+      bf1.add("test")
+      bf1.add("test1")
+
+      bf2 = BloomFit.new(size: 35, hashes: 4)
+      bf2.add("test")
+      bf2.add("test2")
+
+      bf3 = bf1 & bf2
+      assert_equal 35, bf3.size
+      assert_equal 4, bf3.hashes
+      assert_includes bf3, "test"
+      refute_includes bf3, "test1"
+      refute_includes bf3, "test2"
+    end
+
+    it "is aliased as #intersection" do
+      bf1 = BloomFit.new(size: 20, hashes: 4)
+      bf1.add("test")
+      bf1.add("test1")
+
+      bf2 = BloomFit.new(size: 20, hashes: 4)
+      bf2.add("test")
+
+      bf3 = bf1.intersection(bf2)
+      assert_includes bf3, "test"
+      refute_includes bf3, "test1"
+    end
+
+    it "raises when intersection is between incompatible filters" do
+      bf1 = BloomFit.new(size: 10)
+      bf2 = BloomFit.new(size: 20)
+      assert_raises(BloomFit::ConfigurationMismatch) { bf1 & bf2 }
+
+      bf1 = BloomFit.new(size: 10, hashes: 2)
+      bf2 = BloomFit.new(size: 10, hashes: 4)
+      assert_raises(BloomFit::ConfigurationMismatch) { bf1 & bf2 }
+    end
+  end
+
+  describe "#|" do
+    it "returns union with other filter" do
+      bf1 = BloomFit.new
+      bf1.add("test")
+      bf1.add("test1")
+
+      bf2 = BloomFit.new
+      bf2.add("test")
+      bf2.add("test2")
+
+      bf3 = bf1 | bf2
+      assert_includes bf3, "test"
+      assert_includes bf3, "test1"
+      assert_includes bf3, "test2"
+    end
+
+    it "is aliased as #union" do
+      bf1 = BloomFit.new(size: 20, hashes: 4)
+      bf1.add("test")
+      bf1.add("test1")
+
+      bf2 = BloomFit.new(size: 20, hashes: 4)
+      bf2.add("test")
+
+      bf3 = bf1.union(bf2)
+      assert_includes bf3, "test"
+      assert_includes bf3, "test1"
+    end
+
+    it "raises when union is between incompatible filters" do
+      bf1 = BloomFit.new(size: 10)
+      bf2 = BloomFit.new(size: 20)
+      assert_raises(BloomFit::ConfigurationMismatch) { bf1 | bf2 }
+    end
+  end
+
+  describe "#stats" do
+    it "returns current stats" do
+      bf = BloomFit.new(size: 10, hashes: 3)
+      expected = <<~STATS
+        Number of filter buckets (m):  10
+        Number of set bits (n):        0
+        Number of filter hashes (k):   3
+        Predicted false positive rate: 0.00%
+      STATS
+      assert_equal expected, bf.stats
+    end
+  end
+
+  describe "serialization" do
+    after { File.unlink("bf.out") }
+
+    it "marshalls" do
+      bf = BloomFit.new
+      assert bf.save("bf.out")
+    end
+
+    it "loads from marshalled" do
+      subject.add("foo")
+      subject.add("bar")
+      subject.save("bf.out")
+
+      bf2 = BloomFit.load("bf.out")
+      assert_includes bf2, "foo"
+      assert_includes bf2, "bar"
+      refute_includes bf2, "baz"
+
+      assert subject.send(:same_parameters?, bf2)
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,6 @@
+require "minitest/autorun"
+require "minitest/reporters"
+
+Minitest::Reporters.use! # override with MINITEST_REPORTER env var
+
+require "bloom_fit"

metadata

@@ -1,92 +1,37 @@
 --- !ruby/object:Gem::Specification
 name: bloom_fit
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
-- Ilya Grigorik
-- Tatsuya Mori
 - Ryan McGeary
 - Beshad Talayeminaei
+- Ilya Grigorik
+- Tatsuya Mori
 bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
-  name: irb
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake-compiler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '3'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '3'
+dependencies: []
 email:
-- ilya@grigorik.com
-- valdzone@gmail.com
 - ryan@mcgeary.org
 - 'btalayeminaei@gmail.com '
+- ilya@grigorik.com
+- valdzone@gmail.com
 executables: []
 extensions:
 - ext/cbloomfilter/extconf.rb
 extra_rdoc_files: []
 files:
 - README.md
-- Rakefile
 - ext/cbloomfilter/cbloomfilter.c
 - ext/cbloomfilter/crc32.h
 - ext/cbloomfilter/extconf.rb
 - lib/bloom_fit.rb
+- lib/bloom_fit/configuration_mismatch.rb
 - lib/bloom_fit/version.rb
-- spec/bloom_fit_spec.rb
-- spec/helper.rb
+- lib/cbloomfilter.bundle
+- test/bloom_fit_test.rb
+- test/test_helper.rb
 homepage: https://github.com/rmm5t/bloom_fit
 licenses: []
 metadata:
@@ -103,7 +48,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 3.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -114,6 +59,4 @@ rubygems_version: 4.0.9
 specification_version: 4
 summary: BloomFit helps you build correctly sized Bloom filters from expected set
   size and target false positive rate.
-test_files:
-- spec/bloom_fit_spec.rb
-- spec/helper.rb
+test_files: []

data/Rakefile

@@ -1,12 +0,0 @@
-require "bundler/gem_tasks"
-require "bundler/setup"
-require "rspec/core/rake_task"
-require "rake/extensiontask"
-
-Rake::ExtensionTask.new("cbloomfilter")
-RSpec::Core::RakeTask.new(:spec)
-Rake::Task[:spec].prerequisites << :clean
-Rake::Task[:spec].prerequisites << :compile
-
-desc "Default: run unit tests."
-task default: :spec