bloomfilter-rb-without-native 2.1.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ea07d8c1212349fce6e2a2c7871ebac290026c76
+  data.tar.gz: c4b9d7ae7b93d0ea8f1d1ee6de5046869f69505b
+SHA512:
+  metadata.gz: c40a4cda2246babd8e84a3727117833282fac07c24f2339a6dff3da71b77f0d73962ec6a5fc785909fa4a692a6249a310147f530f7c82188cd66eba4222e8335
+  data.tar.gz: 97172d03be304c79160a1754320151130606b4dd848973ebd6a9d91d563d7fa72781e309696815211466ec5dffe1c09fec05ad11c1f3b0d1e33b2b0e20c850f1

data/.gitignore

@@ -0,0 +1,10 @@
+*.o
+*.bundle
+*.swp
+.rvmrc
+.DS_Store
+ext/Makefile
+lib/cbloomfilter.so
+tmp
+Gemfile.lock
+pkg

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,65 @@
+THIS IS A FORK OF https://github.com/igrigorik/bloomfilter-rb
+
+# BloomFilter(s) in Ruby
+
+- Redis-backed getbit/setbit non-counting bloom filter
+- Redis-backed set-based counting (+TTL) bloom filter
+
+Bloom filter is a space-efficient probabilistic data structure that is used to test whether an element is a member of a set. False positives are possible, but false negatives are not. For more detail, check the [wikipedia article](http://en.wikipedia.org/wiki/Bloom_filter). Instead of using k different hash functions, this implementation seeds the CRC32 hash with k different initial values (0, 1, ..., k-1). This may or may not give you a good distribution, it all depends on the data.
+
+Performance of the Bloom filter depends on a number of variables:
+
+- size of the bit array
+- size of the counter bucket
+- number of hash functions
+
+## Resources
+
+- Determining parameters: [Scalable Datasets: Bloom Filters in Ruby](http://www.igvita.com/2008/12/27/scalable-datasets-bloom-filters-in-ruby/)
+- Applications & reasons behind bloom filter: [Flow analysis: Time based bloom filter](http://www.igvita.com/2010/01/06/flow-analysis-time-based-bloom-filters/)
+
+***
+
+## Redis-backed setbit/getbit bloom filter
+
+Uses [getbit](http://redis.io/commands/getbit)/[setbit](http://redis.io/commands/setbit) on Redis strings - efficient, fast, can be shared by multiple/concurrent processes.
+
+```ruby
+bf = BloomFilter::Redis.new
+
+bf.insert('test')
+bf.include?('test')     # => true
+bf.include?('blah')     # => false
+
+bf.delete('test')
+bf.include?('test')     # => false
+```
+
+### Memory footprint
+
+- 1.0% error rate for 1M items, 10 bits/item: *2.5 mb*
+- 1.0% error rate for 150M items, 10 bits per item: *358.52 mb*
+- 0.1% error rate for 150M items, 15 bits per item: *537.33 mb*
+
+***
+
+## Redis-backed counting bloom filter with TTL's
+Uses regular Redis get/set counters to implement a counting filter with optional TTL expiry. Because each "bit" requires its own key in Redis, you do incur a much larger memory overhead.
+
+```ruby
+bf = BloomFilter::CountingRedis.new(:ttl => 2)
+
+bf.insert('test')
+bf.include?('test')     # => true
+
+sleep(2)
+bf.include?('test')     # => false
+```
+
+## Credits
+
+Tatsuya Mori <valdzone@gmail.com> (Original C implementation: http://vald.x0.com/sb/)
+
+## License
+
+MIT License - Copyright (c) 2011 Ilya Grigorik

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler/gem_tasks'
+require 'rake'
+require 'rspec'
+require 'rspec/core/rake_task'
+
+Bundler::GemHelper.install_tasks
+RSpec::Core::RakeTask.new(:spec)
+Rake::Task[:spec].prerequisites << :clean
+Rake::Task[:spec].prerequisites << :compile

data/benchmark/redis-bm.rb

@@ -0,0 +1,34 @@
+$:<< 'lib'
+
+require 'benchmark'
+require 'bloomfilter-rb'
+
+n = 10000
+
+Benchmark.bm do |x|
+  r = BloomFilter::Redis.new
+
+  x.report("insert") do
+    n.times do
+      r.insert("a")
+    end
+  end
+
+  x.report("lookup present") do
+    n.times do
+      r.include?("a")
+    end
+  end
+
+  x.report("lookup missing") do
+    n.times do
+      r.include?("b")
+    end
+  end
+
+end
+
+#       user     system      total        real
+# insert  1.000000   0.380000   1.380000 (  1.942181)
+# lookup present  1.030000   0.470000   1.500000 (  2.577577)
+# lookup missing  0.370000   0.160000   0.530000 (  1.060429)

data/bloomfilter-rb-without-native.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "bloomfilter/version"
+
+Gem::Specification.new do |s|
+  s.name        = "bloomfilter-rb-without-native"
+  s.version     = BloomFilter::VERSION
+  s.authors     = ["Garrett Thornburg"]
+  s.email       = ["film42@gmail.com"]
+  s.homepage    = "https://github.com/film42/bloomfilter-rb-without-native/"
+  s.summary     = "Counting Bloom Filter implemented in Ruby (fork without native code)"
+  s.description = s.summary
+
+  s.add_dependency "redis"
+  s.add_development_dependency "rspec", ">= 3"
+  s.add_development_dependency "rake"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/examples/counting-redis.rb

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+require 'bloomfilter'
+
+bf = BloomFilter::CountingRedis.new(:ttl => 2, :server => {:host => 'localhost'})
+
+bf.insert('test')
+puts bf.include?('test')
+
+sleep(3)
+puts bf.include?('test')
+
+puts bf.stats

data/examples/pure-ruby-bf.rb

@@ -0,0 +1,52 @@
+#
+# Pure ruby implementation of a Bloom filter, just for kicks
+#
+
+require 'bitset'
+require 'zlib'
+
+class BloomFilter
+
+  def initialize(max_entries, num_hashes, seed)
+    @num_hashes = num_hashes
+    @size = max_entries.to_i
+    @bitmap = Bitset.new(@size)
+    @__mask = Bitset.new(@size)
+    @seed = seed
+  end
+
+  def insert(key)
+    mask = make_mask(key)
+    @bitmap |= mask
+  end
+
+  def new?(key)
+    mask = make_mask(key)
+    return ((@bitmap & mask) != mask);
+  end
+
+  def make_mask(key)
+    @__mask.clear
+    0.upto(@num_hashes.to_i - 1) do |i|
+      hash = Zlib.crc32(key, i + @seed)
+      @__mask.set(hash % @size, 1)
+    end
+    return @__mask
+  end
+end
+
+def main
+  bf = BloomFilter.new(1000000, 4, 0)
+  num = 0
+  while line = ARGF.gets
+    data = line.chop
+
+    if bf.new?(data)
+      num += 1
+      bf.insert(data)
+    end
+  end
+  print "#element = #{num}\n"
+end
+
+main

data/examples/simple-native.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+require 'bloomfilter-rb'
+
+WORDS = %w(duck penguin bear panda)
+TEST = %w(penguin moose racooon)
+
+bf = BloomFilter::Native.new(:size => 100, :hashes => 2, :seed => 1, :bucket => 3, :raise => false)
+
+WORDS.each { |w| bf.insert(w) }
+TEST.each do |w|
+  puts "#{w}: #{bf.include?(w)}"
+end
+
+bf.stats
+
+#  penguin: true
+#  moose: false
+#  racooon: false
+#
+#  Number of filter buckets (m): 100
+#  Number of bits per buckets (b): 1
+#  Number of filter elements (n): 4
+#  Number of filter hashes (k) : 4
+#  Raise on overflow? (r) : false
+#  Predicted false positive rate = 0.05%

data/examples/simple-redis.rb

@@ -0,0 +1,31 @@
+require 'set'
+require 'lib/bloomfilter-rb'
+
+items = 1_000_00
+bits = 1
+
+# p BloomFilter::Redis.new(:size => items*bits, :hashes => 7) # 2.5 mb
+# p BloomFilter::Redis.new(:size => items*bits*5, :hashes => 7) # 13 mb
+# p BloomFilter::Redis.new(:size => items*bits*30, :hashes => 7) # 73 mb
+
+# 1%   error rate for 5M items/day, 10 bits per item, for 30 days of data: 358.52 mb
+# 0.1% error rate for 5M items/day, 15 bits per item, for 30 days of data: 537.33 mb
+
+bf = BloomFilter::Redis.new(:size => items*bits, :hashes => 7) # 2.5 mb
+
+seen = Set.new
+err = 0
+num = 100000
+
+num.times do
+  item = rand(items)
+
+  if bf.include?(item) != seen.include?(item)
+    err += 1
+  end
+
+  seen << item
+  bf.insert(item)
+end
+
+p [:error_rate, (err.to_f / num) * 100]

data/lib/bloomfilter-rb-without-native.rb

@@ -0,0 +1,7 @@
+require 'redis'
+require 'zlib'
+
+require 'bloomfilter/filter'
+require 'bloomfilter/counting_redis'
+require 'bloomfilter/redis'
+require 'bloomfilter/version'

data/lib/bloomfilter-rb.rb

@@ -0,0 +1,7 @@
+require 'redis'
+require 'zlib'
+
+require 'bloomfilter/filter'
+require 'bloomfilter/counting_redis'
+require 'bloomfilter/redis'
+require 'bloomfilter/version'

data/lib/bloomfilter.rb

@@ -0,0 +1 @@
+require 'bloomfilter-rb-without-native'

data/lib/bloomfilter/counting_redis.rb

@@ -0,0 +1,64 @@
+module BloomFilter
+  class CountingRedis < Filter
+
+    def initialize(opts = {})
+      @opts = {
+        :identifier => 'rbloom',
+        :size       => 100,
+        :hashes     => 4,
+        :seed       => Time.now.to_i,
+        :bucket     => 3,
+        :ttl        => false,
+        :server     => {}
+      }.merge opts
+      @db = @opts.delete(:db) || ::Redis.new(@opts[:server])
+    end
+
+    def insert(key, ttl=nil)
+      ttl = @opts[:ttl] if ttl.nil?
+
+      indexes_for(key).each do |idx|
+        @db.incr idx
+        @db.expire(idx, ttl) if ttl
+      end
+    end
+    alias :[]= :insert
+
+    def delete(key)
+      indexes_for(key).each do |idx|
+        count = @db.decr(idx).to_i
+        if count <= 0
+          @db.del(idx)
+          @db.setbit(idx, 0) if count < 0
+        end
+      end
+    end
+
+    def include?(*keys)
+      indexes = keys.collect { |key| indexes_for(key) }
+      not @db.mget(*indexes.flatten).include? nil
+    end
+    alias :key? :include?
+
+    def num_set
+      @db.eval("return #redis.call('keys', '#{@opts[:identifier]}:*')")
+    end
+    alias :size :num_set
+
+    def clear
+      @db.flushdb
+    end
+
+    private
+
+      # compute index offsets for provided key
+      def indexes_for(key)
+        indexes = []
+        @opts[:hashes].times do |i|
+          indexes.push @opts[:identifier] + ":" + (Zlib.crc32("#{key}:#{i+@opts[:seed]}") % @opts[:size]).to_s
+        end
+
+        indexes
+      end
+  end
+end

data/lib/bloomfilter/filter.rb

@@ -0,0 +1,13 @@
+module BloomFilter
+  class Filter
+    def stats
+      fp = ((1.0 - Math.exp(-(@opts[:hashes] * size).to_f / @opts[:size])) ** @opts[:hashes]) * 100
+      printf "Number of filter buckets (m): %d\n" % @opts[:size]
+      printf "Number of bits per buckets (b): %d\n" % @opts[:bucket]
+      printf "Number of filter elements (n): %d\n" % size
+      printf "Number of filter hashes (k) : %d\n" % @opts[:hashes]
+      printf "Raise on overflow? (r) : %s\n" % @opts[:raise].to_s
+      printf "Predicted false positive rate = %.2f%\n" % fp
+    end
+  end
+end

data/lib/bloomfilter/redis.rb

@@ -0,0 +1,76 @@
+module BloomFilter
+  class Redis < Filter
+
+    def initialize(opts = {})
+      @opts = {
+        :size    => 100,
+        :hashes  => 4,
+        :seed    => Time.now.to_i,
+        :namespace => 'redis',
+        :eager  => false,
+        :server => {}
+      }.merge opts
+      @db = @opts.delete(:db) || ::Redis.new(@opts[:server])
+
+      if @opts[:eager]
+        @db.setbit @opts[:namespace], @opts[:size]+1, 1
+      end
+    end
+
+    def insert(key, ttl=nil)
+      @db.pipelined do
+        indexes_for(key) { |idx| @db.setbit @opts[:namespace], idx, 1 }
+      end
+    end
+    alias :[]= :insert
+
+    def include?(*keys)
+      keys.each do |key|
+        indexes = []
+        indexes_for(key) { |idx| indexes << idx }
+
+        return false if @db.getbit(@opts[:namespace], indexes.shift) == 0
+
+        result = @db.pipelined do
+          indexes.each do |idx|
+            @db.getbit(@opts[:namespace], idx)
+          end
+        end
+
+        return false if result.include?(0)
+      end
+
+      true
+    end
+    alias :key? :include?
+
+    def delete(key)
+      warn "Deletes are disabled on non-counting filter, see: https://github.com/igrigorik/bloomfilter-rb/issues/37. This method will be deprecated in a future release."
+    end
+
+    def clear
+      @db.set @opts[:namespace], 0
+    end
+
+    def num_set
+      @db.strlen @opts[:namespace]
+    end
+    alias :size :num_set
+
+    def stats
+      printf "Number of filter buckets (m): %d\n" % @opts[:size]
+      printf "Number of filter hashes (k) : %d\n" % @opts[:hashes]
+    end
+
+    private
+
+      # compute index offsets for provided key
+      def indexes_for(key)
+        indexes = []
+        @opts[:hashes].times do |i|
+          yield Zlib.crc32("#{key}:#{i+@opts[:seed]}") % @opts[:size]
+        end
+      end
+
+  end
+end

data/lib/bloomfilter/version.rb

@@ -0,0 +1,3 @@
+module BloomFilter
+  VERSION = "2.1.3"
+end

data/spec/counting_redis_spec.rb

@@ -0,0 +1,61 @@
+require 'helper'
+
+describe BloomFilter::CountingRedis do
+
+  it "should connect to remote redis server" do
+    expect { BloomFilter::CountingRedis.new }.not_to raise_error
+  end
+
+  it "should allow redis client instance to be passed in" do
+    redis_client = double("Redis")
+    bf = BloomFilter::CountingRedis.new(:db => redis_client)
+    expect(bf.instance_variable_get(:@db)).to be(redis_client)
+  end
+
+  context "a default CountingRedis instance" do
+    before do
+      # clear all redis databases
+      subject.instance_variable_get(:@db).flushall
+    end
+    
+    it "should store data in Redis" do
+      subject.insert(:abcd)
+      subject.insert('test')
+
+      expect(subject.include?('test')).to be true
+      expect(subject.key?('test')).to be true
+
+      expect(subject.include?('test', 'test2')).to be false
+      expect(subject.include?('test', 'abcd')).to be true
+      expect(subject.include?('test', 'abcd', 'nada')).to be false
+    end
+
+    it "should delete keys from Redis" do
+      subject.insert('test')
+      expect(subject.include?('test')).to be true
+
+      subject.delete('test')
+      expect(subject.include?('test')).to be false
+    end
+
+    it "should output current stats" do
+      subject.insert('test')
+      expect(subject.size).to eq(4)
+      expect { subject.stats }.not_to raise_error
+    end
+  end
+
+  context "a TTL 1 instance" do
+    subject { BloomFilter::CountingRedis.new(:ttl => 1) }
+
+    it "should accept a TTL value for a key" do
+      subject.instance_variable_get(:@db).flushall
+
+      subject.insert('test')
+      expect(subject.include?('test')).to be true
+
+      sleep(2)
+      expect(subject.include?('test')).to be false
+    end
+  end
+end

data/spec/helper.rb

@@ -0,0 +1,2 @@
+require 'bundler/setup'
+require 'bloomfilter-rb'

data/spec/redis_spec.rb

@@ -0,0 +1,62 @@
+require 'helper'
+
+describe BloomFilter::Redis do
+
+  context "use Redis bitstring for storage" do
+    before do
+      # clear all redis databases
+      subject.instance_variable_get(:@db).flushall
+    end
+
+    it "should store data in Redis" do
+      subject.insert(:abcd)
+      subject.insert('test')
+      expect(subject.include?('test')).to be true
+      expect(subject.key?('test')).to be true
+
+      expect(subject.include?('test', 'test2')).to be false
+      expect(subject.include?('test', 'abcd')).to be true
+    end
+
+    it "should not delete keys from Redis" do
+      subject.insert('test')
+      expect(subject.include?('test')).to be true
+
+      subject.delete('test')
+      expect(subject.include?('test')).to be true
+    end
+
+    it "should clear Redis filter" do
+      subject.insert('test')
+      expect(subject.include?('test')).to be true
+
+      subject.clear
+      expect(subject.include?('test')).to be false
+    end
+
+    it "should output current stats" do
+      subject.clear
+      subject.insert('test')
+      expect { subject.stats }.not_to raise_error
+    end
+
+    it "should connect to remote redis server" do
+      expect { BloomFilter::Redis.new }.not_to raise_error
+    end
+
+    it "should allow redis client instance to be passed in" do
+      redis_client = double("Redis")
+      bf = BloomFilter::Redis.new(:db => redis_client)
+      expect(bf.instance_variable_get(:@db)).to be redis_client
+    end
+
+    it "should allow namespaced BloomFilters" do
+      bf1 = BloomFilter::Redis.new(:namespace => :a)
+      bf2 = BloomFilter::Redis.new(:namespace => :b)
+
+      bf1.insert('test')
+      expect(bf1.include?('test')).to be true
+      expect(bf2.include?('test')).to be false
+    end
+  end
+end

metadata

@@ -0,0 +1,109 @@
+--- !ruby/object:Gem::Specification
+name: bloomfilter-rb-without-native
+version: !ruby/object:Gem::Version
+  version: 2.1.3
+platform: ruby
+authors:
+- Garrett Thornburg
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-07-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: redis
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3'
+  name: rspec
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Counting Bloom Filter implemented in Ruby (fork without native code)
+email:
+- film42@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- README.md
+- Rakefile
+- benchmark/redis-bm.rb
+- bloomfilter-rb-without-native.gemspec
+- examples/counting-redis.rb
+- examples/pure-ruby-bf.rb
+- examples/simple-native.rb
+- examples/simple-redis.rb
+- lib/bloomfilter-rb-without-native.rb
+- lib/bloomfilter-rb.rb
+- lib/bloomfilter.rb
+- lib/bloomfilter/counting_redis.rb
+- lib/bloomfilter/filter.rb
+- lib/bloomfilter/redis.rb
+- lib/bloomfilter/version.rb
+- spec/counting_redis_spec.rb
+- spec/helper.rb
+- spec/redis_spec.rb
+homepage: https://github.com/film42/bloomfilter-rb-without-native/
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.2.2
+signing_key:
+specification_version: 4
+summary: Counting Bloom Filter implemented in Ruby (fork without native code)
+test_files:
+- spec/counting_redis_spec.rb
+- spec/helper.rb
+- spec/redis_spec.rb