redis-bloomfilter 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7ad9f7110655941d31af9601bb5f4bc039f02f2bee7d66ef09c52c32796487e2
-  data.tar.gz: 57daaf41aaee08713cf7b9e0c5bb1e40772af503acf0c5111008ba4daa899b88
+  metadata.gz: 73a3287e8ed9a6cd594471734509e07eb20b731cc6cabbcd67752490b81d78b1
+  data.tar.gz: 345d7a05680558b4d7a3415f910fa57ced190c2c11788882cb47c07a52583236
 SHA512:
-  metadata.gz: 1a68456cb710464e8e2021f2d859c9253a353db34c49f192af5751f8977a0bb1bb05b63d6ef6d8cea3bf836e3235bc97786fe6549fdfedba2eb0fd9ab23d1a05
-  data.tar.gz: 697ce76ea20dbfb916d70787265ccc0c27c60fb726555a449e4d968117fc521685c5e3f3df3257b3951cfa550fd20a1a042daf4162f4ec70ffc6ec98b6f71fe3
+  metadata.gz: d3862a987bf159b4aa3481df9b4eade7f0ee03d69c3c2fbd8fb841f91138a8fffd377af1a5dbf4025755e87852dcd8a365a5ec2a380b14f36ed6037ed400fdcc
+  data.tar.gz: ab51bc7f412025d262c49c8156fb56754a178a76866710b3c04ceaddccfebec3e10e942bf8f8f62f26dcc385b9baeef2a4f7462682abae518f447517ad24d08e

data/lib/bloomfilter_driver/lua.rb

@@ -18,7 +18,7 @@ class Redis
       end
 
       def insert(data)
-        set data, 1
+        set data
       end
 
       def include?(key)
@@ -36,78 +36,20 @@ class Redis
       # Taken from https://github.com/ErikDubbelboer/redis-lua-scaling-bloom-filter
       # This is a scalable implementation of BF. It means the initial size can vary
       def lua_load
-        add_fnc = "
-          local entries   = ARGV[1]
-          local precision = ARGV[2]
-          local set_value = ARGV[4]
-          local index     = math.ceil(redis.call('INCR', KEYS[1] .. ':count') / entries)
-          local key       = KEYS[1] .. ':' .. index
-          local bits = math.floor(-(entries * math.log(precision * math.pow(0.5, index))) / 0.480453013)
-          local k = math.floor(0.693147180 * bits / entries)
-          local hash = redis.sha1hex(ARGV[3])
-          local h = { }
-          h[0] = tonumber(string.sub(hash, 0 , 8 ), 16)
-          h[1] = tonumber(string.sub(hash, 8 , 16), 16)
-          h[2] = tonumber(string.sub(hash, 16, 24), 16)
-          h[3] = tonumber(string.sub(hash, 24, 32), 16)
-          for i=1, k do
-            redis.call('SETBIT', key, (h[i % 2] + i * h[2 + (((i + (i % 2)) % 4) / 2)]) % bits, set_value)
-          end
-        "
-
-        check_fnc = "
-
-          local entries   = ARGV[1]
-          local precision = ARGV[2]
-          local index     = redis.call('GET', KEYS[1] .. ':count')
-          if not index then
-            return 0
-          end
-          index     = math.ceil(redis.call('GET', KEYS[1] .. ':count') / entries)
-          local hash = redis.sha1hex(ARGV[3])
-          local h = { }
-          h[0] = tonumber(string.sub(hash, 0 , 8 ), 16)
-          h[1] = tonumber(string.sub(hash, 8 , 16), 16)
-          h[2] = tonumber(string.sub(hash, 16, 24), 16)
-          h[3] = tonumber(string.sub(hash, 24, 32), 16)
-          local maxk = math.floor(0.693147180 * math.floor((entries * math.log(precision * math.pow(0.5, index))) / -0.480453013) / entries)
-          local b    = { }
-          for i=1, maxk do
-            table.insert(b, h[i % 2] + i * h[2 + (((i + (i % 2)) % 4) / 2)])
-          end
-          for n=1, index do
-            local key   = KEYS[1] .. ':' .. n
-            local found = true
-            local bits = math.floor((entries * math.log(precision * math.pow(0.5, n))) / -0.480453013)
-            local k = math.floor(0.693147180 * bits / entries)
-
-            for i=1, k do
-              if redis.call('GETBIT', key, b[i] % bits) == 0 then
-                found = false
-                break
-              end
-            end
-
-            if found then
-              return 1
-            end
-          end
-
-          return 0
-        "
+        add_fnc = File.read File.expand_path("../../vendor/assets/lua/add.lua", __dir__)
+        check_fnc = File.read File.expand_path("../../vendor/assets/lua/check.lua", __dir__)
 
         @add_fnc_sha   = Digest::SHA1.hexdigest(add_fnc)
         @check_fnc_sha = Digest::SHA1.hexdigest(check_fnc)
 
         loaded = @redis.script(:exists, [@add_fnc_sha, @check_fnc_sha]).uniq
-        if loaded.count != 1 || loaded.first != true
-          @add_fnc_sha   = @redis.script(:load, add_fnc)
-          @check_fnc_sha = @redis.script(:load, check_fnc)
-        end
+        return unless loaded.count != 1 || loaded.first != true
+        @add_fnc_sha   = @redis.script(:load, add_fnc)
+        @check_fnc_sha = @redis.script(:load, check_fnc)
       end
 
-      def set(data, val)
-        @redis.evalsha(@add_fnc_sha, keys: [@options[:key_name]], argv: [@options[:size], @options[:error_rate], data, val])
+      def set(data)
+        @redis.evalsha(@add_fnc_sha, keys: [@options[:key_name]], argv: [@options[:size], @options[:error_rate], data])
       end
     end
   end

data/lib/bloomfilter_driver/ruby.rb

@@ -13,7 +13,7 @@ class Redis
 
       # Insert a new element
       def insert(data)
-        set data, 1
+        set data
       end
 
       # It checks if a key is part of the set
@@ -54,9 +54,9 @@ class Redis
         idxs
       end
 
-      def set(key, val)
+      def set(key)
         @redis.pipelined do
-          indexes_for(key).each { |i| @redis.setbit @options[:key_name], i, val }
+          indexes_for(key).each { |i| @redis.setbit @options[:key_name], i, 1 }
         end
       end
     end

data/lib/redis/bloomfilter/version.rb

@@ -2,7 +2,7 @@
 
 class Redis
   class Bloomfilter
-    VERSION = '1.0.1'
+    VERSION = '1.1.0'
     def self.version
       "redis-bloomfilter version #{VERSION}"
     end

data/vendor/assets/lua/add.lua

@@ -0,0 +1,50 @@
+local entries   = ARGV[1]
+local precision = tonumber(ARGV[2])
+local hash      = redis.sha1hex(ARGV[3])
+local countkey  = KEYS[1] .. ':count'
+local count     = redis.call('GET', countkey)
+if not count then
+  count = 1
+else
+  count = count + 1
+end
+
+local factor = math.ceil((entries + count) / entries)
+-- 0.69314718055995 = ln(2)
+local index  = math.ceil(math.log(factor) / 0.69314718055995)
+local scale  = math.pow(2, index - 1) * entries
+local key    = KEYS[1] .. ':' .. index
+
+-- Based on the math from: http://en.wikipedia.org/wiki/Bloom_filter#Probability_of_false_positives
+-- Combined with: http://www.sciencedirect.com/science/article/pii/S0020019006003127
+-- 0.4804530139182 = ln(2)^2
+local bits = math.floor(-(scale * math.log(precision * math.pow(0.5, index))) / 0.4804530139182)
+
+-- 0.69314718055995 = ln(2)
+local k = math.floor(0.69314718055995 * bits / scale)
+
+-- This uses a variation on:
+-- 'Less Hashing, Same Performance: Building a Better Bloom Filter'
+-- https://www.eecs.harvard.edu/~michaelm/postscripts/tr-02-05.pdf
+local h = { }
+h[0] = tonumber(string.sub(hash, 1 , 8 ), 16)
+h[1] = tonumber(string.sub(hash, 9 , 16), 16)
+h[2] = tonumber(string.sub(hash, 17, 24), 16)
+h[3] = tonumber(string.sub(hash, 25, 32), 16)
+
+local found = true
+for i=1, k do
+  if redis.call('SETBIT', key, (h[i % 2] + i * h[2 + (((i + (i % 2)) % 4) / 2)]) % bits, 1) == 0 then
+    found = false
+  end
+end
+
+-- We only increment the count key when we actually added the item to the filter.
+-- This doesn't mean count is accurate. Since this is a scaling bloom filter
+-- it is possible the item was already present in one of the filters in a lower index.
+-- If you really want to make sure an items isn't added multile times you
+-- can use cas.lua (Check And Set).
+if found == false then
+  -- INCR is a little bit faster than SET.
+  redis.call('INCR', countkey)
+end

data/vendor/assets/lua/check.lua

@@ -0,0 +1,61 @@
+local entries   = ARGV[1]
+local precision = ARGV[2]
+local count     = redis.call('GET', KEYS[1] .. ':count')
+
+if not count then
+  return 0
+end
+
+local factor = math.ceil((entries + count) / entries)
+-- 0.69314718055995 = ln(2)
+local index = math.ceil(math.log(factor) / 0.69314718055995)
+local scale = math.pow(2, index - 1) * entries
+
+local hash = redis.sha1hex(ARGV[3])
+
+-- This uses a variation on:
+-- 'Less Hashing, Same Performance: Building a Better Bloom Filter'
+-- https://www.eecs.harvard.edu/~michaelm/postscripts/tr-02-05.pdf
+local h = { }
+h[0] = tonumber(string.sub(hash, 1 , 8 ), 16)
+h[1] = tonumber(string.sub(hash, 9 , 16), 16)
+h[2] = tonumber(string.sub(hash, 17, 24), 16)
+h[3] = tonumber(string.sub(hash, 25, 32), 16)
+
+-- Based on the math from: http://en.wikipedia.org/wiki/Bloom_filter#Probability_of_false_positives
+-- Combined with: http://www.sciencedirect.com/science/article/pii/S0020019006003127
+-- 0.4804530139182 = ln(2)^2
+local maxbits = math.floor((scale * math.log(precision * math.pow(0.5, index))) / -0.4804530139182)
+
+-- 0.69314718055995 = ln(2)
+local maxk = math.floor(0.69314718055995 * maxbits / scale)
+local b    = { }
+
+for i=1, maxk do
+  table.insert(b, h[i % 2] + i * h[2 + (((i + (i % 2)) % 4) / 2)])
+end
+
+for n=1, index do
+  local key    = KEYS[1] .. ':' .. n
+  local found  = true
+  local scalen = math.pow(2, n - 1) * entries
+
+  -- 0.4804530139182 = ln(2)^2
+  local bits = math.floor((scalen * math.log(precision * math.pow(0.5, n))) / -0.4804530139182)
+
+  -- 0.69314718055995 = ln(2)
+  local k = math.floor(0.69314718055995 * bits / scalen)
+
+  for i=1, k do
+    if redis.call('GETBIT', key, b[i] % bits) == 0 then
+      found = false
+      break
+    end
+  end
+
+  if found then
+    return 1
+  end
+end
+
+return 0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis-bloomfilter
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Francesco Laurita
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-02 00:00:00.000000000 Z
+date: 2018-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -100,6 +100,8 @@ files:
 - redis-bloomfilter.gemspec
 - spec/redis_bloomfilter_spec.rb
 - spec/spec_helper.rb
+- vendor/assets/lua/add.lua
+- vendor/assets/lua/check.lua
 homepage: https://github.com/taganaka/redis-bloomfilter
 licenses:
 - MIT