json-bloomfilter 0.0.5 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile.lock +1 -1
- data/README.md +15 -4
- data/coffee/bloomfilter.coffee +5 -0
- data/js/json-bloomfilter.min.js +1 -1
- data/lib/json/bloomfilter.rb +6 -0
- data/lib/json/bloomfilter/version.rb +1 -1
- data/pkg/json-bloomfilter-0.0.5.gem +0 -0
- data/spec/javascripts/bloomfilter_spec.js +8 -0
- data/spec/json/bloomfilter_spec.rb +8 -0
- metadata +3 -2
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -41,8 +41,13 @@ and the `json-bloomfilter.min.js` will be copied to your local directory. If you
|
|
41
41
|
### Ruby
|
42
42
|
|
43
43
|
```ruby
|
44
|
-
#
|
44
|
+
# use the factory to configure the filter
|
45
|
+
filter = JsonBloomFilter.build 10000, 0.01 # number of expected items, desired error rate
|
46
|
+
|
47
|
+
# or create a define the BloomFilter manually
|
45
48
|
filter = JsonBloomFilter.new size: 100
|
49
|
+
|
50
|
+
# and add entries
|
46
51
|
filter.add "foo"
|
47
52
|
filter.add "bar"
|
48
53
|
filter.test "foo" #=> true
|
@@ -53,7 +58,7 @@ filter.test "doh" #=> probably false
|
|
53
58
|
filter.to_json #=> hash as JSON
|
54
59
|
config = filter.to_hash #=> { "size" => 100, "hashes" => 4, "seed" => 1234567890, "bits" => [...] }
|
55
60
|
|
56
|
-
# use the hash to generate a new
|
61
|
+
# use the hash to generate a new filter with the same config
|
57
62
|
filter2 = JsonBloomFilter.new config
|
58
63
|
filter2.test "foo" #=> true
|
59
64
|
filter2.test "bar" #=> true
|
@@ -63,8 +68,13 @@ filter2.test "doh" #=> probably false
|
|
63
68
|
### Javascript
|
64
69
|
|
65
70
|
```javascript
|
66
|
-
//
|
71
|
+
// use the factory to configure the filter
|
72
|
+
filter = JsonBloomFilter.build(10000, 0.01); // number of expected items, desired error rate
|
73
|
+
|
74
|
+
// or create a define the filter manually
|
67
75
|
filter = new JsonBloomFilter({ size: 100 });
|
76
|
+
|
77
|
+
// and add entries
|
68
78
|
filter.add("foo");
|
69
79
|
filter.add("bar");
|
70
80
|
filter.test("foo"); //=> true
|
@@ -86,7 +96,7 @@ filter2.test("doh") //=> probably false
|
|
86
96
|
|
87
97
|
Valid options for constructor are:
|
88
98
|
|
89
|
-
* `size` (default: 100), the
|
99
|
+
* `size` (default: 100), the bit size of the bit array used
|
90
100
|
* `hashes` (default: 4), the number of hashes used to calculate the bit positions in the bit field
|
91
101
|
* `seed` (default: current UNIX time), the seed for the hashing method
|
92
102
|
|
@@ -102,6 +112,7 @@ Additionally you can pass along:
|
|
102
112
|
|
103
113
|
## Release notes
|
104
114
|
|
115
|
+
* **0.0.6** Adds a factory that takes a size + error rate
|
105
116
|
* **0.0.5** Adds installer of JS file
|
106
117
|
* **0.0.4** Adds JS tests
|
107
118
|
* **0.0.3** Adds Ruby tests
|
data/coffee/bloomfilter.coffee
CHANGED
@@ -9,6 +9,11 @@ JsonBloomfilter = (options = {}) ->
|
|
9
9
|
@bits = new JsonBloomfilter.BitArray(@options["size"], @options["bits"])
|
10
10
|
this
|
11
11
|
|
12
|
+
JsonBloomfilter.build = (capacity, error_rate) ->
|
13
|
+
size = Math.ceil(capacity * Math.log(error_rate) / Math.log(1.0 / Math.pow(2,Math.log(2))))
|
14
|
+
hashes = Math.round(Math.log(2) * size / capacity)
|
15
|
+
new JsonBloomfilter({size: size, hashes: hashes})
|
16
|
+
|
12
17
|
JsonBloomfilter.prototype.add = (key) ->
|
13
18
|
@bits.add(index) for index in @indexesFor(key)
|
14
19
|
return
|
data/js/json-bloomfilter.min.js
CHANGED
@@ -1 +1 @@
|
|
1
|
-
var JsonBloomfilter;JsonBloomfilter=function(options){var key,value;if(options==null){options={}}this.options={size:100,hashes:4,seed:new Date().getTime()/1000,bits:null};for(key in options){value=options[key];this.options[key]=value}this.bits=new JsonBloomfilter.BitArray(this.options.size,this.options.bits);return this};JsonBloomfilter.prototype.add=function(key){var index,_i,_len,_ref;_ref=this.indexesFor(key);for(_i=0,_len=_ref.length;_i<_len;_i++){index=_ref[_i];this.bits.add(index)}};JsonBloomfilter.prototype.test=function(key){var index,_i,_len,_ref;_ref=this.indexesFor(key);for(_i=0,_len=_ref.length;_i<_len;_i++){index=_ref[_i];if(this.bits.get(index)===0){return false}}return true};JsonBloomfilter.prototype.clear=function(){this.bits=new JsonBloomfilter.BitArray(this.options.size)};JsonBloomfilter.prototype.toHash=function(){var hash,key,value,_ref;hash={};_ref=this.options;for(key in _ref){value=_ref[key];hash[key]=value}hash.bits=this.bits.field;return hash};JsonBloomfilter.prototype.toJson=function(){return JSON.stringify(this.toHash())};JsonBloomfilter.prototype.indexesFor=function(key){var index,indexes,_i,_ref;indexes=[];for(index=_i=0,_ref=this.options.hashes-1;0<=_ref?_i<=_ref:_i>=_ref;index=0<=_ref?++_i:--_i){indexes.push(JsonBloomfilter.Zlib.crc32(""+key+":"+(index+this.options.seed))%this.options.size)}return indexes};JsonBloomfilter.BitArray=function(size,field){var arrayLength,i,_i,_ref;if(field==null){field=null}if(!size){throw new Error("Missing argument: size")}this.ELEMENT_WIDTH=32;this.size=size;this.field=field||[];arrayLength=Math.floor(((size-1)/this.ELEMENT_WIDTH)+1);if(!field){for(i=_i=0,_ref=arrayLength-1;0<=_ref?_i<=_ref:_i>=_ref;i=0<=_ref?++_i:--_i){this.field[i]=0}}return this};JsonBloomfilter.BitArray.prototype.add=function(position){return this.set(position,1)};JsonBloomfilter.BitArray.prototype.remove=function(position){return this.set(position,0)};JsonBloomfilter.BitArray.prototype.set=function(position,value){var aPos,bChange;if(position>=this.size){throw new Error("BitArray index out of bounds")}aPos=this.arrayPosition(position);bChange=this.bitChange(position);if(value===1){this.field[aPos]|=bChange}else{if((this.field[aPos]&bChange)!==0){this.field[aPos]^=bChange}}return true};JsonBloomfilter.BitArray.prototype.get=function(position){var aPos,bChange;if(position>=this.size){throw new Error("BitArray index out of bounds")}aPos=this.arrayPosition(position);bChange=this.bitChange(position);if((this.field[aPos]&bChange)>0){return 1}else{return 0}};JsonBloomfilter.BitArray.prototype.arrayPosition=function(position){return Math.floor(position/this.ELEMENT_WIDTH)};JsonBloomfilter.BitArray.prototype.bitChange=function(position){return Math.abs(1<<position%this.ELEMENT_WIDTH)};JsonBloomfilter.BitArray.prototype.toString=function(){var i,output,_i,_ref;output="";for(i=_i=0,_ref=this.size-1;0<=_ref?_i<=_ref:_i>=_ref;i=0<=_ref?++_i:--_i){output+=this.get(i)}return output};JsonBloomfilter.Zlib={CRC32_TABLE:new Array(0,1996959894,3993919788,2567524794,124634137,1886057615,3915621685,2657392035,249268274,2044508324,3772115230,2547177864,162941995,2125561021,3887607047,2428444049,498536548,1789927666,4089016648,2227061214,450548861,1843258603,4107580753,2211677639,325883990,1684777152,4251122042,2321926636,335633487,1661365465,4195302755,2366115317,997073096,1281953886,3579855332,2724688242,1006888145,1258607687,3524101629,2768942443,901097722,1119000684,3686517206,2898065728,853044451,1172266101,3705015759,2882616665,651767980,1373503546,3369554304,3218104598,565507253,1454621731,3485111705,3099436303,671266974,1594198024,3322730930,2970347812,795835527,1483230225,3244367275,3060149565,1994146192,31158534,2563907772,4023717930,1907459465,112637215,2680153253,3904427059,2013776290,251722036,2517215374,3775830040,2137656763,141376813,2439277719,3865271297,1802195444,476864866,2238001368,4066508878,1812370925,453092731,2181625025,4111451223,1706088902,314042704,2344532202,4240017532,1658658271,366619977,2362670323,4224994405,1303535960,984961486,2747007092,3569037538,1256170817,1037604311,2765210733,3554079995,1131014506,879679996,2909243462,3663771856,1141124467,855842277,2852801631,3708648649,1342533948,654459306,3188396048,3373015174,1466479909,544179635,3110523913,3462522015,1591671054,702138776,2966460450,3352799412,1504918807,783551873,3082640443,3233442989,3988292384,2596254646,62317068,1957810842,3939845945,2647816111,81470997,1943803523,3814918930,2489596804,225274430,2053790376,3826175755,2466906013,167816743,2097651377,4027552580,2265490386,503444072,1762050814,4150417245,2154129355,426522225,1852507879,4275313526,2312317920,282753626,1742555852,4189708143,2394877945,397917763,1622183637,3604390888,2714866558,953729732,1340076626,3518719985,2797360999,1068828381,1219638859,3624741850,2936675148,906185462,1090812512,3747672003,2825379669,829329135,1181335161,3412177804,3160834842,628085408,1382605366,3423369109,3138078467,570562233,1426400815,3317316542,2998733608,733239954,1555261956,3268935591,3050360625,752459403,1541320221,2607071920,3965973030,1969922972,40735498,2617837225,3943577151,1913087877,83908371,2512341634,3803740692,2075208622,213261112,2463272603,3855990285,2094854071,198958881,2262029012,4057260610,1759359992,534414190,2176718541,4139329115,1873836001,414664567,2282248934,4279200368,1711684554,285281116,2405801727,4167216745,1634467795,376229701,2685067896,3608007406,1308918612,956543938,2808555105,3495958263,1231636301,1047427035,2932959818,3654703836,1088359270,936918000,2847714899,3736837829,1202900863,817233897,3183342108,3401237130,1404277552,615818150,3134207493,3453421203,1423857449,601450431,3009837614,3294710456,1567103746,711928724,3020668471,3272380065,1510334235,755167117),crc32:function(string){var bytes,crc,i,iTop,n;bytes=this.bytesFor(string);crc=0;n=0;crc=crc^(-1);i=0;iTop=bytes.length;while(i<iTop){n=(crc^bytes[i])&255;crc=(crc>>>8)^this.CRC32_TABLE[n];i++}crc=crc^(-1);if(crc<0){crc+=4294967296}return crc},bytesFor:function(string){var bytes,i;bytes=[];i=0;while(i<string.length){bytes.push(string.charCodeAt(i));++i}return bytes}};
|
1
|
+
var JsonBloomfilter;JsonBloomfilter=function(options){var key,value;if(options==null){options={}}this.options={size:100,hashes:4,seed:new Date().getTime()/1000,bits:null};for(key in options){value=options[key];this.options[key]=value}this.bits=new JsonBloomfilter.BitArray(this.options.size,this.options.bits);return this};JsonBloomfilter.build=function(capacity,error_rate){var hashes,size;size=Math.ceil(capacity*Math.log(error_rate)/Math.log(1/Math.pow(2,Math.log(2))));hashes=Math.round(Math.log(2)*size/capacity);return new JsonBloomfilter({size:size,hashes:hashes})};JsonBloomfilter.prototype.add=function(key){var index,_i,_len,_ref;_ref=this.indexesFor(key);for(_i=0,_len=_ref.length;_i<_len;_i++){index=_ref[_i];this.bits.add(index)}};JsonBloomfilter.prototype.test=function(key){var index,_i,_len,_ref;_ref=this.indexesFor(key);for(_i=0,_len=_ref.length;_i<_len;_i++){index=_ref[_i];if(this.bits.get(index)===0){return false}}return true};JsonBloomfilter.prototype.clear=function(){this.bits=new JsonBloomfilter.BitArray(this.options.size)};JsonBloomfilter.prototype.toHash=function(){var hash,key,value,_ref;hash={};_ref=this.options;for(key in _ref){value=_ref[key];hash[key]=value}hash.bits=this.bits.field;return hash};JsonBloomfilter.prototype.toJson=function(){return JSON.stringify(this.toHash())};JsonBloomfilter.prototype.indexesFor=function(key){var index,indexes,_i,_ref;indexes=[];for(index=_i=0,_ref=this.options.hashes-1;0<=_ref?_i<=_ref:_i>=_ref;index=0<=_ref?++_i:--_i){indexes.push(JsonBloomfilter.Zlib.crc32(""+key+":"+(index+this.options.seed))%this.options.size)}return indexes};JsonBloomfilter.BitArray=function(size,field){var arrayLength,i,_i,_ref;if(field==null){field=null}if(!size){throw new Error("Missing argument: size")}this.ELEMENT_WIDTH=32;this.size=size;this.field=field||[];arrayLength=Math.floor(((size-1)/this.ELEMENT_WIDTH)+1);if(!field){for(i=_i=0,_ref=arrayLength-1;0<=_ref?_i<=_ref:_i>=_ref;i=0<=_ref?++_i:--_i){this.field[i]=0}}return this};JsonBloomfilter.BitArray.prototype.add=function(position){return this.set(position,1)};JsonBloomfilter.BitArray.prototype.remove=function(position){return this.set(position,0)};JsonBloomfilter.BitArray.prototype.set=function(position,value){var aPos,bChange;if(position>=this.size){throw new Error("BitArray index out of bounds")}aPos=this.arrayPosition(position);bChange=this.bitChange(position);if(value===1){this.field[aPos]|=bChange}else{if((this.field[aPos]&bChange)!==0){this.field[aPos]^=bChange}}return true};JsonBloomfilter.BitArray.prototype.get=function(position){var aPos,bChange;if(position>=this.size){throw new Error("BitArray index out of bounds")}aPos=this.arrayPosition(position);bChange=this.bitChange(position);if((this.field[aPos]&bChange)>0){return 1}else{return 0}};JsonBloomfilter.BitArray.prototype.arrayPosition=function(position){return Math.floor(position/this.ELEMENT_WIDTH)};JsonBloomfilter.BitArray.prototype.bitChange=function(position){return Math.abs(1<<position%this.ELEMENT_WIDTH)};JsonBloomfilter.BitArray.prototype.toString=function(){var i,output,_i,_ref;output="";for(i=_i=0,_ref=this.size-1;0<=_ref?_i<=_ref:_i>=_ref;i=0<=_ref?++_i:--_i){output+=this.get(i)}return output};JsonBloomfilter.Zlib={CRC32_TABLE:new Array(0,1996959894,3993919788,2567524794,124634137,1886057615,3915621685,2657392035,249268274,2044508324,3772115230,2547177864,162941995,2125561021,3887607047,2428444049,498536548,1789927666,4089016648,2227061214,450548861,1843258603,4107580753,2211677639,325883990,1684777152,4251122042,2321926636,335633487,1661365465,4195302755,2366115317,997073096,1281953886,3579855332,2724688242,1006888145,1258607687,3524101629,2768942443,901097722,1119000684,3686517206,2898065728,853044451,1172266101,3705015759,2882616665,651767980,1373503546,3369554304,3218104598,565507253,1454621731,3485111705,3099436303,671266974,1594198024,3322730930,2970347812,795835527,1483230225,3244367275,3060149565,1994146192,31158534,2563907772,4023717930,1907459465,112637215,2680153253,3904427059,2013776290,251722036,2517215374,3775830040,2137656763,141376813,2439277719,3865271297,1802195444,476864866,2238001368,4066508878,1812370925,453092731,2181625025,4111451223,1706088902,314042704,2344532202,4240017532,1658658271,366619977,2362670323,4224994405,1303535960,984961486,2747007092,3569037538,1256170817,1037604311,2765210733,3554079995,1131014506,879679996,2909243462,3663771856,1141124467,855842277,2852801631,3708648649,1342533948,654459306,3188396048,3373015174,1466479909,544179635,3110523913,3462522015,1591671054,702138776,2966460450,3352799412,1504918807,783551873,3082640443,3233442989,3988292384,2596254646,62317068,1957810842,3939845945,2647816111,81470997,1943803523,3814918930,2489596804,225274430,2053790376,3826175755,2466906013,167816743,2097651377,4027552580,2265490386,503444072,1762050814,4150417245,2154129355,426522225,1852507879,4275313526,2312317920,282753626,1742555852,4189708143,2394877945,397917763,1622183637,3604390888,2714866558,953729732,1340076626,3518719985,2797360999,1068828381,1219638859,3624741850,2936675148,906185462,1090812512,3747672003,2825379669,829329135,1181335161,3412177804,3160834842,628085408,1382605366,3423369109,3138078467,570562233,1426400815,3317316542,2998733608,733239954,1555261956,3268935591,3050360625,752459403,1541320221,2607071920,3965973030,1969922972,40735498,2617837225,3943577151,1913087877,83908371,2512341634,3803740692,2075208622,213261112,2463272603,3855990285,2094854071,198958881,2262029012,4057260610,1759359992,534414190,2176718541,4139329115,1873836001,414664567,2282248934,4279200368,1711684554,285281116,2405801727,4167216745,1634467795,376229701,2685067896,3608007406,1308918612,956543938,2808555105,3495958263,1231636301,1047427035,2932959818,3654703836,1088359270,936918000,2847714899,3736837829,1202900863,817233897,3183342108,3401237130,1404277552,615818150,3134207493,3453421203,1423857449,601450431,3009837614,3294710456,1567103746,711928724,3020668471,3272380065,1510334235,755167117),crc32:function(string){var bytes,crc,i,iTop,n;bytes=this.bytesFor(string);crc=0;n=0;crc=crc^(-1);i=0;iTop=bytes.length;while(i<iTop){n=(crc^bytes[i])&255;crc=(crc>>>8)^this.CRC32_TABLE[n];i++}crc=crc^(-1);if(crc<0){crc+=4294967296}return crc},bytesFor:function(string){var bytes,i;bytes=[];i=0;while(i<string.length){bytes.push(string.charCodeAt(i));++i}return bytes}};
|
data/lib/json/bloomfilter.rb
CHANGED
@@ -4,6 +4,12 @@ require "json"
|
|
4
4
|
class JsonBloomfilter
|
5
5
|
DEFAULTS = { "size" => 100, "hashes" => 4, "seed" => Time.new.to_i, "bits" => nil }
|
6
6
|
|
7
|
+
def self.build capacity, error_rate
|
8
|
+
size = (capacity * Math.log(error_rate) / Math.log(1.0 / 2**Math.log(2))).ceil
|
9
|
+
hashes = (Math.log(2) * size / capacity).round
|
10
|
+
JsonBloomfilter.new size: size, hashes: hashes
|
11
|
+
end
|
12
|
+
|
7
13
|
def initialize options = {}
|
8
14
|
@options = merge_defaults_with options
|
9
15
|
@bits = BitArray.new(@options["size"], @options["bits"])
|
Binary file
|
@@ -1,5 +1,13 @@
|
|
1
1
|
describe("JsonBloomfilter", function() {
|
2
2
|
|
3
|
+
describe(".build", function() {
|
4
|
+
it("should generate a BloomFilter with the right number of hashes and size", function() {
|
5
|
+
bf = JsonBloomfilter.build(1000, 0.01);
|
6
|
+
expect(bf.toHash()["hashes"]).toBe(7);
|
7
|
+
expect(bf.toHash()["size"]).toBe(9586);
|
8
|
+
});
|
9
|
+
});
|
10
|
+
|
3
11
|
describe("#initialize", function() {
|
4
12
|
it("should take the appropriate options", function() {
|
5
13
|
seed = (new Date().getTime()/1000) - 24*60*60;
|
@@ -2,6 +2,14 @@ require "spec_helper"
|
|
2
2
|
|
3
3
|
describe JsonBloomfilter do
|
4
4
|
|
5
|
+
describe ".build" do
|
6
|
+
it "should generate a BloomFilter with the right number of hashes and size" do
|
7
|
+
bf = JsonBloomfilter.build 1000, 0.01
|
8
|
+
expect(bf.to_hash["hashes"]).to be == 7
|
9
|
+
expect(bf.to_hash["size"]).to be == 9586
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
5
13
|
describe "#initialize" do
|
6
14
|
it "should take the appropriate options" do
|
7
15
|
seed = Time.now.to_i - 24*60*60
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: json-bloomfilter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -153,6 +153,7 @@ files:
|
|
153
153
|
- pkg/json-bloomfilter-0.0.2.gem
|
154
154
|
- pkg/json-bloomfilter-0.0.3.gem
|
155
155
|
- pkg/json-bloomfilter-0.0.4.gem
|
156
|
+
- pkg/json-bloomfilter-0.0.5.gem
|
156
157
|
- spec/javascripts/bitarray_spec.js
|
157
158
|
- spec/javascripts/bloomfilter_spec.js
|
158
159
|
- spec/javascripts/support/jasmine.yml
|
@@ -181,7 +182,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
181
182
|
version: '0'
|
182
183
|
segments:
|
183
184
|
- 0
|
184
|
-
hash: -
|
185
|
+
hash: -1108615832793611037
|
185
186
|
requirements: []
|
186
187
|
rubyforge_project:
|
187
188
|
rubygems_version: 1.8.24
|