json-bloomfilter 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile.lock +1 -1
- data/README.md +15 -4
- data/coffee/bloomfilter.coffee +5 -0
- data/js/json-bloomfilter.min.js +1 -1
- data/lib/json/bloomfilter.rb +6 -0
- data/lib/json/bloomfilter/version.rb +1 -1
- data/pkg/json-bloomfilter-0.0.5.gem +0 -0
- data/spec/javascripts/bloomfilter_spec.js +8 -0
- data/spec/json/bloomfilter_spec.rb +8 -0
- metadata +3 -2
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -41,8 +41,13 @@ and the `json-bloomfilter.min.js` will be copied to your local directory. If you
|
|
41
41
|
### Ruby
|
42
42
|
|
43
43
|
```ruby
|
44
|
-
#
|
44
|
+
# use the factory to configure the filter
|
45
|
+
filter = JsonBloomFilter.build 10000, 0.01 # number of expected items, desired error rate
|
46
|
+
|
47
|
+
# or create a define the BloomFilter manually
|
45
48
|
filter = JsonBloomFilter.new size: 100
|
49
|
+
|
50
|
+
# and add entries
|
46
51
|
filter.add "foo"
|
47
52
|
filter.add "bar"
|
48
53
|
filter.test "foo" #=> true
|
@@ -53,7 +58,7 @@ filter.test "doh" #=> probably false
|
|
53
58
|
filter.to_json #=> hash as JSON
|
54
59
|
config = filter.to_hash #=> { "size" => 100, "hashes" => 4, "seed" => 1234567890, "bits" => [...] }
|
55
60
|
|
56
|
-
# use the hash to generate a new
|
61
|
+
# use the hash to generate a new filter with the same config
|
57
62
|
filter2 = JsonBloomFilter.new config
|
58
63
|
filter2.test "foo" #=> true
|
59
64
|
filter2.test "bar" #=> true
|
@@ -63,8 +68,13 @@ filter2.test "doh" #=> probably false
|
|
63
68
|
### Javascript
|
64
69
|
|
65
70
|
```javascript
|
66
|
-
//
|
71
|
+
// use the factory to configure the filter
|
72
|
+
filter = JsonBloomFilter.build(10000, 0.01); // number of expected items, desired error rate
|
73
|
+
|
74
|
+
// or create a define the filter manually
|
67
75
|
filter = new JsonBloomFilter({ size: 100 });
|
76
|
+
|
77
|
+
// and add entries
|
68
78
|
filter.add("foo");
|
69
79
|
filter.add("bar");
|
70
80
|
filter.test("foo"); //=> true
|
@@ -86,7 +96,7 @@ filter2.test("doh") //=> probably false
|
|
86
96
|
|
87
97
|
Valid options for constructor are:
|
88
98
|
|
89
|
-
* `size` (default: 100), the
|
99
|
+
* `size` (default: 100), the bit size of the bit array used
|
90
100
|
* `hashes` (default: 4), the number of hashes used to calculate the bit positions in the bit field
|
91
101
|
* `seed` (default: current UNIX time), the seed for the hashing method
|
92
102
|
|
@@ -102,6 +112,7 @@ Additionally you can pass along:
|
|
102
112
|
|
103
113
|
## Release notes
|
104
114
|
|
115
|
+
* **0.0.6** Adds a factory that takes a size + error rate
|
105
116
|
* **0.0.5** Adds installer of JS file
|
106
117
|
* **0.0.4** Adds JS tests
|
107
118
|
* **0.0.3** Adds Ruby tests
|
data/coffee/bloomfilter.coffee
CHANGED
@@ -9,6 +9,11 @@ JsonBloomfilter = (options = {}) ->
|
|
9
9
|
@bits = new JsonBloomfilter.BitArray(@options["size"], @options["bits"])
|
10
10
|
this
|
11
11
|
|
12
|
+
JsonBloomfilter.build = (capacity, error_rate) ->
|
13
|
+
size = Math.ceil(capacity * Math.log(error_rate) / Math.log(1.0 / Math.pow(2,Math.log(2))))
|
14
|
+
hashes = Math.round(Math.log(2) * size / capacity)
|
15
|
+
new JsonBloomfilter({size: size, hashes: hashes})
|
16
|
+
|
12
17
|
JsonBloomfilter.prototype.add = (key) ->
|
13
18
|
@bits.add(index) for index in @indexesFor(key)
|
14
19
|
return
|
data/js/json-bloomfilter.min.js
CHANGED
@@ -1 +1 @@
|
|
1
|
-
var JsonBloomfilter;JsonBloomfilter=function(options){var key,value;if(options==null){options={}}this.options={size:100,hashes:4,seed:new Date().getTime()/1000,bits:null};for(key in options){value=options[key];this.options[key]=value}this.bits=new JsonBloomfilter.BitArray(this.options.size,this.options.bits);return this};JsonBloomfilter.prototype.add=function(key){var index,_i,_len,_ref;_ref=this.indexesFor(key);for(_i=0,_len=_ref.length;_i<_len;_i++){index=_ref[_i];this.bits.add(index)}};JsonBloomfilter.prototype.test=function(key){var index,_i,_len,_ref;_ref=this.indexesFor(key);for(_i=0,_len=_ref.length;_i<_len;_i++){index=_ref[_i];if(this.bits.get(index)===0){return false}}return true};JsonBloomfilter.prototype.clear=function(){this.bits=new JsonBloomfilter.BitArray(this.options.size)};JsonBloomfilter.prototype.toHash=function(){var hash,key,value,_ref;hash={};_ref=this.options;for(key in _ref){value=_ref[key];hash[key]=value}hash.bits=this.bits.field;return hash};JsonBloomfilter.prototype.toJson=function(){return JSON.stringify(this.toHash())};JsonBloomfilter.prototype.indexesFor=function(key){var index,indexes,_i,_ref;indexes=[];for(index=_i=0,_ref=this.options.hashes-1;0<=_ref?_i<=_ref:_i>=_ref;index=0<=_ref?++_i:--_i){indexes.push(JsonBloomfilter.Zlib.crc32(""+key+":"+(index+this.options.seed))%this.options.size)}return indexes};JsonBloomfilter.BitArray=function(size,field){var arrayLength,i,_i,_ref;if(field==null){field=null}if(!size){throw new Error("Missing argument: size")}this.ELEMENT_WIDTH=32;this.size=size;this.field=field||[];arrayLength=Math.floor(((size-1)/this.ELEMENT_WIDTH)+1);if(!field){for(i=_i=0,_ref=arrayLength-1;0<=_ref?_i<=_ref:_i>=_ref;i=0<=_ref?++_i:--_i){this.field[i]=0}}return this};JsonBloomfilter.BitArray.prototype.add=function(position){return this.set(position,1)};JsonBloomfilter.BitArray.prototype.remove=function(position){return this.set(position,0)};JsonBloomfilter.BitArray.prototype.set=function(position,value){var aPos,bChange;if(position>=this.size){throw new Error("BitArray index out of bounds")}aPos=this.arrayPosition(position);bChange=this.bitChange(position);if(value===1){this.field[aPos]|=bChange}else{if((this.field[aPos]&bChange)!==0){this.field[aPos]^=bChange}}return true};JsonBloomfilter.BitArray.prototype.get=function(position){var aPos,bChange;if(position>=this.size){throw new Error("BitArray index out of bounds")}aPos=this.arrayPosition(position);bChange=this.bitChange(position);if((this.field[aPos]&bChange)>0){return 1}else{return 0}};JsonBloomfilter.BitArray.prototype.arrayPosition=function(position){return Math.floor(position/this.ELEMENT_WIDTH)};JsonBloomfilter.BitArray.prototype.bitChange=function(position){return Math.abs(1<<position%this.ELEMENT_WIDTH)};JsonBloomfilter.BitArray.prototype.toString=function(){var i,output,_i,_ref;output="";for(i=_i=0,_ref=this.size-1;0<=_ref?_i<=_ref:_i>=_ref;i=0<=_ref?++_i:--_i){output+=this.get(i)}return output};JsonBloomfilter.Zlib={CRC32_TABLE:new Array(0,1996959894,3993919788,2567524794,124634137,1886057615,3915621685,2657392035,249268274,2044508324,3772115230,2547177864,162941995,2125561021,3887607047,2428444049,498536548,1789927666,4089016648,2227061214,450548861,1843258603,4107580753,2211677639,325883990,1684777152,4251122042,2321926636,335633487,1661365465,4195302755,2366115317,997073096,1281953886,3579855332,2724688242,1006888145,1258607687,3524101629,2768942443,901097722,1119000684,3686517206,2898065728,853044451,1172266101,3705015759,2882616665,651767980,1373503546,3369554304,3218104598,565507253,1454621731,3485111705,3099436303,671266974,1594198024,3322730930,2970347812,795835527,1483230225,3244367275,3060149565,1994146192,31158534,2563907772,4023717930,1907459465,112637215,2680153253,3904427059,2013776290,251722036,2517215374,3775830040,2137656763,141376813,2439277719,3865271297,1802195444,476864866,2238001368,4066508878,1812370925,453092731,2181625025,4111451223,1706088902,314042704,2344532202,4240017532,1658658271,366619977,2362670323,4224994405,1303535960,984961486,2747007092,3569037538,1256170817,1037604311,2765210733,3554079995,1131014506,879679996,2909243462,3663771856,1141124467,855842277,2852801631,3708648649,1342533948,654459306,3188396048,3373015174,1466479909,544179635,3110523913,3462522015,1591671054,702138776,2966460450,3352799412,1504918807,783551873,3082640443,3233442989,3988292384,2596254646,62317068,1957810842,3939845945,2647816111,81470997,1943803523,3814918930,2489596804,225274430,2053790376,3826175755,2466906013,167816743,2097651377,4027552580,2265490386,503444072,1762050814,4150417245,2154129355,426522225,1852507879,4275313526,2312317920,282753626,1742555852,4189708143,2394877945,397917763,1622183637,3604390888,2714866558,953729732,1340076626,3518719985,2797360999,1068828381,1219638859,3624741850,2936675148,906185462,1090812512,3747672003,2825379669,829329135,1181335161,3412177804,3160834842,628085408,1382605366,3423369109,3138078467,570562233,1426400815,3317316542,2998733608,733239954,1555261956,3268935591,3050360625,752459403,1541320221,2607071920,3965973030,1969922972,40735498,2617837225,3943577151,1913087877,83908371,2512341634,3803740692,2075208622,213261112,2463272603,3855990285,2094854071,198958881,2262029012,4057260610,1759359992,534414190,2176718541,4139329115,1873836001,414664567,2282248934,4279200368,1711684554,285281116,2405801727,4167216745,1634467795,376229701,2685067896,3608007406,1308918612,956543938,2808555105,3495958263,1231636301,1047427035,2932959818,3654703836,1088359270,936918000,2847714899,3736837829,1202900863,817233897,3183342108,3401237130,1404277552,615818150,3134207493,3453421203,1423857449,601450431,3009837614,3294710456,1567103746,711928724,3020668471,3272380065,1510334235,755167117),crc32:function(string){var bytes,crc,i,iTop,n;bytes=this.bytesFor(string);crc=0;n=0;crc=crc^(-1);i=0;iTop=bytes.length;while(i<iTop){n=(crc^bytes[i])&255;crc=(crc>>>8)^this.CRC32_TABLE[n];i++}crc=crc^(-1);if(crc<0){crc+=4294967296}return crc},bytesFor:function(string){var bytes,i;bytes=[];i=0;while(i<string.length){bytes.push(string.charCodeAt(i));++i}return bytes}};
|
1
|
+
var JsonBloomfilter;JsonBloomfilter=function(options){var key,value;if(options==null){options={}}this.options={size:100,hashes:4,seed:new Date().getTime()/1000,bits:null};for(key in options){value=options[key];this.options[key]=value}this.bits=new JsonBloomfilter.BitArray(this.options.size,this.options.bits);return this};JsonBloomfilter.build=function(capacity,error_rate){var hashes,size;size=Math.ceil(capacity*Math.log(error_rate)/Math.log(1/Math.pow(2,Math.log(2))));hashes=Math.round(Math.log(2)*size/capacity);return new JsonBloomfilter({size:size,hashes:hashes})};JsonBloomfilter.prototype.add=function(key){var index,_i,_len,_ref;_ref=this.indexesFor(key);for(_i=0,_len=_ref.length;_i<_len;_i++){index=_ref[_i];this.bits.add(index)}};JsonBloomfilter.prototype.test=function(key){var index,_i,_len,_ref;_ref=this.indexesFor(key);for(_i=0,_len=_ref.length;_i<_len;_i++){index=_ref[_i];if(this.bits.get(index)===0){return false}}return true};JsonBloomfilter.prototype.clear=function(){this.bits=new JsonBloomfilter.BitArray(this.options.size)};JsonBloomfilter.prototype.toHash=function(){var hash,key,value,_ref;hash={};_ref=this.options;for(key in _ref){value=_ref[key];hash[key]=value}hash.bits=this.bits.field;return hash};JsonBloomfilter.prototype.toJson=function(){return JSON.stringify(this.toHash())};JsonBloomfilter.prototype.indexesFor=function(key){var index,indexes,_i,_ref;indexes=[];for(index=_i=0,_ref=this.options.hashes-1;0<=_ref?_i<=_ref:_i>=_ref;index=0<=_ref?++_i:--_i){indexes.push(JsonBloomfilter.Zlib.crc32(""+key+":"+(index+this.options.seed))%this.options.size)}return indexes};JsonBloomfilter.BitArray=function(size,field){var arrayLength,i,_i,_ref;if(field==null){field=null}if(!size){throw new Error("Missing argument: size")}this.ELEMENT_WIDTH=32;this.size=size;this.field=field||[];arrayLength=Math.floor(((size-1)/this.ELEMENT_WIDTH)+1);if(!field){for(i=_i=0,_ref=arrayLength-1;0<=_ref?_i<=_ref:_i>=_ref;i=0<=_ref?++_i:--_i){this.field[i]=0}}return this};JsonBloomfilter.BitArray.prototype.add=function(position){return this.set(position,1)};JsonBloomfilter.BitArray.prototype.remove=function(position){return this.set(position,0)};JsonBloomfilter.BitArray.prototype.set=function(position,value){var aPos,bChange;if(position>=this.size){throw new Error("BitArray index out of bounds")}aPos=this.arrayPosition(position);bChange=this.bitChange(position);if(value===1){this.field[aPos]|=bChange}else{if((this.field[aPos]&bChange)!==0){this.field[aPos]^=bChange}}return true};JsonBloomfilter.BitArray.prototype.get=function(position){var aPos,bChange;if(position>=this.size){throw new Error("BitArray index out of bounds")}aPos=this.arrayPosition(position);bChange=this.bitChange(position);if((this.field[aPos]&bChange)>0){return 1}else{return 0}};JsonBloomfilter.BitArray.prototype.arrayPosition=function(position){return Math.floor(position/this.ELEMENT_WIDTH)};JsonBloomfilter.BitArray.prototype.bitChange=function(position){return Math.abs(1<<position%this.ELEMENT_WIDTH)};JsonBloomfilter.BitArray.prototype.toString=function(){var i,output,_i,_ref;output="";for(i=_i=0,_ref=this.size-1;0<=_ref?_i<=_ref:_i>=_ref;i=0<=_ref?++_i:--_i){output+=this.get(i)}return output};JsonBloomfilter.Zlib={CRC32_TABLE:new Array(0,1996959894,3993919788,2567524794,124634137,1886057615,3915621685,2657392035,249268274,2044508324,3772115230,2547177864,162941995,2125561021,3887607047,2428444049,498536548,1789927666,4089016648,2227061214,450548861,1843258603,4107580753,2211677639,325883990,1684777152,4251122042,2321926636,335633487,1661365465,4195302755,2366115317,997073096,1281953886,3579855332,2724688242,1006888145,1258607687,3524101629,2768942443,901097722,1119000684,3686517206,2898065728,853044451,1172266101,3705015759,2882616665,651767980,1373503546,3369554304,3218104598,565507253,1454621731,3485111705,3099436303,671266974,1594198024,3322730930,2970347812,795835527,1483230225,3244367275,3060149565,1994146192,31158534,2563907772,4023717930,1907459465,112637215,2680153253,3904427059,2013776290,251722036,2517215374,3775830040,2137656763,141376813,2439277719,3865271297,1802195444,476864866,2238001368,4066508878,1812370925,453092731,2181625025,4111451223,1706088902,314042704,2344532202,4240017532,1658658271,366619977,2362670323,4224994405,1303535960,984961486,2747007092,3569037538,1256170817,1037604311,2765210733,3554079995,1131014506,879679996,2909243462,3663771856,1141124467,855842277,2852801631,3708648649,1342533948,654459306,3188396048,3373015174,1466479909,544179635,3110523913,3462522015,1591671054,702138776,2966460450,3352799412,1504918807,783551873,3082640443,3233442989,3988292384,2596254646,62317068,1957810842,3939845945,2647816111,81470997,1943803523,3814918930,2489596804,225274430,2053790376,3826175755,2466906013,167816743,2097651377,4027552580,2265490386,503444072,1762050814,4150417245,2154129355,426522225,1852507879,4275313526,2312317920,282753626,1742555852,4189708143,2394877945,397917763,1622183637,3604390888,2714866558,953729732,1340076626,3518719985,2797360999,1068828381,1219638859,3624741850,2936675148,906185462,1090812512,3747672003,2825379669,829329135,1181335161,3412177804,3160834842,628085408,1382605366,3423369109,3138078467,570562233,1426400815,3317316542,2998733608,733239954,1555261956,3268935591,3050360625,752459403,1541320221,2607071920,3965973030,1969922972,40735498,2617837225,3943577151,1913087877,83908371,2512341634,3803740692,2075208622,213261112,2463272603,3855990285,2094854071,198958881,2262029012,4057260610,1759359992,534414190,2176718541,4139329115,1873836001,414664567,2282248934,4279200368,1711684554,285281116,2405801727,4167216745,1634467795,376229701,2685067896,3608007406,1308918612,956543938,2808555105,3495958263,1231636301,1047427035,2932959818,3654703836,1088359270,936918000,2847714899,3736837829,1202900863,817233897,3183342108,3401237130,1404277552,615818150,3134207493,3453421203,1423857449,601450431,3009837614,3294710456,1567103746,711928724,3020668471,3272380065,1510334235,755167117),crc32:function(string){var bytes,crc,i,iTop,n;bytes=this.bytesFor(string);crc=0;n=0;crc=crc^(-1);i=0;iTop=bytes.length;while(i<iTop){n=(crc^bytes[i])&255;crc=(crc>>>8)^this.CRC32_TABLE[n];i++}crc=crc^(-1);if(crc<0){crc+=4294967296}return crc},bytesFor:function(string){var bytes,i;bytes=[];i=0;while(i<string.length){bytes.push(string.charCodeAt(i));++i}return bytes}};
|
data/lib/json/bloomfilter.rb
CHANGED
@@ -4,6 +4,12 @@ require "json"
|
|
4
4
|
class JsonBloomfilter
|
5
5
|
DEFAULTS = { "size" => 100, "hashes" => 4, "seed" => Time.new.to_i, "bits" => nil }
|
6
6
|
|
7
|
+
def self.build capacity, error_rate
|
8
|
+
size = (capacity * Math.log(error_rate) / Math.log(1.0 / 2**Math.log(2))).ceil
|
9
|
+
hashes = (Math.log(2) * size / capacity).round
|
10
|
+
JsonBloomfilter.new size: size, hashes: hashes
|
11
|
+
end
|
12
|
+
|
7
13
|
def initialize options = {}
|
8
14
|
@options = merge_defaults_with options
|
9
15
|
@bits = BitArray.new(@options["size"], @options["bits"])
|
Binary file
|
@@ -1,5 +1,13 @@
|
|
1
1
|
describe("JsonBloomfilter", function() {
|
2
2
|
|
3
|
+
describe(".build", function() {
|
4
|
+
it("should generate a BloomFilter with the right number of hashes and size", function() {
|
5
|
+
bf = JsonBloomfilter.build(1000, 0.01);
|
6
|
+
expect(bf.toHash()["hashes"]).toBe(7);
|
7
|
+
expect(bf.toHash()["size"]).toBe(9586);
|
8
|
+
});
|
9
|
+
});
|
10
|
+
|
3
11
|
describe("#initialize", function() {
|
4
12
|
it("should take the appropriate options", function() {
|
5
13
|
seed = (new Date().getTime()/1000) - 24*60*60;
|
@@ -2,6 +2,14 @@ require "spec_helper"
|
|
2
2
|
|
3
3
|
describe JsonBloomfilter do
|
4
4
|
|
5
|
+
describe ".build" do
|
6
|
+
it "should generate a BloomFilter with the right number of hashes and size" do
|
7
|
+
bf = JsonBloomfilter.build 1000, 0.01
|
8
|
+
expect(bf.to_hash["hashes"]).to be == 7
|
9
|
+
expect(bf.to_hash["size"]).to be == 9586
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
5
13
|
describe "#initialize" do
|
6
14
|
it "should take the appropriate options" do
|
7
15
|
seed = Time.now.to_i - 24*60*60
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: json-bloomfilter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -153,6 +153,7 @@ files:
|
|
153
153
|
- pkg/json-bloomfilter-0.0.2.gem
|
154
154
|
- pkg/json-bloomfilter-0.0.3.gem
|
155
155
|
- pkg/json-bloomfilter-0.0.4.gem
|
156
|
+
- pkg/json-bloomfilter-0.0.5.gem
|
156
157
|
- spec/javascripts/bitarray_spec.js
|
157
158
|
- spec/javascripts/bloomfilter_spec.js
|
158
159
|
- spec/javascripts/support/jasmine.yml
|
@@ -181,7 +182,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
181
182
|
version: '0'
|
182
183
|
segments:
|
183
184
|
- 0
|
184
|
-
hash: -
|
185
|
+
hash: -1108615832793611037
|
185
186
|
requirements: []
|
186
187
|
rubyforge_project:
|
187
188
|
rubygems_version: 1.8.24
|