json-bloomfilter 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile.lock +27 -1
- data/Rakefile +74 -0
- data/coffee/bitarray.coffee +46 -0
- data/coffee/bloomfilter.coffee +40 -0
- data/coffee/zlib.coffee +31 -0
- data/js/bloomfilter.min.js +1 -0
- data/json-bloomfilter.gemspec +8 -3
- data/lib/json/bitarray.rb +45 -0
- data/lib/json/bloomfilter/version.rb +3 -0
- data/lib/json/bloomfilter.rb +43 -0
- data/pkg/json-bloomfilter-0.0.1.gem +0 -0
- data/pkg/json-bloomfilter-0.0.2.gem +0 -0
- data/spec/json-bloomfilter_spec.rb +1 -1
- data/spec/spec_helper.rb +1 -1
- metadata +95 -11
- data/lib/json-bloomfilter/version.rb +0 -3
- data/lib/json-bloomfilter.rb +0 -3
data/Gemfile.lock
CHANGED
@@ -1,13 +1,29 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
json-bloomfilter (0.0.
|
4
|
+
json-bloomfilter (0.0.2)
|
5
5
|
|
6
6
|
GEM
|
7
7
|
remote: http://rubygems.org/
|
8
8
|
specs:
|
9
|
+
POpen4 (0.1.4)
|
10
|
+
Platform (>= 0.4.0)
|
11
|
+
open4
|
12
|
+
Platform (0.4.0)
|
13
|
+
coffee-script (2.2.0)
|
14
|
+
coffee-script-source
|
15
|
+
execjs
|
16
|
+
coffee-script-source (1.4.0)
|
9
17
|
diff-lcs (1.1.3)
|
18
|
+
execjs (1.4.0)
|
19
|
+
multi_json (~> 1.0)
|
20
|
+
fssm (0.2.9)
|
21
|
+
libv8 (3.11.8.13)
|
22
|
+
multi_json (1.5.0)
|
23
|
+
open4 (1.3.0)
|
10
24
|
rake (10.0.3)
|
25
|
+
rb-fsevent (0.9.3)
|
26
|
+
ref (1.0.2)
|
11
27
|
rspec (2.12.0)
|
12
28
|
rspec-core (~> 2.12.0)
|
13
29
|
rspec-expectations (~> 2.12.0)
|
@@ -16,11 +32,21 @@ GEM
|
|
16
32
|
rspec-expectations (2.12.1)
|
17
33
|
diff-lcs (~> 1.1.3)
|
18
34
|
rspec-mocks (2.12.1)
|
35
|
+
therubyracer (0.11.3)
|
36
|
+
libv8 (~> 3.11.8.12)
|
37
|
+
ref
|
38
|
+
yui-compressor (0.9.6)
|
39
|
+
POpen4 (>= 0.1.4)
|
19
40
|
|
20
41
|
PLATFORMS
|
21
42
|
ruby
|
22
43
|
|
23
44
|
DEPENDENCIES
|
45
|
+
coffee-script
|
46
|
+
fssm
|
24
47
|
json-bloomfilter!
|
25
48
|
rake
|
49
|
+
rb-fsevent
|
26
50
|
rspec
|
51
|
+
therubyracer
|
52
|
+
yui-compressor
|
data/Rakefile
CHANGED
@@ -6,4 +6,78 @@ task :default => [:test]
|
|
6
6
|
desc "Run tests"
|
7
7
|
task :test do
|
8
8
|
sh "bundle exec rspec spec"
|
9
|
+
end
|
10
|
+
|
11
|
+
|
12
|
+
desc "Autobuild JS/SCSS"
|
13
|
+
task :watch do
|
14
|
+
require "bundler/setup"
|
15
|
+
require "fssm"
|
16
|
+
|
17
|
+
monitor = FSSM::Monitor.new
|
18
|
+
|
19
|
+
monitor.path "coffee", "**/*.coffee" do
|
20
|
+
compile_block = -> basename, filename do
|
21
|
+
puts "Compiling coffee to minified js..."
|
22
|
+
coffee_file = "coffee/#{filename}"
|
23
|
+
puts "\t#{coffee_file}"
|
24
|
+
Rake::Task["js:compile_file"].tap(&:reenable).invoke(coffee_file)
|
25
|
+
Rake::Task["js:minify_file"].tap(&:reenable).invoke(coffee_file)
|
26
|
+
end
|
27
|
+
|
28
|
+
update &compile_block
|
29
|
+
delete &compile_block
|
30
|
+
create &compile_block
|
31
|
+
end
|
32
|
+
|
33
|
+
monitor.run
|
34
|
+
end
|
35
|
+
|
36
|
+
|
37
|
+
namespace :js do
|
38
|
+
desc "Compiles CS & minifies JS"
|
39
|
+
task :compile do
|
40
|
+
puts "Compiling coffee to minified js..."
|
41
|
+
Rake::Task["js:compile_cs"].tap(&:reenable).invoke()
|
42
|
+
Rake::Task["js:minify_js"].tap(&:reenable).invoke()
|
43
|
+
puts "\t\t->done\n\n"
|
44
|
+
end
|
45
|
+
|
46
|
+
task :compile_cs, :filename do |t, args|
|
47
|
+
require "coffee-script"
|
48
|
+
|
49
|
+
coffee_script = ""
|
50
|
+
|
51
|
+
%w{bloomfilter bitarray zlib}.each do |library|
|
52
|
+
filename = "coffee/#{library}.coffee"
|
53
|
+
puts "\t#{filename}"
|
54
|
+
coffee_script += File.read(filename) + "\n"
|
55
|
+
end
|
56
|
+
|
57
|
+
js_file = "js/bloomfilter.js"
|
58
|
+
directory_name = File.dirname js_file
|
59
|
+
FileUtils.mkdir_p(directory_name) unless File.exists? directory_name
|
60
|
+
puts "\t\t->#{js_file}"
|
61
|
+
File.open(js_file, "w+") do |f|
|
62
|
+
f.print CoffeeScript.compile(coffee_script, no_wrap: true)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
task :minify_js, :filename do |t, args|
|
67
|
+
require "yui/compressor"
|
68
|
+
filename = args[:filename]
|
69
|
+
compressor = YUI::JavaScriptCompressor.new
|
70
|
+
|
71
|
+
js_file = "js/bloomfilter.js"
|
72
|
+
min_file = "js/bloomfilter.min.js"
|
73
|
+
|
74
|
+
puts "\t\t->#{min_file}"
|
75
|
+
directory_name = File.dirname min_file
|
76
|
+
FileUtils.mkdir_p(directory_name) unless File.exists? directory_name
|
77
|
+
File.open(min_file, "w+") do |f|
|
78
|
+
f.print compressor.compress(File.read(js_file))
|
79
|
+
end
|
80
|
+
File.delete js_file
|
81
|
+
end
|
82
|
+
|
9
83
|
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
JsonBloomfilter.BitArray =
|
2
|
+
size: null
|
3
|
+
field: null
|
4
|
+
|
5
|
+
ELEMENT_WIDTH: 32
|
6
|
+
|
7
|
+
new: (size, field = null) ->
|
8
|
+
@size = size
|
9
|
+
arrayLength = Math.floor(((size - 1) / @ELEMENT_WIDTH) + 1)
|
10
|
+
@field = field || []
|
11
|
+
@field[i] = 0 for i in [0..arrayLength] unless field
|
12
|
+
this
|
13
|
+
|
14
|
+
add: (position) ->
|
15
|
+
@set(position, 1)
|
16
|
+
|
17
|
+
remove: (position) ->
|
18
|
+
@set(position, 0)
|
19
|
+
|
20
|
+
set: (position, value) ->
|
21
|
+
aPos = @arrayPosition(position)
|
22
|
+
bChange = @bitChange(position)
|
23
|
+
if value == 1
|
24
|
+
@field[aPos] |= bChange
|
25
|
+
else if @field[aPos] & bChange != 0
|
26
|
+
@field[aPos] ^= bChange
|
27
|
+
true
|
28
|
+
|
29
|
+
get: (position) ->
|
30
|
+
aPos = @arrayPosition(position)
|
31
|
+
bChange = @bitChange(position)
|
32
|
+
if (@field[aPos] & bChange) > 0
|
33
|
+
return 1
|
34
|
+
else
|
35
|
+
return 0
|
36
|
+
|
37
|
+
arrayPosition: (position) ->
|
38
|
+
Math.floor(position / @ELEMENT_WIDTH)
|
39
|
+
|
40
|
+
bitChange: (position) ->
|
41
|
+
Math.abs(1 << position % @ELEMENT_WIDTH)
|
42
|
+
|
43
|
+
toString: ->
|
44
|
+
output = ""
|
45
|
+
output += @get(i) for i in [0..@size-1]
|
46
|
+
output
|
@@ -0,0 +1,40 @@
|
|
1
|
+
JsonBloomfilter =
|
2
|
+
bits: null
|
3
|
+
options:
|
4
|
+
size: 100,
|
5
|
+
hashes: 4,
|
6
|
+
seed: (new Date().getTime() / 1000),
|
7
|
+
bits: null
|
8
|
+
|
9
|
+
new: (options = {}) ->
|
10
|
+
@options[key] = value for key, value of options
|
11
|
+
@bits = @BitArray.new(@options["size"], @options["bits"])
|
12
|
+
this
|
13
|
+
|
14
|
+
add: (key) ->
|
15
|
+
@bits.add(index) for index in @indexesFor(key)
|
16
|
+
return
|
17
|
+
|
18
|
+
test: (key) ->
|
19
|
+
for index in @indexesFor(key)
|
20
|
+
return false if @bits.get(index) == 0
|
21
|
+
true
|
22
|
+
|
23
|
+
clear: ->
|
24
|
+
@bits = @BitArray.new(@options["size"], null)
|
25
|
+
return
|
26
|
+
|
27
|
+
toHash: ->
|
28
|
+
hash = {}
|
29
|
+
hash[key] = value for key, value of @options
|
30
|
+
hash["bits"] = @bits.field
|
31
|
+
hash
|
32
|
+
|
33
|
+
toJson: ->
|
34
|
+
JSON.stringify @toHash()
|
35
|
+
|
36
|
+
indexesFor: (key) ->
|
37
|
+
indexes = []
|
38
|
+
for index in [0..@options["hashes"]-1]
|
39
|
+
indexes.push (@Zlib.crc32("#{key}:#{index+@options["seed"]}") % @options["size"])
|
40
|
+
indexes
|
data/coffee/zlib.coffee
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
# Based of:
|
2
|
+
# https://github.com/mikepulaski/node-crc32
|
3
|
+
# http://stackoverflow.com/questions/6226189/how-to-convert-a-string-to-bytearray/10132540#10132540
|
4
|
+
|
5
|
+
JsonBloomfilter.Zlib =
|
6
|
+
CRC32_TABLE: new Array(0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D)
|
7
|
+
|
8
|
+
crc32: (string) ->
|
9
|
+
bytes = @bytesFor(string)
|
10
|
+
crc = 0
|
11
|
+
n = 0
|
12
|
+
crc = crc ^ (-1)
|
13
|
+
i = 0
|
14
|
+
iTop = bytes.length
|
15
|
+
|
16
|
+
while i < iTop
|
17
|
+
n = (crc ^ bytes[i]) & 0xFF
|
18
|
+
crc = (crc >>> 8) ^ @CRC32_TABLE[n]
|
19
|
+
i++
|
20
|
+
crc = crc ^ (-1)
|
21
|
+
crc += 4294967296 if crc < 0
|
22
|
+
crc
|
23
|
+
|
24
|
+
bytesFor: (string) ->
|
25
|
+
bytes = []
|
26
|
+
i = 0
|
27
|
+
|
28
|
+
while i < string.length
|
29
|
+
bytes.push string.charCodeAt(i)
|
30
|
+
++i
|
31
|
+
bytes
|
@@ -0,0 +1 @@
|
|
1
|
+
var JsonBloomfilter;JsonBloomfilter={bits:null,options:{size:100,hashes:4,seed:new Date().getTime()/1000,bits:null},"new":function(options){var key,value;if(options==null){options={}}for(key in options){value=options[key];this.options[key]=value}this.bits=this.BitArray["new"](this.options.size,this.options.bits);return this},add:function(key){var index,_i,_len,_ref;_ref=this.indexesFor(key);for(_i=0,_len=_ref.length;_i<_len;_i++){index=_ref[_i];this.bits.add(index)}},test:function(key){var index,_i,_len,_ref;_ref=this.indexesFor(key);for(_i=0,_len=_ref.length;_i<_len;_i++){index=_ref[_i];if(this.bits.get(index)===0){return false}}return true},clear:function(){this.bits=this.BitArray["new"](this.options.size,null)},toHash:function(){var hash,key,value,_ref;hash={};_ref=this.options;for(key in _ref){value=_ref[key];hash[key]=value}hash.bits=this.bits.field;return hash},toJson:function(){return JSON.stringify(this.toHash())},indexesFor:function(key){var index,indexes,_i,_ref;indexes=[];for(index=_i=0,_ref=this.options.hashes-1;0<=_ref?_i<=_ref:_i>=_ref;index=0<=_ref?++_i:--_i){indexes.push(this.Zlib.crc32(""+key+":"+(index+this.options.seed))%this.options.size)}return indexes}};JsonBloomfilter.BitArray={size:null,field:null,ELEMENT_WIDTH:32,"new":function(size,field){var arrayLength,i,_i;if(field==null){field=null}this.size=size;arrayLength=Math.floor(((size-1)/this.ELEMENT_WIDTH)+1);this.field=field||[];if(!field){for(i=_i=0;0<=arrayLength?_i<=arrayLength:_i>=arrayLength;i=0<=arrayLength?++_i:--_i){this.field[i]=0}}return this},add:function(position){return this.set(position,1)},remove:function(position){return this.set(position,0)},set:function(position,value){var aPos,bChange;aPos=this.arrayPosition(position);bChange=this.bitChange(position);if(value===1){this.field[aPos]|=bChange}else{if(this.field[aPos]&bChange!==0){this.field[aPos]^=bChange}}return true},get:function(position){var aPos,bChange;aPos=this.arrayPosition(position);bChange=this.bitChange(position);if((this.field[aPos]&bChange)>0){return 1}else{return 0}},arrayPosition:function(position){return Math.floor(position/this.ELEMENT_WIDTH)},bitChange:function(position){return Math.abs(1<<position%this.ELEMENT_WIDTH)},toString:function(){var i,output,_i,_ref;output="";for(i=_i=0,_ref=this.size-1;0<=_ref?_i<=_ref:_i>=_ref;i=0<=_ref?++_i:--_i){output+=this.get(i)}return output}};JsonBloomfilter.Zlib={CRC32_TABLE:new Array(0,1996959894,3993919788,2567524794,124634137,1886057615,3915621685,2657392035,249268274,2044508324,3772115230,2547177864,162941995,2125561021,3887607047,2428444049,498536548,1789927666,4089016648,2227061214,450548861,1843258603,4107580753,2211677639,325883990,1684777152,4251122042,2321926636,335633487,1661365465,4195302755,2366115317,997073096,1281953886,3579855332,2724688242,1006888145,1258607687,3524101629,2768942443,901097722,1119000684,3686517206,2898065728,853044451,1172266101,3705015759,2882616665,651767980,1373503546,3369554304,3218104598,565507253,1454621731,3485111705,3099436303,671266974,1594198024,3322730930,2970347812,795835527,1483230225,3244367275,3060149565,1994146192,31158534,2563907772,4023717930,1907459465,112637215,2680153253,3904427059,2013776290,251722036,2517215374,3775830040,2137656763,141376813,2439277719,3865271297,1802195444,476864866,2238001368,4066508878,1812370925,453092731,2181625025,4111451223,1706088902,314042704,2344532202,4240017532,1658658271,366619977,2362670323,4224994405,1303535960,984961486,2747007092,3569037538,1256170817,1037604311,2765210733,3554079995,1131014506,879679996,2909243462,3663771856,1141124467,855842277,2852801631,3708648649,1342533948,654459306,3188396048,3373015174,1466479909,544179635,3110523913,3462522015,1591671054,702138776,2966460450,3352799412,1504918807,783551873,3082640443,3233442989,3988292384,2596254646,62317068,1957810842,3939845945,2647816111,81470997,1943803523,3814918930,2489596804,225274430,2053790376,3826175755,2466906013,167816743,2097651377,4027552580,2265490386,503444072,1762050814,4150417245,2154129355,426522225,1852507879,4275313526,2312317920,282753626,1742555852,4189708143,2394877945,397917763,1622183637,3604390888,2714866558,953729732,1340076626,3518719985,2797360999,1068828381,1219638859,3624741850,2936675148,906185462,1090812512,3747672003,2825379669,829329135,1181335161,3412177804,3160834842,628085408,1382605366,3423369109,3138078467,570562233,1426400815,3317316542,2998733608,733239954,1555261956,3268935591,3050360625,752459403,1541320221,2607071920,3965973030,1969922972,40735498,2617837225,3943577151,1913087877,83908371,2512341634,3803740692,2075208622,213261112,2463272603,3855990285,2094854071,198958881,2262029012,4057260610,1759359992,534414190,2176718541,4139329115,1873836001,414664567,2282248934,4279200368,1711684554,285281116,2405801727,4167216745,1634467795,376229701,2685067896,3608007406,1308918612,956543938,2808555105,3495958263,1231636301,1047427035,2932959818,3654703836,1088359270,936918000,2847714899,3736837829,1202900863,817233897,3183342108,3401237130,1404277552,615818150,3134207493,3453421203,1423857449,601450431,3009837614,3294710456,1567103746,711928724,3020668471,3272380065,1510334235,755167117),crc32:function(string){var bytes,crc,i,iTop,n;bytes=this.bytesFor(string);crc=0;n=0;crc=crc^(-1);i=0;iTop=bytes.length;while(i<iTop){n=(crc^bytes[i])&255;crc=(crc>>>8)^this.CRC32_TABLE[n];i++}crc=crc^(-1);if(crc<0){crc+=4294967296}return crc},bytesFor:function(string){var bytes,i;bytes=[];i=0;while(i<string.length){bytes.push(string.charCodeAt(i));++i}return bytes}};
|
data/json-bloomfilter.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
$:.push File.expand_path("../lib", __FILE__)
|
3
|
-
require "json
|
3
|
+
require "json/bloomfilter/version"
|
4
4
|
|
5
5
|
Gem::Specification.new do |s|
|
6
6
|
s.name = "json-bloomfilter"
|
@@ -10,12 +10,17 @@ Gem::Specification.new do |s|
|
|
10
10
|
s.homepage = "http://github.com/cbetta/json-bloomfilter"
|
11
11
|
s.summary = "A bloomfilter implementation in both Ruby and Javascript."
|
12
12
|
s.description = "A bloomfilter implementation in both Ruby and Javascript that can be serialised to and loaded from JSON. Very useful when needing to train a bloom filter in one language and using it in the other."
|
13
|
-
|
14
|
-
s.
|
13
|
+
s.license = 'MIT'
|
14
|
+
s.required_ruby_version = '>= 1.9.0'
|
15
15
|
|
16
16
|
s.files = `git ls-files`.split("\n")
|
17
17
|
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
18
18
|
s.require_paths = ["lib"]
|
19
19
|
|
20
20
|
s.add_development_dependency 'rspec'
|
21
|
+
s.add_development_dependency 'yui-compressor'
|
22
|
+
s.add_development_dependency 'therubyracer'
|
23
|
+
s.add_development_dependency 'coffee-script'
|
24
|
+
s.add_development_dependency 'fssm'
|
25
|
+
s.add_development_dependency 'rb-fsevent'
|
21
26
|
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# Adapted from
|
2
|
+
# https://github.com/peterc/bitarray
|
3
|
+
|
4
|
+
class BitArray
|
5
|
+
attr_reader :size
|
6
|
+
attr_reader :field
|
7
|
+
include Enumerable
|
8
|
+
|
9
|
+
ELEMENT_WIDTH = 32
|
10
|
+
|
11
|
+
def initialize(size, field = nil)
|
12
|
+
@size = size
|
13
|
+
@field = field || Array.new(((size - 1) / ELEMENT_WIDTH) + 1, 0)
|
14
|
+
end
|
15
|
+
|
16
|
+
# Set a bit (1/0)
|
17
|
+
def []=(position, value)
|
18
|
+
if value == 1
|
19
|
+
@field[position / ELEMENT_WIDTH] |= 1 << (position % ELEMENT_WIDTH)
|
20
|
+
elsif (@field[position / ELEMENT_WIDTH]) & (1 << (position % ELEMENT_WIDTH)) != 0
|
21
|
+
@field[position / ELEMENT_WIDTH] ^= 1 << (position % ELEMENT_WIDTH)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
# Read a bit (1/0)
|
26
|
+
def [](position)
|
27
|
+
@field[position / ELEMENT_WIDTH] & 1 << (position % ELEMENT_WIDTH) > 0 ? 1 : 0
|
28
|
+
end
|
29
|
+
|
30
|
+
# Iterate over each bit
|
31
|
+
def each(&block)
|
32
|
+
@size.times { |position| yield self[position] }
|
33
|
+
end
|
34
|
+
|
35
|
+
# Returns the field as a string like "0101010100111100," etc.
|
36
|
+
def to_s
|
37
|
+
inject("") { |a, b| a + b.to_s }
|
38
|
+
end
|
39
|
+
|
40
|
+
# Returns the total number of bits that are set
|
41
|
+
# (The technique used here is about 6 times faster than using each or inject direct on the bitfield)
|
42
|
+
def total_set
|
43
|
+
@field.inject(0) { |a, byte| a += byte & 1 and byte >>= 1 until byte == 0; a }
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require_relative "bitarray"
|
2
|
+
require "json"
|
3
|
+
|
4
|
+
class JsonBloomfilter
|
5
|
+
DEFAULTS = { "size" => 100, "hashes" => 4, "seed" => Time.new.to_i, "bits" => nil }
|
6
|
+
|
7
|
+
def initialize options = {}
|
8
|
+
@options = DEFAULTS.merge options
|
9
|
+
@bits = BitArray.new(@options["size"], @options["bits"])
|
10
|
+
end
|
11
|
+
|
12
|
+
def add key
|
13
|
+
indexes_for(key).each { |index| @bits[index] = 1 }
|
14
|
+
nil
|
15
|
+
end
|
16
|
+
|
17
|
+
def test key
|
18
|
+
indexes_for(key).each do |index|
|
19
|
+
return false if @bits[index] == 0
|
20
|
+
end
|
21
|
+
true
|
22
|
+
end
|
23
|
+
|
24
|
+
def clear
|
25
|
+
@bits = BitArray.new(@options["size"], 0)
|
26
|
+
end
|
27
|
+
|
28
|
+
def to_hash
|
29
|
+
@options.merge({ "bits" => @bits.field })
|
30
|
+
end
|
31
|
+
|
32
|
+
def to_json
|
33
|
+
JSON.generate to_hash
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def indexes_for key
|
39
|
+
Array.new(@options["hashes"]).each_with_index.map do |item, index|
|
40
|
+
Zlib.crc32("#{key}:#{index+@options["seed"]}") % @options["size"]
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
Binary file
|
Binary file
|
data/spec/spec_helper.rb
CHANGED
@@ -1 +1 @@
|
|
1
|
-
require "json
|
1
|
+
require "json/bloomfilter"
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: json-bloomfilter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-01-
|
12
|
+
date: 2013-01-18 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rspec
|
@@ -27,6 +27,86 @@ dependencies:
|
|
27
27
|
- - ! '>='
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: yui-compressor
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: therubyracer
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: coffee-script
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: fssm
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
type: :development
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: rb-fsevent
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ! '>='
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0'
|
102
|
+
type: :development
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
30
110
|
description: A bloomfilter implementation in both Ruby and Javascript that can be
|
31
111
|
serialised to and loaded from JSON. Very useful when needing to train a bloom filter
|
32
112
|
in one language and using it in the other.
|
@@ -42,14 +122,21 @@ files:
|
|
42
122
|
- LICENSE
|
43
123
|
- README.md
|
44
124
|
- Rakefile
|
125
|
+
- coffee/bitarray.coffee
|
126
|
+
- coffee/bloomfilter.coffee
|
127
|
+
- coffee/zlib.coffee
|
128
|
+
- js/bloomfilter.min.js
|
45
129
|
- json-bloomfilter.gemspec
|
46
|
-
- lib/json
|
47
|
-
- lib/json
|
130
|
+
- lib/json/bitarray.rb
|
131
|
+
- lib/json/bloomfilter.rb
|
132
|
+
- lib/json/bloomfilter/version.rb
|
48
133
|
- pkg/json-bloomfilter-0.0.1.gem
|
134
|
+
- pkg/json-bloomfilter-0.0.2.gem
|
49
135
|
- spec/json-bloomfilter_spec.rb
|
50
136
|
- spec/spec_helper.rb
|
51
137
|
homepage: http://github.com/cbetta/json-bloomfilter
|
52
|
-
licenses:
|
138
|
+
licenses:
|
139
|
+
- MIT
|
53
140
|
post_install_message:
|
54
141
|
rdoc_options: []
|
55
142
|
require_paths:
|
@@ -59,10 +146,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
59
146
|
requirements:
|
60
147
|
- - ! '>='
|
61
148
|
- !ruby/object:Gem::Version
|
62
|
-
version:
|
63
|
-
segments:
|
64
|
-
- 0
|
65
|
-
hash: 1781331154356047358
|
149
|
+
version: 1.9.0
|
66
150
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
67
151
|
none: false
|
68
152
|
requirements:
|
@@ -71,9 +155,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
71
155
|
version: '0'
|
72
156
|
segments:
|
73
157
|
- 0
|
74
|
-
hash:
|
158
|
+
hash: 1888125834509649748
|
75
159
|
requirements: []
|
76
|
-
rubyforge_project:
|
160
|
+
rubyforge_project:
|
77
161
|
rubygems_version: 1.8.24
|
78
162
|
signing_key:
|
79
163
|
specification_version: 3
|
data/lib/json-bloomfilter.rb
DELETED