lambda-store-hll 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 780babd237ad0634b5e0086ed5ae09056215661767041f79909b8ce75b892a87
4
+ data.tar.gz: 1f9ab99bb22294e8756ab343c2bcd7d08ab3e587c066663641506545978b78a6
5
+ SHA512:
6
+ metadata.gz: 116bd2ea92245fa72c48a4362cc60a19aa220eae4f7be6a11ac4d3eaee4c658410d8fc5a15067afb19fa757adb4427a81b37faee4b7782f0ce759f7285b872d5
7
+ data.tar.gz: d1f5a3968d1555cffe5c0872ce095897626248c6f3c3c563736b395c20b1700e264341d8652249d1b9c252cb4f18049573edcad1e88a11908c6be8110a0046e0
@@ -0,0 +1 @@
1
+ require_relative 'lambda-store-hll/counter.rb'
@@ -0,0 +1,95 @@
1
+ require 'Mmh3'
2
+
3
+ module LambdaStoreHLL
4
+ class Counter
5
+
6
+ def initialize(redis, b: 10)
7
+ raise "Accuracy not supported. Please choose a value of b between 4 and 16" if b < 4 || b > 16
8
+ @b = b
9
+ @m = 2 ** b
10
+ @redis = redis
11
+ @hash_len = 32 - b
12
+ @alpha = 0.7213/(1 + 1.079/@m)
13
+ end
14
+
15
+ ADD_SCRIPT = '
16
+ local index = tonumber(KEYS[2])
17
+ local counter = redis.call("GETRANGE", KEYS[1], index, index)
18
+ local str_len = tonumber(ARGV[2])
19
+
20
+ if(counter == "") then
21
+ local str = ""
22
+ for i=0,str_len-1 do
23
+ str = str .. (i == index and ARGV[1] or string.char(0))
24
+ end
25
+ redis.call("SET", KEYS[1], str)
26
+ return 1
27
+ elseif string.byte(ARGV[1]) > string.byte(counter) then
28
+ redis.call("SETRANGE", KEYS[1], index, ARGV[1])
29
+ return 1
30
+ end
31
+
32
+ return 0
33
+ '
34
+ ADD_SCRIPT_MIN = 'local a=tonumber(KEYS[2])local b=redis.call("GETRANGE",KEYS[1],a,a)local c=tonumber(ARGV[2])if b==""then local d=""for e=0,c-1 do d=d..(e==a and ARGV[1]or string.char(0))end;redis.call("SET",KEYS[1],d)return 1 elseif string.byte(ARGV[1])>string.byte(b)then redis.call("SETRANGE",KEYS[1],a,ARGV[1])return 1 end;return 0'
35
+
36
+ COUNT_SCRIPT = '
37
+ local m = tonumber(ARGV[1])
38
+ local alpha = tonumber(ARGV[2])
39
+ local value = 0
40
+ local length = 0
41
+
42
+ local str = redis.call("GET", KEYS[1])
43
+
44
+ if not str then
45
+ return 0
46
+ end
47
+
48
+ for i=1,m do
49
+ local max = string.byte(str:sub(i,i))
50
+ if max > 0 then
51
+ length = length + 1
52
+ end
53
+ value = value + 0.5 ^ max
54
+ end
55
+
56
+ local estimate = m ^ 2 * alpha / value
57
+
58
+ --linear count
59
+ if estimate < 2.5 * m and (length < m) then
60
+ return (m * math.log(m/(m - length)))
61
+ end
62
+
63
+ return math.floor(estimate + 0.5)
64
+ '
65
+ COUNT_SCRIPT_MIN = 'local a=tonumber(ARGV[1])local b=tonumber(ARGV[2])local c=0;local d=0;local e=redis.call("GET",KEYS[1])if not e then return 0 end;for f=1,a do local g=string.byte(e:sub(f,f))if g>0 then d=d+1 end;c=c+0.5^g end;local h=a^2*b/c;if h<2.5*a and d<a then return a*math.log(a/(a-d))end;return math.floor(h+0.5)'
66
+
67
+ # do the hashing on this end to save excessive amounts of script being sent
68
+ def add(k, v)
69
+ #hash the string
70
+ hash = Mmh3.hash32(v)
71
+ #remove negatives but keep info
72
+ hash = ((hash.abs << 1) | (hash.negative? ? 1 : 0))
73
+
74
+ #get the 10 bit bucket id
75
+ int_bucket = (hash & (@m - 1))
76
+
77
+ @redis.eval(ADD_SCRIPT_MIN, :keys => [k, int_bucket], :argv => [rho(hash / @m).chr, @m])
78
+ end
79
+
80
+ def count(k)
81
+ @redis.eval(COUNT_SCRIPT_MIN, :keys => [k], :argv => [@m, @alpha])
82
+ end
83
+
84
+ def del(k)
85
+ @redis.del(k)
86
+ end
87
+
88
+ private
89
+
90
+ def rho(i)
91
+ return @hash_len + 1 if i == 0
92
+ @hash_len - Math.log(i, 2).floor
93
+ end
94
+ end
95
+ end
metadata ADDED
@@ -0,0 +1,86 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: lambda-store-hll
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.0
5
+ platform: ruby
6
+ authors:
7
+ - Aiden Leeming
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2020-12-01 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: redis
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: mmh3
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ description:
56
+ email:
57
+ executables: []
58
+ extensions: []
59
+ extra_rdoc_files: []
60
+ files:
61
+ - lib/lambda-store-hll.rb
62
+ - lib/lambda-store-hll/counter.rb
63
+ homepage: https://github.com/Aidenjl193/lambda-store-hll
64
+ licenses:
65
+ - MIT
66
+ metadata: {}
67
+ post_install_message:
68
+ rdoc_options: []
69
+ require_paths:
70
+ - lib
71
+ required_ruby_version: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ required_rubygems_version: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - ">="
79
+ - !ruby/object:Gem::Version
80
+ version: '0'
81
+ requirements: []
82
+ rubygems_version: 3.1.4
83
+ signing_key:
84
+ specification_version: 4
85
+ summary: A library to add hyperloglog functionality to lambda store in ruby
86
+ test_files: []