lambda-store-hll 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/lambda-store-hll.rb +1 -0
- data/lib/lambda-store-hll/counter.rb +95 -0
- metadata +86 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 780babd237ad0634b5e0086ed5ae09056215661767041f79909b8ce75b892a87
|
4
|
+
data.tar.gz: 1f9ab99bb22294e8756ab343c2bcd7d08ab3e587c066663641506545978b78a6
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 116bd2ea92245fa72c48a4362cc60a19aa220eae4f7be6a11ac4d3eaee4c658410d8fc5a15067afb19fa757adb4427a81b37faee4b7782f0ce759f7285b872d5
|
7
|
+
data.tar.gz: d1f5a3968d1555cffe5c0872ce095897626248c6f3c3c563736b395c20b1700e264341d8652249d1b9c252cb4f18049573edcad1e88a11908c6be8110a0046e0
|
@@ -0,0 +1 @@
|
|
1
|
+
require_relative 'lambda-store-hll/counter.rb'
|
@@ -0,0 +1,95 @@
|
|
1
|
+
require 'Mmh3'
|
2
|
+
|
3
|
+
module LambdaStoreHLL
|
4
|
+
class Counter
|
5
|
+
|
6
|
+
def initialize(redis, b: 10)
|
7
|
+
raise "Accuracy not supported. Please choose a value of b between 4 and 16" if b < 4 || b > 16
|
8
|
+
@b = b
|
9
|
+
@m = 2 ** b
|
10
|
+
@redis = redis
|
11
|
+
@hash_len = 32 - b
|
12
|
+
@alpha = 0.7213/(1 + 1.079/@m)
|
13
|
+
end
|
14
|
+
|
15
|
+
ADD_SCRIPT = '
|
16
|
+
local index = tonumber(KEYS[2])
|
17
|
+
local counter = redis.call("GETRANGE", KEYS[1], index, index)
|
18
|
+
local str_len = tonumber(ARGV[2])
|
19
|
+
|
20
|
+
if(counter == "") then
|
21
|
+
local str = ""
|
22
|
+
for i=0,str_len-1 do
|
23
|
+
str = str .. (i == index and ARGV[1] or string.char(0))
|
24
|
+
end
|
25
|
+
redis.call("SET", KEYS[1], str)
|
26
|
+
return 1
|
27
|
+
elseif string.byte(ARGV[1]) > string.byte(counter) then
|
28
|
+
redis.call("SETRANGE", KEYS[1], index, ARGV[1])
|
29
|
+
return 1
|
30
|
+
end
|
31
|
+
|
32
|
+
return 0
|
33
|
+
'
|
34
|
+
ADD_SCRIPT_MIN = 'local a=tonumber(KEYS[2])local b=redis.call("GETRANGE",KEYS[1],a,a)local c=tonumber(ARGV[2])if b==""then local d=""for e=0,c-1 do d=d..(e==a and ARGV[1]or string.char(0))end;redis.call("SET",KEYS[1],d)return 1 elseif string.byte(ARGV[1])>string.byte(b)then redis.call("SETRANGE",KEYS[1],a,ARGV[1])return 1 end;return 0'
|
35
|
+
|
36
|
+
COUNT_SCRIPT = '
|
37
|
+
local m = tonumber(ARGV[1])
|
38
|
+
local alpha = tonumber(ARGV[2])
|
39
|
+
local value = 0
|
40
|
+
local length = 0
|
41
|
+
|
42
|
+
local str = redis.call("GET", KEYS[1])
|
43
|
+
|
44
|
+
if not str then
|
45
|
+
return 0
|
46
|
+
end
|
47
|
+
|
48
|
+
for i=1,m do
|
49
|
+
local max = string.byte(str:sub(i,i))
|
50
|
+
if max > 0 then
|
51
|
+
length = length + 1
|
52
|
+
end
|
53
|
+
value = value + 0.5 ^ max
|
54
|
+
end
|
55
|
+
|
56
|
+
local estimate = m ^ 2 * alpha / value
|
57
|
+
|
58
|
+
--linear count
|
59
|
+
if estimate < 2.5 * m and (length < m) then
|
60
|
+
return (m * math.log(m/(m - length)))
|
61
|
+
end
|
62
|
+
|
63
|
+
return math.floor(estimate + 0.5)
|
64
|
+
'
|
65
|
+
COUNT_SCRIPT_MIN = 'local a=tonumber(ARGV[1])local b=tonumber(ARGV[2])local c=0;local d=0;local e=redis.call("GET",KEYS[1])if not e then return 0 end;for f=1,a do local g=string.byte(e:sub(f,f))if g>0 then d=d+1 end;c=c+0.5^g end;local h=a^2*b/c;if h<2.5*a and d<a then return a*math.log(a/(a-d))end;return math.floor(h+0.5)'
|
66
|
+
|
67
|
+
# do the hashing on this end to save excessive amounts of script being sent
|
68
|
+
def add(k, v)
|
69
|
+
#hash the string
|
70
|
+
hash = Mmh3.hash32(v)
|
71
|
+
#remove negatives but keep info
|
72
|
+
hash = ((hash.abs << 1) | (hash.negative? ? 1 : 0))
|
73
|
+
|
74
|
+
#get the 10 bit bucket id
|
75
|
+
int_bucket = (hash & (@m - 1))
|
76
|
+
|
77
|
+
@redis.eval(ADD_SCRIPT_MIN, :keys => [k, int_bucket], :argv => [rho(hash / @m).chr, @m])
|
78
|
+
end
|
79
|
+
|
80
|
+
def count(k)
|
81
|
+
@redis.eval(COUNT_SCRIPT_MIN, :keys => [k], :argv => [@m, @alpha])
|
82
|
+
end
|
83
|
+
|
84
|
+
def del(k)
|
85
|
+
@redis.del(k)
|
86
|
+
end
|
87
|
+
|
88
|
+
private
|
89
|
+
|
90
|
+
def rho(i)
|
91
|
+
return @hash_len + 1 if i == 0
|
92
|
+
@hash_len - Math.log(i, 2).floor
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
metadata
ADDED
@@ -0,0 +1,86 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: lambda-store-hll
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Aiden Leeming
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-12-01 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: redis
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: mmh3
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
description:
|
56
|
+
email:
|
57
|
+
executables: []
|
58
|
+
extensions: []
|
59
|
+
extra_rdoc_files: []
|
60
|
+
files:
|
61
|
+
- lib/lambda-store-hll.rb
|
62
|
+
- lib/lambda-store-hll/counter.rb
|
63
|
+
homepage: https://github.com/Aidenjl193/lambda-store-hll
|
64
|
+
licenses:
|
65
|
+
- MIT
|
66
|
+
metadata: {}
|
67
|
+
post_install_message:
|
68
|
+
rdoc_options: []
|
69
|
+
require_paths:
|
70
|
+
- lib
|
71
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - ">="
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: '0'
|
81
|
+
requirements: []
|
82
|
+
rubygems_version: 3.1.4
|
83
|
+
signing_key:
|
84
|
+
specification_version: 4
|
85
|
+
summary: A library to add hyperloglog functionality to lambda store in ruby
|
86
|
+
test_files: []
|