em-bucketer 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/em-bucketer/base.rb +0 -1
- data/lib/em-bucketer/database/hash.rb +5 -1
- data/lib/em-bucketer/database/redis.rb +3 -0
- data/lib/em-bucketer/in_memory.rb +1 -0
- data/lib/em-bucketer/ordered/base.rb +0 -1
- data/lib/em-bucketer/ordered/database/hash.rb +4 -0
- data/lib/em-bucketer/ordered/database/redis.rb +120 -0
- data/lib/em-bucketer/ordered/in_memory.rb +1 -0
- data/lib/em-bucketer/ordered/redis.rb +69 -0
- data/lib/em-bucketer/ordered.rb +1 -1
- data/lib/em-bucketer/redis.rb +1 -0
- data/lib/em-bucketer/version.rb +1 -1
- data/spec/redis_ordered_spec.rb +18 -0
- metadata +5 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c433f1cc453123f151e4a89363b6a6ebcb02e991
|
4
|
+
data.tar.gz: a541740a352cdc8c3a4e6474d6a7f0d69b4979a2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fa4fcfa73e8d0311c2534cdd759fd8150a2fc96868faa271b96b26f12a125084822cd131816dd886df66e95ebecc89a0b474ae1297843b3bc5f5c5db5a3de50a
|
7
|
+
data.tar.gz: 39cd7430e0855f9048e6294cfa38cf7137d83172a6e5af997c4b7b14b4919bdf411a0d25838336325fe99695b02c7593f232373ad7cc5bee6f08c9f9eed8eefc
|
data/lib/em-bucketer/base.rb
CHANGED
@@ -3,7 +3,6 @@ module EventMachine::Bucketer
|
|
3
3
|
def setup(bucket_threshold_size, bucket_max_age)
|
4
4
|
@bucket_threshold_size = bucket_threshold_size
|
5
5
|
@bucket_max_age = bucket_max_age
|
6
|
-
@buckets = {}
|
7
6
|
@on_bucket_full_callbacks = []
|
8
7
|
@on_bucket_timeout_callbacks = []
|
9
8
|
@buckets_with_timers = Set.new
|
@@ -1,10 +1,14 @@
|
|
1
|
-
require 'em-bucketer/database'
|
1
|
+
require 'em-bucketer/ordered/database'
|
2
2
|
|
3
3
|
module EventMachine::Bucketer
|
4
4
|
module Database
|
5
5
|
module Hash
|
6
6
|
private
|
7
7
|
|
8
|
+
def setup_db
|
9
|
+
@buckets = {}
|
10
|
+
end
|
11
|
+
|
8
12
|
def bucket_size_from_db(bucket_id, &blk)
|
9
13
|
EM::Completion.new.tap do |c|
|
10
14
|
c.callback(&blk) if block_given?
|
@@ -19,6 +19,7 @@ module EventMachine::Bucketer
|
|
19
19
|
# can remain before the on_bucket_timed_out is called
|
20
20
|
def initialize(bucket_threshold_size: BUCKET_THRESHOLD_SIZE_DEFAULT, bucket_max_age: BUCKET_MAX_AGE_DEFAULT)
|
21
21
|
setup(bucket_threshold_size, bucket_max_age)
|
22
|
+
setup_db
|
22
23
|
end
|
23
24
|
end
|
24
25
|
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
require 'em-bucketer/ordered/database'
|
2
|
+
require 'em-hiredis'
|
3
|
+
|
4
|
+
module EventMachine::Bucketer
|
5
|
+
module Ordered::Database
|
6
|
+
module Redis
|
7
|
+
private
|
8
|
+
|
9
|
+
def setup_db
|
10
|
+
redis.register_script(:lpopn, <<-END)
|
11
|
+
local r = redis.call('lrange', KEYS[1], 0, ARGV[1] - 1)
|
12
|
+
redis.call('ltrim', KEYS[1], ARGV[1], -1)
|
13
|
+
return r
|
14
|
+
END
|
15
|
+
redis.register_script(:lpopa, <<-END)
|
16
|
+
local r = redis.call('lrange', KEYS[1], 0, - 1)
|
17
|
+
redis.call('del', KEYS[1])
|
18
|
+
return r
|
19
|
+
END
|
20
|
+
end
|
21
|
+
|
22
|
+
def bucket_size_from_db(bucket_id, &blk)
|
23
|
+
EM::Completion.new.tap do |c|
|
24
|
+
c.callback(&blk) if block_given?
|
25
|
+
redis.llen(redis_key(bucket_id)).callback do |len|
|
26
|
+
c.succeed len.to_i
|
27
|
+
end.errback do |e|
|
28
|
+
c.fail e
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def add_item_to_db(bucket_id, item, &blk)
|
34
|
+
EM::Completion.new.tap do |c|
|
35
|
+
c.callback(&blk) if block_given?
|
36
|
+
redis.rpush(redis_key(bucket_id), Marshal.dump(item)).callback do
|
37
|
+
add_to_known_buckets(bucket_id).callback do
|
38
|
+
c.succeed
|
39
|
+
end.errback do |e|
|
40
|
+
c.fail e
|
41
|
+
end
|
42
|
+
end.errback do |e|
|
43
|
+
c.fail e
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def pop_all_from_db(bucket_id, &blk)
|
49
|
+
EM::Completion.new.tap do |c|
|
50
|
+
c.callback(&blk) if block_given?
|
51
|
+
redis.lpopa([redis_key(bucket_id)]).callback do |data|
|
52
|
+
bucket = data.map { |d| Marshal.load(d) }
|
53
|
+
c.succeed bucket
|
54
|
+
end.errback do |e|
|
55
|
+
c.fail e
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def pop_count_from_db(bucket_id, count, &blk)
|
61
|
+
EM::Completion.new.tap do |c|
|
62
|
+
c.callback(&blk) if block_given?
|
63
|
+
redis.lpopn([redis_key(bucket_id)], count).callback do |data|
|
64
|
+
bucket = data.map { |d| Marshal.load(d) }
|
65
|
+
c.succeed bucket
|
66
|
+
end.errback do |e|
|
67
|
+
c.fail e
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def get_bucket_from_db(bucket_id, &blk)
|
73
|
+
EM::Completion.new.tap do |c|
|
74
|
+
c.callback(&blk) if block_given?
|
75
|
+
redis.lrange(redis_key(bucket_id), 0, -1).callback do |data|
|
76
|
+
bucket = data.map { |d| Marshal.load(d) }
|
77
|
+
c.succeed bucket
|
78
|
+
end.errback do |e|
|
79
|
+
c.fail e
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def empty_bucket_in_db(bucket_id, &blk)
|
85
|
+
EM::Completion.new.tap do |c|
|
86
|
+
c.callback(&blk) if block_given?
|
87
|
+
redis.del(redis_key(bucket_id)).callback do
|
88
|
+
remove_from_known_buckets(bucket_id).callback do
|
89
|
+
c.succeed
|
90
|
+
end.errback do |e|
|
91
|
+
c.fail e
|
92
|
+
end
|
93
|
+
end.errback do |e|
|
94
|
+
c.fail e
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def known_buckets(&blk)
|
100
|
+
redis.smembers(redis_known_buckets_key, &blk)
|
101
|
+
end
|
102
|
+
|
103
|
+
def add_to_known_buckets(bucket_id, &blk)
|
104
|
+
redis.sadd(redis_known_buckets_key, bucket_id, &blk)
|
105
|
+
end
|
106
|
+
|
107
|
+
def remove_from_known_buckets(bucket_id, &blk)
|
108
|
+
redis.srem(redis_known_buckets_key, bucket_id, &blk)
|
109
|
+
end
|
110
|
+
|
111
|
+
def redis_key(bucket_id)
|
112
|
+
"em_bucketer_ordered:#{redis_prefix}:#{bucket_id}"
|
113
|
+
end
|
114
|
+
|
115
|
+
def redis_known_buckets_key
|
116
|
+
"em_bucketer_ordered_known_buckets:#{redis_prefix}"
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
@@ -19,6 +19,7 @@ module EventMachine::Bucketer
|
|
19
19
|
# can remain before the on_bucket_timed_out is called
|
20
20
|
def initialize(bucket_threshold_size: BUCKET_THRESHOLD_SIZE_DEFAULT, bucket_max_age: BUCKET_MAX_AGE_DEFAULT)
|
21
21
|
setup(bucket_threshold_size, bucket_max_age)
|
22
|
+
setup_db
|
22
23
|
end
|
23
24
|
end
|
24
25
|
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
require 'em-bucketer/ordered/database/redis'
|
3
|
+
require 'em-bucketer/ordered/base'
|
4
|
+
|
5
|
+
module EventMachine::Bucketer
|
6
|
+
class Ordered::Redis
|
7
|
+
include Ordered::Database::Redis
|
8
|
+
include Ordered::Base
|
9
|
+
|
10
|
+
BUCKET_THRESHOLD_SIZE_DEFAULT = 1000
|
11
|
+
BUCKET_MAX_AGE_DEFAULT = 3600
|
12
|
+
|
13
|
+
# Creates a new redis Bucketer with the requested
|
14
|
+
# configurations.
|
15
|
+
# *NOTE* The redis bucketer uses Marshal to store
|
16
|
+
# the objects in redis. This puts limitations on
|
17
|
+
# the data that cannot be stored in these buckets.
|
18
|
+
# For example you cannot store an object that
|
19
|
+
# references a proc as an instance variable.
|
20
|
+
#
|
21
|
+
# The redis bucketer also sets all timers on
|
22
|
+
# startup for buckets already in the redis
|
23
|
+
# database. This ensures that even if your
|
24
|
+
# app is restarted the previous timers will still
|
25
|
+
# get set and you won't ever lose a bucket.
|
26
|
+
#
|
27
|
+
# @param redis_prefix [String] The prefix for the
|
28
|
+
# bucket in redis. This is necessary because you
|
29
|
+
# may want to have multiple bucketers using one
|
30
|
+
# redis instance and you don't want them conflicting.
|
31
|
+
# Also this can't just be random because the whole
|
32
|
+
# point of the redis bucketer is that you can restart
|
33
|
+
# your app and get back the same bucketer without any
|
34
|
+
# data loss.
|
35
|
+
# @param bucket_threshold_size [Integer] the max size of the bucket
|
36
|
+
# after which the on_bucket_full callback is called
|
37
|
+
# @param bucket_max_age [Integer] max number of seconds a bucket
|
38
|
+
# can remain before the on_bucket_timed_out is called
|
39
|
+
def initialize(redis_prefix, bucket_threshold_size: BUCKET_THRESHOLD_SIZE_DEFAULT, bucket_max_age: BUCKET_MAX_AGE_DEFAULT)
|
40
|
+
@redis = EM::Hiredis.connect
|
41
|
+
@redis_prefix = redis_prefix
|
42
|
+
setup(bucket_threshold_size, bucket_max_age)
|
43
|
+
setup_db
|
44
|
+
set_timers
|
45
|
+
end
|
46
|
+
|
47
|
+
def set_timers
|
48
|
+
known_buckets.callback do |bucket_ids|
|
49
|
+
bucket_ids.each do |bucket_id|
|
50
|
+
add_timer_if_first(bucket_id)
|
51
|
+
end
|
52
|
+
end.errback do |e|
|
53
|
+
# I think this is okay since it will only happen when
|
54
|
+
# you are initializing the bucketer so it will hopefully
|
55
|
+
# bring the issue to your attention on startup. I
|
56
|
+
# couldn't actually pass this error back through to
|
57
|
+
# anyone so I needed to raise it. It is also a bad
|
58
|
+
# exception since it means you are not properly reloading
|
59
|
+
# your buckets from redis
|
60
|
+
raise e
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
|
66
|
+
# Used by Database::Redis
|
67
|
+
attr_reader :redis_prefix, :redis
|
68
|
+
end
|
69
|
+
end
|
data/lib/em-bucketer/ordered.rb
CHANGED
data/lib/em-bucketer/redis.rb
CHANGED
data/lib/em-bucketer/version.rb
CHANGED
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'em-bucketer/ordered'
|
3
|
+
require 'redis'
|
4
|
+
|
5
|
+
describe EventMachine::Bucketer::Ordered::Redis do
|
6
|
+
let(:prefix) { "test_prefix" }
|
7
|
+
before(:each) do
|
8
|
+
redis = Redis.new
|
9
|
+
redis.keys("em_bucketer_ordered:#{prefix}:*").each do |k|
|
10
|
+
redis.del(k)
|
11
|
+
end
|
12
|
+
redis.del("em_bucketer_ordered_known_buckets:#{prefix}")
|
13
|
+
end
|
14
|
+
|
15
|
+
it_behaves_like "an ordered bucketer" do
|
16
|
+
let(:bucketer) { EM::Bucketer::Ordered::Redis.new(prefix, :bucket_threshold_size => 5) }
|
17
|
+
end
|
18
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: em-bucketer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Richard Heycock
|
@@ -147,13 +147,16 @@ files:
|
|
147
147
|
- lib/em-bucketer/ordered/base.rb
|
148
148
|
- lib/em-bucketer/ordered/database.rb
|
149
149
|
- lib/em-bucketer/ordered/database/hash.rb
|
150
|
+
- lib/em-bucketer/ordered/database/redis.rb
|
150
151
|
- lib/em-bucketer/ordered/in_memory.rb
|
152
|
+
- lib/em-bucketer/ordered/redis.rb
|
151
153
|
- lib/em-bucketer/redis.rb
|
152
154
|
- lib/em-bucketer/version.rb
|
153
155
|
- spec/em_bucketer_examples.rb
|
154
156
|
- spec/em_bucketer_ordered_examples.rb
|
155
157
|
- spec/in_memory_ordered_spec.rb
|
156
158
|
- spec/in_memory_spec.rb
|
159
|
+
- spec/redis_ordered_spec.rb
|
157
160
|
- spec/redis_spec.rb
|
158
161
|
- spec/spec_helper.rb
|
159
162
|
- spec/spec_methods.rb
|
@@ -187,6 +190,7 @@ test_files:
|
|
187
190
|
- spec/em_bucketer_ordered_examples.rb
|
188
191
|
- spec/in_memory_ordered_spec.rb
|
189
192
|
- spec/in_memory_spec.rb
|
193
|
+
- spec/redis_ordered_spec.rb
|
190
194
|
- spec/redis_spec.rb
|
191
195
|
- spec/spec_helper.rb
|
192
196
|
- spec/spec_methods.rb
|