redstream 0.4.4 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +3 -3
- data/CHANGELOG.md +6 -0
- data/Gemfile +12 -0
- data/README.md +28 -3
- data/lib/redstream/consumer.rb +1 -1
- data/lib/redstream/lock.rb +61 -13
- data/lib/redstream/model.rb +28 -11
- data/lib/redstream/producer.rb +5 -18
- data/lib/redstream/version.rb +1 -1
- data/redstream.gemspec +0 -13
- data/spec/redstream/consumer_spec.rb +39 -0
- data/spec/redstream/lock_spec.rb +83 -1
- data/spec/redstream/model_spec.rb +0 -49
- data/spec/redstream/producer_spec.rb +58 -7
- data/spec/redstream/trimmer_spec.rb +1 -1
- data/spec/spec_helper.rb +0 -5
- metadata +2 -170
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 20db64a5cecd9accc59e117b19ecd608236d0dfae1c177c47aff0079335ead7d
|
4
|
+
data.tar.gz: 34f383997e4e15a1ce03c83650445b20683b00a6adefcae01248525087fa90a9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 911800fad8f34ce373e3edd884ce45c1804bd82ea2c0d0ae14b69fbc2f8e998aa19210db6ddb65c14ffb1af9daf03164269f8da4a913364eaf8c3f471e88f013
|
7
|
+
data.tar.gz: 3e3b9b0651c4a6d8d4f570df68d8e2ea982a3109450cf62b7f7fc02ebe7b05d3652a136016ea800e2a66ccb0059a59d2a6b703b6804803093c8030c1ac2cbc31
|
data/.github/workflows/test.yml
CHANGED
@@ -5,10 +5,11 @@ jobs:
|
|
5
5
|
runs-on: ubuntu-latest
|
6
6
|
strategy:
|
7
7
|
matrix:
|
8
|
-
ruby: ['2.
|
8
|
+
ruby: ['2.7', '3.0', '3.2']
|
9
9
|
redis:
|
10
10
|
- redis:5.0
|
11
11
|
- redis:6.0
|
12
|
+
- redis:7.0
|
12
13
|
services:
|
13
14
|
redis:
|
14
15
|
image: ${{ matrix.redis }}
|
@@ -16,7 +17,7 @@ jobs:
|
|
16
17
|
- 6379:6379
|
17
18
|
steps:
|
18
19
|
- uses: actions/checkout@v1
|
19
|
-
- uses:
|
20
|
+
- uses: ruby/setup-ruby@v1
|
20
21
|
with:
|
21
22
|
ruby-version: ${{ matrix.ruby }}
|
22
23
|
- uses: actions/cache@v1
|
@@ -25,7 +26,6 @@ jobs:
|
|
25
26
|
path: vendor/bundler
|
26
27
|
key: ${{ hashFiles('Gemfile.lock') }}-${{ matrix.ruby }}
|
27
28
|
- run: |
|
28
|
-
gem install bundler
|
29
29
|
bundle install --path=vendor/bundler
|
30
30
|
bundle exec rspec
|
31
31
|
bundle exec rubocop
|
data/CHANGELOG.md
CHANGED
data/Gemfile
CHANGED
@@ -1,3 +1,15 @@
|
|
1
1
|
source "https://rubygems.org"
|
2
2
|
|
3
3
|
gemspec
|
4
|
+
|
5
|
+
gem "activerecord"
|
6
|
+
gem "bundler"
|
7
|
+
gem "concurrent-ruby"
|
8
|
+
gem "database_cleaner"
|
9
|
+
gem "factory_bot"
|
10
|
+
gem "rake"
|
11
|
+
gem "rspec"
|
12
|
+
gem "rspec-instafail"
|
13
|
+
gem "rubocop"
|
14
|
+
gem "sqlite3"
|
15
|
+
gem "timecop"
|
data/README.md
CHANGED
@@ -97,9 +97,7 @@ any errors occurring in between `after_save` and `after_commit` result in
|
|
97
97
|
inconsistencies between your primary and secondary datastore. By using these
|
98
98
|
kinds of "delay" messages triggered by `after_save` and fetched after e.g. 5
|
99
99
|
minutes, errors occurring in between `after_save` and `after_commit` can be
|
100
|
-
fixed when the delay message get processed.
|
101
|
-
delay messages after the messages for immediate retrieval have been
|
102
|
-
successfully sent, such that messages will not be processed twice, usually.
|
100
|
+
fixed when the delay message get processed.
|
103
101
|
|
104
102
|
Any messages are fetched in batches, such that e.g. elasticsearch can be
|
105
103
|
updated using its bulk API. For instance, depending on which elasticsearch ruby
|
@@ -233,6 +231,33 @@ array of records to `Redstream::Producer#bulk`, like shown above. If you pass
|
|
233
231
|
an `ActiveRecord::Relation`, the `#bulk` method will convert it to an array,
|
234
232
|
i.e. load the whole result set into memory.
|
235
233
|
|
234
|
+
## Sharding
|
235
|
+
|
236
|
+
When you want to attach multiple consumers to a single stream, you maybe want
|
237
|
+
to add sharding. This can be accomplished by specifying a dynamic stream name
|
238
|
+
where you compute the shard key by hashing the primary key.
|
239
|
+
|
240
|
+
```ruby
|
241
|
+
class Product < ActiveRecord::Base
|
242
|
+
include Redstream::Model
|
243
|
+
|
244
|
+
NUM_SHARDS = 4
|
245
|
+
|
246
|
+
def self.redstream_name(shard)
|
247
|
+
"products-#{shard}"
|
248
|
+
end
|
249
|
+
|
250
|
+
def redstream_name
|
251
|
+
self.class.redstream_name(Digest::SHA1.hexdigest(id.to_s)[0, 4].to_i(16) % NUM_SHARDS)
|
252
|
+
end
|
253
|
+
end
|
254
|
+
```
|
255
|
+
|
256
|
+
The sharding via hashing the primary key is neccessary, because we want each
|
257
|
+
change of a specific object to end up in the same stream. Otherwise the order
|
258
|
+
of changes for a specific object gets mixed up. Subsequently, you can add
|
259
|
+
consumers, etc for each individual stream name.
|
260
|
+
|
236
261
|
## Namespacing
|
237
262
|
|
238
263
|
In case you are using a shared redis, where multiple appications read/write
|
data/lib/redstream/consumer.rb
CHANGED
data/lib/redstream/lock.rb
CHANGED
@@ -21,6 +21,21 @@ module Redstream
|
|
21
21
|
# end
|
22
22
|
|
23
23
|
class Lock
|
24
|
+
class Signal
|
25
|
+
def initialize
|
26
|
+
@mutex = Mutex.new
|
27
|
+
@condition_variable = ConditionVariable.new
|
28
|
+
end
|
29
|
+
|
30
|
+
def wait(timeout)
|
31
|
+
@mutex.synchronize { @condition_variable.wait(@mutex, timeout) }
|
32
|
+
end
|
33
|
+
|
34
|
+
def signal
|
35
|
+
@condition_variable.signal
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
24
39
|
def initialize(name:)
|
25
40
|
@name = name
|
26
41
|
@id = SecureRandom.hex
|
@@ -28,33 +43,45 @@ module Redstream
|
|
28
43
|
|
29
44
|
def acquire(&block)
|
30
45
|
got_lock = get_lock
|
31
|
-
|
46
|
+
|
47
|
+
if got_lock
|
48
|
+
keep_lock(&block)
|
49
|
+
release_lock
|
50
|
+
end
|
51
|
+
|
32
52
|
got_lock
|
33
53
|
end
|
34
54
|
|
55
|
+
def wait(timeout)
|
56
|
+
@wait_redis ||= Redstream.connection_pool.with(&:dup)
|
57
|
+
@wait_redis.brpop("#{Redstream.lock_key_name(@name)}.notify", timeout: timeout)
|
58
|
+
end
|
59
|
+
|
35
60
|
private
|
36
61
|
|
37
62
|
def keep_lock(&block)
|
38
|
-
|
39
|
-
|
63
|
+
stopped = false
|
64
|
+
signal = Signal.new
|
40
65
|
|
41
|
-
Thread.new do
|
42
|
-
until
|
43
|
-
Redstream.connection_pool.with
|
66
|
+
thread = Thread.new do
|
67
|
+
until stopped
|
68
|
+
Redstream.connection_pool.with do |redis|
|
69
|
+
redis.expire(Redstream.lock_key_name(@name), 5)
|
70
|
+
end
|
44
71
|
|
45
|
-
|
72
|
+
signal.wait(3)
|
46
73
|
end
|
47
74
|
end
|
48
75
|
|
49
76
|
block.call
|
50
77
|
ensure
|
51
|
-
|
52
|
-
|
53
|
-
|
78
|
+
stopped = true
|
79
|
+
signal&.signal
|
80
|
+
thread&.join
|
54
81
|
end
|
55
82
|
|
56
83
|
def get_lock
|
57
|
-
@get_lock_script = <<~
|
84
|
+
@get_lock_script = <<~SCRIPT
|
58
85
|
local lock_key_name, id = ARGV[1], ARGV[2]
|
59
86
|
|
60
87
|
local cur = redis.call('get', lock_key_name)
|
@@ -70,9 +97,30 @@ module Redstream
|
|
70
97
|
end
|
71
98
|
|
72
99
|
return false
|
73
|
-
|
100
|
+
SCRIPT
|
101
|
+
|
102
|
+
Redstream.connection_pool.with do |redis|
|
103
|
+
redis.eval(@get_lock_script, argv: [Redstream.lock_key_name(@name), @id])
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def release_lock
|
108
|
+
@release_lock_script = <<~SCRIPT
|
109
|
+
local lock_key_name, id = ARGV[1], ARGV[2]
|
74
110
|
|
75
|
-
|
111
|
+
local cur = redis.call('get', lock_key_name)
|
112
|
+
|
113
|
+
if cur and cur == id then
|
114
|
+
redis.call('del', lock_key_name)
|
115
|
+
end
|
116
|
+
|
117
|
+
redis.call('del', lock_key_name .. '.notify')
|
118
|
+
redis.call('rpush', lock_key_name .. '.notify', '1')
|
119
|
+
SCRIPT
|
120
|
+
|
121
|
+
Redstream.connection_pool.with do |redis|
|
122
|
+
redis.eval(@release_lock_script, argv: [Redstream.lock_key_name(@name), @id])
|
123
|
+
end
|
76
124
|
end
|
77
125
|
end
|
78
126
|
end
|
data/lib/redstream/model.rb
CHANGED
@@ -12,8 +12,6 @@ module Redstream
|
|
12
12
|
# end
|
13
13
|
|
14
14
|
module Model
|
15
|
-
IVAR_DELAY_MESSAGE_ID = :@__redstream_delay_message_id__
|
16
|
-
|
17
15
|
def self.included(base)
|
18
16
|
base.extend(ClassMethods)
|
19
17
|
end
|
@@ -31,20 +29,16 @@ module Redstream
|
|
31
29
|
# responsible for writing to a redis stream
|
32
30
|
|
33
31
|
def redstream_callbacks(producer: Producer.new)
|
34
|
-
after_save { |object|
|
35
|
-
after_touch { |object|
|
36
|
-
after_destroy { |object|
|
32
|
+
after_save { |object| producer.delay(object) if object.saved_changes.present? }
|
33
|
+
after_touch { |object| producer.delay(object) }
|
34
|
+
after_destroy { |object| producer.delay(object) }
|
37
35
|
|
38
36
|
after_commit(on: [:create, :update]) do |object|
|
39
|
-
if object.saved_changes.present?
|
40
|
-
producer.queue(object, delay_message_id: instance_variable_get(IVAR_DELAY_MESSAGE_ID))
|
41
|
-
instance_variable_set(IVAR_DELAY_MESSAGE_ID, nil)
|
42
|
-
end
|
37
|
+
producer.queue(object) if object.saved_changes.present?
|
43
38
|
end
|
44
39
|
|
45
40
|
after_commit(on: :destroy) do |object|
|
46
|
-
producer.queue(object
|
47
|
-
instance_variable_set(IVAR_DELAY_MESSAGE_ID, nil)
|
41
|
+
producer.queue(object)
|
48
42
|
end
|
49
43
|
end
|
50
44
|
|
@@ -64,5 +58,28 @@ module Redstream
|
|
64
58
|
def redstream_payload
|
65
59
|
{ id: id }
|
66
60
|
end
|
61
|
+
|
62
|
+
# Override to customize the stream name. By default, the stream name
|
63
|
+
# is determined by the class name. If you override the instance method,
|
64
|
+
# please also override the class method.
|
65
|
+
#
|
66
|
+
# @example Sharding
|
67
|
+
# class Product
|
68
|
+
# include Redstream::Model
|
69
|
+
#
|
70
|
+
# NUM_SHARDS = 4
|
71
|
+
#
|
72
|
+
# def redstream_name
|
73
|
+
# self.class.redstream_name(Digest::SHA1.hexdigest(id.to_s)[0, 4].to_i(16) % NUM_SHARDS)
|
74
|
+
# end
|
75
|
+
#
|
76
|
+
# def self.redstream_name(shard)
|
77
|
+
# "products-#{shard}
|
78
|
+
# end
|
79
|
+
# end
|
80
|
+
|
81
|
+
def redstream_name
|
82
|
+
self.class.redstream_name
|
83
|
+
end
|
67
84
|
end
|
68
85
|
end
|
data/lib/redstream/producer.rb
CHANGED
@@ -31,7 +31,6 @@ module Redstream
|
|
31
31
|
|
32
32
|
def initialize(wait: false)
|
33
33
|
@wait = wait
|
34
|
-
@stream_name_cache = {}
|
35
34
|
|
36
35
|
super()
|
37
36
|
end
|
@@ -72,7 +71,7 @@ module Redstream
|
|
72
71
|
Redstream.connection_pool.with do |redis|
|
73
72
|
redis.pipelined do |pipeline|
|
74
73
|
slice.each do |object|
|
75
|
-
pipeline.xadd(Redstream.stream_key_name("#{
|
74
|
+
pipeline.xadd(Redstream.stream_key_name("#{object.redstream_name}.delay"), { payload: JSON.dump(object.redstream_payload) })
|
76
75
|
end
|
77
76
|
end
|
78
77
|
end
|
@@ -96,7 +95,7 @@ module Redstream
|
|
96
95
|
Redstream.connection_pool.with do |redis|
|
97
96
|
redis.pipelined do |pipeline|
|
98
97
|
slice.each do |object|
|
99
|
-
pipeline.xadd(Redstream.stream_key_name(
|
98
|
+
pipeline.xadd(Redstream.stream_key_name(object.redstream_name), { payload: JSON.dump(object.redstream_payload) })
|
100
99
|
end
|
101
100
|
end
|
102
101
|
end
|
@@ -115,7 +114,7 @@ module Redstream
|
|
115
114
|
|
116
115
|
def delay(object)
|
117
116
|
Redstream.connection_pool.with do |redis|
|
118
|
-
res = redis.xadd(Redstream.stream_key_name("#{
|
117
|
+
res = redis.xadd(Redstream.stream_key_name("#{object.redstream_name}.delay"), { payload: JSON.dump(object.redstream_payload) })
|
119
118
|
redis.wait(@wait, 0) if @wait
|
120
119
|
res
|
121
120
|
end
|
@@ -126,25 +125,13 @@ module Redstream
|
|
126
125
|
# Writes a single message to a stream in redis for immediate retrieval.
|
127
126
|
#
|
128
127
|
# @param object The object hat will be updated, deleted, etc.
|
129
|
-
# @param delay_message_id The delay message id to delete
|
130
128
|
|
131
|
-
def queue(object
|
129
|
+
def queue(object)
|
132
130
|
Redstream.connection_pool.with do |redis|
|
133
|
-
redis.
|
134
|
-
pipeline.xadd(Redstream.stream_key_name(stream_name(object)), { payload: JSON.dump(object.redstream_payload) })
|
135
|
-
pipeline.xdel(Redstream.stream_key_name("#{stream_name(object)}.delay"), delay_message_id) if delay_message_id
|
136
|
-
end
|
131
|
+
redis.xadd(Redstream.stream_key_name(object.redstream_name), { payload: JSON.dump(object.redstream_payload) })
|
137
132
|
end
|
138
133
|
|
139
134
|
true
|
140
135
|
end
|
141
|
-
|
142
|
-
private
|
143
|
-
|
144
|
-
def stream_name(object)
|
145
|
-
synchronize do
|
146
|
-
@stream_name_cache[object.class] ||= object.class.redstream_name
|
147
|
-
end
|
148
|
-
end
|
149
136
|
end
|
150
137
|
end
|
data/lib/redstream/version.rb
CHANGED
data/redstream.gemspec
CHANGED
@@ -16,19 +16,6 @@ Gem::Specification.new do |spec|
|
|
16
16
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
17
|
spec.require_paths = ["lib"]
|
18
18
|
|
19
|
-
spec.add_development_dependency "activerecord"
|
20
|
-
spec.add_development_dependency "bundler"
|
21
|
-
spec.add_development_dependency "concurrent-ruby"
|
22
|
-
spec.add_development_dependency "database_cleaner"
|
23
|
-
spec.add_development_dependency "factory_bot"
|
24
|
-
spec.add_development_dependency "mocha"
|
25
|
-
spec.add_development_dependency "rake"
|
26
|
-
spec.add_development_dependency "rspec"
|
27
|
-
spec.add_development_dependency "rspec-instafail"
|
28
|
-
spec.add_development_dependency "rubocop"
|
29
|
-
spec.add_development_dependency "sqlite3"
|
30
|
-
spec.add_development_dependency "timecop"
|
31
|
-
|
32
19
|
spec.add_dependency "activesupport"
|
33
20
|
spec.add_dependency "connection_pool"
|
34
21
|
spec.add_dependency "json"
|
@@ -86,5 +86,44 @@ RSpec.describe Redstream::Consumer do
|
|
86
86
|
|
87
87
|
expect(redis.get(Redstream.offset_key_name(stream_name: "products", consumer_name: "consumer"))).to eq(all_messages.last[0])
|
88
88
|
end
|
89
|
+
|
90
|
+
it "does not starve" do
|
91
|
+
product = create(:product)
|
92
|
+
stopped = false
|
93
|
+
results = Concurrent::Array.new
|
94
|
+
|
95
|
+
thread1 = Thread.new do
|
96
|
+
until stopped
|
97
|
+
Redstream::Consumer.new(name: "consumer", stream_name: "products").run_once do
|
98
|
+
results.push("thread1")
|
99
|
+
|
100
|
+
product.touch
|
101
|
+
|
102
|
+
sleep 0.1
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
thread2 = Thread.new do
|
108
|
+
until stopped
|
109
|
+
Redstream::Consumer.new(name: "consumer", stream_name: "products").run_once do
|
110
|
+
results.push("thread2")
|
111
|
+
|
112
|
+
product.touch
|
113
|
+
|
114
|
+
sleep 0.1
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
sleep 6
|
120
|
+
|
121
|
+
stopped = true
|
122
|
+
[thread1, thread2].each(&:join)
|
123
|
+
|
124
|
+
expect(results.size).to be > 10
|
125
|
+
expect(results.count("thread1").to_f / results.size).to be > 0.2
|
126
|
+
expect(results.count("thread2").to_f / results.size).to be > 0.2
|
127
|
+
end
|
89
128
|
end
|
90
129
|
end
|
data/spec/redstream/lock_spec.rb
CHANGED
@@ -34,7 +34,7 @@ RSpec.describe Redstream::Lock do
|
|
34
34
|
end
|
35
35
|
end
|
36
36
|
|
37
|
-
sleep
|
37
|
+
sleep 5
|
38
38
|
|
39
39
|
threads << Thread.new do
|
40
40
|
Redstream::Lock.new(name: "lock").acquire do
|
@@ -62,5 +62,87 @@ RSpec.describe Redstream::Lock do
|
|
62
62
|
expect(calls).to eq(2)
|
63
63
|
expect(lock_results).to eq([1, 1])
|
64
64
|
end
|
65
|
+
|
66
|
+
it "releases the lock and notifies" do
|
67
|
+
lock = Redstream::Lock.new(name: "lock")
|
68
|
+
|
69
|
+
expect(redis.llen("#{Redstream.lock_key_name("lock")}.notify")).to eq(0)
|
70
|
+
|
71
|
+
lock.acquire do
|
72
|
+
# nothing
|
73
|
+
end
|
74
|
+
|
75
|
+
expect(redis.exists?(Redstream.lock_key_name("lock"))).to eq(false)
|
76
|
+
expect(redis.llen("#{Redstream.lock_key_name("lock")}.notify")).to eq(1)
|
77
|
+
end
|
78
|
+
|
79
|
+
it "does not release the lock when the lock is already taken again" do
|
80
|
+
lock = Redstream::Lock.new(name: "lock")
|
81
|
+
|
82
|
+
lock.acquire do
|
83
|
+
redis.set(Redstream.lock_key_name("lock"), "other")
|
84
|
+
end
|
85
|
+
|
86
|
+
expect(redis.get(Redstream.lock_key_name("lock"))).to eq("other")
|
87
|
+
end
|
88
|
+
|
89
|
+
it "acquires the lock as soon as it gets released" do
|
90
|
+
time = nil
|
91
|
+
|
92
|
+
thread = Thread.new do
|
93
|
+
Redstream::Lock.new(name: "lock").acquire do
|
94
|
+
time = Time.now.to_f
|
95
|
+
|
96
|
+
sleep 2
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
sleep 1
|
101
|
+
|
102
|
+
lock = Redstream::Lock.new(name: "lock")
|
103
|
+
lock.wait(10) until lock.acquire { "nothing" }
|
104
|
+
|
105
|
+
thread.join
|
106
|
+
|
107
|
+
expect(Time.now.to_f - time).to be < 3
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
describe "#wait" do
|
112
|
+
it "blocks for the specified time max" do
|
113
|
+
stopped = false
|
114
|
+
|
115
|
+
thread = Thread.new do
|
116
|
+
Redstream::Lock.new(name: "lock").acquire do
|
117
|
+
sleep 0.1 until stopped
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
time = Time.now.to_f
|
122
|
+
|
123
|
+
Redstream::Lock.new(name: "lock").wait(2)
|
124
|
+
|
125
|
+
expect(Time.now.to_f - time).to be < 3
|
126
|
+
|
127
|
+
stopped = true
|
128
|
+
|
129
|
+
thread.join
|
130
|
+
end
|
131
|
+
|
132
|
+
it "wakes up when the lock gets released" do
|
133
|
+
thread = Thread.new do
|
134
|
+
Redstream::Lock.new(name: "lock").acquire do
|
135
|
+
sleep 2
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
time = Time.now.to_f
|
140
|
+
|
141
|
+
Redstream::Lock.new(name: "lock").wait(10)
|
142
|
+
|
143
|
+
expect(Time.now.to_f - time).to be < 3
|
144
|
+
|
145
|
+
thread.join
|
146
|
+
end
|
65
147
|
end
|
66
148
|
end
|
@@ -8,12 +8,6 @@ RSpec.describe Redstream::Model do
|
|
8
8
|
end
|
9
9
|
end
|
10
10
|
|
11
|
-
it "assigns the delay message id" do
|
12
|
-
Product.transaction do
|
13
|
-
expect(create(:product).instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_present
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
11
|
it "adds the correct payload for the delay message" do
|
18
12
|
Product.transaction do
|
19
13
|
product = create(:product)
|
@@ -26,13 +20,6 @@ RSpec.describe Redstream::Model do
|
|
26
20
|
expect { create(:product) }.to change { redis.xlen(Redstream.stream_key_name("products")) }
|
27
21
|
end
|
28
22
|
|
29
|
-
it "deletes the delay message on commit" do
|
30
|
-
product = create(:product)
|
31
|
-
|
32
|
-
expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
|
33
|
-
expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_nil
|
34
|
-
end
|
35
|
-
|
36
23
|
it "does not add a delay message after_save if there are no changes" do
|
37
24
|
product = create(:product)
|
38
25
|
|
@@ -55,16 +42,6 @@ RSpec.describe Redstream::Model do
|
|
55
42
|
end
|
56
43
|
end
|
57
44
|
|
58
|
-
it "assigns the delay message id" do
|
59
|
-
product = create(:product)
|
60
|
-
|
61
|
-
Product.transaction do
|
62
|
-
product.touch
|
63
|
-
|
64
|
-
expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_present
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
45
|
it "sets the correct payload for the delay message" do
|
69
46
|
product = create(:product)
|
70
47
|
|
@@ -80,14 +57,6 @@ RSpec.describe Redstream::Model do
|
|
80
57
|
|
81
58
|
expect { product.touch }.to change { redis.xlen(Redstream.stream_key_name("products")) }
|
82
59
|
end
|
83
|
-
|
84
|
-
it "deletes the delay message after touch on commit" do
|
85
|
-
product = create(:product)
|
86
|
-
product.touch
|
87
|
-
|
88
|
-
expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
|
89
|
-
expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_nil
|
90
|
-
end
|
91
60
|
end
|
92
61
|
|
93
62
|
describe "after_destroy" do
|
@@ -99,16 +68,6 @@ RSpec.describe Redstream::Model do
|
|
99
68
|
end
|
100
69
|
end
|
101
70
|
|
102
|
-
it "assigns the delay message id" do
|
103
|
-
product = create(:product)
|
104
|
-
|
105
|
-
Product.transaction do
|
106
|
-
product.destroy
|
107
|
-
|
108
|
-
expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_present
|
109
|
-
end
|
110
|
-
end
|
111
|
-
|
112
71
|
it "sets the correct payload for the delay message" do
|
113
72
|
product = create(:product)
|
114
73
|
|
@@ -124,13 +83,5 @@ RSpec.describe Redstream::Model do
|
|
124
83
|
|
125
84
|
expect { product.destroy }.to change { redis.xlen(Redstream.stream_key_name("products")) }.by(1)
|
126
85
|
end
|
127
|
-
|
128
|
-
it "deletes the delay message after destroy on commit" do
|
129
|
-
product = create(:product)
|
130
|
-
product.destroy
|
131
|
-
|
132
|
-
expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
|
133
|
-
expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_nil
|
134
|
-
end
|
135
86
|
end
|
136
87
|
end
|
@@ -11,15 +11,15 @@ RSpec.describe Redstream::Producer do
|
|
11
11
|
expect(redis.xrange(stream_key_name, "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
|
12
12
|
end
|
13
13
|
|
14
|
-
it "
|
14
|
+
it "uses a custom stream name when specified" do
|
15
15
|
product = create(:product)
|
16
16
|
|
17
|
-
|
17
|
+
allow(product).to receive(:redstream_name).and_return("stream-name")
|
18
18
|
|
19
|
-
|
20
|
-
producer.queue(product, delay_message_id: id)
|
19
|
+
stream_key_name = Redstream.stream_key_name("stream-name")
|
21
20
|
|
22
|
-
expect(redis.xlen(
|
21
|
+
expect { Redstream::Producer.new.queue(product) }.to change { redis.xlen(stream_key_name) }.by(1)
|
22
|
+
expect(redis.xrange(stream_key_name, "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
|
23
23
|
end
|
24
24
|
end
|
25
25
|
|
@@ -33,7 +33,18 @@ RSpec.describe Redstream::Producer do
|
|
33
33
|
expect(redis.xrange(stream_key_name, "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
|
34
34
|
end
|
35
35
|
|
36
|
-
it "
|
36
|
+
it "uses a custom stream name when specified" do
|
37
|
+
product = create(:product)
|
38
|
+
|
39
|
+
allow(product).to receive(:redstream_name).and_return("stream-name")
|
40
|
+
|
41
|
+
stream_key_name = Redstream.stream_key_name("stream-name.delay")
|
42
|
+
|
43
|
+
expect { Redstream::Producer.new.delay(product) }.to change { redis.xlen(stream_key_name) }.by(1)
|
44
|
+
expect(redis.xrange(stream_key_name, "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
|
45
|
+
end
|
46
|
+
|
47
|
+
it "respects wait" do
|
37
48
|
product = create(:product)
|
38
49
|
|
39
50
|
stream_key_name = Redstream.stream_key_name("products.delay")
|
@@ -48,9 +59,11 @@ RSpec.describe Redstream::Producer do
|
|
48
59
|
|
49
60
|
stream_key_name = Redstream.stream_key_name("products")
|
50
61
|
|
51
|
-
expect(redis.xlen("#{stream_key_name}.delay")).to eq(
|
62
|
+
expect(redis.xlen("#{stream_key_name}.delay")).to eq(2)
|
52
63
|
|
53
64
|
Redstream::Producer.new.bulk(Product.all) do
|
65
|
+
expect(redis.xlen("#{stream_key_name}.delay")).to eq(4)
|
66
|
+
|
54
67
|
messages = redis.xrange("#{stream_key_name}.delay", "-", "+").last(2).map { |message| message[1] }
|
55
68
|
|
56
69
|
expect(messages).to eq([
|
@@ -60,6 +73,24 @@ RSpec.describe Redstream::Producer do
|
|
60
73
|
end
|
61
74
|
end
|
62
75
|
|
76
|
+
it "uses a custom stream name when specified" do
|
77
|
+
allow_any_instance_of(Product).to receive(:redstream_name).and_return("stream-name")
|
78
|
+
|
79
|
+
create_list(:product, 2)
|
80
|
+
|
81
|
+
stream_key_name = Redstream.stream_key_name("stream-name")
|
82
|
+
|
83
|
+
expect(redis.xlen(stream_key_name)).to eq(2)
|
84
|
+
expect(redis.xlen("#{stream_key_name}.delay")).to eq(2)
|
85
|
+
|
86
|
+
Redstream::Producer.new.bulk(Product.all) do
|
87
|
+
# nothing
|
88
|
+
end
|
89
|
+
|
90
|
+
expect(redis.xlen(stream_key_name)).to eq(4)
|
91
|
+
expect(redis.xlen("#{stream_key_name}.delay")).to eq(4)
|
92
|
+
end
|
93
|
+
|
63
94
|
it "adds bulk queue messages for scopes" do
|
64
95
|
products = create_list(:product, 2)
|
65
96
|
|
@@ -95,6 +126,16 @@ RSpec.describe Redstream::Producer do
|
|
95
126
|
{ "payload" => JSON.dump(products[1].redstream_payload) }
|
96
127
|
])
|
97
128
|
end
|
129
|
+
|
130
|
+
it "uses a custom stream nameadds bulk queue messages for scopes" do
|
131
|
+
allow_any_instance_of(Product).to receive(:redstream_name).and_return("stream-name")
|
132
|
+
|
133
|
+
create_list(:product, 2)
|
134
|
+
|
135
|
+
stream_key_name = Redstream.stream_key_name("stream-name")
|
136
|
+
|
137
|
+
expect { Redstream::Producer.new.bulk_queue(Product.all) }.to change { redis.xlen(stream_key_name) }.by(2)
|
138
|
+
end
|
98
139
|
end
|
99
140
|
|
100
141
|
describe "#bulk_delay" do
|
@@ -113,6 +154,16 @@ RSpec.describe Redstream::Producer do
|
|
113
154
|
])
|
114
155
|
end
|
115
156
|
|
157
|
+
it "uses a custom stream name when specified" do
|
158
|
+
allow_any_instance_of(Product).to receive(:redstream_name).and_return("stream-name")
|
159
|
+
|
160
|
+
create_list(:product, 2)
|
161
|
+
|
162
|
+
stream_key_name = Redstream.stream_key_name("stream-name.delay")
|
163
|
+
|
164
|
+
expect { Redstream::Producer.new.bulk_delay(Product.all) }.to change { redis.xlen(stream_key_name) }.by(2)
|
165
|
+
end
|
166
|
+
|
116
167
|
it "should respect wait for delay" do
|
117
168
|
create(:product)
|
118
169
|
|
@@ -23,7 +23,7 @@ RSpec.describe Redstream::Trimmer do
|
|
23
23
|
|
24
24
|
it "sleeps for the specified time if there's nothing to trim" do
|
25
25
|
trimmer = Redstream::Trimmer.new(interval: 1, stream_name: "default", consumer_names: ["unknown_consumer"])
|
26
|
-
trimmer.
|
26
|
+
allow(trimmer).to receive(:sleep).with(1).and_return(true)
|
27
27
|
trimmer.run_once
|
28
28
|
end
|
29
29
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -4,13 +4,8 @@ require "factory_bot"
|
|
4
4
|
require "database_cleaner"
|
5
5
|
require "timecop"
|
6
6
|
require "concurrent"
|
7
|
-
require "mocha"
|
8
7
|
require "rspec/instafail"
|
9
8
|
|
10
|
-
RSpec.configure do |config|
|
11
|
-
config.mock_with :mocha
|
12
|
-
end
|
13
|
-
|
14
9
|
ActiveRecord::Base.establish_connection(adapter: "sqlite3", database: "/tmp/redstream.sqlite3")
|
15
10
|
|
16
11
|
ActiveRecord::Base.connection.execute "DROP TABLE IF EXISTS products"
|
metadata
CHANGED
@@ -1,183 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redstream
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Benjamin Vetter
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-09-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
-
- !ruby/object:Gem::Dependency
|
14
|
-
name: activerecord
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - ">="
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '0'
|
20
|
-
type: :development
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - ">="
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: '0'
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: bundler
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - ">="
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: '0'
|
34
|
-
type: :development
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - ">="
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: '0'
|
41
|
-
- !ruby/object:Gem::Dependency
|
42
|
-
name: concurrent-ruby
|
43
|
-
requirement: !ruby/object:Gem::Requirement
|
44
|
-
requirements:
|
45
|
-
- - ">="
|
46
|
-
- !ruby/object:Gem::Version
|
47
|
-
version: '0'
|
48
|
-
type: :development
|
49
|
-
prerelease: false
|
50
|
-
version_requirements: !ruby/object:Gem::Requirement
|
51
|
-
requirements:
|
52
|
-
- - ">="
|
53
|
-
- !ruby/object:Gem::Version
|
54
|
-
version: '0'
|
55
|
-
- !ruby/object:Gem::Dependency
|
56
|
-
name: database_cleaner
|
57
|
-
requirement: !ruby/object:Gem::Requirement
|
58
|
-
requirements:
|
59
|
-
- - ">="
|
60
|
-
- !ruby/object:Gem::Version
|
61
|
-
version: '0'
|
62
|
-
type: :development
|
63
|
-
prerelease: false
|
64
|
-
version_requirements: !ruby/object:Gem::Requirement
|
65
|
-
requirements:
|
66
|
-
- - ">="
|
67
|
-
- !ruby/object:Gem::Version
|
68
|
-
version: '0'
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
|
-
name: factory_bot
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - ">="
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: '0'
|
76
|
-
type: :development
|
77
|
-
prerelease: false
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
79
|
-
requirements:
|
80
|
-
- - ">="
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
version: '0'
|
83
|
-
- !ruby/object:Gem::Dependency
|
84
|
-
name: mocha
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
86
|
-
requirements:
|
87
|
-
- - ">="
|
88
|
-
- !ruby/object:Gem::Version
|
89
|
-
version: '0'
|
90
|
-
type: :development
|
91
|
-
prerelease: false
|
92
|
-
version_requirements: !ruby/object:Gem::Requirement
|
93
|
-
requirements:
|
94
|
-
- - ">="
|
95
|
-
- !ruby/object:Gem::Version
|
96
|
-
version: '0'
|
97
|
-
- !ruby/object:Gem::Dependency
|
98
|
-
name: rake
|
99
|
-
requirement: !ruby/object:Gem::Requirement
|
100
|
-
requirements:
|
101
|
-
- - ">="
|
102
|
-
- !ruby/object:Gem::Version
|
103
|
-
version: '0'
|
104
|
-
type: :development
|
105
|
-
prerelease: false
|
106
|
-
version_requirements: !ruby/object:Gem::Requirement
|
107
|
-
requirements:
|
108
|
-
- - ">="
|
109
|
-
- !ruby/object:Gem::Version
|
110
|
-
version: '0'
|
111
|
-
- !ruby/object:Gem::Dependency
|
112
|
-
name: rspec
|
113
|
-
requirement: !ruby/object:Gem::Requirement
|
114
|
-
requirements:
|
115
|
-
- - ">="
|
116
|
-
- !ruby/object:Gem::Version
|
117
|
-
version: '0'
|
118
|
-
type: :development
|
119
|
-
prerelease: false
|
120
|
-
version_requirements: !ruby/object:Gem::Requirement
|
121
|
-
requirements:
|
122
|
-
- - ">="
|
123
|
-
- !ruby/object:Gem::Version
|
124
|
-
version: '0'
|
125
|
-
- !ruby/object:Gem::Dependency
|
126
|
-
name: rspec-instafail
|
127
|
-
requirement: !ruby/object:Gem::Requirement
|
128
|
-
requirements:
|
129
|
-
- - ">="
|
130
|
-
- !ruby/object:Gem::Version
|
131
|
-
version: '0'
|
132
|
-
type: :development
|
133
|
-
prerelease: false
|
134
|
-
version_requirements: !ruby/object:Gem::Requirement
|
135
|
-
requirements:
|
136
|
-
- - ">="
|
137
|
-
- !ruby/object:Gem::Version
|
138
|
-
version: '0'
|
139
|
-
- !ruby/object:Gem::Dependency
|
140
|
-
name: rubocop
|
141
|
-
requirement: !ruby/object:Gem::Requirement
|
142
|
-
requirements:
|
143
|
-
- - ">="
|
144
|
-
- !ruby/object:Gem::Version
|
145
|
-
version: '0'
|
146
|
-
type: :development
|
147
|
-
prerelease: false
|
148
|
-
version_requirements: !ruby/object:Gem::Requirement
|
149
|
-
requirements:
|
150
|
-
- - ">="
|
151
|
-
- !ruby/object:Gem::Version
|
152
|
-
version: '0'
|
153
|
-
- !ruby/object:Gem::Dependency
|
154
|
-
name: sqlite3
|
155
|
-
requirement: !ruby/object:Gem::Requirement
|
156
|
-
requirements:
|
157
|
-
- - ">="
|
158
|
-
- !ruby/object:Gem::Version
|
159
|
-
version: '0'
|
160
|
-
type: :development
|
161
|
-
prerelease: false
|
162
|
-
version_requirements: !ruby/object:Gem::Requirement
|
163
|
-
requirements:
|
164
|
-
- - ">="
|
165
|
-
- !ruby/object:Gem::Version
|
166
|
-
version: '0'
|
167
|
-
- !ruby/object:Gem::Dependency
|
168
|
-
name: timecop
|
169
|
-
requirement: !ruby/object:Gem::Requirement
|
170
|
-
requirements:
|
171
|
-
- - ">="
|
172
|
-
- !ruby/object:Gem::Version
|
173
|
-
version: '0'
|
174
|
-
type: :development
|
175
|
-
prerelease: false
|
176
|
-
version_requirements: !ruby/object:Gem::Requirement
|
177
|
-
requirements:
|
178
|
-
- - ">="
|
179
|
-
- !ruby/object:Gem::Version
|
180
|
-
version: '0'
|
181
13
|
- !ruby/object:Gem::Dependency
|
182
14
|
name: activesupport
|
183
15
|
requirement: !ruby/object:Gem::Requirement
|