redstream 0.1.1 → 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +28 -0
- data/.rubocop.yml +13 -4
- data/.travis.yml +2 -1
- data/CHANGELOG.md +12 -0
- data/README.md +6 -4
- data/lib/redstream/delayer.rb +3 -3
- data/lib/redstream/model.rb +17 -5
- data/lib/redstream/producer.rb +25 -16
- data/lib/redstream/version.rb +1 -1
- data/spec/redstream/consumer_spec.rb +3 -1
- data/spec/redstream/delayer_spec.rb +4 -4
- data/spec/redstream/model_spec.rb +73 -10
- data/spec/redstream/producer_spec.rb +75 -1
- data/spec/redstream/trimmer_spec.rb +3 -3
- data/spec/redstream_spec.rb +5 -5
- data/spec/spec_helper.rb +1 -1
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cb01602a9b0e51f896989d59b5e2b16f333c7ea0317f31c5235696bc49145013
|
4
|
+
data.tar.gz: 747f63e94c847be8d371e60098f9348f30531587b821ac019c873c10d51dd766
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 19541261544b2d32168b8e290e402b5c9dedcf54868dee1dbb556bfa5cacf41a091fbff391605a734a6c1bf5eeaf6b33127660b190245166e1340a777c5e433d
|
7
|
+
data.tar.gz: 74426b36ca249b2da61f436f195d6811245038835492321009d063f78226cfdc7e099a4e17fe9910588ebc3f9bc6e21bec8e600959e0e5f8c2e77425ccda7023
|
@@ -0,0 +1,28 @@
|
|
1
|
+
name: test
|
2
|
+
on: [push, pull_request]
|
3
|
+
jobs:
|
4
|
+
build:
|
5
|
+
runs-on: ubuntu-latest
|
6
|
+
strategy:
|
7
|
+
matrix:
|
8
|
+
ruby: ['2.5', '2.6', '2.7', '3.0']
|
9
|
+
services:
|
10
|
+
redis:
|
11
|
+
image: redis
|
12
|
+
ports:
|
13
|
+
- 6379:6379
|
14
|
+
steps:
|
15
|
+
- uses: actions/checkout@v1
|
16
|
+
- uses: actions/setup-ruby@v1
|
17
|
+
with:
|
18
|
+
ruby-version: ${{ matrix.ruby }}
|
19
|
+
- uses: actions/cache@v1
|
20
|
+
id: cache
|
21
|
+
with:
|
22
|
+
path: vendor/bundler
|
23
|
+
key: ${{ hashFiles('Gemfile.lock') }}-${{ matrix.ruby }}
|
24
|
+
- run: |
|
25
|
+
gem install bundler
|
26
|
+
bundle install --path=vendor/bundler
|
27
|
+
bundle exec rspec
|
28
|
+
bundle exec rubocop
|
data/.rubocop.yml
CHANGED
@@ -1,3 +1,15 @@
|
|
1
|
+
AllCops:
|
2
|
+
NewCops: enable
|
3
|
+
|
4
|
+
Gemspec/RequireMFA:
|
5
|
+
Enabled: false
|
6
|
+
|
7
|
+
Lint/AssignmentInCondition:
|
8
|
+
Enabled: false
|
9
|
+
|
10
|
+
Gemspec/RequiredRubyVersion:
|
11
|
+
Enabled: false
|
12
|
+
|
1
13
|
Lint/AmbiguousBlockAssociation:
|
2
14
|
Enabled: false
|
3
15
|
|
@@ -13,9 +25,6 @@ Layout/ArgumentAlignment:
|
|
13
25
|
Layout/FirstArrayElementIndentation:
|
14
26
|
EnforcedStyle: consistent
|
15
27
|
|
16
|
-
Style/BracesAroundHashParameters:
|
17
|
-
EnforcedStyle: no_braces
|
18
|
-
|
19
28
|
Style/PercentLiteralDelimiters:
|
20
29
|
Enabled: false
|
21
30
|
|
@@ -52,7 +61,7 @@ Style/SymbolArray:
|
|
52
61
|
Layout/RescueEnsureAlignment:
|
53
62
|
Enabled: false
|
54
63
|
|
55
|
-
|
64
|
+
Layout/LineLength:
|
56
65
|
Enabled: false
|
57
66
|
|
58
67
|
Metrics/MethodLength:
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,17 @@
|
|
1
1
|
# CHANGELOG
|
2
2
|
|
3
|
+
## v0.4.1
|
4
|
+
* Fix keyword argument usage of redis 4.5.1 in ruby 3
|
5
|
+
|
6
|
+
## v0.4.0
|
7
|
+
* Make delay message id params in queue methods optional
|
8
|
+
|
9
|
+
## v0.3.0
|
10
|
+
* Pipeline deletion of delay messages
|
11
|
+
|
12
|
+
## v0.2.0
|
13
|
+
* Delete delay messages after queue messages are sent
|
14
|
+
|
3
15
|
## v0.1.1
|
4
16
|
* Fix missing queue message in `after_commit on: :destroy`
|
5
17
|
|
data/README.md
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
**Using redis streams to keep your primary database in sync with secondary
|
5
5
|
datastores (e.g. elasticsearch).**
|
6
6
|
|
7
|
-
[![Build Status](https://
|
7
|
+
[![Build Status](https://github.com/mrkamel/redstream/workflows/test/badge.svg?branch=master)](https://github.com/mrkamel/redstream/actions?query=workflow%3Atest)
|
8
8
|
|
9
9
|
## Installation
|
10
10
|
|
@@ -86,9 +86,9 @@ end
|
|
86
86
|
More concretely, `after_save`, `after_touch` and `after_destroy` only write
|
87
87
|
"delay" messages to an additional redis stream. Delay message are like any
|
88
88
|
other messages, but they get processed by a `Redstream::Delayer` and the
|
89
|
-
`Delayer`will wait for some (configurable) delay/time before processing them.
|
89
|
+
`Delayer` will wait for some (configurable) delay/time before processing them.
|
90
90
|
As the `Delayer` is neccessary to fix inconsistencies, the delay must be at
|
91
|
-
least as long as your
|
91
|
+
least as long as your maximum database transaction time. Contrary,
|
92
92
|
`after_commit` writes messages to a redis stream from which the messages can
|
93
93
|
be fetched immediately to keep the secondary datastores updated in
|
94
94
|
near-realtime. The reasoning of all this is simple: usually, i.e. by using only
|
@@ -97,7 +97,9 @@ any errors occurring in between `after_save` and `after_commit` result in
|
|
97
97
|
inconsistencies between your primary and secondary datastore. By using these
|
98
98
|
kinds of "delay" messages triggered by `after_save` and fetched after e.g. 5
|
99
99
|
minutes, errors occurring in between `after_save` and `after_commit` can be
|
100
|
-
fixed when the delay message get processed.
|
100
|
+
fixed when the delay message get processed. Please note that redstream deletes
|
101
|
+
delay messages after the messages for immediate retrieval have been
|
102
|
+
successfully sent, such that messages will not be processed twice, usually.
|
101
103
|
|
102
104
|
Any messages are fetched in batches, such that e.g. elasticsearch can be
|
103
105
|
updated using its bulk API. For instance, depending on which elasticsearch ruby
|
data/lib/redstream/delayer.rb
CHANGED
@@ -48,7 +48,7 @@ module Redstream
|
|
48
48
|
def run_once
|
49
49
|
@consumer.run_once do |messages|
|
50
50
|
messages.each do |message|
|
51
|
-
seconds_to_sleep = message.message_id.to_f / 1_000 + @delay.to_f - Time.now.to_f
|
51
|
+
seconds_to_sleep = (message.message_id.to_f / 1_000) + @delay.to_f - Time.now.to_f
|
52
52
|
|
53
53
|
if seconds_to_sleep > 0
|
54
54
|
if @batch.size > 0
|
@@ -85,11 +85,11 @@ module Redstream
|
|
85
85
|
Redstream.connection_pool.with do |redis|
|
86
86
|
redis.pipelined do
|
87
87
|
@batch.each do |message|
|
88
|
-
redis.xadd
|
88
|
+
redis.xadd(Redstream.stream_key_name(@stream_name), { payload: message.fields["payload"] })
|
89
89
|
end
|
90
90
|
end
|
91
91
|
|
92
|
-
redis.xdel
|
92
|
+
redis.xdel(Redstream.stream_key_name("#{@stream_name}.delay"), @batch.map(&:message_id))
|
93
93
|
end
|
94
94
|
|
95
95
|
@batch = []
|
data/lib/redstream/model.rb
CHANGED
@@ -12,6 +12,8 @@ module Redstream
|
|
12
12
|
# end
|
13
13
|
|
14
14
|
module Model
|
15
|
+
IVAR_DELAY_MESSAGE_ID = :@__redstream_delay_message_id__
|
16
|
+
|
15
17
|
def self.included(base)
|
16
18
|
base.extend(ClassMethods)
|
17
19
|
end
|
@@ -29,11 +31,21 @@ module Redstream
|
|
29
31
|
# responsible for writing to a redis stream
|
30
32
|
|
31
33
|
def redstream_callbacks(producer: Producer.new)
|
32
|
-
after_save
|
33
|
-
after_touch
|
34
|
-
after_destroy { |object| producer.delay(object) }
|
35
|
-
|
36
|
-
after_commit(on: :
|
34
|
+
after_save { |object| instance_variable_set(IVAR_DELAY_MESSAGE_ID, producer.delay(object)) if object.saved_changes.present? }
|
35
|
+
after_touch { |object| instance_variable_set(IVAR_DELAY_MESSAGE_ID, producer.delay(object)) }
|
36
|
+
after_destroy { |object| instance_variable_set(IVAR_DELAY_MESSAGE_ID, producer.delay(object)) }
|
37
|
+
|
38
|
+
after_commit(on: [:create, :update]) do |object|
|
39
|
+
if object.saved_changes.present?
|
40
|
+
producer.queue(object, delay_message_id: instance_variable_get(IVAR_DELAY_MESSAGE_ID))
|
41
|
+
instance_variable_set(IVAR_DELAY_MESSAGE_ID, nil)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
after_commit(on: :destroy) do |object|
|
46
|
+
producer.queue(object, delay_message_id: instance_variable_get(IVAR_DELAY_MESSAGE_ID))
|
47
|
+
instance_variable_set(IVAR_DELAY_MESSAGE_ID, nil)
|
48
|
+
end
|
37
49
|
end
|
38
50
|
|
39
51
|
def redstream_name
|
data/lib/redstream/producer.rb
CHANGED
@@ -52,11 +52,11 @@ module Redstream
|
|
52
52
|
def bulk(records)
|
53
53
|
records_array = Array(records)
|
54
54
|
|
55
|
-
bulk_delay(records_array)
|
55
|
+
delay_message_ids = bulk_delay(records_array)
|
56
56
|
|
57
57
|
yield
|
58
58
|
|
59
|
-
bulk_queue(records_array)
|
59
|
+
bulk_queue(records_array, delay_message_ids: delay_message_ids)
|
60
60
|
end
|
61
61
|
|
62
62
|
# @api private
|
@@ -64,13 +64,15 @@ module Redstream
|
|
64
64
|
# Writes delay messages to a delay stream in redis.
|
65
65
|
#
|
66
66
|
# @param records [#to_a] The object/objects that will be updated or deleted
|
67
|
+
#
|
68
|
+
# @return The redis message ids
|
67
69
|
|
68
70
|
def bulk_delay(records)
|
69
|
-
records.each_slice(250) do |slice|
|
71
|
+
res = records.each_slice(250).flat_map do |slice|
|
70
72
|
Redstream.connection_pool.with do |redis|
|
71
73
|
redis.pipelined do
|
72
|
-
slice.
|
73
|
-
redis.xadd
|
74
|
+
slice.each do |object|
|
75
|
+
redis.xadd(Redstream.stream_key_name("#{stream_name(object)}.delay"), { payload: JSON.dump(object.redstream_payload) })
|
74
76
|
end
|
75
77
|
end
|
76
78
|
end
|
@@ -80,7 +82,7 @@ module Redstream
|
|
80
82
|
redis.wait(@wait, 0) if @wait
|
81
83
|
end
|
82
84
|
|
83
|
-
|
85
|
+
res
|
84
86
|
end
|
85
87
|
|
86
88
|
# @api private
|
@@ -88,13 +90,15 @@ module Redstream
|
|
88
90
|
# Writes messages to a stream in redis for immediate retrieval.
|
89
91
|
#
|
90
92
|
# @param records [#to_a] The object/objects that will be updated deleted
|
93
|
+
# @param delay_message_ids [#to_a] The delay message ids to delete
|
91
94
|
|
92
|
-
def bulk_queue(records)
|
93
|
-
records.each_slice(250) do |slice|
|
95
|
+
def bulk_queue(records, delay_message_ids: nil)
|
96
|
+
records.each_with_index.each_slice(250) do |slice|
|
94
97
|
Redstream.connection_pool.with do |redis|
|
95
98
|
redis.pipelined do
|
96
|
-
slice.each do |object|
|
97
|
-
redis.xadd
|
99
|
+
slice.each do |object, index|
|
100
|
+
redis.xadd(Redstream.stream_key_name(stream_name(object)), { payload: JSON.dump(object.redstream_payload) })
|
101
|
+
redis.xdel(Redstream.stream_key_name("#{stream_name(object)}.delay"), delay_message_ids[index]) if delay_message_ids
|
98
102
|
end
|
99
103
|
end
|
100
104
|
end
|
@@ -107,15 +111,16 @@ module Redstream
|
|
107
111
|
#
|
108
112
|
# Writes a single delay message to a delay stream in redis.
|
109
113
|
#
|
110
|
-
# @param object The object
|
114
|
+
# @param object The object that will be updated, deleted, etc.
|
115
|
+
#
|
116
|
+
# @return The redis message id
|
111
117
|
|
112
118
|
def delay(object)
|
113
119
|
Redstream.connection_pool.with do |redis|
|
114
|
-
redis.xadd
|
120
|
+
res = redis.xadd(Redstream.stream_key_name("#{stream_name(object)}.delay"), { payload: JSON.dump(object.redstream_payload) })
|
115
121
|
redis.wait(@wait, 0) if @wait
|
122
|
+
res
|
116
123
|
end
|
117
|
-
|
118
|
-
true
|
119
124
|
end
|
120
125
|
|
121
126
|
# @api private
|
@@ -123,10 +128,14 @@ module Redstream
|
|
123
128
|
# Writes a single message to a stream in redis for immediate retrieval.
|
124
129
|
#
|
125
130
|
# @param object The object hat will be updated, deleted, etc.
|
131
|
+
# @param delay_message_id The delay message id to delete
|
126
132
|
|
127
|
-
def queue(object)
|
133
|
+
def queue(object, delay_message_id: nil)
|
128
134
|
Redstream.connection_pool.with do |redis|
|
129
|
-
redis.
|
135
|
+
redis.pipelined do
|
136
|
+
redis.xadd(Redstream.stream_key_name(stream_name(object)), { payload: JSON.dump(object.redstream_payload) })
|
137
|
+
redis.xdel(Redstream.stream_key_name("#{stream_name(object)}.delay"), delay_message_id) if delay_message_id
|
138
|
+
end
|
130
139
|
end
|
131
140
|
|
132
141
|
true
|
data/lib/redstream/version.rb
CHANGED
@@ -80,7 +80,9 @@ RSpec.describe Redstream::Consumer do
|
|
80
80
|
|
81
81
|
all_messages = redis.xrange(Redstream.stream_key_name("products"), "-", "+")
|
82
82
|
|
83
|
-
Redstream::Consumer.new(name: "consumer", stream_name: "products").run_once
|
83
|
+
Redstream::Consumer.new(name: "consumer", stream_name: "products").run_once do
|
84
|
+
# nothing
|
85
|
+
end
|
84
86
|
|
85
87
|
expect(redis.get(Redstream.offset_key_name(stream_name: "products", consumer_name: "consumer"))).to eq(all_messages.last[0])
|
86
88
|
end
|
@@ -3,7 +3,7 @@ require File.expand_path("../spec_helper", __dir__)
|
|
3
3
|
RSpec.describe Redstream::Delayer do
|
4
4
|
describe "#run_once" do
|
5
5
|
it "copies expired messages to their target streams" do
|
6
|
-
redis.xadd
|
6
|
+
redis.xadd(Redstream.stream_key_name("target.delay"), { payload: JSON.dump(value: "message") })
|
7
7
|
|
8
8
|
expect(redis.xlen(Redstream.stream_key_name("target"))).to eq(0)
|
9
9
|
|
@@ -14,9 +14,9 @@ RSpec.describe Redstream::Delayer do
|
|
14
14
|
end
|
15
15
|
|
16
16
|
it "delivers and commit before falling asleep" do
|
17
|
-
redis.xadd
|
17
|
+
redis.xadd(Redstream.stream_key_name("target.delay"), { payload: JSON.dump(value: "message") })
|
18
18
|
sleep 3
|
19
|
-
redis.xadd
|
19
|
+
redis.xadd(Redstream.stream_key_name("target.delay"), { payload: JSON.dump(value: "message") })
|
20
20
|
|
21
21
|
thread = Thread.new do
|
22
22
|
Redstream::Delayer.new(stream_name: "target", delay: 1).run_once
|
@@ -33,7 +33,7 @@ RSpec.describe Redstream::Delayer do
|
|
33
33
|
end
|
34
34
|
|
35
35
|
it "does not copy not yet expired messages" do
|
36
|
-
redis.xadd
|
36
|
+
redis.xadd(Redstream.stream_key_name("target.delay"), { payload: JSON.dump(value: "message") })
|
37
37
|
|
38
38
|
thread = Thread.new do
|
39
39
|
Redstream::Delayer.new(stream_name: "target", delay: 2).run_once
|
@@ -3,19 +3,36 @@ require File.expand_path("../spec_helper", __dir__)
|
|
3
3
|
RSpec.describe Redstream::Model do
|
4
4
|
describe "after_save" do
|
5
5
|
it "adds a delay message after_save" do
|
6
|
-
|
6
|
+
Product.transaction do
|
7
|
+
expect { create(:product) }.to change { redis.xlen(Redstream.stream_key_name("products.delay")) }
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
it "assigns the delay message id" do
|
12
|
+
Product.transaction do
|
13
|
+
expect(create(:product).instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_present
|
14
|
+
end
|
7
15
|
end
|
8
16
|
|
9
17
|
it "adds the correct payload for the delay message" do
|
10
|
-
|
18
|
+
Product.transaction do
|
19
|
+
product = create(:product)
|
11
20
|
|
12
|
-
|
21
|
+
expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").first[1]).to eq("payload" => JSON.dump(product.redstream_payload))
|
22
|
+
end
|
13
23
|
end
|
14
24
|
|
15
25
|
it "adds a queue message after_save on commit" do
|
16
26
|
expect { create(:product) }.to change { redis.xlen(Redstream.stream_key_name("products")) }
|
17
27
|
end
|
18
28
|
|
29
|
+
it "deletes the delay message on commit" do
|
30
|
+
product = create(:product)
|
31
|
+
|
32
|
+
expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
|
33
|
+
expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_nil
|
34
|
+
end
|
35
|
+
|
19
36
|
it "does not add a delay message after_save if there are no changes" do
|
20
37
|
product = create(:product)
|
21
38
|
|
@@ -33,35 +50,73 @@ RSpec.describe Redstream::Model do
|
|
33
50
|
it "adds a delay message after touch" do
|
34
51
|
product = create(:product)
|
35
52
|
|
36
|
-
|
53
|
+
Product.transaction do
|
54
|
+
expect { product.touch }.to change { redis.xlen(Redstream.stream_key_name("products.delay")) }
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
it "assigns the delay message id" do
|
59
|
+
product = create(:product)
|
60
|
+
|
61
|
+
Product.transaction do
|
62
|
+
product.touch
|
63
|
+
|
64
|
+
expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_present
|
65
|
+
end
|
37
66
|
end
|
38
67
|
|
39
68
|
it "sets the correct payload for the delay message" do
|
40
69
|
product = create(:product)
|
41
|
-
product.touch
|
42
70
|
|
43
|
-
|
71
|
+
Product.transaction do
|
72
|
+
product.touch
|
73
|
+
|
74
|
+
expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
|
75
|
+
end
|
44
76
|
end
|
45
77
|
|
46
|
-
it "adds a queue message after touch commit" do
|
78
|
+
it "adds a queue message after touch on commit" do
|
47
79
|
product = create(:product)
|
48
80
|
|
49
81
|
expect { product.touch }.to change { redis.xlen(Redstream.stream_key_name("products")) }
|
50
82
|
end
|
83
|
+
|
84
|
+
it "deletes the delay message after touch on commit" do
|
85
|
+
product = create(:product)
|
86
|
+
product.touch
|
87
|
+
|
88
|
+
expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
|
89
|
+
expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_nil
|
90
|
+
end
|
51
91
|
end
|
52
92
|
|
53
93
|
describe "after_destroy" do
|
54
94
|
it "adds a delay message after destroy" do
|
55
95
|
product = create(:product)
|
56
96
|
|
57
|
-
|
97
|
+
Product.transaction do
|
98
|
+
expect { product.destroy }.to change { redis.xlen(Redstream.stream_key_name("products.delay")) }
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
it "assigns the delay message id" do
|
103
|
+
product = create(:product)
|
104
|
+
|
105
|
+
Product.transaction do
|
106
|
+
product.destroy
|
107
|
+
|
108
|
+
expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_present
|
109
|
+
end
|
58
110
|
end
|
59
111
|
|
60
112
|
it "sets the correct payload for the delay message" do
|
61
113
|
product = create(:product)
|
62
|
-
product.destroy
|
63
114
|
|
64
|
-
|
115
|
+
Product.transaction do
|
116
|
+
product.destroy
|
117
|
+
|
118
|
+
expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
|
119
|
+
end
|
65
120
|
end
|
66
121
|
|
67
122
|
it "adds a queue messages after destroy on commit" do
|
@@ -69,5 +124,13 @@ RSpec.describe Redstream::Model do
|
|
69
124
|
|
70
125
|
expect { product.destroy }.to change { redis.xlen(Redstream.stream_key_name("products")) }.by(1)
|
71
126
|
end
|
127
|
+
|
128
|
+
it "deletes the delay message after destroy on commit" do
|
129
|
+
product = create(:product)
|
130
|
+
product.destroy
|
131
|
+
|
132
|
+
expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
|
133
|
+
expect(product.instance_variable_get(Redstream::Model::IVAR_DELAY_MESSAGE_ID)).to be_nil
|
134
|
+
end
|
72
135
|
end
|
73
136
|
end
|
@@ -10,6 +10,17 @@ RSpec.describe Redstream::Producer do
|
|
10
10
|
expect { Redstream::Producer.new.queue(product) }.to change { redis.xlen(stream_key_name) }.by(1)
|
11
11
|
expect(redis.xrange(stream_key_name, "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
|
12
12
|
end
|
13
|
+
|
14
|
+
it "deletes the delay message when given" do
|
15
|
+
product = create(:product)
|
16
|
+
|
17
|
+
producer = Redstream::Producer.new
|
18
|
+
|
19
|
+
id = producer.delay(product)
|
20
|
+
producer.queue(product, delay_message_id: id)
|
21
|
+
|
22
|
+
expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(0)
|
23
|
+
end
|
13
24
|
end
|
14
25
|
|
15
26
|
describe "#delay" do
|
@@ -31,6 +42,57 @@ RSpec.describe Redstream::Producer do
|
|
31
42
|
end
|
32
43
|
end
|
33
44
|
|
45
|
+
describe "#bulk" do
|
46
|
+
it "adds bulk delay messages for scopes" do
|
47
|
+
products = create_list(:product, 2)
|
48
|
+
|
49
|
+
stream_key_name = Redstream.stream_key_name("products")
|
50
|
+
|
51
|
+
expect(redis.xlen("#{stream_key_name}.delay")).to eq(0)
|
52
|
+
|
53
|
+
Redstream::Producer.new.bulk(Product.all) do
|
54
|
+
messages = redis.xrange("#{stream_key_name}.delay", "-", "+").last(2).map { |message| message[1] }
|
55
|
+
|
56
|
+
expect(messages).to eq([
|
57
|
+
{ "payload" => JSON.dump(products[0].redstream_payload) },
|
58
|
+
{ "payload" => JSON.dump(products[1].redstream_payload) }
|
59
|
+
])
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
it "adds bulk queue messages for scopes" do
|
64
|
+
products = create_list(:product, 2)
|
65
|
+
|
66
|
+
stream_key_name = Redstream.stream_key_name("products")
|
67
|
+
|
68
|
+
expect do
|
69
|
+
Redstream::Producer.new.bulk(Product.all) do
|
70
|
+
# nothing
|
71
|
+
end
|
72
|
+
end.to change { redis.xlen(stream_key_name) }.by(2)
|
73
|
+
|
74
|
+
messages = redis.xrange(stream_key_name, "-", "+").last(2).map { |message| message[1] }
|
75
|
+
|
76
|
+
expect(messages).to eq([
|
77
|
+
{ "payload" => JSON.dump(products[0].redstream_payload) },
|
78
|
+
{ "payload" => JSON.dump(products[1].redstream_payload) }
|
79
|
+
])
|
80
|
+
end
|
81
|
+
|
82
|
+
it "deletes the delay messages after the queue messages have been sent" do
|
83
|
+
products = create_list(:product, 2)
|
84
|
+
producer = Redstream::Producer.new
|
85
|
+
|
86
|
+
other_delay_message_id = producer.delay(create(:product))
|
87
|
+
|
88
|
+
producer.bulk(products) do
|
89
|
+
expect(redis.xlen(Redstream.stream_key_name("products.delay"))).to eq(3)
|
90
|
+
end
|
91
|
+
|
92
|
+
expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").map(&:first)).to eq([other_delay_message_id])
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
34
96
|
describe "#bulk_queue" do
|
35
97
|
it "adds bulk queue messages for scopes" do
|
36
98
|
products = create_list(:product, 2)
|
@@ -46,6 +108,18 @@ RSpec.describe Redstream::Producer do
|
|
46
108
|
{ "payload" => JSON.dump(products[1].redstream_payload) }
|
47
109
|
])
|
48
110
|
end
|
111
|
+
|
112
|
+
it "deletes the delay messages after the queue messages have been sent" do
|
113
|
+
products = create_list(:product, 2)
|
114
|
+
producer = Redstream::Producer.new
|
115
|
+
|
116
|
+
delay_message_ids = producer.bulk_delay(products)
|
117
|
+
other_delay_message_id = producer.delay(create(:product))
|
118
|
+
|
119
|
+
producer.bulk_queue(products, delay_message_ids: delay_message_ids)
|
120
|
+
|
121
|
+
expect(redis.xrange(Redstream.stream_key_name("products.delay"), "-", "+").map(&:first)).to eq([other_delay_message_id])
|
122
|
+
end
|
49
123
|
end
|
50
124
|
|
51
125
|
describe "#bulk_delay" do
|
@@ -64,7 +138,7 @@ RSpec.describe Redstream::Producer do
|
|
64
138
|
])
|
65
139
|
end
|
66
140
|
|
67
|
-
it "should
|
141
|
+
it "should respect wait for delay" do
|
68
142
|
create(:product)
|
69
143
|
|
70
144
|
stream_key_name = Redstream.stream_key_name("products.delay")
|
@@ -4,11 +4,11 @@ RSpec.describe Redstream::Trimmer do
|
|
4
4
|
describe "#run_once" do
|
5
5
|
it "trims a stream to the minimum committed id" do
|
6
6
|
ids = Array.new(4) do |i|
|
7
|
-
redis.xadd
|
7
|
+
redis.xadd(Redstream.stream_key_name("default"), { payload: JSON.dump(value: "message#{i}") })
|
8
8
|
end
|
9
9
|
|
10
|
-
redis.set
|
11
|
-
redis.set
|
10
|
+
redis.set(Redstream.offset_key_name(stream_name: "default", consumer_name: "consumer1"), ids[1])
|
11
|
+
redis.set(Redstream.offset_key_name(stream_name: "default", consumer_name: "consumer2"), ids[2])
|
12
12
|
|
13
13
|
trimmer = Redstream::Trimmer.new(
|
14
14
|
interval: 5,
|
data/spec/redstream_spec.rb
CHANGED
@@ -34,7 +34,7 @@ RSpec.describe Redstream do
|
|
34
34
|
it "returns the stream's size" do
|
35
35
|
expect(Redstream.stream_size("products")).to eq(0)
|
36
36
|
|
37
|
-
redis.xadd("redstream:stream:products", key: "value")
|
37
|
+
redis.xadd("redstream:stream:products", { key: "value" })
|
38
38
|
|
39
39
|
expect(Redstream.stream_size("products")).to eq(1)
|
40
40
|
end
|
@@ -44,8 +44,8 @@ RSpec.describe Redstream do
|
|
44
44
|
it "returns the stream's max id" do
|
45
45
|
expect(Redstream.max_stream_id("products")).to be_nil
|
46
46
|
|
47
|
-
_id1 = redis.xadd("redstream:stream:products", key: "value")
|
48
|
-
id2 = redis.xadd("redstream:stream:products", key: "value")
|
47
|
+
_id1 = redis.xadd("redstream:stream:products", { key: "value" })
|
48
|
+
id2 = redis.xadd("redstream:stream:products", { key: "value" })
|
49
49
|
|
50
50
|
expect(Redstream.max_stream_id("products")).to eq(id2)
|
51
51
|
end
|
@@ -55,8 +55,8 @@ RSpec.describe Redstream do
|
|
55
55
|
it "returns the consumer's max id" do
|
56
56
|
expect(Redstream.max_consumer_id(stream_name: "products", consumer_name: "consumer")).to be_nil
|
57
57
|
|
58
|
-
_id1 = redis.xadd("redstream:stream:products", key: "value")
|
59
|
-
id2 = redis.xadd("redstream:stream:products", key: "value")
|
58
|
+
_id1 = redis.xadd("redstream:stream:products", { key: "value" })
|
59
|
+
id2 = redis.xadd("redstream:stream:products", { key: "value" })
|
60
60
|
|
61
61
|
Redstream::Consumer.new(name: "consumer", stream_name: "products").run_once do |messages|
|
62
62
|
# nothing
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redstream
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Benjamin Vetter
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-12-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -242,6 +242,7 @@ executables: []
|
|
242
242
|
extensions: []
|
243
243
|
extra_rdoc_files: []
|
244
244
|
files:
|
245
|
+
- ".github/workflows/test.yml"
|
245
246
|
- ".gitignore"
|
246
247
|
- ".rubocop.yml"
|
247
248
|
- ".travis.yml"
|
@@ -288,7 +289,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
288
289
|
- !ruby/object:Gem::Version
|
289
290
|
version: '0'
|
290
291
|
requirements: []
|
291
|
-
rubygems_version: 3.
|
292
|
+
rubygems_version: 3.2.15
|
292
293
|
signing_key:
|
293
294
|
specification_version: 4
|
294
295
|
summary: Using redis streams to keep your primary database in sync with secondary
|