redstream 0.3.0 → 0.4.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +1 -1
- data/.rubocop.yml +3 -0
- data/CHANGELOG.md +9 -0
- data/lib/redstream/delayer.rb +4 -4
- data/lib/redstream/producer.rb +13 -11
- data/lib/redstream/version.rb +1 -1
- data/redstream.gemspec +1 -1
- data/spec/redstream/delayer_spec.rb +4 -4
- data/spec/redstream/producer_spec.rb +2 -2
- data/spec/redstream/trimmer_spec.rb +3 -3
- data/spec/redstream_spec.rb +5 -5
- metadata +8 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 400e4270e920ace54c661a275cf467c7f929a554b937999175d47198b8b403eb
|
4
|
+
data.tar.gz: e38676096310d5fb55f63d9e49971491e7a184cfb5d633390b7085bcb3762903
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 729fee0ff31002c70c83ff5428352e0d7818d8b333259adce8aa150da8748df10daf40c03e7b2b5c28ba0ac1f48dee516b1e35e6e7201652dbba653a4e84d1a4
|
7
|
+
data.tar.gz: dee5156c1df03d23d5fd9aafdb57b3bf8a8e2f2c983242e4feab970415aa2851b9203f8a77cee02a534e5c2571b360ecb6a3360aae81f06d90e291a2f4591a7c
|
data/.github/workflows/test.yml
CHANGED
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,14 @@
|
|
1
1
|
# CHANGELOG
|
2
2
|
|
3
|
+
## v0.4.2
|
4
|
+
* Fix pipelining with redis-rb 4.6.0
|
5
|
+
|
6
|
+
## v0.4.1
|
7
|
+
* Fix keyword argument usage of redis 4.5.1 in ruby 3
|
8
|
+
|
9
|
+
## v0.4.0
|
10
|
+
* Make delay message id params in queue methods optional
|
11
|
+
|
3
12
|
## v0.3.0
|
4
13
|
* Pipeline deletion of delay messages
|
5
14
|
|
data/lib/redstream/delayer.rb
CHANGED
@@ -48,7 +48,7 @@ module Redstream
|
|
48
48
|
def run_once
|
49
49
|
@consumer.run_once do |messages|
|
50
50
|
messages.each do |message|
|
51
|
-
seconds_to_sleep = message.message_id.to_f / 1_000 + @delay.to_f - Time.now.to_f
|
51
|
+
seconds_to_sleep = (message.message_id.to_f / 1_000) + @delay.to_f - Time.now.to_f
|
52
52
|
|
53
53
|
if seconds_to_sleep > 0
|
54
54
|
if @batch.size > 0
|
@@ -83,13 +83,13 @@ module Redstream
|
|
83
83
|
@logger.debug "Delayed #{@batch.size} messages for #{@delay.to_f} seconds on stream #{@stream_name}"
|
84
84
|
|
85
85
|
Redstream.connection_pool.with do |redis|
|
86
|
-
redis.pipelined do
|
86
|
+
redis.pipelined do |pipeline|
|
87
87
|
@batch.each do |message|
|
88
|
-
|
88
|
+
pipeline.xadd(Redstream.stream_key_name(@stream_name), { payload: message.fields["payload"] })
|
89
89
|
end
|
90
90
|
end
|
91
91
|
|
92
|
-
redis.xdel
|
92
|
+
redis.xdel(Redstream.stream_key_name("#{@stream_name}.delay"), @batch.map(&:message_id))
|
93
93
|
end
|
94
94
|
|
95
95
|
@batch = []
|
data/lib/redstream/producer.rb
CHANGED
@@ -70,9 +70,9 @@ module Redstream
|
|
70
70
|
def bulk_delay(records)
|
71
71
|
res = records.each_slice(250).flat_map do |slice|
|
72
72
|
Redstream.connection_pool.with do |redis|
|
73
|
-
redis.pipelined do
|
73
|
+
redis.pipelined do |pipeline|
|
74
74
|
slice.each do |object|
|
75
|
-
|
75
|
+
pipeline.xadd(Redstream.stream_key_name("#{stream_name(object)}.delay"), { payload: JSON.dump(object.redstream_payload) })
|
76
76
|
end
|
77
77
|
end
|
78
78
|
end
|
@@ -90,14 +90,15 @@ module Redstream
|
|
90
90
|
# Writes messages to a stream in redis for immediate retrieval.
|
91
91
|
#
|
92
92
|
# @param records [#to_a] The object/objects that will be updated deleted
|
93
|
+
# @param delay_message_ids [#to_a] The delay message ids to delete
|
93
94
|
|
94
|
-
def bulk_queue(records, delay_message_ids:)
|
95
|
+
def bulk_queue(records, delay_message_ids: nil)
|
95
96
|
records.each_with_index.each_slice(250) do |slice|
|
96
97
|
Redstream.connection_pool.with do |redis|
|
97
|
-
redis.pipelined do
|
98
|
+
redis.pipelined do |pipeline|
|
98
99
|
slice.each do |object, index|
|
99
|
-
|
100
|
-
|
100
|
+
pipeline.xadd(Redstream.stream_key_name(stream_name(object)), { payload: JSON.dump(object.redstream_payload) })
|
101
|
+
pipeline.xdel(Redstream.stream_key_name("#{stream_name(object)}.delay"), delay_message_ids[index]) if delay_message_ids
|
101
102
|
end
|
102
103
|
end
|
103
104
|
end
|
@@ -116,7 +117,7 @@ module Redstream
|
|
116
117
|
|
117
118
|
def delay(object)
|
118
119
|
Redstream.connection_pool.with do |redis|
|
119
|
-
res = redis.xadd(Redstream.stream_key_name("#{stream_name(object)}.delay"), payload: JSON.dump(object.redstream_payload))
|
120
|
+
res = redis.xadd(Redstream.stream_key_name("#{stream_name(object)}.delay"), { payload: JSON.dump(object.redstream_payload) })
|
120
121
|
redis.wait(@wait, 0) if @wait
|
121
122
|
res
|
122
123
|
end
|
@@ -127,12 +128,13 @@ module Redstream
|
|
127
128
|
# Writes a single message to a stream in redis for immediate retrieval.
|
128
129
|
#
|
129
130
|
# @param object The object hat will be updated, deleted, etc.
|
131
|
+
# @param delay_message_id The delay message id to delete
|
130
132
|
|
131
|
-
def queue(object, delay_message_id:)
|
133
|
+
def queue(object, delay_message_id: nil)
|
132
134
|
Redstream.connection_pool.with do |redis|
|
133
|
-
redis.pipelined do
|
134
|
-
|
135
|
-
|
135
|
+
redis.pipelined do |pipeline|
|
136
|
+
pipeline.xadd(Redstream.stream_key_name(stream_name(object)), { payload: JSON.dump(object.redstream_payload) })
|
137
|
+
pipeline.xdel(Redstream.stream_key_name("#{stream_name(object)}.delay"), delay_message_id) if delay_message_id
|
136
138
|
end
|
137
139
|
end
|
138
140
|
|
data/lib/redstream/version.rb
CHANGED
data/redstream.gemspec
CHANGED
@@ -3,7 +3,7 @@ require File.expand_path("../spec_helper", __dir__)
|
|
3
3
|
RSpec.describe Redstream::Delayer do
|
4
4
|
describe "#run_once" do
|
5
5
|
it "copies expired messages to their target streams" do
|
6
|
-
redis.xadd
|
6
|
+
redis.xadd(Redstream.stream_key_name("target.delay"), { payload: JSON.dump(value: "message") })
|
7
7
|
|
8
8
|
expect(redis.xlen(Redstream.stream_key_name("target"))).to eq(0)
|
9
9
|
|
@@ -14,9 +14,9 @@ RSpec.describe Redstream::Delayer do
|
|
14
14
|
end
|
15
15
|
|
16
16
|
it "delivers and commit before falling asleep" do
|
17
|
-
redis.xadd
|
17
|
+
redis.xadd(Redstream.stream_key_name("target.delay"), { payload: JSON.dump(value: "message") })
|
18
18
|
sleep 3
|
19
|
-
redis.xadd
|
19
|
+
redis.xadd(Redstream.stream_key_name("target.delay"), { payload: JSON.dump(value: "message") })
|
20
20
|
|
21
21
|
thread = Thread.new do
|
22
22
|
Redstream::Delayer.new(stream_name: "target", delay: 1).run_once
|
@@ -33,7 +33,7 @@ RSpec.describe Redstream::Delayer do
|
|
33
33
|
end
|
34
34
|
|
35
35
|
it "does not copy not yet expired messages" do
|
36
|
-
redis.xadd
|
36
|
+
redis.xadd(Redstream.stream_key_name("target.delay"), { payload: JSON.dump(value: "message") })
|
37
37
|
|
38
38
|
thread = Thread.new do
|
39
39
|
Redstream::Delayer.new(stream_name: "target", delay: 2).run_once
|
@@ -7,7 +7,7 @@ RSpec.describe Redstream::Producer do
|
|
7
7
|
|
8
8
|
stream_key_name = Redstream.stream_key_name("products")
|
9
9
|
|
10
|
-
expect { Redstream::Producer.new.queue(product
|
10
|
+
expect { Redstream::Producer.new.queue(product) }.to change { redis.xlen(stream_key_name) }.by(1)
|
11
11
|
expect(redis.xrange(stream_key_name, "-", "+").last[1]).to eq("payload" => JSON.dump(product.redstream_payload))
|
12
12
|
end
|
13
13
|
|
@@ -99,7 +99,7 @@ RSpec.describe Redstream::Producer do
|
|
99
99
|
|
100
100
|
stream_key_name = Redstream.stream_key_name("products")
|
101
101
|
|
102
|
-
expect { Redstream::Producer.new.bulk_queue(Product.all
|
102
|
+
expect { Redstream::Producer.new.bulk_queue(Product.all) }.to change { redis.xlen(stream_key_name) }.by(2)
|
103
103
|
|
104
104
|
messages = redis.xrange(stream_key_name, "-", "+").last(2).map { |message| message[1] }
|
105
105
|
|
@@ -4,11 +4,11 @@ RSpec.describe Redstream::Trimmer do
|
|
4
4
|
describe "#run_once" do
|
5
5
|
it "trims a stream to the minimum committed id" do
|
6
6
|
ids = Array.new(4) do |i|
|
7
|
-
redis.xadd
|
7
|
+
redis.xadd(Redstream.stream_key_name("default"), { payload: JSON.dump(value: "message#{i}") })
|
8
8
|
end
|
9
9
|
|
10
|
-
redis.set
|
11
|
-
redis.set
|
10
|
+
redis.set(Redstream.offset_key_name(stream_name: "default", consumer_name: "consumer1"), ids[1])
|
11
|
+
redis.set(Redstream.offset_key_name(stream_name: "default", consumer_name: "consumer2"), ids[2])
|
12
12
|
|
13
13
|
trimmer = Redstream::Trimmer.new(
|
14
14
|
interval: 5,
|
data/spec/redstream_spec.rb
CHANGED
@@ -34,7 +34,7 @@ RSpec.describe Redstream do
|
|
34
34
|
it "returns the stream's size" do
|
35
35
|
expect(Redstream.stream_size("products")).to eq(0)
|
36
36
|
|
37
|
-
redis.xadd("redstream:stream:products", key: "value")
|
37
|
+
redis.xadd("redstream:stream:products", { key: "value" })
|
38
38
|
|
39
39
|
expect(Redstream.stream_size("products")).to eq(1)
|
40
40
|
end
|
@@ -44,8 +44,8 @@ RSpec.describe Redstream do
|
|
44
44
|
it "returns the stream's max id" do
|
45
45
|
expect(Redstream.max_stream_id("products")).to be_nil
|
46
46
|
|
47
|
-
_id1 = redis.xadd("redstream:stream:products", key: "value")
|
48
|
-
id2 = redis.xadd("redstream:stream:products", key: "value")
|
47
|
+
_id1 = redis.xadd("redstream:stream:products", { key: "value" })
|
48
|
+
id2 = redis.xadd("redstream:stream:products", { key: "value" })
|
49
49
|
|
50
50
|
expect(Redstream.max_stream_id("products")).to eq(id2)
|
51
51
|
end
|
@@ -55,8 +55,8 @@ RSpec.describe Redstream do
|
|
55
55
|
it "returns the consumer's max id" do
|
56
56
|
expect(Redstream.max_consumer_id(stream_name: "products", consumer_name: "consumer")).to be_nil
|
57
57
|
|
58
|
-
_id1 = redis.xadd("redstream:stream:products", key: "value")
|
59
|
-
id2 = redis.xadd("redstream:stream:products", key: "value")
|
58
|
+
_id1 = redis.xadd("redstream:stream:products", { key: "value" })
|
59
|
+
id2 = redis.xadd("redstream:stream:products", { key: "value" })
|
60
60
|
|
61
61
|
Redstream::Consumer.new(name: "consumer", stream_name: "products").run_once do |messages|
|
62
62
|
# nothing
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redstream
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Benjamin Vetter
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-02-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -226,14 +226,14 @@ dependencies:
|
|
226
226
|
requirements:
|
227
227
|
- - ">="
|
228
228
|
- !ruby/object:Gem::Version
|
229
|
-
version: 4.
|
229
|
+
version: 4.6.0
|
230
230
|
type: :runtime
|
231
231
|
prerelease: false
|
232
232
|
version_requirements: !ruby/object:Gem::Requirement
|
233
233
|
requirements:
|
234
234
|
- - ">="
|
235
235
|
- !ruby/object:Gem::Version
|
236
|
-
version: 4.
|
236
|
+
version: 4.6.0
|
237
237
|
description: Using redis streams to keep your primary database in sync with secondary
|
238
238
|
datastores
|
239
239
|
email:
|
@@ -274,7 +274,7 @@ homepage: https://github.com/mrkamel/redstream
|
|
274
274
|
licenses:
|
275
275
|
- MIT
|
276
276
|
metadata: {}
|
277
|
-
post_install_message:
|
277
|
+
post_install_message:
|
278
278
|
rdoc_options: []
|
279
279
|
require_paths:
|
280
280
|
- lib
|
@@ -289,8 +289,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
289
289
|
- !ruby/object:Gem::Version
|
290
290
|
version: '0'
|
291
291
|
requirements: []
|
292
|
-
rubygems_version: 3.
|
293
|
-
signing_key:
|
292
|
+
rubygems_version: 3.3.3
|
293
|
+
signing_key:
|
294
294
|
specification_version: 4
|
295
295
|
summary: Using redis streams to keep your primary database in sync with secondary
|
296
296
|
datastores
|