redstream 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (27) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +14 -0
  3. data/.travis.yml +10 -0
  4. data/Gemfile +5 -0
  5. data/LICENSE.txt +22 -0
  6. data/README.md +253 -0
  7. data/Rakefile +9 -0
  8. data/docker-compose.yml +6 -0
  9. data/lib/redstream.rb +134 -0
  10. data/lib/redstream/consumer.rb +115 -0
  11. data/lib/redstream/delayer.rb +100 -0
  12. data/lib/redstream/lock.rb +80 -0
  13. data/lib/redstream/message.rb +52 -0
  14. data/lib/redstream/model.rb +57 -0
  15. data/lib/redstream/producer.rb +145 -0
  16. data/lib/redstream/trimmer.rb +91 -0
  17. data/lib/redstream/version.rb +5 -0
  18. data/redstream.gemspec +38 -0
  19. data/spec/redstream/consumer_spec.rb +90 -0
  20. data/spec/redstream/delayer_spec.rb +53 -0
  21. data/spec/redstream/lock_spec.rb +68 -0
  22. data/spec/redstream/model_spec.rb +57 -0
  23. data/spec/redstream/producer_spec.rb +79 -0
  24. data/spec/redstream/trimmer_spec.rb +32 -0
  25. data/spec/redstream_spec.rb +117 -0
  26. data/spec/spec_helper.rb +66 -0
  27. metadata +289 -0
data/spec/redstream/trimmer_spec.rb ADDED
@@ -0,0 +1,32 @@
1
+
2
+ require File.expand_path("../spec_helper", __dir__)
3
+
4
+ RSpec.describe Redstream::Trimmer do
5
+ describe "#run_once" do
6
+ it "trims a stream to the minimum committed id" do
7
+ ids = Array.new(4) do |i|
8
+ redis.xadd Redstream.stream_key_name("default"), payload: JSON.dump(value: "message#{i}")
9
+ end
10
+
11
+ redis.set Redstream.offset_key_name(stream_name: "default", consumer_name: "consumer1"), ids[1]
12
+ redis.set Redstream.offset_key_name(stream_name: "default", consumer_name: "consumer2"), ids[2]
13
+
14
+ trimmer = Redstream::Trimmer.new(
15
+ interval: 5,
16
+ stream_name: "default",
17
+ consumer_names: ["consumer1", "consumer2", "consumer_without_committed_id"]
18
+ )
19
+
20
+ trimmer.run_once
21
+
22
+ expect(redis.xlen(Redstream.stream_key_name("default"))).to eq(2)
23
+ end
24
+
25
+ it "sleeps for the specified time if there's nothing to trim" do
26
+ trimmer = Redstream::Trimmer.new(interval: 1, stream_name: "default", consumer_names: ["unknown_consumer"])
27
+ trimmer.expects(:sleep).with(1).returns(true)
28
+ trimmer.run_once
29
+ end
30
+ end
31
+ end
32
+
data/spec/redstream_spec.rb ADDED
@@ -0,0 +1,117 @@
1
+
2
+ require File.expand_path("spec_helper", __dir__)
3
+
4
+ RSpec.describe Redstream do
5
+ describe ".connection_pool=" do
6
+ it "sets the connection pool" do
7
+ begin
8
+ connection_pool = Redstream.connection_pool
9
+
10
+ Redstream.connection_pool = "pool"
11
+ expect(Redstream.connection_pool).to eq("pool")
12
+ ensure
13
+ Redstream.connection_pool = connection_pool
14
+ end
15
+ end
16
+ end
17
+
18
+ describe ".connection_pool" do
19
+ it "returns the connection pool" do
20
+ begin
21
+ connection_pool = Redstream.connection_pool
22
+
23
+ Redstream.connection_pool = nil
24
+ expect(Redstream.connection_pool).to be_a(ConnectionPool)
25
+
26
+ Redstream.connection_pool = "pool"
27
+ expect(Redstream.connection_pool).to eq("pool")
28
+ ensure
29
+ Redstream.connection_pool = connection_pool
30
+ end
31
+ end
32
+ end
33
+
34
+ describe ".max_stream_id" do
35
+ it "returns the stream's max id" do
36
+ expect(Redstream.max_stream_id("products")).to be_nil
37
+
38
+ id1 = redis.xadd("redstream:stream:products", key: "value")
39
+ id2 = redis.xadd("redstream:stream:products", key: "value")
40
+
41
+ expect(Redstream.max_stream_id("products")).to eq(id2)
42
+ end
43
+ end
44
+
45
+ describe ".max_consumer_id" do
46
+ it "returns the consumer's max id" do
47
+ expect(Redstream.max_consumer_id(stream_name: "products", consumer_name: "consumer")).to be_nil
48
+
49
+ id1 = redis.xadd("redstream:stream:products", key: "value")
50
+ id2 = redis.xadd("redstream:stream:products", key: "value")
51
+
52
+ Redstream::Consumer.new(name: "consumer", stream_name: "products").run_once do |messages|
53
+ # nothing
54
+ end
55
+
56
+ expect(Redstream.max_consumer_id(stream_name: "products", consumer_name: "consumer")).to eq(id2)
57
+ end
58
+ end
59
+
60
+ describe ".stream_key_name" do
61
+ context "without namespace" do
62
+ it "returns the stream key name" do
63
+ expect(Redstream.stream_key_name("products")).to eq("redstream:stream:products")
64
+ end
65
+ end
66
+
67
+ context "with namespace" do
68
+ it "returns the stream key name" do
69
+ begin
70
+ Redstream.namespace = "namespace"
71
+ expect(Redstream.stream_key_name("products")).to eq("namespace:redstream:stream:products")
72
+ ensure
73
+ Redstream.namespace = nil
74
+ end
75
+ end
76
+ end
77
+ end
78
+
79
+ describe ".offset_key_name" do
80
+ context "without namespace" do
81
+ it "returns the offset key name" do
82
+ expect(Redstream.offset_key_name(stream_name: "stream", consumer_name: "consumer")).to eq("redstream:offset:stream:consumer")
83
+ end
84
+ end
85
+
86
+ context "with namespace" do
87
+ it "returns the offset key name" do
88
+ begin
89
+ Redstream.namespace = "namespace"
90
+ expect(Redstream.offset_key_name(stream_name: "stream", consumer_name: "consumer")).to eq("namespace:redstream:offset:stream:consumer")
91
+ ensure
92
+ Redstream.namespace = nil
93
+ end
94
+ end
95
+ end
96
+ end
97
+
98
+ describe ".lock_key_name" do
99
+ context "without namespace" do
100
+ it "returns the lock key name" do
101
+ expect(Redstream.lock_key_name("name")).to eq("redstream:lock:name")
102
+ end
103
+ end
104
+
105
+ context "with namespace" do
106
+ it "returns the lock key name" do
107
+ begin
108
+ Redstream.namespace = "namespace"
109
+ expect(Redstream.lock_key_name("name")).to eq("namespace:redstream:lock:name")
110
+ ensure
111
+ Redstream.namespace = nil
112
+ end
113
+ end
114
+ end
115
+ end
116
+ end
117
+
data/spec/spec_helper.rb ADDED
@@ -0,0 +1,66 @@
1
+
2
+ require File.expand_path("../lib/redstream", __dir__)
3
+ require "active_record"
4
+ require "factory_bot"
5
+ require "database_cleaner"
6
+ require "timecop"
7
+ require "concurrent"
8
+ require "mocha"
9
+ require "rspec/instafail"
10
+
11
+ RSpec.configure do |config|
12
+ config.mock_with :mocha
13
+ end
14
+
15
+ ActiveRecord::Base.establish_connection(adapter: "sqlite3", database: "/tmp/redstream.sqlite3")
16
+
17
+ ActiveRecord::Base.connection.execute "DROP TABLE IF EXISTS products"
18
+
19
+ ActiveRecord::Base.connection.create_table :products do |t|
20
+ t.string :title
21
+ t.timestamps
22
+ end
23
+
24
+ class Product < ActiveRecord::Base
25
+ include Redstream::Model
26
+
27
+ redstream_callbacks
28
+
29
+ def redstream_payload
30
+ { id: id }
31
+ end
32
+ end
33
+
34
+ FactoryBot.define do
35
+ factory :product do
36
+ title { "title" }
37
+ end
38
+ end
39
+
40
+ module SpecHelper
41
+ def redis
42
+ @redis ||= Redis.new
43
+ end
44
+ end
45
+
46
+ RSpec.configure do |config|
47
+ config.add_formatter(RSpec::Instafail)
48
+ config.add_formatter(:progress)
49
+
50
+ config.include SpecHelper
51
+ config.include FactoryBot::Syntax::Methods
52
+
53
+ config.before(:suite) do
54
+ DatabaseCleaner.strategy = :truncation
55
+ end
56
+
57
+ config.around do |example|
58
+ DatabaseCleaner.cleaning { example.run }
59
+ end
60
+
61
+ config.after do
62
+ Redis.new.flushdb
63
+ end
64
+ end
65
+
66
+
metadata ADDED
@@ -0,0 +1,289 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: redstream
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Benjamin Vetter
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2019-04-26 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: activerecord
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: database_cleaner
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: sqlite3
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - '='
88
+ - !ruby/object:Gem::Version
89
+ version: 1.3.13
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - '='
95
+ - !ruby/object:Gem::Version
96
+ version: 1.3.13
97
+ - !ruby/object:Gem::Dependency
98
+ name: factory_bot
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: timecop
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: concurrent-ruby
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ">="
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
139
+ - !ruby/object:Gem::Dependency
140
+ name: rspec-instafail
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ">="
144
+ - !ruby/object:Gem::Version
145
+ version: '0'
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
153
+ - !ruby/object:Gem::Dependency
154
+ name: mocha
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - ">="
158
+ - !ruby/object:Gem::Version
159
+ version: '0'
160
+ type: :development
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - ">="
165
+ - !ruby/object:Gem::Version
166
+ version: '0'
167
+ - !ruby/object:Gem::Dependency
168
+ name: connection_pool
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - ">="
172
+ - !ruby/object:Gem::Version
173
+ version: '0'
174
+ type: :runtime
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - ">="
179
+ - !ruby/object:Gem::Version
180
+ version: '0'
181
+ - !ruby/object:Gem::Dependency
182
+ name: activesupport
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - ">="
186
+ - !ruby/object:Gem::Version
187
+ version: '0'
188
+ type: :runtime
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - ">="
193
+ - !ruby/object:Gem::Version
194
+ version: '0'
195
+ - !ruby/object:Gem::Dependency
196
+ name: redis
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - ">="
200
+ - !ruby/object:Gem::Version
201
+ version: 4.1.0
202
+ type: :runtime
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - ">="
207
+ - !ruby/object:Gem::Version
208
+ version: 4.1.0
209
+ - !ruby/object:Gem::Dependency
210
+ name: json
211
+ requirement: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - ">="
214
+ - !ruby/object:Gem::Version
215
+ version: '0'
216
+ type: :runtime
217
+ prerelease: false
218
+ version_requirements: !ruby/object:Gem::Requirement
219
+ requirements:
220
+ - - ">="
221
+ - !ruby/object:Gem::Version
222
+ version: '0'
223
+ description: Using redis streams to keep your primary database in sync with secondary
224
+ datastores
225
+ email:
226
+ - vetter@plainpicture.de
227
+ executables: []
228
+ extensions: []
229
+ extra_rdoc_files: []
230
+ files:
231
+ - ".gitignore"
232
+ - ".travis.yml"
233
+ - Gemfile
234
+ - LICENSE.txt
235
+ - README.md
236
+ - Rakefile
237
+ - docker-compose.yml
238
+ - lib/redstream.rb
239
+ - lib/redstream/consumer.rb
240
+ - lib/redstream/delayer.rb
241
+ - lib/redstream/lock.rb
242
+ - lib/redstream/message.rb
243
+ - lib/redstream/model.rb
244
+ - lib/redstream/producer.rb
245
+ - lib/redstream/trimmer.rb
246
+ - lib/redstream/version.rb
247
+ - redstream.gemspec
248
+ - spec/redstream/consumer_spec.rb
249
+ - spec/redstream/delayer_spec.rb
250
+ - spec/redstream/lock_spec.rb
251
+ - spec/redstream/model_spec.rb
252
+ - spec/redstream/producer_spec.rb
253
+ - spec/redstream/trimmer_spec.rb
254
+ - spec/redstream_spec.rb
255
+ - spec/spec_helper.rb
256
+ homepage: https://github.com/mrkamel/redstream
257
+ licenses:
258
+ - MIT
259
+ metadata: {}
260
+ post_install_message:
261
+ rdoc_options: []
262
+ require_paths:
263
+ - lib
264
+ required_ruby_version: !ruby/object:Gem::Requirement
265
+ requirements:
266
+ - - ">="
267
+ - !ruby/object:Gem::Version
268
+ version: '0'
269
+ required_rubygems_version: !ruby/object:Gem::Requirement
270
+ requirements:
271
+ - - ">="
272
+ - !ruby/object:Gem::Version
273
+ version: '0'
274
+ requirements: []
275
+ rubyforge_project:
276
+ rubygems_version: 2.7.3
277
+ signing_key:
278
+ specification_version: 4
279
+ summary: Using redis streams to keep your primary database in sync with secondary
280
+ datastores
281
+ test_files:
282
+ - spec/redstream/consumer_spec.rb
283
+ - spec/redstream/delayer_spec.rb
284
+ - spec/redstream/lock_spec.rb
285
+ - spec/redstream/model_spec.rb
286
+ - spec/redstream/producer_spec.rb
287
+ - spec/redstream/trimmer_spec.rb
288
+ - spec/redstream_spec.rb
289
+ - spec/spec_helper.rb