redstream 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +14 -0
  3. data/.travis.yml +10 -0
  4. data/Gemfile +5 -0
  5. data/LICENSE.txt +22 -0
  6. data/README.md +253 -0
  7. data/Rakefile +9 -0
  8. data/docker-compose.yml +6 -0
  9. data/lib/redstream.rb +134 -0
  10. data/lib/redstream/consumer.rb +115 -0
  11. data/lib/redstream/delayer.rb +100 -0
  12. data/lib/redstream/lock.rb +80 -0
  13. data/lib/redstream/message.rb +52 -0
  14. data/lib/redstream/model.rb +57 -0
  15. data/lib/redstream/producer.rb +145 -0
  16. data/lib/redstream/trimmer.rb +91 -0
  17. data/lib/redstream/version.rb +5 -0
  18. data/redstream.gemspec +38 -0
  19. data/spec/redstream/consumer_spec.rb +90 -0
  20. data/spec/redstream/delayer_spec.rb +53 -0
  21. data/spec/redstream/lock_spec.rb +68 -0
  22. data/spec/redstream/model_spec.rb +57 -0
  23. data/spec/redstream/producer_spec.rb +79 -0
  24. data/spec/redstream/trimmer_spec.rb +32 -0
  25. data/spec/redstream_spec.rb +117 -0
  26. data/spec/spec_helper.rb +66 -0
  27. metadata +289 -0
data/spec/redstream/trimmer_spec.rb ADDED
@@ -0,0 +1,32 @@
1
+
2
+ require File.expand_path("../spec_helper", __dir__)
3
+
4
+ RSpec.describe Redstream::Trimmer do
5
+ describe "#run_once" do
6
+ it "trims a stream to the minimum committed id" do
7
+ ids = Array.new(4) do |i|
8
+ redis.xadd Redstream.stream_key_name("default"), payload: JSON.dump(value: "message#{i}")
9
+ end
10
+
11
+ redis.set Redstream.offset_key_name(stream_name: "default", consumer_name: "consumer1"), ids[1]
12
+ redis.set Redstream.offset_key_name(stream_name: "default", consumer_name: "consumer2"), ids[2]
13
+
14
+ trimmer = Redstream::Trimmer.new(
15
+ interval: 5,
16
+ stream_name: "default",
17
+ consumer_names: ["consumer1", "consumer2", "consumer_without_committed_id"]
18
+ )
19
+
20
+ trimmer.run_once
21
+
22
+ expect(redis.xlen(Redstream.stream_key_name("default"))).to eq(2)
23
+ end
24
+
25
+ it "sleeps for the specified time if there's nothing to trim" do
26
+ trimmer = Redstream::Trimmer.new(interval: 1, stream_name: "default", consumer_names: ["unknown_consumer"])
27
+ trimmer.expects(:sleep).with(1).returns(true)
28
+ trimmer.run_once
29
+ end
30
+ end
31
+ end
32
+
data/spec/redstream_spec.rb ADDED
@@ -0,0 +1,117 @@
1
+
2
+ require File.expand_path("spec_helper", __dir__)
3
+
4
+ RSpec.describe Redstream do
5
+ describe ".connection_pool=" do
6
+ it "sets the connection pool" do
7
+ begin
8
+ connection_pool = Redstream.connection_pool
9
+
10
+ Redstream.connection_pool = "pool"
11
+ expect(Redstream.connection_pool).to eq("pool")
12
+ ensure
13
+ Redstream.connection_pool = connection_pool
14
+ end
15
+ end
16
+ end
17
+
18
+ describe ".connection_pool" do
19
+ it "returns the connection pool" do
20
+ begin
21
+ connection_pool = Redstream.connection_pool
22
+
23
+ Redstream.connection_pool = nil
24
+ expect(Redstream.connection_pool).to be_a(ConnectionPool)
25
+
26
+ Redstream.connection_pool = "pool"
27
+ expect(Redstream.connection_pool).to eq("pool")
28
+ ensure
29
+ Redstream.connection_pool = connection_pool
30
+ end
31
+ end
32
+ end
33
+
34
+ describe ".max_stream_id" do
35
+ it "returns the stream's max id" do
36
+ expect(Redstream.max_stream_id("products")).to be_nil
37
+
38
+ id1 = redis.xadd("redstream:stream:products", key: "value")
39
+ id2 = redis.xadd("redstream:stream:products", key: "value")
40
+
41
+ expect(Redstream.max_stream_id("products")).to eq(id2)
42
+ end
43
+ end
44
+
45
+ describe ".max_consumer_id" do
46
+ it "returns the consumer's max id" do
47
+ expect(Redstream.max_consumer_id(stream_name: "products", consumer_name: "consumer")).to be_nil
48
+
49
+ id1 = redis.xadd("redstream:stream:products", key: "value")
50
+ id2 = redis.xadd("redstream:stream:products", key: "value")
51
+
52
+ Redstream::Consumer.new(name: "consumer", stream_name: "products").run_once do |messages|
53
+ # nothing
54
+ end
55
+
56
+ expect(Redstream.max_consumer_id(stream_name: "products", consumer_name: "consumer")).to eq(id2)
57
+ end
58
+ end
59
+
60
+ describe ".stream_key_name" do
61
+ context "without namespace" do
62
+ it "returns the stream key name" do
63
+ expect(Redstream.stream_key_name("products")).to eq("redstream:stream:products")
64
+ end
65
+ end
66
+
67
+ context "with namespace" do
68
+ it "returns the stream key name" do
69
+ begin
70
+ Redstream.namespace = "namespace"
71
+ expect(Redstream.stream_key_name("products")).to eq("namespace:redstream:stream:products")
72
+ ensure
73
+ Redstream.namespace = nil
74
+ end
75
+ end
76
+ end
77
+ end
78
+
79
+ describe ".offset_key_name" do
80
+ context "without namespace" do
81
+ it "returns the offset key name" do
82
+ expect(Redstream.offset_key_name(stream_name: "stream", consumer_name: "consumer")).to eq("redstream:offset:stream:consumer")
83
+ end
84
+ end
85
+
86
+ context "with namespace" do
87
+ it "returns the offset key name" do
88
+ begin
89
+ Redstream.namespace = "namespace"
90
+ expect(Redstream.offset_key_name(stream_name: "stream", consumer_name: "consumer")).to eq("namespace:redstream:offset:stream:consumer")
91
+ ensure
92
+ Redstream.namespace = nil
93
+ end
94
+ end
95
+ end
96
+ end
97
+
98
+ describe ".lock_key_name" do
99
+ context "without namespace" do
100
+ it "returns the lock key name" do
101
+ expect(Redstream.lock_key_name("name")).to eq("redstream:lock:name")
102
+ end
103
+ end
104
+
105
+ context "with namespace" do
106
+ it "returns the lock key name" do
107
+ begin
108
+ Redstream.namespace = "namespace"
109
+ expect(Redstream.lock_key_name("name")).to eq("namespace:redstream:lock:name")
110
+ ensure
111
+ Redstream.namespace = nil
112
+ end
113
+ end
114
+ end
115
+ end
116
+ end
117
+
data/spec/spec_helper.rb ADDED
@@ -0,0 +1,66 @@
1
+
2
+ require File.expand_path("../lib/redstream", __dir__)
3
+ require "active_record"
4
+ require "factory_bot"
5
+ require "database_cleaner"
6
+ require "timecop"
7
+ require "concurrent"
8
+ require "mocha"
9
+ require "rspec/instafail"
10
+
11
+ RSpec.configure do |config|
12
+ config.mock_with :mocha
13
+ end
14
+
15
+ ActiveRecord::Base.establish_connection(adapter: "sqlite3", database: "/tmp/redstream.sqlite3")
16
+
17
+ ActiveRecord::Base.connection.execute "DROP TABLE IF EXISTS products"
18
+
19
+ ActiveRecord::Base.connection.create_table :products do |t|
20
+ t.string :title
21
+ t.timestamps
22
+ end
23
+
24
+ class Product < ActiveRecord::Base
25
+ include Redstream::Model
26
+
27
+ redstream_callbacks
28
+
29
+ def redstream_payload
30
+ { id: id }
31
+ end
32
+ end
33
+
34
+ FactoryBot.define do
35
+ factory :product do
36
+ title { "title" }
37
+ end
38
+ end
39
+
40
+ module SpecHelper
41
+ def redis
42
+ @redis ||= Redis.new
43
+ end
44
+ end
45
+
46
+ RSpec.configure do |config|
47
+ config.add_formatter(RSpec::Instafail)
48
+ config.add_formatter(:progress)
49
+
50
+ config.include SpecHelper
51
+ config.include FactoryBot::Syntax::Methods
52
+
53
+ config.before(:suite) do
54
+ DatabaseCleaner.strategy = :truncation
55
+ end
56
+
57
+ config.around do |example|
58
+ DatabaseCleaner.cleaning { example.run }
59
+ end
60
+
61
+ config.after do
62
+ Redis.new.flushdb
63
+ end
64
+ end
65
+
66
+
metadata ADDED
@@ -0,0 +1,289 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: redstream
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Benjamin Vetter
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2019-04-26 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: activerecord
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: database_cleaner
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: sqlite3
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - '='
88
+ - !ruby/object:Gem::Version
89
+ version: 1.3.13
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - '='
95
+ - !ruby/object:Gem::Version
96
+ version: 1.3.13
97
+ - !ruby/object:Gem::Dependency
98
+ name: factory_bot
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: timecop
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: concurrent-ruby
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ">="
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
139
+ - !ruby/object:Gem::Dependency
140
+ name: rspec-instafail
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ">="
144
+ - !ruby/object:Gem::Version
145
+ version: '0'
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
153
+ - !ruby/object:Gem::Dependency
154
+ name: mocha
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - ">="
158
+ - !ruby/object:Gem::Version
159
+ version: '0'
160
+ type: :development
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - ">="
165
+ - !ruby/object:Gem::Version
166
+ version: '0'
167
+ - !ruby/object:Gem::Dependency
168
+ name: connection_pool
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - ">="
172
+ - !ruby/object:Gem::Version
173
+ version: '0'
174
+ type: :runtime
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - ">="
179
+ - !ruby/object:Gem::Version
180
+ version: '0'
181
+ - !ruby/object:Gem::Dependency
182
+ name: activesupport
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - ">="
186
+ - !ruby/object:Gem::Version
187
+ version: '0'
188
+ type: :runtime
189
+ prerelease: false
190
+ version_requirements: !ruby/object:Gem::Requirement
191
+ requirements:
192
+ - - ">="
193
+ - !ruby/object:Gem::Version
194
+ version: '0'
195
+ - !ruby/object:Gem::Dependency
196
+ name: redis
197
+ requirement: !ruby/object:Gem::Requirement
198
+ requirements:
199
+ - - ">="
200
+ - !ruby/object:Gem::Version
201
+ version: 4.1.0
202
+ type: :runtime
203
+ prerelease: false
204
+ version_requirements: !ruby/object:Gem::Requirement
205
+ requirements:
206
+ - - ">="
207
+ - !ruby/object:Gem::Version
208
+ version: 4.1.0
209
+ - !ruby/object:Gem::Dependency
210
+ name: json
211
+ requirement: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - ">="
214
+ - !ruby/object:Gem::Version
215
+ version: '0'
216
+ type: :runtime
217
+ prerelease: false
218
+ version_requirements: !ruby/object:Gem::Requirement
219
+ requirements:
220
+ - - ">="
221
+ - !ruby/object:Gem::Version
222
+ version: '0'
223
+ description: Using redis streams to keep your primary database in sync with secondary
224
+ datastores
225
+ email:
226
+ - vetter@plainpicture.de
227
+ executables: []
228
+ extensions: []
229
+ extra_rdoc_files: []
230
+ files:
231
+ - ".gitignore"
232
+ - ".travis.yml"
233
+ - Gemfile
234
+ - LICENSE.txt
235
+ - README.md
236
+ - Rakefile
237
+ - docker-compose.yml
238
+ - lib/redstream.rb
239
+ - lib/redstream/consumer.rb
240
+ - lib/redstream/delayer.rb
241
+ - lib/redstream/lock.rb
242
+ - lib/redstream/message.rb
243
+ - lib/redstream/model.rb
244
+ - lib/redstream/producer.rb
245
+ - lib/redstream/trimmer.rb
246
+ - lib/redstream/version.rb
247
+ - redstream.gemspec
248
+ - spec/redstream/consumer_spec.rb
249
+ - spec/redstream/delayer_spec.rb
250
+ - spec/redstream/lock_spec.rb
251
+ - spec/redstream/model_spec.rb
252
+ - spec/redstream/producer_spec.rb
253
+ - spec/redstream/trimmer_spec.rb
254
+ - spec/redstream_spec.rb
255
+ - spec/spec_helper.rb
256
+ homepage: https://github.com/mrkamel/redstream
257
+ licenses:
258
+ - MIT
259
+ metadata: {}
260
+ post_install_message:
261
+ rdoc_options: []
262
+ require_paths:
263
+ - lib
264
+ required_ruby_version: !ruby/object:Gem::Requirement
265
+ requirements:
266
+ - - ">="
267
+ - !ruby/object:Gem::Version
268
+ version: '0'
269
+ required_rubygems_version: !ruby/object:Gem::Requirement
270
+ requirements:
271
+ - - ">="
272
+ - !ruby/object:Gem::Version
273
+ version: '0'
274
+ requirements: []
275
+ rubyforge_project:
276
+ rubygems_version: 2.7.3
277
+ signing_key:
278
+ specification_version: 4
279
+ summary: Using redis streams to keep your primary database in sync with secondary
280
+ datastores
281
+ test_files:
282
+ - spec/redstream/consumer_spec.rb
283
+ - spec/redstream/delayer_spec.rb
284
+ - spec/redstream/lock_spec.rb
285
+ - spec/redstream/model_spec.rb
286
+ - spec/redstream/producer_spec.rb
287
+ - spec/redstream/trimmer_spec.rb
288
+ - spec/redstream_spec.rb
289
+ - spec/spec_helper.rb