message_bus 2.1.6 → 2.2.0.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of message_bus might be problematic. Click here for more details.

Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +13 -92
  3. data/.rubocop_todo.yml +659 -0
  4. data/.travis.yml +1 -1
  5. data/CHANGELOG +61 -0
  6. data/Dockerfile +18 -0
  7. data/Gemfile +3 -1
  8. data/Guardfile +0 -1
  9. data/README.md +188 -101
  10. data/Rakefile +12 -1
  11. data/assets/message-bus.js +1 -1
  12. data/docker-compose.yml +46 -0
  13. data/examples/bench/config.ru +8 -9
  14. data/examples/bench/unicorn.conf.rb +1 -1
  15. data/examples/chat/chat.rb +150 -153
  16. data/examples/minimal/config.ru +2 -3
  17. data/lib/message_bus.rb +224 -36
  18. data/lib/message_bus/backends.rb +7 -0
  19. data/lib/message_bus/backends/base.rb +184 -0
  20. data/lib/message_bus/backends/memory.rb +304 -226
  21. data/lib/message_bus/backends/postgres.rb +359 -318
  22. data/lib/message_bus/backends/redis.rb +380 -337
  23. data/lib/message_bus/client.rb +99 -41
  24. data/lib/message_bus/connection_manager.rb +29 -21
  25. data/lib/message_bus/diagnostics.rb +50 -41
  26. data/lib/message_bus/distributed_cache.rb +5 -7
  27. data/lib/message_bus/message.rb +2 -2
  28. data/lib/message_bus/rack/diagnostics.rb +65 -55
  29. data/lib/message_bus/rack/middleware.rb +64 -44
  30. data/lib/message_bus/rack/thin_ext.rb +13 -9
  31. data/lib/message_bus/rails/railtie.rb +2 -0
  32. data/lib/message_bus/timer_thread.rb +2 -2
  33. data/lib/message_bus/version.rb +2 -1
  34. data/message_bus.gemspec +3 -2
  35. data/spec/assets/support/jasmine_helper.rb +1 -1
  36. data/spec/lib/fake_async_middleware.rb +1 -6
  37. data/spec/lib/message_bus/assets/asset_encoding_spec.rb +3 -3
  38. data/spec/lib/message_bus/backend_spec.rb +409 -0
  39. data/spec/lib/message_bus/client_spec.rb +8 -11
  40. data/spec/lib/message_bus/connection_manager_spec.rb +8 -14
  41. data/spec/lib/message_bus/distributed_cache_spec.rb +0 -4
  42. data/spec/lib/message_bus/multi_process_spec.rb +6 -7
  43. data/spec/lib/message_bus/rack/middleware_spec.rb +47 -43
  44. data/spec/lib/message_bus/timer_thread_spec.rb +0 -2
  45. data/spec/lib/message_bus_spec.rb +59 -43
  46. data/spec/spec_helper.rb +16 -4
  47. metadata +12 -9
  48. data/spec/lib/message_bus/backends/postgres_spec.rb +0 -221
  49. data/spec/lib/message_bus/backends/redis_spec.rb +0 -271
@@ -1,7 +1,7 @@
1
1
  require_relative '../../../spec_helper'
2
2
  asset_directory = File.expand_path('../../../../../assets', __FILE__)
3
3
  asset_file_paths = Dir.glob(File.join(asset_directory, 'message-bus.js'))
4
- asset_file_names = asset_file_paths.map{|e| File.basename(e) }
4
+ asset_file_names = asset_file_paths.map { |e| File.basename(e) }
5
5
 
6
6
  describe asset_file_names do
7
7
  it 'should contain .js files' do
@@ -9,10 +9,10 @@ describe asset_file_names do
9
9
  end
10
10
  end
11
11
 
12
- asset_file_paths.each do | path |
12
+ asset_file_paths.each do |path|
13
13
  describe "Asset file #{File.basename(path).inspect}" do
14
14
  it 'should be encodable as UTF8' do
15
- binary_data = File.open(path, 'rb'){|f| f.read }
15
+ binary_data = File.open(path, 'rb') { |f| f.read }
16
16
  binary_data.encode(Encoding::UTF_8)
17
17
  end
18
18
  end
@@ -0,0 +1,409 @@
1
+ require_relative '../../spec_helper'
2
+ require 'message_bus'
3
+
4
+ describe PUB_SUB_CLASS do
5
+ def new_test_bus
6
+ PUB_SUB_CLASS.new(MESSAGE_BUS_CONFIG)
7
+ end
8
+
9
+ before do
10
+ @bus = new_test_bus
11
+ @bus.reset!
12
+ end
13
+
14
+ describe "API parity" do
15
+ it "has the same public methods as the base class" do
16
+ @bus.public_methods.sort.must_equal MessageBus::Backends::Base.new(MESSAGE_BUS_CONFIG).public_methods.sort
17
+ end
18
+ end
19
+
20
+ it "should be able to access the backlog" do
21
+ @bus.publish "/foo", "bar"
22
+ @bus.publish "/foo", "baz"
23
+
24
+ @bus.backlog("/foo", 0).to_a.must_equal [
25
+ MessageBus::Message.new(1, 1, '/foo', 'bar'),
26
+ MessageBus::Message.new(2, 2, '/foo', 'baz')
27
+ ]
28
+ end
29
+
30
+ it "should initialize with max_backlog_size" do
31
+ PUB_SUB_CLASS.new({}, 2000).max_backlog_size.must_equal 2000
32
+ end
33
+
34
+ it "should truncate channels correctly" do
35
+ @bus.max_backlog_size = 2
36
+ [
37
+ "one",
38
+ "two",
39
+ "three",
40
+ "four",
41
+ ].each do |t|
42
+ @bus.publish "/foo", t
43
+ end
44
+
45
+ @bus.backlog("/foo").to_a.must_equal [
46
+ MessageBus::Message.new(3, 3, '/foo', 'three'),
47
+ MessageBus::Message.new(4, 4, '/foo', 'four'),
48
+ ]
49
+ end
50
+
51
+ it "should truncate global backlog correctly" do
52
+ @bus.max_global_backlog_size = 2
53
+ @bus.publish "/foo", "one"
54
+ @bus.publish "/bar", "two"
55
+ @bus.publish "/baz", "three"
56
+
57
+ @bus.global_backlog.length.must_equal 2
58
+ end
59
+
60
+ it "should be able to grab a message by id" do
61
+ id1 = @bus.publish "/foo", "bar"
62
+ id2 = @bus.publish "/foo", "baz"
63
+ @bus.get_message("/foo", id2).must_equal MessageBus::Message.new(2, 2, "/foo", "baz")
64
+ @bus.get_message("/foo", id1).must_equal MessageBus::Message.new(1, 1, "/foo", "bar")
65
+ end
66
+
67
+ it "should have the correct number of messages for multi threaded access" do
68
+ threads = []
69
+ 4.times do
70
+ threads << Thread.new do
71
+ 25.times {
72
+ @bus.publish "/foo", "foo"
73
+ }
74
+ end
75
+ end
76
+
77
+ threads.each(&:join)
78
+ @bus.backlog("/foo").length == 100
79
+ end
80
+
81
+ it "should be able to encode and decode messages properly" do
82
+ m = MessageBus::Message.new 1, 2, '||', '||'
83
+ MessageBus::Message.decode(m.encode).must_equal m
84
+ end
85
+
86
+ it "should allow us to get last id on a channel" do
87
+ @bus.last_id("/foo").must_equal 0
88
+ @bus.publish("/foo", "one")
89
+ @bus.last_id("/foo").must_equal 1
90
+ end
91
+
92
+ describe "readonly" do
93
+ after do
94
+ @bus.pub_redis.slaveof "no", "one"
95
+ end
96
+
97
+ it "should be able to store messages in memory for a period while in read only" do
98
+ test_only :redis
99
+ skip "This spec changes redis behavior that in turn means other specs run slow"
100
+
101
+ @bus.pub_redis.slaveof "127.0.0.80", "666"
102
+ @bus.max_in_memory_publish_backlog = 2
103
+
104
+ current_threads = Thread.list
105
+ current_threads_length = current_threads.count
106
+
107
+ 3.times do
108
+ result = @bus.publish "/foo", "bar"
109
+ assert_nil result
110
+ Thread.list.length.must_equal(current_threads_length + 1)
111
+ end
112
+
113
+ @bus.pub_redis.slaveof "no", "one"
114
+ sleep 0.01
115
+
116
+ (Thread.list - current_threads).each(&:join)
117
+ Thread.list.length.must_equal current_threads_length
118
+
119
+ @bus.backlog("/foo", 0).map(&:data).must_equal ["bar", "bar"]
120
+ end
121
+ end
122
+
123
+ it "can set backlog age" do
124
+ @bus.max_backlog_age = 1
125
+
126
+ expected_backlog_size = 0
127
+
128
+ # Start at time = 0s
129
+ @bus.publish "/foo", "bar"
130
+ expected_backlog_size += 1
131
+
132
+ @bus.global_backlog.length.must_equal expected_backlog_size
133
+ @bus.backlog("/foo", 0).length.must_equal expected_backlog_size
134
+
135
+ sleep 1.25 # Should now be at time =~ 1.25s. Our backlog should have expired by now.
136
+ expected_backlog_size = 0
137
+
138
+ case MESSAGE_BUS_CONFIG[:backend]
139
+ when :postgres
140
+ # Force triggering backlog expiry: postgres backend doesn't expire backlogs on a timer, but at publication time.
141
+ @bus.global_backlog.length.wont_equal expected_backlog_size
142
+ @bus.backlog("/foo", 0).length.wont_equal expected_backlog_size
143
+ @bus.publish "/foo", "baz"
144
+ expected_backlog_size += 1
145
+ end
146
+
147
+ # Assert that the backlog did expire, and now has only the new publication in it.
148
+ @bus.global_backlog.length.must_equal expected_backlog_size
149
+ @bus.backlog("/foo", 0).length.must_equal expected_backlog_size
150
+
151
+ sleep 0.75 # Should now be at time =~ 2s
152
+
153
+ @bus.publish "/foo", "baz" # Publish something else before another expiry
154
+ expected_backlog_size += 1
155
+
156
+ sleep 0.75 # Should now be at time =~ 2.75s
157
+ # Our oldest message is now 1.5s old, but we didn't cease publishing for a period of 1s at a time, so we should not have expired the backlog.
158
+
159
+ @bus.publish "/foo", "baz" # Publish something else to ward off another expiry
160
+ expected_backlog_size += 1
161
+
162
+ case MESSAGE_BUS_CONFIG[:backend]
163
+ when :postgres
164
+ # Postgres expires individual messages that have lived longer than the TTL, not whole backlogs
165
+ expected_backlog_size -= 1
166
+ else
167
+ # Assert that the backlog did not expire, and has all of our publications since the last expiry.
168
+ end
169
+ @bus.global_backlog.length.must_equal expected_backlog_size
170
+ @bus.backlog("/foo", 0).length.must_equal expected_backlog_size
171
+ end
172
+
173
+ it "can set backlog age on publish" do
174
+ @bus.max_backlog_age = 100
175
+
176
+ expected_backlog_size = 0
177
+
178
+ initial_id = @bus.last_id("/foo")
179
+
180
+ # Start at time = 0s
181
+ @bus.publish "/foo", "bar", max_backlog_age: 1
182
+ expected_backlog_size += 1
183
+
184
+ @bus.global_backlog.length.must_equal expected_backlog_size
185
+ @bus.backlog("/foo", 0).length.must_equal expected_backlog_size
186
+
187
+ sleep 1.25 # Should now be at time =~ 1.25s. Our backlog should have expired by now.
188
+ expected_backlog_size = 0
189
+
190
+ case MESSAGE_BUS_CONFIG[:backend]
191
+ when :postgres
192
+ # Force triggering backlog expiry: postgres backend doesn't expire backlogs on a timer, but at publication time.
193
+ @bus.global_backlog.length.wont_equal expected_backlog_size
194
+ @bus.backlog("/foo", 0).length.wont_equal expected_backlog_size
195
+ @bus.publish "/foo", "baz", max_backlog_age: 1
196
+ expected_backlog_size += 1
197
+ end
198
+
199
+ # Assert that the backlog did expire, and now has only the new publication in it.
200
+ @bus.global_backlog.length.must_equal expected_backlog_size
201
+ @bus.backlog("/foo", 0).length.must_equal expected_backlog_size
202
+
203
+ # for the time being we can give pg a pass here
204
+ # TODO: make the implementation here consistent
205
+ if MESSAGE_BUS_CONFIG[:backend] != :postgres
206
+ # ids are not opaque we expect them to be reset on our channel if it
207
+ # got cleared due to an expire, the reason for this is cause we will leak entries due to tracking
208
+ # this in turn can bloat storage for the backend
209
+ @bus.last_id("/foo").must_equal initial_id
210
+ end
211
+
212
+ sleep 0.75 # Should now be at time =~ 2s
213
+
214
+ @bus.publish "/foo", "baz", max_backlog_age: 1 # Publish something else before another expiry
215
+ expected_backlog_size += 1
216
+
217
+ sleep 0.75 # Should now be at time =~ 2.75s
218
+ # Our oldest message is now 1.5s old, but we didn't cease publishing for a period of 1s at a time, so we should not have expired the backlog.
219
+
220
+ @bus.publish "/foo", "baz", max_backlog_age: 1 # Publish something else to ward off another expiry
221
+ expected_backlog_size += 1
222
+
223
+ case MESSAGE_BUS_CONFIG[:backend]
224
+ when :postgres
225
+ # Postgres expires individual messages that have lived longer than the TTL, not whole backlogs
226
+ expected_backlog_size -= 1
227
+ else
228
+ # Assert that the backlog did not expire, and has all of our publications since the last expiry.
229
+ end
230
+ @bus.global_backlog.length.must_equal expected_backlog_size
231
+ @bus.backlog("/foo", 0).length.must_equal expected_backlog_size
232
+ end
233
+
234
+ it "can set backlog size on publish" do
235
+ @bus.max_backlog_size = 100
236
+
237
+ @bus.publish "/foo", "bar", max_backlog_size: 2
238
+ @bus.publish "/foo", "bar", max_backlog_size: 2
239
+ @bus.publish "/foo", "bar", max_backlog_size: 2
240
+
241
+ @bus.backlog("/foo").length.must_equal 2
242
+ end
243
+
244
+ it "should be able to access the global backlog" do
245
+ @bus.publish "/foo", "bar"
246
+ @bus.publish "/hello", "world"
247
+ @bus.publish "/foo", "baz"
248
+ @bus.publish "/hello", "planet"
249
+
250
+ expected_messages = case MESSAGE_BUS_CONFIG[:backend]
251
+ when :redis
252
+ # Redis has channel-specific message IDs
253
+ [
254
+ MessageBus::Message.new(1, 1, "/foo", "bar"),
255
+ MessageBus::Message.new(2, 1, "/hello", "world"),
256
+ MessageBus::Message.new(3, 2, "/foo", "baz"),
257
+ MessageBus::Message.new(4, 2, "/hello", "planet")
258
+ ]
259
+ else
260
+ [
261
+ MessageBus::Message.new(1, 1, "/foo", "bar"),
262
+ MessageBus::Message.new(2, 2, "/hello", "world"),
263
+ MessageBus::Message.new(3, 3, "/foo", "baz"),
264
+ MessageBus::Message.new(4, 4, "/hello", "planet")
265
+ ]
266
+ end
267
+
268
+ @bus.global_backlog.to_a.must_equal expected_messages
269
+ end
270
+
271
+ it "should correctly omit dropped messages from the global backlog" do
272
+ @bus.max_backlog_size = 1
273
+ @bus.publish "/foo", "a1"
274
+ @bus.publish "/foo", "b1"
275
+ @bus.publish "/bar", "a1"
276
+ @bus.publish "/bar", "b1"
277
+
278
+ expected_messages = case MESSAGE_BUS_CONFIG[:backend]
279
+ when :redis
280
+ # Redis has channel-specific message IDs
281
+ [
282
+ MessageBus::Message.new(2, 2, "/foo", "b1"),
283
+ MessageBus::Message.new(4, 2, "/bar", "b1")
284
+ ]
285
+ else
286
+ [
287
+ MessageBus::Message.new(2, 2, "/foo", "b1"),
288
+ MessageBus::Message.new(4, 4, "/bar", "b1")
289
+ ]
290
+ end
291
+
292
+ @bus.global_backlog.to_a.must_equal expected_messages
293
+ end
294
+
295
+ it "should cope with a storage reset cleanly" do
296
+ @bus.publish("/foo", "one")
297
+ got = []
298
+
299
+ t = Thread.new do
300
+ @bus.subscribe("/foo") do |msg|
301
+ got << msg
302
+ end
303
+ end
304
+
305
+ # sleep 50ms to allow the bus to correctly subscribe,
306
+ # I thought about adding a subscribed callback, but outside of testing it matters less
307
+ sleep 0.05
308
+
309
+ @bus.publish("/foo", "two")
310
+
311
+ @bus.reset!
312
+
313
+ @bus.publish("/foo", "three")
314
+
315
+ wait_for(100) do
316
+ got.length == 2
317
+ end
318
+
319
+ t.kill
320
+
321
+ got.map { |m| m.data }.must_equal ["two", "three"]
322
+ got[1].global_id.must_equal 1
323
+ end
324
+
325
+ it "should support clear_every setting" do
326
+ test_never :redis
327
+
328
+ @bus.clear_every = 5
329
+ @bus.max_global_backlog_size = 2
330
+ @bus.publish "/foo", "11"
331
+ @bus.publish "/bar", "21"
332
+ @bus.publish "/baz", "31"
333
+ @bus.publish "/bar", "41"
334
+ @bus.global_backlog.length.must_equal 4
335
+
336
+ @bus.publish "/baz", "51"
337
+ @bus.global_backlog.length.must_equal 2
338
+ end
339
+
340
+ it "should be able to subscribe globally with recovery" do
341
+ @bus.publish("/foo", "11")
342
+ @bus.publish("/bar", "12")
343
+ got = []
344
+
345
+ t = Thread.new do
346
+ @bus.global_subscribe(0) do |msg|
347
+ got << msg
348
+ end
349
+ end
350
+
351
+ @bus.publish("/bar", "13")
352
+
353
+ wait_for(100) do
354
+ got.length == 3
355
+ end
356
+
357
+ t.kill
358
+
359
+ got.length.must_equal 3
360
+ got.map { |m| m.data }.must_equal ["11", "12", "13"]
361
+ end
362
+
363
+ it "should handle subscribe on single channel, with recovery" do
364
+ @bus.publish("/foo", "11")
365
+ @bus.publish("/bar", "12")
366
+ got = []
367
+
368
+ t = Thread.new do
369
+ @bus.subscribe("/foo", 0) do |msg|
370
+ got << msg
371
+ end
372
+ end
373
+
374
+ @bus.publish("/foo", "13")
375
+
376
+ wait_for(100) do
377
+ got.length == 2
378
+ end
379
+
380
+ t.kill
381
+
382
+ got.map { |m| m.data }.must_equal ["11", "13"]
383
+ end
384
+
385
+ it "should not get backlog if subscribe is called without params" do
386
+ @bus.publish("/foo", "11")
387
+ got = []
388
+
389
+ t = Thread.new do
390
+ @bus.subscribe("/foo") do |msg|
391
+ got << msg
392
+ end
393
+ end
394
+
395
+ # sleep 50ms to allow the bus to correctly subscribe,
396
+ # I thought about adding a subscribed callback, but outside of testing it matters less
397
+ sleep 0.05
398
+
399
+ @bus.publish("/foo", "12")
400
+
401
+ wait_for(100) do
402
+ got.length == 1
403
+ end
404
+
405
+ t.kill
406
+
407
+ got.map { |m| m.data }.must_equal ["12"]
408
+ end
409
+ end
@@ -2,9 +2,7 @@ require_relative '../../spec_helper'
2
2
  require 'message_bus'
3
3
 
4
4
  describe MessageBus::Client do
5
-
6
5
  describe "subscriptions" do
7
-
8
6
  def setup_client(client_id)
9
7
  MessageBus::Client.new client_id: client_id, message_bus: @bus
10
8
  end
@@ -29,14 +27,15 @@ describe MessageBus::Client do
29
27
 
30
28
  while line = lines.shift
31
29
  break if line == ""
30
+
32
31
  name, val = line.split(": ")
33
32
  headers[name] = val
34
33
  end
35
34
 
36
- length = nil
37
35
  while line = lines.shift
38
36
  length = line.to_i(16)
39
37
  break if length == 0
38
+
40
39
  rest = lines.join("\r\n")
41
40
  chunks << rest[0...length]
42
41
  lines = (rest[length + 2..-1] || "").split("\r\n")
@@ -95,7 +94,6 @@ describe MessageBus::Client do
95
94
  # end with []
96
95
  chunk2 = parse_chunk(chunks[1])
97
96
  chunk2.length.must_equal 0
98
-
99
97
  end
100
98
 
101
99
  it "does not bleed data accross sites" do
@@ -117,7 +115,6 @@ describe MessageBus::Client do
117
115
  end
118
116
 
119
117
  it "allows negative subscribes to look behind" do
120
-
121
118
  @bus.publish '/hello', 'world'
122
119
  @bus.publish '/hello', 'sam'
123
120
 
@@ -139,19 +136,21 @@ describe MessageBus::Client do
139
136
  another_client = setup_client('def')
140
137
  clients = [@client, another_client]
141
138
 
142
- clients.each { |client| client.subscribe('/hello', nil) }
139
+ channel = SecureRandom.hex
140
+
141
+ clients.each { |client| client.subscribe(channel, nil) }
143
142
 
144
- @bus.publish("/hello", "world", client_ids: ['abc'])
143
+ @bus.publish(channel, "world", client_ids: ['abc'])
145
144
 
146
145
  log = @client.backlog
147
146
  log.length.must_equal 1
148
- log[0].channel.must_equal '/hello'
147
+ log[0].channel.must_equal channel
149
148
  log[0].data.must_equal 'world'
150
149
 
151
150
  log = another_client.backlog
152
151
  log.length.must_equal 1
153
152
  log[0].channel.must_equal '/__status'
154
- log[0].data.must_equal('/hello' => 1)
153
+ log[0].data.must_equal(channel => 1)
155
154
  end
156
155
 
157
156
  it "should provide a list of subscriptions" do
@@ -199,8 +198,6 @@ describe MessageBus::Client do
199
198
  @client.group_ids = [77, 0, 10]
200
199
  @client.allowed?(@message).must_equal true
201
200
  end
202
-
203
201
  end
204
202
  end
205
-
206
203
  end