message_bus 3.3.6 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.eslintrc.js +3 -2
- data/.github/workflows/ci.yml +79 -32
- data/.prettierrc +1 -0
- data/CHANGELOG +104 -53
- data/DEV.md +0 -2
- data/Gemfile +0 -27
- data/LICENSE +1 -1
- data/README.md +40 -62
- data/Rakefile +31 -26
- data/assets/message-bus-ajax.js +3 -3
- data/bench/codecs/marshal.rb +1 -1
- data/bench/codecs/packed_string.rb +1 -1
- data/docker-compose.yml +1 -1
- data/examples/bench/bench.lua +2 -2
- data/lib/message_bus/backends/base.rb +8 -3
- data/lib/message_bus/backends/memory.rb +6 -0
- data/lib/message_bus/backends/postgres.rb +29 -16
- data/lib/message_bus/backends/redis.rb +11 -2
- data/lib/message_bus/client.rb +6 -7
- data/lib/message_bus/connection_manager.rb +1 -1
- data/lib/message_bus/distributed_cache.rb +3 -1
- data/lib/message_bus/http_client.rb +2 -2
- data/lib/message_bus/rack/middleware.rb +6 -6
- data/lib/message_bus/rack/thin_ext.rb +2 -1
- data/lib/message_bus/version.rb +1 -1
- data/lib/message_bus.rb +47 -77
- data/message_bus.gemspec +21 -3
- data/package-lock.json +1575 -23
- data/package.json +9 -7
- data/spec/assets/SpecHelper.js +6 -5
- data/spec/assets/message-bus.spec.js +9 -6
- data/spec/helpers.rb +23 -7
- data/spec/integration/http_client_spec.rb +1 -1
- data/spec/lib/fake_async_middleware.rb +1 -0
- data/spec/lib/message_bus/backend_spec.rb +15 -46
- data/spec/lib/message_bus/client_spec.rb +7 -6
- data/spec/lib/message_bus/connection_manager_spec.rb +4 -0
- data/spec/lib/message_bus/distributed_cache_spec.rb +5 -7
- data/spec/lib/message_bus/multi_process_spec.rb +21 -10
- data/spec/lib/message_bus/rack/middleware_spec.rb +8 -44
- data/spec/lib/message_bus/timer_thread_spec.rb +1 -5
- data/spec/lib/message_bus_spec.rb +22 -9
- data/spec/performance/publish.rb +4 -4
- data/spec/spec_helper.rb +8 -9
- data/spec/support/jasmine-browser.json +16 -0
- data/vendor/assets/javascripts/message-bus-ajax.js +3 -3
- metadata +220 -19
- data/assets/application.jsx +0 -121
- data/assets/babel.min.js +0 -25
- data/assets/react-dom.js +0 -19851
- data/assets/react.js +0 -3029
- data/examples/diagnostics/Gemfile +0 -6
- data/examples/diagnostics/config.ru +0 -22
- data/lib/message_bus/diagnostics.rb +0 -62
- data/lib/message_bus/rack/diagnostics.rb +0 -98
- data/spec/assets/support/jasmine.yml +0 -126
- data/spec/assets/support/jasmine_helper.rb +0 -11
data/README.md
CHANGED
@@ -12,7 +12,7 @@ Read the generated docs: <https://www.rubydoc.info/gems/message_bus>
|
|
12
12
|
|
13
13
|
## Ruby version support
|
14
14
|
|
15
|
-
MessageBus only support officially supported versions of Ruby; as of [
|
15
|
+
MessageBus only support officially supported versions of Ruby; as of [2021-03-31](https://www.ruby-lang.org/en/downloads/branches/) this means we only support Ruby version 2.6 and up.
|
16
16
|
|
17
17
|
## Can you handle concurrent requests?
|
18
18
|
|
@@ -26,15 +26,21 @@ MessageBus only support officially supported versions of Ruby; as of [2018-06-20
|
|
26
26
|
|
27
27
|
Add this line to your application's Gemfile:
|
28
28
|
|
29
|
-
|
29
|
+
```ruby
|
30
|
+
gem 'message_bus'
|
31
|
+
```
|
30
32
|
|
31
33
|
And then execute:
|
32
34
|
|
33
|
-
|
35
|
+
```shell
|
36
|
+
$ bundle
|
37
|
+
```
|
34
38
|
|
35
39
|
Or install it yourself as:
|
36
40
|
|
37
|
-
|
41
|
+
```shell
|
42
|
+
$ gem install message_bus
|
43
|
+
```
|
38
44
|
|
39
45
|
## Usage
|
40
46
|
|
@@ -68,19 +74,19 @@ id = MessageBus.last_id("/channel")
|
|
68
74
|
MessageBus.backlog "/channel", id
|
69
75
|
```
|
70
76
|
|
71
|
-
###
|
77
|
+
### Targeted messages
|
72
78
|
|
73
|
-
Messages can be
|
79
|
+
Messages can be targeted to particular clients by supplying the `client_ids` option when publishing a message.
|
74
80
|
|
75
81
|
```ruby
|
76
82
|
MessageBus.publish "/channel", "hello", client_ids: ["XXX", "YYY"] # (using MessageBus.clientId)
|
77
83
|
```
|
78
84
|
|
79
|
-
By configuring the `user_id_lookup` and `group_ids_lookup` options with a Proc or Lambda which will be called with a [Rack specification environment](https://github.com/rack/rack/blob/master/SPEC.rdoc#the-environment-), messages can be
|
85
|
+
By configuring the `user_id_lookup` and `group_ids_lookup` options with a Proc or Lambda which will be called with a [Rack specification environment](https://github.com/rack/rack/blob/master/SPEC.rdoc#the-environment-), messages can be targeted to particular clients users or groups by supplying either the `user_ids` or `group_ids` options when publishing a message.
|
80
86
|
|
81
87
|
```ruby
|
82
88
|
MessageBus.configure(user_id_lookup: proc do |env|
|
83
|
-
# this lookup occurs on JS-client
|
89
|
+
# this lookup occurs on JS-client polling, so that server can retrieve backlog
|
84
90
|
# for the client considering/matching/filtering user_ids set on published messages
|
85
91
|
# if user_id is not set on publish time, any user_id returned here will receive the message
|
86
92
|
# return the user id here
|
@@ -98,7 +104,7 @@ end)
|
|
98
104
|
MessageBus.publish "/channel", "hello", group_ids: [1, 2, 3]
|
99
105
|
|
100
106
|
# example of MessageBus to set user_ids from an initializer in Rails and Devise:
|
101
|
-
# config/
|
107
|
+
# config/initializers/message_bus.rb
|
102
108
|
MessageBus.user_id_lookup do |env|
|
103
109
|
req = Rack::Request.new(env)
|
104
110
|
|
@@ -109,13 +115,13 @@ MessageBus.user_id_lookup do |env|
|
|
109
115
|
end
|
110
116
|
```
|
111
117
|
|
112
|
-
If both `user_ids` and `group_ids` options are supplied when publishing a message, the message will be
|
118
|
+
If both `user_ids` and `group_ids` options are supplied when publishing a message, the message will be targeted at clients with lookup return values that matches on either the `user_ids` **or** the `group_ids` options.
|
113
119
|
|
114
120
|
```ruby
|
115
121
|
MessageBus.publish "/channel", "hello", user_ids: [1, 2, 3], group_ids: [1, 2, 3]
|
116
122
|
```
|
117
123
|
|
118
|
-
If the `client_ids` option is supplied with either the `user_ids` or `group_ids` options when
|
124
|
+
If the `client_ids` option is supplied with either the `user_ids` or `group_ids` options when publishing a message, the `client_ids` option will be applied unconditionally and messages will be filtered further using `user_id` or `group_id` clauses.
|
119
125
|
|
120
126
|
```ruby
|
121
127
|
MessageBus.publish "/channel", "hello", client_ids: ["XXX", "YYY"], user_ids: [1, 2, 3], group_ids: [1, 2, 3]
|
@@ -129,7 +135,7 @@ Custom client message filters can be registered via `MessageBus#register_client_
|
|
129
135
|
|
130
136
|
For example, ensuring that only messages seen by the server in the last 20 seconds are published to the client:
|
131
137
|
|
132
|
-
```
|
138
|
+
```ruby
|
133
139
|
MessageBus.register_client_message_filter('/test') do |message|
|
134
140
|
(Time.now.to_i - message.data[:published_at]) <= 20
|
135
141
|
end
|
@@ -163,32 +169,6 @@ curl -H "Content-Type: application/x-www-form-urlencoded" -X POST --data "/messa
|
|
163
169
|
|
164
170
|
You should see a reply with the messages of that channel you requested (in this case `/message`) starting at the message ID you requested (`0`). The URL parameter `dlp=t` disables long-polling: we do not want this request to stay open.
|
165
171
|
|
166
|
-
### Diagnostics
|
167
|
-
|
168
|
-
MessageBus comes with a diagnostics interface, which you can access at `/message-bus/_diagnostics`. This interface allows you visibility into the runtime behaviour of message_bus.
|
169
|
-
|
170
|
-
In order to use the diagnostics UI in your application, it is necessary to:
|
171
|
-
|
172
|
-
* Enable it
|
173
|
-
* Define a user ID for requests
|
174
|
-
* Define a check for admin role
|
175
|
-
|
176
|
-
as an example, you can do something like this:
|
177
|
-
|
178
|
-
```ruby
|
179
|
-
MessageBus.enable_diagnostics # Must be called after `MessageBus.after_fork` if using a forking webserver
|
180
|
-
|
181
|
-
MessageBus.user_id_lookup do |_env|
|
182
|
-
1
|
183
|
-
end
|
184
|
-
|
185
|
-
MessageBus.is_admin_lookup do |_env|
|
186
|
-
true
|
187
|
-
end
|
188
|
-
```
|
189
|
-
|
190
|
-
Of course, in your real-world application, you would define these values according to your authentication/authorization logic.
|
191
|
-
|
192
172
|
### Transport
|
193
173
|
|
194
174
|
MessageBus ships with 3 transport mechanisms.
|
@@ -245,7 +225,7 @@ end)
|
|
245
225
|
MessageBus.publish "/channel", "some message"
|
246
226
|
|
247
227
|
# you can also choose to pass the `:site_id`.
|
248
|
-
# This takes
|
228
|
+
# This takes precedence over whatever `site_id_lookup`
|
249
229
|
# returns
|
250
230
|
MessageBus.publish "/channel", "some message", site_id: "site-id"
|
251
231
|
|
@@ -280,7 +260,7 @@ MessageBus.start(); // call once at startup
|
|
280
260
|
MessageBus.callbackInterval = 500;
|
281
261
|
|
282
262
|
// you will get all new messages sent to channel
|
283
|
-
MessageBus.subscribe("/channel", function(data){
|
263
|
+
MessageBus.subscribe("/channel", function (data) {
|
284
264
|
// data shipped from server
|
285
265
|
});
|
286
266
|
|
@@ -386,6 +366,16 @@ headers|{}|Extra headers to be include with requests. Properties and values of o
|
|
386
366
|
|
387
367
|
message_bus can be configured to use one of several available storage backends, and each has its own configuration options.
|
388
368
|
|
369
|
+
### Keepalive
|
370
|
+
|
371
|
+
To ensure correct operation of message_bus, every 60 seconds a message is broadcast to itself. If for any reason the message is not consumed by the same process within 3 keepalive intervals a warning log message is raised.
|
372
|
+
|
373
|
+
To control keepalive interval use
|
374
|
+
|
375
|
+
```ruby
|
376
|
+
MessageBus.configure(keepalive_interval: 60)
|
377
|
+
```
|
378
|
+
|
389
379
|
### Redis
|
390
380
|
|
391
381
|
message_bus supports using Redis as a storage backend, and you can configure message_bus to use redis in `config/initializers/message_bus.rb`, like so:
|
@@ -400,17 +390,17 @@ The redis client message_bus uses is [redis-rb](https://github.com/redis/redis-r
|
|
400
390
|
|
401
391
|
Out of the box Redis keeps track of 2000 messages in the global backlog and 1000 messages in a per-channel backlog. Per-channel backlogs get cleared automatically after 7 days of inactivity.
|
402
392
|
|
403
|
-
This is configurable via accessors on the
|
393
|
+
This is configurable via accessors on the Backend instance.
|
404
394
|
|
405
395
|
```ruby
|
406
396
|
# only store 100 messages per channel
|
407
|
-
MessageBus.
|
397
|
+
MessageBus.backend_instance.max_backlog_size = 100
|
408
398
|
|
409
399
|
# only store 100 global messages
|
410
|
-
MessageBus.
|
400
|
+
MessageBus.backend_instance.max_global_backlog_size = 100
|
411
401
|
|
412
402
|
# flush per-channel backlog after 100 seconds of inactivity
|
413
|
-
MessageBus.
|
403
|
+
MessageBus.backend_instance.max_backlog_age = 100
|
414
404
|
```
|
415
405
|
|
416
406
|
### PostgreSQL
|
@@ -435,7 +425,6 @@ MessageBus.configure(backend: :memory)
|
|
435
425
|
|
436
426
|
The `:clear_every` option supported by the PostgreSQL backend is also supported by the in-memory backend.
|
437
427
|
|
438
|
-
|
439
428
|
### Transport codecs
|
440
429
|
|
441
430
|
By default MessageBus serializes messages to the backend using JSON. Under most situation this performs extremely well.
|
@@ -454,9 +443,9 @@ using a packed string encoder.
|
|
454
443
|
|
455
444
|
Keep in mind, much of MessageBus internals and supporting tools expect data to be converted to JSON and back, if you use a naive (and fast) `Marshal` based codec you may need to limit the features you use. Specifically the Postgresql backend expects the codec never to return a string with `\u0000`, additionally some classes like DistributedCache expect keys to be converted to Strings.
|
456
445
|
|
457
|
-
Another example may be very large and complicated messages where Oj in
|
446
|
+
Another example may be very large and complicated messages where Oj in compatibility mode outperforms JSON. To opt for the Oj codec use:
|
458
447
|
|
459
|
-
```
|
448
|
+
```ruby
|
460
449
|
MessageBus.configure(transport_codec: MessageBus::Codec::Oj.new)
|
461
450
|
```
|
462
451
|
|
@@ -503,7 +492,6 @@ For more information see the [Passenger documentation](https://www.phusionpassen
|
|
503
492
|
|
504
493
|
```ruby
|
505
494
|
# path/to/your/config/puma.rb
|
506
|
-
require 'message_bus' # omit this line for Rails 5
|
507
495
|
on_worker_boot do
|
508
496
|
MessageBus.after_fork
|
509
497
|
end
|
@@ -513,7 +501,6 @@ end
|
|
513
501
|
|
514
502
|
```ruby
|
515
503
|
# path/to/your/config/unicorn.rb
|
516
|
-
require 'message_bus'
|
517
504
|
after_fork do |server, worker|
|
518
505
|
MessageBus.after_fork
|
519
506
|
end
|
@@ -554,26 +541,21 @@ MessageBus ships with an optional `DistributedCache` API which provides a simple
|
|
554
541
|
require 'message_bus/distributed_cache'
|
555
542
|
|
556
543
|
# process 1
|
557
|
-
|
558
544
|
cache = MessageBus::DistributedCache.new("animals")
|
559
545
|
|
560
546
|
# process 2
|
561
|
-
|
562
547
|
cache = MessageBus::DistributedCache.new("animals")
|
563
548
|
|
564
549
|
# process 1
|
565
|
-
|
566
550
|
cache["frogs"] = 5
|
567
551
|
|
568
552
|
# process 2
|
569
|
-
|
570
553
|
puts cache["frogs"]
|
571
554
|
# => 5
|
572
555
|
|
573
556
|
cache["frogs"] = nil
|
574
557
|
|
575
558
|
# process 1
|
576
|
-
|
577
559
|
puts cache["frogs"]
|
578
560
|
# => nil
|
579
561
|
```
|
@@ -631,7 +613,7 @@ In e.g. `config/initializers/message_bus.rb`:
|
|
631
613
|
```ruby
|
632
614
|
MessageBus.extra_response_headers_lookup do |env|
|
633
615
|
[
|
634
|
-
|
616
|
+
["Access-Control-Allow-Origin", "http://example.com:3000"],
|
635
617
|
]
|
636
618
|
end
|
637
619
|
```
|
@@ -692,8 +674,8 @@ In certain conditions, a status message will be delivered and look like this:
|
|
692
674
|
"message_id": -1,
|
693
675
|
"channel": "/__status",
|
694
676
|
"data": {
|
695
|
-
"/some/channel":5,
|
696
|
-
"/other/channel":9
|
677
|
+
"/some/channel": 5,
|
678
|
+
"/other/channel": 9
|
697
679
|
}
|
698
680
|
}
|
699
681
|
```
|
@@ -727,7 +709,7 @@ When submitting a PR, please be sure to include notes on it in the `Unreleased`
|
|
727
709
|
|
728
710
|
To run tests you need both Postgres and Redis installed. By default on Redis the tests connect to `localhost:6379` and on Postgres connect the database `localhost:5432/message_bus_test` with the system username; if you wish to override this, you can set alternative values:
|
729
711
|
|
730
|
-
```
|
712
|
+
```shell
|
731
713
|
PGUSER=some_user PGDATABASE=some_db bundle exec rake
|
732
714
|
```
|
733
715
|
|
@@ -742,7 +724,3 @@ While working on documentation, it is useful to automatically re-build it as you
|
|
742
724
|
### Benchmarks
|
743
725
|
|
744
726
|
Some simple benchmarks are implemented in `spec/performance` and can be executed using `rake performance` (or `docker-compose run tests rake performance`). You should run these before and after your changes to avoid introducing performance regressions.
|
745
|
-
|
746
|
-
### Diagnostics Interface
|
747
|
-
|
748
|
-
It is possible to manually test the diagnostics interface by executing `docker-compose up example` and then `open http://localhost:9292`.
|
data/Rakefile
CHANGED
@@ -4,23 +4,17 @@ require 'rake/testtask'
|
|
4
4
|
require 'bundler'
|
5
5
|
require 'bundler/gem_tasks'
|
6
6
|
require 'bundler/setup'
|
7
|
-
require '
|
7
|
+
require 'rubocop/rake_task'
|
8
|
+
require 'yard'
|
8
9
|
|
9
|
-
|
10
|
-
load 'jasmine/tasks/jasmine.rake'
|
10
|
+
Bundler.require(:default, :test)
|
11
11
|
|
12
|
-
require 'rubocop/rake_task'
|
13
12
|
RuboCop::RakeTask.new
|
14
|
-
|
15
|
-
require 'yard'
|
16
13
|
YARD::Rake::YardocTask.new
|
17
14
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
end
|
22
|
-
|
23
|
-
Bundler.require(:default, :test)
|
15
|
+
BACKENDS = Dir["lib/message_bus/backends/*.rb"].map { |file| file.match(%r{backends/(?<backend>.*).rb})[:backend] } - ["base"]
|
16
|
+
SPEC_FILES = Dir['spec/**/*_spec.rb']
|
17
|
+
INTEGRATION_FILES = Dir['spec/integration/**/*_spec.rb']
|
24
18
|
|
25
19
|
module CustomBuild
|
26
20
|
def build_gem
|
@@ -35,20 +29,30 @@ module Bundler
|
|
35
29
|
end
|
36
30
|
end
|
37
31
|
|
38
|
-
|
32
|
+
desc "Generate documentation for Yard, and fail if there are any warnings"
|
33
|
+
task :test_doc do
|
34
|
+
sh "yard --fail-on-warning #{'--no-progress' if ENV['CI']}"
|
35
|
+
end
|
39
36
|
|
40
|
-
|
37
|
+
namespace :jasmine do
|
38
|
+
desc "Run Jasmine tests in headless mode"
|
39
|
+
task 'ci' do
|
40
|
+
if !system("npx jasmine-browser-runner runSpecs")
|
41
|
+
exit 1
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
41
45
|
|
42
46
|
namespace :spec do
|
43
|
-
|
44
|
-
integration_files = Dir['spec/integration/**/*_spec.rb']
|
45
|
-
|
46
|
-
backends.each do |backend|
|
47
|
+
BACKENDS.each do |backend|
|
47
48
|
desc "Run tests on the #{backend} backend"
|
48
49
|
task backend do
|
49
50
|
begin
|
50
51
|
ENV['MESSAGE_BUS_BACKEND'] = backend
|
51
|
-
|
52
|
+
Rake::TestTask.new(backend) do |t|
|
53
|
+
t.test_files = SPEC_FILES - INTEGRATION_FILES
|
54
|
+
end
|
55
|
+
Rake::Task[backend].invoke
|
52
56
|
ensure
|
53
57
|
ENV.delete('MESSAGE_BUS_BACKEND')
|
54
58
|
end
|
@@ -57,6 +61,8 @@ namespace :spec do
|
|
57
61
|
|
58
62
|
desc "Run integration tests"
|
59
63
|
task :integration do
|
64
|
+
require "socket"
|
65
|
+
|
60
66
|
def port_available?(port)
|
61
67
|
server = TCPServer.open("0.0.0.0", port)
|
62
68
|
server.close
|
@@ -69,7 +75,10 @@ namespace :spec do
|
|
69
75
|
ENV['MESSAGE_BUS_BACKEND'] = 'memory'
|
70
76
|
pid = spawn("bundle exec puma -p 9292 spec/fixtures/test/config.ru")
|
71
77
|
sleep 1 while port_available?(9292)
|
72
|
-
|
78
|
+
Rake::TestTask.new(:integration) do |t|
|
79
|
+
t.test_files = INTEGRATION_FILES
|
80
|
+
end
|
81
|
+
Rake::Task[:integration].invoke
|
73
82
|
ensure
|
74
83
|
ENV.delete('MESSAGE_BUS_BACKEND')
|
75
84
|
Process.kill('TERM', pid) if pid
|
@@ -78,12 +87,12 @@ namespace :spec do
|
|
78
87
|
end
|
79
88
|
|
80
89
|
desc "Run tests on all backends, plus client JS tests"
|
81
|
-
task spec:
|
90
|
+
task spec: BACKENDS.map { |backend| "spec:#{backend}" } + ["jasmine:ci", "spec:integration"]
|
82
91
|
|
83
92
|
desc "Run performance benchmarks on all backends"
|
84
93
|
task :performance do
|
85
94
|
begin
|
86
|
-
ENV['MESSAGE_BUS_BACKENDS'] =
|
95
|
+
ENV['MESSAGE_BUS_BACKENDS'] = BACKENDS.join(",")
|
87
96
|
sh "#{FileUtils::RUBY} -e \"ARGV.each{|f| load f}\" #{Dir['spec/performance/*.rb'].to_a.join(' ')}"
|
88
97
|
ensure
|
89
98
|
ENV.delete('MESSAGE_BUS_BACKENDS')
|
@@ -92,7 +101,3 @@ end
|
|
92
101
|
|
93
102
|
desc "Run all tests, link checks and confirms documentation compiles without error"
|
94
103
|
task default: [:spec, :rubocop, :test_doc]
|
95
|
-
|
96
|
-
Rake::Task['release'].enhance do
|
97
|
-
sh "yarn publish"
|
98
|
-
end
|
data/assets/message-bus-ajax.js
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
// as a fallback if jQuery is not present
|
3
3
|
//
|
4
4
|
// Only implements methods & options used by MessageBus
|
5
|
-
(function(global
|
5
|
+
(function(global) {
|
6
6
|
'use strict';
|
7
7
|
if (!global.MessageBus){
|
8
8
|
throw new Error("MessageBus must be loaded before the ajax adapter");
|
@@ -16,7 +16,7 @@
|
|
16
16
|
for (var name in options.headers){
|
17
17
|
xhr.setRequestHeader(name, options.headers[name]);
|
18
18
|
}
|
19
|
-
xhr.setRequestHeader('Content-Type', 'application/
|
19
|
+
xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
|
20
20
|
if (options.messageBus.chunked){
|
21
21
|
options.messageBus.onProgressListener(xhr);
|
22
22
|
}
|
@@ -31,7 +31,7 @@
|
|
31
31
|
options.complete();
|
32
32
|
}
|
33
33
|
}
|
34
|
-
xhr.send(
|
34
|
+
xhr.send(new URLSearchParams(options.data).toString());
|
35
35
|
return xhr;
|
36
36
|
};
|
37
37
|
|
data/bench/codecs/marshal.rb
CHANGED
data/docker-compose.yml
CHANGED
@@ -36,7 +36,7 @@ services:
|
|
36
36
|
example:
|
37
37
|
build:
|
38
38
|
context: .
|
39
|
-
command: bash -c "cd examples/
|
39
|
+
command: bash -c "cd examples/chat && bundle install && bundle exec rackup --server puma --host 0.0.0.0"
|
40
40
|
environment:
|
41
41
|
BUNDLE_TO: /usr/local/bundle
|
42
42
|
REDISURL: redis://redis:6379
|
data/examples/bench/bench.lua
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
-- wrk returns lots of read errors, this is unavoidable cause
|
2
2
|
--
|
3
|
-
-- 1. There is no internal
|
3
|
+
-- 1. There is no internal implementation of chunked encoding in wrk (which would be ideal)
|
4
4
|
--
|
5
5
|
-- 2. MessageBus gem does not provide http keepalive (by design), and can not provide content length
|
6
|
-
-- if MessageBus provided keepalive it would have to be able to
|
6
|
+
-- if MessageBus provided keepalive it would have to be able to re-dispatch requests to rack, something
|
7
7
|
-- that is not supported by the underlying rack hijack protocol, once a req is hijacked it can not be
|
8
8
|
-- returned
|
9
9
|
--
|
@@ -62,7 +62,7 @@ module MessageBus
|
|
62
62
|
attr_accessor :max_backlog_size
|
63
63
|
# @return [Integer] the largest permitted size (number of messages) for the global backlog; beyond this capacity, old messages will be dropped.
|
64
64
|
attr_accessor :max_global_backlog_size
|
65
|
-
# @return [Integer] the longest amount of time a message may live in a backlog before
|
65
|
+
# @return [Integer] the longest amount of time a message may live in a backlog before being removed, in seconds.
|
66
66
|
attr_accessor :max_backlog_age
|
67
67
|
# Typically, backlogs are trimmed whenever we publish to them. This setting allows some tolerance in order to improve performance.
|
68
68
|
# @return [Integer] the interval of publications between which the backlog will not be cleared.
|
@@ -74,7 +74,7 @@ module MessageBus
|
|
74
74
|
# @param [Integer] max_backlog_size the largest permitted size (number of messages) for per-channel backlogs; beyond this capacity, old messages will be dropped.
|
75
75
|
def initialize(config = {}, max_backlog_size = 1000); end
|
76
76
|
|
77
|
-
# Performs routines specific to the backend that are necessary after a process fork, typically
|
77
|
+
# Performs routines specific to the backend that are necessary after a process fork, typically triggered by a forking webserver. Typically this re-opens sockets to the backend.
|
78
78
|
def after_fork
|
79
79
|
raise ConcreteClassMustImplementError
|
80
80
|
end
|
@@ -84,6 +84,11 @@ module MessageBus
|
|
84
84
|
raise ConcreteClassMustImplementError
|
85
85
|
end
|
86
86
|
|
87
|
+
# Closes all open connections to the storage.
|
88
|
+
def destroy
|
89
|
+
raise ConcreteClassMustImplementError
|
90
|
+
end
|
91
|
+
|
87
92
|
# Deletes all backlogs and their data. Does not delete non-backlog data that message_bus may persist, depending on the concrete backend implementation. Use with extreme caution.
|
88
93
|
# @abstract
|
89
94
|
def expire_all_backlogs!
|
@@ -96,7 +101,7 @@ module MessageBus
|
|
96
101
|
# @param [JSON] data some data to publish to the channel. Must be an object that can be encoded as JSON
|
97
102
|
# @param [Hash] opts
|
98
103
|
# @option opts [Boolean] :queue_in_memory (true) whether or not to hold the message in an in-memory buffer if publication fails, to be re-tried later
|
99
|
-
# @option opts [Integer] :max_backlog_age (`self.max_backlog_age`) the longest amount of time a message may live in a backlog before
|
104
|
+
# @option opts [Integer] :max_backlog_age (`self.max_backlog_age`) the longest amount of time a message may live in a backlog before being removed, in seconds
|
100
105
|
# @option opts [Integer] :max_backlog_size (`self.max_backlog_size`) the largest permitted size (number of messages) for the channel backlog; beyond this capacity, old messages will be dropped
|
101
106
|
#
|
102
107
|
# @return [Integer] the channel-specific ID the message was given
|
@@ -212,6 +212,12 @@ module MessageBus
|
|
212
212
|
client.reset!
|
213
213
|
end
|
214
214
|
|
215
|
+
# No-op; this backend doesn't maintain any storage connections.
|
216
|
+
# (see Base#destroy)
|
217
|
+
def destroy
|
218
|
+
nil
|
219
|
+
end
|
220
|
+
|
215
221
|
# (see Base#expire_all_backlogs!)
|
216
222
|
def expire_all_backlogs!
|
217
223
|
client.expire_all_backlogs!
|
@@ -44,6 +44,8 @@ module MessageBus
|
|
44
44
|
@listening_on = {}
|
45
45
|
@available = []
|
46
46
|
@allocated = {}
|
47
|
+
@subscribe_connection = nil
|
48
|
+
@subscribed = false
|
47
49
|
@mutex = Mutex.new
|
48
50
|
@pid = Process.pid
|
49
51
|
end
|
@@ -85,10 +87,12 @@ module MessageBus
|
|
85
87
|
hold { |conn| exec_prepared(conn, 'get_message', [channel, id]) { |r| r.getvalue(0, 0) } }
|
86
88
|
end
|
87
89
|
|
88
|
-
def
|
90
|
+
def after_fork
|
89
91
|
sync do
|
90
|
-
@
|
92
|
+
@pid = Process.pid
|
93
|
+
INHERITED_CONNECTIONS.concat(@available)
|
91
94
|
@available.clear
|
95
|
+
@listening_on.clear
|
92
96
|
end
|
93
97
|
end
|
94
98
|
|
@@ -100,6 +104,13 @@ module MessageBus
|
|
100
104
|
end
|
101
105
|
end
|
102
106
|
|
107
|
+
def destroy
|
108
|
+
sync do
|
109
|
+
@available.each(&:close)
|
110
|
+
@available.clear
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
103
114
|
# use with extreme care, will nuke all of the data
|
104
115
|
def expire_all_backlogs!
|
105
116
|
reset!
|
@@ -131,7 +142,7 @@ module MessageBus
|
|
131
142
|
listener = Listener.new
|
132
143
|
yield listener
|
133
144
|
|
134
|
-
conn = raw_pg_connection
|
145
|
+
conn = @subscribe_connection = raw_pg_connection
|
135
146
|
conn.exec "LISTEN #{channel}"
|
136
147
|
listener.do_sub.call
|
137
148
|
while listening_on?(channel, obj)
|
@@ -145,6 +156,9 @@ module MessageBus
|
|
145
156
|
|
146
157
|
conn.exec "UNLISTEN #{channel}"
|
147
158
|
nil
|
159
|
+
ensure
|
160
|
+
@subscribe_connection&.close
|
161
|
+
@subscribe_connection = nil
|
148
162
|
end
|
149
163
|
|
150
164
|
def unsubscribe
|
@@ -170,11 +184,7 @@ module MessageBus
|
|
170
184
|
def hold
|
171
185
|
current_pid = Process.pid
|
172
186
|
if current_pid != @pid
|
173
|
-
|
174
|
-
sync do
|
175
|
-
INHERITED_CONNECTIONS.concat(@available)
|
176
|
-
@available.clear
|
177
|
-
end
|
187
|
+
after_fork
|
178
188
|
end
|
179
189
|
|
180
190
|
if conn = sync { @allocated[Thread.current] }
|
@@ -249,12 +259,14 @@ module MessageBus
|
|
249
259
|
# after 7 days inactive backlogs will be removed
|
250
260
|
@max_backlog_age = 604800
|
251
261
|
@clear_every = config[:clear_every] || 1
|
262
|
+
@mutex = Mutex.new
|
263
|
+
@client = nil
|
252
264
|
end
|
253
265
|
|
254
|
-
# Reconnects to Postgres; used after a process fork, typically
|
266
|
+
# Reconnects to Postgres; used after a process fork, typically triggered by a forking webserver
|
255
267
|
# @see Base#after_fork
|
256
268
|
def after_fork
|
257
|
-
client.
|
269
|
+
client.after_fork
|
258
270
|
end
|
259
271
|
|
260
272
|
# (see Base#reset!)
|
@@ -262,6 +274,11 @@ module MessageBus
|
|
262
274
|
client.reset!
|
263
275
|
end
|
264
276
|
|
277
|
+
# (see Base#destroy)
|
278
|
+
def destroy
|
279
|
+
client.destroy
|
280
|
+
end
|
281
|
+
|
265
282
|
# (see Base#expire_all_backlogs!)
|
266
283
|
def expire_all_backlogs!
|
267
284
|
client.expire_all_backlogs!
|
@@ -277,7 +294,7 @@ module MessageBus
|
|
277
294
|
msg = MessageBus::Message.new backlog_id, backlog_id, channel, data
|
278
295
|
payload = msg.encode
|
279
296
|
c.publish postgresql_channel_name, payload
|
280
|
-
if backlog_id % clear_every == 0
|
297
|
+
if backlog_id && backlog_id % clear_every == 0
|
281
298
|
max_backlog_size = (opts && opts[:max_backlog_size]) || self.max_backlog_size
|
282
299
|
max_backlog_age = (opts && opts[:max_backlog_age]) || self.max_backlog_age
|
283
300
|
c.clear_global_backlog(backlog_id, @max_global_backlog_size)
|
@@ -397,11 +414,7 @@ module MessageBus
|
|
397
414
|
private
|
398
415
|
|
399
416
|
def client
|
400
|
-
@client ||=
|
401
|
-
end
|
402
|
-
|
403
|
-
def new_connection
|
404
|
-
Client.new(@config)
|
417
|
+
@client || @mutex.synchronize { @client ||= Client.new(@config) }
|
405
418
|
end
|
406
419
|
|
407
420
|
def postgresql_channel_name
|
@@ -58,14 +58,16 @@ module MessageBus
|
|
58
58
|
@in_memory_backlog = []
|
59
59
|
@lock = Mutex.new
|
60
60
|
@flush_backlog_thread = nil
|
61
|
+
@pub_redis = nil
|
62
|
+
@subscribed = false
|
61
63
|
# after 7 days inactive backlogs will be removed
|
62
64
|
@max_backlog_age = 604800
|
63
65
|
end
|
64
66
|
|
65
|
-
# Reconnects to Redis; used after a process fork, typically
|
67
|
+
# Reconnects to Redis; used after a process fork, typically triggered by a forking webserver
|
66
68
|
# @see Base#after_fork
|
67
69
|
def after_fork
|
68
|
-
pub_redis
|
70
|
+
@pub_redis&.disconnect!
|
69
71
|
end
|
70
72
|
|
71
73
|
# (see Base#reset!)
|
@@ -75,6 +77,11 @@ module MessageBus
|
|
75
77
|
end
|
76
78
|
end
|
77
79
|
|
80
|
+
# (see Base#destroy)
|
81
|
+
def destroy
|
82
|
+
@pub_redis&.disconnect!
|
83
|
+
end
|
84
|
+
|
78
85
|
# Deletes all backlogs and their data. Does not delete ID pointers, so new publications will get IDs that continue from the last publication before the expiry. Use with extreme caution.
|
79
86
|
# @see Base#expire_all_backlogs!
|
80
87
|
def expire_all_backlogs!
|
@@ -254,6 +261,7 @@ LUA
|
|
254
261
|
new_redis.publish(redis_channel_name, UNSUB_MESSAGE)
|
255
262
|
ensure
|
256
263
|
new_redis&.disconnect!
|
264
|
+
@subscribed = false
|
257
265
|
end
|
258
266
|
end
|
259
267
|
|
@@ -296,6 +304,7 @@ LUA
|
|
296
304
|
|
297
305
|
on.message do |_c, m|
|
298
306
|
if m == UNSUB_MESSAGE
|
307
|
+
@subscribed = false
|
299
308
|
global_redis.unsubscribe
|
300
309
|
return
|
301
310
|
end
|