message_bus 3.3.8 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.eslintrc.js +1 -8
- data/.github/workflows/ci.yml +3 -1
- data/CHANGELOG +107 -46
- data/README.md +38 -55
- data/Rakefile +22 -22
- data/docker-compose.yml +1 -1
- data/lib/message_bus/backends/base.rb +9 -0
- data/lib/message_bus/backends/memory.rb +7 -0
- data/lib/message_bus/backends/postgres.rb +35 -9
- data/lib/message_bus/backends/redis.rb +18 -5
- data/lib/message_bus/client.rb +23 -22
- data/lib/message_bus/rack/middleware.rb +0 -6
- data/lib/message_bus/version.rb +1 -1
- data/lib/message_bus.rb +49 -72
- data/message_bus.gemspec +0 -2
- data/spec/lib/message_bus/backend_spec.rb +19 -5
- data/spec/lib/message_bus/multi_process_spec.rb +3 -2
- data/spec/lib/message_bus/rack/middleware_spec.rb +0 -48
- data/spec/lib/message_bus/timer_thread_spec.rb +1 -5
- data/spec/lib/message_bus_spec.rb +10 -2
- data/spec/performance/backlog.rb +80 -0
- data/spec/performance/publish.rb +36 -5
- data/spec/spec_helper.rb +1 -1
- metadata +4 -30
- data/assets/application.jsx +0 -121
- data/assets/babel.min.js +0 -25
- data/assets/react-dom.js +0 -19851
- data/assets/react.js +0 -3029
- data/examples/diagnostics/Gemfile +0 -6
- data/examples/diagnostics/config.ru +0 -22
- data/lib/message_bus/diagnostics.rb +0 -62
- data/lib/message_bus/rack/diagnostics.rb +0 -120
data/Rakefile
CHANGED
@@ -5,18 +5,16 @@ require 'bundler'
|
|
5
5
|
require 'bundler/gem_tasks'
|
6
6
|
require 'bundler/setup'
|
7
7
|
require 'rubocop/rake_task'
|
8
|
+
require 'yard'
|
8
9
|
|
9
|
-
|
10
|
+
Bundler.require(:default, :test)
|
10
11
|
|
11
|
-
|
12
|
+
RuboCop::RakeTask.new
|
12
13
|
YARD::Rake::YardocTask.new
|
13
14
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
end
|
18
|
-
|
19
|
-
Bundler.require(:default, :test)
|
15
|
+
BACKENDS = Dir["lib/message_bus/backends/*.rb"].map { |file| file.match(%r{backends/(?<backend>.*).rb})[:backend] } - ["base"]
|
16
|
+
SPEC_FILES = Dir['spec/**/*_spec.rb']
|
17
|
+
INTEGRATION_FILES = Dir['spec/integration/**/*_spec.rb']
|
20
18
|
|
21
19
|
module CustomBuild
|
22
20
|
def build_gem
|
@@ -31,6 +29,11 @@ module Bundler
|
|
31
29
|
end
|
32
30
|
end
|
33
31
|
|
32
|
+
desc "Generate documentation for Yard, and fail if there are any warnings"
|
33
|
+
task :test_doc do
|
34
|
+
sh "yard --fail-on-warning #{'--no-progress' if ENV['CI']}"
|
35
|
+
end
|
36
|
+
|
34
37
|
namespace :jasmine do
|
35
38
|
desc "Run Jasmine tests in headless mode"
|
36
39
|
task 'ci' do
|
@@ -40,18 +43,16 @@ namespace :jasmine do
|
|
40
43
|
end
|
41
44
|
end
|
42
45
|
|
43
|
-
backends = Dir["lib/message_bus/backends/*.rb"].map { |file| file.match(%r{backends/(?<backend>.*).rb})[:backend] } - ["base"]
|
44
|
-
|
45
46
|
namespace :spec do
|
46
|
-
|
47
|
-
integration_files = Dir['spec/integration/**/*_spec.rb']
|
48
|
-
|
49
|
-
backends.each do |backend|
|
47
|
+
BACKENDS.each do |backend|
|
50
48
|
desc "Run tests on the #{backend} backend"
|
51
49
|
task backend do
|
52
50
|
begin
|
53
51
|
ENV['MESSAGE_BUS_BACKEND'] = backend
|
54
|
-
|
52
|
+
Rake::TestTask.new(backend) do |t|
|
53
|
+
t.test_files = SPEC_FILES - INTEGRATION_FILES
|
54
|
+
end
|
55
|
+
Rake::Task[backend].invoke
|
55
56
|
ensure
|
56
57
|
ENV.delete('MESSAGE_BUS_BACKEND')
|
57
58
|
end
|
@@ -74,7 +75,10 @@ namespace :spec do
|
|
74
75
|
ENV['MESSAGE_BUS_BACKEND'] = 'memory'
|
75
76
|
pid = spawn("bundle exec puma -p 9292 spec/fixtures/test/config.ru")
|
76
77
|
sleep 1 while port_available?(9292)
|
77
|
-
|
78
|
+
Rake::TestTask.new(:integration) do |t|
|
79
|
+
t.test_files = INTEGRATION_FILES
|
80
|
+
end
|
81
|
+
Rake::Task[:integration].invoke
|
78
82
|
ensure
|
79
83
|
ENV.delete('MESSAGE_BUS_BACKEND')
|
80
84
|
Process.kill('TERM', pid) if pid
|
@@ -83,12 +87,12 @@ namespace :spec do
|
|
83
87
|
end
|
84
88
|
|
85
89
|
desc "Run tests on all backends, plus client JS tests"
|
86
|
-
task spec:
|
90
|
+
task spec: BACKENDS.map { |backend| "spec:#{backend}" } + ["jasmine:ci", "spec:integration"]
|
87
91
|
|
88
92
|
desc "Run performance benchmarks on all backends"
|
89
93
|
task :performance do
|
90
94
|
begin
|
91
|
-
ENV['MESSAGE_BUS_BACKENDS'] =
|
95
|
+
ENV['MESSAGE_BUS_BACKENDS'] = BACKENDS.join(",")
|
92
96
|
sh "#{FileUtils::RUBY} -e \"ARGV.each{|f| load f}\" #{Dir['spec/performance/*.rb'].to_a.join(' ')}"
|
93
97
|
ensure
|
94
98
|
ENV.delete('MESSAGE_BUS_BACKENDS')
|
@@ -97,7 +101,3 @@ end
|
|
97
101
|
|
98
102
|
desc "Run all tests, link checks and confirms documentation compiles without error"
|
99
103
|
task default: [:spec, :rubocop, :test_doc]
|
100
|
-
|
101
|
-
Rake::Task['release'].enhance do
|
102
|
-
sh "yarn publish"
|
103
|
-
end
|
data/docker-compose.yml
CHANGED
@@ -36,7 +36,7 @@ services:
|
|
36
36
|
example:
|
37
37
|
build:
|
38
38
|
context: .
|
39
|
-
command: bash -c "cd examples/
|
39
|
+
command: bash -c "cd examples/chat && bundle install && bundle exec rackup --server puma --host 0.0.0.0"
|
40
40
|
environment:
|
41
41
|
BUNDLE_TO: /usr/local/bundle
|
42
42
|
REDISURL: redis://redis:6379
|
@@ -118,6 +118,15 @@ module MessageBus
|
|
118
118
|
raise ConcreteClassMustImplementError
|
119
119
|
end
|
120
120
|
|
121
|
+
# Get the ID of the last message published on multiple channels
|
122
|
+
#
|
123
|
+
# @param [Array<String>] channels - array of channels to fetch
|
124
|
+
#
|
125
|
+
# @return [Array<Integer>] the channel-specific IDs of the last message published to each requested channel
|
126
|
+
def last_ids(*channels)
|
127
|
+
raise ConcreteClassMustImplementError
|
128
|
+
end
|
129
|
+
|
121
130
|
# Get messages from a channel backlog
|
122
131
|
#
|
123
132
|
# @param [String] channel the name of the channel in question
|
@@ -244,6 +244,13 @@ module MessageBus
|
|
244
244
|
client.max_id(channel)
|
245
245
|
end
|
246
246
|
|
247
|
+
# (see Base#last_ids)
|
248
|
+
def last_ids(*channels)
|
249
|
+
channels.map do |c|
|
250
|
+
last_id(c)
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
247
254
|
# (see Base#backlog)
|
248
255
|
def backlog(channel, last_id = 0)
|
249
256
|
items = client.backlog channel, last_id.to_i
|
@@ -45,6 +45,7 @@ module MessageBus
|
|
45
45
|
@available = []
|
46
46
|
@allocated = {}
|
47
47
|
@subscribe_connection = nil
|
48
|
+
@subscribed = false
|
48
49
|
@mutex = Mutex.new
|
49
50
|
@pid = Process.pid
|
50
51
|
end
|
@@ -131,6 +132,21 @@ module MessageBus
|
|
131
132
|
end
|
132
133
|
end
|
133
134
|
|
135
|
+
def max_ids(*channels)
|
136
|
+
block = proc do |pg_result|
|
137
|
+
ids = Array.new(channels.size, 0)
|
138
|
+
pg_result.ntuples.times do |i|
|
139
|
+
channel = pg_result.getvalue(i, 0)
|
140
|
+
max_id = pg_result.getvalue(i, 1)
|
141
|
+
channel_index = channels.index(channel)
|
142
|
+
ids[channel_index] = max_id.to_i
|
143
|
+
end
|
144
|
+
ids
|
145
|
+
end
|
146
|
+
|
147
|
+
hold { |conn| exec_prepared(conn, 'max_channel_ids', [PG::TextEncoder::Array.new.encode(channels)], &block) }
|
148
|
+
end
|
149
|
+
|
134
150
|
def publish(channel, data)
|
135
151
|
hold { |conn| exec_prepared(conn, 'publish', [channel, data]) }
|
136
152
|
end
|
@@ -174,9 +190,16 @@ module MessageBus
|
|
174
190
|
end
|
175
191
|
|
176
192
|
def create_table(conn)
|
177
|
-
|
178
|
-
|
179
|
-
|
193
|
+
sync do
|
194
|
+
begin
|
195
|
+
conn.exec("SELECT 'message_bus'::regclass")
|
196
|
+
rescue PG::UndefinedTable
|
197
|
+
conn.exec 'CREATE TABLE message_bus (id bigserial PRIMARY KEY, channel text NOT NULL, value text NOT NULL CHECK (octet_length(value) >= 2), added_at timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL)'
|
198
|
+
conn.exec 'CREATE INDEX table_channel_id_index ON message_bus (channel, id)'
|
199
|
+
conn.exec 'CREATE INDEX table_added_at_index ON message_bus (added_at)'
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
180
203
|
nil
|
181
204
|
end
|
182
205
|
|
@@ -213,11 +236,7 @@ module MessageBus
|
|
213
236
|
def new_pg_connection
|
214
237
|
conn = raw_pg_connection
|
215
238
|
|
216
|
-
|
217
|
-
conn.exec("SELECT 'message_bus'::regclass")
|
218
|
-
rescue PG::UndefinedTable
|
219
|
-
create_table(conn)
|
220
|
-
end
|
239
|
+
create_table(conn)
|
221
240
|
|
222
241
|
conn.exec 'PREPARE insert_message AS INSERT INTO message_bus (channel, value) VALUES ($1, $2) RETURNING id'
|
223
242
|
conn.exec 'PREPARE clear_global_backlog AS DELETE FROM message_bus WHERE (id <= $1)'
|
@@ -227,6 +246,7 @@ module MessageBus
|
|
227
246
|
conn.exec "PREPARE expire AS DELETE FROM message_bus WHERE added_at < CURRENT_TIMESTAMP - ($1::text || ' seconds')::interval"
|
228
247
|
conn.exec 'PREPARE get_message AS SELECT value FROM message_bus WHERE ((channel = $1) AND (id = $2))'
|
229
248
|
conn.exec 'PREPARE max_channel_id AS SELECT max(id) FROM message_bus WHERE (channel = $1)'
|
249
|
+
conn.exec 'PREPARE max_channel_ids AS SELECT channel, max(id) FROM message_bus WHERE (channel = ANY($1)) GROUP BY channel'
|
230
250
|
conn.exec 'PREPARE max_id AS SELECT max(id) FROM message_bus'
|
231
251
|
conn.exec 'PREPARE publish AS SELECT pg_notify($1, $2)'
|
232
252
|
|
@@ -259,6 +279,7 @@ module MessageBus
|
|
259
279
|
@max_backlog_age = 604800
|
260
280
|
@clear_every = config[:clear_every] || 1
|
261
281
|
@mutex = Mutex.new
|
282
|
+
@client = nil
|
262
283
|
end
|
263
284
|
|
264
285
|
# Reconnects to Postgres; used after a process fork, typically triggered by a forking webserver
|
@@ -308,7 +329,12 @@ module MessageBus
|
|
308
329
|
client.max_id(channel)
|
309
330
|
end
|
310
331
|
|
311
|
-
# (see Base#
|
332
|
+
# (see Base#last_ids)
|
333
|
+
def last_ids(*channels)
|
334
|
+
client.max_ids(*channels)
|
335
|
+
end
|
336
|
+
|
337
|
+
# (see Base#backlog)
|
312
338
|
def backlog(channel, last_id = 0)
|
313
339
|
items = client.backlog channel, last_id.to_i
|
314
340
|
|
@@ -30,8 +30,6 @@ module MessageBus
|
|
30
30
|
# messages are removed (when no publication happens during this
|
31
31
|
# time-frame).
|
32
32
|
#
|
33
|
-
# * `clear_every` is not a supported option for this backend.
|
34
|
-
#
|
35
33
|
# @see Base general information about message_bus backends
|
36
34
|
class Redis < Base
|
37
35
|
class BackLogOutOfOrder < StandardError
|
@@ -45,9 +43,11 @@ module MessageBus
|
|
45
43
|
# @param [Hash] redis_config in addition to the options listed, see https://github.com/redis/redis-rb for other available options
|
46
44
|
# @option redis_config [Logger] :logger a logger to which logs will be output
|
47
45
|
# @option redis_config [Boolean] :enable_redis_logger (false) whether or not to enable logging by the underlying Redis library
|
46
|
+
# @option redis_config [Integer] :clear_every (1) the interval of publications between which the backlog will not be cleared
|
48
47
|
# @param [Integer] max_backlog_size the largest permitted size (number of messages) for per-channel backlogs; beyond this capacity, old messages will be dropped.
|
49
48
|
def initialize(redis_config = {}, max_backlog_size = 1000)
|
50
49
|
@redis_config = redis_config.dup
|
50
|
+
@clear_every = redis_config.delete(:clear_every) || 1
|
51
51
|
@logger = @redis_config[:logger]
|
52
52
|
unless @redis_config[:enable_redis_logger]
|
53
53
|
@redis_config[:logger] = nil
|
@@ -58,6 +58,8 @@ module MessageBus
|
|
58
58
|
@in_memory_backlog = []
|
59
59
|
@lock = Mutex.new
|
60
60
|
@flush_backlog_thread = nil
|
61
|
+
@pub_redis = nil
|
62
|
+
@subscribed = false
|
61
63
|
# after 7 days inactive backlogs will be removed
|
62
64
|
@max_backlog_age = 604800
|
63
65
|
end
|
@@ -98,6 +100,7 @@ module MessageBus
|
|
98
100
|
local max_backlog_size = tonumber(ARGV[3])
|
99
101
|
local max_global_backlog_size = tonumber(ARGV[4])
|
100
102
|
local channel = ARGV[5]
|
103
|
+
local clear_every = ARGV[6]
|
101
104
|
|
102
105
|
local global_id_key = KEYS[1]
|
103
106
|
local backlog_id_key = KEYS[2]
|
@@ -118,11 +121,11 @@ module MessageBus
|
|
118
121
|
|
119
122
|
redis.call("EXPIRE", backlog_id_key, max_backlog_age)
|
120
123
|
|
121
|
-
if backlog_id > max_backlog_size then
|
124
|
+
if backlog_id > max_backlog_size and backlog_id % clear_every == 0 then
|
122
125
|
redis.call("ZREMRANGEBYSCORE", backlog_key, 1, backlog_id - max_backlog_size)
|
123
126
|
end
|
124
127
|
|
125
|
-
if global_id > max_global_backlog_size then
|
128
|
+
if global_id > max_global_backlog_size and global_id % clear_every == 0 then
|
126
129
|
redis.call("ZREMRANGEBYSCORE", global_backlog_key, 1, global_id - max_global_backlog_size)
|
127
130
|
end
|
128
131
|
|
@@ -153,7 +156,8 @@ LUA
|
|
153
156
|
max_backlog_age,
|
154
157
|
max_backlog_size,
|
155
158
|
max_global_backlog_size,
|
156
|
-
channel
|
159
|
+
channel,
|
160
|
+
clear_every
|
157
161
|
],
|
158
162
|
keys: [
|
159
163
|
global_id_key,
|
@@ -192,6 +196,13 @@ LUA
|
|
192
196
|
pub_redis.get(backlog_id_key).to_i
|
193
197
|
end
|
194
198
|
|
199
|
+
# (see Base#last_ids)
|
200
|
+
def last_ids(*channels)
|
201
|
+
return [] if channels.size == 0
|
202
|
+
backlog_id_keys = channels.map { |c| backlog_id_key(c) }
|
203
|
+
pub_redis.mget(*backlog_id_keys).map(&:to_i)
|
204
|
+
end
|
205
|
+
|
195
206
|
# (see Base#backlog)
|
196
207
|
def backlog(channel, last_id = 0)
|
197
208
|
redis = pub_redis
|
@@ -259,6 +270,7 @@ LUA
|
|
259
270
|
new_redis.publish(redis_channel_name, UNSUB_MESSAGE)
|
260
271
|
ensure
|
261
272
|
new_redis&.disconnect!
|
273
|
+
@subscribed = false
|
262
274
|
end
|
263
275
|
end
|
264
276
|
|
@@ -301,6 +313,7 @@ LUA
|
|
301
313
|
|
302
314
|
on.message do |_c, m|
|
303
315
|
if m == UNSUB_MESSAGE
|
316
|
+
@subscribed = false
|
304
317
|
global_redis.unsubscribe
|
305
318
|
return
|
306
319
|
end
|
data/lib/message_bus/client.rb
CHANGED
@@ -178,31 +178,32 @@ class MessageBus::Client
|
|
178
178
|
r = []
|
179
179
|
new_message_ids = nil
|
180
180
|
|
181
|
-
@subscriptions.
|
182
|
-
id = v.to_i
|
181
|
+
last_bus_ids = @bus.last_ids(*@subscriptions.keys, site_id: site_id)
|
183
182
|
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
183
|
+
@subscriptions.each do |k, v|
|
184
|
+
last_client_id = v.to_i
|
185
|
+
last_bus_id = last_bus_ids[k]
|
186
|
+
|
187
|
+
if last_client_id < -1 # Client requesting backlog relative to bus position
|
188
|
+
last_client_id = last_bus_id + last_client_id + 1
|
189
|
+
last_client_id = 0 if last_client_id < 0
|
190
|
+
elsif last_client_id == -1 # Client not requesting backlog
|
191
|
+
next
|
192
|
+
elsif last_client_id == last_bus_id # Client already up-to-date
|
193
|
+
next
|
194
|
+
elsif last_client_id > last_bus_id # Client ahead of the bus
|
195
|
+
@subscriptions[k] = -1
|
196
|
+
next
|
188
197
|
end
|
189
198
|
|
190
|
-
|
191
|
-
|
192
|
-
messages = @bus.backlog(k, id, site_id)
|
199
|
+
messages = @bus.backlog(k, last_client_id, site_id)
|
193
200
|
|
194
|
-
|
195
|
-
if
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
if allowed?(msg)
|
201
|
-
r << msg
|
202
|
-
else
|
203
|
-
new_message_ids ||= {}
|
204
|
-
new_message_ids[k] = msg.message_id
|
205
|
-
end
|
201
|
+
messages.each do |msg|
|
202
|
+
if allowed?(msg)
|
203
|
+
r << msg
|
204
|
+
else
|
205
|
+
new_message_ids ||= {}
|
206
|
+
new_message_ids[k] = msg.message_id
|
206
207
|
end
|
207
208
|
end
|
208
209
|
end
|
@@ -212,7 +213,7 @@ class MessageBus::Client
|
|
212
213
|
@subscriptions.each do |k, v|
|
213
214
|
if v.to_i == -1 || (new_message_ids && new_message_ids[k])
|
214
215
|
status_message ||= {}
|
215
|
-
@subscriptions[k] = status_message[k] =
|
216
|
+
@subscriptions[k] = status_message[k] = last_bus_ids[k]
|
216
217
|
end
|
217
218
|
end
|
218
219
|
|
@@ -41,7 +41,6 @@ class MessageBus::Rack::Middleware
|
|
41
41
|
@started_listener = false
|
42
42
|
@base_route = "#{@bus.base_route}message-bus/"
|
43
43
|
@base_route_length = @base_route.length
|
44
|
-
@diagnostics_route = "#{@base_route}_diagnostics"
|
45
44
|
@broadcast_route = "#{@base_route}broadcast"
|
46
45
|
start_listener unless @bus.off?
|
47
46
|
end
|
@@ -79,11 +78,6 @@ class MessageBus::Rack::Middleware
|
|
79
78
|
return [200, { "Content-Type" => "text/html" }, ["sent"]]
|
80
79
|
end
|
81
80
|
|
82
|
-
if env['PATH_INFO'].start_with? @diagnostics_route
|
83
|
-
diags = MessageBus::Rack::Diagnostics.new(@app, message_bus: @bus)
|
84
|
-
return diags.call(env)
|
85
|
-
end
|
86
|
-
|
87
81
|
client_id = env['PATH_INFO'][@base_route_length..-1].split("/")[0]
|
88
82
|
return [404, {}, ["not found"]] unless client_id
|
89
83
|
|
data/lib/message_bus/version.rb
CHANGED
data/lib/message_bus.rb
CHANGED
@@ -7,9 +7,7 @@ require_relative "message_bus/version"
|
|
7
7
|
require_relative "message_bus/message"
|
8
8
|
require_relative "message_bus/client"
|
9
9
|
require_relative "message_bus/connection_manager"
|
10
|
-
require_relative "message_bus/diagnostics"
|
11
10
|
require_relative "message_bus/rack/middleware"
|
12
|
-
require_relative "message_bus/rack/diagnostics"
|
13
11
|
require_relative "message_bus/timer_thread"
|
14
12
|
require_relative "message_bus/codec/base"
|
15
13
|
require_relative "message_bus/backends"
|
@@ -47,21 +45,6 @@ module MessageBus::Implementation
|
|
47
45
|
@subscriber_thread = nil
|
48
46
|
end
|
49
47
|
|
50
|
-
# @param [Boolean] val whether or not to cache static assets for the diagnostics pages
|
51
|
-
# @return [void]
|
52
|
-
def cache_assets=(val)
|
53
|
-
configure(cache_assets: val)
|
54
|
-
end
|
55
|
-
|
56
|
-
# @return [Boolean] whether or not to cache static assets for the diagnostics pages
|
57
|
-
def cache_assets
|
58
|
-
if defined? @config[:cache_assets]
|
59
|
-
@config[:cache_assets]
|
60
|
-
else
|
61
|
-
true
|
62
|
-
end
|
63
|
-
end
|
64
|
-
|
65
48
|
# @param [Logger] logger a logger object to be used by the bus
|
66
49
|
# @return [void]
|
67
50
|
def logger=(logger)
|
@@ -297,15 +280,20 @@ module MessageBus::Implementation
|
|
297
280
|
@config[:transport_codec] ||= MessageBus::Codec::Json.new
|
298
281
|
end
|
299
282
|
|
300
|
-
# @param [MessageBus::Backend::Base]
|
283
|
+
# @param [MessageBus::Backend::Base] backend_instance A configured backend
|
301
284
|
# @return [void]
|
285
|
+
def backend_instance=(backend_instance)
|
286
|
+
configure(backend_instance: backend_instance)
|
287
|
+
end
|
288
|
+
|
302
289
|
def reliable_pub_sub=(pub_sub)
|
303
|
-
|
290
|
+
logger.warn "MessageBus.reliable_pub_sub= is deprecated, use MessageBus.backend_instance= instead."
|
291
|
+
self.backend_instance = pub_sub
|
304
292
|
end
|
305
293
|
|
306
294
|
# @return [MessageBus::Backend::Base] the configured backend. If not
|
307
295
|
# explicitly set, will be loaded based on the configuration provided.
|
308
|
-
def
|
296
|
+
def backend_instance
|
309
297
|
@mutex.synchronize do
|
310
298
|
return nil if @destroyed
|
311
299
|
|
@@ -313,7 +301,7 @@ module MessageBus::Implementation
|
|
313
301
|
# passed to backend.
|
314
302
|
logger
|
315
303
|
|
316
|
-
@config[:
|
304
|
+
@config[:backend_instance] ||= begin
|
317
305
|
@config[:backend_options] ||= {}
|
318
306
|
require "message_bus/backends/#{backend}"
|
319
307
|
MessageBus::BACKENDS[backend].new @config
|
@@ -321,17 +309,16 @@ module MessageBus::Implementation
|
|
321
309
|
end
|
322
310
|
end
|
323
311
|
|
312
|
+
def reliable_pub_sub
|
313
|
+
logger.warn "MessageBus.reliable_pub_sub is deprecated, use MessageBus.backend_instance instead."
|
314
|
+
backend_instance
|
315
|
+
end
|
316
|
+
|
324
317
|
# @return [Symbol] the name of the backend implementation configured
|
325
318
|
def backend
|
326
319
|
@config[:backend] || :redis
|
327
320
|
end
|
328
321
|
|
329
|
-
# Enables diagnostics tracking
|
330
|
-
# @return [void]
|
331
|
-
def enable_diagnostics
|
332
|
-
MessageBus::Diagnostics.enable(self)
|
333
|
-
end
|
334
|
-
|
335
322
|
# Publishes a message to a channel
|
336
323
|
#
|
337
324
|
# @param [String] channel the name of the channel to which the message should be published
|
@@ -398,7 +385,7 @@ module MessageBus::Implementation
|
|
398
385
|
end
|
399
386
|
|
400
387
|
encoded_channel_name = encode_channel_name(channel, site_id)
|
401
|
-
|
388
|
+
backend_instance.publish(encoded_channel_name, encoded_data, channel_opts)
|
402
389
|
end
|
403
390
|
|
404
391
|
# Subscribe to messages. Each message will be delivered by yielding to the
|
@@ -414,9 +401,9 @@ module MessageBus::Implementation
|
|
414
401
|
# @return [void]
|
415
402
|
def blocking_subscribe(channel = nil, &blk)
|
416
403
|
if channel
|
417
|
-
|
404
|
+
backend_instance.subscribe(encode_channel_name(channel), &blk)
|
418
405
|
else
|
419
|
-
|
406
|
+
backend_instance.global_subscribe(&blk)
|
420
407
|
end
|
421
408
|
end
|
422
409
|
|
@@ -495,9 +482,9 @@ module MessageBus::Implementation
|
|
495
482
|
def backlog(channel = nil, last_id = nil, site_id = nil)
|
496
483
|
old =
|
497
484
|
if channel
|
498
|
-
|
485
|
+
backend_instance.backlog(encode_channel_name(channel, site_id), last_id)
|
499
486
|
else
|
500
|
-
|
487
|
+
backend_instance.global_backlog(last_id)
|
501
488
|
end
|
502
489
|
|
503
490
|
old.each do |m|
|
@@ -513,7 +500,19 @@ module MessageBus::Implementation
|
|
513
500
|
#
|
514
501
|
# @return [Integer] the channel-specific ID of the last message published to the given channel
|
515
502
|
def last_id(channel, site_id = nil)
|
516
|
-
|
503
|
+
backend_instance.last_id(encode_channel_name(channel, site_id))
|
504
|
+
end
|
505
|
+
|
506
|
+
# Get the ID of the last message published on multiple channels
|
507
|
+
#
|
508
|
+
# @param [Array<String>] channels - array of channels to fetch
|
509
|
+
# @param [String] site_id - the ID of the site by which to filter
|
510
|
+
#
|
511
|
+
# @return [Hash] the channel-specific IDs of the last message published to each requested channel
|
512
|
+
def last_ids(*channels, site_id: nil)
|
513
|
+
encoded_channel_names = channels.map { |c| encode_channel_name(c, site_id) }
|
514
|
+
ids = backend_instance.last_ids(*encoded_channel_names)
|
515
|
+
channels.zip(ids).to_h
|
517
516
|
end
|
518
517
|
|
519
518
|
# Get the last message published on a channel
|
@@ -536,8 +535,8 @@ module MessageBus::Implementation
|
|
536
535
|
def destroy
|
537
536
|
return if @destroyed
|
538
537
|
|
539
|
-
|
540
|
-
|
538
|
+
backend_instance.global_unsubscribe
|
539
|
+
backend_instance.destroy
|
541
540
|
|
542
541
|
@mutex.synchronize do
|
543
542
|
return if @destroyed
|
@@ -555,7 +554,7 @@ module MessageBus::Implementation
|
|
555
554
|
# scheduled tasks.
|
556
555
|
# @return [void]
|
557
556
|
def after_fork
|
558
|
-
|
557
|
+
backend_instance.after_fork
|
559
558
|
ensure_subscriber_thread
|
560
559
|
# will ensure timer is running
|
561
560
|
timer.queue {}
|
@@ -564,12 +563,12 @@ module MessageBus::Implementation
|
|
564
563
|
# @return [Boolean] whether or not the server is actively listening for
|
565
564
|
# publications on the bus
|
566
565
|
def listening?
|
567
|
-
@subscriber_thread
|
566
|
+
@subscriber_thread&.alive?
|
568
567
|
end
|
569
568
|
|
570
569
|
# (see MessageBus::Backend::Base#reset!)
|
571
570
|
def reset!
|
572
|
-
|
571
|
+
backend_instance.reset! if backend_instance
|
573
572
|
end
|
574
573
|
|
575
574
|
# @return [MessageBus::TimerThread] the timer thread used for triggering
|
@@ -697,12 +696,6 @@ module MessageBus::Implementation
|
|
697
696
|
@subscriptions[site_id][channel] << blk
|
698
697
|
ensure_subscriber_thread
|
699
698
|
|
700
|
-
attempts = 100
|
701
|
-
while attempts > 0 && !reliable_pub_sub.subscribed
|
702
|
-
sleep 0.001
|
703
|
-
attempts -= 1
|
704
|
-
end
|
705
|
-
|
706
699
|
raise MessageBus::BusDestroyed if @destroyed
|
707
700
|
|
708
701
|
blk
|
@@ -720,10 +713,17 @@ module MessageBus::Implementation
|
|
720
713
|
|
721
714
|
def ensure_subscriber_thread
|
722
715
|
@mutex.synchronize do
|
723
|
-
return if
|
716
|
+
return if @destroyed
|
717
|
+
next if @subscriber_thread&.alive?
|
724
718
|
|
725
719
|
@subscriber_thread = new_subscriber_thread
|
726
720
|
end
|
721
|
+
|
722
|
+
attempts = 100
|
723
|
+
while attempts > 0 && !backend_instance.subscribed
|
724
|
+
sleep 0.001
|
725
|
+
attempts -= 1
|
726
|
+
end
|
727
727
|
end
|
728
728
|
|
729
729
|
MIN_KEEPALIVE = 20
|
@@ -744,34 +744,11 @@ module MessageBus::Implementation
|
|
744
744
|
if !@destroyed && thread.alive? && keepalive_interval > MIN_KEEPALIVE
|
745
745
|
|
746
746
|
publish("/__mb_keepalive__/", Process.pid, user_ids: [-1])
|
747
|
-
# going for x3 keepalives missed for a restart, need to ensure this only very rarely happens
|
748
|
-
# note: after_fork will sort out a bad @last_message date, but thread will be dead anyway
|
749
747
|
if (Time.now - (@last_message || Time.now)) > keepalive_interval * 3
|
750
|
-
logger.warn "Global messages on #{Process.pid} timed out,
|
751
|
-
# No other clean way to remove this thread, its listening on a socket
|
752
|
-
# no data is arriving
|
753
|
-
#
|
754
|
-
# In production we see this kind of situation ... sometimes ... when there is
|
755
|
-
# a VRRP failover, or weird networking condition
|
756
|
-
pid = Process.pid
|
757
|
-
|
758
|
-
# do the best we can to terminate self cleanly
|
759
|
-
fork do
|
760
|
-
Process.kill('TERM', pid)
|
761
|
-
sleep 10
|
762
|
-
begin
|
763
|
-
Process.kill('KILL', pid)
|
764
|
-
rescue Errno::ESRCH
|
765
|
-
logger.warn "#{Process.pid} successfully terminated by `TERM` signal."
|
766
|
-
end
|
767
|
-
end
|
768
|
-
|
769
|
-
sleep 10
|
770
|
-
Process.kill('KILL', pid)
|
771
|
-
|
772
|
-
else
|
773
|
-
timer.queue(keepalive_interval, &blk) if keepalive_interval > MIN_KEEPALIVE
|
748
|
+
logger.warn "Global messages on #{Process.pid} timed out, message bus is no longer functioning correctly"
|
774
749
|
end
|
750
|
+
|
751
|
+
timer.queue(keepalive_interval, &blk) if keepalive_interval > MIN_KEEPALIVE
|
775
752
|
end
|
776
753
|
end
|
777
754
|
|
@@ -783,7 +760,7 @@ module MessageBus::Implementation
|
|
783
760
|
def global_subscribe_thread
|
784
761
|
# pretend we just got a message
|
785
762
|
@last_message = Time.now
|
786
|
-
|
763
|
+
backend_instance.global_subscribe do |msg|
|
787
764
|
begin
|
788
765
|
@last_message = Time.now
|
789
766
|
decode_message!(msg)
|
data/message_bus.gemspec
CHANGED
@@ -11,8 +11,6 @@ Gem::Specification.new do |gem|
|
|
11
11
|
gem.license = "MIT"
|
12
12
|
gem.files = `git ls-files`.split($\) +
|
13
13
|
["vendor/assets/javascripts/message-bus.js", "vendor/assets/javascripts/message-bus-ajax.js"]
|
14
|
-
gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
|
15
|
-
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
16
14
|
gem.name = "message_bus"
|
17
15
|
gem.require_paths = ["lib"]
|
18
16
|
gem.version = MessageBus::VERSION
|