message_bus 2.1.6 → 2.2.0.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of message_bus might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.rubocop.yml +13 -92
- data/.rubocop_todo.yml +659 -0
- data/.travis.yml +1 -1
- data/CHANGELOG +61 -0
- data/Dockerfile +18 -0
- data/Gemfile +3 -1
- data/Guardfile +0 -1
- data/README.md +188 -101
- data/Rakefile +12 -1
- data/assets/message-bus.js +1 -1
- data/docker-compose.yml +46 -0
- data/examples/bench/config.ru +8 -9
- data/examples/bench/unicorn.conf.rb +1 -1
- data/examples/chat/chat.rb +150 -153
- data/examples/minimal/config.ru +2 -3
- data/lib/message_bus.rb +224 -36
- data/lib/message_bus/backends.rb +7 -0
- data/lib/message_bus/backends/base.rb +184 -0
- data/lib/message_bus/backends/memory.rb +304 -226
- data/lib/message_bus/backends/postgres.rb +359 -318
- data/lib/message_bus/backends/redis.rb +380 -337
- data/lib/message_bus/client.rb +99 -41
- data/lib/message_bus/connection_manager.rb +29 -21
- data/lib/message_bus/diagnostics.rb +50 -41
- data/lib/message_bus/distributed_cache.rb +5 -7
- data/lib/message_bus/message.rb +2 -2
- data/lib/message_bus/rack/diagnostics.rb +65 -55
- data/lib/message_bus/rack/middleware.rb +64 -44
- data/lib/message_bus/rack/thin_ext.rb +13 -9
- data/lib/message_bus/rails/railtie.rb +2 -0
- data/lib/message_bus/timer_thread.rb +2 -2
- data/lib/message_bus/version.rb +2 -1
- data/message_bus.gemspec +3 -2
- data/spec/assets/support/jasmine_helper.rb +1 -1
- data/spec/lib/fake_async_middleware.rb +1 -6
- data/spec/lib/message_bus/assets/asset_encoding_spec.rb +3 -3
- data/spec/lib/message_bus/backend_spec.rb +409 -0
- data/spec/lib/message_bus/client_spec.rb +8 -11
- data/spec/lib/message_bus/connection_manager_spec.rb +8 -14
- data/spec/lib/message_bus/distributed_cache_spec.rb +0 -4
- data/spec/lib/message_bus/multi_process_spec.rb +6 -7
- data/spec/lib/message_bus/rack/middleware_spec.rb +47 -43
- data/spec/lib/message_bus/timer_thread_spec.rb +0 -2
- data/spec/lib/message_bus_spec.rb +59 -43
- data/spec/spec_helper.rb +16 -4
- metadata +12 -9
- data/spec/lib/message_bus/backends/postgres_spec.rb +0 -221
- data/spec/lib/message_bus/backends/redis_spec.rb +0 -271
@@ -1,389 +1,430 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require 'pg'
|
3
|
-
|
4
|
-
module MessageBus::Postgres; end
|
5
2
|
|
6
|
-
|
7
|
-
INHERITED_CONNECTIONS = []
|
3
|
+
require 'pg'
|
8
4
|
|
9
|
-
|
10
|
-
|
5
|
+
require "message_bus/backends/base"
|
6
|
+
|
7
|
+
module MessageBus
|
8
|
+
module Backends
|
9
|
+
# The Postgres backend stores published messages in a single Postgres table
|
10
|
+
# with only global IDs, and an index on channel name and ID for fast
|
11
|
+
# per-channel lookup. All queries are implemented as prepared statements
|
12
|
+
# to reduce the wire-chatter during use. In addition to storage in the
|
13
|
+
# table, messages are published using `pg_notify`; this is used for
|
14
|
+
# actively subscribed message_bus servers to consume published messages in
|
15
|
+
# real-time while connected and forward them to subscribers, while catch-up
|
16
|
+
# is performed from the backlog table.
|
17
|
+
#
|
18
|
+
# @note This backend diverges from the standard in Base in the following ways:
|
19
|
+
#
|
20
|
+
# * Does not support in-memory buffering of messages on publication
|
21
|
+
# * Does not expire backlogs until they are published to
|
22
|
+
#
|
23
|
+
# @see Base general information about message_bus backends
|
24
|
+
class Postgres < Base
|
25
|
+
class Client
|
26
|
+
INHERITED_CONNECTIONS = []
|
27
|
+
|
28
|
+
class Listener
|
29
|
+
attr_reader :do_sub, :do_unsub, :do_message
|
30
|
+
|
31
|
+
def subscribe(&block)
|
32
|
+
@do_sub = block
|
33
|
+
end
|
11
34
|
|
12
|
-
|
13
|
-
|
14
|
-
|
35
|
+
def unsubscribe(&block)
|
36
|
+
@do_unsub = block
|
37
|
+
end
|
15
38
|
|
16
|
-
|
17
|
-
|
18
|
-
|
39
|
+
def message(&block)
|
40
|
+
@do_message = block
|
41
|
+
end
|
42
|
+
end
|
19
43
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
44
|
+
def initialize(config)
|
45
|
+
@config = config
|
46
|
+
@listening_on = {}
|
47
|
+
@available = []
|
48
|
+
@allocated = {}
|
49
|
+
@mutex = Mutex.new
|
50
|
+
@pid = Process.pid
|
51
|
+
end
|
24
52
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
@available = []
|
29
|
-
@allocated = {}
|
30
|
-
@mutex = Mutex.new
|
31
|
-
@pid = Process.pid
|
32
|
-
end
|
53
|
+
def add(channel, value)
|
54
|
+
hold { |conn| exec_prepared(conn, 'insert_message', [channel, value]) { |r| r.getvalue(0, 0).to_i } }
|
55
|
+
end
|
33
56
|
|
34
|
-
|
35
|
-
|
36
|
-
|
57
|
+
def clear_global_backlog(backlog_id, num_to_keep)
|
58
|
+
if backlog_id > num_to_keep
|
59
|
+
hold { |conn| exec_prepared(conn, 'clear_global_backlog', [backlog_id - num_to_keep]) }
|
60
|
+
nil
|
61
|
+
end
|
62
|
+
end
|
37
63
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
end
|
43
|
-
end
|
64
|
+
def clear_channel_backlog(channel, backlog_id, num_to_keep)
|
65
|
+
hold { |conn| exec_prepared(conn, 'clear_channel_backlog', [channel, backlog_id, num_to_keep]) }
|
66
|
+
nil
|
67
|
+
end
|
44
68
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
69
|
+
def expire(max_backlog_age)
|
70
|
+
hold { |conn| exec_prepared(conn, 'expire', [max_backlog_age]) }
|
71
|
+
nil
|
72
|
+
end
|
49
73
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
74
|
+
def backlog(channel, backlog_id)
|
75
|
+
hold do |conn|
|
76
|
+
exec_prepared(conn, 'channel_backlog', [channel, backlog_id]) { |r| r.values.each { |a| a[0] = a[0].to_i } }
|
77
|
+
end || []
|
78
|
+
end
|
54
79
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
80
|
+
def global_backlog(backlog_id)
|
81
|
+
hold do |conn|
|
82
|
+
exec_prepared(conn, 'global_backlog', [backlog_id]) { |r| r.values.each { |a| a[0] = a[0].to_i } }
|
83
|
+
end || []
|
84
|
+
end
|
60
85
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
end || []
|
65
|
-
end
|
86
|
+
def get_value(channel, id)
|
87
|
+
hold { |conn| exec_prepared(conn, 'get_message', [channel, id]) { |r| r.getvalue(0, 0) } }
|
88
|
+
end
|
66
89
|
|
67
|
-
|
68
|
-
|
69
|
-
|
90
|
+
def reconnect
|
91
|
+
sync do
|
92
|
+
@listening_on.clear
|
93
|
+
@available.clear
|
94
|
+
end
|
95
|
+
end
|
70
96
|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
97
|
+
# Dangerous, drops the message_bus table containing the backlog if it exists.
|
98
|
+
def reset!
|
99
|
+
hold do |conn|
|
100
|
+
conn.exec 'DROP TABLE IF EXISTS message_bus'
|
101
|
+
create_table(conn)
|
102
|
+
end
|
103
|
+
end
|
77
104
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
create_table(conn)
|
83
|
-
end
|
84
|
-
end
|
105
|
+
# use with extreme care, will nuke all of the data
|
106
|
+
def expire_all_backlogs!
|
107
|
+
reset!
|
108
|
+
end
|
85
109
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
110
|
+
def max_id(channel = nil)
|
111
|
+
block = proc do |r|
|
112
|
+
if r.ntuples > 0
|
113
|
+
r.getvalue(0, 0).to_i
|
114
|
+
else
|
115
|
+
0
|
116
|
+
end
|
117
|
+
end
|
94
118
|
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
119
|
+
if channel
|
120
|
+
hold { |conn| exec_prepared(conn, 'max_channel_id', [channel], &block) }
|
121
|
+
else
|
122
|
+
hold { |conn| exec_prepared(conn, 'max_id', &block) }
|
123
|
+
end
|
124
|
+
end
|
101
125
|
|
102
|
-
|
103
|
-
|
104
|
-
|
126
|
+
def publish(channel, data)
|
127
|
+
hold { |conn| exec_prepared(conn, 'publish', [channel, data]) }
|
128
|
+
end
|
105
129
|
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
conn = raw_pg_connection
|
113
|
-
conn.exec "LISTEN #{channel}"
|
114
|
-
listener.do_sub.call
|
115
|
-
while listening_on?(channel, obj)
|
116
|
-
conn.wait_for_notify(10) do |_, _, payload|
|
117
|
-
break unless listening_on?(channel, obj)
|
118
|
-
listener.do_message.call(nil, payload)
|
119
|
-
end
|
120
|
-
end
|
121
|
-
listener.do_unsub.call
|
130
|
+
def subscribe(channel)
|
131
|
+
obj = Object.new
|
132
|
+
sync { @listening_on[channel] = obj }
|
133
|
+
listener = Listener.new
|
134
|
+
yield listener
|
122
135
|
|
123
|
-
|
124
|
-
|
125
|
-
|
136
|
+
conn = raw_pg_connection
|
137
|
+
conn.exec "LISTEN #{channel}"
|
138
|
+
listener.do_sub.call
|
139
|
+
while listening_on?(channel, obj)
|
140
|
+
conn.wait_for_notify(10) do |_, _, payload|
|
141
|
+
break unless listening_on?(channel, obj)
|
126
142
|
|
127
|
-
|
128
|
-
|
129
|
-
|
143
|
+
listener.do_message.call(nil, payload)
|
144
|
+
end
|
145
|
+
end
|
146
|
+
listener.do_unsub.call
|
130
147
|
|
131
|
-
|
148
|
+
conn.exec "UNLISTEN #{channel}"
|
149
|
+
nil
|
150
|
+
end
|
132
151
|
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
ensure
|
137
|
-
r.clear if r.respond_to?(:clear)
|
138
|
-
end
|
152
|
+
def unsubscribe
|
153
|
+
sync { @listening_on.clear }
|
154
|
+
end
|
139
155
|
|
140
|
-
|
141
|
-
conn.exec 'CREATE TABLE message_bus (id bigserial PRIMARY KEY, channel text NOT NULL, value text NOT NULL CHECK (octet_length(value) >= 2), added_at timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL)'
|
142
|
-
conn.exec 'CREATE INDEX table_channel_id_index ON message_bus (channel, id)'
|
143
|
-
conn.exec 'CREATE INDEX table_added_at_index ON message_bus (added_at)'
|
144
|
-
nil
|
145
|
-
end
|
156
|
+
private
|
146
157
|
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
@available.clear
|
154
|
-
end
|
155
|
-
end
|
158
|
+
def exec_prepared(conn, *a)
|
159
|
+
r = conn.exec_prepared(*a)
|
160
|
+
yield r if block_given?
|
161
|
+
ensure
|
162
|
+
r.clear if r.respond_to?(:clear)
|
163
|
+
end
|
156
164
|
|
157
|
-
|
158
|
-
|
159
|
-
|
165
|
+
def create_table(conn)
|
166
|
+
conn.exec 'CREATE TABLE message_bus (id bigserial PRIMARY KEY, channel text NOT NULL, value text NOT NULL CHECK (octet_length(value) >= 2), added_at timestamp DEFAULT CURRENT_TIMESTAMP NOT NULL)'
|
167
|
+
conn.exec 'CREATE INDEX table_channel_id_index ON message_bus (channel, id)'
|
168
|
+
conn.exec 'CREATE INDEX table_added_at_index ON message_bus (added_at)'
|
169
|
+
nil
|
170
|
+
end
|
160
171
|
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
sync { INHERITED_CONNECTIONS << conn }
|
171
|
-
elsif conn && !e
|
172
|
-
sync { @available << conn }
|
173
|
-
end
|
174
|
-
end
|
175
|
-
end
|
172
|
+
def hold
|
173
|
+
current_pid = Process.pid
|
174
|
+
if current_pid != @pid
|
175
|
+
@pid = current_pid
|
176
|
+
sync do
|
177
|
+
INHERITED_CONNECTIONS.concat(@available)
|
178
|
+
@available.clear
|
179
|
+
end
|
180
|
+
end
|
176
181
|
|
177
|
-
|
178
|
-
|
179
|
-
|
182
|
+
if conn = sync { @allocated[Thread.current] }
|
183
|
+
return yield(conn)
|
184
|
+
end
|
180
185
|
|
181
|
-
|
182
|
-
|
186
|
+
begin
|
187
|
+
conn = sync { @available.shift } || new_pg_connection
|
188
|
+
sync { @allocated[Thread.current] = conn }
|
189
|
+
yield conn
|
190
|
+
rescue PG::ConnectionBad, PG::UnableToSend => e
|
191
|
+
# don't add this connection back to the pool
|
192
|
+
ensure
|
193
|
+
sync { @allocated.delete(Thread.current) }
|
194
|
+
if Process.pid != current_pid
|
195
|
+
sync { INHERITED_CONNECTIONS << conn }
|
196
|
+
elsif conn && !e
|
197
|
+
sync { @available << conn }
|
198
|
+
end
|
199
|
+
end
|
200
|
+
end
|
183
201
|
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
create_table(conn)
|
188
|
-
end
|
202
|
+
def raw_pg_connection
|
203
|
+
PG::Connection.connect(@config[:backend_options] || {})
|
204
|
+
end
|
189
205
|
|
190
|
-
|
191
|
-
|
192
|
-
conn.exec 'PREPARE clear_channel_backlog AS DELETE FROM message_bus WHERE ((channel = $1) AND (id <= (SELECT id FROM message_bus WHERE ((channel = $1) AND (id <= $2)) ORDER BY id DESC LIMIT 1 OFFSET $3)))'
|
193
|
-
conn.exec 'PREPARE channel_backlog AS SELECT id, value FROM message_bus WHERE ((channel = $1) AND (id > $2)) ORDER BY id'
|
194
|
-
conn.exec 'PREPARE global_backlog AS SELECT id, channel, value FROM message_bus WHERE (id > $1) ORDER BY id'
|
195
|
-
conn.exec "PREPARE expire AS DELETE FROM message_bus WHERE added_at < CURRENT_TIMESTAMP - ($1::text || ' seconds')::interval"
|
196
|
-
conn.exec 'PREPARE get_message AS SELECT value FROM message_bus WHERE ((channel = $1) AND (id = $2))'
|
197
|
-
conn.exec 'PREPARE max_channel_id AS SELECT max(id) FROM message_bus WHERE (channel = $1)'
|
198
|
-
conn.exec 'PREPARE max_id AS SELECT max(id) FROM message_bus'
|
199
|
-
conn.exec 'PREPARE publish AS SELECT pg_notify($1, $2)'
|
200
|
-
|
201
|
-
conn
|
202
|
-
end
|
206
|
+
def new_pg_connection
|
207
|
+
conn = raw_pg_connection
|
203
208
|
|
204
|
-
|
205
|
-
|
206
|
-
|
209
|
+
begin
|
210
|
+
conn.exec("SELECT 'message_bus'::regclass")
|
211
|
+
rescue PG::UndefinedTable
|
212
|
+
create_table(conn)
|
213
|
+
end
|
207
214
|
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
215
|
+
conn.exec 'PREPARE insert_message AS INSERT INTO message_bus (channel, value) VALUES ($1, $2) RETURNING id'
|
216
|
+
conn.exec 'PREPARE clear_global_backlog AS DELETE FROM message_bus WHERE (id <= $1)'
|
217
|
+
conn.exec 'PREPARE clear_channel_backlog AS DELETE FROM message_bus WHERE ((channel = $1) AND (id <= (SELECT id FROM message_bus WHERE ((channel = $1) AND (id <= $2)) ORDER BY id DESC LIMIT 1 OFFSET $3)))'
|
218
|
+
conn.exec 'PREPARE channel_backlog AS SELECT id, value FROM message_bus WHERE ((channel = $1) AND (id > $2)) ORDER BY id'
|
219
|
+
conn.exec 'PREPARE global_backlog AS SELECT id, channel, value FROM message_bus WHERE (id > $1) ORDER BY id'
|
220
|
+
conn.exec "PREPARE expire AS DELETE FROM message_bus WHERE added_at < CURRENT_TIMESTAMP - ($1::text || ' seconds')::interval"
|
221
|
+
conn.exec 'PREPARE get_message AS SELECT value FROM message_bus WHERE ((channel = $1) AND (id = $2))'
|
222
|
+
conn.exec 'PREPARE max_channel_id AS SELECT max(id) FROM message_bus WHERE (channel = $1)'
|
223
|
+
conn.exec 'PREPARE max_id AS SELECT max(id) FROM message_bus'
|
224
|
+
conn.exec 'PREPARE publish AS SELECT pg_notify($1, $2)'
|
225
|
+
|
226
|
+
conn
|
227
|
+
end
|
212
228
|
|
213
|
-
|
214
|
-
|
215
|
-
|
229
|
+
def listening_on?(channel, obj)
|
230
|
+
sync { @listening_on[channel] } == obj
|
231
|
+
end
|
216
232
|
|
217
|
-
|
233
|
+
def sync
|
234
|
+
@mutex.synchronize { yield }
|
235
|
+
end
|
236
|
+
end
|
218
237
|
|
219
|
-
|
220
|
-
|
221
|
-
|
238
|
+
def self.reset!(config)
|
239
|
+
MessageBus::Postgres::Client.new(config).reset!
|
240
|
+
end
|
222
241
|
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
242
|
+
# @param [Hash] config
|
243
|
+
# @option config [Logger] :logger a logger to which logs will be output
|
244
|
+
# @option config [Integer] :clear_every the interval of publications between which the backlog will not be cleared
|
245
|
+
# @option config [Hash] :backend_options see PG::Connection.connect for details of which options may be provided
|
246
|
+
# @param [Integer] max_backlog_size the largest permitted size (number of messages) for per-channel backlogs; beyond this capacity, old messages will be dropped.
|
247
|
+
def initialize(config = {}, max_backlog_size = 1000)
|
248
|
+
@config = config
|
249
|
+
@max_backlog_size = max_backlog_size
|
250
|
+
@max_global_backlog_size = 2000
|
251
|
+
# after 7 days inactive backlogs will be removed
|
252
|
+
@max_backlog_age = 604800
|
253
|
+
@clear_every = config[:clear_every] || 1
|
254
|
+
end
|
232
255
|
|
233
|
-
|
234
|
-
|
235
|
-
|
256
|
+
# Reconnects to Postgres; used after a process fork, typically triggerd by a forking webserver
|
257
|
+
# @see Base#after_fork
|
258
|
+
def after_fork
|
259
|
+
client.reconnect
|
260
|
+
end
|
236
261
|
|
237
|
-
|
238
|
-
|
239
|
-
|
262
|
+
# (see Base#reset!)
|
263
|
+
def reset!
|
264
|
+
client.reset!
|
265
|
+
end
|
240
266
|
|
241
|
-
|
242
|
-
|
243
|
-
|
267
|
+
# (see Base#expire_all_backlogs!)
|
268
|
+
def expire_all_backlogs!
|
269
|
+
client.expire_all_backlogs!
|
270
|
+
end
|
244
271
|
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
272
|
+
# (see Base#publish)
|
273
|
+
# @todo :queue_in_memory NOT SUPPORTED
|
274
|
+
def publish(channel, data, opts = nil)
|
275
|
+
# TODO in memory queue?
|
276
|
+
|
277
|
+
c = client
|
278
|
+
backlog_id = c.add(channel, data)
|
279
|
+
msg = MessageBus::Message.new backlog_id, backlog_id, channel, data
|
280
|
+
payload = msg.encode
|
281
|
+
c.publish postgresql_channel_name, payload
|
282
|
+
if backlog_id % clear_every == 0
|
283
|
+
max_backlog_size = (opts && opts[:max_backlog_size]) || self.max_backlog_size
|
284
|
+
max_backlog_age = (opts && opts[:max_backlog_age]) || self.max_backlog_age
|
285
|
+
c.clear_global_backlog(backlog_id, @max_global_backlog_size)
|
286
|
+
c.expire(max_backlog_age)
|
287
|
+
c.clear_channel_backlog(channel, backlog_id, max_backlog_size)
|
288
|
+
end
|
249
289
|
|
250
|
-
|
251
|
-
|
252
|
-
end
|
290
|
+
backlog_id
|
291
|
+
end
|
253
292
|
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
293
|
+
# (see Base#last_id)
|
294
|
+
def last_id(channel)
|
295
|
+
client.max_id(channel)
|
296
|
+
end
|
258
297
|
|
259
|
-
|
298
|
+
# (see Base#last_id)
|
299
|
+
def backlog(channel, last_id = 0)
|
300
|
+
items = client.backlog channel, last_id.to_i
|
260
301
|
|
261
|
-
|
302
|
+
items.map! do |id, data|
|
303
|
+
MessageBus::Message.new id, id, channel, data
|
304
|
+
end
|
305
|
+
end
|
262
306
|
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
payload = msg.encode
|
267
|
-
client.publish postgresql_channel_name, payload
|
268
|
-
if backlog_id % clear_every == 0
|
269
|
-
client.clear_global_backlog(backlog_id, @max_global_backlog_size)
|
270
|
-
client.expire(@max_backlog_age)
|
271
|
-
client.clear_channel_backlog(channel, backlog_id, @max_backlog_size)
|
272
|
-
end
|
307
|
+
# (see Base#global_backlog)
|
308
|
+
def global_backlog(last_id = 0)
|
309
|
+
items = client.global_backlog last_id.to_i
|
273
310
|
|
274
|
-
|
275
|
-
|
311
|
+
items.map! do |id, channel, data|
|
312
|
+
MessageBus::Message.new id, id, channel, data
|
313
|
+
end
|
314
|
+
end
|
276
315
|
|
277
|
-
|
278
|
-
|
279
|
-
|
316
|
+
# (see Base#get_message)
|
317
|
+
def get_message(channel, message_id)
|
318
|
+
if data = client.get_value(channel, message_id)
|
319
|
+
MessageBus::Message.new message_id, message_id, channel, data
|
320
|
+
else
|
321
|
+
nil
|
322
|
+
end
|
323
|
+
end
|
280
324
|
|
281
|
-
|
282
|
-
|
325
|
+
# (see Base#subscribe)
|
326
|
+
def subscribe(channel, last_id = nil)
|
327
|
+
# trivial implementation for now,
|
328
|
+
# can cut down on connections if we only have one global subscriber
|
329
|
+
raise ArgumentError unless block_given?
|
283
330
|
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
331
|
+
global_subscribe(last_id) do |m|
|
332
|
+
yield m if m.channel == channel
|
333
|
+
end
|
334
|
+
end
|
288
335
|
|
289
|
-
|
290
|
-
|
336
|
+
# (see Base#global_unsubscribe)
|
337
|
+
def global_unsubscribe
|
338
|
+
client.publish(postgresql_channel_name, UNSUB_MESSAGE)
|
339
|
+
@subscribed = false
|
340
|
+
end
|
291
341
|
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
end
|
342
|
+
# (see Base#global_subscribe)
|
343
|
+
def global_subscribe(last_id = nil)
|
344
|
+
raise ArgumentError unless block_given?
|
296
345
|
|
297
|
-
|
298
|
-
if data = client.get_value(channel, message_id)
|
299
|
-
MessageBus::Message.new message_id, message_id, channel, data
|
300
|
-
else
|
301
|
-
nil
|
302
|
-
end
|
303
|
-
end
|
346
|
+
highest_id = last_id
|
304
347
|
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
raise ArgumentError unless block_given?
|
348
|
+
begin
|
349
|
+
client.subscribe(postgresql_channel_name) do |on|
|
350
|
+
h = {}
|
309
351
|
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
352
|
+
on.subscribe do
|
353
|
+
if highest_id
|
354
|
+
process_global_backlog(highest_id) do |m|
|
355
|
+
h[m.global_id] = true
|
356
|
+
yield m
|
357
|
+
end
|
358
|
+
end
|
359
|
+
h = nil if h.empty?
|
360
|
+
@subscribed = true
|
361
|
+
end
|
314
362
|
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
end
|
363
|
+
on.unsubscribe do
|
364
|
+
@subscribed = false
|
365
|
+
end
|
319
366
|
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
367
|
+
on.message do |_c, m|
|
368
|
+
if m == UNSUB_MESSAGE
|
369
|
+
@subscribed = false
|
370
|
+
return
|
371
|
+
end
|
372
|
+
m = MessageBus::Message.decode m
|
373
|
+
|
374
|
+
# we have 3 options
|
375
|
+
#
|
376
|
+
# 1. message came in the correct order GREAT, just deal with it
|
377
|
+
# 2. message came in the incorrect order COMPLICATED, wait a tiny bit and clear backlog
|
378
|
+
# 3. message came in the incorrect order and is lowest than current highest id, reset
|
379
|
+
|
380
|
+
if h
|
381
|
+
# If already yielded during the clear backlog when subscribing,
|
382
|
+
# don't yield a duplicate copy.
|
383
|
+
unless h.delete(m.global_id)
|
384
|
+
h = nil if h.empty?
|
385
|
+
yield m
|
386
|
+
end
|
387
|
+
else
|
388
|
+
yield m
|
389
|
+
end
|
390
|
+
end
|
391
|
+
end
|
392
|
+
rescue => error
|
393
|
+
@config[:logger].warn "#{error} subscribe failed, reconnecting in 1 second. Call stack\n#{error.backtrace.join("\n")}"
|
394
|
+
sleep 1
|
395
|
+
retry
|
396
|
+
end
|
397
|
+
end
|
324
398
|
|
325
|
-
|
326
|
-
end
|
399
|
+
private
|
327
400
|
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
end
|
401
|
+
def client
|
402
|
+
@client ||= new_connection
|
403
|
+
end
|
332
404
|
|
333
|
-
|
334
|
-
|
335
|
-
|
405
|
+
def new_connection
|
406
|
+
Client.new(@config)
|
407
|
+
end
|
336
408
|
|
337
|
-
|
338
|
-
|
339
|
-
|
409
|
+
def postgresql_channel_name
|
410
|
+
db = @config[:db] || 0
|
411
|
+
"_message_bus_#{db}"
|
412
|
+
end
|
340
413
|
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
h[m.global_id] = true
|
345
|
-
yield m
|
346
|
-
end
|
347
|
-
end
|
348
|
-
h = nil if h.empty?
|
349
|
-
@subscribed = true
|
414
|
+
def process_global_backlog(highest_id)
|
415
|
+
if highest_id > client.max_id
|
416
|
+
highest_id = 0
|
350
417
|
end
|
351
418
|
|
352
|
-
|
353
|
-
|
419
|
+
global_backlog(highest_id).each do |old|
|
420
|
+
yield old
|
421
|
+
highest_id = old.global_id
|
354
422
|
end
|
355
423
|
|
356
|
-
|
357
|
-
if m == UNSUB_MESSAGE
|
358
|
-
@subscribed = false
|
359
|
-
return
|
360
|
-
end
|
361
|
-
m = MessageBus::Message.decode m
|
362
|
-
|
363
|
-
# we have 3 options
|
364
|
-
#
|
365
|
-
# 1. message came in the correct order GREAT, just deal with it
|
366
|
-
# 2. message came in the incorrect order COMPLICATED, wait a tiny bit and clear backlog
|
367
|
-
# 3. message came in the incorrect order and is lowest than current highest id, reset
|
368
|
-
|
369
|
-
if h
|
370
|
-
# If already yielded during the clear backlog when subscribing,
|
371
|
-
# don't yield a duplicate copy.
|
372
|
-
unless h.delete(m.global_id)
|
373
|
-
h = nil if h.empty?
|
374
|
-
yield m
|
375
|
-
end
|
376
|
-
else
|
377
|
-
yield m
|
378
|
-
end
|
379
|
-
end
|
424
|
+
highest_id
|
380
425
|
end
|
381
|
-
|
382
|
-
|
383
|
-
sleep 1
|
384
|
-
retry
|
426
|
+
|
427
|
+
MessageBus::BACKENDS[:postgres] = self
|
385
428
|
end
|
386
429
|
end
|
387
|
-
|
388
|
-
MessageBus::BACKENDS[:postgres] = self
|
389
430
|
end
|