pgbus 0.26.1 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/README.md +28 -0
- data/lib/pgbus/client/notify_stream.rb +16 -0
- data/lib/pgbus/dedicated_connection.rb +20 -7
- data/lib/pgbus/engine.rb +8 -0
- data/lib/pgbus/open_transaction.rb +96 -0
- data/lib/pgbus/signals/middleware.rb +23 -0
- data/lib/pgbus/signals.rb +282 -0
- data/lib/pgbus/streams.rb +3 -39
- data/lib/pgbus/testing/minitest.rb +1 -0
- data/lib/pgbus/testing.rb +36 -0
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/streamer/failover_listener.rb +132 -27
- data/lib/pgbus/web/streamer/hub_client.rb +46 -15
- data/lib/pgbus/web/streamer/hub_protocol.rb +6 -0
- data/lib/pgbus/web/streamer/instance.rb +37 -2
- data/lib/pgbus/web/streamer/listener.rb +113 -24
- data/lib/pgbus/web/streamer/master_hub.rb +87 -2
- data/lib/pgbus/web/streamer/signal_delivery.rb +136 -0
- data/lib/pgbus/web/streamer.rb +10 -0
- data/lib/pgbus.rb +21 -0
- data/sig/pgbus/client.rbs +1 -0
- data/sig/pgbus.rbs +13 -0
- metadata +5 -1
|
@@ -43,6 +43,14 @@ module Pgbus
|
|
|
43
43
|
# PG_NOTIFY('pgmq.' || TG_TABLE_NAME || '.' || TG_OP, NULL)
|
|
44
44
|
# For a queue named `pgbus_stream_chat` the trigger table is
|
|
45
45
|
# `q_pgbus_stream_chat`, so the channel is `pgmq.q_pgbus_stream_chat.INSERT`.
|
|
46
|
+
#
|
|
47
|
+
# Process signals (issue #674) LISTEN on raw channels beside those
|
|
48
|
+
# (ensure_signal_listening, `<queue_prefix>_signal_<name>`). A NOTIFY on
|
|
49
|
+
# a channel outside the pgmq pattern goes to the `signal_sink:`, never to
|
|
50
|
+
# the dispatch queue and never dropped; a reconnect that re-LISTENed a
|
|
51
|
+
# signal channel tells the sink once (#gap), since NOTIFYs sent while the
|
|
52
|
+
# connection was down are lost. The sink runs on this thread: it only
|
|
53
|
+
# hands off (a queue push).
|
|
46
54
|
class Listener
|
|
47
55
|
# Kept Ractor-shareable (issue #527): the hand-off to the dispatcher
|
|
48
56
|
# is the seam a Ractor dispatcher would pass by reference. The Strings
|
|
@@ -64,6 +72,10 @@ module Pgbus
|
|
|
64
72
|
|
|
65
73
|
RECONNECT_BACKOFF_SECONDS = 0.5
|
|
66
74
|
|
|
75
|
+
# A process signal channel (issue #674): `<queue_prefix>_signal_<name>`,
|
|
76
|
+
# within PostgreSQL's 63-byte identifier limit.
|
|
77
|
+
SIGNAL_CHANNEL_PATTERN = /\A[a-z0-9_]{1,63}\z/
|
|
78
|
+
|
|
67
79
|
# Grace added to the ack budget when #stop joins the listener thread.
|
|
68
80
|
# See #stop_join_timeout.
|
|
69
81
|
STOP_JOIN_GRACE_SECONDS = 5
|
|
@@ -93,9 +105,12 @@ module Pgbus
|
|
|
93
105
|
# `ack_timeout_ms:` bounds how long ensure_listening waits for this
|
|
94
106
|
# thread's ack, and (plus a grace) how long #stop joins it
|
|
95
107
|
# (streams_listen_ack_timeout_ms; issue #664).
|
|
108
|
+
# `signal_sink:` receives process signals (issue #674): #signal(channel,
|
|
109
|
+
# payload) per NOTIFY on a signal channel and #gap after a reconnect
|
|
110
|
+
# that re-LISTENed one. nil: such NOTIFYs are ignored.
|
|
96
111
|
def initialize(pg_connection:, dispatch_queue:, health_check_ms:,
|
|
97
112
|
connection_factory:, ack_timeout_ms: DEFAULT_ACK_TIMEOUT_MS, dispatch_queue_limit: 0,
|
|
98
|
-
maintenance: nil, logger: Pgbus.logger, clock: nil)
|
|
113
|
+
maintenance: nil, signal_sink: nil, logger: Pgbus.logger, clock: nil)
|
|
99
114
|
raise ArgumentError, "connection_factory is required" unless connection_factory.respond_to?(:call)
|
|
100
115
|
|
|
101
116
|
@conn = pg_connection
|
|
@@ -106,6 +121,7 @@ module Pgbus
|
|
|
106
121
|
@connection_factory = connection_factory
|
|
107
122
|
@dispatch_queue_limit = dispatch_queue_limit
|
|
108
123
|
@maintenance = maintenance
|
|
124
|
+
@signal_sink = signal_sink
|
|
109
125
|
@clock = clock || -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) }
|
|
110
126
|
@maintenance_last = nil
|
|
111
127
|
@dropped_wakes = 0
|
|
@@ -225,19 +241,21 @@ module Pgbus
|
|
|
225
241
|
# default 1.25 s; issue #664), so a stuck listener can't hang the
|
|
226
242
|
# dispatcher forever.
|
|
227
243
|
def ensure_listening(queue_name)
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
return nil if @stopped
|
|
244
|
+
request_listen(:listen, queue_name)
|
|
245
|
+
end
|
|
231
246
|
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
return nil if @stopped
|
|
247
|
+
# Synchronously LISTEN on a process signal's raw channel (issue #674),
|
|
248
|
+
# with ensure_listening's ack contract: once acked, a NOTIFY on it
|
|
249
|
+
# reaches the signal sink.
|
|
250
|
+
def ensure_signal_listening(channel)
|
|
251
|
+
request_listen(:listen_channel, validate_signal_channel!(channel))
|
|
252
|
+
end
|
|
239
253
|
|
|
240
|
-
|
|
254
|
+
# Asynchronous UNLISTEN of a signal channel (the master hub, when its
|
|
255
|
+
# last worker subscriber leaves).
|
|
256
|
+
def remove_signal_listening(channel)
|
|
257
|
+
@commands << [:unlisten_channel, validate_signal_channel!(channel)]
|
|
258
|
+
wake
|
|
241
259
|
end
|
|
242
260
|
|
|
243
261
|
# Asynchronous: lazy GC of LISTENs whose subscriber count
|
|
@@ -250,6 +268,36 @@ module Pgbus
|
|
|
250
268
|
|
|
251
269
|
private
|
|
252
270
|
|
|
271
|
+
# The channel is interpolated into LISTEN "…": only the identifier
|
|
272
|
+
# characters Pgbus::Signals.channel_for produces may reach it (a hub
|
|
273
|
+
# worker's frame included).
|
|
274
|
+
def validate_signal_channel!(channel)
|
|
275
|
+
return channel if channel.is_a?(String) && SIGNAL_CHANNEL_PATTERN.match?(channel)
|
|
276
|
+
|
|
277
|
+
raise ArgumentError, "invalid signal channel #{channel.inspect}"
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
def request_listen(kind, name)
|
|
281
|
+
# A connect the dispatcher drains during shutdown (Instance#shutdown!
|
|
282
|
+
# stops the listener first) must not wait out the ack budget.
|
|
283
|
+
return nil if @stopped
|
|
284
|
+
|
|
285
|
+
ack = Queue.new
|
|
286
|
+
@commands << [kind, name, ack]
|
|
287
|
+
wake
|
|
288
|
+
# A stop that landed between the check above and the push may have
|
|
289
|
+
# let the thread release its queue and exit before the push: no ack
|
|
290
|
+
# would ever come. Pushed before that release, the command is acked.
|
|
291
|
+
return nil if @stopped
|
|
292
|
+
|
|
293
|
+
result = ack.pop(timeout: ack_timeout)
|
|
294
|
+
# Abandoned: the thread sees the closed queue when it gets to the
|
|
295
|
+
# command, and a signal LISTEN that lands this late reports a gap
|
|
296
|
+
# (the caller already went on as if it were listening).
|
|
297
|
+
ack.close if result.nil?
|
|
298
|
+
result
|
|
299
|
+
end
|
|
300
|
+
|
|
253
301
|
def run_loop
|
|
254
302
|
schedule_health_check
|
|
255
303
|
loop do
|
|
@@ -369,7 +417,7 @@ module Pgbus
|
|
|
369
417
|
# Acks held back by a LISTEN that raised: the reconnect that
|
|
370
418
|
# re-LISTENed their channel is done (or the thread is exiting).
|
|
371
419
|
def release_held_acks
|
|
372
|
-
@acks_after_reconnect.each { |ack| ack
|
|
420
|
+
@acks_after_reconnect.each { |ack| release_ack(ack) }
|
|
373
421
|
@acks_after_reconnect.clear
|
|
374
422
|
end
|
|
375
423
|
|
|
@@ -379,7 +427,7 @@ module Pgbus
|
|
|
379
427
|
def release_pending_acks
|
|
380
428
|
loop do
|
|
381
429
|
cmd = @commands.pop(true)
|
|
382
|
-
cmd[2]
|
|
430
|
+
release_ack(cmd[2]) if cmd[2] && %i[listen listen_channel].include?(cmd[0])
|
|
383
431
|
end
|
|
384
432
|
rescue ThreadError
|
|
385
433
|
nil
|
|
@@ -399,10 +447,11 @@ module Pgbus
|
|
|
399
447
|
run_health_check unless (@health_check_due_at - @clock.call).positive?
|
|
400
448
|
cmd = @commands.pop(true)
|
|
401
449
|
case cmd[0]
|
|
402
|
-
when :listen
|
|
450
|
+
when :listen, :listen_channel
|
|
403
451
|
ack = cmd[2]
|
|
404
452
|
begin
|
|
405
|
-
|
|
453
|
+
do_listen_channel(cmd[0] == :listen ? channel_for(cmd[1]) : cmd[1])
|
|
454
|
+
report_late_signal_listen if cmd[0] == :listen_channel && ack&.closed?
|
|
406
455
|
rescue StandardError
|
|
407
456
|
# The connection failed under the LISTEN. run_loop
|
|
408
457
|
# reconnects, and reconnect! re-LISTENs this channel with the
|
|
@@ -412,9 +461,10 @@ module Pgbus
|
|
|
412
461
|
ack = nil
|
|
413
462
|
raise
|
|
414
463
|
ensure
|
|
415
|
-
ack
|
|
464
|
+
release_ack(ack) if ack
|
|
416
465
|
end
|
|
417
|
-
when :unlisten then
|
|
466
|
+
when :unlisten then do_unlisten_channel(channel_for(cmd[1]))
|
|
467
|
+
when :unlisten_channel then do_unlisten_channel(cmd[1])
|
|
418
468
|
end
|
|
419
469
|
rescue ThreadError
|
|
420
470
|
# empty queue
|
|
@@ -447,8 +497,7 @@ module Pgbus
|
|
|
447
497
|
# no-second-wait rule rests on that, issue #630). The exception is a
|
|
448
498
|
# thread on its way out (stop, or an unexpected error): run_loop's
|
|
449
499
|
# ensure releases every waiting caller with nothing LISTENed.
|
|
450
|
-
def
|
|
451
|
-
channel = channel_for(queue_name)
|
|
500
|
+
def do_listen_channel(channel)
|
|
452
501
|
return if @listening_to.include?(channel)
|
|
453
502
|
|
|
454
503
|
@listening_to.add(channel)
|
|
@@ -457,8 +506,7 @@ module Pgbus
|
|
|
457
506
|
|
|
458
507
|
# The channel leaves @listening_to BEFORE the exec, so a reconnect
|
|
459
508
|
# after a failed UNLISTEN does not LISTEN it again.
|
|
460
|
-
def
|
|
461
|
-
channel = channel_for(queue_name)
|
|
509
|
+
def do_unlisten_channel(channel)
|
|
462
510
|
return unless @listening_to.include?(channel)
|
|
463
511
|
|
|
464
512
|
@listening_to.delete(channel)
|
|
@@ -467,7 +515,7 @@ module Pgbus
|
|
|
467
515
|
|
|
468
516
|
def handle_notify(channel, payload = nil)
|
|
469
517
|
queue_name = queue_name_from(channel)
|
|
470
|
-
return unless queue_name
|
|
518
|
+
return handle_signal(channel, payload) unless queue_name
|
|
471
519
|
|
|
472
520
|
# Backpressure applies ONLY to durable wakes (payload nil): they
|
|
473
521
|
# self-heal because the next durable wake for the stream re-reads
|
|
@@ -492,6 +540,42 @@ module Pgbus
|
|
|
492
540
|
@dispatch_queue << WakeMessage.new(queue_name: queue_name, payload: payload)
|
|
493
541
|
end
|
|
494
542
|
|
|
543
|
+
# A waiter that timed out closed its queue (request_listen).
|
|
544
|
+
def release_ack(ack)
|
|
545
|
+
ack.push(:done) unless ack.closed?
|
|
546
|
+
rescue ClosedQueueError
|
|
547
|
+
nil
|
|
548
|
+
end
|
|
549
|
+
|
|
550
|
+
# ensure_signal_listening timed out before this LISTEN ran, so its
|
|
551
|
+
# caller went on as if the channel were live: whatever was sent in
|
|
552
|
+
# between is a gap.
|
|
553
|
+
def report_late_signal_listen
|
|
554
|
+
@signal_sink&.gap
|
|
555
|
+
rescue StandardError => e
|
|
556
|
+
@logger.error { "[Pgbus::Streamer::Listener] signal sink gap raised: #{e.class}: #{e.message}" }
|
|
557
|
+
end
|
|
558
|
+
|
|
559
|
+
# A NOTIFY outside the pgmq pattern: only signal channels are LISTENed
|
|
560
|
+
# besides those. Never dropped, and a raising sink never stops the loop.
|
|
561
|
+
def handle_signal(channel, payload)
|
|
562
|
+
@signal_sink&.signal(channel, payload)
|
|
563
|
+
rescue StandardError => e
|
|
564
|
+
@logger.error { "[Pgbus::Streamer::Listener] signal sink raised: #{e.class}: #{e.message}" }
|
|
565
|
+
end
|
|
566
|
+
|
|
567
|
+
# After a reconnect: NOTIFYs on a signal channel sent while the
|
|
568
|
+
# connection was down are lost, so the sink hears about the gap once
|
|
569
|
+
# the new LISTENs are in place. Stream channels need no such report
|
|
570
|
+
# (a durable wake re-reads from the cursor).
|
|
571
|
+
def report_gap(channels)
|
|
572
|
+
return unless @signal_sink && channels.any? { |channel| !pgmq_channel?(channel) }
|
|
573
|
+
|
|
574
|
+
@signal_sink.gap
|
|
575
|
+
rescue StandardError => e
|
|
576
|
+
@logger.error { "[Pgbus::Streamer::Listener] signal sink gap raised: #{e.class}: #{e.message}" }
|
|
577
|
+
end
|
|
578
|
+
|
|
495
579
|
# The next keepalive is due one cycle after this one ENDS: scheduled
|
|
496
580
|
# from its start, a keepalive slower than a cycle would leave every
|
|
497
581
|
# pass overdue and the loop running SELECT 1 back to back without ever
|
|
@@ -561,6 +645,7 @@ module Pgbus
|
|
|
561
645
|
|
|
562
646
|
@conn = new_conn
|
|
563
647
|
@listening_to = Set.new(channels)
|
|
648
|
+
report_gap(channels)
|
|
564
649
|
return
|
|
565
650
|
end
|
|
566
651
|
ensure
|
|
@@ -579,8 +664,12 @@ module Pgbus
|
|
|
579
664
|
"#{CHANNEL_PREFIX}#{queue_name}#{CHANNEL_SUFFIX}"
|
|
580
665
|
end
|
|
581
666
|
|
|
667
|
+
def pgmq_channel?(channel)
|
|
668
|
+
channel.start_with?(CHANNEL_PREFIX) && channel.end_with?(CHANNEL_SUFFIX)
|
|
669
|
+
end
|
|
670
|
+
|
|
582
671
|
def queue_name_from(channel)
|
|
583
|
-
return nil unless
|
|
672
|
+
return nil unless pgmq_channel?(channel)
|
|
584
673
|
|
|
585
674
|
channel[CHANNEL_PREFIX.length..-(CHANNEL_SUFFIX.length + 1)]
|
|
586
675
|
end
|
|
@@ -41,7 +41,38 @@ module Pgbus
|
|
|
41
41
|
# is guarded by @table_mutex; each worker's outbox by its own mutex.
|
|
42
42
|
# All socket WRITES go through that worker's writer thread (frames must
|
|
43
43
|
# never interleave).
|
|
44
|
+
#
|
|
45
|
+
# Process signals (issue #674): a worker subscribes a signal channel
|
|
46
|
+
# with an "ssub" frame (register first, LISTEN second, "sack" last, as
|
|
47
|
+
# for a stream), refcounted per channel like the queues. The listener
|
|
48
|
+
# hands each signal NOTIFY and each reconnect gap to a SignalForwarder,
|
|
49
|
+
# which queues them for the fanout thread beside the wakes: "signal"
|
|
50
|
+
# frames go to the channel's subscribers and "gap" frames to every
|
|
51
|
+
# worker with a signal subscription, never dropped. The master runs no
|
|
52
|
+
# subscriber blocks itself; the workers subscribe what their own
|
|
53
|
+
# registries hold.
|
|
44
54
|
class MasterHub
|
|
55
|
+
# A signal NOTIFY the hub's listener received, for the fanout thread.
|
|
56
|
+
SignalMessage = Data.define(:channel, :payload)
|
|
57
|
+
# The hub listener reconnected with signal channels: tell the workers.
|
|
58
|
+
SIGNAL_GAP = :signal_gap
|
|
59
|
+
|
|
60
|
+
# The hub listener's signal sink: queue pushes only (it runs on the
|
|
61
|
+
# listener thread).
|
|
62
|
+
class SignalForwarder
|
|
63
|
+
def initialize(dispatch_queue)
|
|
64
|
+
@dispatch_queue = dispatch_queue
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def signal(channel, payload)
|
|
68
|
+
@dispatch_queue << SignalMessage.new(channel: channel, payload: payload)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def gap
|
|
72
|
+
@dispatch_queue << SIGNAL_GAP
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
45
76
|
DEFAULT_DURABLE_QUEUE_LIMIT = 256
|
|
46
77
|
DEFAULT_HARD_QUEUE_LIMIT = 1024
|
|
47
78
|
# Status is rebroadcast every REBROADCAST_TICKS status intervals even
|
|
@@ -76,6 +107,8 @@ module Pgbus
|
|
|
76
107
|
# already-unsubscribed channel (in-flight NOTIFYs after the last
|
|
77
108
|
# unsub, per-record stream names → unbounded, review on #384).
|
|
78
109
|
@queue_refs = {}
|
|
110
|
+
# channel → Set of worker ids, like @queue_refs (issue #674).
|
|
111
|
+
@signal_refs = {}
|
|
79
112
|
@stop_signal = Queue.new
|
|
80
113
|
@next_id = 0
|
|
81
114
|
@dropped_durable_wakes = 0
|
|
@@ -91,7 +124,9 @@ module Pgbus
|
|
|
91
124
|
@table_mutex.synchronize { @evicted_workers }
|
|
92
125
|
end
|
|
93
126
|
|
|
94
|
-
# The factory must return a STARTED listener wired to +dispatch_queue
|
|
127
|
+
# The factory must return a STARTED listener wired to +dispatch_queue+,
|
|
128
|
+
# with a SignalForwarder.new(dispatch_queue) as its signal_sink for
|
|
129
|
+
# process signals.
|
|
95
130
|
# If any step after the listener exists fails (bad socket path, chmod,
|
|
96
131
|
# thread spawn), the listener — and its dedicated LISTEN connection,
|
|
97
132
|
# the exact resource this hub conserves — is stopped before the error
|
|
@@ -172,6 +207,7 @@ module Pgbus
|
|
|
172
207
|
ack_timeout_ms: @config.streams_listen_ack_timeout_ms,
|
|
173
208
|
connection_factory: build_connection,
|
|
174
209
|
dispatch_queue_limit: @config.streams_dispatch_queue_limit,
|
|
210
|
+
signal_sink: SignalForwarder.new(dispatch_queue),
|
|
175
211
|
logger: @logger
|
|
176
212
|
).tap(&:start)
|
|
177
213
|
end
|
|
@@ -201,7 +237,7 @@ module Pgbus
|
|
|
201
237
|
|
|
202
238
|
def register_worker(sock)
|
|
203
239
|
entry = {
|
|
204
|
-
sock: sock, subs: Set.new, outbox: [], durable_count: 0, open: true,
|
|
240
|
+
sock: sock, subs: Set.new, signals: Set.new, outbox: [], durable_count: 0, open: true,
|
|
205
241
|
outbox_mutex: Mutex.new, outbox_cond: ConditionVariable.new
|
|
206
242
|
}
|
|
207
243
|
id = @table_mutex.synchronize do
|
|
@@ -240,6 +276,7 @@ module Pgbus
|
|
|
240
276
|
case frame["t"]
|
|
241
277
|
when "sub" then handle_sub(id, entry, frame["q"])
|
|
242
278
|
when "unsub" then handle_unsub(id, frame["q"])
|
|
279
|
+
when "ssub" then handle_signal_sub(id, entry, frame["c"])
|
|
243
280
|
else
|
|
244
281
|
@logger.warn { "[Pgbus::Streamer::MasterHub] worker #{id} sent unknown frame: #{frame["t"].inspect}" }
|
|
245
282
|
end
|
|
@@ -258,6 +295,16 @@ module Pgbus
|
|
|
258
295
|
enqueue_frame(id, entry, { "t" => "ack", "q" => queue }, droppable: false)
|
|
259
296
|
end
|
|
260
297
|
|
|
298
|
+
# The same ordering for a process signal channel.
|
|
299
|
+
def handle_signal_sub(id, entry, channel)
|
|
300
|
+
@table_mutex.synchronize do
|
|
301
|
+
entry[:signals].add(channel)
|
|
302
|
+
(@signal_refs[channel] ||= Set.new).add(id)
|
|
303
|
+
end
|
|
304
|
+
@listener.ensure_signal_listening(channel)
|
|
305
|
+
enqueue_frame(id, entry, { "t" => "sack", "c" => channel }, droppable: false)
|
|
306
|
+
end
|
|
307
|
+
|
|
261
308
|
def handle_unsub(id, queue)
|
|
262
309
|
release_queue_refs(id, [queue])
|
|
263
310
|
@table_mutex.synchronize { @workers[id]&.[](:subs)&.delete(queue) }
|
|
@@ -278,6 +325,9 @@ module Pgbus
|
|
|
278
325
|
end
|
|
279
326
|
|
|
280
327
|
def deliver(message)
|
|
328
|
+
return deliver_signal(message) if message.is_a?(SignalMessage)
|
|
329
|
+
return deliver_signal_gap if message == SIGNAL_GAP
|
|
330
|
+
|
|
281
331
|
frame = { "t" => "wake", "q" => message.queue_name, "p" => message.payload }
|
|
282
332
|
droppable = message.payload.nil?
|
|
283
333
|
targets = @table_mutex.synchronize do
|
|
@@ -287,6 +337,20 @@ module Pgbus
|
|
|
287
337
|
targets.each { |id, entry| enqueue_frame(id, entry, frame, droppable: droppable) }
|
|
288
338
|
end
|
|
289
339
|
|
|
340
|
+
def deliver_signal(message)
|
|
341
|
+
frame = { "t" => "signal", "c" => message.channel, "p" => message.payload }
|
|
342
|
+
targets = @table_mutex.synchronize do
|
|
343
|
+
refs = @signal_refs[message.channel]
|
|
344
|
+
refs ? refs.filter_map { |id| [id, @workers[id]] if @workers[id] } : []
|
|
345
|
+
end
|
|
346
|
+
targets.each { |id, entry| enqueue_frame(id, entry, frame, droppable: false) }
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
def deliver_signal_gap
|
|
350
|
+
targets = @table_mutex.synchronize { @workers.select { |_, entry| entry[:signals].any? }.to_a }
|
|
351
|
+
targets.each { |id, entry| enqueue_frame(id, entry, { "t" => "gap" }, droppable: false) }
|
|
352
|
+
end
|
|
353
|
+
|
|
290
354
|
# Non-blocking enqueue with the drop/evict policy. Never blocks the
|
|
291
355
|
# fanout thread on one slow worker (the head-of-line lesson from
|
|
292
356
|
# issue #315 item 3, applied cross-process).
|
|
@@ -397,6 +461,7 @@ module Pgbus
|
|
|
397
461
|
end
|
|
398
462
|
close_quietly(entry[:sock])
|
|
399
463
|
release_queue_refs(id, entry[:subs].to_a)
|
|
464
|
+
release_signal_refs(id, entry[:signals].to_a)
|
|
400
465
|
end
|
|
401
466
|
|
|
402
467
|
# Decrement refcounts; UNLISTEN queues that hit zero (async — no
|
|
@@ -414,6 +479,26 @@ module Pgbus
|
|
|
414
479
|
released.each { |q| @listener.remove_listening(q) }
|
|
415
480
|
end
|
|
416
481
|
|
|
482
|
+
# The UNLISTEN is queued under @table_mutex, with the last ref's
|
|
483
|
+
# removal: a new "ssub" for the channel registers under the same
|
|
484
|
+
# mutex and queues its LISTEN after it, so an acked subscriber can
|
|
485
|
+
# never be UNLISTENed by a cleanup that lost the race (the listener
|
|
486
|
+
# runs its commands in order). remove_signal_listening only queues.
|
|
487
|
+
def release_signal_refs(id, channels)
|
|
488
|
+
@table_mutex.synchronize do
|
|
489
|
+
channels.each do |channel|
|
|
490
|
+
refs = @signal_refs[channel]
|
|
491
|
+
next unless refs
|
|
492
|
+
|
|
493
|
+
refs.delete(id)
|
|
494
|
+
next unless refs.empty?
|
|
495
|
+
|
|
496
|
+
@signal_refs.delete(channel)
|
|
497
|
+
@listener.remove_signal_listening(channel)
|
|
498
|
+
end
|
|
499
|
+
end
|
|
500
|
+
end
|
|
501
|
+
|
|
417
502
|
def close_quietly(io)
|
|
418
503
|
io.close if io && !io.closed?
|
|
419
504
|
rescue IOError, Errno::EBADF
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Web
|
|
5
|
+
module Streamer
|
|
6
|
+
# The streamer's signal sink in a web process (issue #674): the Listener
|
|
7
|
+
# (or the HubClient, under :master scope) hands it each process-signal
|
|
8
|
+
# NOTIFY and each LISTEN gap; one thread of its own runs the subscribers
|
|
9
|
+
# (Pgbus::Signals.deliver / deliver_gap), so a slow or raising block
|
|
10
|
+
# never stalls the LISTEN loop or the stream dispatcher.
|
|
11
|
+
#
|
|
12
|
+
# The thread starts on the first signal for a registered name (or the
|
|
13
|
+
# first gap with an on_gap registered): a process without subscribers
|
|
14
|
+
# never has one. Messages still queued at #stop are dropped, as a NOTIFY
|
|
15
|
+
# to a stopped process is.
|
|
16
|
+
class SignalDelivery
|
|
17
|
+
STOP_JOIN_SECONDS = 2
|
|
18
|
+
# Signals waiting behind a slow or stuck subscriber. Past this, a
|
|
19
|
+
# signal is dropped and reported as a gap instead (on_gap is the
|
|
20
|
+
# recovery for a missed signal), so the backlog never grows unbounded.
|
|
21
|
+
BACKLOG_LIMIT = 1_000
|
|
22
|
+
|
|
23
|
+
def initialize(logger: Pgbus.logger, signals: Pgbus::Signals)
|
|
24
|
+
@logger = logger
|
|
25
|
+
@signals = signals
|
|
26
|
+
@queue = Queue.new
|
|
27
|
+
# Guards @thread, @stopped and @gap_pending: the first signal starts
|
|
28
|
+
# the thread, #stop ends it, from different threads.
|
|
29
|
+
@lock = Mutex.new
|
|
30
|
+
@thread = nil
|
|
31
|
+
@stopped = false
|
|
32
|
+
# A gap queued and not yet run: a second gap before it runs (a
|
|
33
|
+
# failover during the first LISTEN, say) adds nothing.
|
|
34
|
+
@gap_pending = false
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Sink API, called on the listener (or hub reader) thread: a queue
|
|
38
|
+
# push and nothing else.
|
|
39
|
+
def signal(channel, payload)
|
|
40
|
+
name = @signals.name_for(channel)
|
|
41
|
+
return unless name
|
|
42
|
+
return overflow(name) if @queue.size >= BACKLOG_LIMIT
|
|
43
|
+
|
|
44
|
+
enqueue([:signal, name, payload])
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def gap
|
|
48
|
+
return unless @signals.gap_handlers?
|
|
49
|
+
|
|
50
|
+
already_queued = @lock.synchronize do
|
|
51
|
+
pending = @gap_pending
|
|
52
|
+
@gap_pending = true
|
|
53
|
+
pending
|
|
54
|
+
end
|
|
55
|
+
return if already_queued
|
|
56
|
+
|
|
57
|
+
begin
|
|
58
|
+
enqueue([:gap])
|
|
59
|
+
rescue StandardError
|
|
60
|
+
# Never queued (the delivery thread could not start): a later gap
|
|
61
|
+
# must not be coalesced into this one.
|
|
62
|
+
@lock.synchronize { @gap_pending = false }
|
|
63
|
+
raise
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def threads
|
|
68
|
+
@lock.synchronize { [@thread].compact }
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def stop
|
|
72
|
+
# Under @lock with enqueue's push: nothing lands behind the sentinel,
|
|
73
|
+
# and what is still queued is dropped rather than run at shutdown.
|
|
74
|
+
thread = @lock.synchronize do
|
|
75
|
+
@stopped = true
|
|
76
|
+
@queue.clear
|
|
77
|
+
@queue << [:stop] if @thread
|
|
78
|
+
@thread
|
|
79
|
+
end
|
|
80
|
+
return unless thread
|
|
81
|
+
|
|
82
|
+
joined = thread.join(STOP_JOIN_SECONDS)
|
|
83
|
+
if joined
|
|
84
|
+
@lock.synchronize { @thread = nil }
|
|
85
|
+
else
|
|
86
|
+
@logger.warn do
|
|
87
|
+
"[Pgbus::Streamer::SignalDelivery] a signal subscriber is still running after #{STOP_JOIN_SECONDS}s"
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
nil
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
private
|
|
94
|
+
|
|
95
|
+
def enqueue(message)
|
|
96
|
+
@lock.synchronize do
|
|
97
|
+
return if @stopped
|
|
98
|
+
|
|
99
|
+
# The delivery thread, owned by this object: started once, on the
|
|
100
|
+
# first message, under @lock; #stop ends and joins it.
|
|
101
|
+
@thread ||= Thread.new { run_loop } # rubocop:disable ThreadSafety/NewThread
|
|
102
|
+
@queue << message
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# One warning per overflow episode: while a gap is already pending it
|
|
107
|
+
# covers every signal dropped behind it.
|
|
108
|
+
def overflow(name)
|
|
109
|
+
return gap if @lock.synchronize { @gap_pending }
|
|
110
|
+
|
|
111
|
+
@logger.warn do
|
|
112
|
+
"[Pgbus::Streamer::SignalDelivery] #{BACKLOG_LIMIT} signals waiting on a slow subscriber; " \
|
|
113
|
+
"dropped #{name.inspect} and reporting a gap"
|
|
114
|
+
end
|
|
115
|
+
gap
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def run_loop
|
|
119
|
+
loop do
|
|
120
|
+
kind, name, payload = @queue.pop
|
|
121
|
+
case kind
|
|
122
|
+
when :stop then break
|
|
123
|
+
when :signal then @signals.deliver(name, payload)
|
|
124
|
+
when :gap
|
|
125
|
+
@lock.synchronize { @gap_pending = false }
|
|
126
|
+
@signals.deliver_gap
|
|
127
|
+
end
|
|
128
|
+
rescue StandardError => e
|
|
129
|
+
# deliver never raises; a bug here must not end delivery.
|
|
130
|
+
@logger.error { "[Pgbus::Streamer::SignalDelivery] delivery failed: #{e.class}: #{e.message}" }
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
data/lib/pgbus/web/streamer.rb
CHANGED
|
@@ -37,6 +37,14 @@ module Pgbus
|
|
|
37
37
|
@current_mutex.synchronize { @current = instance }
|
|
38
38
|
end
|
|
39
39
|
|
|
40
|
+
# A signal registered after this process's streamer started (issue
|
|
41
|
+
# #674): LISTEN on its channel now. No streamer yet: nothing to do, a
|
|
42
|
+
# streamer started later subscribes every registered channel.
|
|
43
|
+
def listen_signal(channel)
|
|
44
|
+
instance = @current_mutex.synchronize { @current }
|
|
45
|
+
instance&.listen_signal(channel)
|
|
46
|
+
end
|
|
47
|
+
|
|
40
48
|
def stream_counter
|
|
41
49
|
@current_mutex.synchronize { @current&.stream_counter }
|
|
42
50
|
end
|
|
@@ -50,6 +58,8 @@ module Pgbus
|
|
|
50
58
|
instance = @current
|
|
51
59
|
@current = nil
|
|
52
60
|
end
|
|
61
|
+
# A later request must start a fresh streamer for process signals.
|
|
62
|
+
Pgbus::Signals.streamer_reset! if instance
|
|
53
63
|
instance&.shutdown!
|
|
54
64
|
end
|
|
55
65
|
end
|
data/lib/pgbus.rb
CHANGED
|
@@ -335,6 +335,27 @@ module Pgbus
|
|
|
335
335
|
EventBus::Publisher.publish_later(routing_key, payload, delay: delay, headers: headers)
|
|
336
336
|
end
|
|
337
337
|
|
|
338
|
+
# Run a block in every web process when any process sends +name+ (issue
|
|
339
|
+
# #674) — see Pgbus::Signals. Register in an initializer. `on_gap:` runs
|
|
340
|
+
# once after a LISTEN gap (reconnect, hub failover), when signals may have
|
|
341
|
+
# been missed.
|
|
342
|
+
#
|
|
343
|
+
# Pgbus.on_signal(:widgets_changed, on_gap: -> { WIDGETS.clear }) { |_payload| WIDGETS.clear }
|
|
344
|
+
def on_signal(name, on_gap: nil, &)
|
|
345
|
+
Signals.subscribe(name, on_gap: on_gap, &)
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
# Send +name+ to every process's on_signal blocks. Inside an open
|
|
349
|
+
# ActiveRecord transaction it is sent at COMMIT, never on rollback (a
|
|
350
|
+
# `transaction(joinable: false)` block sends at once). +payload+ is an
|
|
351
|
+
# optional String under NOTIFY's 7999-byte cap.
|
|
352
|
+
#
|
|
353
|
+
# Pgbus.signal(:widgets_changed)
|
|
354
|
+
# Pgbus.signal(:widgets_changed, "id=42")
|
|
355
|
+
def signal(name, payload = nil)
|
|
356
|
+
Signals.publish(name, payload)
|
|
357
|
+
end
|
|
358
|
+
|
|
338
359
|
# A fresh configuration (never nil, so readers need no lock) and no client.
|
|
339
360
|
def reset!
|
|
340
361
|
@state_lock.synchronize do
|
data/sig/pgbus/client.rbs
CHANGED
|
@@ -37,6 +37,7 @@ module Pgbus
|
|
|
37
37
|
def native_ready?: (String full_name) -> bool
|
|
38
38
|
def notify_enabled?: (untyped queue_name) -> bool
|
|
39
39
|
def notify_stream: (untyped stream_name, untyped payload) -> untyped
|
|
40
|
+
def notify_signal: (Symbol | String name, ?String? payload) -> untyped
|
|
40
41
|
def oldest_claimable_ages: (?untyped queue_name) -> untyped
|
|
41
42
|
def pgmq: () -> untyped
|
|
42
43
|
def pgmq_installed?: () -> bool
|
data/sig/pgbus.rbs
CHANGED
|
@@ -49,6 +49,15 @@ module Pgbus
|
|
|
49
49
|
end
|
|
50
50
|
end
|
|
51
51
|
|
|
52
|
+
# Partial: only the type Pgbus.on_signal returns.
|
|
53
|
+
module Signals
|
|
54
|
+
class Subscriber
|
|
55
|
+
attr_reader name: String
|
|
56
|
+
attr_reader callback: Proc
|
|
57
|
+
attr_reader on_gap: untyped
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
52
61
|
# Partial: only the types Pgbus.batch names. Batch's own methods are not
|
|
53
62
|
# covered by spec/pgbus/rbs_spec.rb yet.
|
|
54
63
|
class Batch
|
|
@@ -113,6 +122,10 @@ module Pgbus
|
|
|
113
122
|
# delay: seconds, as a Numeric or an ActiveSupport::Duration (1.hour).
|
|
114
123
|
def self.publish: (String routing_key, untyped payload, ?headers: Hash[String, untyped]?, ?delay: Numeric | _ToI) -> untyped
|
|
115
124
|
def self.publish_later: (String routing_key, untyped payload, delay: Numeric | _ToI, ?headers: Hash[String, untyped]?) -> untyped
|
|
125
|
+
# Process signals (issue #674): on_signal registers a block run in every web
|
|
126
|
+
# process when any process calls signal(name); on_gap runs after a LISTEN gap.
|
|
127
|
+
def self.on_signal: (Symbol | String name, ?on_gap: (^() -> void)?) { (String? payload) -> void } -> Signals::Subscriber
|
|
128
|
+
def self.signal: (Symbol | String name, ?String? payload) -> nil
|
|
116
129
|
def self.reset!: () -> void
|
|
117
130
|
def self.reset_client!: () -> void
|
|
118
131
|
def self.stopping: () -> bool
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pgbus
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.27.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Mikael Henriksson
|
|
@@ -463,6 +463,7 @@ files:
|
|
|
463
463
|
- lib/pgbus/metrics/prometheus_exporter.rb
|
|
464
464
|
- lib/pgbus/metrics/subscriber.rb
|
|
465
465
|
- lib/pgbus/metrics/workflow_metrics.rb
|
|
466
|
+
- lib/pgbus/open_transaction.rb
|
|
466
467
|
- lib/pgbus/outbox.rb
|
|
467
468
|
- lib/pgbus/outbox/poller.rb
|
|
468
469
|
- lib/pgbus/pgmq_schema.rb
|
|
@@ -507,6 +508,8 @@ files:
|
|
|
507
508
|
- lib/pgbus/ruby_jit.rb
|
|
508
509
|
- lib/pgbus/serializer.rb
|
|
509
510
|
- lib/pgbus/shareable.rb
|
|
511
|
+
- lib/pgbus/signals.rb
|
|
512
|
+
- lib/pgbus/signals/middleware.rb
|
|
510
513
|
- lib/pgbus/stat_buffer.rb
|
|
511
514
|
- lib/pgbus/streams.rb
|
|
512
515
|
- lib/pgbus/streams/broadcast_opts.rb
|
|
@@ -581,6 +584,7 @@ files:
|
|
|
581
584
|
- lib/pgbus/web/streamer/peer_close_watcher.rb
|
|
582
585
|
- lib/pgbus/web/streamer/presence_tracker.rb
|
|
583
586
|
- lib/pgbus/web/streamer/registry.rb
|
|
587
|
+
- lib/pgbus/web/streamer/signal_delivery.rb
|
|
584
588
|
- lib/pgbus/web/streamer/stream_counter.rb
|
|
585
589
|
- lib/pgbus/web/streamer/stream_event_dispatcher.rb
|
|
586
590
|
- lib/pgbus/web/time_format.rb
|