pgbus 0.26.1 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/README.md +28 -0
- data/lib/pgbus/client/notify_stream.rb +16 -0
- data/lib/pgbus/dedicated_connection.rb +20 -7
- data/lib/pgbus/engine.rb +8 -0
- data/lib/pgbus/open_transaction.rb +96 -0
- data/lib/pgbus/signals/middleware.rb +23 -0
- data/lib/pgbus/signals.rb +282 -0
- data/lib/pgbus/streams.rb +3 -39
- data/lib/pgbus/testing/minitest.rb +1 -0
- data/lib/pgbus/testing.rb +36 -0
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/streamer/failover_listener.rb +132 -27
- data/lib/pgbus/web/streamer/hub_client.rb +46 -15
- data/lib/pgbus/web/streamer/hub_protocol.rb +6 -0
- data/lib/pgbus/web/streamer/instance.rb +37 -2
- data/lib/pgbus/web/streamer/listener.rb +113 -24
- data/lib/pgbus/web/streamer/master_hub.rb +87 -2
- data/lib/pgbus/web/streamer/signal_delivery.rb +136 -0
- data/lib/pgbus/web/streamer.rb +10 -0
- data/lib/pgbus.rb +21 -0
- data/sig/pgbus/client.rbs +1 -0
- data/sig/pgbus.rbs +13 -0
- metadata +5 -1
data/lib/pgbus/testing.rb
CHANGED
|
@@ -14,6 +14,11 @@ module Pgbus
|
|
|
14
14
|
# :fake — capture published events in an in-memory store (default for tests)
|
|
15
15
|
# :inline — capture AND immediately dispatch to matching handlers
|
|
16
16
|
# :disabled — pass through to the real publisher (production behavior)
|
|
17
|
+
#
|
|
18
|
+
# Process signals (Pgbus.signal, issue #674) follow the same modes: :fake
|
|
19
|
+
# records them in `Pgbus::Testing.signals`, :inline records them and runs the
|
|
20
|
+
# process's on_signal blocks synchronously (at the COMMIT of an open
|
|
21
|
+
# transaction, as the NOTIFY would be sent), :disabled sends the NOTIFY.
|
|
17
22
|
module Testing
|
|
18
23
|
MODES = %i[fake inline disabled].freeze
|
|
19
24
|
MODE_KEY = :__pgbus_test_mode
|
|
@@ -68,8 +73,15 @@ module Pgbus
|
|
|
68
73
|
end
|
|
69
74
|
end
|
|
70
75
|
|
|
76
|
+
# One Pgbus.signal captured in fake or inline mode.
|
|
77
|
+
SignalRecord = Data.define(:name, :payload)
|
|
78
|
+
|
|
71
79
|
# Eagerly initialize the store so concurrent access never races on creation.
|
|
72
80
|
@store = EventStore.new
|
|
81
|
+
# Signals captured in fake/inline mode: a frozen Array replaced whole
|
|
82
|
+
# under its own lock.
|
|
83
|
+
@signals_lock = Mutex.new
|
|
84
|
+
@signals = [].freeze
|
|
73
85
|
|
|
74
86
|
# The process-wide mode (`fake!` with no block). A module ivar under a lock
|
|
75
87
|
# built at load, not `Thread.main[]`: that reads the main thread's CURRENT
|
|
@@ -120,6 +132,30 @@ module Pgbus
|
|
|
120
132
|
|
|
121
133
|
attr_reader :store
|
|
122
134
|
|
|
135
|
+
# The signals Pgbus.signal captured in fake or inline mode, oldest first.
|
|
136
|
+
def signals
|
|
137
|
+
@signals_lock.synchronize { @signals }
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def clear_signals!
|
|
141
|
+
@signals_lock.synchronize { @signals = [].freeze }
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# Called by Pgbus::Signals.publish instead of a NOTIFY when the mode is
|
|
145
|
+
# :fake or :inline. Recorded (and, inline, delivered) when the NOTIFY
|
|
146
|
+
# would be sent: at the COMMIT of an open joinable transaction, never on
|
|
147
|
+
# its rollback. Inline delivery runs on the caller's thread without the
|
|
148
|
+
# Rails executor, so the caller's CurrentAttributes survive it.
|
|
149
|
+
def record_signal(name, payload)
|
|
150
|
+
record = SignalRecord.new(name: name.to_s, payload: payload)
|
|
151
|
+
inline = inline?
|
|
152
|
+
commit = lambda do
|
|
153
|
+
@signals_lock.synchronize { @signals = [*@signals, record].freeze }
|
|
154
|
+
Pgbus::Signals.deliver(record.name, payload, executor: false) if inline
|
|
155
|
+
end
|
|
156
|
+
commit.call unless Pgbus::OpenTransaction.defer(&commit)
|
|
157
|
+
end
|
|
158
|
+
|
|
123
159
|
private
|
|
124
160
|
|
|
125
161
|
def default_mode
|
data/lib/pgbus/version.rb
CHANGED
|
@@ -21,12 +21,28 @@ module Pgbus
|
|
|
21
21
|
# relies on it. ensure_listening NEVER raises to the dispatcher: on a
|
|
22
22
|
# double failure (hub dead AND local build failing — e.g. DB down) it
|
|
23
23
|
# logs and returns nil, matching the Listener's own ack-timeout
|
|
24
|
-
# contract, which the dispatcher already tolerates
|
|
24
|
+
# contract, which the dispatcher already tolerates, and a background
|
|
25
|
+
# thread retries the local build every `rebuild_interval` seconds until
|
|
26
|
+
# it succeeds or #stop (issue #674: a process signal has no later
|
|
27
|
+
# ensure_listening call to retry it).
|
|
28
|
+
#
|
|
29
|
+
# Process signal channels (issue #674) are recorded and replayed the
|
|
30
|
+
# same way. NOTIFYs sent between the hub's death and the local LISTEN
|
|
31
|
+
# are lost, so once a failover has re-LISTENed a signal channel it calls
|
|
32
|
+
# `on_gap` (the worker's signal delivery runs every on_gap block).
|
|
25
33
|
class FailoverListener
|
|
26
|
-
|
|
34
|
+
REBUILD_INTERVAL_SECONDS = 5
|
|
35
|
+
REBUILD_JOIN_SECONDS = 2
|
|
36
|
+
|
|
37
|
+
def initialize(hub_client:, local_listener_factory:, on_gap: nil,
|
|
38
|
+
rebuild_interval: REBUILD_INTERVAL_SECONDS, logger: Pgbus.logger)
|
|
27
39
|
@hub_client = hub_client
|
|
28
40
|
@local_listener_factory = local_listener_factory
|
|
41
|
+
@on_gap = on_gap
|
|
42
|
+
@rebuild_interval = rebuild_interval
|
|
29
43
|
@logger = logger
|
|
44
|
+
# #stop wakes the rebuild thread's wait through this queue.
|
|
45
|
+
@stop_signal = Queue.new
|
|
30
46
|
# @state_mutex guards the cheap shared state (@subscriptions, @impl,
|
|
31
47
|
# @failed_over) and is only ever held for constant-time work — the
|
|
32
48
|
# dispatcher's ensure/remove path must never wait behind a failover
|
|
@@ -36,8 +52,12 @@ module Pgbus
|
|
|
36
52
|
@state_mutex = Mutex.new
|
|
37
53
|
@failover_mutex = Mutex.new
|
|
38
54
|
@subscriptions = Set.new
|
|
55
|
+
@signal_subscriptions = Set.new
|
|
39
56
|
@impl = hub_client
|
|
40
57
|
@failed_over = false
|
|
58
|
+
# Guarded by @state_mutex.
|
|
59
|
+
@stopped = false
|
|
60
|
+
@rebuild_thread = nil
|
|
41
61
|
end
|
|
42
62
|
|
|
43
63
|
# Interface parity with Listener for Instance#start: the hub client
|
|
@@ -60,6 +80,19 @@ module Pgbus
|
|
|
60
80
|
end
|
|
61
81
|
end
|
|
62
82
|
|
|
83
|
+
# Same failover contract as ensure_listening, for a signal channel.
|
|
84
|
+
def ensure_signal_listening(channel)
|
|
85
|
+
@state_mutex.synchronize { @signal_subscriptions.add(channel) }
|
|
86
|
+
current_impl.ensure_signal_listening(channel)
|
|
87
|
+
rescue HubClient::HubUnavailableError
|
|
88
|
+
fail_over!
|
|
89
|
+
begin
|
|
90
|
+
current_impl.ensure_signal_listening(channel)
|
|
91
|
+
rescue HubClient::HubUnavailableError
|
|
92
|
+
nil
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
63
96
|
def remove_listening(queue)
|
|
64
97
|
@state_mutex.synchronize { @subscriptions.delete(queue) }
|
|
65
98
|
current_impl.remove_listening(queue)
|
|
@@ -79,55 +112,127 @@ module Pgbus
|
|
|
79
112
|
# The blocking build + replay runs OUTSIDE @state_mutex so concurrent
|
|
80
113
|
# ensure/remove/stop calls never stall behind it.
|
|
81
114
|
def fail_over!
|
|
82
|
-
|
|
115
|
+
signals = nil
|
|
83
116
|
@failover_mutex.synchronize do
|
|
84
117
|
return if @state_mutex.synchronize { @failed_over }
|
|
85
118
|
|
|
86
|
-
|
|
87
|
-
@state_mutex.synchronize { @subscriptions.dup }.each { |q| local.ensure_listening(q) }
|
|
88
|
-
# Subscriptions recorded between the snapshot and this swap arrive
|
|
89
|
-
# via their own retried ensure_listening call on the new impl.
|
|
90
|
-
@state_mutex.synchronize do
|
|
91
|
-
@impl = local
|
|
92
|
-
@failed_over = true
|
|
93
|
-
end
|
|
94
|
-
# Ownership transferred to @impl — the rescue must not stop it.
|
|
95
|
-
local = nil
|
|
119
|
+
signals = swap_in_local!
|
|
96
120
|
end
|
|
121
|
+
report_gap if signals == true
|
|
97
122
|
rescue StandardError => e
|
|
98
|
-
# A listener the factory STARTED but that never swapped in (the
|
|
99
|
-
# replay raised) would otherwise leak its thread and LISTEN
|
|
100
|
-
# connection alongside the dead hub client.
|
|
101
|
-
begin
|
|
102
|
-
local&.stop
|
|
103
|
-
rescue StandardError
|
|
104
|
-
nil
|
|
105
|
-
end
|
|
106
123
|
# Hub dead AND the local listener can't be built (DB down, config
|
|
107
|
-
# broken). Mark failed-over so callers stop rebuilding
|
|
108
|
-
# on the dead client — every
|
|
109
|
-
#
|
|
110
|
-
#
|
|
124
|
+
# broken). Mark failed-over so callers stop rebuilding
|
|
125
|
+
# synchronously; @impl stays on the dead client — every
|
|
126
|
+
# ensure_listening resolves nil and the dispatcher rides its
|
|
127
|
+
# existing timeout tolerance — while the rebuild thread retries.
|
|
111
128
|
@state_mutex.synchronize { @failed_over = true }
|
|
112
129
|
@logger.error do
|
|
113
130
|
"[Pgbus::Streamer::FailoverListener] fallback listener failed to build " \
|
|
114
|
-
"(#{e.class}: #{e.message}) — streams
|
|
131
|
+
"(#{e.class}: #{e.message}) — streams and process signals degraded; " \
|
|
132
|
+
"retrying every #{@rebuild_interval}s"
|
|
115
133
|
end
|
|
134
|
+
start_rebuild
|
|
116
135
|
end
|
|
117
136
|
|
|
118
137
|
# Snapshot of this component's live thread(s). Instance#shutdown! captures
|
|
119
138
|
# it BEFORE calling #stop so a join that timed out is still observable
|
|
120
139
|
# after #stop has cleared the reference (issue #443).
|
|
121
140
|
def threads
|
|
122
|
-
|
|
141
|
+
rebuild = @state_mutex.synchronize { @rebuild_thread }
|
|
142
|
+
rebuild&.alive? ? [*current_impl.threads, rebuild] : current_impl.threads
|
|
123
143
|
end
|
|
124
144
|
|
|
125
145
|
def stop
|
|
146
|
+
rebuild = @state_mutex.synchronize do
|
|
147
|
+
@stopped = true
|
|
148
|
+
@rebuild_thread
|
|
149
|
+
end
|
|
150
|
+
@stop_signal << :stop
|
|
151
|
+
rebuild&.join(REBUILD_JOIN_SECONDS)
|
|
152
|
+
# Never waits on a connect in flight: a rebuild that finishes after
|
|
153
|
+
# this sees @stopped and stops its own listener instead of swapping.
|
|
126
154
|
current_impl.stop
|
|
127
155
|
end
|
|
128
156
|
|
|
129
157
|
private
|
|
130
158
|
|
|
159
|
+
# Builds a started local listener, replays every subscription on it
|
|
160
|
+
# and swaps it in. Returns true when it re-LISTENed a signal channel
|
|
161
|
+
# (the caller reports the gap), false when it re-LISTENed none, and
|
|
162
|
+
# :discarded when #stop came first or another build already swapped
|
|
163
|
+
# in (the listener is stopped, not swapped: only the hub client is
|
|
164
|
+
# ever replaced, so a live local listener is never orphaned). Raises when the build or the replay fails, after stopping
|
|
165
|
+
# a listener that never swapped in (it would leak its thread and
|
|
166
|
+
# LISTEN connection beside the dead client). Holds no lock across the
|
|
167
|
+
# build or the replay.
|
|
168
|
+
def swap_in_local!
|
|
169
|
+
local = @local_listener_factory.call
|
|
170
|
+
queues, signal_channels = @state_mutex.synchronize { [@subscriptions.dup, @signal_subscriptions.dup] }
|
|
171
|
+
queues.each { |q| local.ensure_listening(q) }
|
|
172
|
+
signal_channels.each { |channel| local.ensure_signal_listening(channel) }
|
|
173
|
+
swapped = @state_mutex.synchronize do
|
|
174
|
+
next false if @stopped || !@impl.equal?(@hub_client)
|
|
175
|
+
|
|
176
|
+
@impl = local
|
|
177
|
+
@failed_over = true
|
|
178
|
+
end
|
|
179
|
+
unless swapped
|
|
180
|
+
local.stop
|
|
181
|
+
return :discarded
|
|
182
|
+
end
|
|
183
|
+
# Subscriptions recorded during the replay went to the old impl;
|
|
184
|
+
# replay them on the new one.
|
|
185
|
+
late_queues, late_signals = @state_mutex.synchronize do
|
|
186
|
+
[@subscriptions - queues, @signal_subscriptions - signal_channels]
|
|
187
|
+
end
|
|
188
|
+
late_queues.each { |q| local.ensure_listening(q) }
|
|
189
|
+
late_signals.each { |channel| local.ensure_signal_listening(channel) }
|
|
190
|
+
signal_channels.any? || late_signals.any?
|
|
191
|
+
rescue StandardError
|
|
192
|
+
begin
|
|
193
|
+
local&.stop
|
|
194
|
+
rescue StandardError
|
|
195
|
+
nil
|
|
196
|
+
end
|
|
197
|
+
raise
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def start_rebuild
|
|
201
|
+
@state_mutex.synchronize do
|
|
202
|
+
return if @stopped || @rebuild_thread&.alive?
|
|
203
|
+
|
|
204
|
+
# The retry thread, owned by this object: it ends once a local
|
|
205
|
+
# listener swaps in or #stop signals it; #stop joins it.
|
|
206
|
+
@rebuild_thread = Thread.new { rebuild_loop } # rubocop:disable ThreadSafety/NewThread
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def rebuild_loop
|
|
211
|
+
loop do
|
|
212
|
+
break if @stop_signal.pop(timeout: @rebuild_interval)
|
|
213
|
+
|
|
214
|
+
# No @failover_mutex here: a connect in flight must not hold up
|
|
215
|
+
# #stop, nor fail_over! (which finds @failed_over set and returns).
|
|
216
|
+
result = swap_in_local!
|
|
217
|
+
break if result == :discarded
|
|
218
|
+
|
|
219
|
+
report_gap if result
|
|
220
|
+
@logger.info { "[Pgbus::Streamer::FailoverListener] fallback listener built; streams and process signals restored" }
|
|
221
|
+
break
|
|
222
|
+
rescue StandardError => e
|
|
223
|
+
@logger.warn do
|
|
224
|
+
"[Pgbus::Streamer::FailoverListener] fallback listener still failing to build (#{e.class}: #{e.message})"
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Signals sent while the hub was dying never reached this worker.
|
|
230
|
+
def report_gap
|
|
231
|
+
@on_gap&.call
|
|
232
|
+
rescue StandardError => e
|
|
233
|
+
@logger.error { "[Pgbus::Streamer::FailoverListener] on_gap raised: #{e.class}: #{e.message}" }
|
|
234
|
+
end
|
|
235
|
+
|
|
131
236
|
def current_impl
|
|
132
237
|
@state_mutex.synchronize { @impl }
|
|
133
238
|
end
|
|
@@ -18,6 +18,13 @@ module Pgbus
|
|
|
18
18
|
# the FailoverListener's cue to swap in a per-worker Listener. One-way:
|
|
19
19
|
# once a worker has fallen back it stays local until it recycles
|
|
20
20
|
# (settled on #382 — no flap-back complexity).
|
|
21
|
+
#
|
|
22
|
+
# Process signals (issue #674): ensure_signal_listening subscribes a
|
|
23
|
+
# signal channel with the same ack contract ("ssub" → "sack"); the
|
|
24
|
+
# master's "signal" and "gap" frames go to the `signal_sink:` (the
|
|
25
|
+
# worker's SignalDelivery). A master from before signals never acks an
|
|
26
|
+
# "ssub", so the ack deadline fails this worker over to its own LISTEN:
|
|
27
|
+
# mixed versions during a rolling restart degrade, they do not go deaf.
|
|
21
28
|
class HubClient
|
|
22
29
|
class HubUnavailableError < StandardError; end
|
|
23
30
|
|
|
@@ -25,9 +32,10 @@ module Pgbus
|
|
|
25
32
|
# NotifyListener: a just-connected worker isn't treated as degraded
|
|
26
33
|
# before the hub has said anything.
|
|
27
34
|
def initialize(socket_path:, dispatch_queue:, ack_timeout: 2.0,
|
|
28
|
-
on_failure: nil, logger: Pgbus.logger)
|
|
35
|
+
on_failure: nil, signal_sink: nil, logger: Pgbus.logger)
|
|
29
36
|
@socket_path = socket_path
|
|
30
37
|
@dispatch_queue = dispatch_queue
|
|
38
|
+
@signal_sink = signal_sink
|
|
31
39
|
@ack_timeout = ack_timeout
|
|
32
40
|
@on_failure = on_failure
|
|
33
41
|
@logger = logger
|
|
@@ -73,21 +81,12 @@ module Pgbus
|
|
|
73
81
|
# transport — a hub that can't ack in time can't be trusted with the
|
|
74
82
|
# no-lost-broadcast contract either).
|
|
75
83
|
def ensure_listening(queue)
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
waiter = Queue.new
|
|
79
|
-
@ack_mutex.synchronize { @pending_acks[queue] << waiter }
|
|
80
|
-
write_frame({ "t" => "sub", "q" => queue })
|
|
81
|
-
|
|
82
|
-
result = waiter.pop(timeout: @ack_timeout)
|
|
83
|
-
if result.nil?
|
|
84
|
-
discard_waiter(queue, waiter)
|
|
85
|
-
mark_dead("sub ack for #{queue} not received within #{@ack_timeout}s")
|
|
86
|
-
raise HubUnavailableError, "master hub ack timeout for #{queue}"
|
|
87
|
-
end
|
|
88
|
-
raise HubUnavailableError, "master hub died while awaiting ack for #{queue}" if result == :dead
|
|
84
|
+
await_ack(queue, { "t" => "sub", "q" => queue }, queue)
|
|
85
|
+
end
|
|
89
86
|
|
|
90
|
-
|
|
87
|
+
# The same contract for a process signal channel (issue #674).
|
|
88
|
+
def ensure_signal_listening(channel)
|
|
89
|
+
await_ack([:signal, channel], { "t" => "ssub", "c" => channel }, "signal #{channel}")
|
|
91
90
|
end
|
|
92
91
|
|
|
93
92
|
# Lazy GC, fire-and-forget — no correctness path waits on UNLISTEN
|
|
@@ -118,6 +117,24 @@ module Pgbus
|
|
|
118
117
|
|
|
119
118
|
private
|
|
120
119
|
|
|
120
|
+
def await_ack(key, frame, label)
|
|
121
|
+
raise HubUnavailableError, "master hub transport is dead" if @dead
|
|
122
|
+
|
|
123
|
+
waiter = Queue.new
|
|
124
|
+
@ack_mutex.synchronize { @pending_acks[key] << waiter }
|
|
125
|
+
write_frame(frame)
|
|
126
|
+
|
|
127
|
+
result = waiter.pop(timeout: @ack_timeout)
|
|
128
|
+
if result.nil?
|
|
129
|
+
discard_waiter(key, waiter)
|
|
130
|
+
mark_dead("sub ack for #{label} not received within #{@ack_timeout}s")
|
|
131
|
+
raise HubUnavailableError, "master hub ack timeout for #{label}"
|
|
132
|
+
end
|
|
133
|
+
raise HubUnavailableError, "master hub died while awaiting ack for #{label}" if result == :dead
|
|
134
|
+
|
|
135
|
+
:done
|
|
136
|
+
end
|
|
137
|
+
|
|
121
138
|
def reader_loop
|
|
122
139
|
loop do
|
|
123
140
|
frame = HubProtocol.read_frame(@sock)
|
|
@@ -143,6 +160,12 @@ module Pgbus
|
|
|
143
160
|
@dispatch_queue << Listener::WakeMessage.new(queue_name: frame["q"], payload: frame["p"])
|
|
144
161
|
when "ack"
|
|
145
162
|
@ack_mutex.synchronize { @pending_acks[frame["q"]].shift }&.push(:ack)
|
|
163
|
+
when "sack"
|
|
164
|
+
@ack_mutex.synchronize { @pending_acks[[:signal, frame["c"]]].shift }&.push(:ack)
|
|
165
|
+
when "signal"
|
|
166
|
+
to_signal_sink { |sink| sink.signal(frame["c"], frame["p"]) }
|
|
167
|
+
when "gap"
|
|
168
|
+
to_signal_sink(&:gap)
|
|
146
169
|
when "status"
|
|
147
170
|
@hub_healthy = frame["healthy"]
|
|
148
171
|
else
|
|
@@ -150,6 +173,14 @@ module Pgbus
|
|
|
150
173
|
end
|
|
151
174
|
end
|
|
152
175
|
|
|
176
|
+
# A raising sink must not reach reader_loop's rescue, which would mark
|
|
177
|
+
# a healthy transport dead.
|
|
178
|
+
def to_signal_sink
|
|
179
|
+
yield @signal_sink if @signal_sink
|
|
180
|
+
rescue StandardError => e
|
|
181
|
+
@logger.error { "[Pgbus::Streamer::HubClient] signal sink raised: #{e.class}: #{e.message}" }
|
|
182
|
+
end
|
|
183
|
+
|
|
153
184
|
# Frames must never interleave — all writes go through one mutex
|
|
154
185
|
# (writers: dispatcher thread via ensure/remove; no writer thread
|
|
155
186
|
# needed client-side, sub/unsub frames are tiny). Bounded: a master
|
|
@@ -15,9 +15,15 @@ module Pgbus
|
|
|
15
15
|
# Message shapes (JSON objects; "t" is the discriminator):
|
|
16
16
|
# worker → master: {t:"sub", q:} subscribe, synchronous — master acks
|
|
17
17
|
# {t:"unsub", q:} unsubscribe, fire-and-forget
|
|
18
|
+
# {t:"ssub", c:} subscribe a process signal channel (issue #674), master sacks
|
|
18
19
|
# master → worker: {t:"ack", q:} sub acknowledged (LISTEN active)
|
|
19
20
|
# {t:"wake", q:, p: <String|nil>} durable (p:nil) or ephemeral wake
|
|
20
21
|
# {t:"status", healthy: <bool>} listener health broadcast
|
|
22
|
+
# {t:"sack", c:} ssub acknowledged (LISTEN active)
|
|
23
|
+
# {t:"signal", c:, p: <String|nil>} a process signal, never dropped by the
|
|
24
|
+
# backpressure policy (an evicted worker
|
|
25
|
+
# loses its socket, fails over, and reports a gap)
|
|
26
|
+
# {t:"gap"} the hub's LISTEN reconnected: signals may be lost
|
|
21
27
|
#
|
|
22
28
|
# Reads are blocking (each side owns a dedicated reader thread); a short
|
|
23
29
|
# read means the peer died mid-frame and is reported as EOF (nil), never
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "concurrent/atomic/atomic_boolean"
|
|
4
|
+
|
|
3
5
|
module Pgbus
|
|
4
6
|
module Web
|
|
5
7
|
module Streamer
|
|
@@ -22,7 +24,7 @@ module Pgbus
|
|
|
22
24
|
# defaults from the configuration.
|
|
23
25
|
class Instance
|
|
24
26
|
attr_reader :registry, :listener, :dispatcher, :heartbeat, :dispatch_queue, :stream_counter, :pump,
|
|
25
|
-
:autoscaler, :peer_close_watcher
|
|
27
|
+
:autoscaler, :peer_close_watcher, :signal_delivery
|
|
26
28
|
|
|
27
29
|
def initialize(
|
|
28
30
|
client: Pgbus.client,
|
|
@@ -50,6 +52,14 @@ module Pgbus
|
|
|
50
52
|
if @config.streams_pool_autoscale && !@client.shared_connection?
|
|
51
53
|
Pgbus::Streams::PoolAutoscaler.new(client: @client, config: @config, logger: @logger)
|
|
52
54
|
end
|
|
55
|
+
# Runs process-signal subscribers (issue #674) on a thread of its
|
|
56
|
+
# own, started by the first signal: the listener and the hub client
|
|
57
|
+
# only hand it NOTIFYs and gaps.
|
|
58
|
+
@signal_delivery = SignalDelivery.new(logger: @logger)
|
|
59
|
+
# Set once #start has run its first-LISTEN gap: a hub failover
|
|
60
|
+
# during #start (an old master never acks "ssub") is covered by that
|
|
61
|
+
# gap, so it reports none of its own.
|
|
62
|
+
@signals_listening = Concurrent::AtomicBoolean.new(false)
|
|
53
63
|
@listener = build_listener(pg_connection, connection_factory)
|
|
54
64
|
# Off-thread durable fanout writer (issue #321). Built only when
|
|
55
65
|
# streams_writer_threads > 0; nil means fanout writes stay inline on
|
|
@@ -95,9 +105,29 @@ module Pgbus
|
|
|
95
105
|
@dispatcher.start
|
|
96
106
|
@heartbeat.start
|
|
97
107
|
@peer_close_watcher&.start
|
|
108
|
+
# Every process signal registered so far (issue #674); one
|
|
109
|
+
# registered later arrives through #listen_signal. Signals sent
|
|
110
|
+
# before this first LISTEN never reached the process (it may have
|
|
111
|
+
# cached since boot, or since a fork under preload_app!): that is a
|
|
112
|
+
# gap like any other, so every on_gap runs once.
|
|
113
|
+
channels = Pgbus::Signals.channels
|
|
114
|
+
channels.each { |channel| @listener.ensure_signal_listening(channel) }
|
|
115
|
+
# The gate first: a failover gap from here on is reported (and
|
|
116
|
+
# coalesces with this one while it is still queued).
|
|
117
|
+
@signals_listening.make_true
|
|
118
|
+
@signal_delivery.gap if channels.any?
|
|
98
119
|
self
|
|
99
120
|
end
|
|
100
121
|
|
|
122
|
+
# LISTEN on a process signal channel registered after #start (issue
|
|
123
|
+
# #674). Before #start, or after #shutdown!, nothing: #start
|
|
124
|
+
# subscribes every registered channel.
|
|
125
|
+
def listen_signal(channel)
|
|
126
|
+
return unless @shutdown_mutex.synchronize { @started }
|
|
127
|
+
|
|
128
|
+
@listener.ensure_signal_listening(channel)
|
|
129
|
+
end
|
|
130
|
+
|
|
101
131
|
# Enqueue a new SSE client. The dispatcher picks this up on its next
|
|
102
132
|
# iteration and runs the 5-step race-free replay sequence. The StreamApp
|
|
103
133
|
# calls this right after hijacking the socket.
|
|
@@ -161,6 +191,7 @@ module Pgbus
|
|
|
161
191
|
safely { @listener.stop }
|
|
162
192
|
safely { @dispatcher.stop }
|
|
163
193
|
safely { @pump&.stop }
|
|
194
|
+
safely { @signal_delivery.stop }
|
|
164
195
|
leaked = threads.select { |_, list| list.any?(&:alive?) }.keys
|
|
165
196
|
report_leaked_threads(leaked) if leaked.any?
|
|
166
197
|
close_all_connections
|
|
@@ -200,6 +231,7 @@ module Pgbus
|
|
|
200
231
|
# failover is assigned right below; a transport death in the gap
|
|
201
232
|
# is caught by the FailoverListener's synchronous ensure path.
|
|
202
233
|
on_failure: -> { failover&.fail_over! },
|
|
234
|
+
signal_sink: @signal_delivery,
|
|
203
235
|
logger: @logger
|
|
204
236
|
)
|
|
205
237
|
client.connect
|
|
@@ -208,6 +240,7 @@ module Pgbus
|
|
|
208
240
|
local_listener_factory: lambda do
|
|
209
241
|
build_local_listener(build_pg_connection, connection_factory).tap(&:start)
|
|
210
242
|
end,
|
|
243
|
+
on_gap: -> { @signal_delivery.gap if @signals_listening.true? },
|
|
211
244
|
logger: @logger
|
|
212
245
|
)
|
|
213
246
|
rescue HubClient::HubUnavailableError => e
|
|
@@ -229,6 +262,7 @@ module Pgbus
|
|
|
229
262
|
# own prune_dead self-post never block.
|
|
230
263
|
dispatch_queue_limit: @config.streams_dispatch_queue_limit,
|
|
231
264
|
maintenance: build_autoscale_maintenance,
|
|
265
|
+
signal_sink: @signal_delivery,
|
|
232
266
|
logger: @logger,
|
|
233
267
|
# On reconnect the Listener rebuilds its OWN connection via this
|
|
234
268
|
# factory (fresh connect re-resolves DNS, converges on the promoted
|
|
@@ -256,7 +290,8 @@ module Pgbus
|
|
|
256
290
|
"peer_close_watcher" => @peer_close_watcher ? @peer_close_watcher.threads : [],
|
|
257
291
|
"listener" => @listener.threads,
|
|
258
292
|
"dispatcher" => @dispatcher.threads,
|
|
259
|
-
"pump" => @pump ? @pump.threads : []
|
|
293
|
+
"pump" => @pump ? @pump.threads : [],
|
|
294
|
+
"signal_delivery" => @signal_delivery.threads
|
|
260
295
|
}
|
|
261
296
|
end
|
|
262
297
|
|