pgbus 0.26.1 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -25,6 +25,7 @@ module Pgbus
25
25
  def before_setup
26
26
  Pgbus::Testing.fake!
27
27
  Pgbus::Testing.store.clear!
28
+ Pgbus::Testing.clear_signals!
28
29
  super
29
30
  end
30
31
  end
data/lib/pgbus/testing.rb CHANGED
@@ -14,6 +14,11 @@ module Pgbus
14
14
  # :fake — capture published events in an in-memory store (default for tests)
15
15
  # :inline — capture AND immediately dispatch to matching handlers
16
16
  # :disabled — pass through to the real publisher (production behavior)
17
+ #
18
+ # Process signals (Pgbus.signal, issue #674) follow the same modes: :fake
19
+ # records them in `Pgbus::Testing.signals`, :inline records them and runs the
20
+ # process's on_signal blocks synchronously (at the COMMIT of an open
21
+ # transaction, as the NOTIFY would be sent), :disabled sends the NOTIFY.
17
22
  module Testing
18
23
  MODES = %i[fake inline disabled].freeze
19
24
  MODE_KEY = :__pgbus_test_mode
@@ -68,8 +73,15 @@ module Pgbus
68
73
  end
69
74
  end
70
75
 
76
+ # One Pgbus.signal captured in fake or inline mode.
77
+ SignalRecord = Data.define(:name, :payload)
78
+
71
79
  # Eagerly initialize the store so concurrent access never races on creation.
72
80
  @store = EventStore.new
81
+ # Signals captured in fake/inline mode: a frozen Array replaced whole
82
+ # under its own lock.
83
+ @signals_lock = Mutex.new
84
+ @signals = [].freeze
73
85
 
74
86
  # The process-wide mode (`fake!` with no block). A module ivar under a lock
75
87
  # built at load, not `Thread.main[]`: that reads the main thread's CURRENT
@@ -120,6 +132,30 @@ module Pgbus
120
132
 
121
133
  attr_reader :store
122
134
 
135
+ # The signals Pgbus.signal captured in fake or inline mode, oldest first.
136
+ def signals
137
+ @signals_lock.synchronize { @signals }
138
+ end
139
+
140
+ def clear_signals!
141
+ @signals_lock.synchronize { @signals = [].freeze }
142
+ end
143
+
144
+ # Called by Pgbus::Signals.publish instead of a NOTIFY when the mode is
145
+ # :fake or :inline. Recorded (and, inline, delivered) when the NOTIFY
146
+ # would be sent: at the COMMIT of an open joinable transaction, never on
147
+ # its rollback. Inline delivery runs on the caller's thread without the
148
+ # Rails executor, so the caller's CurrentAttributes survive it.
149
+ def record_signal(name, payload)
150
+ record = SignalRecord.new(name: name.to_s, payload: payload)
151
+ inline = inline?
152
+ commit = lambda do
153
+ @signals_lock.synchronize { @signals = [*@signals, record].freeze }
154
+ Pgbus::Signals.deliver(record.name, payload, executor: false) if inline
155
+ end
156
+ commit.call unless Pgbus::OpenTransaction.defer(&commit)
157
+ end
158
+
123
159
  private
124
160
 
125
161
  def default_mode
data/lib/pgbus/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Pgbus
4
- VERSION = "0.26.1"
4
+ VERSION = "0.27.0"
5
5
  end
@@ -21,12 +21,28 @@ module Pgbus
21
21
  # relies on it. ensure_listening NEVER raises to the dispatcher: on a
22
22
  # double failure (hub dead AND local build failing — e.g. DB down) it
23
23
  # logs and returns nil, matching the Listener's own ack-timeout
24
- # contract, which the dispatcher already tolerates.
24
+ # contract, which the dispatcher already tolerates, and a background
25
+ # thread retries the local build every `rebuild_interval` seconds until
26
+ # it succeeds or #stop (issue #674: a process signal has no later
27
+ # ensure_listening call to retry it).
28
+ #
29
+ # Process signal channels (issue #674) are recorded and replayed the
30
+ # same way. NOTIFYs sent between the hub's death and the local LISTEN
31
+ # are lost, so once a failover has re-LISTENed a signal channel it calls
32
+ # `on_gap` (the worker's signal delivery runs every on_gap block).
25
33
  class FailoverListener
26
- def initialize(hub_client:, local_listener_factory:, logger: Pgbus.logger)
34
+ REBUILD_INTERVAL_SECONDS = 5
35
+ REBUILD_JOIN_SECONDS = 2
36
+
37
+ def initialize(hub_client:, local_listener_factory:, on_gap: nil,
38
+ rebuild_interval: REBUILD_INTERVAL_SECONDS, logger: Pgbus.logger)
27
39
  @hub_client = hub_client
28
40
  @local_listener_factory = local_listener_factory
41
+ @on_gap = on_gap
42
+ @rebuild_interval = rebuild_interval
29
43
  @logger = logger
44
+ # #stop wakes the rebuild thread's wait through this queue.
45
+ @stop_signal = Queue.new
30
46
  # @state_mutex guards the cheap shared state (@subscriptions, @impl,
31
47
  # @failed_over) and is only ever held for constant-time work — the
32
48
  # dispatcher's ensure/remove path must never wait behind a failover
@@ -36,8 +52,12 @@ module Pgbus
36
52
  @state_mutex = Mutex.new
37
53
  @failover_mutex = Mutex.new
38
54
  @subscriptions = Set.new
55
+ @signal_subscriptions = Set.new
39
56
  @impl = hub_client
40
57
  @failed_over = false
58
+ # Guarded by @state_mutex.
59
+ @stopped = false
60
+ @rebuild_thread = nil
41
61
  end
42
62
 
43
63
  # Interface parity with Listener for Instance#start: the hub client
@@ -60,6 +80,19 @@ module Pgbus
60
80
  end
61
81
  end
62
82
 
83
+ # Same failover contract as ensure_listening, for a signal channel.
84
+ def ensure_signal_listening(channel)
85
+ @state_mutex.synchronize { @signal_subscriptions.add(channel) }
86
+ current_impl.ensure_signal_listening(channel)
87
+ rescue HubClient::HubUnavailableError
88
+ fail_over!
89
+ begin
90
+ current_impl.ensure_signal_listening(channel)
91
+ rescue HubClient::HubUnavailableError
92
+ nil
93
+ end
94
+ end
95
+
63
96
  def remove_listening(queue)
64
97
  @state_mutex.synchronize { @subscriptions.delete(queue) }
65
98
  current_impl.remove_listening(queue)
@@ -79,55 +112,127 @@ module Pgbus
79
112
  # The blocking build + replay runs OUTSIDE @state_mutex so concurrent
80
113
  # ensure/remove/stop calls never stall behind it.
81
114
  def fail_over!
82
- local = nil
115
+ signals = nil
83
116
  @failover_mutex.synchronize do
84
117
  return if @state_mutex.synchronize { @failed_over }
85
118
 
86
- local = @local_listener_factory.call
87
- @state_mutex.synchronize { @subscriptions.dup }.each { |q| local.ensure_listening(q) }
88
- # Subscriptions recorded between the snapshot and this swap arrive
89
- # via their own retried ensure_listening call on the new impl.
90
- @state_mutex.synchronize do
91
- @impl = local
92
- @failed_over = true
93
- end
94
- # Ownership transferred to @impl — the rescue must not stop it.
95
- local = nil
119
+ signals = swap_in_local!
96
120
  end
121
+ report_gap if signals == true
97
122
  rescue StandardError => e
98
- # A listener the factory STARTED but that never swapped in (the
99
- # replay raised) would otherwise leak its thread and LISTEN
100
- # connection alongside the dead hub client.
101
- begin
102
- local&.stop
103
- rescue StandardError
104
- nil
105
- end
106
123
  # Hub dead AND the local listener can't be built (DB down, config
107
- # broken). Mark failed-over so callers stop rebuilding; @impl stays
108
- # on the dead client — every ensure_listening resolves nil and the
109
- # dispatcher rides its existing timeout tolerance until the worker
110
- # recycles.
124
+ # broken). Mark failed-over so callers stop rebuilding
125
+ # synchronously; @impl stays on the dead client — every
126
+ # ensure_listening resolves nil and the dispatcher rides its
127
+ # existing timeout tolerance — while the rebuild thread retries.
111
128
  @state_mutex.synchronize { @failed_over = true }
112
129
  @logger.error do
113
130
  "[Pgbus::Streamer::FailoverListener] fallback listener failed to build " \
114
- "(#{e.class}: #{e.message}) — streams degraded until this worker recycles"
131
+ "(#{e.class}: #{e.message}) — streams and process signals degraded; " \
132
+ "retrying every #{@rebuild_interval}s"
115
133
  end
134
+ start_rebuild
116
135
  end
117
136
 
118
137
  # Snapshot of this component's live thread(s). Instance#shutdown! captures
119
138
  # it BEFORE calling #stop so a join that timed out is still observable
120
139
  # after #stop has cleared the reference (issue #443).
121
140
  def threads
122
- current_impl.threads
141
+ rebuild = @state_mutex.synchronize { @rebuild_thread }
142
+ rebuild&.alive? ? [*current_impl.threads, rebuild] : current_impl.threads
123
143
  end
124
144
 
125
145
  def stop
146
+ rebuild = @state_mutex.synchronize do
147
+ @stopped = true
148
+ @rebuild_thread
149
+ end
150
+ @stop_signal << :stop
151
+ rebuild&.join(REBUILD_JOIN_SECONDS)
152
+ # Never waits on a connect in flight: a rebuild that finishes after
153
+ # this sees @stopped and stops its own listener instead of swapping.
126
154
  current_impl.stop
127
155
  end
128
156
 
129
157
  private
130
158
 
159
+ # Builds a started local listener, replays every subscription on it
160
+ # and swaps it in. Returns true when it re-LISTENed a signal channel
161
+ # (the caller reports the gap), false when it re-LISTENed none, and
162
+ # :discarded when #stop came first or another build already swapped
163
+ # in (the listener is stopped, not swapped: only the hub client is
164
+ # ever replaced, so a live local listener is never orphaned). Raises when the build or the replay fails, after stopping
165
+ # a listener that never swapped in (it would leak its thread and
166
+ # LISTEN connection beside the dead client). Holds no lock across the
167
+ # build or the replay.
168
+ def swap_in_local!
169
+ local = @local_listener_factory.call
170
+ queues, signal_channels = @state_mutex.synchronize { [@subscriptions.dup, @signal_subscriptions.dup] }
171
+ queues.each { |q| local.ensure_listening(q) }
172
+ signal_channels.each { |channel| local.ensure_signal_listening(channel) }
173
+ swapped = @state_mutex.synchronize do
174
+ next false if @stopped || !@impl.equal?(@hub_client)
175
+
176
+ @impl = local
177
+ @failed_over = true
178
+ end
179
+ unless swapped
180
+ local.stop
181
+ return :discarded
182
+ end
183
+ # Subscriptions recorded during the replay went to the old impl;
184
+ # replay them on the new one.
185
+ late_queues, late_signals = @state_mutex.synchronize do
186
+ [@subscriptions - queues, @signal_subscriptions - signal_channels]
187
+ end
188
+ late_queues.each { |q| local.ensure_listening(q) }
189
+ late_signals.each { |channel| local.ensure_signal_listening(channel) }
190
+ signal_channels.any? || late_signals.any?
191
+ rescue StandardError
192
+ begin
193
+ local&.stop
194
+ rescue StandardError
195
+ nil
196
+ end
197
+ raise
198
+ end
199
+
200
+ def start_rebuild
201
+ @state_mutex.synchronize do
202
+ return if @stopped || @rebuild_thread&.alive?
203
+
204
+ # The retry thread, owned by this object: it ends once a local
205
+ # listener swaps in or #stop signals it; #stop joins it.
206
+ @rebuild_thread = Thread.new { rebuild_loop } # rubocop:disable ThreadSafety/NewThread
207
+ end
208
+ end
209
+
210
+ def rebuild_loop
211
+ loop do
212
+ break if @stop_signal.pop(timeout: @rebuild_interval)
213
+
214
+ # No @failover_mutex here: a connect in flight must not hold up
215
+ # #stop, nor fail_over! (which finds @failed_over set and returns).
216
+ result = swap_in_local!
217
+ break if result == :discarded
218
+
219
+ report_gap if result
220
+ @logger.info { "[Pgbus::Streamer::FailoverListener] fallback listener built; streams and process signals restored" }
221
+ break
222
+ rescue StandardError => e
223
+ @logger.warn do
224
+ "[Pgbus::Streamer::FailoverListener] fallback listener still failing to build (#{e.class}: #{e.message})"
225
+ end
226
+ end
227
+ end
228
+
229
+ # Signals sent while the hub was dying never reached this worker.
230
+ def report_gap
231
+ @on_gap&.call
232
+ rescue StandardError => e
233
+ @logger.error { "[Pgbus::Streamer::FailoverListener] on_gap raised: #{e.class}: #{e.message}" }
234
+ end
235
+
131
236
  def current_impl
132
237
  @state_mutex.synchronize { @impl }
133
238
  end
@@ -18,6 +18,13 @@ module Pgbus
18
18
  # the FailoverListener's cue to swap in a per-worker Listener. One-way:
19
19
  # once a worker has fallen back it stays local until it recycles
20
20
  # (settled on #382 — no flap-back complexity).
21
+ #
22
+ # Process signals (issue #674): ensure_signal_listening subscribes a
23
+ # signal channel with the same ack contract ("ssub" → "sack"); the
24
+ # master's "signal" and "gap" frames go to the `signal_sink:` (the
25
+ # worker's SignalDelivery). A master from before signals never acks an
26
+ # "ssub", so the ack deadline fails this worker over to its own LISTEN:
27
+ # mixed versions during a rolling restart degrade, they do not go deaf.
21
28
  class HubClient
22
29
  class HubUnavailableError < StandardError; end
23
30
 
@@ -25,9 +32,10 @@ module Pgbus
25
32
  # NotifyListener: a just-connected worker isn't treated as degraded
26
33
  # before the hub has said anything.
27
34
  def initialize(socket_path:, dispatch_queue:, ack_timeout: 2.0,
28
- on_failure: nil, logger: Pgbus.logger)
35
+ on_failure: nil, signal_sink: nil, logger: Pgbus.logger)
29
36
  @socket_path = socket_path
30
37
  @dispatch_queue = dispatch_queue
38
+ @signal_sink = signal_sink
31
39
  @ack_timeout = ack_timeout
32
40
  @on_failure = on_failure
33
41
  @logger = logger
@@ -73,21 +81,12 @@ module Pgbus
73
81
  # transport — a hub that can't ack in time can't be trusted with the
74
82
  # no-lost-broadcast contract either).
75
83
  def ensure_listening(queue)
76
- raise HubUnavailableError, "master hub transport is dead" if @dead
77
-
78
- waiter = Queue.new
79
- @ack_mutex.synchronize { @pending_acks[queue] << waiter }
80
- write_frame({ "t" => "sub", "q" => queue })
81
-
82
- result = waiter.pop(timeout: @ack_timeout)
83
- if result.nil?
84
- discard_waiter(queue, waiter)
85
- mark_dead("sub ack for #{queue} not received within #{@ack_timeout}s")
86
- raise HubUnavailableError, "master hub ack timeout for #{queue}"
87
- end
88
- raise HubUnavailableError, "master hub died while awaiting ack for #{queue}" if result == :dead
84
+ await_ack(queue, { "t" => "sub", "q" => queue }, queue)
85
+ end
89
86
 
90
- :done
87
+ # The same contract for a process signal channel (issue #674).
88
+ def ensure_signal_listening(channel)
89
+ await_ack([:signal, channel], { "t" => "ssub", "c" => channel }, "signal #{channel}")
91
90
  end
92
91
 
93
92
  # Lazy GC, fire-and-forget — no correctness path waits on UNLISTEN
@@ -118,6 +117,24 @@ module Pgbus
118
117
 
119
118
  private
120
119
 
120
+ def await_ack(key, frame, label)
121
+ raise HubUnavailableError, "master hub transport is dead" if @dead
122
+
123
+ waiter = Queue.new
124
+ @ack_mutex.synchronize { @pending_acks[key] << waiter }
125
+ write_frame(frame)
126
+
127
+ result = waiter.pop(timeout: @ack_timeout)
128
+ if result.nil?
129
+ discard_waiter(key, waiter)
130
+ mark_dead("sub ack for #{label} not received within #{@ack_timeout}s")
131
+ raise HubUnavailableError, "master hub ack timeout for #{label}"
132
+ end
133
+ raise HubUnavailableError, "master hub died while awaiting ack for #{label}" if result == :dead
134
+
135
+ :done
136
+ end
137
+
121
138
  def reader_loop
122
139
  loop do
123
140
  frame = HubProtocol.read_frame(@sock)
@@ -143,6 +160,12 @@ module Pgbus
143
160
  @dispatch_queue << Listener::WakeMessage.new(queue_name: frame["q"], payload: frame["p"])
144
161
  when "ack"
145
162
  @ack_mutex.synchronize { @pending_acks[frame["q"]].shift }&.push(:ack)
163
+ when "sack"
164
+ @ack_mutex.synchronize { @pending_acks[[:signal, frame["c"]]].shift }&.push(:ack)
165
+ when "signal"
166
+ to_signal_sink { |sink| sink.signal(frame["c"], frame["p"]) }
167
+ when "gap"
168
+ to_signal_sink(&:gap)
146
169
  when "status"
147
170
  @hub_healthy = frame["healthy"]
148
171
  else
@@ -150,6 +173,14 @@ module Pgbus
150
173
  end
151
174
  end
152
175
 
176
+ # A raising sink must not reach reader_loop's rescue, which would mark
177
+ # a healthy transport dead.
178
+ def to_signal_sink
179
+ yield @signal_sink if @signal_sink
180
+ rescue StandardError => e
181
+ @logger.error { "[Pgbus::Streamer::HubClient] signal sink raised: #{e.class}: #{e.message}" }
182
+ end
183
+
153
184
  # Frames must never interleave — all writes go through one mutex
154
185
  # (writers: dispatcher thread via ensure/remove; no writer thread
155
186
  # needed client-side, sub/unsub frames are tiny). Bounded: a master
@@ -15,9 +15,15 @@ module Pgbus
15
15
  # Message shapes (JSON objects; "t" is the discriminator):
16
16
  # worker → master: {t:"sub", q:} subscribe, synchronous — master acks
17
17
  # {t:"unsub", q:} unsubscribe, fire-and-forget
18
+ # {t:"ssub", c:} subscribe a process signal channel (issue #674), master sacks
18
19
  # master → worker: {t:"ack", q:} sub acknowledged (LISTEN active)
19
20
  # {t:"wake", q:, p: <String|nil>} durable (p:nil) or ephemeral wake
20
21
  # {t:"status", healthy: <bool>} listener health broadcast
22
+ # {t:"sack", c:} ssub acknowledged (LISTEN active)
23
+ # {t:"signal", c:, p: <String|nil>} a process signal, never dropped by the
24
+ # backpressure policy (an evicted worker
25
+ # loses its socket, fails over, and reports a gap)
26
+ # {t:"gap"} the hub's LISTEN reconnected: signals may be lost
21
27
  #
22
28
  # Reads are blocking (each side owns a dedicated reader thread); a short
23
29
  # read means the peer died mid-frame and is reported as EOF (nil), never
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "concurrent/atomic/atomic_boolean"
4
+
3
5
  module Pgbus
4
6
  module Web
5
7
  module Streamer
@@ -22,7 +24,7 @@ module Pgbus
22
24
  # defaults from the configuration.
23
25
  class Instance
24
26
  attr_reader :registry, :listener, :dispatcher, :heartbeat, :dispatch_queue, :stream_counter, :pump,
25
- :autoscaler, :peer_close_watcher
27
+ :autoscaler, :peer_close_watcher, :signal_delivery
26
28
 
27
29
  def initialize(
28
30
  client: Pgbus.client,
@@ -50,6 +52,14 @@ module Pgbus
50
52
  if @config.streams_pool_autoscale && !@client.shared_connection?
51
53
  Pgbus::Streams::PoolAutoscaler.new(client: @client, config: @config, logger: @logger)
52
54
  end
55
+ # Runs process-signal subscribers (issue #674) on a thread of its
56
+ # own, started by the first signal: the listener and the hub client
57
+ # only hand it NOTIFYs and gaps.
58
+ @signal_delivery = SignalDelivery.new(logger: @logger)
59
+ # Set once #start has run its first-LISTEN gap: a hub failover
60
+ # during #start (an old master never acks "ssub") is covered by that
61
+ # gap, so it reports none of its own.
62
+ @signals_listening = Concurrent::AtomicBoolean.new(false)
53
63
  @listener = build_listener(pg_connection, connection_factory)
54
64
  # Off-thread durable fanout writer (issue #321). Built only when
55
65
  # streams_writer_threads > 0; nil means fanout writes stay inline on
@@ -95,9 +105,29 @@ module Pgbus
95
105
  @dispatcher.start
96
106
  @heartbeat.start
97
107
  @peer_close_watcher&.start
108
+ # Every process signal registered so far (issue #674); one
109
+ # registered later arrives through #listen_signal. Signals sent
110
+ # before this first LISTEN never reached the process (it may have
111
+ # cached since boot, or since a fork under preload_app!): that is a
112
+ # gap like any other, so every on_gap runs once.
113
+ channels = Pgbus::Signals.channels
114
+ channels.each { |channel| @listener.ensure_signal_listening(channel) }
115
+ # The gate first: a failover gap from here on is reported (and
116
+ # coalesces with this one while it is still queued).
117
+ @signals_listening.make_true
118
+ @signal_delivery.gap if channels.any?
98
119
  self
99
120
  end
100
121
 
122
+ # LISTEN on a process signal channel registered after #start (issue
123
+ # #674). Before #start, or after #shutdown!, nothing: #start
124
+ # subscribes every registered channel.
125
+ def listen_signal(channel)
126
+ return unless @shutdown_mutex.synchronize { @started }
127
+
128
+ @listener.ensure_signal_listening(channel)
129
+ end
130
+
101
131
  # Enqueue a new SSE client. The dispatcher picks this up on its next
102
132
  # iteration and runs the 5-step race-free replay sequence. The StreamApp
103
133
  # calls this right after hijacking the socket.
@@ -161,6 +191,7 @@ module Pgbus
161
191
  safely { @listener.stop }
162
192
  safely { @dispatcher.stop }
163
193
  safely { @pump&.stop }
194
+ safely { @signal_delivery.stop }
164
195
  leaked = threads.select { |_, list| list.any?(&:alive?) }.keys
165
196
  report_leaked_threads(leaked) if leaked.any?
166
197
  close_all_connections
@@ -200,6 +231,7 @@ module Pgbus
200
231
  # failover is assigned right below; a transport death in the gap
201
232
  # is caught by the FailoverListener's synchronous ensure path.
202
233
  on_failure: -> { failover&.fail_over! },
234
+ signal_sink: @signal_delivery,
203
235
  logger: @logger
204
236
  )
205
237
  client.connect
@@ -208,6 +240,7 @@ module Pgbus
208
240
  local_listener_factory: lambda do
209
241
  build_local_listener(build_pg_connection, connection_factory).tap(&:start)
210
242
  end,
243
+ on_gap: -> { @signal_delivery.gap if @signals_listening.true? },
211
244
  logger: @logger
212
245
  )
213
246
  rescue HubClient::HubUnavailableError => e
@@ -229,6 +262,7 @@ module Pgbus
229
262
  # own prune_dead self-post never block.
230
263
  dispatch_queue_limit: @config.streams_dispatch_queue_limit,
231
264
  maintenance: build_autoscale_maintenance,
265
+ signal_sink: @signal_delivery,
232
266
  logger: @logger,
233
267
  # On reconnect the Listener rebuilds its OWN connection via this
234
268
  # factory (fresh connect re-resolves DNS, converges on the promoted
@@ -256,7 +290,8 @@ module Pgbus
256
290
  "peer_close_watcher" => @peer_close_watcher ? @peer_close_watcher.threads : [],
257
291
  "listener" => @listener.threads,
258
292
  "dispatcher" => @dispatcher.threads,
259
- "pump" => @pump ? @pump.threads : []
293
+ "pump" => @pump ? @pump.threads : [],
294
+ "signal_delivery" => @signal_delivery.threads
260
295
  }
261
296
  end
262
297