pgbus 0.12.4 → 0.13.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,85 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ module Pgbus
6
+ module Web
7
+ module Streamer
8
+ # Framing for the master-hub Unix socket (issue #382): 4-byte big-endian
9
+ # payload length + UTF-8 JSON. Unlike the job-side wake pipes (1-byte,
10
+ # lossy-by-design — Process::WakePipe), stream frames can carry an
11
+ # ephemeral broadcast's ONLY copy of its HTML, so the transport is
12
+ # length-prefixed and lossless; drop decisions are made per-message by
13
+ # the MasterHub, never by the wire format.
14
+ #
15
+ # Message shapes (JSON objects; "t" is the discriminator):
16
+ # worker → master: {t:"sub", q:} subscribe, synchronous — master acks
17
+ # {t:"unsub", q:} unsubscribe, fire-and-forget
18
+ # master → worker: {t:"ack", q:} sub acknowledged (LISTEN active)
19
+ # {t:"wake", q:, p: <String|nil>} durable (p:nil) or ephemeral wake
20
+ # {t:"status", healthy: <bool>} listener health broadcast
21
+ #
22
+ # Reads are blocking (each side owns a dedicated reader thread); a short
23
+ # read means the peer died mid-frame and is reported as EOF (nil), never
24
+ # as a truncated message.
25
+ module HubProtocol
26
+ class ProtocolError < StandardError; end
27
+
28
+ HEADER_BYTES = 4
29
+ # Generous ceiling for ephemeral HTML payloads; a frame announcing
30
+ # more than this is a corrupt stream or a runaway producer — sever
31
+ # rather than allocate.
32
+ MAX_FRAME_BYTES = 4 * 1024 * 1024
33
+
34
+ module_function
35
+
36
+ def encode(message)
37
+ json = JSON.generate(message)
38
+ bytes = json.b
39
+ raise ProtocolError, "frame too large: #{bytes.bytesize} bytes (max #{MAX_FRAME_BYTES})" if
40
+ bytes.bytesize > MAX_FRAME_BYTES
41
+
42
+ [bytes.bytesize].pack("N") + bytes
43
+ end
44
+
45
+ # Returns the decoded Hash, or nil on EOF — clean close, peer death
46
+ # mid-frame, OR a connection reset: an abrupt close can surface as
47
+ # ECONNRESET instead of orderly EOF depending on unread data and
48
+ # platform (Ruby 4.0 reports it deterministically where 3.x saw EOF),
49
+ # and both mean the same thing here: the peer is gone. Raises
50
+ # ProtocolError on an oversized announcement or malformed JSON.
51
+ def read_frame(io)
52
+ header = read_exactly(io, HEADER_BYTES)
53
+ return nil unless header
54
+
55
+ length = header.unpack1("N")
56
+ raise ProtocolError, "frame too large: #{length} bytes (max #{MAX_FRAME_BYTES})" if length > MAX_FRAME_BYTES
57
+
58
+ body = read_exactly(io, length)
59
+ return nil unless body
60
+
61
+ body = body.force_encoding(Encoding::UTF_8)
62
+ raise ProtocolError, "malformed frame: invalid UTF-8" unless body.valid_encoding?
63
+
64
+ decoded = JSON.parse(body)
65
+ raise ProtocolError, "malformed frame: expected a JSON object, got #{decoded.class}" unless decoded.is_a?(Hash)
66
+
67
+ decoded
68
+ rescue JSON::ParserError => e
69
+ raise ProtocolError, "malformed frame: #{e.message}"
70
+ rescue Errno::ECONNRESET
71
+ nil
72
+ end
73
+
74
+ # Blocking read of exactly +count+ bytes; nil on EOF (including EOF
75
+ # partway through — IO#read returns the short tail once, then nil).
76
+ def read_exactly(io, count)
77
+ data = io.read(count)
78
+ return nil if data.nil? || data.bytesize < count
79
+
80
+ data
81
+ end
82
+ end
83
+ end
84
+ end
85
+ end
@@ -39,7 +39,6 @@ module Pgbus
39
39
  @dispatch_queue = dispatch_queue || Queue.new
40
40
 
41
41
  @stream_counter = StreamCounter.new
42
- @pg_connection = pg_connection || build_pg_connection
43
42
  # Self-tuning streams-pool autoscaler (issue #323). Opt-in; nil unless
44
43
  # enabled AND on the dedicated connection path (the shared-AR streams
45
44
  # pool aliases the non-thread-safe job pool and resize is a no-op there).
@@ -50,25 +49,7 @@ module Pgbus
50
49
  if @config.streams_pool_autoscale && !@client.shared_connection?
51
50
  Pgbus::Streams::PoolAutoscaler.new(client: @client, config: @config, logger: @logger)
52
51
  end
53
- @listener = Listener.new(
54
- pg_connection: @pg_connection,
55
- dispatch_queue: @dispatch_queue,
56
- health_check_ms: @config.streams_listen_health_check_ms,
57
- # Opt-in dispatch-queue backpressure (issue #315 item 3). 0 =
58
- # unbounded (default). The queue itself stays an unbounded
59
- # Queue.new so the request-thread Connect push and the dispatcher's
60
- # own prune_dead self-post never block.
61
- dispatch_queue_limit: @config.streams_dispatch_queue_limit,
62
- maintenance: build_autoscale_maintenance,
63
- logger: @logger,
64
- # On reconnect the Listener rebuilds its OWN connection via this
65
- # factory (fresh connect re-resolves DNS, converges on the promoted
66
- # primary after a failover) instead of resetting a possibly-dead
67
- # socket. Always provided — even when an initial pg_connection: is
68
- # injected, the reconnect path builds a fresh raw connection. A test
69
- # can inject its own factory to avoid touching real configuration.
70
- connection_factory: connection_factory || -> { build_raw_pg_connection }
71
- )
52
+ @listener = build_listener(pg_connection, connection_factory)
72
53
  # Off-thread durable fanout writer (issue #321). Built only when
73
54
  # streams_writer_threads > 0; nil means fanout writes stay inline on
74
55
  # the dispatcher thread (the default, pre-#321 behavior). The pump
@@ -169,6 +150,74 @@ module Pgbus
169
150
 
170
151
  private
171
152
 
153
+ # Selects the wake source by streams_listen_scope (issue #382).
154
+ # :master with a reachable hub socket → FailoverListener over a
155
+ # HubClient (NO per-worker LISTEN connection is opened). Anything
156
+ # else — scope :process, no socket exported (single mode,
157
+ # non-preforking server, hub failed to start), or a refused connect —
158
+ # keeps today's per-worker Listener.
159
+ def build_listener(pg_connection, connection_factory)
160
+ hub = build_hub_listener(connection_factory)
161
+ return hub if hub
162
+
163
+ build_local_listener(pg_connection || build_pg_connection, connection_factory)
164
+ end
165
+
166
+ def build_hub_listener(connection_factory)
167
+ return nil unless @config.streams_listen_scope == :master
168
+
169
+ socket_path = ENV.fetch("PGBUS_STREAMS_HUB_SOCKET", nil)
170
+ return nil if socket_path.nil? || socket_path.empty?
171
+
172
+ # The worker's ack deadline must exceed the master's own internal
173
+ # ensure_listening budget (its listener's health-check cycle + 1s).
174
+ failover = nil
175
+ client = HubClient.new(
176
+ socket_path: socket_path,
177
+ dispatch_queue: @dispatch_queue,
178
+ ack_timeout: (@config.streams_listen_health_check_ms / 1000.0) + 2.0,
179
+ # failover is assigned right below; a transport death in the gap
180
+ # is caught by the FailoverListener's synchronous ensure path.
181
+ on_failure: -> { failover&.fail_over! },
182
+ logger: @logger
183
+ )
184
+ client.connect
185
+ failover = FailoverListener.new(
186
+ hub_client: client,
187
+ local_listener_factory: lambda do
188
+ build_local_listener(build_pg_connection, connection_factory).tap(&:start)
189
+ end,
190
+ logger: @logger
191
+ )
192
+ rescue HubClient::HubUnavailableError => e
193
+ @logger.info do
194
+ "[Pgbus::Streamer] master hub not reachable (#{e.message}) — using a per-worker listener"
195
+ end
196
+ nil
197
+ end
198
+
199
+ def build_local_listener(pg_connection, connection_factory)
200
+ Listener.new(
201
+ pg_connection: pg_connection,
202
+ dispatch_queue: @dispatch_queue,
203
+ health_check_ms: @config.streams_listen_health_check_ms,
204
+ # Opt-in dispatch-queue backpressure (issue #315 item 3). 0 =
205
+ # unbounded (default). The queue itself stays an unbounded
206
+ # Queue.new so the request-thread Connect push and the dispatcher's
207
+ # own prune_dead self-post never block.
208
+ dispatch_queue_limit: @config.streams_dispatch_queue_limit,
209
+ maintenance: build_autoscale_maintenance,
210
+ logger: @logger,
211
+ # On reconnect the Listener rebuilds its OWN connection via this
212
+ # factory (fresh connect re-resolves DNS, converges on the promoted
213
+ # primary after a failover) instead of resetting a possibly-dead
214
+ # socket. Always provided — even when an initial pg_connection: is
215
+ # injected, the reconnect path builds a fresh raw connection. A test
216
+ # can inject its own factory to avoid touching real configuration.
217
+ connection_factory: connection_factory || -> { build_raw_pg_connection }
218
+ )
219
+ end
220
+
172
221
  def safely
173
222
  yield
174
223
  rescue StandardError => e
@@ -41,8 +41,11 @@ module Pgbus
41
41
  end
42
42
  end
43
43
 
44
- CHANNEL_PREFIX = "pgmq.q_"
45
- CHANNEL_SUFFIX = ".INSERT"
44
+ # Single-sourced from NotifyListener, which owns the pgmq channel
45
+ # format (issue #381 review — the two copies had already drifted apart
46
+ # once in spirit if not in bytes).
47
+ CHANNEL_PREFIX = Pgbus::Process::NotifyListener::CHANNEL_PREFIX
48
+ CHANNEL_SUFFIX = Pgbus::Process::NotifyListener::CHANNEL_SUFFIX
46
49
 
47
50
  RECONNECT_BACKOFF_SECONDS = 0.5
48
51
 
@@ -102,6 +105,18 @@ module Pgbus
102
105
  self
103
106
  end
104
107
 
108
+ # Health signals for the MasterHub's status broadcasts (issue #382).
109
+ # Read cross-thread without synchronization: ivar assignment is atomic
110
+ # in MRI and a momentarily stale value only delays one status tick —
111
+ # these must never touch the connection itself (single-owner, #375).
112
+ def alive?
113
+ !!@thread&.alive?
114
+ end
115
+
116
+ def connected?
117
+ !@conn.nil?
118
+ end
119
+
105
120
  def stop
106
121
  return unless @running
107
122
 
@@ -0,0 +1,414 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "socket"
4
+ require "fileutils"
5
+
6
+ module Pgbus
7
+ module Web
8
+ module Streamer
9
+ # Master-process streams hub (issue #382): ONE LISTEN connection per web
10
+ # host instead of one per Puma worker. Runs in the Puma master (started
11
+ # by the pgbus_streams plugin), owns a single Web::Streamer::Listener on
12
+ # the refcounted union of every worker's stream channels, and fans wakes
13
+ # (including ephemeral payloads) out to workers over a Unix domain
14
+ # socket using HubProtocol frames.
15
+ #
16
+ # Workers are CLIENTS: they connect lazily to +socket_path+ on first SSE
17
+ # use (HubClient). Nothing is inherited across fork, so there is no FD
18
+ # hygiene for this transport, and a server that never starts a hub (no
19
+ # preload_app!, single mode, hub crash) simply has no socket — every
20
+ # worker falls back to its own per-worker Listener (FailoverListener),
21
+ # trading connections for unchanged semantics (settled on #382).
22
+ #
23
+ # The no-lost-wake ack contract, cross-process: a worker's sub is
24
+ # registered in the routing table BEFORE the hub executes LISTEN, and
25
+ # the ack is sent only AFTER ensure_listening returns — so from the
26
+ # moment LISTEN is active every wake reaches the subscribing worker.
27
+ # Over-delivery before the ack is harmless; under-delivery is the only
28
+ # failure mode that matters (same principle as Process::NotifyHub).
29
+ #
30
+ # Backpressure (per-worker outbound queue + writer thread):
31
+ # - durable wakes (payload nil) are droppable beyond durable_queue_limit
32
+ # — the next durable wake re-reads from the min cursor, so they
33
+ # self-heal (mirrors dispatch_queue_limit semantics);
34
+ # - ephemeral wakes are NEVER dropped: they push past the durable cap,
35
+ # and a worker whose queue exceeds hard_queue_limit is EVICTED
36
+ # (socket severed) — which triggers that worker's own fallback
37
+ # listener. A wedged worker degrades itself, never its siblings.
38
+ #
39
+ # Threading: accept thread + fanout thread + status thread, plus one
40
+ # reader and one writer thread per connected worker. The routing table
41
+ # is guarded by @table_mutex; each worker's outbox by its own mutex.
42
+ # All socket WRITES go through that worker's writer thread (frames must
43
+ # never interleave).
44
+ class MasterHub
45
+ DEFAULT_DURABLE_QUEUE_LIMIT = 256
46
+ DEFAULT_HARD_QUEUE_LIMIT = 1024
47
+ # Status is rebroadcast every REBROADCAST_TICKS status intervals even
48
+ # unchanged, so a worker that connected mid-outage converges.
49
+ REBROADCAST_TICKS = 5
50
+
51
+ attr_reader :socket_path
52
+
53
+ def initialize(config:, socket_path:, listener_factory: nil, status_interval: 1.0,
54
+ durable_queue_limit: DEFAULT_DURABLE_QUEUE_LIMIT,
55
+ hard_queue_limit: DEFAULT_HARD_QUEUE_LIMIT, logger: Pgbus.logger)
56
+ @config = config
57
+ @socket_path = socket_path
58
+ @status_interval = status_interval
59
+ @durable_queue_limit = durable_queue_limit
60
+ @hard_queue_limit = hard_queue_limit
61
+ @logger = logger
62
+ @listener_factory = listener_factory || default_listener_factory
63
+ @dispatch_queue = Queue.new
64
+ # Serializes the FULL start and stop sequences: a stop racing an
65
+ # in-progress start (blocked in the listener factory's PG connect)
66
+ # must WAIT for it and then tear everything down — otherwise stop
67
+ # returns having cleaned nothing and start finishes building a live
68
+ # hub afterwards. Loop threads never take this mutex (they read
69
+ # @running via @table_mutex), so holding it across the blocking
70
+ # startup cannot deadlock them.
71
+ @lifecycle_mutex = Mutex.new
72
+ @table_mutex = Mutex.new
73
+ @workers = {}
74
+ # Plain Hash, entries created ONLY at subscribe time — a default
75
+ # proc here would leak one empty Set per wake that arrives for an
76
+ # already-unsubscribed channel (in-flight NOTIFYs after the last
77
+ # unsub, per-record stream names → unbounded, review on #384).
78
+ @queue_refs = {}
79
+ @stop_signal = Queue.new
80
+ @next_id = 0
81
+ @dropped_durable_wakes = 0
82
+ @evicted_workers = 0
83
+ @running = false
84
+ end
85
+
86
+ def dropped_durable_wakes
87
+ @table_mutex.synchronize { @dropped_durable_wakes }
88
+ end
89
+
90
+ def evicted_workers
91
+ @table_mutex.synchronize { @evicted_workers }
92
+ end
93
+
94
+ # The factory must return a STARTED listener wired to +dispatch_queue+.
95
+ # If any step after the listener exists fails (bad socket path, chmod,
96
+ # thread spawn), the listener — and its dedicated LISTEN connection,
97
+ # the exact resource this hub conserves — is stopped before the error
98
+ # propagates; MasterHubBoot's rescue never sees a leaked connection.
99
+ def start
100
+ @lifecycle_mutex.synchronize { locked_start }
101
+ end
102
+
103
+ def stop
104
+ @lifecycle_mutex.synchronize { locked_stop }
105
+ end
106
+
107
+ private
108
+
109
+ def locked_start
110
+ @table_mutex.synchronize { @running = true }
111
+ @listener = @listener_factory.call(dispatch_queue: @dispatch_queue)
112
+ FileUtils.rm_f(@socket_path)
113
+ # Owner-only: the socket carries every stream wake including
114
+ # ephemeral HTML payloads, and there is no peer authentication —
115
+ # the filesystem mode IS the access control. The umask covers the
116
+ # bind itself so there is no window in which the socket exists with
117
+ # wider permissions; the chmod stays as the second guarantee.
118
+ # File.umask is process-wide, but this runs once at hub start in
119
+ # the Puma master and is restored in the ensure.
120
+ old_umask = File.umask(0o177)
121
+ begin
122
+ @server = UNIXServer.new(@socket_path)
123
+ ensure
124
+ File.umask(old_umask)
125
+ end
126
+ File.chmod(0o600, @socket_path)
127
+ @accept_thread = Thread.new { accept_loop }
128
+ @fanout_thread = Thread.new { fanout_loop }
129
+ @status_thread = Thread.new { status_loop }
130
+ self
131
+ rescue StandardError
132
+ @table_mutex.synchronize { @running = false }
133
+ close_quietly(@server)
134
+ @listener&.stop
135
+ @listener = nil
136
+ raise
137
+ end
138
+
139
+ def locked_stop
140
+ @table_mutex.synchronize do
141
+ return self unless @running
142
+
143
+ @running = false
144
+ end
145
+ @stop_signal << :stop
146
+ close_quietly(@server)
147
+ @dispatch_queue << :stop
148
+ worker_ids = @table_mutex.synchronize { @workers.keys }
149
+ worker_ids.each { |id| cleanup_worker(id) }
150
+ [@accept_thread, @fanout_thread, @status_thread].each { |t| t&.join(2) }
151
+ @listener&.stop
152
+ FileUtils.rm_f(@socket_path)
153
+ self
154
+ end
155
+
156
+ def default_listener_factory
157
+ lambda do |dispatch_queue:|
158
+ build_connection = -> { Pgbus::DedicatedConnection.connect(@config.streams_connection_options) }
159
+ conn = build_connection.call
160
+ Pgbus::Process::PrimaryValidator.validate_primary!(conn)
161
+ Listener.new(
162
+ pg_connection: conn,
163
+ dispatch_queue: dispatch_queue,
164
+ health_check_ms: @config.streams_listen_health_check_ms,
165
+ connection_factory: build_connection,
166
+ dispatch_queue_limit: @config.streams_dispatch_queue_limit,
167
+ logger: @logger
168
+ ).tap(&:start)
169
+ end
170
+ end
171
+
172
+ def running?
173
+ @table_mutex.synchronize { @running }
174
+ end
175
+
176
+ def accept_loop
177
+ loop do
178
+ begin
179
+ sock = @server.accept
180
+ rescue IOError, Errno::EBADF, Errno::EINVAL
181
+ # server closed during stop
182
+ break
183
+ end
184
+ begin
185
+ register_worker(sock)
186
+ rescue StandardError => e
187
+ # One bad connection must not stop the hub accepting others.
188
+ @logger.warn { "[Pgbus::Streamer::MasterHub] failed to register a worker: #{e.class}: #{e.message}" }
189
+ close_quietly(sock)
190
+ end
191
+ end
192
+ end
193
+
194
+ def register_worker(sock)
195
+ entry = {
196
+ sock: sock, subs: Set.new, outbox: [], durable_count: 0, open: true,
197
+ outbox_mutex: Mutex.new, outbox_cond: ConditionVariable.new
198
+ }
199
+ id = @table_mutex.synchronize do
200
+ @next_id += 1
201
+ @workers[@next_id] = entry
202
+ @next_id
203
+ end
204
+ entry[:writer] = Thread.new { writer_loop(id, entry) }
205
+ entry[:reader] = Thread.new { reader_loop(id, entry) }
206
+ id
207
+ end
208
+
209
+ def reader_loop(id, entry)
210
+ loop do
211
+ frame = HubProtocol.read_frame(entry[:sock])
212
+ break if frame.nil?
213
+
214
+ handle_frame(id, entry, frame)
215
+ end
216
+ rescue HubProtocol::ProtocolError => e
217
+ @logger.warn { "[Pgbus::Streamer::MasterHub] worker #{id} protocol error: #{e.message}" }
218
+ rescue IOError, Errno::EBADF, Errno::ECONNRESET
219
+ # severed by eviction or stop
220
+ rescue StandardError => e
221
+ # e.g. ensure_listening raising inside handle_sub — the ensure still
222
+ # severs this worker (its fallback takes over), but never silently.
223
+ @logger.warn { "[Pgbus::Streamer::MasterHub] reader for worker #{id} failed: #{e.class}: #{e.message}" }
224
+ ensure
225
+ cleanup_worker(id)
226
+ end
227
+
228
+ def handle_frame(id, entry, frame)
229
+ case frame["t"]
230
+ when "sub" then handle_sub(id, entry, frame["q"])
231
+ when "unsub" then handle_unsub(id, frame["q"])
232
+ else
233
+ @logger.warn { "[Pgbus::Streamer::MasterHub] worker #{id} sent unknown frame: #{frame["t"].inspect}" }
234
+ end
235
+ end
236
+
237
+ # Register FIRST, LISTEN second, ack LAST — the ordering the no-lost-
238
+ # wake contract rests on (see class comment). Runs on this worker's
239
+ # reader thread; ensure_listening blocks bounded by the listener's own
240
+ # ack budget.
241
+ def handle_sub(id, entry, queue)
242
+ @table_mutex.synchronize do
243
+ entry[:subs].add(queue)
244
+ (@queue_refs[queue] ||= Set.new).add(id)
245
+ end
246
+ @listener.ensure_listening(queue)
247
+ enqueue_frame(id, entry, { "t" => "ack", "q" => queue }, droppable: false)
248
+ end
249
+
250
+ def handle_unsub(id, queue)
251
+ release_queue_refs(id, [queue])
252
+ @table_mutex.synchronize { @workers[id]&.[](:subs)&.delete(queue) }
253
+ end
254
+
255
+ def fanout_loop
256
+ loop do
257
+ message = @dispatch_queue.pop
258
+ break if message == :stop
259
+
260
+ begin
261
+ deliver(message)
262
+ rescue StandardError => e
263
+ # One bad message must not stop wake delivery for the host.
264
+ @logger.error { "[Pgbus::Streamer::MasterHub] wake delivery failed: #{e.class}: #{e.message}" }
265
+ end
266
+ end
267
+ end
268
+
269
+ def deliver(message)
270
+ frame = { "t" => "wake", "q" => message.queue_name, "p" => message.payload }
271
+ droppable = message.payload.nil?
272
+ targets = @table_mutex.synchronize do
273
+ refs = @queue_refs[message.queue_name]
274
+ refs ? refs.filter_map { |id| [id, @workers[id]] if @workers[id] } : []
275
+ end
276
+ targets.each { |id, entry| enqueue_frame(id, entry, frame, droppable: droppable) }
277
+ end
278
+
279
+ # Non-blocking enqueue with the drop/evict policy. Never blocks the
280
+ # fanout thread on one slow worker (the head-of-line lesson from
281
+ # issue #315 item 3, applied cross-process).
282
+ def enqueue_frame(id, entry, frame, droppable:)
283
+ evict = false
284
+ entry[:outbox_mutex].synchronize do
285
+ return unless entry[:open]
286
+
287
+ if droppable && entry[:durable_count] >= @durable_queue_limit
288
+ @table_mutex.synchronize { @dropped_durable_wakes += 1 }
289
+ return
290
+ end
291
+
292
+ entry[:outbox] << [frame, droppable]
293
+ entry[:durable_count] += 1 if droppable
294
+ evict = entry[:outbox].size > @hard_queue_limit
295
+ entry[:outbox_cond].signal
296
+ end
297
+ evict_worker(id, entry) if evict
298
+ end
299
+
300
+ # Sever a worker that stopped draining. Closing the socket unblocks
301
+ # its writer (IOError) and its reader (EOF on the client side makes
302
+ # the worker's HubClient fail over to a local listener) — the wedged
303
+ # worker degrades itself, never its siblings.
304
+ def evict_worker(id, entry)
305
+ already = false
306
+ entry[:outbox_mutex].synchronize do
307
+ already = !entry[:open]
308
+ entry[:open] = false
309
+ entry[:outbox_cond].broadcast
310
+ end
311
+ return if already
312
+
313
+ @table_mutex.synchronize { @evicted_workers += 1 }
314
+ @logger.warn do
315
+ "[Pgbus::Streamer::MasterHub] evicting worker #{id}: outbound queue exceeded " \
316
+ "#{@hard_queue_limit} frames (worker not draining) — it falls back to its own listener"
317
+ end
318
+ close_quietly(entry[:sock])
319
+ end
320
+
321
+ def writer_loop(id, entry)
322
+ loop do
323
+ frame = nil
324
+ entry[:outbox_mutex].synchronize do
325
+ entry[:outbox_cond].wait(entry[:outbox_mutex]) while entry[:outbox].empty? && entry[:open]
326
+ return unless entry[:open]
327
+
328
+ frame, droppable = entry[:outbox].shift
329
+ entry[:durable_count] -= 1 if droppable
330
+ end
331
+ entry[:sock].write(HubProtocol.encode(frame))
332
+ end
333
+ rescue IOError, Errno::EPIPE, Errno::ECONNRESET, Errno::EBADF
334
+ # severed / worker died
335
+ rescue StandardError => e
336
+ # e.g. a ProtocolError from encode — never die silently; sever this
337
+ # worker so its fallback takes over.
338
+ @logger.warn { "[Pgbus::Streamer::MasterHub] writer for worker #{id} failed: #{e.class}: #{e.message}" }
339
+ ensure
340
+ cleanup_worker(id)
341
+ end
342
+
343
+ def status_loop
344
+ last_status = nil
345
+ ticks_since_broadcast = 0
346
+ loop do
347
+ # A stop-signal wait instead of sleep, so #stop wakes the thread
348
+ # immediately even with a long status_interval.
349
+ break if @stop_signal.pop(timeout: @status_interval)
350
+ break unless running?
351
+
352
+ begin
353
+ healthy = listener_healthy?
354
+ ticks_since_broadcast += 1
355
+ next unless healthy != last_status || ticks_since_broadcast >= REBROADCAST_TICKS
356
+
357
+ broadcast_status(healthy)
358
+ last_status = healthy
359
+ ticks_since_broadcast = 0
360
+ rescue StandardError => e
361
+ @logger.warn { "[Pgbus::Streamer::MasterHub] status tick failed: #{e.class}: #{e.message}" }
362
+ end
363
+ end
364
+ end
365
+
366
+ def listener_healthy?
367
+ listener = @listener
368
+ !!(listener&.alive? && listener.connected?)
369
+ end
370
+
371
+ def broadcast_status(healthy)
372
+ frame = { "t" => "status", "healthy" => healthy }
373
+ entries = @table_mutex.synchronize { @workers.to_a }
374
+ entries.each { |id, entry| enqueue_frame(id, entry, frame, droppable: false) }
375
+ end
376
+
377
+ # Idempotent teardown for one worker — reachable from its reader's
378
+ # ensure, an eviction, and stop.
379
+ def cleanup_worker(id)
380
+ entry = @table_mutex.synchronize { @workers.delete(id) }
381
+ return unless entry
382
+
383
+ entry[:outbox_mutex].synchronize do
384
+ entry[:open] = false
385
+ entry[:outbox_cond].broadcast
386
+ end
387
+ close_quietly(entry[:sock])
388
+ release_queue_refs(id, entry[:subs].to_a)
389
+ end
390
+
391
+ # Decrement refcounts; UNLISTEN queues that hit zero (async — no
392
+ # correctness path waits on unlisten, mirroring remove_listening).
393
+ def release_queue_refs(id, queues)
394
+ released = @table_mutex.synchronize do
395
+ queues.select do |q|
396
+ refs = @queue_refs[q]
397
+ next false unless refs
398
+
399
+ refs.delete(id)
400
+ @queue_refs.delete(q) if refs.empty?
401
+ end
402
+ end
403
+ released.each { |q| @listener.remove_listening(q) }
404
+ end
405
+
406
+ def close_quietly(io)
407
+ io.close if io && !io.closed?
408
+ rescue IOError, Errno::EBADF
409
+ nil
410
+ end
411
+ end
412
+ end
413
+ end
414
+ end