pgbus 0.12.4 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/README.md +2 -1
- data/Rakefile +17 -1
- data/lib/pgbus/client.rb +15 -0
- data/lib/pgbus/configuration/capsule_dsl.rb +8 -0
- data/lib/pgbus/configuration.rb +65 -5
- data/lib/pgbus/dedicated_connection.rb +29 -2
- data/lib/pgbus/doctor.rb +47 -2
- data/lib/pgbus/event_bus/registry.rb +22 -0
- data/lib/pgbus/process/consumer.rb +57 -22
- data/lib/pgbus/process/notify_hub.rb +273 -0
- data/lib/pgbus/process/notify_listener.rb +47 -2
- data/lib/pgbus/process/supervisor.rb +146 -34
- data/lib/pgbus/process/wake_pipe.rb +129 -0
- data/lib/pgbus/process/wildcard_queue_resolver.rb +35 -0
- data/lib/pgbus/process/worker.rb +58 -35
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/streamer/failover_listener.rb +130 -0
- data/lib/pgbus/web/streamer/hub_client.rb +199 -0
- data/lib/pgbus/web/streamer/hub_protocol.rb +85 -0
- data/lib/pgbus/web/streamer/instance.rb +69 -20
- data/lib/pgbus/web/streamer/listener.rb +17 -2
- data/lib/pgbus/web/streamer/master_hub.rb +414 -0
- data/lib/pgbus/web/streamer/master_hub_boot.rb +149 -0
- data/lib/puma/plugin/pgbus_streams.rb +38 -2
- metadata +9 -1
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Pgbus
|
|
6
|
+
module Web
|
|
7
|
+
module Streamer
|
|
8
|
+
# Framing for the master-hub Unix socket (issue #382): 4-byte big-endian
|
|
9
|
+
# payload length + UTF-8 JSON. Unlike the job-side wake pipes (1-byte,
|
|
10
|
+
# lossy-by-design — Process::WakePipe), stream frames can carry an
|
|
11
|
+
# ephemeral broadcast's ONLY copy of its HTML, so the transport is
|
|
12
|
+
# length-prefixed and lossless; drop decisions are made per-message by
|
|
13
|
+
# the MasterHub, never by the wire format.
|
|
14
|
+
#
|
|
15
|
+
# Message shapes (JSON objects; "t" is the discriminator):
|
|
16
|
+
# worker → master: {t:"sub", q:} subscribe, synchronous — master acks
|
|
17
|
+
# {t:"unsub", q:} unsubscribe, fire-and-forget
|
|
18
|
+
# master → worker: {t:"ack", q:} sub acknowledged (LISTEN active)
|
|
19
|
+
# {t:"wake", q:, p: <String|nil>} durable (p:nil) or ephemeral wake
|
|
20
|
+
# {t:"status", healthy: <bool>} listener health broadcast
|
|
21
|
+
#
|
|
22
|
+
# Reads are blocking (each side owns a dedicated reader thread); a short
|
|
23
|
+
# read means the peer died mid-frame and is reported as EOF (nil), never
|
|
24
|
+
# as a truncated message.
|
|
25
|
+
module HubProtocol
|
|
26
|
+
class ProtocolError < StandardError; end
|
|
27
|
+
|
|
28
|
+
HEADER_BYTES = 4
|
|
29
|
+
# Generous ceiling for ephemeral HTML payloads; a frame announcing
|
|
30
|
+
# more than this is a corrupt stream or a runaway producer — sever
|
|
31
|
+
# rather than allocate.
|
|
32
|
+
MAX_FRAME_BYTES = 4 * 1024 * 1024
|
|
33
|
+
|
|
34
|
+
module_function
|
|
35
|
+
|
|
36
|
+
def encode(message)
|
|
37
|
+
json = JSON.generate(message)
|
|
38
|
+
bytes = json.b
|
|
39
|
+
raise ProtocolError, "frame too large: #{bytes.bytesize} bytes (max #{MAX_FRAME_BYTES})" if
|
|
40
|
+
bytes.bytesize > MAX_FRAME_BYTES
|
|
41
|
+
|
|
42
|
+
[bytes.bytesize].pack("N") + bytes
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Returns the decoded Hash, or nil on EOF — clean close, peer death
|
|
46
|
+
# mid-frame, OR a connection reset: an abrupt close can surface as
|
|
47
|
+
# ECONNRESET instead of orderly EOF depending on unread data and
|
|
48
|
+
# platform (Ruby 4.0 reports it deterministically where 3.x saw EOF),
|
|
49
|
+
# and both mean the same thing here: the peer is gone. Raises
|
|
50
|
+
# ProtocolError on an oversized announcement or malformed JSON.
|
|
51
|
+
def read_frame(io)
|
|
52
|
+
header = read_exactly(io, HEADER_BYTES)
|
|
53
|
+
return nil unless header
|
|
54
|
+
|
|
55
|
+
length = header.unpack1("N")
|
|
56
|
+
raise ProtocolError, "frame too large: #{length} bytes (max #{MAX_FRAME_BYTES})" if length > MAX_FRAME_BYTES
|
|
57
|
+
|
|
58
|
+
body = read_exactly(io, length)
|
|
59
|
+
return nil unless body
|
|
60
|
+
|
|
61
|
+
body = body.force_encoding(Encoding::UTF_8)
|
|
62
|
+
raise ProtocolError, "malformed frame: invalid UTF-8" unless body.valid_encoding?
|
|
63
|
+
|
|
64
|
+
decoded = JSON.parse(body)
|
|
65
|
+
raise ProtocolError, "malformed frame: expected a JSON object, got #{decoded.class}" unless decoded.is_a?(Hash)
|
|
66
|
+
|
|
67
|
+
decoded
|
|
68
|
+
rescue JSON::ParserError => e
|
|
69
|
+
raise ProtocolError, "malformed frame: #{e.message}"
|
|
70
|
+
rescue Errno::ECONNRESET
|
|
71
|
+
nil
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Blocking read of exactly +count+ bytes; nil on EOF (including EOF
|
|
75
|
+
# partway through — IO#read returns the short tail once, then nil).
|
|
76
|
+
def read_exactly(io, count)
|
|
77
|
+
data = io.read(count)
|
|
78
|
+
return nil if data.nil? || data.bytesize < count
|
|
79
|
+
|
|
80
|
+
data
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -39,7 +39,6 @@ module Pgbus
|
|
|
39
39
|
@dispatch_queue = dispatch_queue || Queue.new
|
|
40
40
|
|
|
41
41
|
@stream_counter = StreamCounter.new
|
|
42
|
-
@pg_connection = pg_connection || build_pg_connection
|
|
43
42
|
# Self-tuning streams-pool autoscaler (issue #323). Opt-in; nil unless
|
|
44
43
|
# enabled AND on the dedicated connection path (the shared-AR streams
|
|
45
44
|
# pool aliases the non-thread-safe job pool and resize is a no-op there).
|
|
@@ -50,25 +49,7 @@ module Pgbus
|
|
|
50
49
|
if @config.streams_pool_autoscale && !@client.shared_connection?
|
|
51
50
|
Pgbus::Streams::PoolAutoscaler.new(client: @client, config: @config, logger: @logger)
|
|
52
51
|
end
|
|
53
|
-
@listener =
|
|
54
|
-
pg_connection: @pg_connection,
|
|
55
|
-
dispatch_queue: @dispatch_queue,
|
|
56
|
-
health_check_ms: @config.streams_listen_health_check_ms,
|
|
57
|
-
# Opt-in dispatch-queue backpressure (issue #315 item 3). 0 =
|
|
58
|
-
# unbounded (default). The queue itself stays an unbounded
|
|
59
|
-
# Queue.new so the request-thread Connect push and the dispatcher's
|
|
60
|
-
# own prune_dead self-post never block.
|
|
61
|
-
dispatch_queue_limit: @config.streams_dispatch_queue_limit,
|
|
62
|
-
maintenance: build_autoscale_maintenance,
|
|
63
|
-
logger: @logger,
|
|
64
|
-
# On reconnect the Listener rebuilds its OWN connection via this
|
|
65
|
-
# factory (fresh connect re-resolves DNS, converges on the promoted
|
|
66
|
-
# primary after a failover) instead of resetting a possibly-dead
|
|
67
|
-
# socket. Always provided — even when an initial pg_connection: is
|
|
68
|
-
# injected, the reconnect path builds a fresh raw connection. A test
|
|
69
|
-
# can inject its own factory to avoid touching real configuration.
|
|
70
|
-
connection_factory: connection_factory || -> { build_raw_pg_connection }
|
|
71
|
-
)
|
|
52
|
+
@listener = build_listener(pg_connection, connection_factory)
|
|
72
53
|
# Off-thread durable fanout writer (issue #321). Built only when
|
|
73
54
|
# streams_writer_threads > 0; nil means fanout writes stay inline on
|
|
74
55
|
# the dispatcher thread (the default, pre-#321 behavior). The pump
|
|
@@ -169,6 +150,74 @@ module Pgbus
|
|
|
169
150
|
|
|
170
151
|
private
|
|
171
152
|
|
|
153
|
+
# Selects the wake source by streams_listen_scope (issue #382).
|
|
154
|
+
# :master with a reachable hub socket → FailoverListener over a
|
|
155
|
+
# HubClient (NO per-worker LISTEN connection is opened). Anything
|
|
156
|
+
# else — scope :process, no socket exported (single mode,
|
|
157
|
+
# non-preforking server, hub failed to start), or a refused connect —
|
|
158
|
+
# keeps today's per-worker Listener.
|
|
159
|
+
def build_listener(pg_connection, connection_factory)
|
|
160
|
+
hub = build_hub_listener(connection_factory)
|
|
161
|
+
return hub if hub
|
|
162
|
+
|
|
163
|
+
build_local_listener(pg_connection || build_pg_connection, connection_factory)
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def build_hub_listener(connection_factory)
|
|
167
|
+
return nil unless @config.streams_listen_scope == :master
|
|
168
|
+
|
|
169
|
+
socket_path = ENV.fetch("PGBUS_STREAMS_HUB_SOCKET", nil)
|
|
170
|
+
return nil if socket_path.nil? || socket_path.empty?
|
|
171
|
+
|
|
172
|
+
# The worker's ack deadline must exceed the master's own internal
|
|
173
|
+
# ensure_listening budget (its listener's health-check cycle + 1s).
|
|
174
|
+
failover = nil
|
|
175
|
+
client = HubClient.new(
|
|
176
|
+
socket_path: socket_path,
|
|
177
|
+
dispatch_queue: @dispatch_queue,
|
|
178
|
+
ack_timeout: (@config.streams_listen_health_check_ms / 1000.0) + 2.0,
|
|
179
|
+
# failover is assigned right below; a transport death in the gap
|
|
180
|
+
# is caught by the FailoverListener's synchronous ensure path.
|
|
181
|
+
on_failure: -> { failover&.fail_over! },
|
|
182
|
+
logger: @logger
|
|
183
|
+
)
|
|
184
|
+
client.connect
|
|
185
|
+
failover = FailoverListener.new(
|
|
186
|
+
hub_client: client,
|
|
187
|
+
local_listener_factory: lambda do
|
|
188
|
+
build_local_listener(build_pg_connection, connection_factory).tap(&:start)
|
|
189
|
+
end,
|
|
190
|
+
logger: @logger
|
|
191
|
+
)
|
|
192
|
+
rescue HubClient::HubUnavailableError => e
|
|
193
|
+
@logger.info do
|
|
194
|
+
"[Pgbus::Streamer] master hub not reachable (#{e.message}) — using a per-worker listener"
|
|
195
|
+
end
|
|
196
|
+
nil
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def build_local_listener(pg_connection, connection_factory)
|
|
200
|
+
Listener.new(
|
|
201
|
+
pg_connection: pg_connection,
|
|
202
|
+
dispatch_queue: @dispatch_queue,
|
|
203
|
+
health_check_ms: @config.streams_listen_health_check_ms,
|
|
204
|
+
# Opt-in dispatch-queue backpressure (issue #315 item 3). 0 =
|
|
205
|
+
# unbounded (default). The queue itself stays an unbounded
|
|
206
|
+
# Queue.new so the request-thread Connect push and the dispatcher's
|
|
207
|
+
# own prune_dead self-post never block.
|
|
208
|
+
dispatch_queue_limit: @config.streams_dispatch_queue_limit,
|
|
209
|
+
maintenance: build_autoscale_maintenance,
|
|
210
|
+
logger: @logger,
|
|
211
|
+
# On reconnect the Listener rebuilds its OWN connection via this
|
|
212
|
+
# factory (fresh connect re-resolves DNS, converges on the promoted
|
|
213
|
+
# primary after a failover) instead of resetting a possibly-dead
|
|
214
|
+
# socket. Always provided — even when an initial pg_connection: is
|
|
215
|
+
# injected, the reconnect path builds a fresh raw connection. A test
|
|
216
|
+
# can inject its own factory to avoid touching real configuration.
|
|
217
|
+
connection_factory: connection_factory || -> { build_raw_pg_connection }
|
|
218
|
+
)
|
|
219
|
+
end
|
|
220
|
+
|
|
172
221
|
def safely
|
|
173
222
|
yield
|
|
174
223
|
rescue StandardError => e
|
|
@@ -41,8 +41,11 @@ module Pgbus
|
|
|
41
41
|
end
|
|
42
42
|
end
|
|
43
43
|
|
|
44
|
-
|
|
45
|
-
|
|
44
|
+
# Single-sourced from NotifyListener, which owns the pgmq channel
|
|
45
|
+
# format (issue #381 review — the two copies had already drifted apart
|
|
46
|
+
# once in spirit if not in bytes).
|
|
47
|
+
CHANNEL_PREFIX = Pgbus::Process::NotifyListener::CHANNEL_PREFIX
|
|
48
|
+
CHANNEL_SUFFIX = Pgbus::Process::NotifyListener::CHANNEL_SUFFIX
|
|
46
49
|
|
|
47
50
|
RECONNECT_BACKOFF_SECONDS = 0.5
|
|
48
51
|
|
|
@@ -102,6 +105,18 @@ module Pgbus
|
|
|
102
105
|
self
|
|
103
106
|
end
|
|
104
107
|
|
|
108
|
+
# Health signals for the MasterHub's status broadcasts (issue #382).
|
|
109
|
+
# Read cross-thread without synchronization: ivar assignment is atomic
|
|
110
|
+
# in MRI and a momentarily stale value only delays one status tick —
|
|
111
|
+
# these must never touch the connection itself (single-owner, #375).
|
|
112
|
+
def alive?
|
|
113
|
+
!!@thread&.alive?
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def connected?
|
|
117
|
+
!@conn.nil?
|
|
118
|
+
end
|
|
119
|
+
|
|
105
120
|
def stop
|
|
106
121
|
return unless @running
|
|
107
122
|
|
|
@@ -0,0 +1,414 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "socket"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
|
|
6
|
+
module Pgbus
|
|
7
|
+
module Web
|
|
8
|
+
module Streamer
|
|
9
|
+
# Master-process streams hub (issue #382): ONE LISTEN connection per web
|
|
10
|
+
# host instead of one per Puma worker. Runs in the Puma master (started
|
|
11
|
+
# by the pgbus_streams plugin), owns a single Web::Streamer::Listener on
|
|
12
|
+
# the refcounted union of every worker's stream channels, and fans wakes
|
|
13
|
+
# (including ephemeral payloads) out to workers over a Unix domain
|
|
14
|
+
# socket using HubProtocol frames.
|
|
15
|
+
#
|
|
16
|
+
# Workers are CLIENTS: they connect lazily to +socket_path+ on first SSE
|
|
17
|
+
# use (HubClient). Nothing is inherited across fork, so there is no FD
|
|
18
|
+
# hygiene for this transport, and a server that never starts a hub (no
|
|
19
|
+
# preload_app!, single mode, hub crash) simply has no socket — every
|
|
20
|
+
# worker falls back to its own per-worker Listener (FailoverListener),
|
|
21
|
+
# trading connections for unchanged semantics (settled on #382).
|
|
22
|
+
#
|
|
23
|
+
# The no-lost-wake ack contract, cross-process: a worker's sub is
|
|
24
|
+
# registered in the routing table BEFORE the hub executes LISTEN, and
|
|
25
|
+
# the ack is sent only AFTER ensure_listening returns — so from the
|
|
26
|
+
# moment LISTEN is active every wake reaches the subscribing worker.
|
|
27
|
+
# Over-delivery before the ack is harmless; under-delivery is the only
|
|
28
|
+
# failure mode that matters (same principle as Process::NotifyHub).
|
|
29
|
+
#
|
|
30
|
+
# Backpressure (per-worker outbound queue + writer thread):
|
|
31
|
+
# - durable wakes (payload nil) are droppable beyond durable_queue_limit
|
|
32
|
+
# — the next durable wake re-reads from the min cursor, so they
|
|
33
|
+
# self-heal (mirrors dispatch_queue_limit semantics);
|
|
34
|
+
# - ephemeral wakes are NEVER dropped: they push past the durable cap,
|
|
35
|
+
# and a worker whose queue exceeds hard_queue_limit is EVICTED
|
|
36
|
+
# (socket severed) — which triggers that worker's own fallback
|
|
37
|
+
# listener. A wedged worker degrades itself, never its siblings.
|
|
38
|
+
#
|
|
39
|
+
# Threading: accept thread + fanout thread + status thread, plus one
|
|
40
|
+
# reader and one writer thread per connected worker. The routing table
|
|
41
|
+
# is guarded by @table_mutex; each worker's outbox by its own mutex.
|
|
42
|
+
# All socket WRITES go through that worker's writer thread (frames must
|
|
43
|
+
# never interleave).
|
|
44
|
+
class MasterHub
|
|
45
|
+
DEFAULT_DURABLE_QUEUE_LIMIT = 256
|
|
46
|
+
DEFAULT_HARD_QUEUE_LIMIT = 1024
|
|
47
|
+
# Status is rebroadcast every REBROADCAST_TICKS status intervals even
|
|
48
|
+
# unchanged, so a worker that connected mid-outage converges.
|
|
49
|
+
REBROADCAST_TICKS = 5
|
|
50
|
+
|
|
51
|
+
attr_reader :socket_path
|
|
52
|
+
|
|
53
|
+
def initialize(config:, socket_path:, listener_factory: nil, status_interval: 1.0,
|
|
54
|
+
durable_queue_limit: DEFAULT_DURABLE_QUEUE_LIMIT,
|
|
55
|
+
hard_queue_limit: DEFAULT_HARD_QUEUE_LIMIT, logger: Pgbus.logger)
|
|
56
|
+
@config = config
|
|
57
|
+
@socket_path = socket_path
|
|
58
|
+
@status_interval = status_interval
|
|
59
|
+
@durable_queue_limit = durable_queue_limit
|
|
60
|
+
@hard_queue_limit = hard_queue_limit
|
|
61
|
+
@logger = logger
|
|
62
|
+
@listener_factory = listener_factory || default_listener_factory
|
|
63
|
+
@dispatch_queue = Queue.new
|
|
64
|
+
# Serializes the FULL start and stop sequences: a stop racing an
|
|
65
|
+
# in-progress start (blocked in the listener factory's PG connect)
|
|
66
|
+
# must WAIT for it and then tear everything down — otherwise stop
|
|
67
|
+
# returns having cleaned nothing and start finishes building a live
|
|
68
|
+
# hub afterwards. Loop threads never take this mutex (they read
|
|
69
|
+
# @running via @table_mutex), so holding it across the blocking
|
|
70
|
+
# startup cannot deadlock them.
|
|
71
|
+
@lifecycle_mutex = Mutex.new
|
|
72
|
+
@table_mutex = Mutex.new
|
|
73
|
+
@workers = {}
|
|
74
|
+
# Plain Hash, entries created ONLY at subscribe time — a default
|
|
75
|
+
# proc here would leak one empty Set per wake that arrives for an
|
|
76
|
+
# already-unsubscribed channel (in-flight NOTIFYs after the last
|
|
77
|
+
# unsub, per-record stream names → unbounded, review on #384).
|
|
78
|
+
@queue_refs = {}
|
|
79
|
+
@stop_signal = Queue.new
|
|
80
|
+
@next_id = 0
|
|
81
|
+
@dropped_durable_wakes = 0
|
|
82
|
+
@evicted_workers = 0
|
|
83
|
+
@running = false
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def dropped_durable_wakes
|
|
87
|
+
@table_mutex.synchronize { @dropped_durable_wakes }
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def evicted_workers
|
|
91
|
+
@table_mutex.synchronize { @evicted_workers }
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# The factory must return a STARTED listener wired to +dispatch_queue+.
|
|
95
|
+
# If any step after the listener exists fails (bad socket path, chmod,
|
|
96
|
+
# thread spawn), the listener — and its dedicated LISTEN connection,
|
|
97
|
+
# the exact resource this hub conserves — is stopped before the error
|
|
98
|
+
# propagates; MasterHubBoot's rescue never sees a leaked connection.
|
|
99
|
+
def start
|
|
100
|
+
@lifecycle_mutex.synchronize { locked_start }
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def stop
|
|
104
|
+
@lifecycle_mutex.synchronize { locked_stop }
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
private
|
|
108
|
+
|
|
109
|
+
def locked_start
|
|
110
|
+
@table_mutex.synchronize { @running = true }
|
|
111
|
+
@listener = @listener_factory.call(dispatch_queue: @dispatch_queue)
|
|
112
|
+
FileUtils.rm_f(@socket_path)
|
|
113
|
+
# Owner-only: the socket carries every stream wake including
|
|
114
|
+
# ephemeral HTML payloads, and there is no peer authentication —
|
|
115
|
+
# the filesystem mode IS the access control. The umask covers the
|
|
116
|
+
# bind itself so there is no window in which the socket exists with
|
|
117
|
+
# wider permissions; the chmod stays as the second guarantee.
|
|
118
|
+
# File.umask is process-wide, but this runs once at hub start in
|
|
119
|
+
# the Puma master and is restored in the ensure.
|
|
120
|
+
old_umask = File.umask(0o177)
|
|
121
|
+
begin
|
|
122
|
+
@server = UNIXServer.new(@socket_path)
|
|
123
|
+
ensure
|
|
124
|
+
File.umask(old_umask)
|
|
125
|
+
end
|
|
126
|
+
File.chmod(0o600, @socket_path)
|
|
127
|
+
@accept_thread = Thread.new { accept_loop }
|
|
128
|
+
@fanout_thread = Thread.new { fanout_loop }
|
|
129
|
+
@status_thread = Thread.new { status_loop }
|
|
130
|
+
self
|
|
131
|
+
rescue StandardError
|
|
132
|
+
@table_mutex.synchronize { @running = false }
|
|
133
|
+
close_quietly(@server)
|
|
134
|
+
@listener&.stop
|
|
135
|
+
@listener = nil
|
|
136
|
+
raise
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def locked_stop
|
|
140
|
+
@table_mutex.synchronize do
|
|
141
|
+
return self unless @running
|
|
142
|
+
|
|
143
|
+
@running = false
|
|
144
|
+
end
|
|
145
|
+
@stop_signal << :stop
|
|
146
|
+
close_quietly(@server)
|
|
147
|
+
@dispatch_queue << :stop
|
|
148
|
+
worker_ids = @table_mutex.synchronize { @workers.keys }
|
|
149
|
+
worker_ids.each { |id| cleanup_worker(id) }
|
|
150
|
+
[@accept_thread, @fanout_thread, @status_thread].each { |t| t&.join(2) }
|
|
151
|
+
@listener&.stop
|
|
152
|
+
FileUtils.rm_f(@socket_path)
|
|
153
|
+
self
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def default_listener_factory
|
|
157
|
+
lambda do |dispatch_queue:|
|
|
158
|
+
build_connection = -> { Pgbus::DedicatedConnection.connect(@config.streams_connection_options) }
|
|
159
|
+
conn = build_connection.call
|
|
160
|
+
Pgbus::Process::PrimaryValidator.validate_primary!(conn)
|
|
161
|
+
Listener.new(
|
|
162
|
+
pg_connection: conn,
|
|
163
|
+
dispatch_queue: dispatch_queue,
|
|
164
|
+
health_check_ms: @config.streams_listen_health_check_ms,
|
|
165
|
+
connection_factory: build_connection,
|
|
166
|
+
dispatch_queue_limit: @config.streams_dispatch_queue_limit,
|
|
167
|
+
logger: @logger
|
|
168
|
+
).tap(&:start)
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
def running?
|
|
173
|
+
@table_mutex.synchronize { @running }
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def accept_loop
|
|
177
|
+
loop do
|
|
178
|
+
begin
|
|
179
|
+
sock = @server.accept
|
|
180
|
+
rescue IOError, Errno::EBADF, Errno::EINVAL
|
|
181
|
+
# server closed during stop
|
|
182
|
+
break
|
|
183
|
+
end
|
|
184
|
+
begin
|
|
185
|
+
register_worker(sock)
|
|
186
|
+
rescue StandardError => e
|
|
187
|
+
# One bad connection must not stop the hub accepting others.
|
|
188
|
+
@logger.warn { "[Pgbus::Streamer::MasterHub] failed to register a worker: #{e.class}: #{e.message}" }
|
|
189
|
+
close_quietly(sock)
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def register_worker(sock)
|
|
195
|
+
entry = {
|
|
196
|
+
sock: sock, subs: Set.new, outbox: [], durable_count: 0, open: true,
|
|
197
|
+
outbox_mutex: Mutex.new, outbox_cond: ConditionVariable.new
|
|
198
|
+
}
|
|
199
|
+
id = @table_mutex.synchronize do
|
|
200
|
+
@next_id += 1
|
|
201
|
+
@workers[@next_id] = entry
|
|
202
|
+
@next_id
|
|
203
|
+
end
|
|
204
|
+
entry[:writer] = Thread.new { writer_loop(id, entry) }
|
|
205
|
+
entry[:reader] = Thread.new { reader_loop(id, entry) }
|
|
206
|
+
id
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def reader_loop(id, entry)
|
|
210
|
+
loop do
|
|
211
|
+
frame = HubProtocol.read_frame(entry[:sock])
|
|
212
|
+
break if frame.nil?
|
|
213
|
+
|
|
214
|
+
handle_frame(id, entry, frame)
|
|
215
|
+
end
|
|
216
|
+
rescue HubProtocol::ProtocolError => e
|
|
217
|
+
@logger.warn { "[Pgbus::Streamer::MasterHub] worker #{id} protocol error: #{e.message}" }
|
|
218
|
+
rescue IOError, Errno::EBADF, Errno::ECONNRESET
|
|
219
|
+
# severed by eviction or stop
|
|
220
|
+
rescue StandardError => e
|
|
221
|
+
# e.g. ensure_listening raising inside handle_sub — the ensure still
|
|
222
|
+
# severs this worker (its fallback takes over), but never silently.
|
|
223
|
+
@logger.warn { "[Pgbus::Streamer::MasterHub] reader for worker #{id} failed: #{e.class}: #{e.message}" }
|
|
224
|
+
ensure
|
|
225
|
+
cleanup_worker(id)
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
def handle_frame(id, entry, frame)
|
|
229
|
+
case frame["t"]
|
|
230
|
+
when "sub" then handle_sub(id, entry, frame["q"])
|
|
231
|
+
when "unsub" then handle_unsub(id, frame["q"])
|
|
232
|
+
else
|
|
233
|
+
@logger.warn { "[Pgbus::Streamer::MasterHub] worker #{id} sent unknown frame: #{frame["t"].inspect}" }
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
# Register FIRST, LISTEN second, ack LAST — the ordering the no-lost-
|
|
238
|
+
# wake contract rests on (see class comment). Runs on this worker's
|
|
239
|
+
# reader thread; ensure_listening blocks bounded by the listener's own
|
|
240
|
+
# ack budget.
|
|
241
|
+
def handle_sub(id, entry, queue)
|
|
242
|
+
@table_mutex.synchronize do
|
|
243
|
+
entry[:subs].add(queue)
|
|
244
|
+
(@queue_refs[queue] ||= Set.new).add(id)
|
|
245
|
+
end
|
|
246
|
+
@listener.ensure_listening(queue)
|
|
247
|
+
enqueue_frame(id, entry, { "t" => "ack", "q" => queue }, droppable: false)
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
def handle_unsub(id, queue)
|
|
251
|
+
release_queue_refs(id, [queue])
|
|
252
|
+
@table_mutex.synchronize { @workers[id]&.[](:subs)&.delete(queue) }
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
def fanout_loop
|
|
256
|
+
loop do
|
|
257
|
+
message = @dispatch_queue.pop
|
|
258
|
+
break if message == :stop
|
|
259
|
+
|
|
260
|
+
begin
|
|
261
|
+
deliver(message)
|
|
262
|
+
rescue StandardError => e
|
|
263
|
+
# One bad message must not stop wake delivery for the host.
|
|
264
|
+
@logger.error { "[Pgbus::Streamer::MasterHub] wake delivery failed: #{e.class}: #{e.message}" }
|
|
265
|
+
end
|
|
266
|
+
end
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def deliver(message)
|
|
270
|
+
frame = { "t" => "wake", "q" => message.queue_name, "p" => message.payload }
|
|
271
|
+
droppable = message.payload.nil?
|
|
272
|
+
targets = @table_mutex.synchronize do
|
|
273
|
+
refs = @queue_refs[message.queue_name]
|
|
274
|
+
refs ? refs.filter_map { |id| [id, @workers[id]] if @workers[id] } : []
|
|
275
|
+
end
|
|
276
|
+
targets.each { |id, entry| enqueue_frame(id, entry, frame, droppable: droppable) }
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
# Non-blocking enqueue with the drop/evict policy. Never blocks the
|
|
280
|
+
# fanout thread on one slow worker (the head-of-line lesson from
|
|
281
|
+
# issue #315 item 3, applied cross-process).
|
|
282
|
+
def enqueue_frame(id, entry, frame, droppable:)
|
|
283
|
+
evict = false
|
|
284
|
+
entry[:outbox_mutex].synchronize do
|
|
285
|
+
return unless entry[:open]
|
|
286
|
+
|
|
287
|
+
if droppable && entry[:durable_count] >= @durable_queue_limit
|
|
288
|
+
@table_mutex.synchronize { @dropped_durable_wakes += 1 }
|
|
289
|
+
return
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
entry[:outbox] << [frame, droppable]
|
|
293
|
+
entry[:durable_count] += 1 if droppable
|
|
294
|
+
evict = entry[:outbox].size > @hard_queue_limit
|
|
295
|
+
entry[:outbox_cond].signal
|
|
296
|
+
end
|
|
297
|
+
evict_worker(id, entry) if evict
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
# Sever a worker that stopped draining. Closing the socket unblocks
|
|
301
|
+
# its writer (IOError) and its reader (EOF on the client side makes
|
|
302
|
+
# the worker's HubClient fail over to a local listener) — the wedged
|
|
303
|
+
# worker degrades itself, never its siblings.
|
|
304
|
+
def evict_worker(id, entry)
|
|
305
|
+
already = false
|
|
306
|
+
entry[:outbox_mutex].synchronize do
|
|
307
|
+
already = !entry[:open]
|
|
308
|
+
entry[:open] = false
|
|
309
|
+
entry[:outbox_cond].broadcast
|
|
310
|
+
end
|
|
311
|
+
return if already
|
|
312
|
+
|
|
313
|
+
@table_mutex.synchronize { @evicted_workers += 1 }
|
|
314
|
+
@logger.warn do
|
|
315
|
+
"[Pgbus::Streamer::MasterHub] evicting worker #{id}: outbound queue exceeded " \
|
|
316
|
+
"#{@hard_queue_limit} frames (worker not draining) — it falls back to its own listener"
|
|
317
|
+
end
|
|
318
|
+
close_quietly(entry[:sock])
|
|
319
|
+
end
|
|
320
|
+
|
|
321
|
+
def writer_loop(id, entry)
|
|
322
|
+
loop do
|
|
323
|
+
frame = nil
|
|
324
|
+
entry[:outbox_mutex].synchronize do
|
|
325
|
+
entry[:outbox_cond].wait(entry[:outbox_mutex]) while entry[:outbox].empty? && entry[:open]
|
|
326
|
+
return unless entry[:open]
|
|
327
|
+
|
|
328
|
+
frame, droppable = entry[:outbox].shift
|
|
329
|
+
entry[:durable_count] -= 1 if droppable
|
|
330
|
+
end
|
|
331
|
+
entry[:sock].write(HubProtocol.encode(frame))
|
|
332
|
+
end
|
|
333
|
+
rescue IOError, Errno::EPIPE, Errno::ECONNRESET, Errno::EBADF
|
|
334
|
+
# severed / worker died
|
|
335
|
+
rescue StandardError => e
|
|
336
|
+
# e.g. a ProtocolError from encode — never die silently; sever this
|
|
337
|
+
# worker so its fallback takes over.
|
|
338
|
+
@logger.warn { "[Pgbus::Streamer::MasterHub] writer for worker #{id} failed: #{e.class}: #{e.message}" }
|
|
339
|
+
ensure
|
|
340
|
+
cleanup_worker(id)
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
def status_loop
|
|
344
|
+
last_status = nil
|
|
345
|
+
ticks_since_broadcast = 0
|
|
346
|
+
loop do
|
|
347
|
+
# A stop-signal wait instead of sleep, so #stop wakes the thread
|
|
348
|
+
# immediately even with a long status_interval.
|
|
349
|
+
break if @stop_signal.pop(timeout: @status_interval)
|
|
350
|
+
break unless running?
|
|
351
|
+
|
|
352
|
+
begin
|
|
353
|
+
healthy = listener_healthy?
|
|
354
|
+
ticks_since_broadcast += 1
|
|
355
|
+
next unless healthy != last_status || ticks_since_broadcast >= REBROADCAST_TICKS
|
|
356
|
+
|
|
357
|
+
broadcast_status(healthy)
|
|
358
|
+
last_status = healthy
|
|
359
|
+
ticks_since_broadcast = 0
|
|
360
|
+
rescue StandardError => e
|
|
361
|
+
@logger.warn { "[Pgbus::Streamer::MasterHub] status tick failed: #{e.class}: #{e.message}" }
|
|
362
|
+
end
|
|
363
|
+
end
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
def listener_healthy?
|
|
367
|
+
listener = @listener
|
|
368
|
+
!!(listener&.alive? && listener.connected?)
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
def broadcast_status(healthy)
|
|
372
|
+
frame = { "t" => "status", "healthy" => healthy }
|
|
373
|
+
entries = @table_mutex.synchronize { @workers.to_a }
|
|
374
|
+
entries.each { |id, entry| enqueue_frame(id, entry, frame, droppable: false) }
|
|
375
|
+
end
|
|
376
|
+
|
|
377
|
+
# Idempotent teardown for one worker — reachable from its reader's
|
|
378
|
+
# ensure, an eviction, and stop.
|
|
379
|
+
def cleanup_worker(id)
|
|
380
|
+
entry = @table_mutex.synchronize { @workers.delete(id) }
|
|
381
|
+
return unless entry
|
|
382
|
+
|
|
383
|
+
entry[:outbox_mutex].synchronize do
|
|
384
|
+
entry[:open] = false
|
|
385
|
+
entry[:outbox_cond].broadcast
|
|
386
|
+
end
|
|
387
|
+
close_quietly(entry[:sock])
|
|
388
|
+
release_queue_refs(id, entry[:subs].to_a)
|
|
389
|
+
end
|
|
390
|
+
|
|
391
|
+
# Decrement refcounts; UNLISTEN queues that hit zero (async — no
|
|
392
|
+
# correctness path waits on unlisten, mirroring remove_listening).
|
|
393
|
+
def release_queue_refs(id, queues)
|
|
394
|
+
released = @table_mutex.synchronize do
|
|
395
|
+
queues.select do |q|
|
|
396
|
+
refs = @queue_refs[q]
|
|
397
|
+
next false unless refs
|
|
398
|
+
|
|
399
|
+
refs.delete(id)
|
|
400
|
+
@queue_refs.delete(q) if refs.empty?
|
|
401
|
+
end
|
|
402
|
+
end
|
|
403
|
+
released.each { |q| @listener.remove_listening(q) }
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
def close_quietly(io)
|
|
407
|
+
io.close if io && !io.closed?
|
|
408
|
+
rescue IOError, Errno::EBADF
|
|
409
|
+
nil
|
|
410
|
+
end
|
|
411
|
+
end
|
|
412
|
+
end
|
|
413
|
+
end
|
|
414
|
+
end
|