pgbus 0.12.4 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/README.md +2 -1
- data/Rakefile +12 -1
- data/lib/pgbus/client.rb +15 -0
- data/lib/pgbus/configuration/capsule_dsl.rb +8 -0
- data/lib/pgbus/configuration.rb +36 -5
- data/lib/pgbus/dedicated_connection.rb +29 -2
- data/lib/pgbus/doctor.rb +36 -2
- data/lib/pgbus/event_bus/registry.rb +22 -0
- data/lib/pgbus/process/consumer.rb +57 -22
- data/lib/pgbus/process/notify_hub.rb +273 -0
- data/lib/pgbus/process/notify_listener.rb +47 -2
- data/lib/pgbus/process/supervisor.rb +146 -34
- data/lib/pgbus/process/wake_pipe.rb +129 -0
- data/lib/pgbus/process/wildcard_queue_resolver.rb +35 -0
- data/lib/pgbus/process/worker.rb +58 -35
- data/lib/pgbus/version.rb +1 -1
- metadata +4 -1
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Process
|
|
5
|
+
# Supervisor-owned LISTEN hub (issue #381, worker_notify_scope:
|
|
6
|
+
# :supervisor). Owns ONE NotifyListener — one direct PG connection for the
|
|
7
|
+
# whole host — on the union of every capsule's and consumer's queue
|
|
8
|
+
# channels, and fans wakes out to worker/consumer forks over per-fork
|
|
9
|
+
# pipes (see WakePipe for the fork side and the W/H/P byte protocol).
|
|
10
|
+
#
|
|
11
|
+
# Union: explicit capsule queues from config.workers, wildcard capsules
|
|
12
|
+
# via the shared WildcardQueueResolver, consumer queues via
|
|
13
|
+
# Registry#queue_names_for_topics — each gated by config.role_enabled?.
|
|
14
|
+
# Refreshed every REFRESH_INTERVAL_SECONDS from the supervisor's monitor
|
|
15
|
+
# tick so wildcard churn and late registry subscriptions converge; a
|
|
16
|
+
# transient gap is bounded by the forks' NOTIFY poll ceiling (15s), so
|
|
17
|
+
# under-listening degrades latency, never liveness.
|
|
18
|
+
#
|
|
19
|
+
# Routing: a NOTIFY for queue Q wakes every fork whose registered set
|
|
20
|
+
# contains Q, plus every wildcard fork. Over-waking costs one empty read;
|
|
21
|
+
# under-waking is the only failure mode that matters, so wildcard routing
|
|
22
|
+
# is unconditional.
|
|
23
|
+
#
|
|
24
|
+
# Status: healthy? = listener running + connected + delivering. Broadcast
|
|
25
|
+
# to all pipes on change and every STATUS_REBROADCAST_SECONDS (idempotent
|
|
26
|
+
# 1-byte writes), so a lost byte or a freshly full pipe can't wedge a fork
|
|
27
|
+
# in the wrong mode.
|
|
28
|
+
#
|
|
29
|
+
# Threading: register/deregister/tick/stop run on the supervisor's main
|
|
30
|
+
# loop; the wake callback runs on the listener's thread. The fork table is
|
|
31
|
+
# the only shared state — guarded by @table_mutex. Pipe writes are 1-byte
|
|
32
|
+
# write_nonblock calls (atomic well under PIPE_BUF) and never block: a
|
|
33
|
+
# full pipe already guarantees a pending readable byte, so the wake is
|
|
34
|
+
# skipped, and status repair rides the periodic rebroadcast.
|
|
35
|
+
class NotifyHub
|
|
36
|
+
REFRESH_INTERVAL_SECONDS = 30
|
|
37
|
+
STATUS_REBROADCAST_SECONDS = 5
|
|
38
|
+
RETRY_BASE_SECONDS = 5
|
|
39
|
+
RETRY_MAX_SECONDS = 300
|
|
40
|
+
|
|
41
|
+
def initialize(config:, listener_factory: nil, clock: nil, logger: Pgbus.logger)
|
|
42
|
+
@config = config
|
|
43
|
+
@logger = logger
|
|
44
|
+
@clock = clock || -> { ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) }
|
|
45
|
+
@listener_factory = listener_factory || default_listener_factory
|
|
46
|
+
@table_mutex = Mutex.new
|
|
47
|
+
@forks = {}
|
|
48
|
+
@listener = nil
|
|
49
|
+
@last_status = nil
|
|
50
|
+
@last_broadcast_at = nil
|
|
51
|
+
@last_refresh_at = nil
|
|
52
|
+
@retry_at = 0.0
|
|
53
|
+
@retry_backoff = RETRY_BASE_SECONDS
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def start
|
|
57
|
+
build_listener
|
|
58
|
+
@last_refresh_at = now
|
|
59
|
+
self
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def stop
|
|
63
|
+
stop_listener_quietly
|
|
64
|
+
@table_mutex.synchronize do
|
|
65
|
+
@forks.each_value { |entry| close_quietly(entry[:pipe]) }
|
|
66
|
+
@forks.clear
|
|
67
|
+
end
|
|
68
|
+
self
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Register a fork's wake pipe. +queues+ are PHYSICAL queue names (already
|
|
72
|
+
# prefixed); wildcard forks pass wildcard: true and are woken for every
|
|
73
|
+
# channel. The fork's current status byte is sent immediately so a
|
|
74
|
+
# freshly forked worker starts in the right polling mode.
|
|
75
|
+
def register_fork(pid:, queues:, pipe:, wildcard: false)
|
|
76
|
+
entry = { pipe: pipe, queues: Set.new(queues), wildcard: wildcard }
|
|
77
|
+
@table_mutex.synchronize { @forks[pid] = entry }
|
|
78
|
+
byte = status_byte
|
|
79
|
+
write_byte(entry, byte)
|
|
80
|
+
# The registration write is the broadcast baseline: without stamping
|
|
81
|
+
# it, the next tick would treat @last_status as unknown and re-send
|
|
82
|
+
# the same byte to every fork immediately.
|
|
83
|
+
@last_status = byte
|
|
84
|
+
@last_broadcast_at = now
|
|
85
|
+
self
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def deregister_fork(pid)
|
|
89
|
+
entry = @table_mutex.synchronize { @forks.delete(pid) }
|
|
90
|
+
close_quietly(entry[:pipe]) if entry
|
|
91
|
+
self
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Called ONLY inside a just-forked child: release the child's copies of
|
|
95
|
+
# the parent's hub resources — the LISTEN socket fd (without a libpq
|
|
96
|
+
# Terminate, see NotifyListener#close_inherited_socket!) and the fork
|
|
97
|
+
# table's pipe write ends. Never touches parent state (fork copied it).
|
|
98
|
+
def close_inherited!
|
|
99
|
+
@listener&.close_inherited_socket!
|
|
100
|
+
@listener = nil
|
|
101
|
+
@table_mutex.synchronize do
|
|
102
|
+
@forks.each_value { |entry| close_quietly(entry[:pipe]) }
|
|
103
|
+
@forks.clear
|
|
104
|
+
end
|
|
105
|
+
self
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# One supervisor monitor-loop beat: self-heal the listener, refresh the
|
|
109
|
+
# LISTEN union, keep fork status fresh.
|
|
110
|
+
def tick
|
|
111
|
+
ensure_listener
|
|
112
|
+
refresh_union_if_due
|
|
113
|
+
broadcast_status
|
|
114
|
+
self
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def healthy?
|
|
118
|
+
listener = @listener
|
|
119
|
+
!!(listener&.running? && listener.connected? && listener.delivering?)
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
private
|
|
123
|
+
|
|
124
|
+
attr_reader :config
|
|
125
|
+
|
|
126
|
+
def now
|
|
127
|
+
@clock.call
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def default_listener_factory
|
|
131
|
+
lambda do |physical_queues:, on_wake:|
|
|
132
|
+
NotifyListener.new(
|
|
133
|
+
physical_queues: physical_queues,
|
|
134
|
+
on_wake: on_wake,
|
|
135
|
+
connection_options: config.worker_notify_connection_options,
|
|
136
|
+
health_check_ms: (config.polling_interval * 1000).to_i.clamp(250, 5_000),
|
|
137
|
+
logger: @logger
|
|
138
|
+
)
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def build_listener
|
|
143
|
+
@listener = @listener_factory.call(
|
|
144
|
+
physical_queues: desired_physical_queues,
|
|
145
|
+
on_wake: method(:route)
|
|
146
|
+
)
|
|
147
|
+
@listener.start
|
|
148
|
+
rescue StandardError => e
|
|
149
|
+
@listener = nil
|
|
150
|
+
@logger.error { "[Pgbus::NotifyHub] listener failed to start: #{e.class}: #{e.message}" }
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Mirrors Worker#ensure_notify_listener: restart a dead listener thread
|
|
154
|
+
# on an exponential backoff so a persistent outage doesn't thrash a
|
|
155
|
+
# reconnect per supervisor tick.
|
|
156
|
+
def ensure_listener
|
|
157
|
+
return if @listener&.running?
|
|
158
|
+
return if now < @retry_at
|
|
159
|
+
|
|
160
|
+
stop_listener_quietly
|
|
161
|
+
build_listener
|
|
162
|
+
|
|
163
|
+
@retry_backoff = if @listener&.running?
|
|
164
|
+
RETRY_BASE_SECONDS
|
|
165
|
+
else
|
|
166
|
+
[@retry_backoff * 2, RETRY_MAX_SECONDS].min
|
|
167
|
+
end
|
|
168
|
+
@retry_at = now + @retry_backoff
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def stop_listener_quietly
|
|
172
|
+
@listener&.stop
|
|
173
|
+
rescue StandardError => e
|
|
174
|
+
@logger.warn { "[Pgbus::NotifyHub] failed to stop listener: #{e.class}: #{e.message}" }
|
|
175
|
+
ensure
|
|
176
|
+
@listener = nil
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def refresh_union_if_due
|
|
180
|
+
return unless @listener
|
|
181
|
+
return if @last_refresh_at && (now - @last_refresh_at) < REFRESH_INTERVAL_SECONDS
|
|
182
|
+
|
|
183
|
+
@last_refresh_at = now
|
|
184
|
+
desired = desired_physical_queues.to_set
|
|
185
|
+
current = @listener.listening_to.to_set { |channel| channel_to_physical(channel) }
|
|
186
|
+
(desired - current).each { |q| @listener.add_queue(q) }
|
|
187
|
+
(current - desired).each { |q| @listener.remove_queue(q) }
|
|
188
|
+
rescue StandardError => e
|
|
189
|
+
@logger.warn { "[Pgbus::NotifyHub] union refresh failed: #{e.class}: #{e.message}" }
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def broadcast_status(status = status_byte)
|
|
193
|
+
rebroadcast_due = @last_broadcast_at.nil? || (now - @last_broadcast_at) >= STATUS_REBROADCAST_SECONDS
|
|
194
|
+
return if status == @last_status && !rebroadcast_due
|
|
195
|
+
|
|
196
|
+
entries = @table_mutex.synchronize { @forks.values.dup }
|
|
197
|
+
entries.each { |entry| write_byte(entry, status) }
|
|
198
|
+
@last_status = status
|
|
199
|
+
@last_broadcast_at = now
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def status_byte
|
|
203
|
+
healthy? ? WakePipe::HEALTHY : WakePipe::DEGRADED
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# Runs on the LISTENER thread (NotifyListener on_wake callback).
|
|
207
|
+
def route(channel)
|
|
208
|
+
physical = channel_to_physical(channel)
|
|
209
|
+
entries = @table_mutex.synchronize { @forks.values.dup }
|
|
210
|
+
entries.each do |entry|
|
|
211
|
+
write_byte(entry, WakePipe::WAKE) if entry[:wildcard] || entry[:queues].include?(physical)
|
|
212
|
+
end
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# Never block, never raise: a full pipe already guarantees a pending
|
|
216
|
+
# readable byte (wake semantics are level-triggered), and a dead pipe's
|
|
217
|
+
# fork is on its way through reap → deregister_fork.
|
|
218
|
+
def write_byte(entry, byte)
|
|
219
|
+
entry[:pipe].write_nonblock(byte)
|
|
220
|
+
rescue IO::WaitWritable, Errno::EPIPE, IOError, Errno::EBADF
|
|
221
|
+
nil
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def desired_physical_queues
|
|
225
|
+
names = []
|
|
226
|
+
names.concat(capsule_queue_names) if config.role_enabled?(:workers)
|
|
227
|
+
names.concat(consumer_queue_names) if config.role_enabled?(:consumers)
|
|
228
|
+
names.uniq.map { |q| config.queue_name(q) }
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def capsule_queue_names
|
|
232
|
+
names = []
|
|
233
|
+
wildcard = false
|
|
234
|
+
Array(config.workers).each do |worker_config|
|
|
235
|
+
queues = worker_config[:queues] || worker_config["queues"] || [config.default_queue]
|
|
236
|
+
Array(queues).each do |q|
|
|
237
|
+
q.to_s == "*" ? wildcard = true : names << q.to_s
|
|
238
|
+
end
|
|
239
|
+
end
|
|
240
|
+
names.concat(resolve_wildcard) if wildcard
|
|
241
|
+
names
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# A resolver failure (DB blip at boot) degrades to the explicit union;
|
|
245
|
+
# wildcard queues rejoin on the next successful refresh, and the forks'
|
|
246
|
+
# poll ceiling keeps them processing meanwhile.
|
|
247
|
+
def resolve_wildcard
|
|
248
|
+
WildcardQueueResolver.resolve(config: config)
|
|
249
|
+
rescue StandardError => e
|
|
250
|
+
@logger.warn { "[Pgbus::NotifyHub] wildcard resolution failed: #{e.class}: #{e.message}" }
|
|
251
|
+
[]
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
def consumer_queue_names
|
|
255
|
+
registry = Pgbus::EventBus::Registry.instance
|
|
256
|
+
Array(config.event_consumers).flat_map do |consumer_config|
|
|
257
|
+
topics = consumer_config[:topics] || consumer_config["topics"] || []
|
|
258
|
+
registry.queue_names_for_topics(Array(topics))
|
|
259
|
+
end
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
def channel_to_physical(channel)
|
|
263
|
+
NotifyListener.physical_for(channel)
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def close_quietly(pipe)
|
|
267
|
+
pipe.close if pipe && !pipe.closed?
|
|
268
|
+
rescue IOError, Errno::EBADF
|
|
269
|
+
nil
|
|
270
|
+
end
|
|
271
|
+
end
|
|
272
|
+
end
|
|
273
|
+
end
|
|
@@ -42,6 +42,14 @@ module Pgbus
|
|
|
42
42
|
CHANNEL_PREFIX = "pgmq.q_"
|
|
43
43
|
CHANNEL_SUFFIX = ".INSERT"
|
|
44
44
|
|
|
45
|
+
# Inverse of #channel_for: map a NOTIFY channel back to the physical
|
|
46
|
+
# queue name. Class-level because the channel format is owned here —
|
|
47
|
+
# NotifyHub (wake routing + union refresh) and Worker (queue-set sync)
|
|
48
|
+
# both consume it.
|
|
49
|
+
def self.physical_for(channel)
|
|
50
|
+
channel.delete_prefix(CHANNEL_PREFIX).delete_suffix(CHANNEL_SUFFIX)
|
|
51
|
+
end
|
|
52
|
+
|
|
45
53
|
RECONNECT_BACKOFF_SECONDS = 0.5
|
|
46
54
|
|
|
47
55
|
# Grace added to one health-check cycle when #stop joins the listener
|
|
@@ -70,6 +78,16 @@ module Pgbus
|
|
|
70
78
|
@state_mutex.synchronize { @listening_to.dup }
|
|
71
79
|
end
|
|
72
80
|
|
|
81
|
+
# Whether a live PG connection is currently published. running? stays
|
|
82
|
+
# true during a reconnect (the thread is alive, looping in reconnect!),
|
|
83
|
+
# so this is the signal that distinguishes "parked in wait_for_notify"
|
|
84
|
+
# from "between connections". The supervisor NotifyHub (issue #381)
|
|
85
|
+
# consults it to broadcast degraded status to forks the moment the
|
|
86
|
+
# shared connection drops, and healthy again once it is rebuilt.
|
|
87
|
+
def connected?
|
|
88
|
+
@state_mutex.synchronize { !@conn.nil? }
|
|
89
|
+
end
|
|
90
|
+
|
|
73
91
|
# Whether the start-time self-probe confirmed this connection can actually
|
|
74
92
|
# receive a NOTIFY. False when a transaction-mode pooler or replica
|
|
75
93
|
# silently drops LISTEN: the thread is still alive (running? == true) but
|
|
@@ -125,6 +143,30 @@ module Pgbus
|
|
|
125
143
|
@state_mutex.synchronize { @running }
|
|
126
144
|
end
|
|
127
145
|
|
|
146
|
+
# Called ONLY inside a just-forked child (issue #381 hub hygiene): drop
|
|
147
|
+
# this process's copy of the LISTEN socket fd WITHOUT PQfinish — #close
|
|
148
|
+
# would send a libpq Terminate over the socket shared with the parent,
|
|
149
|
+
# killing the parent's LISTEN session. Closing the IO wrapper just
|
|
150
|
+
# closes the child's fd. The listener thread does not exist in the
|
|
151
|
+
# child (fork copies only the calling thread), so there is no
|
|
152
|
+
# concurrent owner and the single-owner rule (#375) does not apply.
|
|
153
|
+
def close_inherited_socket!
|
|
154
|
+
conn = @state_mutex.synchronize do
|
|
155
|
+
c = @conn
|
|
156
|
+
@conn = nil
|
|
157
|
+
@running = false
|
|
158
|
+
c
|
|
159
|
+
end
|
|
160
|
+
conn&.socket_io&.close
|
|
161
|
+
rescue StandardError => e
|
|
162
|
+
# Best-effort (a lingering fd copy is benign until the parent dies),
|
|
163
|
+
# but never silent: the child keeps booting either way.
|
|
164
|
+
@logger.warn do
|
|
165
|
+
"[Pgbus::NotifyListener] inherited socket cleanup failed: #{e.class}: #{e.message}"
|
|
166
|
+
end
|
|
167
|
+
nil
|
|
168
|
+
end
|
|
169
|
+
|
|
128
170
|
private
|
|
129
171
|
|
|
130
172
|
def run_loop
|
|
@@ -190,8 +232,11 @@ module Pgbus
|
|
|
190
232
|
return reconnect! unless conn
|
|
191
233
|
|
|
192
234
|
timeout_s = @health_check_ms / 1000.0
|
|
193
|
-
got_notify = conn.wait_for_notify(timeout_s) do |
|
|
194
|
-
|
|
235
|
+
got_notify = conn.wait_for_notify(timeout_s) do |channel, _pid, _payload|
|
|
236
|
+
# The channel rides along so a hub caller (issue #381) can route the
|
|
237
|
+
# wake to the fork(s) reading that queue; fork-owned listeners take
|
|
238
|
+
# ->(_channel) and ignore it.
|
|
239
|
+
@on_wake.call(channel)
|
|
195
240
|
end
|
|
196
241
|
# Skip the keepalive when a stop landed during the wait: the loop is
|
|
197
242
|
# about to exit and close this connection anyway, so the round-trip
|
|
@@ -21,23 +21,29 @@ module Pgbus
|
|
|
21
21
|
RESTART_BACKOFF_MAX = 60
|
|
22
22
|
|
|
23
23
|
attr_reader :config
|
|
24
|
+
# The host-level shared LISTEN hub (issue #381). nil under :fork scope,
|
|
25
|
+
# when notify wakeups are off entirely, or when no worker/consumer role
|
|
26
|
+
# is enabled. Readable as a test seam.
|
|
27
|
+
attr_reader :notify_hub
|
|
24
28
|
# forks is readable everywhere; the writer exists only so tests can seed a
|
|
25
29
|
# known set of children before exercising the reap/watchdog paths (in
|
|
26
30
|
# production forks is populated by fork_* as children spawn).
|
|
27
31
|
attr_accessor :forks
|
|
28
32
|
|
|
29
|
-
# The child-fork bookkeeping (`forks`, `pending_restarts`)
|
|
30
|
-
# `shutting_down` flag accept injected seeds so tests
|
|
31
|
-
# monitor/reap loops from a known state without poking
|
|
32
|
-
# default to the empty/false values production always
|
|
33
|
+
# The child-fork bookkeeping (`forks`, `pending_restarts`), the
|
|
34
|
+
# `shutting_down` flag, and `notify_hub` accept injected seeds so tests
|
|
35
|
+
# can drive the monitor/reap loops from a known state without poking
|
|
36
|
+
# private ivars. All default to the empty/false values production always
|
|
37
|
+
# starts from.
|
|
33
38
|
def initialize(config: Pgbus.configuration, forks: {}, shutting_down: false,
|
|
34
|
-
pending_restarts: [], last_watchdog_at: nil)
|
|
39
|
+
pending_restarts: [], last_watchdog_at: nil, notify_hub: nil)
|
|
35
40
|
@config = config
|
|
36
41
|
@forks = forks
|
|
37
42
|
@shutting_down = shutting_down
|
|
38
43
|
@last_watchdog_at = last_watchdog_at || monotonic_now
|
|
39
44
|
@pending_restarts = pending_restarts
|
|
40
45
|
@crash_counts = Hash.new(0)
|
|
46
|
+
@notify_hub = notify_hub
|
|
41
47
|
end
|
|
42
48
|
|
|
43
49
|
def shutting_down?
|
|
@@ -84,6 +90,12 @@ module Pgbus
|
|
|
84
90
|
# genuinely-fatal finding; :report only logs. Off by default.
|
|
85
91
|
run_doctor_preflight unless config.doctor_on_boot.nil?
|
|
86
92
|
|
|
93
|
+
# Host-level shared LISTEN (issue #381): under :supervisor scope, ONE
|
|
94
|
+
# NotifyListener lives here and forks are woken over pipes — started
|
|
95
|
+
# before any child forks so every fork_worker/fork_consumer can hand
|
|
96
|
+
# its child a wake pipe.
|
|
97
|
+
start_notify_hub
|
|
98
|
+
|
|
87
99
|
boot_processes
|
|
88
100
|
monitor_loop
|
|
89
101
|
ensure
|
|
@@ -122,7 +134,9 @@ module Pgbus
|
|
|
122
134
|
"[Pgbus] boot: pgmq_schema_mode=#{config.pgmq_schema_mode} pgmq_version=#{installed_pgmq_version}"
|
|
123
135
|
end
|
|
124
136
|
Pgbus.logger.info do
|
|
125
|
-
"[Pgbus] boot: listen_notify=#{config.listen_notify}
|
|
137
|
+
"[Pgbus] boot: listen_notify=#{config.listen_notify} " \
|
|
138
|
+
"worker_notify_wakeup=#{config.worker_notify_wakeup?} " \
|
|
139
|
+
"worker_notify_scope=#{config.worker_notify_scope}"
|
|
126
140
|
end
|
|
127
141
|
Pgbus.logger.info { "[Pgbus] boot: roles=#{enabled_roles.join(",")}" }
|
|
128
142
|
log_capsule_banner
|
|
@@ -250,17 +264,18 @@ module Pgbus
|
|
|
250
264
|
# iteration, the parent drains the reader in monitor_loop. This lets
|
|
251
265
|
# the watchdog detect a wedged worker without the database.
|
|
252
266
|
liveness_reader, liveness_writer = IO.pipe
|
|
267
|
+
# Wake channel, opposite direction (issue #381): the NotifyHub writes
|
|
268
|
+
# W/H/P bytes, the child's WakePipe watcher reads them. Only under
|
|
269
|
+
# :supervisor scope (hub present).
|
|
270
|
+
wake_reader, wake_writer = IO.pipe if @notify_hub
|
|
253
271
|
|
|
254
272
|
pid = fork do
|
|
255
|
-
# Child owns the writer
|
|
256
|
-
#
|
|
257
|
-
#
|
|
258
|
-
#
|
|
259
|
-
# are never inherited: the parent closes each writer below before
|
|
260
|
-
# forking the next worker, so a dead sibling's pipe still reaches
|
|
261
|
-
# EOF correctly.) Finally close this fork's own reader copy.
|
|
262
|
-
close_inherited_liveness_readers
|
|
273
|
+
# Child owns the liveness writer + wake reader; close this fork's
|
|
274
|
+
# own copies of the parent-side ends. Sibling pipe ends and the
|
|
275
|
+
# hub's LISTEN socket are released in setup_child_process, which
|
|
276
|
+
# every child type runs.
|
|
263
277
|
liveness_reader.close
|
|
278
|
+
wake_writer&.close
|
|
264
279
|
restore_signals
|
|
265
280
|
setup_child_process
|
|
266
281
|
load_rails_app
|
|
@@ -269,7 +284,7 @@ module Pgbus
|
|
|
269
284
|
queues: queues, threads: threads, config: config,
|
|
270
285
|
single_active_consumer: single_active, consumer_priority: priority,
|
|
271
286
|
execution_mode: exec_mode, group_mode: grp_mode,
|
|
272
|
-
liveness_pipe: liveness_writer
|
|
287
|
+
liveness_pipe: liveness_writer, wake_pipe: wake_reader
|
|
273
288
|
)
|
|
274
289
|
worker.run
|
|
275
290
|
end
|
|
@@ -277,24 +292,50 @@ module Pgbus
|
|
|
277
292
|
unless pid
|
|
278
293
|
close_pipe(liveness_reader)
|
|
279
294
|
close_pipe(liveness_writer)
|
|
295
|
+
close_pipe(wake_reader)
|
|
296
|
+
close_pipe(wake_writer)
|
|
280
297
|
Pgbus.logger.error { "[Pgbus] Failed to fork worker for queues=#{queues.join(",")}" }
|
|
281
298
|
return
|
|
282
299
|
end
|
|
283
300
|
|
|
284
|
-
# Parent keeps the reader
|
|
285
|
-
# EOF once
|
|
301
|
+
# Parent keeps the liveness reader + wake writer, discards its copies
|
|
302
|
+
# of the child-side ends so each pipe reaches EOF once its sole owner
|
|
303
|
+
# closes.
|
|
286
304
|
close_pipe(liveness_writer)
|
|
305
|
+
close_pipe(wake_reader)
|
|
306
|
+
register_fork_with_hub(pid, wake_writer, queues)
|
|
287
307
|
@forks[pid] = {
|
|
288
308
|
type: :worker, config: worker_config, slot: slot, spawned_at: monotonic_now,
|
|
289
|
-
liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false
|
|
309
|
+
liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false,
|
|
310
|
+
wake_writer: wake_writer
|
|
290
311
|
}
|
|
291
312
|
Pgbus.logger.info { "[Pgbus] Forked worker pid=#{pid} queues=#{queues.join(",")} mode=#{exec_mode}" }
|
|
292
313
|
rescue Errno::EAGAIN, Errno::ENOMEM => e
|
|
293
314
|
close_pipe(liveness_reader)
|
|
294
315
|
close_pipe(liveness_writer)
|
|
316
|
+
close_pipe(wake_reader)
|
|
317
|
+
close_pipe(wake_writer)
|
|
295
318
|
ErrorReporter.report(e, { action: "fork_worker", queues: queues })
|
|
296
319
|
end
|
|
297
320
|
|
|
321
|
+
# Hand the hub a worker fork's routing entry: explicit queues as
|
|
322
|
+
# physical names, "*" as the unconditional wildcard flag (the hub wakes
|
|
323
|
+
# wildcard forks for every channel, so the fork's own resolved set never
|
|
324
|
+
# needs to be reported upstream). Registration must never abort the fork
|
|
325
|
+
# bookkeeping that follows it — queue_name can raise on a malformed
|
|
326
|
+
# name — so on error the fork registers with an empty set and rides its
|
|
327
|
+
# poll ceiling (symmetric with register_consumer_with_hub).
|
|
328
|
+
def register_fork_with_hub(pid, wake_writer, queues)
|
|
329
|
+
return unless @notify_hub && wake_writer
|
|
330
|
+
|
|
331
|
+
wildcard = queues.include?("*")
|
|
332
|
+
physical = queues.reject { |q| q == "*" }.map { |q| config.queue_name(q) }
|
|
333
|
+
@notify_hub.register_fork(pid: pid, queues: physical, wildcard: wildcard, pipe: wake_writer)
|
|
334
|
+
rescue StandardError => e
|
|
335
|
+
ErrorReporter.report(e, { action: "register_fork_with_hub", queues: queues })
|
|
336
|
+
@notify_hub.register_fork(pid: pid, queues: [], wildcard: false, pipe: wake_writer)
|
|
337
|
+
end
|
|
338
|
+
|
|
298
339
|
def fork_dispatcher
|
|
299
340
|
pid = fork do
|
|
300
341
|
restore_signals
|
|
@@ -386,7 +427,9 @@ module Pgbus
|
|
|
386
427
|
end
|
|
387
428
|
|
|
388
429
|
def fork_consumer(consumer_config, slot: nil)
|
|
389
|
-
|
|
430
|
+
# Array() so a consumer entry without :topics can't NoMethodError the
|
|
431
|
+
# supervisor on the topics.join log lines below.
|
|
432
|
+
topics = Array(consumer_config[:topics])
|
|
390
433
|
threads = consumer_config[:threads] || 3
|
|
391
434
|
|
|
392
435
|
# OS-level liveness channel: the consumer writes a byte each loop
|
|
@@ -394,41 +437,66 @@ module Pgbus
|
|
|
394
437
|
# watchdog detect a wedged consumer without the database (issue #274),
|
|
395
438
|
# exactly as fork_worker does for workers.
|
|
396
439
|
liveness_reader, liveness_writer = IO.pipe
|
|
440
|
+
# Wake channel from the NotifyHub (issue #381), as in fork_worker.
|
|
441
|
+
wake_reader, wake_writer = IO.pipe if @notify_hub
|
|
397
442
|
|
|
398
443
|
pid = fork do
|
|
399
|
-
# Child owns the writer
|
|
400
|
-
#
|
|
401
|
-
# Consumer (see fork_worker for the full rationale).
|
|
402
|
-
close_inherited_liveness_readers
|
|
444
|
+
# Child owns the liveness writer + wake reader; close this fork's
|
|
445
|
+
# own copies of the parent-side ends (see fork_worker).
|
|
403
446
|
liveness_reader.close
|
|
447
|
+
wake_writer&.close
|
|
404
448
|
restore_signals
|
|
405
449
|
setup_child_process
|
|
406
450
|
load_rails_app
|
|
407
|
-
consumer = Consumer.new(topics: topics, threads: threads, config: config,
|
|
451
|
+
consumer = Consumer.new(topics: topics, threads: threads, config: config,
|
|
452
|
+
liveness_pipe: liveness_writer, wake_pipe: wake_reader)
|
|
408
453
|
consumer.run
|
|
409
454
|
end
|
|
410
455
|
|
|
411
456
|
unless pid
|
|
412
457
|
close_pipe(liveness_reader)
|
|
413
458
|
close_pipe(liveness_writer)
|
|
459
|
+
close_pipe(wake_reader)
|
|
460
|
+
close_pipe(wake_writer)
|
|
414
461
|
Pgbus.logger.error { "[Pgbus] Failed to fork consumer for topics=#{topics.join(",")}" }
|
|
415
462
|
return
|
|
416
463
|
end
|
|
417
464
|
|
|
418
|
-
# Parent keeps the reader
|
|
419
|
-
#
|
|
465
|
+
# Parent keeps the liveness reader + wake writer, discards its copies
|
|
466
|
+
# of the child-side ends.
|
|
420
467
|
close_pipe(liveness_writer)
|
|
468
|
+
close_pipe(wake_reader)
|
|
469
|
+
register_consumer_with_hub(pid, wake_writer, topics)
|
|
421
470
|
@forks[pid] = {
|
|
422
471
|
type: :consumer, config: consumer_config, slot: slot, spawned_at: monotonic_now,
|
|
423
|
-
liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false
|
|
472
|
+
liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false,
|
|
473
|
+
wake_writer: wake_writer
|
|
424
474
|
}
|
|
425
475
|
Pgbus.logger.info { "[Pgbus] Forked consumer pid=#{pid} topics=#{topics.join(",")}" }
|
|
426
476
|
rescue Errno::EAGAIN, Errno::ENOMEM => e
|
|
427
477
|
close_pipe(liveness_reader)
|
|
428
478
|
close_pipe(liveness_writer)
|
|
479
|
+
close_pipe(wake_reader)
|
|
480
|
+
close_pipe(wake_writer)
|
|
429
481
|
ErrorReporter.report(e, { action: "fork_consumer", topics: topics })
|
|
430
482
|
end
|
|
431
483
|
|
|
484
|
+
# A consumer's routing entry mirrors Consumer#setup_subscriptions: the
|
|
485
|
+
# registry derives the queue set from the topic list. Registry lookups
|
|
486
|
+
# never abort a fork — on error the fork registers with an empty set and
|
|
487
|
+
# rides its poll ceiling until the next supervisor restart.
|
|
488
|
+
def register_consumer_with_hub(pid, wake_writer, topics)
|
|
489
|
+
return unless @notify_hub && wake_writer
|
|
490
|
+
|
|
491
|
+
physical = EventBus::Registry.instance
|
|
492
|
+
.queue_names_for_topics(Array(topics))
|
|
493
|
+
.map { |q| config.queue_name(q) }
|
|
494
|
+
@notify_hub.register_fork(pid: pid, queues: physical, wildcard: false, pipe: wake_writer)
|
|
495
|
+
rescue StandardError => e
|
|
496
|
+
ErrorReporter.report(e, { action: "register_consumer_with_hub", topics: topics })
|
|
497
|
+
@notify_hub.register_fork(pid: pid, queues: [], wildcard: false, pipe: wake_writer)
|
|
498
|
+
end
|
|
499
|
+
|
|
432
500
|
def boot_outbox_poller
|
|
433
501
|
return unless config.outbox_enabled
|
|
434
502
|
|
|
@@ -465,6 +533,9 @@ module Pgbus
|
|
|
465
533
|
unless @shutting_down
|
|
466
534
|
process_pending_restarts
|
|
467
535
|
check_stalled_workers
|
|
536
|
+
# One hub beat per monitor pass: listener self-heal, LISTEN union
|
|
537
|
+
# refresh, and fork status broadcast (issue #381).
|
|
538
|
+
@notify_hub&.tick
|
|
468
539
|
end
|
|
469
540
|
interruptible_sleep(FORK_WAIT)
|
|
470
541
|
end
|
|
@@ -480,8 +551,11 @@ module Pgbus
|
|
|
480
551
|
|
|
481
552
|
# Close the liveness reader as the fork leaves @forks so a crash-loop
|
|
482
553
|
# (restart deferred up to RESTART_BACKOFF_MAX) can't leak an FD per
|
|
483
|
-
# crash. Scrub the
|
|
554
|
+
# crash. Scrub the keys so a closed IO never rides into a restart.
|
|
555
|
+
# The wake writer is closed by the hub's deregister (same IO object).
|
|
484
556
|
close_pipe(info.delete(:liveness_reader))
|
|
557
|
+
info.delete(:wake_writer)
|
|
558
|
+
@notify_hub&.deregister_fork(pid)
|
|
485
559
|
|
|
486
560
|
if @shutting_down
|
|
487
561
|
Pgbus.logger.info { "[Pgbus] Child #{info[:type]} pid=#{pid} exited (status=#{status.exitstatus})" }
|
|
@@ -655,6 +729,13 @@ module Pgbus
|
|
|
655
729
|
end
|
|
656
730
|
|
|
657
731
|
def setup_child_process
|
|
732
|
+
# Every child type (worker, consumer, dispatcher, scheduler, outbox
|
|
733
|
+
# poller) releases its copies of the parent's per-fork resources —
|
|
734
|
+
# a dispatcher child holding a sibling worker's wake WRITER would
|
|
735
|
+
# keep that sibling's pipe from ever reaching EOF after the
|
|
736
|
+
# supervisor dies, pinning the sibling to the 15s NOTIFY ceiling
|
|
737
|
+
# with no wake source (issue #381 review).
|
|
738
|
+
close_inherited_parent_resources
|
|
658
739
|
# Reset the PGMQ client so this forked process gets a fresh
|
|
659
740
|
# PG::Connection instead of inheriting the parent's (which is
|
|
660
741
|
# in undefined state post-fork and not thread-safe to share).
|
|
@@ -758,11 +839,40 @@ module Pgbus
|
|
|
758
839
|
nil
|
|
759
840
|
end
|
|
760
841
|
|
|
761
|
-
#
|
|
762
|
-
# parent's FD table
|
|
763
|
-
#
|
|
764
|
-
|
|
765
|
-
|
|
842
|
+
# Called only inside a just-forked child: close every sibling pipe end
|
|
843
|
+
# inherited from the parent's FD table (liveness readers AND wake
|
|
844
|
+
# writers), and release the child's copy of the NotifyHub's LISTEN
|
|
845
|
+
# socket without a libpq Terminate (PQfinish would kill the PARENT's
|
|
846
|
+
# session over the shared socket — see
|
|
847
|
+
# NotifyListener#close_inherited_socket!).
|
|
848
|
+
def close_inherited_parent_resources
|
|
849
|
+
@forks.each_value do |info|
|
|
850
|
+
close_pipe(info[:liveness_reader])
|
|
851
|
+
close_pipe(info[:wake_writer])
|
|
852
|
+
end
|
|
853
|
+
@notify_hub&.close_inherited!
|
|
854
|
+
@notify_hub = nil
|
|
855
|
+
end
|
|
856
|
+
|
|
857
|
+
# Build the host-level shared LISTEN hub (issue #381). Only under
|
|
858
|
+
# :supervisor scope, with notify wakeups on, and with at least one role
|
|
859
|
+
# that reads queues. A hub that fails to start degrades to no hub: forks
|
|
860
|
+
# get no wake pipe and fall back to fast polling, exactly like a failed
|
|
861
|
+
# per-fork listener under :fork scope.
|
|
862
|
+
def start_notify_hub
|
|
863
|
+
return unless config.worker_notify_wakeup?
|
|
864
|
+
return unless config.worker_notify_scope == :supervisor
|
|
865
|
+
return unless config.role_enabled?(:workers) || config.role_enabled?(:consumers)
|
|
866
|
+
|
|
867
|
+
hub = NotifyHub.new(config: config)
|
|
868
|
+
hub.start
|
|
869
|
+
@notify_hub = hub
|
|
870
|
+
rescue StandardError => e
|
|
871
|
+
@notify_hub = nil
|
|
872
|
+
ErrorReporter.report(e, { action: "start_notify_hub" })
|
|
873
|
+
Pgbus.logger.error do
|
|
874
|
+
"[Pgbus] NotifyHub failed to start — forks fall back to polling: #{e.class}: #{e.message}"
|
|
875
|
+
end
|
|
766
876
|
end
|
|
767
877
|
|
|
768
878
|
def shutdown
|
|
@@ -778,9 +888,11 @@ module Pgbus
|
|
|
778
888
|
signal_children("KILL") unless @forks.empty?
|
|
779
889
|
|
|
780
890
|
# Close any liveness readers still open on un-reaped children so the
|
|
781
|
-
# supervisor never leaks FDs across a restart of itself.
|
|
891
|
+
# supervisor never leaks FDs across a restart of itself. (Wake writers
|
|
892
|
+
# are closed by the hub's stop below.)
|
|
782
893
|
@forks.each_value { |info| close_pipe(info[:liveness_reader]) }
|
|
783
894
|
|
|
895
|
+
@notify_hub&.stop
|
|
784
896
|
@health_server&.stop
|
|
785
897
|
@heartbeat&.stop
|
|
786
898
|
restore_signals
|