pgbus 0.12.3 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/README.md +2 -1
- data/Rakefile +12 -1
- data/lib/pgbus/client.rb +15 -0
- data/lib/pgbus/configuration/capsule_dsl.rb +8 -0
- data/lib/pgbus/configuration.rb +36 -5
- data/lib/pgbus/dedicated_connection.rb +29 -2
- data/lib/pgbus/doctor.rb +36 -2
- data/lib/pgbus/event_bus/registry.rb +22 -0
- data/lib/pgbus/process/consumer.rb +57 -22
- data/lib/pgbus/process/notify_hub.rb +273 -0
- data/lib/pgbus/process/notify_listener.rb +103 -26
- data/lib/pgbus/process/supervisor.rb +146 -34
- data/lib/pgbus/process/wake_pipe.rb +129 -0
- data/lib/pgbus/process/wildcard_queue_resolver.rb +35 -0
- data/lib/pgbus/process/worker.rb +58 -35
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/streamer/listener.rb +49 -30
- metadata +4 -1
|
@@ -21,23 +21,29 @@ module Pgbus
|
|
|
21
21
|
RESTART_BACKOFF_MAX = 60
|
|
22
22
|
|
|
23
23
|
attr_reader :config
|
|
24
|
+
# The host-level shared LISTEN hub (issue #381). nil under :fork scope,
|
|
25
|
+
# when notify wakeups are off entirely, or when no worker/consumer role
|
|
26
|
+
# is enabled. Readable as a test seam.
|
|
27
|
+
attr_reader :notify_hub
|
|
24
28
|
# forks is readable everywhere; the writer exists only so tests can seed a
|
|
25
29
|
# known set of children before exercising the reap/watchdog paths (in
|
|
26
30
|
# production forks is populated by fork_* as children spawn).
|
|
27
31
|
attr_accessor :forks
|
|
28
32
|
|
|
29
|
-
# The child-fork bookkeeping (`forks`, `pending_restarts`)
|
|
30
|
-
# `shutting_down` flag accept injected seeds so tests
|
|
31
|
-
# monitor/reap loops from a known state without poking
|
|
32
|
-
# default to the empty/false values production always
|
|
33
|
+
# The child-fork bookkeeping (`forks`, `pending_restarts`), the
|
|
34
|
+
# `shutting_down` flag, and `notify_hub` accept injected seeds so tests
|
|
35
|
+
# can drive the monitor/reap loops from a known state without poking
|
|
36
|
+
# private ivars. All default to the empty/false values production always
|
|
37
|
+
# starts from.
|
|
33
38
|
def initialize(config: Pgbus.configuration, forks: {}, shutting_down: false,
|
|
34
|
-
pending_restarts: [], last_watchdog_at: nil)
|
|
39
|
+
pending_restarts: [], last_watchdog_at: nil, notify_hub: nil)
|
|
35
40
|
@config = config
|
|
36
41
|
@forks = forks
|
|
37
42
|
@shutting_down = shutting_down
|
|
38
43
|
@last_watchdog_at = last_watchdog_at || monotonic_now
|
|
39
44
|
@pending_restarts = pending_restarts
|
|
40
45
|
@crash_counts = Hash.new(0)
|
|
46
|
+
@notify_hub = notify_hub
|
|
41
47
|
end
|
|
42
48
|
|
|
43
49
|
def shutting_down?
|
|
@@ -84,6 +90,12 @@ module Pgbus
|
|
|
84
90
|
# genuinely-fatal finding; :report only logs. Off by default.
|
|
85
91
|
run_doctor_preflight unless config.doctor_on_boot.nil?
|
|
86
92
|
|
|
93
|
+
# Host-level shared LISTEN (issue #381): under :supervisor scope, ONE
|
|
94
|
+
# NotifyListener lives here and forks are woken over pipes — started
|
|
95
|
+
# before any child forks so every fork_worker/fork_consumer can hand
|
|
96
|
+
# its child a wake pipe.
|
|
97
|
+
start_notify_hub
|
|
98
|
+
|
|
87
99
|
boot_processes
|
|
88
100
|
monitor_loop
|
|
89
101
|
ensure
|
|
@@ -122,7 +134,9 @@ module Pgbus
|
|
|
122
134
|
"[Pgbus] boot: pgmq_schema_mode=#{config.pgmq_schema_mode} pgmq_version=#{installed_pgmq_version}"
|
|
123
135
|
end
|
|
124
136
|
Pgbus.logger.info do
|
|
125
|
-
"[Pgbus] boot: listen_notify=#{config.listen_notify}
|
|
137
|
+
"[Pgbus] boot: listen_notify=#{config.listen_notify} " \
|
|
138
|
+
"worker_notify_wakeup=#{config.worker_notify_wakeup?} " \
|
|
139
|
+
"worker_notify_scope=#{config.worker_notify_scope}"
|
|
126
140
|
end
|
|
127
141
|
Pgbus.logger.info { "[Pgbus] boot: roles=#{enabled_roles.join(",")}" }
|
|
128
142
|
log_capsule_banner
|
|
@@ -250,17 +264,18 @@ module Pgbus
|
|
|
250
264
|
# iteration, the parent drains the reader in monitor_loop. This lets
|
|
251
265
|
# the watchdog detect a wedged worker without the database.
|
|
252
266
|
liveness_reader, liveness_writer = IO.pipe
|
|
267
|
+
# Wake channel, opposite direction (issue #381): the NotifyHub writes
|
|
268
|
+
# W/H/P bytes, the child's WakePipe watcher reads them. Only under
|
|
269
|
+
# :supervisor scope (hub present).
|
|
270
|
+
wake_reader, wake_writer = IO.pipe if @notify_hub
|
|
253
271
|
|
|
254
272
|
pid = fork do
|
|
255
|
-
# Child owns the writer
|
|
256
|
-
#
|
|
257
|
-
#
|
|
258
|
-
#
|
|
259
|
-
# are never inherited: the parent closes each writer below before
|
|
260
|
-
# forking the next worker, so a dead sibling's pipe still reaches
|
|
261
|
-
# EOF correctly.) Finally close this fork's own reader copy.
|
|
262
|
-
close_inherited_liveness_readers
|
|
273
|
+
# Child owns the liveness writer + wake reader; close this fork's
|
|
274
|
+
# own copies of the parent-side ends. Sibling pipe ends and the
|
|
275
|
+
# hub's LISTEN socket are released in setup_child_process, which
|
|
276
|
+
# every child type runs.
|
|
263
277
|
liveness_reader.close
|
|
278
|
+
wake_writer&.close
|
|
264
279
|
restore_signals
|
|
265
280
|
setup_child_process
|
|
266
281
|
load_rails_app
|
|
@@ -269,7 +284,7 @@ module Pgbus
|
|
|
269
284
|
queues: queues, threads: threads, config: config,
|
|
270
285
|
single_active_consumer: single_active, consumer_priority: priority,
|
|
271
286
|
execution_mode: exec_mode, group_mode: grp_mode,
|
|
272
|
-
liveness_pipe: liveness_writer
|
|
287
|
+
liveness_pipe: liveness_writer, wake_pipe: wake_reader
|
|
273
288
|
)
|
|
274
289
|
worker.run
|
|
275
290
|
end
|
|
@@ -277,24 +292,50 @@ module Pgbus
|
|
|
277
292
|
unless pid
|
|
278
293
|
close_pipe(liveness_reader)
|
|
279
294
|
close_pipe(liveness_writer)
|
|
295
|
+
close_pipe(wake_reader)
|
|
296
|
+
close_pipe(wake_writer)
|
|
280
297
|
Pgbus.logger.error { "[Pgbus] Failed to fork worker for queues=#{queues.join(",")}" }
|
|
281
298
|
return
|
|
282
299
|
end
|
|
283
300
|
|
|
284
|
-
# Parent keeps the reader
|
|
285
|
-
# EOF once
|
|
301
|
+
# Parent keeps the liveness reader + wake writer, discards its copies
|
|
302
|
+
# of the child-side ends so each pipe reaches EOF once its sole owner
|
|
303
|
+
# closes.
|
|
286
304
|
close_pipe(liveness_writer)
|
|
305
|
+
close_pipe(wake_reader)
|
|
306
|
+
register_fork_with_hub(pid, wake_writer, queues)
|
|
287
307
|
@forks[pid] = {
|
|
288
308
|
type: :worker, config: worker_config, slot: slot, spawned_at: monotonic_now,
|
|
289
|
-
liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false
|
|
309
|
+
liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false,
|
|
310
|
+
wake_writer: wake_writer
|
|
290
311
|
}
|
|
291
312
|
Pgbus.logger.info { "[Pgbus] Forked worker pid=#{pid} queues=#{queues.join(",")} mode=#{exec_mode}" }
|
|
292
313
|
rescue Errno::EAGAIN, Errno::ENOMEM => e
|
|
293
314
|
close_pipe(liveness_reader)
|
|
294
315
|
close_pipe(liveness_writer)
|
|
316
|
+
close_pipe(wake_reader)
|
|
317
|
+
close_pipe(wake_writer)
|
|
295
318
|
ErrorReporter.report(e, { action: "fork_worker", queues: queues })
|
|
296
319
|
end
|
|
297
320
|
|
|
321
|
+
# Hand the hub a worker fork's routing entry: explicit queues as
|
|
322
|
+
# physical names, "*" as the unconditional wildcard flag (the hub wakes
|
|
323
|
+
# wildcard forks for every channel, so the fork's own resolved set never
|
|
324
|
+
# needs to be reported upstream). Registration must never abort the fork
|
|
325
|
+
# bookkeeping that follows it — queue_name can raise on a malformed
|
|
326
|
+
# name — so on error the fork registers with an empty set and rides its
|
|
327
|
+
# poll ceiling (symmetric with register_consumer_with_hub).
|
|
328
|
+
def register_fork_with_hub(pid, wake_writer, queues)
|
|
329
|
+
return unless @notify_hub && wake_writer
|
|
330
|
+
|
|
331
|
+
wildcard = queues.include?("*")
|
|
332
|
+
physical = queues.reject { |q| q == "*" }.map { |q| config.queue_name(q) }
|
|
333
|
+
@notify_hub.register_fork(pid: pid, queues: physical, wildcard: wildcard, pipe: wake_writer)
|
|
334
|
+
rescue StandardError => e
|
|
335
|
+
ErrorReporter.report(e, { action: "register_fork_with_hub", queues: queues })
|
|
336
|
+
@notify_hub.register_fork(pid: pid, queues: [], wildcard: false, pipe: wake_writer)
|
|
337
|
+
end
|
|
338
|
+
|
|
298
339
|
def fork_dispatcher
|
|
299
340
|
pid = fork do
|
|
300
341
|
restore_signals
|
|
@@ -386,7 +427,9 @@ module Pgbus
|
|
|
386
427
|
end
|
|
387
428
|
|
|
388
429
|
def fork_consumer(consumer_config, slot: nil)
|
|
389
|
-
|
|
430
|
+
# Array() so a consumer entry without :topics can't NoMethodError the
|
|
431
|
+
# supervisor on the topics.join log lines below.
|
|
432
|
+
topics = Array(consumer_config[:topics])
|
|
390
433
|
threads = consumer_config[:threads] || 3
|
|
391
434
|
|
|
392
435
|
# OS-level liveness channel: the consumer writes a byte each loop
|
|
@@ -394,41 +437,66 @@ module Pgbus
|
|
|
394
437
|
# watchdog detect a wedged consumer without the database (issue #274),
|
|
395
438
|
# exactly as fork_worker does for workers.
|
|
396
439
|
liveness_reader, liveness_writer = IO.pipe
|
|
440
|
+
# Wake channel from the NotifyHub (issue #381), as in fork_worker.
|
|
441
|
+
wake_reader, wake_writer = IO.pipe if @notify_hub
|
|
397
442
|
|
|
398
443
|
pid = fork do
|
|
399
|
-
# Child owns the writer
|
|
400
|
-
#
|
|
401
|
-
# Consumer (see fork_worker for the full rationale).
|
|
402
|
-
close_inherited_liveness_readers
|
|
444
|
+
# Child owns the liveness writer + wake reader; close this fork's
|
|
445
|
+
# own copies of the parent-side ends (see fork_worker).
|
|
403
446
|
liveness_reader.close
|
|
447
|
+
wake_writer&.close
|
|
404
448
|
restore_signals
|
|
405
449
|
setup_child_process
|
|
406
450
|
load_rails_app
|
|
407
|
-
consumer = Consumer.new(topics: topics, threads: threads, config: config,
|
|
451
|
+
consumer = Consumer.new(topics: topics, threads: threads, config: config,
|
|
452
|
+
liveness_pipe: liveness_writer, wake_pipe: wake_reader)
|
|
408
453
|
consumer.run
|
|
409
454
|
end
|
|
410
455
|
|
|
411
456
|
unless pid
|
|
412
457
|
close_pipe(liveness_reader)
|
|
413
458
|
close_pipe(liveness_writer)
|
|
459
|
+
close_pipe(wake_reader)
|
|
460
|
+
close_pipe(wake_writer)
|
|
414
461
|
Pgbus.logger.error { "[Pgbus] Failed to fork consumer for topics=#{topics.join(",")}" }
|
|
415
462
|
return
|
|
416
463
|
end
|
|
417
464
|
|
|
418
|
-
# Parent keeps the reader
|
|
419
|
-
#
|
|
465
|
+
# Parent keeps the liveness reader + wake writer, discards its copies
|
|
466
|
+
# of the child-side ends.
|
|
420
467
|
close_pipe(liveness_writer)
|
|
468
|
+
close_pipe(wake_reader)
|
|
469
|
+
register_consumer_with_hub(pid, wake_writer, topics)
|
|
421
470
|
@forks[pid] = {
|
|
422
471
|
type: :consumer, config: consumer_config, slot: slot, spawned_at: monotonic_now,
|
|
423
|
-
liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false
|
|
472
|
+
liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false,
|
|
473
|
+
wake_writer: wake_writer
|
|
424
474
|
}
|
|
425
475
|
Pgbus.logger.info { "[Pgbus] Forked consumer pid=#{pid} topics=#{topics.join(",")}" }
|
|
426
476
|
rescue Errno::EAGAIN, Errno::ENOMEM => e
|
|
427
477
|
close_pipe(liveness_reader)
|
|
428
478
|
close_pipe(liveness_writer)
|
|
479
|
+
close_pipe(wake_reader)
|
|
480
|
+
close_pipe(wake_writer)
|
|
429
481
|
ErrorReporter.report(e, { action: "fork_consumer", topics: topics })
|
|
430
482
|
end
|
|
431
483
|
|
|
484
|
+
# A consumer's routing entry mirrors Consumer#setup_subscriptions: the
|
|
485
|
+
# registry derives the queue set from the topic list. Registry lookups
|
|
486
|
+
# never abort a fork — on error the fork registers with an empty set and
|
|
487
|
+
# rides its poll ceiling until the next supervisor restart.
|
|
488
|
+
def register_consumer_with_hub(pid, wake_writer, topics)
|
|
489
|
+
return unless @notify_hub && wake_writer
|
|
490
|
+
|
|
491
|
+
physical = EventBus::Registry.instance
|
|
492
|
+
.queue_names_for_topics(Array(topics))
|
|
493
|
+
.map { |q| config.queue_name(q) }
|
|
494
|
+
@notify_hub.register_fork(pid: pid, queues: physical, wildcard: false, pipe: wake_writer)
|
|
495
|
+
rescue StandardError => e
|
|
496
|
+
ErrorReporter.report(e, { action: "register_consumer_with_hub", topics: topics })
|
|
497
|
+
@notify_hub.register_fork(pid: pid, queues: [], wildcard: false, pipe: wake_writer)
|
|
498
|
+
end
|
|
499
|
+
|
|
432
500
|
def boot_outbox_poller
|
|
433
501
|
return unless config.outbox_enabled
|
|
434
502
|
|
|
@@ -465,6 +533,9 @@ module Pgbus
|
|
|
465
533
|
unless @shutting_down
|
|
466
534
|
process_pending_restarts
|
|
467
535
|
check_stalled_workers
|
|
536
|
+
# One hub beat per monitor pass: listener self-heal, LISTEN union
|
|
537
|
+
# refresh, and fork status broadcast (issue #381).
|
|
538
|
+
@notify_hub&.tick
|
|
468
539
|
end
|
|
469
540
|
interruptible_sleep(FORK_WAIT)
|
|
470
541
|
end
|
|
@@ -480,8 +551,11 @@ module Pgbus
|
|
|
480
551
|
|
|
481
552
|
# Close the liveness reader as the fork leaves @forks so a crash-loop
|
|
482
553
|
# (restart deferred up to RESTART_BACKOFF_MAX) can't leak an FD per
|
|
483
|
-
# crash. Scrub the
|
|
554
|
+
# crash. Scrub the keys so a closed IO never rides into a restart.
|
|
555
|
+
# The wake writer is closed by the hub's deregister (same IO object).
|
|
484
556
|
close_pipe(info.delete(:liveness_reader))
|
|
557
|
+
info.delete(:wake_writer)
|
|
558
|
+
@notify_hub&.deregister_fork(pid)
|
|
485
559
|
|
|
486
560
|
if @shutting_down
|
|
487
561
|
Pgbus.logger.info { "[Pgbus] Child #{info[:type]} pid=#{pid} exited (status=#{status.exitstatus})" }
|
|
@@ -655,6 +729,13 @@ module Pgbus
|
|
|
655
729
|
end
|
|
656
730
|
|
|
657
731
|
def setup_child_process
|
|
732
|
+
# Every child type (worker, consumer, dispatcher, scheduler, outbox
|
|
733
|
+
# poller) releases its copies of the parent's per-fork resources —
|
|
734
|
+
# a dispatcher child holding a sibling worker's wake WRITER would
|
|
735
|
+
# keep that sibling's pipe from ever reaching EOF after the
|
|
736
|
+
# supervisor dies, pinning the sibling to the 15s NOTIFY ceiling
|
|
737
|
+
# with no wake source (issue #381 review).
|
|
738
|
+
close_inherited_parent_resources
|
|
658
739
|
# Reset the PGMQ client so this forked process gets a fresh
|
|
659
740
|
# PG::Connection instead of inheriting the parent's (which is
|
|
660
741
|
# in undefined state post-fork and not thread-safe to share).
|
|
@@ -758,11 +839,40 @@ module Pgbus
|
|
|
758
839
|
nil
|
|
759
840
|
end
|
|
760
841
|
|
|
761
|
-
#
|
|
762
|
-
# parent's FD table
|
|
763
|
-
#
|
|
764
|
-
|
|
765
|
-
|
|
842
|
+
# Called only inside a just-forked child: close every sibling pipe end
|
|
843
|
+
# inherited from the parent's FD table (liveness readers AND wake
|
|
844
|
+
# writers), and release the child's copy of the NotifyHub's LISTEN
|
|
845
|
+
# socket without a libpq Terminate (PQfinish would kill the PARENT's
|
|
846
|
+
# session over the shared socket — see
|
|
847
|
+
# NotifyListener#close_inherited_socket!).
|
|
848
|
+
def close_inherited_parent_resources
|
|
849
|
+
@forks.each_value do |info|
|
|
850
|
+
close_pipe(info[:liveness_reader])
|
|
851
|
+
close_pipe(info[:wake_writer])
|
|
852
|
+
end
|
|
853
|
+
@notify_hub&.close_inherited!
|
|
854
|
+
@notify_hub = nil
|
|
855
|
+
end
|
|
856
|
+
|
|
857
|
+
# Build the host-level shared LISTEN hub (issue #381). Only under
|
|
858
|
+
# :supervisor scope, with notify wakeups on, and with at least one role
|
|
859
|
+
# that reads queues. A hub that fails to start degrades to no hub: forks
|
|
860
|
+
# get no wake pipe and fall back to fast polling, exactly like a failed
|
|
861
|
+
# per-fork listener under :fork scope.
|
|
862
|
+
def start_notify_hub
|
|
863
|
+
return unless config.worker_notify_wakeup?
|
|
864
|
+
return unless config.worker_notify_scope == :supervisor
|
|
865
|
+
return unless config.role_enabled?(:workers) || config.role_enabled?(:consumers)
|
|
866
|
+
|
|
867
|
+
hub = NotifyHub.new(config: config)
|
|
868
|
+
hub.start
|
|
869
|
+
@notify_hub = hub
|
|
870
|
+
rescue StandardError => e
|
|
871
|
+
@notify_hub = nil
|
|
872
|
+
ErrorReporter.report(e, { action: "start_notify_hub" })
|
|
873
|
+
Pgbus.logger.error do
|
|
874
|
+
"[Pgbus] NotifyHub failed to start — forks fall back to polling: #{e.class}: #{e.message}"
|
|
875
|
+
end
|
|
766
876
|
end
|
|
767
877
|
|
|
768
878
|
def shutdown
|
|
@@ -778,9 +888,11 @@ module Pgbus
|
|
|
778
888
|
signal_children("KILL") unless @forks.empty?
|
|
779
889
|
|
|
780
890
|
# Close any liveness readers still open on un-reaped children so the
|
|
781
|
-
# supervisor never leaks FDs across a restart of itself.
|
|
891
|
+
# supervisor never leaks FDs across a restart of itself. (Wake writers
|
|
892
|
+
# are closed by the hub's stop below.)
|
|
782
893
|
@forks.each_value { |info| close_pipe(info[:liveness_reader]) }
|
|
783
894
|
|
|
895
|
+
@notify_hub&.stop
|
|
784
896
|
@health_server&.stop
|
|
785
897
|
@heartbeat&.stop
|
|
786
898
|
restore_signals
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Process
|
|
5
|
+
# Fork-side receiver for supervisor-mediated wake-ups (issue #381,
|
|
6
|
+
# worker_notify_scope: :supervisor). The supervisor's NotifyHub holds the
|
|
7
|
+
# write end of a per-fork pipe; this class owns the inherited read end and
|
|
8
|
+
# a single watcher thread that translates the byte protocol into the
|
|
9
|
+
# fork's existing primitives:
|
|
10
|
+
#
|
|
11
|
+
# W — a NOTIFY arrived for a queue this fork reads → wake_signal.notify!
|
|
12
|
+
# H — the shared listener is healthy (connected + delivering) → the
|
|
13
|
+
# fork may sleep up to the NOTIFY poll ceiling
|
|
14
|
+
# P — the shared listener is degraded (dead thread, mid-reconnect,
|
|
15
|
+
# pooler-deaf) → the fork falls back to fast polling
|
|
16
|
+
#
|
|
17
|
+
# Status starts optimistic (mirrors NotifyListener's @delivering default)
|
|
18
|
+
# so a just-forked worker isn't pinned to fast polling before the hub's
|
|
19
|
+
# first broadcast. EOF on the pipe means the supervisor is gone: mark
|
|
20
|
+
# not-delivering and let the fork run on plain polling.
|
|
21
|
+
#
|
|
22
|
+
# A nil reader (scope :fork, or the supervisor never armed the pipe)
|
|
23
|
+
# yields an inert instance: start is a no-op and delivering? is false, so
|
|
24
|
+
# wake_timeout math treats it exactly like an absent listener.
|
|
25
|
+
class WakePipe
|
|
26
|
+
WAKE = "W"
|
|
27
|
+
HEALTHY = "H"
|
|
28
|
+
DEGRADED = "P"
|
|
29
|
+
|
|
30
|
+
def initialize(reader, wake_signal:, logger: Pgbus.logger)
|
|
31
|
+
@reader = reader
|
|
32
|
+
@wake_signal = wake_signal
|
|
33
|
+
@logger = logger
|
|
34
|
+
@state_mutex = Mutex.new
|
|
35
|
+
@running = false
|
|
36
|
+
@thread = nil
|
|
37
|
+
@delivering = !reader.nil?
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def delivering?
|
|
41
|
+
@state_mutex.synchronize { @delivering }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def running?
|
|
45
|
+
@state_mutex.synchronize { @running }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def start
|
|
49
|
+
return self if @reader.nil?
|
|
50
|
+
|
|
51
|
+
@state_mutex.synchronize do
|
|
52
|
+
return self if @running
|
|
53
|
+
|
|
54
|
+
@running = true
|
|
55
|
+
end
|
|
56
|
+
@thread = Thread.new { run_loop }
|
|
57
|
+
self
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def stop
|
|
61
|
+
@state_mutex.synchronize do
|
|
62
|
+
return self unless @running
|
|
63
|
+
|
|
64
|
+
@running = false
|
|
65
|
+
end
|
|
66
|
+
# Closing the pipe FD interrupts the watcher's blocking readpartial —
|
|
67
|
+
# Ruby raises IOError in the blocked thread at the interpreter level.
|
|
68
|
+
# (Safe for a plain IO, unlike PG::Connection#close, which is PQfinish
|
|
69
|
+
# under a concurrent libpq call — issue #375. That constraint is about
|
|
70
|
+
# libpq, not IO.)
|
|
71
|
+
close_reader_quietly
|
|
72
|
+
@thread&.join(2)
|
|
73
|
+
@thread = nil
|
|
74
|
+
self
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
private
|
|
78
|
+
|
|
79
|
+
def run_loop
|
|
80
|
+
loop do
|
|
81
|
+
break unless running?
|
|
82
|
+
|
|
83
|
+
handle_bytes(@reader.readpartial(4096))
|
|
84
|
+
end
|
|
85
|
+
rescue EOFError
|
|
86
|
+
# Supervisor exited: no more wakes will ever arrive on this pipe.
|
|
87
|
+
mark_not_delivering("supervisor wake pipe reached EOF — falling back to polling")
|
|
88
|
+
rescue IOError, Errno::EBADF
|
|
89
|
+
# Reader closed under us. Expected during #stop (running? already
|
|
90
|
+
# false); anything else degrades to polling like EOF.
|
|
91
|
+
mark_not_delivering("supervisor wake pipe closed — falling back to polling") if running?
|
|
92
|
+
rescue StandardError => e
|
|
93
|
+
mark_not_delivering("supervisor wake pipe failed (#{e.class}: #{e.message}) — falling back to polling")
|
|
94
|
+
ensure
|
|
95
|
+
@state_mutex.synchronize { @running = false }
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# One coalesced read may carry several bytes. Status transitions apply
|
|
99
|
+
# in order; any number of W bytes collapses into one notify! (WakeSignal
|
|
100
|
+
# coalesces concurrent notifies anyway).
|
|
101
|
+
def handle_bytes(data)
|
|
102
|
+
wake = false
|
|
103
|
+
data.each_char do |byte|
|
|
104
|
+
case byte
|
|
105
|
+
when WAKE then wake = true
|
|
106
|
+
when HEALTHY then update_delivering(true)
|
|
107
|
+
when DEGRADED then update_delivering(false)
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
@wake_signal.notify! if wake
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def update_delivering(value)
|
|
114
|
+
@state_mutex.synchronize { @delivering = value }
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def mark_not_delivering(message)
|
|
118
|
+
update_delivering(false)
|
|
119
|
+
@logger.warn { "[Pgbus::WakePipe] #{message}" }
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def close_reader_quietly
|
|
123
|
+
@reader.close if @reader && !@reader.closed?
|
|
124
|
+
rescue IOError, Errno::EBADF
|
|
125
|
+
nil
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Process
|
|
5
|
+
# Resolves a wildcard ("*") queue list to the concrete logical queue names
|
|
6
|
+
# a job worker may adopt: every pgmq.meta queue minus dead-letter queues,
|
|
7
|
+
# stream queues (issue #309/#366 — a job worker adopting one would claim
|
|
8
|
+
# durable broadcasts and DLQ-move them out of replay history), and
|
|
9
|
+
# event-subscriber queues (issue #333 — event payloads, not ActiveJob
|
|
10
|
+
# jobs), with the configured prefix stripped.
|
|
11
|
+
#
|
|
12
|
+
# Shared by Worker#resolve_wildcard_queues (per-fork adoption) and the
|
|
13
|
+
# supervisor-owned NotifyHub (issue #381 — the LISTEN union for wildcard
|
|
14
|
+
# capsules), so both sides of the wake path agree on what "*" means.
|
|
15
|
+
module WildcardQueueResolver
|
|
16
|
+
module_function
|
|
17
|
+
|
|
18
|
+
def resolve(config: Pgbus.configuration)
|
|
19
|
+
prefix = "#{config.queue_prefix}_"
|
|
20
|
+
|
|
21
|
+
# Reset first so a stream created since the last resolve is excluded.
|
|
22
|
+
Pgbus::StreamQueue.reset_cache!
|
|
23
|
+
stream_names = Pgbus::StreamQueue.known_names
|
|
24
|
+
event_names = Pgbus::EventBus::Registry.instance.event_queue_names
|
|
25
|
+
|
|
26
|
+
conn = config.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection
|
|
27
|
+
conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
|
|
28
|
+
.reject { |q| q.end_with?(Pgbus::DEAD_LETTER_SUFFIX) }
|
|
29
|
+
.reject { |q| stream_names.include?(q) }
|
|
30
|
+
.reject { |q| event_names.include?(q) }
|
|
31
|
+
.map { |q| q.delete_prefix(prefix) }
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
data/lib/pgbus/process/worker.rb
CHANGED
|
@@ -19,6 +19,11 @@ module Pgbus
|
|
|
19
19
|
# between calls, and simulate start_notify_listener assigning the listener
|
|
20
20
|
# from inside a stub (production mutates all three in the run loop).
|
|
21
21
|
attr_accessor :notify_listener, :notify_retry_at, :notify_retry_backoff
|
|
22
|
+
# Supervisor-mediated wake source (issue #381, worker_notify_scope:
|
|
23
|
+
# :supervisor). Non-nil iff the supervisor forked us with a wake pipe;
|
|
24
|
+
# its presence switches the whole notify lifecycle from a fork-local
|
|
25
|
+
# NotifyListener to the shared hub. Readable as a test seam.
|
|
26
|
+
attr_reader :wake_pipe
|
|
22
27
|
|
|
23
28
|
# The collaborators below (rate_counter, wake_signal, stat_buffer) and the
|
|
24
29
|
# recycle clock (started_at_monotonic) accept injected seeds so tests can
|
|
@@ -30,7 +35,7 @@ module Pgbus
|
|
|
30
35
|
rate_counter: nil, wake_signal: nil, stat_buffer: :default,
|
|
31
36
|
notify_listener: nil, notify_retry_at: 0.0,
|
|
32
37
|
notify_retry_backoff: NOTIFY_RETRY_BASE_SECONDS,
|
|
33
|
-
started_at_monotonic: nil)
|
|
38
|
+
started_at_monotonic: nil, wake_pipe: nil)
|
|
34
39
|
@queues = Array(queues)
|
|
35
40
|
@initial_queues = @queues.dup.freeze
|
|
36
41
|
@wildcard = @queues.include?("*")
|
|
@@ -98,6 +103,10 @@ module Pgbus
|
|
|
98
103
|
# so the watchdog can detect a wedged worker even when the database (and
|
|
99
104
|
# thus the Heartbeat's loop_tick_at) is unavailable.
|
|
100
105
|
@liveness_pipe = liveness_pipe
|
|
106
|
+
# Supervisor wake pipe (read end, inherited across fork). When present
|
|
107
|
+
# the fork owns NO LISTEN connection: wakes and listener-health status
|
|
108
|
+
# arrive as bytes from the supervisor's NotifyHub.
|
|
109
|
+
@wake_pipe = wake_pipe ? WakePipe.new(wake_pipe, wake_signal: @wake_signal) : nil
|
|
101
110
|
end
|
|
102
111
|
|
|
103
112
|
def stats
|
|
@@ -150,7 +159,7 @@ module Pgbus
|
|
|
150
159
|
setup_signals
|
|
151
160
|
start_heartbeat
|
|
152
161
|
resolve_wildcard_queues
|
|
153
|
-
|
|
162
|
+
start_wake_source
|
|
154
163
|
@lifecycle.transition_to!(:running)
|
|
155
164
|
Pgbus.logger.info do
|
|
156
165
|
"[Pgbus] Worker started: queues=#{queues.join(",")} threads=#{threads} " \
|
|
@@ -404,34 +413,10 @@ module Pgbus
|
|
|
404
413
|
def resolve_wildcard_queues
|
|
405
414
|
return unless @wildcard
|
|
406
415
|
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
# be adopted by a wildcard worker: a worker would claim durable
|
|
412
|
-
# broadcasts, fail to deserialize them, and DLQ-move them out of the
|
|
413
|
-
# stream's replay history. known_names includes fingerprint-matched
|
|
414
|
-
# dormant pre-registry streams (issue #366) so they are excluded even
|
|
415
|
-
# before backfill. Reset first so a stream created since the last
|
|
416
|
-
# resolve is excluded.
|
|
417
|
-
Pgbus::StreamQueue.reset_cache!
|
|
418
|
-
stream_names = Pgbus::StreamQueue.known_names
|
|
419
|
-
|
|
420
|
-
# Event-bus subscriber queues also share the job namespace (pgbus_<handler>)
|
|
421
|
-
# but carry event payloads, not ActiveJob jobs. A wildcard worker that
|
|
422
|
-
# adopts one hands the event to the executor, which fails to deserialize
|
|
423
|
-
# it and DLQ-moves it out of the consumer's reach — so an app running the
|
|
424
|
-
# event bus had to hand-maintain an explicit queue list. Exclude them,
|
|
425
|
-
# like stream queues (issue #333).
|
|
426
|
-
event_names = Pgbus::EventBus::Registry.instance.event_queue_names
|
|
427
|
-
|
|
428
|
-
conn = Pgbus.configuration.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection
|
|
429
|
-
all_queues = conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
|
|
430
|
-
resolved = all_queues
|
|
431
|
-
.reject { |q| q.end_with?(dlq_suffix) }
|
|
432
|
-
.reject { |q| stream_names.include?(q) }
|
|
433
|
-
.reject { |q| event_names.include?(q) }
|
|
434
|
-
.map { |q| q.delete_prefix(prefix) }
|
|
416
|
+
# Exclusion rationale (streams, event queues, DLQs) lives with the
|
|
417
|
+
# shared resolver — the NotifyHub uses the same one, so the LISTEN
|
|
418
|
+
# union and the fork's adopted set can't drift (issue #381).
|
|
419
|
+
resolved = WildcardQueueResolver.resolve(config: config)
|
|
435
420
|
|
|
436
421
|
if resolved.empty?
|
|
437
422
|
Pgbus.logger.warn { "[Pgbus] Wildcard queue '*' resolved to no queues — falling back to default" }
|
|
@@ -627,6 +612,10 @@ module Pgbus
|
|
|
627
612
|
end
|
|
628
613
|
|
|
629
614
|
def listener_delivering?
|
|
615
|
+
# Supervisor scope: the hub's H/P broadcasts (via WakePipe) stand in
|
|
616
|
+
# for the local listener's health — same fast-poll fallback rules.
|
|
617
|
+
return @wake_pipe.delivering? if @wake_pipe
|
|
618
|
+
|
|
630
619
|
@notify_listener&.running? && @notify_listener.delivering?
|
|
631
620
|
end
|
|
632
621
|
|
|
@@ -642,12 +631,35 @@ module Pgbus
|
|
|
642
631
|
config.polling_interval
|
|
643
632
|
end
|
|
644
633
|
|
|
634
|
+
# :supervisor scope: the fork opens NO LISTEN connection; the WakePipe
|
|
635
|
+
# watcher is the wake source. A missing pipe under that scope means the
|
|
636
|
+
# hub failed to start — plain polling, NEVER a local listener, or a hub
|
|
637
|
+
# outage would balloon the host back to one direct connection per fork
|
|
638
|
+
# (the exact budget the scope exists to protect). :fork scope: the
|
|
639
|
+
# fork-local NotifyListener, exactly as before.
|
|
640
|
+
def start_wake_source
|
|
641
|
+
return @wake_pipe.start if @wake_pipe
|
|
642
|
+
|
|
643
|
+
start_notify_listener if local_listener_scope?
|
|
644
|
+
end
|
|
645
|
+
|
|
646
|
+
# Local NotifyListener lifecycle (start + self-heal) is allowed only
|
|
647
|
+
# under :fork scope.
|
|
648
|
+
def local_listener_scope?
|
|
649
|
+
config.worker_notify_scope == :fork
|
|
650
|
+
end
|
|
651
|
+
|
|
652
|
+
def stop_wake_source
|
|
653
|
+
@wake_pipe&.stop
|
|
654
|
+
@notify_listener&.stop
|
|
655
|
+
end
|
|
656
|
+
|
|
645
657
|
def start_notify_listener
|
|
646
658
|
return unless notify_wakeup?
|
|
647
659
|
|
|
648
660
|
@notify_listener = NotifyListener.new(
|
|
649
661
|
physical_queues: physical_queue_names,
|
|
650
|
-
on_wake: -> { @wake_signal.notify! },
|
|
662
|
+
on_wake: ->(_channel) { @wake_signal.notify! },
|
|
651
663
|
connection_options: config.worker_notify_connection_options,
|
|
652
664
|
health_check_ms: (config.polling_interval * 1000).to_i.clamp(250, 5_000),
|
|
653
665
|
logger: Pgbus.logger
|
|
@@ -666,6 +678,12 @@ module Pgbus
|
|
|
666
678
|
# its queue subscription reconciled (wildcard workers) and the backoff
|
|
667
679
|
# reset; a still-failing restart doubles the backoff up to the cap.
|
|
668
680
|
def ensure_notify_listener
|
|
681
|
+
# Supervisor scope: listener self-healing is the hub's job (it runs
|
|
682
|
+
# once per host in the supervisor's monitor tick), and a pipe-less
|
|
683
|
+
# fork under that scope must stay on plain polling — self-healing a
|
|
684
|
+
# listener it was never allowed to start would leak a connection.
|
|
685
|
+
return if @wake_pipe
|
|
686
|
+
return unless local_listener_scope?
|
|
669
687
|
return unless notify_wakeup?
|
|
670
688
|
return if @notify_listener&.running?
|
|
671
689
|
return if monotonic_now < @notify_retry_at
|
|
@@ -706,13 +724,18 @@ module Pgbus
|
|
|
706
724
|
Pgbus.logger.warn { "[Pgbus] NotifyListener queue sync failed: #{e.class}: #{e.message}" }
|
|
707
725
|
end
|
|
708
726
|
|
|
727
|
+
# Through config.queue_name (not raw concatenation) so a logical name
|
|
728
|
+
# needing normalization (e.g. "orders-handler" → "orders_handler")
|
|
729
|
+
# yields the SAME physical name the queue table was created with — the
|
|
730
|
+
# NOTIFY channel derives from the table name, so a raw concat would
|
|
731
|
+
# LISTEN on a channel that never fires. Keeps the fork-local (:fork
|
|
732
|
+
# scope) channels identical to the NotifyHub's (issue #381 review).
|
|
709
733
|
def physical_queue_names
|
|
710
|
-
|
|
711
|
-
queues.map { |q| "#{prefix}#{q}" }
|
|
734
|
+
queues.map { |q| config.queue_name(q) }
|
|
712
735
|
end
|
|
713
736
|
|
|
714
737
|
def channel_to_physical(channel)
|
|
715
|
-
|
|
738
|
+
NotifyListener.physical_for(channel)
|
|
716
739
|
end
|
|
717
740
|
|
|
718
741
|
def start_heartbeat
|
|
@@ -767,7 +790,7 @@ module Pgbus
|
|
|
767
790
|
|
|
768
791
|
def shutdown
|
|
769
792
|
Pgbus.logger.info { "[Pgbus] Worker draining thread pool..." }
|
|
770
|
-
|
|
793
|
+
stop_wake_source
|
|
771
794
|
@pool.shutdown
|
|
772
795
|
@pool.wait_for_termination(30)
|
|
773
796
|
@stat_buffer&.stop
|
data/lib/pgbus/version.rb
CHANGED