pgbus 0.9.8 → 0.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +329 -0
- data/README.md +336 -25
- data/lib/generators/pgbus/{add_job_locks_generator.rb → add_uniqueness_keys_generator.rb} +5 -5
- data/lib/pgbus/active_job/adapter.rb +1 -1
- data/lib/pgbus/config_loader.rb +29 -2
- data/lib/pgbus/configuration.rb +202 -73
- data/lib/pgbus/doctor.rb +28 -3
- data/lib/pgbus/execution_pools/async_pool.rb +2 -2
- data/lib/pgbus/generators/config_converter.rb +7 -5
- data/lib/pgbus/generators/migration_detector.rb +20 -16
- data/lib/pgbus/instrumentation.rb +2 -1
- data/lib/pgbus/integrations/appsignal/subscriber.rb +17 -2
- data/lib/pgbus/metrics/subscriber.rb +26 -2
- data/lib/pgbus/metrics.rb +3 -2
- data/lib/pgbus/pgmq_schema/pgmq_v1.11.1.sql +2126 -0
- data/lib/pgbus/pgmq_schema.rb +7 -2
- data/lib/pgbus/process/consumer.rb +141 -12
- data/lib/pgbus/process/supervisor.rb +33 -10
- data/lib/pgbus/process/worker.rb +6 -16
- data/lib/pgbus/recurring/schedule.rb +1 -2
- data/lib/pgbus/serializer.rb +4 -4
- data/lib/pgbus/streams/broadcastable_override.rb +0 -8
- data/lib/pgbus/streams/signed_name.rb +2 -2
- data/lib/pgbus/uniqueness.rb +11 -12
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +20 -4
- data/lib/pgbus/web/payload_filter.rb +3 -3
- data/lib/pgbus/web/streamer/instance.rb +6 -4
- data/lib/pgbus/web/streamer/listener.rb +21 -36
- data/lib/pgbus.rb +46 -6
- metadata +4 -6
- data/app/models/pgbus/job_lock.rb +0 -98
- data/lib/generators/pgbus/templates/add_job_locks.rb.erb +0 -21
- data/lib/rubocop/cop/pgbus/no_ruby_timeout.rb +0 -42
- data/lib/rubocop/pgbus.rb +0 -5
data/lib/pgbus/pgmq_schema.rb
CHANGED
|
@@ -11,7 +11,7 @@ module Pgbus
|
|
|
11
11
|
# Vendored SQL files live in lib/pgbus/pgmq_schema/pgmq_v{VERSION}.sql
|
|
12
12
|
# and are exact copies of the upstream pgmq-extension/sql/pgmq.sql at each release.
|
|
13
13
|
module PgmqSchema
|
|
14
|
-
class VersionNotFoundError <
|
|
14
|
+
class VersionNotFoundError < Pgbus::Error; end
|
|
15
15
|
|
|
16
16
|
SCHEMA_DIR = File.expand_path("pgmq_schema", __dir__).freeze
|
|
17
17
|
|
|
@@ -112,7 +112,11 @@ module Pgbus
|
|
|
112
112
|
EXECUTE 'DROP FUNCTION IF EXISTS ' || r.func_sig || ' CASCADE';
|
|
113
113
|
END LOOP;
|
|
114
114
|
|
|
115
|
-
-- Drop custom types in pgmq schema
|
|
115
|
+
-- Drop custom composite types in pgmq schema (e.g. message_record,
|
|
116
|
+
-- queue_record, metrics_result). Standalone `CREATE TYPE ... AS (...)`
|
|
117
|
+
-- types get a pg_class shell with relkind = 'c'; only skip row-types
|
|
118
|
+
-- backed by an actual relation (table/view/etc.), which must be
|
|
119
|
+
-- dropped via DROP TABLE rather than DROP TYPE.
|
|
116
120
|
FOR r IN
|
|
117
121
|
SELECT n.nspname || '.' || t.typname AS type_name
|
|
118
122
|
FROM pg_catalog.pg_type t
|
|
@@ -122,6 +126,7 @@ module Pgbus
|
|
|
122
126
|
AND NOT EXISTS (
|
|
123
127
|
SELECT 1 FROM pg_catalog.pg_class c
|
|
124
128
|
WHERE c.reltype = t.oid
|
|
129
|
+
AND c.relkind <> 'c'
|
|
125
130
|
)
|
|
126
131
|
LOOP
|
|
127
132
|
EXECUTE 'DROP TYPE IF EXISTS ' || r.type_name || ' CASCADE';
|
|
@@ -8,12 +8,16 @@ module Pgbus
|
|
|
8
8
|
include SignalHandler
|
|
9
9
|
|
|
10
10
|
attr_reader :topics, :threads, :config, :execution_mode,
|
|
11
|
-
:queue_names, :wake_signal, :notify_retry_backoff
|
|
11
|
+
:queue_names, :wake_signal, :notify_retry_backoff, :circuit_breaker
|
|
12
12
|
# notify_listener is writable so tests can simulate a start_notify_listener
|
|
13
13
|
# success from inside a stub (production sets it in start_notify_listener).
|
|
14
14
|
# notify_retry_at is writable so a test can re-arm the backoff window
|
|
15
15
|
# between successive ensure_notify_listener calls.
|
|
16
16
|
attr_accessor :notify_listener, :notify_retry_at
|
|
17
|
+
# stat_buffer is writable so a test can swap in a buffer double after
|
|
18
|
+
# construction and assert graceful_shutdown / check_recycle / shutdown flush
|
|
19
|
+
# it (mirrors Worker#stat_buffer).
|
|
20
|
+
attr_accessor :stat_buffer
|
|
17
21
|
|
|
18
22
|
def shutting_down?
|
|
19
23
|
@shutting_down
|
|
@@ -50,7 +54,8 @@ module Pgbus
|
|
|
50
54
|
# ivars. All default to the values production initializes to; `queue_names`
|
|
51
55
|
# defaults to nil, meaning "derive from the registry in setup_subscriptions".
|
|
52
56
|
def initialize(topics:, threads: 3, config: Pgbus.configuration, execution_mode: :threads,
|
|
53
|
-
queue_names: nil,
|
|
57
|
+
queue_names: nil, liveness_pipe: nil, stat_buffer: :default,
|
|
58
|
+
notify_listener: nil, notify_retry_at: 0.0,
|
|
54
59
|
notify_retry_backoff: NOTIFY_RETRY_BASE_SECONDS,
|
|
55
60
|
started_at_monotonic: nil)
|
|
56
61
|
@topics = Array(topics)
|
|
@@ -60,6 +65,7 @@ module Pgbus
|
|
|
60
65
|
@shutting_down = false
|
|
61
66
|
@recycling = false
|
|
62
67
|
@jobs_processed = Concurrent::AtomicFixnum.new(0)
|
|
68
|
+
@loop_tick_at = Concurrent::AtomicReference.new(nil)
|
|
63
69
|
@started_at_monotonic = started_at_monotonic || monotonic_now
|
|
64
70
|
@wake_signal = WakeSignal.new
|
|
65
71
|
@pool = ExecutionPools.build(
|
|
@@ -68,10 +74,36 @@ module Pgbus
|
|
|
68
74
|
on_state_change: -> { @wake_signal.notify! }
|
|
69
75
|
)
|
|
70
76
|
@registry = EventBus::Registry.instance
|
|
77
|
+
@circuit_breaker = Pgbus::CircuitBreaker.new(config: config)
|
|
78
|
+
# stat_buffer: :default means "build one iff config.stats_enabled";
|
|
79
|
+
# passing an explicit value (including nil) overrides that for tests.
|
|
80
|
+
@stat_buffer =
|
|
81
|
+
if stat_buffer == :default
|
|
82
|
+
if config.stats_enabled
|
|
83
|
+
Pgbus::StatBuffer.new(
|
|
84
|
+
flush_size: config.stats_flush_size,
|
|
85
|
+
flush_interval: config.stats_flush_interval
|
|
86
|
+
)
|
|
87
|
+
end
|
|
88
|
+
else
|
|
89
|
+
stat_buffer
|
|
90
|
+
end
|
|
71
91
|
@queue_names = queue_names
|
|
72
92
|
@notify_listener = notify_listener
|
|
73
93
|
@notify_retry_at = notify_retry_at
|
|
74
94
|
@notify_retry_backoff = notify_retry_backoff
|
|
95
|
+
# OS-level liveness channel to the supervisor watchdog. nil unless the
|
|
96
|
+
# supervisor forked us with one. Written from stamp_loop_tick so the
|
|
97
|
+
# watchdog can detect a wedged consumer even when the database is down.
|
|
98
|
+
@liveness_pipe = liveness_pipe
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# The last wall-clock loop-tick stamp (Time.now.to_f) fed to the
|
|
102
|
+
# heartbeat's loop_tick_supplier. Wall-clock so it stays comparable across
|
|
103
|
+
# the process boundary the supervisor watchdog reads it over. nil until the
|
|
104
|
+
# first stamp_loop_tick.
|
|
105
|
+
def last_loop_tick
|
|
106
|
+
@loop_tick_at.get
|
|
75
107
|
end
|
|
76
108
|
|
|
77
109
|
def run
|
|
@@ -85,6 +117,7 @@ module Pgbus
|
|
|
85
117
|
end
|
|
86
118
|
|
|
87
119
|
loop do
|
|
120
|
+
stamp_loop_tick
|
|
88
121
|
process_signals
|
|
89
122
|
check_recycle
|
|
90
123
|
ensure_notify_listener
|
|
@@ -92,6 +125,7 @@ module Pgbus
|
|
|
92
125
|
break if @shutting_down
|
|
93
126
|
|
|
94
127
|
consume
|
|
128
|
+
@stat_buffer&.flush_if_due
|
|
95
129
|
end
|
|
96
130
|
|
|
97
131
|
shutdown
|
|
@@ -99,6 +133,11 @@ module Pgbus
|
|
|
99
133
|
|
|
100
134
|
def graceful_shutdown
|
|
101
135
|
@shutting_down = true
|
|
136
|
+
# Flush buffered stats at drain entry so the window since the last flush
|
|
137
|
+
# isn't lost if the supervisor watchdog SIGKILLs a stalled consumer
|
|
138
|
+
# before shutdown runs. Same-thread (signals dispatched via
|
|
139
|
+
# process_signals, not trap context), so the DB write is safe.
|
|
140
|
+
@stat_buffer&.flush
|
|
102
141
|
@wake_signal.notify!
|
|
103
142
|
end
|
|
104
143
|
|
|
@@ -121,12 +160,7 @@ module Pgbus
|
|
|
121
160
|
idle = @pool.available_capacity
|
|
122
161
|
return @wake_signal.wait(timeout: wake_timeout) if idle <= 0
|
|
123
162
|
|
|
124
|
-
tagged_messages =
|
|
125
|
-
queue = @queue_names.first
|
|
126
|
-
(Pgbus.client.read_batch(queue, qty: idle) || []).map { |m| [queue, m] }
|
|
127
|
-
else
|
|
128
|
-
fetch_multi_consumer(idle)
|
|
129
|
-
end
|
|
163
|
+
tagged_messages = fetch_messages(idle)
|
|
130
164
|
|
|
131
165
|
if tagged_messages.empty?
|
|
132
166
|
@wake_signal.wait(timeout: wake_timeout)
|
|
@@ -138,10 +172,38 @@ module Pgbus
|
|
|
138
172
|
end
|
|
139
173
|
end
|
|
140
174
|
|
|
175
|
+
# Returns an array of [queue_name, message] pairs. Queues whose circuit
|
|
176
|
+
# breaker has tripped are skipped so a poison queue is left to cool down
|
|
177
|
+
# instead of being hammered every tick (mirrors Worker#fetch_messages).
|
|
178
|
+
def fetch_messages(qty)
|
|
179
|
+
active_queues = @queue_names.reject { |q| @circuit_breaker.paused?(q) }
|
|
180
|
+
return [] if active_queues.empty?
|
|
181
|
+
|
|
182
|
+
if active_queues.size == 1
|
|
183
|
+
queue = active_queues.first
|
|
184
|
+
(Pgbus.client.read_batch(queue, qty: qty) || []).map { |m| [queue, m] }
|
|
185
|
+
else
|
|
186
|
+
fetch_multi_consumer(active_queues, qty)
|
|
187
|
+
end
|
|
188
|
+
rescue Pgbus::ConnectionCircuitOpenError
|
|
189
|
+
# The client-level connection breaker is open: the database has failed
|
|
190
|
+
# enough consecutive connection attempts that reads fail fast. Idle this
|
|
191
|
+
# poll without an ErrorReporter call so the whole consumer pool doesn't
|
|
192
|
+
# flood the error tracker for the duration of a database outage. The
|
|
193
|
+
# open/close transitions are logged once by the client, not per poll.
|
|
194
|
+
[]
|
|
195
|
+
rescue StandardError => e
|
|
196
|
+
ErrorReporter.report(e, { action: "fetch_messages", queues: active_queues })
|
|
197
|
+
[]
|
|
198
|
+
end
|
|
199
|
+
|
|
141
200
|
def handle_message(message, queue_name)
|
|
201
|
+
execution_start = monotonic_now
|
|
202
|
+
|
|
142
203
|
if message.read_ct.to_i > config.max_retries
|
|
143
204
|
Pgbus.logger.warn { "[Pgbus] Consumer moving message #{message.msg_id} to DLQ after #{message.read_ct} reads" }
|
|
144
205
|
Pgbus.client.move_to_dead_letter(queue_name, message)
|
|
206
|
+
record_stat(message, queue_name, "dead_lettered", execution_start)
|
|
145
207
|
return
|
|
146
208
|
end
|
|
147
209
|
|
|
@@ -155,11 +217,15 @@ module Pgbus
|
|
|
155
217
|
end
|
|
156
218
|
|
|
157
219
|
Pgbus.client.archive_message(queue_name, message.msg_id.to_i)
|
|
220
|
+
@circuit_breaker.record_success(queue_name)
|
|
221
|
+
record_stat(message, queue_name, "success", execution_start)
|
|
158
222
|
rescue StandardError => e
|
|
159
223
|
Pgbus.logger.error { "[Pgbus] Consumer error: #{e.class}: #{e.message}" }
|
|
160
224
|
# Message stays in queue; VT will expire and it becomes available again.
|
|
161
225
|
# read_ct tracks delivery attempts — when it exceeds max_retries,
|
|
162
226
|
# the next read will route to DLQ above.
|
|
227
|
+
@circuit_breaker.record_failure(queue_name)
|
|
228
|
+
record_stat(message, queue_name, "failed", execution_start)
|
|
163
229
|
ensure
|
|
164
230
|
# Count every message the consumer handles — success, DLQ-routed, AND
|
|
165
231
|
# rescued failure — mirroring Worker#process_message, which increments
|
|
@@ -169,16 +235,40 @@ module Pgbus
|
|
|
169
235
|
@jobs_processed.increment
|
|
170
236
|
end
|
|
171
237
|
|
|
238
|
+
# Record a job stat for the handled message, mirroring the shape the
|
|
239
|
+
# executor pushes (Executor#record_stat) so consumer and worker throughput
|
|
240
|
+
# land in the same pgbus_job_stats table. No-op unless stats are enabled.
|
|
241
|
+
def record_stat(message, queue_name, status, start_time)
|
|
242
|
+
return unless config.stats_enabled
|
|
243
|
+
|
|
244
|
+
attrs = {
|
|
245
|
+
job_class: "EventConsumer",
|
|
246
|
+
queue_name: queue_name,
|
|
247
|
+
status: status,
|
|
248
|
+
duration_ms: ((monotonic_now - start_time) * 1000).round,
|
|
249
|
+
enqueue_latency_ms: nil,
|
|
250
|
+
retry_count: [message.read_ct.to_i - 1, 0].max
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
if @stat_buffer
|
|
254
|
+
@stat_buffer.push(attrs)
|
|
255
|
+
else
|
|
256
|
+
JobStat.record!(**attrs)
|
|
257
|
+
end
|
|
258
|
+
rescue StandardError => e
|
|
259
|
+
Pgbus.logger.debug { "[Pgbus] Consumer stat recording failed: #{e.message}" }
|
|
260
|
+
end
|
|
261
|
+
|
|
172
262
|
# `qty` is the total pool capacity. pgmq-ruby treats `qty:` as per-queue,
|
|
173
263
|
# so we also pass `limit: qty` to cap the total across all queues —
|
|
174
264
|
# otherwise we get `queue_count * qty` messages and overflow the
|
|
175
265
|
# execution pool, crashing the consumer fork (issue #123).
|
|
176
|
-
def fetch_multi_consumer(qty)
|
|
177
|
-
messages = Pgbus.client.read_multi(
|
|
266
|
+
def fetch_multi_consumer(active_queues, qty)
|
|
267
|
+
messages = Pgbus.client.read_multi(active_queues, qty: qty, limit: qty) || []
|
|
178
268
|
prefix = "#{config.queue_prefix}_"
|
|
179
269
|
|
|
180
270
|
messages.map do |m|
|
|
181
|
-
logical = m.queue_name&.delete_prefix(prefix) ||
|
|
271
|
+
logical = m.queue_name&.delete_prefix(prefix) || active_queues.first
|
|
182
272
|
[logical, m]
|
|
183
273
|
end
|
|
184
274
|
end
|
|
@@ -203,6 +293,9 @@ module Pgbus
|
|
|
203
293
|
|
|
204
294
|
@recycling = true
|
|
205
295
|
@shutting_down = true
|
|
296
|
+
# Flush buffered stats on recycle-triggered drain for the same reason as
|
|
297
|
+
# graceful_shutdown: shrink the SIGKILL loss window. Same-thread, safe.
|
|
298
|
+
@stat_buffer&.flush
|
|
206
299
|
Pgbus::Instrumentation.instrument(
|
|
207
300
|
"pgbus.consumer.recycle",
|
|
208
301
|
reason: reason,
|
|
@@ -324,15 +417,36 @@ module Pgbus
|
|
|
324
417
|
def start_heartbeat
|
|
325
418
|
@heartbeat = Heartbeat.new(
|
|
326
419
|
kind: "consumer",
|
|
327
|
-
metadata: { topics: topics, threads: threads, pid: ::Process.pid }
|
|
420
|
+
metadata: { topics: topics, threads: threads, pid: ::Process.pid },
|
|
421
|
+
on_beat: -> { on_heartbeat },
|
|
422
|
+
loop_tick_supplier: -> { @loop_tick_at.get }
|
|
328
423
|
)
|
|
329
424
|
@heartbeat.start
|
|
330
425
|
end
|
|
331
426
|
|
|
427
|
+
# Runs once per heartbeat interval (not per message), so it's the right
|
|
428
|
+
# place to emit connection-pool observability without touching any per-job
|
|
429
|
+
# hot path. Reading the pool must never crash the beat — pool_stats already
|
|
430
|
+
# rescues to {}, and this whole method is guarded so an unexpected error
|
|
431
|
+
# can't take down the heartbeat thread (mirrors Worker#on_heartbeat).
|
|
432
|
+
def on_heartbeat
|
|
433
|
+
emit_pool_stats
|
|
434
|
+
rescue StandardError => e
|
|
435
|
+
Pgbus.logger.debug { "[Pgbus] Consumer heartbeat hook error: #{e.class}: #{e.message}" }
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
def emit_pool_stats
|
|
439
|
+
stats = Pgbus.client.pool_stats
|
|
440
|
+
return if stats.empty?
|
|
441
|
+
|
|
442
|
+
Pgbus::Instrumentation.instrument("pgbus.client.pool", stats)
|
|
443
|
+
end
|
|
444
|
+
|
|
332
445
|
def shutdown
|
|
333
446
|
@notify_listener&.stop
|
|
334
447
|
@pool.shutdown
|
|
335
448
|
@pool.wait_for_termination(30)
|
|
449
|
+
@stat_buffer&.stop
|
|
336
450
|
@heartbeat&.stop
|
|
337
451
|
restore_signals
|
|
338
452
|
Pgbus.logger.info { "[Pgbus] Consumer stopped. Processed: #{@jobs_processed.value}" }
|
|
@@ -341,6 +455,21 @@ module Pgbus
|
|
|
341
455
|
def monotonic_now
|
|
342
456
|
::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
343
457
|
end
|
|
458
|
+
|
|
459
|
+
# Stamp the loop-progress beacon with a wall-clock timestamp (required
|
|
460
|
+
# because the supervisor watchdog reads it cross-fork and the dashboard
|
|
461
|
+
# reads it cross-host). Also pokes the OS-level liveness pipe when the
|
|
462
|
+
# supervisor forked us with one, giving the watchdog a database-independent
|
|
463
|
+
# signal. The write is non-blocking and never raises in the hot path
|
|
464
|
+
# (mirrors Worker#stamp_loop_tick).
|
|
465
|
+
def stamp_loop_tick
|
|
466
|
+
@loop_tick_at.set(Time.now.to_f)
|
|
467
|
+
return unless @liveness_pipe
|
|
468
|
+
|
|
469
|
+
@liveness_pipe.write_nonblock("\0", exception: false)
|
|
470
|
+
rescue Errno::EPIPE, IOError, Errno::EBADF
|
|
471
|
+
nil
|
|
472
|
+
end
|
|
344
473
|
end
|
|
345
474
|
end
|
|
346
475
|
end
|
|
@@ -308,7 +308,7 @@ module Pgbus
|
|
|
308
308
|
end
|
|
309
309
|
|
|
310
310
|
def boot_scheduler
|
|
311
|
-
return
|
|
311
|
+
return unless config.recurring_enabled
|
|
312
312
|
return unless recurring_tasks_configured?
|
|
313
313
|
|
|
314
314
|
fork_scheduler
|
|
@@ -381,22 +381,43 @@ module Pgbus
|
|
|
381
381
|
topics = consumer_config[:topics]
|
|
382
382
|
threads = consumer_config[:threads] || 3
|
|
383
383
|
|
|
384
|
+
# OS-level liveness channel: the consumer writes a byte each loop
|
|
385
|
+
# iteration, the parent drains the reader in monitor_loop. This lets the
|
|
386
|
+
# watchdog detect a wedged consumer without the database (issue #274),
|
|
387
|
+
# exactly as fork_worker does for workers.
|
|
388
|
+
liveness_reader, liveness_writer = IO.pipe
|
|
389
|
+
|
|
384
390
|
pid = fork do
|
|
391
|
+
# Child owns the writer end. Close every earlier sibling's inherited
|
|
392
|
+
# liveness reader and this fork's own reader copy before becoming a
|
|
393
|
+
# Consumer (see fork_worker for the full rationale).
|
|
394
|
+
close_inherited_liveness_readers
|
|
395
|
+
liveness_reader.close
|
|
385
396
|
restore_signals
|
|
386
397
|
setup_child_process
|
|
387
398
|
load_rails_app
|
|
388
|
-
consumer = Consumer.new(topics: topics, threads: threads, config: config)
|
|
399
|
+
consumer = Consumer.new(topics: topics, threads: threads, config: config, liveness_pipe: liveness_writer)
|
|
389
400
|
consumer.run
|
|
390
401
|
end
|
|
391
402
|
|
|
392
403
|
unless pid
|
|
404
|
+
close_pipe(liveness_reader)
|
|
405
|
+
close_pipe(liveness_writer)
|
|
393
406
|
Pgbus.logger.error { "[Pgbus] Failed to fork consumer for topics=#{topics.join(",")}" }
|
|
394
407
|
return
|
|
395
408
|
end
|
|
396
409
|
|
|
397
|
-
|
|
410
|
+
# Parent keeps the reader, discards its writer copy so the pipe reaches
|
|
411
|
+
# EOF once the (sole remaining) child writer closes.
|
|
412
|
+
close_pipe(liveness_writer)
|
|
413
|
+
@forks[pid] = {
|
|
414
|
+
type: :consumer, config: consumer_config, slot: slot, spawned_at: monotonic_now,
|
|
415
|
+
liveness_reader: liveness_reader, last_pipe_tick_at: monotonic_now, pipe_seen: false
|
|
416
|
+
}
|
|
398
417
|
Pgbus.logger.info { "[Pgbus] Forked consumer pid=#{pid} topics=#{topics.join(",")}" }
|
|
399
418
|
rescue Errno::EAGAIN, Errno::ENOMEM => e
|
|
419
|
+
close_pipe(liveness_reader)
|
|
420
|
+
close_pipe(liveness_writer)
|
|
400
421
|
ErrorReporter.report(e, { action: "fork_consumer", topics: topics })
|
|
401
422
|
end
|
|
402
423
|
|
|
@@ -525,12 +546,14 @@ module Pgbus
|
|
|
525
546
|
threshold = config.stall_threshold
|
|
526
547
|
return unless threshold&.positive?
|
|
527
548
|
|
|
528
|
-
|
|
529
|
-
|
|
549
|
+
# Workers AND consumers both stamp a loop beacon and carry a liveness
|
|
550
|
+
# pipe (issue #274), so both are watched for a wedged claim/consume loop.
|
|
551
|
+
watched_pids = @forks.select { |_, info| %i[worker consumer].include?(info[:type]) }.keys
|
|
552
|
+
return if watched_pids.empty?
|
|
530
553
|
|
|
531
|
-
db_ages = db_loop_tick_ages(
|
|
554
|
+
db_ages = db_loop_tick_ages(watched_pids)
|
|
532
555
|
|
|
533
|
-
|
|
556
|
+
watched_pids.each do |pid|
|
|
534
557
|
info = @forks[pid]
|
|
535
558
|
next unless info
|
|
536
559
|
|
|
@@ -540,13 +563,13 @@ module Pgbus
|
|
|
540
563
|
Pgbus.logger.warn { "[Pgbus] Supervisor watchdog check failed: #{e.message}" }
|
|
541
564
|
end
|
|
542
565
|
|
|
543
|
-
# Read each
|
|
544
|
-
# {pid => wall-clock age in seconds} map. Isolated in its own rescue so a
|
|
566
|
+
# Read each watched fork's loop_tick_at from the process table and return
|
|
567
|
+
# a {pid => wall-clock age in seconds} map. Isolated in its own rescue so a
|
|
545
568
|
# database outage degrades to the OS-pipe fallback instead of skipping
|
|
546
569
|
# the whole watchdog — the exact failure this pipe channel exists to fix.
|
|
547
570
|
def db_loop_tick_ages(worker_pids)
|
|
548
571
|
ages = {}
|
|
549
|
-
ProcessEntry.where(kind:
|
|
572
|
+
ProcessEntry.where(kind: %w[worker consumer], pid: worker_pids).to_a.each do |entry|
|
|
550
573
|
meta = entry.metadata
|
|
551
574
|
next unless meta.is_a?(Hash)
|
|
552
575
|
|
data/lib/pgbus/process/worker.rb
CHANGED
|
@@ -141,21 +141,11 @@ module Pgbus
|
|
|
141
141
|
# restarting, failover, DNS blip) or its thread can die mid-run. Rather
|
|
142
142
|
# than downgrade to blind polling until process restart, the loop retries
|
|
143
143
|
# from ensure_notify_listener on an exponential backoff: NOTIFY_RETRY_BASE
|
|
144
|
-
# doubling up to NOTIFY_RETRY_MAX. Constant-tuned, matching
|
|
145
|
-
#
|
|
144
|
+
# doubling up to NOTIFY_RETRY_MAX. Constant-tuned, matching the
|
|
145
|
+
# CircuitBreaker/Dispatcher precedent.
|
|
146
146
|
NOTIFY_RETRY_BASE_SECONDS = 5
|
|
147
147
|
NOTIFY_RETRY_MAX_SECONDS = 300
|
|
148
148
|
|
|
149
|
-
# Upper bound on the drain phase. Waiting for quiesced? must be
|
|
150
|
-
# bounded: a permanently-stuck job would otherwise hold the loop open
|
|
151
|
-
# forever — recycling never completes, TERM shutdown hangs the whole
|
|
152
|
-
# process tree, and the loop keeps stamping loop_tick so the
|
|
153
|
-
# supervisor watchdog never intervenes. After this many seconds the
|
|
154
|
-
# loop falls through to shutdown, whose wait_for_termination(30)
|
|
155
|
-
# bounds the remaining in-flight work. Tuned via constant rather than
|
|
156
|
-
# configuration, matching CircuitBreaker/Dispatcher precedent.
|
|
157
|
-
DRAIN_TIMEOUT = 30
|
|
158
|
-
|
|
159
149
|
def run
|
|
160
150
|
setup_signals
|
|
161
151
|
start_heartbeat
|
|
@@ -178,7 +168,7 @@ module Pgbus
|
|
|
178
168
|
# quiesced? (all slots free), not idle? (any slot free) — exiting
|
|
179
169
|
# with work still in flight abandons those jobs to the 30s
|
|
180
170
|
# wait_for_termination timeout in shutdown. Bounded by
|
|
181
|
-
#
|
|
171
|
+
# config.drain_timeout so a stuck job can't wedge the loop forever.
|
|
182
172
|
break if @lifecycle.draining? && (@pool.quiesced? || drain_deadline_exceeded?)
|
|
183
173
|
|
|
184
174
|
claim_and_execute if @lifecycle.can_process?
|
|
@@ -223,7 +213,7 @@ module Pgbus
|
|
|
223
213
|
# deleted), the first restore waits RESTORE_COOLDOWN_BASE seconds; each
|
|
224
214
|
# consecutive failed restore doubles the wait up to RESTORE_COOLDOWN_MAX.
|
|
225
215
|
# A successful fetch resets the streak so a recreated queue restores
|
|
226
|
-
# promptly. Constant-tuned, matching NOTIFY_RETRY
|
|
216
|
+
# promptly. Constant-tuned, matching the NOTIFY_RETRY precedent.
|
|
227
217
|
RESTORE_COOLDOWN_BASE = 30
|
|
228
218
|
RESTORE_COOLDOWN_MAX = 300
|
|
229
219
|
|
|
@@ -531,10 +521,10 @@ module Pgbus
|
|
|
531
521
|
# state (graceful_shutdown, recycling) without hooking each one.
|
|
532
522
|
def drain_deadline_exceeded?
|
|
533
523
|
@drain_started_at ||= monotonic_now
|
|
534
|
-
return false unless monotonic_now - @drain_started_at >
|
|
524
|
+
return false unless monotonic_now - @drain_started_at > config.drain_timeout
|
|
535
525
|
|
|
536
526
|
Pgbus.logger.warn do
|
|
537
|
-
"[Pgbus] Worker drain deadline (#{
|
|
527
|
+
"[Pgbus] Worker drain deadline (#{config.drain_timeout}s) reached with #{@in_flight.value} job(s) " \
|
|
538
528
|
"still in flight — proceeding to shutdown"
|
|
539
529
|
end
|
|
540
530
|
true
|
|
@@ -202,8 +202,7 @@ module Pgbus
|
|
|
202
202
|
|
|
203
203
|
payload.merge(
|
|
204
204
|
Pgbus::Uniqueness::METADATA_KEY => key,
|
|
205
|
-
Pgbus::Uniqueness::STRATEGY_KEY => config[:strategy].to_s
|
|
206
|
-
Pgbus::Uniqueness::TTL_KEY => config[:lock_ttl]
|
|
205
|
+
Pgbus::Uniqueness::STRATEGY_KEY => config[:strategy].to_s
|
|
207
206
|
)
|
|
208
207
|
end
|
|
209
208
|
end
|
data/lib/pgbus/serializer.rb
CHANGED
|
@@ -55,21 +55,21 @@ module Pgbus
|
|
|
55
55
|
# can be resolved — prevents loading arbitrary objects from crafted payloads.
|
|
56
56
|
def locate_global_id(gid_string)
|
|
57
57
|
gid = GlobalID.parse(gid_string)
|
|
58
|
-
raise
|
|
58
|
+
raise Pgbus::SerializationError, "Invalid GlobalID: #{gid_string.inspect}" unless gid
|
|
59
59
|
|
|
60
60
|
allowed = Pgbus.configuration.allowed_global_id_models
|
|
61
61
|
if allowed && allowed.empty?
|
|
62
|
-
raise
|
|
62
|
+
raise Pgbus::SerializationError,
|
|
63
63
|
"GlobalID deserialization is disabled (allowed_global_id_models is empty). " \
|
|
64
64
|
"Set to nil to allow all models, or add permitted classes."
|
|
65
65
|
end
|
|
66
66
|
if allowed&.any? { |entry| !entry.is_a?(Class) && !entry.is_a?(Module) }
|
|
67
|
-
raise
|
|
67
|
+
raise Pgbus::SerializationError,
|
|
68
68
|
"allowed_global_id_models must contain Class/Module objects, " \
|
|
69
69
|
"got: #{allowed.map(&:class).uniq.join(", ")}"
|
|
70
70
|
end
|
|
71
71
|
if allowed&.none? { |klass| gid.model_class <= klass }
|
|
72
|
-
raise
|
|
72
|
+
raise Pgbus::SerializationError,
|
|
73
73
|
"GlobalID model #{gid.model_class} is not in allowed_global_id_models. " \
|
|
74
74
|
"Add it to Pgbus.configuration.allowed_global_id_models to permit deserialization."
|
|
75
75
|
end
|
|
@@ -155,14 +155,6 @@ module Pgbus
|
|
|
155
155
|
ensure
|
|
156
156
|
previous.each { |tl_key, value| Thread.current[tl_key] = value }
|
|
157
157
|
end
|
|
158
|
-
|
|
159
|
-
# Kept for backward compatibility — the class-level durable callbacks
|
|
160
|
-
# (broadcasts_to with durable:) and any external callers may still use it.
|
|
161
|
-
def with_pgbus_durable(value, &)
|
|
162
|
-
return yield if value.nil?
|
|
163
|
-
|
|
164
|
-
with_pgbus_broadcast_opts(durable: value, &)
|
|
165
|
-
end
|
|
166
158
|
end
|
|
167
159
|
end
|
|
168
160
|
end
|
|
@@ -14,10 +14,10 @@ module Pgbus
|
|
|
14
14
|
# `"gid://app/Order/42:messages"`). Verification returns that string;
|
|
15
15
|
# tampered or unsigned input raises `InvalidSignedName`.
|
|
16
16
|
module SignedName
|
|
17
|
-
class InvalidSignedName <
|
|
17
|
+
class InvalidSignedName < Pgbus::Error
|
|
18
18
|
end
|
|
19
19
|
|
|
20
|
-
class MissingSecret <
|
|
20
|
+
class MissingSecret < Pgbus::Error
|
|
21
21
|
end
|
|
22
22
|
|
|
23
23
|
def self.verify!(token)
|
data/lib/pgbus/uniqueness.rb
CHANGED
|
@@ -43,28 +43,28 @@ module Pgbus
|
|
|
43
43
|
|
|
44
44
|
METADATA_KEY = "pgbus_uniqueness_key"
|
|
45
45
|
STRATEGY_KEY = "pgbus_uniqueness_strategy"
|
|
46
|
-
TTL_KEY = "pgbus_uniqueness_lock_ttl"
|
|
47
|
-
|
|
48
|
-
# TTL is kept for metadata compatibility but no longer drives lock expiry.
|
|
49
|
-
# The lock exists until the job completes or is dead-lettered.
|
|
50
|
-
DEFAULT_LOCK_TTL = 24 * 60 * 60
|
|
51
46
|
|
|
52
47
|
VALID_STRATEGIES = %i[until_executed while_executing].freeze
|
|
53
48
|
VALID_CONFLICTS = %i[reject discard log].freeze
|
|
54
49
|
|
|
55
50
|
class_methods do
|
|
56
|
-
def ensures_uniqueness(strategy: :until_executed, key: nil,
|
|
51
|
+
def ensures_uniqueness(strategy: :until_executed, key: nil, on_conflict: :reject, **opts)
|
|
52
|
+
# lock_ttl was validated and stored in message metadata but never read
|
|
53
|
+
# by the lock lifecycle (the lock lives until the job completes or is
|
|
54
|
+
# dead-lettered, not until a TTL expires). Removed in 1.0.0.
|
|
55
|
+
if opts.key?(:lock_ttl)
|
|
56
|
+
raise ArgumentError,
|
|
57
|
+
"lock_ttl: was removed in pgbus 1.0.0 — it was validated but never read by anything. " \
|
|
58
|
+
"Remove it from ensures_uniqueness. See https://pgbus.dev/docs/upgrading-pgbus"
|
|
59
|
+
end
|
|
60
|
+
raise ArgumentError, "unknown keyword: #{opts.keys.first.inspect}" if opts.any?
|
|
57
61
|
raise ArgumentError, "strategy must be one of: #{VALID_STRATEGIES.join(", ")}" unless VALID_STRATEGIES.include?(strategy)
|
|
58
62
|
raise ArgumentError, "on_conflict must be one of: #{VALID_CONFLICTS.join(", ")}" unless VALID_CONFLICTS.include?(on_conflict)
|
|
59
|
-
|
|
60
|
-
valid_ttl = lock_ttl.is_a?(Numeric) || (defined?(ActiveSupport::Duration) && lock_ttl.is_a?(ActiveSupport::Duration))
|
|
61
|
-
raise ArgumentError, "lock_ttl must be a positive number or Duration" unless valid_ttl && lock_ttl.positive?
|
|
62
63
|
raise ArgumentError, "key must be callable (Proc or lambda)" if key && !key.respond_to?(:call)
|
|
63
64
|
|
|
64
65
|
@pgbus_uniqueness = {
|
|
65
66
|
strategy: strategy,
|
|
66
67
|
key: key || ->(*) { name },
|
|
67
|
-
lock_ttl: lock_ttl,
|
|
68
68
|
on_conflict: on_conflict
|
|
69
69
|
}.freeze
|
|
70
70
|
end
|
|
@@ -102,8 +102,7 @@ module Pgbus
|
|
|
102
102
|
|
|
103
103
|
payload_hash.merge(
|
|
104
104
|
METADATA_KEY => key,
|
|
105
|
-
STRATEGY_KEY => config[:strategy].to_s
|
|
106
|
-
TTL_KEY => config[:lock_ttl]
|
|
105
|
+
STRATEGY_KEY => config[:strategy].to_s
|
|
107
106
|
)
|
|
108
107
|
end
|
|
109
108
|
|
data/lib/pgbus/version.rb
CHANGED
|
@@ -1250,15 +1250,31 @@ module Pgbus
|
|
|
1250
1250
|
# stall_threshold applies directly. The dispatcher and scheduler sleep
|
|
1251
1251
|
# between iterations, so their beacon is naturally up to one sleep
|
|
1252
1252
|
# interval stale even when healthy — widen the window by that interval to
|
|
1253
|
-
# avoid false stalls.
|
|
1253
|
+
# avoid false stalls. Consumers idle on an empty-read wait that can reach
|
|
1254
|
+
# NOTIFY_FALLBACK_POLL_SECONDS (15s) when a live listener drives wake-up,
|
|
1255
|
+
# so widen by that max wait when notify is on, or by the poll interval
|
|
1256
|
+
# otherwise. Kinds with no loop beacon (nil) are never stalled.
|
|
1254
1257
|
def stall_threshold_for(kind)
|
|
1255
|
-
|
|
1258
|
+
config = Pgbus.configuration
|
|
1259
|
+
base = config.stall_threshold
|
|
1256
1260
|
return nil unless base&.positive?
|
|
1257
1261
|
|
|
1258
1262
|
case kind
|
|
1259
1263
|
when "worker" then base
|
|
1260
|
-
when "dispatcher" then base +
|
|
1261
|
-
when "scheduler" then base +
|
|
1264
|
+
when "dispatcher" then base + config.dispatch_interval
|
|
1265
|
+
when "scheduler" then base + config.recurring_schedule_interval
|
|
1266
|
+
when "consumer" then base + consumer_stall_slack(config)
|
|
1267
|
+
end
|
|
1268
|
+
end
|
|
1269
|
+
|
|
1270
|
+
# Widen the consumer's stall window by its maximum empty-read wait. With
|
|
1271
|
+
# notify wake-up live, that wait rises to NOTIFY_FALLBACK_POLL_SECONDS as a
|
|
1272
|
+
# safety net; without it, the consumer polls at polling_interval.
|
|
1273
|
+
def consumer_stall_slack(config)
|
|
1274
|
+
if config.worker_notify_wakeup?
|
|
1275
|
+
[config.polling_interval, Pgbus::Process::Consumer::NOTIFY_FALLBACK_POLL_SECONDS].max
|
|
1276
|
+
else
|
|
1277
|
+
config.polling_interval
|
|
1262
1278
|
end
|
|
1263
1279
|
end
|
|
1264
1280
|
|
|
@@ -16,7 +16,7 @@ module Pgbus
|
|
|
16
16
|
FILTERED = "[FILTERED]"
|
|
17
17
|
|
|
18
18
|
def filter(value)
|
|
19
|
-
return value unless Pgbus.configuration.
|
|
19
|
+
return value unless Pgbus.configuration.web_filter_sensitive
|
|
20
20
|
|
|
21
21
|
case value
|
|
22
22
|
when Hash
|
|
@@ -29,7 +29,7 @@ module Pgbus
|
|
|
29
29
|
end
|
|
30
30
|
|
|
31
31
|
def filter_json(value)
|
|
32
|
-
return value unless Pgbus.configuration.
|
|
32
|
+
return value unless Pgbus.configuration.web_filter_sensitive
|
|
33
33
|
|
|
34
34
|
case value
|
|
35
35
|
when nil then nil
|
|
@@ -46,7 +46,7 @@ module Pgbus
|
|
|
46
46
|
end
|
|
47
47
|
|
|
48
48
|
def parameter_filter
|
|
49
|
-
patterns = Pgbus.configuration.
|
|
49
|
+
patterns = Pgbus.configuration.web_filter_parameters ||
|
|
50
50
|
rails_filter_parameters ||
|
|
51
51
|
DEFAULT_FILTER_PATTERNS
|
|
52
52
|
ActiveSupport::ParameterFilter.new(patterns)
|
|
@@ -28,7 +28,8 @@ module Pgbus
|
|
|
28
28
|
pg_connection: nil,
|
|
29
29
|
logger: Pgbus.logger,
|
|
30
30
|
registry: nil,
|
|
31
|
-
dispatch_queue: nil
|
|
31
|
+
dispatch_queue: nil,
|
|
32
|
+
connection_factory: nil
|
|
32
33
|
)
|
|
33
34
|
@client = client
|
|
34
35
|
@config = config
|
|
@@ -46,9 +47,10 @@ module Pgbus
|
|
|
46
47
|
# On reconnect the Listener rebuilds its OWN connection via this
|
|
47
48
|
# factory (fresh connect re-resolves DNS, converges on the promoted
|
|
48
49
|
# primary after a failover) instead of resetting a possibly-dead
|
|
49
|
-
# socket.
|
|
50
|
-
#
|
|
51
|
-
|
|
50
|
+
# socket. Always provided — even when an initial pg_connection: is
|
|
51
|
+
# injected, the reconnect path builds a fresh raw connection. A test
|
|
52
|
+
# can inject its own factory to avoid touching real configuration.
|
|
53
|
+
connection_factory: connection_factory || -> { build_raw_pg_connection }
|
|
52
54
|
)
|
|
53
55
|
@dispatcher = StreamEventDispatcher.new(
|
|
54
56
|
client: @client,
|