pgbus 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.md +36 -14
- data/Rakefile +7 -1
- data/app/assets/javascripts/pgbus/stream_source_element.js +17 -3
- data/app/assets/javascripts/pgbus/stream_source_element.min.js +4 -4
- data/app/assets/javascripts/pgbus/stream_source_element.min.js.map +1 -1
- data/app/controllers/pgbus/dead_letter_controller.rb +3 -1
- data/app/controllers/pgbus/jobs_controller.rb +2 -1
- data/app/models/pgbus/blocked_execution.rb +3 -2
- data/app/models/pgbus/queue_state.rb +6 -0
- data/app/models/pgbus/uniqueness_key.rb +11 -0
- data/config/locales/da.yml +1 -0
- data/config/locales/de.yml +1 -0
- data/config/locales/en.yml +1 -0
- data/config/locales/es.yml +1 -0
- data/config/locales/fi.yml +1 -0
- data/config/locales/fr.yml +1 -0
- data/config/locales/it.yml +1 -0
- data/config/locales/ja.yml +1 -0
- data/config/locales/nb.yml +1 -0
- data/config/locales/nl.yml +1 -0
- data/config/locales/pt.yml +1 -0
- data/config/locales/sv.yml +1 -0
- data/lib/generators/pgbus/fold_queue_name_case_generator.rb +57 -0
- data/lib/generators/pgbus/templates/fold_queue_name_case.rb.erb +12 -0
- data/lib/generators/pgbus/templates/tune_autovacuum.rb.erb +2 -2
- data/lib/generators/pgbus/templates/tune_fillfactor.rb.erb +1 -1
- data/lib/pgbus/active_job/adapter.rb +3 -2
- data/lib/pgbus/active_job/executor.rb +142 -20
- data/lib/pgbus/active_job/quiet_logs.rb +57 -0
- data/lib/pgbus/autovacuum_tuning.rb +2 -2
- data/lib/pgbus/cli/dlq.rb +2 -1
- data/lib/pgbus/client/async_commit.rb +149 -0
- data/lib/pgbus/client/native_statements.rb +28 -16
- data/lib/pgbus/client.rb +145 -17
- data/lib/pgbus/concurrency/blocked_execution.rb +9 -1
- data/lib/pgbus/configuration.rb +71 -6
- data/lib/pgbus/dead_letter_header.rb +3 -0
- data/lib/pgbus/doctor.rb +55 -2
- data/lib/pgbus/failed_event_recorder.rb +5 -2
- data/lib/pgbus/generators/migration_detector.rb +29 -2
- data/lib/pgbus/instrumentation.rb +16 -1
- data/lib/pgbus/metrics/subscriber.rb +27 -1
- data/lib/pgbus/process/completer.rb +4 -2
- data/lib/pgbus/process/worker.rb +28 -3
- data/lib/pgbus/queue_name_fold.rb +145 -0
- data/lib/pgbus/queue_name_validator.rb +20 -2
- data/lib/pgbus/recurring/schedule.rb +22 -13
- data/lib/pgbus/stat_buffer.rb +67 -26
- data/lib/pgbus/streams/cursor.rb +17 -6
- data/lib/pgbus/streams/envelope.rb +16 -0
- data/lib/pgbus/streams/key.rb +1 -1
- data/lib/pgbus/table_maintenance.rb +1 -1
- data/lib/pgbus/uniqueness.rb +43 -16
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source/dead_letter.rb +14 -5
- data/lib/pgbus/web/data_source/resend.rb +185 -0
- data/lib/pgbus/web/data_source/resend_slot.rb +139 -0
- data/lib/pgbus/web/data_source.rb +55 -37
- data/lib/pgbus/web/dead_letter_reason.rb +6 -0
- data/lib/pgbus/web/streamer/connection.rb +36 -29
- data/lib/pgbus/web/streamer/falcon_connection.rb +29 -25
- data/lib/pgbus/web/streamer/heartbeat.rb +14 -3
- data/lib/pgbus/web/streamer/instance.rb +8 -5
- data/lib/pgbus/web/streamer/listener.rb +239 -62
- data/lib/pgbus/web/streamer/outbound_pump.rb +6 -4
- data/lib/pgbus/web/streamer/registry.rb +55 -29
- data/lib/pgbus/web/streamer/stream_event_dispatcher.rb +73 -12
- data/lib/tasks/pgbus_queues.rake +27 -0
- data/sig/pgbus/active_job/adapter.rbs +1 -1
- data/sig/pgbus/active_job/quiet_logs.rbs +11 -0
- data/sig/pgbus/client.rbs +6 -5
- data/sig/pgbus/configuration.rbs +4 -0
- metadata +11 -3
data/lib/pgbus/doctor.rb
CHANGED
|
@@ -5,7 +5,7 @@ require "pgbus/mcp/health_analyzer"
|
|
|
5
5
|
|
|
6
6
|
module Pgbus
|
|
7
7
|
# Preflight diagnostics for a pgbus deployment — the single command that
|
|
8
|
-
# answers "is this environment healthy enough to run?". Runs
|
|
8
|
+
# answers "is this environment healthy enough to run?". Runs fourteen checks and
|
|
9
9
|
# returns a machine-readable result plus a human report, so `pgbus doctor`
|
|
10
10
|
# and `rake pgbus:doctor` can gate a deploy or CI run (exit 0 on success,
|
|
11
11
|
# 1 on any failure).
|
|
@@ -36,13 +36,15 @@ module Pgbus
|
|
|
36
36
|
"Database" => :check_database,
|
|
37
37
|
"PGMQ schema" => :check_pgmq_schema,
|
|
38
38
|
"Queues" => :check_queues,
|
|
39
|
+
"Queue name case" => :check_queue_name_case,
|
|
39
40
|
"LISTEN/NOTIFY" => :check_notify,
|
|
40
41
|
"Process liveness" => :check_processes,
|
|
41
42
|
"GlobalID allowlist" => :check_allowed_global_id_models,
|
|
42
43
|
"Broadcast queue" => :check_broadcast_queue,
|
|
43
44
|
"Primary affinity" => :check_primary,
|
|
44
45
|
"Dedicated connections" => :check_dedicated_connections,
|
|
45
|
-
"Connection budget" => :check_connection_budget
|
|
46
|
+
"Connection budget" => :check_connection_budget,
|
|
47
|
+
"Completion commit" => :check_completion_commit
|
|
46
48
|
}.freeze
|
|
47
49
|
|
|
48
50
|
# Process liveness reads the pgbus_processes table (via HealthAnalyzer), so
|
|
@@ -128,6 +130,7 @@ module Pgbus
|
|
|
128
130
|
resolved_pool_size: safe { @config.resolved_pool_size },
|
|
129
131
|
listen_notify: @config.listen_notify,
|
|
130
132
|
roles: @config.roles || "all",
|
|
133
|
+
completion_commit: @config.completion_synchronous_commit ? "sync" : "async",
|
|
131
134
|
capsules: capsule_summary,
|
|
132
135
|
database_url: redacted_database_url,
|
|
133
136
|
connection_params: redacted_connection_params
|
|
@@ -252,6 +255,30 @@ module Pgbus
|
|
|
252
255
|
Check.new(name: "Queues", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
253
256
|
end
|
|
254
257
|
|
|
258
|
+
# How many mixed-case queue names the queue-name case check lists.
|
|
259
|
+
CASE_NAMES_SHOWN = 5
|
|
260
|
+
private_constant :CASE_NAMES_SHOWN
|
|
261
|
+
|
|
262
|
+
# 4b. Queue-name case (issue #633). PGMQ spells every queue table in lower
|
|
263
|
+
# case, and pgbus now spells every queue name that way; a pgmq.meta row an
|
|
264
|
+
# older pgbus created as "pgbus_Orders" shares pgmq.q_pgbus_orders with the
|
|
265
|
+
# current spelling. Its messages are reachable, so this warns rather than
|
|
266
|
+
# fails: the dashboard lists the queue twice until the fold migration runs.
|
|
267
|
+
def check_queue_name_case
|
|
268
|
+
names = @client.mixed_case_queue_names
|
|
269
|
+
return Check.new(name: "Queue name case", status: :ok, detail: "every queue is spelled in lower case") if names.empty?
|
|
270
|
+
|
|
271
|
+
shown = names.first(CASE_NAMES_SHOWN).join(", ")
|
|
272
|
+
shown += ", …" if names.size > CASE_NAMES_SHOWN
|
|
273
|
+
Check.new(name: "Queue name case", status: :warn,
|
|
274
|
+
detail: "#{names.size} queue(s) in pgmq.meta spelled in mixed case (#{shown}); their tables are " \
|
|
275
|
+
"lower case and pgbus now uses that spelling — once every process runs this pgbus, run " \
|
|
276
|
+
"`rails generate pgbus:fold_queue_name_case` and migrate " \
|
|
277
|
+
"(`rake pgbus:queues:fold_case` once that migration has run)")
|
|
278
|
+
rescue StandardError => e
|
|
279
|
+
Check.new(name: "Queue name case", status: :warn, detail: "could not read pgmq.meta (#{e.class}: #{e.message})")
|
|
280
|
+
end
|
|
281
|
+
|
|
255
282
|
# 5. LISTEN/NOTIFY — when configured on, every configured queue should carry
|
|
256
283
|
# the insert-NOTIFY trigger. A missing trigger is a warning, not a failure:
|
|
257
284
|
# pgbus still works (it falls back to polling), it just loses instant wakeup.
|
|
@@ -395,6 +422,32 @@ module Pgbus
|
|
|
395
422
|
Check.new(name: "Dedicated connections", status: :fail, detail: "#{e.class}: #{e.message}")
|
|
396
423
|
end
|
|
397
424
|
|
|
425
|
+
# config.completion_synchronous_commit (issue #619): informational, except
|
|
426
|
+
# that false does nothing with native_statements off or on ActiveRecord's
|
|
427
|
+
# shared connection, which warns.
|
|
428
|
+
def check_completion_commit
|
|
429
|
+
return Check.new(name: "Completion commit", status: :ok, detail: "synchronous (default)") if
|
|
430
|
+
@config.completion_synchronous_commit
|
|
431
|
+
|
|
432
|
+
unless @config.native_statements
|
|
433
|
+
return Check.new(name: "Completion commit", status: :warn,
|
|
434
|
+
detail: "completion_synchronous_commit = false has no effect: completions run through " \
|
|
435
|
+
"pgmq's functions (native_statements = false)")
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
if @client.respond_to?(:shared_connection?) && @client.shared_connection?
|
|
439
|
+
return Check.new(name: "Completion commit", status: :warn,
|
|
440
|
+
detail: "completion_synchronous_commit = false has no effect: pgbus shares ActiveRecord's " \
|
|
441
|
+
"connection (connection_params Proc); give pgbus its own connections to use it")
|
|
442
|
+
end
|
|
443
|
+
|
|
444
|
+
Check.new(name: "Completion commit", status: :ok,
|
|
445
|
+
detail: "asynchronous: completions commit with synchronous_commit off; a Postgres server crash can " \
|
|
446
|
+
"lose the last completions, whose jobs run again after visibility_timeout")
|
|
447
|
+
rescue StandardError => e
|
|
448
|
+
Check.new(name: "Completion commit", status: :warn, detail: "#{e.class}: #{e.message}")
|
|
449
|
+
end
|
|
450
|
+
|
|
398
451
|
# How many direct LISTEN connections this config pins at steady state
|
|
399
452
|
# (issue #381) — informational (always :ok) so operators can do capacity
|
|
400
453
|
# math on the pooler's direct-connection budget from the doctor output.
|
|
@@ -85,11 +85,14 @@ module Pgbus
|
|
|
85
85
|
# queue is dropped: PGMQ restarts msg_ids on recreate, so a surviving
|
|
86
86
|
# row would aim the dashboard's retry/discard at an unrelated message.
|
|
87
87
|
# Queue names are validated word characters, so the array literal is safe.
|
|
88
|
+
# Matched in any case (issue #633): the dropped queue's name is lower
|
|
89
|
+
# case, while a row may carry the executor's configured spelling
|
|
90
|
+
# ("Critical") or a physical name from before #633.
|
|
88
91
|
def clear_queue!(queue_names)
|
|
89
92
|
connection.exec_delete(
|
|
90
|
-
"DELETE FROM pgbus_failed_events WHERE queue_name = ANY($1::text[])",
|
|
93
|
+
"DELETE FROM pgbus_failed_events WHERE lower(queue_name) = ANY($1::text[])",
|
|
91
94
|
"FailedEvent Clear Queue",
|
|
92
|
-
["{#{queue_names.join(",")}}"]
|
|
95
|
+
["{#{queue_names.map(&:downcase).join(",")}}"]
|
|
93
96
|
)
|
|
94
97
|
rescue StandardError => e
|
|
95
98
|
Pgbus.logger.error do
|
|
@@ -102,7 +102,8 @@ module Pgbus
|
|
|
102
102
|
add_batch_executions: "pgbus:add_batch_executions",
|
|
103
103
|
add_batch_callback_jobs: "pgbus:add_batch_callback_jobs",
|
|
104
104
|
tune_autovacuum: "pgbus:tune_autovacuum",
|
|
105
|
-
tune_fillfactor: "pgbus:tune_fillfactor"
|
|
105
|
+
tune_fillfactor: "pgbus:tune_fillfactor",
|
|
106
|
+
fold_queue_name_case: "pgbus:fold_queue_name_case"
|
|
106
107
|
}.freeze
|
|
107
108
|
|
|
108
109
|
# Human-friendly description of each migration for the generator
|
|
@@ -128,9 +129,15 @@ module Pgbus
|
|
|
128
129
|
add_batch_executions: "batch execution-row tracking (self-healing completion, on_failure rename)",
|
|
129
130
|
add_batch_callback_jobs: "batch callback-job columns (configured ActiveJob instances as batch callbacks)",
|
|
130
131
|
tune_autovacuum: "autovacuum tuning for PGMQ queue and archive tables",
|
|
131
|
-
tune_fillfactor: "fillfactor=70 on PGMQ queue tables (reduces page density during update churn)"
|
|
132
|
+
tune_fillfactor: "fillfactor=70 on PGMQ queue tables (reduces page density during update churn)",
|
|
133
|
+
fold_queue_name_case: "lower-case spelling for queues created in mixed case (PGMQ's table names are lower case)"
|
|
132
134
|
}.freeze
|
|
133
135
|
|
|
136
|
+
# A pgbus pgmq.meta row ($1: the lower-case "<queue_prefix>_") spelled
|
|
137
|
+
# with an upper-case letter (issue #633).
|
|
138
|
+
MIXED_CASE_QUEUE_SQL = "SELECT 1 FROM pgmq.meta WHERE queue_name <> lower(queue_name) " \
|
|
139
|
+
"AND left(lower(queue_name), length($1)) = $1 LIMIT 1"
|
|
140
|
+
|
|
134
141
|
def initialize(connection)
|
|
135
142
|
@connection = connection
|
|
136
143
|
end
|
|
@@ -143,6 +150,9 @@ module Pgbus
|
|
|
143
150
|
return [FRESH_INSTALL] if fresh_install?
|
|
144
151
|
|
|
145
152
|
[
|
|
153
|
+
# First: the tuning migrations below build table names from
|
|
154
|
+
# pgmq.meta, which a mixed-case row spells unlike its table.
|
|
155
|
+
*queue_name_case_migrations,
|
|
146
156
|
*uniqueness_key_migrations,
|
|
147
157
|
*job_stats_migrations,
|
|
148
158
|
*stream_stats_migrations,
|
|
@@ -328,6 +338,23 @@ module Pgbus
|
|
|
328
338
|
[:tune_autovacuum]
|
|
329
339
|
end
|
|
330
340
|
|
|
341
|
+
# Issue #633: a queue an older pgbus created in mixed case shares its
|
|
342
|
+
# lower-case table with the spelling pgbus now uses. An unreadable
|
|
343
|
+
# pgmq.meta queues nothing: the doctor's "Queue name case" check says why.
|
|
344
|
+
def queue_name_case_migrations
|
|
345
|
+
return [] unless pgmq_schema_exists?
|
|
346
|
+
return [] unless mixed_case_queue?
|
|
347
|
+
|
|
348
|
+
[:fold_queue_name_case]
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
def mixed_case_queue?
|
|
352
|
+
prefix = "#{Pgbus.configuration.queue_prefix.to_s.downcase}_"
|
|
353
|
+
connection.select_value(MIXED_CASE_QUEUE_SQL.gsub("$1", connection.quote(prefix))).present?
|
|
354
|
+
rescue StandardError
|
|
355
|
+
false
|
|
356
|
+
end
|
|
357
|
+
|
|
331
358
|
# Fillfactor tuning: check if any PGMQ queue table already has
|
|
332
359
|
# fillfactor applied. If not, queue the migration.
|
|
333
360
|
def fillfactor_migrations
|
|
@@ -16,7 +16,22 @@ module Pgbus
|
|
|
16
16
|
# pgbus.job_completed — job archived successfully
|
|
17
17
|
# pgbus.job_failed — job raised; carries :exception_object
|
|
18
18
|
# pgbus.job_dead_lettered — job exceeded max_retries and was DLQ-routed;
|
|
19
|
-
# :error is the last recorded error class (or nil)
|
|
19
|
+
# :error is the last recorded error class (or nil);
|
|
20
|
+
# :reason is max_retries_exceeded, or uniqueness_conflict
|
|
21
|
+
# for a :while_executing copy with no failed attempt whose
|
|
22
|
+
# key's row is not its own (skipped while another message
|
|
23
|
+
# held the key); also uniqueness_conflict when that row
|
|
24
|
+
# lookup itself fails
|
|
25
|
+
# pgbus.job_skipped — a :while_executing copy was read while another
|
|
26
|
+
# message held its key and left for a later delivery
|
|
27
|
+
# (on_conflict :reject or :log)
|
|
28
|
+
# payload: queue, job_class, job_id, provider_job_id,
|
|
29
|
+
# msg_id, read_ct, on_conflict, vt (seconds until it
|
|
30
|
+
# is delivered again)
|
|
31
|
+
# pgbus.job_discarded — a copy was completed without running
|
|
32
|
+
# payload: queue, job_class, job_id, provider_job_id,
|
|
33
|
+
# msg_id, reason ("uniqueness": a :while_executing copy
|
|
34
|
+
# under on_conflict: :discard)
|
|
20
35
|
# pgbus.job_visibility_extended — heartbeat re-armed a running job's visibility timeout
|
|
21
36
|
# payload: queue, job_class, msg_id, vt, extensions
|
|
22
37
|
# pgbus.event_processed — event handler succeeded
|
|
@@ -60,6 +60,8 @@ module Pgbus
|
|
|
60
60
|
subscribe("pgbus.job_completed") { |event| on_job_completed(event) },
|
|
61
61
|
subscribe("pgbus.job_failed") { |event| on_job_failed(event) },
|
|
62
62
|
subscribe("pgbus.job_dead_lettered") { |event| on_job_dead_lettered(event) },
|
|
63
|
+
subscribe("pgbus.job_skipped") { |event| on_job_skipped(event) },
|
|
64
|
+
subscribe("pgbus.job_discarded") { |event| on_job_discarded(event) },
|
|
63
65
|
subscribe("pgbus.job_visibility_extended") { |event| on_job_visibility_extended(event) },
|
|
64
66
|
subscribe("pgbus.event_processed") { |event| on_event_processed(event) },
|
|
65
67
|
subscribe("pgbus.event_failed") { |event| on_event_failed(event) },
|
|
@@ -121,11 +123,35 @@ module Pgbus
|
|
|
121
123
|
)
|
|
122
124
|
end
|
|
123
125
|
|
|
126
|
+
# reason: max_retries_exceeded, or uniqueness_conflict for a
|
|
127
|
+
# :while_executing duplicate skipped until it ran out of reads (#629).
|
|
124
128
|
def on_job_dead_lettered(event)
|
|
125
129
|
payload = event.payload
|
|
126
130
|
backend.increment(
|
|
127
131
|
"#{METRIC_PREFIX}queue_job_count", 1,
|
|
128
|
-
compact(queue: payload[:queue], job_class: payload[:job_class], status: "dead_lettered"
|
|
132
|
+
compact(queue: payload[:queue], job_class: payload[:job_class], status: "dead_lettered",
|
|
133
|
+
reason: payload[:reason])
|
|
134
|
+
)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# A :while_executing duplicate left for a later delivery because
|
|
138
|
+
# another message held its key (issue #629): counted per skip, so one
|
|
139
|
+
# copy waiting out a long holder counts several times.
|
|
140
|
+
def on_job_skipped(event)
|
|
141
|
+
payload = event.payload
|
|
142
|
+
backend.increment(
|
|
143
|
+
"#{METRIC_PREFIX}queue_job_count", 1,
|
|
144
|
+
compact(queue: payload[:queue], job_class: payload[:job_class], status: "skipped", reason: "uniqueness")
|
|
145
|
+
)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# A :while_executing duplicate dropped under on_conflict: :discard.
|
|
149
|
+
def on_job_discarded(event)
|
|
150
|
+
payload = event.payload
|
|
151
|
+
backend.increment(
|
|
152
|
+
"#{METRIC_PREFIX}queue_job_count", 1,
|
|
153
|
+
compact(queue: payload[:queue], job_class: payload[:job_class], status: "discarded",
|
|
154
|
+
reason: payload[:reason])
|
|
129
155
|
)
|
|
130
156
|
end
|
|
131
157
|
|
|
@@ -94,7 +94,8 @@ module Pgbus
|
|
|
94
94
|
return waiter.value if waiter
|
|
95
95
|
|
|
96
96
|
# Stopped: complete on the caller's thread, with the same results.
|
|
97
|
-
@client.public_send(@batch_op, queue, [msg_id], prefixed: false
|
|
97
|
+
done = @client.public_send(@batch_op, queue, [msg_id], prefixed: false, job_completion: true)
|
|
98
|
+
done.empty? ? :already_archived : :archived
|
|
98
99
|
end
|
|
99
100
|
|
|
100
101
|
# Flushes whatever is pending, then joins the flusher. Past timeout the
|
|
@@ -157,7 +158,8 @@ module Pgbus
|
|
|
157
158
|
# is absent. A duplicated id (the same message held by two threads after
|
|
158
159
|
# a redelivery) is archived once, so only its first waiter gets :archived.
|
|
159
160
|
def flush_slice(queue, slice)
|
|
160
|
-
archived = @client.public_send(@batch_op, queue, slice.map(&:first).uniq, prefixed: false)
|
|
161
|
+
archived = @client.public_send(@batch_op, queue, slice.map(&:first).uniq, prefixed: false, job_completion: true)
|
|
162
|
+
.to_set(&:to_i)
|
|
161
163
|
slice.each { |msg_id, waiter| waiter.resolve(archived.delete?(msg_id) ? :archived : :already_archived) }
|
|
162
164
|
rescue Exception => e # rubocop:disable Lint/RescueException -- delivered to the waiters, never swallowed
|
|
163
165
|
log_flush_failure(queue, slice, e) unless e.is_a?(StandardError)
|
data/lib/pgbus/process/worker.rb
CHANGED
|
@@ -194,6 +194,7 @@ module Pgbus
|
|
|
194
194
|
POOL_TERMINATION_WAIT = 5
|
|
195
195
|
|
|
196
196
|
def run
|
|
197
|
+
quiet_active_job_logs
|
|
197
198
|
setup_signals
|
|
198
199
|
start_heartbeat
|
|
199
200
|
resolve_wildcard_queues
|
|
@@ -282,6 +283,18 @@ module Pgbus
|
|
|
282
283
|
|
|
283
284
|
private
|
|
284
285
|
|
|
286
|
+
# config.quiet_active_job_logs (issue #618): only worker processes perform
|
|
287
|
+
# jobs, so only they drop the per-job "Performing …" / "Performed …"
|
|
288
|
+
# lines. ActiveJob::LogSubscriber attaches when active_job/base loads,
|
|
289
|
+
# which an app with no job class to eager-load has not done yet, so load
|
|
290
|
+
# it first (defined? does not trigger ActiveJob's autoload of Base).
|
|
291
|
+
def quiet_active_job_logs
|
|
292
|
+
return unless config.quiet_active_job_logs && defined?(::ActiveJob::Base)
|
|
293
|
+
|
|
294
|
+
require "active_job/base"
|
|
295
|
+
Pgbus::ActiveJob::QuietLogs.apply!
|
|
296
|
+
end
|
|
297
|
+
|
|
285
298
|
# One loop step (issue #486): feed buffered claims to free slots, read
|
|
286
299
|
# the deficit (free slots + read_ahead - buffered, capped by
|
|
287
300
|
# prefetch_limit), feed again. With read_ahead 0 the buffer never
|
|
@@ -510,11 +523,18 @@ module Pgbus
|
|
|
510
523
|
def process_message(message, queue_name, source_queue: nil)
|
|
511
524
|
result = @executor.execute(message, queue_name, source_queue: source_queue)
|
|
512
525
|
@jobs_processed.increment
|
|
513
|
-
|
|
514
|
-
|
|
526
|
+
# A skipped uniqueness duplicate comes back later: counting it now
|
|
527
|
+
# would count one job twice (issue #629).
|
|
528
|
+
@rate_counter.increment(:processed) unless result == :skipped
|
|
529
|
+
case result
|
|
530
|
+
when :failed
|
|
515
531
|
@jobs_failed.increment
|
|
516
532
|
@rate_counter.increment(:failed)
|
|
517
533
|
@circuit_breaker.record_failure(queue_name)
|
|
534
|
+
when :skipped, :discarded
|
|
535
|
+
# No job ran: neither a failure nor a success that would wipe the
|
|
536
|
+
# failure streak of a queue whose real jobs keep failing.
|
|
537
|
+
nil
|
|
518
538
|
else
|
|
519
539
|
@circuit_breaker.record_success(queue_name)
|
|
520
540
|
end
|
|
@@ -569,7 +589,12 @@ module Pgbus
|
|
|
569
589
|
prefix = "#{config.queue_prefix}_"
|
|
570
590
|
if (match = MISSING_QUEUE_REGEX.match(error.message))
|
|
571
591
|
physical_name = match[1]
|
|
572
|
-
|
|
592
|
+
# Postgres names the table as PGMQ spells it (lower case, issue #633;
|
|
593
|
+
# separators normalized), so find the queue as configured.
|
|
594
|
+
# A priority sub-queue (_pN) belongs to its base queue.
|
|
595
|
+
logical_name = @queues.find { |q| config.priority_queue_names(q).include?(physical_name) } ||
|
|
596
|
+
@queues.find { |q| config.queue_name(q) == physical_name } ||
|
|
597
|
+
physical_name.delete_prefix(prefix)
|
|
573
598
|
if @queues.delete(logical_name)
|
|
574
599
|
Pgbus.logger.warn { "[Pgbus] Evicted deleted queue '#{logical_name}' (#{physical_name}) from worker" }
|
|
575
600
|
end
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
# Issue #633 upgrade path. PGMQ lower-cases its table names
|
|
5
|
+
# (pgmq.format_table_name), so a queue an older pgbus created as
|
|
6
|
+
# "pgbus_Orders" lives in pgmq.q_pgbus_orders while pgmq.meta records
|
|
7
|
+
# "pgbus_Orders". pgbus now spells every queue in lower case; this renames
|
|
8
|
+
# the old rows to match their tables.
|
|
9
|
+
#
|
|
10
|
+
# Only rows change. The queue tables are shared by both spellings, so nothing
|
|
11
|
+
# here drops, renames or copies a table: dropping either spelling's queue
|
|
12
|
+
# through PGMQ would remove the shared table.
|
|
13
|
+
#
|
|
14
|
+
# Run by the pgbus:fold_queue_name_case migration and by
|
|
15
|
+
# `rake pgbus:queues:fold_case` (for a row an older process recreated after
|
|
16
|
+
# the migration ran). Idempotent. Takes an ActiveRecord connection and
|
|
17
|
+
# folds pgbus's queues only (those named "<prefix>_…" in any case), so
|
|
18
|
+
# another PGMQ user of the same schema keeps its spelling and its NOTIFY
|
|
19
|
+
# channel.
|
|
20
|
+
module QueueNameFold
|
|
21
|
+
module_function
|
|
22
|
+
|
|
23
|
+
def run!(connection, prefix: Pgbus.configuration.queue_prefix)
|
|
24
|
+
fold_pgmq_meta(connection, "#{prefix.to_s.downcase}_")
|
|
25
|
+
fold_unique_column(connection, :pgbus_stream_queues, prefer: "")
|
|
26
|
+
fold_unique_column(connection, :pgbus_queue_states, prefer: "paused DESC,")
|
|
27
|
+
fold_column(connection, :pgbus_uniqueness_keys, :queue_name)
|
|
28
|
+
fold_failed_event_source_queues(connection)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# One mixed-case pgmq.meta row at a time: add the lower-case row, move its
|
|
32
|
+
# topic bindings, delete the old row (and its throttle and binding rows),
|
|
33
|
+
# then re-arm the insert-NOTIFY trigger under the new name. PGMQ 1.13.1's
|
|
34
|
+
# trigger NOTIFYs on the channel of the name it was armed with, which
|
|
35
|
+
# listeners now spell in lower case. Where both spellings exist, the
|
|
36
|
+
# lower-case row's throttle wins.
|
|
37
|
+
def fold_pgmq_meta(connection, own_prefix)
|
|
38
|
+
prefix = connection.quote(own_prefix)
|
|
39
|
+
connection.execute(<<~SQL)
|
|
40
|
+
DO $$
|
|
41
|
+
DECLARE
|
|
42
|
+
q RECORD;
|
|
43
|
+
folded TEXT;
|
|
44
|
+
throttle INTEGER;
|
|
45
|
+
has_throttle BOOLEAN := to_regclass('pgmq.notify_insert_throttle') IS NOT NULL;
|
|
46
|
+
has_bindings BOOLEAN := to_regclass('pgmq.topic_bindings') IS NOT NULL;
|
|
47
|
+
BEGIN
|
|
48
|
+
FOR q IN
|
|
49
|
+
SELECT queue_name, is_partitioned, is_unlogged, created_at
|
|
50
|
+
FROM pgmq.meta
|
|
51
|
+
WHERE queue_name <> lower(queue_name)
|
|
52
|
+
AND left(lower(queue_name), length(#{prefix})) = #{prefix}
|
|
53
|
+
ORDER BY queue_name
|
|
54
|
+
LOOP
|
|
55
|
+
folded := lower(q.queue_name);
|
|
56
|
+
|
|
57
|
+
INSERT INTO pgmq.meta (queue_name, is_partitioned, is_unlogged, created_at)
|
|
58
|
+
VALUES (folded, q.is_partitioned, q.is_unlogged, q.created_at)
|
|
59
|
+
ON CONFLICT DO NOTHING;
|
|
60
|
+
|
|
61
|
+
throttle := NULL;
|
|
62
|
+
IF has_throttle THEN
|
|
63
|
+
SELECT COALESCE(
|
|
64
|
+
(SELECT throttle_interval_ms FROM pgmq.notify_insert_throttle WHERE queue_name = folded),
|
|
65
|
+
(SELECT throttle_interval_ms FROM pgmq.notify_insert_throttle WHERE queue_name = q.queue_name)
|
|
66
|
+
) INTO throttle;
|
|
67
|
+
DELETE FROM pgmq.notify_insert_throttle WHERE queue_name = q.queue_name;
|
|
68
|
+
END IF;
|
|
69
|
+
|
|
70
|
+
IF has_bindings THEN
|
|
71
|
+
INSERT INTO pgmq.topic_bindings (pattern, queue_name, bound_at)
|
|
72
|
+
SELECT pattern, folded, bound_at FROM pgmq.topic_bindings WHERE queue_name = q.queue_name
|
|
73
|
+
ON CONFLICT DO NOTHING;
|
|
74
|
+
DELETE FROM pgmq.topic_bindings WHERE queue_name = q.queue_name;
|
|
75
|
+
END IF;
|
|
76
|
+
|
|
77
|
+
DELETE FROM pgmq.meta WHERE queue_name = q.queue_name;
|
|
78
|
+
|
|
79
|
+
IF throttle IS NOT NULL THEN
|
|
80
|
+
PERFORM pgmq.enable_notify_insert(folded, throttle);
|
|
81
|
+
END IF;
|
|
82
|
+
END LOOP;
|
|
83
|
+
END $$;
|
|
84
|
+
SQL
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# A column with a unique index: keep one row per lower-case name (the
|
|
88
|
+
# `prefer` order first, then the row already in lower case, then the
|
|
89
|
+
# oldest), delete the rest, then lower-case what is left.
|
|
90
|
+
def fold_unique_column(connection, table, prefer:)
|
|
91
|
+
return unless connection.table_exists?(table)
|
|
92
|
+
|
|
93
|
+
connection.execute(<<~SQL)
|
|
94
|
+
DELETE FROM #{table} WHERE id IN (
|
|
95
|
+
SELECT id FROM (
|
|
96
|
+
SELECT id, row_number() OVER (
|
|
97
|
+
PARTITION BY lower(queue_name)
|
|
98
|
+
ORDER BY #{prefer} (queue_name = lower(queue_name)) DESC, id
|
|
99
|
+
) AS rank
|
|
100
|
+
FROM #{table}
|
|
101
|
+
WHERE lower(queue_name) IN (SELECT lower(queue_name) FROM #{table} WHERE queue_name <> lower(queue_name))
|
|
102
|
+
) ranked
|
|
103
|
+
WHERE rank > 1
|
|
104
|
+
)
|
|
105
|
+
SQL
|
|
106
|
+
fold_column(connection, table, :queue_name)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def fold_column(connection, table, column)
|
|
110
|
+
return unless connection.table_exists?(table) && connection.column_exists?(table, column)
|
|
111
|
+
|
|
112
|
+
connection.execute("UPDATE #{table} SET #{column} = lower(#{column}) WHERE #{column} <> lower(#{column})")
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# source_queue is the physical queue a failure was read from, now spelled in
|
|
116
|
+
# lower case. queue_name is the executor's logical name, as configured, and
|
|
117
|
+
# stays. A message that failed under both spellings (before and after the
|
|
118
|
+
# deploy) has two rows: one per (queue_name, msg_id, folded source) stays,
|
|
119
|
+
# the lower-case one if there is one (written after the deploy), else the
|
|
120
|
+
# newest, so the (queue_name, msg_id, source_queue) unique index holds.
|
|
121
|
+
def fold_failed_event_source_queues(connection)
|
|
122
|
+
return unless connection.table_exists?(:pgbus_failed_events) && connection.column_exists?(:pgbus_failed_events, :source_queue)
|
|
123
|
+
|
|
124
|
+
connection.execute(<<~SQL)
|
|
125
|
+
DELETE FROM pgbus_failed_events WHERE id IN (
|
|
126
|
+
SELECT id FROM (
|
|
127
|
+
SELECT id, row_number() OVER (
|
|
128
|
+
PARTITION BY queue_name, msg_id, lower(source_queue)
|
|
129
|
+
ORDER BY (source_queue = lower(source_queue)) DESC, id DESC
|
|
130
|
+
) AS rank
|
|
131
|
+
FROM pgbus_failed_events
|
|
132
|
+
WHERE (queue_name, msg_id, lower(source_queue)) IN (
|
|
133
|
+
SELECT queue_name, msg_id, lower(source_queue) FROM pgbus_failed_events
|
|
134
|
+
WHERE source_queue <> lower(source_queue)
|
|
135
|
+
)
|
|
136
|
+
) ranked
|
|
137
|
+
WHERE rank > 1
|
|
138
|
+
)
|
|
139
|
+
SQL
|
|
140
|
+
fold_column(connection, :pgbus_failed_events, :source_queue)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
private_class_method :fold_pgmq_meta, :fold_unique_column, :fold_column, :fold_failed_event_source_queues
|
|
144
|
+
end
|
|
145
|
+
end
|
|
@@ -9,7 +9,9 @@ module Pgbus
|
|
|
9
9
|
module QueueNameValidator
|
|
10
10
|
# PostgreSQL's NAMEDATALEN caps identifiers at 63 bytes, but the
|
|
11
11
|
# effective limit is tighter: pgmq-ruby (our transport gem) rejects
|
|
12
|
-
# any queue name with `length >=
|
|
12
|
+
# any queue name with `length >= PGMQ::QueueName::MAX_LENGTH` (48) in
|
|
13
|
+
# `PGMQ::QueueName.validate!`, whose error has said "maximum length of
|
|
14
|
+
# 47 characters" since pgmq-ruby 0.8.2 (48 before).
|
|
13
15
|
# That leaves an actual usable ceiling of 47 characters for the
|
|
14
16
|
# fully-prefixed name (`<queue_prefix>_<logical_name>`), which is what
|
|
15
17
|
# this constant expresses. pgmq-ruby picked 48 to leave headroom for
|
|
@@ -26,6 +28,10 @@ module Pgbus
|
|
|
26
28
|
# Only alphanumeric characters and underscores are allowed.
|
|
27
29
|
VALID_QUEUE_NAME_PATTERN = /\A[a-zA-Z0-9_]+\z/
|
|
28
30
|
|
|
31
|
+
# A valid name already in lower case: what normalize returns.
|
|
32
|
+
CANONICAL_QUEUE_NAME = /\A[a-z0-9_]+\z/
|
|
33
|
+
private_constant :CANONICAL_QUEUE_NAME
|
|
34
|
+
|
|
29
35
|
module_function
|
|
30
36
|
|
|
31
37
|
# Validates a queue name for safe SQL identifier use.
|
|
@@ -58,14 +64,26 @@ module Pgbus
|
|
|
58
64
|
# (`Pgbus.stream([user, :notifications])` → `"user_gid:notifications"`),
|
|
59
65
|
# so they must map to a safe character rather than be stripped —
|
|
60
66
|
# otherwise `"a:b"` and `"ab"` would collide on the same queue.
|
|
67
|
+
#
|
|
68
|
+
# The result is lower case (issue #633): PGMQ lower-cases its table
|
|
69
|
+
# names (`pgmq.format_table_name`), so "Orders" and "orders" share
|
|
70
|
+
# `pgmq.q_orders` while `pgmq.meta` records each spelling as its own
|
|
71
|
+
# queue. Spelling every name the way its table is gives one name per
|
|
72
|
+
# table. `validate!` and `sanitize!` keep case: they also run on names
|
|
73
|
+
# read back from `pgmq.meta`, which an install from before #633 may
|
|
74
|
+
# still hold in mixed case (`rails generate pgbus:fold_queue_name_case`).
|
|
61
75
|
def normalize(name)
|
|
62
76
|
name = name.to_s
|
|
63
|
-
|
|
77
|
+
# Every enqueue and read names its queue through here: a name already
|
|
78
|
+
# spelled as its table costs one scan and no allocation.
|
|
79
|
+
return name if name.length <= MAX_QUEUE_NAME_LENGTH && CANONICAL_QUEUE_NAME.match?(name)
|
|
80
|
+
return validate!(name).downcase if VALID_QUEUE_NAME_PATTERN.match?(name)
|
|
64
81
|
|
|
65
82
|
normalized = name.gsub(/[-.:]/, "_") # hyphens/dots/colons → underscores
|
|
66
83
|
.gsub(/[^a-zA-Z0-9_]/, "") # strip remaining invalid chars
|
|
67
84
|
.gsub(/_+/, "_") # collapse consecutive underscores
|
|
68
85
|
.gsub(/\A_|_\z/, "") # strip leading/trailing underscores
|
|
86
|
+
normalized.downcase!
|
|
69
87
|
validate!(normalized)
|
|
70
88
|
normalized
|
|
71
89
|
end
|
|
@@ -26,7 +26,7 @@ module Pgbus
|
|
|
26
26
|
RecurringExecution.record(task.key, run_at) do
|
|
27
27
|
payload = build_payload(task)
|
|
28
28
|
headers = build_headers(task, run_at)
|
|
29
|
-
payload = inject_uniqueness_metadata(task, payload)
|
|
29
|
+
payload = inject_uniqueness_metadata(task, payload, acquired_key)
|
|
30
30
|
|
|
31
31
|
Pgbus.client.ensure_queue(queue)
|
|
32
32
|
msg_id = Pgbus.client.send_message(queue, payload, headers: headers)
|
|
@@ -138,7 +138,7 @@ module Pgbus
|
|
|
138
138
|
|
|
139
139
|
config = job_class.pgbus_uniqueness
|
|
140
140
|
return nil unless config
|
|
141
|
-
return nil unless config[:strategy]
|
|
141
|
+
return nil unless Uniqueness::ENQUEUE_LOCK_STRATEGIES.include?(config[:strategy])
|
|
142
142
|
|
|
143
143
|
key = resolve_uniqueness_key(job_class, config, task)
|
|
144
144
|
return nil unless key
|
|
@@ -198,12 +198,15 @@ module Pgbus
|
|
|
198
198
|
args = task.arguments || []
|
|
199
199
|
return default_uniqueness_key(job_class, args) unless key_proc
|
|
200
200
|
|
|
201
|
-
begin
|
|
201
|
+
key = begin
|
|
202
202
|
args.empty? ? key_proc.call : key_proc.call(*args)
|
|
203
203
|
rescue StandardError => e
|
|
204
204
|
Pgbus.logger.warn { "[Pgbus] Could not resolve uniqueness key for #{task.key}: #{e.message}" }
|
|
205
205
|
raise
|
|
206
206
|
end
|
|
207
|
+
# As Uniqueness.resolve_key does for an ordinary enqueue: a record
|
|
208
|
+
# becomes its GlobalID, so both paths key one record the same way.
|
|
209
|
+
key.respond_to?(:to_global_id) ? key.to_global_id.to_s : key
|
|
207
210
|
end
|
|
208
211
|
|
|
209
212
|
# Class-name default for the scheduler path, args-qualified so distinct
|
|
@@ -216,10 +219,17 @@ module Pgbus
|
|
|
216
219
|
end
|
|
217
220
|
|
|
218
221
|
# Inject uniqueness metadata into the payload so the executor releases
|
|
219
|
-
# the lock after the job completes
|
|
220
|
-
#
|
|
221
|
-
#
|
|
222
|
-
|
|
222
|
+
# the lock: after the job completes (:until_executed) or when it starts
|
|
223
|
+
# (:until_executing, issue #628). For :while_executing the executor
|
|
224
|
+
# takes the lock at execution start, and only when the payload carries
|
|
225
|
+
# the key, so the scheduler takes none but carries key, strategy and
|
|
226
|
+
# on_conflict (issue #629): a run that outlasts its interval is not
|
|
227
|
+
# overlapped by the next. A key proc that raises propagates, so the
|
|
228
|
+
# tick enqueues nothing rather than a run without that protection.
|
|
229
|
+
# The key is the one the row was acquired under when there is one: a
|
|
230
|
+
# key proc resolved again could answer differently, and the executor
|
|
231
|
+
# would release a key nobody holds and orphan the acquired row.
|
|
232
|
+
def inject_uniqueness_metadata(task, payload, acquired_key = nil)
|
|
223
233
|
return payload unless task.class_name
|
|
224
234
|
|
|
225
235
|
job_class = task.class_name.safe_constantize
|
|
@@ -227,15 +237,14 @@ module Pgbus
|
|
|
227
237
|
|
|
228
238
|
config = job_class.pgbus_uniqueness
|
|
229
239
|
return payload unless config
|
|
230
|
-
return payload unless config[:strategy]
|
|
240
|
+
return payload unless Uniqueness::VALID_STRATEGIES.include?(config[:strategy])
|
|
231
241
|
|
|
232
|
-
key = resolve_uniqueness_key(job_class, config, task)
|
|
242
|
+
key = acquired_key.is_a?(String) ? acquired_key : resolve_uniqueness_key(job_class, config, task)
|
|
233
243
|
return payload unless key
|
|
234
244
|
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
)
|
|
245
|
+
metadata = { Uniqueness::METADATA_KEY => key, Uniqueness::STRATEGY_KEY => config[:strategy].to_s }
|
|
246
|
+
metadata[Uniqueness::ON_CONFLICT_KEY] = config[:on_conflict].to_s if config[:strategy] == :while_executing
|
|
247
|
+
payload.merge(metadata)
|
|
239
248
|
end
|
|
240
249
|
end
|
|
241
250
|
end
|