pgbus 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.md +36 -14
- data/Rakefile +7 -1
- data/app/assets/javascripts/pgbus/stream_source_element.js +17 -3
- data/app/assets/javascripts/pgbus/stream_source_element.min.js +4 -4
- data/app/assets/javascripts/pgbus/stream_source_element.min.js.map +1 -1
- data/app/controllers/pgbus/dead_letter_controller.rb +3 -1
- data/app/controllers/pgbus/jobs_controller.rb +2 -1
- data/app/models/pgbus/blocked_execution.rb +3 -2
- data/app/models/pgbus/queue_state.rb +6 -0
- data/app/models/pgbus/uniqueness_key.rb +11 -0
- data/config/locales/da.yml +1 -0
- data/config/locales/de.yml +1 -0
- data/config/locales/en.yml +1 -0
- data/config/locales/es.yml +1 -0
- data/config/locales/fi.yml +1 -0
- data/config/locales/fr.yml +1 -0
- data/config/locales/it.yml +1 -0
- data/config/locales/ja.yml +1 -0
- data/config/locales/nb.yml +1 -0
- data/config/locales/nl.yml +1 -0
- data/config/locales/pt.yml +1 -0
- data/config/locales/sv.yml +1 -0
- data/lib/generators/pgbus/fold_queue_name_case_generator.rb +57 -0
- data/lib/generators/pgbus/templates/fold_queue_name_case.rb.erb +12 -0
- data/lib/generators/pgbus/templates/tune_autovacuum.rb.erb +2 -2
- data/lib/generators/pgbus/templates/tune_fillfactor.rb.erb +1 -1
- data/lib/pgbus/active_job/adapter.rb +3 -2
- data/lib/pgbus/active_job/executor.rb +142 -20
- data/lib/pgbus/active_job/quiet_logs.rb +57 -0
- data/lib/pgbus/autovacuum_tuning.rb +2 -2
- data/lib/pgbus/cli/dlq.rb +2 -1
- data/lib/pgbus/client/async_commit.rb +149 -0
- data/lib/pgbus/client/native_statements.rb +28 -16
- data/lib/pgbus/client.rb +145 -17
- data/lib/pgbus/concurrency/blocked_execution.rb +9 -1
- data/lib/pgbus/configuration.rb +71 -6
- data/lib/pgbus/dead_letter_header.rb +3 -0
- data/lib/pgbus/doctor.rb +55 -2
- data/lib/pgbus/failed_event_recorder.rb +5 -2
- data/lib/pgbus/generators/migration_detector.rb +29 -2
- data/lib/pgbus/instrumentation.rb +16 -1
- data/lib/pgbus/metrics/subscriber.rb +27 -1
- data/lib/pgbus/process/completer.rb +4 -2
- data/lib/pgbus/process/worker.rb +28 -3
- data/lib/pgbus/queue_name_fold.rb +145 -0
- data/lib/pgbus/queue_name_validator.rb +20 -2
- data/lib/pgbus/recurring/schedule.rb +22 -13
- data/lib/pgbus/stat_buffer.rb +67 -26
- data/lib/pgbus/streams/cursor.rb +17 -6
- data/lib/pgbus/streams/envelope.rb +16 -0
- data/lib/pgbus/streams/key.rb +1 -1
- data/lib/pgbus/table_maintenance.rb +1 -1
- data/lib/pgbus/uniqueness.rb +43 -16
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source/dead_letter.rb +14 -5
- data/lib/pgbus/web/data_source/resend.rb +185 -0
- data/lib/pgbus/web/data_source/resend_slot.rb +139 -0
- data/lib/pgbus/web/data_source.rb +55 -37
- data/lib/pgbus/web/dead_letter_reason.rb +6 -0
- data/lib/pgbus/web/streamer/connection.rb +36 -29
- data/lib/pgbus/web/streamer/falcon_connection.rb +29 -25
- data/lib/pgbus/web/streamer/heartbeat.rb +14 -3
- data/lib/pgbus/web/streamer/instance.rb +8 -5
- data/lib/pgbus/web/streamer/listener.rb +239 -62
- data/lib/pgbus/web/streamer/outbound_pump.rb +6 -4
- data/lib/pgbus/web/streamer/registry.rb +55 -29
- data/lib/pgbus/web/streamer/stream_event_dispatcher.rb +73 -12
- data/lib/tasks/pgbus_queues.rake +27 -0
- data/sig/pgbus/active_job/adapter.rbs +1 -1
- data/sig/pgbus/active_job/quiet_logs.rbs +11 -0
- data/sig/pgbus/client.rbs +6 -5
- data/sig/pgbus/configuration.rbs +4 -0
- metadata +11 -3
data/config/locales/fi.yml
CHANGED
|
@@ -224,6 +224,7 @@ fi:
|
|
|
224
224
|
retried_before:
|
|
225
225
|
one: Yritetty uudelleen DLQ:sta kerran aiemmin.
|
|
226
226
|
other: Yritetty uudelleen DLQ:sta %{count} kertaa aiemmin.
|
|
227
|
+
uniqueness_conflict_html: Ohitettu kaikilla %{attempts} lukukerralla (enint. %{max}), koska työn toinen kopio piti sen yksilöllisyysavainta — yhtään yritystä ei ajettu
|
|
227
228
|
show:
|
|
228
229
|
back: Takaisin DLQ:hon
|
|
229
230
|
context: Konteksti
|
data/config/locales/fr.yml
CHANGED
|
@@ -224,6 +224,7 @@ fr:
|
|
|
224
224
|
retried_before:
|
|
225
225
|
one: Déjà relancé depuis la DLQ une fois.
|
|
226
226
|
other: Déjà relancé depuis la DLQ %{count} fois.
|
|
227
|
+
uniqueness_conflict_html: Ignoré à chacune des %{attempts} lectures (max %{max}) car une autre copie du job détenait sa clé d’unicité — aucune tentative n’a été exécutée
|
|
227
228
|
show:
|
|
228
229
|
back: Retour à la DLQ
|
|
229
230
|
context: Contexte
|
data/config/locales/it.yml
CHANGED
|
@@ -224,6 +224,7 @@ it:
|
|
|
224
224
|
retried_before:
|
|
225
225
|
one: Già riprovato dalla DLQ una volta.
|
|
226
226
|
other: Già riprovato dalla DLQ %{count} volte.
|
|
227
|
+
uniqueness_conflict_html: Saltato in tutte le %{attempts} letture (max %{max}) mentre un’altra copia del job deteneva la sua chiave di unicità — nessun tentativo eseguito
|
|
227
228
|
show:
|
|
228
229
|
back: Torna a DLQ
|
|
229
230
|
context: Contesto
|
data/config/locales/ja.yml
CHANGED
|
@@ -224,6 +224,7 @@ ja:
|
|
|
224
224
|
retried_before:
|
|
225
225
|
one: 以前に DLQ から 1 回リトライされました。
|
|
226
226
|
other: 以前に DLQ から %{count} 回リトライされました。
|
|
227
|
+
uniqueness_conflict_html: 別のコピーが一意性キーを保持していたため、%{attempts} 回の読み取りすべてでスキップされました(上限 %{max} 回) — 試行は一度も実行されていません
|
|
227
228
|
show:
|
|
228
229
|
back: DLQに戻る
|
|
229
230
|
context: コンテキスト
|
data/config/locales/nb.yml
CHANGED
|
@@ -224,6 +224,7 @@ nb:
|
|
|
224
224
|
retried_before:
|
|
225
225
|
one: Forsøkt på nytt fra DLQ én gang før.
|
|
226
226
|
other: Forsøkt på nytt fra DLQ %{count} ganger før.
|
|
227
|
+
uniqueness_conflict_html: Hoppet over ved alle %{attempts} lesinger (maks. %{max}) mens en annen kopi av jobben holdt unikhetsnøkkelen — ingen forsøk kjørte
|
|
227
228
|
show:
|
|
228
229
|
back: Tilbake til DLQ
|
|
229
230
|
context: Kontekst
|
data/config/locales/nl.yml
CHANGED
|
@@ -224,6 +224,7 @@ nl:
|
|
|
224
224
|
retried_before:
|
|
225
225
|
one: Al eens eerder vanuit de DLQ opnieuw geprobeerd.
|
|
226
226
|
other: Al %{count} keer eerder vanuit de DLQ opnieuw geprobeerd.
|
|
227
|
+
uniqueness_conflict_html: Bij alle %{attempts} keer lezen overgeslagen (max %{max}) omdat een andere kopie van de job de uniciteitssleutel had — geen poging uitgevoerd
|
|
227
228
|
show:
|
|
228
229
|
back: Terug naar DLQ
|
|
229
230
|
context: Context
|
data/config/locales/pt.yml
CHANGED
|
@@ -224,6 +224,7 @@ pt:
|
|
|
224
224
|
retried_before:
|
|
225
225
|
one: Já reenviado da DLQ uma vez.
|
|
226
226
|
other: Já reenviado da DLQ %{count} vezes.
|
|
227
|
+
uniqueness_conflict_html: Ignorado em todas as %{attempts} leituras (máx. %{max}) enquanto outra cópia do job detinha sua chave de unicidade — nenhuma tentativa foi executada
|
|
227
228
|
show:
|
|
228
229
|
back: Voltar para DLQ
|
|
229
230
|
context: Contexto
|
data/config/locales/sv.yml
CHANGED
|
@@ -224,6 +224,7 @@ sv:
|
|
|
224
224
|
retried_before:
|
|
225
225
|
one: Omförsökt från DLQ en gång tidigare.
|
|
226
226
|
other: Omförsökt från DLQ %{count} gånger tidigare.
|
|
227
|
+
uniqueness_conflict_html: Hoppades över vid alla %{attempts} läsningar (max %{max}) medan en annan kopia av jobbet höll dess unikhetsnyckel — inget försök kördes
|
|
227
228
|
show:
|
|
228
229
|
back: Tillbaka till DLQ
|
|
229
230
|
context: Kontext
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
require_relative "migration_path"
|
|
6
|
+
|
|
7
|
+
module Pgbus
|
|
8
|
+
module Generators
|
|
9
|
+
# Issue #633: PGMQ lower-cases its table names, so a queue pgbus created
|
|
10
|
+
# as "pgbus_Orders" before 0.23.0 lives in pgmq.q_pgbus_orders while
|
|
11
|
+
# pgmq.meta records the mixed-case spelling. pgbus now spells every queue
|
|
12
|
+
# in lower case; this migration brings an install's existing rows to that
|
|
13
|
+
# spelling. No table is dropped, renamed or copied.
|
|
14
|
+
class FoldQueueNameCaseGenerator < Rails::Generators::Base
|
|
15
|
+
include ActiveRecord::Generators::Migration
|
|
16
|
+
include MigrationPath
|
|
17
|
+
|
|
18
|
+
source_root File.expand_path("templates", __dir__)
|
|
19
|
+
|
|
20
|
+
desc "Spell existing PGMQ queues (pgmq.meta, NOTIFY triggers, pgbus's queue-name columns) in lower case, " \
|
|
21
|
+
"the way their tables are"
|
|
22
|
+
|
|
23
|
+
class_option :database,
|
|
24
|
+
type: :string,
|
|
25
|
+
default: nil,
|
|
26
|
+
desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
|
|
27
|
+
|
|
28
|
+
def create_migration_file
|
|
29
|
+
migration_template "fold_queue_name_case.rb.erb",
|
|
30
|
+
File.join(pgbus_migrate_path, "fold_pgbus_queue_name_case.rb")
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def display_post_install
|
|
34
|
+
say ""
|
|
35
|
+
say "Pgbus queue-name fold migration created!", :green
|
|
36
|
+
say ""
|
|
37
|
+
say "PGMQ spells every queue table in lower case. This migration renames the"
|
|
38
|
+
say "pgmq.meta rows (and NOTIFY triggers) of queues created with upper-case"
|
|
39
|
+
say "letters to the lower-case spelling their tables already use. Messages stay"
|
|
40
|
+
say "where they are."
|
|
41
|
+
say ""
|
|
42
|
+
say "Next steps:"
|
|
43
|
+
say " 1. Deploy pgbus 0.23.0+ everywhere first: an older process recreates its mixed-case row"
|
|
44
|
+
say " 2. Run: rails db:migrate#{migrate_command_suffix}"
|
|
45
|
+
say " 3. Check: rake pgbus:doctor (\"Queue name case\" is ok); if it still names a queue,"
|
|
46
|
+
say " run: rake pgbus:queues:fold_case"
|
|
47
|
+
say ""
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def migration_version
|
|
53
|
+
"[#{ActiveRecord::Migration.current_version}]"
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
class FoldPgbusQueueNameCase < ActiveRecord::Migration<%= migration_version %>
|
|
2
|
+
# Issue #633. PGMQ lower-cases its table names, so a queue created as
|
|
3
|
+
# "pgbus_Orders" lives in pgmq.q_pgbus_orders while pgmq.meta records
|
|
4
|
+
# "pgbus_Orders". Pgbus::QueueNameFold renames the old rows to the lower-case
|
|
5
|
+
# spelling pgbus now uses; it changes rows only, never a table.
|
|
6
|
+
def up
|
|
7
|
+
Pgbus::QueueNameFold.run!(connection)
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
# The original spellings are not kept: PGMQ never used them for a table.
|
|
11
|
+
def down; end
|
|
12
|
+
end
|
|
@@ -25,8 +25,8 @@ class TunePgbusAutovacuum < ActiveRecord::Migration<%= migration_version %>
|
|
|
25
25
|
q RECORD;
|
|
26
26
|
BEGIN
|
|
27
27
|
FOR q IN SELECT queue_name FROM pgmq.meta LOOP
|
|
28
|
-
EXECUTE format('ALTER TABLE pgmq
|
|
29
|
-
EXECUTE format('ALTER TABLE pgmq
|
|
28
|
+
EXECUTE format('ALTER TABLE pgmq.%I RESET (autovacuum_vacuum_scale_factor, autovacuum_vacuum_cost_delay, autovacuum_analyze_scale_factor)', 'q_' || lower(q.queue_name));
|
|
29
|
+
EXECUTE format('ALTER TABLE pgmq.%I RESET (autovacuum_vacuum_scale_factor, autovacuum_vacuum_cost_delay, autovacuum_analyze_scale_factor)', 'a_' || lower(q.queue_name));
|
|
30
30
|
END LOOP;
|
|
31
31
|
END $$;
|
|
32
32
|
SQL
|
|
@@ -22,7 +22,7 @@ class TunePgbusFillfactor < ActiveRecord::Migration<%= migration_version %>
|
|
|
22
22
|
q RECORD;
|
|
23
23
|
BEGIN
|
|
24
24
|
FOR q IN SELECT queue_name FROM pgmq.meta LOOP
|
|
25
|
-
EXECUTE format('ALTER TABLE pgmq
|
|
25
|
+
EXECUTE format('ALTER TABLE pgmq.%I RESET (fillfactor)', 'q_' || lower(q.queue_name));
|
|
26
26
|
END LOOP;
|
|
27
27
|
END $$;
|
|
28
28
|
SQL
|
|
@@ -215,7 +215,7 @@ module Pgbus
|
|
|
215
215
|
return true
|
|
216
216
|
when :discard
|
|
217
217
|
Pgbus.logger.info { "[Pgbus] Discarding job #{active_job.class.name}: concurrency limit for #{key}" }
|
|
218
|
-
# The job will never run: roll back an
|
|
218
|
+
# The job will never run: roll back an enqueue-time uniqueness lock
|
|
219
219
|
# acquired earlier in this enqueue (no executor will release it), and
|
|
220
220
|
# uncount it from its batch so completion is not waited on forever.
|
|
221
221
|
rollback_acquired_uniqueness_lock
|
|
@@ -226,7 +226,8 @@ module Pgbus
|
|
|
226
226
|
false
|
|
227
227
|
end
|
|
228
228
|
|
|
229
|
-
# Releases an
|
|
229
|
+
# Releases an enqueue-time lock (:until_executed, :until_executing) this
|
|
230
|
+
# enqueue acquired, if any.
|
|
230
231
|
# Used when the job is dropped before a message is sent (concurrency
|
|
231
232
|
# :discard, or send_message raising) — otherwise the lock is orphaned
|
|
232
233
|
# because no executor will ever release it.
|
|
@@ -35,12 +35,16 @@ module Pgbus
|
|
|
35
35
|
# it: the DLQ copy carries it in its headers (issue #495).
|
|
36
36
|
last_error = FailedEventRecorder.last_error(queue_name: queue_name, msg_id: message.msg_id.to_i,
|
|
37
37
|
source_queue: physical_queue(queue_name, source_queue))
|
|
38
|
-
|
|
38
|
+
# Before the release below: the lookup asks whether the key's row is
|
|
39
|
+
# still this message's own (a crashed attempt's), see dead_letter_reason.
|
|
40
|
+
reason = dead_letter_reason(message, queue_name, payload, last_error)
|
|
41
|
+
handle_dead_letter(message, queue_name, payload, source_queue: source_queue, last_error: last_error,
|
|
42
|
+
reason: reason)
|
|
39
43
|
FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i,
|
|
40
44
|
source_queue: physical_queue(queue_name, source_queue))
|
|
41
45
|
signal_concurrency(payload)
|
|
42
46
|
signal_batch_discarded(payload)
|
|
43
|
-
release_uniqueness_lock(Uniqueness.extract_key(payload), queue_name, message)
|
|
47
|
+
release_uniqueness_lock(Uniqueness.extract_key(payload), queue_name, message) if holds_uniqueness_row?(payload)
|
|
44
48
|
record_stat(payload, queue_name, "dead_lettered", execution_start, message: message)
|
|
45
49
|
instrument(
|
|
46
50
|
"pgbus.job_dead_lettered",
|
|
@@ -50,7 +54,8 @@ module Pgbus
|
|
|
50
54
|
provider_job_id: payload["provider_job_id"],
|
|
51
55
|
read_ct: read_count,
|
|
52
56
|
msg_id: message.msg_id.to_i,
|
|
53
|
-
error: last_error&.dig(:error_class)
|
|
57
|
+
error: last_error&.dig(:error_class),
|
|
58
|
+
reason: reason
|
|
54
59
|
)
|
|
55
60
|
Pgbus.logger.debug { "[Pgbus::Executor] dead_lettered #{log_tag(message, queue_name)} job_class=#{job_class}" }
|
|
56
61
|
return :dead_lettered
|
|
@@ -66,17 +71,23 @@ module Pgbus
|
|
|
66
71
|
# released on completion or DLQ.
|
|
67
72
|
nil
|
|
68
73
|
when :while_executing
|
|
69
|
-
# Acquire the lock now, bound to this message. If another
|
|
70
|
-
#
|
|
71
|
-
#
|
|
74
|
+
# Acquire the lock now, bound to this message. If another message
|
|
75
|
+
# holds it, the job's on_conflict decides (issue #629): :discard
|
|
76
|
+
# drops this copy, :reject and :log leave it for a backed-off
|
|
77
|
+
# redelivery. A row left by a crashed attempt of THIS message is
|
|
72
78
|
# re-acquired, not treated as a duplicate.
|
|
73
79
|
acquired = Uniqueness.acquire_execution_lock(
|
|
74
80
|
uniqueness_key, payload, msg_id: message.msg_id.to_i, queue_name: queue_name
|
|
75
81
|
)
|
|
76
|
-
unless acquired
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
82
|
+
return handle_uniqueness_conflict(message, queue_name, payload, uniqueness_key, source_queue) unless acquired
|
|
83
|
+
when :until_executing
|
|
84
|
+
# A debounce (issue #628): the row guarded the queued copy, and
|
|
85
|
+
# goes before perform so an enqueue that arrives while this runs
|
|
86
|
+
# is accepted. release_held_by! takes only the row this message
|
|
87
|
+
# holds, never a newer copy's (a dashboard re-send holds its own,
|
|
88
|
+
# #631). A redelivery (row already gone) matches nothing and runs; a failed release is logged and the
|
|
89
|
+
# job runs anyway — the reaper or the DLQ release takes the row.
|
|
90
|
+
release_uniqueness_lock(uniqueness_key, queue_name, message) if holds_uniqueness_row?(payload)
|
|
80
91
|
end
|
|
81
92
|
end
|
|
82
93
|
|
|
@@ -132,7 +143,9 @@ module Pgbus
|
|
|
132
143
|
end
|
|
133
144
|
Pgbus.logger.debug { "[Pgbus::Executor] #{@completed_word} #{log_tag(message, queue_name)} job_class=#{job_class}" }
|
|
134
145
|
job_succeeded = true
|
|
135
|
-
|
|
146
|
+
# :until_executing released its row before perform; a newer copy's
|
|
147
|
+
# row is not this message's to take.
|
|
148
|
+
release_uniqueness_lock(uniqueness_key, queue_name, message) unless uniqueness_strategy == :until_executing
|
|
136
149
|
# Only FailedEventRecorder.record! writes this table, and only after a
|
|
137
150
|
# failed attempt, so a first delivery has no row: skip the per-job
|
|
138
151
|
# DELETE round trip (issue #484). A stale row from a dropped-and-
|
|
@@ -228,6 +241,15 @@ module Pgbus
|
|
|
228
241
|
Pgbus.logger.warn { "[Pgbus] Uniqueness release failed: #{e.message}" }
|
|
229
242
|
end
|
|
230
243
|
|
|
244
|
+
# False for an :until_executing retry_on re-enqueue (executions > 0): it
|
|
245
|
+
# went out through the free key without acquiring a row (the attempt's
|
|
246
|
+
# was released at its start), so the unbound clause of release_held_by!
|
|
247
|
+
# could only match a newer copy's row — one parked behind its
|
|
248
|
+
# concurrency limit before this retry was sent (issue #628).
|
|
249
|
+
def holds_uniqueness_row?(payload)
|
|
250
|
+
Uniqueness.extract_strategy(payload) != :until_executing || payload["executions"].to_i.zero?
|
|
251
|
+
end
|
|
252
|
+
|
|
231
253
|
# Keep the message invisible while perform runs (see VisibilityHeartbeat).
|
|
232
254
|
# Wraps only the perform: the heartbeat must be gone before archive or
|
|
233
255
|
# the retry backoff touches the same message's VT.
|
|
@@ -368,13 +390,88 @@ module Pgbus
|
|
|
368
390
|
RetryBackoff.compute_delay(attempt: attempt)
|
|
369
391
|
end
|
|
370
392
|
|
|
393
|
+
set_message_visibility(message, queue_name, delay, source_queue)
|
|
394
|
+
rescue StandardError => e
|
|
395
|
+
Pgbus.logger.debug { "[Pgbus] Retry backoff VT update failed: #{e.message}" }
|
|
396
|
+
end
|
|
397
|
+
|
|
398
|
+
def set_message_visibility(message, queue_name, delay, source_queue)
|
|
371
399
|
if source_queue
|
|
372
400
|
client.set_visibility_timeout(source_queue, message.msg_id.to_i, vt: delay, prefixed: false)
|
|
373
401
|
else
|
|
374
402
|
client.set_visibility_timeout(queue_name, message.msg_id.to_i, vt: delay)
|
|
375
403
|
end
|
|
376
|
-
|
|
377
|
-
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
# A :while_executing message whose key another message holds (issue
|
|
407
|
+
# #629). The duplicate holds no uniqueness row, so nothing is released.
|
|
408
|
+
def handle_uniqueness_conflict(message, queue_name, payload, key, source_queue)
|
|
409
|
+
policy = Uniqueness.extract_on_conflict(payload)
|
|
410
|
+
if policy == :discard
|
|
411
|
+
discard_duplicate(message, queue_name, payload, key, source_queue)
|
|
412
|
+
else
|
|
413
|
+
skip_duplicate(message, queue_name, payload, key, source_queue, policy)
|
|
414
|
+
end
|
|
415
|
+
end
|
|
416
|
+
|
|
417
|
+
# :discard — the duplicate will never run: complete it, then the
|
|
418
|
+
# bookkeeping the dead-letter path does for such a message. Lost to
|
|
419
|
+
# another worker that completed it first: that worker owns the signals.
|
|
420
|
+
def discard_duplicate(message, queue_name, payload, key, source_queue)
|
|
421
|
+
msg_id = message.msg_id.to_i
|
|
422
|
+
if complete_from(queue_name, msg_id, source_queue: source_queue) == :already_archived
|
|
423
|
+
Pgbus.logger.warn do
|
|
424
|
+
"[Pgbus::Executor] already #{@completed_word} elsewhere, skipping signals " \
|
|
425
|
+
"#{log_tag(message, queue_name)} job_class=#{payload["job_class"]}"
|
|
426
|
+
end
|
|
427
|
+
return :duplicate
|
|
428
|
+
end
|
|
429
|
+
if message.read_ct.to_i > 1
|
|
430
|
+
FailedEventRecorder.clear!(queue_name: queue_name, msg_id: msg_id,
|
|
431
|
+
source_queue: physical_queue(queue_name, source_queue))
|
|
432
|
+
end
|
|
433
|
+
signal_concurrency(payload)
|
|
434
|
+
signal_batch_discarded(payload)
|
|
435
|
+
instrument("pgbus.job_discarded", queue: queue_name, job_class: payload["job_class"], job_id: payload["job_id"],
|
|
436
|
+
provider_job_id: payload["provider_job_id"], msg_id: msg_id, reason: "uniqueness")
|
|
437
|
+
Pgbus.logger.info do
|
|
438
|
+
"[Pgbus] Discarded duplicate #{payload["job_class"]}: uniqueness key #{key} is held by another message " \
|
|
439
|
+
"(#{log_tag(message, queue_name)})"
|
|
440
|
+
end
|
|
441
|
+
:discarded
|
|
442
|
+
end
|
|
443
|
+
|
|
444
|
+
# :reject / :log — the copy still runs once the holder is done, but each
|
|
445
|
+
# skip backs its redelivery off so the skips stretch over the holder's
|
|
446
|
+
# run. read_ct cannot be decremented, so a skip still counts toward
|
|
447
|
+
# max_retries; the dead-letter reason says when that is why it died.
|
|
448
|
+
def skip_duplicate(message, queue_name, payload, key, source_queue, policy)
|
|
449
|
+
delay = skip_delay(message, payload)
|
|
450
|
+
begin
|
|
451
|
+
set_message_visibility(message, queue_name, delay, source_queue)
|
|
452
|
+
rescue StandardError => e
|
|
453
|
+
Pgbus.logger.warn { "[Pgbus] Skip backoff VT update failed: #{e.message}" }
|
|
454
|
+
end
|
|
455
|
+
text = "[Pgbus] Skipping duplicate execution for #{payload["job_class"]}: uniqueness key #{key} is held by " \
|
|
456
|
+
"another message (#{log_tag(message, queue_name)}); next delivery in #{delay}s"
|
|
457
|
+
policy == :log ? Pgbus.logger.warn { text } : Pgbus.logger.info { text }
|
|
458
|
+
instrument("pgbus.job_skipped", queue: queue_name, job_class: payload["job_class"], job_id: payload["job_id"],
|
|
459
|
+
provider_job_id: payload["provider_job_id"], msg_id: message.msg_id.to_i,
|
|
460
|
+
read_ct: message.read_ct.to_i, on_conflict: policy, vt: delay)
|
|
461
|
+
:skipped
|
|
462
|
+
end
|
|
463
|
+
|
|
464
|
+
# visibility_timeout * 2^(read_ct - 1), jittered and capped at the job's
|
|
465
|
+
# (or the global) retry_backoff_max, and never below visibility_timeout:
|
|
466
|
+
# a skip never comes back sooner than the read's own timeout would.
|
|
467
|
+
def skip_delay(message, payload)
|
|
468
|
+
vt = config.visibility_timeout
|
|
469
|
+
job_class = resolve_job_class(payload)
|
|
470
|
+
overrides = (job_class.respond_to?(:pgbus_retry_backoff_config) && job_class.pgbus_retry_backoff_config) || {}
|
|
471
|
+
max = [overrides[:max] || config.retry_backoff_max, vt].max
|
|
472
|
+
delay = RetryBackoff.compute_delay(attempt: message.read_ct.to_i, base: vt, max: max,
|
|
473
|
+
jitter: overrides[:jitter] || config.retry_backoff_jitter)
|
|
474
|
+
[delay, vt].max
|
|
378
475
|
end
|
|
379
476
|
|
|
380
477
|
def resolve_job_class(payload)
|
|
@@ -433,7 +530,9 @@ module Pgbus
|
|
|
433
530
|
# config.completed_jobs = :delete it deletes the message instead
|
|
434
531
|
# (delete_message, or the completer's delete_batch): pgmq.delete is as
|
|
435
532
|
# idempotent and returns false the same way, so the symbols keep their
|
|
436
|
-
# names and "archived" means "completed".
|
|
533
|
+
# names and "archived" means "completed". job_completion: true lets
|
|
534
|
+
# config.completion_synchronous_commit = false commit it with
|
|
535
|
+
# synchronous_commit off (issue #619).
|
|
437
536
|
#
|
|
438
537
|
# A `false` reported after our OWN retry is ambiguous, not a duplicate:
|
|
439
538
|
# the first archive may well have committed with only its reply lost to
|
|
@@ -450,13 +549,13 @@ module Pgbus
|
|
|
450
549
|
# Spelled out, no `**` forward: on Ruby 3.3 that
|
|
451
550
|
# allocates per job even without keywords (#484 budget).
|
|
452
551
|
elsif @delete_completed && source_queue
|
|
453
|
-
client.delete_message(source_queue, msg_id, prefixed: false)
|
|
552
|
+
client.delete_message(source_queue, msg_id, prefixed: false, job_completion: true)
|
|
454
553
|
elsif @delete_completed
|
|
455
|
-
client.delete_message(queue_name, msg_id)
|
|
554
|
+
client.delete_message(queue_name, msg_id, job_completion: true)
|
|
456
555
|
elsif source_queue
|
|
457
|
-
client.archive_message(source_queue, msg_id, prefixed: false)
|
|
556
|
+
client.archive_message(source_queue, msg_id, prefixed: false, job_completion: true)
|
|
458
557
|
else
|
|
459
|
-
client.archive_message(queue_name, msg_id)
|
|
558
|
+
client.archive_message(queue_name, msg_id, job_completion: true)
|
|
460
559
|
end
|
|
461
560
|
return :archived unless archived == false
|
|
462
561
|
|
|
@@ -492,14 +591,37 @@ module Pgbus
|
|
|
492
591
|
defined?(PGMQ::Errors::ConnectionError) && error.is_a?(PGMQ::Errors::ConnectionError)
|
|
493
592
|
end
|
|
494
593
|
|
|
495
|
-
|
|
594
|
+
# uniqueness_conflict for a :while_executing message no attempt of
|
|
595
|
+
# which failed: it was skipped on every read until read_ct passed
|
|
596
|
+
# max_retries (issue #629). Not whether the key is held at this read:
|
|
597
|
+
# the backed-off skips span minutes and the holder may have finished
|
|
598
|
+
# inside the last one. The one other way to die without a failed-event
|
|
599
|
+
# row is a crash loop, and a crashed attempt leaves the key's row bound
|
|
600
|
+
# to this message: that keeps max_retries_exceeded. A failed lookup
|
|
601
|
+
# keeps the stronger signal, the missing failed-event row.
|
|
602
|
+
def dead_letter_reason(message, queue_name, payload, last_error)
|
|
603
|
+
return DeadLetterHeader::REASON_MAX_RETRIES if last_error
|
|
604
|
+
|
|
605
|
+
key = Uniqueness.extract_key(payload)
|
|
606
|
+
return DeadLetterHeader::REASON_MAX_RETRIES unless key && Uniqueness.extract_strategy(payload) == :while_executing
|
|
607
|
+
|
|
608
|
+
own = UniquenessKey.bound_to?(key, queue_name: Uniqueness.lock_queue_name(queue_name, config: config),
|
|
609
|
+
msg_id: message.msg_id.to_i)
|
|
610
|
+
own ? DeadLetterHeader::REASON_MAX_RETRIES : DeadLetterHeader::REASON_UNIQUENESS_CONFLICT
|
|
611
|
+
rescue StandardError => e
|
|
612
|
+
Pgbus.logger.warn { "[Pgbus] Uniqueness row lookup failed, dead-lettering as uniqueness_conflict: #{e.message}" }
|
|
613
|
+
DeadLetterHeader::REASON_UNIQUENESS_CONFLICT
|
|
614
|
+
end
|
|
615
|
+
|
|
616
|
+
def handle_dead_letter(message, queue_name, payload, source_queue: nil, last_error: nil,
|
|
617
|
+
reason: DeadLetterHeader::REASON_MAX_RETRIES)
|
|
496
618
|
Pgbus.logger.warn do
|
|
497
619
|
job_class = payload["job_class"] || "unknown"
|
|
498
620
|
error = last_error ? " (last error: #{last_error[:error_class]})" : ""
|
|
499
621
|
"[Pgbus] Moving job #{job_class} to dead letter queue after #{message.read_ct} attempts#{error}"
|
|
500
622
|
end
|
|
501
623
|
headers = DeadLetterHeader.build(
|
|
502
|
-
existing: message.headers, reason:
|
|
624
|
+
existing: message.headers, reason: reason, source: "worker",
|
|
503
625
|
source_queue: source_queue || config.queue_name(queue_name), source_msg_id: message.msg_id.to_i,
|
|
504
626
|
source_enqueued_at: message.enqueued_at, attempts: message.read_ct.to_i,
|
|
505
627
|
max_retries: config.max_retries, error: last_error
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module ActiveJob
|
|
5
|
+
# config.quiet_active_job_logs (issue #618): a worker process stops
|
|
6
|
+
# ActiveJob's per-job "Performing …" / "Performed …" info lines by
|
|
7
|
+
# unsubscribing ActiveJob::LogSubscriber from perform_start.active_job and
|
|
8
|
+
# perform.active_job. The perform listener also writes ActiveJob's
|
|
9
|
+
# "Error performing …" error line, for a job that raised (with its
|
|
10
|
+
# backtrace) and for one a before_perform callback halted, so those go too.
|
|
11
|
+
# An exception that escapes ActiveJob's handlers is still logged by
|
|
12
|
+
# ErrorReporter (class and message at error) and recorded in
|
|
13
|
+
# pgbus_failed_events; one retry_on catches still logs ActiveJob's
|
|
14
|
+
# "Retrying …" line (and "Stopped retrying …" at error when it gives up);
|
|
15
|
+
# a halted job logs nothing.
|
|
16
|
+
# The other ActiveJob events (Enqueued, Retrying, Stopped retrying,
|
|
17
|
+
# Discarded) still log as the host configured them, and the web process is
|
|
18
|
+
# untouched. Swapping
|
|
19
|
+
# ActiveJob::Base.logger instead would reach the web process too, and by
|
|
20
|
+
# default that logger is Rails.logger.
|
|
21
|
+
module QuietLogs
|
|
22
|
+
EVENTS = %w[perform_start.active_job perform.active_job].freeze
|
|
23
|
+
|
|
24
|
+
# Built at load; the applied notifiers are read and written under it.
|
|
25
|
+
@mutex = Mutex.new
|
|
26
|
+
@applied = {}.compare_by_identity
|
|
27
|
+
|
|
28
|
+
class << self
|
|
29
|
+
# Unsubscribes the LogSubscriber listeners once per notifier. Returns
|
|
30
|
+
# true when it did, false when that notifier was already done.
|
|
31
|
+
def apply!(notifier: ::ActiveSupport::Notifications.notifier)
|
|
32
|
+
@mutex.synchronize do
|
|
33
|
+
return false if @applied.key?(notifier)
|
|
34
|
+
|
|
35
|
+
@applied[notifier] = true
|
|
36
|
+
# all_listeners_for, not listeners_for: the latter skips a
|
|
37
|
+
# silenced subscriber (its logger above info), which would come
|
|
38
|
+
# back the moment the level drops.
|
|
39
|
+
EVENTS.each do |name|
|
|
40
|
+
notifier.all_listeners_for(name).each do |listener|
|
|
41
|
+
notifier.unsubscribe(listener) if log_subscriber?(listener)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
true
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
# Fanout::Subscribers::Evented exposes #delegate on Rails 7.1 and later.
|
|
51
|
+
def log_subscriber?(listener)
|
|
52
|
+
listener.respond_to?(:delegate) && listener.delegate.is_a?(::ActiveJob::LogSubscriber)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -66,8 +66,8 @@ module Pgbus
|
|
|
66
66
|
q RECORD;
|
|
67
67
|
BEGIN
|
|
68
68
|
FOR q IN SELECT queue_name FROM pgmq.meta LOOP
|
|
69
|
-
EXECUTE format('ALTER TABLE pgmq
|
|
70
|
-
EXECUTE format('ALTER TABLE pgmq
|
|
69
|
+
EXECUTE format('ALTER TABLE pgmq.%I SET (#{settings_clause(QUEUE_SETTINGS)})', 'q_' || lower(q.queue_name));
|
|
70
|
+
EXECUTE format('ALTER TABLE pgmq.%I SET (#{settings_clause(ARCHIVE_SETTINGS)})', 'a_' || lower(q.queue_name));
|
|
71
71
|
END LOOP;
|
|
72
72
|
END $$;
|
|
73
73
|
SQL
|
data/lib/pgbus/cli/dlq.rb
CHANGED
|
@@ -70,7 +70,8 @@ module Pgbus
|
|
|
70
70
|
if data_source.retry_dlq_message(detail[:queue_name], detail[:msg_id])
|
|
71
71
|
puts "Re-enqueued msg_id #{detail[:msg_id]} to #{origin_queue(detail[:queue_name])}."
|
|
72
72
|
else
|
|
73
|
-
warn "Failed to retry msg_id #{detail[:msg_id]}
|
|
73
|
+
warn "Failed to retry msg_id #{detail[:msg_id]}: it is gone, the retry failed, " \
|
|
74
|
+
"another copy of the job holds its uniqueness key, or its concurrency limit is full."
|
|
74
75
|
exit 1
|
|
75
76
|
end
|
|
76
77
|
end
|