pgbus 0.22.0 → 0.23.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +52 -0
  3. data/README.md +41 -14
  4. data/Rakefile +7 -1
  5. data/app/assets/javascripts/pgbus/stream_source_element.js +17 -3
  6. data/app/assets/javascripts/pgbus/stream_source_element.min.js +4 -4
  7. data/app/assets/javascripts/pgbus/stream_source_element.min.js.map +1 -1
  8. data/app/controllers/pgbus/dead_letter_controller.rb +3 -1
  9. data/app/controllers/pgbus/jobs_controller.rb +2 -1
  10. data/app/models/pgbus/blocked_execution.rb +3 -2
  11. data/app/models/pgbus/queue_state.rb +6 -0
  12. data/app/models/pgbus/uniqueness_key.rb +11 -0
  13. data/config/locales/da.yml +1 -0
  14. data/config/locales/de.yml +1 -0
  15. data/config/locales/en.yml +1 -0
  16. data/config/locales/es.yml +1 -0
  17. data/config/locales/fi.yml +1 -0
  18. data/config/locales/fr.yml +1 -0
  19. data/config/locales/it.yml +1 -0
  20. data/config/locales/ja.yml +1 -0
  21. data/config/locales/nb.yml +1 -0
  22. data/config/locales/nl.yml +1 -0
  23. data/config/locales/pt.yml +1 -0
  24. data/config/locales/sv.yml +1 -0
  25. data/lib/generators/pgbus/fold_queue_name_case_generator.rb +57 -0
  26. data/lib/generators/pgbus/templates/fold_queue_name_case.rb.erb +12 -0
  27. data/lib/generators/pgbus/templates/tune_autovacuum.rb.erb +2 -2
  28. data/lib/generators/pgbus/templates/tune_fillfactor.rb.erb +1 -1
  29. data/lib/pgbus/active_job/adapter.rb +3 -2
  30. data/lib/pgbus/active_job/executor.rb +142 -20
  31. data/lib/pgbus/active_job/quiet_logs.rb +57 -0
  32. data/lib/pgbus/autovacuum_tuning.rb +2 -2
  33. data/lib/pgbus/cli/dlq.rb +2 -1
  34. data/lib/pgbus/client/async_commit.rb +149 -0
  35. data/lib/pgbus/client/native_statements.rb +28 -16
  36. data/lib/pgbus/client.rb +181 -17
  37. data/lib/pgbus/concurrency/blocked_execution.rb +9 -1
  38. data/lib/pgbus/configuration.rb +71 -6
  39. data/lib/pgbus/dead_letter_header.rb +3 -0
  40. data/lib/pgbus/doctor.rb +55 -2
  41. data/lib/pgbus/failed_event_recorder.rb +5 -2
  42. data/lib/pgbus/generators/migration_detector.rb +29 -2
  43. data/lib/pgbus/instrumentation.rb +16 -1
  44. data/lib/pgbus/metrics/subscriber.rb +27 -1
  45. data/lib/pgbus/process/completer.rb +4 -2
  46. data/lib/pgbus/process/worker.rb +28 -3
  47. data/lib/pgbus/queue_name_fold.rb +145 -0
  48. data/lib/pgbus/queue_name_validator.rb +20 -2
  49. data/lib/pgbus/recurring/schedule.rb +22 -13
  50. data/lib/pgbus/stat_buffer.rb +67 -26
  51. data/lib/pgbus/streams/cursor.rb +17 -6
  52. data/lib/pgbus/streams/envelope.rb +16 -0
  53. data/lib/pgbus/streams/key.rb +1 -1
  54. data/lib/pgbus/table_maintenance.rb +1 -1
  55. data/lib/pgbus/uniqueness.rb +43 -16
  56. data/lib/pgbus/version.rb +1 -1
  57. data/lib/pgbus/web/data_source/dead_letter.rb +14 -5
  58. data/lib/pgbus/web/data_source/resend.rb +185 -0
  59. data/lib/pgbus/web/data_source/resend_slot.rb +139 -0
  60. data/lib/pgbus/web/data_source.rb +55 -37
  61. data/lib/pgbus/web/dead_letter_reason.rb +6 -0
  62. data/lib/pgbus/web/streamer/connection.rb +36 -29
  63. data/lib/pgbus/web/streamer/falcon_connection.rb +29 -25
  64. data/lib/pgbus/web/streamer/heartbeat.rb +14 -3
  65. data/lib/pgbus/web/streamer/instance.rb +8 -5
  66. data/lib/pgbus/web/streamer/listener.rb +239 -62
  67. data/lib/pgbus/web/streamer/outbound_pump.rb +6 -4
  68. data/lib/pgbus/web/streamer/registry.rb +55 -29
  69. data/lib/pgbus/web/streamer/stream_event_dispatcher.rb +73 -12
  70. data/lib/tasks/pgbus_queues.rake +27 -0
  71. data/sig/pgbus/active_job/adapter.rbs +1 -1
  72. data/sig/pgbus/active_job/quiet_logs.rbs +11 -0
  73. data/sig/pgbus/client.rbs +6 -5
  74. data/sig/pgbus/configuration.rbs +4 -0
  75. metadata +11 -3
@@ -224,6 +224,7 @@ de:
224
224
  retried_before:
225
225
  one: Schon einmal aus der DLQ wiederholt.
226
226
  other: Schon %{count}-mal aus der DLQ wiederholt.
227
+ uniqueness_conflict_html: Bei allen %{attempts} Lesevorgängen übersprungen (max. %{max}), weil eine andere Kopie des Jobs seinen Eindeutigkeitsschlüssel hielt — kein Versuch lief
227
228
  show:
228
229
  back: Zurück zur DLQ
229
230
  context: Kontext
@@ -224,6 +224,7 @@ en:
224
224
  retried_before:
225
225
  one: Retried from the DLQ once before.
226
226
  other: Retried from the DLQ %{count} times before.
227
+ uniqueness_conflict_html: Skipped on all %{attempts} reads (max %{max}) while another copy of the job held its uniqueness key — no attempt ran
227
228
  show:
228
229
  back: Back to DLQ
229
230
  context: Context
@@ -224,6 +224,7 @@ es:
224
224
  retried_before:
225
225
  one: Reintentado desde la DLQ una vez antes.
226
226
  other: Reintentado desde la DLQ %{count} veces antes.
227
+ uniqueness_conflict_html: Omitido en las %{attempts} lecturas (máx. %{max}) mientras otra copia del job tenía su clave de unicidad — no se ejecutó ningún intento
227
228
  show:
228
229
  back: Volver a DLQ
229
230
  context: Contexto
@@ -224,6 +224,7 @@ fi:
224
224
  retried_before:
225
225
  one: Yritetty uudelleen DLQ:sta kerran aiemmin.
226
226
  other: Yritetty uudelleen DLQ:sta %{count} kertaa aiemmin.
227
+ uniqueness_conflict_html: Ohitettu kaikilla %{attempts} lukukerralla (enint. %{max}), koska työn toinen kopio piti sen yksilöllisyysavainta — yhtään yritystä ei ajettu
227
228
  show:
228
229
  back: Takaisin DLQ:hon
229
230
  context: Konteksti
@@ -224,6 +224,7 @@ fr:
224
224
  retried_before:
225
225
  one: Déjà relancé depuis la DLQ une fois.
226
226
  other: Déjà relancé depuis la DLQ %{count} fois.
227
+ uniqueness_conflict_html: Ignoré à chacune des %{attempts} lectures (max %{max}) car une autre copie du job détenait sa clé d’unicité — aucune tentative n’a été exécutée
227
228
  show:
228
229
  back: Retour à la DLQ
229
230
  context: Contexte
@@ -224,6 +224,7 @@ it:
224
224
  retried_before:
225
225
  one: Già riprovato dalla DLQ una volta.
226
226
  other: Già riprovato dalla DLQ %{count} volte.
227
+ uniqueness_conflict_html: Saltato in tutte le %{attempts} letture (max %{max}) mentre un’altra copia del job deteneva la sua chiave di unicità — nessun tentativo eseguito
227
228
  show:
228
229
  back: Torna a DLQ
229
230
  context: Contesto
@@ -224,6 +224,7 @@ ja:
224
224
  retried_before:
225
225
  one: 以前に DLQ から 1 回リトライされました。
226
226
  other: 以前に DLQ から %{count} 回リトライされました。
227
+ uniqueness_conflict_html: 別のコピーが一意性キーを保持していたため、%{attempts} 回の読み取りすべてでスキップされました(上限 %{max} 回) — 試行は一度も実行されていません
227
228
  show:
228
229
  back: DLQに戻る
229
230
  context: コンテキスト
@@ -224,6 +224,7 @@ nb:
224
224
  retried_before:
225
225
  one: Forsøkt på nytt fra DLQ én gang før.
226
226
  other: Forsøkt på nytt fra DLQ %{count} ganger før.
227
+ uniqueness_conflict_html: Hoppet over ved alle %{attempts} lesinger (maks. %{max}) mens en annen kopi av jobben holdt unikhetsnøkkelen — ingen forsøk kjørte
227
228
  show:
228
229
  back: Tilbake til DLQ
229
230
  context: Kontekst
@@ -224,6 +224,7 @@ nl:
224
224
  retried_before:
225
225
  one: Al eens eerder vanuit de DLQ opnieuw geprobeerd.
226
226
  other: Al %{count} keer eerder vanuit de DLQ opnieuw geprobeerd.
227
+ uniqueness_conflict_html: Bij alle %{attempts} keer lezen overgeslagen (max %{max}) omdat een andere kopie van de job de uniciteitssleutel had — geen poging uitgevoerd
227
228
  show:
228
229
  back: Terug naar DLQ
229
230
  context: Context
@@ -224,6 +224,7 @@ pt:
224
224
  retried_before:
225
225
  one: Já reenviado da DLQ uma vez.
226
226
  other: Já reenviado da DLQ %{count} vezes.
227
+ uniqueness_conflict_html: Ignorado em todas as %{attempts} leituras (máx. %{max}) enquanto outra cópia do job detinha sua chave de unicidade — nenhuma tentativa foi executada
227
228
  show:
228
229
  back: Voltar para DLQ
229
230
  context: Contexto
@@ -224,6 +224,7 @@ sv:
224
224
  retried_before:
225
225
  one: Omförsökt från DLQ en gång tidigare.
226
226
  other: Omförsökt från DLQ %{count} gånger tidigare.
227
+ uniqueness_conflict_html: Hoppades över vid alla %{attempts} läsningar (max %{max}) medan en annan kopia av jobbet höll dess unikhetsnyckel — inget försök kördes
227
228
  show:
228
229
  back: Tillbaka till DLQ
229
230
  context: Kontext
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rails/generators"
4
+ require "rails/generators/active_record"
5
+ require_relative "migration_path"
6
+
7
+ module Pgbus
8
+ module Generators
9
+ # Issue #633: PGMQ lower-cases its table names, so a queue pgbus created
10
+ # as "pgbus_Orders" before 0.23.0 lives in pgmq.q_pgbus_orders while
11
+ # pgmq.meta records the mixed-case spelling. pgbus now spells every queue
12
+ # in lower case; this migration brings an install's existing rows to that
13
+ # spelling. No table is dropped, renamed or copied.
14
+ class FoldQueueNameCaseGenerator < Rails::Generators::Base
15
+ include ActiveRecord::Generators::Migration
16
+ include MigrationPath
17
+
18
+ source_root File.expand_path("templates", __dir__)
19
+
20
+ desc "Spell existing PGMQ queues (pgmq.meta, NOTIFY triggers, pgbus's queue-name columns) in lower case, " \
21
+ "the way their tables are"
22
+
23
+ class_option :database,
24
+ type: :string,
25
+ default: nil,
26
+ desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
27
+
28
+ def create_migration_file
29
+ migration_template "fold_queue_name_case.rb.erb",
30
+ File.join(pgbus_migrate_path, "fold_pgbus_queue_name_case.rb")
31
+ end
32
+
33
+ def display_post_install
34
+ say ""
35
+ say "Pgbus queue-name fold migration created!", :green
36
+ say ""
37
+ say "PGMQ spells every queue table in lower case. This migration renames the"
38
+ say "pgmq.meta rows (and NOTIFY triggers) of queues created with upper-case"
39
+ say "letters to the lower-case spelling their tables already use. Messages stay"
40
+ say "where they are."
41
+ say ""
42
+ say "Next steps:"
43
+ say " 1. Deploy pgbus 0.23.0+ everywhere first: an older process recreates its mixed-case row"
44
+ say " 2. Run: rails db:migrate#{migrate_command_suffix}"
45
+ say " 3. Check: rake pgbus:doctor (\"Queue name case\" is ok); if it still names a queue,"
46
+ say " run: rake pgbus:queues:fold_case"
47
+ say ""
48
+ end
49
+
50
+ private
51
+
52
+ def migration_version
53
+ "[#{ActiveRecord::Migration.current_version}]"
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,12 @@
1
+ class FoldPgbusQueueNameCase < ActiveRecord::Migration<%= migration_version %>
2
+ # Issue #633. PGMQ lower-cases its table names, so a queue created as
3
+ # "pgbus_Orders" lives in pgmq.q_pgbus_orders while pgmq.meta records
4
+ # "pgbus_Orders". Pgbus::QueueNameFold renames the old rows to the lower-case
5
+ # spelling pgbus now uses; it changes rows only, never a table.
6
+ def up
7
+ Pgbus::QueueNameFold.run!(connection)
8
+ end
9
+
10
+ # The original spellings are not kept: PGMQ never used them for a table.
11
+ def down; end
12
+ end
@@ -25,8 +25,8 @@ class TunePgbusAutovacuum < ActiveRecord::Migration<%= migration_version %>
25
25
  q RECORD;
26
26
  BEGIN
27
27
  FOR q IN SELECT queue_name FROM pgmq.meta LOOP
28
- EXECUTE format('ALTER TABLE pgmq.q_%I RESET (autovacuum_vacuum_scale_factor, autovacuum_vacuum_cost_delay, autovacuum_analyze_scale_factor)', q.queue_name);
29
- EXECUTE format('ALTER TABLE pgmq.a_%I RESET (autovacuum_vacuum_scale_factor, autovacuum_vacuum_cost_delay, autovacuum_analyze_scale_factor)', q.queue_name);
28
+ EXECUTE format('ALTER TABLE pgmq.%I RESET (autovacuum_vacuum_scale_factor, autovacuum_vacuum_cost_delay, autovacuum_analyze_scale_factor)', 'q_' || lower(q.queue_name));
29
+ EXECUTE format('ALTER TABLE pgmq.%I RESET (autovacuum_vacuum_scale_factor, autovacuum_vacuum_cost_delay, autovacuum_analyze_scale_factor)', 'a_' || lower(q.queue_name));
30
30
  END LOOP;
31
31
  END $$;
32
32
  SQL
@@ -22,7 +22,7 @@ class TunePgbusFillfactor < ActiveRecord::Migration<%= migration_version %>
22
22
  q RECORD;
23
23
  BEGIN
24
24
  FOR q IN SELECT queue_name FROM pgmq.meta LOOP
25
- EXECUTE format('ALTER TABLE pgmq.q_%I RESET (fillfactor)', q.queue_name);
25
+ EXECUTE format('ALTER TABLE pgmq.%I RESET (fillfactor)', 'q_' || lower(q.queue_name));
26
26
  END LOOP;
27
27
  END $$;
28
28
  SQL
@@ -215,7 +215,7 @@ module Pgbus
215
215
  return true
216
216
  when :discard
217
217
  Pgbus.logger.info { "[Pgbus] Discarding job #{active_job.class.name}: concurrency limit for #{key}" }
218
- # The job will never run: roll back an :until_executed uniqueness lock
218
+ # The job will never run: roll back an enqueue-time uniqueness lock
219
219
  # acquired earlier in this enqueue (no executor will release it), and
220
220
  # uncount it from its batch so completion is not waited on forever.
221
221
  rollback_acquired_uniqueness_lock
@@ -226,7 +226,8 @@ module Pgbus
226
226
  false
227
227
  end
228
228
 
229
- # Releases an :until_executed lock this enqueue acquired, if any.
229
+ # Releases an enqueue-time lock (:until_executed, :until_executing) this
230
+ # enqueue acquired, if any.
230
231
  # Used when the job is dropped before a message is sent (concurrency
231
232
  # :discard, or send_message raising) — otherwise the lock is orphaned
232
233
  # because no executor will ever release it.
@@ -35,12 +35,16 @@ module Pgbus
35
35
  # it: the DLQ copy carries it in its headers (issue #495).
36
36
  last_error = FailedEventRecorder.last_error(queue_name: queue_name, msg_id: message.msg_id.to_i,
37
37
  source_queue: physical_queue(queue_name, source_queue))
38
- handle_dead_letter(message, queue_name, payload, source_queue: source_queue, last_error: last_error)
38
+ # Before the release below: the lookup asks whether the key's row is
39
+ # still this message's own (a crashed attempt's), see dead_letter_reason.
40
+ reason = dead_letter_reason(message, queue_name, payload, last_error)
41
+ handle_dead_letter(message, queue_name, payload, source_queue: source_queue, last_error: last_error,
42
+ reason: reason)
39
43
  FailedEventRecorder.clear!(queue_name: queue_name, msg_id: message.msg_id.to_i,
40
44
  source_queue: physical_queue(queue_name, source_queue))
41
45
  signal_concurrency(payload)
42
46
  signal_batch_discarded(payload)
43
- release_uniqueness_lock(Uniqueness.extract_key(payload), queue_name, message)
47
+ release_uniqueness_lock(Uniqueness.extract_key(payload), queue_name, message) if holds_uniqueness_row?(payload)
44
48
  record_stat(payload, queue_name, "dead_lettered", execution_start, message: message)
45
49
  instrument(
46
50
  "pgbus.job_dead_lettered",
@@ -50,7 +54,8 @@ module Pgbus
50
54
  provider_job_id: payload["provider_job_id"],
51
55
  read_ct: read_count,
52
56
  msg_id: message.msg_id.to_i,
53
- error: last_error&.dig(:error_class)
57
+ error: last_error&.dig(:error_class),
58
+ reason: reason
54
59
  )
55
60
  Pgbus.logger.debug { "[Pgbus::Executor] dead_lettered #{log_tag(message, queue_name)} job_class=#{job_class}" }
56
61
  return :dead_lettered
@@ -66,17 +71,23 @@ module Pgbus
66
71
  # released on completion or DLQ.
67
72
  nil
68
73
  when :while_executing
69
- # Acquire the lock now, bound to this message. If another worker is
70
- # already executing this job, skip it — VT will expire and it'll be
71
- # retried. A row left by a crashed attempt of THIS message is
74
+ # Acquire the lock now, bound to this message. If another message
75
+ # holds it, the job's on_conflict decides (issue #629): :discard
76
+ # drops this copy, :reject and :log leave it for a backed-off
77
+ # redelivery. A row left by a crashed attempt of THIS message is
72
78
  # re-acquired, not treated as a duplicate.
73
79
  acquired = Uniqueness.acquire_execution_lock(
74
80
  uniqueness_key, payload, msg_id: message.msg_id.to_i, queue_name: queue_name
75
81
  )
76
- unless acquired
77
- Pgbus.logger.info { "[Pgbus] Skipping duplicate execution for #{job_class}" }
78
- return :skipped
79
- end
82
+ return handle_uniqueness_conflict(message, queue_name, payload, uniqueness_key, source_queue) unless acquired
83
+ when :until_executing
84
+ # A debounce (issue #628): the row guarded the queued copy, and
85
+ # goes before perform so an enqueue that arrives while this runs
86
+ # is accepted. release_held_by! takes only the row this message
87
+ # holds, never a newer copy's (a dashboard re-send holds its own,
88
+ # #631). A redelivery (row already gone) matches nothing and runs; a failed release is logged and the
89
+ # job runs anyway — the reaper or the DLQ release takes the row.
90
+ release_uniqueness_lock(uniqueness_key, queue_name, message) if holds_uniqueness_row?(payload)
80
91
  end
81
92
  end
82
93
 
@@ -132,7 +143,9 @@ module Pgbus
132
143
  end
133
144
  Pgbus.logger.debug { "[Pgbus::Executor] #{@completed_word} #{log_tag(message, queue_name)} job_class=#{job_class}" }
134
145
  job_succeeded = true
135
- release_uniqueness_lock(uniqueness_key, queue_name, message)
146
+ # :until_executing released its row before perform; a newer copy's
147
+ # row is not this message's to take.
148
+ release_uniqueness_lock(uniqueness_key, queue_name, message) unless uniqueness_strategy == :until_executing
136
149
  # Only FailedEventRecorder.record! writes this table, and only after a
137
150
  # failed attempt, so a first delivery has no row: skip the per-job
138
151
  # DELETE round trip (issue #484). A stale row from a dropped-and-
@@ -228,6 +241,15 @@ module Pgbus
228
241
  Pgbus.logger.warn { "[Pgbus] Uniqueness release failed: #{e.message}" }
229
242
  end
230
243
 
244
+ # False for an :until_executing retry_on re-enqueue (executions > 0): it
245
+ # went out through the free key without acquiring a row (the attempt's
246
+ # was released at its start), so the unbound clause of release_held_by!
247
+ # could only match a newer copy's row — one parked behind its
248
+ # concurrency limit before this retry was sent (issue #628).
249
+ def holds_uniqueness_row?(payload)
250
+ Uniqueness.extract_strategy(payload) != :until_executing || payload["executions"].to_i.zero?
251
+ end
252
+
231
253
  # Keep the message invisible while perform runs (see VisibilityHeartbeat).
232
254
  # Wraps only the perform: the heartbeat must be gone before archive or
233
255
  # the retry backoff touches the same message's VT.
@@ -368,13 +390,88 @@ module Pgbus
368
390
  RetryBackoff.compute_delay(attempt: attempt)
369
391
  end
370
392
 
393
+ set_message_visibility(message, queue_name, delay, source_queue)
394
+ rescue StandardError => e
395
+ Pgbus.logger.debug { "[Pgbus] Retry backoff VT update failed: #{e.message}" }
396
+ end
397
+
398
+ def set_message_visibility(message, queue_name, delay, source_queue)
371
399
  if source_queue
372
400
  client.set_visibility_timeout(source_queue, message.msg_id.to_i, vt: delay, prefixed: false)
373
401
  else
374
402
  client.set_visibility_timeout(queue_name, message.msg_id.to_i, vt: delay)
375
403
  end
376
- rescue StandardError => e
377
- Pgbus.logger.debug { "[Pgbus] Retry backoff VT update failed: #{e.message}" }
404
+ end
405
+
406
+ # A :while_executing message whose key another message holds (issue
407
+ # #629). The duplicate holds no uniqueness row, so nothing is released.
408
+ def handle_uniqueness_conflict(message, queue_name, payload, key, source_queue)
409
+ policy = Uniqueness.extract_on_conflict(payload)
410
+ if policy == :discard
411
+ discard_duplicate(message, queue_name, payload, key, source_queue)
412
+ else
413
+ skip_duplicate(message, queue_name, payload, key, source_queue, policy)
414
+ end
415
+ end
416
+
417
+ # :discard — the duplicate will never run: complete it, then the
418
+ # bookkeeping the dead-letter path does for such a message. Lost to
419
+ # another worker that completed it first: that worker owns the signals.
420
+ def discard_duplicate(message, queue_name, payload, key, source_queue)
421
+ msg_id = message.msg_id.to_i
422
+ if complete_from(queue_name, msg_id, source_queue: source_queue) == :already_archived
423
+ Pgbus.logger.warn do
424
+ "[Pgbus::Executor] already #{@completed_word} elsewhere, skipping signals " \
425
+ "#{log_tag(message, queue_name)} job_class=#{payload["job_class"]}"
426
+ end
427
+ return :duplicate
428
+ end
429
+ if message.read_ct.to_i > 1
430
+ FailedEventRecorder.clear!(queue_name: queue_name, msg_id: msg_id,
431
+ source_queue: physical_queue(queue_name, source_queue))
432
+ end
433
+ signal_concurrency(payload)
434
+ signal_batch_discarded(payload)
435
+ instrument("pgbus.job_discarded", queue: queue_name, job_class: payload["job_class"], job_id: payload["job_id"],
436
+ provider_job_id: payload["provider_job_id"], msg_id: msg_id, reason: "uniqueness")
437
+ Pgbus.logger.info do
438
+ "[Pgbus] Discarded duplicate #{payload["job_class"]}: uniqueness key #{key} is held by another message " \
439
+ "(#{log_tag(message, queue_name)})"
440
+ end
441
+ :discarded
442
+ end
443
+
444
+ # :reject / :log — the copy still runs once the holder is done, but each
445
+ # skip backs its redelivery off so the skips stretch over the holder's
446
+ # run. read_ct cannot be decremented, so a skip still counts toward
447
+ # max_retries; the dead-letter reason says when that is why it died.
448
+ def skip_duplicate(message, queue_name, payload, key, source_queue, policy)
449
+ delay = skip_delay(message, payload)
450
+ begin
451
+ set_message_visibility(message, queue_name, delay, source_queue)
452
+ rescue StandardError => e
453
+ Pgbus.logger.warn { "[Pgbus] Skip backoff VT update failed: #{e.message}" }
454
+ end
455
+ text = "[Pgbus] Skipping duplicate execution for #{payload["job_class"]}: uniqueness key #{key} is held by " \
456
+ "another message (#{log_tag(message, queue_name)}); next delivery in #{delay}s"
457
+ policy == :log ? Pgbus.logger.warn { text } : Pgbus.logger.info { text }
458
+ instrument("pgbus.job_skipped", queue: queue_name, job_class: payload["job_class"], job_id: payload["job_id"],
459
+ provider_job_id: payload["provider_job_id"], msg_id: message.msg_id.to_i,
460
+ read_ct: message.read_ct.to_i, on_conflict: policy, vt: delay)
461
+ :skipped
462
+ end
463
+
464
+ # visibility_timeout * 2^(read_ct - 1), jittered and capped at the job's
465
+ # (or the global) retry_backoff_max, and never below visibility_timeout:
466
+ # a skip never comes back sooner than the read's own timeout would.
467
+ def skip_delay(message, payload)
468
+ vt = config.visibility_timeout
469
+ job_class = resolve_job_class(payload)
470
+ overrides = (job_class.respond_to?(:pgbus_retry_backoff_config) && job_class.pgbus_retry_backoff_config) || {}
471
+ max = [overrides[:max] || config.retry_backoff_max, vt].max
472
+ delay = RetryBackoff.compute_delay(attempt: message.read_ct.to_i, base: vt, max: max,
473
+ jitter: overrides[:jitter] || config.retry_backoff_jitter)
474
+ [delay, vt].max
378
475
  end
379
476
 
380
477
  def resolve_job_class(payload)
@@ -433,7 +530,9 @@ module Pgbus
433
530
  # config.completed_jobs = :delete it deletes the message instead
434
531
  # (delete_message, or the completer's delete_batch): pgmq.delete is as
435
532
  # idempotent and returns false the same way, so the symbols keep their
436
- # names and "archived" means "completed".
533
+ # names and "archived" means "completed". job_completion: true lets
534
+ # config.completion_synchronous_commit = false commit it with
535
+ # synchronous_commit off (issue #619).
437
536
  #
438
537
  # A `false` reported after our OWN retry is ambiguous, not a duplicate:
439
538
  # the first archive may well have committed with only its reply lost to
@@ -450,13 +549,13 @@ module Pgbus
450
549
  # Spelled out, no `**` forward: on Ruby 3.3 that
451
550
  # allocates per job even without keywords (#484 budget).
452
551
  elsif @delete_completed && source_queue
453
- client.delete_message(source_queue, msg_id, prefixed: false)
552
+ client.delete_message(source_queue, msg_id, prefixed: false, job_completion: true)
454
553
  elsif @delete_completed
455
- client.delete_message(queue_name, msg_id)
554
+ client.delete_message(queue_name, msg_id, job_completion: true)
456
555
  elsif source_queue
457
- client.archive_message(source_queue, msg_id, prefixed: false)
556
+ client.archive_message(source_queue, msg_id, prefixed: false, job_completion: true)
458
557
  else
459
- client.archive_message(queue_name, msg_id)
558
+ client.archive_message(queue_name, msg_id, job_completion: true)
460
559
  end
461
560
  return :archived unless archived == false
462
561
 
@@ -492,14 +591,37 @@ module Pgbus
492
591
  defined?(PGMQ::Errors::ConnectionError) && error.is_a?(PGMQ::Errors::ConnectionError)
493
592
  end
494
593
 
495
- def handle_dead_letter(message, queue_name, payload, source_queue: nil, last_error: nil)
594
+ # uniqueness_conflict for a :while_executing message no attempt of
595
+ # which failed: it was skipped on every read until read_ct passed
596
+ # max_retries (issue #629). Not whether the key is held at this read:
597
+ # the backed-off skips span minutes and the holder may have finished
598
+ # inside the last one. The one other way to die without a failed-event
599
+ # row is a crash loop, and a crashed attempt leaves the key's row bound
600
+ # to this message: that keeps max_retries_exceeded. A failed lookup
601
+ # keeps the stronger signal, the missing failed-event row.
602
+ def dead_letter_reason(message, queue_name, payload, last_error)
603
+ return DeadLetterHeader::REASON_MAX_RETRIES if last_error
604
+
605
+ key = Uniqueness.extract_key(payload)
606
+ return DeadLetterHeader::REASON_MAX_RETRIES unless key && Uniqueness.extract_strategy(payload) == :while_executing
607
+
608
+ own = UniquenessKey.bound_to?(key, queue_name: Uniqueness.lock_queue_name(queue_name, config: config),
609
+ msg_id: message.msg_id.to_i)
610
+ own ? DeadLetterHeader::REASON_MAX_RETRIES : DeadLetterHeader::REASON_UNIQUENESS_CONFLICT
611
+ rescue StandardError => e
612
+ Pgbus.logger.warn { "[Pgbus] Uniqueness row lookup failed, dead-lettering as uniqueness_conflict: #{e.message}" }
613
+ DeadLetterHeader::REASON_UNIQUENESS_CONFLICT
614
+ end
615
+
616
+ def handle_dead_letter(message, queue_name, payload, source_queue: nil, last_error: nil,
617
+ reason: DeadLetterHeader::REASON_MAX_RETRIES)
496
618
  Pgbus.logger.warn do
497
619
  job_class = payload["job_class"] || "unknown"
498
620
  error = last_error ? " (last error: #{last_error[:error_class]})" : ""
499
621
  "[Pgbus] Moving job #{job_class} to dead letter queue after #{message.read_ct} attempts#{error}"
500
622
  end
501
623
  headers = DeadLetterHeader.build(
502
- existing: message.headers, reason: DeadLetterHeader::REASON_MAX_RETRIES, source: "worker",
624
+ existing: message.headers, reason: reason, source: "worker",
503
625
  source_queue: source_queue || config.queue_name(queue_name), source_msg_id: message.msg_id.to_i,
504
626
  source_enqueued_at: message.enqueued_at, attempts: message.read_ct.to_i,
505
627
  max_retries: config.max_retries, error: last_error
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ module ActiveJob
5
+ # config.quiet_active_job_logs (issue #618): a worker process stops
6
+ # ActiveJob's per-job "Performing …" / "Performed …" info lines by
7
+ # unsubscribing ActiveJob::LogSubscriber from perform_start.active_job and
8
+ # perform.active_job. The perform listener also writes ActiveJob's
9
+ # "Error performing …" error line, for a job that raised (with its
10
+ # backtrace) and for one a before_perform callback halted, so those go too.
11
+ # An exception that escapes ActiveJob's handlers is still logged by
12
+ # ErrorReporter (class and message at error) and recorded in
13
+ # pgbus_failed_events; one retry_on catches still logs ActiveJob's
14
+ # "Retrying …" line (and "Stopped retrying …" at error when it gives up);
15
+ # a halted job logs nothing.
16
+ # The other ActiveJob events (Enqueued, Retrying, Stopped retrying,
17
+ # Discarded) still log as the host configured them, and the web process is
18
+ # untouched. Swapping
19
+ # ActiveJob::Base.logger instead would reach the web process too, and by
20
+ # default that logger is Rails.logger.
21
+ module QuietLogs
22
+ EVENTS = %w[perform_start.active_job perform.active_job].freeze
23
+
24
+ # Built at load; the applied notifiers are read and written under it.
25
+ @mutex = Mutex.new
26
+ @applied = {}.compare_by_identity
27
+
28
+ class << self
29
+ # Unsubscribes the LogSubscriber listeners once per notifier. Returns
30
+ # true when it did, false when that notifier was already done.
31
+ def apply!(notifier: ::ActiveSupport::Notifications.notifier)
32
+ @mutex.synchronize do
33
+ return false if @applied.key?(notifier)
34
+
35
+ @applied[notifier] = true
36
+ # all_listeners_for, not listeners_for: the latter skips a
37
+ # silenced subscriber (its logger above info), which would come
38
+ # back the moment the level drops.
39
+ EVENTS.each do |name|
40
+ notifier.all_listeners_for(name).each do |listener|
41
+ notifier.unsubscribe(listener) if log_subscriber?(listener)
42
+ end
43
+ end
44
+ true
45
+ end
46
+ end
47
+
48
+ private
49
+
50
+ # Fanout::Subscribers::Evented exposes #delegate on Rails 7.1 and later.
51
+ def log_subscriber?(listener)
52
+ listener.respond_to?(:delegate) && listener.delegate.is_a?(::ActiveJob::LogSubscriber)
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end
@@ -66,8 +66,8 @@ module Pgbus
66
66
  q RECORD;
67
67
  BEGIN
68
68
  FOR q IN SELECT queue_name FROM pgmq.meta LOOP
69
- EXECUTE format('ALTER TABLE pgmq.q_%I SET (#{settings_clause(QUEUE_SETTINGS)})', q.queue_name);
70
- EXECUTE format('ALTER TABLE pgmq.a_%I SET (#{settings_clause(ARCHIVE_SETTINGS)})', q.queue_name);
69
+ EXECUTE format('ALTER TABLE pgmq.%I SET (#{settings_clause(QUEUE_SETTINGS)})', 'q_' || lower(q.queue_name));
70
+ EXECUTE format('ALTER TABLE pgmq.%I SET (#{settings_clause(ARCHIVE_SETTINGS)})', 'a_' || lower(q.queue_name));
71
71
  END LOOP;
72
72
  END $$;
73
73
  SQL
data/lib/pgbus/cli/dlq.rb CHANGED
@@ -70,7 +70,8 @@ module Pgbus
70
70
  if data_source.retry_dlq_message(detail[:queue_name], detail[:msg_id])
71
71
  puts "Re-enqueued msg_id #{detail[:msg_id]} to #{origin_queue(detail[:queue_name])}."
72
72
  else
73
- warn "Failed to retry msg_id #{detail[:msg_id]}."
73
+ warn "Failed to retry msg_id #{detail[:msg_id]}: it is gone, the retry failed, " \
74
+ "another copy of the job holds its uniqueness key, or its concurrency limit is full."
74
75
  exit 1
75
76
  end
76
77
  end