pgbus 0.22.0 → 0.23.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +52 -0
  3. data/README.md +41 -14
  4. data/Rakefile +7 -1
  5. data/app/assets/javascripts/pgbus/stream_source_element.js +17 -3
  6. data/app/assets/javascripts/pgbus/stream_source_element.min.js +4 -4
  7. data/app/assets/javascripts/pgbus/stream_source_element.min.js.map +1 -1
  8. data/app/controllers/pgbus/dead_letter_controller.rb +3 -1
  9. data/app/controllers/pgbus/jobs_controller.rb +2 -1
  10. data/app/models/pgbus/blocked_execution.rb +3 -2
  11. data/app/models/pgbus/queue_state.rb +6 -0
  12. data/app/models/pgbus/uniqueness_key.rb +11 -0
  13. data/config/locales/da.yml +1 -0
  14. data/config/locales/de.yml +1 -0
  15. data/config/locales/en.yml +1 -0
  16. data/config/locales/es.yml +1 -0
  17. data/config/locales/fi.yml +1 -0
  18. data/config/locales/fr.yml +1 -0
  19. data/config/locales/it.yml +1 -0
  20. data/config/locales/ja.yml +1 -0
  21. data/config/locales/nb.yml +1 -0
  22. data/config/locales/nl.yml +1 -0
  23. data/config/locales/pt.yml +1 -0
  24. data/config/locales/sv.yml +1 -0
  25. data/lib/generators/pgbus/fold_queue_name_case_generator.rb +57 -0
  26. data/lib/generators/pgbus/templates/fold_queue_name_case.rb.erb +12 -0
  27. data/lib/generators/pgbus/templates/tune_autovacuum.rb.erb +2 -2
  28. data/lib/generators/pgbus/templates/tune_fillfactor.rb.erb +1 -1
  29. data/lib/pgbus/active_job/adapter.rb +3 -2
  30. data/lib/pgbus/active_job/executor.rb +142 -20
  31. data/lib/pgbus/active_job/quiet_logs.rb +57 -0
  32. data/lib/pgbus/autovacuum_tuning.rb +2 -2
  33. data/lib/pgbus/cli/dlq.rb +2 -1
  34. data/lib/pgbus/client/async_commit.rb +149 -0
  35. data/lib/pgbus/client/native_statements.rb +28 -16
  36. data/lib/pgbus/client.rb +181 -17
  37. data/lib/pgbus/concurrency/blocked_execution.rb +9 -1
  38. data/lib/pgbus/configuration.rb +71 -6
  39. data/lib/pgbus/dead_letter_header.rb +3 -0
  40. data/lib/pgbus/doctor.rb +55 -2
  41. data/lib/pgbus/failed_event_recorder.rb +5 -2
  42. data/lib/pgbus/generators/migration_detector.rb +29 -2
  43. data/lib/pgbus/instrumentation.rb +16 -1
  44. data/lib/pgbus/metrics/subscriber.rb +27 -1
  45. data/lib/pgbus/process/completer.rb +4 -2
  46. data/lib/pgbus/process/worker.rb +28 -3
  47. data/lib/pgbus/queue_name_fold.rb +145 -0
  48. data/lib/pgbus/queue_name_validator.rb +20 -2
  49. data/lib/pgbus/recurring/schedule.rb +22 -13
  50. data/lib/pgbus/stat_buffer.rb +67 -26
  51. data/lib/pgbus/streams/cursor.rb +17 -6
  52. data/lib/pgbus/streams/envelope.rb +16 -0
  53. data/lib/pgbus/streams/key.rb +1 -1
  54. data/lib/pgbus/table_maintenance.rb +1 -1
  55. data/lib/pgbus/uniqueness.rb +43 -16
  56. data/lib/pgbus/version.rb +1 -1
  57. data/lib/pgbus/web/data_source/dead_letter.rb +14 -5
  58. data/lib/pgbus/web/data_source/resend.rb +185 -0
  59. data/lib/pgbus/web/data_source/resend_slot.rb +139 -0
  60. data/lib/pgbus/web/data_source.rb +55 -37
  61. data/lib/pgbus/web/dead_letter_reason.rb +6 -0
  62. data/lib/pgbus/web/streamer/connection.rb +36 -29
  63. data/lib/pgbus/web/streamer/falcon_connection.rb +29 -25
  64. data/lib/pgbus/web/streamer/heartbeat.rb +14 -3
  65. data/lib/pgbus/web/streamer/instance.rb +8 -5
  66. data/lib/pgbus/web/streamer/listener.rb +239 -62
  67. data/lib/pgbus/web/streamer/outbound_pump.rb +6 -4
  68. data/lib/pgbus/web/streamer/registry.rb +55 -29
  69. data/lib/pgbus/web/streamer/stream_event_dispatcher.rb +73 -12
  70. data/lib/tasks/pgbus_queues.rake +27 -0
  71. data/sig/pgbus/active_job/adapter.rbs +1 -1
  72. data/sig/pgbus/active_job/quiet_logs.rbs +11 -0
  73. data/sig/pgbus/client.rbs +6 -5
  74. data/sig/pgbus/configuration.rbs +4 -0
  75. metadata +11 -3
@@ -52,6 +52,37 @@ module Pgbus
52
52
  # Batch::Sweep then cannot tell a deleted batch job from one whose message
53
53
  # vanished another way (a purged queue): both count as completed, silently.
54
54
  attr_reader :completed_jobs
55
+ # completion_synchronous_commit (issue #619, opt-in): false commits a job's
56
+ # completion (the executor's archive or delete, the completer's
57
+ # archive_batch or delete_batch) with `SET LOCAL synchronous_commit TO off`,
58
+ # in one libpq pipeline: BEGIN, the SET LOCAL, the prepared statement and
59
+ # COMMIT go out with one sync, so the job stops waiting for its WAL flush.
60
+ # A completion is then acknowledged before its WAL record reaches disk: a
61
+ # Postgres server crash (not a client crash, not a planned switchover), or a
62
+ # failover to a standby that has not yet received that WAL (it also stops
63
+ # waiting for a synchronous standby), within about three times
64
+ # wal_writer_delay can lose completions committed in that window, and each
65
+ # lost completion redelivers its job after visibility_timeout. Sends, retries, dead-letter moves, the EventBus
66
+ # consumer's archive and dashboard actions keep full durability, and a
67
+ # completion inside a caller's transaction is left to that transaction.
68
+ # Native statements on pgbus's own connections only: with
69
+ # native_statements = false, or on ActiveRecord's shared connection (a
70
+ # connection_params Proc), it has no effect (pgbus doctor warns). Needs
71
+ # libpq 14+ and pg 1.5+ (pipeline mode), checked by validate!. true
72
+ # (default) commits as before.
73
+ attr_reader :completion_synchronous_commit
74
+ # quiet_active_job_logs (issue #618): true makes each worker process
75
+ # unsubscribe ActiveJob's LogSubscriber from perform_start.active_job and
76
+ # perform.active_job (Pgbus::ActiveJob::QuietLogs), so the per-job
77
+ # "Performing …" / "Performed …" info lines stop, and with them ActiveJob's
78
+ # "Error performing …" line (the same perform listener writes it): an
79
+ # exception that escapes ActiveJob's handlers is still logged by
80
+ # ErrorReporter and recorded in pgbus_failed_events, one retry_on catches
81
+ # still logs ActiveJob's "Retrying …" line, and a job a before_perform
82
+ # callback halted logs nothing.
83
+ # Enqueued, Retrying, Stopped retrying and Discarded still log, and the web
84
+ # process is untouched. Without ActiveJob it does nothing.
85
+ attr_reader :quiet_active_job_logs
55
86
  # visibility_heartbeat / visibility_heartbeat_interval: while a job runs,
56
87
  # its message's visibility timeout is re-armed every interval seconds
57
88
  # (default: a third of visibility_timeout), so a job that outlives the
@@ -367,6 +398,8 @@ module Pgbus
367
398
  @completion_batching = false
368
399
  @completion_batch_size = 100
369
400
  @completed_jobs = :archive
401
+ @completion_synchronous_commit = true
402
+ @quiet_active_job_logs = false
370
403
  @execution_mode = :threads
371
404
 
372
405
  @max_jobs_per_worker = nil
@@ -564,12 +597,13 @@ module Pgbus
564
597
  @streams_idle_timeout = 3_600 # 1 hour
565
598
  # 250ms — this value plays two roles: (1) the TCP keepalive
566
599
  # interval for the streamer's PG LISTEN connection, and (2) the
567
- # upper bound on how long Dispatcher#handle_connect waits for
568
- # the Listener to acknowledge a synchronous ensure_listening
569
- # call. 5s was unbounded enough to drop messages on a
570
- # realistic subscribe burst; 250ms keeps the connect-path race
571
- # window tight while still leaving headroom over a typical
572
- # PG keepalive interval.
600
+ # budget (plus 1s) a stream's first connect waits for a stuck local
601
+ # Listener to acknowledge its ensure_listening (with
602
+ # streams_listen_scope :master, HubClient waits this cycle plus 2s
603
+ # before failing over). A live listener
604
+ # is woken through a pipe and acks within a thread hop, and a
605
+ # connect on a stream that already has a subscriber does not
606
+ # wait at all (issue #630), so connects no longer pay this cycle.
573
607
  @streams_listen_health_check_ms = 250
574
608
  @streams_write_deadline_ms = 5_000
575
609
  # Deadline (ms) for a socket write made INSIDE the dispatcher's
@@ -856,6 +890,22 @@ module Pgbus
856
890
  @completed_jobs = mode
857
891
  end
858
892
 
893
+ def completion_synchronous_commit=(value)
894
+ unless [true, false].include?(value)
895
+ raise Pgbus::ConfigurationError, "completion_synchronous_commit must be true or false (got #{value.inspect})"
896
+ end
897
+
898
+ @completion_synchronous_commit = value
899
+ end
900
+
901
+ def quiet_active_job_logs=(value)
902
+ unless [true, false].include?(value)
903
+ raise Pgbus::ConfigurationError, "quiet_active_job_logs must be true or false (got #{value.inspect})"
904
+ end
905
+
906
+ @quiet_active_job_logs = value
907
+ end
908
+
859
909
  def native_statements=(value)
860
910
  raise Pgbus::ConfigurationError, "native_statements must be true or false" unless [true, false].include?(value)
861
911
 
@@ -1052,6 +1102,7 @@ module Pgbus
1052
1102
 
1053
1103
  validate_read_ahead!
1054
1104
  validate_completion_batching!
1105
+ validate_completion_synchronous_commit!
1055
1106
  validate_capsule_processes!
1056
1107
 
1057
1108
  if priority_levels && !(priority_levels.is_a?(Integer) && priority_levels >= 1 && priority_levels <= 10)
@@ -1843,6 +1894,20 @@ module Pgbus
1843
1894
  "execution_mode :async"
1844
1895
  end
1845
1896
 
1897
+ # The asynchronous completion commit runs in libpq pipeline mode (issue
1898
+ # #619): libpq 14+ and a pg gem that wraps it (1.5+). Checked here, once,
1899
+ # rather than on every completion.
1900
+ def validate_completion_synchronous_commit!
1901
+ return if completion_synchronous_commit
1902
+
1903
+ require "pg" unless defined?(::PG::Connection)
1904
+ return if ::PG::Connection.method_defined?(:enter_pipeline_mode) && ::PG.library_version >= 140_000
1905
+
1906
+ raise Pgbus::ConfigurationError,
1907
+ "completion_synchronous_commit = false needs libpq 14+ and pg 1.5+ (pipeline mode); " \
1908
+ "this process has libpq #{::PG.library_version} and pg #{::PG::VERSION}"
1909
+ end
1910
+
1846
1911
  # Shared resolver for a dedicated connection (streamer LISTEN / worker
1847
1912
  # NotifyListener) that can override the base `connection_options` to bypass a
1848
1913
  # pooler. Precedence: a full `url` wins outright; otherwise `host`/`port`
@@ -20,6 +20,9 @@ module Pgbus
20
20
  ORIGINAL_KEY = "pgbus_original_headers"
21
21
  VERSION = 1
22
22
  REASON_MAX_RETRIES = "max_retries_exceeded"
23
+ # A :while_executing duplicate skipped on every read while another
24
+ # message held its key, until read_ct passed max_retries (issue #629).
25
+ REASON_UNIQUENESS_CONFLICT = "uniqueness_conflict"
23
26
  # The release that started writing the block: rows without it predate it.
24
27
  SINCE = "0.18.0"
25
28
  MESSAGE_LIMIT = 1_000
data/lib/pgbus/doctor.rb CHANGED
@@ -5,7 +5,7 @@ require "pgbus/mcp/health_analyzer"
5
5
 
6
6
  module Pgbus
7
7
  # Preflight diagnostics for a pgbus deployment — the single command that
8
- # answers "is this environment healthy enough to run?". Runs twelve checks and
8
+ # answers "is this environment healthy enough to run?". Runs fourteen checks and
9
9
  # returns a machine-readable result plus a human report, so `pgbus doctor`
10
10
  # and `rake pgbus:doctor` can gate a deploy or CI run (exit 0 on success,
11
11
  # 1 on any failure).
@@ -36,13 +36,15 @@ module Pgbus
36
36
  "Database" => :check_database,
37
37
  "PGMQ schema" => :check_pgmq_schema,
38
38
  "Queues" => :check_queues,
39
+ "Queue name case" => :check_queue_name_case,
39
40
  "LISTEN/NOTIFY" => :check_notify,
40
41
  "Process liveness" => :check_processes,
41
42
  "GlobalID allowlist" => :check_allowed_global_id_models,
42
43
  "Broadcast queue" => :check_broadcast_queue,
43
44
  "Primary affinity" => :check_primary,
44
45
  "Dedicated connections" => :check_dedicated_connections,
45
- "Connection budget" => :check_connection_budget
46
+ "Connection budget" => :check_connection_budget,
47
+ "Completion commit" => :check_completion_commit
46
48
  }.freeze
47
49
 
48
50
  # Process liveness reads the pgbus_processes table (via HealthAnalyzer), so
@@ -128,6 +130,7 @@ module Pgbus
128
130
  resolved_pool_size: safe { @config.resolved_pool_size },
129
131
  listen_notify: @config.listen_notify,
130
132
  roles: @config.roles || "all",
133
+ completion_commit: @config.completion_synchronous_commit ? "sync" : "async",
131
134
  capsules: capsule_summary,
132
135
  database_url: redacted_database_url,
133
136
  connection_params: redacted_connection_params
@@ -252,6 +255,30 @@ module Pgbus
252
255
  Check.new(name: "Queues", status: :fail, detail: "#{e.class}: #{e.message}")
253
256
  end
254
257
 
258
+ # How many mixed-case queue names the queue-name case check lists.
259
+ CASE_NAMES_SHOWN = 5
260
+ private_constant :CASE_NAMES_SHOWN
261
+
262
+ # 4b. Queue-name case (issue #633). PGMQ spells every queue table in lower
263
+ # case, and pgbus now spells every queue name that way; a pgmq.meta row an
264
+ # older pgbus created as "pgbus_Orders" shares pgmq.q_pgbus_orders with the
265
+ # current spelling. Its messages are reachable, so this warns rather than
266
+ # fails: the dashboard lists the queue twice until the fold migration runs.
267
+ def check_queue_name_case
268
+ names = @client.mixed_case_queue_names
269
+ return Check.new(name: "Queue name case", status: :ok, detail: "every queue is spelled in lower case") if names.empty?
270
+
271
+ shown = names.first(CASE_NAMES_SHOWN).join(", ")
272
+ shown += ", …" if names.size > CASE_NAMES_SHOWN
273
+ Check.new(name: "Queue name case", status: :warn,
274
+ detail: "#{names.size} queue(s) in pgmq.meta spelled in mixed case (#{shown}); their tables are " \
275
+ "lower case and pgbus now uses that spelling — once every process runs this pgbus, run " \
276
+ "`rails generate pgbus:fold_queue_name_case` and migrate " \
277
+ "(`rake pgbus:queues:fold_case` once that migration has run)")
278
+ rescue StandardError => e
279
+ Check.new(name: "Queue name case", status: :warn, detail: "could not read pgmq.meta (#{e.class}: #{e.message})")
280
+ end
281
+
255
282
  # 5. LISTEN/NOTIFY — when configured on, every configured queue should carry
256
283
  # the insert-NOTIFY trigger. A missing trigger is a warning, not a failure:
257
284
  # pgbus still works (it falls back to polling), it just loses instant wakeup.
@@ -395,6 +422,32 @@ module Pgbus
395
422
  Check.new(name: "Dedicated connections", status: :fail, detail: "#{e.class}: #{e.message}")
396
423
  end
397
424
 
425
+ # config.completion_synchronous_commit (issue #619): informational, except
426
+ # that false does nothing with native_statements off or on ActiveRecord's
427
+ # shared connection, which warns.
428
+ def check_completion_commit
429
+ return Check.new(name: "Completion commit", status: :ok, detail: "synchronous (default)") if
430
+ @config.completion_synchronous_commit
431
+
432
+ unless @config.native_statements
433
+ return Check.new(name: "Completion commit", status: :warn,
434
+ detail: "completion_synchronous_commit = false has no effect: completions run through " \
435
+ "pgmq's functions (native_statements = false)")
436
+ end
437
+
438
+ if @client.respond_to?(:shared_connection?) && @client.shared_connection?
439
+ return Check.new(name: "Completion commit", status: :warn,
440
+ detail: "completion_synchronous_commit = false has no effect: pgbus shares ActiveRecord's " \
441
+ "connection (connection_params Proc); give pgbus its own connections to use it")
442
+ end
443
+
444
+ Check.new(name: "Completion commit", status: :ok,
445
+ detail: "asynchronous: completions commit with synchronous_commit off; a Postgres server crash can " \
446
+ "lose the last completions, whose jobs run again after visibility_timeout")
447
+ rescue StandardError => e
448
+ Check.new(name: "Completion commit", status: :warn, detail: "#{e.class}: #{e.message}")
449
+ end
450
+
398
451
  # How many direct LISTEN connections this config pins at steady state
399
452
  # (issue #381) — informational (always :ok) so operators can do capacity
400
453
  # math on the pooler's direct-connection budget from the doctor output.
@@ -85,11 +85,14 @@ module Pgbus
85
85
  # queue is dropped: PGMQ restarts msg_ids on recreate, so a surviving
86
86
  # row would aim the dashboard's retry/discard at an unrelated message.
87
87
  # Queue names are validated word characters, so the array literal is safe.
88
+ # Matched in any case (issue #633): the dropped queue's name is lower
89
+ # case, while a row may carry the executor's configured spelling
90
+ # ("Critical") or a physical name from before #633.
88
91
  def clear_queue!(queue_names)
89
92
  connection.exec_delete(
90
- "DELETE FROM pgbus_failed_events WHERE queue_name = ANY($1::text[])",
93
+ "DELETE FROM pgbus_failed_events WHERE lower(queue_name) = ANY($1::text[])",
91
94
  "FailedEvent Clear Queue",
92
- ["{#{queue_names.join(",")}}"]
95
+ ["{#{queue_names.map(&:downcase).join(",")}}"]
93
96
  )
94
97
  rescue StandardError => e
95
98
  Pgbus.logger.error do
@@ -102,7 +102,8 @@ module Pgbus
102
102
  add_batch_executions: "pgbus:add_batch_executions",
103
103
  add_batch_callback_jobs: "pgbus:add_batch_callback_jobs",
104
104
  tune_autovacuum: "pgbus:tune_autovacuum",
105
- tune_fillfactor: "pgbus:tune_fillfactor"
105
+ tune_fillfactor: "pgbus:tune_fillfactor",
106
+ fold_queue_name_case: "pgbus:fold_queue_name_case"
106
107
  }.freeze
107
108
 
108
109
  # Human-friendly description of each migration for the generator
@@ -128,9 +129,15 @@ module Pgbus
128
129
  add_batch_executions: "batch execution-row tracking (self-healing completion, on_failure rename)",
129
130
  add_batch_callback_jobs: "batch callback-job columns (configured ActiveJob instances as batch callbacks)",
130
131
  tune_autovacuum: "autovacuum tuning for PGMQ queue and archive tables",
131
- tune_fillfactor: "fillfactor=70 on PGMQ queue tables (reduces page density during update churn)"
132
+ tune_fillfactor: "fillfactor=70 on PGMQ queue tables (reduces page density during update churn)",
133
+ fold_queue_name_case: "lower-case spelling for queues created in mixed case (PGMQ's table names are lower case)"
132
134
  }.freeze
133
135
 
136
+ # A pgbus pgmq.meta row ($1: the lower-case "<queue_prefix>_") spelled
137
+ # with an upper-case letter (issue #633).
138
+ MIXED_CASE_QUEUE_SQL = "SELECT 1 FROM pgmq.meta WHERE queue_name <> lower(queue_name) " \
139
+ "AND left(lower(queue_name), length($1)) = $1 LIMIT 1"
140
+
134
141
  def initialize(connection)
135
142
  @connection = connection
136
143
  end
@@ -143,6 +150,9 @@ module Pgbus
143
150
  return [FRESH_INSTALL] if fresh_install?
144
151
 
145
152
  [
153
+ # First: the tuning migrations below build table names from
154
+ # pgmq.meta, which a mixed-case row spells unlike its table.
155
+ *queue_name_case_migrations,
146
156
  *uniqueness_key_migrations,
147
157
  *job_stats_migrations,
148
158
  *stream_stats_migrations,
@@ -328,6 +338,23 @@ module Pgbus
328
338
  [:tune_autovacuum]
329
339
  end
330
340
 
341
+ # Issue #633: a queue an older pgbus created in mixed case shares its
342
+ # lower-case table with the spelling pgbus now uses. An unreadable
343
+ # pgmq.meta queues nothing: the doctor's "Queue name case" check says why.
344
+ def queue_name_case_migrations
345
+ return [] unless pgmq_schema_exists?
346
+ return [] unless mixed_case_queue?
347
+
348
+ [:fold_queue_name_case]
349
+ end
350
+
351
+ def mixed_case_queue?
352
+ prefix = "#{Pgbus.configuration.queue_prefix.to_s.downcase}_"
353
+ connection.select_value(MIXED_CASE_QUEUE_SQL.gsub("$1", connection.quote(prefix))).present?
354
+ rescue StandardError
355
+ false
356
+ end
357
+
331
358
  # Fillfactor tuning: check if any PGMQ queue table already has
332
359
  # fillfactor applied. If not, queue the migration.
333
360
  def fillfactor_migrations
@@ -16,7 +16,22 @@ module Pgbus
16
16
  # pgbus.job_completed — job archived successfully
17
17
  # pgbus.job_failed — job raised; carries :exception_object
18
18
  # pgbus.job_dead_lettered — job exceeded max_retries and was DLQ-routed;
19
- # :error is the last recorded error class (or nil)
19
+ # :error is the last recorded error class (or nil);
20
+ # :reason is max_retries_exceeded, or uniqueness_conflict
21
+ # for a :while_executing copy with no failed attempt whose
22
+ # key's row is not its own (skipped while another message
23
+ # held the key); also uniqueness_conflict when that row
24
+ # lookup itself fails
25
+ # pgbus.job_skipped — a :while_executing copy was read while another
26
+ # message held its key and left for a later delivery
27
+ # (on_conflict :reject or :log)
28
+ # payload: queue, job_class, job_id, provider_job_id,
29
+ # msg_id, read_ct, on_conflict, vt (seconds until it
30
+ # is delivered again)
31
+ # pgbus.job_discarded — a copy was completed without running
32
+ # payload: queue, job_class, job_id, provider_job_id,
33
+ # msg_id, reason ("uniqueness": a :while_executing copy
34
+ # under on_conflict: :discard)
20
35
  # pgbus.job_visibility_extended — heartbeat re-armed a running job's visibility timeout
21
36
  # payload: queue, job_class, msg_id, vt, extensions
22
37
  # pgbus.event_processed — event handler succeeded
@@ -60,6 +60,8 @@ module Pgbus
60
60
  subscribe("pgbus.job_completed") { |event| on_job_completed(event) },
61
61
  subscribe("pgbus.job_failed") { |event| on_job_failed(event) },
62
62
  subscribe("pgbus.job_dead_lettered") { |event| on_job_dead_lettered(event) },
63
+ subscribe("pgbus.job_skipped") { |event| on_job_skipped(event) },
64
+ subscribe("pgbus.job_discarded") { |event| on_job_discarded(event) },
63
65
  subscribe("pgbus.job_visibility_extended") { |event| on_job_visibility_extended(event) },
64
66
  subscribe("pgbus.event_processed") { |event| on_event_processed(event) },
65
67
  subscribe("pgbus.event_failed") { |event| on_event_failed(event) },
@@ -121,11 +123,35 @@ module Pgbus
121
123
  )
122
124
  end
123
125
 
126
+ # reason: max_retries_exceeded, or uniqueness_conflict for a
127
+ # :while_executing duplicate skipped until it ran out of reads (#629).
124
128
  def on_job_dead_lettered(event)
125
129
  payload = event.payload
126
130
  backend.increment(
127
131
  "#{METRIC_PREFIX}queue_job_count", 1,
128
- compact(queue: payload[:queue], job_class: payload[:job_class], status: "dead_lettered")
132
+ compact(queue: payload[:queue], job_class: payload[:job_class], status: "dead_lettered",
133
+ reason: payload[:reason])
134
+ )
135
+ end
136
+
137
+ # A :while_executing duplicate left for a later delivery because
138
+ # another message held its key (issue #629): counted per skip, so one
139
+ # copy waiting out a long holder counts several times.
140
+ def on_job_skipped(event)
141
+ payload = event.payload
142
+ backend.increment(
143
+ "#{METRIC_PREFIX}queue_job_count", 1,
144
+ compact(queue: payload[:queue], job_class: payload[:job_class], status: "skipped", reason: "uniqueness")
145
+ )
146
+ end
147
+
148
+ # A :while_executing duplicate dropped under on_conflict: :discard.
149
+ def on_job_discarded(event)
150
+ payload = event.payload
151
+ backend.increment(
152
+ "#{METRIC_PREFIX}queue_job_count", 1,
153
+ compact(queue: payload[:queue], job_class: payload[:job_class], status: "discarded",
154
+ reason: payload[:reason])
129
155
  )
130
156
  end
131
157
 
@@ -94,7 +94,8 @@ module Pgbus
94
94
  return waiter.value if waiter
95
95
 
96
96
  # Stopped: complete on the caller's thread, with the same results.
97
- @client.public_send(@batch_op, queue, [msg_id], prefixed: false).empty? ? :already_archived : :archived
97
+ done = @client.public_send(@batch_op, queue, [msg_id], prefixed: false, job_completion: true)
98
+ done.empty? ? :already_archived : :archived
98
99
  end
99
100
 
100
101
  # Flushes whatever is pending, then joins the flusher. Past timeout the
@@ -157,7 +158,8 @@ module Pgbus
157
158
  # is absent. A duplicated id (the same message held by two threads after
158
159
  # a redelivery) is archived once, so only its first waiter gets :archived.
159
160
  def flush_slice(queue, slice)
160
- archived = @client.public_send(@batch_op, queue, slice.map(&:first).uniq, prefixed: false).to_set(&:to_i)
161
+ archived = @client.public_send(@batch_op, queue, slice.map(&:first).uniq, prefixed: false, job_completion: true)
162
+ .to_set(&:to_i)
161
163
  slice.each { |msg_id, waiter| waiter.resolve(archived.delete?(msg_id) ? :archived : :already_archived) }
162
164
  rescue Exception => e # rubocop:disable Lint/RescueException -- delivered to the waiters, never swallowed
163
165
  log_flush_failure(queue, slice, e) unless e.is_a?(StandardError)
@@ -194,6 +194,7 @@ module Pgbus
194
194
  POOL_TERMINATION_WAIT = 5
195
195
 
196
196
  def run
197
+ quiet_active_job_logs
197
198
  setup_signals
198
199
  start_heartbeat
199
200
  resolve_wildcard_queues
@@ -282,6 +283,18 @@ module Pgbus
282
283
 
283
284
  private
284
285
 
286
+ # config.quiet_active_job_logs (issue #618): only worker processes perform
287
+ # jobs, so only they drop the per-job "Performing …" / "Performed …"
288
+ # lines. ActiveJob::LogSubscriber attaches when active_job/base loads,
289
+ # which an app with no job class to eager-load has not done yet, so load
290
+ # it first (defined? does not trigger ActiveJob's autoload of Base).
291
+ def quiet_active_job_logs
292
+ return unless config.quiet_active_job_logs && defined?(::ActiveJob::Base)
293
+
294
+ require "active_job/base"
295
+ Pgbus::ActiveJob::QuietLogs.apply!
296
+ end
297
+
285
298
  # One loop step (issue #486): feed buffered claims to free slots, read
286
299
  # the deficit (free slots + read_ahead - buffered, capped by
287
300
  # prefetch_limit), feed again. With read_ahead 0 the buffer never
@@ -510,11 +523,18 @@ module Pgbus
510
523
  def process_message(message, queue_name, source_queue: nil)
511
524
  result = @executor.execute(message, queue_name, source_queue: source_queue)
512
525
  @jobs_processed.increment
513
- @rate_counter.increment(:processed)
514
- if result == :failed
526
+ # A skipped uniqueness duplicate comes back later: counting it now
527
+ # would count one job twice (issue #629).
528
+ @rate_counter.increment(:processed) unless result == :skipped
529
+ case result
530
+ when :failed
515
531
  @jobs_failed.increment
516
532
  @rate_counter.increment(:failed)
517
533
  @circuit_breaker.record_failure(queue_name)
534
+ when :skipped, :discarded
535
+ # No job ran: neither a failure nor a success that would wipe the
536
+ # failure streak of a queue whose real jobs keep failing.
537
+ nil
518
538
  else
519
539
  @circuit_breaker.record_success(queue_name)
520
540
  end
@@ -569,7 +589,12 @@ module Pgbus
569
589
  prefix = "#{config.queue_prefix}_"
570
590
  if (match = MISSING_QUEUE_REGEX.match(error.message))
571
591
  physical_name = match[1]
572
- logical_name = physical_name.delete_prefix(prefix)
592
+ # Postgres names the table as PGMQ spells it (lower case, issue #633;
593
+ # separators normalized), so find the queue as configured.
594
+ # A priority sub-queue (_pN) belongs to its base queue.
595
+ logical_name = @queues.find { |q| config.priority_queue_names(q).include?(physical_name) } ||
596
+ @queues.find { |q| config.queue_name(q) == physical_name } ||
597
+ physical_name.delete_prefix(prefix)
573
598
  if @queues.delete(logical_name)
574
599
  Pgbus.logger.warn { "[Pgbus] Evicted deleted queue '#{logical_name}' (#{physical_name}) from worker" }
575
600
  end
@@ -0,0 +1,145 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ # Issue #633 upgrade path. PGMQ lower-cases its table names
5
+ # (pgmq.format_table_name), so a queue an older pgbus created as
6
+ # "pgbus_Orders" lives in pgmq.q_pgbus_orders while pgmq.meta records
7
+ # "pgbus_Orders". pgbus now spells every queue in lower case; this renames
8
+ # the old rows to match their tables.
9
+ #
10
+ # Only rows change. The queue tables are shared by both spellings, so nothing
11
+ # here drops, renames or copies a table: dropping either spelling's queue
12
+ # through PGMQ would remove the shared table.
13
+ #
14
+ # Run by the pgbus:fold_queue_name_case migration and by
15
+ # `rake pgbus:queues:fold_case` (for a row an older process recreated after
16
+ # the migration ran). Idempotent. Takes an ActiveRecord connection and
17
+ # folds pgbus's queues only (those named "<prefix>_…" in any case), so
18
+ # another PGMQ user of the same schema keeps its spelling and its NOTIFY
19
+ # channel.
20
+ module QueueNameFold
21
+ module_function
22
+
23
+ def run!(connection, prefix: Pgbus.configuration.queue_prefix)
24
+ fold_pgmq_meta(connection, "#{prefix.to_s.downcase}_")
25
+ fold_unique_column(connection, :pgbus_stream_queues, prefer: "")
26
+ fold_unique_column(connection, :pgbus_queue_states, prefer: "paused DESC,")
27
+ fold_column(connection, :pgbus_uniqueness_keys, :queue_name)
28
+ fold_failed_event_source_queues(connection)
29
+ end
30
+
31
+ # One mixed-case pgmq.meta row at a time: add the lower-case row, move its
32
+ # topic bindings, delete the old row (and its throttle and binding rows),
33
+ # then re-arm the insert-NOTIFY trigger under the new name. PGMQ 1.13.1's
34
+ # trigger NOTIFYs on the channel of the name it was armed with, which
35
+ # listeners now spell in lower case. Where both spellings exist, the
36
+ # lower-case row's throttle wins.
37
+ def fold_pgmq_meta(connection, own_prefix)
38
+ prefix = connection.quote(own_prefix)
39
+ connection.execute(<<~SQL)
40
+ DO $$
41
+ DECLARE
42
+ q RECORD;
43
+ folded TEXT;
44
+ throttle INTEGER;
45
+ has_throttle BOOLEAN := to_regclass('pgmq.notify_insert_throttle') IS NOT NULL;
46
+ has_bindings BOOLEAN := to_regclass('pgmq.topic_bindings') IS NOT NULL;
47
+ BEGIN
48
+ FOR q IN
49
+ SELECT queue_name, is_partitioned, is_unlogged, created_at
50
+ FROM pgmq.meta
51
+ WHERE queue_name <> lower(queue_name)
52
+ AND left(lower(queue_name), length(#{prefix})) = #{prefix}
53
+ ORDER BY queue_name
54
+ LOOP
55
+ folded := lower(q.queue_name);
56
+
57
+ INSERT INTO pgmq.meta (queue_name, is_partitioned, is_unlogged, created_at)
58
+ VALUES (folded, q.is_partitioned, q.is_unlogged, q.created_at)
59
+ ON CONFLICT DO NOTHING;
60
+
61
+ throttle := NULL;
62
+ IF has_throttle THEN
63
+ SELECT COALESCE(
64
+ (SELECT throttle_interval_ms FROM pgmq.notify_insert_throttle WHERE queue_name = folded),
65
+ (SELECT throttle_interval_ms FROM pgmq.notify_insert_throttle WHERE queue_name = q.queue_name)
66
+ ) INTO throttle;
67
+ DELETE FROM pgmq.notify_insert_throttle WHERE queue_name = q.queue_name;
68
+ END IF;
69
+
70
+ IF has_bindings THEN
71
+ INSERT INTO pgmq.topic_bindings (pattern, queue_name, bound_at)
72
+ SELECT pattern, folded, bound_at FROM pgmq.topic_bindings WHERE queue_name = q.queue_name
73
+ ON CONFLICT DO NOTHING;
74
+ DELETE FROM pgmq.topic_bindings WHERE queue_name = q.queue_name;
75
+ END IF;
76
+
77
+ DELETE FROM pgmq.meta WHERE queue_name = q.queue_name;
78
+
79
+ IF throttle IS NOT NULL THEN
80
+ PERFORM pgmq.enable_notify_insert(folded, throttle);
81
+ END IF;
82
+ END LOOP;
83
+ END $$;
84
+ SQL
85
+ end
86
+
87
+ # A column with a unique index: keep one row per lower-case name (the
88
+ # `prefer` order first, then the row already in lower case, then the
89
+ # oldest), delete the rest, then lower-case what is left.
90
+ def fold_unique_column(connection, table, prefer:)
91
+ return unless connection.table_exists?(table)
92
+
93
+ connection.execute(<<~SQL)
94
+ DELETE FROM #{table} WHERE id IN (
95
+ SELECT id FROM (
96
+ SELECT id, row_number() OVER (
97
+ PARTITION BY lower(queue_name)
98
+ ORDER BY #{prefer} (queue_name = lower(queue_name)) DESC, id
99
+ ) AS rank
100
+ FROM #{table}
101
+ WHERE lower(queue_name) IN (SELECT lower(queue_name) FROM #{table} WHERE queue_name <> lower(queue_name))
102
+ ) ranked
103
+ WHERE rank > 1
104
+ )
105
+ SQL
106
+ fold_column(connection, table, :queue_name)
107
+ end
108
+
109
+ def fold_column(connection, table, column)
110
+ return unless connection.table_exists?(table) && connection.column_exists?(table, column)
111
+
112
+ connection.execute("UPDATE #{table} SET #{column} = lower(#{column}) WHERE #{column} <> lower(#{column})")
113
+ end
114
+
115
+ # source_queue is the physical queue a failure was read from, now spelled in
116
+ # lower case. queue_name is the executor's logical name, as configured, and
117
+ # stays. A message that failed under both spellings (before and after the
118
+ # deploy) has two rows: one per (queue_name, msg_id, folded source) stays,
119
+ # the lower-case one if there is one (written after the deploy), else the
120
+ # newest, so the (queue_name, msg_id, source_queue) unique index holds.
121
+ def fold_failed_event_source_queues(connection)
122
+ return unless connection.table_exists?(:pgbus_failed_events) && connection.column_exists?(:pgbus_failed_events, :source_queue)
123
+
124
+ connection.execute(<<~SQL)
125
+ DELETE FROM pgbus_failed_events WHERE id IN (
126
+ SELECT id FROM (
127
+ SELECT id, row_number() OVER (
128
+ PARTITION BY queue_name, msg_id, lower(source_queue)
129
+ ORDER BY (source_queue = lower(source_queue)) DESC, id DESC
130
+ ) AS rank
131
+ FROM pgbus_failed_events
132
+ WHERE (queue_name, msg_id, lower(source_queue)) IN (
133
+ SELECT queue_name, msg_id, lower(source_queue) FROM pgbus_failed_events
134
+ WHERE source_queue <> lower(source_queue)
135
+ )
136
+ ) ranked
137
+ WHERE rank > 1
138
+ )
139
+ SQL
140
+ fold_column(connection, :pgbus_failed_events, :source_queue)
141
+ end
142
+
143
+ private_class_method :fold_pgmq_meta, :fold_unique_column, :fold_column, :fold_failed_event_source_queues
144
+ end
145
+ end
@@ -9,7 +9,9 @@ module Pgbus
9
9
  module QueueNameValidator
10
10
  # PostgreSQL's NAMEDATALEN caps identifiers at 63 bytes, but the
11
11
  # effective limit is tighter: pgmq-ruby (our transport gem) rejects
12
- # any queue name with `length >= 48` in `PGMQ::Client#validate_queue_name!`.
12
+ # any queue name with `length >= PGMQ::QueueName::MAX_LENGTH` (48) in
13
+ # `PGMQ::QueueName.validate!`, whose error has said "maximum length of
14
+ # 47 characters" since pgmq-ruby 0.8.2 (48 before).
13
15
  # That leaves an actual usable ceiling of 47 characters for the
14
16
  # fully-prefixed name (`<queue_prefix>_<logical_name>`), which is what
15
17
  # this constant expresses. pgmq-ruby picked 48 to leave headroom for
@@ -26,6 +28,10 @@ module Pgbus
26
28
  # Only alphanumeric characters and underscores are allowed.
27
29
  VALID_QUEUE_NAME_PATTERN = /\A[a-zA-Z0-9_]+\z/
28
30
 
31
+ # A valid name already in lower case: what normalize returns.
32
+ CANONICAL_QUEUE_NAME = /\A[a-z0-9_]+\z/
33
+ private_constant :CANONICAL_QUEUE_NAME
34
+
29
35
  module_function
30
36
 
31
37
  # Validates a queue name for safe SQL identifier use.
@@ -58,14 +64,26 @@ module Pgbus
58
64
  # (`Pgbus.stream([user, :notifications])` → `"user_gid:notifications"`),
59
65
  # so they must map to a safe character rather than be stripped —
60
66
  # otherwise `"a:b"` and `"ab"` would collide on the same queue.
67
+ #
68
+ # The result is lower case (issue #633): PGMQ lower-cases its table
69
+ # names (`pgmq.format_table_name`), so "Orders" and "orders" share
70
+ # `pgmq.q_orders` while `pgmq.meta` records each spelling as its own
71
+ # queue. Spelling every name the way its table is gives one name per
72
+ # table. `validate!` and `sanitize!` keep case: they also run on names
73
+ # read back from `pgmq.meta`, which an install from before #633 may
74
+ # still hold in mixed case (`rails generate pgbus:fold_queue_name_case`).
61
75
  def normalize(name)
62
76
  name = name.to_s
63
- return validate!(name) if VALID_QUEUE_NAME_PATTERN.match?(name)
77
+ # Every enqueue and read names its queue through here: a name already
78
+ # spelled as its table costs one scan and no allocation.
79
+ return name if name.length <= MAX_QUEUE_NAME_LENGTH && CANONICAL_QUEUE_NAME.match?(name)
80
+ return validate!(name).downcase if VALID_QUEUE_NAME_PATTERN.match?(name)
64
81
 
65
82
  normalized = name.gsub(/[-.:]/, "_") # hyphens/dots/colons → underscores
66
83
  .gsub(/[^a-zA-Z0-9_]/, "") # strip remaining invalid chars
67
84
  .gsub(/_+/, "_") # collapse consecutive underscores
68
85
  .gsub(/\A_|_\z/, "") # strip leading/trailing underscores
86
+ normalized.downcase!
69
87
  validate!(normalized)
70
88
  normalized
71
89
  end