pgbus 0.22.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +46 -0
  3. data/README.md +36 -14
  4. data/Rakefile +7 -1
  5. data/app/assets/javascripts/pgbus/stream_source_element.js +17 -3
  6. data/app/assets/javascripts/pgbus/stream_source_element.min.js +4 -4
  7. data/app/assets/javascripts/pgbus/stream_source_element.min.js.map +1 -1
  8. data/app/controllers/pgbus/dead_letter_controller.rb +3 -1
  9. data/app/controllers/pgbus/jobs_controller.rb +2 -1
  10. data/app/models/pgbus/blocked_execution.rb +3 -2
  11. data/app/models/pgbus/queue_state.rb +6 -0
  12. data/app/models/pgbus/uniqueness_key.rb +11 -0
  13. data/config/locales/da.yml +1 -0
  14. data/config/locales/de.yml +1 -0
  15. data/config/locales/en.yml +1 -0
  16. data/config/locales/es.yml +1 -0
  17. data/config/locales/fi.yml +1 -0
  18. data/config/locales/fr.yml +1 -0
  19. data/config/locales/it.yml +1 -0
  20. data/config/locales/ja.yml +1 -0
  21. data/config/locales/nb.yml +1 -0
  22. data/config/locales/nl.yml +1 -0
  23. data/config/locales/pt.yml +1 -0
  24. data/config/locales/sv.yml +1 -0
  25. data/lib/generators/pgbus/fold_queue_name_case_generator.rb +57 -0
  26. data/lib/generators/pgbus/templates/fold_queue_name_case.rb.erb +12 -0
  27. data/lib/generators/pgbus/templates/tune_autovacuum.rb.erb +2 -2
  28. data/lib/generators/pgbus/templates/tune_fillfactor.rb.erb +1 -1
  29. data/lib/pgbus/active_job/adapter.rb +3 -2
  30. data/lib/pgbus/active_job/executor.rb +142 -20
  31. data/lib/pgbus/active_job/quiet_logs.rb +57 -0
  32. data/lib/pgbus/autovacuum_tuning.rb +2 -2
  33. data/lib/pgbus/cli/dlq.rb +2 -1
  34. data/lib/pgbus/client/async_commit.rb +149 -0
  35. data/lib/pgbus/client/native_statements.rb +28 -16
  36. data/lib/pgbus/client.rb +145 -17
  37. data/lib/pgbus/concurrency/blocked_execution.rb +9 -1
  38. data/lib/pgbus/configuration.rb +71 -6
  39. data/lib/pgbus/dead_letter_header.rb +3 -0
  40. data/lib/pgbus/doctor.rb +55 -2
  41. data/lib/pgbus/failed_event_recorder.rb +5 -2
  42. data/lib/pgbus/generators/migration_detector.rb +29 -2
  43. data/lib/pgbus/instrumentation.rb +16 -1
  44. data/lib/pgbus/metrics/subscriber.rb +27 -1
  45. data/lib/pgbus/process/completer.rb +4 -2
  46. data/lib/pgbus/process/worker.rb +28 -3
  47. data/lib/pgbus/queue_name_fold.rb +145 -0
  48. data/lib/pgbus/queue_name_validator.rb +20 -2
  49. data/lib/pgbus/recurring/schedule.rb +22 -13
  50. data/lib/pgbus/stat_buffer.rb +67 -26
  51. data/lib/pgbus/streams/cursor.rb +17 -6
  52. data/lib/pgbus/streams/envelope.rb +16 -0
  53. data/lib/pgbus/streams/key.rb +1 -1
  54. data/lib/pgbus/table_maintenance.rb +1 -1
  55. data/lib/pgbus/uniqueness.rb +43 -16
  56. data/lib/pgbus/version.rb +1 -1
  57. data/lib/pgbus/web/data_source/dead_letter.rb +14 -5
  58. data/lib/pgbus/web/data_source/resend.rb +185 -0
  59. data/lib/pgbus/web/data_source/resend_slot.rb +139 -0
  60. data/lib/pgbus/web/data_source.rb +55 -37
  61. data/lib/pgbus/web/dead_letter_reason.rb +6 -0
  62. data/lib/pgbus/web/streamer/connection.rb +36 -29
  63. data/lib/pgbus/web/streamer/falcon_connection.rb +29 -25
  64. data/lib/pgbus/web/streamer/heartbeat.rb +14 -3
  65. data/lib/pgbus/web/streamer/instance.rb +8 -5
  66. data/lib/pgbus/web/streamer/listener.rb +239 -62
  67. data/lib/pgbus/web/streamer/outbound_pump.rb +6 -4
  68. data/lib/pgbus/web/streamer/registry.rb +55 -29
  69. data/lib/pgbus/web/streamer/stream_event_dispatcher.rb +73 -12
  70. data/lib/tasks/pgbus_queues.rake +27 -0
  71. data/sig/pgbus/active_job/adapter.rbs +1 -1
  72. data/sig/pgbus/active_job/quiet_logs.rbs +11 -0
  73. data/sig/pgbus/client.rbs +6 -5
  74. data/sig/pgbus/configuration.rbs +4 -0
  75. metadata +11 -3
@@ -7,6 +7,8 @@ module Pgbus
7
7
  class StatBuffer
8
8
  DEFAULT_FLUSH_SIZE = 100
9
9
  DEFAULT_FLUSH_INTERVAL = 5 # seconds
10
+ BASE_COLUMNS = "job_class, queue_name, status, duration_ms"
11
+ ALL_COLUMNS = "#{BASE_COLUMNS}, enqueue_latency_ms, retry_count".freeze
10
12
 
11
13
  attr_reader :flush_size, :flush_interval
12
14
 
@@ -65,41 +67,80 @@ module Pgbus
65
67
 
66
68
  private
67
69
 
70
+ # One multi-row INSERT per flush, built here rather than by
71
+ # JobStat.insert_all: ActiveRecord's per-value attribute check and type
72
+ # cast and its RETURNING "id" cost ~1 ms of Ruby CPU per 100 rows, 12-15 %
73
+ # of a no-op job's on-CPU time (issue #622). The integer columns are
74
+ # written through Integer(); the strings (job_class comes from the
75
+ # message) go through the connection's quoting, once per distinct value.
76
+ #
77
+ # Always attempt all 6 columns. If the latency migration hasn't run, the
78
+ # statement fails, and we fall back to base columns. This avoids relying
79
+ # on the memoized latency_columns? flag which can be stuck at false if
80
+ # evaluated before the connection is ready.
68
81
  def write_to_database(entries)
69
82
  return unless JobStat.table_exists?
70
83
 
71
- # Always attempt all 6 columns. If the latency migration hasn't
72
- # run, the first flush will fail, and we fall back to base columns.
73
- # This avoids relying on the memoized latency_columns? flag which
74
- # can be stuck at false if evaluated before the connection is ready.
75
- rows = entries.map do |e|
76
- { job_class: e[:job_class], queue_name: e[:queue_name],
77
- status: e[:status], duration_ms: e[:duration_ms],
78
- enqueue_latency_ms: e[:enqueue_latency_ms], retry_count: e[:retry_count] }
79
- end
84
+ JobStat.connection_pool.with_connection do |conn|
85
+ sql = insert_sql(conn, entries, latency: true)
86
+ conn.execute(sql, "Pgbus::JobStat Bulk Insert") if sql
87
+ rescue ActiveRecord::StatementInvalid => e
88
+ raise unless missing_latency_column?(e)
80
89
 
81
- JobStat.insert_all(rows)
82
- rescue ActiveRecord::StatementInvalid, ActiveModel::UnknownAttributeError => e
83
- # Column doesn't exist — fall back to base columns only
84
- if e.message.include?("enqueue_latency_ms") || e.message.include?("retry_count")
85
- Pgbus.logger.warn { "[Pgbus] Latency columns missing — recording stats without latency data" } unless @degraded_warned
86
- @degraded_warned = true
87
- base_rows = entries.map do |entry|
88
- { job_class: entry[:job_class], queue_name: entry[:queue_name],
89
- status: entry[:status], duration_ms: entry[:duration_ms] }
90
- end
91
- begin
92
- JobStat.insert_all(base_rows)
93
- rescue StandardError => fallback_error
94
- Pgbus.logger.warn { "[Pgbus] Stat buffer fallback flush failed: #{fallback_error.message}" }
95
- end
96
- else
97
- Pgbus.logger.warn { "[Pgbus] Stat buffer flush failed: #{e.message}" }
90
+ write_base_columns(conn, entries)
98
91
  end
99
92
  rescue StandardError => e
100
93
  Pgbus.logger.warn { "[Pgbus] Stat buffer flush failed: #{e.message}" }
101
94
  end
102
95
 
96
+ # PostgreSQL's undefined_column (42703, PG::UndefinedColumn: the cause
97
+ # ActiveRecord keeps) naming a latency column in its first line. The
98
+ # class, not the wording, says a column is missing (the wording follows
99
+ # the server's lc_messages); only the first line names which, because the
100
+ # "LINE 1:" excerpt of the statement below it shows both latency columns
101
+ # whatever went wrong. pg is loaded once PostgreSQL has raised anything.
102
+ def missing_latency_column?(error)
103
+ return false unless defined?(::PG::UndefinedColumn) && error.cause.is_a?(::PG::UndefinedColumn)
104
+
105
+ primary = error.cause.message.lines.first.to_s
106
+ primary.include?("enqueue_latency_ms") || primary.include?("retry_count")
107
+ end
108
+
109
+ def write_base_columns(conn, entries)
110
+ Pgbus.logger.warn { "[Pgbus] Latency columns missing — recording stats without latency data" } unless @degraded_warned
111
+ @degraded_warned = true
112
+ sql = insert_sql(conn, entries, latency: false)
113
+ conn.execute(sql, "Pgbus::JobStat Bulk Insert") if sql
114
+ rescue StandardError => e
115
+ Pgbus.logger.warn { "[Pgbus] Stat buffer fallback flush failed: #{e.message}" }
116
+ end
117
+
118
+ def insert_sql(conn, entries, latency:)
119
+ quoted = Hash.new { |memo, value| memo[value] = conn.quote(value) }
120
+ columns = latency ? ALL_COLUMNS : BASE_COLUMNS
121
+ values = entries.filter_map { |entry| sql_row(entry, quoted, latency) }
122
+ return if values.empty?
123
+
124
+ "INSERT INTO #{JobStat.quoted_table_name} (#{columns}) VALUES #{values.join(", ")}"
125
+ end
126
+
127
+ # One entry's tuple, or nil when an integer column holds something
128
+ # Integer() refuses: that entry is dropped, the rest of the flush is still
129
+ # written. Logged on the six-column pass only: the base-column retry runs
130
+ # on the same entries, so anything it drops was dropped and logged already.
131
+ def sql_row(entry, quoted, latency)
132
+ row = "(#{quoted[entry[:job_class]]}, #{quoted[entry[:queue_name]]}, #{quoted[entry[:status]]}, #{sql_integer(entry[:duration_ms])}"
133
+ row << ", #{sql_integer(entry[:enqueue_latency_ms])}, #{sql_integer(entry[:retry_count])}" if latency
134
+ row << ")"
135
+ rescue ArgumentError, TypeError => e
136
+ Pgbus.logger.warn { "[Pgbus] Stat buffer dropped a #{entry[:job_class]} stat: #{e.message}" } if latency
137
+ nil
138
+ end
139
+
140
+ def sql_integer(value)
141
+ value.nil? ? "NULL" : Integer(value).to_s
142
+ end
143
+
103
144
  def monotonic_now
104
145
  ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
105
146
  end
@@ -12,6 +12,12 @@ module Pgbus
12
12
  # because EventSource cannot send custom headers on the initial request)
13
13
  #
14
14
  # Returns 0 when no cursor is present (i.e. "deliver everything from now").
15
+ #
16
+ # A negative Last-Event-ID is not a cursor: ephemeral broadcasts carry a
17
+ # negative `id:` (they have no archive position), and a native EventSource
18
+ # that saw one last echoes it on reconnect. It is skipped, so the
19
+ # element's ?since= (its last durable id) decides (issue #621). A negative
20
+ # ?since= is still rejected: the element never sends one.
15
21
  module Cursor
16
22
  # PGMQ msg_id is BIGINT — strictly within signed 64-bit range.
17
23
  MAX_MSG_ID = (2**63) - 1
@@ -34,7 +40,7 @@ module Pgbus
34
40
  end
35
41
 
36
42
  def self.pick(last_event_id, query_since)
37
- return last_event_id if present?(last_event_id)
43
+ return last_event_id if present?(last_event_id) && !ephemeral_id?(last_event_id)
38
44
  return query_since if present?(query_since)
39
45
 
40
46
  nil
@@ -42,15 +48,20 @@ module Pgbus
42
48
 
43
49
  def self.present?(value)
44
50
  return false if value.nil?
45
- # Any Integer is "present" so precedence is decided here, not by
46
- # the sign of the value. validate! rejects negatives separately
47
- # so a caller passing a literal -1 still crashes loud rather than
48
- # silently falling through to query_since.
51
+ # Any Integer is "present"; pick skips a negative last_event_id (an
52
+ # ephemeral frame's id) and validate! rejects a negative ?since=.
49
53
  return true if value.is_a?(Integer)
50
54
 
51
55
  !value.to_s.strip.empty?
52
56
  end
53
57
 
58
+ # An ephemeral frame's id: a negative Integer, or its decimal text.
59
+ def self.ephemeral_id?(value)
60
+ return value.negative? if value.is_a?(Integer)
61
+
62
+ value.to_s.strip.match?(/\A-\d+\z/)
63
+ end
64
+
54
65
  def self.coerce(raw)
55
66
  return raw if raw.is_a?(Integer)
56
67
 
@@ -65,7 +76,7 @@ module Pgbus
65
76
  raise InvalidCursor, "cursor #{value} is out of BIGINT range" if value > MAX_MSG_ID
66
77
  end
67
78
 
68
- private_class_method :pick, :present?, :coerce, :validate!
79
+ private_class_method :pick, :present?, :ephemeral_id?, :coerce, :validate!
69
80
  end
70
81
  end
71
82
  end
@@ -70,6 +70,16 @@ module Pgbus
70
70
  "retry: #{milliseconds}\n\n"
71
71
  end
72
72
 
73
+ # The SSE event name for a frame: the envelope's typed event when
74
+ # present, else the default turbo-stream. Plain ReadAfter::Envelopes
75
+ # (no event field) and StreamEnvelopes with a nil event both fall back
76
+ # to the default (issue #170). Shared by the connections and the
77
+ # dispatcher, which frames each envelope once per wake (issue #621).
78
+ def self.sse_event_for(envelope)
79
+ event = envelope.respond_to?(:event) ? envelope.event : nil
80
+ event && !event.to_s.empty? ? event.to_s : Pgbus::Streams::DEFAULT_SSE_EVENT
81
+ end
82
+
73
83
  def self.http_response_headers
74
84
  RESPONSE_HEADERS
75
85
  end
@@ -82,6 +92,12 @@ module Pgbus
82
92
  # empty strings, so a payload ending in \n round-trips as an empty
83
93
  # final `data:` line (the client's rejoin restores the newline).
84
94
  def self.data_lines(str)
95
+ # Most payloads are one line (a rendered turbo-stream): no split, map
96
+ # or join, one String (issue #621). Two memchr scans beat the regex
97
+ # match? (232 vs 388 ns on a 170-byte line, YJIT) and cost a multi-line
98
+ # payload ~3 % before its split.
99
+ return "data: #{str}\n" unless str.include?("\n") || str.include?("\r")
100
+
85
101
  lines = str.split(DATA_LINE_BREAK, -1)
86
102
  lines = [""] if lines.empty? # "".split → [] — an empty payload still gets its data: line
87
103
  lines.map { |line| "data: #{line}\n" }.join
@@ -9,7 +9,7 @@ module Pgbus
9
9
  # PGMQ queue names are bounded by two ceilings: PostgreSQL's
10
10
  # NAMEDATALEN (63 chars for `pgmq.q_<name>`) and pgmq-ruby's own
11
11
  # stricter runtime check (`length >= 48` in
12
- # `PGMQ::Client#validate_queue_name!`). The effective budget is the
12
+ # `PGMQ::QueueName.validate!`). The effective budget is the
13
13
  # lower of the two, exposed as `QueueNameValidator::MAX_QUEUE_NAME_LENGTH`
14
14
  # (currently 47). Any stream name composed from UUID primary keys
15
15
  # and turbo-rails-style dom ids blows past that budget almost
@@ -37,7 +37,7 @@ module Pgbus
37
37
  q RECORD;
38
38
  BEGIN
39
39
  FOR q IN SELECT queue_name FROM pgmq.meta LOOP
40
- EXECUTE format('ALTER TABLE pgmq.q_%I SET (fillfactor = #{FILLFACTOR})', q.queue_name);
40
+ EXECUTE format('ALTER TABLE pgmq.%I SET (fillfactor = #{FILLFACTOR})', 'q_' || lower(q.queue_name));
41
41
  END LOOP;
42
42
  END $$;
43
43
  SQL
@@ -13,13 +13,15 @@ module Pgbus
13
13
  # 1. Enqueue: INSERT INTO pgbus_uniqueness_keys ON CONFLICT DO NOTHING
14
14
  # (logical queue, msg_id=0). After send_message, bind! writes the
15
15
  # real msg_id. The lock row lives as long as the job is in the queue
16
- # or executing.
16
+ # or executing (:until_executing: only while it is queued — the
17
+ # executor releases it before perform).
17
18
  # 2. Execution: PGMQ's visibility timeout is the execution lock —
18
19
  # no separate claim_for_execution step needed.
19
- # 3. Retry: a retry_job / retry_on re-enqueue passes through the key the
20
- # running attempt holds, and the row is handed to the retry: unbound
21
- # and re-stamped before the retry is sent or parked (unbind!), bound
22
- # to it once sent (bind!).
20
+ # 3. Retry (:until_executed): a retry_job / retry_on re-enqueue passes
21
+ # through the key the running attempt holds, and the row is handed to
22
+ # the retry: unbound and re-stamped before the retry is sent or parked
23
+ # (unbind!), bound to it once sent (bind!). An :until_executing
24
+ # attempt released its row at start, so its retry goes out with none.
23
25
  # 4. Completion/DLQ: the finishing message deletes the row only while it
24
26
  # holds it (release_held_by!), so a row handed to a retry, or a
25
27
  # successor's, stays. The dashboard's discards do the same.
@@ -31,9 +33,20 @@ module Pgbus
31
33
  # :until_executed — Lock acquired at enqueue, held through execution, released on
32
34
  # completion or DLQ. Prevents duplicate enqueue AND duplicate execution.
33
35
  #
36
+ # :until_executing — Lock acquired at enqueue, released when execution starts
37
+ # (a debounce). Further enqueues are dropped while a copy is
38
+ # queued; one that arrives while a copy runs is accepted, so a
39
+ # run starting after it observes the state written before that
40
+ # enqueue. At most one copy is queued at a time.
41
+ # Two runs can overlap; compose with limits_concurrency to: 1,
42
+ # on_conflict: :block for "debounce and no overlap".
43
+ #
34
44
  # :while_executing — Lock acquired at execution start, released on completion.
35
45
  # Allows duplicate enqueue (multiple copies in queue) but only one
36
- # executes at a time.
46
+ # executes at a time. A copy read while another message holds
47
+ # the key follows on_conflict (issue #629): :discard completes
48
+ # it without running; :reject and :log leave it for a
49
+ # backed-off redelivery (:log warns).
37
50
  #
38
51
  # Usage:
39
52
  # class ImportOrderJob < ApplicationJob
@@ -50,12 +63,17 @@ module Pgbus
50
63
 
51
64
  METADATA_KEY = "pgbus_uniqueness_key"
52
65
  STRATEGY_KEY = "pgbus_uniqueness_strategy"
66
+ # The on_conflict policy of a :while_executing job, read by the executor
67
+ # when the lock is held by another message (issue #629).
68
+ ON_CONFLICT_KEY = "pgbus_uniqueness_on_conflict"
53
69
  # Synthetic queue stored before produce when the caller does not know the
54
70
  # real queue yet. Never a live PGMQ queue — the reaper must not probe
55
71
  # pgmq.q_<prefix>_pending for these rows (issue #418).
56
72
  PLACEHOLDER_QUEUE = "pending"
57
73
 
58
- VALID_STRATEGIES = %i[until_executed while_executing].freeze
74
+ VALID_STRATEGIES = %i[until_executed until_executing while_executing].freeze
75
+ # The strategies that take the lock at enqueue, before the message exists.
76
+ ENQUEUE_LOCK_STRATEGIES = %i[until_executed until_executing].freeze
59
77
  VALID_CONFLICTS = %i[reject discard log].freeze
60
78
 
61
79
  # Class-body macro: it runs while the job class loads (at boot, or under the
@@ -129,7 +147,8 @@ module Pgbus
129
147
  end
130
148
 
131
149
  # Guards the class-name default key against the per-record collapse
132
- # footgun (#333). When an :until_executed job was declared with NO explicit
150
+ # footgun (#333). When an enqueue-locked job (:until_executed, or
151
+ # :until_executing, issue #628) was declared with NO explicit
133
152
  # key (so the key is the class name) AND is enqueued WITH arguments, every
134
153
  # distinct argument set would resolve to the same class-name key and
135
154
  # collapse into one per-class singleton — almost never what the caller
@@ -139,11 +158,11 @@ module Pgbus
139
158
  # per-invocation at execution start, not by class-name identity at enqueue).
140
159
  def guard_class_name_default!(active_job, config, args)
141
160
  return if config[:explicit_key]
142
- return unless config[:strategy] == :until_executed
161
+ return unless ENQUEUE_LOCK_STRATEGIES.include?(config[:strategy])
143
162
  return if args.nil? || args.empty?
144
163
 
145
164
  raise ArgumentError,
146
- "#{active_job.class.name} uses ensures_uniqueness strategy: :until_executed with no key: " \
165
+ "#{active_job.class.name} uses ensures_uniqueness strategy: #{config[:strategy].inspect} with no key: " \
147
166
  "but is enqueued with arguments — the default key is the class name, which would collapse " \
148
167
  "every distinct argument set into one per-class singleton. Pass an explicit " \
149
168
  "key: ->(*args) { ... } that includes the arguments. See https://pgbus.dev/docs/upgrading-pgbus"
@@ -156,10 +175,9 @@ module Pgbus
156
175
  key = resolve_key(active_job)
157
176
  return payload_hash unless key
158
177
 
159
- payload_hash.merge(
160
- METADATA_KEY => key,
161
- STRATEGY_KEY => config[:strategy].to_s
162
- )
178
+ metadata = { METADATA_KEY => key, STRATEGY_KEY => config[:strategy].to_s }
179
+ metadata[ON_CONFLICT_KEY] = config[:on_conflict].to_s if config[:strategy] == :while_executing
180
+ payload_hash.merge(metadata)
163
181
  end
164
182
 
165
183
  def extract_key(payload)
@@ -170,19 +188,28 @@ module Pgbus
170
188
  payload&.dig(STRATEGY_KEY)&.to_sym
171
189
  end
172
190
 
191
+ # The :while_executing policy at the skip point. :reject when the payload
192
+ # carries none (enqueued before the key existed) or one this version does not know:
193
+ # that keeps the redelivery such a message had.
194
+ def extract_on_conflict(payload)
195
+ policy = payload&.dig(ON_CONFLICT_KEY)&.to_sym
196
+ VALID_CONFLICTS.include?(policy) ? policy : :reject
197
+ end
198
+
173
199
  def uniqueness_config(active_job)
174
200
  return nil unless active_job.class.respond_to?(:pgbus_uniqueness)
175
201
 
176
202
  active_job.class.pgbus_uniqueness
177
203
  end
178
204
 
179
- # Acquire the uniqueness lock at enqueue time (:until_executed only).
205
+ # Acquire the uniqueness lock at enqueue time (:until_executed and
206
+ # :until_executing).
180
207
  # Uses pg_advisory_xact_lock to serialize concurrent attempts.
181
208
  # Returns :acquired, :locked, or :no_lock.
182
209
  def acquire_enqueue_lock(key, active_job, queue_name: nil, msg_id: nil)
183
210
  config = uniqueness_config(active_job)
184
211
  return :no_lock unless config
185
- return :no_lock unless config[:strategy] == :until_executed
212
+ return :no_lock unless ENQUEUE_LOCK_STRATEGIES.include?(config[:strategy])
186
213
 
187
214
  acquired = if msg_id && queue_name
188
215
  UniquenessKey.acquire!(key, queue_name: lock_queue_name(queue_name), msg_id: msg_id)
data/lib/pgbus/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Pgbus
4
- VERSION = "0.22.0"
4
+ VERSION = "0.23.0"
5
5
  end
@@ -80,12 +80,21 @@ module Pgbus
80
80
  )
81
81
  return false unless row
82
82
 
83
- @client.transaction do |txn|
84
- # A live message must not claim to be dead: strip the block, count the trip.
85
- txn.produce(original_queue, row["message"], headers: DeadLetterHeader.strip_for_retry(row["headers"]))
86
- txn.delete(queue_name, msg_id.to_i)
83
+ payload = row["message"].is_a?(String) ? JSON.parse(row["message"]) : row["message"]
84
+ # The row Discard would release is the one the copy takes over.
85
+ holder = lock_holder(queue_name, row["msg_id"], row["headers"], row["enqueued_at"])
86
+ # A live message must not claim to be dead: strip the block, count the trip.
87
+ headers = DeadLetterHeader.strip_for_retry(row["headers"])
88
+ drop_source = -> { @client.delete_message(queue_name, msg_id.to_i, prefixed: false) }
89
+ outcome = resend_job(payload, lock_queue: lock_queue_for_table(original_queue), holder: holder,
90
+ drop_source: drop_source, headers: JSON.parse(headers)) do
91
+ @client.transaction do |txn|
92
+ sent = txn.produce(original_queue, row["message"], headers: headers)
93
+ txn.delete(queue_name, msg_id.to_i)
94
+ sent
95
+ end
87
96
  end
88
- true
97
+ outcome != false
89
98
  rescue StandardError => e
90
99
  Pgbus.logger.debug { "[Pgbus::Web] Error retrying DLQ message #{msg_id}: #{e.message}" }
91
100
  false
@@ -0,0 +1,185 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ module Web
5
+ class DataSource
6
+ # Re-sending a job payload as a new message — DLQ Retry, and failed-job
7
+ # Retry when the original message is gone (issues #631, #636). The
8
+ # payload names its uniqueness key and strategy, and its
9
+ # limits_concurrency key. The copy releases the uniqueness row it holds
10
+ # when it starts (:until_executing) or finishes (:until_executed), and
11
+ # signals the semaphore when it finishes. Re-sent without a row or a
12
+ # slot of its own, that release could free a newer parked copy's key,
13
+ # and that signal gave back a slot another copy held and promoted a
14
+ # parked job beside it. So the copy goes out the way Adapter#enqueue
15
+ # sends a job: the uniqueness key taken unbound before the send and
16
+ # bound to the new message after it; a slot taken before the send, or
17
+ # the copy parked behind a full limit (on_conflict: :block), or the
18
+ # retry refused (:discard, :raise) with the source left in place.
19
+ module Resend
20
+ private
21
+
22
+ # Yields to send the payload; the block returns the new msg_id.
23
+ # Returns :sent, :parked (a BlockedExecution the limit's next signal
24
+ # promotes; nothing is sent and the block does not run), or false
25
+ # when another copy holds the uniqueness key or the limit is full
26
+ # under :discard / :raise (nothing is sent, so the source — the DLQ
27
+ # message or the failed row — stays for a later retry).
28
+ # +lock_queue+ is the queue as the executor spells the lock
29
+ # (Uniqueness.lock_queue_name), which is also the logical queue a
30
+ # parked copy is promoted onto; +holder+ is the [lock queue, msg_id,
31
+ # enqueued_at] of the message the payload was read from, as the
32
+ # dashboard's discards name it (#lock_holder). +drop_source+ removes
33
+ # the source once the copy is parked (a sent copy's source is the
34
+ # block's to remove). +headers+ (a Hash) are the ones the block
35
+ # sends; a parked copy carries them to its promotion.
36
+ def resend_job(payload, lock_queue:, holder:, drop_source:, headers: nil, &)
37
+ key = resend_lock_key(payload)
38
+ stamp = key && take_resend_lock(key, lock_queue, holder)
39
+ if key && !stamp
40
+ Pgbus.logger.warn { "[Pgbus::Web] Not re-sending #{payload["job_class"]}: uniqueness key #{key} is held by another copy" }
41
+ return false
42
+ end
43
+
44
+ # Only an error raised by the produce (:produce), or by the park's
45
+ # commit (:park: the COMMIT may have landed with its reply lost),
46
+ # can hide a live copy: one from the slot upsert, the park's insert
47
+ # or the source's removal proves nothing went out, however
48
+ # database-shaped it is.
49
+ sending = [nil]
50
+ begin
51
+ outcome = resend_holding_slot(payload, lock_queue, drop_source, sending, headers, &)
52
+ rescue StandardError => e
53
+ give_back_resend_lock(key, stamp, e, sending: sending.first) if key
54
+ raise
55
+ end
56
+
57
+ case outcome
58
+ when :refused
59
+ give_back_resend_lock(key, stamp) if key
60
+ false
61
+ when :parked
62
+ # Unbound, as a parked enqueue's row is: the promoted copy's
63
+ # release takes it (UniquenessKey.release_held_by!).
64
+ :parked
65
+ else
66
+ bind_resent_lock(key, lock_queue, outcome, stamp) if key
67
+ :sent
68
+ end
69
+ ensure
70
+ UniquenessKey.clear_bind_stamp!(key) if key
71
+ end
72
+
73
+ # A failed job's message back to work: its vt reset to 0 while it is
74
+ # still queued (a second copy would run beside it; a failed attempt
75
+ # keeps its concurrency slot, so it needs no other), else its payload
76
+ # re-sent or parked. The failed row goes either way; false (row kept)
77
+ # while the re-send's key is held or its limit refuses it.
78
+ def retry_failed_job(event)
79
+ physical = failed_message_queue(event)
80
+ return resend_failed_event(event) unless physical
81
+
82
+ @client.set_visibility_timeout(physical, event["msg_id"].to_i, vt: 0, prefixed: false)
83
+ delete_failed_row(event)
84
+ true
85
+ end
86
+
87
+ # A failed job's payload as a new message on its queue, its failed row
88
+ # deleted once it is sent or parked. Its message is gone, so it can
89
+ # hold only a row still bound to it (nil send time: no unbound row is
90
+ # its own). The delete comes after the key is bound: PGMQ sends on its
91
+ # own connection, so a failed delete cannot undo the send, and must
92
+ # not give back the key of a live copy.
93
+ def resend_failed_event(event) # rubocop:disable Naming/PredicateMethod
94
+ payload = JSON.parse(event["payload"])
95
+ headers = event["headers"]
96
+ headers = JSON.parse(headers) if headers.is_a?(String)
97
+ lock_queue = Uniqueness.lock_queue_name(event["queue_name"])
98
+
99
+ outcome = resend_job(payload, lock_queue: lock_queue, holder: [lock_queue, event["msg_id"].to_i, nil],
100
+ drop_source: -> { delete_failed_row(event) }, headers: headers) do
101
+ @client.send_message(event["queue_name"], payload, headers: headers)
102
+ end
103
+ delete_failed_row(event) if outcome == :sent
104
+ outcome != false
105
+ end
106
+
107
+ def delete_failed_row(event)
108
+ connection.exec_delete(
109
+ "DELETE FROM pgbus_failed_events WHERE id = $1", "Pgbus Delete Failed Event", [event["id"].to_i]
110
+ )
111
+ end
112
+
113
+ # The key for the copy: the row the source message still holds is
114
+ # taken over (it went without releasing it — a release that failed,
115
+ # or a failed job whose message is gone), as a retry takes its
116
+ # attempt's (Adapter#hand_lock_to_retry); else a free key is
117
+ # acquired. Returns the row's created_at stamp (bind! and the
118
+ # give-back match the row by it), nil when another copy holds the
119
+ # key. A newer copy parked after the source released its row was
120
+ # acquired after the source was sent, so the takeover never matches it.
121
+ def take_resend_lock(key, lock_queue, holder)
122
+ from_queue, from_msg_id, from_enqueued_at = holder
123
+ stamp = UniquenessKey.unbind!(key, from_queue_name: from_queue, from_msg_id: from_msg_id,
124
+ from_enqueued_at: from_enqueued_at, queue_name: lock_queue)
125
+ return stamp if stamp
126
+ return unless UniquenessKey.acquire!(key, queue_name: lock_queue, msg_id: 0)
127
+
128
+ Thread.current[:pgbus_uniqueness_created_at]&.[](key)
129
+ end
130
+
131
+ # The copy did not go out (the limit refused it), or the send raised.
132
+ # Proven unsent, the row taken for it goes (matched by its stamp, so
133
+ # never a successor's). An error from the produce that may hide a
134
+ # committed send (a lost connection or reply, as
135
+ # Adapter#ambiguous_delivery? counts it), or any error from the park's
136
+ # commit, keeps the row: the copy may be live, and the reaper takes
137
+ # the row if it is not. A failed give-back is logged, so the send's
138
+ # error is the one that surfaces.
139
+ def give_back_resend_lock(key, stamp, error = nil, sending: nil)
140
+ if sending == :park || (sending == :produce && ambiguous_resend?(error))
141
+ Pgbus.logger.warn { "[Pgbus::Web] Re-send of #{key} may have been sent (#{error.class}); its key stays held" }
142
+ return
143
+ end
144
+
145
+ UniquenessKey.release_if_unbound!(key, acquired_before: stamp)
146
+ rescue StandardError => e
147
+ Pgbus.logger.warn { "[Pgbus::Web] Giving back the uniqueness key #{key} failed: #{e.message}" }
148
+ end
149
+
150
+ # ActiveRecord wraps a PG error (StatementInvalid, ConnectionFailed),
151
+ # so the cause is asked too.
152
+ def ambiguous_resend?(error)
153
+ [error, error&.cause].compact.any? do |err|
154
+ (defined?(PGMQ::Errors::ConnectionError) && err.is_a?(PGMQ::Errors::ConnectionError)) ||
155
+ (defined?(PG::Error) && err.is_a?(PG::Error))
156
+ end
157
+ end
158
+
159
+ # The key a re-sent copy has to hold: only an enqueue-locked one. An
160
+ # :until_executing payload with executions > 0 is a retry_on
161
+ # re-enqueue, which went out with no row and whose copy releases none
162
+ # (Executor#holds_uniqueness_row?): a row taken for it would never go.
163
+ def resend_lock_key(payload)
164
+ return unless payload.is_a?(Hash)
165
+
166
+ key = Uniqueness.extract_key(payload)
167
+ strategy = Uniqueness.extract_strategy(payload)
168
+ return unless key && Uniqueness::ENQUEUE_LOCK_STRATEGIES.include?(strategy)
169
+ return if strategy == :until_executing && payload["executions"].to_i.positive?
170
+
171
+ key.to_s
172
+ end
173
+
174
+ # The copy is live: a failed bind is logged, not raised. The row stays
175
+ # unbound and stamped before the send, so the copy's own release
176
+ # still takes it.
177
+ def bind_resent_lock(key, lock_queue, msg_id, stamp)
178
+ UniquenessKey.bind!(key, queue_name: lock_queue, msg_id: msg_id.to_i, acquired_at: stamp)
179
+ rescue StandardError => e
180
+ Pgbus.logger.warn { "[Pgbus::Web] Uniqueness bind of re-sent #{key} failed: #{e.message}" }
181
+ end
182
+ end
183
+ end
184
+ end
185
+ end