pgbus 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.md +36 -14
- data/Rakefile +7 -1
- data/app/assets/javascripts/pgbus/stream_source_element.js +17 -3
- data/app/assets/javascripts/pgbus/stream_source_element.min.js +4 -4
- data/app/assets/javascripts/pgbus/stream_source_element.min.js.map +1 -1
- data/app/controllers/pgbus/dead_letter_controller.rb +3 -1
- data/app/controllers/pgbus/jobs_controller.rb +2 -1
- data/app/models/pgbus/blocked_execution.rb +3 -2
- data/app/models/pgbus/queue_state.rb +6 -0
- data/app/models/pgbus/uniqueness_key.rb +11 -0
- data/config/locales/da.yml +1 -0
- data/config/locales/de.yml +1 -0
- data/config/locales/en.yml +1 -0
- data/config/locales/es.yml +1 -0
- data/config/locales/fi.yml +1 -0
- data/config/locales/fr.yml +1 -0
- data/config/locales/it.yml +1 -0
- data/config/locales/ja.yml +1 -0
- data/config/locales/nb.yml +1 -0
- data/config/locales/nl.yml +1 -0
- data/config/locales/pt.yml +1 -0
- data/config/locales/sv.yml +1 -0
- data/lib/generators/pgbus/fold_queue_name_case_generator.rb +57 -0
- data/lib/generators/pgbus/templates/fold_queue_name_case.rb.erb +12 -0
- data/lib/generators/pgbus/templates/tune_autovacuum.rb.erb +2 -2
- data/lib/generators/pgbus/templates/tune_fillfactor.rb.erb +1 -1
- data/lib/pgbus/active_job/adapter.rb +3 -2
- data/lib/pgbus/active_job/executor.rb +142 -20
- data/lib/pgbus/active_job/quiet_logs.rb +57 -0
- data/lib/pgbus/autovacuum_tuning.rb +2 -2
- data/lib/pgbus/cli/dlq.rb +2 -1
- data/lib/pgbus/client/async_commit.rb +149 -0
- data/lib/pgbus/client/native_statements.rb +28 -16
- data/lib/pgbus/client.rb +145 -17
- data/lib/pgbus/concurrency/blocked_execution.rb +9 -1
- data/lib/pgbus/configuration.rb +71 -6
- data/lib/pgbus/dead_letter_header.rb +3 -0
- data/lib/pgbus/doctor.rb +55 -2
- data/lib/pgbus/failed_event_recorder.rb +5 -2
- data/lib/pgbus/generators/migration_detector.rb +29 -2
- data/lib/pgbus/instrumentation.rb +16 -1
- data/lib/pgbus/metrics/subscriber.rb +27 -1
- data/lib/pgbus/process/completer.rb +4 -2
- data/lib/pgbus/process/worker.rb +28 -3
- data/lib/pgbus/queue_name_fold.rb +145 -0
- data/lib/pgbus/queue_name_validator.rb +20 -2
- data/lib/pgbus/recurring/schedule.rb +22 -13
- data/lib/pgbus/stat_buffer.rb +67 -26
- data/lib/pgbus/streams/cursor.rb +17 -6
- data/lib/pgbus/streams/envelope.rb +16 -0
- data/lib/pgbus/streams/key.rb +1 -1
- data/lib/pgbus/table_maintenance.rb +1 -1
- data/lib/pgbus/uniqueness.rb +43 -16
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source/dead_letter.rb +14 -5
- data/lib/pgbus/web/data_source/resend.rb +185 -0
- data/lib/pgbus/web/data_source/resend_slot.rb +139 -0
- data/lib/pgbus/web/data_source.rb +55 -37
- data/lib/pgbus/web/dead_letter_reason.rb +6 -0
- data/lib/pgbus/web/streamer/connection.rb +36 -29
- data/lib/pgbus/web/streamer/falcon_connection.rb +29 -25
- data/lib/pgbus/web/streamer/heartbeat.rb +14 -3
- data/lib/pgbus/web/streamer/instance.rb +8 -5
- data/lib/pgbus/web/streamer/listener.rb +239 -62
- data/lib/pgbus/web/streamer/outbound_pump.rb +6 -4
- data/lib/pgbus/web/streamer/registry.rb +55 -29
- data/lib/pgbus/web/streamer/stream_event_dispatcher.rb +73 -12
- data/lib/tasks/pgbus_queues.rake +27 -0
- data/sig/pgbus/active_job/adapter.rbs +1 -1
- data/sig/pgbus/active_job/quiet_logs.rbs +11 -0
- data/sig/pgbus/client.rbs +6 -5
- data/sig/pgbus/configuration.rbs +4 -0
- metadata +11 -3
data/lib/pgbus/stat_buffer.rb
CHANGED
|
@@ -7,6 +7,8 @@ module Pgbus
|
|
|
7
7
|
class StatBuffer
|
|
8
8
|
DEFAULT_FLUSH_SIZE = 100
|
|
9
9
|
DEFAULT_FLUSH_INTERVAL = 5 # seconds
|
|
10
|
+
BASE_COLUMNS = "job_class, queue_name, status, duration_ms"
|
|
11
|
+
ALL_COLUMNS = "#{BASE_COLUMNS}, enqueue_latency_ms, retry_count".freeze
|
|
10
12
|
|
|
11
13
|
attr_reader :flush_size, :flush_interval
|
|
12
14
|
|
|
@@ -65,41 +67,80 @@ module Pgbus
|
|
|
65
67
|
|
|
66
68
|
private
|
|
67
69
|
|
|
70
|
+
# One multi-row INSERT per flush, built here rather than by
|
|
71
|
+
# JobStat.insert_all: ActiveRecord's per-value attribute check and type
|
|
72
|
+
# cast and its RETURNING "id" cost ~1 ms of Ruby CPU per 100 rows, 12-15 %
|
|
73
|
+
# of a no-op job's on-CPU time (issue #622). The integer columns are
|
|
74
|
+
# written through Integer(); the strings (job_class comes from the
|
|
75
|
+
# message) go through the connection's quoting, once per distinct value.
|
|
76
|
+
#
|
|
77
|
+
# Always attempt all 6 columns. If the latency migration hasn't run, the
|
|
78
|
+
# statement fails, and we fall back to base columns. This avoids relying
|
|
79
|
+
# on the memoized latency_columns? flag which can be stuck at false if
|
|
80
|
+
# evaluated before the connection is ready.
|
|
68
81
|
def write_to_database(entries)
|
|
69
82
|
return unless JobStat.table_exists?
|
|
70
83
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
{ job_class: e[:job_class], queue_name: e[:queue_name],
|
|
77
|
-
status: e[:status], duration_ms: e[:duration_ms],
|
|
78
|
-
enqueue_latency_ms: e[:enqueue_latency_ms], retry_count: e[:retry_count] }
|
|
79
|
-
end
|
|
84
|
+
JobStat.connection_pool.with_connection do |conn|
|
|
85
|
+
sql = insert_sql(conn, entries, latency: true)
|
|
86
|
+
conn.execute(sql, "Pgbus::JobStat Bulk Insert") if sql
|
|
87
|
+
rescue ActiveRecord::StatementInvalid => e
|
|
88
|
+
raise unless missing_latency_column?(e)
|
|
80
89
|
|
|
81
|
-
|
|
82
|
-
rescue ActiveRecord::StatementInvalid, ActiveModel::UnknownAttributeError => e
|
|
83
|
-
# Column doesn't exist — fall back to base columns only
|
|
84
|
-
if e.message.include?("enqueue_latency_ms") || e.message.include?("retry_count")
|
|
85
|
-
Pgbus.logger.warn { "[Pgbus] Latency columns missing — recording stats without latency data" } unless @degraded_warned
|
|
86
|
-
@degraded_warned = true
|
|
87
|
-
base_rows = entries.map do |entry|
|
|
88
|
-
{ job_class: entry[:job_class], queue_name: entry[:queue_name],
|
|
89
|
-
status: entry[:status], duration_ms: entry[:duration_ms] }
|
|
90
|
-
end
|
|
91
|
-
begin
|
|
92
|
-
JobStat.insert_all(base_rows)
|
|
93
|
-
rescue StandardError => fallback_error
|
|
94
|
-
Pgbus.logger.warn { "[Pgbus] Stat buffer fallback flush failed: #{fallback_error.message}" }
|
|
95
|
-
end
|
|
96
|
-
else
|
|
97
|
-
Pgbus.logger.warn { "[Pgbus] Stat buffer flush failed: #{e.message}" }
|
|
90
|
+
write_base_columns(conn, entries)
|
|
98
91
|
end
|
|
99
92
|
rescue StandardError => e
|
|
100
93
|
Pgbus.logger.warn { "[Pgbus] Stat buffer flush failed: #{e.message}" }
|
|
101
94
|
end
|
|
102
95
|
|
|
96
|
+
# PostgreSQL's undefined_column (42703, PG::UndefinedColumn: the cause
|
|
97
|
+
# ActiveRecord keeps) naming a latency column in its first line. The
|
|
98
|
+
# class, not the wording, says a column is missing (the wording follows
|
|
99
|
+
# the server's lc_messages); only the first line names which, because the
|
|
100
|
+
# "LINE 1:" excerpt of the statement below it shows both latency columns
|
|
101
|
+
# whatever went wrong. pg is loaded once PostgreSQL has raised anything.
|
|
102
|
+
def missing_latency_column?(error)
|
|
103
|
+
return false unless defined?(::PG::UndefinedColumn) && error.cause.is_a?(::PG::UndefinedColumn)
|
|
104
|
+
|
|
105
|
+
primary = error.cause.message.lines.first.to_s
|
|
106
|
+
primary.include?("enqueue_latency_ms") || primary.include?("retry_count")
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def write_base_columns(conn, entries)
|
|
110
|
+
Pgbus.logger.warn { "[Pgbus] Latency columns missing — recording stats without latency data" } unless @degraded_warned
|
|
111
|
+
@degraded_warned = true
|
|
112
|
+
sql = insert_sql(conn, entries, latency: false)
|
|
113
|
+
conn.execute(sql, "Pgbus::JobStat Bulk Insert") if sql
|
|
114
|
+
rescue StandardError => e
|
|
115
|
+
Pgbus.logger.warn { "[Pgbus] Stat buffer fallback flush failed: #{e.message}" }
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def insert_sql(conn, entries, latency:)
|
|
119
|
+
quoted = Hash.new { |memo, value| memo[value] = conn.quote(value) }
|
|
120
|
+
columns = latency ? ALL_COLUMNS : BASE_COLUMNS
|
|
121
|
+
values = entries.filter_map { |entry| sql_row(entry, quoted, latency) }
|
|
122
|
+
return if values.empty?
|
|
123
|
+
|
|
124
|
+
"INSERT INTO #{JobStat.quoted_table_name} (#{columns}) VALUES #{values.join(", ")}"
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# One entry's tuple, or nil when an integer column holds something
|
|
128
|
+
# Integer() refuses: that entry is dropped, the rest of the flush is still
|
|
129
|
+
# written. Logged on the six-column pass only: the base-column retry runs
|
|
130
|
+
# on the same entries, so anything it drops was dropped and logged already.
|
|
131
|
+
def sql_row(entry, quoted, latency)
|
|
132
|
+
row = "(#{quoted[entry[:job_class]]}, #{quoted[entry[:queue_name]]}, #{quoted[entry[:status]]}, #{sql_integer(entry[:duration_ms])}"
|
|
133
|
+
row << ", #{sql_integer(entry[:enqueue_latency_ms])}, #{sql_integer(entry[:retry_count])}" if latency
|
|
134
|
+
row << ")"
|
|
135
|
+
rescue ArgumentError, TypeError => e
|
|
136
|
+
Pgbus.logger.warn { "[Pgbus] Stat buffer dropped a #{entry[:job_class]} stat: #{e.message}" } if latency
|
|
137
|
+
nil
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def sql_integer(value)
|
|
141
|
+
value.nil? ? "NULL" : Integer(value).to_s
|
|
142
|
+
end
|
|
143
|
+
|
|
103
144
|
def monotonic_now
|
|
104
145
|
::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
105
146
|
end
|
data/lib/pgbus/streams/cursor.rb
CHANGED
|
@@ -12,6 +12,12 @@ module Pgbus
|
|
|
12
12
|
# because EventSource cannot send custom headers on the initial request)
|
|
13
13
|
#
|
|
14
14
|
# Returns 0 when no cursor is present (i.e. "deliver everything from now").
|
|
15
|
+
#
|
|
16
|
+
# A negative Last-Event-ID is not a cursor: ephemeral broadcasts carry a
|
|
17
|
+
# negative `id:` (they have no archive position), and a native EventSource
|
|
18
|
+
# that saw one last echoes it on reconnect. It is skipped, so the
|
|
19
|
+
# element's ?since= (its last durable id) decides (issue #621). A negative
|
|
20
|
+
# ?since= is still rejected: the element never sends one.
|
|
15
21
|
module Cursor
|
|
16
22
|
# PGMQ msg_id is BIGINT — strictly within signed 64-bit range.
|
|
17
23
|
MAX_MSG_ID = (2**63) - 1
|
|
@@ -34,7 +40,7 @@ module Pgbus
|
|
|
34
40
|
end
|
|
35
41
|
|
|
36
42
|
def self.pick(last_event_id, query_since)
|
|
37
|
-
return last_event_id if present?(last_event_id)
|
|
43
|
+
return last_event_id if present?(last_event_id) && !ephemeral_id?(last_event_id)
|
|
38
44
|
return query_since if present?(query_since)
|
|
39
45
|
|
|
40
46
|
nil
|
|
@@ -42,15 +48,20 @@ module Pgbus
|
|
|
42
48
|
|
|
43
49
|
def self.present?(value)
|
|
44
50
|
return false if value.nil?
|
|
45
|
-
# Any Integer is "present"
|
|
46
|
-
#
|
|
47
|
-
# so a caller passing a literal -1 still crashes loud rather than
|
|
48
|
-
# silently falling through to query_since.
|
|
51
|
+
# Any Integer is "present"; pick skips a negative last_event_id (an
|
|
52
|
+
# ephemeral frame's id) and validate! rejects a negative ?since=.
|
|
49
53
|
return true if value.is_a?(Integer)
|
|
50
54
|
|
|
51
55
|
!value.to_s.strip.empty?
|
|
52
56
|
end
|
|
53
57
|
|
|
58
|
+
# An ephemeral frame's id: a negative Integer, or its decimal text.
|
|
59
|
+
def self.ephemeral_id?(value)
|
|
60
|
+
return value.negative? if value.is_a?(Integer)
|
|
61
|
+
|
|
62
|
+
value.to_s.strip.match?(/\A-\d+\z/)
|
|
63
|
+
end
|
|
64
|
+
|
|
54
65
|
def self.coerce(raw)
|
|
55
66
|
return raw if raw.is_a?(Integer)
|
|
56
67
|
|
|
@@ -65,7 +76,7 @@ module Pgbus
|
|
|
65
76
|
raise InvalidCursor, "cursor #{value} is out of BIGINT range" if value > MAX_MSG_ID
|
|
66
77
|
end
|
|
67
78
|
|
|
68
|
-
private_class_method :pick, :present?, :coerce, :validate!
|
|
79
|
+
private_class_method :pick, :present?, :ephemeral_id?, :coerce, :validate!
|
|
69
80
|
end
|
|
70
81
|
end
|
|
71
82
|
end
|
|
@@ -70,6 +70,16 @@ module Pgbus
|
|
|
70
70
|
"retry: #{milliseconds}\n\n"
|
|
71
71
|
end
|
|
72
72
|
|
|
73
|
+
# The SSE event name for a frame: the envelope's typed event when
|
|
74
|
+
# present, else the default turbo-stream. Plain ReadAfter::Envelopes
|
|
75
|
+
# (no event field) and StreamEnvelopes with a nil event both fall back
|
|
76
|
+
# to the default (issue #170). Shared by the connections and the
|
|
77
|
+
# dispatcher, which frames each envelope once per wake (issue #621).
|
|
78
|
+
def self.sse_event_for(envelope)
|
|
79
|
+
event = envelope.respond_to?(:event) ? envelope.event : nil
|
|
80
|
+
event && !event.to_s.empty? ? event.to_s : Pgbus::Streams::DEFAULT_SSE_EVENT
|
|
81
|
+
end
|
|
82
|
+
|
|
73
83
|
def self.http_response_headers
|
|
74
84
|
RESPONSE_HEADERS
|
|
75
85
|
end
|
|
@@ -82,6 +92,12 @@ module Pgbus
|
|
|
82
92
|
# empty strings, so a payload ending in \n round-trips as an empty
|
|
83
93
|
# final `data:` line (the client's rejoin restores the newline).
|
|
84
94
|
def self.data_lines(str)
|
|
95
|
+
# Most payloads are one line (a rendered turbo-stream): no split, map
|
|
96
|
+
# or join, one String (issue #621). Two memchr scans beat the regex
|
|
97
|
+
# match? (232 vs 388 ns on a 170-byte line, YJIT) and cost a multi-line
|
|
98
|
+
# payload ~3 % before its split.
|
|
99
|
+
return "data: #{str}\n" unless str.include?("\n") || str.include?("\r")
|
|
100
|
+
|
|
85
101
|
lines = str.split(DATA_LINE_BREAK, -1)
|
|
86
102
|
lines = [""] if lines.empty? # "".split → [] — an empty payload still gets its data: line
|
|
87
103
|
lines.map { |line| "data: #{line}\n" }.join
|
data/lib/pgbus/streams/key.rb
CHANGED
|
@@ -9,7 +9,7 @@ module Pgbus
|
|
|
9
9
|
# PGMQ queue names are bounded by two ceilings: PostgreSQL's
|
|
10
10
|
# NAMEDATALEN (63 chars for `pgmq.q_<name>`) and pgmq-ruby's own
|
|
11
11
|
# stricter runtime check (`length >= 48` in
|
|
12
|
-
# `PGMQ::
|
|
12
|
+
# `PGMQ::QueueName.validate!`). The effective budget is the
|
|
13
13
|
# lower of the two, exposed as `QueueNameValidator::MAX_QUEUE_NAME_LENGTH`
|
|
14
14
|
# (currently 47). Any stream name composed from UUID primary keys
|
|
15
15
|
# and turbo-rails-style dom ids blows past that budget almost
|
|
@@ -37,7 +37,7 @@ module Pgbus
|
|
|
37
37
|
q RECORD;
|
|
38
38
|
BEGIN
|
|
39
39
|
FOR q IN SELECT queue_name FROM pgmq.meta LOOP
|
|
40
|
-
EXECUTE format('ALTER TABLE pgmq
|
|
40
|
+
EXECUTE format('ALTER TABLE pgmq.%I SET (fillfactor = #{FILLFACTOR})', 'q_' || lower(q.queue_name));
|
|
41
41
|
END LOOP;
|
|
42
42
|
END $$;
|
|
43
43
|
SQL
|
data/lib/pgbus/uniqueness.rb
CHANGED
|
@@ -13,13 +13,15 @@ module Pgbus
|
|
|
13
13
|
# 1. Enqueue: INSERT INTO pgbus_uniqueness_keys ON CONFLICT DO NOTHING
|
|
14
14
|
# (logical queue, msg_id=0). After send_message, bind! writes the
|
|
15
15
|
# real msg_id. The lock row lives as long as the job is in the queue
|
|
16
|
-
# or executing
|
|
16
|
+
# or executing (:until_executing: only while it is queued — the
|
|
17
|
+
# executor releases it before perform).
|
|
17
18
|
# 2. Execution: PGMQ's visibility timeout is the execution lock —
|
|
18
19
|
# no separate claim_for_execution step needed.
|
|
19
|
-
# 3. Retry: a retry_job / retry_on re-enqueue passes
|
|
20
|
-
# running attempt holds, and the row is handed to
|
|
21
|
-
# and re-stamped before the retry is sent or parked
|
|
22
|
-
# to it once sent (bind!).
|
|
20
|
+
# 3. Retry (:until_executed): a retry_job / retry_on re-enqueue passes
|
|
21
|
+
# through the key the running attempt holds, and the row is handed to
|
|
22
|
+
# the retry: unbound and re-stamped before the retry is sent or parked
|
|
23
|
+
# (unbind!), bound to it once sent (bind!). An :until_executing
|
|
24
|
+
# attempt released its row at start, so its retry goes out with none.
|
|
23
25
|
# 4. Completion/DLQ: the finishing message deletes the row only while it
|
|
24
26
|
# holds it (release_held_by!), so a row handed to a retry, or a
|
|
25
27
|
# successor's, stays. The dashboard's discards do the same.
|
|
@@ -31,9 +33,20 @@ module Pgbus
|
|
|
31
33
|
# :until_executed — Lock acquired at enqueue, held through execution, released on
|
|
32
34
|
# completion or DLQ. Prevents duplicate enqueue AND duplicate execution.
|
|
33
35
|
#
|
|
36
|
+
# :until_executing — Lock acquired at enqueue, released when execution starts
|
|
37
|
+
# (a debounce). Further enqueues are dropped while a copy is
|
|
38
|
+
# queued; one that arrives while a copy runs is accepted, so a
|
|
39
|
+
# run starting after it observes the state written before that
|
|
40
|
+
# enqueue. At most one copy is queued at a time.
|
|
41
|
+
# Two runs can overlap; compose with limits_concurrency to: 1,
|
|
42
|
+
# on_conflict: :block for "debounce and no overlap".
|
|
43
|
+
#
|
|
34
44
|
# :while_executing — Lock acquired at execution start, released on completion.
|
|
35
45
|
# Allows duplicate enqueue (multiple copies in queue) but only one
|
|
36
|
-
# executes at a time.
|
|
46
|
+
# executes at a time. A copy read while another message holds
|
|
47
|
+
# the key follows on_conflict (issue #629): :discard completes
|
|
48
|
+
# it without running; :reject and :log leave it for a
|
|
49
|
+
# backed-off redelivery (:log warns).
|
|
37
50
|
#
|
|
38
51
|
# Usage:
|
|
39
52
|
# class ImportOrderJob < ApplicationJob
|
|
@@ -50,12 +63,17 @@ module Pgbus
|
|
|
50
63
|
|
|
51
64
|
METADATA_KEY = "pgbus_uniqueness_key"
|
|
52
65
|
STRATEGY_KEY = "pgbus_uniqueness_strategy"
|
|
66
|
+
# The on_conflict policy of a :while_executing job, read by the executor
|
|
67
|
+
# when the lock is held by another message (issue #629).
|
|
68
|
+
ON_CONFLICT_KEY = "pgbus_uniqueness_on_conflict"
|
|
53
69
|
# Synthetic queue stored before produce when the caller does not know the
|
|
54
70
|
# real queue yet. Never a live PGMQ queue — the reaper must not probe
|
|
55
71
|
# pgmq.q_<prefix>_pending for these rows (issue #418).
|
|
56
72
|
PLACEHOLDER_QUEUE = "pending"
|
|
57
73
|
|
|
58
|
-
VALID_STRATEGIES = %i[until_executed while_executing].freeze
|
|
74
|
+
VALID_STRATEGIES = %i[until_executed until_executing while_executing].freeze
|
|
75
|
+
# The strategies that take the lock at enqueue, before the message exists.
|
|
76
|
+
ENQUEUE_LOCK_STRATEGIES = %i[until_executed until_executing].freeze
|
|
59
77
|
VALID_CONFLICTS = %i[reject discard log].freeze
|
|
60
78
|
|
|
61
79
|
# Class-body macro: it runs while the job class loads (at boot, or under the
|
|
@@ -129,7 +147,8 @@ module Pgbus
|
|
|
129
147
|
end
|
|
130
148
|
|
|
131
149
|
# Guards the class-name default key against the per-record collapse
|
|
132
|
-
# footgun (#333). When an
|
|
150
|
+
# footgun (#333). When an enqueue-locked job (:until_executed, or
|
|
151
|
+
# :until_executing, issue #628) was declared with NO explicit
|
|
133
152
|
# key (so the key is the class name) AND is enqueued WITH arguments, every
|
|
134
153
|
# distinct argument set would resolve to the same class-name key and
|
|
135
154
|
# collapse into one per-class singleton — almost never what the caller
|
|
@@ -139,11 +158,11 @@ module Pgbus
|
|
|
139
158
|
# per-invocation at execution start, not by class-name identity at enqueue).
|
|
140
159
|
def guard_class_name_default!(active_job, config, args)
|
|
141
160
|
return if config[:explicit_key]
|
|
142
|
-
return unless config[:strategy]
|
|
161
|
+
return unless ENQUEUE_LOCK_STRATEGIES.include?(config[:strategy])
|
|
143
162
|
return if args.nil? || args.empty?
|
|
144
163
|
|
|
145
164
|
raise ArgumentError,
|
|
146
|
-
"#{active_job.class.name} uses ensures_uniqueness strategy: :
|
|
165
|
+
"#{active_job.class.name} uses ensures_uniqueness strategy: #{config[:strategy].inspect} with no key: " \
|
|
147
166
|
"but is enqueued with arguments — the default key is the class name, which would collapse " \
|
|
148
167
|
"every distinct argument set into one per-class singleton. Pass an explicit " \
|
|
149
168
|
"key: ->(*args) { ... } that includes the arguments. See https://pgbus.dev/docs/upgrading-pgbus"
|
|
@@ -156,10 +175,9 @@ module Pgbus
|
|
|
156
175
|
key = resolve_key(active_job)
|
|
157
176
|
return payload_hash unless key
|
|
158
177
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
)
|
|
178
|
+
metadata = { METADATA_KEY => key, STRATEGY_KEY => config[:strategy].to_s }
|
|
179
|
+
metadata[ON_CONFLICT_KEY] = config[:on_conflict].to_s if config[:strategy] == :while_executing
|
|
180
|
+
payload_hash.merge(metadata)
|
|
163
181
|
end
|
|
164
182
|
|
|
165
183
|
def extract_key(payload)
|
|
@@ -170,19 +188,28 @@ module Pgbus
|
|
|
170
188
|
payload&.dig(STRATEGY_KEY)&.to_sym
|
|
171
189
|
end
|
|
172
190
|
|
|
191
|
+
# The :while_executing policy at the skip point. :reject when the payload
|
|
192
|
+
# carries none (enqueued before the key existed) or one this version does not know:
|
|
193
|
+
# that keeps the redelivery such a message had.
|
|
194
|
+
def extract_on_conflict(payload)
|
|
195
|
+
policy = payload&.dig(ON_CONFLICT_KEY)&.to_sym
|
|
196
|
+
VALID_CONFLICTS.include?(policy) ? policy : :reject
|
|
197
|
+
end
|
|
198
|
+
|
|
173
199
|
def uniqueness_config(active_job)
|
|
174
200
|
return nil unless active_job.class.respond_to?(:pgbus_uniqueness)
|
|
175
201
|
|
|
176
202
|
active_job.class.pgbus_uniqueness
|
|
177
203
|
end
|
|
178
204
|
|
|
179
|
-
# Acquire the uniqueness lock at enqueue time (:until_executed
|
|
205
|
+
# Acquire the uniqueness lock at enqueue time (:until_executed and
|
|
206
|
+
# :until_executing).
|
|
180
207
|
# Uses pg_advisory_xact_lock to serialize concurrent attempts.
|
|
181
208
|
# Returns :acquired, :locked, or :no_lock.
|
|
182
209
|
def acquire_enqueue_lock(key, active_job, queue_name: nil, msg_id: nil)
|
|
183
210
|
config = uniqueness_config(active_job)
|
|
184
211
|
return :no_lock unless config
|
|
185
|
-
return :no_lock unless config[:strategy]
|
|
212
|
+
return :no_lock unless ENQUEUE_LOCK_STRATEGIES.include?(config[:strategy])
|
|
186
213
|
|
|
187
214
|
acquired = if msg_id && queue_name
|
|
188
215
|
UniquenessKey.acquire!(key, queue_name: lock_queue_name(queue_name), msg_id: msg_id)
|
data/lib/pgbus/version.rb
CHANGED
|
@@ -80,12 +80,21 @@ module Pgbus
|
|
|
80
80
|
)
|
|
81
81
|
return false unless row
|
|
82
82
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
83
|
+
payload = row["message"].is_a?(String) ? JSON.parse(row["message"]) : row["message"]
|
|
84
|
+
# The row Discard would release is the one the copy takes over.
|
|
85
|
+
holder = lock_holder(queue_name, row["msg_id"], row["headers"], row["enqueued_at"])
|
|
86
|
+
# A live message must not claim to be dead: strip the block, count the trip.
|
|
87
|
+
headers = DeadLetterHeader.strip_for_retry(row["headers"])
|
|
88
|
+
drop_source = -> { @client.delete_message(queue_name, msg_id.to_i, prefixed: false) }
|
|
89
|
+
outcome = resend_job(payload, lock_queue: lock_queue_for_table(original_queue), holder: holder,
|
|
90
|
+
drop_source: drop_source, headers: JSON.parse(headers)) do
|
|
91
|
+
@client.transaction do |txn|
|
|
92
|
+
sent = txn.produce(original_queue, row["message"], headers: headers)
|
|
93
|
+
txn.delete(queue_name, msg_id.to_i)
|
|
94
|
+
sent
|
|
95
|
+
end
|
|
87
96
|
end
|
|
88
|
-
|
|
97
|
+
outcome != false
|
|
89
98
|
rescue StandardError => e
|
|
90
99
|
Pgbus.logger.debug { "[Pgbus::Web] Error retrying DLQ message #{msg_id}: #{e.message}" }
|
|
91
100
|
false
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Web
|
|
5
|
+
class DataSource
|
|
6
|
+
# Re-sending a job payload as a new message — DLQ Retry, and failed-job
|
|
7
|
+
# Retry when the original message is gone (issues #631, #636). The
|
|
8
|
+
# payload names its uniqueness key and strategy, and its
|
|
9
|
+
# limits_concurrency key. The copy releases the uniqueness row it holds
|
|
10
|
+
# when it starts (:until_executing) or finishes (:until_executed), and
|
|
11
|
+
# signals the semaphore when it finishes. Re-sent without a row or a
|
|
12
|
+
# slot of its own, that release could free a newer parked copy's key,
|
|
13
|
+
# and that signal gave back a slot another copy held and promoted a
|
|
14
|
+
# parked job beside it. So the copy goes out the way Adapter#enqueue
|
|
15
|
+
# sends a job: the uniqueness key taken unbound before the send and
|
|
16
|
+
# bound to the new message after it; a slot taken before the send, or
|
|
17
|
+
# the copy parked behind a full limit (on_conflict: :block), or the
|
|
18
|
+
# retry refused (:discard, :raise) with the source left in place.
|
|
19
|
+
module Resend
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
# Yields to send the payload; the block returns the new msg_id.
|
|
23
|
+
# Returns :sent, :parked (a BlockedExecution the limit's next signal
|
|
24
|
+
# promotes; nothing is sent and the block does not run), or false
|
|
25
|
+
# when another copy holds the uniqueness key or the limit is full
|
|
26
|
+
# under :discard / :raise (nothing is sent, so the source — the DLQ
|
|
27
|
+
# message or the failed row — stays for a later retry).
|
|
28
|
+
# +lock_queue+ is the queue as the executor spells the lock
|
|
29
|
+
# (Uniqueness.lock_queue_name), which is also the logical queue a
|
|
30
|
+
# parked copy is promoted onto; +holder+ is the [lock queue, msg_id,
|
|
31
|
+
# enqueued_at] of the message the payload was read from, as the
|
|
32
|
+
# dashboard's discards name it (#lock_holder). +drop_source+ removes
|
|
33
|
+
# the source once the copy is parked (a sent copy's source is the
|
|
34
|
+
# block's to remove). +headers+ (a Hash) are the ones the block
|
|
35
|
+
# sends; a parked copy carries them to its promotion.
|
|
36
|
+
def resend_job(payload, lock_queue:, holder:, drop_source:, headers: nil, &)
|
|
37
|
+
key = resend_lock_key(payload)
|
|
38
|
+
stamp = key && take_resend_lock(key, lock_queue, holder)
|
|
39
|
+
if key && !stamp
|
|
40
|
+
Pgbus.logger.warn { "[Pgbus::Web] Not re-sending #{payload["job_class"]}: uniqueness key #{key} is held by another copy" }
|
|
41
|
+
return false
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Only an error raised by the produce (:produce), or by the park's
|
|
45
|
+
# commit (:park: the COMMIT may have landed with its reply lost),
|
|
46
|
+
# can hide a live copy: one from the slot upsert, the park's insert
|
|
47
|
+
# or the source's removal proves nothing went out, however
|
|
48
|
+
# database-shaped it is.
|
|
49
|
+
sending = [nil]
|
|
50
|
+
begin
|
|
51
|
+
outcome = resend_holding_slot(payload, lock_queue, drop_source, sending, headers, &)
|
|
52
|
+
rescue StandardError => e
|
|
53
|
+
give_back_resend_lock(key, stamp, e, sending: sending.first) if key
|
|
54
|
+
raise
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
case outcome
|
|
58
|
+
when :refused
|
|
59
|
+
give_back_resend_lock(key, stamp) if key
|
|
60
|
+
false
|
|
61
|
+
when :parked
|
|
62
|
+
# Unbound, as a parked enqueue's row is: the promoted copy's
|
|
63
|
+
# release takes it (UniquenessKey.release_held_by!).
|
|
64
|
+
:parked
|
|
65
|
+
else
|
|
66
|
+
bind_resent_lock(key, lock_queue, outcome, stamp) if key
|
|
67
|
+
:sent
|
|
68
|
+
end
|
|
69
|
+
ensure
|
|
70
|
+
UniquenessKey.clear_bind_stamp!(key) if key
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# A failed job's message back to work: its vt reset to 0 while it is
|
|
74
|
+
# still queued (a second copy would run beside it; a failed attempt
|
|
75
|
+
# keeps its concurrency slot, so it needs no other), else its payload
|
|
76
|
+
# re-sent or parked. The failed row goes either way; false (row kept)
|
|
77
|
+
# while the re-send's key is held or its limit refuses it.
|
|
78
|
+
def retry_failed_job(event)
|
|
79
|
+
physical = failed_message_queue(event)
|
|
80
|
+
return resend_failed_event(event) unless physical
|
|
81
|
+
|
|
82
|
+
@client.set_visibility_timeout(physical, event["msg_id"].to_i, vt: 0, prefixed: false)
|
|
83
|
+
delete_failed_row(event)
|
|
84
|
+
true
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# A failed job's payload as a new message on its queue, its failed row
|
|
88
|
+
# deleted once it is sent or parked. Its message is gone, so it can
|
|
89
|
+
# hold only a row still bound to it (nil send time: no unbound row is
|
|
90
|
+
# its own). The delete comes after the key is bound: PGMQ sends on its
|
|
91
|
+
# own connection, so a failed delete cannot undo the send, and must
|
|
92
|
+
# not give back the key of a live copy.
|
|
93
|
+
def resend_failed_event(event) # rubocop:disable Naming/PredicateMethod
|
|
94
|
+
payload = JSON.parse(event["payload"])
|
|
95
|
+
headers = event["headers"]
|
|
96
|
+
headers = JSON.parse(headers) if headers.is_a?(String)
|
|
97
|
+
lock_queue = Uniqueness.lock_queue_name(event["queue_name"])
|
|
98
|
+
|
|
99
|
+
outcome = resend_job(payload, lock_queue: lock_queue, holder: [lock_queue, event["msg_id"].to_i, nil],
|
|
100
|
+
drop_source: -> { delete_failed_row(event) }, headers: headers) do
|
|
101
|
+
@client.send_message(event["queue_name"], payload, headers: headers)
|
|
102
|
+
end
|
|
103
|
+
delete_failed_row(event) if outcome == :sent
|
|
104
|
+
outcome != false
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def delete_failed_row(event)
|
|
108
|
+
connection.exec_delete(
|
|
109
|
+
"DELETE FROM pgbus_failed_events WHERE id = $1", "Pgbus Delete Failed Event", [event["id"].to_i]
|
|
110
|
+
)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# The key for the copy: the row the source message still holds is
|
|
114
|
+
# taken over (it went without releasing it — a release that failed,
|
|
115
|
+
# or a failed job whose message is gone), as a retry takes its
|
|
116
|
+
# attempt's (Adapter#hand_lock_to_retry); else a free key is
|
|
117
|
+
# acquired. Returns the row's created_at stamp (bind! and the
|
|
118
|
+
# give-back match the row by it), nil when another copy holds the
|
|
119
|
+
# key. A newer copy parked after the source released its row was
|
|
120
|
+
# acquired after the source was sent, so the takeover never matches it.
|
|
121
|
+
def take_resend_lock(key, lock_queue, holder)
|
|
122
|
+
from_queue, from_msg_id, from_enqueued_at = holder
|
|
123
|
+
stamp = UniquenessKey.unbind!(key, from_queue_name: from_queue, from_msg_id: from_msg_id,
|
|
124
|
+
from_enqueued_at: from_enqueued_at, queue_name: lock_queue)
|
|
125
|
+
return stamp if stamp
|
|
126
|
+
return unless UniquenessKey.acquire!(key, queue_name: lock_queue, msg_id: 0)
|
|
127
|
+
|
|
128
|
+
Thread.current[:pgbus_uniqueness_created_at]&.[](key)
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# The copy did not go out (the limit refused it), or the send raised.
|
|
132
|
+
# Proven unsent, the row taken for it goes (matched by its stamp, so
|
|
133
|
+
# never a successor's). An error from the produce that may hide a
|
|
134
|
+
# committed send (a lost connection or reply, as
|
|
135
|
+
# Adapter#ambiguous_delivery? counts it), or any error from the park's
|
|
136
|
+
# commit, keeps the row: the copy may be live, and the reaper takes
|
|
137
|
+
# the row if it is not. A failed give-back is logged, so the send's
|
|
138
|
+
# error is the one that surfaces.
|
|
139
|
+
def give_back_resend_lock(key, stamp, error = nil, sending: nil)
|
|
140
|
+
if sending == :park || (sending == :produce && ambiguous_resend?(error))
|
|
141
|
+
Pgbus.logger.warn { "[Pgbus::Web] Re-send of #{key} may have been sent (#{error.class}); its key stays held" }
|
|
142
|
+
return
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
UniquenessKey.release_if_unbound!(key, acquired_before: stamp)
|
|
146
|
+
rescue StandardError => e
|
|
147
|
+
Pgbus.logger.warn { "[Pgbus::Web] Giving back the uniqueness key #{key} failed: #{e.message}" }
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# ActiveRecord wraps a PG error (StatementInvalid, ConnectionFailed),
|
|
151
|
+
# so the cause is asked too.
|
|
152
|
+
def ambiguous_resend?(error)
|
|
153
|
+
[error, error&.cause].compact.any? do |err|
|
|
154
|
+
(defined?(PGMQ::Errors::ConnectionError) && err.is_a?(PGMQ::Errors::ConnectionError)) ||
|
|
155
|
+
(defined?(PG::Error) && err.is_a?(PG::Error))
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# The key a re-sent copy has to hold: only an enqueue-locked one. An
|
|
160
|
+
# :until_executing payload with executions > 0 is a retry_on
|
|
161
|
+
# re-enqueue, which went out with no row and whose copy releases none
|
|
162
|
+
# (Executor#holds_uniqueness_row?): a row taken for it would never go.
|
|
163
|
+
def resend_lock_key(payload)
|
|
164
|
+
return unless payload.is_a?(Hash)
|
|
165
|
+
|
|
166
|
+
key = Uniqueness.extract_key(payload)
|
|
167
|
+
strategy = Uniqueness.extract_strategy(payload)
|
|
168
|
+
return unless key && Uniqueness::ENQUEUE_LOCK_STRATEGIES.include?(strategy)
|
|
169
|
+
return if strategy == :until_executing && payload["executions"].to_i.positive?
|
|
170
|
+
|
|
171
|
+
key.to_s
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# The copy is live: a failed bind is logged, not raised. The row stays
|
|
175
|
+
# unbound and stamped before the send, so the copy's own release
|
|
176
|
+
# still takes it.
|
|
177
|
+
def bind_resent_lock(key, lock_queue, msg_id, stamp)
|
|
178
|
+
UniquenessKey.bind!(key, queue_name: lock_queue, msg_id: msg_id.to_i, acquired_at: stamp)
|
|
179
|
+
rescue StandardError => e
|
|
180
|
+
Pgbus.logger.warn { "[Pgbus::Web] Uniqueness bind of re-sent #{key} failed: #{e.message}" }
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
end
|