pgbus 0.23.1 → 0.23.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -0
- data/README.md +1 -1
- data/app/assets/javascripts/pgbus/stream_source_element.js +55 -11
- data/app/assets/javascripts/pgbus/stream_source_element.min.js +4 -4
- data/app/assets/javascripts/pgbus/stream_source_element.min.js.map +1 -1
- data/app/frontend/pgbus/application.min.js.map +1 -1
- data/app/frontend/pgbus/modules/charts.min.js.map +1 -1
- data/app/frontend/pgbus/vendor/turbo.min.js.map +1 -1
- data/app/models/pgbus/batch_entry.rb +70 -29
- data/app/models/pgbus/batch_execution.rb +8 -6
- data/app/models/pgbus/blocked_execution.rb +17 -15
- data/app/models/pgbus/job_stat.rb +16 -13
- data/app/models/pgbus/semaphore.rb +16 -14
- data/app/models/pgbus/uniqueness_key.rb +58 -45
- data/lib/pgbus/active_job/adapter.rb +7 -5
- data/lib/pgbus/batch.rb +76 -37
- data/lib/pgbus/concurrency/blocked_execution.rb +28 -24
- data/lib/pgbus/concurrency/semaphore.rb +17 -9
- data/lib/pgbus/failed_event_recorder.rb +34 -25
- data/lib/pgbus/version.rb +1 -1
- metadata +1 -1
|
@@ -38,19 +38,55 @@ module Pgbus
|
|
|
38
38
|
.update_all(["total_jobs = GREATEST(total_jobs - 1, 0)"])
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
+
# Resolve a batch member and count it, in one statement and no
|
|
42
|
+
# transaction (issue #650): delete the job's execution row and add one to
|
|
43
|
+
# +column+ on the batch that row belonged to, RETURNING the batch row.
|
|
44
|
+
# Returns that row as a BatchEntry, or nil when the job had no execution
|
|
45
|
+
# row (already resolved, or a pre-migration in-flight job) or its batch
|
|
46
|
+
# is gone. +column+ is interpolated, so only COUNTER_COLUMNS pass.
|
|
47
|
+
def self.count_and_resolve!(job_id, column)
|
|
48
|
+
raise ArgumentError, "Invalid column: #{column}" unless COUNTER_COLUMNS.include?(column)
|
|
49
|
+
|
|
50
|
+
sql = <<~SQL.squish
|
|
51
|
+
WITH gone AS (
|
|
52
|
+
DELETE FROM #{BatchExecution.table_name} WHERE job_id = $1 RETURNING batch_id
|
|
53
|
+
)
|
|
54
|
+
UPDATE #{table_name} b
|
|
55
|
+
SET #{column} = b.#{column} + 1
|
|
56
|
+
FROM gone
|
|
57
|
+
WHERE b.batch_id = gone.batch_id
|
|
58
|
+
RETURNING b.*
|
|
59
|
+
SQL
|
|
60
|
+
row = connection_pool.with_connection do |conn|
|
|
61
|
+
conn.exec_query(sql, "Pgbus Batch Count", [job_id]).first
|
|
62
|
+
end
|
|
63
|
+
row && instantiate(row)
|
|
64
|
+
end
|
|
65
|
+
|
|
41
66
|
# Single-winner finish: status is processing AND no execution rows remain.
|
|
42
|
-
# Join-free NOT EXISTS so the subquery stays in this UPDATE's WHERE
|
|
43
|
-
#
|
|
67
|
+
# Join-free NOT EXISTS so the subquery stays in this UPDATE's WHERE (it
|
|
68
|
+
# was a NOT IN before issue #650; the same rows, batch_id is NOT NULL on
|
|
69
|
+
# pgbus_batch_executions). Returns the finished row as a BatchEntry
|
|
70
|
+
# (RETURNING *), or nil when this call did not finish the batch.
|
|
44
71
|
def self.finish_if_empty!(batch_id)
|
|
45
72
|
# Counters must already be terminal so a pre-migration in-flight batch
|
|
46
73
|
# (zero execution rows, total_jobs = N, counters short of N) is not
|
|
47
74
|
# closed empty. total_jobs = 0 with zero counters IS terminal: that is a
|
|
48
75
|
# batch whose enqueue block crashed before it enqueued anything, and
|
|
49
76
|
# nothing else will ever close it.
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
77
|
+
sql = <<~SQL.squish
|
|
78
|
+
UPDATE #{table_name} SET status = 'finished', finished_at = $2
|
|
79
|
+
WHERE batch_id = $1 AND status = 'processing'
|
|
80
|
+
AND completed_jobs + failed_jobs = total_jobs
|
|
81
|
+
AND NOT EXISTS (
|
|
82
|
+
SELECT 1 FROM #{BatchExecution.table_name} e WHERE e.batch_id = #{table_name}.batch_id
|
|
83
|
+
)
|
|
84
|
+
RETURNING *
|
|
85
|
+
SQL
|
|
86
|
+
row = connection_pool.with_connection do |conn|
|
|
87
|
+
conn.exec_query(sql, "Pgbus Batch Finish", [batch_id, Time.current]).first
|
|
88
|
+
end
|
|
89
|
+
row && instantiate(row)
|
|
54
90
|
end
|
|
55
91
|
|
|
56
92
|
# Finish the batch if every job already reached a terminal state. Used
|
|
@@ -60,16 +96,19 @@ module Pgbus
|
|
|
60
96
|
# idempotent against concurrent completion signals.
|
|
61
97
|
# Returns { just_finished:, record: } or nil if batch not found.
|
|
62
98
|
def self.check_finished!(batch_id)
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
99
|
+
connection_pool.with_connection do
|
|
100
|
+
# The schema probe inside the lease: its first call reads the table list.
|
|
101
|
+
return Batch.try_finish!(batch_id) if Batch.executions_migrated?
|
|
102
|
+
|
|
103
|
+
transaction do
|
|
104
|
+
record = lock.find_by(batch_id: batch_id)
|
|
105
|
+
return nil unless record
|
|
106
|
+
return { record: record, just_finished: false } if record.status == "finished"
|
|
107
|
+
return { record: record, just_finished: false } unless record.completed_jobs + record.discarded_jobs == record.total_jobs
|
|
108
|
+
|
|
109
|
+
record.update!(status: "finished", finished_at: Time.current)
|
|
110
|
+
{ record: record, just_finished: true }
|
|
111
|
+
end
|
|
73
112
|
end
|
|
74
113
|
end
|
|
75
114
|
|
|
@@ -80,23 +119,25 @@ module Pgbus
|
|
|
80
119
|
def self.increment_counter!(batch_id, column)
|
|
81
120
|
raise ArgumentError, "Invalid column: #{column}" unless COUNTER_COLUMNS.include?(column)
|
|
82
121
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
122
|
+
connection_pool.with_connection do
|
|
123
|
+
transaction do
|
|
124
|
+
record = lock.find_by(batch_id: batch_id)
|
|
125
|
+
return nil unless record
|
|
86
126
|
|
|
87
|
-
|
|
127
|
+
record.increment!(column)
|
|
88
128
|
|
|
89
|
-
|
|
129
|
+
return { record: record, just_finished: false } if Batch.executions_migrated?
|
|
90
130
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
131
|
+
# total_jobs grows per job while the block is still open (issue #423),
|
|
132
|
+
# so completed == total can be momentarily true on a pending batch.
|
|
133
|
+
# Only a processing batch may auto-finish; check_finished! at the end
|
|
134
|
+
# of the block covers the pending case.
|
|
135
|
+
counters_match = record.completed_jobs + record.discarded_jobs == record.total_jobs
|
|
136
|
+
just_finished = counters_match && record.status == "processing"
|
|
137
|
+
record.update!(status: "finished", finished_at: Time.current) if just_finished
|
|
98
138
|
|
|
99
|
-
|
|
139
|
+
{ record: record, just_finished: just_finished }
|
|
140
|
+
end
|
|
100
141
|
end
|
|
101
142
|
end
|
|
102
143
|
end
|
|
@@ -13,12 +13,14 @@ module Pgbus
|
|
|
13
13
|
# live message before the msg_id backfill lands); backfill! overwrites it
|
|
14
14
|
# with the physical target after send.
|
|
15
15
|
def self.insert_for!(batch_id:, job_id:, queue_name: nil)
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
16
|
+
connection_pool.with_connection do |conn|
|
|
17
|
+
conn.exec_query(
|
|
18
|
+
"INSERT INTO #{table_name} (batch_id, job_id, queue_name, created_at) " \
|
|
19
|
+
"VALUES ($1, $2, $3, $4) ON CONFLICT (job_id) DO NOTHING",
|
|
20
|
+
"BatchExecution Insert",
|
|
21
|
+
[batch_id, job_id, queue_name, Time.current]
|
|
22
|
+
)
|
|
23
|
+
end
|
|
22
24
|
end
|
|
23
25
|
|
|
24
26
|
def self.backfill!(job_id, msg_id:, queue_name:)
|
|
@@ -10,21 +10,23 @@ module Pgbus
|
|
|
10
10
|
# A parked job never ages out — `expires_at` orders the sweep, nothing more.
|
|
11
11
|
# Returns { queue_name:, payload:, priority: } or nil.
|
|
12
12
|
def self.release_next!(concurrency_key)
|
|
13
|
-
result =
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
13
|
+
result = connection_pool.with_connection do |conn|
|
|
14
|
+
conn.exec_query(
|
|
15
|
+
<<~SQL,
|
|
16
|
+
DELETE FROM pgbus_blocked_executions
|
|
17
|
+
WHERE id = (
|
|
18
|
+
SELECT id FROM pgbus_blocked_executions
|
|
19
|
+
WHERE concurrency_key = $1
|
|
20
|
+
ORDER BY priority ASC, created_at ASC
|
|
21
|
+
LIMIT 1
|
|
22
|
+
FOR UPDATE SKIP LOCKED
|
|
23
|
+
)
|
|
24
|
+
RETURNING queue_name, payload, priority
|
|
25
|
+
SQL
|
|
26
|
+
"Pgbus Blocked Release",
|
|
27
|
+
[concurrency_key]
|
|
28
|
+
)
|
|
29
|
+
end
|
|
28
30
|
|
|
29
31
|
row = result.first
|
|
30
32
|
return nil unless row
|
|
@@ -10,19 +10,22 @@ module Pgbus
|
|
|
10
10
|
scope :dead_lettered, -> { where(status: "dead_lettered") }
|
|
11
11
|
|
|
12
12
|
# Record a job execution stat. Called by the executor after each job.
|
|
13
|
+
# One lease for the probe and the insert, returned at the end (issue #649).
|
|
13
14
|
def self.record!(job_class:, queue_name:, status:, duration_ms:, enqueue_latency_ms: nil, retry_count: 0)
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
15
|
+
connection_pool.with_connection do
|
|
16
|
+
next unless table_exists?
|
|
17
|
+
|
|
18
|
+
attrs = {
|
|
19
|
+
job_class: job_class,
|
|
20
|
+
queue_name: queue_name,
|
|
21
|
+
status: status,
|
|
22
|
+
duration_ms: duration_ms
|
|
23
|
+
}
|
|
24
|
+
attrs[:enqueue_latency_ms] = enqueue_latency_ms if latency_columns?
|
|
25
|
+
attrs[:retry_count] = retry_count if latency_columns?
|
|
26
|
+
|
|
27
|
+
create!(attrs)
|
|
28
|
+
end
|
|
26
29
|
rescue StandardError => e
|
|
27
30
|
Pgbus.logger.debug { "[Pgbus] Failed to record job stat: #{e.message}" }
|
|
28
31
|
end
|
|
@@ -43,7 +46,7 @@ module Pgbus
|
|
|
43
46
|
def self.table_exists?
|
|
44
47
|
return @table_exists if defined?(@table_exists)
|
|
45
48
|
|
|
46
|
-
@table_exists =
|
|
49
|
+
@table_exists = connection_pool.with_connection { |conn| conn.table_exists?(table_name) }
|
|
47
50
|
rescue StandardError => e
|
|
48
51
|
Pgbus.logger.debug { "[Pgbus] Failed to check job stat table: #{e.message}" }
|
|
49
52
|
false
|
|
@@ -13,20 +13,22 @@ module Pgbus
|
|
|
13
13
|
# promotion is judged against the real limit rather than a guessed 1.
|
|
14
14
|
# On a fresh row there is no recorded limit, so nil falls back to 1.
|
|
15
15
|
def self.acquire!(key, max_value, expires_at)
|
|
16
|
-
result =
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
16
|
+
result = connection_pool.with_connection do |conn|
|
|
17
|
+
conn.exec_query(
|
|
18
|
+
<<~SQL,
|
|
19
|
+
INSERT INTO pgbus_semaphores (key, value, max_value, expires_at)
|
|
20
|
+
VALUES ($1, 1, COALESCE($2, 1), $3)
|
|
21
|
+
ON CONFLICT (key) DO UPDATE
|
|
22
|
+
SET value = pgbus_semaphores.value + 1,
|
|
23
|
+
max_value = COALESCE($2, pgbus_semaphores.max_value),
|
|
24
|
+
expires_at = GREATEST(pgbus_semaphores.expires_at, EXCLUDED.expires_at)
|
|
25
|
+
WHERE pgbus_semaphores.value < COALESCE($2, pgbus_semaphores.max_value)
|
|
26
|
+
RETURNING value
|
|
27
|
+
SQL
|
|
28
|
+
"Pgbus Semaphore Acquire",
|
|
29
|
+
[key, max_value, expires_at]
|
|
30
|
+
)
|
|
31
|
+
end
|
|
30
32
|
|
|
31
33
|
result.any? ? :acquired : :blocked
|
|
32
34
|
end
|
|
@@ -21,11 +21,13 @@ module Pgbus
|
|
|
21
21
|
else
|
|
22
22
|
"DO NOTHING"
|
|
23
23
|
end
|
|
24
|
-
result =
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
24
|
+
result = connection_pool.with_connection do |conn|
|
|
25
|
+
conn.exec_query(
|
|
26
|
+
"INSERT INTO #{table_name} (lock_key, queue_name, msg_id) " \
|
|
27
|
+
"VALUES ($1, $2, $3) ON CONFLICT (lock_key) #{on_conflict} RETURNING lock_key, created_at",
|
|
28
|
+
"UniquenessKey Acquire", [lock_key, queue_name, msg_id]
|
|
29
|
+
)
|
|
30
|
+
end
|
|
29
31
|
row = result.rows.first
|
|
30
32
|
return false unless row
|
|
31
33
|
|
|
@@ -52,7 +54,7 @@ module Pgbus
|
|
|
52
54
|
sql += " AND created_at = $4"
|
|
53
55
|
binds << created_at
|
|
54
56
|
end
|
|
55
|
-
|
|
57
|
+
connection_pool.with_connection { |conn| conn.exec_update(sql, "UniquenessKey Bind", binds) }
|
|
56
58
|
end
|
|
57
59
|
|
|
58
60
|
# Drop the bind ownership stamp without touching the lock row. Used when
|
|
@@ -65,10 +67,9 @@ module Pgbus
|
|
|
65
67
|
# Release a uniqueness lock after job completion or DLQ.
|
|
66
68
|
def self.release!(lock_key)
|
|
67
69
|
Thread.current[:pgbus_uniqueness_created_at]&.delete(lock_key)
|
|
68
|
-
|
|
69
|
-
"DELETE FROM #{table_name} WHERE lock_key = $1",
|
|
70
|
-
|
|
71
|
-
)
|
|
70
|
+
connection_pool.with_connection do |conn|
|
|
71
|
+
conn.exec_delete("DELETE FROM #{table_name} WHERE lock_key = $1", "UniquenessKey Release", [lock_key])
|
|
72
|
+
end
|
|
72
73
|
end
|
|
73
74
|
|
|
74
75
|
# Release a lock only while it still points at this message. Used by the
|
|
@@ -81,10 +82,12 @@ module Pgbus
|
|
|
81
82
|
# queue. The queue is part of the match.
|
|
82
83
|
def self.release_if_bound!(lock_key, queue_name:, msg_id:)
|
|
83
84
|
Thread.current[:pgbus_uniqueness_created_at]&.delete(lock_key)
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
85
|
+
connection_pool.with_connection do |conn|
|
|
86
|
+
conn.exec_delete(
|
|
87
|
+
"DELETE FROM #{table_name} WHERE lock_key = $1 AND queue_name = $2 AND msg_id = $3",
|
|
88
|
+
"UniquenessKey Release If Bound", [lock_key, queue_name.to_s, msg_id.to_i]
|
|
89
|
+
)
|
|
90
|
+
end
|
|
88
91
|
end
|
|
89
92
|
|
|
90
93
|
# The row a message holds: bound to that message (queue + msg_id, see
|
|
@@ -109,10 +112,10 @@ module Pgbus
|
|
|
109
112
|
# be a retry's it handed over), and the reaper takes an orphan.
|
|
110
113
|
def self.release_held_by!(lock_key, queue_name:, msg_id:, enqueued_at:)
|
|
111
114
|
Thread.current[:pgbus_uniqueness_created_at]&.delete(lock_key)
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
"UniquenessKey Release Held By",
|
|
115
|
-
|
|
115
|
+
binds = held_by_binds(lock_key, queue_name, msg_id, sent_at_literal(enqueued_at))
|
|
116
|
+
connection_pool.with_connection do |conn|
|
|
117
|
+
conn.exec_delete("DELETE FROM #{table_name} WHERE #{HELD_BY_CONDITION}", "UniquenessKey Release Held By", binds)
|
|
118
|
+
end
|
|
116
119
|
end
|
|
117
120
|
|
|
118
121
|
# Hand the running attempt's lock to its retry, which goes out through the
|
|
@@ -126,11 +129,13 @@ module Pgbus
|
|
|
126
129
|
# attempt's enqueued_at only a row bound to it is taken.
|
|
127
130
|
def self.unbind!(lock_key, from_queue_name:, from_msg_id:, from_enqueued_at:, queue_name:)
|
|
128
131
|
binds = held_by_binds(lock_key, from_queue_name, from_msg_id, sent_at_literal(from_enqueued_at))
|
|
129
|
-
result =
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
132
|
+
result = connection_pool.with_connection do |conn|
|
|
133
|
+
conn.exec_query(
|
|
134
|
+
"UPDATE #{table_name} SET queue_name = $5, msg_id = 0, created_at = CURRENT_TIMESTAMP " \
|
|
135
|
+
"WHERE #{HELD_BY_CONDITION} RETURNING created_at",
|
|
136
|
+
"UniquenessKey Unbind", binds + [queue_name.to_s]
|
|
137
|
+
)
|
|
138
|
+
end
|
|
134
139
|
result.rows.first&.first
|
|
135
140
|
end
|
|
136
141
|
|
|
@@ -138,17 +143,21 @@ module Pgbus
|
|
|
138
143
|
# discards and queue purges — as [lock_key, queue_name, msg_id,
|
|
139
144
|
# enqueued_at] tuples, one statement per thousand.
|
|
140
145
|
def self.release_all_held_by!(holders)
|
|
141
|
-
holders.
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
146
|
+
return 0 if holders.empty?
|
|
147
|
+
|
|
148
|
+
connection_pool.with_connection do |conn|
|
|
149
|
+
holders.each_slice(1000).sum do |slice|
|
|
150
|
+
values = slice.map do |lock_key, queue_name, msg_id, enqueued_at|
|
|
151
|
+
"(#{conn.quote(lock_key.to_s)}, #{conn.quote(queue_name.to_s)}, #{msg_id.to_i}, " \
|
|
152
|
+
"#{conn.quote(sent_at_literal(enqueued_at))}::timestamptz)"
|
|
153
|
+
end
|
|
154
|
+
conn.exec_delete(
|
|
155
|
+
"DELETE FROM #{table_name} AS u USING (VALUES #{values.join(", ")}) AS m(lock_key, queue_name, msg_id, sent_at) " \
|
|
156
|
+
"WHERE u.lock_key = m.lock_key AND ((u.queue_name = m.queue_name AND u.msg_id = m.msg_id AND u.msg_id > 0) OR " \
|
|
157
|
+
"(u.msg_id <= 0 AND u.created_at <= m.sent_at))",
|
|
158
|
+
"UniquenessKey Release All Held By"
|
|
159
|
+
)
|
|
145
160
|
end
|
|
146
|
-
connection.exec_delete(
|
|
147
|
-
"DELETE FROM #{table_name} AS u USING (VALUES #{values.join(", ")}) AS m(lock_key, queue_name, msg_id, sent_at) " \
|
|
148
|
-
"WHERE u.lock_key = m.lock_key AND ((u.queue_name = m.queue_name AND u.msg_id = m.msg_id AND u.msg_id > 0) OR " \
|
|
149
|
-
"(u.msg_id <= 0 AND u.created_at <= m.sent_at))",
|
|
150
|
-
"UniquenessKey Release All Held By"
|
|
151
|
-
)
|
|
152
161
|
end
|
|
153
162
|
end
|
|
154
163
|
|
|
@@ -185,10 +194,12 @@ module Pgbus
|
|
|
185
194
|
# belong to it — pass the parked row's created_at as the ceiling.
|
|
186
195
|
def self.release_if_unbound!(lock_key, acquired_before:)
|
|
187
196
|
Thread.current[:pgbus_uniqueness_created_at]&.delete(lock_key)
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
197
|
+
connection_pool.with_connection do |conn|
|
|
198
|
+
conn.exec_delete(
|
|
199
|
+
"DELETE FROM #{table_name} WHERE lock_key = $1 AND msg_id = 0 AND created_at <= $2",
|
|
200
|
+
"UniquenessKey Release If Unbound", [lock_key, acquired_before]
|
|
201
|
+
)
|
|
202
|
+
end
|
|
192
203
|
end
|
|
193
204
|
|
|
194
205
|
# True when the key's row is bound to this message (queue + msg_id): its
|
|
@@ -196,18 +207,20 @@ module Pgbus
|
|
|
196
207
|
# this when it dead-letters a :while_executing message that never failed
|
|
197
208
|
# (issue #629): a crash loop, not a uniqueness conflict.
|
|
198
209
|
def self.bound_to?(lock_key, queue_name:, msg_id:)
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
210
|
+
row = connection_pool.with_connection do |conn|
|
|
211
|
+
conn.select_value(
|
|
212
|
+
"SELECT 1 FROM #{table_name} WHERE lock_key = $1 AND queue_name = $2 AND msg_id = $3 LIMIT 1",
|
|
213
|
+
"UniquenessKey Bound To", [lock_key, queue_name.to_s, msg_id.to_i]
|
|
214
|
+
)
|
|
215
|
+
end
|
|
216
|
+
!row.nil?
|
|
203
217
|
end
|
|
204
218
|
|
|
205
219
|
# Check if a key is currently locked.
|
|
206
220
|
def self.locked?(lock_key)
|
|
207
|
-
result =
|
|
208
|
-
"SELECT 1 FROM #{table_name} WHERE lock_key = $1 LIMIT 1",
|
|
209
|
-
|
|
210
|
-
)
|
|
221
|
+
result = connection_pool.with_connection do |conn|
|
|
222
|
+
conn.select_value("SELECT 1 FROM #{table_name} WHERE lock_key = $1 LIMIT 1", "UniquenessKey Check", [lock_key])
|
|
223
|
+
end
|
|
211
224
|
!result.nil?
|
|
212
225
|
end
|
|
213
226
|
end
|
|
@@ -88,11 +88,13 @@ module Pgbus
|
|
|
88
88
|
# signalling right now waits and then sees the parked row instead
|
|
89
89
|
# of stranding it (rails/solid_queue#712).
|
|
90
90
|
acquired = false
|
|
91
|
-
Pgbus::Semaphore.
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
91
|
+
Pgbus::Semaphore.connection_pool.with_connection do
|
|
92
|
+
Pgbus::Semaphore.transaction(requires_new: true) do
|
|
93
|
+
acquired = Concurrency::Semaphore.acquire(key, concurrency[:limit], slot_lease(concurrency, delay)) ==
|
|
94
|
+
:acquired
|
|
95
|
+
blocked = handle_conflict(concurrency, active_job, key, queue, payload_hash, priority: priority) unless
|
|
96
|
+
acquired
|
|
97
|
+
end
|
|
96
98
|
end
|
|
97
99
|
|
|
98
100
|
if acquired
|
data/lib/pgbus/batch.rb
CHANGED
|
@@ -198,20 +198,26 @@ module Pgbus
|
|
|
198
198
|
end
|
|
199
199
|
|
|
200
200
|
# Record a completed job. Returns the batch row after update.
|
|
201
|
+
# One lease for the whole signal, returned at its end (issue #649): the
|
|
202
|
+
# executor calls this after the job's own lease went back.
|
|
201
203
|
def self.job_completed(batch_id, job_id: nil)
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
204
|
+
BatchEntry.connection_pool.with_connection do
|
|
205
|
+
if executions_migrated?
|
|
206
|
+
job_id ? resolve_execution(batch_id, job_id, "completed_jobs") : signal_without_row(batch_id, "completed_jobs")
|
|
207
|
+
else
|
|
208
|
+
update_counter(batch_id, "completed_jobs")
|
|
209
|
+
end
|
|
206
210
|
end
|
|
207
211
|
end
|
|
208
212
|
|
|
209
213
|
# Record a discarded/dead-lettered job. Returns the batch row after update.
|
|
210
214
|
def self.job_discarded(batch_id, job_id: nil)
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
+
BatchEntry.connection_pool.with_connection do
|
|
216
|
+
if executions_migrated?
|
|
217
|
+
job_id ? resolve_execution(batch_id, job_id, "failed_jobs") : signal_without_row(batch_id, "failed_jobs")
|
|
218
|
+
else
|
|
219
|
+
update_counter(batch_id, "discarded_jobs")
|
|
220
|
+
end
|
|
215
221
|
end
|
|
216
222
|
end
|
|
217
223
|
|
|
@@ -353,16 +359,20 @@ module Pgbus
|
|
|
353
359
|
batch_id = payloads.first&.fetch(METADATA_KEY, nil)
|
|
354
360
|
return if payloads.empty? || batch_id.nil?
|
|
355
361
|
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
362
|
+
BatchEntry.connection_pool.with_connection do
|
|
363
|
+
# The schema probe inside the lease: its first call reads the table
|
|
364
|
+
# list through the connection.
|
|
365
|
+
migrated = executions_migrated?
|
|
366
|
+
BatchEntry.transaction do
|
|
367
|
+
BatchEntry.increment_total_jobs!(batch_id, payloads.size)
|
|
368
|
+
next unless migrated
|
|
360
369
|
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
370
|
+
payloads.each do |payload|
|
|
371
|
+
job_id = payload["job_id"]
|
|
372
|
+
next unless job_id
|
|
364
373
|
|
|
365
|
-
|
|
374
|
+
BatchExecution.insert_for!(batch_id: batch_id, job_id: job_id, queue_name: payload["queue_name"])
|
|
375
|
+
end
|
|
366
376
|
end
|
|
367
377
|
end
|
|
368
378
|
end
|
|
@@ -417,10 +427,12 @@ module Pgbus
|
|
|
417
427
|
return unless batch_id
|
|
418
428
|
|
|
419
429
|
job_id = payload["job_id"]
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
BatchEntry.
|
|
423
|
-
|
|
430
|
+
BatchEntry.connection_pool.with_connection do
|
|
431
|
+
migrated = executions_migrated?
|
|
432
|
+
BatchEntry.transaction do
|
|
433
|
+
BatchEntry.decrement_total_jobs!(batch_id)
|
|
434
|
+
BatchExecution.where(job_id: job_id).delete_all if migrated && job_id
|
|
435
|
+
end
|
|
424
436
|
end
|
|
425
437
|
end
|
|
426
438
|
|
|
@@ -429,30 +441,40 @@ module Pgbus
|
|
|
429
441
|
# Only batch members have an execution row: without this, every plain
|
|
430
442
|
# enqueue paid a no-op UPDATE round trip after the send (issue #603).
|
|
431
443
|
return unless payload[METADATA_KEY]
|
|
432
|
-
return unless executions_migrated?
|
|
433
444
|
|
|
434
445
|
job_id = payload["job_id"]
|
|
435
446
|
return unless job_id
|
|
436
447
|
|
|
437
|
-
|
|
448
|
+
# The adapter calls this after its send, outside any lease of its own,
|
|
449
|
+
# so the schema probe stays fail-soft as it always was: a cold process
|
|
450
|
+
# that cannot get a connection skips the backfill rather than failing
|
|
451
|
+
# an enqueue whose message is already live.
|
|
452
|
+
return unless execution_rows_migrated_softly?
|
|
453
|
+
|
|
454
|
+
BatchExecution.connection_pool.with_connection do
|
|
455
|
+
BatchExecution.backfill!(job_id, msg_id: msg_id, queue_name: queue_name)
|
|
456
|
+
end
|
|
438
457
|
end
|
|
439
458
|
|
|
440
459
|
# Single-winner finish via execution-row absence. After a winning UPDATE,
|
|
441
460
|
# re-check exists? in a fresh statement (Postgres READ COMMITTED can let a
|
|
442
461
|
# blocked CAS win from a stale NOT EXISTS snapshot — solid_queue's finalize).
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
462
|
+
# The winner's record comes from the UPDATE's RETURNING; a loser hands back
|
|
463
|
+
# +record+ (the row a completion signal already holds, issue #650) and
|
|
464
|
+
# reads the row only when it has none (the sweep, check_finished!).
|
|
465
|
+
def self.try_finish!(batch_id, record: nil)
|
|
466
|
+
BatchEntry.connection_pool.with_connection do
|
|
467
|
+
result = BatchEntry.transaction do
|
|
468
|
+
finished = BatchEntry.finish_if_empty!(batch_id)
|
|
469
|
+
next { just_finished: false, record: record || BatchEntry.find_by(batch_id: batch_id) } unless finished
|
|
470
|
+
|
|
471
|
+
raise ActiveRecord::Rollback if BatchExecution.where(batch_id: batch_id).exists?
|
|
472
|
+
|
|
473
|
+
{ just_finished: true, record: finished }
|
|
474
|
+
end
|
|
449
475
|
|
|
450
|
-
{ just_finished:
|
|
476
|
+
result || { just_finished: false, record: record || BatchEntry.find_by(batch_id: batch_id) }
|
|
451
477
|
end
|
|
452
|
-
|
|
453
|
-
return { just_finished: false, record: BatchEntry.find_by(batch_id: batch_id) } if result.nil?
|
|
454
|
-
|
|
455
|
-
result
|
|
456
478
|
end
|
|
457
479
|
|
|
458
480
|
def self.sweep_stalled(stalled_for: Pgbus.configuration.batch_stall_threshold, batch_size: 500, client: Pgbus.client)
|
|
@@ -462,11 +484,16 @@ module Pgbus
|
|
|
462
484
|
class << self
|
|
463
485
|
private
|
|
464
486
|
|
|
487
|
+
# Two statements and one short finish transaction per member (issue
|
|
488
|
+
# #650): count_and_resolve! deletes the execution row and counts it
|
|
489
|
+
# into the batch, RETURNING the row the finish and the callbacks use.
|
|
490
|
+
# No row back means the job had no execution row: an already-resolved
|
|
491
|
+
# signal, or a pre-migration in-flight batch counted the old way.
|
|
465
492
|
def resolve_execution(batch_id, job_id, column)
|
|
466
|
-
BatchEntry.
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
493
|
+
record = BatchEntry.count_and_resolve!(job_id, column)
|
|
494
|
+
return finish_if_needed(try_finish!(record.batch_id, record: record)) if record
|
|
495
|
+
|
|
496
|
+
BatchEntry.increment_counter!(batch_id, column) if legacy_untracked_batch?(batch_id)
|
|
470
497
|
finish_if_needed(try_finish!(batch_id))
|
|
471
498
|
end
|
|
472
499
|
|
|
@@ -480,6 +507,18 @@ module Pgbus
|
|
|
480
507
|
record && !counters_match_total?(record)
|
|
481
508
|
end
|
|
482
509
|
|
|
510
|
+
# executions_migrated? with its first probe inside a lease it returns;
|
|
511
|
+
# false when no connection can be had (it already is false when the
|
|
512
|
+
# probe itself raises).
|
|
513
|
+
def execution_rows_migrated_softly?
|
|
514
|
+
# Memoized true: no lease for the probe, the backfill takes the one.
|
|
515
|
+
return true if @schema_lock.synchronize { @executions_migrated }
|
|
516
|
+
|
|
517
|
+
BatchExecution.connection_pool.with_connection { executions_migrated? }
|
|
518
|
+
rescue StandardError
|
|
519
|
+
false
|
|
520
|
+
end
|
|
521
|
+
|
|
483
522
|
def counters_match_total?(record)
|
|
484
523
|
failures = record.respond_to?(:discarded_jobs) ? record.discarded_jobs.to_i : record.failed_jobs.to_i
|
|
485
524
|
record.total_jobs.positive? && (record.completed_jobs + failures) == record.total_jobs
|