pgbus 0.14.1 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +35 -0
- data/README.md +33 -0
- data/Rakefile +6 -1
- data/app/helpers/pgbus/application_helper.rb +7 -0
- data/app/models/pgbus/batch_entry.rb +78 -6
- data/app/models/pgbus/batch_execution.rb +28 -0
- data/app/models/pgbus/blocked_execution.rb +2 -2
- data/app/models/pgbus/uniqueness_key.rb +50 -4
- data/app/views/pgbus/batches/_batches_table.html.erb +3 -3
- data/app/views/pgbus/batches/show.html.erb +7 -7
- data/app/views/pgbus/dead_letter/show.html.erb +17 -0
- data/app/views/pgbus/events/_pending_table.html.erb +21 -0
- data/app/views/pgbus/jobs/show.html.erb +17 -0
- data/config/locales/da.yml +5 -2
- data/config/locales/de.yml +5 -2
- data/config/locales/en.yml +5 -2
- data/config/locales/es.yml +5 -2
- data/config/locales/fi.yml +5 -2
- data/config/locales/fr.yml +5 -2
- data/config/locales/it.yml +5 -2
- data/config/locales/ja.yml +5 -2
- data/config/locales/nb.yml +5 -2
- data/config/locales/nl.yml +5 -2
- data/config/locales/pt.yml +5 -2
- data/config/locales/sv.yml +5 -2
- data/lib/generators/pgbus/add_batch_callback_jobs_generator.rb +47 -0
- data/lib/generators/pgbus/add_batch_executions_generator.rb +44 -0
- data/lib/generators/pgbus/templates/add_batch_callback_jobs.rb.erb +13 -0
- data/lib/generators/pgbus/templates/add_batch_executions.rb.erb +50 -0
- data/lib/generators/pgbus/templates/initializer.rb.erb +2 -0
- data/lib/generators/pgbus/templates/migration.rb.erb +23 -2
- data/lib/pgbus/active_job/adapter.rb +146 -28
- data/lib/pgbus/active_job/batch_id.rb +48 -0
- data/lib/pgbus/active_job/current_attributes.rb +52 -0
- data/lib/pgbus/active_job/executor.rb +45 -9
- data/lib/pgbus/batch/sweep.rb +163 -0
- data/lib/pgbus/batch.rb +448 -63
- data/lib/pgbus/client/fair_read.rb +187 -0
- data/lib/pgbus/client.rb +221 -18
- data/lib/pgbus/concurrency/blocked_execution.rb +14 -1
- data/lib/pgbus/configuration.rb +81 -1
- data/lib/pgbus/current_attributes.rb +156 -0
- data/lib/pgbus/engine.rb +2 -0
- data/lib/pgbus/event.rb +7 -2
- data/lib/pgbus/event_bus/handler.rb +12 -2
- data/lib/pgbus/event_bus/publisher.rb +37 -2
- data/lib/pgbus/event_bus/subscriber.rb +4 -0
- data/lib/pgbus/fair_share.rb +110 -0
- data/lib/pgbus/generators/migration_detector.rb +30 -0
- data/lib/pgbus/instrumentation.rb +5 -0
- data/lib/pgbus/outbox/poller.rb +6 -7
- data/lib/pgbus/outbox.rb +8 -1
- data/lib/pgbus/process/consumer.rb +38 -1
- data/lib/pgbus/process/dispatcher.rb +41 -17
- data/lib/pgbus/process/worker.rb +41 -0
- data/lib/pgbus/recurring/schedule.rb +12 -1
- data/lib/pgbus/testing.rb +2 -1
- data/lib/pgbus/uniqueness.rb +35 -7
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +47 -9
- data/lib/pgbus/web/job_context.rb +80 -0
- data/lib/pgbus.rb +2 -0
- metadata +13 -1
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class Client
|
|
5
|
+
# Fair share reads (issue #426): a weighted, work-conserving interleave
|
|
6
|
+
# across fair-share keys within one queue. Replaces `read_batch` on the
|
|
7
|
+
# worker when `config.fair_share` is set.
|
|
8
|
+
#
|
|
9
|
+
# Scheduling rule per read of `qty`: for every key that has visible
|
|
10
|
+
# messages, rank that key's visible messages 1..qty oldest-visible first
|
|
11
|
+
# (`vt, msg_id`); a message's virtual time is `rank / weight`; take the
|
|
12
|
+
# `qty` messages with the lowest `(virtual_time, msg_id)`. Weight 3 vs 1
|
|
13
|
+
# yields a 3:1 split under contention; a lone key fills the whole batch.
|
|
14
|
+
# This is batch-level weighted fair queuing — proportional within each
|
|
15
|
+
# batch, memoryless across batches (no deficit carry-over).
|
|
16
|
+
#
|
|
17
|
+
# Cost model: key enumeration is a loose index scan over
|
|
18
|
+
# (key, vt, msg_id) — one probe per key with visible work; keys whose
|
|
19
|
+
# messages are all invisible (in flight / delayed / in retry backoff) are
|
|
20
|
+
# skipped at the index level because `vt` is in the index. Per-key
|
|
21
|
+
# candidates are a bounded `LIMIT qty` index range. Roughly
|
|
22
|
+
# O(K · log n + K · qty), K = keys with visible work, independent of
|
|
23
|
+
# backlog depth. Within a key the order is `(vt, msg_id)` — FIFO for
|
|
24
|
+
# immediate enqueues; retried/delayed messages sort by when they became
|
|
25
|
+
# visible — a deliberate deviation from pgmq.read's pure msg_id order so
|
|
26
|
+
# the per-key lookup never sorts a tenant's whole visible backlog.
|
|
27
|
+
#
|
|
28
|
+
# Candidates are selected before locking (same shape as
|
|
29
|
+
# pgmq.read_grouped_rr), so under concurrent readers a batch can come back
|
|
30
|
+
# short; the worker loop re-reads immediately while it has capacity.
|
|
31
|
+
module FairRead
|
|
32
|
+
# The read expressions are written against the alias `m`; the index
|
|
33
|
+
# expression is the same shape minus the alias so the planner matches it.
|
|
34
|
+
FAIR_INDEX_KEY_EXPR = "COALESCE(message->>'#{FairShare::METADATA_KEY}', '')".freeze
|
|
35
|
+
FAIR_KEY_EXPR = "COALESCE(m.message->>'#{FairShare::METADATA_KEY}', '')".freeze
|
|
36
|
+
FAIR_WEIGHT_EXPR = "COALESCE((m.message->>'#{FairShare::WEIGHT_KEY}')::numeric, 1)".freeze
|
|
37
|
+
FAIR_INDEX_SUFFIX = "_fair_idx"
|
|
38
|
+
|
|
39
|
+
def read_batch_fair(queue_name, qty:, vt: nil)
|
|
40
|
+
full_name = fair_queue_name(queue_name)
|
|
41
|
+
guarded_read { fair_read_step(full_name, qty, vt || config.visibility_timeout) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Idempotent, memoized per process. Uses CREATE INDEX CONCURRENTLY so
|
|
45
|
+
# enabling fair share on an existing, populated queue never blocks
|
|
46
|
+
# enqueues. A queue table that does not exist yet is left alone (its
|
|
47
|
+
# creation path builds the index non-concurrently); any other failure is
|
|
48
|
+
# logged with the remediation and NOT memoized so the next ensure retries.
|
|
49
|
+
def ensure_fair_index(queue_name)
|
|
50
|
+
full_name = fair_queue_name(queue_name)
|
|
51
|
+
return if fair_indexes_ensured[full_name]
|
|
52
|
+
|
|
53
|
+
with_stale_connection_retry do
|
|
54
|
+
synchronized { exec_ddl(fair_index_sql(full_name, concurrently: true)) }
|
|
55
|
+
end
|
|
56
|
+
fair_indexes_ensured[full_name] = true
|
|
57
|
+
rescue StandardError => e
|
|
58
|
+
if duplicate_relation_error?(e)
|
|
59
|
+
fair_indexes_ensured[full_name] = true
|
|
60
|
+
elsif undefined_table_error?(e)
|
|
61
|
+
Pgbus.logger.debug { "[Pgbus] Fair index deferred — queue table #{full_name} not created yet" }
|
|
62
|
+
else
|
|
63
|
+
Pgbus.logger.error do
|
|
64
|
+
"[Pgbus] Could not create fair index #{fair_index_name(full_name)} on pgmq.q_#{full_name}: " \
|
|
65
|
+
"#{e.class}: #{e.message}. A failed CONCURRENTLY build leaves an INVALID index behind — " \
|
|
66
|
+
"run `DROP INDEX IF EXISTS pgmq.#{fair_index_name(full_name)}` and restart the worker to retry."
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
# One instrumented, retried, serialized, timeout-bounded fair read of a
|
|
74
|
+
# physical (already prefixed + sanitized) queue table. Not breaker-guarded
|
|
75
|
+
# itself so callers that loop over sub-queues can wrap the loop once.
|
|
76
|
+
def fair_read_step(full_name, qty, vt_seconds)
|
|
77
|
+
Instrumentation.instrument("pgbus.client.read_batch_fair", queue: full_name, qty: qty) do
|
|
78
|
+
with_stale_connection_retry do
|
|
79
|
+
synchronized { with_read_timeout { exec_fair_read(full_name, qty, vt_seconds) } }
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def fair_queue_name(queue_name)
|
|
85
|
+
QueueNameValidator.sanitize!(config.queue_name(queue_name))
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def fair_indexes_ensured
|
|
89
|
+
@fair_indexes_ensured ||= Concurrent::Map.new
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Runs inside create_queue_physically (caller owns the mutex) on a
|
|
93
|
+
# freshly created, empty table — plain CREATE INDEX is instant there.
|
|
94
|
+
def create_fair_index_if_needed(full_name)
|
|
95
|
+
return unless FairShare.enabled?(config)
|
|
96
|
+
|
|
97
|
+
exec_ddl(fair_index_sql(full_name, concurrently: false))
|
|
98
|
+
fair_indexes_ensured[full_name] = true
|
|
99
|
+
rescue StandardError => e
|
|
100
|
+
raise unless duplicate_relation_error?(e)
|
|
101
|
+
|
|
102
|
+
fair_indexes_ensured[full_name] = true
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def exec_ddl(sql)
|
|
106
|
+
@pgmq.with_connection { |conn| conn.exec(sql) }
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def exec_fair_read(full_name, qty, vt_seconds)
|
|
110
|
+
rows = @pgmq.with_connection do |conn|
|
|
111
|
+
conn.exec_params(fair_read_sql(full_name), [qty.to_i, vt_seconds.to_i]).to_a
|
|
112
|
+
end
|
|
113
|
+
rows.map { |row| PGMQ::Message.new(row) }
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def fair_index_name(full_name)
|
|
117
|
+
"q_#{full_name}#{FAIR_INDEX_SUFFIX}"
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def fair_index_sql(full_name, concurrently:)
|
|
121
|
+
"CREATE INDEX #{"CONCURRENTLY " if concurrently}IF NOT EXISTS #{fair_index_name(full_name)} " \
|
|
122
|
+
"ON pgmq.q_#{full_name} ((#{FAIR_INDEX_KEY_EXPR}), vt, msg_id)"
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def fair_read_sql(full_name)
|
|
126
|
+
table = "pgmq.q_#{full_name}"
|
|
127
|
+
<<~SQL
|
|
128
|
+
WITH RECURSIVE fair_keys AS (
|
|
129
|
+
(SELECT #{FAIR_KEY_EXPR} AS k
|
|
130
|
+
FROM #{table} m
|
|
131
|
+
WHERE m.vt <= now()
|
|
132
|
+
ORDER BY 1 LIMIT 1)
|
|
133
|
+
UNION ALL
|
|
134
|
+
SELECT (SELECT #{FAIR_KEY_EXPR}
|
|
135
|
+
FROM #{table} m
|
|
136
|
+
WHERE #{FAIR_KEY_EXPR} > fk.k
|
|
137
|
+
AND m.vt <= now()
|
|
138
|
+
ORDER BY 1 LIMIT 1)
|
|
139
|
+
FROM fair_keys fk
|
|
140
|
+
WHERE fk.k IS NOT NULL
|
|
141
|
+
),
|
|
142
|
+
candidates AS (
|
|
143
|
+
SELECT c.msg_id, c.rn, c.w
|
|
144
|
+
FROM fair_keys fk
|
|
145
|
+
CROSS JOIN LATERAL (
|
|
146
|
+
SELECT m.msg_id,
|
|
147
|
+
ROW_NUMBER() OVER (ORDER BY m.vt, m.msg_id) AS rn,
|
|
148
|
+
#{FAIR_WEIGHT_EXPR} AS w
|
|
149
|
+
FROM #{table} m
|
|
150
|
+
WHERE #{FAIR_KEY_EXPR} = fk.k
|
|
151
|
+
AND m.vt <= now()
|
|
152
|
+
ORDER BY m.vt, m.msg_id
|
|
153
|
+
LIMIT $1
|
|
154
|
+
) c
|
|
155
|
+
WHERE fk.k IS NOT NULL
|
|
156
|
+
),
|
|
157
|
+
picked AS (
|
|
158
|
+
SELECT msg_id, ROW_NUMBER() OVER (ORDER BY rn / w, msg_id) AS selection_order
|
|
159
|
+
FROM candidates
|
|
160
|
+
ORDER BY rn / w, msg_id
|
|
161
|
+
LIMIT $1
|
|
162
|
+
),
|
|
163
|
+
locked AS (
|
|
164
|
+
SELECT m.msg_id, p.selection_order
|
|
165
|
+
FROM #{table} m
|
|
166
|
+
JOIN picked p ON p.msg_id = m.msg_id
|
|
167
|
+
WHERE m.vt <= now()
|
|
168
|
+
FOR UPDATE OF m SKIP LOCKED
|
|
169
|
+
),
|
|
170
|
+
updated AS (
|
|
171
|
+
UPDATE #{table} m
|
|
172
|
+
SET vt = clock_timestamp() + make_interval(secs => $2),
|
|
173
|
+
read_ct = read_ct + 1,
|
|
174
|
+
last_read_at = clock_timestamp()
|
|
175
|
+
FROM locked l
|
|
176
|
+
WHERE m.msg_id = l.msg_id
|
|
177
|
+
RETURNING m.msg_id, m.read_ct, m.enqueued_at, m.last_read_at, m.vt, m.message, m.headers,
|
|
178
|
+
l.selection_order
|
|
179
|
+
)
|
|
180
|
+
SELECT msg_id, read_ct, enqueued_at, last_read_at, vt, message, headers
|
|
181
|
+
FROM updated
|
|
182
|
+
ORDER BY selection_order
|
|
183
|
+
SQL
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
data/lib/pgbus/client.rb
CHANGED
|
@@ -12,6 +12,7 @@ require_relative "client/resizable_pool"
|
|
|
12
12
|
module Pgbus
|
|
13
13
|
class Client
|
|
14
14
|
include ReadAfter
|
|
15
|
+
include FairRead
|
|
15
16
|
include EnsureStreamQueue
|
|
16
17
|
include NotifyStream
|
|
17
18
|
|
|
@@ -339,6 +340,10 @@ module Pgbus
|
|
|
339
340
|
end
|
|
340
341
|
end
|
|
341
342
|
|
|
343
|
+
def target_queue(queue_name, priority = nil)
|
|
344
|
+
@queue_strategy.target_queue(queue_name, priority)
|
|
345
|
+
end
|
|
346
|
+
|
|
342
347
|
def send_message(queue_name, payload, headers: nil, delay: 0, priority: nil)
|
|
343
348
|
target = @queue_strategy.target_queue(queue_name, priority)
|
|
344
349
|
Instrumentation.instrument("pgbus.client.send_message", queue: target) do
|
|
@@ -381,13 +386,17 @@ module Pgbus
|
|
|
381
386
|
msg_id
|
|
382
387
|
end
|
|
383
388
|
|
|
384
|
-
|
|
385
|
-
|
|
389
|
+
# Bulk enqueue. Routes through the queue strategy exactly like
|
|
390
|
+
# #send_message: under priority routing the bare `pgbus_<queue>` table is
|
|
391
|
+
# never created (only `_p0.._pN` are), so targeting it raised
|
|
392
|
+
# PG::UndefinedTable and no worker would have read it either.
|
|
393
|
+
def send_batch(queue_name, payloads, headers: nil, delay: 0, priority: nil)
|
|
394
|
+
target = @queue_strategy.target_queue(queue_name, priority)
|
|
386
395
|
serialized, serialized_headers = serialize_batch(payloads, headers)
|
|
387
|
-
Instrumentation.instrument("pgbus.client.send_batch", queue:
|
|
396
|
+
Instrumentation.instrument("pgbus.client.send_batch", queue: target, size: payloads.size) do
|
|
388
397
|
with_stale_connection_retry do
|
|
389
398
|
ensure_queue(queue_name)
|
|
390
|
-
synchronized { @pgmq.produce_batch(
|
|
399
|
+
synchronized { @pgmq.produce_batch(target, serialized, headers: serialized_headers, delay: delay) }
|
|
391
400
|
end
|
|
392
401
|
end
|
|
393
402
|
end
|
|
@@ -420,11 +429,17 @@ module Pgbus
|
|
|
420
429
|
# Non-priority fast path delegates to read_batch, which is already gated
|
|
421
430
|
# by the connection-health breaker — no extra guard needed here.
|
|
422
431
|
unless @queue_strategy.priority?
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
432
|
+
msgs = if FairShare.enabled?(config)
|
|
433
|
+
read_batch_fair(queue_name, qty: qty, vt: vt)
|
|
434
|
+
else
|
|
435
|
+
read_batch(queue_name, qty: qty, vt: vt)
|
|
436
|
+
end
|
|
437
|
+
return (msgs || []).map { |m| [config.queue_name(queue_name), m] }
|
|
426
438
|
end
|
|
427
439
|
|
|
440
|
+
# Fair share (issue #426): strict between levels, fair within a level.
|
|
441
|
+
return read_batch_prioritized_fair(queue_name, qty: qty, vt: vt) if FairShare.enabled?(config)
|
|
442
|
+
|
|
428
443
|
# The priority loop issues its own reads, so gate the whole loop: an open
|
|
429
444
|
# breaker fails fast before any sub-queue is touched, and the loop as a
|
|
430
445
|
# unit records one success/failure with the latch.
|
|
@@ -449,6 +464,25 @@ module Pgbus
|
|
|
449
464
|
end
|
|
450
465
|
end
|
|
451
466
|
|
|
467
|
+
# Priority sub-queues drained p0 → pN, each level read with the fair
|
|
468
|
+
# interleave. guarded_read wraps the loop as one unit (see above).
|
|
469
|
+
def read_batch_prioritized_fair(queue_name, qty:, vt: nil)
|
|
470
|
+
guarded_read do
|
|
471
|
+
remaining = qty
|
|
472
|
+
results = []
|
|
473
|
+
|
|
474
|
+
config.priority_queue_names(queue_name).each do |pq_name|
|
|
475
|
+
break if remaining <= 0
|
|
476
|
+
|
|
477
|
+
msgs = fair_read_step(QueueNameValidator.sanitize!(pq_name), remaining, vt || config.visibility_timeout)
|
|
478
|
+
msgs.each { |m| results << [pq_name, m] }
|
|
479
|
+
remaining -= msgs.size
|
|
480
|
+
end
|
|
481
|
+
|
|
482
|
+
results
|
|
483
|
+
end
|
|
484
|
+
end
|
|
485
|
+
|
|
452
486
|
def read_with_poll(queue_name, qty:, vt: nil, max_poll_seconds: 5, poll_interval_ms: 100)
|
|
453
487
|
full_name = config.queue_name(queue_name)
|
|
454
488
|
guarded_read do
|
|
@@ -682,21 +716,55 @@ module Pgbus
|
|
|
682
716
|
# false — the message definitely does not exist
|
|
683
717
|
# nil — could not determine (e.g. queue table missing or unknown error).
|
|
684
718
|
# Callers MUST treat nil as "exists" for safety.
|
|
685
|
-
def
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
719
|
+
def message_in_queue?(queue_name, msg_id:)
|
|
720
|
+
message_exists?(queue_name, msg_id: msg_id)
|
|
721
|
+
end
|
|
722
|
+
|
|
723
|
+
# Look up a queue row by ActiveJob job_id in the JSON payload. Used by the
|
|
724
|
+
# batch sweep: move_to_dead_letter produces a new DLQ msg_id, so the
|
|
725
|
+
# source msg_id is not a DLQ identity.
|
|
726
|
+
#
|
|
727
|
+
# Same tri-state as message_exists?: true / false / nil (unknown). A
|
|
728
|
+
# logical name expands to every physical table the strategy owns (the
|
|
729
|
+
# `_pN` sub-queues under priority routing); an already-prefixed physical
|
|
730
|
+
# name probes itself only.
|
|
731
|
+
def message_with_job_id?(queue_name, job_id:)
|
|
732
|
+
tables = lookup_physical_queue_names(queue_name).map { |name| QueueNameValidator.sanitize!(name) }
|
|
733
|
+
determined = false
|
|
734
|
+
|
|
735
|
+
synchronized do
|
|
736
|
+
with_raw_connection do |conn|
|
|
737
|
+
tables.each do |sanitized|
|
|
738
|
+
present = probe_job_id_presence(conn, sanitized, job_id)
|
|
739
|
+
return true if present == true
|
|
689
740
|
|
|
741
|
+
determined = true unless present == :missing
|
|
742
|
+
end
|
|
743
|
+
end
|
|
744
|
+
end
|
|
745
|
+
|
|
746
|
+
determined ? false : nil # rubocop:disable Style/ReturnNilInPredicateMethodDefinition -- tri-state: nil means unknown
|
|
747
|
+
end
|
|
748
|
+
|
|
749
|
+
# DLQ companion of a logical or already-physical queue name. Does not
|
|
750
|
+
# re-prefix a name that resolve_full_queue_name already expanded.
|
|
751
|
+
def dead_letter_physical_name(queue_name)
|
|
752
|
+
full = resolve_full_queue_name(queue_name)
|
|
753
|
+
# Priority sub-queues share the logical queue's DLQ (`pgbus_default_dlq`),
|
|
754
|
+
# not a per-priority `…_pN_dlq`. Only strip when the strategy actually
|
|
755
|
+
# creates `_pN` tables — a logical queue named `orders_p0` must keep it.
|
|
756
|
+
base = @queue_strategy.priority? ? full.sub(/_p\d+\z/, "") : full
|
|
757
|
+
"#{base}#{Pgbus::DEAD_LETTER_SUFFIX}"
|
|
758
|
+
end
|
|
759
|
+
|
|
760
|
+
# Same tri-state as message_exists?: true / false / nil (unknown).
|
|
761
|
+
def message_archived?(queue_name, msg_id:)
|
|
690
762
|
full_name = resolve_full_queue_name(queue_name)
|
|
691
763
|
sanitized = QueueNameValidator.sanitize!(full_name)
|
|
692
764
|
|
|
693
765
|
synchronized do
|
|
694
766
|
with_raw_connection do |conn|
|
|
695
|
-
|
|
696
|
-
msg_id_present?(conn, sanitized, msg_id.to_i)
|
|
697
|
-
else
|
|
698
|
-
uniqueness_key_present?(conn, sanitized, uniqueness_key)
|
|
699
|
-
end
|
|
767
|
+
msg_id_archived?(conn, sanitized, msg_id.to_i)
|
|
700
768
|
end
|
|
701
769
|
end
|
|
702
770
|
rescue ActiveRecord::StatementInvalid => e
|
|
@@ -709,6 +777,64 @@ module Pgbus
|
|
|
709
777
|
nil
|
|
710
778
|
end
|
|
711
779
|
|
|
780
|
+
def message_exists?(queue_name, msg_id: nil, uniqueness_key: nil)
|
|
781
|
+
has_msg_id = !msg_id.nil?
|
|
782
|
+
has_uniqueness_key = !uniqueness_key.nil?
|
|
783
|
+
raise ArgumentError, "pass msg_id, uniqueness_key, or both" unless has_msg_id || has_uniqueness_key
|
|
784
|
+
|
|
785
|
+
tables = lookup_physical_queue_names(queue_name).map { |name| QueueNameValidator.sanitize!(name) }
|
|
786
|
+
determined = false
|
|
787
|
+
|
|
788
|
+
synchronized do
|
|
789
|
+
with_raw_connection do |conn|
|
|
790
|
+
tables.each do |sanitized|
|
|
791
|
+
present = probe_queue_presence(
|
|
792
|
+
conn, sanitized,
|
|
793
|
+
msg_id: has_msg_id ? msg_id.to_i : nil,
|
|
794
|
+
uniqueness_key: has_uniqueness_key ? uniqueness_key : nil
|
|
795
|
+
)
|
|
796
|
+
return true if present == true
|
|
797
|
+
|
|
798
|
+
determined = true unless present == :missing
|
|
799
|
+
end
|
|
800
|
+
end
|
|
801
|
+
end
|
|
802
|
+
|
|
803
|
+
determined ? false : nil # rubocop:disable Style/ReturnNilInPredicateMethodDefinition -- tri-state: nil means unknown
|
|
804
|
+
end
|
|
805
|
+
|
|
806
|
+
# Which uniqueness keys currently appear in any live PGMQ queue payload.
|
|
807
|
+
# Used by the dispatcher reaper for unbound locks (pending / msg_id=0)
|
|
808
|
+
# so it never probes the synthetic `pending` queue (issue #418).
|
|
809
|
+
#
|
|
810
|
+
# Per-queue UndefinedTable is skipped (table dropped between listing and
|
|
811
|
+
# select). Failure to list pgmq.meta — or any non-undefined error — raises;
|
|
812
|
+
# callers must treat that as "unknown" and keep the lock.
|
|
813
|
+
def uniqueness_keys_present(lock_keys)
|
|
814
|
+
keys = Array(lock_keys).compact.map(&:to_s).uniq
|
|
815
|
+
return Set.new if keys.empty?
|
|
816
|
+
|
|
817
|
+
found = Set.new
|
|
818
|
+
synchronized do
|
|
819
|
+
with_raw_connection do |conn|
|
|
820
|
+
names = conn.exec("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
|
|
821
|
+
.map { |row| row["queue_name"] }
|
|
822
|
+
names.each do |name|
|
|
823
|
+
break if found.size == keys.size
|
|
824
|
+
|
|
825
|
+
sanitized = begin
|
|
826
|
+
QueueNameValidator.sanitize!(name)
|
|
827
|
+
rescue ArgumentError
|
|
828
|
+
next
|
|
829
|
+
end
|
|
830
|
+
|
|
831
|
+
scan_uniqueness_keys(conn, sanitized, keys, found)
|
|
832
|
+
end
|
|
833
|
+
end
|
|
834
|
+
end
|
|
835
|
+
found
|
|
836
|
+
end
|
|
837
|
+
|
|
712
838
|
def purge_archive(queue_name, older_than:, batch_size: 1000)
|
|
713
839
|
full_name = config.queue_name(queue_name)
|
|
714
840
|
sanitized = QueueNameValidator.sanitize!(full_name)
|
|
@@ -933,9 +1059,76 @@ module Pgbus
|
|
|
933
1059
|
name.start_with?(prefix) ? name : config.queue_name(name)
|
|
934
1060
|
end
|
|
935
1061
|
|
|
936
|
-
|
|
1062
|
+
# Logical names expand through QueueFactory so a priority queue's _pN
|
|
1063
|
+
# tables are included; already-prefixed physical names stay as-is.
|
|
1064
|
+
def lookup_physical_queue_names(queue_name)
|
|
1065
|
+
name = queue_name.to_s
|
|
1066
|
+
prefix = "#{config.queue_prefix}_"
|
|
1067
|
+
name.start_with?(prefix) ? [name] : @queue_strategy.physical_queue_names(name)
|
|
1068
|
+
end
|
|
1069
|
+
|
|
1070
|
+
def probe_queue_presence(conn, sanitized, msg_id:, uniqueness_key:)
|
|
1071
|
+
if msg_id
|
|
1072
|
+
msg_id_present?(conn, sanitized, msg_id, uniqueness_key: uniqueness_key)
|
|
1073
|
+
else
|
|
1074
|
+
uniqueness_key_present?(conn, sanitized, uniqueness_key)
|
|
1075
|
+
end
|
|
1076
|
+
rescue ActiveRecord::StatementInvalid => e
|
|
1077
|
+
raise unless undefined_table_error?(e)
|
|
1078
|
+
|
|
1079
|
+
:missing
|
|
1080
|
+
rescue StandardError => e
|
|
1081
|
+
raise unless defined?(PG::UndefinedTable) && e.is_a?(PG::UndefinedTable)
|
|
1082
|
+
|
|
1083
|
+
:missing
|
|
1084
|
+
end
|
|
1085
|
+
|
|
1086
|
+
def probe_job_id_presence(conn, sanitized, job_id)
|
|
1087
|
+
job_id_present?(conn, sanitized, job_id)
|
|
1088
|
+
rescue ActiveRecord::StatementInvalid => e
|
|
1089
|
+
raise unless undefined_table_error?(e)
|
|
1090
|
+
|
|
1091
|
+
:missing
|
|
1092
|
+
rescue StandardError => e
|
|
1093
|
+
raise unless defined?(PG::UndefinedTable) && e.is_a?(PG::UndefinedTable)
|
|
1094
|
+
|
|
1095
|
+
:missing
|
|
1096
|
+
end
|
|
1097
|
+
|
|
1098
|
+
def scan_uniqueness_keys(conn, sanitized, keys, found)
|
|
1099
|
+
placeholders = keys.each_index.map { |i| "$#{i + 1}" }.join(", ")
|
|
1100
|
+
result = conn.exec_params(
|
|
1101
|
+
"SELECT DISTINCT message::jsonb ->> 'pgbus_uniqueness_key' AS k " \
|
|
1102
|
+
"FROM pgmq.q_#{sanitized} " \
|
|
1103
|
+
"WHERE message::jsonb ->> 'pgbus_uniqueness_key' IN (#{placeholders})",
|
|
1104
|
+
keys
|
|
1105
|
+
)
|
|
1106
|
+
result.each { |row| found.add(row["k"]) if row["k"] }
|
|
1107
|
+
rescue ActiveRecord::StatementInvalid => e
|
|
1108
|
+
raise unless undefined_table_error?(e)
|
|
1109
|
+
rescue StandardError => e
|
|
1110
|
+
raise unless defined?(PG::UndefinedTable) && e.is_a?(PG::UndefinedTable)
|
|
1111
|
+
end
|
|
1112
|
+
|
|
1113
|
+
def msg_id_present?(conn, sanitized, msg_id, uniqueness_key: nil)
|
|
1114
|
+
result = if uniqueness_key
|
|
1115
|
+
conn.exec_params(
|
|
1116
|
+
"SELECT 1 FROM pgmq.q_#{sanitized} WHERE msg_id = $1 " \
|
|
1117
|
+
"AND message::jsonb ->> 'pgbus_uniqueness_key' = $2 LIMIT 1",
|
|
1118
|
+
[msg_id, uniqueness_key]
|
|
1119
|
+
)
|
|
1120
|
+
else
|
|
1121
|
+
conn.exec_params(
|
|
1122
|
+
"SELECT 1 FROM pgmq.q_#{sanitized} WHERE msg_id = $1 LIMIT 1",
|
|
1123
|
+
[msg_id]
|
|
1124
|
+
)
|
|
1125
|
+
end
|
|
1126
|
+
result.ntuples.positive?
|
|
1127
|
+
end
|
|
1128
|
+
|
|
1129
|
+
def msg_id_archived?(conn, sanitized, msg_id)
|
|
937
1130
|
result = conn.exec_params(
|
|
938
|
-
"SELECT 1 FROM pgmq.
|
|
1131
|
+
"SELECT 1 FROM pgmq.a_#{sanitized} WHERE msg_id = $1 LIMIT 1",
|
|
939
1132
|
[msg_id]
|
|
940
1133
|
)
|
|
941
1134
|
result.ntuples.positive?
|
|
@@ -950,6 +1143,15 @@ module Pgbus
|
|
|
950
1143
|
result.ntuples.positive?
|
|
951
1144
|
end
|
|
952
1145
|
|
|
1146
|
+
def job_id_present?(conn, sanitized, job_id)
|
|
1147
|
+
result = conn.exec_params(
|
|
1148
|
+
"SELECT 1 FROM pgmq.q_#{sanitized} " \
|
|
1149
|
+
"WHERE message::jsonb ->> 'job_id' = $1 LIMIT 1",
|
|
1150
|
+
[job_id]
|
|
1151
|
+
)
|
|
1152
|
+
result.ntuples.positive?
|
|
1153
|
+
end
|
|
1154
|
+
|
|
953
1155
|
# Detect "relation does not exist" via the underlying PG error type.
|
|
954
1156
|
# Falls back to message matching only if PG::UndefinedTable is undefined
|
|
955
1157
|
# (very old pg gem) — never relies on locale-sensitive text.
|
|
@@ -1175,6 +1377,7 @@ module Pgbus
|
|
|
1175
1377
|
create_queue_table(full_name)
|
|
1176
1378
|
enable_notify_if_needed(full_name, NOTIFY_THROTTLE_MS)
|
|
1177
1379
|
create_fifo_index_if_needed(full_name)
|
|
1380
|
+
create_fair_index_if_needed(full_name)
|
|
1178
1381
|
end
|
|
1179
1382
|
end
|
|
1180
1383
|
|
|
@@ -28,12 +28,25 @@ module Pgbus
|
|
|
28
28
|
# otherwise. This avoids losing a blocked row if enqueue fails.
|
|
29
29
|
def promote_next(concurrency_key, client:, delay: 0)
|
|
30
30
|
released = nil
|
|
31
|
+
msg_id = nil
|
|
31
32
|
Pgbus::BlockedExecution.transaction do
|
|
32
33
|
released = release_next(concurrency_key)
|
|
33
34
|
raise ActiveRecord::Rollback unless released
|
|
34
35
|
|
|
35
36
|
actual_delay = resolve_delay(released[:payload], delay)
|
|
36
|
-
|
|
37
|
+
# Carry the enqueuer's priority through: under priority routing it
|
|
38
|
+
# picks the _pN sub-queue, not just the release order (issue #423).
|
|
39
|
+
msg_id = client.send_message(released[:queue_name], released[:payload],
|
|
40
|
+
delay: actual_delay, priority: released[:priority])
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
if released && msg_id
|
|
44
|
+
begin
|
|
45
|
+
Batch.backfill_execution(released[:payload], msg_id,
|
|
46
|
+
client.target_queue(released[:queue_name], released[:priority]))
|
|
47
|
+
rescue StandardError => e
|
|
48
|
+
Pgbus.logger.warn { "[Pgbus] Batch execution backfill failed after promote: #{e.message}" }
|
|
49
|
+
end
|
|
37
50
|
end
|
|
38
51
|
|
|
39
52
|
!!released
|
data/lib/pgbus/configuration.rb
CHANGED
|
@@ -70,11 +70,40 @@ module Pgbus
|
|
|
70
70
|
# :round_robin = use read_grouped_rr (fair round-robin across groups).
|
|
71
71
|
attr_reader :group_mode
|
|
72
72
|
|
|
73
|
+
# Fair share scheduling across tenants (issue #426). nil = disabled.
|
|
74
|
+
# Any #call-able receiving the ActiveJob instance at enqueue time and
|
|
75
|
+
# returning nil (unkeyed), a key (String/Symbol/Integer), or [key, weight]
|
|
76
|
+
# (weight: positive Numeric, default 1). Keyed jobs are read with
|
|
77
|
+
# Client#read_batch_fair — a weighted, work-conserving interleave across
|
|
78
|
+
# keys within each queue. Mutually exclusive with group_mode.
|
|
79
|
+
attr_reader :fair_share
|
|
80
|
+
|
|
81
|
+
# Fair share for event-bus consumers (issue #427). nil = disabled. Any
|
|
82
|
+
# #call-able receiving the Pgbus::Event at publish time (routing_key,
|
|
83
|
+
# payload as passed to publish, headers) and returning nil, a key, or
|
|
84
|
+
# [key, weight] exactly like fair_share. The key rides in the event
|
|
85
|
+
# envelope and consumers read subscriber queues with
|
|
86
|
+
# Client#read_batch_fair. Independent of fair_share (jobs).
|
|
87
|
+
attr_reader :event_fair_share
|
|
88
|
+
|
|
89
|
+
# Persist ActiveSupport::CurrentAttributes across enqueue → perform
|
|
90
|
+
# (issue #430). nil = off (default). :auto = every CurrentAttributes
|
|
91
|
+
# subclass; an Array of classes/names; or a Hash of class/name =>
|
|
92
|
+
# { only: [...] } / { except: [...] }. Normalized to :auto or an Array of
|
|
93
|
+
# { name:, only:, except: } specs (classes are resolved lazily by name).
|
|
94
|
+
attr_reader :current_attributes
|
|
95
|
+
|
|
73
96
|
# Archive compaction. Only the user-facing retention window is configurable;
|
|
74
97
|
# the loop interval and batch size are tuned via constants on
|
|
75
98
|
# Pgbus::Process::Dispatcher.
|
|
76
99
|
attr_reader :archive_retention
|
|
77
100
|
|
|
101
|
+
# Batch execution-row sweep + finished-batch cleanup. Retention nil disables
|
|
102
|
+
# cleanup (same sentinel as archive_retention). Sweep interval is required.
|
|
103
|
+
# Stall threshold: how long a pending batch may go without a new execution
|
|
104
|
+
# row (or an orphan row without a msg_id) before the sweep repairs it.
|
|
105
|
+
attr_reader :batch_retention, :batch_sweep_interval, :batch_stall_threshold
|
|
106
|
+
|
|
78
107
|
# Transactional outbox
|
|
79
108
|
attr_accessor :outbox_enabled, :outbox_poll_interval, :outbox_batch_size
|
|
80
109
|
attr_reader :outbox_retention # rubocop:disable Style/AccessorGrouping
|
|
@@ -262,8 +291,14 @@ module Pgbus
|
|
|
262
291
|
@priority_levels = nil
|
|
263
292
|
@default_priority = 1
|
|
264
293
|
@group_mode = nil
|
|
294
|
+
@fair_share = nil
|
|
295
|
+
@event_fair_share = nil
|
|
296
|
+
@current_attributes = nil
|
|
265
297
|
|
|
266
298
|
@archive_retention = 7 * 24 * 3600 # 7 days
|
|
299
|
+
@batch_retention = 7 * 24 * 3600 # 7 days
|
|
300
|
+
@batch_sweep_interval = 300 # 5 minutes
|
|
301
|
+
@batch_stall_threshold = 300 # 5 minutes, solid_queue's stalled_for default
|
|
267
302
|
|
|
268
303
|
@outbox_enabled = false
|
|
269
304
|
@outbox_poll_interval = 1.0
|
|
@@ -599,6 +634,28 @@ module Pgbus
|
|
|
599
634
|
@group_mode = coerced
|
|
600
635
|
end
|
|
601
636
|
|
|
637
|
+
def current_attributes=(value)
|
|
638
|
+
@current_attributes = Pgbus::CurrentAttributes.normalize(value)
|
|
639
|
+
end
|
|
640
|
+
|
|
641
|
+
def fair_share=(callable)
|
|
642
|
+
unless callable.nil? || callable.respond_to?(:call)
|
|
643
|
+
raise Pgbus::ConfigurationError,
|
|
644
|
+
"fair_share must be nil or respond to #call (got #{callable.class})"
|
|
645
|
+
end
|
|
646
|
+
|
|
647
|
+
@fair_share = callable
|
|
648
|
+
end
|
|
649
|
+
|
|
650
|
+
def event_fair_share=(callable)
|
|
651
|
+
unless callable.nil? || callable.respond_to?(:call)
|
|
652
|
+
raise Pgbus::ConfigurationError,
|
|
653
|
+
"event_fair_share must be nil or respond to #call (got #{callable.class})"
|
|
654
|
+
end
|
|
655
|
+
|
|
656
|
+
@event_fair_share = callable
|
|
657
|
+
end
|
|
658
|
+
|
|
602
659
|
VALID_CONNECTION_GUC_MODES = %i[options session].freeze
|
|
603
660
|
|
|
604
661
|
def connection_guc_mode=(mode)
|
|
@@ -774,6 +831,7 @@ module Pgbus
|
|
|
774
831
|
validate_job_path_gaps!
|
|
775
832
|
validate_streams!
|
|
776
833
|
validate_metrics_backend!
|
|
834
|
+
validate_fair_share!
|
|
777
835
|
|
|
778
836
|
self
|
|
779
837
|
end
|
|
@@ -818,7 +876,9 @@ module Pgbus
|
|
|
818
876
|
|
|
819
877
|
# Interval knobs: positive Numeric, never nil (mirror polling_interval).
|
|
820
878
|
{ dispatch_interval: dispatch_interval, outbox_poll_interval: outbox_poll_interval,
|
|
821
|
-
recurring_schedule_interval: recurring_schedule_interval
|
|
879
|
+
recurring_schedule_interval: recurring_schedule_interval,
|
|
880
|
+
batch_sweep_interval: batch_sweep_interval,
|
|
881
|
+
batch_stall_threshold: batch_stall_threshold }.each do |name, value|
|
|
822
882
|
raise Pgbus::ConfigurationError, "#{name} must be > 0" unless value.is_a?(Numeric) && value.positive?
|
|
823
883
|
end
|
|
824
884
|
|
|
@@ -852,6 +912,14 @@ module Pgbus
|
|
|
852
912
|
raise Pgbus::ConfigurationError, "connects_to must be a Hash or nil"
|
|
853
913
|
end
|
|
854
914
|
|
|
915
|
+
def validate_fair_share!
|
|
916
|
+
return unless fair_share && group_mode
|
|
917
|
+
|
|
918
|
+
raise Pgbus::ConfigurationError,
|
|
919
|
+
"fair_share and group_mode are mutually exclusive — fair_share interleaves across keys " \
|
|
920
|
+
"within a queue, group_mode serializes PGMQ FIFO groups; pick one"
|
|
921
|
+
end
|
|
922
|
+
|
|
855
923
|
def validate_streams!
|
|
856
924
|
unless streams_default_retention.is_a?(Numeric) && streams_default_retention >= 0
|
|
857
925
|
raise Pgbus::ConfigurationError, "streams_default_retention must be a non-negative number"
|
|
@@ -1170,6 +1238,18 @@ module Pgbus
|
|
|
1170
1238
|
@archive_retention = coerce_duration!(value, :archive_retention)
|
|
1171
1239
|
end
|
|
1172
1240
|
|
|
1241
|
+
def batch_retention=(value)
|
|
1242
|
+
@batch_retention = coerce_duration!(value, :batch_retention)
|
|
1243
|
+
end
|
|
1244
|
+
|
|
1245
|
+
def batch_sweep_interval=(value)
|
|
1246
|
+
@batch_sweep_interval = coerce_duration!(value, :batch_sweep_interval)
|
|
1247
|
+
end
|
|
1248
|
+
|
|
1249
|
+
def batch_stall_threshold=(value)
|
|
1250
|
+
@batch_stall_threshold = coerce_duration!(value, :batch_stall_threshold)
|
|
1251
|
+
end
|
|
1252
|
+
|
|
1173
1253
|
def outbox_retention=(value)
|
|
1174
1254
|
@outbox_retention = coerce_duration!(value, :outbox_retention)
|
|
1175
1255
|
end
|