pgbus 0.14.2 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.md +33 -0
- data/Rakefile +6 -1
- data/app/helpers/pgbus/application_helper.rb +7 -0
- data/app/views/pgbus/dead_letter/show.html.erb +17 -0
- data/app/views/pgbus/events/_pending_table.html.erb +21 -0
- data/app/views/pgbus/jobs/show.html.erb +17 -0
- data/config/locales/da.yml +3 -0
- data/config/locales/de.yml +3 -0
- data/config/locales/en.yml +3 -0
- data/config/locales/es.yml +3 -0
- data/config/locales/fi.yml +3 -0
- data/config/locales/fr.yml +3 -0
- data/config/locales/it.yml +3 -0
- data/config/locales/ja.yml +3 -0
- data/config/locales/nb.yml +3 -0
- data/config/locales/nl.yml +3 -0
- data/config/locales/pt.yml +3 -0
- data/config/locales/sv.yml +3 -0
- data/lib/pgbus/active_job/adapter.rb +5 -1
- data/lib/pgbus/active_job/current_attributes.rb +52 -0
- data/lib/pgbus/client/fair_read.rb +187 -0
- data/lib/pgbus/client.rb +30 -3
- data/lib/pgbus/configuration.rb +57 -0
- data/lib/pgbus/current_attributes.rb +156 -0
- data/lib/pgbus/engine.rb +1 -0
- data/lib/pgbus/event.rb +7 -2
- data/lib/pgbus/event_bus/handler.rb +12 -2
- data/lib/pgbus/event_bus/publisher.rb +37 -2
- data/lib/pgbus/event_bus/subscriber.rb +4 -0
- data/lib/pgbus/fair_share.rb +110 -0
- data/lib/pgbus/outbox.rb +8 -1
- data/lib/pgbus/process/consumer.rb +38 -1
- data/lib/pgbus/process/worker.rb +41 -0
- data/lib/pgbus/testing.rb +2 -1
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/job_context.rb +80 -0
- data/lib/pgbus.rb +2 -0
- metadata +6 -1
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class Client
|
|
5
|
+
# Fair share reads (issue #426): a weighted, work-conserving interleave
|
|
6
|
+
# across fair-share keys within one queue. Replaces `read_batch` on the
|
|
7
|
+
# worker when `config.fair_share` is set.
|
|
8
|
+
#
|
|
9
|
+
# Scheduling rule per read of `qty`: for every key that has visible
|
|
10
|
+
# messages, rank that key's visible messages 1..qty oldest-visible first
|
|
11
|
+
# (`vt, msg_id`); a message's virtual time is `rank / weight`; take the
|
|
12
|
+
# `qty` messages with the lowest `(virtual_time, msg_id)`. Weight 3 vs 1
|
|
13
|
+
# yields a 3:1 split under contention; a lone key fills the whole batch.
|
|
14
|
+
# This is batch-level weighted fair queuing — proportional within each
|
|
15
|
+
# batch, memoryless across batches (no deficit carry-over).
|
|
16
|
+
#
|
|
17
|
+
# Cost model: key enumeration is a loose index scan over
|
|
18
|
+
# (key, vt, msg_id) — one probe per key with visible work; keys whose
|
|
19
|
+
# messages are all invisible (in flight / delayed / in retry backoff) are
|
|
20
|
+
# skipped at the index level because `vt` is in the index. Per-key
|
|
21
|
+
# candidates are a bounded `LIMIT qty` index range. Roughly
|
|
22
|
+
# O(K · log n + K · qty), K = keys with visible work, independent of
|
|
23
|
+
# backlog depth. Within a key the order is `(vt, msg_id)` — FIFO for
|
|
24
|
+
# immediate enqueues; retried/delayed messages sort by when they became
|
|
25
|
+
# visible — a deliberate deviation from pgmq.read's pure msg_id order so
|
|
26
|
+
# the per-key lookup never sorts a tenant's whole visible backlog.
|
|
27
|
+
#
|
|
28
|
+
# Candidates are selected before locking (same shape as
|
|
29
|
+
# pgmq.read_grouped_rr), so under concurrent readers a batch can come back
|
|
30
|
+
# short; the worker loop re-reads immediately while it has capacity.
|
|
31
|
+
module FairRead
|
|
32
|
+
# The read expressions are written against the alias `m`; the index
|
|
33
|
+
# expression is the same shape minus the alias so the planner matches it.
|
|
34
|
+
FAIR_INDEX_KEY_EXPR = "COALESCE(message->>'#{FairShare::METADATA_KEY}', '')".freeze
|
|
35
|
+
FAIR_KEY_EXPR = "COALESCE(m.message->>'#{FairShare::METADATA_KEY}', '')".freeze
|
|
36
|
+
FAIR_WEIGHT_EXPR = "COALESCE((m.message->>'#{FairShare::WEIGHT_KEY}')::numeric, 1)".freeze
|
|
37
|
+
FAIR_INDEX_SUFFIX = "_fair_idx"
|
|
38
|
+
|
|
39
|
+
def read_batch_fair(queue_name, qty:, vt: nil)
|
|
40
|
+
full_name = fair_queue_name(queue_name)
|
|
41
|
+
guarded_read { fair_read_step(full_name, qty, vt || config.visibility_timeout) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Idempotent, memoized per process. Uses CREATE INDEX CONCURRENTLY so
|
|
45
|
+
# enabling fair share on an existing, populated queue never blocks
|
|
46
|
+
# enqueues. A queue table that does not exist yet is left alone (its
|
|
47
|
+
# creation path builds the index non-concurrently); any other failure is
|
|
48
|
+
# logged with the remediation and NOT memoized so the next ensure retries.
|
|
49
|
+
def ensure_fair_index(queue_name)
|
|
50
|
+
full_name = fair_queue_name(queue_name)
|
|
51
|
+
return if fair_indexes_ensured[full_name]
|
|
52
|
+
|
|
53
|
+
with_stale_connection_retry do
|
|
54
|
+
synchronized { exec_ddl(fair_index_sql(full_name, concurrently: true)) }
|
|
55
|
+
end
|
|
56
|
+
fair_indexes_ensured[full_name] = true
|
|
57
|
+
rescue StandardError => e
|
|
58
|
+
if duplicate_relation_error?(e)
|
|
59
|
+
fair_indexes_ensured[full_name] = true
|
|
60
|
+
elsif undefined_table_error?(e)
|
|
61
|
+
Pgbus.logger.debug { "[Pgbus] Fair index deferred — queue table #{full_name} not created yet" }
|
|
62
|
+
else
|
|
63
|
+
Pgbus.logger.error do
|
|
64
|
+
"[Pgbus] Could not create fair index #{fair_index_name(full_name)} on pgmq.q_#{full_name}: " \
|
|
65
|
+
"#{e.class}: #{e.message}. A failed CONCURRENTLY build leaves an INVALID index behind — " \
|
|
66
|
+
"run `DROP INDEX IF EXISTS pgmq.#{fair_index_name(full_name)}` and restart the worker to retry."
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
# One instrumented, retried, serialized, timeout-bounded fair read of a
|
|
74
|
+
# physical (already prefixed + sanitized) queue table. Not breaker-guarded
|
|
75
|
+
# itself so callers that loop over sub-queues can wrap the loop once.
|
|
76
|
+
def fair_read_step(full_name, qty, vt_seconds)
|
|
77
|
+
Instrumentation.instrument("pgbus.client.read_batch_fair", queue: full_name, qty: qty) do
|
|
78
|
+
with_stale_connection_retry do
|
|
79
|
+
synchronized { with_read_timeout { exec_fair_read(full_name, qty, vt_seconds) } }
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def fair_queue_name(queue_name)
|
|
85
|
+
QueueNameValidator.sanitize!(config.queue_name(queue_name))
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def fair_indexes_ensured
|
|
89
|
+
@fair_indexes_ensured ||= Concurrent::Map.new
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Runs inside create_queue_physically (caller owns the mutex) on a
|
|
93
|
+
# freshly created, empty table — plain CREATE INDEX is instant there.
|
|
94
|
+
def create_fair_index_if_needed(full_name)
|
|
95
|
+
return unless FairShare.enabled?(config)
|
|
96
|
+
|
|
97
|
+
exec_ddl(fair_index_sql(full_name, concurrently: false))
|
|
98
|
+
fair_indexes_ensured[full_name] = true
|
|
99
|
+
rescue StandardError => e
|
|
100
|
+
raise unless duplicate_relation_error?(e)
|
|
101
|
+
|
|
102
|
+
fair_indexes_ensured[full_name] = true
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def exec_ddl(sql)
|
|
106
|
+
@pgmq.with_connection { |conn| conn.exec(sql) }
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def exec_fair_read(full_name, qty, vt_seconds)
|
|
110
|
+
rows = @pgmq.with_connection do |conn|
|
|
111
|
+
conn.exec_params(fair_read_sql(full_name), [qty.to_i, vt_seconds.to_i]).to_a
|
|
112
|
+
end
|
|
113
|
+
rows.map { |row| PGMQ::Message.new(row) }
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def fair_index_name(full_name)
|
|
117
|
+
"q_#{full_name}#{FAIR_INDEX_SUFFIX}"
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def fair_index_sql(full_name, concurrently:)
|
|
121
|
+
"CREATE INDEX #{"CONCURRENTLY " if concurrently}IF NOT EXISTS #{fair_index_name(full_name)} " \
|
|
122
|
+
"ON pgmq.q_#{full_name} ((#{FAIR_INDEX_KEY_EXPR}), vt, msg_id)"
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def fair_read_sql(full_name)
|
|
126
|
+
table = "pgmq.q_#{full_name}"
|
|
127
|
+
<<~SQL
|
|
128
|
+
WITH RECURSIVE fair_keys AS (
|
|
129
|
+
(SELECT #{FAIR_KEY_EXPR} AS k
|
|
130
|
+
FROM #{table} m
|
|
131
|
+
WHERE m.vt <= now()
|
|
132
|
+
ORDER BY 1 LIMIT 1)
|
|
133
|
+
UNION ALL
|
|
134
|
+
SELECT (SELECT #{FAIR_KEY_EXPR}
|
|
135
|
+
FROM #{table} m
|
|
136
|
+
WHERE #{FAIR_KEY_EXPR} > fk.k
|
|
137
|
+
AND m.vt <= now()
|
|
138
|
+
ORDER BY 1 LIMIT 1)
|
|
139
|
+
FROM fair_keys fk
|
|
140
|
+
WHERE fk.k IS NOT NULL
|
|
141
|
+
),
|
|
142
|
+
candidates AS (
|
|
143
|
+
SELECT c.msg_id, c.rn, c.w
|
|
144
|
+
FROM fair_keys fk
|
|
145
|
+
CROSS JOIN LATERAL (
|
|
146
|
+
SELECT m.msg_id,
|
|
147
|
+
ROW_NUMBER() OVER (ORDER BY m.vt, m.msg_id) AS rn,
|
|
148
|
+
#{FAIR_WEIGHT_EXPR} AS w
|
|
149
|
+
FROM #{table} m
|
|
150
|
+
WHERE #{FAIR_KEY_EXPR} = fk.k
|
|
151
|
+
AND m.vt <= now()
|
|
152
|
+
ORDER BY m.vt, m.msg_id
|
|
153
|
+
LIMIT $1
|
|
154
|
+
) c
|
|
155
|
+
WHERE fk.k IS NOT NULL
|
|
156
|
+
),
|
|
157
|
+
picked AS (
|
|
158
|
+
SELECT msg_id, ROW_NUMBER() OVER (ORDER BY rn / w, msg_id) AS selection_order
|
|
159
|
+
FROM candidates
|
|
160
|
+
ORDER BY rn / w, msg_id
|
|
161
|
+
LIMIT $1
|
|
162
|
+
),
|
|
163
|
+
locked AS (
|
|
164
|
+
SELECT m.msg_id, p.selection_order
|
|
165
|
+
FROM #{table} m
|
|
166
|
+
JOIN picked p ON p.msg_id = m.msg_id
|
|
167
|
+
WHERE m.vt <= now()
|
|
168
|
+
FOR UPDATE OF m SKIP LOCKED
|
|
169
|
+
),
|
|
170
|
+
updated AS (
|
|
171
|
+
UPDATE #{table} m
|
|
172
|
+
SET vt = clock_timestamp() + make_interval(secs => $2),
|
|
173
|
+
read_ct = read_ct + 1,
|
|
174
|
+
last_read_at = clock_timestamp()
|
|
175
|
+
FROM locked l
|
|
176
|
+
WHERE m.msg_id = l.msg_id
|
|
177
|
+
RETURNING m.msg_id, m.read_ct, m.enqueued_at, m.last_read_at, m.vt, m.message, m.headers,
|
|
178
|
+
l.selection_order
|
|
179
|
+
)
|
|
180
|
+
SELECT msg_id, read_ct, enqueued_at, last_read_at, vt, message, headers
|
|
181
|
+
FROM updated
|
|
182
|
+
ORDER BY selection_order
|
|
183
|
+
SQL
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
data/lib/pgbus/client.rb
CHANGED
|
@@ -12,6 +12,7 @@ require_relative "client/resizable_pool"
|
|
|
12
12
|
module Pgbus
|
|
13
13
|
class Client
|
|
14
14
|
include ReadAfter
|
|
15
|
+
include FairRead
|
|
15
16
|
include EnsureStreamQueue
|
|
16
17
|
include NotifyStream
|
|
17
18
|
|
|
@@ -428,11 +429,17 @@ module Pgbus
|
|
|
428
429
|
# Non-priority fast path delegates to read_batch, which is already gated
|
|
429
430
|
# by the connection-health breaker — no extra guard needed here.
|
|
430
431
|
unless @queue_strategy.priority?
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
432
|
+
msgs = if FairShare.enabled?(config)
|
|
433
|
+
read_batch_fair(queue_name, qty: qty, vt: vt)
|
|
434
|
+
else
|
|
435
|
+
read_batch(queue_name, qty: qty, vt: vt)
|
|
436
|
+
end
|
|
437
|
+
return (msgs || []).map { |m| [config.queue_name(queue_name), m] }
|
|
434
438
|
end
|
|
435
439
|
|
|
440
|
+
# Fair share (issue #426): strict between levels, fair within a level.
|
|
441
|
+
return read_batch_prioritized_fair(queue_name, qty: qty, vt: vt) if FairShare.enabled?(config)
|
|
442
|
+
|
|
436
443
|
# The priority loop issues its own reads, so gate the whole loop: an open
|
|
437
444
|
# breaker fails fast before any sub-queue is touched, and the loop as a
|
|
438
445
|
# unit records one success/failure with the latch.
|
|
@@ -457,6 +464,25 @@ module Pgbus
|
|
|
457
464
|
end
|
|
458
465
|
end
|
|
459
466
|
|
|
467
|
+
# Priority sub-queues drained p0 → pN, each level read with the fair
|
|
468
|
+
# interleave. guarded_read wraps the loop as one unit (see above).
|
|
469
|
+
def read_batch_prioritized_fair(queue_name, qty:, vt: nil)
|
|
470
|
+
guarded_read do
|
|
471
|
+
remaining = qty
|
|
472
|
+
results = []
|
|
473
|
+
|
|
474
|
+
config.priority_queue_names(queue_name).each do |pq_name|
|
|
475
|
+
break if remaining <= 0
|
|
476
|
+
|
|
477
|
+
msgs = fair_read_step(QueueNameValidator.sanitize!(pq_name), remaining, vt || config.visibility_timeout)
|
|
478
|
+
msgs.each { |m| results << [pq_name, m] }
|
|
479
|
+
remaining -= msgs.size
|
|
480
|
+
end
|
|
481
|
+
|
|
482
|
+
results
|
|
483
|
+
end
|
|
484
|
+
end
|
|
485
|
+
|
|
460
486
|
def read_with_poll(queue_name, qty:, vt: nil, max_poll_seconds: 5, poll_interval_ms: 100)
|
|
461
487
|
full_name = config.queue_name(queue_name)
|
|
462
488
|
guarded_read do
|
|
@@ -1351,6 +1377,7 @@ module Pgbus
|
|
|
1351
1377
|
create_queue_table(full_name)
|
|
1352
1378
|
enable_notify_if_needed(full_name, NOTIFY_THROTTLE_MS)
|
|
1353
1379
|
create_fifo_index_if_needed(full_name)
|
|
1380
|
+
create_fair_index_if_needed(full_name)
|
|
1354
1381
|
end
|
|
1355
1382
|
end
|
|
1356
1383
|
|
data/lib/pgbus/configuration.rb
CHANGED
|
@@ -70,6 +70,29 @@ module Pgbus
|
|
|
70
70
|
# :round_robin = use read_grouped_rr (fair round-robin across groups).
|
|
71
71
|
attr_reader :group_mode
|
|
72
72
|
|
|
73
|
+
# Fair share scheduling across tenants (issue #426). nil = disabled.
|
|
74
|
+
# Any #call-able receiving the ActiveJob instance at enqueue time and
|
|
75
|
+
# returning nil (unkeyed), a key (String/Symbol/Integer), or [key, weight]
|
|
76
|
+
# (weight: positive Numeric, default 1). Keyed jobs are read with
|
|
77
|
+
# Client#read_batch_fair — a weighted, work-conserving interleave across
|
|
78
|
+
# keys within each queue. Mutually exclusive with group_mode.
|
|
79
|
+
attr_reader :fair_share
|
|
80
|
+
|
|
81
|
+
# Fair share for event-bus consumers (issue #427). nil = disabled. Any
|
|
82
|
+
# #call-able receiving the Pgbus::Event at publish time (routing_key,
|
|
83
|
+
# payload as passed to publish, headers) and returning nil, a key, or
|
|
84
|
+
# [key, weight] exactly like fair_share. The key rides in the event
|
|
85
|
+
# envelope and consumers read subscriber queues with
|
|
86
|
+
# Client#read_batch_fair. Independent of fair_share (jobs).
|
|
87
|
+
attr_reader :event_fair_share
|
|
88
|
+
|
|
89
|
+
# Persist ActiveSupport::CurrentAttributes across enqueue → perform
|
|
90
|
+
# (issue #430). nil = off (default). :auto = every CurrentAttributes
|
|
91
|
+
# subclass; an Array of classes/names; or a Hash of class/name =>
|
|
92
|
+
# { only: [...] } / { except: [...] }. Normalized to :auto or an Array of
|
|
93
|
+
# { name:, only:, except: } specs (classes are resolved lazily by name).
|
|
94
|
+
attr_reader :current_attributes
|
|
95
|
+
|
|
73
96
|
# Archive compaction. Only the user-facing retention window is configurable;
|
|
74
97
|
# the loop interval and batch size are tuned via constants on
|
|
75
98
|
# Pgbus::Process::Dispatcher.
|
|
@@ -268,6 +291,9 @@ module Pgbus
|
|
|
268
291
|
@priority_levels = nil
|
|
269
292
|
@default_priority = 1
|
|
270
293
|
@group_mode = nil
|
|
294
|
+
@fair_share = nil
|
|
295
|
+
@event_fair_share = nil
|
|
296
|
+
@current_attributes = nil
|
|
271
297
|
|
|
272
298
|
@archive_retention = 7 * 24 * 3600 # 7 days
|
|
273
299
|
@batch_retention = 7 * 24 * 3600 # 7 days
|
|
@@ -608,6 +634,28 @@ module Pgbus
|
|
|
608
634
|
@group_mode = coerced
|
|
609
635
|
end
|
|
610
636
|
|
|
637
|
+
def current_attributes=(value)
|
|
638
|
+
@current_attributes = Pgbus::CurrentAttributes.normalize(value)
|
|
639
|
+
end
|
|
640
|
+
|
|
641
|
+
def fair_share=(callable)
|
|
642
|
+
unless callable.nil? || callable.respond_to?(:call)
|
|
643
|
+
raise Pgbus::ConfigurationError,
|
|
644
|
+
"fair_share must be nil or respond to #call (got #{callable.class})"
|
|
645
|
+
end
|
|
646
|
+
|
|
647
|
+
@fair_share = callable
|
|
648
|
+
end
|
|
649
|
+
|
|
650
|
+
def event_fair_share=(callable)
|
|
651
|
+
unless callable.nil? || callable.respond_to?(:call)
|
|
652
|
+
raise Pgbus::ConfigurationError,
|
|
653
|
+
"event_fair_share must be nil or respond to #call (got #{callable.class})"
|
|
654
|
+
end
|
|
655
|
+
|
|
656
|
+
@event_fair_share = callable
|
|
657
|
+
end
|
|
658
|
+
|
|
611
659
|
VALID_CONNECTION_GUC_MODES = %i[options session].freeze
|
|
612
660
|
|
|
613
661
|
def connection_guc_mode=(mode)
|
|
@@ -783,6 +831,7 @@ module Pgbus
|
|
|
783
831
|
validate_job_path_gaps!
|
|
784
832
|
validate_streams!
|
|
785
833
|
validate_metrics_backend!
|
|
834
|
+
validate_fair_share!
|
|
786
835
|
|
|
787
836
|
self
|
|
788
837
|
end
|
|
@@ -863,6 +912,14 @@ module Pgbus
|
|
|
863
912
|
raise Pgbus::ConfigurationError, "connects_to must be a Hash or nil"
|
|
864
913
|
end
|
|
865
914
|
|
|
915
|
+
def validate_fair_share!
|
|
916
|
+
return unless fair_share && group_mode
|
|
917
|
+
|
|
918
|
+
raise Pgbus::ConfigurationError,
|
|
919
|
+
"fair_share and group_mode are mutually exclusive — fair_share interleaves across keys " \
|
|
920
|
+
"within a queue, group_mode serializes PGMQ FIFO groups; pick one"
|
|
921
|
+
end
|
|
922
|
+
|
|
866
923
|
def validate_streams!
|
|
867
924
|
unless streams_default_retention.is_a?(Numeric) && streams_default_retention >= 0
|
|
868
925
|
raise Pgbus::ConfigurationError, "streams_default_retention must be a non-negative number"
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
# Persists ActiveSupport::CurrentAttributes across enqueue → perform
|
|
5
|
+
# (issue #430).
|
|
6
|
+
#
|
|
7
|
+
# `capture` snapshots the assigned attributes of every persisted class
|
|
8
|
+
# (config.current_attributes: :auto, an explicit list, or per-class
|
|
9
|
+
# only:/except: filters) serialized through ActiveJob::Arguments — so
|
|
10
|
+
# GlobalID models, Symbols, Times round-trip like job arguments and fall
|
|
11
|
+
# under the allowed_global_id_models allowlist on the way back. `restore`
|
|
12
|
+
# nests `klass.set(attrs)` for the duration of a block.
|
|
13
|
+
#
|
|
14
|
+
# The ActiveJob side (Pgbus::ActiveJob::CurrentAttributes) calls capture
|
|
15
|
+
# from `serialize` and restore around `perform_now`, so the hop works under
|
|
16
|
+
# the pgbus worker and under Rails' :test / :inline adapters alike.
|
|
17
|
+
module CurrentAttributes
|
|
18
|
+
METADATA_KEY = "pgbus_current"
|
|
19
|
+
|
|
20
|
+
@missing_warned = Concurrent::Map.new
|
|
21
|
+
|
|
22
|
+
class << self
|
|
23
|
+
def enabled?(config = Pgbus.configuration)
|
|
24
|
+
!config.current_attributes.nil?
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# { "Current" => serialized_attrs, ... } or nil when there is nothing
|
|
28
|
+
# to persist. `override` is a per-job-class spec (false disables; an
|
|
29
|
+
# Array/Hash replaces the config's list for this job).
|
|
30
|
+
def capture(config = Pgbus.configuration, override: nil)
|
|
31
|
+
return nil if override == false
|
|
32
|
+
return nil unless override || enabled?(config)
|
|
33
|
+
|
|
34
|
+
captured = {}
|
|
35
|
+
persisted_specs(config, override: override).each do |spec|
|
|
36
|
+
klass = resolve_class(spec[:name]) or next
|
|
37
|
+
attrs = filter_attrs(klass.attributes, spec)
|
|
38
|
+
next if attrs.empty?
|
|
39
|
+
|
|
40
|
+
captured[klass.name] = serialize_attrs(klass, attrs)
|
|
41
|
+
end
|
|
42
|
+
captured.empty? ? nil : captured
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Set every stored class's attributes for the block; previous values
|
|
46
|
+
# come back afterwards (ActiveSupport::CurrentAttributes#set semantics).
|
|
47
|
+
def restore(stored, &block)
|
|
48
|
+
return yield if stored.nil? || stored.empty?
|
|
49
|
+
|
|
50
|
+
stored.reduce(block) do |inner, (name, attrs)|
|
|
51
|
+
klass = resolve_class(name)
|
|
52
|
+
next inner unless klass
|
|
53
|
+
|
|
54
|
+
values = deserialize_attrs(klass, attrs)
|
|
55
|
+
-> { klass.set(values) { inner.call } }
|
|
56
|
+
end.call
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Config/override value → nil | :auto | [{ name:, only:, except: }].
|
|
60
|
+
# Raises Pgbus::ConfigurationError for anything else.
|
|
61
|
+
def normalize(value)
|
|
62
|
+
case value
|
|
63
|
+
when nil then nil
|
|
64
|
+
when :auto then :auto
|
|
65
|
+
when Array then value.map { |entry| class_spec(entry, {}) }
|
|
66
|
+
when Hash then value.map { |entry, filters| class_spec(entry, filters) }
|
|
67
|
+
else
|
|
68
|
+
raise Pgbus::ConfigurationError,
|
|
69
|
+
"current_attributes must be nil, :auto, an Array of CurrentAttributes classes/names, " \
|
|
70
|
+
"or a Hash of class/name => { only: [...] } | { except: [...] } (got #{value.inspect})"
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Normalized [{ name:, only:, except: }] for the config (or override).
|
|
75
|
+
def persisted_specs(config = Pgbus.configuration, override: nil)
|
|
76
|
+
source = override.nil? ? config.current_attributes : normalize(override)
|
|
77
|
+
return [] if source.nil? || source == false
|
|
78
|
+
return source unless source == :auto
|
|
79
|
+
|
|
80
|
+
ActiveSupport::CurrentAttributes.descendants.filter_map do |klass|
|
|
81
|
+
{ name: klass.name, only: nil, except: nil } if klass.name
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
|
|
87
|
+
def class_spec(entry, filters)
|
|
88
|
+
name = case entry
|
|
89
|
+
when Class, String then entry.to_s
|
|
90
|
+
else
|
|
91
|
+
raise Pgbus::ConfigurationError,
|
|
92
|
+
"current_attributes entries must be classes or class names (got #{entry.inspect})"
|
|
93
|
+
end
|
|
94
|
+
unless filters.is_a?(Hash) && (filters.keys - %i[only except]).empty? && filters.size <= 1
|
|
95
|
+
raise Pgbus::ConfigurationError,
|
|
96
|
+
"current_attributes filters for #{name} must be { only: [...] } or { except: [...] } (got #{filters.inspect})"
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
{ name: name, only: filter_list(name, filters[:only], :only), except: filter_list(name, filters[:except], :except) }
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def filter_list(name, list, kind)
|
|
103
|
+
return nil if list.nil?
|
|
104
|
+
unless list.is_a?(Array) && list.all? { |a| a.is_a?(Symbol) || a.is_a?(String) }
|
|
105
|
+
raise Pgbus::ConfigurationError,
|
|
106
|
+
"current_attributes #{kind}: for #{name} must be an Array of attribute names (got #{list.inspect})"
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
list.map(&:to_sym)
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def resolve_class(name)
|
|
113
|
+
name.constantize
|
|
114
|
+
rescue NameError
|
|
115
|
+
@missing_warned.compute_if_absent(name) do
|
|
116
|
+
Pgbus.logger.warn { "[Pgbus] current_attributes: #{name} is not defined — skipping" }
|
|
117
|
+
true
|
|
118
|
+
end
|
|
119
|
+
nil
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def filter_attrs(attrs, spec)
|
|
123
|
+
attrs = attrs.compact
|
|
124
|
+
attrs = attrs.slice(*spec[:only]) if spec[:only]
|
|
125
|
+
attrs = attrs.except(*spec[:except]) if spec[:except]
|
|
126
|
+
attrs
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def serialize_attrs(klass, attrs)
|
|
130
|
+
::ActiveJob::Arguments.serialize([attrs]).first
|
|
131
|
+
rescue ::ActiveJob::SerializationError, URI::Error
|
|
132
|
+
culprit, value = attrs.find do |_name, v|
|
|
133
|
+
::ActiveJob::Arguments.serialize([v])
|
|
134
|
+
false
|
|
135
|
+
rescue ::ActiveJob::SerializationError, URI::Error
|
|
136
|
+
true
|
|
137
|
+
end
|
|
138
|
+
raise Pgbus::CurrentAttributesError,
|
|
139
|
+
"#{klass.name}##{culprit} (#{value.class}) cannot be serialized for job persistence — " \
|
|
140
|
+
"make it GlobalID/JSON-serializable or exclude it: " \
|
|
141
|
+
"config.current_attributes = { #{klass.name} => { except: [:#{culprit}] } }"
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def deserialize_attrs(klass, attrs)
|
|
145
|
+
instance = klass.instance
|
|
146
|
+
known, unknown = attrs.partition { |name, _| name.start_with?("_aj_") || instance.respond_to?("#{name}=") }
|
|
147
|
+
unless unknown.empty?
|
|
148
|
+
Pgbus.logger.debug do
|
|
149
|
+
"[Pgbus] current_attributes: #{klass.name} no longer defines #{unknown.map(&:first).join(", ")} — dropped"
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
::ActiveJob::Arguments.deserialize([known.to_h]).first
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
end
|
data/lib/pgbus/engine.rb
CHANGED
data/lib/pgbus/event.rb
CHANGED
|
@@ -4,14 +4,19 @@ require "time"
|
|
|
4
4
|
|
|
5
5
|
module Pgbus
|
|
6
6
|
class Event
|
|
7
|
-
|
|
7
|
+
# +context+ is the publisher's persisted ActiveSupport::CurrentAttributes
|
|
8
|
+
# (issue #431) in their stored form — { "Current" => serialized attrs } —
|
|
9
|
+
# or nil. Handlers normally just read +Current+ (it is restored around
|
|
10
|
+
# +handle+); the raw form is here for tests and explicit access.
|
|
11
|
+
attr_reader :event_id, :payload, :published_at, :routing_key, :headers, :context
|
|
8
12
|
|
|
9
|
-
def initialize(event_id:, payload:, published_at: nil, routing_key: nil, headers: nil)
|
|
13
|
+
def initialize(event_id:, payload:, published_at: nil, routing_key: nil, headers: nil, context: nil)
|
|
10
14
|
@event_id = event_id
|
|
11
15
|
@payload = payload
|
|
12
16
|
@published_at = published_at || Time.now.utc
|
|
13
17
|
@routing_key = routing_key
|
|
14
18
|
@headers = headers || {}
|
|
19
|
+
@context = context
|
|
15
20
|
end
|
|
16
21
|
|
|
17
22
|
def [](key)
|
|
@@ -43,7 +43,10 @@ module Pgbus
|
|
|
43
43
|
msg_id: message.msg_id.to_i
|
|
44
44
|
}
|
|
45
45
|
Instrumentation.instrument("pgbus.event_processed", instrument_payload) do
|
|
46
|
-
|
|
46
|
+
# Publisher's Current attributes (issue #431) are set for the handler
|
|
47
|
+
# and reverted after (CurrentAttributes#set semantics); the Rails
|
|
48
|
+
# executor wrap above additionally resets at completion.
|
|
49
|
+
Pgbus::CurrentAttributes.restore(event.context) { handle(event) }
|
|
47
50
|
end
|
|
48
51
|
complete_claim!(event.event_id) if self.class.idempotent?
|
|
49
52
|
:handled
|
|
@@ -88,10 +91,17 @@ module Pgbus
|
|
|
88
91
|
payload = raw["payload"]
|
|
89
92
|
payload = Serializer.locate_global_id(payload["_global_id"]) if payload.is_a?(Hash) && payload["_global_id"]
|
|
90
93
|
|
|
94
|
+
# Same allowlist boundary as Serializer.deserialize_job_data: every
|
|
95
|
+
# _aj_globalid in the persisted context is checked BEFORE anything is
|
|
96
|
+
# located, so a crafted envelope cannot load an arbitrary model.
|
|
97
|
+
context = raw[Pgbus::CurrentAttributes::METADATA_KEY]
|
|
98
|
+
Serializer.assert_job_global_ids_allowed!(context) if context
|
|
99
|
+
|
|
91
100
|
Event.new(
|
|
92
101
|
event_id: raw["event_id"],
|
|
93
102
|
payload: payload,
|
|
94
|
-
published_at: raw["published_at"] ? Time.parse(raw["published_at"]) : nil
|
|
103
|
+
published_at: raw["published_at"] ? Time.parse(raw["published_at"]) : nil,
|
|
104
|
+
context: context
|
|
95
105
|
)
|
|
96
106
|
end
|
|
97
107
|
|
|
@@ -9,6 +9,8 @@ module Pgbus
|
|
|
9
9
|
|
|
10
10
|
def publish(routing_key, payload, headers: nil, delay: 0)
|
|
11
11
|
event_data = build_event_data(payload, routing_key: routing_key)
|
|
12
|
+
event_data = tag_fair_share(event_data, payload, routing_key: routing_key, headers: headers)
|
|
13
|
+
event_data = tag_current(event_data)
|
|
12
14
|
|
|
13
15
|
if defined?(Pgbus::Testing) && !Pgbus::Testing.disabled?
|
|
14
16
|
event = Pgbus::Event.new(
|
|
@@ -16,14 +18,15 @@ module Pgbus
|
|
|
16
18
|
payload: event_data["payload"],
|
|
17
19
|
published_at: event_data["published_at"] ? Time.parse(event_data["published_at"]) : nil,
|
|
18
20
|
routing_key: routing_key,
|
|
19
|
-
headers: headers
|
|
21
|
+
headers: headers,
|
|
22
|
+
context: event_data[Pgbus::CurrentAttributes::METADATA_KEY]
|
|
20
23
|
)
|
|
21
24
|
|
|
22
25
|
Pgbus::Testing.store.push_event(event)
|
|
23
26
|
|
|
24
27
|
if Pgbus::Testing.inline? && delay.to_i <= 0
|
|
25
28
|
Pgbus::EventBus::Registry.instance.handlers_for(routing_key).each do |subscriber|
|
|
26
|
-
subscriber.handler_class.new.handle(event)
|
|
29
|
+
Pgbus::CurrentAttributes.restore(event.context) { subscriber.handler_class.new.handle(event) }
|
|
27
30
|
end
|
|
28
31
|
end
|
|
29
32
|
|
|
@@ -37,6 +40,38 @@ module Pgbus
|
|
|
37
40
|
publish(routing_key, payload, headers: headers, delay: delay)
|
|
38
41
|
end
|
|
39
42
|
|
|
43
|
+
# Fair share for consumers (issue #427): when config.event_fair_share is
|
|
44
|
+
# set, hand the callable a Pgbus::Event (routing key, the payload object
|
|
45
|
+
# as passed to publish — not its serialized form — and headers) and merge
|
|
46
|
+
# the resolved key/weight into the envelope. Shared with Outbox.publish_event
|
|
47
|
+
# so the key is resolved where the publisher's context (Current.*) exists
|
|
48
|
+
# and rides the outbox row to the bus. Returns event_data itself when off.
|
|
49
|
+
def tag_fair_share(event_data, payload, routing_key:, headers:)
|
|
50
|
+
return event_data unless FairShare.event_enabled?
|
|
51
|
+
|
|
52
|
+
event = Pgbus::Event.new(
|
|
53
|
+
event_id: event_data["event_id"],
|
|
54
|
+
payload: payload,
|
|
55
|
+
routing_key: routing_key,
|
|
56
|
+
headers: headers
|
|
57
|
+
)
|
|
58
|
+
FairShare.inject_event_metadata(event, event_data)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Current attributes publish → handler (issue #431): when
|
|
62
|
+
# config.current_attributes is set, snapshot the publisher's persisted
|
|
63
|
+
# Current classes (Pgbus::CurrentAttributes.capture — same filters,
|
|
64
|
+
# serialization and allowlist gating as jobs) into the envelope under
|
|
65
|
+
# the same +pgbus_current+ key jobs use. Shared with Outbox.publish_event
|
|
66
|
+
# so the capture happens where Current is set, not at relay time.
|
|
67
|
+
# Returns event_data itself when off or nothing is assigned.
|
|
68
|
+
def tag_current(event_data)
|
|
69
|
+
captured = Pgbus::CurrentAttributes.capture
|
|
70
|
+
return event_data unless captured
|
|
71
|
+
|
|
72
|
+
event_data.merge(Pgbus::CurrentAttributes::METADATA_KEY => captured)
|
|
73
|
+
end
|
|
74
|
+
|
|
40
75
|
def build_event_data(payload, routing_key: nil)
|
|
41
76
|
event_id = SecureRandom.uuid
|
|
42
77
|
|
|
@@ -13,6 +13,10 @@ module Pgbus
|
|
|
13
13
|
|
|
14
14
|
def setup!
|
|
15
15
|
Pgbus.client.ensure_queue(queue_name)
|
|
16
|
+
# Subscriber queues are created here, not by a worker, so this is their
|
|
17
|
+
# creation hook for the fair-share index (issue #427). Idempotent and
|
|
18
|
+
# memoized per process; instant on a fresh table, CONCURRENTLY otherwise.
|
|
19
|
+
Pgbus.client.ensure_fair_index(queue_name) if FairShare.event_enabled?
|
|
16
20
|
Pgbus.client.bind_topic(pattern, queue_name)
|
|
17
21
|
end
|
|
18
22
|
|