pgbus 0.11.4 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/lib/generators/pgbus/templates/upgrade_pgmq.rb.erb +7 -0
- data/lib/pgbus/client/resizable_pool.rb +9 -0
- data/lib/pgbus/client.rb +77 -14
- data/lib/pgbus/concurrency.rb +14 -11
- data/lib/pgbus/configuration.rb +44 -0
- data/lib/pgbus/mcp/health_analyzer.rb +11 -5
- data/lib/pgbus/pgmq_schema/pgmq_v1.12.0.sql +2166 -0
- data/lib/pgbus/pgmq_schema.rb +40 -0
- data/lib/pgbus/recurring/schedule.rb +32 -11
- data/lib/pgbus/support.rb +22 -0
- data/lib/pgbus/uniqueness.rb +18 -7
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +13 -0
- metadata +5 -3
data/lib/pgbus/pgmq_schema.rb
CHANGED
|
@@ -135,6 +135,46 @@ module Pgbus
|
|
|
135
135
|
SQL
|
|
136
136
|
end
|
|
137
137
|
|
|
138
|
+
# SQL to re-install the per-queue NOTIFY insert triggers that
|
|
139
|
+
# drop_pgmq_functions_sql cascades away (issue #360). Dropping
|
|
140
|
+
# pgmq.notify_queue_listeners() with CASCADE also drops the
|
|
141
|
+
# trigger_notify_queue_insert_listeners trigger from every queue table;
|
|
142
|
+
# install_sql re-creates the function but nothing re-creates the
|
|
143
|
+
# triggers, so NOTIFY-gated wakeups silently die fleet-wide until each
|
|
144
|
+
# queue happens to be re-ensured by a process restart.
|
|
145
|
+
#
|
|
146
|
+
# The dropped state is fully recoverable: pgmq.notify_insert_throttle is
|
|
147
|
+
# a TABLE (preserved by the upgrade — its rows record exactly which
|
|
148
|
+
# queues had notify enabled and at what throttle, FK-bound to pgmq.meta
|
|
149
|
+
# so it can't reference a dropped queue), and pgmq.enable_notify_insert
|
|
150
|
+
# is idempotent. Replaying it per recorded row restores every trigger at
|
|
151
|
+
# its original interval. No-ops when the throttle table or the enable
|
|
152
|
+
# function is absent (a vendored version without the notify feature).
|
|
153
|
+
def reinstall_notify_triggers_sql
|
|
154
|
+
<<~SQL
|
|
155
|
+
DO $$
|
|
156
|
+
DECLARE
|
|
157
|
+
r RECORD;
|
|
158
|
+
BEGIN
|
|
159
|
+
IF to_regclass('pgmq.notify_insert_throttle') IS NULL THEN
|
|
160
|
+
RETURN;
|
|
161
|
+
END IF;
|
|
162
|
+
IF to_regprocedure('pgmq.enable_notify_insert(text, integer)') IS NULL THEN
|
|
163
|
+
RETURN;
|
|
164
|
+
END IF;
|
|
165
|
+
|
|
166
|
+
-- The FOR loop iterates a snapshot, so enable_notify_insert's
|
|
167
|
+
-- internal DELETE + re-INSERT of the same throttle row is safe.
|
|
168
|
+
FOR r IN
|
|
169
|
+
SELECT queue_name, throttle_interval_ms
|
|
170
|
+
FROM pgmq.notify_insert_throttle
|
|
171
|
+
LOOP
|
|
172
|
+
PERFORM pgmq.enable_notify_insert(r.queue_name, r.throttle_interval_ms);
|
|
173
|
+
END LOOP;
|
|
174
|
+
END $$;
|
|
175
|
+
SQL
|
|
176
|
+
end
|
|
177
|
+
|
|
138
178
|
private
|
|
139
179
|
|
|
140
180
|
# Strips extension-specific blocks (pg_extension_config_dump, pg_depend checks)
|
|
@@ -139,7 +139,7 @@ module Pgbus
|
|
|
139
139
|
return nil unless config
|
|
140
140
|
return nil unless config[:strategy] == :until_executed
|
|
141
141
|
|
|
142
|
-
key = resolve_uniqueness_key(config, task)
|
|
142
|
+
key = resolve_uniqueness_key(job_class, config, task)
|
|
143
143
|
return nil unless key
|
|
144
144
|
|
|
145
145
|
acquired = UniquenessKey.acquire!(key, queue_name: resolve_queue(task), msg_id: 0)
|
|
@@ -155,6 +155,7 @@ module Pgbus
|
|
|
155
155
|
"[Pgbus] Uniqueness lock check failed for #{task.key}: #{e.class}: #{e.message} — " \
|
|
156
156
|
"skipping enqueue to prevent duplicates"
|
|
157
157
|
end
|
|
158
|
+
ErrorReporter.report(e, { action: "recurring_uniqueness_lock", task: task.key })
|
|
158
159
|
:already_locked # Fail closed — skip enqueue when lock check errors
|
|
159
160
|
end
|
|
160
161
|
|
|
@@ -168,19 +169,39 @@ module Pgbus
|
|
|
168
169
|
end
|
|
169
170
|
|
|
170
171
|
# Resolve the uniqueness key for a recurring task.
|
|
171
|
-
#
|
|
172
|
-
|
|
172
|
+
# With no explicit key: the key is the SCHEDULED job class's name —
|
|
173
|
+
# resolved here, not via a stored proc, so a declaration inherited from
|
|
174
|
+
# a base class keys each subclass separately (issue #357) — qualified
|
|
175
|
+
# with the task's arguments so two recurring tasks pointing at the same
|
|
176
|
+
# job class with different args: don't share one lock (the scheduler
|
|
177
|
+
# can't use #333's raise-at-enqueue guard, so it disambiguates instead).
|
|
178
|
+
#
|
|
179
|
+
# Every failure here — default path or user-supplied key proc — must
|
|
180
|
+
# propagate to acquire_uniqueness_lock's fail-closed rescue (skip this
|
|
181
|
+
# tick, log, report). Returning nil instead would mean "no key
|
|
182
|
+
# configured", and the task would be enqueued WITHOUT a lock: a broken
|
|
183
|
+
# key proc silently disabling the very protection it configures. The
|
|
184
|
+
# warn adds the failing-proc context before re-raising.
|
|
185
|
+
def resolve_uniqueness_key(job_class, config, task)
|
|
173
186
|
key_proc = config[:key]
|
|
174
187
|
args = task.arguments || []
|
|
188
|
+
return default_uniqueness_key(job_class, args) unless key_proc
|
|
175
189
|
|
|
176
|
-
|
|
177
|
-
key_proc.call
|
|
178
|
-
|
|
179
|
-
|
|
190
|
+
begin
|
|
191
|
+
args.empty? ? key_proc.call : key_proc.call(*args)
|
|
192
|
+
rescue StandardError => e
|
|
193
|
+
Pgbus.logger.warn { "[Pgbus] Could not resolve uniqueness key for #{task.key}: #{e.message}" }
|
|
194
|
+
raise
|
|
180
195
|
end
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# Class-name default for the scheduler path, args-qualified so distinct
|
|
199
|
+
# argument sets get distinct locks. JSON keeps the key deterministic
|
|
200
|
+
# and readable in the dashboard/locks table.
|
|
201
|
+
def default_uniqueness_key(job_class, args)
|
|
202
|
+
return job_class.name if args.empty?
|
|
203
|
+
|
|
204
|
+
"#{job_class.name}:#{JSON.generate(args)}"
|
|
184
205
|
end
|
|
185
206
|
|
|
186
207
|
# Inject uniqueness metadata into the payload so the executor releases
|
|
@@ -197,7 +218,7 @@ module Pgbus
|
|
|
197
218
|
return payload unless config
|
|
198
219
|
return payload unless config[:strategy] == :until_executed
|
|
199
220
|
|
|
200
|
-
key = resolve_uniqueness_key(config, task)
|
|
221
|
+
key = resolve_uniqueness_key(job_class, config, task)
|
|
201
222
|
return payload unless key
|
|
202
223
|
|
|
203
224
|
payload.merge(
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
# Shared internal helpers.
|
|
5
|
+
module Support
|
|
6
|
+
module_function
|
|
7
|
+
|
|
8
|
+
# Call a user-supplied key proc with a job's arguments, preserving
|
|
9
|
+
# ActiveJob's keyword-argument convention: a trailing Hash whose keys are
|
|
10
|
+
# all Symbols is splatted as keywords; everything else is positional.
|
|
11
|
+
# Single implementation for Uniqueness.resolve_key and
|
|
12
|
+
# Concurrency.resolve_key so the dispatch semantics cannot drift apart.
|
|
13
|
+
def call_key_proc(key_proc, args)
|
|
14
|
+
last = args.last
|
|
15
|
+
if last.is_a?(Hash) && last.each_key.all?(Symbol)
|
|
16
|
+
key_proc.call(*args[...-1], **last)
|
|
17
|
+
else
|
|
18
|
+
key_proc.call(*args)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
data/lib/pgbus/uniqueness.rb
CHANGED
|
@@ -60,7 +60,7 @@ module Pgbus
|
|
|
60
60
|
raise ArgumentError, "unknown keyword: #{opts.keys.first.inspect}" if opts.any?
|
|
61
61
|
raise ArgumentError, "strategy must be one of: #{VALID_STRATEGIES.join(", ")}" unless VALID_STRATEGIES.include?(strategy)
|
|
62
62
|
raise ArgumentError, "on_conflict must be one of: #{VALID_CONFLICTS.join(", ")}" unless VALID_CONFLICTS.include?(on_conflict)
|
|
63
|
-
raise ArgumentError, "key must be callable (Proc or lambda)" if key && !key.respond_to?(:call)
|
|
63
|
+
raise ArgumentError, "key must be callable (Proc or lambda)" if !key.nil? && !key.respond_to?(:call)
|
|
64
64
|
|
|
65
65
|
# Record whether an explicit key was given. With NO explicit key the key
|
|
66
66
|
# defaults to the class name; that is safe for a no-argument job (one
|
|
@@ -70,16 +70,26 @@ module Pgbus
|
|
|
70
70
|
# order into ONE per-class singleton. resolve_key raises at resolve time
|
|
71
71
|
# when the class-name default meets non-empty arguments (see #333); the
|
|
72
72
|
# no-arg case keeps working. explicit_key marks which is which.
|
|
73
|
+
#
|
|
74
|
+
# No proc is stored for the default — resolve_key derives it from the
|
|
75
|
+
# ENQUEUED job's class. A stored `->(*) { name }` would capture the
|
|
76
|
+
# DECLARING class, so a base-class declaration would collapse every
|
|
77
|
+
# subclass into one shared key once configs became inheritable (#357).
|
|
73
78
|
@pgbus_uniqueness = {
|
|
74
79
|
strategy: strategy,
|
|
75
|
-
key: key
|
|
80
|
+
key: key,
|
|
76
81
|
explicit_key: !key.nil?,
|
|
77
82
|
on_conflict: on_conflict
|
|
78
83
|
}.freeze
|
|
79
84
|
end
|
|
80
85
|
|
|
86
|
+
# The nearest declaration in the ancestor chain wins: a class's own
|
|
87
|
+
# `ensures_uniqueness` beats an inherited one, and a base-class
|
|
88
|
+
# declaration reaches every subclass (issue #357 — class-level ivars
|
|
89
|
+
# don't inherit, so without the superclass walk a base declaration was
|
|
90
|
+
# silently inert for subclasses).
|
|
81
91
|
def pgbus_uniqueness
|
|
82
|
-
@pgbus_uniqueness
|
|
92
|
+
@pgbus_uniqueness || (superclass.pgbus_uniqueness if superclass.respond_to?(:pgbus_uniqueness))
|
|
83
93
|
end
|
|
84
94
|
end
|
|
85
95
|
|
|
@@ -90,11 +100,12 @@ module Pgbus
|
|
|
90
100
|
|
|
91
101
|
args = active_job.arguments
|
|
92
102
|
guard_class_name_default!(active_job, config, args)
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
config[:key].call(*args[...-1], **last)
|
|
103
|
+
key = if config[:explicit_key]
|
|
104
|
+
Support.call_key_proc(config[:key], args)
|
|
96
105
|
else
|
|
97
|
-
|
|
106
|
+
# Class-name default, resolved from the ENQUEUED job's class so
|
|
107
|
+
# an inherited declaration keys each subclass separately (#357).
|
|
108
|
+
active_job.class.name
|
|
98
109
|
end
|
|
99
110
|
|
|
100
111
|
# Automatically serialize GlobalID-compatible objects (e.g. ActiveRecord models)
|
data/lib/pgbus/version.rb
CHANGED
|
@@ -65,6 +65,19 @@ module Pgbus
|
|
|
65
65
|
end
|
|
66
66
|
private :fetch_queues_with_metrics
|
|
67
67
|
|
|
68
|
+
# Physical queue names registered as stream queues (the
|
|
69
|
+
# pgbus_stream_queues registry). Stream delivery is a non-consuming
|
|
70
|
+
# peek, so stream messages sit visible with read_ct=0 forever — health
|
|
71
|
+
# verdicts must not read that as a wedge (issue #359). Loaded fresh per
|
|
72
|
+
# call (reset + one query): verdicts run on coarse intervals and a
|
|
73
|
+
# long-lived process must see streams registered since the last check.
|
|
74
|
+
# Degrades to an empty Set when the registry table is absent
|
|
75
|
+
# (pre-migration installs) or unreadable — StreamQueue swallows both.
|
|
76
|
+
def stream_queue_names
|
|
77
|
+
StreamQueue.reset_cache!
|
|
78
|
+
StreamQueue.all_names
|
|
79
|
+
end
|
|
80
|
+
|
|
68
81
|
# name is the full PGMQ queue name (e.g. "pgbus_default") as returned
|
|
69
82
|
# by queues_with_metrics. No prefix is added.
|
|
70
83
|
def queue_detail(name)
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: pgbus
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.12.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Mikael Henriksson
|
|
@@ -63,14 +63,14 @@ dependencies:
|
|
|
63
63
|
requirements:
|
|
64
64
|
- - "~>"
|
|
65
65
|
- !ruby/object:Gem::Version
|
|
66
|
-
version: 0.7.
|
|
66
|
+
version: 0.7.1
|
|
67
67
|
type: :runtime
|
|
68
68
|
prerelease: false
|
|
69
69
|
version_requirements: !ruby/object:Gem::Requirement
|
|
70
70
|
requirements:
|
|
71
71
|
- - "~>"
|
|
72
72
|
- !ruby/object:Gem::Version
|
|
73
|
-
version: 0.7.
|
|
73
|
+
version: 0.7.1
|
|
74
74
|
- !ruby/object:Gem::Dependency
|
|
75
75
|
name: railties
|
|
76
76
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -321,6 +321,7 @@ files:
|
|
|
321
321
|
- lib/pgbus/pgmq_schema.rb
|
|
322
322
|
- lib/pgbus/pgmq_schema/pgmq_v1.11.0.sql
|
|
323
323
|
- lib/pgbus/pgmq_schema/pgmq_v1.11.1.sql
|
|
324
|
+
- lib/pgbus/pgmq_schema/pgmq_v1.12.0.sql
|
|
324
325
|
- lib/pgbus/process/consumer.rb
|
|
325
326
|
- lib/pgbus/process/consumer_priority.rb
|
|
326
327
|
- lib/pgbus/process/dispatcher.rb
|
|
@@ -364,6 +365,7 @@ files:
|
|
|
364
365
|
- lib/pgbus/streams/turbo_broadcastable.rb
|
|
365
366
|
- lib/pgbus/streams/turbo_stream_override.rb
|
|
366
367
|
- lib/pgbus/streams/watermark_cache_middleware.rb
|
|
368
|
+
- lib/pgbus/support.rb
|
|
367
369
|
- lib/pgbus/table_maintenance.rb
|
|
368
370
|
- lib/pgbus/testing.rb
|
|
369
371
|
- lib/pgbus/testing/assertions.rb
|