pgbus 0.18.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +27 -0
- data/README.md +5 -0
- data/app/controllers/pgbus/frontends_controller.rb +10 -8
- data/app/helpers/pgbus/application_helper.rb +4 -3
- data/app/models/pgbus/job_stat.rb +4 -0
- data/app/models/pgbus/processed_event.rb +3 -1
- data/app/models/pgbus/stream_queue.rb +34 -3
- data/app/models/pgbus/stream_stat.rb +3 -0
- data/lib/pgbus/active_job/current_attributes.rb +3 -1
- data/lib/pgbus/batch.rb +17 -9
- data/lib/pgbus/cli.rb +3 -0
- data/lib/pgbus/client/read_after.rb +10 -1
- data/lib/pgbus/client.rb +54 -9
- data/lib/pgbus/concurrency.rb +3 -0
- data/lib/pgbus/configuration.rb +84 -9
- data/lib/pgbus/current_attributes.rb +3 -1
- data/lib/pgbus/dedup_cache.rb +15 -7
- data/lib/pgbus/engine.rb +11 -0
- data/lib/pgbus/event_bus/handler.rb +16 -4
- data/lib/pgbus/execution_pools/async_pool.rb +23 -2
- data/lib/pgbus/execution_pools/thread_pool.rb +22 -0
- data/lib/pgbus/execution_pools.rb +26 -0
- data/lib/pgbus/integrations/appsignal/probe.rb +17 -9
- data/lib/pgbus/integrations/appsignal/subscriber.rb +30 -18
- data/lib/pgbus/integrations/appsignal.rb +26 -16
- data/lib/pgbus/mcp/rack_app.rb +1 -1
- data/lib/pgbus/metrics/subscriber.rb +35 -21
- data/lib/pgbus/process/consumer.rb +1 -1
- data/lib/pgbus/process/consumer_priority.rb +1 -1
- data/lib/pgbus/process/lifecycle.rb +5 -5
- data/lib/pgbus/process/notify_listener.rb +32 -18
- data/lib/pgbus/process/wake_pipe.rb +4 -1
- data/lib/pgbus/process/worker.rb +4 -2
- data/lib/pgbus/retry_backoff.rb +3 -0
- data/lib/pgbus/shareable.rb +70 -0
- data/lib/pgbus/streams/broadcastable_override.rb +3 -0
- data/lib/pgbus/streams/presence.rb +6 -7
- data/lib/pgbus/streams.rb +38 -27
- data/lib/pgbus/testing.rb +19 -3
- data/lib/pgbus/uniqueness.rb +3 -0
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/visibility_heartbeat.rb +51 -34
- data/lib/pgbus/web/authentication.rb +19 -2
- data/lib/pgbus/web/data_source/list_counts.rb +8 -6
- data/lib/pgbus/web/health_server.rb +4 -1
- data/lib/pgbus/web/process_activity.rb +1 -1
- data/lib/pgbus/web/streamer/heartbeat.rb +4 -1
- data/lib/pgbus/web/streamer/hub_client.rb +4 -1
- data/lib/pgbus/web/streamer/listener.rb +10 -1
- data/lib/pgbus/web/streamer/master_hub.rb +11 -5
- data/lib/pgbus/web/streamer/master_hub_boot.rb +3 -1
- data/lib/pgbus/web/streamer/outbound_pump.rb +4 -1
- data/lib/pgbus/web/streamer/peer_close_watcher.rb +4 -1
- data/lib/pgbus/web/streamer/stream_event_dispatcher.rb +13 -1
- data/lib/pgbus.rb +92 -30
- data/sig/pgbus/active_job/adapter.rbs +57 -0
- data/sig/pgbus/client.rbs +78 -0
- data/sig/pgbus/configuration.rbs +316 -0
- data/sig/pgbus/streams/stream.rbs +25 -0
- data/sig/pgbus/streams.rbs +22 -0
- data/sig/pgbus.rbs +73 -0
- metadata +8 -1
|
@@ -9,8 +9,25 @@ module Pgbus
|
|
|
9
9
|
before_action :authenticate_pgbus!
|
|
10
10
|
end
|
|
11
11
|
|
|
12
|
+
# The "dashboard has no authentication" warning is logged once per
|
|
13
|
+
# process. Request threads check and set the flag under @warn_lock, so
|
|
14
|
+
# concurrent first requests cannot both log it (issue #527).
|
|
15
|
+
@warn_lock = Mutex.new
|
|
16
|
+
@auth_warned = false
|
|
17
|
+
|
|
12
18
|
class << self
|
|
13
|
-
|
|
19
|
+
def auth_warned
|
|
20
|
+
@warn_lock.synchronize { @auth_warned }
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def auth_warned=(value)
|
|
24
|
+
@warn_lock.synchronize { @auth_warned = value }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# True for the one caller that flips the flag from false to true.
|
|
28
|
+
def claim_auth_warning!
|
|
29
|
+
@warn_lock.synchronize { @auth_warned ? false : (@auth_warned = true) }
|
|
30
|
+
end
|
|
14
31
|
end
|
|
15
32
|
|
|
16
33
|
# Default base controller — anything else means the app deliberately
|
|
@@ -40,13 +57,13 @@ module Pgbus
|
|
|
40
57
|
# place; don't nag apps that gate the dashboard via a custom controller
|
|
41
58
|
# (they had to set a redundant web_auth lambda just to silence us — #334).
|
|
42
59
|
return unless default_base_controller?
|
|
60
|
+
return unless Pgbus::Web::Authentication.claim_auth_warning!
|
|
43
61
|
|
|
44
62
|
Pgbus.logger.warn do
|
|
45
63
|
"[Pgbus] Dashboard is accessible without authentication. " \
|
|
46
64
|
"Configure Pgbus.configuration.web_auth to restrict access. " \
|
|
47
65
|
"See: https://github.com/zoolutions/pgbus#dashboard-authentication"
|
|
48
66
|
end
|
|
49
|
-
Pgbus::Web::Authentication.auth_warned = true
|
|
50
67
|
end
|
|
51
68
|
|
|
52
69
|
# True when base_controller_class is (still) the default ActionController::Base.
|
|
@@ -30,12 +30,14 @@ module Pgbus
|
|
|
30
30
|
) b ON s.key = b.concurrency_key
|
|
31
31
|
SQL
|
|
32
32
|
|
|
33
|
+
# Model names, resolved when counted (the models autoload lazily):
|
|
34
|
+
# names rather than lambdas, so the constant is plain frozen data.
|
|
33
35
|
LIST_MODELS = {
|
|
34
|
-
batches:
|
|
35
|
-
job_locks:
|
|
36
|
-
outbox:
|
|
37
|
-
processed_events:
|
|
38
|
-
recurring_tasks:
|
|
36
|
+
batches: :BatchEntry,
|
|
37
|
+
job_locks: :UniquenessKey,
|
|
38
|
+
outbox: :OutboxEntry,
|
|
39
|
+
processed_events: :ProcessedEvent,
|
|
40
|
+
recurring_tasks: :RecurringTask
|
|
39
41
|
}.freeze
|
|
40
42
|
|
|
41
43
|
# list – :batches, :job_locks, :concurrency_keys, :outbox, :processed_events or :recurring_tasks.
|
|
@@ -62,7 +64,7 @@ module Pgbus
|
|
|
62
64
|
private
|
|
63
65
|
|
|
64
66
|
def bounded_count(list)
|
|
65
|
-
rows = list == :concurrency_keys ? concurrency_keys_probe : LIST_MODELS.fetch(list).
|
|
67
|
+
rows = list == :concurrency_keys ? concurrency_keys_probe : Pgbus.const_get(LIST_MODELS.fetch(list)).limit(COUNT_CAP + 1).count
|
|
66
68
|
Count.new(total: [rows.to_i, COUNT_CAP].min, capped: rows.to_i > COUNT_CAP)
|
|
67
69
|
rescue StandardError => e
|
|
68
70
|
Pgbus.logger.debug { "[Pgbus::Web] Error counting #{list}: #{e.message}" }
|
|
@@ -65,7 +65,10 @@ module Pgbus
|
|
|
65
65
|
|
|
66
66
|
@server = TCPServer.new(@bind, @configured_port)
|
|
67
67
|
@port = @server.addr[1]
|
|
68
|
-
|
|
68
|
+
# The accept loop, owned by this server: started once (the @server
|
|
69
|
+
# check above) by the supervisor, joined (then killed if it hangs) by
|
|
70
|
+
# #stop.
|
|
71
|
+
@thread = Thread.new { accept_loop } # rubocop:disable ThreadSafety/NewThread
|
|
69
72
|
logger.info { "[Pgbus::Web::HealthServer] listening on #{@bind}:#{@port} (/livez, /readyz)" }
|
|
70
73
|
end
|
|
71
74
|
|
|
@@ -11,7 +11,7 @@ module Pgbus
|
|
|
11
11
|
# :uptime and :limit (lifetime), Strings for queue and topic lists.
|
|
12
12
|
module ProcessActivity
|
|
13
13
|
KIND_ORDER = %w[worker consumer dispatcher scheduler outbox_poller].freeze
|
|
14
|
-
RECYCLE_LIMITS = [%w[jobs max_jobs jobs_processed done], %w[memory max_memory_mb memory_mb used]].freeze
|
|
14
|
+
RECYCLE_LIMITS = [%w[jobs max_jobs jobs_processed done].freeze, %w[memory max_memory_mb memory_mb used].freeze].freeze
|
|
15
15
|
|
|
16
16
|
module_function
|
|
17
17
|
|
|
@@ -48,7 +48,10 @@ module Pgbus
|
|
|
48
48
|
return if @running
|
|
49
49
|
|
|
50
50
|
@running = true
|
|
51
|
-
|
|
51
|
+
# The heartbeat's one thread, owned by this object: Instance#start
|
|
52
|
+
# starts it once (under Streamer's @current_mutex); #stop signals
|
|
53
|
+
# and joins it.
|
|
54
|
+
@thread = Thread.new { run_loop } # rubocop:disable ThreadSafety/NewThread
|
|
52
55
|
self
|
|
53
56
|
end
|
|
54
57
|
|
|
@@ -43,7 +43,10 @@ module Pgbus
|
|
|
43
43
|
|
|
44
44
|
def connect
|
|
45
45
|
@sock = UNIXSocket.new(@socket_path)
|
|
46
|
-
|
|
46
|
+
# The client's reader thread, owned by this object: #connect starts
|
|
47
|
+
# it once per socket; it ends when the socket closes, and #stop
|
|
48
|
+
# closes the socket and joins it.
|
|
49
|
+
@reader = Thread.new { reader_loop } # rubocop:disable ThreadSafety/NewThread
|
|
47
50
|
self
|
|
48
51
|
rescue SystemCallError, IOError, ArgumentError, ThreadError => e
|
|
49
52
|
# ArgumentError: a socket path over the platform sun_path limit;
|
|
@@ -35,8 +35,14 @@ module Pgbus
|
|
|
35
35
|
# For a queue named `pgbus_stream_chat` the trigger table is
|
|
36
36
|
# `q_pgbus_stream_chat`, so the channel is `pgmq.q_pgbus_stream_chat.INSERT`.
|
|
37
37
|
class Listener
|
|
38
|
+
# Kept Ractor-shareable (issue #527): the hand-off to the dispatcher
|
|
39
|
+
# is the seam a Ractor dispatcher would pass by reference. The Strings
|
|
40
|
+
# are frozen in place: they are the sender's own (the NOTIFY's, or the
|
|
41
|
+
# hub frame's).
|
|
38
42
|
WakeMessage = Data.define(:queue_name, :payload) do
|
|
39
43
|
def initialize(queue_name:, payload: nil)
|
|
44
|
+
queue_name.freeze
|
|
45
|
+
payload&.freeze
|
|
40
46
|
super
|
|
41
47
|
end
|
|
42
48
|
end
|
|
@@ -101,7 +107,10 @@ module Pgbus
|
|
|
101
107
|
return if @running
|
|
102
108
|
|
|
103
109
|
@running = true
|
|
104
|
-
|
|
110
|
+
# The LISTEN thread, owned by this object: Instance#start starts it
|
|
111
|
+
# once (under Streamer's @current_mutex); #stop clears @running and
|
|
112
|
+
# joins it.
|
|
113
|
+
@thread = Thread.new { run_loop } # rubocop:disable ThreadSafety/NewThread
|
|
105
114
|
self
|
|
106
115
|
end
|
|
107
116
|
|
|
@@ -124,9 +124,12 @@ module Pgbus
|
|
|
124
124
|
File.umask(old_umask)
|
|
125
125
|
end
|
|
126
126
|
File.chmod(0o600, @socket_path)
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
127
|
+
# The hub's three loops, owned by this hub: #start runs once at hub
|
|
128
|
+
# start in the Puma master, under @lifecycle_mutex; #stop signals
|
|
129
|
+
# each loop and joins all three.
|
|
130
|
+
@accept_thread = Thread.new { accept_loop } # rubocop:disable ThreadSafety/NewThread
|
|
131
|
+
@fanout_thread = Thread.new { fanout_loop } # rubocop:disable ThreadSafety/NewThread
|
|
132
|
+
@status_thread = Thread.new { status_loop } # rubocop:disable ThreadSafety/NewThread
|
|
130
133
|
self
|
|
131
134
|
rescue StandardError
|
|
132
135
|
@table_mutex.synchronize { @running = false }
|
|
@@ -201,8 +204,11 @@ module Pgbus
|
|
|
201
204
|
@workers[@next_id] = entry
|
|
202
205
|
@next_id
|
|
203
206
|
end
|
|
204
|
-
|
|
205
|
-
entry
|
|
207
|
+
# One writer and one reader per connected Puma worker, owned by its
|
|
208
|
+
# @workers entry: both end when cleanup_worker closes the entry's
|
|
209
|
+
# socket (a worker hangs up, or #stop cleans up every worker).
|
|
210
|
+
entry[:writer] = Thread.new { writer_loop(id, entry) } # rubocop:disable ThreadSafety/NewThread
|
|
211
|
+
entry[:reader] = Thread.new { reader_loop(id, entry) } # rubocop:disable ThreadSafety/NewThread
|
|
206
212
|
id
|
|
207
213
|
end
|
|
208
214
|
|
|
@@ -50,7 +50,9 @@ module Pgbus
|
|
|
50
50
|
def start
|
|
51
51
|
ENV["PGBUS_STREAMS_HUB_SOCKET"] = @socket_path
|
|
52
52
|
@state_mutex.synchronize { @running = true }
|
|
53
|
-
|
|
53
|
+
# The boot thread, owned by this object: the Puma plugin starts it
|
|
54
|
+
# once in the master; #stop clears @running and joins it.
|
|
55
|
+
@thread = Thread.new { wait_and_start } # rubocop:disable ThreadSafety/NewThread
|
|
54
56
|
self
|
|
55
57
|
end
|
|
56
58
|
|
|
@@ -62,7 +62,10 @@ module Pgbus
|
|
|
62
62
|
|
|
63
63
|
@started = true
|
|
64
64
|
@partitions.each do |partition|
|
|
65
|
-
|
|
65
|
+
# One writer per partition, owned by this pump: Instance#start
|
|
66
|
+
# starts them once (the @started check above); #stop signals each
|
|
67
|
+
# partition and joins every thread.
|
|
68
|
+
@threads << Thread.new { run_worker(partition) } # rubocop:disable ThreadSafety/NewThread
|
|
66
69
|
end
|
|
67
70
|
self
|
|
68
71
|
end
|
|
@@ -57,7 +57,10 @@ module Pgbus
|
|
|
57
57
|
|
|
58
58
|
@wake_reader, @wake_writer = IO.pipe if @wake_reader.closed?
|
|
59
59
|
@running = true
|
|
60
|
-
|
|
60
|
+
# The watcher's one thread, owned by this object: Instance#start
|
|
61
|
+
# starts it once (under Streamer's @current_mutex); #stop wakes it
|
|
62
|
+
# through the pipe and joins it.
|
|
63
|
+
@thread = Thread.new { run_loop } # rubocop:disable ThreadSafety/NewThread
|
|
61
64
|
self
|
|
62
65
|
end
|
|
63
66
|
|
|
@@ -62,8 +62,17 @@ module Pgbus
|
|
|
62
62
|
# nil means the default (turbo-stream); a typed
|
|
63
63
|
# name (e.g. "presence", "reactive") lets clients
|
|
64
64
|
# route without sniffing the HTML (issue #170).
|
|
65
|
+
#
|
|
66
|
+
# Kept Ractor-shareable, like WakeMessage (issue #527): its Strings are
|
|
67
|
+
# frozen in place (the dispatcher's own, from JSON.parse or the read),
|
|
68
|
+
# and `exclude` is normalized to the String it is compared as.
|
|
65
69
|
StreamEnvelope = Data.define(:msg_id, :enqueued_at, :payload, :source, :visible_to, :exclude, :event) do
|
|
66
70
|
def initialize(msg_id:, enqueued_at:, payload:, source:, visible_to: nil, exclude: nil, event: nil)
|
|
71
|
+
enqueued_at&.freeze
|
|
72
|
+
payload&.freeze
|
|
73
|
+
source&.freeze
|
|
74
|
+
event&.freeze
|
|
75
|
+
exclude = exclude&.to_s&.freeze
|
|
67
76
|
super
|
|
68
77
|
end
|
|
69
78
|
end
|
|
@@ -147,7 +156,10 @@ module Pgbus
|
|
|
147
156
|
return if @running
|
|
148
157
|
|
|
149
158
|
@running = true
|
|
150
|
-
|
|
159
|
+
# The dispatcher's one thread, owned by this object: Instance#start
|
|
160
|
+
# starts it once (under Streamer's @current_mutex); #stop sends the
|
|
161
|
+
# sentinel and joins it.
|
|
162
|
+
@thread = Thread.new { run_loop } # rubocop:disable ThreadSafety/NewThread
|
|
151
163
|
self
|
|
152
164
|
end
|
|
153
165
|
|
data/lib/pgbus.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "concurrent/map"
|
|
3
4
|
require "zeitwerk"
|
|
4
5
|
|
|
5
6
|
module Pgbus
|
|
@@ -83,17 +84,37 @@ module Pgbus
|
|
|
83
84
|
end
|
|
84
85
|
|
|
85
86
|
class << self
|
|
87
|
+
# The gem's Zeitwerk loader, the app/models loader used outside Rails, and
|
|
88
|
+
# the configuration: built once when this file loads (see the bottom of
|
|
89
|
+
# the module body) and replaced only by reset!/teardown under @state_lock.
|
|
90
|
+
# Readers need no lock: a reference is published whole.
|
|
91
|
+
attr_reader :loader, :models_loader, :configuration
|
|
92
|
+
|
|
86
93
|
# Process-global flag set by Worker#graceful_shutdown so the adapter
|
|
87
94
|
# can report stopping? to ActiveJob::Continuation (Rails 8.1+).
|
|
88
|
-
attr_writer :stopping
|
|
89
|
-
|
|
90
95
|
def stopping
|
|
91
|
-
@stopping
|
|
96
|
+
@state_lock.synchronize { @stopping }
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def stopping=(value)
|
|
100
|
+
@state_lock.synchronize { @stopping = value ? true : false }
|
|
92
101
|
end
|
|
93
102
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
103
|
+
private
|
|
104
|
+
|
|
105
|
+
# True while a Rails app boots (its initializers and after_initialize
|
|
106
|
+
# blocks run): the engine finalizes at the end of that.
|
|
107
|
+
def rails_booting?
|
|
108
|
+
return false unless defined?(::Rails::Application) && ::Rails.respond_to?(:application)
|
|
109
|
+
|
|
110
|
+
app = ::Rails.application
|
|
111
|
+
!app.nil? && !app.initialized?
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Zeitwerk::Loader.for_gem works out the gem root from its caller, so this
|
|
115
|
+
# must stay in lib/pgbus.rb.
|
|
116
|
+
def build_loader
|
|
117
|
+
Zeitwerk::Loader.for_gem.tap do |loader|
|
|
97
118
|
loader.inflector.inflect(
|
|
98
119
|
"pgbus" => "Pgbus",
|
|
99
120
|
"cli" => "CLI",
|
|
@@ -127,35 +148,32 @@ module Pgbus
|
|
|
127
148
|
# root and tries to autoload Puma::Plugin, which collides with the real
|
|
128
149
|
# Puma::Plugin class defined by the puma gem itself.
|
|
129
150
|
loader.ignore("#{__dir__}/puma")
|
|
130
|
-
loader
|
|
131
151
|
end
|
|
132
152
|
end
|
|
133
153
|
|
|
134
154
|
# Separate loader for app/models used only in non-Rails contexts (specs,
|
|
135
155
|
# standalone scripts). When the Engine boots, Rails' autoloader takes over
|
|
136
156
|
# app/models and this loader is torn down to avoid conflicts.
|
|
137
|
-
def
|
|
157
|
+
def build_models_loader
|
|
138
158
|
models_dir = File.expand_path("../app/models", __dir__)
|
|
139
159
|
return nil unless File.directory?(models_dir)
|
|
140
160
|
|
|
141
|
-
|
|
142
|
-
loader = Zeitwerk::Loader.new
|
|
161
|
+
Zeitwerk::Loader.new.tap do |loader|
|
|
143
162
|
loader.tag = "pgbus-models"
|
|
144
163
|
loader.push_dir(models_dir)
|
|
145
164
|
loader.setup
|
|
146
|
-
loader
|
|
147
165
|
end
|
|
148
166
|
end
|
|
149
167
|
|
|
150
|
-
|
|
151
|
-
return unless @models_loader
|
|
152
|
-
|
|
153
|
-
@models_loader.unregister
|
|
154
|
-
@models_loader = nil
|
|
155
|
-
end
|
|
168
|
+
public
|
|
156
169
|
|
|
157
|
-
def
|
|
158
|
-
|
|
170
|
+
def teardown_models_loader!
|
|
171
|
+
loader = @state_lock.synchronize do
|
|
172
|
+
current = @models_loader
|
|
173
|
+
@models_loader = nil
|
|
174
|
+
current
|
|
175
|
+
end
|
|
176
|
+
loader&.unregister
|
|
159
177
|
end
|
|
160
178
|
|
|
161
179
|
# True when the current process is running a rake task (db:*, assets:*, …
|
|
@@ -188,11 +206,32 @@ module Pgbus
|
|
|
188
206
|
# until a worker path consumes it. Checking the flag after the yield
|
|
189
207
|
# lets a block opt out for itself via `c.eager_validation = false`.
|
|
190
208
|
configuration.validate! if configuration.eager_validation
|
|
209
|
+
# Without Rails this is the end of boot, so it finalizes; so does any
|
|
210
|
+
# configure once a Rails app has booted. While a Rails app boots, the
|
|
211
|
+
# engine's after_initialize finalizes instead, after every initializer
|
|
212
|
+
# and after_initialize block has had its say.
|
|
213
|
+
configuration.finalize! if configuration.finalized? || !rails_booting?
|
|
191
214
|
configuration
|
|
192
215
|
end
|
|
193
216
|
|
|
217
|
+
# The end of boot: validates the configuration (unless eager_validation
|
|
218
|
+
# is off) and freezes its plain-data settings into Ractor-shareable values
|
|
219
|
+
# (Configuration#finalize!). Callables and the logger stay as they are, on
|
|
220
|
+
# the main Ractor. Runs on its own: from the engine's after_initialize in a
|
|
221
|
+
# Rails app, at the end of each Pgbus.configure without Rails, and in
|
|
222
|
+
# `pgbus start` after its flags. Call it yourself only when you configure
|
|
223
|
+
# pgbus through setters alone, without Rails.
|
|
224
|
+
def finalize!
|
|
225
|
+
configuration.validate! if configuration.eager_validation
|
|
226
|
+
configuration.finalize!
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Built on first use, under @state_lock: two threads making the first
|
|
230
|
+
# call at once must not each build a Client (the loser's pool would be
|
|
231
|
+
# orphaned, its connections never closed). Not built at boot: a Client
|
|
232
|
+
# resolves connection options, which a rake db: task must not need.
|
|
194
233
|
def client
|
|
195
|
-
@client ||= Client.new(configuration)
|
|
234
|
+
@state_lock.synchronize { @client ||= Client.new(configuration) }
|
|
196
235
|
end
|
|
197
236
|
|
|
198
237
|
# Entry point for the streams subsystem — `Pgbus.stream(name).broadcast(html)`
|
|
@@ -218,8 +257,7 @@ module Pgbus
|
|
|
218
257
|
name = Streams::Stream.name_from(streamables)
|
|
219
258
|
resolved = durable.nil? ? configuration.stream_durable?(name) : durable
|
|
220
259
|
cache_key = "#{name}:#{resolved ? "d" : "e"}"
|
|
221
|
-
|
|
222
|
-
@stream_cache.compute_if_absent(cache_key) { Streams::Stream.new(streamables, durable: resolved) }
|
|
260
|
+
stream_cache.compute_if_absent(cache_key) { Streams::Stream.new(streamables, durable: resolved) }
|
|
223
261
|
end
|
|
224
262
|
|
|
225
263
|
# Compose a short, pgbus-safe stream identifier from any mix of
|
|
@@ -278,11 +316,14 @@ module Pgbus
|
|
|
278
316
|
EventBus::Publisher.publish_later(routing_key, payload, delay: delay, headers: headers)
|
|
279
317
|
end
|
|
280
318
|
|
|
319
|
+
# A fresh configuration (never nil, so readers need no lock) and no client.
|
|
281
320
|
def reset!
|
|
282
|
-
@
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
321
|
+
@state_lock.synchronize do
|
|
322
|
+
@client&.close
|
|
323
|
+
@client = nil
|
|
324
|
+
@configuration = Configuration.new
|
|
325
|
+
@stream_cache = Concurrent::Map.new
|
|
326
|
+
end
|
|
286
327
|
end
|
|
287
328
|
|
|
288
329
|
# Discard the inherited PGMQ client after fork.
|
|
@@ -290,9 +331,13 @@ module Pgbus
|
|
|
290
331
|
# state post-fork and attempting to acquire it can deadlock.
|
|
291
332
|
# The next call to Pgbus.client will lazily create a fresh one.
|
|
292
333
|
# Cached streams hold the client they were built with, so they go too.
|
|
334
|
+
# (@state_lock itself is safe to take in the child: Ruby abandons a Mutex
|
|
335
|
+
# whose owning thread did not survive the fork.)
|
|
293
336
|
def reset_client!
|
|
294
|
-
@
|
|
295
|
-
|
|
337
|
+
@state_lock.synchronize do
|
|
338
|
+
@client = nil
|
|
339
|
+
@stream_cache = Concurrent::Map.new
|
|
340
|
+
end
|
|
296
341
|
end
|
|
297
342
|
|
|
298
343
|
def logger
|
|
@@ -302,13 +347,30 @@ module Pgbus
|
|
|
302
347
|
def logger=(value)
|
|
303
348
|
configuration.logger = value
|
|
304
349
|
end
|
|
350
|
+
|
|
351
|
+
private
|
|
352
|
+
|
|
353
|
+
# Process-local, like the client it holds; replaced (never mutated in
|
|
354
|
+
# place) by reset!/reset_client!.
|
|
355
|
+
attr_reader :stream_cache
|
|
305
356
|
end
|
|
306
357
|
|
|
307
|
-
|
|
358
|
+
# Module state, built here at load rather than by `||=` on first use, so no
|
|
359
|
+
# two threads can each build their own (issue #527). @state_lock guards every
|
|
360
|
+
# later write: the client memo, reset!/reset_client!, the stopping flag.
|
|
361
|
+
@state_lock = Mutex.new
|
|
362
|
+
@stopping = false
|
|
363
|
+
@client = nil
|
|
364
|
+
@stream_cache = Concurrent::Map.new
|
|
365
|
+
@loader = build_loader
|
|
366
|
+
@loader.setup
|
|
367
|
+
|
|
368
|
+
# After loader.setup: Configuration is Zeitwerk-managed.
|
|
369
|
+
@configuration = Configuration.new
|
|
308
370
|
|
|
309
371
|
# In non-Rails contexts, set up model autoloading via a separate loader.
|
|
310
372
|
# This is torn down by the Engine initializer when Rails boots.
|
|
311
|
-
models_loader
|
|
373
|
+
@models_loader = defined?(Rails::Engine) ? nil : build_models_loader
|
|
312
374
|
end
|
|
313
375
|
|
|
314
376
|
require "active_job/queue_adapters/pgbus_adapter" if defined?(ActiveJob)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
module Pgbus
|
|
2
|
+
module ActiveJob
|
|
3
|
+
# What the queue adapter delegates to.
|
|
4
|
+
class Adapter
|
|
5
|
+
def enqueue: (untyped active_job) -> untyped
|
|
6
|
+
def enqueue_at: (untyped active_job, Numeric timestamp) -> untyped
|
|
7
|
+
def enqueue_all: (Array[untyped] active_jobs) -> Integer
|
|
8
|
+
end
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
# The job-class options, included on ActiveJob::Base by the engine.
|
|
12
|
+
module Concurrency
|
|
13
|
+
module ClassMethods
|
|
14
|
+
def limits_concurrency: (to: Integer, ?key: untyped, ?duration: Numeric, ?on_conflict: :block | :discard | :raise) -> untyped
|
|
15
|
+
def pgbus_concurrency: () -> Hash[Symbol, untyped]?
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
module Uniqueness
|
|
20
|
+
module ClassMethods
|
|
21
|
+
# Any other keyword raises ArgumentError (lock_ttl: with a pointer to 1.0's removal).
|
|
22
|
+
def ensures_uniqueness: (?strategy: :until_executed | :while_executing, ?key: untyped,
|
|
23
|
+
?on_conflict: :reject | :discard | :log) -> untyped
|
|
24
|
+
def pgbus_uniqueness: () -> Hash[Symbol, untyped]?
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
module RetryBackoff
|
|
29
|
+
module JobMixin
|
|
30
|
+
module ClassMethods
|
|
31
|
+
def pgbus_retry_backoff: (?base: Numeric?, ?max: Numeric?, ?jitter: Numeric?) -> untyped
|
|
32
|
+
def pgbus_retry_backoff_config: () -> Hash[Symbol, Numeric]?
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
module VisibilityHeartbeat
|
|
38
|
+
module JobMixin
|
|
39
|
+
module ClassMethods
|
|
40
|
+
def pgbus_visibility_heartbeat: (?boolish enabled) -> bool
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
module ActiveJob
|
|
47
|
+
module QueueAdapters
|
|
48
|
+
# `config.active_job.queue_adapter = :pgbus`
|
|
49
|
+
class PgbusAdapter
|
|
50
|
+
def enqueue: (untyped active_job) -> untyped
|
|
51
|
+
def enqueue_at: (untyped active_job, Numeric timestamp) -> untyped
|
|
52
|
+
def enqueue_all: (Array[untyped] active_jobs) -> Integer
|
|
53
|
+
def enqueue_after_transaction_commit?: () -> bool
|
|
54
|
+
def stopping?: () -> bool
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# The PGMQ client (Pgbus.client). Generated from reflection, untyped; always go
|
|
2
|
+
# through it rather than PGMQ directly. spec/pgbus/rbs_spec.rb fails when a
|
|
3
|
+
# public method is missing here.
|
|
4
|
+
module Pgbus
|
|
5
|
+
class Client
|
|
6
|
+
def archive_batch: (untyped queue_name, untyped msg_ids, ?prefixed: untyped) -> untyped
|
|
7
|
+
def archive_message: (untyped queue_name, untyped msg_id, ?prefixed: untyped) -> untyped
|
|
8
|
+
def bind_topic: (untyped pattern, untyped queue_name) -> untyped
|
|
9
|
+
def close: () -> untyped
|
|
10
|
+
def config: () -> untyped
|
|
11
|
+
def configured_queues: () -> untyped
|
|
12
|
+
def connection_health: () -> untyped
|
|
13
|
+
def convert_archive_partitioned: (untyped queue_name, ?partition_interval: untyped, ?retention_interval: untyped, ?leading_partition: untyped) -> untyped
|
|
14
|
+
def create_fifo_index: (untyped queue_name) -> untyped
|
|
15
|
+
def create_fifo_indexes_all: () -> untyped
|
|
16
|
+
def dead_letter_physical_name: (untyped queue_name) -> untyped
|
|
17
|
+
def delete_batch: (untyped queue_name, untyped msg_ids, ?prefixed: untyped) -> untyped
|
|
18
|
+
def delete_message: (untyped queue_name, untyped msg_id, ?prefixed: untyped) -> untyped
|
|
19
|
+
def drop_queue: (untyped queue_name, ?prefixed: untyped) -> untyped
|
|
20
|
+
def ensure_all_queues: () -> untyped
|
|
21
|
+
def ensure_dead_letter_queue: (untyped name) -> untyped
|
|
22
|
+
def ensure_fair_index: (untyped queue_name) -> untyped
|
|
23
|
+
def ensure_queue: (untyped name) -> untyped
|
|
24
|
+
def ensure_stream_queue: (untyped stream_name) -> untyped
|
|
25
|
+
def in_recovery?: () -> bool
|
|
26
|
+
def list_notify_insert_throttles: () -> untyped
|
|
27
|
+
def list_queues: () -> untyped
|
|
28
|
+
def message_archived?: (untyped queue_name, msg_id: untyped) -> bool
|
|
29
|
+
def message_exists?: (untyped queue_name, ?msg_id: untyped, ?uniqueness_key: untyped) -> bool
|
|
30
|
+
def message_in_queue?: (untyped queue_name, msg_id: untyped) -> bool
|
|
31
|
+
def message_with_job_id?: (untyped queue_name, job_id: untyped) -> bool
|
|
32
|
+
def metrics: (?untyped queue_name) -> untyped
|
|
33
|
+
def move_to_dead_letter: (untyped queue_name, untyped message, ?headers: untyped) -> untyped
|
|
34
|
+
def notify_enabled?: (untyped queue_name) -> bool
|
|
35
|
+
def notify_stream: (untyped stream_name, untyped payload) -> untyped
|
|
36
|
+
def oldest_claimable_ages: (?untyped queue_name) -> untyped
|
|
37
|
+
def pgmq: () -> untyped
|
|
38
|
+
def pgmq_installed?: () -> bool
|
|
39
|
+
def pgmq_schema_version: () -> untyped
|
|
40
|
+
def physical_queue_names: (untyped logical_name) -> untyped
|
|
41
|
+
def ping: () -> untyped
|
|
42
|
+
def pool_stats: () -> untyped
|
|
43
|
+
def publish_to_topic: (untyped routing_key, untyped payload, ?headers: untyped, ?delay: untyped) -> untyped
|
|
44
|
+
def purge_archive: (untyped queue_name, older_than: untyped, ?batch_size: untyped) -> untyped
|
|
45
|
+
def purge_queue: (untyped queue_name, ?prefixed: untyped) -> untyped
|
|
46
|
+
def read_after: (untyped stream_name, after_id: untyped, ?limit: untyped) -> untyped
|
|
47
|
+
def read_batch: (untyped queue_name, qty: untyped, ?vt: untyped) -> untyped
|
|
48
|
+
def read_batch_fair: (untyped queue_name, qty: untyped, ?vt: untyped) -> untyped
|
|
49
|
+
def read_batch_prioritized: (untyped queue_name, qty: untyped, ?vt: untyped) -> untyped
|
|
50
|
+
def read_batch_prioritized_fair: (untyped queue_name, qty: untyped, ?vt: untyped) -> untyped
|
|
51
|
+
def read_grouped: (untyped queue_name, qty: untyped, ?vt: untyped) -> untyped
|
|
52
|
+
def read_grouped_head: (untyped queue_name, qty: untyped, ?vt: untyped) -> untyped
|
|
53
|
+
def read_grouped_rr: (untyped queue_name, qty: untyped, ?vt: untyped) -> untyped
|
|
54
|
+
def read_message: (untyped queue_name, ?vt: untyped) -> untyped
|
|
55
|
+
def read_multi: (untyped queue_names, qty: untyped, ?vt: untyped, ?limit: untyped) -> untyped
|
|
56
|
+
def read_with_poll: (untyped queue_name, qty: untyped, ?vt: untyped, ?max_poll_seconds: untyped, ?poll_interval_ms: untyped) -> untyped
|
|
57
|
+
def reload: () -> untyped
|
|
58
|
+
def resize_streams_pool: (untyped new_size) -> untyped
|
|
59
|
+
def send_batch: (untyped queue_name, untyped payloads, ?headers: untyped, ?delay: untyped, ?priority: untyped) -> untyped
|
|
60
|
+
def send_message: (untyped queue_name, untyped payload, ?headers: untyped, ?delay: untyped, ?priority: untyped) -> untyped
|
|
61
|
+
def send_stream_message: (untyped stream_name, untyped payload, ?headers: untyped, ?delay: untyped) -> untyped
|
|
62
|
+
def set_visibility_timeout: (untyped queue_name, untyped msg_id, vt: untyped, ?prefixed: untyped) -> untyped
|
|
63
|
+
def shared_connection?: () -> bool
|
|
64
|
+
def stream_current_msg_id: (untyped stream_name) -> untyped
|
|
65
|
+
def stream_oldest_msg_id: (untyped stream_name) -> untyped
|
|
66
|
+
def streams_pool_stats: () -> untyped
|
|
67
|
+
def streams_swap_stats: () -> untyped
|
|
68
|
+
def synchronizing?: () -> bool
|
|
69
|
+
def target_queue: (untyped queue_name, ?untyped priority) -> untyped
|
|
70
|
+
def transaction: () ?{ (*untyped) -> untyped } -> untyped
|
|
71
|
+
def uniqueness_keys_present: (untyped lock_keys) -> untyped
|
|
72
|
+
def update_notify_insert: (untyped queue_name, throttle_interval_ms: untyped) -> untyped
|
|
73
|
+
def verify_connection!: () -> untyped
|
|
74
|
+
def wait_for_notify: (untyped queue_name, ?timeout: untyped) ?{ (*untyped) -> untyped } -> untyped
|
|
75
|
+
def self.load_pgmq_gem!: () -> untyped
|
|
76
|
+
def self.pgmq_install_mutex: () -> untyped
|
|
77
|
+
end
|
|
78
|
+
end
|