pgbus 0.18.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +27 -0
- data/README.md +5 -0
- data/app/controllers/pgbus/frontends_controller.rb +10 -8
- data/app/helpers/pgbus/application_helper.rb +4 -3
- data/app/models/pgbus/job_stat.rb +4 -0
- data/app/models/pgbus/processed_event.rb +3 -1
- data/app/models/pgbus/stream_queue.rb +34 -3
- data/app/models/pgbus/stream_stat.rb +3 -0
- data/lib/pgbus/active_job/current_attributes.rb +3 -1
- data/lib/pgbus/batch.rb +17 -9
- data/lib/pgbus/cli.rb +3 -0
- data/lib/pgbus/client/read_after.rb +10 -1
- data/lib/pgbus/client.rb +54 -9
- data/lib/pgbus/concurrency.rb +3 -0
- data/lib/pgbus/configuration.rb +84 -9
- data/lib/pgbus/current_attributes.rb +3 -1
- data/lib/pgbus/dedup_cache.rb +15 -7
- data/lib/pgbus/engine.rb +11 -0
- data/lib/pgbus/event_bus/handler.rb +16 -4
- data/lib/pgbus/execution_pools/async_pool.rb +23 -2
- data/lib/pgbus/execution_pools/thread_pool.rb +22 -0
- data/lib/pgbus/execution_pools.rb +26 -0
- data/lib/pgbus/integrations/appsignal/probe.rb +17 -9
- data/lib/pgbus/integrations/appsignal/subscriber.rb +30 -18
- data/lib/pgbus/integrations/appsignal.rb +26 -16
- data/lib/pgbus/mcp/rack_app.rb +1 -1
- data/lib/pgbus/metrics/subscriber.rb +35 -21
- data/lib/pgbus/process/consumer.rb +1 -1
- data/lib/pgbus/process/consumer_priority.rb +1 -1
- data/lib/pgbus/process/lifecycle.rb +5 -5
- data/lib/pgbus/process/notify_listener.rb +32 -18
- data/lib/pgbus/process/wake_pipe.rb +4 -1
- data/lib/pgbus/process/worker.rb +4 -2
- data/lib/pgbus/retry_backoff.rb +3 -0
- data/lib/pgbus/shareable.rb +70 -0
- data/lib/pgbus/streams/broadcastable_override.rb +3 -0
- data/lib/pgbus/streams/presence.rb +6 -7
- data/lib/pgbus/streams.rb +38 -27
- data/lib/pgbus/testing.rb +19 -3
- data/lib/pgbus/uniqueness.rb +3 -0
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/visibility_heartbeat.rb +51 -34
- data/lib/pgbus/web/authentication.rb +19 -2
- data/lib/pgbus/web/data_source/list_counts.rb +8 -6
- data/lib/pgbus/web/health_server.rb +4 -1
- data/lib/pgbus/web/process_activity.rb +1 -1
- data/lib/pgbus/web/streamer/heartbeat.rb +4 -1
- data/lib/pgbus/web/streamer/hub_client.rb +4 -1
- data/lib/pgbus/web/streamer/listener.rb +10 -1
- data/lib/pgbus/web/streamer/master_hub.rb +11 -5
- data/lib/pgbus/web/streamer/master_hub_boot.rb +3 -1
- data/lib/pgbus/web/streamer/outbound_pump.rb +4 -1
- data/lib/pgbus/web/streamer/peer_close_watcher.rb +4 -1
- data/lib/pgbus/web/streamer/stream_event_dispatcher.rb +13 -1
- data/lib/pgbus.rb +92 -30
- data/sig/pgbus/active_job/adapter.rbs +57 -0
- data/sig/pgbus/client.rbs +78 -0
- data/sig/pgbus/configuration.rbs +316 -0
- data/sig/pgbus/streams/stream.rbs +25 -0
- data/sig/pgbus/streams.rbs +22 -0
- data/sig/pgbus.rbs +73 -0
- metadata +8 -1
|
@@ -17,15 +17,45 @@ module Pgbus
|
|
|
17
17
|
METRIC_PREFIX = "pgbus_"
|
|
18
18
|
private_constant :METRIC_PREFIX
|
|
19
19
|
|
|
20
|
-
|
|
20
|
+
# Installed once at boot (the engine's after_initialize) and torn down by
|
|
21
|
+
# reset! in specs; @install_lock makes the check-then-install atomic so
|
|
22
|
+
# two callers can never subscribe twice (issue #527). Handlers read
|
|
23
|
+
# @backend through the private reader: a reference swapped whole.
|
|
24
|
+
@install_lock = Mutex.new
|
|
25
|
+
@subscriptions = [].freeze
|
|
21
26
|
@backend = nil
|
|
27
|
+
@installed = false
|
|
22
28
|
|
|
23
29
|
class << self
|
|
24
30
|
def install!(backend:)
|
|
25
|
-
|
|
31
|
+
@install_lock.synchronize do
|
|
32
|
+
next false if @installed
|
|
26
33
|
|
|
27
|
-
|
|
28
|
-
|
|
34
|
+
@backend = backend
|
|
35
|
+
@subscriptions = subscribe_all
|
|
36
|
+
@installed = true
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def installed?
|
|
41
|
+
@install_lock.synchronize { @installed }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def reset!
|
|
45
|
+
@install_lock.synchronize do
|
|
46
|
+
@subscriptions.each { |s| ActiveSupport::Notifications.unsubscribe(s) }
|
|
47
|
+
@subscriptions = [].freeze
|
|
48
|
+
@backend = nil
|
|
49
|
+
@installed = false
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
attr_reader :backend
|
|
56
|
+
|
|
57
|
+
def subscribe_all
|
|
58
|
+
[
|
|
29
59
|
subscribe("pgbus.executor.execute") { |event| on_executor_execute(event) },
|
|
30
60
|
subscribe("pgbus.job_completed") { |event| on_job_completed(event) },
|
|
31
61
|
subscribe("pgbus.job_failed") { |event| on_job_failed(event) },
|
|
@@ -43,25 +73,9 @@ module Pgbus
|
|
|
43
73
|
subscribe("pgbus.worker.recycle") { |event| on_worker_recycle(event) },
|
|
44
74
|
subscribe("pgbus.consumer.recycle") { |event| on_consumer_recycle(event) },
|
|
45
75
|
subscribe("pgbus.client.pool") { |event| on_client_pool(event) }
|
|
46
|
-
]
|
|
47
|
-
@installed = true
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
def installed?
|
|
51
|
-
@installed == true
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
def reset!
|
|
55
|
-
@subscriptions&.each { |s| ActiveSupport::Notifications.unsubscribe(s) }
|
|
56
|
-
@subscriptions = []
|
|
57
|
-
@backend = nil
|
|
58
|
-
@installed = false
|
|
76
|
+
].freeze
|
|
59
77
|
end
|
|
60
78
|
|
|
61
|
-
private
|
|
62
|
-
|
|
63
|
-
attr_reader :backend
|
|
64
|
-
|
|
65
79
|
def subscribe(name, &block)
|
|
66
80
|
ActiveSupport::Notifications.subscribe(name) do |*args|
|
|
67
81
|
event = ActiveSupport::Notifications::Event.new(*args)
|
|
@@ -17,8 +17,8 @@ module Pgbus
|
|
|
17
17
|
# 0.1s polling interval.
|
|
18
18
|
CACHE_TTL = 5
|
|
19
19
|
|
|
20
|
-
@cache = {}
|
|
21
20
|
@cache_mutex = Mutex.new
|
|
21
|
+
@cache = {} # rubocop:disable ThreadSafety/MutableClassInstanceVariable -- only read or written under @cache_mutex
|
|
22
22
|
|
|
23
23
|
# Check if this worker should yield to a higher-priority worker.
|
|
24
24
|
# Returns true if a higher-priority healthy worker exists for
|
|
@@ -22,11 +22,11 @@ module Pgbus
|
|
|
22
22
|
STATES = %i[starting running paused draining stopped].freeze
|
|
23
23
|
|
|
24
24
|
TRANSITIONS = {
|
|
25
|
-
starting: %i[running stopped],
|
|
26
|
-
running: %i[paused draining stopped],
|
|
27
|
-
paused: %i[running draining stopped],
|
|
28
|
-
draining: %i[stopped],
|
|
29
|
-
stopped: []
|
|
25
|
+
starting: %i[running stopped].freeze,
|
|
26
|
+
running: %i[paused draining stopped].freeze,
|
|
27
|
+
paused: %i[running draining stopped].freeze,
|
|
28
|
+
draining: %i[stopped].freeze,
|
|
29
|
+
stopped: [].freeze
|
|
30
30
|
}.freeze
|
|
31
31
|
|
|
32
32
|
attr_reader :state
|
|
@@ -64,6 +64,9 @@ module Pgbus
|
|
|
64
64
|
@health_check_ms = health_check_ms
|
|
65
65
|
@logger = logger
|
|
66
66
|
@state_mutex = Mutex.new
|
|
67
|
+
# Serializes #start and #stop whole (see #start). Never taken by the
|
|
68
|
+
# listener thread, so #stop can join it while holding this.
|
|
69
|
+
@lifecycle_mutex = Mutex.new
|
|
67
70
|
@listening_to = Set.new
|
|
68
71
|
@commands = Queue.new
|
|
69
72
|
@running = false
|
|
@@ -98,33 +101,44 @@ module Pgbus
|
|
|
98
101
|
@state_mutex.synchronize { @delivering }
|
|
99
102
|
end
|
|
100
103
|
|
|
104
|
+
# Under @lifecycle_mutex, like #stop: a stop that ran between the
|
|
105
|
+
# @running check and Thread.new found no thread to join, and the thread
|
|
106
|
+
# started after it was joined by nobody (and a restart could run a
|
|
107
|
+
# second run_loop beside it).
|
|
101
108
|
def start
|
|
102
|
-
@
|
|
103
|
-
|
|
109
|
+
@lifecycle_mutex.synchronize do
|
|
110
|
+
@state_mutex.synchronize do
|
|
111
|
+
return self if @running
|
|
104
112
|
|
|
105
|
-
|
|
113
|
+
@running = true
|
|
114
|
+
end
|
|
115
|
+
@physical_queues.each { |q| @commands << [:listen, q] }
|
|
116
|
+
# The listener's one thread, owned by this object: started once (the
|
|
117
|
+
# @running check above, with #start and #stop serialized), the sole
|
|
118
|
+
# owner of @conn, and joined by #stop after it clears @running.
|
|
119
|
+
@thread = Thread.new { run_loop } # rubocop:disable ThreadSafety/NewThread
|
|
106
120
|
end
|
|
107
|
-
@physical_queues.each { |q| @commands << [:listen, q] }
|
|
108
|
-
@thread = Thread.new { run_loop }
|
|
109
121
|
self
|
|
110
122
|
end
|
|
111
123
|
|
|
112
124
|
def stop
|
|
113
|
-
@
|
|
114
|
-
|
|
125
|
+
@lifecycle_mutex.synchronize do
|
|
126
|
+
@state_mutex.synchronize do
|
|
127
|
+
return self unless @running
|
|
115
128
|
|
|
116
|
-
|
|
129
|
+
@running = false
|
|
130
|
+
end
|
|
131
|
+
@commands << [:stop]
|
|
132
|
+
# Deliberately does NOT touch @conn. PG::Connection is not thread-safe
|
|
133
|
+
# and #close is PQfinish: it frees the PGconn and its OpenSSL objects
|
|
134
|
+
# out from under whatever libpq call the listener thread is making on
|
|
135
|
+
# the same connection. That is a use-after-free — a process-killing
|
|
136
|
+
# SEGV, not a rescuable PG::Error (issue #375). The listener thread is
|
|
137
|
+
# the sole owner of @conn for its entire life and closes it in
|
|
138
|
+
# run_loop's ensure; clearing @running above is the whole stop signal.
|
|
139
|
+
@thread&.join(stop_join_timeout)
|
|
140
|
+
@thread = nil
|
|
117
141
|
end
|
|
118
|
-
@commands << [:stop]
|
|
119
|
-
# Deliberately does NOT touch @conn. PG::Connection is not thread-safe
|
|
120
|
-
# and #close is PQfinish: it frees the PGconn and its OpenSSL objects
|
|
121
|
-
# out from under whatever libpq call the listener thread is making on
|
|
122
|
-
# the same connection. That is a use-after-free — a process-killing
|
|
123
|
-
# SEGV, not a rescuable PG::Error (issue #375). The listener thread is
|
|
124
|
-
# the sole owner of @conn for its entire life and closes it in
|
|
125
|
-
# run_loop's ensure; clearing @running above is the whole stop signal.
|
|
126
|
-
@thread&.join(stop_join_timeout)
|
|
127
|
-
@thread = nil
|
|
128
142
|
self
|
|
129
143
|
end
|
|
130
144
|
|
|
@@ -53,7 +53,10 @@ module Pgbus
|
|
|
53
53
|
|
|
54
54
|
@running = true
|
|
55
55
|
end
|
|
56
|
-
|
|
56
|
+
# The pipe watcher's one thread, owned by this object: started once
|
|
57
|
+
# (the @running check above, under @state_mutex) and stopped by #stop,
|
|
58
|
+
# which closes the pipe under it and joins it.
|
|
59
|
+
@thread = Thread.new { run_loop } # rubocop:disable ThreadSafety/NewThread
|
|
57
60
|
self
|
|
58
61
|
end
|
|
59
62
|
|
data/lib/pgbus/process/worker.rb
CHANGED
|
@@ -37,7 +37,9 @@ module Pgbus
|
|
|
37
37
|
notify_retry_backoff: NOTIFY_RETRY_BASE_SECONDS,
|
|
38
38
|
started_at_monotonic: nil, wake_pipe: nil, read_ahead: nil,
|
|
39
39
|
recycle_limits: {}, capsule: nil, process: nil)
|
|
40
|
-
|
|
40
|
+
# The worker's own list: eviction deletes from it, and the queues it is
|
|
41
|
+
# handed come from a configuration frozen by Pgbus.finalize! at boot.
|
|
42
|
+
@queues = Array(queues).dup
|
|
41
43
|
@initial_queues = @queues.dup.freeze
|
|
42
44
|
@wildcard = @queues.include?("*")
|
|
43
45
|
@threads = threads
|
|
@@ -237,7 +239,7 @@ module Pgbus
|
|
|
237
239
|
Pgbus.stopping = true
|
|
238
240
|
@lifecycle.transition_to!(:stopped)
|
|
239
241
|
@wake_signal.notify!
|
|
240
|
-
@pool.
|
|
242
|
+
ExecutionPools.terminate(@pool, grace: config.immediate_shutdown_grace)
|
|
241
243
|
end
|
|
242
244
|
|
|
243
245
|
WILDCARD_REFRESH_INTERVAL = 30 # seconds
|
data/lib/pgbus/retry_backoff.rb
CHANGED
|
@@ -20,6 +20,9 @@ module Pgbus
|
|
|
20
20
|
module JobMixin
|
|
21
21
|
extend ActiveSupport::Concern
|
|
22
22
|
|
|
23
|
+
# Class-body macro: it runs while the job class loads (at boot, or under the
|
|
24
|
+
# autoloader's lock), stores a frozen Hash, and the reader only reads it.
|
|
25
|
+
# rubocop:disable-next ThreadSafety/ClassInstanceVariable
|
|
23
26
|
class_methods do
|
|
24
27
|
def pgbus_retry_backoff(base: nil, max: nil, jitter: nil)
|
|
25
28
|
raise ArgumentError, "retry_backoff base must be > 0" if !base.nil? && (!base.is_a?(Numeric) || base <= 0)
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
# Deep-frozen, Ractor-shareable copies of plain data, for the settings
|
|
5
|
+
# Configuration#finalize! freezes at boot (issue #527).
|
|
6
|
+
module Shareable
|
|
7
|
+
# Leaves that are plain data. Modules and classes count too: they are
|
|
8
|
+
# shareable by nature.
|
|
9
|
+
LEAVES = [String, Symbol, Numeric, TrueClass, FalseClass, NilClass, Regexp, Module].freeze
|
|
10
|
+
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
# True when +value+ is plain data all the way down: the LEAVES, and Arrays,
|
|
14
|
+
# Sets, Ranges and default-less Hashes of them. A Proc, a Logger or any
|
|
15
|
+
# other object anywhere inside makes it false, and so does a collection
|
|
16
|
+
# that contains itself (+path+ holds the collections being walked), so
|
|
17
|
+
# finalizing leaves it as it is instead of recursing without end.
|
|
18
|
+
def data?(value, path = {}.compare_by_identity)
|
|
19
|
+
case value
|
|
20
|
+
when Array, Set, Hash, Range
|
|
21
|
+
return false if path.key?(value)
|
|
22
|
+
|
|
23
|
+
path[value] = true
|
|
24
|
+
begin
|
|
25
|
+
collection_data?(value, path)
|
|
26
|
+
ensure
|
|
27
|
+
path.delete(value)
|
|
28
|
+
end
|
|
29
|
+
else LEAVES.any? { |leaf| value.is_a?(leaf) }
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def collection_data?(value, path)
|
|
34
|
+
case value
|
|
35
|
+
when Array, Set then value.all? { |item| data?(item, path) }
|
|
36
|
+
when Hash
|
|
37
|
+
value.default_proc.nil? && data?(value.default, path) && value.all? { |k, v| data?(k, path) && data?(v, path) }
|
|
38
|
+
else data?(value.begin, path) && data?(value.end, path)
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# A deep-frozen copy of plain data (see data?), Ractor-shareable where the
|
|
43
|
+
# engine has Ractors. The caller's own objects are copied, never frozen in
|
|
44
|
+
# place.
|
|
45
|
+
def freeze_copy(value)
|
|
46
|
+
return ::Ractor.make_shareable(value, copy: true) if defined?(::Ractor) && ::Ractor.respond_to?(:make_shareable)
|
|
47
|
+
|
|
48
|
+
deep_freeze(value)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# The copy freeze_copy makes where the engine has no Ractor: everything
|
|
52
|
+
# data? accepts, a Hash's default and a Range's endpoints included.
|
|
53
|
+
def deep_freeze(value)
|
|
54
|
+
case value
|
|
55
|
+
when Array then value.map { |item| deep_freeze(item) }.freeze
|
|
56
|
+
when Set then value.to_set { |item| deep_freeze(item) }.freeze
|
|
57
|
+
when Hash then deep_freeze_hash(value)
|
|
58
|
+
when Range then Range.new(deep_freeze(value.begin), deep_freeze(value.end), value.exclude_end?).freeze
|
|
59
|
+
when String then value.frozen? ? value : value.dup.freeze
|
|
60
|
+
else value
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def deep_freeze_hash(hash)
|
|
65
|
+
copy = hash.to_h { |k, v| [deep_freeze(k), deep_freeze(v)] }
|
|
66
|
+
copy.default = deep_freeze(hash.default)
|
|
67
|
+
copy.freeze
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
@@ -56,6 +56,9 @@ module Pgbus
|
|
|
56
56
|
end
|
|
57
57
|
end
|
|
58
58
|
|
|
59
|
+
# broadcasts_to / broadcasts_refreshes_to are class-body macros: they write
|
|
60
|
+
# @pgbus_durable_streams while the model class loads, and it is only read after.
|
|
61
|
+
# rubocop:disable-next ThreadSafety/ClassInstanceVariable
|
|
59
62
|
module ClassMethods
|
|
60
63
|
def broadcasts_to(stream, durable: nil, inserts_by: :append, target: broadcast_target_default, **rendering)
|
|
61
64
|
if durable.nil?
|
|
@@ -77,11 +77,13 @@ module Pgbus
|
|
|
77
77
|
# True when pgbus_presence_members has the seat columns (the
|
|
78
78
|
# add_presence_seats migration ran). A true answer is kept for the
|
|
79
79
|
# life of the process; a false one for SCHEMA_RECHECK_SECONDS.
|
|
80
|
+
#
|
|
81
|
+
# The answer is read and written under @schema_mutex only (issue
|
|
82
|
+
# #527). A successful probe is kept as above; a probe that raises is
|
|
83
|
+
# not, so the next call probes again.
|
|
80
84
|
def seats_supported?
|
|
81
|
-
return @seats_supported if schema_answer_fresh?
|
|
82
|
-
|
|
83
85
|
@schema_mutex.synchronize do
|
|
84
|
-
next @seats_supported if schema_answer_fresh?
|
|
86
|
+
next @seats_supported if schema_answer_fresh?(@seats_supported, @probed_at)
|
|
85
87
|
|
|
86
88
|
rows = connection.exec_params(<<~SQL, [])
|
|
87
89
|
SELECT 1 AS present FROM information_schema.columns
|
|
@@ -169,10 +171,7 @@ module Pgbus
|
|
|
169
171
|
|
|
170
172
|
private
|
|
171
173
|
|
|
172
|
-
|
|
173
|
-
def schema_answer_fresh?
|
|
174
|
-
supported = @seats_supported
|
|
175
|
-
probed_at = @probed_at
|
|
174
|
+
def schema_answer_fresh?(supported, probed_at)
|
|
176
175
|
return false if supported.nil? || probed_at.nil?
|
|
177
176
|
|
|
178
177
|
supported || monotonic_now - probed_at < SCHEMA_RECHECK_SECONDS
|
data/lib/pgbus/streams.rb
CHANGED
|
@@ -33,38 +33,49 @@ module Pgbus
|
|
|
33
33
|
# `message`. Broadcasts may override it with a typed name (issue #170).
|
|
34
34
|
DEFAULT_SSE_EVENT = "turbo-stream"
|
|
35
35
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
36
|
+
class << self
|
|
37
|
+
# Process-wide registry of server-side audience filter predicates.
|
|
38
|
+
# Register filters at boot time via:
|
|
39
|
+
# Pgbus::Streams.filters.register(:admin_only) { |user| user.admin? }
|
|
40
|
+
# See lib/pgbus/streams/filters.rb for the full API.
|
|
41
|
+
#
|
|
42
|
+
# Process-wide publish-side coalescer for high-frequency broadcasts
|
|
43
|
+
# (issue #171); the flush re-enters the normal broadcast path so a
|
|
44
|
+
# coalesced frame is just a deferred ordinary broadcast.
|
|
45
|
+
#
|
|
46
|
+
# Both are built when this file loads, not by `||=` on first use: two
|
|
47
|
+
# first callers could each build a registry, and a filter registered on
|
|
48
|
+
# the one that lost was gone (issue #527). Replaced only by the resets.
|
|
49
|
+
attr_reader :filters, :coalescer
|
|
50
|
+
|
|
51
|
+
# Fresh filters registry. Used by tests; not intended for runtime.
|
|
52
|
+
def reset_filters!
|
|
53
|
+
@reset_lock.synchronize { @filters = Filters.new }
|
|
54
|
+
end
|
|
43
55
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
56
|
+
# Fresh coalescer. Used by tests; not intended for runtime.
|
|
57
|
+
def reset_coalescer!
|
|
58
|
+
@reset_lock.synchronize { @coalescer = build_coalescer }
|
|
59
|
+
end
|
|
48
60
|
|
|
49
|
-
|
|
50
|
-
# (issue #171). Lazily built; the flush re-enters the normal broadcast
|
|
51
|
-
# path so a coalesced frame is just a deferred ordinary broadcast.
|
|
52
|
-
def self.coalescer
|
|
53
|
-
# `target:` is part of the flush signature but unused here — it's only
|
|
54
|
-
# a coalescing key; the target is already encoded in the payload's
|
|
55
|
-
# turbo-stream tag. Absorbed via ** so it isn't a broadcast argument.
|
|
56
|
-
@coalescer ||= Coalescer.new(
|
|
57
|
-
flush: lambda do |stream_name:, payload:, opts:, **|
|
|
58
|
-
Pgbus.stream(stream_name).broadcast(payload, **opts)
|
|
59
|
-
end
|
|
60
|
-
)
|
|
61
|
-
end
|
|
61
|
+
private
|
|
62
62
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
63
|
+
def build_coalescer
|
|
64
|
+
# `target:` is part of the flush signature but unused here — it's only
|
|
65
|
+
# a coalescing key; the target is already encoded in the payload's
|
|
66
|
+
# turbo-stream tag. Absorbed via ** so it isn't a broadcast argument.
|
|
67
|
+
Coalescer.new(
|
|
68
|
+
flush: lambda do |stream_name:, payload:, opts:, **|
|
|
69
|
+
Pgbus.stream(stream_name).broadcast(payload, **opts)
|
|
70
|
+
end
|
|
71
|
+
)
|
|
72
|
+
end
|
|
66
73
|
end
|
|
67
74
|
|
|
75
|
+
@reset_lock = Mutex.new
|
|
76
|
+
@filters = Filters.new
|
|
77
|
+
@coalescer = build_coalescer
|
|
78
|
+
|
|
68
79
|
# A handle on a single logical stream. The name can be any string, an
|
|
69
80
|
# object responding to `to_gid_param`, or an array of streamables (which
|
|
70
81
|
# are joined with colons — turbo-rails-compatible).
|
data/lib/pgbus/testing.rb
CHANGED
|
@@ -71,6 +71,15 @@ module Pgbus
|
|
|
71
71
|
# Eagerly initialize the store so concurrent access never races on creation.
|
|
72
72
|
@store = EventStore.new
|
|
73
73
|
|
|
74
|
+
# The process-wide mode (`fake!` with no block). A module ivar under a lock
|
|
75
|
+
# built at load, not `Thread.main[]`: that reads the main thread's CURRENT
|
|
76
|
+
# fiber only, so a mode set inside a fiber (an Async task) or read while the
|
|
77
|
+
# main thread runs one was lost (issue #527). The block form stays
|
|
78
|
+
# `Thread.current[MODE_KEY]`, which is fiber-local on every Ruby: a `fake!`
|
|
79
|
+
# block inside one Async task must not leak to its siblings.
|
|
80
|
+
@mode_lock = Mutex.new
|
|
81
|
+
@default_mode = nil
|
|
82
|
+
|
|
74
83
|
class << self
|
|
75
84
|
def mode!(mode, &block)
|
|
76
85
|
raise ArgumentError, "Unknown mode: #{mode}. Valid modes: #{MODES.join(", ")}" unless MODES.include?(mode)
|
|
@@ -78,7 +87,7 @@ module Pgbus
|
|
|
78
87
|
# Record the mode BEFORE syncing streams: a StreamerLeakError raised
|
|
79
88
|
# by the teardown must still leave the process in the requested mode.
|
|
80
89
|
unless block
|
|
81
|
-
|
|
90
|
+
@mode_lock.synchronize { @default_mode = mode }
|
|
82
91
|
sync_streams_test_mode!(mode)
|
|
83
92
|
return
|
|
84
93
|
end
|
|
@@ -90,12 +99,15 @@ module Pgbus
|
|
|
90
99
|
ensure
|
|
91
100
|
if block
|
|
92
101
|
Thread.current[MODE_KEY] = old
|
|
93
|
-
|
|
102
|
+
# The mode in effect again: the outer block's, else the
|
|
103
|
+
# process-wide one (not :disabled, which would let the streamer
|
|
104
|
+
# into a fake-mode test's later SSE requests).
|
|
105
|
+
sync_streams_test_mode!(old || default_mode)
|
|
94
106
|
end
|
|
95
107
|
end
|
|
96
108
|
|
|
97
109
|
def mode
|
|
98
|
-
Thread.current[MODE_KEY] ||
|
|
110
|
+
Thread.current[MODE_KEY] || default_mode
|
|
99
111
|
end
|
|
100
112
|
|
|
101
113
|
def fake!(&) = mode!(:fake, &)
|
|
@@ -110,6 +122,10 @@ module Pgbus
|
|
|
110
122
|
|
|
111
123
|
private
|
|
112
124
|
|
|
125
|
+
def default_mode
|
|
126
|
+
@mode_lock.synchronize { @default_mode } || :disabled
|
|
127
|
+
end
|
|
128
|
+
|
|
113
129
|
# When entering fake/inline mode, enable streams_test_mode to prevent
|
|
114
130
|
# rack.hijack from spawning background threads that acquire DB
|
|
115
131
|
# connections outside the test transaction (see issue #133). When
|
data/lib/pgbus/uniqueness.rb
CHANGED
|
@@ -52,6 +52,9 @@ module Pgbus
|
|
|
52
52
|
VALID_STRATEGIES = %i[until_executed while_executing].freeze
|
|
53
53
|
VALID_CONFLICTS = %i[reject discard log].freeze
|
|
54
54
|
|
|
55
|
+
# Class-body macro: it runs while the job class loads (at boot, or under the
|
|
56
|
+
# autoloader's lock), stores a frozen Hash, and the reader only reads it.
|
|
57
|
+
# rubocop:disable-next ThreadSafety/ClassInstanceVariable
|
|
55
58
|
class_methods do
|
|
56
59
|
def ensures_uniqueness(strategy: :until_executed, key: nil, on_conflict: :reject, **opts)
|
|
57
60
|
# lock_ttl was validated and stored in message metadata but never read
|
data/lib/pgbus/version.rb
CHANGED
|
@@ -45,7 +45,9 @@ module Pgbus
|
|
|
45
45
|
extend ActiveSupport::Concern
|
|
46
46
|
|
|
47
47
|
included do
|
|
48
|
-
|
|
48
|
+
# A per-class setting the job's own class body writes at load
|
|
49
|
+
# (`pgbus_visibility_heartbeat false`); jobs only read it while they run.
|
|
50
|
+
class_attribute :pgbus_visibility_heartbeat_enabled, instance_writer: false, default: nil # rubocop:disable ThreadSafety/ClassAndModuleAttributes
|
|
49
51
|
end
|
|
50
52
|
|
|
51
53
|
class_methods do
|
|
@@ -55,6 +57,14 @@ module Pgbus
|
|
|
55
57
|
end
|
|
56
58
|
end
|
|
57
59
|
|
|
60
|
+
# Built at load, never lazily: a `@mutex ||= Mutex.new` is itself a race in
|
|
61
|
+
# which two first callers each lock a Mutex of their own (issue #527). It
|
|
62
|
+
# guards @entries, @running, @thread and @pid, which are only touched inside
|
|
63
|
+
# #synchronize. A forked child can take it even when a parent thread held it
|
|
64
|
+
# at the fork: Ruby abandons a Mutex whose owner did not survive the fork.
|
|
65
|
+
@mutex = Mutex.new
|
|
66
|
+
@entries = {} # rubocop:disable ThreadSafety/MutableClassInstanceVariable -- only read or written inside #synchronize
|
|
67
|
+
|
|
58
68
|
class << self
|
|
59
69
|
# Track the message for the duration of the block.
|
|
60
70
|
#
|
|
@@ -106,13 +116,13 @@ module Pgbus
|
|
|
106
116
|
# can drive it.
|
|
107
117
|
def tick!(now: monotonic_now, config: Pgbus.configuration)
|
|
108
118
|
interval = config.effective_visibility_heartbeat_interval
|
|
109
|
-
due = synchronize { entries.values.select { |entry| now - entry.extended_at >= interval } }
|
|
119
|
+
due = synchronize { @entries.values.select { |entry| now - entry.extended_at >= interval } }
|
|
110
120
|
due.each { |entry| extend!(entry, now: now, config: config) }
|
|
111
121
|
due.size
|
|
112
122
|
end
|
|
113
123
|
|
|
114
124
|
def tracked_count
|
|
115
|
-
synchronize { entries.size }
|
|
125
|
+
synchronize { @entries.size }
|
|
116
126
|
end
|
|
117
127
|
|
|
118
128
|
# Stop the background thread. Tracked entries are kept: a job still
|
|
@@ -141,9 +151,20 @@ module Pgbus
|
|
|
141
151
|
|
|
142
152
|
def register(entry, config)
|
|
143
153
|
synchronize do
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
154
|
+
# Entries registered before a fork belong to the parent's jobs; the
|
|
155
|
+
# thread did not survive the fork either.
|
|
156
|
+
unless @pid == ::Process.pid
|
|
157
|
+
@pid = ::Process.pid
|
|
158
|
+
@entries = {}
|
|
159
|
+
@running = false
|
|
160
|
+
@thread = nil
|
|
161
|
+
end
|
|
162
|
+
@entries[key_for(entry)] = entry
|
|
163
|
+
# Start the ticker once per process.
|
|
164
|
+
unless @running && @thread&.alive?
|
|
165
|
+
@running = true
|
|
166
|
+
@thread = start_ticker(config)
|
|
167
|
+
end
|
|
147
168
|
end
|
|
148
169
|
end
|
|
149
170
|
|
|
@@ -153,7 +174,7 @@ module Pgbus
|
|
|
153
174
|
def unregister(entry)
|
|
154
175
|
synchronize do
|
|
155
176
|
key = key_for(entry)
|
|
156
|
-
entries.delete(key) if entries[key].equal?(entry)
|
|
177
|
+
@entries.delete(key) if @entries[key].equal?(entry)
|
|
157
178
|
end
|
|
158
179
|
end
|
|
159
180
|
|
|
@@ -186,31 +207,38 @@ module Pgbus
|
|
|
186
207
|
end
|
|
187
208
|
end
|
|
188
209
|
|
|
189
|
-
#
|
|
190
|
-
|
|
191
|
-
|
|
210
|
+
# One ticker per process, owned by this module: #register starts it under
|
|
211
|
+
# the lock (at most one live thread, see @running/@thread) and #stop
|
|
212
|
+
# clears @running, wakes it and joins it; Worker#shutdown calls #stop.
|
|
213
|
+
def start_ticker(config)
|
|
214
|
+
thread = Thread.new { run_loop(config) } # rubocop:disable ThreadSafety/NewThread
|
|
215
|
+
thread.name = "pgbus-visibility-heartbeat"
|
|
216
|
+
thread
|
|
217
|
+
end
|
|
192
218
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
219
|
+
# True while the calling thread is the ticker #register started last
|
|
220
|
+
# and #stop has not stopped it. A stopped ticker can wake after a
|
|
221
|
+
# #register has already started its successor; @running alone would be
|
|
222
|
+
# true again for it, and it would tick beside the new one.
|
|
223
|
+
def current_ticker?
|
|
224
|
+
synchronize { @running && @thread.equal?(Thread.current) }
|
|
196
225
|
end
|
|
197
226
|
|
|
198
227
|
def run_loop(config)
|
|
199
|
-
while
|
|
228
|
+
while current_ticker?
|
|
200
229
|
# Half the interval keeps every extension inside [interval, 1.5 * interval]
|
|
201
230
|
# of the previous one — at most half the visibility timeout.
|
|
202
231
|
sleep([config.effective_visibility_heartbeat_interval / 2.0, 0.05].max)
|
|
203
|
-
break unless
|
|
232
|
+
break unless current_ticker?
|
|
204
233
|
|
|
205
234
|
tick!(config: config)
|
|
206
235
|
end
|
|
207
236
|
rescue StandardError => e
|
|
208
237
|
Pgbus.logger.error { "[Pgbus::VisibilityHeartbeat] ticker died: #{e.class}: #{e.message}" }
|
|
209
|
-
|
|
238
|
+
# Only its own state: a stopped ticker must not stop its successor.
|
|
239
|
+
synchronize { @running = false if @thread.equal?(Thread.current) }
|
|
210
240
|
end
|
|
211
241
|
|
|
212
|
-
# Entries registered before a fork belong to the parent's jobs; the
|
|
213
|
-
# thread did not survive the fork either.
|
|
214
242
|
# A live holder keeps its semaphore from being swept, so `duration`
|
|
215
243
|
# bounds heartbeat silence rather than run time. Its own rescue: a
|
|
216
244
|
# failed touch must not cost the message its visibility extension.
|
|
@@ -236,29 +264,18 @@ module Pgbus
|
|
|
236
264
|
end
|
|
237
265
|
end
|
|
238
266
|
|
|
239
|
-
def forget_parent_entries!
|
|
240
|
-
return if @pid == ::Process.pid
|
|
241
|
-
|
|
242
|
-
@pid = ::Process.pid
|
|
243
|
-
@entries = {}
|
|
244
|
-
@running = false
|
|
245
|
-
@thread = nil
|
|
246
|
-
end
|
|
247
|
-
|
|
248
|
-
def entries
|
|
249
|
-
@entries ||= {}
|
|
250
|
-
end
|
|
251
|
-
|
|
252
267
|
def registered?(entry)
|
|
253
|
-
synchronize { entries[key_for(entry)].equal?(entry) }
|
|
268
|
+
synchronize { @entries[key_for(entry)].equal?(entry) }
|
|
254
269
|
end
|
|
255
270
|
|
|
256
271
|
def key_for(entry)
|
|
257
272
|
[entry.queue_name, entry.msg_id]
|
|
258
273
|
end
|
|
259
274
|
|
|
260
|
-
|
|
261
|
-
|
|
275
|
+
# A literal block, not `(&)`: ThreadSafety/ClassInstanceVariable accepts
|
|
276
|
+
# @mutex as the receiver of a synchronize block only.
|
|
277
|
+
def synchronize
|
|
278
|
+
@mutex.synchronize { yield } # rubocop:disable Style/ExplicitBlockArgument
|
|
262
279
|
end
|
|
263
280
|
|
|
264
281
|
def monotonic_now
|