pgbus 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.md +36 -14
- data/Rakefile +7 -1
- data/app/assets/javascripts/pgbus/stream_source_element.js +17 -3
- data/app/assets/javascripts/pgbus/stream_source_element.min.js +4 -4
- data/app/assets/javascripts/pgbus/stream_source_element.min.js.map +1 -1
- data/app/controllers/pgbus/dead_letter_controller.rb +3 -1
- data/app/controllers/pgbus/jobs_controller.rb +2 -1
- data/app/models/pgbus/blocked_execution.rb +3 -2
- data/app/models/pgbus/queue_state.rb +6 -0
- data/app/models/pgbus/uniqueness_key.rb +11 -0
- data/config/locales/da.yml +1 -0
- data/config/locales/de.yml +1 -0
- data/config/locales/en.yml +1 -0
- data/config/locales/es.yml +1 -0
- data/config/locales/fi.yml +1 -0
- data/config/locales/fr.yml +1 -0
- data/config/locales/it.yml +1 -0
- data/config/locales/ja.yml +1 -0
- data/config/locales/nb.yml +1 -0
- data/config/locales/nl.yml +1 -0
- data/config/locales/pt.yml +1 -0
- data/config/locales/sv.yml +1 -0
- data/lib/generators/pgbus/fold_queue_name_case_generator.rb +57 -0
- data/lib/generators/pgbus/templates/fold_queue_name_case.rb.erb +12 -0
- data/lib/generators/pgbus/templates/tune_autovacuum.rb.erb +2 -2
- data/lib/generators/pgbus/templates/tune_fillfactor.rb.erb +1 -1
- data/lib/pgbus/active_job/adapter.rb +3 -2
- data/lib/pgbus/active_job/executor.rb +142 -20
- data/lib/pgbus/active_job/quiet_logs.rb +57 -0
- data/lib/pgbus/autovacuum_tuning.rb +2 -2
- data/lib/pgbus/cli/dlq.rb +2 -1
- data/lib/pgbus/client/async_commit.rb +149 -0
- data/lib/pgbus/client/native_statements.rb +28 -16
- data/lib/pgbus/client.rb +145 -17
- data/lib/pgbus/concurrency/blocked_execution.rb +9 -1
- data/lib/pgbus/configuration.rb +71 -6
- data/lib/pgbus/dead_letter_header.rb +3 -0
- data/lib/pgbus/doctor.rb +55 -2
- data/lib/pgbus/failed_event_recorder.rb +5 -2
- data/lib/pgbus/generators/migration_detector.rb +29 -2
- data/lib/pgbus/instrumentation.rb +16 -1
- data/lib/pgbus/metrics/subscriber.rb +27 -1
- data/lib/pgbus/process/completer.rb +4 -2
- data/lib/pgbus/process/worker.rb +28 -3
- data/lib/pgbus/queue_name_fold.rb +145 -0
- data/lib/pgbus/queue_name_validator.rb +20 -2
- data/lib/pgbus/recurring/schedule.rb +22 -13
- data/lib/pgbus/stat_buffer.rb +67 -26
- data/lib/pgbus/streams/cursor.rb +17 -6
- data/lib/pgbus/streams/envelope.rb +16 -0
- data/lib/pgbus/streams/key.rb +1 -1
- data/lib/pgbus/table_maintenance.rb +1 -1
- data/lib/pgbus/uniqueness.rb +43 -16
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source/dead_letter.rb +14 -5
- data/lib/pgbus/web/data_source/resend.rb +185 -0
- data/lib/pgbus/web/data_source/resend_slot.rb +139 -0
- data/lib/pgbus/web/data_source.rb +55 -37
- data/lib/pgbus/web/dead_letter_reason.rb +6 -0
- data/lib/pgbus/web/streamer/connection.rb +36 -29
- data/lib/pgbus/web/streamer/falcon_connection.rb +29 -25
- data/lib/pgbus/web/streamer/heartbeat.rb +14 -3
- data/lib/pgbus/web/streamer/instance.rb +8 -5
- data/lib/pgbus/web/streamer/listener.rb +239 -62
- data/lib/pgbus/web/streamer/outbound_pump.rb +6 -4
- data/lib/pgbus/web/streamer/registry.rb +55 -29
- data/lib/pgbus/web/streamer/stream_event_dispatcher.rb +73 -12
- data/lib/tasks/pgbus_queues.rake +27 -0
- data/sig/pgbus/active_job/adapter.rbs +1 -1
- data/sig/pgbus/active_job/quiet_logs.rbs +11 -0
- data/sig/pgbus/client.rbs +6 -5
- data/sig/pgbus/configuration.rbs +4 -0
- metadata +11 -3
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Web
|
|
5
|
+
class DataSource
|
|
6
|
+
# The limits_concurrency half of a re-send (issue #636; Resend#resend_job
|
|
7
|
+
# is the whole). A re-sent copy signals its key's semaphore when it
|
|
8
|
+
# finishes, so it has to hold a slot first, taken the way
|
|
9
|
+
# Adapter#enqueue_with_concurrency takes one: the check and the park
|
|
10
|
+
# commit together under the semaphore row lock (rails/solid_queue#712),
|
|
11
|
+
# and the send follows the commit.
|
|
12
|
+
module ResendSlot
|
|
13
|
+
private
|
|
14
|
+
|
|
15
|
+
# Returns the block's msg_id once the copy is sent holding a slot,
|
|
16
|
+
# :parked when it waits behind a full limit (on_conflict: :block), and
|
|
17
|
+
# :refused when the limit is full under :discard or :raise. A payload
|
|
18
|
+
# with no concurrency key is sent as it is. +sending+ is :produce
|
|
19
|
+
# around the produce and :park around the park's commit, so the
|
|
20
|
+
# caller can tell an error that may hide a live copy from one that
|
|
21
|
+
# proves nothing went out.
|
|
22
|
+
def resend_holding_slot(payload, lock_queue, drop_source, sending, headers, &)
|
|
23
|
+
key = payload.is_a?(Hash) && Concurrency.extract_key(payload)
|
|
24
|
+
unless key
|
|
25
|
+
sending[0] = :produce
|
|
26
|
+
return yield
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
limits = resend_limits(payload)
|
|
30
|
+
case (taken = take_resend_slot(key, limits, lock_queue, park_payload(payload, headers), sending))
|
|
31
|
+
when :acquired then send_holding_resend_slot(key, sending, &)
|
|
32
|
+
when :full
|
|
33
|
+
Pgbus.logger.warn do
|
|
34
|
+
"[Pgbus::Web] Not re-sending #{payload["job_class"]}: concurrency limit for #{key} is full " \
|
|
35
|
+
"(on_conflict: :#{limits[:on_conflict]})"
|
|
36
|
+
end
|
|
37
|
+
:refused
|
|
38
|
+
else drop_parked_source(taken, key, drop_source)
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# The limits live on the job class, not in the payload. A class that
|
|
43
|
+
# no longer declares them, or no longer resolves, is judged the way a
|
|
44
|
+
# promotion judges its parked job (Concurrency.config_for): against
|
|
45
|
+
# the limit the semaphore row records, parked when that is full.
|
|
46
|
+
def resend_limits(payload)
|
|
47
|
+
klass = Object.const_get(payload["job_class"].to_s)
|
|
48
|
+
config = klass.respond_to?(:pgbus_concurrency) && klass.pgbus_concurrency
|
|
49
|
+
config || { limit: nil, duration: Concurrency::DEFAULT_DURATION, on_conflict: :block }
|
|
50
|
+
rescue NameError
|
|
51
|
+
{ limit: nil, duration: Concurrency::DEFAULT_DURATION, on_conflict: :block }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# :acquired, :full, or the id of the BlockedExecution the copy was
|
|
55
|
+
# parked as. The copy is sent right away, so its lease is the
|
|
56
|
+
# duration alone (Adapter#slot_lease with no delay). Parked on the
|
|
57
|
+
# logical queue at the job's own priority (the serialized "priority",
|
|
58
|
+
# else the default, as Adapter#handle_conflict parks it): promote_next
|
|
59
|
+
# sends it through Client#send_message, which prefixes the name and
|
|
60
|
+
# picks the _pN sub-queue by that priority. Once the park is inserted
|
|
61
|
+
# only its commit remains, and a COMMIT whose reply is lost may have
|
|
62
|
+
# landed: +sending+ says so until the transaction returns.
|
|
63
|
+
def take_resend_slot(key, limits, lock_queue, payload, sending)
|
|
64
|
+
taken = Pgbus::Semaphore.transaction(requires_new: true) do
|
|
65
|
+
lease = Concurrency.effective_duration(limits[:duration])
|
|
66
|
+
next :acquired if Concurrency::Semaphore.acquire(key, limits[:limit], lease) == :acquired
|
|
67
|
+
next :full unless limits[:on_conflict] == :block
|
|
68
|
+
|
|
69
|
+
Concurrency::BlockedExecution.insert(
|
|
70
|
+
concurrency_key: key, queue_name: lock_queue, payload: payload,
|
|
71
|
+
priority: payload["priority"] || Pgbus.configuration.default_priority, duration: limits[:duration]
|
|
72
|
+
).rows.dig(0, 0).tap { sending[0] = :park }
|
|
73
|
+
end
|
|
74
|
+
sending[0] = nil
|
|
75
|
+
taken
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# pgbus_blocked_executions has no headers column: the headers the
|
|
79
|
+
# copy would have been sent with (a DLQ retry's retries count) ride
|
|
80
|
+
# in the parked payload, and promote_next sends them as headers again
|
|
81
|
+
# (Concurrency::BlockedExecution::HEADERS_KEY).
|
|
82
|
+
def park_payload(payload, headers)
|
|
83
|
+
return payload if headers.nil? || headers.empty?
|
|
84
|
+
|
|
85
|
+
payload.merge(Concurrency::BlockedExecution::HEADERS_KEY => headers)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Hand the slot back only when the failure proves nothing was
|
|
89
|
+
# produced (Adapter#send_holding_slot): on an ambiguous outcome the
|
|
90
|
+
# copy may be live, so the hold stands and the lease expiry plus the
|
|
91
|
+
# dispatcher's sweep reclaim it.
|
|
92
|
+
def send_holding_resend_slot(key, sending)
|
|
93
|
+
sending[0] = :produce
|
|
94
|
+
yield
|
|
95
|
+
rescue StandardError => e
|
|
96
|
+
if ambiguous_resend?(e)
|
|
97
|
+
Pgbus.logger.warn { "[Pgbus::Web] Re-send of #{key} may have been sent (#{e.class}); its concurrency slot stays held" }
|
|
98
|
+
else
|
|
99
|
+
begin
|
|
100
|
+
Concurrency::Semaphore.release(key)
|
|
101
|
+
rescue StandardError => release_error
|
|
102
|
+
Pgbus.logger.warn { "[Pgbus::Web] Giving back the concurrency slot of #{key} failed: #{release_error.message}" }
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
raise
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# The park has committed; the source goes next (it lives on another
|
|
109
|
+
# connection, or after the commit, so the two cannot share a
|
|
110
|
+
# transaction). If that fails, the park is withdrawn while it still
|
|
111
|
+
# waits, and the error raised: nothing changed, the source stays. A
|
|
112
|
+
# removal whose outcome is unknown (a lost reply: its DELETE may have
|
|
113
|
+
# committed) keeps the park, since withdrawing it could leave neither
|
|
114
|
+
# copy; so does a park that cannot be withdrawn (already promoted: a
|
|
115
|
+
# live copy). Either way the source may stay, for the operator to
|
|
116
|
+
# discard: a duplicate is preferred over a lost job.
|
|
117
|
+
def drop_parked_source(park_id, key, drop_source)
|
|
118
|
+
drop_source.call
|
|
119
|
+
:parked
|
|
120
|
+
rescue StandardError => e
|
|
121
|
+
raise if !ambiguous_resend?(e) && withdraw_park(park_id)
|
|
122
|
+
|
|
123
|
+
Pgbus.logger.warn do
|
|
124
|
+
"[Pgbus::Web] Re-sent copy of #{key} is parked, but removing its source failed (#{e.class}: #{e.message}); " \
|
|
125
|
+
"the parked copy stands, and the source may stay to be discarded"
|
|
126
|
+
end
|
|
127
|
+
:parked
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def withdraw_park(park_id)
|
|
131
|
+
Pgbus::BlockedExecution.where(id: park_id).delete_all.positive?
|
|
132
|
+
rescue StandardError => e
|
|
133
|
+
Pgbus.logger.warn { "[Pgbus::Web] Withdrawing the parked re-send #{park_id} failed: #{e.message}" }
|
|
134
|
+
false
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
@@ -7,6 +7,8 @@ module Pgbus
|
|
|
7
7
|
class DataSource
|
|
8
8
|
include JobList
|
|
9
9
|
include DeadLetter
|
|
10
|
+
include Resend
|
|
11
|
+
include ResendSlot
|
|
10
12
|
include ListCounts
|
|
11
13
|
include QueueSummary
|
|
12
14
|
include EventList
|
|
@@ -175,9 +177,17 @@ module Pgbus
|
|
|
175
177
|
@client.purge_queue(name, prefixed: false)
|
|
176
178
|
end
|
|
177
179
|
|
|
180
|
+
# The messages' uniqueness locks are released only once the table is
|
|
181
|
+
# gone: their holders are read just before the drop (the client runs
|
|
182
|
+
# the block only when it drops the table, not for a mixed-case
|
|
183
|
+
# duplicate from before issue #633, whose table and messages stay with
|
|
184
|
+
# the lower-case queue) and released after it succeeds, so a failed
|
|
185
|
+
# drop leaves both messages and locks.
|
|
178
186
|
def drop_queue(name)
|
|
179
|
-
|
|
180
|
-
@client.drop_queue(name, prefixed: false)
|
|
187
|
+
holders = nil
|
|
188
|
+
result = @client.drop_queue(name, prefixed: false) { holders = queue_lock_holders(name) }
|
|
189
|
+
release_lock_holders(holders) if holders
|
|
190
|
+
result
|
|
181
191
|
end
|
|
182
192
|
|
|
183
193
|
def pause_queue(name, reason: nil)
|
|
@@ -299,20 +309,7 @@ module Pgbus
|
|
|
299
309
|
# duplicate (the original is still in the queue waiting for retry).
|
|
300
310
|
# Falls back to enqueueing a fresh copy only if the original is gone
|
|
301
311
|
# (e.g., already moved to DLQ).
|
|
302
|
-
|
|
303
|
-
if physical
|
|
304
|
-
@client.set_visibility_timeout(physical, event["msg_id"].to_i, vt: 0, prefixed: false)
|
|
305
|
-
else
|
|
306
|
-
payload = JSON.parse(event["payload"])
|
|
307
|
-
headers = event["headers"]
|
|
308
|
-
headers = JSON.parse(headers) if headers.is_a?(String)
|
|
309
|
-
@client.send_message(event["queue_name"], payload, headers: headers)
|
|
310
|
-
end
|
|
311
|
-
|
|
312
|
-
connection.exec_delete(
|
|
313
|
-
"DELETE FROM pgbus_failed_events WHERE id = $1", "Pgbus Delete Failed Event", [id.to_i]
|
|
314
|
-
)
|
|
315
|
-
true
|
|
312
|
+
retry_failed_job(event)
|
|
316
313
|
rescue StandardError => e
|
|
317
314
|
Pgbus.logger.debug { "[Pgbus::Web] Error retrying failed event #{id}: #{e.message}" }
|
|
318
315
|
false
|
|
@@ -336,27 +333,24 @@ module Pgbus
|
|
|
336
333
|
|
|
337
334
|
# Failed jobs only, like discard_all_failed: an event's message is still
|
|
338
335
|
# in its handler queue, so a re-sent copy would run the handler twice.
|
|
336
|
+
# Each job goes back the way Retry sends it (#retry_failed_job): a
|
|
337
|
+
# message still queued gets its vt reset, one that is gone is re-sent.
|
|
338
|
+
# A row it cannot retry (the re-send's key is held, its concurrency
|
|
339
|
+
# limit refuses it, or a call fails) stays, and the walk goes on past it
|
|
340
|
+
# by id.
|
|
339
341
|
def retry_all_failed
|
|
340
342
|
count = 0
|
|
343
|
+
last_id = 0
|
|
341
344
|
loop do
|
|
342
345
|
batch = connection.select_all(
|
|
343
|
-
"SELECT * FROM pgbus_failed_events WHERE #{NOT_HANDLER_QUEUE} ORDER BY id LIMIT 100",
|
|
344
|
-
[handler_queue_names_literal]
|
|
346
|
+
"SELECT * FROM pgbus_failed_events WHERE #{NOT_HANDLER_QUEUE} AND id > $2 ORDER BY id LIMIT 100",
|
|
347
|
+
"Pgbus Retry Batch", [handler_queue_names_literal, last_id]
|
|
345
348
|
).to_a
|
|
346
349
|
break if batch.empty?
|
|
347
350
|
|
|
348
351
|
batch.each do |event|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
headers = JSON.parse(headers) if headers.is_a?(String)
|
|
352
|
-
|
|
353
|
-
connection.transaction do
|
|
354
|
-
@client.send_message(event["queue_name"], payload, headers: headers)
|
|
355
|
-
connection.exec_delete(
|
|
356
|
-
"DELETE FROM pgbus_failed_events WHERE id = $1", "Pgbus Delete Failed Event", [event["id"].to_i]
|
|
357
|
-
)
|
|
358
|
-
end
|
|
359
|
-
count += 1
|
|
352
|
+
last_id = event["id"].to_i
|
|
353
|
+
count += 1 if retry_failed_job(event)
|
|
360
354
|
rescue StandardError => e
|
|
361
355
|
Pgbus.logger.error { "[Pgbus::Web] Failed to retry event #{event["id"]}: #{e.message}" }
|
|
362
356
|
end
|
|
@@ -634,9 +628,10 @@ module Pgbus
|
|
|
634
628
|
# Drop every job parked behind a key. The jobs never run, so the
|
|
635
629
|
# bookkeeping they would have resolved has to be resolved here: a parked
|
|
636
630
|
# batch child is marked failed (otherwise its batch waits forever and
|
|
637
|
-
# on_failure never fires — issue #413) and an :until_executed
|
|
638
|
-
# lock is released (otherwise it is
|
|
639
|
-
# release it — issue #423).
|
|
631
|
+
# on_failure never fires — issue #413) and an :until_executed or
|
|
632
|
+
# :until_executing uniqueness lock is released (otherwise it is
|
|
633
|
+
# orphaned, since no executor will ever release it — issue #423).
|
|
634
|
+
# Returns the number of jobs discarded.
|
|
640
635
|
def discard_parked_jobs(key)
|
|
641
636
|
return 0 if key.to_s.strip.empty?
|
|
642
637
|
|
|
@@ -1162,11 +1157,17 @@ module Pgbus
|
|
|
1162
1157
|
# one acquired after this row was parked.
|
|
1163
1158
|
def release_parked_uniqueness_lock(payload, parked_at)
|
|
1164
1159
|
key = payload[Uniqueness::METADATA_KEY]
|
|
1165
|
-
return unless key && payload[Uniqueness::STRATEGY_KEY]
|
|
1160
|
+
return unless key && enqueue_locked_strategy?(payload[Uniqueness::STRATEGY_KEY])
|
|
1166
1161
|
|
|
1167
1162
|
UniquenessKey.release_if_unbound!(key, acquired_before: parked_at)
|
|
1168
1163
|
end
|
|
1169
1164
|
|
|
1165
|
+
# The strategies whose row is taken at enqueue and so is held by a parked
|
|
1166
|
+
# job: :until_executed, and :until_executing (issue #628).
|
|
1167
|
+
def enqueue_locked_strategy?(strategy)
|
|
1168
|
+
Uniqueness::ENQUEUE_LOCK_STRATEGIES.any? { |s| s.to_s == strategy.to_s }
|
|
1169
|
+
end
|
|
1170
|
+
|
|
1170
1171
|
# A row parked before the double-encoding fix stores a jsonb *string*
|
|
1171
1172
|
# holding the document — the same two-pass unwrap BlockedExecution
|
|
1172
1173
|
# .release_next! does.
|
|
@@ -1613,10 +1614,19 @@ module Pgbus
|
|
|
1613
1614
|
# Release the locks a collection of formatted messages hold, in a single
|
|
1614
1615
|
# query.
|
|
1615
1616
|
def release_locks_for_messages(messages)
|
|
1616
|
-
|
|
1617
|
+
release_lock_holders(lock_holders_for(messages))
|
|
1618
|
+
rescue StandardError => e
|
|
1619
|
+
Pgbus.logger.debug { "[Pgbus::Web] Error releasing locks for messages: #{e.message}" }
|
|
1620
|
+
end
|
|
1621
|
+
|
|
1622
|
+
def lock_holders_for(messages)
|
|
1623
|
+
messages.filter_map do |m|
|
|
1617
1624
|
key = extract_uniqueness_key_from_payload_str(m[:message])
|
|
1618
1625
|
[key, *lock_holder(m[:queue_name], m[:msg_id], m[:headers], m[:enqueued_at])] if key
|
|
1619
1626
|
end
|
|
1627
|
+
end
|
|
1628
|
+
|
|
1629
|
+
def release_lock_holders(holders)
|
|
1620
1630
|
UniquenessKey.release_all_held_by!(holders) if holders.any?
|
|
1621
1631
|
rescue StandardError => e
|
|
1622
1632
|
Pgbus.logger.debug { "[Pgbus::Web] Error releasing locks for messages: #{e.message}" }
|
|
@@ -1802,10 +1812,18 @@ module Pgbus
|
|
|
1802
1812
|
# Release all uniqueness keys associated with a queue before purge/drop.
|
|
1803
1813
|
# Scans queue messages for uniqueness metadata and deletes matching rows.
|
|
1804
1814
|
def release_uniqueness_keys_for_queue(queue_name)
|
|
1805
|
-
|
|
1806
|
-
|
|
1815
|
+
holders = queue_lock_holders(queue_name)
|
|
1816
|
+
release_lock_holders(holders) if holders
|
|
1817
|
+
end
|
|
1818
|
+
|
|
1819
|
+
# The uniqueness-lock holders of a queue's messages, or nil when they
|
|
1820
|
+
# cannot be read (the purge or drop goes ahead; the reaper frees what
|
|
1821
|
+
# is left).
|
|
1822
|
+
def queue_lock_holders(queue_name)
|
|
1823
|
+
lock_holders_for(query_queue_messages_raw(queue_name, 10_000, 0))
|
|
1807
1824
|
rescue StandardError => e
|
|
1808
|
-
Pgbus.logger.debug { "[Pgbus::Web] Error
|
|
1825
|
+
Pgbus.logger.debug { "[Pgbus::Web] Error reading uniqueness keys for queue #{queue_name}: #{e.message}" }
|
|
1826
|
+
nil
|
|
1809
1827
|
end
|
|
1810
1828
|
end
|
|
1811
1829
|
end
|
|
@@ -45,6 +45,12 @@ module Pgbus
|
|
|
45
45
|
|
|
46
46
|
def reason_for(block, attempts, max)
|
|
47
47
|
error_class = block["error_class"]
|
|
48
|
+
# A :while_executing copy skipped on every read (issue #629): no
|
|
49
|
+
# attempt ran, so "crashed before it could report" would mislead.
|
|
50
|
+
if block["reason"] == DeadLetterHeader::REASON_UNIQUENESS_CONFLICT && !error_class
|
|
51
|
+
return ["uniqueness_conflict", { attempts: attempts, max: max }]
|
|
52
|
+
end
|
|
53
|
+
|
|
48
54
|
unless error_class
|
|
49
55
|
key = block["source"] == "consumer" ? "event_no_error" : "no_error_recorded"
|
|
50
56
|
return [key, { attempts: attempts, max: max }]
|
|
@@ -62,21 +62,24 @@ module Pgbus
|
|
|
62
62
|
# existing caller is unchanged. The Dispatcher overrides it with the
|
|
63
63
|
# short streams_fanout_write_deadline_ms for hot-loop fanout writes,
|
|
64
64
|
# bounding head-of-line blocking on a slow client (issue #315 item 3).
|
|
65
|
-
|
|
65
|
+
#
|
|
66
|
+
# frames: msg_id → the frozen SSE frame, built once per wake by the
|
|
67
|
+
# Dispatcher for every connection on the stream (issue #621). An
|
|
68
|
+
# envelope the Hash does not hold (or no Hash: the connect replay, an
|
|
69
|
+
# in-flight buffer flush) is framed here, so the bytes are the same
|
|
70
|
+
# either way.
|
|
71
|
+
def enqueue(envelopes, deadline_ms: @write_deadline_ms, frames: nil)
|
|
66
72
|
written = []
|
|
67
73
|
envelopes.each do |envelope|
|
|
68
|
-
|
|
69
|
-
|
|
74
|
+
msg_id = envelope.msg_id
|
|
75
|
+
ephemeral = msg_id.negative?
|
|
76
|
+
next if !ephemeral && msg_id <= @last_msg_id_sent.get
|
|
70
77
|
|
|
71
|
-
bytes =
|
|
72
|
-
id: envelope.msg_id,
|
|
73
|
-
event: sse_event_for(envelope),
|
|
74
|
-
data: envelope.payload
|
|
75
|
-
)
|
|
78
|
+
bytes = frames&.[](msg_id) || frame_for(envelope)
|
|
76
79
|
|
|
77
80
|
result = @writer.write(self, bytes, deadline_ms: deadline_ms)
|
|
78
81
|
if result == :ok
|
|
79
|
-
@last_msg_id_sent.set(
|
|
82
|
+
@last_msg_id_sent.set(msg_id) unless ephemeral
|
|
80
83
|
@last_write_at = monotonic
|
|
81
84
|
written << envelope
|
|
82
85
|
else
|
|
@@ -88,11 +91,26 @@ module Pgbus
|
|
|
88
91
|
end
|
|
89
92
|
|
|
90
93
|
def write_comment(text)
|
|
91
|
-
|
|
94
|
+
write_raw(Pgbus::Streams::Envelope.comment(text))
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# The one raw writer: bytes that are already a whole SSE frame (the
|
|
98
|
+
# heartbeat's shared comment, the shutdown sentinel). A successful
|
|
99
|
+
# write counts as activity for the idle reaper; a failed one marks the
|
|
100
|
+
# connection dead. The Heartbeat passes the short fan-out deadline
|
|
101
|
+
# (issue #621); everything else keeps the connection's own.
|
|
102
|
+
def write_raw(bytes, deadline_ms: @write_deadline_ms)
|
|
103
|
+
result = @writer.write(self, bytes, deadline_ms: deadline_ms)
|
|
104
|
+
if result == :ok
|
|
105
|
+
@last_write_at = monotonic
|
|
106
|
+
else
|
|
107
|
+
mark_dead!
|
|
108
|
+
end
|
|
109
|
+
result
|
|
92
110
|
end
|
|
93
111
|
|
|
94
112
|
def write_sentinel(bytes)
|
|
95
|
-
|
|
113
|
+
write_raw(bytes)
|
|
96
114
|
end
|
|
97
115
|
|
|
98
116
|
# The pgbus:connected frame carrying this connection's id. Written by
|
|
@@ -100,7 +118,7 @@ module Pgbus
|
|
|
100
118
|
# seat joined (issue #521), so a client that acts on the id right
|
|
101
119
|
# after the event finds its row.
|
|
102
120
|
def write_connected
|
|
103
|
-
|
|
121
|
+
write_raw(Pgbus::Streams::Envelope.connected(id: @id))
|
|
104
122
|
end
|
|
105
123
|
|
|
106
124
|
def idle_for
|
|
@@ -143,23 +161,12 @@ module Pgbus
|
|
|
143
161
|
|
|
144
162
|
private
|
|
145
163
|
|
|
146
|
-
def
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
end
|
|
153
|
-
result
|
|
154
|
-
end
|
|
155
|
-
|
|
156
|
-
# The SSE event name for a frame: the envelope's typed event when
|
|
157
|
-
# present, else the default turbo-stream. Plain ReadAfter::Envelopes
|
|
158
|
-
# (no event field) and StreamEnvelopes with a nil event both fall
|
|
159
|
-
# back to the default. (issue #170)
|
|
160
|
-
def sse_event_for(envelope)
|
|
161
|
-
event = envelope.respond_to?(:event) ? envelope.event : nil
|
|
162
|
-
event && !event.to_s.empty? ? event.to_s : Pgbus::Streams::DEFAULT_SSE_EVENT
|
|
164
|
+
def frame_for(envelope)
|
|
165
|
+
Pgbus::Streams::Envelope.message(
|
|
166
|
+
id: envelope.msg_id,
|
|
167
|
+
event: Pgbus::Streams::Envelope.sse_event_for(envelope),
|
|
168
|
+
data: envelope.payload
|
|
169
|
+
)
|
|
163
170
|
end
|
|
164
171
|
|
|
165
172
|
def monotonic
|
|
@@ -49,20 +49,20 @@ module Pgbus
|
|
|
49
49
|
# (so the Dispatcher's safe_enqueue can call either class) but ignored:
|
|
50
50
|
# write_to_body is non-blocking under Falcon's fiber reactor, so there
|
|
51
51
|
# is no head-of-line blocking to bound here (issue #315 item 3).
|
|
52
|
-
|
|
52
|
+
# frames: as Connection#enqueue (issue #621). Ephemeral envelopes
|
|
53
|
+
# (negative msg_id) bypass the cursor and never advance it, as there.
|
|
54
|
+
def enqueue(envelopes, deadline_ms: @write_deadline_ms, frames: nil) # rubocop:disable Lint/UnusedMethodArgument
|
|
53
55
|
written = []
|
|
54
56
|
envelopes.each do |envelope|
|
|
55
|
-
|
|
57
|
+
msg_id = envelope.msg_id
|
|
58
|
+
ephemeral = msg_id.negative?
|
|
59
|
+
next if !ephemeral && msg_id <= @last_msg_id_sent.get
|
|
56
60
|
|
|
57
|
-
bytes =
|
|
58
|
-
id: envelope.msg_id,
|
|
59
|
-
event: sse_event_for(envelope),
|
|
60
|
-
data: envelope.payload
|
|
61
|
-
)
|
|
61
|
+
bytes = frames&.[](msg_id) || frame_for(envelope)
|
|
62
62
|
|
|
63
63
|
result = write_to_body(bytes)
|
|
64
64
|
if result == :ok
|
|
65
|
-
@last_msg_id_sent.set(
|
|
65
|
+
@last_msg_id_sent.set(msg_id) unless ephemeral
|
|
66
66
|
@last_write_at = monotonic
|
|
67
67
|
written << envelope
|
|
68
68
|
else
|
|
@@ -74,16 +74,28 @@ module Pgbus
|
|
|
74
74
|
end
|
|
75
75
|
|
|
76
76
|
def write_comment(text)
|
|
77
|
-
|
|
77
|
+
write_raw(Pgbus::Streams::Envelope.comment(text))
|
|
78
78
|
end
|
|
79
79
|
|
|
80
80
|
# See Connection#write_connected (issue #521).
|
|
81
81
|
def write_connected
|
|
82
|
-
|
|
82
|
+
write_raw(Pgbus::Streams::Envelope.connected(id: @id))
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# See Connection#write_raw (issue #621). deadline_ms is ignored, as in
|
|
86
|
+
# #enqueue: the body write does not block.
|
|
87
|
+
def write_raw(bytes, deadline_ms: @write_deadline_ms) # rubocop:disable Lint/UnusedMethodArgument
|
|
88
|
+
result = write_to_body(bytes)
|
|
89
|
+
if result == :ok
|
|
90
|
+
@last_write_at = monotonic
|
|
91
|
+
else
|
|
92
|
+
mark_dead!
|
|
93
|
+
end
|
|
94
|
+
result
|
|
83
95
|
end
|
|
84
96
|
|
|
85
97
|
def write_sentinel(bytes)
|
|
86
|
-
|
|
98
|
+
write_raw(bytes)
|
|
87
99
|
end
|
|
88
100
|
|
|
89
101
|
def close
|
|
@@ -112,20 +124,12 @@ module Pgbus
|
|
|
112
124
|
|
|
113
125
|
private
|
|
114
126
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
def write_frame(bytes)
|
|
122
|
-
result = write_to_body(bytes)
|
|
123
|
-
if result == :ok
|
|
124
|
-
@last_write_at = monotonic
|
|
125
|
-
else
|
|
126
|
-
mark_dead!
|
|
127
|
-
end
|
|
128
|
-
result
|
|
127
|
+
def frame_for(envelope)
|
|
128
|
+
Pgbus::Streams::Envelope.message(
|
|
129
|
+
id: envelope.msg_id,
|
|
130
|
+
event: Pgbus::Streams::Envelope.sse_event_for(envelope),
|
|
131
|
+
data: envelope.payload
|
|
132
|
+
)
|
|
129
133
|
end
|
|
130
134
|
|
|
131
135
|
def write_to_body(bytes)
|
|
@@ -10,7 +10,15 @@ module Pgbus
|
|
|
10
10
|
# connection. This keeps proxies and load balancers from timing
|
|
11
11
|
# out idle HTTP responses; most reverse proxies close HTTP
|
|
12
12
|
# responses that sit idle for 30-60s, which would silently drop
|
|
13
|
-
# SSE clients.
|
|
13
|
+
# SSE clients. The frame is built once per tick and shared by
|
|
14
|
+
# every connection, and each write runs under the short fan-out
|
|
15
|
+
# deadline (streams_fanout_write_deadline_ms, issue #621): the
|
|
16
|
+
# writes are serial on this one thread, and each socket write is
|
|
17
|
+
# bounded by 250 ms once it holds the connection's mutex, not by
|
|
18
|
+
# streams_write_deadline_ms (a concurrent fan-out or replay write
|
|
19
|
+
# on the same connection can still delay its turn). A client that
|
|
20
|
+
# cannot take the heartbeat comment in that window is marked dead
|
|
21
|
+
# and reconnects (durable streams replay from Last-Event-ID).
|
|
14
22
|
#
|
|
15
23
|
# 2. Mark connections that have been idle longer than the
|
|
16
24
|
# configured idle_timeout as dead. The StreamEventDispatcher's next pass
|
|
@@ -30,8 +38,10 @@ module Pgbus
|
|
|
30
38
|
# presence touch every tick, even with no presence connection here,
|
|
31
39
|
# so the dispatcher can expire seats left by other processes.
|
|
32
40
|
def initialize(registry:, dispatch_queue:, interval:, idle_timeout:, logger: Pgbus.logger, clock: nil,
|
|
33
|
-
presence_expiry: false
|
|
41
|
+
presence_expiry: false,
|
|
42
|
+
fanout_write_deadline_ms: Pgbus.configuration.streams_fanout_write_deadline_ms)
|
|
34
43
|
@registry = registry
|
|
44
|
+
@fanout_write_deadline_ms = fanout_write_deadline_ms
|
|
35
45
|
@queue = dispatch_queue
|
|
36
46
|
@interval = interval
|
|
37
47
|
@idle_timeout = idle_timeout
|
|
@@ -76,6 +86,7 @@ module Pgbus
|
|
|
76
86
|
# code goes through the background thread.
|
|
77
87
|
def tick
|
|
78
88
|
now = @clock.call
|
|
89
|
+
frame = Pgbus::Streams::Envelope.comment("heartbeat #{now.to_i}").freeze
|
|
79
90
|
presence_conns = []
|
|
80
91
|
@registry.each_connection do |connection|
|
|
81
92
|
if connection.dead?
|
|
@@ -91,7 +102,7 @@ module Pgbus
|
|
|
91
102
|
next
|
|
92
103
|
end
|
|
93
104
|
|
|
94
|
-
result = connection.
|
|
105
|
+
result = connection.write_raw(frame, deadline_ms: @fanout_write_deadline_ms)
|
|
95
106
|
if connection.dead? || result != :ok
|
|
96
107
|
enqueue_disconnect(connection)
|
|
97
108
|
next
|
|
@@ -76,7 +76,8 @@ module Pgbus
|
|
|
76
76
|
interval: @config.streams_heartbeat_interval,
|
|
77
77
|
idle_timeout: @config.streams_idle_timeout,
|
|
78
78
|
logger: @logger,
|
|
79
|
-
presence_expiry: Array(@config.streams_presence_patterns).any
|
|
79
|
+
presence_expiry: Array(@config.streams_presence_patterns).any?,
|
|
80
|
+
fanout_write_deadline_ms: @config.streams_fanout_write_deadline_ms
|
|
80
81
|
)
|
|
81
82
|
@peer_close_watcher = build_peer_close_watcher
|
|
82
83
|
@started = false
|
|
@@ -243,8 +244,10 @@ module Pgbus
|
|
|
243
244
|
@logger.warn { "[Pgbus::Streamer::Instance] component stop raised: #{e.class}: #{e.message}" }
|
|
244
245
|
end
|
|
245
246
|
|
|
246
|
-
# Captured BEFORE the stops:
|
|
247
|
-
# reference in #stop whether or not the join succeeded
|
|
247
|
+
# Captured BEFORE the stops: the other components nil their thread
|
|
248
|
+
# reference in #stop whether or not the join succeeded (the Listener
|
|
249
|
+
# keeps a thread that outlived its budget, so it cannot be restarted
|
|
250
|
+
# beside it).
|
|
248
251
|
def component_threads
|
|
249
252
|
{
|
|
250
253
|
"heartbeat" => @heartbeat.threads,
|
|
@@ -340,8 +343,8 @@ module Pgbus
|
|
|
340
343
|
when Proc
|
|
341
344
|
# The Proc branch in connection_options typically returns
|
|
342
345
|
# ActiveRecord::Base.connection.raw_connection — a pooled
|
|
343
|
-
# AR connection. That's fatal for the streamer: LISTEN and
|
|
344
|
-
#
|
|
346
|
+
# AR connection. That's fatal for the streamer: LISTEN and the
|
|
347
|
+
# listener's wait on its socket bind the session to the connection object,
|
|
345
348
|
# so if AR's pool hands the same raw_connection to another
|
|
346
349
|
# thread mid-wait, we get concurrent libpq calls → segfault
|
|
347
350
|
# or result corruption. Bail out with a clear error instead
|