pgbus 0.22.0 → 0.23.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +52 -0
  3. data/README.md +41 -14
  4. data/Rakefile +7 -1
  5. data/app/assets/javascripts/pgbus/stream_source_element.js +17 -3
  6. data/app/assets/javascripts/pgbus/stream_source_element.min.js +4 -4
  7. data/app/assets/javascripts/pgbus/stream_source_element.min.js.map +1 -1
  8. data/app/controllers/pgbus/dead_letter_controller.rb +3 -1
  9. data/app/controllers/pgbus/jobs_controller.rb +2 -1
  10. data/app/models/pgbus/blocked_execution.rb +3 -2
  11. data/app/models/pgbus/queue_state.rb +6 -0
  12. data/app/models/pgbus/uniqueness_key.rb +11 -0
  13. data/config/locales/da.yml +1 -0
  14. data/config/locales/de.yml +1 -0
  15. data/config/locales/en.yml +1 -0
  16. data/config/locales/es.yml +1 -0
  17. data/config/locales/fi.yml +1 -0
  18. data/config/locales/fr.yml +1 -0
  19. data/config/locales/it.yml +1 -0
  20. data/config/locales/ja.yml +1 -0
  21. data/config/locales/nb.yml +1 -0
  22. data/config/locales/nl.yml +1 -0
  23. data/config/locales/pt.yml +1 -0
  24. data/config/locales/sv.yml +1 -0
  25. data/lib/generators/pgbus/fold_queue_name_case_generator.rb +57 -0
  26. data/lib/generators/pgbus/templates/fold_queue_name_case.rb.erb +12 -0
  27. data/lib/generators/pgbus/templates/tune_autovacuum.rb.erb +2 -2
  28. data/lib/generators/pgbus/templates/tune_fillfactor.rb.erb +1 -1
  29. data/lib/pgbus/active_job/adapter.rb +3 -2
  30. data/lib/pgbus/active_job/executor.rb +142 -20
  31. data/lib/pgbus/active_job/quiet_logs.rb +57 -0
  32. data/lib/pgbus/autovacuum_tuning.rb +2 -2
  33. data/lib/pgbus/cli/dlq.rb +2 -1
  34. data/lib/pgbus/client/async_commit.rb +149 -0
  35. data/lib/pgbus/client/native_statements.rb +28 -16
  36. data/lib/pgbus/client.rb +181 -17
  37. data/lib/pgbus/concurrency/blocked_execution.rb +9 -1
  38. data/lib/pgbus/configuration.rb +71 -6
  39. data/lib/pgbus/dead_letter_header.rb +3 -0
  40. data/lib/pgbus/doctor.rb +55 -2
  41. data/lib/pgbus/failed_event_recorder.rb +5 -2
  42. data/lib/pgbus/generators/migration_detector.rb +29 -2
  43. data/lib/pgbus/instrumentation.rb +16 -1
  44. data/lib/pgbus/metrics/subscriber.rb +27 -1
  45. data/lib/pgbus/process/completer.rb +4 -2
  46. data/lib/pgbus/process/worker.rb +28 -3
  47. data/lib/pgbus/queue_name_fold.rb +145 -0
  48. data/lib/pgbus/queue_name_validator.rb +20 -2
  49. data/lib/pgbus/recurring/schedule.rb +22 -13
  50. data/lib/pgbus/stat_buffer.rb +67 -26
  51. data/lib/pgbus/streams/cursor.rb +17 -6
  52. data/lib/pgbus/streams/envelope.rb +16 -0
  53. data/lib/pgbus/streams/key.rb +1 -1
  54. data/lib/pgbus/table_maintenance.rb +1 -1
  55. data/lib/pgbus/uniqueness.rb +43 -16
  56. data/lib/pgbus/version.rb +1 -1
  57. data/lib/pgbus/web/data_source/dead_letter.rb +14 -5
  58. data/lib/pgbus/web/data_source/resend.rb +185 -0
  59. data/lib/pgbus/web/data_source/resend_slot.rb +139 -0
  60. data/lib/pgbus/web/data_source.rb +55 -37
  61. data/lib/pgbus/web/dead_letter_reason.rb +6 -0
  62. data/lib/pgbus/web/streamer/connection.rb +36 -29
  63. data/lib/pgbus/web/streamer/falcon_connection.rb +29 -25
  64. data/lib/pgbus/web/streamer/heartbeat.rb +14 -3
  65. data/lib/pgbus/web/streamer/instance.rb +8 -5
  66. data/lib/pgbus/web/streamer/listener.rb +239 -62
  67. data/lib/pgbus/web/streamer/outbound_pump.rb +6 -4
  68. data/lib/pgbus/web/streamer/registry.rb +55 -29
  69. data/lib/pgbus/web/streamer/stream_event_dispatcher.rb +73 -12
  70. data/lib/tasks/pgbus_queues.rake +27 -0
  71. data/sig/pgbus/active_job/adapter.rbs +1 -1
  72. data/sig/pgbus/active_job/quiet_logs.rbs +11 -0
  73. data/sig/pgbus/client.rbs +6 -5
  74. data/sig/pgbus/configuration.rbs +4 -0
  75. metadata +11 -3
@@ -0,0 +1,139 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ module Web
5
+ class DataSource
6
+ # The limits_concurrency half of a re-send (issue #636; Resend#resend_job
7
+ # is the whole). A re-sent copy signals its key's semaphore when it
8
+ # finishes, so it has to hold a slot first, taken the way
9
+ # Adapter#enqueue_with_concurrency takes one: the check and the park
10
+ # commit together under the semaphore row lock (rails/solid_queue#712),
11
+ # and the send follows the commit.
12
+ module ResendSlot
13
+ private
14
+
15
+ # Returns the block's msg_id once the copy is sent holding a slot,
16
+ # :parked when it waits behind a full limit (on_conflict: :block), and
17
+ # :refused when the limit is full under :discard or :raise. A payload
18
+ # with no concurrency key is sent as it is. +sending+ is :produce
19
+ # around the produce and :park around the park's commit, so the
20
+ # caller can tell an error that may hide a live copy from one that
21
+ # proves nothing went out.
22
+ def resend_holding_slot(payload, lock_queue, drop_source, sending, headers, &)
23
+ key = payload.is_a?(Hash) && Concurrency.extract_key(payload)
24
+ unless key
25
+ sending[0] = :produce
26
+ return yield
27
+ end
28
+
29
+ limits = resend_limits(payload)
30
+ case (taken = take_resend_slot(key, limits, lock_queue, park_payload(payload, headers), sending))
31
+ when :acquired then send_holding_resend_slot(key, sending, &)
32
+ when :full
33
+ Pgbus.logger.warn do
34
+ "[Pgbus::Web] Not re-sending #{payload["job_class"]}: concurrency limit for #{key} is full " \
35
+ "(on_conflict: :#{limits[:on_conflict]})"
36
+ end
37
+ :refused
38
+ else drop_parked_source(taken, key, drop_source)
39
+ end
40
+ end
41
+
42
+ # The limits live on the job class, not in the payload. A class that
43
+ # no longer declares them, or no longer resolves, is judged the way a
44
+ # promotion judges its parked job (Concurrency.config_for): against
45
+ # the limit the semaphore row records, parked when that is full.
46
+ def resend_limits(payload)
47
+ klass = Object.const_get(payload["job_class"].to_s)
48
+ config = klass.respond_to?(:pgbus_concurrency) && klass.pgbus_concurrency
49
+ config || { limit: nil, duration: Concurrency::DEFAULT_DURATION, on_conflict: :block }
50
+ rescue NameError
51
+ { limit: nil, duration: Concurrency::DEFAULT_DURATION, on_conflict: :block }
52
+ end
53
+
54
+ # :acquired, :full, or the id of the BlockedExecution the copy was
55
+ # parked as. The copy is sent right away, so its lease is the
56
+ # duration alone (Adapter#slot_lease with no delay). Parked on the
57
+ # logical queue at the job's own priority (the serialized "priority",
58
+ # else the default, as Adapter#handle_conflict parks it): promote_next
59
+ # sends it through Client#send_message, which prefixes the name and
60
+ # picks the _pN sub-queue by that priority. Once the park is inserted
61
+ # only its commit remains, and a COMMIT whose reply is lost may have
62
+ # landed: +sending+ says so until the transaction returns.
63
+ def take_resend_slot(key, limits, lock_queue, payload, sending)
64
+ taken = Pgbus::Semaphore.transaction(requires_new: true) do
65
+ lease = Concurrency.effective_duration(limits[:duration])
66
+ next :acquired if Concurrency::Semaphore.acquire(key, limits[:limit], lease) == :acquired
67
+ next :full unless limits[:on_conflict] == :block
68
+
69
+ Concurrency::BlockedExecution.insert(
70
+ concurrency_key: key, queue_name: lock_queue, payload: payload,
71
+ priority: payload["priority"] || Pgbus.configuration.default_priority, duration: limits[:duration]
72
+ ).rows.dig(0, 0).tap { sending[0] = :park }
73
+ end
74
+ sending[0] = nil
75
+ taken
76
+ end
77
+
78
+ # pgbus_blocked_executions has no headers column: the headers the
79
+ # copy would have been sent with (a DLQ retry's retries count) ride
80
+ # in the parked payload, and promote_next sends them as headers again
81
+ # (Concurrency::BlockedExecution::HEADERS_KEY).
82
+ def park_payload(payload, headers)
83
+ return payload if headers.nil? || headers.empty?
84
+
85
+ payload.merge(Concurrency::BlockedExecution::HEADERS_KEY => headers)
86
+ end
87
+
88
+ # Hand the slot back only when the failure proves nothing was
89
+ # produced (Adapter#send_holding_slot): on an ambiguous outcome the
90
+ # copy may be live, so the hold stands and the lease expiry plus the
91
+ # dispatcher's sweep reclaim it.
92
+ def send_holding_resend_slot(key, sending)
93
+ sending[0] = :produce
94
+ yield
95
+ rescue StandardError => e
96
+ if ambiguous_resend?(e)
97
+ Pgbus.logger.warn { "[Pgbus::Web] Re-send of #{key} may have been sent (#{e.class}); its concurrency slot stays held" }
98
+ else
99
+ begin
100
+ Concurrency::Semaphore.release(key)
101
+ rescue StandardError => release_error
102
+ Pgbus.logger.warn { "[Pgbus::Web] Giving back the concurrency slot of #{key} failed: #{release_error.message}" }
103
+ end
104
+ end
105
+ raise
106
+ end
107
+
108
+ # The park has committed; the source goes next (it lives on another
109
+ # connection, or after the commit, so the two cannot share a
110
+ # transaction). If that fails, the park is withdrawn while it still
111
+ # waits, and the error raised: nothing changed, the source stays. A
112
+ # removal whose outcome is unknown (a lost reply: its DELETE may have
113
+ # committed) keeps the park, since withdrawing it could leave neither
114
+ # copy; so does a park that cannot be withdrawn (already promoted: a
115
+ # live copy). Either way the source may stay, for the operator to
116
+ # discard: a duplicate is preferred over a lost job.
117
+ def drop_parked_source(park_id, key, drop_source)
118
+ drop_source.call
119
+ :parked
120
+ rescue StandardError => e
121
+ raise if !ambiguous_resend?(e) && withdraw_park(park_id)
122
+
123
+ Pgbus.logger.warn do
124
+ "[Pgbus::Web] Re-sent copy of #{key} is parked, but removing its source failed (#{e.class}: #{e.message}); " \
125
+ "the parked copy stands, and the source may stay to be discarded"
126
+ end
127
+ :parked
128
+ end
129
+
130
+ def withdraw_park(park_id)
131
+ Pgbus::BlockedExecution.where(id: park_id).delete_all.positive?
132
+ rescue StandardError => e
133
+ Pgbus.logger.warn { "[Pgbus::Web] Withdrawing the parked re-send #{park_id} failed: #{e.message}" }
134
+ false
135
+ end
136
+ end
137
+ end
138
+ end
139
+ end
@@ -7,6 +7,8 @@ module Pgbus
7
7
  class DataSource
8
8
  include JobList
9
9
  include DeadLetter
10
+ include Resend
11
+ include ResendSlot
10
12
  include ListCounts
11
13
  include QueueSummary
12
14
  include EventList
@@ -175,9 +177,17 @@ module Pgbus
175
177
  @client.purge_queue(name, prefixed: false)
176
178
  end
177
179
 
180
+ # The messages' uniqueness locks are released only once the table is
181
+ # gone: their holders are read just before the drop (the client runs
182
+ # the block only when it drops the table, not for a mixed-case
183
+ # duplicate from before issue #633, whose table and messages stay with
184
+ # the lower-case queue) and released after it succeeds, so a failed
185
+ # drop leaves both messages and locks.
178
186
  def drop_queue(name)
179
- release_uniqueness_keys_for_queue(name)
180
- @client.drop_queue(name, prefixed: false)
187
+ holders = nil
188
+ result = @client.drop_queue(name, prefixed: false) { holders = queue_lock_holders(name) }
189
+ release_lock_holders(holders) if holders
190
+ result
181
191
  end
182
192
 
183
193
  def pause_queue(name, reason: nil)
@@ -299,20 +309,7 @@ module Pgbus
299
309
  # duplicate (the original is still in the queue waiting for retry).
300
310
  # Falls back to enqueueing a fresh copy only if the original is gone
301
311
  # (e.g., already moved to DLQ).
302
- physical = failed_message_queue(event)
303
- if physical
304
- @client.set_visibility_timeout(physical, event["msg_id"].to_i, vt: 0, prefixed: false)
305
- else
306
- payload = JSON.parse(event["payload"])
307
- headers = event["headers"]
308
- headers = JSON.parse(headers) if headers.is_a?(String)
309
- @client.send_message(event["queue_name"], payload, headers: headers)
310
- end
311
-
312
- connection.exec_delete(
313
- "DELETE FROM pgbus_failed_events WHERE id = $1", "Pgbus Delete Failed Event", [id.to_i]
314
- )
315
- true
312
+ retry_failed_job(event)
316
313
  rescue StandardError => e
317
314
  Pgbus.logger.debug { "[Pgbus::Web] Error retrying failed event #{id}: #{e.message}" }
318
315
  false
@@ -336,27 +333,24 @@ module Pgbus
336
333
 
337
334
  # Failed jobs only, like discard_all_failed: an event's message is still
338
335
  # in its handler queue, so a re-sent copy would run the handler twice.
336
+ # Each job goes back the way Retry sends it (#retry_failed_job): a
337
+ # message still queued gets its vt reset, one that is gone is re-sent.
338
+ # A row it cannot retry (the re-send's key is held, its concurrency
339
+ # limit refuses it, or a call fails) stays, and the walk goes on past it
340
+ # by id.
339
341
  def retry_all_failed
340
342
  count = 0
343
+ last_id = 0
341
344
  loop do
342
345
  batch = connection.select_all(
343
- "SELECT * FROM pgbus_failed_events WHERE #{NOT_HANDLER_QUEUE} ORDER BY id LIMIT 100", "Pgbus Retry Batch",
344
- [handler_queue_names_literal]
346
+ "SELECT * FROM pgbus_failed_events WHERE #{NOT_HANDLER_QUEUE} AND id > $2 ORDER BY id LIMIT 100",
347
+ "Pgbus Retry Batch", [handler_queue_names_literal, last_id]
345
348
  ).to_a
346
349
  break if batch.empty?
347
350
 
348
351
  batch.each do |event|
349
- payload = JSON.parse(event["payload"])
350
- headers = event["headers"]
351
- headers = JSON.parse(headers) if headers.is_a?(String)
352
-
353
- connection.transaction do
354
- @client.send_message(event["queue_name"], payload, headers: headers)
355
- connection.exec_delete(
356
- "DELETE FROM pgbus_failed_events WHERE id = $1", "Pgbus Delete Failed Event", [event["id"].to_i]
357
- )
358
- end
359
- count += 1
352
+ last_id = event["id"].to_i
353
+ count += 1 if retry_failed_job(event)
360
354
  rescue StandardError => e
361
355
  Pgbus.logger.error { "[Pgbus::Web] Failed to retry event #{event["id"]}: #{e.message}" }
362
356
  end
@@ -634,9 +628,10 @@ module Pgbus
634
628
  # Drop every job parked behind a key. The jobs never run, so the
635
629
  # bookkeeping they would have resolved has to be resolved here: a parked
636
630
  # batch child is marked failed (otherwise its batch waits forever and
637
- # on_failure never fires — issue #413) and an :until_executed uniqueness
638
- # lock is released (otherwise it is orphaned, since no executor will ever
639
- # release it — issue #423). Returns the number of jobs discarded.
631
+ # on_failure never fires — issue #413) and an :until_executed or
632
+ # :until_executing uniqueness lock is released (otherwise it is
633
+ # orphaned, since no executor will ever release it — issue #423).
634
+ # Returns the number of jobs discarded.
640
635
  def discard_parked_jobs(key)
641
636
  return 0 if key.to_s.strip.empty?
642
637
 
@@ -1162,11 +1157,17 @@ module Pgbus
1162
1157
  # one acquired after this row was parked.
1163
1158
  def release_parked_uniqueness_lock(payload, parked_at)
1164
1159
  key = payload[Uniqueness::METADATA_KEY]
1165
- return unless key && payload[Uniqueness::STRATEGY_KEY].to_s == "until_executed"
1160
+ return unless key && enqueue_locked_strategy?(payload[Uniqueness::STRATEGY_KEY])
1166
1161
 
1167
1162
  UniquenessKey.release_if_unbound!(key, acquired_before: parked_at)
1168
1163
  end
1169
1164
 
1165
+ # The strategies whose row is taken at enqueue and so is held by a parked
1166
+ # job: :until_executed, and :until_executing (issue #628).
1167
+ def enqueue_locked_strategy?(strategy)
1168
+ Uniqueness::ENQUEUE_LOCK_STRATEGIES.any? { |s| s.to_s == strategy.to_s }
1169
+ end
1170
+
1170
1171
  # A row parked before the double-encoding fix stores a jsonb *string*
1171
1172
  # holding the document — the same two-pass unwrap BlockedExecution
1172
1173
  # .release_next! does.
@@ -1613,10 +1614,19 @@ module Pgbus
1613
1614
  # Release the locks a collection of formatted messages hold, in a single
1614
1615
  # query.
1615
1616
  def release_locks_for_messages(messages)
1616
- holders = messages.filter_map do |m|
1617
+ release_lock_holders(lock_holders_for(messages))
1618
+ rescue StandardError => e
1619
+ Pgbus.logger.debug { "[Pgbus::Web] Error releasing locks for messages: #{e.message}" }
1620
+ end
1621
+
1622
+ def lock_holders_for(messages)
1623
+ messages.filter_map do |m|
1617
1624
  key = extract_uniqueness_key_from_payload_str(m[:message])
1618
1625
  [key, *lock_holder(m[:queue_name], m[:msg_id], m[:headers], m[:enqueued_at])] if key
1619
1626
  end
1627
+ end
1628
+
1629
+ def release_lock_holders(holders)
1620
1630
  UniquenessKey.release_all_held_by!(holders) if holders.any?
1621
1631
  rescue StandardError => e
1622
1632
  Pgbus.logger.debug { "[Pgbus::Web] Error releasing locks for messages: #{e.message}" }
@@ -1802,10 +1812,18 @@ module Pgbus
1802
1812
  # Release all uniqueness keys associated with a queue before purge/drop.
1803
1813
  # Scans queue messages for uniqueness metadata and deletes matching rows.
1804
1814
  def release_uniqueness_keys_for_queue(queue_name)
1805
- messages = query_queue_messages_raw(queue_name, 10_000, 0)
1806
- release_locks_for_messages(messages)
1815
+ holders = queue_lock_holders(queue_name)
1816
+ release_lock_holders(holders) if holders
1817
+ end
1818
+
1819
+ # The uniqueness-lock holders of a queue's messages, or nil when they
1820
+ # cannot be read (the purge or drop goes ahead; the reaper frees what
1821
+ # is left).
1822
+ def queue_lock_holders(queue_name)
1823
+ lock_holders_for(query_queue_messages_raw(queue_name, 10_000, 0))
1807
1824
  rescue StandardError => e
1808
- Pgbus.logger.debug { "[Pgbus::Web] Error releasing uniqueness keys for queue #{queue_name}: #{e.message}" }
1825
+ Pgbus.logger.debug { "[Pgbus::Web] Error reading uniqueness keys for queue #{queue_name}: #{e.message}" }
1826
+ nil
1809
1827
  end
1810
1828
  end
1811
1829
  end
@@ -45,6 +45,12 @@ module Pgbus
45
45
 
46
46
  def reason_for(block, attempts, max)
47
47
  error_class = block["error_class"]
48
+ # A :while_executing copy skipped on every read (issue #629): no
49
+ # attempt ran, so "crashed before it could report" would mislead.
50
+ if block["reason"] == DeadLetterHeader::REASON_UNIQUENESS_CONFLICT && !error_class
51
+ return ["uniqueness_conflict", { attempts: attempts, max: max }]
52
+ end
53
+
48
54
  unless error_class
49
55
  key = block["source"] == "consumer" ? "event_no_error" : "no_error_recorded"
50
56
  return [key, { attempts: attempts, max: max }]
@@ -62,21 +62,24 @@ module Pgbus
62
62
  # existing caller is unchanged. The Dispatcher overrides it with the
63
63
  # short streams_fanout_write_deadline_ms for hot-loop fanout writes,
64
64
  # bounding head-of-line blocking on a slow client (issue #315 item 3).
65
- def enqueue(envelopes, deadline_ms: @write_deadline_ms)
65
+ #
66
+ # frames: msg_id → the frozen SSE frame, built once per wake by the
67
+ # Dispatcher for every connection on the stream (issue #621). An
68
+ # envelope the Hash does not hold (or no Hash: the connect replay, an
69
+ # in-flight buffer flush) is framed here, so the bytes are the same
70
+ # either way.
71
+ def enqueue(envelopes, deadline_ms: @write_deadline_ms, frames: nil)
66
72
  written = []
67
73
  envelopes.each do |envelope|
68
- ephemeral = envelope.msg_id.negative?
69
- next if !ephemeral && envelope.msg_id <= @last_msg_id_sent.get
74
+ msg_id = envelope.msg_id
75
+ ephemeral = msg_id.negative?
76
+ next if !ephemeral && msg_id <= @last_msg_id_sent.get
70
77
 
71
- bytes = Pgbus::Streams::Envelope.message(
72
- id: envelope.msg_id,
73
- event: sse_event_for(envelope),
74
- data: envelope.payload
75
- )
78
+ bytes = frames&.[](msg_id) || frame_for(envelope)
76
79
 
77
80
  result = @writer.write(self, bytes, deadline_ms: deadline_ms)
78
81
  if result == :ok
79
- @last_msg_id_sent.set(envelope.msg_id) unless ephemeral
82
+ @last_msg_id_sent.set(msg_id) unless ephemeral
80
83
  @last_write_at = monotonic
81
84
  written << envelope
82
85
  else
@@ -88,11 +91,26 @@ module Pgbus
88
91
  end
89
92
 
90
93
  def write_comment(text)
91
- write_frame(Pgbus::Streams::Envelope.comment(text))
94
+ write_raw(Pgbus::Streams::Envelope.comment(text))
95
+ end
96
+
97
+ # The one raw writer: bytes that are already a whole SSE frame (the
98
+ # heartbeat's shared comment, the shutdown sentinel). A successful
99
+ # write counts as activity for the idle reaper; a failed one marks the
100
+ # connection dead. The Heartbeat passes the short fan-out deadline
101
+ # (issue #621); everything else keeps the connection's own.
102
+ def write_raw(bytes, deadline_ms: @write_deadline_ms)
103
+ result = @writer.write(self, bytes, deadline_ms: deadline_ms)
104
+ if result == :ok
105
+ @last_write_at = monotonic
106
+ else
107
+ mark_dead!
108
+ end
109
+ result
92
110
  end
93
111
 
94
112
  def write_sentinel(bytes)
95
- @writer.write(self, bytes, deadline_ms: @write_deadline_ms)
113
+ write_raw(bytes)
96
114
  end
97
115
 
98
116
  # The pgbus:connected frame carrying this connection's id. Written by
@@ -100,7 +118,7 @@ module Pgbus
100
118
  # seat joined (issue #521), so a client that acts on the id right
101
119
  # after the event finds its row.
102
120
  def write_connected
103
- write_frame(Pgbus::Streams::Envelope.connected(id: @id))
121
+ write_raw(Pgbus::Streams::Envelope.connected(id: @id))
104
122
  end
105
123
 
106
124
  def idle_for
@@ -143,23 +161,12 @@ module Pgbus
143
161
 
144
162
  private
145
163
 
146
- def write_frame(bytes)
147
- result = @writer.write(self, bytes, deadline_ms: @write_deadline_ms)
148
- if result == :ok
149
- @last_write_at = monotonic
150
- else
151
- mark_dead!
152
- end
153
- result
154
- end
155
-
156
- # The SSE event name for a frame: the envelope's typed event when
157
- # present, else the default turbo-stream. Plain ReadAfter::Envelopes
158
- # (no event field) and StreamEnvelopes with a nil event both fall
159
- # back to the default. (issue #170)
160
- def sse_event_for(envelope)
161
- event = envelope.respond_to?(:event) ? envelope.event : nil
162
- event && !event.to_s.empty? ? event.to_s : Pgbus::Streams::DEFAULT_SSE_EVENT
164
+ def frame_for(envelope)
165
+ Pgbus::Streams::Envelope.message(
166
+ id: envelope.msg_id,
167
+ event: Pgbus::Streams::Envelope.sse_event_for(envelope),
168
+ data: envelope.payload
169
+ )
163
170
  end
164
171
 
165
172
  def monotonic
@@ -49,20 +49,20 @@ module Pgbus
49
49
  # (so the Dispatcher's safe_enqueue can call either class) but ignored:
50
50
  # write_to_body is non-blocking under Falcon's fiber reactor, so there
51
51
  # is no head-of-line blocking to bound here (issue #315 item 3).
52
- def enqueue(envelopes, deadline_ms: @write_deadline_ms) # rubocop:disable Lint/UnusedMethodArgument
52
+ # frames: as Connection#enqueue (issue #621). Ephemeral envelopes
53
+ # (negative msg_id) bypass the cursor and never advance it, as there.
54
+ def enqueue(envelopes, deadline_ms: @write_deadline_ms, frames: nil) # rubocop:disable Lint/UnusedMethodArgument
53
55
  written = []
54
56
  envelopes.each do |envelope|
55
- next if envelope.msg_id <= @last_msg_id_sent.get
57
+ msg_id = envelope.msg_id
58
+ ephemeral = msg_id.negative?
59
+ next if !ephemeral && msg_id <= @last_msg_id_sent.get
56
60
 
57
- bytes = Pgbus::Streams::Envelope.message(
58
- id: envelope.msg_id,
59
- event: sse_event_for(envelope),
60
- data: envelope.payload
61
- )
61
+ bytes = frames&.[](msg_id) || frame_for(envelope)
62
62
 
63
63
  result = write_to_body(bytes)
64
64
  if result == :ok
65
- @last_msg_id_sent.set(envelope.msg_id)
65
+ @last_msg_id_sent.set(msg_id) unless ephemeral
66
66
  @last_write_at = monotonic
67
67
  written << envelope
68
68
  else
@@ -74,16 +74,28 @@ module Pgbus
74
74
  end
75
75
 
76
76
  def write_comment(text)
77
- write_frame(Pgbus::Streams::Envelope.comment(text))
77
+ write_raw(Pgbus::Streams::Envelope.comment(text))
78
78
  end
79
79
 
80
80
  # See Connection#write_connected (issue #521).
81
81
  def write_connected
82
- write_frame(Pgbus::Streams::Envelope.connected(id: @id))
82
+ write_raw(Pgbus::Streams::Envelope.connected(id: @id))
83
+ end
84
+
85
+ # See Connection#write_raw (issue #621). deadline_ms is ignored, as in
86
+ # #enqueue: the body write does not block.
87
+ def write_raw(bytes, deadline_ms: @write_deadline_ms) # rubocop:disable Lint/UnusedMethodArgument
88
+ result = write_to_body(bytes)
89
+ if result == :ok
90
+ @last_write_at = monotonic
91
+ else
92
+ mark_dead!
93
+ end
94
+ result
83
95
  end
84
96
 
85
97
  def write_sentinel(bytes)
86
- write_to_body(bytes)
98
+ write_raw(bytes)
87
99
  end
88
100
 
89
101
  def close
@@ -112,20 +124,12 @@ module Pgbus
112
124
 
113
125
  private
114
126
 
115
- # See Connection#sse_event_for. (issue #170)
116
- def sse_event_for(envelope)
117
- event = envelope.respond_to?(:event) ? envelope.event : nil
118
- event && !event.to_s.empty? ? event.to_s : Pgbus::Streams::DEFAULT_SSE_EVENT
119
- end
120
-
121
- def write_frame(bytes)
122
- result = write_to_body(bytes)
123
- if result == :ok
124
- @last_write_at = monotonic
125
- else
126
- mark_dead!
127
- end
128
- result
127
+ def frame_for(envelope)
128
+ Pgbus::Streams::Envelope.message(
129
+ id: envelope.msg_id,
130
+ event: Pgbus::Streams::Envelope.sse_event_for(envelope),
131
+ data: envelope.payload
132
+ )
129
133
  end
130
134
 
131
135
  def write_to_body(bytes)
@@ -10,7 +10,15 @@ module Pgbus
10
10
  # connection. This keeps proxies and load balancers from timing
11
11
  # out idle HTTP responses; most reverse proxies close HTTP
12
12
  # responses that sit idle for 30-60s, which would silently drop
13
- # SSE clients.
13
+ # SSE clients. The frame is built once per tick and shared by
14
+ # every connection, and each write runs under the short fan-out
15
+ # deadline (streams_fanout_write_deadline_ms, issue #621): the
16
+ # writes are serial on this one thread, and each socket write is
17
+ # bounded by 250 ms once it holds the connection's mutex, not by
18
+ # streams_write_deadline_ms (a concurrent fan-out or replay write
19
+ # on the same connection can still delay its turn). A client that
20
+ # cannot take the heartbeat comment in that window is marked dead
21
+ # and reconnects (durable streams replay from Last-Event-ID).
14
22
  #
15
23
  # 2. Mark connections that have been idle longer than the
16
24
  # configured idle_timeout as dead. The StreamEventDispatcher's next pass
@@ -30,8 +38,10 @@ module Pgbus
30
38
  # presence touch every tick, even with no presence connection here,
31
39
  # so the dispatcher can expire seats left by other processes.
32
40
  def initialize(registry:, dispatch_queue:, interval:, idle_timeout:, logger: Pgbus.logger, clock: nil,
33
- presence_expiry: false)
41
+ presence_expiry: false,
42
+ fanout_write_deadline_ms: Pgbus.configuration.streams_fanout_write_deadline_ms)
34
43
  @registry = registry
44
+ @fanout_write_deadline_ms = fanout_write_deadline_ms
35
45
  @queue = dispatch_queue
36
46
  @interval = interval
37
47
  @idle_timeout = idle_timeout
@@ -76,6 +86,7 @@ module Pgbus
76
86
  # code goes through the background thread.
77
87
  def tick
78
88
  now = @clock.call
89
+ frame = Pgbus::Streams::Envelope.comment("heartbeat #{now.to_i}").freeze
79
90
  presence_conns = []
80
91
  @registry.each_connection do |connection|
81
92
  if connection.dead?
@@ -91,7 +102,7 @@ module Pgbus
91
102
  next
92
103
  end
93
104
 
94
- result = connection.write_comment("heartbeat #{now.to_i}")
105
+ result = connection.write_raw(frame, deadline_ms: @fanout_write_deadline_ms)
95
106
  if connection.dead? || result != :ok
96
107
  enqueue_disconnect(connection)
97
108
  next
@@ -76,7 +76,8 @@ module Pgbus
76
76
  interval: @config.streams_heartbeat_interval,
77
77
  idle_timeout: @config.streams_idle_timeout,
78
78
  logger: @logger,
79
- presence_expiry: Array(@config.streams_presence_patterns).any?
79
+ presence_expiry: Array(@config.streams_presence_patterns).any?,
80
+ fanout_write_deadline_ms: @config.streams_fanout_write_deadline_ms
80
81
  )
81
82
  @peer_close_watcher = build_peer_close_watcher
82
83
  @started = false
@@ -243,8 +244,10 @@ module Pgbus
243
244
  @logger.warn { "[Pgbus::Streamer::Instance] component stop raised: #{e.class}: #{e.message}" }
244
245
  end
245
246
 
246
- # Captured BEFORE the stops: every component nils its thread
247
- # reference in #stop whether or not the join succeeded.
247
+ # Captured BEFORE the stops: the other components nil their thread
248
+ # reference in #stop whether or not the join succeeded (the Listener
249
+ # keeps a thread that outlived its budget, so it cannot be restarted
250
+ # beside it).
248
251
  def component_threads
249
252
  {
250
253
  "heartbeat" => @heartbeat.threads,
@@ -340,8 +343,8 @@ module Pgbus
340
343
  when Proc
341
344
  # The Proc branch in connection_options typically returns
342
345
  # ActiveRecord::Base.connection.raw_connection — a pooled
343
- # AR connection. That's fatal for the streamer: LISTEN and
344
- # wait_for_notify bind the session to the connection object,
346
+ # AR connection. That's fatal for the streamer: LISTEN and the
347
+ # listener's wait on its socket bind the session to the connection object,
345
348
  # so if AR's pool hands the same raw_connection to another
346
349
  # thread mid-wait, we get concurrent libpq calls → segfault
347
350
  # or result corruption. Bail out with a clear error instead