pgbus 0.9.0 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/app/assets/javascripts/pgbus/stream_source_element.js +150 -5
- data/app/views/pgbus/batches/_batches_table.html.erb +2 -1
- data/lib/pgbus/configuration.rb +107 -0
- data/lib/pgbus/execution_pools/async_pool.rb +44 -6
- data/lib/pgbus/process/notify_listener.rb +268 -0
- data/lib/pgbus/process/worker.rb +76 -3
- data/lib/pgbus/streams/coalescer.rb +88 -0
- data/lib/pgbus/streams/envelope.rb +20 -1
- data/lib/pgbus/streams/key.rb +35 -2
- data/lib/pgbus/streams/renderer.rb +67 -0
- data/lib/pgbus/streams.rb +150 -1
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/stream_app.rb +8 -4
- data/lib/pgbus/web/streamer/connection.rb +15 -1
- data/lib/pgbus/web/streamer/falcon_connection.rb +9 -1
- data/lib/pgbus/web/streamer/heartbeat.rb +23 -1
- data/lib/pgbus/web/streamer/stream_event_dispatcher.rb +129 -14
- data/lib/pgbus.rb +11 -0
- metadata +4 -1
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Process
|
|
5
|
+
# Owns a single dedicated PG::Connection that LISTENs on the INSERT NOTIFY
|
|
6
|
+
# channel of every queue a Worker/Consumer reads, and fires a WakeSignal the
|
|
7
|
+
# moment any of them receives a row. This converts the worker/consumer loop
|
|
8
|
+
# from "blind-read every polling_interval" into "sleep until a real insert,
|
|
9
|
+
# poll only as a fallback" — eliminating the empty-read storm that dominates
|
|
10
|
+
# DB load on idle queues.
|
|
11
|
+
#
|
|
12
|
+
# pgmq-ruby's `wait_for_notify(queue, timeout:)` is single-queue and wraps
|
|
13
|
+
# the wait in `with_connection`, which only watches one channel and holds the
|
|
14
|
+
# pooled connection for the whole wait. Neither fits a worker that reads N
|
|
15
|
+
# queues on a small shared pool. So we own ONE raw PG::Connection and
|
|
16
|
+
# hand-roll per-channel LISTEN on it.
|
|
17
|
+
#
|
|
18
|
+
# A persistent LISTEN connection silently dies under a transaction-pool
|
|
19
|
+
# PgBouncer (LISTEN does not survive COMMIT boundaries). Point this
|
|
20
|
+
# connection at a DIRECT port via `config.worker_notify_*` overrides.
|
|
21
|
+
# The health-check-on-timeout catches a connection killed out from under us
|
|
22
|
+
# and re-LISTENs everything.
|
|
23
|
+
#
|
|
24
|
+
# NOTIFY channel naming (pgmq trigger): PG_NOTIFY('pgmq.' || table || '.' ||
|
|
25
|
+
# TG_OP). For queue `pgbus_default` the table is `q_pgbus_default`, so the
|
|
26
|
+
# channel is `pgmq.q_pgbus_default.INSERT`.
|
|
27
|
+
#
|
|
28
|
+
# Thread safety: @running, @conn, and @listening_to are guarded by
|
|
29
|
+
# @state_mutex. The listener thread owns @conn during wait_for_notify (a
|
|
30
|
+
# blocking IO call where the mutex MUST NOT be held), so wait_once reads
|
|
31
|
+
# the connection out of the mutex first and operates on a local. Reconnect
|
|
32
|
+
# publishes the new connection + channel set under the mutex.
|
|
33
|
+
class NotifyListener
|
|
34
|
+
CHANNEL_PREFIX = "pgmq.q_"
|
|
35
|
+
CHANNEL_SUFFIX = ".INSERT"
|
|
36
|
+
|
|
37
|
+
RECONNECT_BACKOFF_SECONDS = 0.5
|
|
38
|
+
|
|
39
|
+
def initialize(physical_queues:, on_wake:, connection_options:,
|
|
40
|
+
health_check_ms: 1000, logger: Pgbus.logger)
|
|
41
|
+
@physical_queues = Array(physical_queues)
|
|
42
|
+
@on_wake = on_wake
|
|
43
|
+
@connection_options = connection_options
|
|
44
|
+
@health_check_ms = health_check_ms
|
|
45
|
+
@logger = logger
|
|
46
|
+
@state_mutex = Mutex.new
|
|
47
|
+
@listening_to = Set.new
|
|
48
|
+
@commands = Queue.new
|
|
49
|
+
@running = false
|
|
50
|
+
@thread = nil
|
|
51
|
+
@conn = nil
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def listening_to
|
|
55
|
+
@state_mutex.synchronize { @listening_to.dup }
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def start
|
|
59
|
+
@state_mutex.synchronize do
|
|
60
|
+
return self if @running
|
|
61
|
+
|
|
62
|
+
@running = true
|
|
63
|
+
end
|
|
64
|
+
@physical_queues.each { |q| @commands << [:listen, q] }
|
|
65
|
+
@thread = Thread.new { run_loop }
|
|
66
|
+
self
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def stop
|
|
70
|
+
conn_to_close = nil
|
|
71
|
+
@state_mutex.synchronize do
|
|
72
|
+
return self unless @running
|
|
73
|
+
|
|
74
|
+
@running = false
|
|
75
|
+
conn_to_close = @conn
|
|
76
|
+
end
|
|
77
|
+
@commands << [:stop]
|
|
78
|
+
# Interrupt the blocking wait by closing the socket; the rescue in
|
|
79
|
+
# wait_once sees @running == false and exits cleanly.
|
|
80
|
+
begin
|
|
81
|
+
conn_to_close&.close if conn_to_close.respond_to?(:close)
|
|
82
|
+
rescue StandardError
|
|
83
|
+
nil
|
|
84
|
+
end
|
|
85
|
+
@thread&.join(5)
|
|
86
|
+
@thread = nil
|
|
87
|
+
self
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def add_queue(physical_queue)
|
|
91
|
+
@commands << [:listen, physical_queue]
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def remove_queue(physical_queue)
|
|
95
|
+
@commands << [:unlisten, physical_queue]
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
private
|
|
99
|
+
|
|
100
|
+
def running?
|
|
101
|
+
@state_mutex.synchronize { @running }
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def run_loop
|
|
105
|
+
conn = build_connection
|
|
106
|
+
@state_mutex.synchronize { @conn = conn }
|
|
107
|
+
drain_commands
|
|
108
|
+
|
|
109
|
+
loop do
|
|
110
|
+
break unless running?
|
|
111
|
+
|
|
112
|
+
drain_commands
|
|
113
|
+
break unless running?
|
|
114
|
+
|
|
115
|
+
wait_once
|
|
116
|
+
end
|
|
117
|
+
rescue StandardError => e
|
|
118
|
+
@logger.error { "[Pgbus::NotifyListener] fatal: #{e.class}: #{e.message}" } if running?
|
|
119
|
+
ensure
|
|
120
|
+
# Clear @running so #start can spawn a fresh thread after a fatal exit
|
|
121
|
+
# (e.g. build_connection raising at boot). Without this, the dead
|
|
122
|
+
# thread's @running stays true and #start returns early forever.
|
|
123
|
+
@state_mutex.synchronize { @running = false }
|
|
124
|
+
safe_unlisten_all
|
|
125
|
+
safe_close
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def wait_once
|
|
129
|
+
conn = @state_mutex.synchronize { @conn }
|
|
130
|
+
return reconnect! unless conn
|
|
131
|
+
|
|
132
|
+
timeout_s = @health_check_ms / 1000.0
|
|
133
|
+
got_notify = conn.wait_for_notify(timeout_s) do |_channel, _pid, _payload|
|
|
134
|
+
@on_wake.call
|
|
135
|
+
end
|
|
136
|
+
run_health_check(conn) unless got_notify
|
|
137
|
+
rescue IOError, PG::Error => e
|
|
138
|
+
return unless running?
|
|
139
|
+
|
|
140
|
+
@logger.warn { "[Pgbus::NotifyListener] connection error (#{e.class}: #{e.message}) — reconnecting" }
|
|
141
|
+
reconnect!
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def drain_commands
|
|
145
|
+
loop do
|
|
146
|
+
cmd = @commands.pop(true)
|
|
147
|
+
case cmd[0]
|
|
148
|
+
when :listen then do_listen(cmd[1])
|
|
149
|
+
when :unlisten then do_unlisten(cmd[1])
|
|
150
|
+
when :stop
|
|
151
|
+
@state_mutex.synchronize { @running = false }
|
|
152
|
+
return
|
|
153
|
+
end
|
|
154
|
+
rescue ThreadError
|
|
155
|
+
return
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def do_listen(physical_queue)
|
|
160
|
+
channel = channel_for(physical_queue)
|
|
161
|
+
conn = @state_mutex.synchronize do
|
|
162
|
+
return if @listening_to.include?(channel)
|
|
163
|
+
|
|
164
|
+
@conn
|
|
165
|
+
end
|
|
166
|
+
return unless conn
|
|
167
|
+
|
|
168
|
+
conn.exec(%(LISTEN "#{channel}"))
|
|
169
|
+
@state_mutex.synchronize { @listening_to.add(channel) }
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
def do_unlisten(physical_queue)
|
|
173
|
+
channel = channel_for(physical_queue)
|
|
174
|
+
conn = @state_mutex.synchronize do
|
|
175
|
+
return unless @listening_to.include?(channel)
|
|
176
|
+
|
|
177
|
+
@conn
|
|
178
|
+
end
|
|
179
|
+
return unless conn
|
|
180
|
+
|
|
181
|
+
conn.exec(%(UNLISTEN "#{channel}"))
|
|
182
|
+
@state_mutex.synchronize { @listening_to.delete(channel) }
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def run_health_check(conn)
|
|
186
|
+
conn.exec("SELECT 1")
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
# Retry reconnect until either we succeed (new conn + every channel
|
|
190
|
+
# re-LISTENed) or @running flips to false. Without the loop, a single
|
|
191
|
+
# PG::Error during build/LISTEN left @conn nil and the listener degraded
|
|
192
|
+
# silently — wait_once would re-enter and fail forever or run with an
|
|
193
|
+
# incomplete subscription set.
|
|
194
|
+
def reconnect!
|
|
195
|
+
channels = @state_mutex.synchronize { @listening_to.to_a }
|
|
196
|
+
loop do
|
|
197
|
+
return unless running?
|
|
198
|
+
|
|
199
|
+
safe_close
|
|
200
|
+
new_conn = nil
|
|
201
|
+
begin
|
|
202
|
+
new_conn = build_connection
|
|
203
|
+
channels.each { |channel| new_conn.exec(%(LISTEN "#{channel}")) }
|
|
204
|
+
rescue PG::Error => e
|
|
205
|
+
# build_connection may have succeeded before a later LISTEN raised.
|
|
206
|
+
# Without this close, the partially-built conn is orphaned and the
|
|
207
|
+
# next retry just allocates another one — leaking PG connections on
|
|
208
|
+
# repeated failures.
|
|
209
|
+
close_quietly(new_conn)
|
|
210
|
+
@logger.error { "[Pgbus::NotifyListener] reconnect failed: #{e.class}: #{e.message}" }
|
|
211
|
+
sleep RECONNECT_BACKOFF_SECONDS
|
|
212
|
+
next
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
@state_mutex.synchronize do
|
|
216
|
+
@conn = new_conn
|
|
217
|
+
@listening_to = Set.new(channels)
|
|
218
|
+
end
|
|
219
|
+
return
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def build_connection
|
|
224
|
+
require "pg" unless defined?(::PG::Connection)
|
|
225
|
+
case @connection_options
|
|
226
|
+
when String then ::PG.connect(@connection_options)
|
|
227
|
+
when Hash then ::PG.connect(**@connection_options)
|
|
228
|
+
else
|
|
229
|
+
raise Pgbus::ConfigurationError,
|
|
230
|
+
"NotifyListener cannot build a PG connection from #{@connection_options.class}. " \
|
|
231
|
+
"Set worker_notify_database_url / worker_notify_host / worker_notify_port, " \
|
|
232
|
+
"or a base database_url, so the listener owns a dedicated connection."
|
|
233
|
+
end
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
def safe_unlisten_all
|
|
237
|
+
channels, conn = @state_mutex.synchronize { [@listening_to.to_a, @conn] }
|
|
238
|
+
channels.each do |channel|
|
|
239
|
+
conn&.exec(%(UNLISTEN "#{channel}"))
|
|
240
|
+
rescue PG::Error
|
|
241
|
+
nil
|
|
242
|
+
end
|
|
243
|
+
@state_mutex.synchronize { @listening_to.clear }
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def safe_close
|
|
247
|
+
conn = @state_mutex.synchronize do
|
|
248
|
+
c = @conn
|
|
249
|
+
@conn = nil
|
|
250
|
+
c
|
|
251
|
+
end
|
|
252
|
+
close_quietly(conn)
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# Close an unpublished PG::Connection (one that never made it into @conn,
|
|
256
|
+
# e.g. a half-built reconnect attempt where LISTEN raised). Best-effort.
|
|
257
|
+
def close_quietly(conn)
|
|
258
|
+
conn&.close if conn.respond_to?(:close)
|
|
259
|
+
rescue StandardError
|
|
260
|
+
nil
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
def channel_for(physical_queue)
|
|
264
|
+
"#{CHANNEL_PREFIX}#{physical_queue}#{CHANNEL_SUFFIX}"
|
|
265
|
+
end
|
|
266
|
+
end
|
|
267
|
+
end
|
|
268
|
+
end
|
data/lib/pgbus/process/worker.rb
CHANGED
|
@@ -13,6 +13,7 @@ module Pgbus
|
|
|
13
13
|
single_active_consumer: false, consumer_priority: 0,
|
|
14
14
|
execution_mode: :threads, group_mode: nil)
|
|
15
15
|
@queues = Array(queues)
|
|
16
|
+
@initial_queues = @queues.dup.freeze
|
|
16
17
|
@wildcard = @queues.include?("*")
|
|
17
18
|
@threads = threads
|
|
18
19
|
@config = config
|
|
@@ -49,6 +50,7 @@ module Pgbus
|
|
|
49
50
|
)
|
|
50
51
|
@circuit_breaker = Pgbus::CircuitBreaker.new(config: config)
|
|
51
52
|
@queue_lock = QueueLock.new if @single_active_consumer
|
|
53
|
+
@notify_listener = nil
|
|
52
54
|
end
|
|
53
55
|
|
|
54
56
|
def stats
|
|
@@ -66,14 +68,17 @@ module Pgbus
|
|
|
66
68
|
}.merge(@pool.metadata)
|
|
67
69
|
end
|
|
68
70
|
|
|
71
|
+
NOTIFY_FALLBACK_POLL_SECONDS = 15
|
|
72
|
+
|
|
69
73
|
def run
|
|
70
74
|
setup_signals
|
|
71
75
|
start_heartbeat
|
|
72
76
|
resolve_wildcard_queues
|
|
77
|
+
start_notify_listener
|
|
73
78
|
@lifecycle.transition_to!(:running)
|
|
74
79
|
Pgbus.logger.info do
|
|
75
80
|
"[Pgbus] Worker started: queues=#{queues.join(",")} threads=#{threads} " \
|
|
76
|
-
"mode=#{@execution_mode} pid=#{::Process.pid}"
|
|
81
|
+
"mode=#{@execution_mode} notify_wakeup=#{notify_wakeup?} pid=#{::Process.pid}"
|
|
77
82
|
end
|
|
78
83
|
|
|
79
84
|
loop do
|
|
@@ -117,7 +122,7 @@ module Pgbus
|
|
|
117
122
|
private
|
|
118
123
|
|
|
119
124
|
def claim_and_execute
|
|
120
|
-
poll_interval =
|
|
125
|
+
poll_interval = wake_timeout
|
|
121
126
|
|
|
122
127
|
idle = @pool.available_capacity
|
|
123
128
|
return @wake_signal.wait(timeout: poll_interval) if idle <= 0
|
|
@@ -147,9 +152,19 @@ module Pgbus
|
|
|
147
152
|
# Returns an array of [queue_name, message] pairs so we always know
|
|
148
153
|
# which queue each message came from.
|
|
149
154
|
def fetch_messages(qty)
|
|
155
|
+
restore_evicted_queues if queues.empty? && !@wildcard
|
|
156
|
+
|
|
150
157
|
active_queues = queues.reject { |q| @circuit_breaker.paused?(q) }
|
|
151
158
|
active_queues = active_queues.select { |q| @queue_lock.try_lock(q) } if @single_active_consumer
|
|
152
|
-
|
|
159
|
+
|
|
160
|
+
if active_queues.empty?
|
|
161
|
+
Pgbus.logger.debug do
|
|
162
|
+
paused = queues.select { |q| @circuit_breaker.paused?(q) }
|
|
163
|
+
"[Pgbus] Worker fetch: all queues filtered — queues=#{queues.join(",")} " \
|
|
164
|
+
"paused=#{paused.join(",")}"
|
|
165
|
+
end
|
|
166
|
+
return []
|
|
167
|
+
end
|
|
153
168
|
|
|
154
169
|
if priority_enabled?
|
|
155
170
|
fetch_prioritized(active_queues, qty)
|
|
@@ -295,6 +310,7 @@ module Pgbus
|
|
|
295
310
|
Pgbus.logger.info { "[Pgbus] Wildcard queue '*' resolved to: #{@queues.join(", ")}" } unless @last_wildcard_resolve
|
|
296
311
|
end
|
|
297
312
|
@last_wildcard_resolve = monotonic_now
|
|
313
|
+
sync_notify_listener_queues
|
|
298
314
|
rescue StandardError => e
|
|
299
315
|
Pgbus.logger.error { "[Pgbus] Failed to resolve wildcard queues: #{e.message} — falling back to default" }
|
|
300
316
|
@queues = [config.default_queue] unless @last_wildcard_resolve
|
|
@@ -321,6 +337,15 @@ module Pgbus
|
|
|
321
337
|
end
|
|
322
338
|
end
|
|
323
339
|
Pgbus.logger.error { "[Pgbus] Queue table missing: #{error.message}" }
|
|
340
|
+
restore_evicted_queues if @queues.empty? && !@wildcard
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
def restore_evicted_queues
|
|
344
|
+
@queues = @initial_queues.dup
|
|
345
|
+
Pgbus.logger.warn do
|
|
346
|
+
"[Pgbus] Worker queue list was empty after eviction — " \
|
|
347
|
+
"restoring initial queues: #{@queues.join(", ")}"
|
|
348
|
+
end
|
|
324
349
|
end
|
|
325
350
|
|
|
326
351
|
def detect_zombie(queue_name, message)
|
|
@@ -401,6 +426,16 @@ module Pgbus
|
|
|
401
426
|
end
|
|
402
427
|
end
|
|
403
428
|
|
|
429
|
+
def notify_wakeup?
|
|
430
|
+
config.respond_to?(:worker_notify_wakeup?) && config.worker_notify_wakeup?
|
|
431
|
+
end
|
|
432
|
+
|
|
433
|
+
def wake_timeout
|
|
434
|
+
return effective_polling_interval unless notify_wakeup? && @notify_listener
|
|
435
|
+
|
|
436
|
+
[effective_polling_interval, config.polling_interval, NOTIFY_FALLBACK_POLL_SECONDS].max
|
|
437
|
+
end
|
|
438
|
+
|
|
404
439
|
def effective_polling_interval
|
|
405
440
|
return config.polling_interval if @consumer_priority.zero?
|
|
406
441
|
|
|
@@ -413,6 +448,43 @@ module Pgbus
|
|
|
413
448
|
config.polling_interval
|
|
414
449
|
end
|
|
415
450
|
|
|
451
|
+
def start_notify_listener
|
|
452
|
+
return unless notify_wakeup?
|
|
453
|
+
|
|
454
|
+
@notify_listener = NotifyListener.new(
|
|
455
|
+
physical_queues: physical_queue_names,
|
|
456
|
+
on_wake: -> { @wake_signal.notify! },
|
|
457
|
+
connection_options: config.worker_notify_connection_options,
|
|
458
|
+
health_check_ms: (config.polling_interval * 1000).to_i.clamp(250, 5_000),
|
|
459
|
+
logger: Pgbus.logger
|
|
460
|
+
).start
|
|
461
|
+
rescue StandardError => e
|
|
462
|
+
@notify_listener = nil
|
|
463
|
+
Pgbus.logger.error do
|
|
464
|
+
"[Pgbus] NotifyListener failed to start, falling back to polling: #{e.class}: #{e.message}"
|
|
465
|
+
end
|
|
466
|
+
end
|
|
467
|
+
|
|
468
|
+
def sync_notify_listener_queues
|
|
469
|
+
return unless @notify_listener
|
|
470
|
+
|
|
471
|
+
desired = physical_queue_names.to_set
|
|
472
|
+
current = @notify_listener.listening_to.to_set { |c| channel_to_physical(c) }
|
|
473
|
+
(desired - current).each { |q| @notify_listener.add_queue(q) }
|
|
474
|
+
(current - desired).each { |q| @notify_listener.remove_queue(q) }
|
|
475
|
+
rescue StandardError => e
|
|
476
|
+
Pgbus.logger.warn { "[Pgbus] NotifyListener queue sync failed: #{e.class}: #{e.message}" }
|
|
477
|
+
end
|
|
478
|
+
|
|
479
|
+
def physical_queue_names
|
|
480
|
+
prefix = "#{config.queue_prefix}_"
|
|
481
|
+
queues.map { |q| "#{prefix}#{q}" }
|
|
482
|
+
end
|
|
483
|
+
|
|
484
|
+
def channel_to_physical(channel)
|
|
485
|
+
channel.delete_prefix(NotifyListener::CHANNEL_PREFIX).delete_suffix(NotifyListener::CHANNEL_SUFFIX)
|
|
486
|
+
end
|
|
487
|
+
|
|
416
488
|
def start_heartbeat
|
|
417
489
|
@heartbeat = Heartbeat.new(
|
|
418
490
|
kind: "worker",
|
|
@@ -427,6 +499,7 @@ module Pgbus
|
|
|
427
499
|
|
|
428
500
|
def shutdown
|
|
429
501
|
Pgbus.logger.info { "[Pgbus] Worker draining thread pool..." }
|
|
502
|
+
@notify_listener&.stop
|
|
430
503
|
@pool.shutdown
|
|
431
504
|
@pool.wait_for_termination(30)
|
|
432
505
|
@stat_buffer&.stop
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "concurrent"
|
|
4
|
+
|
|
5
|
+
module Pgbus
|
|
6
|
+
module Streams
|
|
7
|
+
# Publish-side coalescing for high-frequency broadcasts (issue #171).
|
|
8
|
+
#
|
|
9
|
+
# A chatty reactive component — a live cursor, a typing indicator, a
|
|
10
|
+
# progress bar — can fan out many small broadcasts per second. For
|
|
11
|
+
# per-keystroke / per-frame updates that's wasteful: every frame
|
|
12
|
+
# becomes a PGMQ insert (or a NOTIFY) and a fan-out to every connection.
|
|
13
|
+
#
|
|
14
|
+
# The Coalescer batches per (stream, target) within a short window and
|
|
15
|
+
# flushes only the *latest* payload, so superseded frames never hit the
|
|
16
|
+
# bus at all. This is last-write-wins and is only safe for idempotent
|
|
17
|
+
# actions (replace / update of a stable target) — which is exactly the
|
|
18
|
+
# high-frequency case. It is strictly opt-in (`coalesce:` on broadcast).
|
|
19
|
+
#
|
|
20
|
+
# Debounce semantics: the FIRST submit for a (stream, target) schedules
|
|
21
|
+
# a flush `window_ms` later; subsequent submits within that window only
|
|
22
|
+
# overwrite the buffered payload. So latency is bounded to one window
|
|
23
|
+
# and a continuous stream of updates can't starve the flush (it is a
|
|
24
|
+
# trailing-edge-with-max-wait debounce, not a resettable one).
|
|
25
|
+
#
|
|
26
|
+
# Thread-safe: many request threads may submit concurrently. The buffer
|
|
27
|
+
# and the per-key pending-flush set are guarded by a single mutex; the
|
|
28
|
+
# flush itself runs off the mutex on the scheduler's thread.
|
|
29
|
+
class Coalescer
|
|
30
|
+
# Default coalescing window when `coalesce: true` is passed without an
|
|
31
|
+
# explicit millisecond value.
|
|
32
|
+
DEFAULT_WINDOW_MS = 50
|
|
33
|
+
|
|
34
|
+
Entry = Struct.new(:payload, :opts)
|
|
35
|
+
|
|
36
|
+
# scheduler: responds to `schedule(delay_seconds) { ... }`. Defaults
|
|
37
|
+
# to a Concurrent::ScheduledTask-backed scheduler.
|
|
38
|
+
# flush: ->(stream_name:, target:, payload:, opts:) called once per
|
|
39
|
+
# window per key with the latest buffered frame.
|
|
40
|
+
def initialize(flush:, scheduler: nil)
|
|
41
|
+
@flush = flush
|
|
42
|
+
@scheduler = scheduler || ScheduledTaskScheduler.new
|
|
43
|
+
@mutex = Mutex.new
|
|
44
|
+
@buffer = {} # key => Entry (latest)
|
|
45
|
+
@pending = {} # key => true while a flush is scheduled
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Buffers a frame for (stream_name, target). Overwrites any frame
|
|
49
|
+
# already buffered for the same key within the current window. The
|
|
50
|
+
# first submit per window schedules the flush.
|
|
51
|
+
def submit(stream_name:, target:, payload:, opts:, window_ms: DEFAULT_WINDOW_MS)
|
|
52
|
+
key = [stream_name, target]
|
|
53
|
+
schedule = false
|
|
54
|
+
|
|
55
|
+
@mutex.synchronize do
|
|
56
|
+
@buffer[key] = Entry.new(payload, opts)
|
|
57
|
+
unless @pending[key]
|
|
58
|
+
@pending[key] = true
|
|
59
|
+
schedule = true
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
@scheduler.schedule(window_ms / 1000.0) { flush_key(key, stream_name, target) } if schedule
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
private
|
|
67
|
+
|
|
68
|
+
def flush_key(key, stream_name, target)
|
|
69
|
+
entry = @mutex.synchronize do
|
|
70
|
+
@pending.delete(key)
|
|
71
|
+
@buffer.delete(key)
|
|
72
|
+
end
|
|
73
|
+
return unless entry
|
|
74
|
+
|
|
75
|
+
@flush.call(stream_name: stream_name, target: target, payload: entry.payload, opts: entry.opts)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Default scheduler backed by Concurrent::ScheduledTask. Kept as a
|
|
79
|
+
# tiny adapter so the Coalescer can be unit-tested with a synchronous
|
|
80
|
+
# fake scheduler (no real timers, no sleeps).
|
|
81
|
+
class ScheduledTaskScheduler
|
|
82
|
+
def schedule(delay_seconds, &)
|
|
83
|
+
Concurrent::ScheduledTask.execute(delay_seconds, &)
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
3
5
|
module Pgbus
|
|
4
6
|
module Streams
|
|
5
7
|
# Encodes Server-Sent Events frames per https://html.spec.whatwg.org/multipage/server-sent-events.html.
|
|
@@ -29,13 +31,30 @@ module Pgbus
|
|
|
29
31
|
raise ArgumentError, "id is required" if id.nil?
|
|
30
32
|
raise ArgumentError, "event is required" if event.nil? || event.to_s.empty?
|
|
31
33
|
|
|
32
|
-
|
|
34
|
+
# Strip newlines from BOTH event and data, not just data: each is
|
|
35
|
+
# interpolated into its own SSE field line, so an unescaped \r/\n in
|
|
36
|
+
# either would terminate the field early and let a crafted value
|
|
37
|
+
# inject extra SSE fields (a forged id:/data:) into the frame.
|
|
38
|
+
"id: #{id}\nevent: #{strip_newlines(event.to_s)}\ndata: #{strip_newlines(data.to_s)}\n\n"
|
|
33
39
|
end
|
|
34
40
|
|
|
35
41
|
def self.comment(text)
|
|
36
42
|
": #{strip_newlines(text.to_s)}\n\n"
|
|
37
43
|
end
|
|
38
44
|
|
|
45
|
+
# Emits a `pgbus:connected` frame carrying the server-minted
|
|
46
|
+
# connection id as JSON. Sent once, right after the open handshake,
|
|
47
|
+
# so the page can read its own connection id and send it back as
|
|
48
|
+
# `X-Pgbus-Connection` on action requests (actor-echo suppression,
|
|
49
|
+
# issue #165). Deliberately omits an `id:` line: this is connection
|
|
50
|
+
# metadata, not a broadcast, and giving it a cursor id would corrupt
|
|
51
|
+
# the client's Last-Event-ID replay position on reconnect.
|
|
52
|
+
def self.connected(id:)
|
|
53
|
+
raise ArgumentError, "id is required" if id.nil? || id.to_s.empty?
|
|
54
|
+
|
|
55
|
+
"event: pgbus:connected\ndata: #{JSON.generate({ connectionId: id.to_s })}\n\n"
|
|
56
|
+
end
|
|
57
|
+
|
|
39
58
|
def self.retry_directive(milliseconds)
|
|
40
59
|
unless milliseconds.is_a?(Integer) && !milliseconds.negative?
|
|
41
60
|
raise ArgumentError, "retry must be a non-negative integer (got #{milliseconds.inspect})"
|
data/lib/pgbus/streams/key.rb
CHANGED
|
@@ -63,9 +63,42 @@ module Pgbus
|
|
|
63
63
|
# `to_stream_key`/`to_gid_param` implementation that forgot to
|
|
64
64
|
# sanitize) raise an ArgumentError at the call site.
|
|
65
65
|
def stream_key(*parts, digest_bits: DEFAULT_DIGEST_BITS)
|
|
66
|
-
|
|
66
|
+
flattened = Array(parts).flatten
|
|
67
|
+
|
|
68
|
+
# Idempotency for an already-built key: a single String argument
|
|
69
|
+
# is treated as a pre-built pgbus stream key and returned
|
|
70
|
+
# unchanged (after the budget check). This lets a consumer hold
|
|
71
|
+
# one `stream_key` value and pass it to both `turbo_stream_from`
|
|
72
|
+
# and the broadcaster without the colon separator guard raising
|
|
73
|
+
# on the second call. The guard only protects against ambiguous
|
|
74
|
+
# *joins* (`stream_key('a:b', :c)` vs `stream_key('a', 'b:c')`),
|
|
75
|
+
# and there is no second fragment here to collapse against, so the
|
|
76
|
+
# hazard cannot arise. Symbols and records are NOT keys — a colon
|
|
77
|
+
# in those never came from `stream_key` and stays a mistake.
|
|
78
|
+
return stream_key!(flattened.first) if flattened.length == 1 && flattened.first.is_a?(String)
|
|
79
|
+
|
|
80
|
+
fragments = flattened.map { |part| normalize(part, digest_bits: digest_bits) }
|
|
67
81
|
fragments.each { |fragment| reject_colons!(fragment) }
|
|
68
|
-
|
|
82
|
+
validate_budget!(fragments.join(":"))
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Accepts an already-built stream key verbatim, skipping the
|
|
86
|
+
# per-fragment colon guard (a pre-built key legitimately contains
|
|
87
|
+
# ':' separators). Still enforces the queue-name budget so an
|
|
88
|
+
# oversized key fails at the call site rather than deep inside
|
|
89
|
+
# Client#ensure_stream_queue. Use this when you hold a key string
|
|
90
|
+
# and want to be explicit that no re-keying should happen — e.g.
|
|
91
|
+
# passing the same value to `turbo_stream_from` and a broadcaster.
|
|
92
|
+
def stream_key!(key)
|
|
93
|
+
raise ArgumentError, "stream_key! key must be a String, got #{key.class}" unless key.is_a?(String)
|
|
94
|
+
|
|
95
|
+
validate_budget!(key)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Returns the key when it fits the pgbus queue-name budget; raises
|
|
99
|
+
# ArgumentError with an actionable message otherwise. Shared by
|
|
100
|
+
# `stream_key` and `stream_key!` so both paths fail identically.
|
|
101
|
+
def validate_budget!(key)
|
|
69
102
|
budget = queue_name_budget
|
|
70
103
|
return key if key.length <= budget
|
|
71
104
|
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "cgi"
|
|
4
|
+
|
|
5
|
+
module Pgbus
|
|
6
|
+
module Streams
|
|
7
|
+
# Turns a renderable into a complete `<turbo-stream>` action tag,
|
|
8
|
+
# ready to hand to Stream#broadcast. This centralises the off-request
|
|
9
|
+
# render + tag-building that every consumer would otherwise stitch
|
|
10
|
+
# together by hand (the #1 footgun in server-driven UI: rendering a
|
|
11
|
+
# component outside a request without a usable view context).
|
|
12
|
+
#
|
|
13
|
+
# pgbus deliberately has no hard dependency on turbo-rails, ActionView,
|
|
14
|
+
# or Phlex, so this builder is self-contained and matches Turbo's wire
|
|
15
|
+
# format directly. The browser's Turbo runtime consumes the tag; the
|
|
16
|
+
# exact string is the contract, not any particular Ruby library.
|
|
17
|
+
#
|
|
18
|
+
# Renderable resolution (first match wins):
|
|
19
|
+
# - String → used verbatim (already-rendered markup)
|
|
20
|
+
# - responds to :call → Phlex::HTML#call (the issue's example shape)
|
|
21
|
+
# - responds to :render_in → ViewComponent / phlex-rails
|
|
22
|
+
# (`render_in(view_context)`; a nil context is passed because
|
|
23
|
+
# off-request there is no controller — components that need URL
|
|
24
|
+
# helpers should be rendered by the app and the string passed in)
|
|
25
|
+
# - else → to_s
|
|
26
|
+
#
|
|
27
|
+
# Tag format mirrors Turbo::Streams::TagBuilder:
|
|
28
|
+
# - content actions wrap the markup in a <template>
|
|
29
|
+
# - content-less actions (remove) emit no <template>
|
|
30
|
+
module Renderer
|
|
31
|
+
# Turbo stream actions that carry no content (no <template> wrapper).
|
|
32
|
+
CONTENTLESS_ACTIONS = %w[remove].freeze
|
|
33
|
+
|
|
34
|
+
module_function
|
|
35
|
+
|
|
36
|
+
# Builds a `<turbo-stream action target><template>...</template></turbo-stream>`
|
|
37
|
+
# string. `renderable` may be nil for content-less actions.
|
|
38
|
+
def turbo_stream_tag(action:, target:, renderable: nil)
|
|
39
|
+
raise ArgumentError, "target is required" if target.nil? || target.to_s.empty?
|
|
40
|
+
|
|
41
|
+
action = action.to_s
|
|
42
|
+
attrs = %(action="#{escape(action)}" target="#{escape(target)}")
|
|
43
|
+
|
|
44
|
+
return "<turbo-stream #{attrs}></turbo-stream>" if CONTENTLESS_ACTIONS.include?(action)
|
|
45
|
+
|
|
46
|
+
content = render(renderable)
|
|
47
|
+
"<turbo-stream #{attrs}><template>#{content}</template></turbo-stream>"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Resolves a renderable to an HTML string. See module docs for the
|
|
51
|
+
# resolution order. Returns "" for nil (a content action with no
|
|
52
|
+
# renderable still emits an empty <template>).
|
|
53
|
+
def render(renderable)
|
|
54
|
+
return "" if renderable.nil?
|
|
55
|
+
return renderable if renderable.is_a?(String)
|
|
56
|
+
return renderable.call.to_s if renderable.respond_to?(:call)
|
|
57
|
+
return renderable.render_in(nil).to_s if renderable.respond_to?(:render_in)
|
|
58
|
+
|
|
59
|
+
renderable.to_s
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def escape(value)
|
|
63
|
+
CGI.escape_html(value.to_s)
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|