pgbus 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +238 -0
- data/Rakefile +8 -1
- data/app/controllers/pgbus/insights_controller.rb +6 -0
- data/app/helpers/pgbus/streams_helper.rb +115 -0
- data/app/javascript/pgbus/stream_source_element.js +212 -0
- data/app/models/pgbus/stream_stat.rb +118 -0
- data/app/views/pgbus/insights/show.html.erb +59 -0
- data/config/locales/en.yml +16 -0
- data/config/routes.rb +11 -0
- data/lib/generators/pgbus/add_presence_generator.rb +55 -0
- data/lib/generators/pgbus/add_stream_stats_generator.rb +54 -0
- data/lib/generators/pgbus/templates/add_presence.rb.erb +26 -0
- data/lib/generators/pgbus/templates/add_stream_stats.rb.erb +18 -0
- data/lib/pgbus/client/ensure_stream_queue.rb +54 -0
- data/lib/pgbus/client/read_after.rb +100 -0
- data/lib/pgbus/client.rb +6 -0
- data/lib/pgbus/configuration.rb +65 -0
- data/lib/pgbus/engine.rb +31 -0
- data/lib/pgbus/process/dispatcher.rb +62 -4
- data/lib/pgbus/streams/cursor.rb +71 -0
- data/lib/pgbus/streams/envelope.rb +58 -0
- data/lib/pgbus/streams/filters.rb +98 -0
- data/lib/pgbus/streams/presence.rb +216 -0
- data/lib/pgbus/streams/signed_name.rb +69 -0
- data/lib/pgbus/streams/turbo_broadcastable.rb +53 -0
- data/lib/pgbus/streams/watermark_cache_middleware.rb +28 -0
- data/lib/pgbus/streams.rb +151 -0
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +29 -0
- data/lib/pgbus/web/stream_app.rb +179 -0
- data/lib/pgbus/web/streamer/connection.rb +122 -0
- data/lib/pgbus/web/streamer/dispatcher.rb +467 -0
- data/lib/pgbus/web/streamer/heartbeat.rb +105 -0
- data/lib/pgbus/web/streamer/instance.rb +176 -0
- data/lib/pgbus/web/streamer/io_writer.rb +73 -0
- data/lib/pgbus/web/streamer/listener.rb +228 -0
- data/lib/pgbus/web/streamer/registry.rb +103 -0
- data/lib/pgbus/web/streamer.rb +53 -0
- data/lib/pgbus.rb +28 -0
- data/lib/puma/plugin/pgbus_streams.rb +54 -0
- data/lib/tasks/pgbus_streams.rake +52 -0
- metadata +29 -1
data/lib/pgbus/engine.rb
CHANGED
|
@@ -46,6 +46,7 @@ module Pgbus
|
|
|
46
46
|
|
|
47
47
|
rake_tasks do
|
|
48
48
|
load File.expand_path("../tasks/pgbus_pgmq.rake", __dir__)
|
|
49
|
+
load File.expand_path("../tasks/pgbus_streams.rake", __dir__)
|
|
49
50
|
end
|
|
50
51
|
|
|
51
52
|
initializer "pgbus.i18n" do
|
|
@@ -56,5 +57,35 @@ module Pgbus
|
|
|
56
57
|
require "pgbus/web/authentication"
|
|
57
58
|
require "pgbus/web/data_source"
|
|
58
59
|
end
|
|
60
|
+
|
|
61
|
+
# Install the watermark cache middleware ahead of the app's own
|
|
62
|
+
# middleware so the thread-local cache is cleared between every
|
|
63
|
+
# Rack request. Without this, repeated page renders served by the
|
|
64
|
+
# same Puma thread would see stale current_msg_id values.
|
|
65
|
+
initializer "pgbus.streams.middleware" do |app|
|
|
66
|
+
app.middleware.use Pgbus::Streams::WatermarkCacheMiddleware if Pgbus.configuration.streams_enabled
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Install the Turbo::StreamsChannel patch after turbo-rails has been
|
|
70
|
+
# loaded. The patch redirects broadcast_stream_to through Pgbus.stream
|
|
71
|
+
# instead of ActionCable. When turbo-rails is not loaded, this is a
|
|
72
|
+
# no-op and pgbus_stream_from still works via the explicit
|
|
73
|
+
# Pgbus.stream(...).broadcast(...) API.
|
|
74
|
+
initializer "pgbus.streams.turbo_broadcastable", after: :load_config_initializers do
|
|
75
|
+
ActiveSupport.on_load(:after_initialize) do
|
|
76
|
+
if Pgbus.configuration.streams_enabled
|
|
77
|
+
# Touch the constant first so Zeitwerk autoloads
|
|
78
|
+
# lib/pgbus/streams/turbo_broadcastable.rb. The file defines
|
|
79
|
+
# `Pgbus::Streams::TurboBroadcastable` (the autoloaded const)
|
|
80
|
+
# AND `Pgbus::Streams.install_turbo_broadcastable_patch!`
|
|
81
|
+
# (a side-effect class method on the parent module). Without
|
|
82
|
+
# the constant reference, Zeitwerk doesn't load the file and
|
|
83
|
+
# the method call below raises NoMethodError. Assigning to
|
|
84
|
+
# `_` keeps RuboCop's Lint/Void from deleting the line.
|
|
85
|
+
_autoload_trigger = Pgbus::Streams::TurboBroadcastable
|
|
86
|
+
Pgbus::Streams.install_turbo_broadcastable_patch!
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
59
90
|
end
|
|
60
91
|
end
|
|
@@ -33,6 +33,7 @@ module Pgbus
|
|
|
33
33
|
@last_batch_cleanup_at = monotonic_now
|
|
34
34
|
@last_recurring_cleanup_at = monotonic_now
|
|
35
35
|
@last_archive_compaction_at = monotonic_now
|
|
36
|
+
@last_stream_archive_compaction_at = monotonic_now
|
|
36
37
|
@last_outbox_cleanup_at = monotonic_now
|
|
37
38
|
@last_job_lock_cleanup_at = monotonic_now
|
|
38
39
|
@last_stats_cleanup_at = monotonic_now
|
|
@@ -79,6 +80,7 @@ module Pgbus
|
|
|
79
80
|
run_if_due(now, :@last_batch_cleanup_at, BATCH_CLEANUP_INTERVAL) { cleanup_batches }
|
|
80
81
|
run_if_due(now, :@last_recurring_cleanup_at, RECURRING_CLEANUP_INTERVAL) { cleanup_recurring_executions }
|
|
81
82
|
run_if_due(now, :@last_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { compact_archives }
|
|
83
|
+
run_if_due(now, :@last_stream_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { prune_stream_archives }
|
|
82
84
|
run_if_due(now, :@last_outbox_cleanup_at, OUTBOX_CLEANUP_INTERVAL) { cleanup_outbox }
|
|
83
85
|
run_if_due(now, :@last_job_lock_cleanup_at, JOB_LOCK_CLEANUP_INTERVAL) { cleanup_job_locks }
|
|
84
86
|
run_if_due(now, :@last_stats_cleanup_at, STATS_CLEANUP_INTERVAL) { cleanup_stats }
|
|
@@ -140,13 +142,20 @@ module Pgbus
|
|
|
140
142
|
end
|
|
141
143
|
|
|
142
144
|
def cleanup_stats
|
|
143
|
-
return unless config.stats_enabled
|
|
144
|
-
|
|
145
145
|
retention = config.stats_retention
|
|
146
146
|
return unless retention&.positive?
|
|
147
147
|
|
|
148
|
-
|
|
149
|
-
|
|
148
|
+
cutoff = Time.current - retention
|
|
149
|
+
|
|
150
|
+
if config.stats_enabled
|
|
151
|
+
deleted = JobStat.cleanup!(older_than: cutoff)
|
|
152
|
+
Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} old job stats" } if deleted.positive?
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
return unless config.streams_stats_enabled
|
|
156
|
+
|
|
157
|
+
deleted = StreamStat.cleanup!(older_than: cutoff)
|
|
158
|
+
Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} old stream stats" } if deleted.positive?
|
|
150
159
|
end
|
|
151
160
|
|
|
152
161
|
def cleanup_job_locks
|
|
@@ -241,6 +250,55 @@ module Pgbus
|
|
|
241
250
|
Pgbus.logger.warn { "[Pgbus] Archive compaction failed: #{e.message}" }
|
|
242
251
|
end
|
|
243
252
|
|
|
253
|
+
# Prunes per-stream archive tables. Unlike compact_archives (which
|
|
254
|
+
# uses the global archive_retention, typically 7 days), streams need
|
|
255
|
+
# per-stream retention because a chat-history stream and a
|
|
256
|
+
# presence-ping stream have wildly different storage requirements.
|
|
257
|
+
# Lookup order for a given queue: exact string match, then regex
|
|
258
|
+
# match, then streams_default_retention (5 minutes by default).
|
|
259
|
+
def prune_stream_archives
|
|
260
|
+
prefix = config.streams_queue_prefix
|
|
261
|
+
return if prefix.nil? || prefix.empty?
|
|
262
|
+
|
|
263
|
+
batch_size = config.archive_compaction_batch_size || 1000
|
|
264
|
+
conn = config.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection
|
|
265
|
+
queue_names = conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
|
|
266
|
+
|
|
267
|
+
queue_names.each do |full_name|
|
|
268
|
+
next unless full_name.start_with?("#{prefix}_")
|
|
269
|
+
|
|
270
|
+
retention = retention_for_stream_queue(full_name)
|
|
271
|
+
next unless retention.positive?
|
|
272
|
+
|
|
273
|
+
cutoff = Time.current - retention
|
|
274
|
+
stripped = full_name.delete_prefix("#{config.queue_prefix}_")
|
|
275
|
+
deleted = Pgbus.client.purge_archive(stripped, older_than: cutoff, batch_size: batch_size)
|
|
276
|
+
if deleted.positive?
|
|
277
|
+
Pgbus.logger.debug do
|
|
278
|
+
"[Pgbus] Compacted #{deleted} stream archive entries from #{full_name}"
|
|
279
|
+
end
|
|
280
|
+
end
|
|
281
|
+
rescue StandardError => e
|
|
282
|
+
Pgbus.logger.warn { "[Pgbus] Stream archive compaction failed for #{full_name}: #{e.message}" }
|
|
283
|
+
end
|
|
284
|
+
rescue StandardError => e
|
|
285
|
+
Pgbus.logger.warn { "[Pgbus] Stream archive compaction failed: #{e.message}" }
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
def retention_for_stream_queue(full_name)
|
|
289
|
+
retention_map = config.streams_retention || {}
|
|
290
|
+
# Exact-match first — cheapest and most common path
|
|
291
|
+
exact = retention_map[full_name]
|
|
292
|
+
return exact.to_f if exact
|
|
293
|
+
|
|
294
|
+
# Regex-match second for pattern-based overrides (e.g. /^chat_/)
|
|
295
|
+
retention_map.each do |key, value|
|
|
296
|
+
return value.to_f if key.is_a?(Regexp) && key.match?(full_name)
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
config.streams_default_retention.to_f
|
|
300
|
+
end
|
|
301
|
+
|
|
244
302
|
def cleanup_recurring_executions
|
|
245
303
|
retention = config.recurring_execution_retention
|
|
246
304
|
return unless retention&.positive?
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Streams
|
|
5
|
+
# Parses an SSE replay cursor from a Rack request. The cursor is the highest
|
|
6
|
+
# PGMQ msg_id the client has already seen; the streamer replays everything
|
|
7
|
+
# strictly greater on (re)connect.
|
|
8
|
+
#
|
|
9
|
+
# Two sources, by precedence:
|
|
10
|
+
# 1. Last-Event-ID header (sent automatically by EventSource on reconnect)
|
|
11
|
+
# 2. ?since= query param (sent by the custom element on first connect,
|
|
12
|
+
# because EventSource cannot send custom headers on the initial request)
|
|
13
|
+
#
|
|
14
|
+
# Returns 0 when no cursor is present (i.e. "deliver everything from now").
|
|
15
|
+
module Cursor
|
|
16
|
+
# PGMQ msg_id is BIGINT — strictly within signed 64-bit range.
|
|
17
|
+
MAX_MSG_ID = (2**63) - 1
|
|
18
|
+
|
|
19
|
+
class InvalidCursor < ArgumentError
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Strict integer pattern: optional leading minus, then digits. No plus prefix,
|
|
23
|
+
# no decimal, no whitespace. Negatives are caught by validate! with a clearer
|
|
24
|
+
# error message than "must be numeric".
|
|
25
|
+
INTEGER_PATTERN = /\A-?\d+\z/
|
|
26
|
+
|
|
27
|
+
def self.parse(query_since:, last_event_id:)
|
|
28
|
+
raw = pick(last_event_id, query_since)
|
|
29
|
+
return 0 if raw.nil?
|
|
30
|
+
|
|
31
|
+
value = coerce(raw)
|
|
32
|
+
validate!(value)
|
|
33
|
+
value
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def self.pick(last_event_id, query_since)
|
|
37
|
+
return last_event_id if present?(last_event_id)
|
|
38
|
+
return query_since if present?(query_since)
|
|
39
|
+
|
|
40
|
+
nil
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def self.present?(value)
|
|
44
|
+
return false if value.nil?
|
|
45
|
+
# Any Integer is "present" so precedence is decided here, not by
|
|
46
|
+
# the sign of the value. validate! rejects negatives separately
|
|
47
|
+
# so a caller passing a literal -1 still crashes loud rather than
|
|
48
|
+
# silently falling through to query_since.
|
|
49
|
+
return true if value.is_a?(Integer)
|
|
50
|
+
|
|
51
|
+
!value.to_s.strip.empty?
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def self.coerce(raw)
|
|
55
|
+
return raw if raw.is_a?(Integer)
|
|
56
|
+
|
|
57
|
+
str = raw.to_s
|
|
58
|
+
raise InvalidCursor, "cursor must be numeric (got #{raw.inspect})" unless str.match?(INTEGER_PATTERN)
|
|
59
|
+
|
|
60
|
+
Integer(str, 10)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def self.validate!(value)
|
|
64
|
+
raise InvalidCursor, "cursor must not be negative (got #{value})" if value.negative?
|
|
65
|
+
raise InvalidCursor, "cursor #{value} is out of BIGINT range" if value > MAX_MSG_ID
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
private_class_method :pick, :present?, :coerce, :validate!
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Streams
|
|
5
|
+
# Encodes Server-Sent Events frames per https://html.spec.whatwg.org/multipage/server-sent-events.html.
|
|
6
|
+
#
|
|
7
|
+
# Pgbus uses three frame types:
|
|
8
|
+
# - `message(id:, event:, data:)` — a real broadcast (carries an `id:` so the client
|
|
9
|
+
# can resume via `Last-Event-ID` on reconnect)
|
|
10
|
+
# - `comment(text)` — a heartbeat or sentinel that the SSE parser ignores
|
|
11
|
+
# - `retry_directive(ms)` — tells `EventSource` how long to wait before reconnecting
|
|
12
|
+
#
|
|
13
|
+
# All frames end with `\n\n` (the SSE event terminator). `data:` lines must not
|
|
14
|
+
# contain newlines — the SSE spec uses `\n` as the field terminator, so a multi-line
|
|
15
|
+
# payload would arrive as multiple events. We strip `\r` and `\n` from data and
|
|
16
|
+
# comment text rather than splitting into multiple `data:` lines, because Turbo
|
|
17
|
+
# Stream HTML is already flat and the simpler encoding is easier to debug.
|
|
18
|
+
module Envelope
|
|
19
|
+
NEWLINES = /[\r\n]+/
|
|
20
|
+
|
|
21
|
+
RESPONSE_HEADERS = "HTTP/1.1 200 OK\r\n" \
|
|
22
|
+
"content-type: text/event-stream\r\n" \
|
|
23
|
+
"cache-control: no-cache, no-transform\r\n" \
|
|
24
|
+
"x-accel-buffering: no\r\n" \
|
|
25
|
+
"connection: keep-alive\r\n" \
|
|
26
|
+
"\r\n"
|
|
27
|
+
|
|
28
|
+
def self.message(id:, event:, data:)
|
|
29
|
+
raise ArgumentError, "id is required" if id.nil?
|
|
30
|
+
raise ArgumentError, "event is required" if event.nil? || event.to_s.empty?
|
|
31
|
+
|
|
32
|
+
"id: #{id}\nevent: #{event}\ndata: #{strip_newlines(data.to_s)}\n\n"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def self.comment(text)
|
|
36
|
+
": #{strip_newlines(text.to_s)}\n\n"
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def self.retry_directive(milliseconds)
|
|
40
|
+
unless milliseconds.is_a?(Integer) && !milliseconds.negative?
|
|
41
|
+
raise ArgumentError, "retry must be a non-negative integer (got #{milliseconds.inspect})"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
"retry: #{milliseconds}\n\n"
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def self.http_response_headers
|
|
48
|
+
RESPONSE_HEADERS
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def self.strip_newlines(str)
|
|
52
|
+
str.gsub(NEWLINES, "")
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private_class_method :strip_newlines
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Streams
|
|
5
|
+
# Process-wide registry of server-side audience filter predicates.
|
|
6
|
+
# Used by `Pgbus.stream(name).broadcast(html, visible_to: :label)`
|
|
7
|
+
# to restrict delivery to connections whose authorize-hook context
|
|
8
|
+
# matches the predicate.
|
|
9
|
+
#
|
|
10
|
+
# Typical setup at boot time:
|
|
11
|
+
#
|
|
12
|
+
# Pgbus::Streams.filters.register(:admin_only) { |user| user.admin? }
|
|
13
|
+
# Pgbus::Streams.filters.register(:workspace_member) do |user, stream|
|
|
14
|
+
# user.workspaces.pluck(:id).include?(stream.split(":").last.to_i)
|
|
15
|
+
# end
|
|
16
|
+
#
|
|
17
|
+
# Broadcasts reference filters by label:
|
|
18
|
+
#
|
|
19
|
+
# Pgbus.stream("workspace:42").broadcast(html, visible_to: :admin_only)
|
|
20
|
+
#
|
|
21
|
+
# The Dispatcher looks up the filter in the registry, evaluates it
|
|
22
|
+
# against each connection's context (populated from the StreamApp's
|
|
23
|
+
# authorize hook return value), and only delivers to connections
|
|
24
|
+
# where the predicate returns true.
|
|
25
|
+
#
|
|
26
|
+
# Why a registry of labels instead of passing a Proc directly to
|
|
27
|
+
# broadcast: predicates can't be serialized to JSON, so they can't
|
|
28
|
+
# travel through PGMQ. The label is serialized; the predicate lives
|
|
29
|
+
# in-process on the subscriber side. This also means the predicate
|
|
30
|
+
# is evaluated on the same process that holds the SSE connection,
|
|
31
|
+
# so the user context (typically an ActiveRecord model or a session
|
|
32
|
+
# hash) is always available.
|
|
33
|
+
class Filters
|
|
34
|
+
def initialize(logger: nil)
|
|
35
|
+
@mutex = Mutex.new
|
|
36
|
+
@filters = {}
|
|
37
|
+
@logger = logger
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def register(label, callable = nil, &block)
|
|
41
|
+
raise ArgumentError, "filter label must be a Symbol (got #{label.class})" unless label.is_a?(Symbol)
|
|
42
|
+
|
|
43
|
+
predicate = callable || block
|
|
44
|
+
raise ArgumentError, "filter must be given a block or callable" if predicate.nil?
|
|
45
|
+
|
|
46
|
+
@mutex.synchronize { @filters[label] = predicate }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def lookup(label)
|
|
50
|
+
@mutex.synchronize { @filters[label] }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Evaluates the named filter against a context. The context is
|
|
54
|
+
# whatever the StreamApp's authorize hook returned when the
|
|
55
|
+
# connection was established — typically a user model.
|
|
56
|
+
#
|
|
57
|
+
# Policy decisions:
|
|
58
|
+
# - label=nil → visible (no filter attached to the broadcast)
|
|
59
|
+
# - unknown label → NOT visible + warning log. Fail-closed so
|
|
60
|
+
# a typo or renamed filter doesn't turn a restricted
|
|
61
|
+
# broadcast into a public one. The whole point of audience
|
|
62
|
+
# filtering is data isolation; failing open on a typo
|
|
63
|
+
# defeats the feature. The warning log is loud enough that
|
|
64
|
+
# typos still get noticed in dev (check the log or wonder
|
|
65
|
+
# why no subscriber sees your broadcast).
|
|
66
|
+
# - predicate raises → NOT visible (fail-closed on runtime
|
|
67
|
+
# error to avoid leaking data on an exception path).
|
|
68
|
+
def visible?(label, context)
|
|
69
|
+
return true if label.nil?
|
|
70
|
+
|
|
71
|
+
predicate = lookup(label)
|
|
72
|
+
if predicate.nil?
|
|
73
|
+
log_warn("unknown filter label #{label.inspect} — broadcast dropped (fail-closed)")
|
|
74
|
+
return false
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
begin
|
|
78
|
+
!!predicate.call(context)
|
|
79
|
+
rescue StandardError => e
|
|
80
|
+
log_error("filter #{label.inspect} raised #{e.class}: #{e.message} — dropping broadcast (fail-closed)")
|
|
81
|
+
false
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
|
|
87
|
+
def log_warn(message)
|
|
88
|
+
logger = @logger || (Pgbus.logger if defined?(Pgbus) && Pgbus.respond_to?(:logger))
|
|
89
|
+
logger&.warn { "[Pgbus::Streams::Filters] #{message}" }
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def log_error(message)
|
|
93
|
+
logger = @logger || (Pgbus.logger if defined?(Pgbus) && Pgbus.respond_to?(:logger))
|
|
94
|
+
logger&.error { "[Pgbus::Streams::Filters] #{message}" }
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Streams
|
|
5
|
+
# Presence tracking for SSE streams. A "presence member" is an
|
|
6
|
+
# arbitrary identifier (typically a user id) marked as currently
|
|
7
|
+
# subscribed to a given stream. Members live in pgbus_presence_members
|
|
8
|
+
# with a (stream_name, member_id) primary key and a `last_seen_at`
|
|
9
|
+
# timestamp; the table is the source of truth across all Puma workers
|
|
10
|
+
# and Falcon reactors.
|
|
11
|
+
#
|
|
12
|
+
# Typical usage in a controller:
|
|
13
|
+
#
|
|
14
|
+
# def show
|
|
15
|
+
# @room = Room.find(params[:id])
|
|
16
|
+
# Pgbus.stream(@room).presence.join(
|
|
17
|
+
# member_id: current_user.id.to_s,
|
|
18
|
+
# metadata: { name: current_user.name, avatar: current_user.avatar_url }
|
|
19
|
+
# )
|
|
20
|
+
# end
|
|
21
|
+
#
|
|
22
|
+
# And when the user leaves:
|
|
23
|
+
#
|
|
24
|
+
# Pgbus.stream(@room).presence.leave(member_id: current_user.id.to_s)
|
|
25
|
+
#
|
|
26
|
+
# Reading the current member list:
|
|
27
|
+
#
|
|
28
|
+
# Pgbus.stream(@room).presence.members
|
|
29
|
+
# # => [{ "id" => "7", "metadata" => {"name" => ..., "avatar" => ...},
|
|
30
|
+
# # "joined_at" => "...", "last_seen_at" => "..." }]
|
|
31
|
+
#
|
|
32
|
+
# The join/leave operations also fire a stream broadcast (via the
|
|
33
|
+
# caller-provided block) so connected clients see the change in real
|
|
34
|
+
# time. The library emits the broadcast through the regular pgbus
|
|
35
|
+
# stream pipeline, which means it's transactional, replayable, and
|
|
36
|
+
# filterable just like any other broadcast.
|
|
37
|
+
#
|
|
38
|
+
# NOT included in this v1:
|
|
39
|
+
# - Automatic connection-driven join/leave (the application must
|
|
40
|
+
# call join/leave explicitly).
|
|
41
|
+
# - Automatic stale-member sweeping (call Presence.sweep!(stream)
|
|
42
|
+
# from a cron or after a heartbeat to expire idle members).
|
|
43
|
+
# - Built-in DOM events on the <pgbus-stream-source> element
|
|
44
|
+
# (the application's broadcast block decides the HTML).
|
|
45
|
+
class Presence
|
|
46
|
+
def initialize(stream)
|
|
47
|
+
@stream = stream
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Adds (or refreshes) a member on this stream. Idempotent: calling
|
|
51
|
+
# join twice with the same member_id updates last_seen_at and
|
|
52
|
+
# metadata without creating a duplicate row. Yields the member
|
|
53
|
+
# hash to the optional block so the caller can render an
|
|
54
|
+
# `<turbo-stream action="append">` to broadcast the join.
|
|
55
|
+
def join(member_id:, metadata: {})
|
|
56
|
+
member_id = member_id.to_s
|
|
57
|
+
record = upsert_member(member_id, metadata)
|
|
58
|
+
|
|
59
|
+
if block_given?
|
|
60
|
+
html = yield(record)
|
|
61
|
+
@stream.broadcast(html) if html.is_a?(String) && !html.empty?
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
record
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Removes a member. Yields the (former) member hash to the
|
|
68
|
+
# optional block so the caller can broadcast a `remove` action.
|
|
69
|
+
# Returns the deleted record, or nil if the member wasn't present.
|
|
70
|
+
def leave(member_id:)
|
|
71
|
+
member_id = member_id.to_s
|
|
72
|
+
record = delete_member(member_id)
|
|
73
|
+
return nil unless record
|
|
74
|
+
|
|
75
|
+
if block_given?
|
|
76
|
+
html = yield(record)
|
|
77
|
+
@stream.broadcast(html) if html.is_a?(String) && !html.empty?
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
record
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Refreshes the last_seen_at timestamp without re-broadcasting.
|
|
84
|
+
# Used as a heartbeat mechanism: clients ping touch periodically
|
|
85
|
+
# to stay in the member list, and a sweeper expires anyone who
|
|
86
|
+
# hasn't pinged in N seconds.
|
|
87
|
+
def touch(member_id:)
|
|
88
|
+
member_id = member_id.to_s
|
|
89
|
+
connection.exec_params(<<~SQL, [@stream.name, member_id])
|
|
90
|
+
UPDATE pgbus_presence_members
|
|
91
|
+
SET last_seen_at = NOW()
|
|
92
|
+
WHERE stream_name = $1 AND member_id = $2
|
|
93
|
+
SQL
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Returns the current list of members on this stream as an
|
|
97
|
+
# array of hashes (id, metadata, joined_at, last_seen_at).
|
|
98
|
+
def members
|
|
99
|
+
rows = connection.exec_params(<<~SQL, [@stream.name])
|
|
100
|
+
SELECT member_id, metadata, joined_at, last_seen_at
|
|
101
|
+
FROM pgbus_presence_members
|
|
102
|
+
WHERE stream_name = $1
|
|
103
|
+
ORDER BY joined_at
|
|
104
|
+
SQL
|
|
105
|
+
rows.to_a.map do |row|
|
|
106
|
+
{
|
|
107
|
+
"id" => row["member_id"],
|
|
108
|
+
"metadata" => parse_metadata(row["metadata"]),
|
|
109
|
+
"joined_at" => row["joined_at"],
|
|
110
|
+
"last_seen_at" => row["last_seen_at"]
|
|
111
|
+
}
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Returns the count of current members. Faster than members.size
|
|
116
|
+
# because it doesn't deserialize metadata.
|
|
117
|
+
def count
|
|
118
|
+
rows = connection.exec_params(<<~SQL, [@stream.name])
|
|
119
|
+
SELECT COUNT(*) AS n FROM pgbus_presence_members WHERE stream_name = $1
|
|
120
|
+
SQL
|
|
121
|
+
rows.first["n"].to_i
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Removes members whose last_seen_at is older than the given cutoff.
|
|
125
|
+
# Returns the number of expired rows. Use as a sweeper:
|
|
126
|
+
#
|
|
127
|
+
# Pgbus.stream("room:42").presence.sweep!(older_than: 60.seconds.ago)
|
|
128
|
+
#
|
|
129
|
+
# Atomically claims the deletion via DELETE RETURNING so multiple
|
|
130
|
+
# workers running the sweep concurrently won't double-emit leave
|
|
131
|
+
# events.
|
|
132
|
+
def sweep!(older_than:)
|
|
133
|
+
rows = connection.exec_params(<<~SQL, [@stream.name, older_than])
|
|
134
|
+
DELETE FROM pgbus_presence_members
|
|
135
|
+
WHERE stream_name = $1 AND last_seen_at < $2
|
|
136
|
+
RETURNING member_id, metadata, joined_at, last_seen_at
|
|
137
|
+
SQL
|
|
138
|
+
rows.to_a.size
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
private
|
|
142
|
+
|
|
143
|
+
def upsert_member(member_id, metadata)
|
|
144
|
+
rows = connection.exec_params(<<~SQL, [@stream.name, member_id, JSON.generate(metadata)])
|
|
145
|
+
INSERT INTO pgbus_presence_members (stream_name, member_id, metadata, joined_at, last_seen_at)
|
|
146
|
+
VALUES ($1, $2, $3::jsonb, NOW(), NOW())
|
|
147
|
+
ON CONFLICT (stream_name, member_id)
|
|
148
|
+
DO UPDATE SET metadata = EXCLUDED.metadata, last_seen_at = NOW()
|
|
149
|
+
RETURNING member_id, metadata, joined_at, last_seen_at
|
|
150
|
+
SQL
|
|
151
|
+
row = rows.first
|
|
152
|
+
{
|
|
153
|
+
"id" => row["member_id"],
|
|
154
|
+
"metadata" => parse_metadata(row["metadata"]),
|
|
155
|
+
"joined_at" => row["joined_at"],
|
|
156
|
+
"last_seen_at" => row["last_seen_at"]
|
|
157
|
+
}
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def delete_member(member_id)
|
|
161
|
+
rows = connection.exec_params(<<~SQL, [@stream.name, member_id])
|
|
162
|
+
DELETE FROM pgbus_presence_members
|
|
163
|
+
WHERE stream_name = $1 AND member_id = $2
|
|
164
|
+
RETURNING member_id, metadata, joined_at, last_seen_at
|
|
165
|
+
SQL
|
|
166
|
+
row = rows.first
|
|
167
|
+
return nil unless row
|
|
168
|
+
|
|
169
|
+
{
|
|
170
|
+
"id" => row["member_id"],
|
|
171
|
+
"metadata" => parse_metadata(row["metadata"]),
|
|
172
|
+
"joined_at" => row["joined_at"],
|
|
173
|
+
"last_seen_at" => row["last_seen_at"]
|
|
174
|
+
}
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def parse_metadata(value)
|
|
178
|
+
return value if value.is_a?(Hash)
|
|
179
|
+
return {} if value.nil? || value.empty?
|
|
180
|
+
|
|
181
|
+
JSON.parse(value)
|
|
182
|
+
rescue JSON::ParserError => e
|
|
183
|
+
# Resilience fallback: return an empty hash rather than
|
|
184
|
+
# crashing the presence member list on one corrupt row. Debug
|
|
185
|
+
# log so operators can still find the corruption without
|
|
186
|
+
# impacting production throughput (debug is off by default).
|
|
187
|
+
# Truncate the value so a huge metadata payload doesn't bloat
|
|
188
|
+
# the log line.
|
|
189
|
+
truncated = value.to_s[0, 200]
|
|
190
|
+
Pgbus.logger&.debug do
|
|
191
|
+
"[Pgbus::Streams::Presence] parse_metadata failed (#{e.class}: #{e.message}); raw: #{truncated.inspect}"
|
|
192
|
+
end
|
|
193
|
+
{}
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def connection
|
|
197
|
+
# Respect multi-database setups (connects_to). When
|
|
198
|
+
# Pgbus.configuration.connects_to is set, the pgbus tables
|
|
199
|
+
# live in a separate database accessed via Pgbus::BusRecord;
|
|
200
|
+
# the default path uses ActiveRecord::Base. Matches the
|
|
201
|
+
# canonical pattern in Pgbus::Process::QueueLock#connection
|
|
202
|
+
# and Pgbus::Configuration's connection probe.
|
|
203
|
+
unless defined?(::ActiveRecord::Base)
|
|
204
|
+
raise Pgbus::ConfigurationError,
|
|
205
|
+
"Pgbus::Streams::Presence requires ActiveRecord (no AR connection available)"
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
if Pgbus.configuration.connects_to
|
|
209
|
+
Pgbus::BusRecord.connection.raw_connection
|
|
210
|
+
else
|
|
211
|
+
::ActiveRecord::Base.connection.raw_connection
|
|
212
|
+
end
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
end
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_support/message_verifier"
|
|
4
|
+
|
|
5
|
+
module Pgbus
|
|
6
|
+
module Streams
|
|
7
|
+
# Verifies tamper-proof stream identifiers carried in URLs (the
|
|
8
|
+
# `signed-stream-name` attribute set by `pgbus_stream_from`). Reuses
|
|
9
|
+
# `Turbo.signed_stream_verifier_key` when turbo-rails is loaded so that
|
|
10
|
+
# existing `broadcasts_to :room` calls Just Work; falls back to
|
|
11
|
+
# `Pgbus.configuration.streams_signed_name_secret` otherwise.
|
|
12
|
+
#
|
|
13
|
+
# The signed payload is the logical stream name as a string (e.g.
|
|
14
|
+
# `"gid://app/Order/42:messages"`). Verification returns that string;
|
|
15
|
+
# tampered or unsigned input raises `InvalidSignedName`.
|
|
16
|
+
module SignedName
|
|
17
|
+
class InvalidSignedName < StandardError
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
class MissingSecret < StandardError
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def self.verify!(token)
|
|
24
|
+
raise InvalidSignedName, "signed stream name is blank" if token.nil? || token.to_s.strip.empty?
|
|
25
|
+
|
|
26
|
+
verifier.verified(token) || raise(InvalidSignedName, "signed stream name failed verification")
|
|
27
|
+
rescue ActiveSupport::MessageVerifier::InvalidSignature => e
|
|
28
|
+
raise InvalidSignedName, "signed stream name failed verification: #{e.message}"
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def self.sign(stream_name)
|
|
32
|
+
verifier.generate(stream_name)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Returns an ActiveSupport::MessageVerifier configured with whichever
|
|
36
|
+
# secret is appropriate for this process. Memoization is intentionally
|
|
37
|
+
# NOT used because tests rotate keys and the cost of constructing a
|
|
38
|
+
# verifier is negligible compared to the SHA256 work it does.
|
|
39
|
+
def self.verifier
|
|
40
|
+
ActiveSupport::MessageVerifier.new(secret, digest: "SHA256", serializer: JSON)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def self.secret
|
|
44
|
+
turbo_secret || configuration_secret || raise_missing_secret!
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def self.turbo_secret
|
|
48
|
+
return nil unless defined?(::Turbo)
|
|
49
|
+
|
|
50
|
+
::Turbo.signed_stream_verifier_key
|
|
51
|
+
rescue ArgumentError
|
|
52
|
+
# Real turbo-rails raises ArgumentError when the key is unset.
|
|
53
|
+
nil
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def self.configuration_secret
|
|
57
|
+
Pgbus.configuration.streams_signed_name_secret
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def self.raise_missing_secret!
|
|
61
|
+
raise MissingSecret,
|
|
62
|
+
"no signing secret available — set Pgbus.configuration.streams_signed_name_secret " \
|
|
63
|
+
"or Turbo.signed_stream_verifier_key"
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
private_class_method :verifier, :secret, :turbo_secret, :configuration_secret, :raise_missing_secret!
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|