pgbus 0.5.1 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +244 -1
- data/Rakefile +8 -1
- data/app/controllers/pgbus/insights_controller.rb +6 -0
- data/app/helpers/pgbus/streams_helper.rb +115 -0
- data/app/javascript/pgbus/stream_source_element.js +212 -0
- data/app/models/pgbus/stream_stat.rb +123 -0
- data/app/views/pgbus/insights/show.html.erb +59 -0
- data/config/locales/en.yml +16 -0
- data/config/routes.rb +11 -0
- data/lib/generators/pgbus/add_job_stats_queue_index_generator.rb +53 -0
- data/lib/generators/pgbus/add_presence_generator.rb +55 -0
- data/lib/generators/pgbus/add_stream_stats_generator.rb +54 -0
- data/lib/generators/pgbus/templates/add_job_stats_latency.rb.erb +4 -1
- data/lib/generators/pgbus/templates/add_job_stats_queue_index.rb.erb +11 -0
- data/lib/generators/pgbus/templates/add_presence.rb.erb +26 -0
- data/lib/generators/pgbus/templates/add_stream_stats.rb.erb +18 -0
- data/lib/generators/pgbus/update_generator.rb +176 -23
- data/lib/pgbus/client/ensure_stream_queue.rb +54 -0
- data/lib/pgbus/client/read_after.rb +100 -0
- data/lib/pgbus/client.rb +6 -0
- data/lib/pgbus/configuration.rb +65 -0
- data/lib/pgbus/engine.rb +31 -0
- data/lib/pgbus/generators/config_converter.rb +22 -2
- data/lib/pgbus/generators/database_target_detector.rb +94 -0
- data/lib/pgbus/generators/migration_detector.rb +217 -0
- data/lib/pgbus/process/dispatcher.rb +62 -4
- data/lib/pgbus/streams/cursor.rb +71 -0
- data/lib/pgbus/streams/envelope.rb +58 -0
- data/lib/pgbus/streams/filters.rb +98 -0
- data/lib/pgbus/streams/presence.rb +216 -0
- data/lib/pgbus/streams/signed_name.rb +69 -0
- data/lib/pgbus/streams/turbo_broadcastable.rb +53 -0
- data/lib/pgbus/streams/watermark_cache_middleware.rb +28 -0
- data/lib/pgbus/streams.rb +151 -0
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +88 -10
- data/lib/pgbus/web/stream_app.rb +179 -0
- data/lib/pgbus/web/streamer/connection.rb +122 -0
- data/lib/pgbus/web/streamer/dispatcher.rb +467 -0
- data/lib/pgbus/web/streamer/heartbeat.rb +105 -0
- data/lib/pgbus/web/streamer/instance.rb +176 -0
- data/lib/pgbus/web/streamer/io_writer.rb +73 -0
- data/lib/pgbus/web/streamer/listener.rb +228 -0
- data/lib/pgbus/web/streamer/registry.rb +103 -0
- data/lib/pgbus/web/streamer.rb +53 -0
- data/lib/pgbus.rb +28 -0
- data/lib/puma/plugin/pgbus_streams.rb +54 -0
- data/lib/tasks/pgbus_streams.rake +52 -0
- metadata +33 -1
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Generators
|
|
5
|
+
# Detects whether the host application has configured pgbus to use
|
|
6
|
+
# a separate database via connects_to, and returns the database name
|
|
7
|
+
# that --database flags should be set to for sub-generators.
|
|
8
|
+
#
|
|
9
|
+
# Detection sources, in priority order:
|
|
10
|
+
#
|
|
11
|
+
# 1. Pgbus.configuration.connects_to (runtime — authoritative if
|
|
12
|
+
# the initializer has already booted)
|
|
13
|
+
# 2. config/initializers/pgbus.rb (text scan)
|
|
14
|
+
# 3. config/application.rb (text scan — fallback for apps that
|
|
15
|
+
# call connects_to in application config instead of the pgbus
|
|
16
|
+
# initializer)
|
|
17
|
+
#
|
|
18
|
+
# Returns a String (the database name, e.g. "pgbus") or nil if no
|
|
19
|
+
# separate database is configured.
|
|
20
|
+
class DatabaseTargetDetector
|
|
21
|
+
# Matches:
|
|
22
|
+
# c.connects_to = { database: { writing: :pgbus } }
|
|
23
|
+
# c.connects_to(database: { writing: :queue_db })
|
|
24
|
+
# Pgbus.configuration.connects_to = { database: { writing: :pgbus } }
|
|
25
|
+
#
|
|
26
|
+
# Rejects:
|
|
27
|
+
# c.connects_to = { role: :writing } (different API shape)
|
|
28
|
+
# connects_to :something (no database: key)
|
|
29
|
+
#
|
|
30
|
+
# The [^;\n]*? and [^}]*? laziness keeps the scan within a
|
|
31
|
+
# reasonable window so a stray "writing:" later in the file
|
|
32
|
+
# can't cross-contaminate.
|
|
33
|
+
CONNECTS_TO_PATTERN = /connects_to\b[^;\n]*?database\s*:\s*\{[^}]*?writing\s*:\s*:?(?<name>[a-zA-Z_][a-zA-Z0-9_]*)/m
|
|
34
|
+
|
|
35
|
+
def initialize(destination_root:)
|
|
36
|
+
@destination_root = destination_root
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Returns the database name string if a separate DB is configured,
|
|
40
|
+
# nil otherwise. Checks runtime config first, then falls back to
|
|
41
|
+
# static file scanning.
|
|
42
|
+
def detect
|
|
43
|
+
runtime_database_name || scan_initializer || scan_application_config
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
attr_reader :destination_root
|
|
49
|
+
|
|
50
|
+
# Runtime path: Pgbus.configuration.connects_to. Only works if the
|
|
51
|
+
# host app has loaded pgbus AND booted the initializer. The update
|
|
52
|
+
# generator runs after Rails app boot so this is usually available.
|
|
53
|
+
def runtime_database_name
|
|
54
|
+
return nil unless defined?(Pgbus) && Pgbus.respond_to?(:configuration)
|
|
55
|
+
|
|
56
|
+
extract_database_name(Pgbus.configuration.connects_to)
|
|
57
|
+
rescue StandardError
|
|
58
|
+
nil
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def scan_initializer
|
|
62
|
+
scan_file(File.join(destination_root, "config", "initializers", "pgbus.rb"))
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def scan_application_config
|
|
66
|
+
scan_file(File.join(destination_root, "config", "application.rb"))
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def scan_file(path)
|
|
70
|
+
return nil unless File.exist?(path)
|
|
71
|
+
|
|
72
|
+
content = File.read(path)
|
|
73
|
+
match = content.match(CONNECTS_TO_PATTERN)
|
|
74
|
+
return nil unless match
|
|
75
|
+
|
|
76
|
+
match[:name]
|
|
77
|
+
rescue StandardError
|
|
78
|
+
nil
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Parses `{ database: { writing: :name } }` or the String variant
|
|
82
|
+
# into the database name. Returns nil for anything we don't
|
|
83
|
+
# recognize.
|
|
84
|
+
def extract_database_name(connects_to)
|
|
85
|
+
return nil unless connects_to.is_a?(Hash)
|
|
86
|
+
|
|
87
|
+
db = connects_to[:database] || connects_to["database"]
|
|
88
|
+
return nil unless db.is_a?(Hash)
|
|
89
|
+
|
|
90
|
+
(db[:writing] || db["writing"])&.to_s
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Generators
|
|
5
|
+
# Inspects a live ActiveRecord connection and determines which of
|
|
6
|
+
# pgbus's migration generators need to run to bring the schema up
|
|
7
|
+
# to date.
|
|
8
|
+
#
|
|
9
|
+
# Usage:
|
|
10
|
+
#
|
|
11
|
+
# detector = Pgbus::Generators::MigrationDetector.new(connection)
|
|
12
|
+
# detector.missing_migrations
|
|
13
|
+
# # => [:add_uniqueness_keys, :add_job_stats_queue_index, ...]
|
|
14
|
+
#
|
|
15
|
+
# The returned symbols correspond to generator names that the
|
|
16
|
+
# pgbus:update generator invokes via Thor composition. Each symbol
|
|
17
|
+
# maps to exactly one generator (see GENERATOR_MAP).
|
|
18
|
+
#
|
|
19
|
+
# Detection rules:
|
|
20
|
+
#
|
|
21
|
+
# 1. Fresh install (no core tables) → returns [:fresh_install] as a
|
|
22
|
+
# sentinel. The caller should tell the user to run pgbus:install
|
|
23
|
+
# instead of stacking 8 migrations.
|
|
24
|
+
#
|
|
25
|
+
# 2. Core tables missing → queued unconditionally. These are features
|
|
26
|
+
# pgbus assumes are present.
|
|
27
|
+
#
|
|
28
|
+
# 3. Opt-in feature tables missing → queued unconditionally. The user
|
|
29
|
+
# asked for update-to-latest, so we add the migration files but
|
|
30
|
+
# don't enable the feature in config. They'll opt in separately.
|
|
31
|
+
#
|
|
32
|
+
# 4. Columns missing on existing tables → queued. These are in-place
|
|
33
|
+
# schema upgrades (e.g. add_job_stats_latency adds
|
|
34
|
+
# enqueue_latency_ms + retry_count).
|
|
35
|
+
#
|
|
36
|
+
# 5. Indexes missing on existing tables → queued. Additive, safe.
|
|
37
|
+
#
|
|
38
|
+
# 6. Modern replacements for legacy tables (e.g. pgbus_job_locks →
|
|
39
|
+
# pgbus_uniqueness_keys) → queue the migration path only if the
|
|
40
|
+
# legacy table still exists. Otherwise queue the fresh install.
|
|
41
|
+
class MigrationDetector
|
|
42
|
+
# Sentinel returned when the database looks empty of pgbus tables.
|
|
43
|
+
# The caller (pgbus:update generator) should redirect the user to
|
|
44
|
+
# pgbus:install rather than trying to stack the full schema as
|
|
45
|
+
# individual add_* migrations.
|
|
46
|
+
FRESH_INSTALL = :fresh_install
|
|
47
|
+
|
|
48
|
+
# The set of tables that the base pgbus:install migration creates.
|
|
49
|
+
# If NONE of these exist, we treat the DB as a fresh install.
|
|
50
|
+
CORE_INSTALL_TABLES = %w[
|
|
51
|
+
pgbus_processed_events
|
|
52
|
+
pgbus_processes
|
|
53
|
+
pgbus_failed_events
|
|
54
|
+
pgbus_semaphores
|
|
55
|
+
pgbus_blocked_executions
|
|
56
|
+
pgbus_batches
|
|
57
|
+
].freeze
|
|
58
|
+
|
|
59
|
+
# generator_key → Rails generator name. Passed to Thor's invoke.
|
|
60
|
+
#
|
|
61
|
+
# Note: uniqueness_keys uses the migrate_job_locks generator for
|
|
62
|
+
# both the fresh-install and upgrade-from-job_locks paths. The
|
|
63
|
+
# template is idempotent: `unless table_exists?(:pgbus_uniqueness_keys)`
|
|
64
|
+
# creates it, and `if table_exists?(:pgbus_job_locks)` drops the
|
|
65
|
+
# legacy table. One generator covers both cases.
|
|
66
|
+
GENERATOR_MAP = {
|
|
67
|
+
uniqueness_keys: "pgbus:migrate_job_locks",
|
|
68
|
+
add_job_stats: "pgbus:add_job_stats",
|
|
69
|
+
add_job_stats_latency: "pgbus:add_job_stats_latency",
|
|
70
|
+
add_job_stats_queue_index: "pgbus:add_job_stats_queue_index",
|
|
71
|
+
add_stream_stats: "pgbus:add_stream_stats",
|
|
72
|
+
add_presence: "pgbus:add_presence",
|
|
73
|
+
add_queue_states: "pgbus:add_queue_states",
|
|
74
|
+
add_outbox: "pgbus:add_outbox",
|
|
75
|
+
add_recurring: "pgbus:add_recurring",
|
|
76
|
+
add_failed_events_index: "pgbus:add_failed_events_index"
|
|
77
|
+
}.freeze
|
|
78
|
+
|
|
79
|
+
# Human-friendly description of each migration for the generator
|
|
80
|
+
# output. Keeps the update generator's run log readable.
|
|
81
|
+
DESCRIPTIONS = {
|
|
82
|
+
uniqueness_keys: "uniqueness keys table (job deduplication, also upgrades legacy job_locks if present)",
|
|
83
|
+
add_job_stats: "job stats table (Insights dashboard)",
|
|
84
|
+
add_job_stats_latency: "job stats latency columns (enqueue_latency_ms, retry_count)",
|
|
85
|
+
add_job_stats_queue_index: "job stats (queue_name, created_at) index",
|
|
86
|
+
add_stream_stats: "stream stats table (opt-in real-time Insights)",
|
|
87
|
+
add_presence: "presence members table (Turbo Streams presence)",
|
|
88
|
+
add_queue_states: "queue states table (pause/resume)",
|
|
89
|
+
add_outbox: "outbox entries table (transactional outbox)",
|
|
90
|
+
add_recurring: "recurring tasks + executions tables",
|
|
91
|
+
add_failed_events_index: "unique index on pgbus_failed_events (queue_name, msg_id)"
|
|
92
|
+
}.freeze
|
|
93
|
+
|
|
94
|
+
def initialize(connection)
|
|
95
|
+
@connection = connection
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Returns an Array of generator keys (symbols) in the order they
|
|
99
|
+
# should run. Dependencies are resolved implicitly via order: the
|
|
100
|
+
# base table creation for a feature always comes before the
|
|
101
|
+
# column/index add-ons.
|
|
102
|
+
def missing_migrations
|
|
103
|
+
return [FRESH_INSTALL] if fresh_install?
|
|
104
|
+
|
|
105
|
+
[
|
|
106
|
+
*uniqueness_key_migrations,
|
|
107
|
+
*job_stats_migrations,
|
|
108
|
+
*stream_stats_migrations,
|
|
109
|
+
*presence_migrations,
|
|
110
|
+
*queue_states_migrations,
|
|
111
|
+
*outbox_migrations,
|
|
112
|
+
*recurring_migrations,
|
|
113
|
+
*failed_events_index_migrations
|
|
114
|
+
]
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
private
|
|
118
|
+
|
|
119
|
+
attr_reader :connection
|
|
120
|
+
|
|
121
|
+
def fresh_install?
|
|
122
|
+
CORE_INSTALL_TABLES.none? { |t| table_exists?(t) }
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Legacy pgbus_job_locks → modern pgbus_uniqueness_keys.
|
|
126
|
+
# The migrate_job_locks template is idempotent: it creates
|
|
127
|
+
# uniqueness_keys if missing and drops job_locks if present.
|
|
128
|
+
# One symbol covers both cases; see GENERATOR_MAP.
|
|
129
|
+
def uniqueness_key_migrations
|
|
130
|
+
return [] if table_exists?("pgbus_uniqueness_keys")
|
|
131
|
+
|
|
132
|
+
[:uniqueness_keys]
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def job_stats_migrations
|
|
136
|
+
migrations = []
|
|
137
|
+
|
|
138
|
+
unless table_exists?("pgbus_job_stats")
|
|
139
|
+
migrations << :add_job_stats
|
|
140
|
+
# The latency columns and queue index are add-ons to job_stats.
|
|
141
|
+
# If we're creating the base table now, the add_job_stats
|
|
142
|
+
# template doesn't include them — add the upgrade migrations
|
|
143
|
+
# so a fresh install lands on the current schema.
|
|
144
|
+
migrations << :add_job_stats_latency
|
|
145
|
+
migrations << :add_job_stats_queue_index
|
|
146
|
+
return migrations
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# Base table exists — check each add-on independently.
|
|
150
|
+
cols = column_names("pgbus_job_stats")
|
|
151
|
+
migrations << :add_job_stats_latency unless cols.include?("enqueue_latency_ms") && cols.include?("retry_count")
|
|
152
|
+
|
|
153
|
+
migrations << :add_job_stats_queue_index unless index_exists?("pgbus_job_stats", "idx_pgbus_job_stats_queue_time")
|
|
154
|
+
|
|
155
|
+
migrations
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def stream_stats_migrations
|
|
159
|
+
table_exists?("pgbus_stream_stats") ? [] : [:add_stream_stats]
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def presence_migrations
|
|
163
|
+
table_exists?("pgbus_presence_members") ? [] : [:add_presence]
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def queue_states_migrations
|
|
167
|
+
table_exists?("pgbus_queue_states") ? [] : [:add_queue_states]
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def outbox_migrations
|
|
171
|
+
table_exists?("pgbus_outbox_entries") ? [] : [:add_outbox]
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def recurring_migrations
|
|
175
|
+
# pgbus_recurring_tasks + pgbus_recurring_executions are the two
|
|
176
|
+
# tables the recurring generator creates. If BOTH exist, nothing
|
|
177
|
+
# to do. If either is missing, we need the generator (which is
|
|
178
|
+
# idempotent via if_not_exists on both tables).
|
|
179
|
+
return [] if table_exists?("pgbus_recurring_tasks") && table_exists?("pgbus_recurring_executions")
|
|
180
|
+
|
|
181
|
+
[:add_recurring]
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
def failed_events_index_migrations
|
|
185
|
+
# pgbus_failed_events is created by pgbus:install, but the
|
|
186
|
+
# unique (queue_name, msg_id) index was added later via its own
|
|
187
|
+
# generator. If the table exists without the unique index,
|
|
188
|
+
# FailedEventRecorder's upsert silently swallows ON CONFLICT
|
|
189
|
+
# errors — so this is a real bug waiting to bite.
|
|
190
|
+
return [] unless table_exists?("pgbus_failed_events")
|
|
191
|
+
return [] if index_exists?("pgbus_failed_events", "idx_pgbus_failed_events_queue_msg")
|
|
192
|
+
|
|
193
|
+
[:add_failed_events_index]
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# --- schema probes -------------------------------------------------
|
|
197
|
+
|
|
198
|
+
def table_exists?(name)
|
|
199
|
+
connection.table_exists?(name)
|
|
200
|
+
rescue StandardError
|
|
201
|
+
false
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def column_names(table)
|
|
205
|
+
connection.columns(table).map(&:name)
|
|
206
|
+
rescue StandardError
|
|
207
|
+
[]
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def index_exists?(table, index_name)
|
|
211
|
+
connection.indexes(table).any? { |idx| idx.name == index_name }
|
|
212
|
+
rescue StandardError
|
|
213
|
+
false
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
end
|
|
@@ -33,6 +33,7 @@ module Pgbus
|
|
|
33
33
|
@last_batch_cleanup_at = monotonic_now
|
|
34
34
|
@last_recurring_cleanup_at = monotonic_now
|
|
35
35
|
@last_archive_compaction_at = monotonic_now
|
|
36
|
+
@last_stream_archive_compaction_at = monotonic_now
|
|
36
37
|
@last_outbox_cleanup_at = monotonic_now
|
|
37
38
|
@last_job_lock_cleanup_at = monotonic_now
|
|
38
39
|
@last_stats_cleanup_at = monotonic_now
|
|
@@ -79,6 +80,7 @@ module Pgbus
|
|
|
79
80
|
run_if_due(now, :@last_batch_cleanup_at, BATCH_CLEANUP_INTERVAL) { cleanup_batches }
|
|
80
81
|
run_if_due(now, :@last_recurring_cleanup_at, RECURRING_CLEANUP_INTERVAL) { cleanup_recurring_executions }
|
|
81
82
|
run_if_due(now, :@last_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { compact_archives }
|
|
83
|
+
run_if_due(now, :@last_stream_archive_compaction_at, ARCHIVE_COMPACTION_INTERVAL) { prune_stream_archives }
|
|
82
84
|
run_if_due(now, :@last_outbox_cleanup_at, OUTBOX_CLEANUP_INTERVAL) { cleanup_outbox }
|
|
83
85
|
run_if_due(now, :@last_job_lock_cleanup_at, JOB_LOCK_CLEANUP_INTERVAL) { cleanup_job_locks }
|
|
84
86
|
run_if_due(now, :@last_stats_cleanup_at, STATS_CLEANUP_INTERVAL) { cleanup_stats }
|
|
@@ -140,13 +142,20 @@ module Pgbus
|
|
|
140
142
|
end
|
|
141
143
|
|
|
142
144
|
def cleanup_stats
|
|
143
|
-
return unless config.stats_enabled
|
|
144
|
-
|
|
145
145
|
retention = config.stats_retention
|
|
146
146
|
return unless retention&.positive?
|
|
147
147
|
|
|
148
|
-
|
|
149
|
-
|
|
148
|
+
cutoff = Time.current - retention
|
|
149
|
+
|
|
150
|
+
if config.stats_enabled
|
|
151
|
+
deleted = JobStat.cleanup!(older_than: cutoff)
|
|
152
|
+
Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} old job stats" } if deleted.positive?
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
return unless config.streams_stats_enabled
|
|
156
|
+
|
|
157
|
+
deleted = StreamStat.cleanup!(older_than: cutoff)
|
|
158
|
+
Pgbus.logger.debug { "[Pgbus] Cleaned up #{deleted} old stream stats" } if deleted.positive?
|
|
150
159
|
end
|
|
151
160
|
|
|
152
161
|
def cleanup_job_locks
|
|
@@ -241,6 +250,55 @@ module Pgbus
|
|
|
241
250
|
Pgbus.logger.warn { "[Pgbus] Archive compaction failed: #{e.message}" }
|
|
242
251
|
end
|
|
243
252
|
|
|
253
|
+
# Prunes per-stream archive tables. Unlike compact_archives (which
|
|
254
|
+
# uses the global archive_retention, typically 7 days), streams need
|
|
255
|
+
# per-stream retention because a chat-history stream and a
|
|
256
|
+
# presence-ping stream have wildly different storage requirements.
|
|
257
|
+
# Lookup order for a given queue: exact string match, then regex
|
|
258
|
+
# match, then streams_default_retention (5 minutes by default).
|
|
259
|
+
def prune_stream_archives
|
|
260
|
+
prefix = config.streams_queue_prefix
|
|
261
|
+
return if prefix.nil? || prefix.empty?
|
|
262
|
+
|
|
263
|
+
batch_size = config.archive_compaction_batch_size || 1000
|
|
264
|
+
conn = config.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection
|
|
265
|
+
queue_names = conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
|
|
266
|
+
|
|
267
|
+
queue_names.each do |full_name|
|
|
268
|
+
next unless full_name.start_with?("#{prefix}_")
|
|
269
|
+
|
|
270
|
+
retention = retention_for_stream_queue(full_name)
|
|
271
|
+
next unless retention.positive?
|
|
272
|
+
|
|
273
|
+
cutoff = Time.current - retention
|
|
274
|
+
stripped = full_name.delete_prefix("#{config.queue_prefix}_")
|
|
275
|
+
deleted = Pgbus.client.purge_archive(stripped, older_than: cutoff, batch_size: batch_size)
|
|
276
|
+
if deleted.positive?
|
|
277
|
+
Pgbus.logger.debug do
|
|
278
|
+
"[Pgbus] Compacted #{deleted} stream archive entries from #{full_name}"
|
|
279
|
+
end
|
|
280
|
+
end
|
|
281
|
+
rescue StandardError => e
|
|
282
|
+
Pgbus.logger.warn { "[Pgbus] Stream archive compaction failed for #{full_name}: #{e.message}" }
|
|
283
|
+
end
|
|
284
|
+
rescue StandardError => e
|
|
285
|
+
Pgbus.logger.warn { "[Pgbus] Stream archive compaction failed: #{e.message}" }
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
def retention_for_stream_queue(full_name)
|
|
289
|
+
retention_map = config.streams_retention || {}
|
|
290
|
+
# Exact-match first — cheapest and most common path
|
|
291
|
+
exact = retention_map[full_name]
|
|
292
|
+
return exact.to_f if exact
|
|
293
|
+
|
|
294
|
+
# Regex-match second for pattern-based overrides (e.g. /^chat_/)
|
|
295
|
+
retention_map.each do |key, value|
|
|
296
|
+
return value.to_f if key.is_a?(Regexp) && key.match?(full_name)
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
config.streams_default_retention.to_f
|
|
300
|
+
end
|
|
301
|
+
|
|
244
302
|
def cleanup_recurring_executions
|
|
245
303
|
retention = config.recurring_execution_retention
|
|
246
304
|
return unless retention&.positive?
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Streams
|
|
5
|
+
# Parses an SSE replay cursor from a Rack request. The cursor is the highest
|
|
6
|
+
# PGMQ msg_id the client has already seen; the streamer replays everything
|
|
7
|
+
# strictly greater on (re)connect.
|
|
8
|
+
#
|
|
9
|
+
# Two sources, by precedence:
|
|
10
|
+
# 1. Last-Event-ID header (sent automatically by EventSource on reconnect)
|
|
11
|
+
# 2. ?since= query param (sent by the custom element on first connect,
|
|
12
|
+
# because EventSource cannot send custom headers on the initial request)
|
|
13
|
+
#
|
|
14
|
+
# Returns 0 when no cursor is present (i.e. "deliver everything from now").
|
|
15
|
+
module Cursor
|
|
16
|
+
# PGMQ msg_id is BIGINT — strictly within signed 64-bit range.
|
|
17
|
+
MAX_MSG_ID = (2**63) - 1
|
|
18
|
+
|
|
19
|
+
class InvalidCursor < ArgumentError
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Strict integer pattern: optional leading minus, then digits. No plus prefix,
|
|
23
|
+
# no decimal, no whitespace. Negatives are caught by validate! with a clearer
|
|
24
|
+
# error message than "must be numeric".
|
|
25
|
+
INTEGER_PATTERN = /\A-?\d+\z/
|
|
26
|
+
|
|
27
|
+
def self.parse(query_since:, last_event_id:)
|
|
28
|
+
raw = pick(last_event_id, query_since)
|
|
29
|
+
return 0 if raw.nil?
|
|
30
|
+
|
|
31
|
+
value = coerce(raw)
|
|
32
|
+
validate!(value)
|
|
33
|
+
value
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def self.pick(last_event_id, query_since)
|
|
37
|
+
return last_event_id if present?(last_event_id)
|
|
38
|
+
return query_since if present?(query_since)
|
|
39
|
+
|
|
40
|
+
nil
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def self.present?(value)
|
|
44
|
+
return false if value.nil?
|
|
45
|
+
# Any Integer is "present" so precedence is decided here, not by
|
|
46
|
+
# the sign of the value. validate! rejects negatives separately
|
|
47
|
+
# so a caller passing a literal -1 still crashes loud rather than
|
|
48
|
+
# silently falling through to query_since.
|
|
49
|
+
return true if value.is_a?(Integer)
|
|
50
|
+
|
|
51
|
+
!value.to_s.strip.empty?
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def self.coerce(raw)
|
|
55
|
+
return raw if raw.is_a?(Integer)
|
|
56
|
+
|
|
57
|
+
str = raw.to_s
|
|
58
|
+
raise InvalidCursor, "cursor must be numeric (got #{raw.inspect})" unless str.match?(INTEGER_PATTERN)
|
|
59
|
+
|
|
60
|
+
Integer(str, 10)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def self.validate!(value)
|
|
64
|
+
raise InvalidCursor, "cursor must not be negative (got #{value})" if value.negative?
|
|
65
|
+
raise InvalidCursor, "cursor #{value} is out of BIGINT range" if value > MAX_MSG_ID
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
private_class_method :pick, :present?, :coerce, :validate!
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Streams
|
|
5
|
+
# Encodes Server-Sent Events frames per https://html.spec.whatwg.org/multipage/server-sent-events.html.
|
|
6
|
+
#
|
|
7
|
+
# Pgbus uses three frame types:
|
|
8
|
+
# - `message(id:, event:, data:)` — a real broadcast (carries an `id:` so the client
|
|
9
|
+
# can resume via `Last-Event-ID` on reconnect)
|
|
10
|
+
# - `comment(text)` — a heartbeat or sentinel that the SSE parser ignores
|
|
11
|
+
# - `retry_directive(ms)` — tells `EventSource` how long to wait before reconnecting
|
|
12
|
+
#
|
|
13
|
+
# All frames end with `\n\n` (the SSE event terminator). `data:` lines must not
|
|
14
|
+
# contain newlines — the SSE spec uses `\n` as the field terminator, so a multi-line
|
|
15
|
+
# payload would arrive as multiple events. We strip `\r` and `\n` from data and
|
|
16
|
+
# comment text rather than splitting into multiple `data:` lines, because Turbo
|
|
17
|
+
# Stream HTML is already flat and the simpler encoding is easier to debug.
|
|
18
|
+
module Envelope
|
|
19
|
+
NEWLINES = /[\r\n]+/
|
|
20
|
+
|
|
21
|
+
RESPONSE_HEADERS = "HTTP/1.1 200 OK\r\n" \
|
|
22
|
+
"content-type: text/event-stream\r\n" \
|
|
23
|
+
"cache-control: no-cache, no-transform\r\n" \
|
|
24
|
+
"x-accel-buffering: no\r\n" \
|
|
25
|
+
"connection: keep-alive\r\n" \
|
|
26
|
+
"\r\n"
|
|
27
|
+
|
|
28
|
+
def self.message(id:, event:, data:)
|
|
29
|
+
raise ArgumentError, "id is required" if id.nil?
|
|
30
|
+
raise ArgumentError, "event is required" if event.nil? || event.to_s.empty?
|
|
31
|
+
|
|
32
|
+
"id: #{id}\nevent: #{event}\ndata: #{strip_newlines(data.to_s)}\n\n"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def self.comment(text)
|
|
36
|
+
": #{strip_newlines(text.to_s)}\n\n"
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def self.retry_directive(milliseconds)
|
|
40
|
+
unless milliseconds.is_a?(Integer) && !milliseconds.negative?
|
|
41
|
+
raise ArgumentError, "retry must be a non-negative integer (got #{milliseconds.inspect})"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
"retry: #{milliseconds}\n\n"
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def self.http_response_headers
|
|
48
|
+
RESPONSE_HEADERS
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def self.strip_newlines(str)
|
|
52
|
+
str.gsub(NEWLINES, "")
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private_class_method :strip_newlines
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
module Streams
|
|
5
|
+
# Process-wide registry of server-side audience filter predicates.
|
|
6
|
+
# Used by `Pgbus.stream(name).broadcast(html, visible_to: :label)`
|
|
7
|
+
# to restrict delivery to connections whose authorize-hook context
|
|
8
|
+
# matches the predicate.
|
|
9
|
+
#
|
|
10
|
+
# Typical setup at boot time:
|
|
11
|
+
#
|
|
12
|
+
# Pgbus::Streams.filters.register(:admin_only) { |user| user.admin? }
|
|
13
|
+
# Pgbus::Streams.filters.register(:workspace_member) do |user, stream|
|
|
14
|
+
# user.workspaces.pluck(:id).include?(stream.split(":").last.to_i)
|
|
15
|
+
# end
|
|
16
|
+
#
|
|
17
|
+
# Broadcasts reference filters by label:
|
|
18
|
+
#
|
|
19
|
+
# Pgbus.stream("workspace:42").broadcast(html, visible_to: :admin_only)
|
|
20
|
+
#
|
|
21
|
+
# The Dispatcher looks up the filter in the registry, evaluates it
|
|
22
|
+
# against each connection's context (populated from the StreamApp's
|
|
23
|
+
# authorize hook return value), and only delivers to connections
|
|
24
|
+
# where the predicate returns true.
|
|
25
|
+
#
|
|
26
|
+
# Why a registry of labels instead of passing a Proc directly to
|
|
27
|
+
# broadcast: predicates can't be serialized to JSON, so they can't
|
|
28
|
+
# travel through PGMQ. The label is serialized; the predicate lives
|
|
29
|
+
# in-process on the subscriber side. This also means the predicate
|
|
30
|
+
# is evaluated on the same process that holds the SSE connection,
|
|
31
|
+
# so the user context (typically an ActiveRecord model or a session
|
|
32
|
+
# hash) is always available.
|
|
33
|
+
class Filters
|
|
34
|
+
def initialize(logger: nil)
|
|
35
|
+
@mutex = Mutex.new
|
|
36
|
+
@filters = {}
|
|
37
|
+
@logger = logger
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def register(label, callable = nil, &block)
|
|
41
|
+
raise ArgumentError, "filter label must be a Symbol (got #{label.class})" unless label.is_a?(Symbol)
|
|
42
|
+
|
|
43
|
+
predicate = callable || block
|
|
44
|
+
raise ArgumentError, "filter must be given a block or callable" if predicate.nil?
|
|
45
|
+
|
|
46
|
+
@mutex.synchronize { @filters[label] = predicate }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def lookup(label)
|
|
50
|
+
@mutex.synchronize { @filters[label] }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Evaluates the named filter against a context. The context is
|
|
54
|
+
# whatever the StreamApp's authorize hook returned when the
|
|
55
|
+
# connection was established — typically a user model.
|
|
56
|
+
#
|
|
57
|
+
# Policy decisions:
|
|
58
|
+
# - label=nil → visible (no filter attached to the broadcast)
|
|
59
|
+
# - unknown label → NOT visible + warning log. Fail-closed so
|
|
60
|
+
# a typo or renamed filter doesn't turn a restricted
|
|
61
|
+
# broadcast into a public one. The whole point of audience
|
|
62
|
+
# filtering is data isolation; failing open on a typo
|
|
63
|
+
# defeats the feature. The warning log is loud enough that
|
|
64
|
+
# typos still get noticed in dev (check the log or wonder
|
|
65
|
+
# why no subscriber sees your broadcast).
|
|
66
|
+
# - predicate raises → NOT visible (fail-closed on runtime
|
|
67
|
+
# error to avoid leaking data on an exception path).
|
|
68
|
+
def visible?(label, context)
|
|
69
|
+
return true if label.nil?
|
|
70
|
+
|
|
71
|
+
predicate = lookup(label)
|
|
72
|
+
if predicate.nil?
|
|
73
|
+
log_warn("unknown filter label #{label.inspect} — broadcast dropped (fail-closed)")
|
|
74
|
+
return false
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
begin
|
|
78
|
+
!!predicate.call(context)
|
|
79
|
+
rescue StandardError => e
|
|
80
|
+
log_error("filter #{label.inspect} raised #{e.class}: #{e.message} — dropping broadcast (fail-closed)")
|
|
81
|
+
false
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
|
|
87
|
+
def log_warn(message)
|
|
88
|
+
logger = @logger || (Pgbus.logger if defined?(Pgbus) && Pgbus.respond_to?(:logger))
|
|
89
|
+
logger&.warn { "[Pgbus::Streams::Filters] #{message}" }
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def log_error(message)
|
|
93
|
+
logger = @logger || (Pgbus.logger if defined?(Pgbus) && Pgbus.respond_to?(:logger))
|
|
94
|
+
logger&.error { "[Pgbus::Streams::Filters] #{message}" }
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|