pgbus 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +15 -0
  3. data/README.md +238 -0
  4. data/Rakefile +8 -1
  5. data/app/controllers/pgbus/insights_controller.rb +6 -0
  6. data/app/helpers/pgbus/streams_helper.rb +115 -0
  7. data/app/javascript/pgbus/stream_source_element.js +212 -0
  8. data/app/models/pgbus/stream_stat.rb +118 -0
  9. data/app/views/pgbus/insights/show.html.erb +59 -0
  10. data/config/locales/en.yml +16 -0
  11. data/config/routes.rb +11 -0
  12. data/lib/generators/pgbus/add_presence_generator.rb +55 -0
  13. data/lib/generators/pgbus/add_stream_stats_generator.rb +54 -0
  14. data/lib/generators/pgbus/templates/add_presence.rb.erb +26 -0
  15. data/lib/generators/pgbus/templates/add_stream_stats.rb.erb +18 -0
  16. data/lib/pgbus/client/ensure_stream_queue.rb +54 -0
  17. data/lib/pgbus/client/read_after.rb +100 -0
  18. data/lib/pgbus/client.rb +6 -0
  19. data/lib/pgbus/configuration/capsule_dsl.rb +6 -20
  20. data/lib/pgbus/configuration.rb +126 -14
  21. data/lib/pgbus/engine.rb +31 -0
  22. data/lib/pgbus/process/dispatcher.rb +62 -4
  23. data/lib/pgbus/streams/cursor.rb +71 -0
  24. data/lib/pgbus/streams/envelope.rb +58 -0
  25. data/lib/pgbus/streams/filters.rb +98 -0
  26. data/lib/pgbus/streams/presence.rb +216 -0
  27. data/lib/pgbus/streams/signed_name.rb +69 -0
  28. data/lib/pgbus/streams/turbo_broadcastable.rb +53 -0
  29. data/lib/pgbus/streams/watermark_cache_middleware.rb +28 -0
  30. data/lib/pgbus/streams.rb +151 -0
  31. data/lib/pgbus/version.rb +1 -1
  32. data/lib/pgbus/web/data_source.rb +29 -0
  33. data/lib/pgbus/web/stream_app.rb +179 -0
  34. data/lib/pgbus/web/streamer/connection.rb +122 -0
  35. data/lib/pgbus/web/streamer/dispatcher.rb +467 -0
  36. data/lib/pgbus/web/streamer/heartbeat.rb +105 -0
  37. data/lib/pgbus/web/streamer/instance.rb +176 -0
  38. data/lib/pgbus/web/streamer/io_writer.rb +73 -0
  39. data/lib/pgbus/web/streamer/listener.rb +228 -0
  40. data/lib/pgbus/web/streamer/registry.rb +103 -0
  41. data/lib/pgbus/web/streamer.rb +53 -0
  42. data/lib/pgbus.rb +28 -0
  43. data/lib/puma/plugin/pgbus_streams.rb +54 -0
  44. data/lib/tasks/pgbus_streams.rake +52 -0
  45. metadata +29 -1
@@ -0,0 +1,118 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ # Records one row per stream event (broadcast / connect / disconnect)
5
+ # when `config.streams_stats_enabled` is true. Mirrors JobStat in
6
+ # shape and aggregation API so the Insights controller can query
7
+ # both with the same patterns.
8
+ #
9
+ # Writes are fire-and-forget: any error is swallowed so a stat
10
+ # recording failure cannot affect the dispatcher's hot path.
11
+ class StreamStat < BusRecord
12
+ self.table_name = "pgbus_stream_stats"
13
+
14
+ EVENT_TYPES = %w[broadcast connect disconnect].freeze
15
+
16
+ scope :since, ->(time) { where("created_at >= ?", time) }
17
+ scope :broadcasts, -> { where(event_type: "broadcast") }
18
+ scope :connects, -> { where(event_type: "connect") }
19
+ scope :disconnects, -> { where(event_type: "disconnect") }
20
+
21
+ # Records a stream event. Called from the Dispatcher when the
22
+ # `streams_stats_enabled` flag is set. Errors are swallowed so a
23
+ # missing table or a connection blip cannot kill the dispatcher.
24
+ def self.record!(stream_name:, event_type:, duration_ms: 0, fanout: nil)
25
+ return unless table_exists?
26
+
27
+ create!(
28
+ stream_name: stream_name,
29
+ event_type: event_type,
30
+ duration_ms: duration_ms.to_i,
31
+ fanout: fanout
32
+ )
33
+ rescue StandardError => e
34
+ Pgbus.logger.debug { "[Pgbus] Failed to record stream stat: #{e.message}" }
35
+ end
36
+
37
+ # Memoized — intentionally never invalidated at runtime. If the
38
+ # pgbus_stream_stats migration runs while the app is already
39
+ # running, a restart is required for stat recording to begin.
40
+ #
41
+ # We only memoize a *successful* probe. A transient error (PG
42
+ # hiccup during boot, connection refused during a failover) is
43
+ # treated as "don't know yet" — the next call retries. Caching
44
+ # false on the first hiccup would permanently disable stream
45
+ # stats for the process lifetime, which is a worse failure mode
46
+ # than a few retries.
47
+ def self.table_exists?
48
+ return @table_exists if defined?(@table_exists)
49
+
50
+ @table_exists = connection.table_exists?(table_name)
51
+ rescue StandardError => e
52
+ Pgbus.logger.debug { "[Pgbus] Failed to check stream stat table: #{e.message}" }
53
+ false
54
+ end
55
+
56
+ # Single-query aggregate summary over the given window.
57
+ # Returns totals by event type, average fanout for broadcasts,
58
+ # and an "active" estimate (connects − disconnects in window).
59
+ def self.summary(minutes: 60)
60
+ row = since(minutes.minutes.ago).pick(
61
+ Arel.sql("COUNT(*) FILTER (WHERE event_type = 'broadcast')"),
62
+ Arel.sql("COUNT(*) FILTER (WHERE event_type = 'connect')"),
63
+ Arel.sql("COUNT(*) FILTER (WHERE event_type = 'disconnect')"),
64
+ Arel.sql("ROUND(AVG(fanout) FILTER (WHERE event_type = 'broadcast')::numeric, 1)"),
65
+ Arel.sql("ROUND(AVG(duration_ms) FILTER (WHERE event_type = 'broadcast')::numeric, 1)"),
66
+ Arel.sql("ROUND(AVG(duration_ms) FILTER (WHERE event_type = 'connect')::numeric, 1)")
67
+ )
68
+
69
+ {
70
+ broadcasts: row[0].to_i,
71
+ connects: row[1].to_i,
72
+ disconnects: row[2].to_i,
73
+ active_estimate: [row[1].to_i - row[2].to_i, 0].max,
74
+ avg_fanout: row[3]&.to_f || 0,
75
+ avg_broadcast_ms: row[4]&.to_f || 0,
76
+ avg_connect_ms: row[5]&.to_f || 0
77
+ }
78
+ end
79
+
80
+ # Top N streams by broadcast count in the window, with avg fanout.
81
+ def self.top_streams(limit: 10, minutes: 60)
82
+ broadcasts
83
+ .since(minutes.minutes.ago)
84
+ .group(:stream_name)
85
+ .order(Arel.sql("COUNT(*) DESC"))
86
+ .limit(limit)
87
+ .pluck(
88
+ :stream_name,
89
+ Arel.sql("COUNT(*)"),
90
+ Arel.sql("ROUND(AVG(fanout)::numeric, 1)"),
91
+ Arel.sql("ROUND(AVG(duration_ms)::numeric, 1)")
92
+ )
93
+ .map do |name, count, avg_fanout, avg_ms|
94
+ {
95
+ stream_name: name,
96
+ count: count.to_i,
97
+ avg_fanout: avg_fanout&.to_f || 0,
98
+ avg_ms: avg_ms&.to_f || 0
99
+ }
100
+ end
101
+ end
102
+
103
+ # Throughput: broadcast events per minute bucketed by minute.
104
+ def self.throughput(minutes: 60)
105
+ broadcasts
106
+ .since(minutes.minutes.ago)
107
+ .group("date_trunc('minute', created_at)")
108
+ .order(Arel.sql("date_trunc('minute', created_at)"))
109
+ .count
110
+ end
111
+
112
+ # Cleanup old stats. Called from Pgbus::Process::Dispatcher on the
113
+ # same cadence as JobStat.cleanup! using the shared stats_retention.
114
+ def self.cleanup!(older_than:)
115
+ where("created_at < ?", older_than).delete_all
116
+ end
117
+ end
118
+ end
@@ -159,6 +159,65 @@
159
159
  </table>
160
160
  </div>
161
161
 
162
+ <% if @stream_stats_available %>
163
+ <!-- Stream stats -->
164
+ <div class="mt-8">
165
+ <h2 class="text-lg font-semibold text-gray-900 dark:text-white mb-4"><%= t("pgbus.insights.show.streams.title") %></h2>
166
+
167
+ <div class="grid grid-cols-2 gap-4 sm:grid-cols-3 lg:grid-cols-5 mb-6">
168
+ <div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
169
+ <dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.broadcasts") %></dt>
170
+ <dd class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white"><%= pgbus_number(@stream_summary[:broadcasts]) %></dd>
171
+ </div>
172
+ <div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
173
+ <dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.connects") %></dt>
174
+ <dd class="mt-1 text-2xl font-semibold text-green-600 dark:text-green-400"><%= pgbus_number(@stream_summary[:connects]) %></dd>
175
+ </div>
176
+ <div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
177
+ <dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.disconnects") %></dt>
178
+ <dd class="mt-1 text-2xl font-semibold text-orange-600 dark:text-orange-400"><%= pgbus_number(@stream_summary[:disconnects]) %></dd>
179
+ </div>
180
+ <div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
181
+ <dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.active") %></dt>
182
+ <dd class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white"><%= pgbus_number(@stream_summary[:active_estimate]) %></dd>
183
+ </div>
184
+ <div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
185
+ <dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.avg_fanout") %></dt>
186
+ <dd class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white"><%= @stream_summary[:avg_fanout]&.round(1) || 0 %></dd>
187
+ </div>
188
+ </div>
189
+
190
+ <div class="rounded-lg bg-white dark:bg-gray-800 shadow ring-1 ring-gray-200 dark:ring-gray-700">
191
+ <div class="px-5 py-4 border-b border-gray-200 dark:border-gray-700">
192
+ <h3 class="text-sm font-medium text-gray-700 dark:text-gray-300"><%= t("pgbus.insights.show.streams.top.title") %></h3>
193
+ </div>
194
+ <table class="pgbus-table min-w-full divide-y divide-gray-200 dark:divide-gray-700">
195
+ <thead class="bg-gray-50 dark:bg-gray-900">
196
+ <tr>
197
+ <th class="px-4 py-3 text-left text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.top.headers.stream") %></th>
198
+ <th class="px-4 py-3 text-right text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.top.headers.broadcasts") %></th>
199
+ <th class="px-4 py-3 text-right text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.top.headers.avg_fanout") %></th>
200
+ <th class="px-4 py-3 text-right text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.top.headers.avg_ms") %></th>
201
+ </tr>
202
+ </thead>
203
+ <tbody class="divide-y divide-gray-100 dark:divide-gray-700">
204
+ <% @top_streams.each do |row| %>
205
+ <tr class="hover:bg-gray-50 dark:hover:bg-gray-700/50">
206
+ <td data-label="Stream" class="px-4 py-3 text-sm font-medium text-gray-700 dark:text-gray-300"><%= row[:stream_name] %></td>
207
+ <td data-label="Broadcasts" class="px-4 py-3 text-sm text-right font-mono text-gray-700 dark:text-gray-300"><%= pgbus_number(row[:count]) %></td>
208
+ <td data-label="Avg Fanout" class="px-4 py-3 text-sm text-right font-mono text-gray-700 dark:text-gray-300"><%= row[:avg_fanout]&.round(1) || 0 %></td>
209
+ <td data-label="Avg Duration" class="px-4 py-3 text-sm text-right font-mono text-gray-700 dark:text-gray-300"><%= pgbus_ms_duration(row[:avg_ms]) %></td>
210
+ </tr>
211
+ <% end %>
212
+ <% if @top_streams.empty? %>
213
+ <tr><td colspan="4" class="px-4 py-8 text-center text-sm text-gray-400 dark:text-gray-500"><%= t("pgbus.insights.show.streams.top.empty") %></td></tr>
214
+ <% end %>
215
+ </tbody>
216
+ </table>
217
+ </div>
218
+ </div>
219
+ <% end %>
220
+
162
221
  <script type="module" nonce="<%= content_security_policy_nonce %>">
163
222
  import { renderCharts, observeThemeChanges } from "charts";
164
223
 
@@ -161,6 +161,22 @@ en:
161
161
  job_class: Job Class
162
162
  max: Max
163
163
  title: Slowest Job Classes (avg duration)
164
+ streams:
165
+ summary:
166
+ active: Active
167
+ avg_fanout: Avg Fanout
168
+ broadcasts: Broadcasts
169
+ connects: Connects
170
+ disconnects: Disconnects
171
+ title: Real-time Streams
172
+ top:
173
+ empty: No stream activity recorded in the selected window
174
+ headers:
175
+ avg_fanout: Avg Fanout
176
+ avg_ms: Avg Dispatch
177
+ broadcasts: Broadcasts
178
+ stream: Stream
179
+ title: Top Streams by Broadcast Volume
164
180
  summary:
165
181
  avg_duration: Avg Duration
166
182
  avg_latency: Avg Latency
data/config/routes.rb CHANGED
@@ -1,6 +1,17 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  Pgbus::Engine.routes.draw do
4
+ # SSE streaming endpoint for the pgbus_stream_from turbo-rails replacement.
5
+ # Mounted as a bare Rack app so it bypasses the entire Rails middleware
6
+ # stack — see lib/pgbus/web/stream_app.rb for the rationale. Named with
7
+ # `as: :streams` so `Pgbus::Engine.routes.url_helpers.streams_path`
8
+ # resolves to whatever mount point the host app chose for the engine,
9
+ # not a hardcoded `/pgbus/streams`. The helper under
10
+ # app/helpers/pgbus/streams_helper.rb appends `/:signed_name` to that
11
+ # base so the full URL works whether the engine is at /pgbus, /admin,
12
+ # or anywhere else.
13
+ mount Pgbus::Web::StreamApp.new => "/streams", as: :streams if Pgbus.configuration.streams_enabled
14
+
4
15
  root to: "dashboard#show"
5
16
 
6
17
  resources :queues, only: %i[index show destroy], param: :name do
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rails/generators"
4
+ require "rails/generators/active_record"
5
+
6
+ module Pgbus
7
+ module Generators
8
+ class AddPresenceGenerator < Rails::Generators::Base
9
+ include ActiveRecord::Generators::Migration
10
+
11
+ source_root File.expand_path("templates", __dir__)
12
+
13
+ desc "Add presence_members table for Pgbus::Streams presence tracking"
14
+
15
+ class_option :database,
16
+ type: :string,
17
+ default: nil,
18
+ desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
19
+
20
+ def create_migration_file
21
+ if separate_database?
22
+ migration_template "add_presence.rb.erb",
23
+ "db/pgbus_migrate/add_pgbus_presence.rb"
24
+ else
25
+ migration_template "add_presence.rb.erb",
26
+ "db/migrate/add_pgbus_presence.rb"
27
+ end
28
+ end
29
+
30
+ def display_post_install
31
+ say ""
32
+ say "Pgbus presence installed!", :green
33
+ say ""
34
+ say "Next steps:"
35
+ say " 1. Run: rails db:migrate#{":#{options[:database]}" if separate_database?}"
36
+ say " 2. Use in your code:"
37
+ say " Pgbus.stream(@room).presence.join(member_id: current_user.id.to_s)"
38
+ say " Pgbus.stream(@room).presence.members"
39
+ say " 3. Run a periodic sweeper to expire idle members:"
40
+ say " Pgbus.stream(@room).presence.sweep!(older_than: 60.seconds.ago)"
41
+ say ""
42
+ end
43
+
44
+ private
45
+
46
+ def migration_version
47
+ "[#{ActiveRecord::Migration.current_version}]"
48
+ end
49
+
50
+ def separate_database?
51
+ options[:database].present?
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rails/generators"
4
+ require "rails/generators/active_record"
5
+
6
+ module Pgbus
7
+ module Generators
8
+ class AddStreamStatsGenerator < Rails::Generators::Base
9
+ include ActiveRecord::Generators::Migration
10
+
11
+ source_root File.expand_path("templates", __dir__)
12
+
13
+ desc "Add stream stats table for Insights dashboard (opt-in, disabled by default)"
14
+
15
+ class_option :database,
16
+ type: :string,
17
+ default: nil,
18
+ desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
19
+
20
+ def create_migration_file
21
+ if separate_database?
22
+ migration_template "add_stream_stats.rb.erb",
23
+ "db/pgbus_migrate/add_pgbus_stream_stats.rb"
24
+ else
25
+ migration_template "add_stream_stats.rb.erb",
26
+ "db/migrate/add_pgbus_stream_stats.rb"
27
+ end
28
+ end
29
+
30
+ def display_post_install
31
+ say ""
32
+ say "Pgbus stream stats table installed!", :green
33
+ say ""
34
+ say "Next steps:"
35
+ say " 1. Run: rails db:migrate#{":#{options[:database]}" if separate_database?}"
36
+ say " 2. Opt in by setting `config.streams_stats_enabled = true` in your"
37
+ say " pgbus initializer (disabled by default — stream event volume can"
38
+ say " be high, so stats recording is off unless you ask for it)."
39
+ say " 3. View stream insights at /pgbus/insights"
40
+ say ""
41
+ end
42
+
43
+ private
44
+
45
+ def migration_version
46
+ "[#{ActiveRecord::Migration.current_version}]"
47
+ end
48
+
49
+ def separate_database?
50
+ options[:database].present?
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,26 @@
1
+ class AddPgbusPresence < ActiveRecord::Migration<%= migration_version %>
2
+ def change
3
+ create_table :pgbus_presence_members, id: false do |t|
4
+ t.string :stream_name, null: false
5
+ t.string :member_id, null: false
6
+ t.jsonb :metadata, null: false, default: {}
7
+ t.datetime :joined_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
8
+ t.datetime :last_seen_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
9
+ end
10
+
11
+ # Composite primary key (stream_name, member_id) — a member can only
12
+ # appear once per stream. Used by the upsert path in
13
+ # Pgbus::Streams::Presence#join.
14
+ add_index :pgbus_presence_members,
15
+ %i[stream_name member_id],
16
+ unique: true,
17
+ name: "idx_pgbus_presence_members_pk"
18
+
19
+ # Sweeping by last_seen_at expires stale members. The Dispatcher's
20
+ # periodic sweeper queries WHERE stream_name = ? AND last_seen_at < ?,
21
+ # so this composite index makes that fast.
22
+ add_index :pgbus_presence_members,
23
+ %i[stream_name last_seen_at],
24
+ name: "idx_pgbus_presence_members_sweep"
25
+ end
26
+ end
@@ -0,0 +1,18 @@
1
+ class AddPgbusStreamStats < ActiveRecord::Migration<%= migration_version %>
2
+ def change
3
+ create_table :pgbus_stream_stats do |t|
4
+ t.string :stream_name, null: false
5
+ t.string :event_type, null: false # 'broadcast' | 'connect' | 'disconnect'
6
+ t.integer :duration_ms, null: false, default: 0
7
+ t.integer :fanout # nil for connect/disconnect
8
+ t.datetime :created_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
9
+ end
10
+
11
+ add_index :pgbus_stream_stats, :created_at,
12
+ name: "idx_pgbus_stream_stats_time"
13
+ add_index :pgbus_stream_stats, %i[stream_name created_at],
14
+ name: "idx_pgbus_stream_stats_stream_time"
15
+ add_index :pgbus_stream_stats, %i[event_type created_at],
16
+ name: "idx_pgbus_stream_stats_type_time"
17
+ end
18
+ end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ class Client
5
+ # Idempotent stream-queue setup. Creates the PGMQ queue (delegating to
6
+ # `ensure_queue` which already handles schema bootstrap and dedup),
7
+ # overrides the NOTIFY throttle to 0 so every broadcast fires its
8
+ # own NOTIFY, and adds an `msg_id` index on the archive table that
9
+ # PGMQ does not ship with.
10
+ #
11
+ # PGMQ's archive tables (`pgmq.a_<name>`) only carry an `archived_at`
12
+ # index by default. `Client#read_after`'s replay query filters by
13
+ # `WHERE msg_id > $1`, which becomes a sequential scan once the archive
14
+ # grows past a few thousand rows. We add the index here, scoped to
15
+ # stream queues only, so users with chat-history-style retention don't
16
+ # hit a performance cliff.
17
+ #
18
+ # Called from `Pgbus.stream(name).broadcast(...)` on first publish per
19
+ # stream and from the streamer on first subscription per stream.
20
+ module EnsureStreamQueue
21
+ def ensure_stream_queue(stream_name)
22
+ ensure_queue(stream_name)
23
+ full_name = config.queue_name(stream_name)
24
+
25
+ # PGMQ's default NOTIFY throttle is 250ms — meant to coalesce
26
+ # high-frequency worker queue inserts. Streams are latency-
27
+ # sensitive and need every broadcast to fire a NOTIFY, even
28
+ # when several are batched within a single millisecond.
29
+ # Override the throttle to 0 specifically for stream queues.
30
+ synchronized { @pgmq.enable_notify_insert(full_name, throttle_interval_ms: 0) } if config.listen_notify
31
+
32
+ # CREATE INDEX IF NOT EXISTS is idempotent in Postgres but still
33
+ # requires a roundtrip and a brief ACCESS SHARE lock on the archive
34
+ # table. Broadcast-per-after_commit loops can hit this 1000x/sec on
35
+ # the same stream, so memoize per-process after the first success.
36
+ return if @stream_indexes_created[stream_name]
37
+
38
+ sanitized = QueueNameValidator.sanitize!(full_name)
39
+ sql = <<~SQL
40
+ CREATE INDEX IF NOT EXISTS a_#{sanitized}_msg_id_idx
41
+ ON pgmq.a_#{sanitized} (msg_id)
42
+ SQL
43
+
44
+ synchronized do
45
+ with_raw_connection do |conn|
46
+ conn.exec(sql)
47
+ end
48
+ end
49
+
50
+ @stream_indexes_created[stream_name] = true
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,100 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pgbus
4
+ class Client
5
+ # Non-consuming peek across PGMQ live (`q_`) and archive (`a_`) tables. Used
6
+ # exclusively by `Pgbus::Web::Streamer` for SSE replay — workers continue to
7
+ # use `read_batch` (claim semantics). The two read paths are disjoint.
8
+ #
9
+ # The cursor is the highest msg_id the client has already seen. Replay returns
10
+ # everything strictly greater, ordered by msg_id ASC, capped by `limit`.
11
+ module ReadAfter
12
+ Envelope = Data.define(:msg_id, :enqueued_at, :payload, :source)
13
+
14
+ DEFAULT_LIMIT = 500
15
+
16
+ def read_after(stream_name, after_id:, limit: DEFAULT_LIMIT)
17
+ sanitized = sanitized_queue(stream_name)
18
+ sql = build_read_after_sql(sanitized)
19
+
20
+ rows = synchronized do
21
+ with_raw_connection do |conn|
22
+ conn.exec_params(sql, [after_id.to_i, limit.to_i]).to_a
23
+ end
24
+ end
25
+
26
+ rows.map { |row| build_envelope(row) }
27
+ end
28
+
29
+ def stream_current_msg_id(stream_name)
30
+ sanitized = sanitized_queue(stream_name)
31
+ sql = "SELECT COALESCE(MAX(msg_id), 0) AS max FROM pgmq.q_#{sanitized}"
32
+ synchronized do
33
+ with_raw_connection do |conn|
34
+ conn.exec(sql).first.fetch("max").to_i
35
+ end
36
+ end
37
+ end
38
+
39
+ def stream_oldest_msg_id(stream_name)
40
+ sanitized = sanitized_queue(stream_name)
41
+ sql = <<~SQL
42
+ SELECT LEAST(
43
+ (SELECT MIN(msg_id) FROM pgmq.q_#{sanitized}),
44
+ (SELECT MIN(msg_id) FROM pgmq.a_#{sanitized})
45
+ ) AS least
46
+ SQL
47
+ synchronized do
48
+ with_raw_connection do |conn|
49
+ value = conn.exec(sql).first.fetch("least")
50
+ value&.to_i
51
+ end
52
+ end
53
+ end
54
+
55
+ private
56
+
57
+ # Builds the union of live and archive tables. The outer ORDER BY + LIMIT
58
+ # ensures we never return more than `limit` rows total even if both
59
+ # subqueries hit it. The 'live'/'archive' constants are how the streamer
60
+ # tells whether a row was peeked from the queue or replayed from history;
61
+ # the streamer doesn't currently distinguish them, but we keep the column
62
+ # so debugging is straightforward when archive replay misbehaves.
63
+ def build_read_after_sql(sanitized)
64
+ <<~SQL
65
+ (
66
+ SELECT msg_id, enqueued_at, message, 'live'::text AS source
67
+ FROM pgmq.q_#{sanitized}
68
+ WHERE msg_id > $1
69
+ ORDER BY msg_id ASC
70
+ LIMIT $2
71
+ )
72
+ UNION ALL
73
+ (
74
+ SELECT msg_id, enqueued_at, message, 'archive'::text AS source
75
+ FROM pgmq.a_#{sanitized}
76
+ WHERE msg_id > $1
77
+ ORDER BY msg_id ASC
78
+ LIMIT $2
79
+ )
80
+ ORDER BY msg_id ASC
81
+ LIMIT $2
82
+ SQL
83
+ end
84
+
85
+ def sanitized_queue(stream_name)
86
+ full = config.queue_name(stream_name)
87
+ QueueNameValidator.sanitize!(full)
88
+ end
89
+
90
+ def build_envelope(row)
91
+ Envelope.new(
92
+ msg_id: row.fetch("msg_id").to_i,
93
+ enqueued_at: row.fetch("enqueued_at"),
94
+ payload: row.fetch("message"),
95
+ source: row.fetch("source")
96
+ )
97
+ end
98
+ end
99
+ end
100
+ end
data/lib/pgbus/client.rb CHANGED
@@ -1,9 +1,14 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "json"
4
+ require_relative "client/read_after"
5
+ require_relative "client/ensure_stream_queue"
4
6
 
5
7
  module Pgbus
6
8
  class Client
9
+ include ReadAfter
10
+ include EnsureStreamQueue
11
+
7
12
  attr_reader :pgmq, :config
8
13
 
9
14
  PGMQ_REQUIRE_MUTEX = Mutex.new
@@ -49,6 +54,7 @@ module Pgbus
49
54
  end
50
55
 
51
56
  @queues_created = Concurrent::Map.new
57
+ @stream_indexes_created = Concurrent::Map.new
52
58
  @queue_strategy = QueueFactory.for(config)
53
59
  @schema_ensured = false
54
60
  end
@@ -62,9 +62,12 @@ module Pgbus
62
62
  validate_input_type!
63
63
  validate_input_not_empty!
64
64
 
65
- capsules = split_capsules(@input).map { |segment| parse_capsule(segment) }
66
- validate_no_duplicate_queues_across_capsules!(capsules)
67
- capsules
65
+ # Pure tokenization: split, parse each capsule, return them in order.
66
+ # Cross-capsule overlap rules live in Pgbus::Configuration#workers=
67
+ # because they depend on whether the resulting capsules are named or
68
+ # anonymous, and naming is a Configuration concern (not a parser one).
69
+ # Within-capsule duplicate-queue checks still happen in parse_capsule.
70
+ split_capsules(@input).map { |segment| parse_capsule(segment) }
68
71
  end
69
72
 
70
73
  private
@@ -168,23 +171,6 @@ module Pgbus
168
171
  seen[q] = true
169
172
  end
170
173
  end
171
-
172
- def validate_no_duplicate_queues_across_capsules!(capsules)
173
- return if capsules.size < 2
174
-
175
- seen = {}
176
- capsules.each_with_index do |capsule, idx|
177
- capsule[:queues].each do |q|
178
- if seen[q]
179
- label = (q == WILDCARD ? "wildcard '*'" : "queue #{q.inspect}")
180
- raise ParseError,
181
- "#{label} appears in two capsules (positions #{seen[q]} and #{idx + 1}) " \
182
- "in #{@input.inspect} — each queue can only be assigned to one capsule"
183
- end
184
- seen[q] = idx + 1
185
- end
186
- end
187
- end
188
174
  end
189
175
  end
190
176
  end