pgbus 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +15 -0
- data/README.md +238 -0
- data/Rakefile +8 -1
- data/app/controllers/pgbus/insights_controller.rb +6 -0
- data/app/helpers/pgbus/streams_helper.rb +115 -0
- data/app/javascript/pgbus/stream_source_element.js +212 -0
- data/app/models/pgbus/stream_stat.rb +118 -0
- data/app/views/pgbus/insights/show.html.erb +59 -0
- data/config/locales/en.yml +16 -0
- data/config/routes.rb +11 -0
- data/lib/generators/pgbus/add_presence_generator.rb +55 -0
- data/lib/generators/pgbus/add_stream_stats_generator.rb +54 -0
- data/lib/generators/pgbus/templates/add_presence.rb.erb +26 -0
- data/lib/generators/pgbus/templates/add_stream_stats.rb.erb +18 -0
- data/lib/pgbus/client/ensure_stream_queue.rb +54 -0
- data/lib/pgbus/client/read_after.rb +100 -0
- data/lib/pgbus/client.rb +6 -0
- data/lib/pgbus/configuration/capsule_dsl.rb +6 -20
- data/lib/pgbus/configuration.rb +126 -14
- data/lib/pgbus/engine.rb +31 -0
- data/lib/pgbus/process/dispatcher.rb +62 -4
- data/lib/pgbus/streams/cursor.rb +71 -0
- data/lib/pgbus/streams/envelope.rb +58 -0
- data/lib/pgbus/streams/filters.rb +98 -0
- data/lib/pgbus/streams/presence.rb +216 -0
- data/lib/pgbus/streams/signed_name.rb +69 -0
- data/lib/pgbus/streams/turbo_broadcastable.rb +53 -0
- data/lib/pgbus/streams/watermark_cache_middleware.rb +28 -0
- data/lib/pgbus/streams.rb +151 -0
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +29 -0
- data/lib/pgbus/web/stream_app.rb +179 -0
- data/lib/pgbus/web/streamer/connection.rb +122 -0
- data/lib/pgbus/web/streamer/dispatcher.rb +467 -0
- data/lib/pgbus/web/streamer/heartbeat.rb +105 -0
- data/lib/pgbus/web/streamer/instance.rb +176 -0
- data/lib/pgbus/web/streamer/io_writer.rb +73 -0
- data/lib/pgbus/web/streamer/listener.rb +228 -0
- data/lib/pgbus/web/streamer/registry.rb +103 -0
- data/lib/pgbus/web/streamer.rb +53 -0
- data/lib/pgbus.rb +28 -0
- data/lib/puma/plugin/pgbus_streams.rb +54 -0
- data/lib/tasks/pgbus_streams.rake +52 -0
- metadata +29 -1
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
# Records one row per stream event (broadcast / connect / disconnect)
|
|
5
|
+
# when `config.streams_stats_enabled` is true. Mirrors JobStat in
|
|
6
|
+
# shape and aggregation API so the Insights controller can query
|
|
7
|
+
# both with the same patterns.
|
|
8
|
+
#
|
|
9
|
+
# Writes are fire-and-forget: any error is swallowed so a stat
|
|
10
|
+
# recording failure cannot affect the dispatcher's hot path.
|
|
11
|
+
class StreamStat < BusRecord
|
|
12
|
+
self.table_name = "pgbus_stream_stats"
|
|
13
|
+
|
|
14
|
+
EVENT_TYPES = %w[broadcast connect disconnect].freeze
|
|
15
|
+
|
|
16
|
+
scope :since, ->(time) { where("created_at >= ?", time) }
|
|
17
|
+
scope :broadcasts, -> { where(event_type: "broadcast") }
|
|
18
|
+
scope :connects, -> { where(event_type: "connect") }
|
|
19
|
+
scope :disconnects, -> { where(event_type: "disconnect") }
|
|
20
|
+
|
|
21
|
+
# Records a stream event. Called from the Dispatcher when the
|
|
22
|
+
# `streams_stats_enabled` flag is set. Errors are swallowed so a
|
|
23
|
+
# missing table or a connection blip cannot kill the dispatcher.
|
|
24
|
+
def self.record!(stream_name:, event_type:, duration_ms: 0, fanout: nil)
|
|
25
|
+
return unless table_exists?
|
|
26
|
+
|
|
27
|
+
create!(
|
|
28
|
+
stream_name: stream_name,
|
|
29
|
+
event_type: event_type,
|
|
30
|
+
duration_ms: duration_ms.to_i,
|
|
31
|
+
fanout: fanout
|
|
32
|
+
)
|
|
33
|
+
rescue StandardError => e
|
|
34
|
+
Pgbus.logger.debug { "[Pgbus] Failed to record stream stat: #{e.message}" }
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Memoized — intentionally never invalidated at runtime. If the
|
|
38
|
+
# pgbus_stream_stats migration runs while the app is already
|
|
39
|
+
# running, a restart is required for stat recording to begin.
|
|
40
|
+
#
|
|
41
|
+
# We only memoize a *successful* probe. A transient error (PG
|
|
42
|
+
# hiccup during boot, connection refused during a failover) is
|
|
43
|
+
# treated as "don't know yet" — the next call retries. Caching
|
|
44
|
+
# false on the first hiccup would permanently disable stream
|
|
45
|
+
# stats for the process lifetime, which is a worse failure mode
|
|
46
|
+
# than a few retries.
|
|
47
|
+
def self.table_exists?
|
|
48
|
+
return @table_exists if defined?(@table_exists)
|
|
49
|
+
|
|
50
|
+
@table_exists = connection.table_exists?(table_name)
|
|
51
|
+
rescue StandardError => e
|
|
52
|
+
Pgbus.logger.debug { "[Pgbus] Failed to check stream stat table: #{e.message}" }
|
|
53
|
+
false
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Single-query aggregate summary over the given window.
|
|
57
|
+
# Returns totals by event type, average fanout for broadcasts,
|
|
58
|
+
# and an "active" estimate (connects − disconnects in window).
|
|
59
|
+
def self.summary(minutes: 60)
|
|
60
|
+
row = since(minutes.minutes.ago).pick(
|
|
61
|
+
Arel.sql("COUNT(*) FILTER (WHERE event_type = 'broadcast')"),
|
|
62
|
+
Arel.sql("COUNT(*) FILTER (WHERE event_type = 'connect')"),
|
|
63
|
+
Arel.sql("COUNT(*) FILTER (WHERE event_type = 'disconnect')"),
|
|
64
|
+
Arel.sql("ROUND(AVG(fanout) FILTER (WHERE event_type = 'broadcast')::numeric, 1)"),
|
|
65
|
+
Arel.sql("ROUND(AVG(duration_ms) FILTER (WHERE event_type = 'broadcast')::numeric, 1)"),
|
|
66
|
+
Arel.sql("ROUND(AVG(duration_ms) FILTER (WHERE event_type = 'connect')::numeric, 1)")
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
{
|
|
70
|
+
broadcasts: row[0].to_i,
|
|
71
|
+
connects: row[1].to_i,
|
|
72
|
+
disconnects: row[2].to_i,
|
|
73
|
+
active_estimate: [row[1].to_i - row[2].to_i, 0].max,
|
|
74
|
+
avg_fanout: row[3]&.to_f || 0,
|
|
75
|
+
avg_broadcast_ms: row[4]&.to_f || 0,
|
|
76
|
+
avg_connect_ms: row[5]&.to_f || 0
|
|
77
|
+
}
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Top N streams by broadcast count in the window, with avg fanout.
|
|
81
|
+
def self.top_streams(limit: 10, minutes: 60)
|
|
82
|
+
broadcasts
|
|
83
|
+
.since(minutes.minutes.ago)
|
|
84
|
+
.group(:stream_name)
|
|
85
|
+
.order(Arel.sql("COUNT(*) DESC"))
|
|
86
|
+
.limit(limit)
|
|
87
|
+
.pluck(
|
|
88
|
+
:stream_name,
|
|
89
|
+
Arel.sql("COUNT(*)"),
|
|
90
|
+
Arel.sql("ROUND(AVG(fanout)::numeric, 1)"),
|
|
91
|
+
Arel.sql("ROUND(AVG(duration_ms)::numeric, 1)")
|
|
92
|
+
)
|
|
93
|
+
.map do |name, count, avg_fanout, avg_ms|
|
|
94
|
+
{
|
|
95
|
+
stream_name: name,
|
|
96
|
+
count: count.to_i,
|
|
97
|
+
avg_fanout: avg_fanout&.to_f || 0,
|
|
98
|
+
avg_ms: avg_ms&.to_f || 0
|
|
99
|
+
}
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Throughput: broadcast events per minute bucketed by minute.
|
|
104
|
+
def self.throughput(minutes: 60)
|
|
105
|
+
broadcasts
|
|
106
|
+
.since(minutes.minutes.ago)
|
|
107
|
+
.group("date_trunc('minute', created_at)")
|
|
108
|
+
.order(Arel.sql("date_trunc('minute', created_at)"))
|
|
109
|
+
.count
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Cleanup old stats. Called from Pgbus::Process::Dispatcher on the
|
|
113
|
+
# same cadence as JobStat.cleanup! using the shared stats_retention.
|
|
114
|
+
def self.cleanup!(older_than:)
|
|
115
|
+
where("created_at < ?", older_than).delete_all
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
@@ -159,6 +159,65 @@
|
|
|
159
159
|
</table>
|
|
160
160
|
</div>
|
|
161
161
|
|
|
162
|
+
<% if @stream_stats_available %>
|
|
163
|
+
<!-- Stream stats -->
|
|
164
|
+
<div class="mt-8">
|
|
165
|
+
<h2 class="text-lg font-semibold text-gray-900 dark:text-white mb-4"><%= t("pgbus.insights.show.streams.title") %></h2>
|
|
166
|
+
|
|
167
|
+
<div class="grid grid-cols-2 gap-4 sm:grid-cols-3 lg:grid-cols-5 mb-6">
|
|
168
|
+
<div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
|
|
169
|
+
<dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.broadcasts") %></dt>
|
|
170
|
+
<dd class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white"><%= pgbus_number(@stream_summary[:broadcasts]) %></dd>
|
|
171
|
+
</div>
|
|
172
|
+
<div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
|
|
173
|
+
<dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.connects") %></dt>
|
|
174
|
+
<dd class="mt-1 text-2xl font-semibold text-green-600 dark:text-green-400"><%= pgbus_number(@stream_summary[:connects]) %></dd>
|
|
175
|
+
</div>
|
|
176
|
+
<div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
|
|
177
|
+
<dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.disconnects") %></dt>
|
|
178
|
+
<dd class="mt-1 text-2xl font-semibold text-orange-600 dark:text-orange-400"><%= pgbus_number(@stream_summary[:disconnects]) %></dd>
|
|
179
|
+
</div>
|
|
180
|
+
<div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
|
|
181
|
+
<dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.active") %></dt>
|
|
182
|
+
<dd class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white"><%= pgbus_number(@stream_summary[:active_estimate]) %></dd>
|
|
183
|
+
</div>
|
|
184
|
+
<div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
|
|
185
|
+
<dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.avg_fanout") %></dt>
|
|
186
|
+
<dd class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white"><%= @stream_summary[:avg_fanout]&.round(1) || 0 %></dd>
|
|
187
|
+
</div>
|
|
188
|
+
</div>
|
|
189
|
+
|
|
190
|
+
<div class="rounded-lg bg-white dark:bg-gray-800 shadow ring-1 ring-gray-200 dark:ring-gray-700">
|
|
191
|
+
<div class="px-5 py-4 border-b border-gray-200 dark:border-gray-700">
|
|
192
|
+
<h3 class="text-sm font-medium text-gray-700 dark:text-gray-300"><%= t("pgbus.insights.show.streams.top.title") %></h3>
|
|
193
|
+
</div>
|
|
194
|
+
<table class="pgbus-table min-w-full divide-y divide-gray-200 dark:divide-gray-700">
|
|
195
|
+
<thead class="bg-gray-50 dark:bg-gray-900">
|
|
196
|
+
<tr>
|
|
197
|
+
<th class="px-4 py-3 text-left text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.top.headers.stream") %></th>
|
|
198
|
+
<th class="px-4 py-3 text-right text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.top.headers.broadcasts") %></th>
|
|
199
|
+
<th class="px-4 py-3 text-right text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.top.headers.avg_fanout") %></th>
|
|
200
|
+
<th class="px-4 py-3 text-right text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.top.headers.avg_ms") %></th>
|
|
201
|
+
</tr>
|
|
202
|
+
</thead>
|
|
203
|
+
<tbody class="divide-y divide-gray-100 dark:divide-gray-700">
|
|
204
|
+
<% @top_streams.each do |row| %>
|
|
205
|
+
<tr class="hover:bg-gray-50 dark:hover:bg-gray-700/50">
|
|
206
|
+
<td data-label="Stream" class="px-4 py-3 text-sm font-medium text-gray-700 dark:text-gray-300"><%= row[:stream_name] %></td>
|
|
207
|
+
<td data-label="Broadcasts" class="px-4 py-3 text-sm text-right font-mono text-gray-700 dark:text-gray-300"><%= pgbus_number(row[:count]) %></td>
|
|
208
|
+
<td data-label="Avg Fanout" class="px-4 py-3 text-sm text-right font-mono text-gray-700 dark:text-gray-300"><%= row[:avg_fanout]&.round(1) || 0 %></td>
|
|
209
|
+
<td data-label="Avg Duration" class="px-4 py-3 text-sm text-right font-mono text-gray-700 dark:text-gray-300"><%= pgbus_ms_duration(row[:avg_ms]) %></td>
|
|
210
|
+
</tr>
|
|
211
|
+
<% end %>
|
|
212
|
+
<% if @top_streams.empty? %>
|
|
213
|
+
<tr><td colspan="4" class="px-4 py-8 text-center text-sm text-gray-400 dark:text-gray-500"><%= t("pgbus.insights.show.streams.top.empty") %></td></tr>
|
|
214
|
+
<% end %>
|
|
215
|
+
</tbody>
|
|
216
|
+
</table>
|
|
217
|
+
</div>
|
|
218
|
+
</div>
|
|
219
|
+
<% end %>
|
|
220
|
+
|
|
162
221
|
<script type="module" nonce="<%= content_security_policy_nonce %>">
|
|
163
222
|
import { renderCharts, observeThemeChanges } from "charts";
|
|
164
223
|
|
data/config/locales/en.yml
CHANGED
|
@@ -161,6 +161,22 @@ en:
|
|
|
161
161
|
job_class: Job Class
|
|
162
162
|
max: Max
|
|
163
163
|
title: Slowest Job Classes (avg duration)
|
|
164
|
+
streams:
|
|
165
|
+
summary:
|
|
166
|
+
active: Active
|
|
167
|
+
avg_fanout: Avg Fanout
|
|
168
|
+
broadcasts: Broadcasts
|
|
169
|
+
connects: Connects
|
|
170
|
+
disconnects: Disconnects
|
|
171
|
+
title: Real-time Streams
|
|
172
|
+
top:
|
|
173
|
+
empty: No stream activity recorded in the selected window
|
|
174
|
+
headers:
|
|
175
|
+
avg_fanout: Avg Fanout
|
|
176
|
+
avg_ms: Avg Dispatch
|
|
177
|
+
broadcasts: Broadcasts
|
|
178
|
+
stream: Stream
|
|
179
|
+
title: Top Streams by Broadcast Volume
|
|
164
180
|
summary:
|
|
165
181
|
avg_duration: Avg Duration
|
|
166
182
|
avg_latency: Avg Latency
|
data/config/routes.rb
CHANGED
|
@@ -1,6 +1,17 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
Pgbus::Engine.routes.draw do
|
|
4
|
+
# SSE streaming endpoint for the pgbus_stream_from turbo-rails replacement.
|
|
5
|
+
# Mounted as a bare Rack app so it bypasses the entire Rails middleware
|
|
6
|
+
# stack — see lib/pgbus/web/stream_app.rb for the rationale. Named with
|
|
7
|
+
# `as: :streams` so `Pgbus::Engine.routes.url_helpers.streams_path`
|
|
8
|
+
# resolves to whatever mount point the host app chose for the engine,
|
|
9
|
+
# not a hardcoded `/pgbus/streams`. The helper under
|
|
10
|
+
# app/helpers/pgbus/streams_helper.rb appends `/:signed_name` to that
|
|
11
|
+
# base so the full URL works whether the engine is at /pgbus, /admin,
|
|
12
|
+
# or anywhere else.
|
|
13
|
+
mount Pgbus::Web::StreamApp.new => "/streams", as: :streams if Pgbus.configuration.streams_enabled
|
|
14
|
+
|
|
4
15
|
root to: "dashboard#show"
|
|
5
16
|
|
|
6
17
|
resources :queues, only: %i[index show destroy], param: :name do
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
|
|
6
|
+
module Pgbus
|
|
7
|
+
module Generators
|
|
8
|
+
class AddPresenceGenerator < Rails::Generators::Base
|
|
9
|
+
include ActiveRecord::Generators::Migration
|
|
10
|
+
|
|
11
|
+
source_root File.expand_path("templates", __dir__)
|
|
12
|
+
|
|
13
|
+
desc "Add presence_members table for Pgbus::Streams presence tracking"
|
|
14
|
+
|
|
15
|
+
class_option :database,
|
|
16
|
+
type: :string,
|
|
17
|
+
default: nil,
|
|
18
|
+
desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
|
|
19
|
+
|
|
20
|
+
def create_migration_file
|
|
21
|
+
if separate_database?
|
|
22
|
+
migration_template "add_presence.rb.erb",
|
|
23
|
+
"db/pgbus_migrate/add_pgbus_presence.rb"
|
|
24
|
+
else
|
|
25
|
+
migration_template "add_presence.rb.erb",
|
|
26
|
+
"db/migrate/add_pgbus_presence.rb"
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def display_post_install
|
|
31
|
+
say ""
|
|
32
|
+
say "Pgbus presence installed!", :green
|
|
33
|
+
say ""
|
|
34
|
+
say "Next steps:"
|
|
35
|
+
say " 1. Run: rails db:migrate#{":#{options[:database]}" if separate_database?}"
|
|
36
|
+
say " 2. Use in your code:"
|
|
37
|
+
say " Pgbus.stream(@room).presence.join(member_id: current_user.id.to_s)"
|
|
38
|
+
say " Pgbus.stream(@room).presence.members"
|
|
39
|
+
say " 3. Run a periodic sweeper to expire idle members:"
|
|
40
|
+
say " Pgbus.stream(@room).presence.sweep!(older_than: 60.seconds.ago)"
|
|
41
|
+
say ""
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def migration_version
|
|
47
|
+
"[#{ActiveRecord::Migration.current_version}]"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def separate_database?
|
|
51
|
+
options[:database].present?
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
|
|
6
|
+
module Pgbus
|
|
7
|
+
module Generators
|
|
8
|
+
class AddStreamStatsGenerator < Rails::Generators::Base
|
|
9
|
+
include ActiveRecord::Generators::Migration
|
|
10
|
+
|
|
11
|
+
source_root File.expand_path("templates", __dir__)
|
|
12
|
+
|
|
13
|
+
desc "Add stream stats table for Insights dashboard (opt-in, disabled by default)"
|
|
14
|
+
|
|
15
|
+
class_option :database,
|
|
16
|
+
type: :string,
|
|
17
|
+
default: nil,
|
|
18
|
+
desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
|
|
19
|
+
|
|
20
|
+
def create_migration_file
|
|
21
|
+
if separate_database?
|
|
22
|
+
migration_template "add_stream_stats.rb.erb",
|
|
23
|
+
"db/pgbus_migrate/add_pgbus_stream_stats.rb"
|
|
24
|
+
else
|
|
25
|
+
migration_template "add_stream_stats.rb.erb",
|
|
26
|
+
"db/migrate/add_pgbus_stream_stats.rb"
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def display_post_install
|
|
31
|
+
say ""
|
|
32
|
+
say "Pgbus stream stats table installed!", :green
|
|
33
|
+
say ""
|
|
34
|
+
say "Next steps:"
|
|
35
|
+
say " 1. Run: rails db:migrate#{":#{options[:database]}" if separate_database?}"
|
|
36
|
+
say " 2. Opt in by setting `config.streams_stats_enabled = true` in your"
|
|
37
|
+
say " pgbus initializer (disabled by default — stream event volume can"
|
|
38
|
+
say " be high, so stats recording is off unless you ask for it)."
|
|
39
|
+
say " 3. View stream insights at /pgbus/insights"
|
|
40
|
+
say ""
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
private
|
|
44
|
+
|
|
45
|
+
def migration_version
|
|
46
|
+
"[#{ActiveRecord::Migration.current_version}]"
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def separate_database?
|
|
50
|
+
options[:database].present?
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
class AddPgbusPresence < ActiveRecord::Migration<%= migration_version %>
|
|
2
|
+
def change
|
|
3
|
+
create_table :pgbus_presence_members, id: false do |t|
|
|
4
|
+
t.string :stream_name, null: false
|
|
5
|
+
t.string :member_id, null: false
|
|
6
|
+
t.jsonb :metadata, null: false, default: {}
|
|
7
|
+
t.datetime :joined_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
|
|
8
|
+
t.datetime :last_seen_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
# Composite primary key (stream_name, member_id) — a member can only
|
|
12
|
+
# appear once per stream. Used by the upsert path in
|
|
13
|
+
# Pgbus::Streams::Presence#join.
|
|
14
|
+
add_index :pgbus_presence_members,
|
|
15
|
+
%i[stream_name member_id],
|
|
16
|
+
unique: true,
|
|
17
|
+
name: "idx_pgbus_presence_members_pk"
|
|
18
|
+
|
|
19
|
+
# Sweeping by last_seen_at expires stale members. The Dispatcher's
|
|
20
|
+
# periodic sweeper queries WHERE stream_name = ? AND last_seen_at < ?,
|
|
21
|
+
# so this composite index makes that fast.
|
|
22
|
+
add_index :pgbus_presence_members,
|
|
23
|
+
%i[stream_name last_seen_at],
|
|
24
|
+
name: "idx_pgbus_presence_members_sweep"
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
class AddPgbusStreamStats < ActiveRecord::Migration<%= migration_version %>
|
|
2
|
+
def change
|
|
3
|
+
create_table :pgbus_stream_stats do |t|
|
|
4
|
+
t.string :stream_name, null: false
|
|
5
|
+
t.string :event_type, null: false # 'broadcast' | 'connect' | 'disconnect'
|
|
6
|
+
t.integer :duration_ms, null: false, default: 0
|
|
7
|
+
t.integer :fanout # nil for connect/disconnect
|
|
8
|
+
t.datetime :created_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
add_index :pgbus_stream_stats, :created_at,
|
|
12
|
+
name: "idx_pgbus_stream_stats_time"
|
|
13
|
+
add_index :pgbus_stream_stats, %i[stream_name created_at],
|
|
14
|
+
name: "idx_pgbus_stream_stats_stream_time"
|
|
15
|
+
add_index :pgbus_stream_stats, %i[event_type created_at],
|
|
16
|
+
name: "idx_pgbus_stream_stats_type_time"
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class Client
|
|
5
|
+
# Idempotent stream-queue setup. Creates the PGMQ queue (delegating to
|
|
6
|
+
# `ensure_queue` which already handles schema bootstrap and dedup),
|
|
7
|
+
# overrides the NOTIFY throttle to 0 so every broadcast fires its
|
|
8
|
+
# own NOTIFY, and adds an `msg_id` index on the archive table that
|
|
9
|
+
# PGMQ does not ship with.
|
|
10
|
+
#
|
|
11
|
+
# PGMQ's archive tables (`pgmq.a_<name>`) only carry an `archived_at`
|
|
12
|
+
# index by default. `Client#read_after`'s replay query filters by
|
|
13
|
+
# `WHERE msg_id > $1`, which becomes a sequential scan once the archive
|
|
14
|
+
# grows past a few thousand rows. We add the index here, scoped to
|
|
15
|
+
# stream queues only, so users with chat-history-style retention don't
|
|
16
|
+
# hit a performance cliff.
|
|
17
|
+
#
|
|
18
|
+
# Called from `Pgbus.stream(name).broadcast(...)` on first publish per
|
|
19
|
+
# stream and from the streamer on first subscription per stream.
|
|
20
|
+
module EnsureStreamQueue
|
|
21
|
+
def ensure_stream_queue(stream_name)
|
|
22
|
+
ensure_queue(stream_name)
|
|
23
|
+
full_name = config.queue_name(stream_name)
|
|
24
|
+
|
|
25
|
+
# PGMQ's default NOTIFY throttle is 250ms — meant to coalesce
|
|
26
|
+
# high-frequency worker queue inserts. Streams are latency-
|
|
27
|
+
# sensitive and need every broadcast to fire a NOTIFY, even
|
|
28
|
+
# when several are batched within a single millisecond.
|
|
29
|
+
# Override the throttle to 0 specifically for stream queues.
|
|
30
|
+
synchronized { @pgmq.enable_notify_insert(full_name, throttle_interval_ms: 0) } if config.listen_notify
|
|
31
|
+
|
|
32
|
+
# CREATE INDEX IF NOT EXISTS is idempotent in Postgres but still
|
|
33
|
+
# requires a roundtrip and a brief ACCESS SHARE lock on the archive
|
|
34
|
+
# table. Broadcast-per-after_commit loops can hit this 1000x/sec on
|
|
35
|
+
# the same stream, so memoize per-process after the first success.
|
|
36
|
+
return if @stream_indexes_created[stream_name]
|
|
37
|
+
|
|
38
|
+
sanitized = QueueNameValidator.sanitize!(full_name)
|
|
39
|
+
sql = <<~SQL
|
|
40
|
+
CREATE INDEX IF NOT EXISTS a_#{sanitized}_msg_id_idx
|
|
41
|
+
ON pgmq.a_#{sanitized} (msg_id)
|
|
42
|
+
SQL
|
|
43
|
+
|
|
44
|
+
synchronized do
|
|
45
|
+
with_raw_connection do |conn|
|
|
46
|
+
conn.exec(sql)
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
@stream_indexes_created[stream_name] = true
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class Client
|
|
5
|
+
# Non-consuming peek across PGMQ live (`q_`) and archive (`a_`) tables. Used
|
|
6
|
+
# exclusively by `Pgbus::Web::Streamer` for SSE replay — workers continue to
|
|
7
|
+
# use `read_batch` (claim semantics). The two read paths are disjoint.
|
|
8
|
+
#
|
|
9
|
+
# The cursor is the highest msg_id the client has already seen. Replay returns
|
|
10
|
+
# everything strictly greater, ordered by msg_id ASC, capped by `limit`.
|
|
11
|
+
module ReadAfter
|
|
12
|
+
Envelope = Data.define(:msg_id, :enqueued_at, :payload, :source)
|
|
13
|
+
|
|
14
|
+
DEFAULT_LIMIT = 500
|
|
15
|
+
|
|
16
|
+
def read_after(stream_name, after_id:, limit: DEFAULT_LIMIT)
|
|
17
|
+
sanitized = sanitized_queue(stream_name)
|
|
18
|
+
sql = build_read_after_sql(sanitized)
|
|
19
|
+
|
|
20
|
+
rows = synchronized do
|
|
21
|
+
with_raw_connection do |conn|
|
|
22
|
+
conn.exec_params(sql, [after_id.to_i, limit.to_i]).to_a
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
rows.map { |row| build_envelope(row) }
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def stream_current_msg_id(stream_name)
|
|
30
|
+
sanitized = sanitized_queue(stream_name)
|
|
31
|
+
sql = "SELECT COALESCE(MAX(msg_id), 0) AS max FROM pgmq.q_#{sanitized}"
|
|
32
|
+
synchronized do
|
|
33
|
+
with_raw_connection do |conn|
|
|
34
|
+
conn.exec(sql).first.fetch("max").to_i
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def stream_oldest_msg_id(stream_name)
|
|
40
|
+
sanitized = sanitized_queue(stream_name)
|
|
41
|
+
sql = <<~SQL
|
|
42
|
+
SELECT LEAST(
|
|
43
|
+
(SELECT MIN(msg_id) FROM pgmq.q_#{sanitized}),
|
|
44
|
+
(SELECT MIN(msg_id) FROM pgmq.a_#{sanitized})
|
|
45
|
+
) AS least
|
|
46
|
+
SQL
|
|
47
|
+
synchronized do
|
|
48
|
+
with_raw_connection do |conn|
|
|
49
|
+
value = conn.exec(sql).first.fetch("least")
|
|
50
|
+
value&.to_i
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
# Builds the union of live and archive tables. The outer ORDER BY + LIMIT
|
|
58
|
+
# ensures we never return more than `limit` rows total even if both
|
|
59
|
+
# subqueries hit it. The 'live'/'archive' constants are how the streamer
|
|
60
|
+
# tells whether a row was peeked from the queue or replayed from history;
|
|
61
|
+
# the streamer doesn't currently distinguish them, but we keep the column
|
|
62
|
+
# so debugging is straightforward when archive replay misbehaves.
|
|
63
|
+
def build_read_after_sql(sanitized)
|
|
64
|
+
<<~SQL
|
|
65
|
+
(
|
|
66
|
+
SELECT msg_id, enqueued_at, message, 'live'::text AS source
|
|
67
|
+
FROM pgmq.q_#{sanitized}
|
|
68
|
+
WHERE msg_id > $1
|
|
69
|
+
ORDER BY msg_id ASC
|
|
70
|
+
LIMIT $2
|
|
71
|
+
)
|
|
72
|
+
UNION ALL
|
|
73
|
+
(
|
|
74
|
+
SELECT msg_id, enqueued_at, message, 'archive'::text AS source
|
|
75
|
+
FROM pgmq.a_#{sanitized}
|
|
76
|
+
WHERE msg_id > $1
|
|
77
|
+
ORDER BY msg_id ASC
|
|
78
|
+
LIMIT $2
|
|
79
|
+
)
|
|
80
|
+
ORDER BY msg_id ASC
|
|
81
|
+
LIMIT $2
|
|
82
|
+
SQL
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def sanitized_queue(stream_name)
|
|
86
|
+
full = config.queue_name(stream_name)
|
|
87
|
+
QueueNameValidator.sanitize!(full)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def build_envelope(row)
|
|
91
|
+
Envelope.new(
|
|
92
|
+
msg_id: row.fetch("msg_id").to_i,
|
|
93
|
+
enqueued_at: row.fetch("enqueued_at"),
|
|
94
|
+
payload: row.fetch("message"),
|
|
95
|
+
source: row.fetch("source")
|
|
96
|
+
)
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
data/lib/pgbus/client.rb
CHANGED
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
|
+
require_relative "client/read_after"
|
|
5
|
+
require_relative "client/ensure_stream_queue"
|
|
4
6
|
|
|
5
7
|
module Pgbus
|
|
6
8
|
class Client
|
|
9
|
+
include ReadAfter
|
|
10
|
+
include EnsureStreamQueue
|
|
11
|
+
|
|
7
12
|
attr_reader :pgmq, :config
|
|
8
13
|
|
|
9
14
|
PGMQ_REQUIRE_MUTEX = Mutex.new
|
|
@@ -49,6 +54,7 @@ module Pgbus
|
|
|
49
54
|
end
|
|
50
55
|
|
|
51
56
|
@queues_created = Concurrent::Map.new
|
|
57
|
+
@stream_indexes_created = Concurrent::Map.new
|
|
52
58
|
@queue_strategy = QueueFactory.for(config)
|
|
53
59
|
@schema_ensured = false
|
|
54
60
|
end
|
|
@@ -62,9 +62,12 @@ module Pgbus
|
|
|
62
62
|
validate_input_type!
|
|
63
63
|
validate_input_not_empty!
|
|
64
64
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
capsules
|
|
65
|
+
# Pure tokenization: split, parse each capsule, return them in order.
|
|
66
|
+
# Cross-capsule overlap rules live in Pgbus::Configuration#workers=
|
|
67
|
+
# because they depend on whether the resulting capsules are named or
|
|
68
|
+
# anonymous, and naming is a Configuration concern (not a parser one).
|
|
69
|
+
# Within-capsule duplicate-queue checks still happen in parse_capsule.
|
|
70
|
+
split_capsules(@input).map { |segment| parse_capsule(segment) }
|
|
68
71
|
end
|
|
69
72
|
|
|
70
73
|
private
|
|
@@ -168,23 +171,6 @@ module Pgbus
|
|
|
168
171
|
seen[q] = true
|
|
169
172
|
end
|
|
170
173
|
end
|
|
171
|
-
|
|
172
|
-
def validate_no_duplicate_queues_across_capsules!(capsules)
|
|
173
|
-
return if capsules.size < 2
|
|
174
|
-
|
|
175
|
-
seen = {}
|
|
176
|
-
capsules.each_with_index do |capsule, idx|
|
|
177
|
-
capsule[:queues].each do |q|
|
|
178
|
-
if seen[q]
|
|
179
|
-
label = (q == WILDCARD ? "wildcard '*'" : "queue #{q.inspect}")
|
|
180
|
-
raise ParseError,
|
|
181
|
-
"#{label} appears in two capsules (positions #{seen[q]} and #{idx + 1}) " \
|
|
182
|
-
"in #{@input.inspect} — each queue can only be assigned to one capsule"
|
|
183
|
-
end
|
|
184
|
-
seen[q] = idx + 1
|
|
185
|
-
end
|
|
186
|
-
end
|
|
187
|
-
end
|
|
188
174
|
end
|
|
189
175
|
end
|
|
190
176
|
end
|