pgbus 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +238 -0
- data/Rakefile +8 -1
- data/app/controllers/pgbus/insights_controller.rb +6 -0
- data/app/helpers/pgbus/streams_helper.rb +115 -0
- data/app/javascript/pgbus/stream_source_element.js +212 -0
- data/app/models/pgbus/stream_stat.rb +118 -0
- data/app/views/pgbus/insights/show.html.erb +59 -0
- data/config/locales/en.yml +16 -0
- data/config/routes.rb +11 -0
- data/lib/generators/pgbus/add_presence_generator.rb +55 -0
- data/lib/generators/pgbus/add_stream_stats_generator.rb +54 -0
- data/lib/generators/pgbus/templates/add_presence.rb.erb +26 -0
- data/lib/generators/pgbus/templates/add_stream_stats.rb.erb +18 -0
- data/lib/pgbus/client/ensure_stream_queue.rb +54 -0
- data/lib/pgbus/client/read_after.rb +100 -0
- data/lib/pgbus/client.rb +6 -0
- data/lib/pgbus/configuration.rb +65 -0
- data/lib/pgbus/engine.rb +31 -0
- data/lib/pgbus/process/dispatcher.rb +62 -4
- data/lib/pgbus/streams/cursor.rb +71 -0
- data/lib/pgbus/streams/envelope.rb +58 -0
- data/lib/pgbus/streams/filters.rb +98 -0
- data/lib/pgbus/streams/presence.rb +216 -0
- data/lib/pgbus/streams/signed_name.rb +69 -0
- data/lib/pgbus/streams/turbo_broadcastable.rb +53 -0
- data/lib/pgbus/streams/watermark_cache_middleware.rb +28 -0
- data/lib/pgbus/streams.rb +151 -0
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source.rb +29 -0
- data/lib/pgbus/web/stream_app.rb +179 -0
- data/lib/pgbus/web/streamer/connection.rb +122 -0
- data/lib/pgbus/web/streamer/dispatcher.rb +467 -0
- data/lib/pgbus/web/streamer/heartbeat.rb +105 -0
- data/lib/pgbus/web/streamer/instance.rb +176 -0
- data/lib/pgbus/web/streamer/io_writer.rb +73 -0
- data/lib/pgbus/web/streamer/listener.rb +228 -0
- data/lib/pgbus/web/streamer/registry.rb +103 -0
- data/lib/pgbus/web/streamer.rb +53 -0
- data/lib/pgbus.rb +28 -0
- data/lib/puma/plugin/pgbus_streams.rb +54 -0
- data/lib/tasks/pgbus_streams.rake +52 -0
- metadata +29 -1
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
# Records one row per stream event (broadcast / connect / disconnect)
|
|
5
|
+
# when `config.streams_stats_enabled` is true. Mirrors JobStat in
|
|
6
|
+
# shape and aggregation API so the Insights controller can query
|
|
7
|
+
# both with the same patterns.
|
|
8
|
+
#
|
|
9
|
+
# Writes are fire-and-forget: any error is swallowed so a stat
|
|
10
|
+
# recording failure cannot affect the dispatcher's hot path.
|
|
11
|
+
class StreamStat < BusRecord
|
|
12
|
+
self.table_name = "pgbus_stream_stats"
|
|
13
|
+
|
|
14
|
+
EVENT_TYPES = %w[broadcast connect disconnect].freeze
|
|
15
|
+
|
|
16
|
+
scope :since, ->(time) { where("created_at >= ?", time) }
|
|
17
|
+
scope :broadcasts, -> { where(event_type: "broadcast") }
|
|
18
|
+
scope :connects, -> { where(event_type: "connect") }
|
|
19
|
+
scope :disconnects, -> { where(event_type: "disconnect") }
|
|
20
|
+
|
|
21
|
+
# Records a stream event. Called from the Dispatcher when the
|
|
22
|
+
# `streams_stats_enabled` flag is set. Errors are swallowed so a
|
|
23
|
+
# missing table or a connection blip cannot kill the dispatcher.
|
|
24
|
+
def self.record!(stream_name:, event_type:, duration_ms: 0, fanout: nil)
|
|
25
|
+
return unless table_exists?
|
|
26
|
+
|
|
27
|
+
create!(
|
|
28
|
+
stream_name: stream_name,
|
|
29
|
+
event_type: event_type,
|
|
30
|
+
duration_ms: duration_ms.to_i,
|
|
31
|
+
fanout: fanout
|
|
32
|
+
)
|
|
33
|
+
rescue StandardError => e
|
|
34
|
+
Pgbus.logger.debug { "[Pgbus] Failed to record stream stat: #{e.message}" }
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Memoized — intentionally never invalidated at runtime. If the
|
|
38
|
+
# pgbus_stream_stats migration runs while the app is already
|
|
39
|
+
# running, a restart is required for stat recording to begin.
|
|
40
|
+
#
|
|
41
|
+
# We only memoize a *successful* probe. A transient error (PG
|
|
42
|
+
# hiccup during boot, connection refused during a failover) is
|
|
43
|
+
# treated as "don't know yet" — the next call retries. Caching
|
|
44
|
+
# false on the first hiccup would permanently disable stream
|
|
45
|
+
# stats for the process lifetime, which is a worse failure mode
|
|
46
|
+
# than a few retries.
|
|
47
|
+
def self.table_exists?
|
|
48
|
+
return @table_exists if defined?(@table_exists)
|
|
49
|
+
|
|
50
|
+
@table_exists = connection.table_exists?(table_name)
|
|
51
|
+
rescue StandardError => e
|
|
52
|
+
Pgbus.logger.debug { "[Pgbus] Failed to check stream stat table: #{e.message}" }
|
|
53
|
+
false
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Single-query aggregate summary over the given window.
|
|
57
|
+
# Returns totals by event type, average fanout for broadcasts,
|
|
58
|
+
# and an "active" estimate (connects − disconnects in window).
|
|
59
|
+
def self.summary(minutes: 60)
|
|
60
|
+
row = since(minutes.minutes.ago).pick(
|
|
61
|
+
Arel.sql("COUNT(*) FILTER (WHERE event_type = 'broadcast')"),
|
|
62
|
+
Arel.sql("COUNT(*) FILTER (WHERE event_type = 'connect')"),
|
|
63
|
+
Arel.sql("COUNT(*) FILTER (WHERE event_type = 'disconnect')"),
|
|
64
|
+
Arel.sql("ROUND(AVG(fanout) FILTER (WHERE event_type = 'broadcast')::numeric, 1)"),
|
|
65
|
+
Arel.sql("ROUND(AVG(duration_ms) FILTER (WHERE event_type = 'broadcast')::numeric, 1)"),
|
|
66
|
+
Arel.sql("ROUND(AVG(duration_ms) FILTER (WHERE event_type = 'connect')::numeric, 1)")
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
{
|
|
70
|
+
broadcasts: row[0].to_i,
|
|
71
|
+
connects: row[1].to_i,
|
|
72
|
+
disconnects: row[2].to_i,
|
|
73
|
+
active_estimate: [row[1].to_i - row[2].to_i, 0].max,
|
|
74
|
+
avg_fanout: row[3]&.to_f || 0,
|
|
75
|
+
avg_broadcast_ms: row[4]&.to_f || 0,
|
|
76
|
+
avg_connect_ms: row[5]&.to_f || 0
|
|
77
|
+
}
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Top N streams by broadcast count in the window, with avg fanout.
|
|
81
|
+
def self.top_streams(limit: 10, minutes: 60)
|
|
82
|
+
broadcasts
|
|
83
|
+
.since(minutes.minutes.ago)
|
|
84
|
+
.group(:stream_name)
|
|
85
|
+
.order(Arel.sql("COUNT(*) DESC"))
|
|
86
|
+
.limit(limit)
|
|
87
|
+
.pluck(
|
|
88
|
+
:stream_name,
|
|
89
|
+
Arel.sql("COUNT(*)"),
|
|
90
|
+
Arel.sql("ROUND(AVG(fanout)::numeric, 1)"),
|
|
91
|
+
Arel.sql("ROUND(AVG(duration_ms)::numeric, 1)")
|
|
92
|
+
)
|
|
93
|
+
.map do |name, count, avg_fanout, avg_ms|
|
|
94
|
+
{
|
|
95
|
+
stream_name: name,
|
|
96
|
+
count: count.to_i,
|
|
97
|
+
avg_fanout: avg_fanout&.to_f || 0,
|
|
98
|
+
avg_ms: avg_ms&.to_f || 0
|
|
99
|
+
}
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Throughput: broadcast events per minute bucketed by minute.
|
|
104
|
+
def self.throughput(minutes: 60)
|
|
105
|
+
broadcasts
|
|
106
|
+
.since(minutes.minutes.ago)
|
|
107
|
+
.group("date_trunc('minute', created_at)")
|
|
108
|
+
.order(Arel.sql("date_trunc('minute', created_at)"))
|
|
109
|
+
.count
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Cleanup old stats. Called from Pgbus::Process::Dispatcher on the
|
|
113
|
+
# same cadence as JobStat.cleanup! using the shared stats_retention.
|
|
114
|
+
def self.cleanup!(older_than:)
|
|
115
|
+
where("created_at < ?", older_than).delete_all
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
@@ -159,6 +159,65 @@
|
|
|
159
159
|
</table>
|
|
160
160
|
</div>
|
|
161
161
|
|
|
162
|
+
<% if @stream_stats_available %>
|
|
163
|
+
<!-- Stream stats -->
|
|
164
|
+
<div class="mt-8">
|
|
165
|
+
<h2 class="text-lg font-semibold text-gray-900 dark:text-white mb-4"><%= t("pgbus.insights.show.streams.title") %></h2>
|
|
166
|
+
|
|
167
|
+
<div class="grid grid-cols-2 gap-4 sm:grid-cols-3 lg:grid-cols-5 mb-6">
|
|
168
|
+
<div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
|
|
169
|
+
<dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.broadcasts") %></dt>
|
|
170
|
+
<dd class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white"><%= pgbus_number(@stream_summary[:broadcasts]) %></dd>
|
|
171
|
+
</div>
|
|
172
|
+
<div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
|
|
173
|
+
<dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.connects") %></dt>
|
|
174
|
+
<dd class="mt-1 text-2xl font-semibold text-green-600 dark:text-green-400"><%= pgbus_number(@stream_summary[:connects]) %></dd>
|
|
175
|
+
</div>
|
|
176
|
+
<div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
|
|
177
|
+
<dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.disconnects") %></dt>
|
|
178
|
+
<dd class="mt-1 text-2xl font-semibold text-orange-600 dark:text-orange-400"><%= pgbus_number(@stream_summary[:disconnects]) %></dd>
|
|
179
|
+
</div>
|
|
180
|
+
<div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
|
|
181
|
+
<dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.active") %></dt>
|
|
182
|
+
<dd class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white"><%= pgbus_number(@stream_summary[:active_estimate]) %></dd>
|
|
183
|
+
</div>
|
|
184
|
+
<div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
|
|
185
|
+
<dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.avg_fanout") %></dt>
|
|
186
|
+
<dd class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white"><%= @stream_summary[:avg_fanout]&.round(1) || 0 %></dd>
|
|
187
|
+
</div>
|
|
188
|
+
</div>
|
|
189
|
+
|
|
190
|
+
<div class="rounded-lg bg-white dark:bg-gray-800 shadow ring-1 ring-gray-200 dark:ring-gray-700">
|
|
191
|
+
<div class="px-5 py-4 border-b border-gray-200 dark:border-gray-700">
|
|
192
|
+
<h3 class="text-sm font-medium text-gray-700 dark:text-gray-300"><%= t("pgbus.insights.show.streams.top.title") %></h3>
|
|
193
|
+
</div>
|
|
194
|
+
<table class="pgbus-table min-w-full divide-y divide-gray-200 dark:divide-gray-700">
|
|
195
|
+
<thead class="bg-gray-50 dark:bg-gray-900">
|
|
196
|
+
<tr>
|
|
197
|
+
<th class="px-4 py-3 text-left text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.top.headers.stream") %></th>
|
|
198
|
+
<th class="px-4 py-3 text-right text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.top.headers.broadcasts") %></th>
|
|
199
|
+
<th class="px-4 py-3 text-right text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.top.headers.avg_fanout") %></th>
|
|
200
|
+
<th class="px-4 py-3 text-right text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.top.headers.avg_ms") %></th>
|
|
201
|
+
</tr>
|
|
202
|
+
</thead>
|
|
203
|
+
<tbody class="divide-y divide-gray-100 dark:divide-gray-700">
|
|
204
|
+
<% @top_streams.each do |row| %>
|
|
205
|
+
<tr class="hover:bg-gray-50 dark:hover:bg-gray-700/50">
|
|
206
|
+
<td data-label="Stream" class="px-4 py-3 text-sm font-medium text-gray-700 dark:text-gray-300"><%= row[:stream_name] %></td>
|
|
207
|
+
<td data-label="Broadcasts" class="px-4 py-3 text-sm text-right font-mono text-gray-700 dark:text-gray-300"><%= pgbus_number(row[:count]) %></td>
|
|
208
|
+
<td data-label="Avg Fanout" class="px-4 py-3 text-sm text-right font-mono text-gray-700 dark:text-gray-300"><%= row[:avg_fanout]&.round(1) || 0 %></td>
|
|
209
|
+
<td data-label="Avg Duration" class="px-4 py-3 text-sm text-right font-mono text-gray-700 dark:text-gray-300"><%= pgbus_ms_duration(row[:avg_ms]) %></td>
|
|
210
|
+
</tr>
|
|
211
|
+
<% end %>
|
|
212
|
+
<% if @top_streams.empty? %>
|
|
213
|
+
<tr><td colspan="4" class="px-4 py-8 text-center text-sm text-gray-400 dark:text-gray-500"><%= t("pgbus.insights.show.streams.top.empty") %></td></tr>
|
|
214
|
+
<% end %>
|
|
215
|
+
</tbody>
|
|
216
|
+
</table>
|
|
217
|
+
</div>
|
|
218
|
+
</div>
|
|
219
|
+
<% end %>
|
|
220
|
+
|
|
162
221
|
<script type="module" nonce="<%= content_security_policy_nonce %>">
|
|
163
222
|
import { renderCharts, observeThemeChanges } from "charts";
|
|
164
223
|
|
data/config/locales/en.yml
CHANGED
|
@@ -161,6 +161,22 @@ en:
|
|
|
161
161
|
job_class: Job Class
|
|
162
162
|
max: Max
|
|
163
163
|
title: Slowest Job Classes (avg duration)
|
|
164
|
+
streams:
|
|
165
|
+
summary:
|
|
166
|
+
active: Active
|
|
167
|
+
avg_fanout: Avg Fanout
|
|
168
|
+
broadcasts: Broadcasts
|
|
169
|
+
connects: Connects
|
|
170
|
+
disconnects: Disconnects
|
|
171
|
+
title: Real-time Streams
|
|
172
|
+
top:
|
|
173
|
+
empty: No stream activity recorded in the selected window
|
|
174
|
+
headers:
|
|
175
|
+
avg_fanout: Avg Fanout
|
|
176
|
+
avg_ms: Avg Dispatch
|
|
177
|
+
broadcasts: Broadcasts
|
|
178
|
+
stream: Stream
|
|
179
|
+
title: Top Streams by Broadcast Volume
|
|
164
180
|
summary:
|
|
165
181
|
avg_duration: Avg Duration
|
|
166
182
|
avg_latency: Avg Latency
|
data/config/routes.rb
CHANGED
|
@@ -1,6 +1,17 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
Pgbus::Engine.routes.draw do
|
|
4
|
+
# SSE streaming endpoint for the pgbus_stream_from turbo-rails replacement.
|
|
5
|
+
# Mounted as a bare Rack app so it bypasses the entire Rails middleware
|
|
6
|
+
# stack — see lib/pgbus/web/stream_app.rb for the rationale. Named with
|
|
7
|
+
# `as: :streams` so `Pgbus::Engine.routes.url_helpers.streams_path`
|
|
8
|
+
# resolves to whatever mount point the host app chose for the engine,
|
|
9
|
+
# not a hardcoded `/pgbus/streams`. The helper under
|
|
10
|
+
# app/helpers/pgbus/streams_helper.rb appends `/:signed_name` to that
|
|
11
|
+
# base so the full URL works whether the engine is at /pgbus, /admin,
|
|
12
|
+
# or anywhere else.
|
|
13
|
+
mount Pgbus::Web::StreamApp.new => "/streams", as: :streams if Pgbus.configuration.streams_enabled
|
|
14
|
+
|
|
4
15
|
root to: "dashboard#show"
|
|
5
16
|
|
|
6
17
|
resources :queues, only: %i[index show destroy], param: :name do
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
|
|
6
|
+
module Pgbus
|
|
7
|
+
module Generators
|
|
8
|
+
class AddPresenceGenerator < Rails::Generators::Base
|
|
9
|
+
include ActiveRecord::Generators::Migration
|
|
10
|
+
|
|
11
|
+
source_root File.expand_path("templates", __dir__)
|
|
12
|
+
|
|
13
|
+
desc "Add presence_members table for Pgbus::Streams presence tracking"
|
|
14
|
+
|
|
15
|
+
class_option :database,
|
|
16
|
+
type: :string,
|
|
17
|
+
default: nil,
|
|
18
|
+
desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
|
|
19
|
+
|
|
20
|
+
def create_migration_file
|
|
21
|
+
if separate_database?
|
|
22
|
+
migration_template "add_presence.rb.erb",
|
|
23
|
+
"db/pgbus_migrate/add_pgbus_presence.rb"
|
|
24
|
+
else
|
|
25
|
+
migration_template "add_presence.rb.erb",
|
|
26
|
+
"db/migrate/add_pgbus_presence.rb"
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def display_post_install
|
|
31
|
+
say ""
|
|
32
|
+
say "Pgbus presence installed!", :green
|
|
33
|
+
say ""
|
|
34
|
+
say "Next steps:"
|
|
35
|
+
say " 1. Run: rails db:migrate#{":#{options[:database]}" if separate_database?}"
|
|
36
|
+
say " 2. Use in your code:"
|
|
37
|
+
say " Pgbus.stream(@room).presence.join(member_id: current_user.id.to_s)"
|
|
38
|
+
say " Pgbus.stream(@room).presence.members"
|
|
39
|
+
say " 3. Run a periodic sweeper to expire idle members:"
|
|
40
|
+
say " Pgbus.stream(@room).presence.sweep!(older_than: 60.seconds.ago)"
|
|
41
|
+
say ""
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def migration_version
|
|
47
|
+
"[#{ActiveRecord::Migration.current_version}]"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def separate_database?
|
|
51
|
+
options[:database].present?
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
|
|
6
|
+
module Pgbus
|
|
7
|
+
module Generators
|
|
8
|
+
class AddStreamStatsGenerator < Rails::Generators::Base
|
|
9
|
+
include ActiveRecord::Generators::Migration
|
|
10
|
+
|
|
11
|
+
source_root File.expand_path("templates", __dir__)
|
|
12
|
+
|
|
13
|
+
desc "Add stream stats table for Insights dashboard (opt-in, disabled by default)"
|
|
14
|
+
|
|
15
|
+
class_option :database,
|
|
16
|
+
type: :string,
|
|
17
|
+
default: nil,
|
|
18
|
+
desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
|
|
19
|
+
|
|
20
|
+
def create_migration_file
|
|
21
|
+
if separate_database?
|
|
22
|
+
migration_template "add_stream_stats.rb.erb",
|
|
23
|
+
"db/pgbus_migrate/add_pgbus_stream_stats.rb"
|
|
24
|
+
else
|
|
25
|
+
migration_template "add_stream_stats.rb.erb",
|
|
26
|
+
"db/migrate/add_pgbus_stream_stats.rb"
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def display_post_install
|
|
31
|
+
say ""
|
|
32
|
+
say "Pgbus stream stats table installed!", :green
|
|
33
|
+
say ""
|
|
34
|
+
say "Next steps:"
|
|
35
|
+
say " 1. Run: rails db:migrate#{":#{options[:database]}" if separate_database?}"
|
|
36
|
+
say " 2. Opt in by setting `config.streams_stats_enabled = true` in your"
|
|
37
|
+
say " pgbus initializer (disabled by default — stream event volume can"
|
|
38
|
+
say " be high, so stats recording is off unless you ask for it)."
|
|
39
|
+
say " 3. View stream insights at /pgbus/insights"
|
|
40
|
+
say ""
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
private
|
|
44
|
+
|
|
45
|
+
def migration_version
|
|
46
|
+
"[#{ActiveRecord::Migration.current_version}]"
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def separate_database?
|
|
50
|
+
options[:database].present?
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
class AddPgbusPresence < ActiveRecord::Migration<%= migration_version %>
|
|
2
|
+
def change
|
|
3
|
+
create_table :pgbus_presence_members, id: false do |t|
|
|
4
|
+
t.string :stream_name, null: false
|
|
5
|
+
t.string :member_id, null: false
|
|
6
|
+
t.jsonb :metadata, null: false, default: {}
|
|
7
|
+
t.datetime :joined_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
|
|
8
|
+
t.datetime :last_seen_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
# Composite primary key (stream_name, member_id) — a member can only
|
|
12
|
+
# appear once per stream. Used by the upsert path in
|
|
13
|
+
# Pgbus::Streams::Presence#join.
|
|
14
|
+
add_index :pgbus_presence_members,
|
|
15
|
+
%i[stream_name member_id],
|
|
16
|
+
unique: true,
|
|
17
|
+
name: "idx_pgbus_presence_members_pk"
|
|
18
|
+
|
|
19
|
+
# Sweeping by last_seen_at expires stale members. The Dispatcher's
|
|
20
|
+
# periodic sweeper queries WHERE stream_name = ? AND last_seen_at < ?,
|
|
21
|
+
# so this composite index makes that fast.
|
|
22
|
+
add_index :pgbus_presence_members,
|
|
23
|
+
%i[stream_name last_seen_at],
|
|
24
|
+
name: "idx_pgbus_presence_members_sweep"
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
class AddPgbusStreamStats < ActiveRecord::Migration<%= migration_version %>
|
|
2
|
+
def change
|
|
3
|
+
create_table :pgbus_stream_stats do |t|
|
|
4
|
+
t.string :stream_name, null: false
|
|
5
|
+
t.string :event_type, null: false # 'broadcast' | 'connect' | 'disconnect'
|
|
6
|
+
t.integer :duration_ms, null: false, default: 0
|
|
7
|
+
t.integer :fanout # nil for connect/disconnect
|
|
8
|
+
t.datetime :created_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
add_index :pgbus_stream_stats, :created_at,
|
|
12
|
+
name: "idx_pgbus_stream_stats_time"
|
|
13
|
+
add_index :pgbus_stream_stats, %i[stream_name created_at],
|
|
14
|
+
name: "idx_pgbus_stream_stats_stream_time"
|
|
15
|
+
add_index :pgbus_stream_stats, %i[event_type created_at],
|
|
16
|
+
name: "idx_pgbus_stream_stats_type_time"
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class Client
|
|
5
|
+
# Idempotent stream-queue setup. Creates the PGMQ queue (delegating to
|
|
6
|
+
# `ensure_queue` which already handles schema bootstrap and dedup),
|
|
7
|
+
# overrides the NOTIFY throttle to 0 so every broadcast fires its
|
|
8
|
+
# own NOTIFY, and adds an `msg_id` index on the archive table that
|
|
9
|
+
# PGMQ does not ship with.
|
|
10
|
+
#
|
|
11
|
+
# PGMQ's archive tables (`pgmq.a_<name>`) only carry an `archived_at`
|
|
12
|
+
# index by default. `Client#read_after`'s replay query filters by
|
|
13
|
+
# `WHERE msg_id > $1`, which becomes a sequential scan once the archive
|
|
14
|
+
# grows past a few thousand rows. We add the index here, scoped to
|
|
15
|
+
# stream queues only, so users with chat-history-style retention don't
|
|
16
|
+
# hit a performance cliff.
|
|
17
|
+
#
|
|
18
|
+
# Called from `Pgbus.stream(name).broadcast(...)` on first publish per
|
|
19
|
+
# stream and from the streamer on first subscription per stream.
|
|
20
|
+
module EnsureStreamQueue
|
|
21
|
+
def ensure_stream_queue(stream_name)
|
|
22
|
+
ensure_queue(stream_name)
|
|
23
|
+
full_name = config.queue_name(stream_name)
|
|
24
|
+
|
|
25
|
+
# PGMQ's default NOTIFY throttle is 250ms — meant to coalesce
|
|
26
|
+
# high-frequency worker queue inserts. Streams are latency-
|
|
27
|
+
# sensitive and need every broadcast to fire a NOTIFY, even
|
|
28
|
+
# when several are batched within a single millisecond.
|
|
29
|
+
# Override the throttle to 0 specifically for stream queues.
|
|
30
|
+
synchronized { @pgmq.enable_notify_insert(full_name, throttle_interval_ms: 0) } if config.listen_notify
|
|
31
|
+
|
|
32
|
+
# CREATE INDEX IF NOT EXISTS is idempotent in Postgres but still
|
|
33
|
+
# requires a roundtrip and a brief ACCESS SHARE lock on the archive
|
|
34
|
+
# table. Broadcast-per-after_commit loops can hit this 1000x/sec on
|
|
35
|
+
# the same stream, so memoize per-process after the first success.
|
|
36
|
+
return if @stream_indexes_created[stream_name]
|
|
37
|
+
|
|
38
|
+
sanitized = QueueNameValidator.sanitize!(full_name)
|
|
39
|
+
sql = <<~SQL
|
|
40
|
+
CREATE INDEX IF NOT EXISTS a_#{sanitized}_msg_id_idx
|
|
41
|
+
ON pgmq.a_#{sanitized} (msg_id)
|
|
42
|
+
SQL
|
|
43
|
+
|
|
44
|
+
synchronized do
|
|
45
|
+
with_raw_connection do |conn|
|
|
46
|
+
conn.exec(sql)
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
@stream_indexes_created[stream_name] = true
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class Client
|
|
5
|
+
# Non-consuming peek across PGMQ live (`q_`) and archive (`a_`) tables. Used
|
|
6
|
+
# exclusively by `Pgbus::Web::Streamer` for SSE replay — workers continue to
|
|
7
|
+
# use `read_batch` (claim semantics). The two read paths are disjoint.
|
|
8
|
+
#
|
|
9
|
+
# The cursor is the highest msg_id the client has already seen. Replay returns
|
|
10
|
+
# everything strictly greater, ordered by msg_id ASC, capped by `limit`.
|
|
11
|
+
module ReadAfter
|
|
12
|
+
Envelope = Data.define(:msg_id, :enqueued_at, :payload, :source)
|
|
13
|
+
|
|
14
|
+
DEFAULT_LIMIT = 500
|
|
15
|
+
|
|
16
|
+
def read_after(stream_name, after_id:, limit: DEFAULT_LIMIT)
|
|
17
|
+
sanitized = sanitized_queue(stream_name)
|
|
18
|
+
sql = build_read_after_sql(sanitized)
|
|
19
|
+
|
|
20
|
+
rows = synchronized do
|
|
21
|
+
with_raw_connection do |conn|
|
|
22
|
+
conn.exec_params(sql, [after_id.to_i, limit.to_i]).to_a
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
rows.map { |row| build_envelope(row) }
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def stream_current_msg_id(stream_name)
|
|
30
|
+
sanitized = sanitized_queue(stream_name)
|
|
31
|
+
sql = "SELECT COALESCE(MAX(msg_id), 0) AS max FROM pgmq.q_#{sanitized}"
|
|
32
|
+
synchronized do
|
|
33
|
+
with_raw_connection do |conn|
|
|
34
|
+
conn.exec(sql).first.fetch("max").to_i
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def stream_oldest_msg_id(stream_name)
|
|
40
|
+
sanitized = sanitized_queue(stream_name)
|
|
41
|
+
sql = <<~SQL
|
|
42
|
+
SELECT LEAST(
|
|
43
|
+
(SELECT MIN(msg_id) FROM pgmq.q_#{sanitized}),
|
|
44
|
+
(SELECT MIN(msg_id) FROM pgmq.a_#{sanitized})
|
|
45
|
+
) AS least
|
|
46
|
+
SQL
|
|
47
|
+
synchronized do
|
|
48
|
+
with_raw_connection do |conn|
|
|
49
|
+
value = conn.exec(sql).first.fetch("least")
|
|
50
|
+
value&.to_i
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
# Builds the union of live and archive tables. The outer ORDER BY + LIMIT
|
|
58
|
+
# ensures we never return more than `limit` rows total even if both
|
|
59
|
+
# subqueries hit it. The 'live'/'archive' constants are how the streamer
|
|
60
|
+
# tells whether a row was peeked from the queue or replayed from history;
|
|
61
|
+
# the streamer doesn't currently distinguish them, but we keep the column
|
|
62
|
+
# so debugging is straightforward when archive replay misbehaves.
|
|
63
|
+
def build_read_after_sql(sanitized)
|
|
64
|
+
<<~SQL
|
|
65
|
+
(
|
|
66
|
+
SELECT msg_id, enqueued_at, message, 'live'::text AS source
|
|
67
|
+
FROM pgmq.q_#{sanitized}
|
|
68
|
+
WHERE msg_id > $1
|
|
69
|
+
ORDER BY msg_id ASC
|
|
70
|
+
LIMIT $2
|
|
71
|
+
)
|
|
72
|
+
UNION ALL
|
|
73
|
+
(
|
|
74
|
+
SELECT msg_id, enqueued_at, message, 'archive'::text AS source
|
|
75
|
+
FROM pgmq.a_#{sanitized}
|
|
76
|
+
WHERE msg_id > $1
|
|
77
|
+
ORDER BY msg_id ASC
|
|
78
|
+
LIMIT $2
|
|
79
|
+
)
|
|
80
|
+
ORDER BY msg_id ASC
|
|
81
|
+
LIMIT $2
|
|
82
|
+
SQL
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def sanitized_queue(stream_name)
|
|
86
|
+
full = config.queue_name(stream_name)
|
|
87
|
+
QueueNameValidator.sanitize!(full)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def build_envelope(row)
|
|
91
|
+
Envelope.new(
|
|
92
|
+
msg_id: row.fetch("msg_id").to_i,
|
|
93
|
+
enqueued_at: row.fetch("enqueued_at"),
|
|
94
|
+
payload: row.fetch("message"),
|
|
95
|
+
source: row.fetch("source")
|
|
96
|
+
)
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
data/lib/pgbus/client.rb
CHANGED
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
|
+
require_relative "client/read_after"
|
|
5
|
+
require_relative "client/ensure_stream_queue"
|
|
4
6
|
|
|
5
7
|
module Pgbus
|
|
6
8
|
class Client
|
|
9
|
+
include ReadAfter
|
|
10
|
+
include EnsureStreamQueue
|
|
11
|
+
|
|
7
12
|
attr_reader :pgmq, :config
|
|
8
13
|
|
|
9
14
|
PGMQ_REQUIRE_MUTEX = Mutex.new
|
|
@@ -49,6 +54,7 @@ module Pgbus
|
|
|
49
54
|
end
|
|
50
55
|
|
|
51
56
|
@queues_created = Concurrent::Map.new
|
|
57
|
+
@stream_indexes_created = Concurrent::Map.new
|
|
52
58
|
@queue_strategy = QueueFactory.for(config)
|
|
53
59
|
@schema_ensured = false
|
|
54
60
|
end
|
data/lib/pgbus/configuration.rb
CHANGED
|
@@ -84,6 +84,13 @@ module Pgbus
|
|
|
84
84
|
attr_accessor :web_auth, :web_refresh_interval, :web_per_page, :web_live_updates, :web_data_source,
|
|
85
85
|
:insights_default_minutes, :base_controller_class, :return_to_app_url
|
|
86
86
|
|
|
87
|
+
# Streams (turbo-rails replacement, SSE-based)
|
|
88
|
+
attr_accessor :streams_enabled, :streams_queue_prefix, :streams_signed_name_secret,
|
|
89
|
+
:streams_default_retention, :streams_retention, :streams_heartbeat_interval,
|
|
90
|
+
:streams_max_connections, :streams_idle_timeout, :streams_listen_health_check_ms,
|
|
91
|
+
:streams_write_deadline_ms, :streams_falcon_streaming_body,
|
|
92
|
+
:streams_stats_enabled
|
|
93
|
+
|
|
87
94
|
def initialize
|
|
88
95
|
@database_url = nil
|
|
89
96
|
@connection_params = nil
|
|
@@ -150,6 +157,34 @@ module Pgbus
|
|
|
150
157
|
@insights_default_minutes = 30 * 24 * 60 # 30 days
|
|
151
158
|
@base_controller_class = "::ActionController::Base"
|
|
152
159
|
@return_to_app_url = nil
|
|
160
|
+
|
|
161
|
+
@streams_enabled = true
|
|
162
|
+
@streams_queue_prefix = "pgbus_stream"
|
|
163
|
+
@streams_signed_name_secret = nil
|
|
164
|
+
@streams_default_retention = 5 * 60 # 5 minutes
|
|
165
|
+
@streams_retention = {}
|
|
166
|
+
@streams_heartbeat_interval = 15
|
|
167
|
+
@streams_max_connections = 2_000
|
|
168
|
+
@streams_idle_timeout = 3_600 # 1 hour
|
|
169
|
+
# 250ms — this value plays two roles: (1) the TCP keepalive
|
|
170
|
+
# interval for the streamer's PG LISTEN connection, and (2) the
|
|
171
|
+
# upper bound on how long Dispatcher#handle_connect waits for
|
|
172
|
+
# the Listener to acknowledge a synchronous ensure_listening
|
|
173
|
+
# call. 5s was unbounded enough to drop messages on a
|
|
174
|
+
# realistic subscribe burst; 250ms keeps the connect-path race
|
|
175
|
+
# window tight while still leaving headroom over a typical
|
|
176
|
+
# PG keepalive interval.
|
|
177
|
+
@streams_listen_health_check_ms = 250
|
|
178
|
+
@streams_write_deadline_ms = 5_000
|
|
179
|
+
@streams_falcon_streaming_body = false
|
|
180
|
+
# Opt-in: when true, the Dispatcher writes one row to
|
|
181
|
+
# pgbus_stream_stats per broadcast/connect/disconnect. Default
|
|
182
|
+
# off because stream event volume can be much higher than job
|
|
183
|
+
# volume and the Insights surface is only useful if operators
|
|
184
|
+
# actually look at it. Separate from #stats_enabled (which
|
|
185
|
+
# gates pgbus_job_stats recording) on purpose — operators
|
|
186
|
+
# usually want job stats on and stream stats off, or vice versa.
|
|
187
|
+
@streams_stats_enabled = false
|
|
153
188
|
end
|
|
154
189
|
|
|
155
190
|
def queue_name(name)
|
|
@@ -208,9 +243,39 @@ module Pgbus
|
|
|
208
243
|
raise ArgumentError, "insights_default_minutes must be a positive integer"
|
|
209
244
|
end
|
|
210
245
|
|
|
246
|
+
validate_streams!
|
|
247
|
+
|
|
211
248
|
self
|
|
212
249
|
end
|
|
213
250
|
|
|
251
|
+
def validate_streams!
|
|
252
|
+
unless streams_default_retention.is_a?(Numeric) && streams_default_retention >= 0
|
|
253
|
+
raise ArgumentError, "streams_default_retention must be a non-negative number"
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
unless streams_max_connections.is_a?(Integer) && streams_max_connections.positive?
|
|
257
|
+
raise ArgumentError, "streams_max_connections must be a positive integer"
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
unless streams_heartbeat_interval.is_a?(Numeric) && streams_heartbeat_interval.positive?
|
|
261
|
+
raise ArgumentError, "streams_heartbeat_interval must be a positive number"
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
unless streams_idle_timeout.is_a?(Numeric) && streams_idle_timeout.positive?
|
|
265
|
+
raise ArgumentError, "streams_idle_timeout must be a positive number"
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
unless streams_listen_health_check_ms.is_a?(Integer) && streams_listen_health_check_ms.positive?
|
|
269
|
+
raise ArgumentError, "streams_listen_health_check_ms must be a positive integer"
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
unless streams_write_deadline_ms.is_a?(Integer) && streams_write_deadline_ms.positive?
|
|
273
|
+
raise ArgumentError, "streams_write_deadline_ms must be a positive integer"
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
raise ArgumentError, "streams_retention must be a Hash" unless streams_retention.is_a?(Hash)
|
|
277
|
+
end
|
|
278
|
+
|
|
214
279
|
# Set the worker capsule list. Accepts:
|
|
215
280
|
#
|
|
216
281
|
# String — parsed via Pgbus::Configuration::CapsuleDSL into capsules
|