pgbus 0.5.1 → 0.6.1

data/app/models/pgbus/stream_stat.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Pgbus
+  # Records one row per stream event (broadcast / connect / disconnect)
+  # when `config.streams_stats_enabled` is true. Mirrors JobStat in
+  # shape and aggregation API so the Insights controller can query
+  # both with the same patterns.
+  #
+  # Writes are fire-and-forget: any error is swallowed so a stat
+  # recording failure cannot affect the dispatcher's hot path.
+  class StreamStat < BusRecord
+    self.table_name = "pgbus_stream_stats"
+
+    EVENT_TYPES = %w[broadcast connect disconnect].freeze
+
+    scope :since, ->(time) { where("created_at >= ?", time) }
+    # NOTE: scope names intentionally avoid `broadcasts` / `connects` /
+    # `disconnects`. `turbo-rails` auto-includes `Turbo::Broadcastable`
+    # into every AR model, which defines a class method `broadcasts`.
+    # AR's dangerous_class_method? guard then rejects a colliding
+    # scope name at load time, crashing eager_load. See issue #92.
+    scope :broadcast_events, -> { where(event_type: "broadcast") }
+    scope :connect_events, -> { where(event_type: "connect") }
+    scope :disconnect_events, -> { where(event_type: "disconnect") }
+
+    # Records a stream event. Called from the Dispatcher when the
+    # `streams_stats_enabled` flag is set. Errors are swallowed so a
+    # missing table or a connection blip cannot kill the dispatcher.
+    def self.record!(stream_name:, event_type:, duration_ms: 0, fanout: nil)
+      return unless table_exists?
+
+      create!(
+        stream_name: stream_name,
+        event_type: event_type,
+        duration_ms: duration_ms.to_i,
+        fanout: fanout
+      )
+    rescue StandardError => e
+      Pgbus.logger.debug { "[Pgbus] Failed to record stream stat: #{e.message}" }
+    end
+
+    # Memoized — intentionally never invalidated at runtime. If the
+    # pgbus_stream_stats migration runs while the app is already
+    # running, a restart is required for stat recording to begin.
+    #
+    # We only memoize a *successful* probe. A transient error (PG
+    # hiccup during boot, connection refused during a failover) is
+    # treated as "don't know yet" — the next call retries. Caching
+    # false on the first hiccup would permanently disable stream
+    # stats for the process lifetime, which is a worse failure mode
+    # than a few retries.
+    def self.table_exists?
+      return @table_exists if defined?(@table_exists)
+
+      @table_exists = connection.table_exists?(table_name)
+    rescue StandardError => e
+      Pgbus.logger.debug { "[Pgbus] Failed to check stream stat table: #{e.message}" }
+      false
+    end
+
+    # Single-query aggregate summary over the given window.
+    # Returns totals by event type, average fanout for broadcasts,
+    # and an "active" estimate (connects − disconnects in window).
+    def self.summary(minutes: 60)
+      row = since(minutes.minutes.ago).pick(
+        Arel.sql("COUNT(*) FILTER (WHERE event_type = 'broadcast')"),
+        Arel.sql("COUNT(*) FILTER (WHERE event_type = 'connect')"),
+        Arel.sql("COUNT(*) FILTER (WHERE event_type = 'disconnect')"),
+        Arel.sql("ROUND(AVG(fanout) FILTER (WHERE event_type = 'broadcast')::numeric, 1)"),
+        Arel.sql("ROUND(AVG(duration_ms) FILTER (WHERE event_type = 'broadcast')::numeric, 1)"),
+        Arel.sql("ROUND(AVG(duration_ms) FILTER (WHERE event_type = 'connect')::numeric, 1)")
+      )
+
+      {
+        broadcasts: row[0].to_i,
+        connects: row[1].to_i,
+        disconnects: row[2].to_i,
+        active_estimate: [row[1].to_i - row[2].to_i, 0].max,
+        avg_fanout: row[3]&.to_f || 0,
+        avg_broadcast_ms: row[4]&.to_f || 0,
+        avg_connect_ms: row[5]&.to_f || 0
+      }
+    end
+
+    # Top N streams by broadcast count in the window, with avg fanout.
+    def self.top_streams(limit: 10, minutes: 60)
+      broadcast_events
+        .since(minutes.minutes.ago)
+        .group(:stream_name)
+        .order(Arel.sql("COUNT(*) DESC"))
+        .limit(limit)
+        .pluck(
+          :stream_name,
+          Arel.sql("COUNT(*)"),
+          Arel.sql("ROUND(AVG(fanout)::numeric, 1)"),
+          Arel.sql("ROUND(AVG(duration_ms)::numeric, 1)")
+        )
+        .map do |name, count, avg_fanout, avg_ms|
+          {
+            stream_name: name,
+            count: count.to_i,
+            avg_fanout: avg_fanout&.to_f || 0,
+            avg_ms: avg_ms&.to_f || 0
+          }
+        end
+    end
+
+    # Throughput: broadcast events per minute bucketed by minute.
+    def self.throughput(minutes: 60)
+      broadcast_events
+        .since(minutes.minutes.ago)
+        .group("date_trunc('minute', created_at)")
+        .order(Arel.sql("date_trunc('minute', created_at)"))
+        .count
+    end
+
+    # Cleanup old stats. Called from Pgbus::Process::Dispatcher on the
+    # same cadence as JobStat.cleanup! using the shared stats_retention.
+    def self.cleanup!(older_than:)
+      where("created_at < ?", older_than).delete_all
+    end
+  end
+end

data/app/views/pgbus/insights/show.html.erb

@@ -159,6 +159,65 @@
   </table>
 </div>
 
+<% if @stream_stats_available %>
+<!-- Stream stats -->
+<div class="mt-8">
+  <h2 class="text-lg font-semibold text-gray-900 dark:text-white mb-4"><%= t("pgbus.insights.show.streams.title") %></h2>
+
+  <div class="grid grid-cols-2 gap-4 sm:grid-cols-3 lg:grid-cols-5 mb-6">
+    <div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
+      <dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.broadcasts") %></dt>
+      <dd class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white"><%= pgbus_number(@stream_summary[:broadcasts]) %></dd>
+    </div>
+    <div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
+      <dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.connects") %></dt>
+      <dd class="mt-1 text-2xl font-semibold text-green-600 dark:text-green-400"><%= pgbus_number(@stream_summary[:connects]) %></dd>
+    </div>
+    <div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
+      <dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.disconnects") %></dt>
+      <dd class="mt-1 text-2xl font-semibold text-orange-600 dark:text-orange-400"><%= pgbus_number(@stream_summary[:disconnects]) %></dd>
+    </div>
+    <div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
+      <dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.active") %></dt>
+      <dd class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white"><%= pgbus_number(@stream_summary[:active_estimate]) %></dd>
+    </div>
+    <div class="rounded-lg bg-white dark:bg-gray-800 p-4 shadow ring-1 ring-gray-200 dark:ring-gray-700">
+      <dt class="text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.summary.avg_fanout") %></dt>
+      <dd class="mt-1 text-2xl font-semibold text-gray-900 dark:text-white"><%= @stream_summary[:avg_fanout]&.round(1) || 0 %></dd>
+    </div>
+  </div>
+
+  <div class="rounded-lg bg-white dark:bg-gray-800 shadow ring-1 ring-gray-200 dark:ring-gray-700">
+    <div class="px-5 py-4 border-b border-gray-200 dark:border-gray-700">
+      <h3 class="text-sm font-medium text-gray-700 dark:text-gray-300"><%= t("pgbus.insights.show.streams.top.title") %></h3>
+    </div>
+    <table class="pgbus-table min-w-full divide-y divide-gray-200 dark:divide-gray-700">
+      <thead class="bg-gray-50 dark:bg-gray-900">
+        <tr>
+          <th class="px-4 py-3 text-left text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.top.headers.stream") %></th>
+          <th class="px-4 py-3 text-right text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.top.headers.broadcasts") %></th>
+          <th class="px-4 py-3 text-right text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.top.headers.avg_fanout") %></th>
+          <th class="px-4 py-3 text-right text-xs font-medium uppercase text-gray-500 dark:text-gray-400"><%= t("pgbus.insights.show.streams.top.headers.avg_ms") %></th>
+        </tr>
+      </thead>
+      <tbody class="divide-y divide-gray-100 dark:divide-gray-700">
+        <% @top_streams.each do |row| %>
+          <tr class="hover:bg-gray-50 dark:hover:bg-gray-700/50">
+            <td data-label="Stream" class="px-4 py-3 text-sm font-medium text-gray-700 dark:text-gray-300"><%= row[:stream_name] %></td>
+            <td data-label="Broadcasts" class="px-4 py-3 text-sm text-right font-mono text-gray-700 dark:text-gray-300"><%= pgbus_number(row[:count]) %></td>
+            <td data-label="Avg Fanout" class="px-4 py-3 text-sm text-right font-mono text-gray-700 dark:text-gray-300"><%= row[:avg_fanout]&.round(1) || 0 %></td>
+            <td data-label="Avg Duration" class="px-4 py-3 text-sm text-right font-mono text-gray-700 dark:text-gray-300"><%= pgbus_ms_duration(row[:avg_ms]) %></td>
+          </tr>
+        <% end %>
+        <% if @top_streams.empty? %>
+          <tr><td colspan="4" class="px-4 py-8 text-center text-sm text-gray-400 dark:text-gray-500"><%= t("pgbus.insights.show.streams.top.empty") %></td></tr>
+        <% end %>
+      </tbody>
+    </table>
+  </div>
+</div>
+<% end %>
+
 <script type="module" nonce="<%= content_security_policy_nonce %>">
   import { renderCharts, observeThemeChanges } from "charts";
 

data/config/locales/en.yml

@@ -161,6 +161,22 @@ en:
             job_class: Job Class
             max: Max
           title: Slowest Job Classes (avg duration)
+        streams:
+          summary:
+            active: Active
+            avg_fanout: Avg Fanout
+            broadcasts: Broadcasts
+            connects: Connects
+            disconnects: Disconnects
+          title: Real-time Streams
+          top:
+            empty: No stream activity recorded in the selected window
+            headers:
+              avg_fanout: Avg Fanout
+              avg_ms: Avg Dispatch
+              broadcasts: Broadcasts
+              stream: Stream
+            title: Top Streams by Broadcast Volume
         summary:
           avg_duration: Avg Duration
           avg_latency: Avg Latency

data/config/routes.rb

@@ -1,6 +1,17 @@
 # frozen_string_literal: true
 
 Pgbus::Engine.routes.draw do
+  # SSE streaming endpoint for the pgbus_stream_from turbo-rails replacement.
+  # Mounted as a bare Rack app so it bypasses the entire Rails middleware
+  # stack — see lib/pgbus/web/stream_app.rb for the rationale. Named with
+  # `as: :streams` so `Pgbus::Engine.routes.url_helpers.streams_path`
+  # resolves to whatever mount point the host app chose for the engine,
+  # not a hardcoded `/pgbus/streams`. The helper under
+  # app/helpers/pgbus/streams_helper.rb appends `/:signed_name` to that
+  # base so the full URL works whether the engine is at /pgbus, /admin,
+  # or anywhere else.
+  mount Pgbus::Web::StreamApp.new => "/streams", as: :streams if Pgbus.configuration.streams_enabled
+
   root to: "dashboard#show"
 
   resources :queues, only: %i[index show destroy], param: :name do

data/lib/generators/pgbus/add_job_stats_queue_index_generator.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module Pgbus
+  module Generators
+    class AddJobStatsQueueIndexGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Add composite index on pgbus_job_stats (queue_name, created_at) for Insights latency-by-queue aggregation"
+
+      class_option :database,
+                   type: :string,
+                   default: nil,
+                   desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
+
+      def create_migration_file
+        if separate_database?
+          migration_template "add_job_stats_queue_index.rb.erb",
+                             "db/pgbus_migrate/add_pgbus_job_stats_queue_index.rb"
+        else
+          migration_template "add_job_stats_queue_index.rb.erb",
+                             "db/migrate/add_pgbus_job_stats_queue_index.rb"
+        end
+      end
+
+      def display_post_install
+        say ""
+        say "Pgbus job stats queue index added!", :green
+        say ""
+        say "Next steps:"
+        say "  1. Run: rails db:migrate#{":#{options[:database]}" if separate_database?}"
+        say "  2. The Insights 'latency by queue' aggregation will now use the index"
+        say "     instead of sequentially scanning pgbus_job_stats. Install this on"
+        say "     heavy-traffic deployments with a large job stats retention window."
+        say ""
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::Migration.current_version}]"
+      end
+
+      def separate_database?
+        options[:database].present?
+      end
+    end
+  end
+end

data/lib/generators/pgbus/add_presence_generator.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module Pgbus
+  module Generators
+    class AddPresenceGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Add presence_members table for Pgbus::Streams presence tracking"
+
+      class_option :database,
+                   type: :string,
+                   default: nil,
+                   desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
+
+      def create_migration_file
+        if separate_database?
+          migration_template "add_presence.rb.erb",
+                             "db/pgbus_migrate/add_pgbus_presence.rb"
+        else
+          migration_template "add_presence.rb.erb",
+                             "db/migrate/add_pgbus_presence.rb"
+        end
+      end
+
+      def display_post_install
+        say ""
+        say "Pgbus presence installed!", :green
+        say ""
+        say "Next steps:"
+        say "  1. Run: rails db:migrate#{":#{options[:database]}" if separate_database?}"
+        say "  2. Use in your code:"
+        say "       Pgbus.stream(@room).presence.join(member_id: current_user.id.to_s)"
+        say "       Pgbus.stream(@room).presence.members"
+        say "  3. Run a periodic sweeper to expire idle members:"
+        say "       Pgbus.stream(@room).presence.sweep!(older_than: 60.seconds.ago)"
+        say ""
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::Migration.current_version}]"
+      end
+
+      def separate_database?
+        options[:database].present?
+      end
+    end
+  end
+end

data/lib/generators/pgbus/add_stream_stats_generator.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module Pgbus
+  module Generators
+    class AddStreamStatsGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Add stream stats table for Insights dashboard (opt-in, disabled by default)"
+
+      class_option :database,
+                   type: :string,
+                   default: nil,
+                   desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
+
+      def create_migration_file
+        if separate_database?
+          migration_template "add_stream_stats.rb.erb",
+                             "db/pgbus_migrate/add_pgbus_stream_stats.rb"
+        else
+          migration_template "add_stream_stats.rb.erb",
+                             "db/migrate/add_pgbus_stream_stats.rb"
+        end
+      end
+
+      def display_post_install
+        say ""
+        say "Pgbus stream stats table installed!", :green
+        say ""
+        say "Next steps:"
+        say "  1. Run: rails db:migrate#{":#{options[:database]}" if separate_database?}"
+        say "  2. Opt in by setting `config.streams_stats_enabled = true` in your"
+        say "     pgbus initializer (disabled by default — stream event volume can"
+        say "     be high, so stats recording is off unless you ask for it)."
+        say "  3. View stream insights at /pgbus/insights"
+        say ""
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::Migration.current_version}]"
+      end
+
+      def separate_database?
+        options[:database].present?
+      end
+    end
+  end
+end

data/lib/generators/pgbus/templates/add_job_stats_latency.rb.erb

@@ -3,7 +3,10 @@ class AddPgbusJobStatsLatency < ActiveRecord::Migration<%= migration_version %>
     add_column :pgbus_job_stats, :enqueue_latency_ms, :bigint
     add_column :pgbus_job_stats, :retry_count, :integer, default: 0
 
+    # idempotent: the standalone add_job_stats_queue_index generator
+    # creates the same index. Either order is safe.
     add_index :pgbus_job_stats, [:queue_name, :created_at],
-      name: "idx_pgbus_job_stats_queue_time"
+      name: "idx_pgbus_job_stats_queue_time",
+      if_not_exists: true
   end
 end

data/lib/generators/pgbus/templates/add_job_stats_queue_index.rb.erb

@@ -0,0 +1,11 @@
+class AddPgbusJobStatsQueueIndex < ActiveRecord::Migration<%= migration_version %>
+  def change
+    # Supports InsightsController#latency_by_queue and any other
+    # per-queue aggregation over pgbus_job_stats. Without this, a
+    # 60-day window over 1M jobs/day seq-scans the whole table for
+    # every Insights page load with the latency tab open.
+    add_index :pgbus_job_stats, %i[queue_name created_at],
+              name: "idx_pgbus_job_stats_queue_time",
+              if_not_exists: true
+  end
+end

data/lib/generators/pgbus/templates/add_presence.rb.erb

@@ -0,0 +1,26 @@
+class AddPgbusPresence < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :pgbus_presence_members, id: false do |t|
+      t.string :stream_name, null: false
+      t.string :member_id, null: false
+      t.jsonb :metadata, null: false, default: {}
+      t.datetime :joined_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
+      t.datetime :last_seen_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
+    end
+
+    # Composite primary key (stream_name, member_id) — a member can only
+    # appear once per stream. Used by the upsert path in
+    # Pgbus::Streams::Presence#join.
+    add_index :pgbus_presence_members,
+              %i[stream_name member_id],
+              unique: true,
+              name: "idx_pgbus_presence_members_pk"
+
+    # Sweeping by last_seen_at expires stale members. The Dispatcher's
+    # periodic sweeper queries WHERE stream_name = ? AND last_seen_at < ?,
+    # so this composite index makes that fast.
+    add_index :pgbus_presence_members,
+              %i[stream_name last_seen_at],
+              name: "idx_pgbus_presence_members_sweep"
+  end
+end

data/lib/generators/pgbus/templates/add_stream_stats.rb.erb

@@ -0,0 +1,18 @@
+class AddPgbusStreamStats < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :pgbus_stream_stats do |t|
+      t.string :stream_name, null: false
+      t.string :event_type, null: false # 'broadcast' | 'connect' | 'disconnect'
+      t.integer :duration_ms, null: false, default: 0
+      t.integer :fanout # nil for connect/disconnect
+      t.datetime :created_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
+    end
+
+    add_index :pgbus_stream_stats, :created_at,
+              name: "idx_pgbus_stream_stats_time"
+    add_index :pgbus_stream_stats, %i[stream_name created_at],
+              name: "idx_pgbus_stream_stats_stream_time"
+    add_index :pgbus_stream_stats, %i[event_type created_at],
+              name: "idx_pgbus_stream_stats_type_time"
+  end
+end