pgbus 0.5.1 → 0.6.1

data/lib/generators/pgbus/update_generator.rb

@@ -2,58 +2,171 @@
 
 require "rails/generators"
 require "pgbus/generators/config_converter"
+require "pgbus/generators/migration_detector"
+require "pgbus/generators/database_target_detector"
 
 module Pgbus
   module Generators
-    # Converts an existing config/pgbus.yml to a Ruby initializer at
-    # config/initializers/pgbus.rb using the modern DSL.
+    # Upgrade command with two independent jobs:
     #
-    # The original YAML file is left in place — the user reviews the
-    # generated initializer and deletes the YAML when ready.
+    #   1. Config conversion: if config/pgbus.yml exists, convert it to
+    #      config/initializers/pgbus.rb using the modern Ruby DSL. Skip
+    #      silently if the initializer already exists or the YAML is
+    #      absent — safe to re-run.
+    #
+    #   2. Migration detection: inspect the live database and add any
+    #      missing pgbus migrations to db/migrate (or db/pgbus_migrate
+    #      if a separate database is configured). Invokes each matching
+    #      sub-generator in-process via Thor's invoke, so this mirrors
+    #      what the user would get running each generator by hand.
     #
     # Usage:
     #
     #   bin/rails generate pgbus:update
-    #   bin/rails generate pgbus:update --force      # overwrite existing initializer
-    #   bin/rails generate pgbus:update --source=path/to/pgbus.yml
+    #   bin/rails generate pgbus:update --dry-run
+    #   bin/rails generate pgbus:update --skip-config
+    #   bin/rails generate pgbus:update --skip-migrations
+    #   bin/rails generate pgbus:update --database=pgbus
+    #   bin/rails generate pgbus:update --quiet
     class UpdateGenerator < Rails::Generators::Base
-      desc "Convert config/pgbus.yml to config/initializers/pgbus.rb using the Ruby DSL"
+      desc "Upgrade pgbus: convert YAML config + add any missing migrations"
 
       class_option :source,
                    type: :string,
                    default: "config/pgbus.yml",
-                   desc: "Path to the existing YAML config (default: config/pgbus.yml)"
+                   desc: "Path to an existing YAML config to convert (default: config/pgbus.yml)"
 
       class_option :destination,
                    type: :string,
                    default: "config/initializers/pgbus.rb",
                    desc: "Path to the generated initializer (default: config/initializers/pgbus.rb)"
 
-      def convert
+      class_option :skip_config,
+                   type: :boolean,
+                   default: false,
+                   desc: "Skip the YAML → Ruby initializer conversion step"
+
+      class_option :skip_migrations,
+                   type: :boolean,
+                   default: false,
+                   desc: "Skip the migration detection step"
+
+      class_option :database,
+                   type: :string,
+                   default: nil,
+                   desc: "Use a separate database for pgbus tables (default: auto-detect " \
+                         "from Pgbus.configuration.connects_to or config/initializers/pgbus.rb)"
+
+      class_option :dry_run,
+                   type: :boolean,
+                   default: false,
+                   desc: "Print what would be done without creating any files"
+
+      class_option :quiet,
+                   type: :boolean,
+                   default: false,
+                   desc: "Suppress verbose per-step output"
+
+      def convert_yaml_if_present
+        return if options[:skip_config]
+
         source_path = File.expand_path(options[:source], destination_root)
         destination_path = File.expand_path(options[:destination], destination_root)
 
-        # Thor::Error is the idiomatic way to abort a Rails generator. Thor
-        # catches it, prints the message in red, and exits with status 1
-        # without a Ruby backtrace. exit 1 would skip the framework's
-        # cleanup hooks and is hard to test.
-        raise Thor::Error, "Source file not found: #{options[:source]}" unless File.exist?(source_path)
+        unless File.exist?(source_path)
+          log "YAML config not found at #{options[:source]}; skipping config conversion."
+          return
+        end
+
+        if File.exist?(destination_path)
+          log "Initializer already exists at #{options[:destination]}; skipping config conversion."
+          return
+        end
 
         ruby_source = load_and_convert(source_path)
-        create_file destination_path, ruby_source
+        if options[:dry_run]
+          log_change "[dry-run] would create #{options[:destination]}"
+        else
+          create_file destination_path, ruby_source
+        end
+      end
+
+      def detect_and_install_missing_migrations
+        return if options[:skip_migrations]
+
+        unless active_record_available?
+          log "ActiveRecord not loaded — skipping migration detection. Run this generator from a Rails app."
+          return
+        end
+
+        connection = resolve_connection
+        unless connection
+          log "No ActiveRecord connection available — skipping migration detection."
+          return
+        end
+
+        detector = MigrationDetector.new(connection)
+        missing = detector.missing_migrations
+
+        if missing.empty?
+          log "Database schema is up to date — no migrations needed."
+          return
+        end
+
+        if missing == [MigrationDetector::FRESH_INSTALL]
+          say ""
+          say "Database looks empty of pgbus tables — this is a fresh install.", :yellow
+          say "Run `rails generate pgbus:install` instead of `pgbus:update`.", :yellow
+          say ""
+          return
+        end
+
+        database_name = options[:database] || detected_database_name
+        log "Auto-detected separate database: #{database_name}" if options[:database].nil? && database_name
+
+        log "Found #{missing.size} missing migration(s):"
+        missing.each do |key|
+          description = MigrationDetector::DESCRIPTIONS[key] || key.to_s
+          log "  - #{key}: #{description}"
+        end
+
+        # Two loops on purpose: print the full plan first so operators
+        # see what's coming, then execute. Combining would interleave
+        # "  - add_presence: foo" with "Invoking pgbus:add_presence..."
+        # which hides the shape of the upgrade from the reader.
+        missing.each do |key| # rubocop:disable Style/CombinableLoops
+          generator = MigrationDetector::GENERATOR_MAP[key]
+          unless generator
+            say "  !  no generator mapped for #{key}, skipping", :red
+            next
+          end
+
+          if options[:dry_run]
+            log_change "[dry-run] would invoke #{generator}#{" --database=#{database_name}" if database_name}"
+            next
+          end
+
+          invoke_args = []
+          invoke_args << "--database=#{database_name}" if database_name
+          log "Invoking #{generator}#{" --database=#{database_name}" if database_name}..."
+          invoke generator, invoke_args
+        end
       end
 
       def display_post_install
+        return if options[:quiet]
+
         say ""
-        say "Pgbus initializer generated at #{options[:destination]}!", :green
-        say ""
-        say "Next steps:"
-        say "  1. Review the generated initializer for correctness"
-        say "  2. Boot your app and verify everything still works"
-        say "  3. Delete #{options[:source]} when satisfied (Pgbus will stop reading it)"
+        say "Pgbus update complete.", :green
         say ""
-        say "If you spot a setting that didn't translate cleanly, please open an issue:"
-        say "  https://github.com/mhenrixon/pgbus/issues", :cyan
+        if options[:dry_run]
+          say "Dry-run: no files were created.", :yellow
+        else
+          say "Next steps:"
+          say "  1. Review the generated migration files in db/migrate (or db/pgbus_migrate)"
+          say "  2. Run: rails db:migrate#{":#{effective_database_name}" if effective_database_name}"
+          say "  3. Restart pgbus: bin/pgbus start"
+        end
         say ""
       end
 
@@ -70,6 +183,46 @@ module Pgbus
       rescue ConfigConverter::Error, Psych::Exception, Errno::ENOENT, Errno::EACCES => e
         raise Thor::Error, "Failed to convert #{options[:source]}: #{e.message}"
       end
+
+      def active_record_available?
+        defined?(::ActiveRecord::Base) && ::ActiveRecord::Base.respond_to?(:connection)
+      end
+
+      # Resolve the AR connection to inspect. If pgbus is configured to
+      # use a separate database (via connects_to), use BusRecord's
+      # connection so the detector probes the right schema.
+      def resolve_connection
+        if defined?(Pgbus) && Pgbus.respond_to?(:configuration) && Pgbus.configuration.connects_to
+          Pgbus::BusRecord.connection
+        else
+          ::ActiveRecord::Base.connection
+        end
+      rescue StandardError => e
+        say "  !  could not resolve AR connection: #{e.class}: #{e.message}", :red
+        nil
+      end
+
+      def detected_database_name
+        @detected_database_name ||= DatabaseTargetDetector.new(
+          destination_root: destination_root
+        ).detect
+      end
+
+      def effective_database_name
+        options[:database] || detected_database_name
+      end
+
+      def log(message)
+        return if options[:quiet]
+
+        say message
+      end
+
+      def log_change(message)
+        return if options[:quiet]
+
+        say message, :yellow
+      end
     end
   end
 end

data/lib/pgbus/client/ensure_stream_queue.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Client
+    # Idempotent stream-queue setup. Creates the PGMQ queue (delegating to
+    # `ensure_queue` which already handles schema bootstrap and dedup),
+    # overrides the NOTIFY throttle to 0 so every broadcast fires its
+    # own NOTIFY, and adds an `msg_id` index on the archive table that
+    # PGMQ does not ship with.
+    #
+    # PGMQ's archive tables (`pgmq.a_<name>`) only carry an `archived_at`
+    # index by default. `Client#read_after`'s replay query filters by
+    # `WHERE msg_id > $1`, which becomes a sequential scan once the archive
+    # grows past a few thousand rows. We add the index here, scoped to
+    # stream queues only, so users with chat-history-style retention don't
+    # hit a performance cliff.
+    #
+    # Called from `Pgbus.stream(name).broadcast(...)` on first publish per
+    # stream and from the streamer on first subscription per stream.
+    module EnsureStreamQueue
+      def ensure_stream_queue(stream_name)
+        ensure_queue(stream_name)
+        full_name = config.queue_name(stream_name)
+
+        # PGMQ's default NOTIFY throttle is 250ms — meant to coalesce
+        # high-frequency worker queue inserts. Streams are latency-
+        # sensitive and need every broadcast to fire a NOTIFY, even
+        # when several are batched within a single millisecond.
+        # Override the throttle to 0 specifically for stream queues.
+        synchronized { @pgmq.enable_notify_insert(full_name, throttle_interval_ms: 0) } if config.listen_notify
+
+        # CREATE INDEX IF NOT EXISTS is idempotent in Postgres but still
+        # requires a roundtrip and a brief ACCESS SHARE lock on the archive
+        # table. Broadcast-per-after_commit loops can hit this 1000x/sec on
+        # the same stream, so memoize per-process after the first success.
+        return if @stream_indexes_created[stream_name]
+
+        sanitized = QueueNameValidator.sanitize!(full_name)
+        sql = <<~SQL
+          CREATE INDEX IF NOT EXISTS a_#{sanitized}_msg_id_idx
+          ON pgmq.a_#{sanitized} (msg_id)
+        SQL
+
+        synchronized do
+          with_raw_connection do |conn|
+            conn.exec(sql)
+          end
+        end
+
+        @stream_indexes_created[stream_name] = true
+      end
+    end
+  end
+end

data/lib/pgbus/client/read_after.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Client
+    # Non-consuming peek across PGMQ live (`q_`) and archive (`a_`) tables. Used
+    # exclusively by `Pgbus::Web::Streamer` for SSE replay — workers continue to
+    # use `read_batch` (claim semantics). The two read paths are disjoint.
+    #
+    # The cursor is the highest msg_id the client has already seen. Replay returns
+    # everything strictly greater, ordered by msg_id ASC, capped by `limit`.
+    module ReadAfter
+      Envelope = Data.define(:msg_id, :enqueued_at, :payload, :source)
+
+      DEFAULT_LIMIT = 500
+
+      def read_after(stream_name, after_id:, limit: DEFAULT_LIMIT)
+        sanitized = sanitized_queue(stream_name)
+        sql = build_read_after_sql(sanitized)
+
+        rows = synchronized do
+          with_raw_connection do |conn|
+            conn.exec_params(sql, [after_id.to_i, limit.to_i]).to_a
+          end
+        end
+
+        rows.map { |row| build_envelope(row) }
+      end
+
+      def stream_current_msg_id(stream_name)
+        sanitized = sanitized_queue(stream_name)
+        sql = "SELECT COALESCE(MAX(msg_id), 0) AS max FROM pgmq.q_#{sanitized}"
+        synchronized do
+          with_raw_connection do |conn|
+            conn.exec(sql).first.fetch("max").to_i
+          end
+        end
+      end
+
+      def stream_oldest_msg_id(stream_name)
+        sanitized = sanitized_queue(stream_name)
+        sql = <<~SQL
+          SELECT LEAST(
+            (SELECT MIN(msg_id) FROM pgmq.q_#{sanitized}),
+            (SELECT MIN(msg_id) FROM pgmq.a_#{sanitized})
+          ) AS least
+        SQL
+        synchronized do
+          with_raw_connection do |conn|
+            value = conn.exec(sql).first.fetch("least")
+            value&.to_i
+          end
+        end
+      end
+
+      private
+
+      # Builds the union of live and archive tables. The outer ORDER BY + LIMIT
+      # ensures we never return more than `limit` rows total even if both
+      # subqueries hit it. The 'live'/'archive' constants are how the streamer
+      # tells whether a row was peeked from the queue or replayed from history;
+      # the streamer doesn't currently distinguish them, but we keep the column
+      # so debugging is straightforward when archive replay misbehaves.
+      def build_read_after_sql(sanitized)
+        <<~SQL
+          (
+            SELECT msg_id, enqueued_at, message, 'live'::text AS source
+            FROM pgmq.q_#{sanitized}
+            WHERE msg_id > $1
+            ORDER BY msg_id ASC
+            LIMIT $2
+          )
+          UNION ALL
+          (
+            SELECT msg_id, enqueued_at, message, 'archive'::text AS source
+            FROM pgmq.a_#{sanitized}
+            WHERE msg_id > $1
+            ORDER BY msg_id ASC
+            LIMIT $2
+          )
+          ORDER BY msg_id ASC
+          LIMIT $2
+        SQL
+      end
+
+      def sanitized_queue(stream_name)
+        full = config.queue_name(stream_name)
+        QueueNameValidator.sanitize!(full)
+      end
+
+      def build_envelope(row)
+        Envelope.new(
+          msg_id: row.fetch("msg_id").to_i,
+          enqueued_at: row.fetch("enqueued_at"),
+          payload: row.fetch("message"),
+          source: row.fetch("source")
+        )
+      end
+    end
+  end
+end

data/lib/pgbus/client.rb

@@ -1,9 +1,14 @@
 # frozen_string_literal: true
 
 require "json"
+require_relative "client/read_after"
+require_relative "client/ensure_stream_queue"
 
 module Pgbus
   class Client
+    include ReadAfter
+    include EnsureStreamQueue
+
     attr_reader :pgmq, :config
 
     PGMQ_REQUIRE_MUTEX = Mutex.new
@@ -49,6 +54,7 @@ module Pgbus
       end
 
       @queues_created = Concurrent::Map.new
+      @stream_indexes_created = Concurrent::Map.new
       @queue_strategy = QueueFactory.for(config)
       @schema_ensured = false
     end

data/lib/pgbus/configuration.rb

@@ -84,6 +84,13 @@ module Pgbus
     attr_accessor :web_auth, :web_refresh_interval, :web_per_page, :web_live_updates, :web_data_source,
                   :insights_default_minutes, :base_controller_class, :return_to_app_url
 
+    # Streams (turbo-rails replacement, SSE-based)
+    attr_accessor :streams_enabled, :streams_queue_prefix, :streams_signed_name_secret,
+                  :streams_default_retention, :streams_retention, :streams_heartbeat_interval,
+                  :streams_max_connections, :streams_idle_timeout, :streams_listen_health_check_ms,
+                  :streams_write_deadline_ms, :streams_falcon_streaming_body,
+                  :streams_stats_enabled
+
     def initialize
       @database_url = nil
       @connection_params = nil
@@ -150,6 +157,34 @@ module Pgbus
       @insights_default_minutes = 30 * 24 * 60 # 30 days
       @base_controller_class = "::ActionController::Base"
       @return_to_app_url = nil
+
+      @streams_enabled = true
+      @streams_queue_prefix = "pgbus_stream"
+      @streams_signed_name_secret = nil
+      @streams_default_retention = 5 * 60 # 5 minutes
+      @streams_retention = {}
+      @streams_heartbeat_interval = 15
+      @streams_max_connections = 2_000
+      @streams_idle_timeout = 3_600 # 1 hour
+      # 250ms — this value plays two roles: (1) the TCP keepalive
+      # interval for the streamer's PG LISTEN connection, and (2) the
+      # upper bound on how long Dispatcher#handle_connect waits for
+      # the Listener to acknowledge a synchronous ensure_listening
+      # call. 5s was unbounded enough to drop messages on a
+      # realistic subscribe burst; 250ms keeps the connect-path race
+      # window tight while still leaving headroom over a typical
+      # PG keepalive interval.
+      @streams_listen_health_check_ms = 250
+      @streams_write_deadline_ms = 5_000
+      @streams_falcon_streaming_body = false
+      # Opt-in: when true, the Dispatcher writes one row to
+      # pgbus_stream_stats per broadcast/connect/disconnect. Default
+      # off because stream event volume can be much higher than job
+      # volume and the Insights surface is only useful if operators
+      # actually look at it. Separate from #stats_enabled (which
+      # gates pgbus_job_stats recording) on purpose — operators
+      # usually want job stats on and stream stats off, or vice versa.
+      @streams_stats_enabled = false
     end
 
     def queue_name(name)
@@ -208,9 +243,39 @@ module Pgbus
         raise ArgumentError, "insights_default_minutes must be a positive integer"
       end
 
+      validate_streams!
+
       self
     end
 
+    def validate_streams!
+      unless streams_default_retention.is_a?(Numeric) && streams_default_retention >= 0
+        raise ArgumentError, "streams_default_retention must be a non-negative number"
+      end
+
+      unless streams_max_connections.is_a?(Integer) && streams_max_connections.positive?
+        raise ArgumentError, "streams_max_connections must be a positive integer"
+      end
+
+      unless streams_heartbeat_interval.is_a?(Numeric) && streams_heartbeat_interval.positive?
+        raise ArgumentError, "streams_heartbeat_interval must be a positive number"
+      end
+
+      unless streams_idle_timeout.is_a?(Numeric) && streams_idle_timeout.positive?
+        raise ArgumentError, "streams_idle_timeout must be a positive number"
+      end
+
+      unless streams_listen_health_check_ms.is_a?(Integer) && streams_listen_health_check_ms.positive?
+        raise ArgumentError, "streams_listen_health_check_ms must be a positive integer"
+      end
+
+      unless streams_write_deadline_ms.is_a?(Integer) && streams_write_deadline_ms.positive?
+        raise ArgumentError, "streams_write_deadline_ms must be a positive integer"
+      end
+
+      raise ArgumentError, "streams_retention must be a Hash" unless streams_retention.is_a?(Hash)
+    end
+
     # Set the worker capsule list. Accepts:
     #
     #   String — parsed via Pgbus::Configuration::CapsuleDSL into capsules

data/lib/pgbus/engine.rb

@@ -46,6 +46,7 @@ module Pgbus
 
     rake_tasks do
       load File.expand_path("../tasks/pgbus_pgmq.rake", __dir__)
+      load File.expand_path("../tasks/pgbus_streams.rake", __dir__)
     end
 
     initializer "pgbus.i18n" do
@@ -56,5 +57,35 @@ module Pgbus
       require "pgbus/web/authentication"
       require "pgbus/web/data_source"
     end
+
+    # Install the watermark cache middleware ahead of the app's own
+    # middleware so the thread-local cache is cleared between every
+    # Rack request. Without this, repeated page renders served by the
+    # same Puma thread would see stale current_msg_id values.
+    initializer "pgbus.streams.middleware" do |app|
+      app.middleware.use Pgbus::Streams::WatermarkCacheMiddleware if Pgbus.configuration.streams_enabled
+    end
+
+    # Install the Turbo::StreamsChannel patch after turbo-rails has been
+    # loaded. The patch redirects broadcast_stream_to through Pgbus.stream
+    # instead of ActionCable. When turbo-rails is not loaded, this is a
+    # no-op and pgbus_stream_from still works via the explicit
+    # Pgbus.stream(...).broadcast(...) API.
+    initializer "pgbus.streams.turbo_broadcastable", after: :load_config_initializers do
+      ActiveSupport.on_load(:after_initialize) do
+        if Pgbus.configuration.streams_enabled
+          # Touch the constant first so Zeitwerk autoloads
+          # lib/pgbus/streams/turbo_broadcastable.rb. The file defines
+          # `Pgbus::Streams::TurboBroadcastable` (the autoloaded const)
+          # AND `Pgbus::Streams.install_turbo_broadcastable_patch!`
+          # (a side-effect class method on the parent module). Without
+          # the constant reference, Zeitwerk doesn't load the file and
+          # the method call below raises NoMethodError. Assigning to
+          # `_` keeps RuboCop's Lint/Void from deleting the line.
+          _autoload_trigger = Pgbus::Streams::TurboBroadcastable
+          Pgbus::Streams.install_turbo_broadcastable_patch!
+        end
+      end
+    end
   end
 end

data/lib/pgbus/generators/config_converter.rb

@@ -151,15 +151,23 @@ module Pgbus
       end
 
       # Returns [constant_settings, varying_settings].
-      # constant_settings: { "key" => value }   (same value across all envs)
+      # constant_settings: { "key" => value }   (same value in EVERY env)
       # varying_settings:  { "key" => { env => value, ... } }
+      #
+      # A setting is only "constant" when it is present in every env
+      # and all envs agree on the value. If any env is missing the
+      # setting entirely (e.g. `polling_interval: 0.01` set only under
+      # `test:`), emitting it as an unconditional line would silently
+      # apply the value to envs that never asked for it — see #93.
       def partition_by_variance(all_settings)
         constant = {}
         varying = {}
         all_settings.each do |key, env_values|
           present_values = env_values.reject { |_, v| v == :__missing__ }
           unique_values = present_values.values.uniq
-          if unique_values.size <= 1
+          all_envs_present = present_values.size == env_values.size
+
+          if all_envs_present && unique_values.size <= 1
             constant[key] = unique_values.first
           else
             varying[key] = present_values
@@ -182,6 +190,18 @@ module Pgbus
 
       def render_varying_setting(key, env_values)
         envs = env_values.keys
+
+        # Single-env coverage: the setting exists in exactly one env.
+        # Emit an `if Rails.env.X?` modifier rather than a case block
+        # so other envs fall back to the gem default. This is the
+        # fix for #93 — without it, a `test:`-only `polling_interval`
+        # would leak into dev and prod as an unconditional assignment.
+        if envs.size == 1
+          env = envs.first
+          value = env_values[env]
+          return ["c.#{key} = #{render_value(key, value)} if Rails.env.#{env}?"]
+        end
+
         if envs.size == 2 && envs.include?("development")
           # Special case: "everything except dev" — common pattern
           non_dev_value = env_values.except("development").values.first