pgbus 0.7.8 → 0.8.0

data/lib/pgbus/integrations/appsignal/probe.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Integrations
+    module Appsignal
+      # Minutely probe that pushes pgbus-wide gauges into AppSignal.
+      #
+      # All readings come from Pgbus::Web::DataSource so the probe doesn't
+      # duplicate query logic. DataSource is built to be resilient — every
+      # method rescues StandardError and returns a safe default — but we
+      # still wrap each section in our own rescue so a probe iteration
+      # never raises out into the AppSignal probe runner.
+      module Probe
+        METRIC_PREFIX = "pgbus_"
+        private_constant :METRIC_PREFIX
+
+        class << self
+          def install! # rubocop:disable Naming/PredicateMethod
+            return false if @installed
+
+            ::Appsignal::Probes.register :pgbus, new_probe_instance
+            @installed = true
+            true
+          end
+
+          def installed?
+            @installed == true
+          end
+
+          def reset!
+            ::Appsignal::Probes.unregister(:pgbus) if defined?(::Appsignal::Probes) &&
+                                                      ::Appsignal::Probes.respond_to?(:unregister)
+            @installed = false
+          end
+
+          # Visible for testing — returns a fresh runnable probe.
+          def new_probe_instance
+            Runner.new
+          end
+        end
+
+        # The actual probe object; AppSignal calls #call once per minute.
+        class Runner
+          def initialize(data_source: nil)
+            @data_source = data_source
+          end
+
+          def call
+            return unless data_source
+
+            track_queues
+            track_processes
+            track_summary
+            track_streams
+          end
+
+          private
+
+          def data_source
+            @data_source ||=
+              (::Pgbus::Web::DataSource.new if defined?(::Pgbus::Web::DataSource))
+          end
+
+          def track_queues
+            data_source.queues_with_metrics.each do |q|
+              tags = { queue: q[:name] }
+              gauge "queue_depth", q[:queue_length], tags
+              gauge "queue_visible_depth", q[:queue_visible_length], tags
+              gauge "queue_paused", q[:paused] ? 1 : 0, tags
+              age = q[:oldest_msg_age_sec]
+              gauge "queue_oldest_message_age_seconds", age, tags if age
+            end
+          rescue StandardError => e
+            log_failure("queue metrics", e)
+          end
+
+          def track_processes
+            gauge "active_processes", data_source.processes.count
+          rescue StandardError => e
+            log_failure("process metrics", e)
+          end
+
+          def track_summary
+            stats = data_source.summary_stats
+            gauge "total_queues", stats[:total_queues]
+            gauge "total_depth", stats[:total_depth]
+            gauge "total_visible", stats[:total_visible]
+            gauge "dlq_depth", stats[:dlq_depth]
+            gauge "failed_events_total", stats[:failed_count]
+            gauge "throughput_rate", stats[:throughput_rate]
+            gauge "total_dead_tuples", stats[:total_dead_tuples]
+            gauge "tables_needing_vacuum", stats[:tables_needing_vacuum]
+            gauge "oldest_transaction_age_seconds", stats[:oldest_transaction_age_sec]
+          rescue StandardError => e
+            log_failure("summary metrics", e)
+          end
+
+          def track_streams
+            return unless data_source.respond_to?(:stream_stats_available?) &&
+                          data_source.stream_stats_available?
+
+            summary = data_source.stream_stats_summary
+            gauge "stream_broadcasts_60m", summary[:broadcasts]
+            gauge "stream_connects_60m", summary[:connects]
+            gauge "stream_disconnects_60m", summary[:disconnects]
+            gauge "stream_active_connections", summary[:active_estimate]
+            gauge "stream_avg_fanout", summary[:avg_fanout]
+            gauge "stream_avg_broadcast_ms", summary[:avg_broadcast_ms]
+          rescue StandardError => e
+            log_failure("stream metrics", e)
+          end
+
+          def gauge(key, value, tags = {})
+            return if value.nil?
+
+            ::Appsignal.set_gauge("#{METRIC_PREFIX}#{key}", value, tags)
+          end
+
+          def log_failure(label, error)
+            Pgbus.logger.debug do
+              "[Pgbus::AppSignal::Probe] #{label} failed: #{error.class}: #{error.message}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/integrations/appsignal/subscriber.rb

@@ -0,0 +1,303 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Pgbus
+  module Integrations
+    module Appsignal
+      # Translates Pgbus::Instrumentation events into AppSignal transactions
+      # and custom metrics.
+      #
+      # Job and event-handler events open a BACKGROUND_JOB transaction so they
+      # appear under AppSignal's "Performance > Background jobs" view, with
+      # action `<JobClass>#perform` or `<HandlerClass>#handle`. All other
+      # events are reported as counters or distributions only.
+      #
+      # All metric names are prefixed `pgbus_`. Tag keys avoid high-cardinality
+      # values (no msg_id, no event_id) so AppSignal's metric storage stays
+      # efficient.
+      module Subscriber
+        BACKGROUND_JOB = "background_job"
+        METRIC_PREFIX = "pgbus_"
+        private_constant :BACKGROUND_JOB, :METRIC_PREFIX
+
+        # Tracked so we can detach in reset! (used by specs).
+        @subscriptions = []
+
+        class << self
+          def install!
+            return false if @installed
+
+            @subscriptions = [
+              subscribe("pgbus.executor.execute") { |event| on_executor_execute(event) },
+              subscribe("pgbus.job_completed") { |event| on_job_completed(event) },
+              subscribe("pgbus.job_failed") { |event| on_job_failed(event) },
+              subscribe("pgbus.job_dead_lettered") { |event| on_job_dead_lettered(event) },
+              subscribe("pgbus.event_processed") { |event| on_event_processed(event) },
+              subscribe("pgbus.event_failed") { |event| on_event_failed(event) },
+              subscribe("pgbus.client.send_message") { |event| on_send_message(event) },
+              subscribe("pgbus.client.send_batch") { |event| on_send_batch(event) },
+              subscribe("pgbus.client.read_batch") { |event| on_read_batch(event) },
+              subscribe("pgbus.stream.broadcast") { |event| on_stream_broadcast(event) },
+              subscribe("pgbus.outbox.publish") { |event| on_outbox_publish(event) },
+              subscribe("pgbus.recurring.enqueue") { |event| on_recurring_enqueue(event) },
+              subscribe("pgbus.worker.recycle") { |event| on_worker_recycle(event) }
+            ]
+            @installed = true
+          end
+
+          def installed?
+            @installed == true
+          end
+
+          def reset!
+            @subscriptions&.each { |s| ActiveSupport::Notifications.unsubscribe(s) }
+            @subscriptions = []
+            @installed = false
+          end
+
+          private
+
+          def subscribe(name, &block)
+            # silence rubocop unused
+            ActiveSupport::Notifications.subscribe(name) do |*args|
+              event = ActiveSupport::Notifications::Event.new(*args)
+              safely { block.call(event) }
+            end
+          end
+
+          # Errors in the subscriber must never affect the producer thread.
+          # AppSignal can be misconfigured, the agent can be down, etc. — log
+          # and move on.
+          def safely
+            yield
+          rescue StandardError => e
+            Pgbus.logger.warn do
+              "[Pgbus::AppSignal] subscriber error: #{e.class}: #{e.message}"
+            end
+          end
+
+          # ── Job execution ───────────────────────────────────────────────
+
+          def on_executor_execute(event)
+            payload = event.payload
+            transaction = ::Appsignal::Transaction.create(BACKGROUND_JOB)
+            transaction.set_action_if_nil("#{payload[:job_class] || "UnknownJob"}#perform")
+            apply_queue_start(transaction, payload[:enqueued_at])
+            transaction.add_tags(job_tags(payload))
+            transaction.add_params_if_nil { { arguments: payload[:arguments] } }
+            ::Appsignal.add_distribution_value(
+              "#{METRIC_PREFIX}job_duration_ms",
+              event.duration,
+              { queue: payload[:queue], job_class: payload[:job_class] }
+            )
+          ensure
+            ::Appsignal::Transaction.complete_current!
+          end
+
+          def on_job_completed(event)
+            payload = event.payload
+            ::Appsignal.increment_counter(
+              "#{METRIC_PREFIX}queue_job_count",
+              1,
+              { queue: payload[:queue], job_class: payload[:job_class], status: "processed" }
+            )
+          end
+
+          def on_job_failed(event)
+            payload = event.payload
+            ::Appsignal.increment_counter(
+              "#{METRIC_PREFIX}queue_job_count",
+              1,
+              { queue: payload[:queue], job_class: payload[:job_class], status: "failed" }
+            )
+            err = payload[:exception_object]
+            ::Appsignal.set_error(err) if err && ::Appsignal.respond_to?(:set_error)
+          end
+
+          def on_job_dead_lettered(event)
+            payload = event.payload
+            ::Appsignal.increment_counter(
+              "#{METRIC_PREFIX}queue_job_count",
+              1,
+              { queue: payload[:queue], job_class: payload[:job_class], status: "dead_lettered" }
+            )
+          end
+
+          # ── Event handler ───────────────────────────────────────────────
+
+          def on_event_processed(event)
+            payload = event.payload
+            transaction = ::Appsignal::Transaction.create(BACKGROUND_JOB)
+            transaction.set_action_if_nil("#{payload[:handler] || "UnknownHandler"}#handle")
+            apply_queue_start(transaction, payload[:published_at])
+            transaction.add_tags(handler_tags(payload))
+            ::Appsignal.add_distribution_value(
+              "#{METRIC_PREFIX}event_duration_ms",
+              event.duration,
+              { handler: payload[:handler], routing_key: payload[:routing_key] }
+            )
+            ::Appsignal.increment_counter(
+              "#{METRIC_PREFIX}event_count",
+              1,
+              { handler: payload[:handler], routing_key: payload[:routing_key], status: "processed" }
+            )
+          ensure
+            ::Appsignal::Transaction.complete_current!
+          end
+
+          def on_event_failed(event)
+            payload = event.payload
+            ::Appsignal.increment_counter(
+              "#{METRIC_PREFIX}event_count",
+              1,
+              { handler: payload[:handler], routing_key: payload[:routing_key], status: "failed" }
+            )
+            err = payload[:exception_object]
+            ::Appsignal.set_error(err) if err && ::Appsignal.respond_to?(:set_error)
+          end
+
+          # ── Client (PGMQ wrapper) ───────────────────────────────────────
+
+          def on_send_message(event)
+            payload = event.payload
+            ::Appsignal.increment_counter(
+              "#{METRIC_PREFIX}messages_sent",
+              1,
+              { queue: payload[:queue] }
+            )
+            ::Appsignal.add_distribution_value(
+              "#{METRIC_PREFIX}send_duration_ms",
+              event.duration,
+              { queue: payload[:queue] }
+            )
+          end
+
+          def on_send_batch(event)
+            payload = event.payload
+            count = payload[:count] || payload[:batch_size] || 1
+            ::Appsignal.increment_counter(
+              "#{METRIC_PREFIX}messages_sent",
+              count,
+              { queue: payload[:queue] }
+            )
+            ::Appsignal.add_distribution_value(
+              "#{METRIC_PREFIX}send_batch_duration_ms",
+              event.duration,
+              { queue: payload[:queue] }
+            )
+          end
+
+          def on_read_batch(event)
+            payload = event.payload
+            count = payload[:count] || payload[:fetched] || 0
+            ::Appsignal.increment_counter(
+              "#{METRIC_PREFIX}messages_read",
+              count,
+              { queue: payload[:queue] }
+            )
+          end
+
+          # ── Streams ─────────────────────────────────────────────────────
+
+          def on_stream_broadcast(event)
+            payload = event.payload
+            ::Appsignal.increment_counter(
+              "#{METRIC_PREFIX}stream_broadcast_count",
+              1,
+              { stream: payload[:stream], deferred: payload[:deferred] ? "true" : "false" }
+            )
+            return unless payload[:bytes]
+
+            ::Appsignal.add_distribution_value(
+              "#{METRIC_PREFIX}stream_broadcast_bytes",
+              payload[:bytes],
+              { stream: payload[:stream] }
+            )
+          end
+
+          # ── Outbox ──────────────────────────────────────────────────────
+
+          def on_outbox_publish(event)
+            payload = event.payload
+            ::Appsignal.increment_counter(
+              "#{METRIC_PREFIX}outbox_published",
+              1,
+              { kind: payload[:kind] || "job" }
+            )
+            ::Appsignal.add_distribution_value(
+              "#{METRIC_PREFIX}outbox_publish_duration_ms",
+              event.duration,
+              { kind: payload[:kind] || "job" }
+            )
+          end
+
+          # ── Recurring scheduler ─────────────────────────────────────────
+
+          def on_recurring_enqueue(event)
+            payload = event.payload
+            ::Appsignal.increment_counter(
+              "#{METRIC_PREFIX}recurring_enqueued",
+              1,
+              { task: payload[:task], class_name: payload[:class_name] }
+            )
+          end
+
+          # ── Worker lifecycle ────────────────────────────────────────────
+
+          def on_worker_recycle(event)
+            payload = event.payload
+            ::Appsignal.increment_counter(
+              "#{METRIC_PREFIX}worker_recycled",
+              1,
+              { reason: payload[:reason] }
+            )
+          end
+
+          # ── Helpers ─────────────────────────────────────────────────────
+
+          # AppSignal expects queue-start as Unix epoch milliseconds. Pgbus
+          # carries it as either an ISO-8601 String or a Time — both happen
+          # in practice (executor passes the JSON string, handler passes a
+          # parsed Time).
+          def apply_queue_start(transaction, value)
+            return unless value
+
+            millis =
+              case value
+              when Time
+                (value.to_f * 1_000).to_i
+              when String
+                (Time.parse(value).to_f * 1_000).to_i
+              when Numeric
+                value.to_i
+              end
+            transaction.set_queue_start(millis) if millis
+          rescue ArgumentError
+            # Unparseable timestamp — skip rather than blow up.
+          end
+
+          def job_tags(payload)
+            tags = {
+              "queue" => payload[:queue],
+              "job_class" => payload[:job_class],
+              "attempts" => payload[:read_ct]
+            }
+            tags["active_job_id"] = payload[:job_id] if payload[:job_id]
+            tags["provider_job_id"] = payload[:provider_job_id] if payload[:provider_job_id]
+            tags["request_id"] = payload[:provider_job_id] || payload[:job_id]
+            tags.compact
+          end
+
+          def handler_tags(payload)
+            {
+              "handler" => payload[:handler],
+              "routing_key" => payload[:routing_key],
+              "attempts" => payload[:read_ct]
+            }.compact
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/integrations/appsignal.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "pgbus/integrations/appsignal/subscriber"
+require "pgbus/integrations/appsignal/probe"
+
+module Pgbus
+  module Integrations
+    # AppSignal integration for pgbus.
+    #
+    # Loaded automatically by Pgbus::Engine when the appsignal gem is present
+    # and config.appsignal_enabled is true (default). To opt out:
+    #
+    #   Pgbus.configure do |c|
+    #     c.appsignal_enabled = false
+    #   end
+    #
+    # The integration:
+    #   * Subscribes to pgbus.* ActiveSupport::Notifications and translates
+    #     them into AppSignal background-job transactions and metrics.
+    #   * Registers a minutely probe that reports queue depth, DLQ size,
+    #     dead-tuple counts, MVCC horizon age, and stream stats from
+    #     Pgbus::Web::DataSource.
+    #
+    # All metric names are prefixed `pgbus_` so they group cleanly in
+    # AppSignal's custom-metrics view.
+    module Appsignal
+      module_function
+
+      def install! # rubocop:disable Naming/PredicateMethod
+        return false unless defined?(::Appsignal)
+        return false if @installed
+
+        Subscriber.install!
+        Probe.install! if Pgbus.configuration.appsignal_probe_enabled
+        @installed = true
+        Pgbus.logger.info { "[Pgbus] AppSignal integration installed" }
+        true
+      end
+
+      def installed?
+        @installed == true
+      end
+
+      # Test hook: tear everything down so a fresh install! can run.
+      def reset!
+        Subscriber.reset! if defined?(Subscriber)
+        Probe.reset! if defined?(Probe)
+        @installed = false
+      end
+    end
+  end
+end

data/lib/pgbus/outbox.rb

@@ -5,22 +5,26 @@ module Pgbus
     module_function
 
     def publish(queue_name, payload, headers: nil, priority: nil, delay: 0)
-      OutboxEntry.create!(
-        queue_name: queue_name,
-        payload: payload,
-        headers: headers,
-        priority: priority || Pgbus.configuration.default_priority,
-        delay: delay
-      )
+      Instrumentation.instrument("pgbus.outbox.publish", queue: queue_name, kind: :job) do
+        OutboxEntry.create!(
+          queue_name: queue_name,
+          payload: payload,
+          headers: headers,
+          priority: priority || Pgbus.configuration.default_priority,
+          delay: delay
+        )
+      end
     end
 
     def publish_event(routing_key, payload, headers: nil)
-      event_data = EventBus::Publisher.build_event_data(payload)
-      OutboxEntry.create!(
-        routing_key: routing_key,
-        payload: event_data,
-        headers: headers
-      )
+      Instrumentation.instrument("pgbus.outbox.publish", routing_key: routing_key, kind: :event) do
+        event_data = EventBus::Publisher.build_event_data(payload)
+        OutboxEntry.create!(
+          routing_key: routing_key,
+          payload: event_data,
+          headers: headers
+        )
+      end
     end
 
     def flush!

data/lib/pgbus/process/dispatcher.rb

@@ -15,6 +15,7 @@ module Pgbus
       OUTBOX_CLEANUP_INTERVAL = 3600 # Run outbox cleanup every hour
       JOB_LOCK_CLEANUP_INTERVAL = 300 # Run job lock cleanup every 5 minutes
       STATS_CLEANUP_INTERVAL = 3600 # Run stats cleanup every hour
+      ORPHAN_STREAM_SWEEP_INTERVAL = 3600 # Run orphan stream sweep every hour
       TABLE_MAINTENANCE_INTERVAL = Pgbus::TableMaintenance::MAINTENANCE_INTERVAL
 
       # Page size for archive compaction. Each cycle deletes up to this
@@ -38,6 +39,7 @@ module Pgbus
         @last_outbox_cleanup_at = monotonic_now
         @last_job_lock_cleanup_at = monotonic_now
         @last_stats_cleanup_at = monotonic_now
+        @last_orphan_stream_sweep_at = monotonic_now
         @last_table_maintenance_at = monotonic_now
       end
 
@@ -86,6 +88,8 @@ module Pgbus
         run_if_due(now, :@last_outbox_cleanup_at, OUTBOX_CLEANUP_INTERVAL) { cleanup_outbox }
         run_if_due(now, :@last_job_lock_cleanup_at, JOB_LOCK_CLEANUP_INTERVAL) { cleanup_job_locks }
         run_if_due(now, :@last_stats_cleanup_at, STATS_CLEANUP_INTERVAL) { cleanup_stats }
+        sweep_interval = config.streams_orphan_sweep_interval
+        run_if_due(now, :@last_orphan_stream_sweep_at, sweep_interval) { sweep_orphan_streams } if sweep_interval
         run_if_due(now, :@last_table_maintenance_at, TABLE_MAINTENANCE_INTERVAL) { run_table_maintenance }
       end
 
@@ -315,6 +319,40 @@ module Pgbus
         config.streams_default_retention.to_f
       end
 
+      def sweep_orphan_streams
+        prefix = config.streams_queue_prefix
+        return if prefix.nil? || prefix.empty?
+
+        threshold = config.streams_orphan_threshold
+        return unless threshold
+
+        conn = config.connects_to ? Pgbus::BusRecord.connection : ActiveRecord::Base.connection
+        queue_names = conn.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
+
+        dropped = 0
+        queue_names.each do |full_name|
+          next unless full_name.start_with?("#{prefix}_")
+
+          row = conn.select_one(<<~SQL, "Pgbus Orphan Check")
+            SELECT count(*) AS queue_length
+            FROM pgmq.q_#{QueueNameValidator.sanitize!(full_name)}
+          SQL
+
+          next unless row
+          next if row["queue_length"].to_i.positive?
+
+          Pgbus.client.drop_queue(full_name, prefixed: false)
+          dropped += 1
+          Pgbus.logger.info { "[Pgbus] Dropped orphan stream queue: #{full_name}" }
+        rescue StandardError => e
+          Pgbus.logger.warn { "[Pgbus] Orphan stream sweep failed for #{full_name}: #{e.message}" }
+        end
+
+        Pgbus.logger.debug { "[Pgbus] Orphan stream sweep complete: dropped #{dropped} queue(s)" } if dropped.positive?
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus] Orphan stream sweep failed: #{e.message}" }
+      end
+
       def cleanup_recurring_executions
         retention = config.recurring_execution_retention
         return unless retention&.positive?

data/lib/pgbus/process/worker.rb

@@ -302,15 +302,33 @@ module Pgbus
       end
 
       def check_recycle
-        return unless @lifecycle.running? && recycle_needed?
+        return unless @lifecycle.running?
+
+        reason = recycle_reason
+        return unless reason
 
         Pgbus.stopping = true
         @lifecycle.transition_to(:draining)
+        Pgbus::Instrumentation.instrument(
+          "pgbus.worker.recycle",
+          reason: reason,
+          jobs_processed: @jobs_processed.value,
+          memory_mb: current_memory_mb,
+          lifetime_seconds: monotonic_now - @started_at_monotonic
+        )
         @wake_signal.notify!
       end
 
+      def recycle_reason
+        return :max_jobs if exceeded_max_jobs?
+        return :max_memory if exceeded_max_memory?
+        return :max_lifetime if exceeded_max_lifetime?
+
+        nil
+      end
+
       def recycle_needed?
-        exceeded_max_jobs? || exceeded_max_memory? || exceeded_max_lifetime?
+        !recycle_reason.nil?
       end
 
       def exceeded_max_jobs?

data/lib/pgbus/recurring/scheduler.rb

@@ -41,8 +41,16 @@ module Pgbus
 
       def tick(now)
         schedule.due_tasks(now).each do |task, run_at|
-          schedule.enqueue_task(task, run_at: run_at)
-          @last_runs[task.key] = now
+          Pgbus::Instrumentation.instrument(
+            "pgbus.recurring.enqueue",
+            task: task.key,
+            class_name: task.class_name,
+            queue: task.queue_name,
+            run_at: run_at
+          ) do
+            schedule.enqueue_task(task, run_at: run_at)
+            @last_runs[task.key] = now
+          end
         rescue StandardError => e
           Pgbus.logger.error do
             "[Pgbus] Error scheduling recurring task #{task.key}: #{e.class}: #{e.message}"

data/lib/pgbus/streams/turbo_broadcastable.rb

@@ -36,7 +36,8 @@ module Pgbus
     module TurboBroadcastable
       def broadcast_stream_to(*streamables, content:)
         name = stream_name_from(streamables)
-        Pgbus.stream(name).broadcast(content)
+        mode = Pgbus.configuration.streams_default_broadcast_mode
+        Pgbus.stream(name, durable: mode == :durable).broadcast(content)
       end
     end