pgbus 0.9.7 → 0.9.9

data/lib/pgbus/metrics/backends/prometheus.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Metrics
+    module Backends
+      # Dependency-free, in-process Prometheus registry.
+      #
+      # Counters and gauges are stored per (name, sorted-tags) series. Histograms
+      # are stored as a sum+count summary — cheap, dependency-free, and enough for
+      # rate/avg dashboards without pre-declaring buckets. State is Mutex-guarded
+      # (no unsynchronized shared state), since instrumentation fires from many
+      # worker threads while the exporter renders from a Rack thread.
+      #
+      # Render output is Prometheus text exposition format (v0.0.4), served by
+      # Pgbus::Metrics::PrometheusExporter.
+      class Prometheus < Backend
+        Series = Struct.new(:type, :samples) # samples: { tags_key => value }
+
+        def initialize
+          super
+          @mutex = Mutex.new
+          # name => Series
+          @counters = {}
+          @gauges = {}
+          # name => { tags_key => { sum:, count:, tags: } }
+          @histograms = {}
+          # tags_key (sorted array) => original tags hash, for rendering labels
+          @tag_labels = {}
+        end
+
+        def increment(name, value = 1, tags = {})
+          key = tag_key(tags)
+          @mutex.synchronize do
+            series = (@counters[name] ||= {})
+            series[key] = (series[key] || 0) + value
+          end
+          nil
+        end
+
+        def gauge(name, value, tags = {})
+          key = tag_key(tags)
+          @mutex.synchronize do
+            series = (@gauges[name] ||= {})
+            series[key] = value
+          end
+          nil
+        end
+
+        def histogram(name, value, tags = {})
+          key = tag_key(tags)
+          @mutex.synchronize do
+            series = (@histograms[name] ||= {})
+            bucket = (series[key] ||= { sum: 0, count: 0 })
+            bucket[:sum] += value
+            bucket[:count] += 1
+          end
+          nil
+        end
+
+        # Render the full registry as Prometheus text exposition format.
+        def render
+          @mutex.synchronize do
+            lines = []
+            @counters.each { |name, series| render_simple(lines, name, "counter", series) }
+            @gauges.each   { |name, series| render_simple(lines, name, "gauge", series) }
+            @histograms.each { |name, series| render_summary(lines, name, series) }
+            "#{lines.join("\n")}\n"
+          end
+        end
+
+        private
+
+        def render_simple(lines, name, type, series)
+          lines << "# TYPE #{name} #{type}"
+          series.each do |key, value|
+            lines << "#{name}#{labels_for(key)} #{format_number(value)}"
+          end
+        end
+
+        def render_summary(lines, name, series)
+          lines << "# TYPE #{name} summary"
+          series.each do |key, bucket|
+            labels = labels_for(key)
+            lines << "#{name}_sum#{labels} #{format_number(bucket[:sum])}"
+            lines << "#{name}_count#{labels} #{bucket[:count]}"
+          end
+        end
+
+        # Stable, order-independent key for a tag set. Also memoizes the original
+        # (stringified) tag hash so render can reproduce the label set.
+        def tag_key(tags)
+          normalized = tags.each_with_object({}) do |(k, v), acc|
+            acc[k.to_s] = v.to_s
+          end
+          key = normalized.sort.freeze
+          @tag_labels[key] ||= normalized
+          key
+        end
+
+        def labels_for(key)
+          labels = @tag_labels[key]
+          return "" if labels.nil? || labels.empty?
+
+          inner = key.map { |k, _| %(#{k}="#{escape_label(labels[k])}") }.join(",")
+          "{#{inner}}"
+        end
+
+        # Prometheus label-value escaping: backslash, double-quote, newline.
+        def escape_label(value)
+          value.gsub("\\", "\\\\\\\\").gsub('"', '\\"').gsub("\n", '\\n')
+        end
+
+        def format_number(value)
+          if value.is_a?(Float) && value.finite? && value == value.to_i
+            value.to_i.to_s
+          else
+            value.to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/metrics/backends/statsd.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "socket"
+
+module Pgbus
+  module Metrics
+    module Backends
+      # StatsD backend emitting UDP datagrams in DogStatsD dialect.
+      #
+      #   increment -> "name:value|c"
+      #   gauge     -> "name:value|g"
+      #   histogram -> "name:value|ms"
+      #
+      # Tags are appended as DogStatsD `|#k:v,k:v`. No gem dependency — a plain
+      # UDPSocket. UDP send is fire-and-forget; any socket error is logged and
+      # swallowed so a metrics hiccup never propagates into the producer thread.
+      class Statsd < Backend
+        def initialize(host: "127.0.0.1", port: 8125, socket: nil)
+          super()
+          @host = host
+          @port = port
+          @socket = socket
+          @mutex = Mutex.new
+        end
+
+        def increment(name, value = 1, tags = {})
+          emit(name, value, "c", tags)
+        end
+
+        def gauge(name, value, tags = {})
+          emit(name, value, "g", tags)
+        end
+
+        def histogram(name, value, tags = {})
+          emit(name, value, "ms", tags)
+        end
+
+        private
+
+        def emit(name, value, type, tags)
+          datagram = "#{name}:#{value}|#{type}#{format_tags(tags)}"
+          socket.send(datagram, 0, @host, @port)
+          nil
+        rescue StandardError => e
+          Pgbus.logger.warn do
+            "[Pgbus::Metrics::Statsd] send failed: #{e.class}: #{e.message}"
+          end
+          nil
+        end
+
+        def format_tags(tags)
+          pairs = tags.compact.map { |k, v| "#{k}:#{v}" }
+          pairs.empty? ? "" : "|##{pairs.join(",")}"
+        end
+
+        # UDPSocket is not guaranteed thread-safe for concurrent sends; guard
+        # lazy creation so the first burst of instrumentation can't race.
+        def socket
+          @socket || @mutex.synchronize { @socket ||= UDPSocket.new }
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/metrics/prometheus_exporter.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module Metrics
+    # Rack app that renders the in-process Prometheus registry as text exposition
+    # format. Mount it in the host Rails app (or any Rack server) so Prometheus
+    # can scrape it — see the README. Mirrors the "mountable Rack app" shape used
+    # by Pgbus::Web::StreamApp.
+    #
+    #   # config/routes.rb
+    #   mount Pgbus::Metrics::PrometheusExporter.new => "/metrics"
+    #
+    # With no injected backend it reads Pgbus.configuration.metrics_backend, so a
+    # single `config.metrics_backend = :prometheus` wires both the subscriber and
+    # the exporter to the same registry.
+    class PrometheusExporter
+      CONTENT_TYPE = "text/plain; version=0.0.4"
+
+      def initialize(backend: nil)
+        @backend = backend
+      end
+
+      def call(_env)
+        backend = @backend || Pgbus.configuration.metrics_backend
+        unless backend.is_a?(Backends::Prometheus)
+          return [503, { "content-type" => "text/plain" },
+                  ["metrics_backend is not a Prometheus backend\n"]]
+        end
+
+        [200, { "content-type" => CONTENT_TYPE }, [backend.render]]
+      end
+    end
+  end
+end

data/lib/pgbus/metrics/subscriber.rb

@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+
+require "socket"
+
+module Pgbus
+  module Metrics
+    # Consumes Pgbus::Instrumentation events and forwards them to the configured
+    # metrics backend. The event→metric mapping (names, tags, counter vs
+    # histogram) intentionally mirrors the AppSignal subscriber so a team can
+    # switch backends and keep the same dashboards.
+    #
+    # Every handler runs inside `safely`: a raising backend (Prometheus registry
+    # bug, StatsD socket down, custom backend) is logged and swallowed — a metrics
+    # failure must never propagate into the producer thread that emitted the
+    # event.
+    module Subscriber
+      METRIC_PREFIX = "pgbus_"
+      private_constant :METRIC_PREFIX
+
+      @subscriptions = []
+      @backend = nil
+
+      class << self
+        def install!(backend:)
+          return false if @installed
+
+          @backend = backend
+          @subscriptions = [
+            subscribe("pgbus.executor.execute") { |event| on_executor_execute(event) },
+            subscribe("pgbus.job_completed") { |event| on_job_completed(event) },
+            subscribe("pgbus.job_failed") { |event| on_job_failed(event) },
+            subscribe("pgbus.job_dead_lettered") { |event| on_job_dead_lettered(event) },
+            subscribe("pgbus.event_processed") { |event| on_event_processed(event) },
+            subscribe("pgbus.event_failed") { |event| on_event_failed(event) },
+            subscribe("pgbus.client.send_message") { |event| on_send_message(event) },
+            subscribe("pgbus.client.send_batch") { |event| on_send_batch(event) },
+            subscribe("pgbus.client.read_batch") { |event| on_read_batch(event) },
+            subscribe("pgbus.stream.broadcast") { |event| on_stream_broadcast(event) },
+            subscribe("pgbus.outbox.publish") { |event| on_outbox_publish(event) },
+            subscribe("pgbus.recurring.enqueue") { |event| on_recurring_enqueue(event) },
+            subscribe("pgbus.worker.recycle") { |event| on_worker_recycle(event) },
+            subscribe("pgbus.consumer.recycle") { |event| on_consumer_recycle(event) },
+            subscribe("pgbus.client.pool") { |event| on_client_pool(event) }
+          ]
+          @installed = true
+        end
+
+        def installed?
+          @installed == true
+        end
+
+        def reset!
+          @subscriptions&.each { |s| ActiveSupport::Notifications.unsubscribe(s) }
+          @subscriptions = []
+          @backend = nil
+          @installed = false
+        end
+
+        private
+
+        attr_reader :backend
+
+        def subscribe(name, &block)
+          ActiveSupport::Notifications.subscribe(name) do |*args|
+            event = ActiveSupport::Notifications::Event.new(*args)
+            safely { block.call(event) }
+          end
+        end
+
+        # Errors in the subscriber must never affect the producer thread.
+        def safely
+          yield
+        rescue StandardError => e
+          Pgbus.logger.warn do
+            "[Pgbus::Metrics] subscriber error: #{e.class}: #{e.message}"
+          end
+        end
+
+        # ── Job execution ─────────────────────────────────────────────────
+
+        def on_executor_execute(event)
+          payload = event.payload
+          backend.histogram(
+            "#{METRIC_PREFIX}job_duration_ms",
+            event.duration,
+            compact(queue: payload[:queue], job_class: payload[:job_class])
+          )
+        end
+
+        def on_job_completed(event)
+          payload = event.payload
+          backend.increment(
+            "#{METRIC_PREFIX}queue_job_count", 1,
+            compact(queue: payload[:queue], job_class: payload[:job_class], status: "processed")
+          )
+        end
+
+        def on_job_failed(event)
+          payload = event.payload
+          backend.increment(
+            "#{METRIC_PREFIX}queue_job_count", 1,
+            compact(queue: payload[:queue], job_class: payload[:job_class], status: "failed")
+          )
+        end
+
+        def on_job_dead_lettered(event)
+          payload = event.payload
+          backend.increment(
+            "#{METRIC_PREFIX}queue_job_count", 1,
+            compact(queue: payload[:queue], job_class: payload[:job_class], status: "dead_lettered")
+          )
+        end
+
+        # ── Event handler ─────────────────────────────────────────────────
+
+        def on_event_processed(event)
+          payload = event.payload
+          tags = compact(handler: payload[:handler], routing_key: payload[:routing_key])
+          backend.histogram("#{METRIC_PREFIX}event_duration_ms", event.duration, tags)
+          backend.increment("#{METRIC_PREFIX}event_count", 1, tags.merge(status: "processed"))
+        end
+
+        def on_event_failed(event)
+          payload = event.payload
+          backend.increment(
+            "#{METRIC_PREFIX}event_count", 1,
+            compact(handler: payload[:handler], routing_key: payload[:routing_key], status: "failed")
+          )
+        end
+
+        # ── Client (PGMQ wrapper) ─────────────────────────────────────────
+
+        def on_send_message(event)
+          backend.increment("#{METRIC_PREFIX}messages_sent", 1, compact(queue: event.payload[:queue]))
+        end
+
+        def on_send_batch(event)
+          payload = event.payload
+          count = payload[:count] || payload[:batch_size] || 1
+          backend.increment("#{METRIC_PREFIX}messages_sent", count, compact(queue: payload[:queue]))
+        end
+
+        def on_read_batch(event)
+          payload = event.payload
+          count = payload[:count] || payload[:fetched] || 0
+          backend.increment("#{METRIC_PREFIX}messages_read", count, compact(queue: payload[:queue]))
+        end
+
+        # ── Streams ───────────────────────────────────────────────────────
+
+        def on_stream_broadcast(event)
+          payload = event.payload
+          backend.increment(
+            "#{METRIC_PREFIX}stream_broadcast_count", 1,
+            compact(stream: payload[:stream], deferred: payload[:deferred] ? "true" : "false")
+          )
+        end
+
+        # ── Outbox ────────────────────────────────────────────────────────
+
+        def on_outbox_publish(event)
+          backend.increment(
+            "#{METRIC_PREFIX}outbox_published", 1,
+            compact(kind: event.payload[:kind] || "job")
+          )
+        end
+
+        # ── Recurring scheduler ───────────────────────────────────────────
+
+        def on_recurring_enqueue(event)
+          payload = event.payload
+          backend.increment(
+            "#{METRIC_PREFIX}recurring_enqueued", 1,
+            compact(task: payload[:task], class_name: payload[:class_name])
+          )
+        end
+
+        # ── Worker lifecycle ──────────────────────────────────────────────
+
+        def on_worker_recycle(event)
+          backend.increment(
+            "#{METRIC_PREFIX}worker_recycled", 1,
+            compact(reason: event.payload[:reason], kind: "worker")
+          )
+        end
+
+        def on_consumer_recycle(event)
+          backend.increment(
+            "#{METRIC_PREFIX}worker_recycled", 1,
+            compact(reason: event.payload[:reason], kind: "consumer")
+          )
+        end
+
+        # ── Connection pool ───────────────────────────────────────────────
+
+        def on_client_pool(event)
+          payload = event.payload
+          tags = { hostname: hostname }
+          backend.gauge("#{METRIC_PREFIX}pool_size", payload[:size], tags)
+          backend.gauge("#{METRIC_PREFIX}pool_available", payload[:available], tags)
+        end
+
+        def hostname
+          Socket.gethostname
+        end
+
+        # Drop nil-valued tags so backends don't emit empty labels.
+        def compact(tags)
+          tags.compact
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/metrics.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Pgbus
+  # Backend-agnostic metrics adapter.
+  #
+  # Pgbus emits a family of `pgbus.*` ActiveSupport::Notifications events;
+  # `Pgbus::Instrumentation` is the single source of truth for the catalog. The
+  # AppSignal integration is one consumer of those
+  # events; this namespace is a second, vendor-neutral consumer that forwards the
+  # same event→metric mapping to a configurable backend so teams on Prometheus,
+  # Datadog, or plain StatsD get metrics without hand-writing subscribers.
+  #
+  # Opt in via `config.metrics_backend = :prometheus | :statsd | <Backend>`. The
+  # default is nil, which installs no subscription (zero overhead). Installation
+  # happens from the engine, mirroring the AppSignal initializer.
+  module Metrics
+    module_function
+
+    # Coerce a configuration value into a concrete Backend instance.
+    #
+    #   nil          -> Backend::Null (no-op)
+    #   :prometheus  -> Backends::Prometheus
+    #   :statsd      -> Backends::Statsd (host/port from configuration)
+    #   Backend inst -> returned unchanged
+    def resolve_backend(value)
+      case value
+      when nil
+        Backend::Null.new
+      when :prometheus
+        Backends::Prometheus.new
+      when :statsd
+        config = Pgbus.configuration
+        Backends::Statsd.new(host: config.statsd_host, port: config.statsd_port)
+      when Backend
+        value
+      else
+        raise ArgumentError,
+              "metrics_backend must be nil, :prometheus, :statsd, or a " \
+              "Pgbus::Metrics::Backend instance (got #{value.inspect})"
+      end
+    end
+  end
+end