pgbus 0.9.5 → 0.9.6

data/lib/pgbus/integrations/appsignal/dashboards/pgbus_streams.json

@@ -1,65 +1,156 @@
 {
-  "title": "Pgbus — Streams",
-  "description": "Real-time SSE pub/sub. Broadcasts, fanout, active connections, and the outbox/recurring scheduler.",
-  "graphs": [
-    {
-      "title": "Stream broadcasts per minute",
-      "line_label": "%stream %deferred",
-      "format": "number",
-      "kind": "timeseries",
-      "draw_null_as_zero": true,
-      "metrics": [
-        { "name": "pgbus_stream_broadcast_count", "fields": ["COUNTER"], "tags": [] }
-      ]
-    },
-    {
-      "title": "Active SSE connections",
-      "description": "Estimated from connect/disconnect events in the last 60 minutes. Use as a rough capacity gauge — exact count requires the SSE process telemetry.",
-      "format": "number",
-      "kind": "timeseries",
-      "metrics": [
-        { "name": "pgbus_stream_active_connections", "fields": ["GAUGE"], "tags": [] }
-      ]
-    },
-    {
-      "title": "Average fanout per broadcast",
-      "description": "Mean number of connections that received each broadcast over the last hour.",
-      "format": "number",
-      "kind": "timeseries",
-      "metrics": [
-        { "name": "pgbus_stream_avg_fanout", "fields": ["GAUGE"], "tags": [] }
-      ]
-    },
-    {
-      "title": "Broadcast payload size (bytes)",
-      "description": "Distribution of payload bytes. Use to spot accidentally-streaming-an-entire-page bugs.",
-      "line_label": "%stream",
-      "format": "size",
-      "format_input": "byte",
-      "kind": "timeseries",
-      "metrics": [
-        { "name": "pgbus_stream_broadcast_bytes", "fields": ["MEAN", "P95"], "tags": [] }
-      ]
-    },
-    {
-      "title": "Outbox publishes per minute",
-      "line_label": "%kind",
-      "format": "number",
-      "kind": "timeseries",
-      "draw_null_as_zero": true,
-      "metrics": [
-        { "name": "pgbus_outbox_published", "fields": ["COUNTER"], "tags": [] }
-      ]
-    },
-    {
-      "title": "Recurring tasks enqueued per minute",
-      "line_label": "%task",
-      "format": "number",
-      "kind": "timeseries",
-      "draw_null_as_zero": true,
-      "metrics": [
-        { "name": "pgbus_recurring_enqueued", "fields": ["COUNTER"], "tags": [] }
-      ]
-    }
-  ]
+  "metric_keys": [
+    "pgbus_stream_broadcast_count"
+  ],
+  "dashboard": {
+    "title": "Pgbus — Streams",
+    "description": "Real-time SSE pub/sub. Broadcasts, fanout, active connections,\nand the outbox/recurring scheduler.\n",
+    "visuals": [
+      {
+        "type": "timeseries",
+        "display": "LINE",
+        "title": "Stream broadcasts per minute",
+        "line_label": "%stream% - %deferred%",
+        "format": "number",
+        "draw_null_as_zero": true,
+        "metrics": [
+          {
+            "name": "pgbus_stream_broadcast_count",
+            "fields": [
+              {
+                "field": "COUNTER"
+              }
+            ],
+            "tags": [
+              {
+                "key": "stream",
+                "value": "*"
+              },
+              {
+                "key": "deferred",
+                "value": "*"
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "type": "timeseries",
+        "display": "LINE",
+        "title": "Active SSE connections",
+        "description": "Estimated from connect/disconnect events in the last 60 minutes.",
+        "line_label": "Active connections",
+        "format": "number",
+        "draw_null_as_zero": true,
+        "metrics": [
+          {
+            "name": "pgbus_stream_active_connections",
+            "fields": [
+              {
+                "field": "GAUGE"
+              }
+            ],
+            "tags": []
+          }
+        ]
+      },
+      {
+        "type": "timeseries",
+        "display": "LINE",
+        "title": "Average fanout per broadcast",
+        "description": "Mean number of connections that received each broadcast over the last hour.",
+        "line_label": "Avg fanout",
+        "format": "number",
+        "draw_null_as_zero": false,
+        "metrics": [
+          {
+            "name": "pgbus_stream_avg_fanout",
+            "fields": [
+              {
+                "field": "GAUGE"
+              }
+            ],
+            "tags": []
+          }
+        ]
+      },
+      {
+        "type": "timeseries",
+        "display": "LINE",
+        "title": "Broadcast payload size (bytes)",
+        "description": "Distribution of payload bytes. Use to spot accidentally-streaming-an-entire-page bugs.",
+        "line_label": "%stream%",
+        "format": "size",
+        "format_input": "byte",
+        "draw_null_as_zero": false,
+        "metrics": [
+          {
+            "name": "pgbus_stream_broadcast_bytes",
+            "fields": [
+              {
+                "field": "MEAN"
+              },
+              {
+                "field": "P95"
+              }
+            ],
+            "tags": [
+              {
+                "key": "stream",
+                "value": "*"
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "type": "timeseries",
+        "display": "LINE",
+        "title": "Outbox publishes per minute",
+        "line_label": "%kind%",
+        "format": "number",
+        "draw_null_as_zero": true,
+        "metrics": [
+          {
+            "name": "pgbus_outbox_published",
+            "fields": [
+              {
+                "field": "COUNTER"
+              }
+            ],
+            "tags": [
+              {
+                "key": "kind",
+                "value": "*"
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "type": "timeseries",
+        "display": "LINE",
+        "title": "Recurring tasks enqueued per minute",
+        "line_label": "%task%",
+        "format": "number",
+        "draw_null_as_zero": true,
+        "metrics": [
+          {
+            "name": "pgbus_recurring_enqueued",
+            "fields": [
+              {
+                "field": "COUNTER"
+              }
+            ],
+            "tags": [
+              {
+                "key": "task",
+                "value": "*"
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  }
 }

data/lib/pgbus/integrations/appsignal/dashboards/pgbus_throughput.json

@@ -1,81 +1,220 @@
 {
-  "title": "Pgbus — Throughput & Latency",
-  "description": "Job and event throughput, perform-duration percentiles, and PGMQ send/read counts. Drives the most common 'is the worker keeping up?' question.",
-  "graphs": [
-    {
-      "title": "Jobs processed per minute",
-      "description": "Successful, failed, and dead-lettered jobs. A spike in failed without a spike in processed usually means a deploy regression.",
-      "line_label": "%queue %job_class %status",
-      "format": "number",
-      "format_input": null,
-      "kind": "timeseries",
-      "draw_null_as_zero": true,
-      "metrics": [
-        { "name": "pgbus_queue_job_count", "fields": ["COUNTER"], "tags": [] }
-      ]
-    },
-    {
-      "title": "Job perform duration (ms)",
-      "description": "Distribution of how long perform_now takes per job class. P95 and P99 are the lines to watch.",
-      "line_label": "%job_class",
-      "format": "duration",
-      "format_input": "millisecond",
-      "kind": "timeseries",
-      "metrics": [
-        { "name": "pgbus_job_duration_ms", "fields": ["MEAN", "P95", "P99"], "tags": [] }
-      ]
-    },
-    {
-      "title": "Events processed per minute",
-      "description": "Event-bus handler invocations grouped by routing key.",
-      "line_label": "%routing_key %handler %status",
-      "format": "number",
-      "kind": "timeseries",
-      "draw_null_as_zero": true,
-      "metrics": [
-        { "name": "pgbus_event_count", "fields": ["COUNTER"], "tags": [] }
-      ]
-    },
-    {
-      "title": "Event handler duration (ms)",
-      "line_label": "%handler",
-      "format": "duration",
-      "format_input": "millisecond",
-      "kind": "timeseries",
-      "metrics": [
-        { "name": "pgbus_event_duration_ms", "fields": ["MEAN", "P95", "P99"], "tags": [] }
-      ]
-    },
-    {
-      "title": "PGMQ messages sent (per minute)",
-      "line_label": "%queue",
-      "format": "number",
-      "kind": "timeseries",
-      "draw_null_as_zero": true,
-      "metrics": [
-        { "name": "pgbus_messages_sent", "fields": ["COUNTER"], "tags": [] }
-      ]
-    },
-    {
-      "title": "Send duration (ms)",
-      "line_label": "%queue",
-      "format": "duration",
-      "format_input": "millisecond",
-      "kind": "timeseries",
-      "metrics": [
-        { "name": "pgbus_send_duration_ms", "fields": ["MEAN", "P95", "P99"], "tags": [] }
-      ]
-    },
-    {
-      "title": "PGMQ messages read (per minute)",
-      "description": "Messages fetched from queues by workers. Compare against 'sent' to spot backlog growth.",
-      "line_label": "%queue",
-      "format": "number",
-      "kind": "timeseries",
-      "draw_null_as_zero": true,
-      "metrics": [
-        { "name": "pgbus_messages_read", "fields": ["COUNTER"], "tags": [] }
-      ]
-    }
-  ]
+  "metric_keys": [
+    "pgbus_queue_job_count",
+    "pgbus_event_count",
+    "pgbus_messages_sent",
+    "pgbus_messages_read"
+  ],
+  "dashboard": {
+    "title": "Pgbus — Throughput & Latency",
+    "description": "Job and event throughput, perform-duration percentiles, and PGMQ send/read counts.\nDrives the most common 'is the worker keeping up?' question.\n",
+    "visuals": [
+      {
+        "type": "timeseries",
+        "display": "LINE",
+        "title": "Jobs processed per minute",
+        "description": "Successful, failed, and dead-lettered jobs. A spike in failed without a spike in processed usually means a deploy regression.",
+        "line_label": "%queue% - %job_class% - %status%",
+        "format": "number",
+        "draw_null_as_zero": true,
+        "metrics": [
+          {
+            "name": "pgbus_queue_job_count",
+            "fields": [
+              {
+                "field": "COUNTER"
+              }
+            ],
+            "tags": [
+              {
+                "key": "queue",
+                "value": "*"
+              },
+              {
+                "key": "job_class",
+                "value": "*"
+              },
+              {
+                "key": "status",
+                "value": "*"
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "type": "timeseries",
+        "display": "LINE",
+        "title": "Job perform duration (ms)",
+        "description": "Distribution of how long perform_now takes per job class. P95 and P99 are the lines to watch.",
+        "line_label": "%job_class%",
+        "format": "duration",
+        "draw_null_as_zero": false,
+        "metrics": [
+          {
+            "name": "pgbus_job_duration_ms",
+            "fields": [
+              {
+                "field": "MEAN"
+              },
+              {
+                "field": "P95"
+              },
+              {
+                "field": "P99"
+              }
+            ],
+            "tags": [
+              {
+                "key": "job_class",
+                "value": "*"
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "type": "timeseries",
+        "display": "LINE",
+        "title": "Events processed per minute",
+        "description": "Event-bus handler invocations grouped by routing key.",
+        "line_label": "%routing_key% - %handler% - %status%",
+        "format": "number",
+        "draw_null_as_zero": true,
+        "metrics": [
+          {
+            "name": "pgbus_event_count",
+            "fields": [
+              {
+                "field": "COUNTER"
+              }
+            ],
+            "tags": [
+              {
+                "key": "routing_key",
+                "value": "*"
+              },
+              {
+                "key": "handler",
+                "value": "*"
+              },
+              {
+                "key": "status",
+                "value": "*"
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "type": "timeseries",
+        "display": "LINE",
+        "title": "Event handler duration (ms)",
+        "line_label": "%handler%",
+        "format": "duration",
+        "draw_null_as_zero": false,
+        "metrics": [
+          {
+            "name": "pgbus_event_duration_ms",
+            "fields": [
+              {
+                "field": "MEAN"
+              },
+              {
+                "field": "P95"
+              },
+              {
+                "field": "P99"
+              }
+            ],
+            "tags": [
+              {
+                "key": "handler",
+                "value": "*"
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "type": "timeseries",
+        "display": "LINE",
+        "title": "PGMQ messages sent (per minute)",
+        "line_label": "%queue%",
+        "format": "number",
+        "draw_null_as_zero": true,
+        "metrics": [
+          {
+            "name": "pgbus_messages_sent",
+            "fields": [
+              {
+                "field": "COUNTER"
+              }
+            ],
+            "tags": [
+              {
+                "key": "queue",
+                "value": "*"
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "type": "timeseries",
+        "display": "LINE",
+        "title": "Send duration (ms)",
+        "line_label": "%queue%",
+        "format": "duration",
+        "draw_null_as_zero": false,
+        "metrics": [
+          {
+            "name": "pgbus_send_duration_ms",
+            "fields": [
+              {
+                "field": "MEAN"
+              },
+              {
+                "field": "P95"
+              },
+              {
+                "field": "P99"
+              }
+            ],
+            "tags": [
+              {
+                "key": "queue",
+                "value": "*"
+              }
+            ]
+          }
+        ]
+      },
+      {
+        "type": "timeseries",
+        "display": "LINE",
+        "title": "PGMQ messages read (per minute)",
+        "description": "Messages fetched from queues by workers. Compare against 'sent' to spot backlog growth.",
+        "line_label": "%queue%",
+        "format": "number",
+        "draw_null_as_zero": true,
+        "metrics": [
+          {
+            "name": "pgbus_messages_read",
+            "fields": [
+              {
+                "field": "COUNTER"
+              }
+            ],
+            "tags": [
+              {
+                "key": "queue",
+                "value": "*"
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  }
 }

data/lib/pgbus/integrations/appsignal/probe.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "socket"
+
 module Pgbus
   module Integrations
     module Appsignal
@@ -10,6 +12,14 @@ module Pgbus
       # method rescues StandardError and returns a safe default — but we
       # still wrap each section in our own rescue so a probe iteration
       # never raises out into the AppSignal probe runner.
+      #
+      # Tagging policy: most pgbus metrics are cluster-wide (the queue
+      # depth in PostgreSQL is the same regardless of which host reads it),
+      # so cluster-wide gauges are emitted WITHOUT a hostname tag — every
+      # host sends the same value and AppSignal's last-write-wins
+      # semantics keep the dashboard correct. The only gauge that is
+      # genuinely per-host is `active_processes`, which the probe filters
+      # to this host before tagging.
       module Probe
         METRIC_PREFIX = "pgbus_"
         private_constant :METRIC_PREFIX
@@ -43,6 +53,7 @@ module Pgbus
         class Runner
           def initialize(data_source: nil)
             @data_source = data_source
+            @hostname = Socket.gethostname
           end
 
           def call
@@ -68,14 +79,18 @@ module Pgbus
               gauge "queue_visible_depth", q[:queue_visible_length], tags
               gauge "queue_paused", q[:paused] ? 1 : 0, tags
               age = q[:oldest_msg_age_sec]
-              gauge "queue_oldest_message_age_seconds", age, tags if age
+              if age
+                gauge "queue_oldest_message_age_seconds", age, tags
+                gauge "queue_latency", age * 1_000, tags
+              end
             end
           rescue StandardError => e
             log_failure("queue metrics", e)
           end
 
           def track_processes
-            gauge "active_processes", data_source.processes.count
+            local_count = data_source.processes.count { |p| p[:hostname] == @hostname }
+            gauge "active_processes", local_count, { hostname: @hostname }
           rescue StandardError => e
             log_failure("process metrics", e)
           end

data/lib/pgbus/integrations/appsignal.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "json"
 require "pgbus/integrations/appsignal/subscriber"
 require "pgbus/integrations/appsignal/probe"
 
@@ -24,6 +25,9 @@ module Pgbus
     # All metric names are prefixed `pgbus_` so they group cleanly in
     # AppSignal's custom-metrics view.
     module Appsignal
+      DASHBOARD_PATH = File.expand_path("appsignal/dashboard.json", __dir__).freeze
+      DASHBOARDS_DIR = File.expand_path("appsignal/dashboards", __dir__).freeze
+
       module_function
 
       def install! # rubocop:disable Naming/PredicateMethod
@@ -41,11 +45,24 @@ module Pgbus
         @installed == true
       end
 
+      def dashboard_definition
+        @dashboard_definition ||= JSON.parse(File.read(DASHBOARD_PATH))
+      end
+
+      def dashboard_definitions
+        @dashboard_definitions ||=
+          Dir[File.join(DASHBOARDS_DIR, "*.json")].map do |path|
+            JSON.parse(File.read(path))
+          end
+      end
+
       # Test hook: tear everything down so a fresh install! can run.
       def reset!
         Subscriber.reset! if defined?(Subscriber)
         Probe.reset! if defined?(Probe)
         @installed = false
+        @dashboard_definition = nil
+        @dashboard_definitions = nil
       end
     end
   end

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.9.5"
+  VERSION = "0.9.6"
 end

data/lib/pgbus/web/data_source.rb

@@ -41,7 +41,17 @@ module Pgbus
       # Queues — query via ActiveRecord for reliability in web processes
       # (avoids PGMQ client connection issues when the web server uses a
       # different connection lifecycle than the worker processes).
+      # Memoized for the lifetime of this data-source instance (one per web
+      # request — see ApplicationController#data_source). A single page can ask
+      # for queue metrics more than once (e.g. the DLQ page reads both the
+      # message rows and the total count); without the memo each call re-runs
+      # the meta query + batched metrics. Mutations redirect to a fresh request
+      # with a new instance, so a per-request memo never serves stale data.
       def queues_with_metrics
+        @queues_with_metrics ||= fetch_queues_with_metrics
+      end
+
+      def fetch_queues_with_metrics
         queue_names = connection.select_values("SELECT queue_name FROM pgmq.meta ORDER BY queue_name")
         return [] if queue_names.empty?
 
@@ -53,6 +63,7 @@ module Pgbus
         Pgbus.logger.error { "[Pgbus::Web] Error fetching queue metrics: #{e.class}: #{e.message}" }
         []
       end
+      private :fetch_queues_with_metrics
 
       # name is the full PGMQ queue name (e.g. "pgbus_default") as returned
       # by queues_with_metrics. No prefix is added.
@@ -276,6 +287,16 @@ module Pgbus
         []
       end
 
+      def dlq_total_count
+        dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
+        queues_with_metrics
+          .select { |q| q[:name].end_with?(dlq_suffix) }
+          .sum { |q| q[:queue_length] }
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching DLQ count: #{e.message}" }
+        0
+      end
+
       def dlq_message_detail(msg_id)
         dlq_suffix = Pgbus::DEAD_LETTER_SUFFIX
         queues = queues_with_metrics.select { |q| q[:name].end_with?(dlq_suffix) }