pgbus 0.16.3 → 0.16.5

data/lib/pgbus/concurrency.rb

@@ -7,6 +7,10 @@ module Pgbus
     extend ActiveSupport::Concern
 
     METADATA_KEY = "pgbus_concurrency_key"
+    # How long a slot holder may go silent (no visibility heartbeat) before
+    # its slot is presumed dead. Also the limit/duration used to promote a
+    # parked job whose class no longer resolves.
+    DEFAULT_DURATION = 15 * 60
 
     class_methods do
       # Limit concurrent execution of jobs with the same key.
@@ -20,9 +24,12 @@ module Pgbus
       #                Default: the ENQUEUED job's class name (resolved at
       #                resolve time, so an inherited declaration keys each
       #                subclass separately — issue #357).
-      #   duration:    Safety expiry for semaphore (default: 15 minutes)
+      #   duration:    How long a running holder may go without a visibility
+      #                heartbeat before its slot is presumed dead and swept
+      #                (default: 15 minutes). Not a cap on run time: the
+      #                heartbeat keeps the semaphore alive while the job runs.
       #   on_conflict: What to do when limit is reached — :block, :discard, or :raise (default: :block)
-      def limits_concurrency(to:, key: nil, duration: 15 * 60, on_conflict: :block) # rubocop:disable Naming/MethodParameterName
+      def limits_concurrency(to:, key: nil, duration: DEFAULT_DURATION, on_conflict: :block) # rubocop:disable Naming/MethodParameterName
         raise ArgumentError, "to: must be a positive integer" unless to.is_a?(Integer) && to.positive?
         raise ArgumentError, "on_conflict must be :block, :discard, or :raise" unless %i[block discard raise].include?(on_conflict)
         raise ArgumentError, "duration must be a positive number" unless duration.is_a?(Numeric) && duration.positive?
@@ -71,6 +78,33 @@ module Pgbus
       def extract_key(payload)
         payload[METADATA_KEY]
       end
+
+      # Limit and duration for a job class. A class with no concurrency
+      # config — or one that no longer resolves — gets `limit: nil`, meaning
+      # "whatever limit the semaphore row already records". Forcing 1 here
+      # would refuse every promotion for a `to: 3` key that still holds two
+      # slots, so its parked jobs could never reach the executor (which is
+      # what dead-letters a missing class).
+      def config_for(job_class)
+        config = job_class.respond_to?(:pgbus_concurrency) && job_class.pgbus_concurrency
+        return { limit: nil, duration: DEFAULT_DURATION } unless config
+
+        { limit: config[:limit], duration: config[:duration] }
+      end
+
+      # A slot's lease is only renewed by the visibility heartbeat, which
+      # first beats one interval after the job starts. A `duration` shorter
+      # than that would expire before the first touch and let the sweep
+      # promote beside a running job, so it is floored at two intervals.
+      def effective_duration(duration, config: Pgbus.configuration)
+        [duration, config.effective_visibility_heartbeat_interval * 2].max
+      end
+
+      def config_for_payload(payload)
+        config_for(Object.const_get(payload["job_class"].to_s))
+      rescue NameError
+        config_for(nil)
+      end
     end
   end
 end

data/lib/pgbus/instrumentation.rb

@@ -29,6 +29,9 @@ module Pgbus
   #   pgbus.serializer.deserialize — job/event deserialization
   #   pgbus.batch_finished         — batch flipped to finished
   #                                  payload: batch_id, total_jobs, completed_jobs, failed_jobs
+  #   pgbus.blocked_execution_discarded — a parked job was discarded from the
+  #                                  dashboard; it will never run
+  #                                  payload: concurrency_key, job_class, job_id
   #   pgbus.batch_sweep            — dispatcher stalled-batch sweep
   #                                  payload: stalled_for, stale_executions, orphan_rows,
   #                                  started_batches, finished_batches

data/lib/pgbus/integrations/appsignal/dashboard.json

@@ -355,6 +355,44 @@
             "tags": []
           }
         ]
+      },
+      {
+        "type": "timeseries",
+        "display": "LINE",
+        "title": "Concurrency",
+        "description": "Jobs parked behind a concurrency key, how long the oldest has waited, and slots held across all keys.",
+        "line_label": "%name%",
+        "format": "number",
+        "draw_null_as_zero": true,
+        "metrics": [
+          {
+            "name": "pgbus_concurrency_blocked_executions",
+            "fields": [
+              {
+                "field": "gauge"
+              }
+            ],
+            "tags": []
+          },
+          {
+            "name": "pgbus_concurrency_blocked_oldest_age_seconds",
+            "fields": [
+              {
+                "field": "gauge"
+              }
+            ],
+            "tags": []
+          },
+          {
+            "name": "pgbus_concurrency_slots_held",
+            "fields": [
+              {
+                "field": "gauge"
+              }
+            ],
+            "tags": []
+          }
+        ]
       }
     ]
   }

data/lib/pgbus/integrations/appsignal/probe.rb

@@ -60,6 +60,11 @@ module Pgbus
           def call
             return unless data_source
 
+            # One Runner lives for the life of the process, so its DataSource
+            # does too — without this, every memoized read (queue metrics, the
+            # concurrency aggregates) would report the first minute's numbers
+            # forever.
+            data_source.reset_cache! if data_source.respond_to?(:reset_cache!)
             track_queues
             track_processes
             track_summary
@@ -116,6 +121,11 @@ module Pgbus
             gauge "total_dead_tuples", stats[:total_dead_tuples]
             gauge "tables_needing_vacuum", stats[:tables_needing_vacuum]
             gauge "oldest_transaction_age_seconds", stats[:oldest_transaction_age_sec]
+            # Unlabelled, same rationale as the /pgbus/api/metrics family:
+            # concurrency keys are per-record and would be unbounded as tags.
+            gauge "concurrency_blocked_executions", stats[:parked_total]
+            gauge "concurrency_blocked_oldest_age_seconds", stats[:oldest_parked_age_sec]
+            gauge "concurrency_slots_held", stats[:slots_held]
           rescue StandardError => e
             log_failure("summary metrics", e)
           end

data/lib/pgbus/mcp/base_tool.rb

@@ -36,8 +36,13 @@ module Pgbus
         # Pull the injected DataSource (or build a default one). Kept as a
         # class method because MCP tool entry points (`self.call`) are class
         # methods.
+        # The server injects ONE DataSource for the life of the process, so its
+        # per-request memos have to be dropped per tool call — otherwise every
+        # call after the first replays the first call's snapshot.
         def data_source_from(server_context)
-          (server_context && server_context[:data_source]) || Pgbus::Web::DataSource.new
+          data_source = (server_context && server_context[:data_source]) || Pgbus::Web::DataSource.new
+          data_source.reset_cache! if data_source.respond_to?(:reset_cache!)
+          data_source
         end
 
         # Whether payloads may be returned for this call. Honors a per-call

data/lib/pgbus/mcp/server.rb

@@ -23,6 +23,7 @@ module Pgbus
         Tools::DlqTool,
         Tools::DlqDetailTool,
         Tools::LocksTool,
+        Tools::ConcurrencyTool,
         Tools::ThroughputTool,
         Tools::StatsTool,
         Tools::RecurringTool
@@ -32,7 +33,7 @@ module Pgbus
         Read-only diagnostic tools for a pgbus (PostgreSQL/PGMQ) deployment.
         Start with pgbus_health for a one-call OK/DEGRADED/STALLED verdict,
         then drill in with pgbus_queues, pgbus_processes, pgbus_jobs,
-        pgbus_dlq, and pgbus_locks. No tool mutates state. Message payloads
+        pgbus_dlq, pgbus_locks, and pgbus_concurrency. No tool mutates state. Message payloads
         are redacted unless the server was started with payloads explicitly
         allowed.
       TEXT

data/lib/pgbus/mcp/tools/concurrency_tool.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module MCP
+    module Tools
+      # Reports concurrency-key pressure: how many jobs are parked behind a
+      # `limits_concurrency` key, how long the oldest has waited, how many slots
+      # are held, and the per-key detail behind those numbers. Maps to
+      # DataSource#concurrency_stats. Carries no job payloads — only key names,
+      # counts and lease state — so there is nothing to redact.
+      class ConcurrencyTool < BaseTool
+        tool_name "pgbus_concurrency"
+        title "Pgbus Concurrency"
+        description <<~DESC
+          Report concurrency keys and the jobs parked behind them: total parked
+          jobs, the oldest parked job's wait in seconds, slots held, and keys at
+          their limit — plus up to 100 key rows with value/limit, lease expiry,
+          whether the lease is still fresh, parked count and oldest wait. A key
+          with a stale lease and a parked backlog is a stuck pipeline: its holder
+          died before releasing the slot.
+        DESC
+
+        input_schema(properties: {}, required: [])
+
+        def self.call(server_context: nil)
+          data_source = data_source_from(server_context)
+          json_response(data_source.concurrency_stats)
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/mcp.rb

@@ -24,6 +24,7 @@ module Pgbus
       pgbus/mcp/tools/dlq_tool
       pgbus/mcp/tools/dlq_detail_tool
       pgbus/mcp/tools/locks_tool
+      pgbus/mcp/tools/concurrency_tool
       pgbus/mcp/tools/throughput_tool
       pgbus/mcp/tools/stats_tool
       pgbus/mcp/tools/recurring_tool

data/lib/pgbus/process/dispatcher.rb

@@ -355,25 +355,18 @@ module Pgbus
       end
 
       def cleanup_concurrency
-        expired_keys = Concurrency::Semaphore.expire_stale
-        expired_keys.each do |row|
-          release_blocked_for_key(row["key"])
-        end
+        expired = Concurrency::Semaphore.expire_stale
+        Pgbus.logger.debug { "[Pgbus] Swept #{expired.size} expired semaphores" } if expired.any?
 
-        orphaned = Concurrency::BlockedExecution.expire_stale
-        Pgbus.logger.debug { "[Pgbus] Expired #{orphaned} orphaned blocked executions" } if orphaned.positive?
+        # A parked job is never dropped: the sweep only promotes what can take
+        # a slot now — behind a swept semaphore or a promote that failed.
+        promoted = Concurrency::BlockedExecution.promote_pending(client: Pgbus.client)
+        Pgbus.logger.debug { "[Pgbus] Promoted #{promoted} blocked executions" } if promoted.positive?
       rescue StandardError => e
         log_maintenance_failure("Concurrency cleanup", e)
         e
       end
 
-      def release_blocked_for_key(key)
-        promoted = Concurrency::BlockedExecution.promote_next(key, client: Pgbus.client)
-        Pgbus.logger.debug { "[Pgbus] Released blocked execution for key: #{key}" } if promoted
-      rescue StandardError => e
-        log_maintenance_failure("Releasing blocked execution for #{key}", e)
-      end
-
       def cleanup_batches
         retention = config.batch_retention
         return unless retention&.positive?

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.16.3"
+  VERSION = "0.16.5"
 end

data/lib/pgbus/visibility_heartbeat.rb

@@ -24,8 +24,10 @@ module Pgbus
   # cadence with `config.visibility_heartbeat_interval`, or opt a job class
   # out with `pgbus_visibility_heartbeat false`.
   module VisibilityHeartbeat
+    # `concurrency` is `[key, duration]` for a concurrency-limited job, else
+    # nil — one member, so the struct stays inside an 80-byte slot.
     Entry = Struct.new(:client, :queue_name, :prefixed, :msg_id, :job_class, :extended_at, :extensions,
-                       keyword_init: true)
+                       :concurrency, keyword_init: true)
 
     # Per-job opt-out, included on ActiveJob::Base by the engine:
     #
@@ -55,11 +57,15 @@ module Pgbus
       # @param prefixed [Boolean] whether queue_name still needs the prefix
       # @param job_class [String, nil] for logging and instrumentation
       # @param config [Pgbus::Configuration]
-      def track(client:, queue_name:, msg_id:, prefixed: true, job_class: nil, config: Pgbus.configuration)
+      # @param concurrency [Array(String, Numeric), nil] semaphore key to keep alive alongside
+      #   the message and how far to push its expiry on each beat
+      def track(client:, queue_name:, msg_id:, prefixed: true, job_class: nil, config: Pgbus.configuration,
+                concurrency: nil)
         return yield unless config.visibility_heartbeat
 
         entry = Entry.new(client: client, queue_name: queue_name, prefixed: prefixed, msg_id: msg_id.to_i,
-                          job_class: job_class, extended_at: monotonic_now, extensions: 0)
+                          job_class: job_class, extended_at: monotonic_now, extensions: 0,
+                          concurrency: concurrency)
         register(entry, config)
         begin
           yield
@@ -123,6 +129,7 @@ module Pgbus
         entry.client.set_visibility_timeout(entry.queue_name, entry.msg_id, vt: vt, prefixed: entry.prefixed)
         entry.extended_at = now
         entry.extensions += 1
+        touch_semaphore(entry)
         Instrumentation.instrument(
           "pgbus.job_visibility_extended",
           queue: entry.queue_name, job_class: entry.job_class, msg_id: entry.msg_id, vt: vt,
@@ -165,6 +172,20 @@ module Pgbus
 
       # Entries registered before a fork belong to the parent's jobs; the
       # thread did not survive the fork either.
+      # A live holder keeps its semaphore from being swept, so `duration`
+      # bounds heartbeat silence rather than run time. Its own rescue: a
+      # failed touch must not cost the message its visibility extension.
+      def touch_semaphore(entry)
+        key, duration = entry.concurrency
+        return unless key
+
+        Concurrency::Semaphore.touch(key, duration || Concurrency::DEFAULT_DURATION)
+      rescue StandardError => e
+        Pgbus.logger.warn do
+          "[Pgbus::VisibilityHeartbeat] could not touch semaphore #{key}: #{e.class}: #{e.message}"
+        end
+      end
+
       def forget_parent_entries!
         return if @pid == ::Process.pid
 

data/lib/pgbus/web/data_source.rb

@@ -5,6 +5,11 @@ require "time"
 module Pgbus
   module Web
     class DataSource
+      # Ceiling on how many parked jobs one dashboard release promotes, so a
+      # key with thousands parked cannot hold the request open. The dispatcher
+      # sweep picks up whatever is left on its next pass.
+      PROMOTE_CAP = 100
+
       def initialize(client: Pgbus.client)
         @client = client
         @last_throughput_snapshot = nil
@@ -35,7 +40,18 @@ module Pgbus
           total_dead_tuples: health[:total_dead_tuples],
           tables_needing_vacuum: health[:tables_needing_vacuum],
           oldest_transaction_age_sec: health[:oldest_transaction_age_sec]
-        }
+        }.merge(concurrency_summary)
+      end
+
+      # Drop the per-instance memos. The memos assume one instance per web
+      # request; the AppSignal probe and the MCP server both hold ONE DataSource
+      # for the life of the process, so without this their gauges and tool
+      # responses would freeze at the first read. Both call this per iteration
+      # / per tool call.
+      def reset_cache!
+        @queues_with_metrics = nil
+        @concurrency_summary = nil
+        self
       end
 
       # Queues — query via ActiveRecord for reliability in web processes
@@ -667,6 +683,83 @@ module Pgbus
         []
       end
 
+      # Concurrency keys — the `limits_concurrency` slot table and the jobs
+      # parked behind it. Aggregate numbers plus up to 100 key rows, busiest
+      # first. The two tables are joined FULL OUTER, not LEFT: a key can have
+      # parked rows and no semaphore (its holder died and the sweep removed the
+      # row) or a semaphore and nothing parked — an operator needs to see both.
+      def concurrency_stats
+        concurrency_summary.merge(keys: concurrency_keys)
+      end
+
+      # Drop a key's semaphore row and promote whatever can now run.
+      #
+      # Promotion goes through Concurrency::BlockedExecution.promote_next, which
+      # takes each slot through the same guarded upsert an enqueue uses — a
+      # dashboard release can no more over-admit than an enqueue can. Re-sending
+      # the parked payloads directly would reintroduce exactly the over-admission
+      # issue #460 closed. Returns the number of jobs promoted.
+      def release_concurrency_key(key)
+        return 0 if key.to_s.strip.empty?
+
+        # Zero the held slots rather than DELETE the row: the row is where the
+        # key's limit is recorded, and `Semaphore.acquire!` falls back to a
+        # limit of 1 on a fresh row. Deleting it would silently demote a
+        # `to: 3` key to 1 whenever the parked job's class no longer resolves
+        # (the payload then carries no limit for promotion to use). A row left
+        # at 0 is reaped by the dispatcher's `Semaphore.expired` sweep.
+        Pgbus::Semaphore.where(key: key).update_all(value: 0)
+        promoted = 0
+        promoted += 1 while promoted < PROMOTE_CAP &&
+                            Concurrency::BlockedExecution.promote_next(key, client: @client)
+
+        # Nothing took the freed slot, so mark the empty row expired: the
+        # dispatcher's `Concurrency::Semaphore.expire_stale` matches on
+        # expires_at ALONE, so a zeroed row would otherwise sit on the Locks
+        # page as a phantom 0/N key until its original lease ran out — up to
+        # the key's whole `duration`. Guarded on value = 0 so a row a promotion
+        # just filled is never expired out from under its holder.
+        Pgbus::Semaphore.where(key: key, value: 0).update_all(expires_at: Time.current)
+        promoted
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error releasing concurrency key #{key}: #{e.message}" }
+        0
+      end
+
+      # Drop every job parked behind a key. The jobs never run, so the
+      # bookkeeping they would have resolved has to be resolved here: a parked
+      # batch child is marked failed (otherwise its batch waits forever and
+      # on_failure never fires — issue #413) and an :until_executed uniqueness
+      # lock is released (otherwise it is orphaned, since no executor will ever
+      # release it — issue #423). Returns the number of jobs discarded.
+      def discard_parked_jobs(key)
+        return 0 if key.to_s.strip.empty?
+
+        rows = []
+        Pgbus::BlockedExecution.transaction do
+          # SKIP LOCKED so this can never race a concurrent promote, which
+          # deletes its row under the same lock before enqueueing it.
+          rows = Pgbus::BlockedExecution.for_key(key).lock("FOR UPDATE SKIP LOCKED").to_a
+          Pgbus::BlockedExecution.where(id: rows.map(&:id)).delete_all if rows.any?
+        end
+
+        # Cleanup runs after the claim commits, because the SKIP LOCKED claim
+        # only holds inside the transaction. The residual: a process killed
+        # between the commit and the cleanup leaves a resolved-nothing orphan
+        # that Batch::Sweep un-counts rather than fails. Running the cleanup
+        # inside the transaction would swap that for the failure this codebase
+        # has already judged worse (Concurrency::BlockedExecution#backfill) —
+        # `Batch.job_discarded` can enqueue a batch callback, and a callback
+        # fired for a batch that then rolls back is not recoverable, while an
+        # orphan row is. Each row is cleaned under its own rescue, so one bad
+        # payload never costs the rest.
+        rows.each { |row| cleanup_discarded_parked_job(row) }
+        rows.size
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error discarding parked jobs for #{key}: #{e.message}" }
+        0
+      end
+
       # Batches
       def batches(limit: 100)
         records = BatchEntry.order(created_at: :desc).limit(limit).to_a
@@ -1026,6 +1119,122 @@ module Pgbus
         Pgbus::BusRecord.connection
       end
 
+      # Aggregate concurrency numbers, memoized for the lifetime of this
+      # instance: summary_stats, the metrics serializer and the AppSignal probe
+      # all read them on the same request, and concurrency_stats merges them in
+      # alongside the key rows. Mutations redirect to a fresh request with a new
+      # instance, so the memo never serves stale numbers.
+      def concurrency_summary
+        @concurrency_summary ||= fetch_concurrency_summary
+      end
+
+      def fetch_concurrency_summary
+        row = connection.select_all(<<~SQL, "Pgbus Concurrency Summary").to_a.first || {}
+          SELECT
+            (SELECT COUNT(*) FROM pgbus_blocked_executions) AS parked_total,
+            (SELECT EXTRACT(EPOCH FROM (now() - MIN(created_at)))::bigint
+               FROM pgbus_blocked_executions) AS oldest_parked_age_sec,
+            (SELECT COALESCE(SUM(value), 0) FROM pgbus_semaphores) AS slots_held,
+            (SELECT COUNT(*) FROM pgbus_semaphores WHERE value >= max_value) AS keys_at_limit
+        SQL
+
+        {
+          parked_total: row["parked_total"].to_i,
+          oldest_parked_age_sec: row["oldest_parked_age_sec"]&.to_i,
+          slots_held: row["slots_held"].to_i,
+          keys_at_limit: row["keys_at_limit"].to_i
+        }
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching concurrency summary: #{e.message}" }
+        { parked_total: 0, oldest_parked_age_sec: nil, slots_held: 0, keys_at_limit: 0 }
+      end
+
+      # `lease_fresh` is the one fact an operator needs before releasing a key:
+      # a live lease means a holder is probably still running, so releasing lets
+      # another job start beside it.
+      def concurrency_keys(limit: 100)
+        rows = connection.select_all(<<~SQL, "Pgbus Concurrency Keys")
+          SELECT COALESCE(s.key, b.concurrency_key) AS key,
+                 s.value AS value,
+                 s.max_value AS max_value,
+                 s.expires_at AS expires_at,
+                 (s.value > 0 AND s.expires_at > now()) AS lease_fresh,
+                 COALESCE(b.parked_count, 0) AS parked_count,
+                 EXTRACT(EPOCH FROM (now() - b.oldest_parked_at))::bigint AS oldest_parked_age_sec
+          FROM pgbus_semaphores s
+          FULL OUTER JOIN (
+            SELECT concurrency_key, COUNT(*) AS parked_count, MIN(created_at) AS oldest_parked_at
+            FROM pgbus_blocked_executions
+            GROUP BY concurrency_key
+          ) b ON s.key = b.concurrency_key
+          ORDER BY COALESCE(b.parked_count, 0) DESC, s.expires_at ASC NULLS LAST
+          LIMIT #{limit.to_i}
+        SQL
+
+        rows.to_a.map { |row| format_concurrency_key(row) }
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching concurrency keys: #{e.message}" }
+        []
+      end
+
+      def format_concurrency_key(row)
+        {
+          key: row["key"],
+          value: row["value"]&.to_i,
+          max_value: row["max_value"]&.to_i,
+          expires_at: row["expires_at"],
+          lease_fresh: [true, "t"].include?(row["lease_fresh"]),
+          parked_count: row["parked_count"].to_i,
+          oldest_parked_age_sec: row["oldest_parked_age_sec"]&.to_i
+        }
+      end
+
+      # Resolve the bookkeeping a discarded parked job will never resolve
+      # itself. Per-row rather than batched: one bad payload must not stop the
+      # rest from being cleaned up, and the rows are already deleted.
+      def cleanup_discarded_parked_job(row)
+        payload = decode_parked_payload(row.payload)
+        batch_id = payload[Batch::METADATA_KEY]
+        Batch.job_discarded(batch_id, job_id: payload["job_id"]) if batch_id
+
+        release_parked_uniqueness_lock(payload, row.created_at)
+
+        Pgbus.logger.warn do
+          "[Pgbus::Web] Discarded parked job #{payload["job_class"]} (#{payload["job_id"]}) " \
+            "for concurrency key #{row.concurrency_key}"
+        end
+        Instrumentation.instrument(
+          "pgbus.blocked_execution_discarded",
+          concurrency_key: row.concurrency_key,
+          job_class: payload["job_class"],
+          job_id: payload["job_id"]
+        )
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus::Web] Parked job cleanup failed: #{e.class}: #{e.message}" }
+      end
+
+      # release_if_unbound!, not release_lock: a parked job's lock is unbound (it
+      # never got a msg_id). If the reaper already removed it and a successor
+      # took the same key, that successor's lock must survive — msg_id = 0
+      # excludes one that was sent, and the parked row's own created_at excludes
+      # one acquired after this row was parked.
+      def release_parked_uniqueness_lock(payload, parked_at)
+        key = payload[Uniqueness::METADATA_KEY]
+        return unless key && payload[Uniqueness::STRATEGY_KEY].to_s == "until_executed"
+
+        UniquenessKey.release_if_unbound!(key, acquired_before: parked_at)
+      end
+
+      # A row parked before the double-encoding fix stores a jsonb *string*
+      # holding the document — the same two-pass unwrap BlockedExecution
+      # .release_next! does.
+      def decode_parked_payload(payload)
+        2.times { payload = JSON.parse(payload) if payload.is_a?(String) }
+        payload.is_a?(Hash) ? payload : {}
+      rescue JSON::ParserError
+        {}
+      end
+
       # Single query to fetch pg_stat_user_tables stats for all queue and
       # archive tables. Avoids 2*N catalog queries on the dashboard.
       def fetch_all_table_stats

data/lib/pgbus/web/metrics_serializer.rb

@@ -22,9 +22,14 @@ module Pgbus
         append_queue_metrics(lines)
         append_job_metrics(lines)
         append_process_metrics(lines)
-        append_summary_metrics(lines)
+        # One summary read, shared: summary_stats is not memoized, so calling it
+        # per family would repeat the queue/health/process queries and advance
+        # the throughput snapshot a second time within one scrape.
+        summary = summary_stats
+        append_summary_metrics(lines, summary)
         append_stream_metrics(lines)
         append_health_metrics(lines)
+        append_concurrency_metrics(lines, summary)
         "#{lines.join("\n")}\n"
       end
 
@@ -147,8 +152,18 @@ module Pgbus
         Pgbus.logger.debug { "[Pgbus::Metrics] Error serializing process metrics: #{e.message}" }
       end
 
-      def append_summary_metrics(lines)
-        stats = @data_source.summary_stats
+      # nil when the read raised — each family then skips rather than blanking
+      # the whole scrape.
+      def summary_stats
+        @data_source.summary_stats
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Metrics] Error reading summary stats: #{e.message}" }
+        nil
+      end
+
+      def append_summary_metrics(lines, stats)
+        return unless stats
+
         gauge(lines, "pgbus_failed_events_total", "Total failed events") do
           [[stats[:failed_count]]]
         end
@@ -219,6 +234,39 @@ module Pgbus
         Pgbus.logger.debug { "[Pgbus::Metrics] Error serializing health metrics: #{e.message}" }
       end
 
+      # Concurrency slot pressure: how many jobs are parked, how long the oldest
+      # has waited, how many slots are held right now. Unlabelled on purpose —
+      # concurrency keys are per-record (one per order, one per sync group), so a
+      # per-key label would make an unbounded series. The per-key detail lives on
+      # the Locks page and in the pgbus_concurrency MCP tool, both bounded.
+      #
+      # Omitted entirely when neither table holds anything: an install that does
+      # not use limits_concurrency reports nothing rather than a flat zero line.
+      def append_concurrency_metrics(lines, stats)
+        return unless stats
+        return if stats[:parked_total].to_i.zero? && stats[:slots_held].to_i.zero? &&
+                  stats[:keys_at_limit].to_i.zero? && stats[:oldest_parked_age_sec].nil?
+
+        gauge(lines, "pgbus_concurrency_blocked_executions",
+              "Jobs parked behind a concurrency key, waiting for a slot") do
+          [[stats[:parked_total].to_i]]
+        end
+
+        if stats[:oldest_parked_age_sec]
+          gauge(lines, "pgbus_concurrency_blocked_oldest_age_seconds",
+                "How long the longest-waiting parked job has waited") do
+            [[stats[:oldest_parked_age_sec]]]
+          end
+        end
+
+        gauge(lines, "pgbus_concurrency_slots_held",
+              "Concurrency slots currently held across all keys") do
+          [[stats[:slots_held].to_i]]
+        end
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Metrics] Error serializing concurrency metrics: #{e.message}" }
+      end
+
       # Emits a Prometheus gauge metric family. The block must return an array
       # of [value] or [value, { label: "val" }] pairs.
       def gauge(lines, name, help)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.16.3
+  version: 0.16.5
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -185,6 +185,8 @@ files:
 - app/views/pgbus/jobs/_failed_table.html.erb
 - app/views/pgbus/jobs/index.html.erb
 - app/views/pgbus/jobs/show.html.erb
+- app/views/pgbus/locks/_concurrency.html.erb
+- app/views/pgbus/locks/_uniqueness.html.erb
 - app/views/pgbus/locks/index.html.erb
 - app/views/pgbus/outbox/index.html.erb
 - app/views/pgbus/processes/_processes_table.html.erb
@@ -317,6 +319,7 @@ files:
 - lib/pgbus/mcp/redactor.rb
 - lib/pgbus/mcp/runner.rb
 - lib/pgbus/mcp/server.rb
+- lib/pgbus/mcp/tools/concurrency_tool.rb
 - lib/pgbus/mcp/tools/dlq_detail_tool.rb
 - lib/pgbus/mcp/tools/dlq_tool.rb
 - lib/pgbus/mcp/tools/health_tool.rb