pgbus 0.16.4 → 0.16.6

data/lib/pgbus/web/data_source.rb

@@ -5,6 +5,11 @@ require "time"
 module Pgbus
   module Web
     class DataSource
+      # Ceiling on how many parked jobs one dashboard release promotes, so a
+      # key with thousands parked cannot hold the request open. The dispatcher
+      # sweep picks up whatever is left on its next pass.
+      PROMOTE_CAP = 100
+
       def initialize(client: Pgbus.client)
         @client = client
         @last_throughput_snapshot = nil
@@ -35,7 +40,18 @@ module Pgbus
           total_dead_tuples: health[:total_dead_tuples],
           tables_needing_vacuum: health[:tables_needing_vacuum],
           oldest_transaction_age_sec: health[:oldest_transaction_age_sec]
-        }
+        }.merge(concurrency_summary)
+      end
+
+      # Drop the per-instance memos. The memos assume one instance per web
+      # request; the AppSignal probe and the MCP server both hold ONE DataSource
+      # for the life of the process, so without this their gauges and tool
+      # responses would freeze at the first read. Both call this per iteration
+      # / per tool call.
+      def reset_cache!
+        @queues_with_metrics = nil
+        @concurrency_summary = nil
+        self
       end
 
       # Queues — query via ActiveRecord for reliability in web processes
@@ -667,6 +683,83 @@ module Pgbus
         []
       end
 
+      # Concurrency keys — the `limits_concurrency` slot table and the jobs
+      # parked behind it. Aggregate numbers plus up to 100 key rows, busiest
+      # first. The two tables are joined FULL OUTER, not LEFT: a key can have
+      # parked rows and no semaphore (its holder died and the sweep removed the
+      # row) or a semaphore and nothing parked — an operator needs to see both.
+      def concurrency_stats
+        concurrency_summary.merge(keys: concurrency_keys)
+      end
+
+      # Drop a key's semaphore row and promote whatever can now run.
+      #
+      # Promotion goes through Concurrency::BlockedExecution.promote_next, which
+      # takes each slot through the same guarded upsert an enqueue uses — a
+      # dashboard release can no more over-admit than an enqueue can. Re-sending
+      # the parked payloads directly would reintroduce exactly the over-admission
+      # issue #460 closed. Returns the number of jobs promoted.
+      def release_concurrency_key(key)
+        return 0 if key.to_s.strip.empty?
+
+        # Zero the held slots rather than DELETE the row: the row is where the
+        # key's limit is recorded, and `Semaphore.acquire!` falls back to a
+        # limit of 1 on a fresh row. Deleting it would silently demote a
+        # `to: 3` key to 1 whenever the parked job's class no longer resolves
+        # (the payload then carries no limit for promotion to use). A row left
+        # at 0 is reaped by the dispatcher's `Semaphore.expired` sweep.
+        Pgbus::Semaphore.where(key: key).update_all(value: 0)
+        promoted = 0
+        promoted += 1 while promoted < PROMOTE_CAP &&
+                            Concurrency::BlockedExecution.promote_next(key, client: @client)
+
+        # Nothing took the freed slot, so mark the empty row expired: the
+        # dispatcher's `Concurrency::Semaphore.expire_stale` matches on
+        # expires_at ALONE, so a zeroed row would otherwise sit on the Locks
+        # page as a phantom 0/N key until its original lease ran out — up to
+        # the key's whole `duration`. Guarded on value = 0 so a row a promotion
+        # just filled is never expired out from under its holder.
+        Pgbus::Semaphore.where(key: key, value: 0).update_all(expires_at: Time.current)
+        promoted
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error releasing concurrency key #{key}: #{e.message}" }
+        0
+      end
+
+      # Drop every job parked behind a key. The jobs never run, so the
+      # bookkeeping they would have resolved has to be resolved here: a parked
+      # batch child is marked failed (otherwise its batch waits forever and
+      # on_failure never fires — issue #413) and an :until_executed uniqueness
+      # lock is released (otherwise it is orphaned, since no executor will ever
+      # release it — issue #423). Returns the number of jobs discarded.
+      def discard_parked_jobs(key)
+        return 0 if key.to_s.strip.empty?
+
+        rows = []
+        Pgbus::BlockedExecution.transaction do
+          # SKIP LOCKED so this can never race a concurrent promote, which
+          # deletes its row under the same lock before enqueueing it.
+          rows = Pgbus::BlockedExecution.for_key(key).lock("FOR UPDATE SKIP LOCKED").to_a
+          Pgbus::BlockedExecution.where(id: rows.map(&:id)).delete_all if rows.any?
+        end
+
+        # Cleanup runs after the claim commits, because the SKIP LOCKED claim
+        # only holds inside the transaction. The residual: a process killed
+        # between the commit and the cleanup leaves a resolved-nothing orphan
+        # that Batch::Sweep un-counts rather than fails. Running the cleanup
+        # inside the transaction would swap that for the failure this codebase
+        # has already judged worse (Concurrency::BlockedExecution#backfill) —
+        # `Batch.job_discarded` can enqueue a batch callback, and a callback
+        # fired for a batch that then rolls back is not recoverable, while an
+        # orphan row is. Each row is cleaned under its own rescue, so one bad
+        # payload never costs the rest.
+        rows.each { |row| cleanup_discarded_parked_job(row) }
+        rows.size
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error discarding parked jobs for #{key}: #{e.message}" }
+        0
+      end
+
       # Batches
       def batches(limit: 100)
         records = BatchEntry.order(created_at: :desc).limit(limit).to_a
@@ -1026,6 +1119,122 @@ module Pgbus
         Pgbus::BusRecord.connection
       end
 
+      # Aggregate concurrency numbers, memoized for the lifetime of this
+      # instance: summary_stats, the metrics serializer and the AppSignal probe
+      # all read them on the same request, and concurrency_stats merges them in
+      # alongside the key rows. Mutations redirect to a fresh request with a new
+      # instance, so the memo never serves stale numbers.
+      def concurrency_summary
+        @concurrency_summary ||= fetch_concurrency_summary
+      end
+
+      def fetch_concurrency_summary
+        row = connection.select_all(<<~SQL, "Pgbus Concurrency Summary").to_a.first || {}
+          SELECT
+            (SELECT COUNT(*) FROM pgbus_blocked_executions) AS parked_total,
+            (SELECT EXTRACT(EPOCH FROM (now() - MIN(created_at)))::bigint
+               FROM pgbus_blocked_executions) AS oldest_parked_age_sec,
+            (SELECT COALESCE(SUM(value), 0) FROM pgbus_semaphores) AS slots_held,
+            (SELECT COUNT(*) FROM pgbus_semaphores WHERE value >= max_value) AS keys_at_limit
+        SQL
+
+        {
+          parked_total: row["parked_total"].to_i,
+          oldest_parked_age_sec: row["oldest_parked_age_sec"]&.to_i,
+          slots_held: row["slots_held"].to_i,
+          keys_at_limit: row["keys_at_limit"].to_i
+        }
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching concurrency summary: #{e.message}" }
+        { parked_total: 0, oldest_parked_age_sec: nil, slots_held: 0, keys_at_limit: 0 }
+      end
+
+      # `lease_fresh` is the one fact an operator needs before releasing a key:
+      # a live lease means a holder is probably still running, so releasing lets
+      # another job start beside it.
+      def concurrency_keys(limit: 100)
+        rows = connection.select_all(<<~SQL, "Pgbus Concurrency Keys")
+          SELECT COALESCE(s.key, b.concurrency_key) AS key,
+                 s.value AS value,
+                 s.max_value AS max_value,
+                 s.expires_at AS expires_at,
+                 (s.value > 0 AND s.expires_at > now()) AS lease_fresh,
+                 COALESCE(b.parked_count, 0) AS parked_count,
+                 EXTRACT(EPOCH FROM (now() - b.oldest_parked_at))::bigint AS oldest_parked_age_sec
+          FROM pgbus_semaphores s
+          FULL OUTER JOIN (
+            SELECT concurrency_key, COUNT(*) AS parked_count, MIN(created_at) AS oldest_parked_at
+            FROM pgbus_blocked_executions
+            GROUP BY concurrency_key
+          ) b ON s.key = b.concurrency_key
+          ORDER BY COALESCE(b.parked_count, 0) DESC, s.expires_at ASC NULLS LAST
+          LIMIT #{limit.to_i}
+        SQL
+
+        rows.to_a.map { |row| format_concurrency_key(row) }
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Web] Error fetching concurrency keys: #{e.message}" }
+        []
+      end
+
+      def format_concurrency_key(row)
+        {
+          key: row["key"],
+          value: row["value"]&.to_i,
+          max_value: row["max_value"]&.to_i,
+          expires_at: row["expires_at"],
+          lease_fresh: [true, "t"].include?(row["lease_fresh"]),
+          parked_count: row["parked_count"].to_i,
+          oldest_parked_age_sec: row["oldest_parked_age_sec"]&.to_i
+        }
+      end
+
+      # Resolve the bookkeeping a discarded parked job will never resolve
+      # itself. Per-row rather than batched: one bad payload must not stop the
+      # rest from being cleaned up, and the rows are already deleted.
+      def cleanup_discarded_parked_job(row)
+        payload = decode_parked_payload(row.payload)
+        batch_id = payload[Batch::METADATA_KEY]
+        Batch.job_discarded(batch_id, job_id: payload["job_id"]) if batch_id
+
+        release_parked_uniqueness_lock(payload, row.created_at)
+
+        Pgbus.logger.warn do
+          "[Pgbus::Web] Discarded parked job #{payload["job_class"]} (#{payload["job_id"]}) " \
+            "for concurrency key #{row.concurrency_key}"
+        end
+        Instrumentation.instrument(
+          "pgbus.blocked_execution_discarded",
+          concurrency_key: row.concurrency_key,
+          job_class: payload["job_class"],
+          job_id: payload["job_id"]
+        )
+      rescue StandardError => e
+        Pgbus.logger.warn { "[Pgbus::Web] Parked job cleanup failed: #{e.class}: #{e.message}" }
+      end
+
+      # release_if_unbound!, not release_lock: a parked job's lock is unbound (it
+      # never got a msg_id). If the reaper already removed it and a successor
+      # took the same key, that successor's lock must survive — msg_id = 0
+      # excludes one that was sent, and the parked row's own created_at excludes
+      # one acquired after this row was parked.
+      def release_parked_uniqueness_lock(payload, parked_at)
+        key = payload[Uniqueness::METADATA_KEY]
+        return unless key && payload[Uniqueness::STRATEGY_KEY].to_s == "until_executed"
+
+        UniquenessKey.release_if_unbound!(key, acquired_before: parked_at)
+      end
+
+      # A row parked before the double-encoding fix stores a jsonb *string*
+      # holding the document — the same two-pass unwrap BlockedExecution
+      # .release_next! does.
+      def decode_parked_payload(payload)
+        2.times { payload = JSON.parse(payload) if payload.is_a?(String) }
+        payload.is_a?(Hash) ? payload : {}
+      rescue JSON::ParserError
+        {}
+      end
+
       # Single query to fetch pg_stat_user_tables stats for all queue and
       # archive tables. Avoids 2*N catalog queries on the dashboard.
       def fetch_all_table_stats

data/lib/pgbus/web/metrics_serializer.rb

@@ -22,9 +22,14 @@ module Pgbus
         append_queue_metrics(lines)
         append_job_metrics(lines)
         append_process_metrics(lines)
-        append_summary_metrics(lines)
+        # One summary read, shared: summary_stats is not memoized, so calling it
+        # per family would repeat the queue/health/process queries and advance
+        # the throughput snapshot a second time within one scrape.
+        summary = summary_stats
+        append_summary_metrics(lines, summary)
         append_stream_metrics(lines)
         append_health_metrics(lines)
+        append_concurrency_metrics(lines, summary)
         "#{lines.join("\n")}\n"
       end
 
@@ -147,8 +152,18 @@ module Pgbus
         Pgbus.logger.debug { "[Pgbus::Metrics] Error serializing process metrics: #{e.message}" }
       end
 
-      def append_summary_metrics(lines)
-        stats = @data_source.summary_stats
+      # nil when the read raised — each family then skips rather than blanking
+      # the whole scrape.
+      def summary_stats
+        @data_source.summary_stats
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Metrics] Error reading summary stats: #{e.message}" }
+        nil
+      end
+
+      def append_summary_metrics(lines, stats)
+        return unless stats
+
         gauge(lines, "pgbus_failed_events_total", "Total failed events") do
           [[stats[:failed_count]]]
         end
@@ -219,6 +234,39 @@ module Pgbus
         Pgbus.logger.debug { "[Pgbus::Metrics] Error serializing health metrics: #{e.message}" }
       end
 
+      # Concurrency slot pressure: how many jobs are parked, how long the oldest
+      # has waited, how many slots are held right now. Unlabelled on purpose —
+      # concurrency keys are per-record (one per order, one per sync group), so a
+      # per-key label would make an unbounded series. The per-key detail lives on
+      # the Locks page and in the pgbus_concurrency MCP tool, both bounded.
+      #
+      # Omitted entirely when neither table holds anything: an install that does
+      # not use limits_concurrency reports nothing rather than a flat zero line.
+      def append_concurrency_metrics(lines, stats)
+        return unless stats
+        return if stats[:parked_total].to_i.zero? && stats[:slots_held].to_i.zero? &&
+                  stats[:keys_at_limit].to_i.zero? && stats[:oldest_parked_age_sec].nil?
+
+        gauge(lines, "pgbus_concurrency_blocked_executions",
+              "Jobs parked behind a concurrency key, waiting for a slot") do
+          [[stats[:parked_total].to_i]]
+        end
+
+        if stats[:oldest_parked_age_sec]
+          gauge(lines, "pgbus_concurrency_blocked_oldest_age_seconds",
+                "How long the longest-waiting parked job has waited") do
+            [[stats[:oldest_parked_age_sec]]]
+          end
+        end
+
+        gauge(lines, "pgbus_concurrency_slots_held",
+              "Concurrency slots currently held across all keys") do
+          [[stats[:slots_held].to_i]]
+        end
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus::Metrics] Error serializing concurrency metrics: #{e.message}" }
+      end
+
       # Emits a Prometheus gauge metric family. The block must return an array
       # of [value] or [value, { label: "val" }] pairs.
       def gauge(lines, name, help)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.16.4
+  version: 0.16.6
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -185,6 +185,8 @@ files:
 - app/views/pgbus/jobs/_failed_table.html.erb
 - app/views/pgbus/jobs/index.html.erb
 - app/views/pgbus/jobs/show.html.erb
+- app/views/pgbus/locks/_concurrency.html.erb
+- app/views/pgbus/locks/_uniqueness.html.erb
 - app/views/pgbus/locks/index.html.erb
 - app/views/pgbus/outbox/index.html.erb
 - app/views/pgbus/processes/_processes_table.html.erb
@@ -292,6 +294,7 @@ files:
 - lib/pgbus/event_bus/handler.rb
 - lib/pgbus/event_bus/publisher.rb
 - lib/pgbus/event_bus/registry.rb
+- lib/pgbus/event_bus/stale_connection_retry.rb
 - lib/pgbus/event_bus/subscriber.rb
 - lib/pgbus/execution_pools.rb
 - lib/pgbus/execution_pools/async_pool.rb
@@ -317,6 +320,7 @@ files:
 - lib/pgbus/mcp/redactor.rb
 - lib/pgbus/mcp/runner.rb
 - lib/pgbus/mcp/server.rb
+- lib/pgbus/mcp/tools/concurrency_tool.rb
 - lib/pgbus/mcp/tools/dlq_detail_tool.rb
 - lib/pgbus/mcp/tools/dlq_tool.rb
 - lib/pgbus/mcp/tools/health_tool.rb
@@ -372,6 +376,7 @@ files:
 - lib/pgbus/serializer.rb
 - lib/pgbus/stat_buffer.rb
 - lib/pgbus/streams.rb
+- lib/pgbus/streams/broadcast_opts.rb
 - lib/pgbus/streams/broadcastable_override.rb
 - lib/pgbus/streams/coalescer.rb
 - lib/pgbus/streams/cursor.rb