dispatch_policy 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0f7f08701682539b873609e4f45432661e9d9e070e5b27593695a90b6266595e
-  data.tar.gz: c32b4f91f257b1e69e0417edc44fb3779c3f0567f4dd152d893fa781ca7d3142
+  metadata.gz: 23433a64c963b0e0908c185ad8dc8e6f97edbd8d476ee712d15023f74ba0e338
+  data.tar.gz: 64ff19e04a6d02b0f1eedb4fb6d74b0e073e3773efb9a3afc92ae1a3e9002aeb
 SHA512:
-  metadata.gz: 982defdd7fda9aae96d31b83666bd291bf4c36dde33e673f43322ee35cbb830c547a9ada0bbcbe142b47a00f10112d6aae24f1631a06f94e97e2e9ef71365239
-  data.tar.gz: e928eb7605905fb7de6867d4933332789ebd36e932be2f6784ed672fdcd485bf3228b659a5947b5965e7e47ee3e2c9089dd6cdd5b6069764008f682eacbacaa2
+  metadata.gz: e168e049dbb0d399dddc6e84427b7b557474d9ce10cb1983d3f4f24c6fde43ffda9c03c179b4590d9f09d225a8adeb5d7295a10788898ebc4ab0bc47a765163c
+  data.tar.gz: d8ef9debaebdf89de7cce28e5fa669484acafd27e4b5e65ff959f6f177c1aaa124cb1215da191c3a6759094aac467980489241062f907574860b7226dd9dbc9a

data/CHANGELOG.md

@@ -1,5 +1,58 @@
 # Changelog
 
+## 0.4.3
+
+### Fixed
+- The `throttle` gate now charges its token bucket for the number of jobs
+  **actually admitted**, not for the optimistic `allowed` it computes at
+  evaluate time. The deduction moved from `#evaluate` to the `#consume`
+  hook (run after the staging DELETE, via `Pipeline.settle`), so the
+  bucket is no longer over-charged — and the effective rate no longer
+  drifts below the configured one — when fewer jobs are admitted than
+  allowed: future-scheduled rows skipped by the `scheduled_at <= now()`
+  filter, a downstream `concurrency` gate capping `admit_count`, or rows a
+  concurrent tick claimed under `SKIP LOCKED`.
+- Inflight rows for jobs that were admitted but have **not started
+  performing yet** (still waiting in the adapter's queue) are no longer
+  reaped at `inflight_stale_after`. Their heartbeat thread only starts in
+  `around_perform`, so under a deep adapter backlog the sweeper used to
+  delete still-valid admissions, making the concurrency gate under-count
+  and over-admit. `sweep_stale_inflight!` is now two-tier: rows
+  heartbeated past admission reap at `inflight_stale_after`; never-started
+  rows reap only past the new, generous `config.inflight_queued_stale_after`
+  (1 hour default).
+- `InflightTracker` now applies the same `job.queue_name || policy.queue_name`
+  fallback at perform time that the staging path uses, so a policy whose
+  `partition_by`/`shard_by` reads `queue_name` derives the same
+  `partition_key` at admission and at perform (otherwise the inflight row
+  and adaptive observations landed under the wrong scope).
+- `CursorPagination` rejects cursors whose value isn't a scalar or whose
+  id isn't an integer (the cursor is an attacker-controllable query
+  param), and ignores a value whose type can't compare against the sort
+  column instead of raising a `PG` error (a forged numeric value on a
+  timestamp sort). Falls back to the first page.
+- `PolicyDSL#tick_admission_budget(nil)` / `#admission_batch_size(nil)` are
+  no-ops that defer to config instead of raising in `Integer(nil)`,
+  matching how `fairness(half_life:)` already guards nil.
+
+### Changed
+- The admin UI's dashboard and policies index collapse their per-policy
+  `N+1` query loops into grouped `Repository` methods
+  (`tick_summaries_by_policy`, `top_denied_reason_by_policy`,
+  `partition_round_trip_stats_by_policy`, `partition_counts_by_policy`),
+  one query each instead of several per policy.
+
+### Added
+- `config.inflight_queued_stale_after` (default 1 hour) — the sweep cutoff
+  for inflight rows admitted but never started. Raise it if your adapter
+  backlog can exceed an hour.
+
+### Removed
+- The broken, unused `Partition.stale_inactive` scope — it filtered on an
+  `in_flight_count` column dropped back in 0.3.0, so any call raised
+  `PG::UndefinedColumn`. The real partition GC is
+  `Repository.sweep_inactive_partitions!`.
+
 ## 0.4.2
 
 ### Fixed

data/README.md

@@ -534,6 +534,7 @@ DispatchPolicy.configure do |c|
   c.idle_pause                = 0.5      # seconds slept when a tick admits nothing
   c.partition_inactive_after  = 86_400   # GC partitions idle this long
   c.inflight_stale_after      = 300      # GC inflight rows whose worker stopped heartbeating
+  c.inflight_queued_stale_after = 3_600  # GC inflight rows admitted but never started (queued)
   c.inflight_heartbeat_interval = 30     # how often the worker bumps heartbeat_at
   c.sweep_every_ticks         = 50       # sweeper cadence (in tick iterations)
   c.metrics_retention         = 86_400   # tick_samples kept this long

data/app/controllers/dispatch_policy/dashboard_controller.rb

@@ -61,29 +61,37 @@ module DispatchPolicy
       one_min_ago = now - 60
       five_min_ago = now - 300
 
+      # Aggregate everything the per-policy rows need in 4 grouped queries
+      # instead of ~4 per policy. With dozens of policies this was the bulk
+      # of the dashboard's query count.
+      m1_by     = Repository.tick_summaries_by_policy(since: one_min_ago)
+      m5_by     = Repository.tick_summaries_by_policy(since: five_min_ago)
+      denied_by = Repository.top_denied_reason_by_policy(since: one_min_ago)
+      rt_by     = Repository.partition_round_trip_stats_by_policy
+
       names = (pending_by_policy.keys + in_flight_by_policy.keys).uniq.sort
       @policies = names.map do |name|
-        info  = pending_by_policy[name] || {}
-        m1    = Repository.tick_summary(policy_name: name, since: one_min_ago)
-        m5    = Repository.tick_summary(policy_name: name, since: five_min_ago)
-        rs    = Repository.denied_reasons_summary(policy_name: name, since: one_min_ago)
-        rt    = Repository.partition_round_trip_stats(policy_name: name)
+        info = pending_by_policy[name] || {}
+        m1   = m1_by[name] || {}
+        m5   = m5_by[name] || {}
+        rt   = rt_by[name] || {}
+        top  = denied_by[name] # [reason, count] or nil
 
         {
           name:           name,
           pending:        info[:pending] || 0,
           in_flight:      in_flight_by_policy[name] || 0,
           last_admit_at:  info[:last_admit_at],
-          admitted_1m:    m1[:jobs_admitted],
-          admitted_5m:    m5[:jobs_admitted],
-          ticks_1m:       m1[:ticks],
-          avg_tick_ms_1m: m1[:avg_duration_ms],
-          forward_failures_1m: m1[:forward_failures],
+          admitted_1m:    m1[:jobs_admitted] || 0,
+          admitted_5m:    m5[:jobs_admitted] || 0,
+          ticks_1m:       m1[:ticks] || 0,
+          avg_tick_ms_1m: m1[:avg_duration_ms] || 0,
+          forward_failures_1m: m1[:forward_failures] || 0,
           oldest_age_seconds:  rt[:oldest_age_seconds],
           p95_age_seconds:     rt[:p95_age_seconds],
-          in_backoff:          rt[:in_backoff],
-          top_denial_reason:   rs.first&.first,
-          top_denial_count:    rs.first&.last
+          in_backoff:          rt[:in_backoff] || 0,
+          top_denial_reason:   top&.first,
+          top_denial_count:    top&.last
         }
       end
     end

data/app/controllers/dispatch_policy/policies_controller.rb

@@ -12,16 +12,19 @@ module DispatchPolicy
       names          = (registry_names + db_names).uniq.sort
 
       in_flight_by_policy = InflightJob.where(policy_name: names).group(:policy_name).count
+      # One grouped query for pending / partition count / paused count
+      # across every policy instead of three per policy.
+      counts_by_policy    = Repository.partition_counts_by_policy
 
       @rows = names.map do |name|
-        partitions = Partition.for_policy(name)
+        counts = counts_by_policy[name] || {}
         {
           name:           name,
           registered:     registry_names.include?(name),
-          pending:        partitions.sum(:pending_count),
+          pending:        counts[:pending] || 0,
           in_flight:      in_flight_by_policy[name] || 0,
-          partitions:     partitions.count,
-          paused_count:   partitions.paused.count
+          partitions:     counts[:partitions] || 0,
+          paused_count:   counts[:paused] || 0
         }
       end
     end

data/app/models/dispatch_policy/partition.rb

@@ -9,10 +9,6 @@ module DispatchPolicy
     scope :active,     -> { where(status: "active") }
     scope :paused,     -> { where(status: "paused") }
     scope :pending,    -> { where("pending_count > 0") }
-    scope :stale_inactive, ->(cutoff) {
-      where("pending_count = 0 AND in_flight_count = 0")
-        .where("last_admit_at < ? OR (last_admit_at IS NULL AND created_at < ?)", cutoff, cutoff)
-    }
 
     def paused?
       status == "paused"

data/lib/dispatch_policy/config.rb

@@ -10,6 +10,7 @@ module DispatchPolicy
                   :busy_pause,
                   :partition_inactive_after,
                   :inflight_stale_after,
+                  :inflight_queued_stale_after,
                   :inflight_heartbeat_interval,
                   :real_adapter,
                   :logger,
@@ -40,6 +41,16 @@ module DispatchPolicy
       @busy_pause                = 0.0
       @partition_inactive_after  = 24 * 60 * 60
       @inflight_stale_after      = 5 * 60
+      # Cutoff for inflight rows that were admitted (pre-inserted by the
+      # Tick) but never started performing — so the heartbeat thread, which
+      # only starts in around_perform, never advanced their heartbeat_at.
+      # These sit in the adapter's queue waiting for a worker; reaping them
+      # at `inflight_stale_after` (5 min) would make the concurrency gate
+      # under-count and over-admit whenever queue latency exceeds that. We
+      # give never-started rows a far more generous cutoff (1h) before
+      # assuming the admission was lost. Raise it if your adapter backlog
+      # can exceed an hour.
+      @inflight_queued_stale_after = 60 * 60
       @inflight_heartbeat_interval = 30
       @real_adapter              = nil
       @logger                    = nil

data/lib/dispatch_policy/cursor_pagination.rb

@@ -66,6 +66,13 @@ module DispatchPolicy
       decoded = JSON.parse(Base64.urlsafe_decode64(cursor))
       return nil unless decoded.is_a?(Array) && decoded.size == 2
 
+      # The cursor is attacker-controllable (a query param). Reject anything
+      # that isn't a (scalar value, integer id) tuple so a hostile payload
+      # like [[1,2], {}] can't reach the WHERE clause and raise a 500 (or
+      # worse). Per-column type compatibility is enforced in #apply.
+      value, id = decoded
+      return nil unless (value.is_a?(String) || value.is_a?(Numeric)) && id.is_a?(Integer)
+
       decoded
     rescue StandardError
       nil
@@ -78,6 +85,15 @@ module DispatchPolicy
       return scope if cursor.nil?
 
       value, last_id = cursor
+      # Ignore a cursor whose value type can't be compared against this
+      # sort's column. The numeric columns (pending_count, total_admitted)
+      # need a Numeric; everything else compares as text (partition_key, or
+      # the ISO8601 timestamps emitted by #extract). A mismatch — e.g. a
+      # numeric value forged for a timestamp sort — would raise PG error;
+      # instead we fall back to the first page.
+      numeric_column = %w[pending_count total_admitted].include?(sort[:cursor_sql])
+      return scope unless numeric_column ? value.is_a?(Numeric) : value.is_a?(String)
+
       case sort[:direction]
       when :desc
         scope.where(

data/lib/dispatch_policy/gates/concurrency.rb

@@ -27,6 +27,13 @@ module DispatchPolicy
         cap = capacity_for(ctx)
         return Decision.deny(retry_after: @full_backoff, reason: "max=0") if cap <= 0
 
+        # This COUNT(*) runs in `evaluate`, BEFORE the admission TX opens, so
+        # the cap holds only when a single tick loop owns a given
+        # (policy, shard): within one tick, pass-2 re-reads the count after
+        # pass-1's inflight pre-insert has committed. Running two tick loops
+        # over the SAME shard would let both read the same pre-admission
+        # count and over-admit — shard the policy instead of duplicating
+        # loops on one shard (see shard_by in the README).
         in_flight = Repository.count_inflight(
           policy_name:   partition["policy_name"],
           partition_key: inflight_partition_key(partition["policy_name"], ctx)

data/lib/dispatch_policy/gates/throttle.rb

@@ -39,11 +39,19 @@ module DispatchPolicy
         elapsed     = [now - refilled_at, 0.0].max
         tokens      = [tokens + (elapsed * refill_rate), capacity.to_f].min
 
-        whole       = tokens.floor
+        # The patch records the post-refill bucket WITHOUT deducting yet.
+        # The actual deduction is deferred to #consume, which runs once
+        # the admission TX knows how many staged rows were really claimed.
+        # Deducting `allowed` here over-charges the bucket whenever fewer
+        # jobs are admitted than allowed — a later gate capping admit_count,
+        # future-scheduled rows skipped by the `scheduled_at <= now()`
+        # filter, or rows another tick grabbed under SKIP LOCKED.
+        patch = { "tokens" => tokens, "refilled_at" => now }
+
+        whole = tokens.floor
         if whole.zero?
           missing      = 1.0 - tokens
           retry_after  = missing / refill_rate
-          patch        = { "tokens" => tokens, "refilled_at" => now }
           return Decision.new(allowed: 0,
                               retry_after: retry_after,
                               gate_state_patch: { "throttle" => patch },
@@ -51,10 +59,22 @@ module DispatchPolicy
         end
 
         allowed = [whole, admit_budget].min
-        patch   = { "tokens" => tokens - allowed, "refilled_at" => now }
         Decision.new(allowed: allowed, gate_state_patch: { "throttle" => patch })
       end
 
+      # Settles the bucket against the number of jobs actually admitted.
+      # `evaluate` recorded the post-refill token count in the decision's
+      # patch; here we subtract exactly `admitted_count` (≤ allowed), so
+      # the bucket is charged for jobs that really left, never for unspent
+      # budget. Called by Pipeline.settle after the claim.
+      def consume(decision, admitted_count)
+        st = decision.gate_state_patch && decision.gate_state_patch["throttle"]
+        return nil unless st
+
+        { "throttle" => { "tokens"      => st["tokens"].to_f - admitted_count,
+                          "refilled_at" => st["refilled_at"] } }
+      end
+
       private
 
       def capacity_for(ctx)

data/lib/dispatch_policy/inflight_tracker.rb

@@ -28,7 +28,15 @@ module DispatchPolicy
       policy = DispatchPolicy.registry.fetch(policy_name)
       return yield unless policy
 
-      ctx           = policy.build_context(job.arguments, queue_name: job.queue_name&.to_s)
+      # Mirror the stage-time fallback in JobExtension.around_enqueue_for:
+      # when the job carries no explicit queue, use the policy's default.
+      # Without this, a policy whose partition_by/shard_by reads queue_name
+      # would compute a DIFFERENT partition_key here than at admission, so
+      # the around_perform inflight row (and adaptive observations) would
+      # land under the wrong scope and the concurrency gate's COUNT(*) would
+      # miss them.
+      queue_name    = job.queue_name&.to_s || policy.queue_name
+      ctx           = policy.build_context(job.arguments, queue_name: queue_name)
       partition_key = policy.partition_key_for(ctx)
 
       Repository.insert_inflight!([{

data/lib/dispatch_policy/pipeline.rb

@@ -5,12 +5,30 @@ module DispatchPolicy
   # partition. Returns a value object describing how many jobs may be
   # admitted right now and which gate-state patches to persist.
   class Pipeline
-    Result = Struct.new(:admit_count, :retry_after, :gate_state_patch, :reasons, keyword_init: true)
+    Result = Struct.new(:admit_count, :retry_after, :gate_state_patch, :reasons, :decisions, keyword_init: true)
 
     def initialize(policy)
       @policy = policy
     end
 
+    # Computes the gate_state patch to persist once the REAL admitted count
+    # is known (after the staging DELETE). Each gate's #consume settles its
+    # state against the actual number of jobs claimed — the throttle
+    # deducts that many tokens rather than the optimistic `allowed` it
+    # returned at evaluate time. Gates that keep no gate_state (concurrency,
+    # adaptive_concurrency — their state lives in their own tables) return
+    # nil from #consume and contribute nothing here.
+    #
+    # `decisions` is the [gate, decision] list carried on the Result.
+    def self.settle(decisions, admitted_count)
+      patch = {}
+      decisions.each do |gate, decision|
+        sub = gate.consume(decision, admitted_count)
+        patch.merge!(sub) if sub
+      end
+      patch
+    end
+
     def call(ctx, partition, max_budget)
       budget          = max_budget
       retry_after     = nil
@@ -41,7 +59,8 @@ module DispatchPolicy
         admit_count:       admit_count,
         retry_after:       retry_after,
         gate_state_patch:  patch,
-        reasons:           reasons
+        reasons:           reasons,
+        decisions:         decisions
       )
     end
   end

data/lib/dispatch_policy/policy_dsl.rb

@@ -45,7 +45,7 @@ module DispatchPolicy
     end
 
     def admission_batch_size(size)
-      @admission_batch_size = Integer(size)
+      @admission_batch_size = Integer(size) if size
     end
 
     # Per-policy override for the EWMA half-life used to weigh recent
@@ -62,7 +62,7 @@ module DispatchPolicy
     # nil, no global cap is enforced and per-partition admission_batch_size
     # is the only ceiling.
     def tick_admission_budget(value)
-      @tick_admission_budget = Integer(value)
+      @tick_admission_budget = Integer(value) if value
     end
 
     # Defines the partition scope. Required — every policy declares

data/lib/dispatch_policy/repository.rb

@@ -192,8 +192,8 @@ module DispatchPolicy
     # through `bulk_record_partition_denies!` instead, which collapses
     # many partitions into a single UPDATE…FROM(VALUES…) at the end of
     # the tick.
-    def claim_staged_jobs!(policy_name:, partition_key:, limit:, gate_state_patch:, retry_after:,
-                           half_life_seconds: nil)
+    def claim_staged_jobs!(policy_name:, partition_key:, limit:, retry_after:,
+                           gate_state_patch: nil, half_life_seconds: nil)
       raise ArgumentError, "claim_staged_jobs! requires limit > 0" unless limit.positive?
 
       sql_select = <<~SQL.squish
@@ -212,11 +212,18 @@ module DispatchPolicy
       SQL
       rows = connection.exec_query(sql_select, "claim_staged_jobs", [policy_name, partition_key, limit]).to_a
 
+      # The gate_state patch may depend on how many rows we actually
+      # claimed (e.g. the throttle charges its bucket for jobs admitted,
+      # not for the optimistic `allowed`). When the caller passes a block
+      # it receives that real count and returns the patch to persist;
+      # gate-less callers pass a fixed `gate_state_patch:` instead.
+      patch = block_given? ? yield(rows.size) : (gate_state_patch || {})
+
       record_partition_admit!(
         policy_name:       policy_name,
         partition_key:     partition_key,
         admitted:          rows.size,
-        gate_state_patch:  gate_state_patch,
+        gate_state_patch:  patch,
         retry_after:       retry_after,
         half_life_seconds: half_life_seconds
       )
@@ -396,14 +403,37 @@ module DispatchPolicy
       Integer(result.rows.first.first)
     end
 
-    def sweep_stale_inflight!(cutoff_seconds:)
+    # Reap inflight rows whose owner is gone. Two tiers, distinguished by
+    # whether the row was ever heartbeated past its admission:
+    #
+    #   heartbeat_at > admitted_at  → the worker started performing and the
+    #     heartbeat thread advanced heartbeat_at at least once. If it then
+    #     went silent for `cutoff_seconds`, the worker died mid-run: reap.
+    #
+    #   heartbeat_at <= admitted_at → never heartbeated past admission. The
+    #     row was pre-inserted by the Tick and the job is still waiting in
+    #     the adapter's queue (or only just started — the first heartbeat
+    #     fires after inflight_heartbeat_interval). Reaping these at the
+    #     short cutoff would under-count the concurrency gate and over-admit
+    #     whenever queue latency exceeds it. Only reap once they're older
+    #     than the far more generous `queued_cutoff_seconds`, by which point
+    #     the admission is presumed lost.
+    #
+    # The Tick pre-insert writes admitted_at and heartbeat_at from the same
+    # now() (a single statement), so a never-started row has them exactly
+    # equal; one heartbeat makes heartbeat_at strictly greater.
+    def sweep_stale_inflight!(cutoff_seconds:, queued_cutoff_seconds: nil)
+      queued_cutoff_seconds ||= cutoff_seconds
       connection.exec_query(
         <<~SQL.squish,
           DELETE FROM #{INFLIGHT_TABLE}
-          WHERE heartbeat_at < now() - ($1 || ' seconds')::interval
+          WHERE (heartbeat_at > admitted_at
+                 AND heartbeat_at < now() - ($1 || ' seconds')::interval)
+             OR (heartbeat_at <= admitted_at
+                 AND admitted_at < now() - ($2 || ' seconds')::interval)
         SQL
         "sweep_stale_inflight",
-        [cutoff_seconds.to_i]
+        [cutoff_seconds.to_i, queued_cutoff_seconds.to_i]
       )
     end
 
@@ -471,6 +501,37 @@ module DispatchPolicy
       }
     end
 
+    # One grouped query returning per-policy tick aggregates, keyed by
+    # policy_name. Replaces calling tick_summary once per policy on the
+    # dashboard (N queries → 1). Only the fields the overview renders.
+    #   { "policy_a" => { jobs_admitted:, forward_failures:, ticks:,
+    #                     avg_duration_ms: }, ... }
+    def tick_summaries_by_policy(since:)
+      result = connection.exec_query(
+        <<~SQL.squish,
+          SELECT
+            policy_name,
+            COALESCE(SUM(jobs_admitted), 0)::int    AS jobs_admitted,
+            COALESCE(SUM(forward_failures), 0)::int AS forward_failures,
+            COUNT(*)::int                           AS ticks,
+            COALESCE(AVG(duration_ms), 0)::int      AS avg_duration_ms
+          FROM #{SAMPLES_TABLE}
+          WHERE sampled_at >= $1
+          GROUP BY policy_name
+        SQL
+        "tick_summaries_by_policy",
+        [since]
+      )
+      result.to_a.each_with_object({}) do |r, h|
+        h[r["policy_name"]] = {
+          jobs_admitted:    r["jobs_admitted"].to_i,
+          forward_failures: r["forward_failures"].to_i,
+          ticks:            r["ticks"].to_i,
+          avg_duration_ms:  r["avg_duration_ms"].to_i
+        }
+      end
+    end
+
     # Aggregate denied_reasons jsonb across samples in window: returns
     # { "throttle" => 12, "concurrency_full" => 3, ... }
     def denied_reasons_summary(policy_name: nil, since:)
@@ -490,6 +551,30 @@ module DispatchPolicy
       result.to_a.each_with_object({}) { |r, h| h[r["key"]] = r["total"].to_i }
     end
 
+    # The single most-denied reason per policy in one query, keyed by
+    # policy_name → [reason, count]. Replaces calling denied_reasons_summary
+    # per policy on the dashboard just to read its top entry.
+    def top_denied_reason_by_policy(since:)
+      result = connection.exec_query(
+        <<~SQL.squish,
+          SELECT DISTINCT ON (policy_name) policy_name, key, total
+          FROM (
+            SELECT policy_name, key, SUM(value::int)::int AS total
+            FROM #{SAMPLES_TABLE},
+                 LATERAL jsonb_each_text(denied_reasons)
+            WHERE sampled_at >= $1
+            GROUP BY policy_name, key
+          ) t
+          ORDER BY policy_name, total DESC
+        SQL
+        "top_denied_reason_by_policy",
+        [since]
+      )
+      result.to_a.each_with_object({}) do |r, h|
+        h[r["policy_name"]] = [r["key"], r["total"].to_i]
+      end
+    end
+
     # Returns time-bucketed series for sparklines. `bucket_seconds` is the
     # bucket width. Each row: { bucket_at:, jobs_admitted:, forward_failures:,
     # pending_total:, ticks: }.
@@ -595,6 +680,62 @@ module DispatchPolicy
       }
     end
 
+    # Per-policy partition counts in one grouped query, keyed by
+    # policy_name → { pending, partitions, paused }. Replaces calling
+    # Partition.for_policy(name).sum/.count/.paused.count once per policy on
+    # the policies index (3N queries → 1).
+    def partition_counts_by_policy
+      result = connection.exec_query(
+        <<~SQL.squish,
+          SELECT
+            policy_name,
+            COALESCE(SUM(pending_count), 0)::int                 AS pending,
+            COUNT(*)::int                                        AS partitions,
+            COUNT(*) FILTER (WHERE status = 'paused')::int       AS paused
+          FROM #{PARTITIONS_TABLE}
+          GROUP BY policy_name
+        SQL
+        "partition_counts_by_policy",
+        []
+      )
+      result.to_a.each_with_object({}) do |r, h|
+        h[r["policy_name"]] = {
+          pending:    r["pending"].to_i,
+          partitions: r["partitions"].to_i,
+          paused:     r["paused"].to_i
+        }
+      end
+    end
+
+    # Per-policy round-trip stats in one grouped query, keyed by
+    # policy_name. Only the fields the dashboard overview renders
+    # (in_backoff, oldest/p95 age); use partition_round_trip_stats for the
+    # full single-policy breakdown. Replaces N per-policy calls on the
+    # dashboard. Same percentile-inversion note as partition_round_trip_stats.
+    def partition_round_trip_stats_by_policy
+      result = connection.exec_query(
+        <<~SQL.squish,
+          SELECT
+            p.policy_name,
+            COUNT(*) FILTER (WHERE p.next_eligible_at IS NOT NULL AND p.next_eligible_at > now())::int AS in_backoff,
+            EXTRACT(EPOCH FROM (now() - MIN(p.last_checked_at)))::float AS oldest_age_seconds,
+            EXTRACT(EPOCH FROM (now() - PERCENTILE_DISC(0.05) WITHIN GROUP (ORDER BY p.last_checked_at)))::float AS p95_age_seconds
+          FROM #{PARTITIONS_TABLE} p
+          WHERE p.status = 'active' AND p.pending_count > 0
+          GROUP BY p.policy_name
+        SQL
+        "partition_round_trip_stats_by_policy",
+        []
+      )
+      result.to_a.each_with_object({}) do |r, h|
+        h[r["policy_name"]] = {
+          in_backoff:         r["in_backoff"].to_i,
+          oldest_age_seconds: r["oldest_age_seconds"]&.to_f,
+          p95_age_seconds:    r["p95_age_seconds"]&.to_f
+        }
+      end
+    end
+
     # ----- adaptive_concurrency stats -----------------------------------------
 
     # Insert a fresh stats row for the given partition if none exists.

data/lib/dispatch_policy/tick.rb

@@ -201,19 +201,24 @@ module DispatchPolicy
         return { admitted: 0, failures: 0, reasons: deduce_reasons(result) }
       end
 
-      admitted = 0
+      admitted      = 0
+      settled_patch = nil
       half_life = @policy.fairness_half_life_seconds || @config.fairness_half_life_seconds
 
       Repository.with_connection do
         ActiveRecord::Base.transaction(requires_new: true) do
+          # The gate_state we persist depends on how many rows actually
+          # got claimed: each gate settles its state against the real
+          # admitted count via Pipeline.settle (the throttle deducts that
+          # many tokens, not the optimistic `allowed`). The block runs
+          # inside claim_staged_jobs! right after the DELETE.
           rows = Repository.claim_staged_jobs!(
             policy_name:       @policy_name,
             partition_key:     partition["partition_key"],
             limit:             result.admit_count,
-            gate_state_patch:  result.gate_state_patch,
             retry_after:       result.retry_after,
             half_life_seconds: half_life
-          )
+          ) { |admitted_count| settled_patch = Pipeline.settle(result.decisions, admitted_count) }
 
           # `claim_staged_jobs!` always runs `record_partition_admit!` so
           # the partition's counters and gate_state commit even when the
@@ -293,11 +298,13 @@ module DispatchPolicy
       # the STALE pre-pass-1 snapshot. For the throttle that means reading
       # the token bucket at its original level and double-spending —
       # admitting above the configured rate and overwriting pass-1's
-      # consumption. The shallow merge matches Postgres jsonb `||`.
-      # Only runs on a committed admit: if the TX raised we fall through to
-      # the rescue below and never touch the in-memory state.
-      if result.gate_state_patch&.any?
-        partition["gate_state"] = (partition["gate_state"] || {}).merge(result.gate_state_patch)
+      # consumption. We mirror the SETTLED patch (post-consume, charged for
+      # the real admitted count), not evaluate's pre-consume snapshot. The
+      # shallow merge matches Postgres jsonb `||`. Only runs on a committed
+      # admit: if the TX raised we fall through to the rescue below and
+      # never touch the in-memory state.
+      if settled_patch&.any?
+        partition["gate_state"] = (partition["gate_state"] || {}).merge(settled_patch)
       end
 
       if admitted.zero?

data/lib/dispatch_policy/tick_loop.rb

@@ -68,7 +68,10 @@ module DispatchPolicy
 
     def sweep!
       cfg = DispatchPolicy.config
-      Repository.sweep_stale_inflight!(cutoff_seconds: cfg.inflight_stale_after)
+      Repository.sweep_stale_inflight!(
+        cutoff_seconds:        cfg.inflight_stale_after,
+        queued_cutoff_seconds: cfg.inflight_queued_stale_after
+      )
       Repository.sweep_inactive_partitions!(cutoff_seconds: cfg.partition_inactive_after)
       Repository.sweep_old_tick_samples!(cutoff_seconds: cfg.metrics_retention)
     rescue StandardError => e

data/lib/dispatch_policy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DispatchPolicy
-  VERSION = "0.4.2"
+  VERSION = "0.4.3"
 end

data/lib/generators/dispatch_policy/install/templates/initializer.rb.tt

@@ -7,5 +7,6 @@ DispatchPolicy.configure do |c|
   c.idle_pause                = 0.5     # seconds slept when no admissions happened
   c.partition_inactive_after  = 24 * 60 * 60 # GC partitions idle this long
   c.inflight_stale_after      = 5 * 60      # GC inflight rows whose worker stopped heartbeating
+  c.inflight_queued_stale_after = 60 * 60   # GC inflight rows admitted but never started (still queued)
   c.sweep_every_ticks         = 50          # how often to run the sweepers
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dispatch_policy
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - José Galisteo