dispatch_policy 0.1.0

data/app/controllers/dispatch_policy/policies_controller.rb

@@ -0,0 +1,269 @@
+# frozen_string_literal: true
+
+module DispatchPolicy
+  class PoliciesController < ApplicationController
+    STALE_PENDING_THRESHOLD = 1.hour
+    PARTITION_LIST_PAGE_SIZE = 25
+
+    before_action :load_policy, only: :show
+
+    def index
+      @policies = DispatchPolicy.registry.map do |name, job_class|
+        scope   = StagedJob.where(policy_name: name)
+        pending = scope.pending
+        {
+          name:            name,
+          job_class:       job_class,
+          policy:          job_class.resolved_dispatch_policy,
+          pending_count:   pending.count,
+          admitted_count:  scope.admitted.count,
+          completed_24h:   scope.completed.where(completed_at: 24.hours.ago..).count,
+          oldest_pending:  pending.minimum(:staged_at),
+          stale_threshold: STALE_PENDING_THRESHOLD
+        }
+      end.sort_by { |p| -p[:pending_count] }
+
+      @active_partitions = PartitionInflightCount.where("in_flight > 0").count
+      @expired_leases    = StagedJob.expired_leases.count
+    end
+
+    def show
+      scope = StagedJob.where(policy_name: @policy_name)
+      @pending_count           = scope.pending.count
+      @pending_eligible_count  = scope.pending.where("not_before_at IS NULL OR not_before_at <= ?", Time.current).count
+      @pending_scheduled_count = @pending_count - @pending_eligible_count
+      @admitted_count          = scope.admitted.count
+      @completed_24h           = scope.completed.where(completed_at: 24.hours.ago..).count
+
+      all_breakdown = partition_breakdown(scope)
+
+      # "Watched" subset (passed via ?watch=a,b,c; the JS layer syncs it
+      # with localStorage so the choice sticks across reloads).
+      @watched_keys        = (params[:watch] || "").split(",").map(&:strip).reject(&:empty?)
+      @partition_breakdown = @watched_keys.any? ? all_breakdown.select { |r| @watched_keys.include?(r[:partition]) } : []
+
+      # Browsable list of every active partition with filter + sort + pagination.
+      @partition_search = params[:q].to_s.strip
+      @partition_page   = [ params[:page].to_i, 1 ].max
+      @partition_sort   = %w[source partition pending in_flight completed_24h last_enqueued_at last_dispatched_at].include?(params[:sort]) ? params[:sort] : "activity"
+      @partition_dir    = params[:dir] == "asc" ? "asc" : "desc"
+
+      list = all_breakdown
+      list = list.select { |r| r[:partition].to_s.downcase.include?(@partition_search.downcase) } if @partition_search.present?
+      list = sort_partition_list(list, @partition_sort, @partition_dir)
+
+      @partition_total_list = list.size
+      offset                = (@partition_page - 1) * PARTITION_LIST_PAGE_SIZE
+      @partition_list       = list[offset, PARTITION_LIST_PAGE_SIZE] || []
+
+      load_adaptive_chart_data
+      @throttle_buckets = ThrottleBucket
+        .where(policy_name: @policy_name).order(:gate_name, :partition_key).limit(50)
+      @pending_jobs = scope.pending.order(:priority, :staged_at).limit(50)
+    end
+
+    private
+
+    def load_policy
+      @policy_name = params[:policy_name]
+      @job_class   = DispatchPolicy.registry[@policy_name] ||
+                     Tick.autoload_job_for(@policy_name)
+      raise ActiveRecord::RecordNotFound unless @job_class
+      @policy = @job_class.resolved_dispatch_policy
+    end
+
+    # Per-(source, partition) breakdown of pending-eligible / pending-scheduled
+    # / in-flight / completed-24h. A "source" is either a gate with a
+    # partition_by (uses gate.partition_key_for(context)) or the policy's
+    # round_robin_by declaration (uses the round_robin_key column directly).
+    # All four counts come from StagedJob groupings; PartitionInflightCount
+    # is an admission-time optimization, not the user-facing truth.
+    def partition_breakdown(scope)
+      sources = partition_sources
+      return [] if sources.empty?
+
+      now       = Time.current
+      now_iso   = now.iso8601
+      since_24h = 24.hours.ago.iso8601
+
+      adaptive_stats = AdaptiveConcurrencyStats.where(policy_name: @policy_name)
+        .pluck(:gate_name, :partition_key, :current_max, :ewma_latency_ms)
+        .each_with_object({}) { |(g, k, c, l), h|
+          h[[ g, k ]] = { current_max: c, ewma_latency_ms: l.to_f.round(1) }
+        }
+
+      rows = Hash.new { |h, k|
+        h[k] = {
+          source:             k[0],
+          partition:          k[1],
+          eligible:           0,
+          scheduled:          0,
+          in_flight:          0,
+          completed_24h:      0,
+          last_enqueued_at:   nil,
+          last_dispatched_at: nil,
+          current_max:        nil,
+          ewma_latency_ms:    nil
+        }
+      }
+
+      # Activity timestamps bounded to the last 24h so the scan stays on
+      # an index-friendly slice of staged_jobs.
+      activity_rows = scope
+        .where("staged_at > ?", since_24h)
+        .group(:context, :round_robin_key)
+        .pluck(
+          :context,
+          :round_robin_key,
+          Arel.sql("MAX(staged_at)"),
+          Arel.sql("MAX(admitted_at)")
+        )
+
+      sources.each do |name, extract|
+        pending_counts = scope.pending.group(:context, :round_robin_key).pluck(
+          :context,
+          :round_robin_key,
+          Arel.sql("count(*) filter (where not_before_at is null or not_before_at <= '#{now_iso}')"),
+          Arel.sql("count(*) filter (where not_before_at > '#{now_iso}')")
+        )
+        pending_counts.each do |ctx, rr_key, eligible, scheduled|
+          partition = extract.call(ctx, rr_key)
+          row = rows[[ name, partition ]]
+          row[:eligible]  += eligible
+          row[:scheduled] += scheduled
+        end
+
+        admitted_counts = scope.admitted.group(:context, :round_robin_key).pluck(
+          :context, :round_robin_key, Arel.sql("count(*)")
+        )
+        admitted_counts.each do |ctx, rr_key, in_flight|
+          partition = extract.call(ctx, rr_key)
+          rows[[ name, partition ]][:in_flight] += in_flight
+        end
+
+        completed_counts = scope.completed.where("completed_at > ?", since_24h)
+          .group(:context, :round_robin_key).pluck(
+            :context, :round_robin_key, Arel.sql("count(*)")
+          )
+        completed_counts.each do |ctx, rr_key, completed|
+          partition = extract.call(ctx, rr_key)
+          rows[[ name, partition ]][:completed_24h] += completed
+        end
+
+        activity_rows.each do |ctx, rr_key, last_staged, last_admitted|
+          partition = extract.call(ctx, rr_key)
+          row       = rows[[ name, partition ]]
+          row[:last_enqueued_at]   = [ row[:last_enqueued_at], last_staged ].compact.max
+          row[:last_dispatched_at] = [ row[:last_dispatched_at], last_admitted ].compact.max
+        end
+      end
+
+      rows.each do |(source, partition), row|
+        stats = adaptive_stats[[ source, partition ]]
+        next unless stats
+        row[:current_max]     = stats[:current_max]
+        row[:ewma_latency_ms] = stats[:ewma_latency_ms]
+      end
+
+      # Two different sources (say round_robin_by account_id + a gate
+      # partitioned by account_id) producing the same partition key yield
+      # identical counts — collapse them into one row with a merged source
+      # label instead of listing the same numbers twice.
+      merged = rows.values
+        .reject { |r| r[:partition].nil? || r[:partition].empty? }
+        .group_by { |r| [ r[:partition], r[:eligible], r[:scheduled], r[:in_flight], r[:completed_24h] ] }
+        .map { |_, group|
+          base = group.first.dup
+          base[:source] = group.map { |r| r[:source] }.uniq.sort.join(" + ")
+          group.each do |r|
+            base[:current_max]        ||= r[:current_max]
+            base[:ewma_latency_ms]    ||= r[:ewma_latency_ms]
+            base[:last_enqueued_at]     = [ base[:last_enqueued_at], r[:last_enqueued_at] ].compact.max
+            base[:last_dispatched_at]   = [ base[:last_dispatched_at], r[:last_dispatched_at] ].compact.max
+          end
+          base
+        }
+
+      merged.sort_by { |r|
+        [ -(r[:eligible] + r[:scheduled] + r[:in_flight] + r[:completed_24h]), r[:source], r[:partition] ]
+      }
+    end
+
+    def sort_partition_list(list, sort, dir)
+      # Put nulls at the bottom regardless of direction (Time#to_f on nil
+      # would crash; -Float::INFINITY sorts first, +Float::INFINITY last).
+      key =
+        case sort
+        when "source"             then ->(r) { [ r[:source], r[:partition] ] }
+        when "partition"          then ->(r) { r[:partition] }
+        when "pending"            then ->(r) { r[:eligible] + r[:scheduled] }
+        when "in_flight"          then ->(r) { r[:in_flight] }
+        when "completed_24h"      then ->(r) { r[:completed_24h] }
+        when "last_enqueued_at"   then ->(r) { r[:last_enqueued_at]&.to_f || 0 }
+        when "last_dispatched_at" then ->(r) { r[:last_dispatched_at]&.to_f || 0 }
+        else ->(r) { r[:eligible] + r[:scheduled] + r[:in_flight] + r[:completed_24h] }
+        end
+      sorted = list.sort_by(&key)
+      dir == "asc" ? sorted : sorted.reverse
+    end
+
+    # Returns [[source_name, ->(ctx, rr_key) { partition_key }], ...]
+    # covering every partition-producing declaration on the policy: every
+    # gate with a partition_by, plus round_robin_by if declared.
+    def partition_sources
+      return [] unless @policy
+
+      sources = @policy.gates.select(&:partition_by).map do |gate|
+        [ gate.name.to_s, ->(ctx, _rr) { gate.partition_key_for((ctx || {}).symbolize_keys) } ]
+      end
+      sources << [ "round_robin_by", ->(_ctx, rr) { rr } ] if @policy.round_robin?
+      sources
+    end
+
+    # Build chart data from PartitionObservation. Two queries:
+    # - Global aggregated (one row per minute): cheap even with 1000s of
+    #   partitions because we SUM/AVG in SQL, not in Ruby.
+    # - Per-partition sparkline data, scoped to only the partitions we're
+    #   going to actually render (breakdown's top N).
+    def load_adaptive_chart_data
+      last_minute   = Time.current.utc.beginning_of_minute
+      @chart_slots  = (0..59).map { |i| last_minute - (59 - i).minutes }
+      @chart_labels = @chart_slots.map { |t| t.strftime("%H:%M") }
+      slot_index    = @chart_slots.each_with_index.to_h
+
+      @adaptive_global    = Array.new(@chart_slots.size)
+      @completions_global = Array.new(@chart_slots.size, 0)
+      global_rows = PartitionObservation
+        .where(policy_name: @policy_name)
+        .where("minute_bucket >= ?", @chart_slots.first)
+        .group(:minute_bucket)
+        .pluck(:minute_bucket, Arel.sql("SUM(total_lag_ms)"), Arel.sql("SUM(observation_count)"))
+      global_rows.each do |bucket, total_lag, obs_count|
+        idx = slot_index[bucket.utc.beginning_of_minute]
+        next unless idx
+        @completions_global[idx] = obs_count
+        @adaptive_global[idx]    = obs_count.positive? ? (total_lag.to_f / obs_count).round(1) : nil
+      end
+
+      partition_keys = (@partition_breakdown || []).map { |r| r[:partition] }.uniq
+      @adaptive_samples    = {}
+      @completions_samples = {}
+      return if partition_keys.empty?
+
+      per_partition_lag    = Hash.new { |h, k| h[k] = Array.new(@chart_slots.size) }
+      per_partition_counts = Hash.new { |h, k| h[k] = Array.new(@chart_slots.size, 0) }
+      rows = PartitionObservation
+        .where(policy_name: @policy_name, partition_key: partition_keys)
+        .where("minute_bucket >= ?", @chart_slots.first)
+        .pluck(:partition_key, :minute_bucket, :total_lag_ms, :observation_count)
+      rows.each do |pk, bucket, total, count|
+        idx = slot_index[bucket.utc.beginning_of_minute]
+        next unless idx
+        per_partition_lag[pk][idx]    = count.positive? ? (total.to_f / count).round(1) : nil
+        per_partition_counts[pk][idx] = count
+      end
+      @adaptive_samples    = per_partition_lag
+      @completions_samples = per_partition_counts
+    end
+  end
+end

data/app/models/dispatch_policy/adaptive_concurrency_stats.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module DispatchPolicy
+  class AdaptiveConcurrencyStats < ApplicationRecord
+    self.table_name = "dispatch_policy_adaptive_concurrency_stats"
+
+    # Seed a stats row if one doesn't exist yet. Mirrors ThrottleBucket.lock.
+    def self.seed!(policy_name:, gate_name:, partition_key:, initial_max:)
+      now = Time.current
+      sql = <<~SQL.squish
+        INSERT INTO #{quoted_table_name}
+          (policy_name, gate_name, partition_key, current_max,
+           ewma_latency_ms, sample_count, created_at, updated_at)
+        VALUES (?, ?, ?, ?, 0, 0, ?, ?)
+        ON CONFLICT (policy_name, gate_name, partition_key) DO NOTHING
+      SQL
+      connection.exec_update(
+        sanitize_sql_array([
+          sql, policy_name, gate_name.to_s, partition_key.to_s,
+          initial_max.to_i, now, now
+        ])
+      )
+    end
+
+    def self.fetch_many(policy_name:, gate_name:, partition_keys:)
+      return {} if partition_keys.empty?
+      where(policy_name: policy_name, gate_name: gate_name.to_s, partition_key: partition_keys)
+        .pluck(:partition_key, :current_max, :ewma_latency_ms)
+        .each_with_object({}) { |(k, c, l), h| h[k] = { current_max: c, ewma_latency_ms: l } }
+    end
+
+    # Single-statement EWMA + AIMD update so concurrent performs can't race
+    # on read-modify-write. Seed first (INSERT ON CONFLICT DO NOTHING), then
+    # apply the adjustment.
+    def self.record_observation!(
+      policy_name:, gate_name:, partition_key:,
+      queue_lag_ms:, succeeded:,
+      alpha:, min:, target_lag_ms:,
+      fail_factor:, slow_factor:, initial_max:
+    )
+      seed!(
+        policy_name:   policy_name,
+        gate_name:     gate_name,
+        partition_key: partition_key,
+        initial_max:   initial_max
+      )
+
+      # Feedback signal is queue_lag (admitted_at → perform_start). When
+      # the adapter queue is empty, lag ≈ 0 → +1 grow. When the queue
+      # backs up, lag rises past target → multiplicative shrink. Failures
+      # shrink harder. Only `min` is enforced so a partition can't lock
+      # out entirely.
+      sql = <<~SQL.squish
+        UPDATE #{quoted_table_name}
+        SET
+          ewma_latency_ms = ewma_latency_ms * (1 - ?) + ? * ?,
+          sample_count    = sample_count + 1,
+          current_max = GREATEST(?, CASE
+            WHEN ? = FALSE                                THEN FLOOR(current_max * ?)::int
+            WHEN (ewma_latency_ms * (1 - ?) + ? * ?) > ?  THEN FLOOR(current_max * ?)::int
+            ELSE current_max + 1
+          END),
+          last_observed_at = ?,
+          updated_at       = ?
+        WHERE policy_name = ? AND gate_name = ? AND partition_key = ?
+      SQL
+
+      now = Time.current
+      connection.exec_update(
+        sanitize_sql_array([
+          sql,
+          alpha, alpha, queue_lag_ms,
+          min.to_i,
+          succeeded, fail_factor,
+          alpha, alpha, queue_lag_ms, target_lag_ms, slow_factor,
+          now, now,
+          policy_name, gate_name.to_s, partition_key.to_s
+        ])
+      )
+    end
+
+    # Quick lookup used by Dispatchable to denormalize current_max into
+    # the generic partition observation row.
+    def self.current_max_for(policy_name:, partition_key:)
+      where(policy_name: policy_name, partition_key: partition_key.to_s)
+        .limit(1).pick(:current_max)
+    end
+  end
+end

data/app/models/dispatch_policy/application_record.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module DispatchPolicy
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/dispatch_policy/partition_inflight_count.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module DispatchPolicy
+  class PartitionInflightCount < ApplicationRecord
+    self.table_name = "dispatch_policy_partition_counts"
+
+    def self.fetch_many(policy_name:, gate_name:, partition_keys:)
+      return {} if partition_keys.empty?
+
+      where(policy_name: policy_name, gate_name: gate_name.to_s, partition_key: partition_keys)
+        .pluck(:partition_key, :in_flight).to_h
+        .tap { |h| partition_keys.each { |k| h[k] ||= 0 } }
+    end
+
+    def self.total_for(policy_name:, gate_name:)
+      where(policy_name: policy_name, gate_name: gate_name.to_s).sum(:in_flight)
+    end
+
+    def self.increment(policy_name:, gate_name:, partition_key:, by: 1)
+      now = Time.current
+      sql = <<~SQL.squish
+        INSERT INTO #{quoted_table_name}
+          (policy_name, gate_name, partition_key, in_flight, created_at, updated_at)
+        VALUES (?, ?, ?, ?, ?, ?)
+        ON CONFLICT (policy_name, gate_name, partition_key)
+        DO UPDATE SET
+          in_flight  = #{quoted_table_name}.in_flight + EXCLUDED.in_flight,
+          updated_at = EXCLUDED.updated_at
+      SQL
+      connection.exec_update(
+        sanitize_sql_array([ sql, policy_name, gate_name.to_s, partition_key.to_s, by, now, now ])
+      )
+    end
+
+    def self.decrement(policy_name:, gate_name:, partition_key:, by: 1)
+      where(policy_name: policy_name, gate_name: gate_name.to_s, partition_key: partition_key.to_s)
+        .update_all([
+          "in_flight = GREATEST(in_flight - ?, 0), updated_at = ?", by, Time.current
+        ])
+    end
+  end
+end

data/app/models/dispatch_policy/partition_observation.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module DispatchPolicy
+  # Minute-bucketed observability per (policy, partition). Any gate with
+  # partition_by gets an observation row here — adaptive, throttle,
+  # concurrency, whatever — so the admin chart shows queue lag / throughput
+  # for all partitioned policies, not just the adaptive ones.
+  #
+  # One row per (policy, partition, minute): total_lag_ms accumulates the
+  # sum of queue_lag_ms observations in that minute, observation_count
+  # increments, max_lag_ms tracks the worst spike. Average lag for the
+  # bucket is derived on read as total / count.
+  class PartitionObservation < ApplicationRecord
+    self.table_name = "dispatch_policy_partition_observations"
+
+    OBSERVATION_TTL = 2 * 60 * 60  # 2 hours
+
+    def self.observe!(policy_name:, partition_key:, queue_lag_ms:, current_max: nil)
+      return if partition_key.nil? || partition_key.to_s.empty?
+
+      now = Time.current
+      lag = queue_lag_ms.to_i
+      sql = <<~SQL.squish
+        INSERT INTO #{quoted_table_name}
+          (policy_name, partition_key, minute_bucket,
+           total_lag_ms, observation_count, max_lag_ms, current_max,
+           created_at, updated_at)
+        VALUES (?, ?, date_trunc('minute', ?::timestamp), ?, 1, ?, ?, ?, ?)
+        ON CONFLICT (policy_name, partition_key, minute_bucket)
+        DO UPDATE SET
+          total_lag_ms      = #{quoted_table_name}.total_lag_ms + EXCLUDED.total_lag_ms,
+          observation_count = #{quoted_table_name}.observation_count + 1,
+          max_lag_ms        = GREATEST(#{quoted_table_name}.max_lag_ms, EXCLUDED.max_lag_ms),
+          current_max       = COALESCE(EXCLUDED.current_max, #{quoted_table_name}.current_max),
+          updated_at        = EXCLUDED.updated_at
+      SQL
+      connection.exec_update(
+        sanitize_sql_array([
+          sql, policy_name, partition_key.to_s, now,
+          lag, lag, current_max, now, now
+        ])
+      )
+    end
+
+    def self.prune!
+      where("minute_bucket < ?", Time.current - OBSERVATION_TTL).delete_all
+    end
+  end
+end

data/app/models/dispatch_policy/staged_job.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module DispatchPolicy
+  class StagedJob < ApplicationRecord
+    self.table_name = "dispatch_policy_staged_jobs"
+
+    scope :pending,   -> { where(admitted_at: nil, completed_at: nil) }
+    scope :admitted,  -> { where.not(admitted_at: nil).where(completed_at: nil) }
+    scope :completed, -> { where.not(completed_at: nil) }
+    scope :active,    -> { where(completed_at: nil) }
+    scope :expired_leases, -> {
+      admitted.where("lease_expires_at IS NOT NULL AND lease_expires_at < ?", Time.current)
+    }
+
+    # Merge the job's ActiveJob metadata (queue_name, priority) into the
+    # context hash so gate lambdas can partition_by :queue_name without
+    # the user having to pass it as a kwarg. User-provided keys win.
+    def self.context_for(job_instance, policy)
+      built = policy.context_builder.call(job_instance.arguments)
+      return built unless built.is_a?(Hash)
+      {
+        queue_name: job_instance.queue_name,
+        priority:   job_instance.priority
+      }.merge(built.symbolize_keys)
+    end
+
+    # Stages a job in the admission queue. Returns the created row, or nil if
+    # the policy declares a dedupe_key and an active row already exists.
+    def self.stage!(job_instance:, policy:)
+      dedupe_key = policy.build_dedupe_key(job_instance.arguments)
+
+      if dedupe_key && exists?(policy_name: policy.name, dedupe_key: dedupe_key, completed_at: nil)
+        return nil
+      end
+
+      create!(
+        job_class:       job_instance.class.name,
+        policy_name:     policy.name,
+        arguments:       job_instance.serialize,
+        snapshot:        policy.build_snapshot(job_instance.arguments),
+        context:         context_for(job_instance, policy),
+        priority:        job_instance.priority || 100,
+        not_before_at:   job_instance.scheduled_at,
+        staged_at:       Time.current,
+        dedupe_key:      dedupe_key,
+        round_robin_key: policy.build_round_robin_key(job_instance.arguments)
+      )
+    rescue ActiveRecord::RecordNotUnique
+      nil
+    end
+
+    # Batch-insert variant of stage!.
+    def self.stage_many!(policy:, jobs:)
+      return 0 if jobs.empty?
+
+      now = Time.current
+      rows = jobs.map do |job_instance|
+        {
+          job_class:       job_instance.class.name,
+          policy_name:     policy.name,
+          arguments:       job_instance.serialize,
+          snapshot:        policy.build_snapshot(job_instance.arguments),
+          context:         context_for(job_instance, policy),
+          priority:        job_instance.priority || 100,
+          not_before_at:   job_instance.scheduled_at,
+          staged_at:       now,
+          dedupe_key:      policy.build_dedupe_key(job_instance.arguments),
+          round_robin_key: policy.build_round_robin_key(job_instance.arguments),
+          partitions:      {},
+          created_at:      now,
+          updated_at:      now
+        }
+      end
+
+      result = insert_all(rows, unique_by: :idx_dp_staged_dedupe_active)
+      result.rows.size
+    end
+
+    def self.mark_completed_by_active_job_id(active_job_id)
+      return 0 if active_job_id.blank?
+      where(active_job_id: active_job_id, completed_at: nil)
+        .update_all(completed_at: Time.current, lease_expires_at: nil)
+    end
+
+    def mark_admitted!(partitions:)
+      now = Time.current
+      job = instantiate_active_job
+      job._dispatch_partitions  = partitions
+      job._dispatch_admitted_at = now
+
+      update!(
+        admitted_at:      now,
+        lease_expires_at: now + DispatchPolicy.config.lease_duration,
+        active_job_id:    job.job_id,
+        partitions:       partitions
+      )
+
+      job
+    end
+
+    def instantiate_active_job
+      ActiveJob::Base.deserialize(arguments)
+    end
+  end
+end

data/app/models/dispatch_policy/throttle_bucket.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module DispatchPolicy
+  class ThrottleBucket < ApplicationRecord
+    self.table_name = "dispatch_policy_throttle_buckets"
+
+    def self.lock(policy_name:, gate_name:, partition_key:, burst:)
+      now = Time.current
+      seed_sql = <<~SQL.squish
+        INSERT INTO #{quoted_table_name}
+          (policy_name, gate_name, partition_key, tokens, refilled_at, created_at, updated_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?)
+        ON CONFLICT (policy_name, gate_name, partition_key) DO NOTHING
+      SQL
+      connection.exec_update(
+        sanitize_sql_array([
+          seed_sql, policy_name, gate_name.to_s, partition_key.to_s,
+          burst.to_f, now, now, now
+        ])
+      )
+
+      where(policy_name: policy_name, gate_name: gate_name.to_s, partition_key: partition_key.to_s)
+        .lock("FOR UPDATE")
+        .first!
+    end
+
+    def refill!(rate:, per:, burst:)
+      now = Time.current
+      elapsed = (now - refilled_at).to_f
+      new_tokens = tokens + (rate * elapsed / per)
+      self.tokens = [ new_tokens, burst.to_f ].min
+      self.refilled_at = now
+    end
+
+    def consume(n = 1)
+      return false if tokens < n
+      self.tokens -= n
+      true
+    end
+  end
+end

data/app/views/dispatch_policy/policies/index.html.erb

@@ -0,0 +1,52 @@
+<h1>DispatchPolicy</h1>
+<p class="muted">Admission-control policies registered in this app.</p>
+
+<div class="summary">
+  <div class="summary-item">
+    <div class="label">Active partitions</div>
+    <div class="value"><%= @active_partitions %></div>
+  </div>
+  <div class="summary-item">
+    <div class="label">Expired leases</div>
+    <div class="value"><%= @expired_leases %></div>
+  </div>
+</div>
+
+<h2>Policies</h2>
+<% if @policies.empty? %>
+  <p class="muted">No policies registered yet.</p>
+<% else %>
+  <table>
+    <thead>
+      <tr>
+        <th>Policy</th>
+        <th>Job class</th>
+        <th>Pending</th>
+        <th>Admitted</th>
+        <th>Completed (24h)</th>
+        <th>Oldest pending</th>
+      </tr>
+    </thead>
+    <tbody>
+      <% @policies.each do |p| %>
+        <tr>
+          <td><%= link_to p[:name], policy_path(policy_name: p[:name]) %></td>
+          <td><code><%= p[:job_class].name %></code></td>
+          <td><%= p[:pending_count] %></td>
+          <td><%= p[:admitted_count] %></td>
+          <td><%= p[:completed_24h] %></td>
+          <td>
+            <% if p[:oldest_pending] %>
+              <% stale = p[:oldest_pending] < Time.current - p[:stale_threshold] %>
+              <span class="<%= 'stale' if stale %>">
+                <%= time_ago_in_words(p[:oldest_pending]) %> ago
+              </span>
+            <% else %>
+              <span class="muted">—</span>
+            <% end %>
+          </td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+<% end %>