@kici-dev/orchestrator 0.5.0 → 0.6.1

package/dist/scaler/manager.d.ts

@@ -6,10 +6,25 @@
  * enforces global limits, tracks spawning agents to prevent over-provisioning,
  * and manages the agent lifecycle from spawn to destroy.
  */
-import type { ResourceRequest } from '@kici-dev/engine';
+import type { ResourceRequest, ScalerCapacitySummary } from '@kici-dev/engine';
+import { ScaleDownReason } from './scaler-events.js';
+import type { ScalerEventEmitterLike } from './event-backend.js';
+import type { ClaimStore, ClaimedCredentials } from './claim-store.js';
 import type { BackendFailureSummary } from './failure-tracker.js';
-import type { ScalerBackend, ScalerConfig, ScaleResult, ScalerEvent, ResourceCap, ValidationResult } from './types.js';
-import type { ScalerStateStore, ScalerStateRecovery } from './scaler-state-store.js';
+import type { ScalerBackend, ScalerConfig, ScalerEntry, ScaleResult, ScalerEvent, ResourceCap, ValidationResult, ResolvedContainerSpawn } from './types.js';
+import type { ScalerStateStore, ScalerStateRecovery, ReapCandidate } from './scaler-state-store.js';
+/**
+ * Raised when a registering agent presents a scaler-minted id that the
+ * orchestrator cannot back with a spawn record — either because the lookup
+ * could not be completed (`cause` set) or because no record exists at all
+ * (`cause` omitted). The WS handler answers it by refusing the registration, so
+ * the agent reconnects rather than registering with no `mandatoryLabels` gate.
+ */
+export declare class ScalerAdoptionLookupError extends Error {
+    readonly agentId: string;
+    readonly cause?: unknown;
+    constructor(agentId: string, cause?: unknown);
+}
 /**
  * Counting semaphore that throttles the number of concurrent async operations.
  *
@@ -111,18 +126,184 @@ export interface ScalerStatus {
         /** Machine-pool reference, if any. */
         machinePool?: string;
         /**
-         * Labels a job MUST declare in `runsOn` to be allowed on this backend.
-         * Empty array = no gate. Surfaced in heartbeat-side scaler capacity
-         * summaries so cross-peer routing applies the same gate.
+         * The union of every entry in {@link labelSetMandatoryLabels}. Surfaced in
+         * heartbeat-side scaler capacity summaries for a peer that predates the
+         * per-label-set gate.
+         *
+         * @deprecated Use {@link labelSetMandatoryLabels}. On a scaler whose label
+         * sets declare different platforms this union names a taint no single set
+         * can satisfy, which is what made a mixed-platform scaler unroutable.
          */
         mandatoryLabels: string[];
+        /**
+         * Labels a job MUST declare in `runsOn` to be allowed on each label set,
+         * index-aligned with `labelSets`. An empty entry means that set has no
+         * gate.
+         */
+        labelSetMandatoryLabels: string[][];
+        /**
+         * True when this scaler was removed from the config and is draining: it
+         * accepts no new work, and disappears once its last agent goes away.
+         */
+        retiring: boolean;
+    }>;
+}
+/**
+ * What the coordinator holding an event agent needs to tear its provision
+ * down: enough to emit `kici.scaler.scale-down` with no local backend and no
+ * config entry.
+ *
+ * Recorded on both registration paths — the agent adopted from a peer and the
+ * one this coordinator spawned itself — so the teardown addresses the targets
+ * recorded at spawn either way.
+ */
+export interface AdoptedSpec {
+    scalerName: string;
+    provisioningTargets: string[];
+}
+/**
+ * The three knobs bounding how often a repeatedly failing external scaler is
+ * asked to provision again.
+ */
+export interface ProvisionBackoffSettings {
+    /** First deferral after one consecutive failure; doubles per further failure. */
+    baseMs: number;
+    /** Ceiling on the doubling. */
+    maxMs: number;
+    /** Consecutive failures past which a refusal names repeated failure. */
+    maxConsecutiveFailures: number;
+}
+/** Everything the {@link ScalerManager} is constructed from. */
+export interface ScalerManagerDeps {
+    config: ScalerConfig;
+    backends: Array<{
+        name: string;
+        backend: ScalerBackend;
     }>;
+    /**
+     * Read-only view of the agent registry, for the warm pool's deficit
+     * calculation. Structural on purpose — the scaler must not depend on the
+     * agent layer's concrete class.
+     *
+     * Both hosts pass one, and a host that skips it does not get an inert warm
+     * pool: the readiness count falls back to 0, so a configured pool never sees
+     * the agents it started and keeps filling until `warmCapacityRemaining`
+     * clamps it at `maxAgents` — starving the job-bound spawns that share the
+     * cap. Optional only because the unit tests drive the spawn path without a
+     * registry.
+     */
+    agentRegistry?: {
+        findAvailable(requiredLabels: string[]): Array<{
+            agentId: string;
+            activeJobs: number;
+            registeredAt: number;
+        }>;
+    };
+    /**
+     * Id of the orchestrator instance this manager runs on. Rows this
+     * coordinator writes to the shared scaler tables are attributed to it, so
+     * every peer can tell which instance owns a given spawn.
+     */
+    instanceId: string;
+    /** Callback for relaying scaler events with runId/jobId context. */
+    onScalerEvent?: (runId: string, jobId: string, event: ScalerEvent) => void;
+    /**
+     * Optional DB-backed state store. Tests omit it; production wires it
+     * up via the orchestrator-core bootstrap.
+     */
+    stateStore?: ScalerStateStore;
+    /**
+     * Late-bound reserved-event emitter, resolved per call because the event
+     * router is built after the manager. Omitted where there is no event plane
+     * (the worker, and unit tests that never emit).
+     */
+    eventEmitter?: () => ScalerEventEmitterLike;
+    /**
+     * Redemption-only claim store. Wired wherever the shared claim table is
+     * reachable, so a provisioning code minted by any coordinator can be
+     * redeemed here. Omitted where there is no database.
+     */
+    claimStore?: ClaimStore;
+    /**
+     * Optional machine-ledger options. When `machinePools` are configured,
+     * the manager initializes a `MachineLedger` keyed off this directory and
+     * the orchestrator's instance id; reservations sum across orchestrators
+     * on the same host.
+     */
+    machineLedger?: {
+        /** Override the on-disk ledger directory. Falls back to `KICI_MACHINE_LEDGER_DIR`. */
+        dir?: string;
+        /** Orchestrator instance id (used in ledger rows for ownership). */
+        instanceId: string;
+    };
+    /** Coordinator drain predicate. When true, requestScale() declines to spawn
+     *  fresh capacity for the held Pending backlog. Defaults to never-draining. */
+    isDraining?: () => boolean;
+    /**
+     * Cluster-wide default deadline (ms) for a single `backend.spawn`. Always
+     * supplied (from config.scalerSpawnTimeoutMs); the per-org resolver, when
+     * present, overrides it per tenant.
+     */
+    spawnTimeoutMs: number;
+    /**
+     * Optional per-org resolver for the spawn deadline: given the job's org
+     * (jobConfig.cacheOrgId, undefined for warm-pool spawns), returns the
+     * effective timeout, falling back to the cluster default internally. Wired
+     * in orchestrator-core (has DB); omitted by the worker + unit tests → the
+     * cluster default spawnTimeoutMs is used.
+     */
+    resolveSpawnTimeoutMs?: (orgId: string | undefined) => Promise<number>;
+    /**
+     * Live resolver for the external-provision backoff knobs, read once per spawn
+     * request so an operator retunes a running cluster. Wired in
+     * orchestrator-core from `ClusterSettingsReader` + the config defaults;
+     * omitted by the worker and unit tests, which then get
+     * {@link DEFAULT_PROVISION_BACKOFF}.
+     */
+    resolveProvisionBackoff?: () => Promise<ProvisionBackoffSettings>;
+    /**
+     * Constructs a backend for a scaler added by a config reload. Supplied by
+     * both hosts from the shared backend factory, so a reload can only build
+     * what startup already exercises. When omitted (unit tests), a reload that
+     * adds a scaler is rejected rather than silently ignored.
+     *
+     * `config` is the config being reloaded, not the boot-time one. The factory
+     * reads `defaults.resources` and the whole `firecracker` network block off
+     * it, so an added scaler built from a stale config would allocate IPs from
+     * the old CIDR and attach TAPs to the old bridge — silently unreachable.
+     */
+    createBackend?: (entry: ScalerEntry, config: ScalerConfig) => Promise<ScalerBackend | null>;
 }
 export declare class ScalerManager {
     private readonly backends;
     private readonly backendRoles;
+    /**
+     * Scalers deleted from the config that still have agents running. They stay
+     * in `backends` — and so stay counted by `getGlobalActiveCount` and reachable
+     * for destroy / log forwarding — but are filtered out of routing, and are
+     * torn down once their last agent goes away.
+     */
+    private readonly retiring;
+    /** When each retiring scaler was marked, for the drain-duration log. */
+    private readonly retiredAt;
+    /** Periodic sweep that tears down drained retiring backends. */
+    private retirementSweep;
     /** Recent scaler spawn failures, surfaced by `kici-admin diagnose`. */
     private readonly failureTracker;
+    /**
+     * Consecutive external-provision failures per scaler NAME, and the deferral
+     * they earned.
+     *
+     * Keyed by name rather than by backend type so one failing scaler never
+     * defers spawns for an unrelated one — two event scalers routinely drive two
+     * different providers, and an outage at one says nothing about the other.
+     *
+     * In-memory and leader-local, like `ScalerFailureTracker`: the state resets on
+     * restart, which is the conservative direction (a fresh coordinator retries
+     * once immediately rather than inheriting a deferral it never observed).
+     */
+    private readonly provisionFailures;
+    private readonly resolveProvisionBackoff;
     private globalMaxAgents;
     /** Per-scaler resource caps (`{ maxCpu, maxMemoryBytes }`), keyed by scaler name. */
     private readonly resourceCaps;
@@ -151,12 +332,31 @@ export declare class ScalerManager {
     private reservationLock;
     /**
      * File-backed cross-process ledger for named machine pools (optional).
-     * Lazily initialized in the constructor when at least one scaler entry
-     * references a pool.
+     * Initialized when at least one scaler entry references a pool — at
+     * construction, or on the reload that first declares one.
+     */
+    private machineLedger;
+    /** Ledger options, retained so a reload that first declares a pool can build one. */
+    private readonly machineLedgerOptions;
+    /** True between `start()` and `shutdownAll()`; a ledger built later must be started too. */
+    private started;
+    /** Builds a backend for a scaler a reload added. Absent in unit tests. */
+    private readonly createBackend?;
+    /**
+     * Read-only agent-registry view backing the warm pool's deficit pass.
+     * Absent in unit tests and in any host that has no registry.
      */
-    private readonly machineLedger;
+    private readonly agentRegistry?;
     /** Per-scaler URL overrides from config, keyed by scaler name */
     private readonly scalerUrls;
+    /**
+     * Per-scaler `provisioningTargets` from config, keyed by scaler name.
+     *
+     * Copied onto every spawn row, because it is the only thing that lets a
+     * coordinator with no config entry for the scaler address the customer's
+     * teardown workflow when the agent it adopted disconnects.
+     */
+    private readonly scalerProvisioningTargets;
     /**
      * Tracks agents being spawned but not yet registered via WS.
      * Keyed by pre-generated agentId.
@@ -166,6 +366,32 @@ export declare class ScalerManager {
      * Maps registered agentId to backendName for lifecycle events.
      */
     private readonly managedAgentIndex;
+    /**
+     * Agents this instance pre-spawned to wait for work, rather than for a
+     * specific queued job. The agent registry tracks readiness (`activeJobs`,
+     * `registeredAt`) and has never tracked provenance — it cannot derive this,
+     * and the spawning entry that carries it is deleted at registration.
+     *
+     * This is not a second copy of the pool's readiness. It is provenance, which
+     * nothing else knows, and only the reaper reads it: it may destroy an agent
+     * the pool itself created, never one that registered for a bound job whose
+     * dispatch has not yet arrived.
+     */
+    private readonly warmAgents;
+    /**
+     * Warm agents whose destroy is issued but has not settled yet.
+     *
+     * An agent leaves `warmAgents` only once its destroy resolves, so until then
+     * `listIdle` keeps returning it and the next reap pass — 30 seconds later,
+     * or a reload that lowers the target — selects the very same agent again. A
+     * backend whose teardown outlives a tick would be handed a second destroy for
+     * an agent it is already destroying, and every extra pass would count another
+     * `kici_orch_scaler_warm_pool_reaped_total`. One entry per agent, cleared
+     * when the destroy settles either way.
+     */
+    private readonly warmDestroying;
+    /** Agents this instance adopted from another instance's spawn record. */
+    private readonly adoptedAgents;
     /**
      * Active log forwarders for scaler-managed agents (container/bare-metal).
      * Keyed by agentId. Each forwarder consumes a LogCapture stream.
@@ -203,6 +429,11 @@ export declare class ScalerManager {
     /** Trailing debounce window for {@link notifyCapacityFreed}. */
     private static readonly CAPACITY_FREED_DEBOUNCE_MS;
     private readonly warmPool;
+    /**
+     * The callback set handed to `warmPool`, kept so the manager's own view of
+     * pool capacity and reapability is reachable without a live agent registry.
+     */
+    private readonly warmPoolCallbacks;
     /**
      * Optional DB-backed state store. When wired (production path), every
      * mutation to `spawningAgents` / `agentJobCorrelation` / `reservations`
@@ -211,53 +442,174 @@ export declare class ScalerManager {
      * Unit tests can omit the store and operate from in-memory Maps only.
      */
     private readonly stateStore?;
+    /** Id of the orchestrator instance this manager runs on. */
+    private readonly instanceId;
+    /**
+     * Late-bound reserved-event emitter. Undefined where there is no event
+     * plane (the worker, and unit tests that never emit).
+     */
+    private readonly eventEmitter?;
+    /**
+     * Redemption-only claim store, reading the shared `scaler_pending_claims`
+     * table. Undefined where there is no database.
+     */
+    private readonly claimStore?;
     /** Cluster-wide default deadline (ms) for a single `backend.spawn`. */
     private readonly spawnTimeoutMs;
     /** Optional per-org spawn-deadline resolver (production path; DB-backed). */
     private readonly resolveSpawnTimeoutMs?;
-    constructor(deps: {
-        config: ScalerConfig;
-        backends: Array<{
-            name: string;
-            backend: ScalerBackend;
-        }>;
-        /** Callback for relaying scaler events with runId/jobId context. */
-        onScalerEvent?: (runId: string, jobId: string, event: ScalerEvent) => void;
-        /**
-         * Optional DB-backed state store. Tests omit it; production wires it
-         * up via the orchestrator-core bootstrap.
-         */
-        stateStore?: ScalerStateStore;
-        /**
-         * Optional machine-ledger options. When `machinePools` are configured,
-         * the manager initializes a `MachineLedger` keyed off this directory and
-         * the orchestrator's instance id; reservations sum across orchestrators
-         * on the same host.
-         */
-        machineLedger?: {
-            /** Override the on-disk ledger directory. Falls back to `KICI_MACHINE_LEDGER_DIR`. */
-            dir?: string;
-            /** Orchestrator instance id (used in ledger rows for ownership). */
-            instanceId: string;
-        };
-        /** Coordinator drain predicate. When true, requestScale() declines to spawn
-         *  fresh capacity for the held Pending backlog. Defaults to never-draining. */
-        isDraining?: () => boolean;
-        /**
-         * Cluster-wide default deadline (ms) for a single `backend.spawn`. Always
-         * supplied (from config.scalerSpawnTimeoutMs); the per-org resolver, when
-         * present, overrides it per tenant.
-         */
-        spawnTimeoutMs: number;
-        /**
-         * Optional per-org resolver for the spawn deadline: given the job's org
-         * (jobConfig.cacheOrgId, undefined for warm-pool spawns), returns the
-         * effective timeout, falling back to the cluster default internally. Wired
-         * in orchestrator-core (has DB); omitted by the worker + unit tests → the
-         * cluster default spawnTimeoutMs is used.
-         */
-        resolveSpawnTimeoutMs?: (orgId: string | undefined) => Promise<number>;
-    });
+    constructor(deps: ScalerManagerDeps);
+    /**
+     * Pre-spawn one warm agent for a pool's label set.
+     *
+     * The pool's counterpart of `requestScale`, with the job term of every
+     * resolution absent: no bound job, no run, and resources resolved
+     * from the label set (then the scaler defaults) alone. It takes the same
+     * reservations through the same {@link reserveForSpawn} helper — a warm
+     * agent occupies the same cpu and memory as a job-bound one, and on an event
+     * backend it claims a cluster slot too, so a pool can no longer carry a
+     * scaler past `maxAgents` fleet-wide.
+     *
+     * A refused reservation is reported back to the pool as a failed spawn, so
+     * the in-flight slot it holds is released and the deficit is retried on the
+     * next tick rather than the pool stalling below target.
+     */
+    private spawnWarmAgent;
+    /**
+     * Whether a warm pool for this label set can count its own agents.
+     *
+     * The pool measures readiness with `AgentRegistry.findAvailable(labels)`, and
+     * that query applies the agent's `mandatoryLabels` gate: an agent is dropped
+     * unless every label in its gate also appears in the query. A scaler-spawned
+     * agent's gate is `labelSetMandatoryLabels()` for the set it was spawned for,
+     * which unions the configured gate with the pool's platform taints.
+     * `warmPoolQueryLabels` puts the structured taints into the query, so that
+     * part of the gate is satisfied by construction, and the legacy shim derives
+     * from this label set's own labels, so it is satisfied by construction too.
+     *
+     * That leaves one residual case: a configured `mandatoryLabels` entry absent
+     * from the label set, which `scaler/config.ts` validation already rejects.
+     * The check stays as the fail-safe — without it a pool in that state would
+     * spawn to `maxAgents` with `ready` stuck at 0, starving the on-demand spawns
+     * that share the cap. A refused pool is inert, exactly as it is before it
+     * fills at all — never partially filled.
+     *
+     * The gate derives from `declaredLabels` and is compared against
+     * `queryLabels`. Deriving it from `queryLabels` would fold a structured taint
+     * back in as a legacy one; comparing it against `declaredLabels` alone would
+     * refuse every structurally-tainted pool.
+     */
+    private warmPoolLabelSetFillable;
+    /**
+     * Whether this scaler pre-spawned (warm-filled) the agent, i.e. whether
+     * {@link canPrespawnedAgentServe} can ever answer false for it. A caller that
+     * picks one agent at a time uses this to decide whether the suitability check
+     * is worth carrying at all.
+     */
+    isPrespawnedAgent(agentId: string): boolean;
+    /**
+     * Whether a pre-spawned (warm) agent can serve this job.
+     *
+     * A warm agent is generic by construction: it was started before the job
+     * existed, at the pool's declared shape and running the pool's agent image.
+     * Both are applied when the agent starts and cannot be changed afterwards —
+     * nothing in the agent reads a job's `resources`, and an already-running
+     * container cannot become a different image. So a job that needs something
+     * else must get its own agent instead of silently running with the pool's.
+     *
+     * Returns true for any agent this scaler did not pre-spawn: a job-bound or
+     * static agent is not this predicate's business. {@link isPrespawnedAgent}
+     * answers that half on its own.
+     *
+     * Only the fields the job actually declares are compared. `resolveEffective`
+     * layers a job's declaration over the label set and the scaler defaults field
+     * by field, and the warm agent's own reservation is what those two layers
+     * resolve to — so a field the job leaves out already matches whatever the
+     * agent was started at, and comparing it would refuse a job that fits.
+     *
+     * A field the job DOES declare must match exactly, in both directions. A
+     * bigger ready agent is not a superset: limits are what the kernel enforces
+     * on the running agent, so a job asking to be capped at 2 cpus, served by an
+     * agent started at 4, runs uncapped. That is the same wrong execution
+     * environment as the under-provisioned case, in the other direction, so `>=`
+     * is not a safe relaxation.
+     *
+     * Both sides of the shape are compared, not just `requests`. The two answer
+     * different questions — `requests` is what the agent bills against the caps,
+     * `limits` is what the kernel enforces — and a declaration may legally set
+     * them apart, so a job matched on `requests` alone can still land on an agent
+     * capped somewhere else entirely.
+     */
+    canPrespawnedAgentServe(agentId: string, job: {
+        resources?: ResourceRequest;
+        hasOwnContainerImage: boolean;
+    }): boolean;
+    /**
+     * Whether one side (`requests` or `limits`) of a job's declared shape leaves
+     * the agent's resolved side unchanged.
+     *
+     * True when the job declares nothing on this side, and true when `resolved`
+     * is absent — both mean "this cannot be told", and the predicate's rule is to
+     * fail open rather than strand a job.
+     */
+    private shapeSideMatches;
+    /**
+     * Whether a warm pool for this label set has a declared agent shape.
+     *
+     * A warm agent starts before any job exists, so its cpu and memory come from
+     * the label set (or the scaler defaults) and from nowhere else. Both are
+     * applied when the agent starts and cannot be changed afterwards, so an
+     * undeclared shape means the pool reserves a guess and no job can be tested
+     * for fit against it. Refusing is the fail-safe reading, and it is what makes
+     * a pool predictable — on a cloud backend it is what lets an operator reason
+     * about instance size and cost at all.
+     *
+     * Limits alone count as a declaration: `resolveEffective` mirrors limits into
+     * requests, so a label set naming only `limits` still resolves to a concrete
+     * shape.
+     *
+     * A refused pool is inert, exactly as it was before it filled at all — never
+     * partially filled.
+     *
+     * A declaration this cannot resolve is refused the same way, and must never
+     * throw: the config schema types `resources.*.memory` as a plain string, so
+     * an unparseable value such as `4Gi` reaches `resolveEffective` and
+     * `parseMemoryString` rejects it. This runs inside the constructor and inside
+     * the commit block of {@link reload}, where an exception would abort
+     * orchestrator startup or leave a half-applied config behind.
+     */
+    private warmPoolShapeDeclared;
+    /**
+     * Agents this backend may still start before it reaches the scaler's
+     * `maxAgents` or the orchestrator's `globalMaxAgents`, whichever binds first.
+     *
+     * The warm pool clamps its deficit to this, so a pool that has reached a
+     * count cap asks for nothing at all rather than `size` spawns on every tick.
+     * Only the count caps are visible here: a pool with count headroom but no
+     * cpu/memory headroom still asks, and {@link reserveForSpawn} refuses each
+     * request under the reservation lock. Those refusals cost a log line and a
+     * `spawn_refusals` increment per tick, never a spawned agent.
+     *
+     * It is a best-effort read, not a reservation: the authority is
+     * {@link reserveForSpawn}, which every warm pre-spawn goes through and which
+     * re-checks both count caps under the reservation lock. The two backend
+     * families read differently here. A local backend (container, bare-metal,
+     * Firecracker) pins its compute to this host, so `getActiveCount()` is the
+     * whole population and the room it reports is exact. An **event** backend is
+     * capped cluster-wide — the reserve sequence claims a spawn row under
+     * `withScalerCapLock` — while `getActiveCount()` sees only the provisions
+     * this coordinator tracks, so the room reported here counts slots a peer may
+     * already hold. The cluster-wide cap is still enforced, by that claim rather
+     * than by this clamp.
+     */
+    private warmCapacityRemaining;
+    /**
+     * Publish the warm-pool gauges from the pool's current fill state.
+     *
+     * Called after each pool tick and after a config reload, so an operator sees
+     * a newly configured target immediately rather than at the next tick.
+     */
+    private publishWarmPoolGauges;
     /** Per-scaler mandatoryLabels (taint-style opt-in gate). */
     private readonly scalerMandatoryLabels;
     /** Declared structured platform per scaler (undefined = host-derive / linux default). */
@@ -280,12 +632,63 @@ export declare class ScalerManager {
      */
     private resolveScalerPlatform;
     /**
-     * Effective taint gate for a scaler: its configured mandatoryLabels plus the
-     * platform-taint labels derived from the pool's resolved structured platform,
-     * unioned with the legacy plain-label denylist shim. A pool declaring a
-     * non-default platform (`windows`, `macos`, `arm64`, …) only accepts jobs that
-     * request that platform. Container / firecracker pools resolve to linux and
-     * carry no taint; calling it uniformly is safe.
+     * The plain platform-taint tokens (`windows`, `macos`, `arm64`) for a scaler,
+     * derived from the same resolved platform `labelSetMandatoryLabels` gates on.
+     *
+     * This is the single source for the taint: it feeds the matcher's view of the
+     * label sets, the taint gate, the labels the spawned agent registers with, and
+     * the warm-pool readiness query. Deriving it anywhere else — in a backend, say
+     * — is what let the gate and the agent's labels disagree, so a tainted pool
+     * spawned agents `AgentRegistry.findAvailable` could never return.
+     *
+     * `backendType` is passed rather than looked up because the constructor
+     * configures warm pools from config entries whose backend may not be indexed.
+     */
+    private platformTaintsFor;
+    /**
+     * The labels a warm pool measures its readiness with: the declared label set
+     * plus the pool's platform taints.
+     *
+     * `AgentRegistry.findAvailable` drops an agent unless its whole
+     * `mandatoryLabels` gate appears in the query, and a tainted pool's gate
+     * carries the taints — so a query built from the declared labels alone counts
+     * none of the pool's own agents and the pool fills to `maxAgents` with `ready`
+     * stuck at 0. Both construction sites (the constructor and the reload path)
+     * call this so the two can never diverge.
+     *
+     * These labels are for the QUERY only. The pool is keyed by, and spawns with,
+     * the DECLARED label set: a backend matches the requested set against its own
+     * `labelSets` to resolve the image / binary / VM config and throws on a set it
+     * does not have, and every in-flight release path normalizes the spawn labels
+     * back to the map key.
+     */
+    private warmPoolQueryLabels;
+    /**
+     * Taint gate for ONE of a scaler's label sets: the configured
+     * `mandatoryLabels` plus the platform taints derived from the pool's resolved
+     * structured platform, unioned with the legacy plain-label denylist shim read
+     * from THIS label set alone.
+     *
+     * This is the gate every routing and stamping decision uses. Deriving the
+     * legacy shim per label set is what keeps a mixed-platform scaler routable: a
+     * pool declaring `[linux, gpu]` alongside `[macos, xcode]` gates only the
+     * second set on `macos`, where the union would gate both and leave the linux
+     * set unreachable by any sensible job.
+     *
+     * `backend` carries only `type` because the label set is passed in — a caller
+     * that holds one label set never has to reach for the whole backend.
+     */
+    private labelSetMandatoryLabels;
+    /**
+     * The scaler-wide taint gate: {@link labelSetMandatoryLabels} unioned across
+     * every label set the backend declares.
+     *
+     * Its only remaining consumers are the deprecated scaler-wide fields — the
+     * `mandatoryLabels` entry on `ScalerStatus['backends']` and the peer
+     * scaler-capacity summary — which stay populated for a peer that predates the
+     * per-label-set gate. Do NOT gate routing or stamp an agent with this: on a
+     * scaler whose label sets declare different platforms the union names a taint
+     * no single set can satisfy.
      */
     private effectiveMandatoryLabels;
     /**
@@ -298,8 +701,32 @@ export declare class ScalerManager {
      * (declared field wins, else host-derive for bare-metal, else linux), and the
      * mandatory labels combine the per-scaler config gate with the platform taint,
      * so the label matcher applies the gate alongside subset matching.
+     *
+     * Retiring scalers are skipped: they were removed from the config and must
+     * take no new work while their remaining agents finish.
      */
     private getEnrichedScalerEntries;
+    /**
+     * Mark a scaler retiring: routing stops immediately, but job-bound agents are
+     * left to finish their work. The backend is torn down by
+     * `sweepRetiredBackends` once its active count reaches zero.
+     */
+    private retireBackend;
+    /**
+     * Tear down every retiring backend that has drained. Runs on an interval and
+     * after each agent teardown, so a wind-down finishes promptly without a
+     * dedicated per-backend watcher.
+     */
+    private sweepRetiredBackends;
+    /**
+     * True while any spawn this manager started for `name` has neither registered
+     * nor failed — including one still queued behind the spawn semaphore, which
+     * has not reached `backend.spawn` at all. Such a spawn counts toward the
+     * scaler's drain even though the backend reports no active agent.
+     */
+    private hasSpawnInFlight;
+    /** Drop every trace of a scaler: the backend and all its per-scaler metadata. */
+    private forgetScaler;
     /**
      * Main entry point: called by the Dispatcher when no agent is available.
      *
@@ -317,7 +744,40 @@ export declare class ScalerManager {
      * surface as an unhandled rejection.
      */
     private runSpawnWithTimeout;
-    requestScale(labels: string[], jobId: string, runId: string, excludeLabels?: string[], resources?: ResourceRequest, orgId?: string): Promise<ScaleResult>;
+    requestScale(labels: string[], jobId: string, runId: string, excludeLabels?: string[], resources?: ResourceRequest, orgId?: string, 
+    /**
+     * The job's resolved container image + registry credentials, when it
+     * declared one. Threaded to the backend so it can spawn the job's own
+     * image instead of the pool's fixed agent image.
+     */
+    containerSpawn?: ResolvedContainerSpawn): Promise<ScaleResult>;
+    /**
+     * Take every reservation one spawn needs, or take none at all.
+     *
+     * The single reserve sequence both spawn paths share — the job-bound
+     * `requestScale` and the warm pool's pre-spawn. The only difference between
+     * them is `boundJobId`, which is `undefined` for a pre-spawn; a warm agent
+     * costs the same host resources as a job-bound one, so it takes the same
+     * reservations.
+     *
+     * Ordering, and why:
+     * - Count caps and the in-memory resource reservation run under the
+     *   in-process reservation lock, so the check and the reserve are atomic
+     *   against a concurrent caller. A local backend counts its own host's
+     *   agents in process, which is what its cap means: the compute is pinned to
+     *   this machine. An event backend provisions cloud instances any
+     *   coordinator can start, so its cap is counted — and its slot claimed —
+     *   cluster-wide, inside one advisory-locked transaction. The in-process
+     *   lock stays the outer guard so the two never contend.
+     * - The cross-process machine-pool ledger is taken outside that lock,
+     *   because the ledger has its own cross-process mutex; holding both at once
+     *   buys nothing.
+     *
+     * Every refusal rolls back exactly what it already took, so a `false` return
+     * leaves no reservation behind. On success the caller owns the release,
+     * which is always `releaseAll(agentId)`.
+     */
+    private reserveForSpawn;
     /**
      * Run a critical section under the reservation lock. Serializes concurrent
      * `requestScale()` calls so the check+reserve sequence is atomic.
@@ -410,30 +870,245 @@ export declare class ScalerManager {
      *   (`dispatchBoundJob` → `dequeueById`) both reject queued jobs whose
      *   `runsOn` does not include every gate label.
      *
-     * Returns `null` for static agents and warm-pool replenishment spawns
-     * (no spawning entry exists).
+     * Returns `null` for a static agent — no spawning entry exists, and no other
+     * instance's spawn record claims the agent either. A warm-pool spawn is not
+     * that case: its spawning entry (and durable row) survives until it
+     * registers, so it resolves with `mandatoryLabels` and no `boundJobId`, which
+     * is exactly what marks the agent warm on `register.ack`.
+     *
+     * A local hit (this instance spawned the agent) is answered from memory. A
+     * miss falls back to the shared spawn record: on an HA cluster behind one
+     * endpoint the agent routinely reaches an instance that did not spawn it,
+     * and adopting is the normal path, not an exception. Adoption is a
+     * conditional UPDATE, so exactly one instance can win.
      */
-    onAgentRegistered(agentId: string, labels: string[]): {
+    onAgentRegistered(agentId: string, labels: string[]): Promise<{
         boundJobId?: string;
         mandatoryLabels: string[];
-    } | null;
+    } | null>;
+    /**
+     * Reclaim the host-local compute of a scaler-spawned agent this coordinator
+     * refused for want of a spawn record.
+     *
+     * Refusing closes the WS before registration completes, so `onAgentDisconnected`
+     * never runs for that agent, and `managedAgentIndex` never held it in the first
+     * place — the two writes that populate it (`completeLocalRegistration` and the
+     * adoption branch) both come after this point. So even the disconnect hook is a
+     * no-op here: it reads that index, finds nothing, and takes the static-agent
+     * exit. The agent keeps its VM or host process, reconnects, is refused again,
+     * and loops until an operator stops it by hand.
+     *
+     * **This force-destroys a live instance, so read what makes it safe.**
+     *
+     * - **Local backends only.** An event agent's compute is a customer cloud
+     *   instance, reachable from any coordinator, and its missing adoption is
+     *   ambiguous: `adoptSpawningAgent` resolves `null` both when no row exists AND
+     *   when a peer coordinator already holds the row, so an event agent that is
+     *   alive and legitimately owned elsewhere reaches this exact branch. Tearing
+     *   it down there would destroy a peer's healthy provision. A local backend's
+     *   compute cannot be reached that way: it lives on the host that spawned it.
+     * - **Host-local evidence decides, not this verdict.** The verdict alone is NOT
+     *   evidence the agent is this host's: `adoptSpawningAgent` adopts `event` rows
+     *   only, so a `container` / `bare-metal` / `firecracker` agent that reconnects
+     *   to a peer behind a shared endpoint is refused here even though its spawn
+     *   row is intact and its compute is alive on the peer's host. What keeps that
+     *   harmless is that each backend reclaims only what its own tracking or its
+     *   own on-host artifacts name — never a shared database row, which on an HA
+     *   pair names an identically-configured scaler on both hosts. A wrongly-routed
+     *   agent id finds nothing here and nothing is touched.
+     * - **Never on a transient failure.** The caller must have positive evidence
+     *   the agent is unowned. A store read that merely *failed* is not evidence —
+     *   the record may exist and be perfectly healthy.
+     *
+     * Fire-and-forget, matching `onAgentDisconnected`'s own teardown: the caller
+     * refuses the registration immediately rather than holding the agent's socket
+     * open for a VM shutdown.
+     */
+    private reclaimUnownedScalerAgent;
+    /**
+     * Finish registration for an agent this instance spawned itself, from the
+     * in-memory spawning entry.
+     */
+    private completeLocalRegistration;
     /**
      * Called from agent-handler.ts when an agent disconnects.
      */
     onAgentDisconnected(agentId: string): void;
     /**
-     * Look up the scaler backend TYPE (`container`, `firecracker`,
-     * `bare-metal`) for a registered agent. Returns null if the agent is
-     * not scaler-managed (a static / stateful agent).
+     * Tear down a stranded provision the `EventProvisionReaper` found.
+     *
+     * Public because the reaper is leader-gated and lives outside the manager,
+     * while the emit and the row delete must stay in one place. The reason comes
+     * from the reaper's verdict — a provision whose agent never registered is a
+     * `spawn-timeout`, one whose agent went away is a `heartbeat-timeout` — so
+     * unlike the disconnect path this does not default to `shutdown`.
+     *
+     * `kici_orch_scaler_external_provision_timeout_total` counts the spawn-timeout
+     * arm. Nothing else increments it in production: `runSpawnWithTimeout` bounds
+     * only the `spawn()` call, and an event backend's `spawn()` returns the moment
+     * the scale-up event is emitted.
+     *
+     * The spawn-timeout arm is also where an external provisioning failure enters
+     * the ordinary failure machinery. The local backends report a dead spawn
+     * through the `scaler.failed` event their `spawn()` emits; an event scaler's
+     * `spawn()` succeeds by definition (it only emits a scale-up), so the failure
+     * is not observable until the provision it asked for never registers — which
+     * is exactly this verdict. Synthesizing the event here gives the fleet
+     * counter, the operator's `diagnose scaler` view, and `last_provisioning_error`
+     * the same input every other backend already supplies, so a job whose external
+     * provisioning failed settles with the provisioning cause instead of a
+     * label-mismatch complaint that sends its operator to fix a correct `runsOn`.
+     */
+    emitOrphanScaleDown(candidate: ReapCandidate, reason: ScaleDownReason): Promise<void>;
+    /**
+     * Refuse a spawn request while the scaler is inside its backoff window.
+     *
+     * Returns the refusal to hand back, or null to proceed. The message names how
+     * many consecutive failures produced the deferral and when it lifts. It
+     * reaches the operator through this method's own log line: the dispatcher
+     * branches on `action` alone and discards `reason`, so a job deferred here
+     * carries no record of the deferral on its queue row. The job whose provision
+     * actually failed is the one that gets `last_provisioning_error`, written by
+     * the `scaler.failed` this backoff's own trigger emits.
+     *
+     * The knobs are read here per request rather than per process, so an operator
+     * lowering the failure limit mid-outage sees the repeated-failure wording on
+     * the next refusal. The window itself was armed by `recordProvisionFailure`
+     * from the base and ceiling live at that moment, so a retuned window applies
+     * from the next recorded failure.
+     */
+    private provisionBackoffRefusal;
+    /**
+     * Record a consecutive provisioning failure for a scaler and arm its next
+     * deferral.
+     *
+     * The delay doubles per consecutive failure and is capped, so a provider
+     * outage settles into a steady retry cadence rather than either hammering the
+     * provider or growing without bound. `2 ** (n - 1)` is bounded by the cap
+     * before it is used, so a long outage cannot overflow the shift.
+     */
+    private recordProvisionFailure;
+    /**
+     * Claim a dead provision as counted, or report that someone already did.
+     *
+     * The same provision is seen twice on the leader — once by its own
+     * stale-spawn prune, once by the leader-gated reaper — and every consequence
+     * of a failure is counted per provision, not per observer. The claim is
+     * synchronous and happens before any await, so two observers arriving in the
+     * same tick cannot both pass it.
+     */
+    private claimFailedProvision;
+    /**
+     * Report one dead external provision, wherever this coordinator noticed it.
+     *
+     * Everything a failed provision owes an operator happens here: the fleet
+     * spawn-failure counter, the structured warn, the `ScalerFailureTracker`
+     * record `kici-admin diagnose scaler` reads, the `last_provisioning_error`
+     * that tells the waiting job why it never ran, and the backoff.
+     *
+     * It is deliberately reachable from BOTH observers rather than from the
+     * reaper alone. `ScalerFailureTracker` and `provisionFailures` are
+     * per-process, and the reaper is leader-gated — so a reaper-only report left
+     * every non-leader with an empty `diagnose scaler` for a scaler failing
+     * fleet-wide, and dispatching at the un-deferred cadence while only the
+     * leader backed off. The local prune is the observation every coordinator has
+     * of its own spawns.
+     *
+     * Synchronous up to the backoff arm, so a caller that must clean up after the
+     * event (the prune drops the agent's event buffer) sees the effect
+     * immediately.
+     */
+    private reportProvisionFailure;
+    /**
+     * Report a pruned event spawn as a dead provision — unless a peer adopted it.
+     *
+     * The prune is NOT proof of failure on an HA cluster. Nothing tells the
+     * spawning coordinator that its agent registered on a peer, so the entry
+     * survives adoption and lands in the prune the stale window later next to a
+     * perfectly live agent — the same asymmetry `deleteUnadoptedSpawningAgent`
+     * exists for. Reporting that as a dead provision writes a `scaler.failed` and
+     * a `last_provisioning_error` onto a job that is running, and backs a healthy
+     * scaler off, which is the misattribution this whole report exists to remove.
+     * The reaper already applies the same test: it returns `spawn-timeout` only
+     * for a candidate with no `adoptedBy`, so without this the two observers of
+     * one provision disagree.
+     *
+     * The question is "was this provision ever adopted?", and the spawn row alone
+     * cannot answer it: the row is deleted on teardown, so its absence is BOTH
+     * "never adopted" and "adopted, then torn down". `provisionAdopter` reads the
+     * live row first and the durable provision-outcome record second, which
+     * survives the delete — so an adopted provision the reaper has since
+     * condemned no longer reads as a dead one.
+     *
+     * The lookup is fire-and-forget so the prune stays off the request path, and
+     * it fails open. Both "the store errored" and "there is no verdict" leave the
+     * failure reported, deliberately: a provision with no positive evidence of
+     * adoption is what a single coordinator sees on a real provisioning outage,
+     * and suppressing there would disarm the per-coordinator backoff on every
+     * follower — an invisible missing report in place of a visible false one. The
+     * only rows with no verdict are those adopted before the outcome table
+     * existed, and they age out within one stale-prune window of the deploy.
+     */
+    private reportPrunedProvisionFailure;
+    /**
+     * Clear a scaler's consecutive-failure state after a successful provision.
+     *
+     * Called from `onAgentRegistered`, which is the single entry point for BOTH a
+     * locally-spawned agent and one this coordinator adopted from a peer. An
+     * adopted agent is a successful provision — the instance that spawned it
+     * is not the one it reached — so counting only local registrations
+     * would leave a healthy scaler deferred on an HA cluster.
+     */
+    private clearProvisionFailures;
+    /**
+     * Emit `kici.scaler.scale-down` straight from a persisted spawn spec, then
+     * drop the durable row.
+     *
+     * Used where the instance holding the agent has no local backend for its
+     * scaler — the ordinary shape on an HA cluster behind one shared endpoint,
+     * where the coordinator the agent reached may not configure the scaler that
+     * spawned it. The row's own targets are therefore primary, with local config
+     * as the fallback for a row that recorded none.
+     *
+     * A failed emit leaves the row in place: it is the only durable pointer at
+     * the customer's running instance, so the reaper needs it to retry from.
+     *
+     * Returns whether the teardown was actually emitted, so a caller that counts
+     * teardowns counts deliveries rather than attempts.
+     */
+    private emitScaleDownForSpec;
+    /**
+     * Look up the scaler backend TYPE (a `ScalerBackendType` — `container`,
+     * `firecracker`, `bare-metal`, `event`, …) for a registered agent.
+     * Returns null if the agent is not scaler-managed (a static / stateful
+     * agent).
      *
      * Used by AgentMetricsAggregator to stamp the `scaler` label on each
      * kici_agent_* series. The label MUST be the backend type, not the
      * operator-chosen scaler name: the Platform catalog filter constrains
-     * it to AGENT_SCALER_VALUES (the four types), so a free-form name is
+     * it to the ScalerBackendType set (plus `stateful`), so a free-form name is
      * dropped as bad_label_value. `managedAgentIndex` stores the scaler
      * name, so resolve it through `backends` to the type.
      */
     getBackendType(agentId: string): string | null;
+    /**
+     * Exchange a provisioning claim code for freshly minted ephemeral credentials.
+     *
+     * Reads the shared claim table directly rather than routing through the
+     * backend that minted the code: the mint needs only the agent-token store,
+     * which is already DB-backed, so redemption succeeds on an instance that has
+     * never heard of the emitting scaler. That is what lets an HA cluster front
+     * its coordinators with one shared endpoint — the provisioned agent reaches
+     * whichever coordinator the load balancer picks.
+     *
+     * Returns `{ error }` when this instance has no claim store (no database), or
+     * when the redeem is rejected (unknown / consumed / expired / minting error).
+     * Called from the `scaler.claim-credentials` WS handler.
+     */
+    claimScalerCredentials(code: string): Promise<{
+        credentials?: ClaimedCredentials;
+        error?: string;
+    }>;
     /**
      * Called from agent-handler.ts when an agent sends config.ack.
      * For Firecracker agents, clears MMDS data (belt-and-suspenders with in-VM iptables).
@@ -458,7 +1133,60 @@ export declare class ScalerManager {
      */
     shutdownAll(): Promise<void>;
     /**
-     * Validate and reload configuration.
+     * Classify every configured scaler against the currently loaded backends.
+     * Pure — it reads state and writes none.
+     */
+    private planReload;
+    /**
+     * Validate the new config: label-set overlaps, backend type changes, and
+     * each kept backend's own view of its label sets.
+     *
+     * `backend.reload` validates and applies in one call, so a backend rejected
+     * late in the loop would otherwise leave the ones before it holding the new
+     * label sets and `maxAgents` while the commit never runs. Each successful
+     * call's previous state is therefore snapshotted and restored when any
+     * backend rejects — a rejected reload leaves every backend as it was.
+     *
+     * The snapshots are returned as well as used here: a later stage can still
+     * fail (an added scaler whose backend does not construct), and the same
+     * rollback is what keeps that rejection from half-applying too.
+     */
+    private validateReload;
+    /** Put every backend a rejected reload had already mutated back as it was. */
+    private restoreReloadedBackends;
+    /**
+     * Construct every added backend into a staging list. On any failure the
+     * partially built set is torn down and the caller keeps the old config.
+     */
+    private buildAddedBackends;
+    /**
+     * Host prep for backends added by a reload: bridge provisioning, a reap of
+     * leftovers from a previous incarnation of the same scaler, and the periodic
+     * TAP sweep a long-running Firecracker backend needs. Every failure degrades
+     * that one scaler; none of them fails the reload.
+     */
+    private prepareAddedBackends;
+    /**
+     * Refresh per-scaler metadata from the new config. Keys are deleted only for
+     * scalers that are neither configured nor retiring — a retiring scaler keeps
+     * its cap, URL, roles and defaults until its last agent is gone.
+     */
+    private applyScalerMetadata;
+    /**
+     * Register every configured machine pool with the ledger, building the
+     * ledger when this reload is the first config to declare a pool. `cap` is
+     * re-registered on every reload, so a changed cap applies without a restart.
+     */
+    private applyMachinePools;
+    /** Rebuild the warm-pool config map from the new config (see WarmPoolManager.reload). */
+    private applyWarmPoolConfig;
+    /**
+     * Validate and reload configuration: plan → validate → build → commit.
+     *
+     * Either the whole new config applies, or nothing does and the previous one
+     * keeps running. A scaler the new config no longer names is retired, not
+     * dropped: it stops taking work at once, its idle warm agents are destroyed,
+     * its job-bound agents finish, and its backend is torn down when it drains.
      */
     reload(newConfig: ScalerConfig): Promise<ValidationResult>;
     /**
@@ -470,6 +1198,15 @@ export declare class ScalerManager {
      */
     recentSpawnFailures(windowMs: number, nowMs: number): Map<string, BackendFailureSummary>;
     getStatus(): ScalerStatus;
+    /**
+     * Scaler capacity advertised to peers in the cluster heartbeat.
+     *
+     * Retiring backends are excluded: they accept no new work locally, so a peer
+     * that saw them would keep selecting this host for jobs it can no longer
+     * serve. Agents still draining on a retiring backend stay visible through the
+     * separate per-agent inventory, so in-flight work is unaffected.
+     */
+    getRoutableCapacity(): ScalerCapacitySummary[];
     /**
      * Get the backend name managing a specific agent.
      * Returns null if the agent is not scaler-managed (standalone).
@@ -511,7 +1248,8 @@ export declare class ScalerManager {
      */
     getScalerContextForAgent(agentId: string): Record<string, unknown> | undefined;
     /**
-     * Start the warm pool idle check interval and the machine-pool ledger reaper.
+     * Start the warm pool idle check interval, the machine-pool ledger reaper,
+     * and the retirement sweep that tears down drained retiring backends.
      */
     start(): void;
     /**
@@ -523,9 +1261,21 @@ export declare class ScalerManager {
      */
     ensureHostsReady(): Promise<void>;
     /**
-     * Remove spawning entries whose `backend.spawn` started more than 5 minutes
-     * ago but never registered via WS (e.g., process/container crashed on
-     * startup). Without cleanup, these entries would leak in spawningAgents
+     * The window a spawning entry may sit unregistered before it is pruned.
+     *
+     * An entry cannot be stale before its own spawn deadline has passed, so the
+     * window is floored at the configured deadline: an operator raising
+     * `KICI_SCALER_SPAWN_TIMEOUT_MS` for a slow cloud must not have this prune
+     * fire first and drop the correlation a provision booting inside the raised
+     * deadline still needs. The floor only ever extends the window — a lowered
+     * deadline keeps the default five minutes, because a local backend's agent
+     * boots and registers well after `spawn()` itself returns.
+     */
+    private stalePruneWindowMs;
+    /**
+     * Remove spawning entries whose `backend.spawn` started longer ago than the
+     * stale window but never registered via WS (e.g., process/container crashed
+     * on startup). Without cleanup, these entries would leak in spawningAgents
      * forever.
      *
      * The window is measured from `spawnStartedAt` (when the throttled spawn
@@ -534,10 +1284,65 @@ export declare class ScalerManager {
      * reaped mid-queue during a large burst. Recovered orphan entries carry
      * `spawnStartedAt` set to their persisted enqueue time, so they are subject to
      * the normal window.
+     *
+     * What happens to the durable row depends on the backend type the spawn
+     * recorded, which the entry carries so a scaler since removed from this
+     * coordinator's config still resolves. A local backend's
+     * compute is pinned to a host this process manages, so a pruned entry means
+     * the spawn is dead and the row goes. An event backend's row is the only
+     * durable pointer at a customer cloud instance, and it is precisely the
+     * reaper's `spawn-timeout` candidate — so the row STAYS, and the reaper
+     * decides. Deleting it here would beat the reaper by the flap grace on any
+     * coordinator serving jobs, so the teardown the row exists to trigger would
+     * never be emitted.
      */
     private pruneStaleSpawningEntries;
+    /**
+     * The spawn row for one agent, as every writer of it must build it.
+     *
+     * Ownership and self-describing columns are assembled only here.
+     * `backend_type` is what `adoptSpawningAgent`'s conditional UPDATE matches
+     * on, so a row without it can never be adopted by any instance — including
+     * the one that spawned it. `owner_instance_id` scopes recovery to that
+     * instance, and `mandatory_labels` carries the platform taint to a
+     * coordinator that has no config entry for this scaler and can therefore
+     * derive it from nothing else. `provisioning_targets` is the same story for
+     * teardown: it is the only address such a coordinator has for the customer's
+     * teardown workflow, and an emit to an empty target list reaches nobody, so
+     * the provision would bill forever. `roles` travels with it so the row stays
+     * a complete description of the spawn.
+     */
+    private spawningAgentSnapshot;
+    /**
+     * Claim one slot against an event scaler's cluster-wide `maxAgents`.
+     *
+     * The count and the claim share a single advisory-locked transaction, so the
+     * row this writes is already visible to the next coordinator that takes the
+     * lock. A store failure is never an admission: returning false costs a job a
+     * queue wait, while admitting on a count nobody recorded provisions a cloud
+     * instance the cap cannot see. Failing closed is not silent — a refusal
+     * increments the spawn-refusal counter and a store failure its own counter,
+     * labelled so a database outage, a contended lock, and a full cluster are
+     * three distinguishable things rather than one.
+     *
+     * The failure arm bounds the admission, not the row: a transaction that
+     * commits and then loses its acknowledgement leaves `slotClaimed` false, so
+     * neither rollback path fires and the row counts against the cap for an
+     * agent that never spawned. Closing that window needs two-phase commit; the
+     * event-provision reaper is the backstop, clearing the row once the spawn
+     * deadline passes with nothing adopted.
+     */
+    private claimClusterSlot;
     private persistSpawningAgent;
     private deleteSpawningAgentFromStore;
+    /**
+     * Stamp `adopted_by = this instance` on the spawning row of an agent that just
+     * registered here. Reuses the cross-instance conditional UPDATE, so a row some
+     * other instance already claimed is left exactly as it is.
+     */
+    private markSpawningAgentAdopted;
+    /** Store-side half of the stale prune: drops the row only if nobody adopted it. */
+    private deleteUnadoptedSpawningAgentFromStore;
     private persistReservation;
     private deleteReservationFromStore;
     private persistAgentJob;
@@ -557,9 +1362,24 @@ export declare class ScalerManager {
      * emitted by the previous coord before correlation are lost (see
      * wishlist for the rationale).
      *
+     * Both reads are scoped to this instance's own rows. An unscoped read
+     * hydrates a peer's in-flight spawns and reservations as our own, so our
+     * spawn-timeout reaper destroys agents that peer is still legitimately
+     * waiting on and our caps double-count its reservations. Rows owned by a
+     * dead instance are the leader-gated reaper's business, not ours.
+     *
      * No-op when no store is wired (unit-test path).
      */
     recoverState(): Promise<ScalerStateRecovery>;
+    /**
+     * Mint a scaler-managed agent id.
+     *
+     * Built from {@link SCALER_AGENT_ID_PREFIX} rather than a second copy of the
+     * literal: the registration guard in `onAgentRegistered` recognises a
+     * scaler-managed agent by that prefix alone when the spawn-record lookup
+     * fails, so a minting prefix that drifts from the constant leaves the guard
+     * matching nothing and failing open.
+     */
     private generateAgentId;
     /**
      * Start log forwarding for a scaler-managed agent if its backend supports LogCapture.
@@ -589,6 +1409,13 @@ export declare class ScalerManager {
      * The backend label is resolved from the spawning entry first, then the
      * managed-agent index — so a late failure after the spawning entry is gone
      * still buckets under the real backend instead of "unknown".
+     *
+     * `attribution` is the last resort for a failure this coordinator did not
+     * spawn and never registered: the reaper is leader-gated, so the instance that
+     * condemns a stranded external provision routinely has neither a spawning
+     * entry nor a managed-agent index row for it, and every in-memory resolver
+     * above comes back empty. It is read only where the maps are silent, so a live
+     * local entry always wins over a durable row that may be a sweep behind.
      */
     private handleScalerEvent;
 }