@kici-dev/orchestrator 0.4.0 → 0.6.0

package/dist/scaler/event-backend.d.ts

@@ -0,0 +1,124 @@
+/**
+ * Event scaler backend — workflow-driven autoscaling.
+ *
+ * Unlike the local backends (container / bare-metal / firecracker), this backend
+ * performs NO local compute. Its `spawn()` emits a `kici.scaler.scale-up` event
+ * and its `destroy()` emits a `kici.scaler.scale-down` event; a customer-authored
+ * provisioning / teardown workflow (subscribed via `kiciEvent()`) consumes those
+ * events to boot and tear down an ephemeral cloud instance. The provisioned
+ * instance's agent registers back with the scaler-chosen `agentId` and the
+ * existing bound-job dispatch runs the pending job — reusing the whole scaling
+ * engine (demand detection, caps, reservations, spawn timeout, teardown).
+ */
+import type { EffectiveLimits, LabelSetConfig, ManagedAgent, ScalerBackend, ScalerDestroyContext, ScalerEntry, ScalerEventCallback, SpawnContext, ValidationResult } from './types.js';
+import type { ClaimStore } from './claim-store.js';
+import type { ScalerScaleUpPayload, ScalerScaleDownPayload } from './scaler-events.js';
+/**
+ * Subset of `EventEmitter` the backend needs. Injected so the backend is unit
+ * testable without a real event router.
+ */
+export interface ScalerEventEmitterLike {
+    emitScalerScaleUp(payload: ScalerScaleUpPayload, targets: string[]): Promise<string>;
+    emitScalerScaleDown(payload: ScalerScaleDownPayload, targets: string[]): Promise<string>;
+}
+export interface EventScalerBackendOptions {
+    /** The parsed scaler entry (`type: 'event'`). */
+    entry: ScalerEntry;
+    /** Emits the reserved scale-up / scale-down events. */
+    emitter: ScalerEventEmitterLike;
+    /** Mints ephemeral credentials when a claim code is redeemed — normally by
+     * the provisioned agent self-bootstrapping, or by a provisioning workflow. */
+    claimStore: ClaimStore;
+    /** Correlation-id generator for scale-up / scale-down requests. */
+    requestId: () => string;
+}
+export declare class EventScalerBackend implements ScalerBackend {
+    readonly type: "event";
+    readonly spawnsOnLocalHost = false;
+    readonly logsSource = "event";
+    labelSets: LabelSetConfig[];
+    maxAgents: number;
+    /** Exposed so the composition root can wire `onClaimCredentials` to it. */
+    readonly claimStore: ClaimStore;
+    private entry;
+    private readonly emitter;
+    private readonly requestId;
+    private readonly agents;
+    constructor(opts: EventScalerBackendOptions);
+    /** The entry this backend is serving; the reload rollback restores it. */
+    get currentEntry(): ScalerEntry;
+    getActiveCount(): number;
+    /**
+     * Register a pending claim, emit a scale-up event to the provisioning targets,
+     * and track the agent as provisioning. Performs no local compute.
+     */
+    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, _onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext, signal?: AbortSignal): Promise<ManagedAgent>;
+    /**
+     * Drop the tracked agent, emit a scale-down event (carrying the teardown
+     * reason), then invalidate the pending claim. Idempotent: a repeat destroy of
+     * an already-torn-down agent is a no-op (no duplicate scale-down).
+     *
+     * The tracking entry is dropped before the first `await` so a concurrent
+     * destroy of the same agent returns at the guard above and cannot emit a
+     * second scale-down.
+     *
+     * The claim invalidation is last and fails open. It is a DB delete, and the
+     * scale-down is the only thing that tears down the customer's cloud instance:
+     * running the delete first lets a DB error suppress the teardown for good,
+     * because the agent is already out of `this.agents` and a retried destroy
+     * returns at the guard. A claim left behind instead is inert — the row is
+     * single-use and TTL-bounded, so it can mint at most one token for an agent
+     * id that is being torn down.
+     */
+    destroy(managedId: string, context?: ScalerDestroyContext): Promise<void>;
+    /**
+     * Drop the tracked agent without emitting anything.
+     *
+     * The counterpart to `adopt()` on the other side of a cross-instance
+     * registration: the spawning coordinator is never told that a peer adopted
+     * its provision, so its entry would otherwise live until the process exits —
+     * inflating `getActiveCount()` against the global cap forever, and handing
+     * `shutdownAll()` a peer's live agent to tear down. Forgetting says "this
+     * coordinator no longer tracks the provision", which is exactly true and says
+     * nothing about whether the customer's instance is still running; the adopter
+     * emits the teardown, and the leader-gated reaper is the backstop.
+     */
+    forget(agentId: string): void;
+    /**
+     * Take over bookkeeping for an agent another instance spawned. Seeds the same
+     * entry `spawn()` would have, so `destroy()` emits its teardown instead of
+     * no-opping on an unknown id.
+     */
+    adopt(agentId: string, labelSet: string[]): void;
+    /** Mark a provisioned agent active once its instance registers over WS. */
+    markActive(agentId: string): void;
+    /** Delegate a credential claim to the backend's claim store. */
+    claim(code: string): Promise<import("./claim-store.js").ClaimedCredentials>;
+    /**
+     * Emits nothing. An event provision is a customer cloud instance whose
+     * lifetime spans the cluster, not this process: the coordinator holding the
+     * agent emits its teardown from the WS-close path, and the leader-gated
+     * reaper is the backstop for one that reaches nobody. This map also carries
+     * provisions a peer has already adopted — the spawning coordinator is never
+     * told — so emitting here would make a routine restart tear down a peer's
+     * running agents.
+     *
+     * The map is deliberately left intact rather than cleared: a WS close landing
+     * during the remaining shutdown steps then still emits its own teardown.
+     */
+    shutdownAll(): Promise<void>;
+    /**
+     * Apply the new config. The backend reads `roles`, `mandatoryLabels`,
+     * `agentTokenTtlSeconds` and `provisioningTargets` off its entry at every
+     * spawn, so the entry is replaced here — otherwise a reload that retargets
+     * the provisioning workflow, or changes the scaler's roles, would keep
+     * emitting scale-up events to the old workflow refs and minting agent labels
+     * from the old roles, while the manager's own routing gate used the new ones.
+     */
+    reload(labelSets: LabelSetConfig[], opts?: {
+        maxAgents?: number;
+        entry?: ScalerEntry;
+    }): ValidationResult;
+    private provisioningTargets;
+}
+//# sourceMappingURL=event-backend.d.ts.map

package/dist/scaler/event-provision-reaper.d.ts

@@ -0,0 +1,350 @@
+import { ScaleDownReason } from './scaler-events.js';
+import type { ReapCandidate } from './scaler-state-store.js';
+/**
+ * How long a provision outcome is kept once its spawn row is gone.
+ *
+ * Deliberately a constant rather than a `cluster_settings` knob: unlike the
+ * claim retention beside it, there is no operational decision here. The only
+ * reader is the stale-spawn prune, and the interval the row must survive is
+ * derived from windows the reaper already resolves — so a knob would be surface
+ * with nothing behind it. The purge floors this against twice the spawn
+ * deadline, so raising `KICI_SCALER_SPAWN_TIMEOUT_MS` for a slow cloud widens
+ * the retention with it.
+ */
+export declare const PROVISION_OUTCOME_RETENTION_MS = 86400000;
+/**
+ * The live values one sweep runs against. Read together at the top of each tick
+ * so a sweep is internally consistent even if an operator changes a knob
+ * halfway through.
+ */
+export interface ReaperWindows {
+    /** How often the leader sweeps. A change reschedules the timer after this tick. */
+    intervalMs: number;
+    /** How long an unadopted provision may stay unregistered before teardown. */
+    spawnTimeoutMs: number;
+    /** How long an adopted provision may stay unseen before teardown. */
+    strandedTimeoutMs: number;
+    /**
+     * How long a peer disconnection, or an agent's absence, must persist before
+     * it counts as real rather than as a reconnect in progress. Shared with the
+     * rerouted-job guard (`reroute_flap_grace_ms`) because it answers the same
+     * question for the same reason.
+     */
+    flapGraceMs: number;
+    /** How long before a candidate whose teardown did not clear it is retried. */
+    reattemptIntervalMs: number;
+    /** How long an expired provisioning claim is kept before it is purged. */
+    claimRetentionMs: number;
+}
+/**
+ * The flap grace the reaper actually runs on.
+ *
+ * `reroute_flap_grace_ms` is shared with the rerouted-job guard, and the two
+ * consumers do not carry the same risk: the reroute guard force-fails a run it
+ * could have waited for, while this one deletes a customer's running instance.
+ * An operator lowering the knob for reroute reasons must not silently collapse
+ * that protection to two consecutive sweeps.
+ *
+ * The floor is two peer-stale-timeout windows — the very derivation the knob's
+ * own default (`DEFAULT_REROUTE_FLAP_GRACE_MS`) comes from, applied as a floor
+ * rather than a default. It has to cover the worst honest gap between an agent
+ * being alive and being visible here: an agent reconnect backing off to its cap
+ * (60s), plus the peer heartbeat interval (30s) during which a freshly
+ * reconnected peer advertises no agents at all. Two stale windows (120s at the
+ * defaults) covers that 90s; one (60s) would not.
+ *
+ * A raised knob is honoured as-is: the floor only ever raises.
+ */
+export declare function reaperFlapGraceMs(flapGraceMs: number, peerStaleTimeoutMs: number): number;
+/** The slice of the peer registry the cluster-view guard reads. */
+export interface ReaperPeerView {
+    /** Every coordinator peer this instance knows, connected or not. */
+    getCoordinatorPeers(): readonly unknown[];
+    /** How many of those are connected right now. */
+    getConnectedCoordinatorPeerCount(): number;
+}
+/**
+ * The reaper's `canReap` predicate, over static config plus the live peer
+ * registry.
+ *
+ * Extracted and exported so the wiring itself is testable. The bug this closes
+ * lived in exactly this expression — `clusterViewSufficient` was unit-tested
+ * throughout, while the call site that fed it the wrong count was not, so the
+ * guard read permissive on a Platform-mode HA pair through two review rounds.
+ * A component whose false positive deletes a customer's running instances
+ * cannot have an untested load-bearing line.
+ *
+ * The peer count is the max of the two sources because neither alone covers
+ * both modes: `KICI_CLUSTER_PEERS` is set only in independent mode, while a
+ * Platform-mode cluster learns its peers from the matchmaker and leaves that
+ * array empty. `getCoordinatorPeers()` filters on role, not on connectivity, so
+ * a peer lost to a partition stays counted and the guard stays armed.
+ */
+export declare function canReapForCluster(configuredPeerCount: number, peers: ReaperPeerView): boolean;
+/**
+ * Whether an adopting coordinator should still be treated as live.
+ *
+ * Mirrors `shouldDeferReroutedJob`, and for the same reason: `PeerRegistry`
+ * flips `connected` the instant a peer WS closes, with no grace of its own, so
+ * sampling it alone cannot tell a link blip between two healthy processes from
+ * a dead coordinator — while the agent is still attached to that peer running a
+ * customer job. A peer we have never heard of is genuinely gone.
+ */
+export declare function adopterIsLive(peer: {
+    connected: boolean;
+    lastHeartbeatAt: number;
+} | undefined, nowMs: number, flapGraceMs: number): boolean;
+/**
+ * Whether this coordinator sees enough of its cluster to tear down provisions
+ * on the cluster's behalf.
+ *
+ * Raft self-elects a coordinator that has zero connected coordinator peers, so
+ * both halves of a partitioned pair become leader on their own side. Each then
+ * reads every agent the other side holds as registered nowhere, and roughly one
+ * flap grace later both emit teardowns for the other's live instances —
+ * symmetric destruction of a running fleet. Standing down costs a delayed
+ * backstop; not standing down costs the customer their fleet.
+ *
+ * `knownCoordinatorPeerCount` must count peers this coordinator has *ever*
+ * handshaken with, not the ones named in static config. `KICI_CLUSTER_PEERS` is
+ * set only in multi-orchestrator independent mode: the documented HA-pair recipe
+ * runs Platform mode and joins by token, so peers arrive through the Platform
+ * matchmaker and the static array stays empty. Keying on it left the guard inert
+ * on precisely the topology it exists to protect. A known peer survives a
+ * partition because `evictStalePeers` only marks it disconnected, so the guard
+ * stays armed for as long as the partition lasts.
+ *
+ * A coordinator that has never met a peer is a single-coordinator deployment,
+ * not a partitioned one, so it is never blocked.
+ */
+export declare function clusterViewSufficient(knownCoordinatorPeerCount: number, connectedCoordinatorPeerCount: number): boolean;
+export interface EventProvisionReaperOptions {
+    /** Event rows that are either adopted, or past the spawn deadline unadopted. */
+    listCandidates: (spawnCutoff: Date) => Promise<ReapCandidate[]>;
+    /**
+     * Whether an instance id is still a live cluster member (self counts as live).
+     * `flapGraceMs` is passed in so the implementation can treat a peer that
+     * disconnected moments ago as live — `peer.connected` alone flips on WS close
+     * with no grace at all.
+     */
+    isPeerLive: (instanceId: string, flapGraceMs: number) => boolean;
+    /** Whether an agent id is registered anywhere in the cluster we can see. */
+    isAgentRegistered: (agentId: string) => boolean;
+    /**
+     * Whether this coordinator can see enough of the cluster to act on its
+     * behalf. False on a node that *knows* coordinator peers but is connected to
+     * none of them, which is exactly the shape a network partition produces.
+     *
+     * "Knows" spans both modes: peers named in static config, plus peers this
+     * instance has handshaken with. Keying on static config alone is what left
+     * this guard inert on a Platform-mode HA pair, which configures none.
+     */
+    canReap: () => boolean;
+    /** Emit the teardown and drop the row. */
+    emitScaleDown: (candidate: ReapCandidate, reason: ScaleDownReason) => Promise<void>;
+    /** Delete pending claims that expired before the cutoff; returns rows deleted. */
+    purgeExpiredClaims: (cutoff: Date) => Promise<number>;
+    /**
+     * Delete provision outcomes older than the cutoff whose spawn row is gone;
+     * returns rows deleted. Runs on the same leader-gated sweep as the claim
+     * purge, which is the orchestrator's one precedent for a timer-driven purge
+     * of scaler bookkeeping.
+     */
+    purgeProvisionOutcomes: (cutoff: Date) => Promise<number>;
+    /**
+     * Read the live windows for one sweep. Backed by `cluster_settings`, so an
+     * operator retunes the reaper on a running cluster; each read falls back to
+     * the orchestrator's configured default when the column is NULL.
+     */
+    resolveWindows: () => Promise<ReaperWindows>;
+    /**
+     * Interval used for the very first schedule, before any knob has been read.
+     * `onBecomeLeader` is synchronous (it is a Raft callback), so the timer starts
+     * at the configured default and the first tick reconciles it to the live value.
+     */
+    bootIntervalMs: number;
+}
+/**
+ * Leader-gated teardown backstop for event-scaler provisions.
+ *
+ * An event scaler emits a scale-up and then has no further local handle on the
+ * instance a customer workflow booted. `spawn()` returns as soon as the event
+ * is emitted, so the spawn-call deadline says nothing about whether the VM ever
+ * appeared, and nothing else tears the provision down: an agent that never
+ * registers — a cloud API 500, a cancelled run, a denied quota — leaves a
+ * permanent row holding a `maxAgents` slot and no `kici.scaler.scale-down` is
+ * ever emitted. Two more shapes strand a provision on a cluster: the spawning
+ * coordinator dies before the agent registers, and the adopting coordinator
+ * dies mid-job.
+ *
+ * The same pass purges pending provisioning claims past their expiry. A claim
+ * row outlives its agent whenever a registration unwinds after the claim was
+ * written, or whenever the coordinator that would have invalidated it crashed
+ * first, and nothing else deletes one on a timer.
+ *
+ * Every teardown deletes a customer's cloud instance, so the whole design is
+ * biased towards leaving a doubtful row alone:
+ *
+ *  - **Nothing is ever condemned on one sample.** "The agent is registered
+ *    nowhere" is an observation that flips false for a live agent on routine
+ *    events — a local WS reconnect, or the ~30s after any peer reconnect during
+ *    which that peer's advertised agent list is empty. So the reaper times how
+ *    long the observation has *persisted* (leader-local, never a database
+ *    column a re-adopt could refresh) and requires it to hold for at least the
+ *    flap grace before any arm can fire.
+ *  - **A peer disconnection gets the same grace.** `peer.connected` flips on WS
+ *    close with no grace of its own, so the adopter-liveness question is asked
+ *    the way `shouldDeferReroutedJob` asks it: connected, or last heard from
+ *    inside the flap window.
+ *  - **A node that cannot see its cluster does not act for it.** A coordinator
+ *    isolated by a partition self-elects after ~60s with zero connected peers;
+ *    without a guard its first sweep would read every adopter as dead and
+ *    condemn every adopted provision in the cluster, including the ones the
+ *    majority side is still running. The guard counts *known* peers — static
+ *    config plus everyone handshaken with — never static config alone.
+ *
+ * Modelled on `PendingScaleSweeper` / `HostRosterReaper`: one timer, started and
+ * stopped by the Raft leadership callbacks, so several coordinators never each
+ * tear the same provision down. Unlike those two, every window it runs on is a
+ * live `cluster_settings` knob re-read per sweep — including the interval, which
+ * reschedules the timer at the end of the sweep that observed the change.
+ */
+export declare class EventProvisionReaper {
+    private readonly opts;
+    private timer;
+    private isLeader;
+    /** The interval the live timer was created with, so a change is detectable. */
+    private scheduledIntervalMs;
+    /** Whether the previous sweep was blocked, so the log fires on the edge only. */
+    private reapBlocked;
+    /**
+     * agentId → when this reaper last emitted a teardown for it. A candidate that
+     * comes back means the emit did not clear the row — most often because
+     * neither the row nor the leader's config names a provisioning target, which
+     * `ScalerManager.emitScaleDownForSpec` deliberately answers by keeping the
+     * row and logging. Without this throttle that row would be re-emitted and
+     * re-logged at error level on every single tick, forever.
+     */
+    private readonly lastAttemptAt;
+    /**
+     * agentId → when this reaper first observed the agent registered nowhere.
+     *
+     * This is the debounce that makes "unseen" mean sustained absence rather than
+     * one unlucky sample. It is leader-local observation, deliberately not a
+     * database column: `adopted_at` is refreshed by a coordinator re-adopting its
+     * own agent, so a restart loop on the adopter would keep resetting the clock
+     * on its own stranded provision.
+     */
+    private readonly strandedSince;
+    /**
+     * How many entries of `strandedSince` the last sweep found absent for at
+     * least the flap grace.
+     *
+     * The gauge publishes this rather than `strandedSince.size`, because the two
+     * answer different questions. `size` counts every agent currently observed
+     * absent, and an agent transits that set on every ordinary reconnect — a 30s
+     * peer heartbeat against a 60s sweep, and `PeerRegistry` emptying a peer's
+     * agent list on any disconnect, mean one flap enrols every peer-adopted agent
+     * at once. So `size` is routinely non-zero on a healthy cluster, while
+     * `verdict()` refuses to act on anything below the flap grace. Publishing the
+     * past-grace count makes the gauge measure the same thing the reaper does.
+     */
+    private unseenPastGrace;
+    /**
+     * agentIds already reported as having an unknown owner, so the warning fires
+     * once per row rather than once per sweep. Pruned with the other per-agent
+     * maps when the row leaves the candidate set.
+     */
+    private readonly unknownOwnerLogged;
+    constructor(opts: EventProvisionReaperOptions);
+    onBecomeLeader(): void;
+    onLoseLeadership(): void;
+    stop(): void;
+    /**
+     * The interval the timer currently runs at, or null when no timer is armed.
+     * Public so a caller can observe that a live knob change actually took.
+     */
+    currentIntervalMs(): number | null;
+    /** One sweep pass. Public for tests. Never throws (logs and keeps ticking). */
+    tick(): Promise<void>;
+    /** Tear down every candidate the verdict condemns. */
+    private reapProvisions;
+    /** The candidate half of one sweep, once the blocked check has passed. */
+    private sweepCandidates;
+    /** How many absence clocks have run for at least the flap grace. */
+    private countUnseenPastGrace;
+    /** Emit a teardown for every candidate the verdict condemns. */
+    private condemnCandidates;
+    /**
+     * Publish what this sweep observed.
+     *
+     * The unseen count is the only outward sign of the one failure this design
+     * cannot fix on its own: the absence clock is leader-local, so a cluster with
+     * enough leader churn — or a peer link flapping faster than the stranded
+     * window — restarts it before it can ever expire, and a genuinely stranded
+     * provision bills forever with no log line to show for it.
+     */
+    private publishGauges;
+    /** Delete pending claims past their expiry plus the retention grace. */
+    private purgeClaims;
+    /**
+     * Delete provision outcomes nothing can ask about any more.
+     *
+     * The retention is floored against twice the spawn deadline because that
+     * deadline also floors the stale-spawn prune window: recovery rehydrates a
+     * spawning entry from the spawn row, and that entry asks about its provision
+     * one prune later. The store's own `NOT EXISTS` predicate is what makes the
+     * purge safe while the spawn row is still there; this floor covers the window
+     * after it goes.
+     *
+     * `spawnTimeoutMs` needs no finiteness guard the way `reconcileInterval`'s
+     * period does: `config.scalerSpawnTimeoutMs` is parsed as an integer of at
+     * least 1000, and this is the one window `resolveWindows` takes straight from
+     * config rather than from a `cluster_settings` column.
+     */
+    private purgeOutcomes;
+    /**
+     * Adopt a changed sweep interval by rebuilding the timer.
+     *
+     * Re-read every sweep rather than only at a leadership transition: a knob an
+     * operator can set but that is silently ignored until the next election is
+     * worse than no knob at all. A value that is not a usable interval is ignored
+     * — a zero or negative `setInterval` period would spin the sweep as fast as
+     * the event loop allows.
+     */
+    private reconcileInterval;
+    /** Replace the timer with one running at `intervalMs`. */
+    private schedule;
+    /**
+     * Drop per-agent tracking for rows that are gone, so a long-lived leader's
+     * maps cannot grow without bound.
+     */
+    private pruneTracking;
+    /**
+     * `null` means leave it alone.
+     *
+     * A registered agent is always spared, and seeing it registered resets its
+     * stranded clock: that is the one signal saying the provision is genuinely
+     * alive, and it outranks every other test.
+     *
+     * Otherwise the agent is unseen, and every arm below needs that observation
+     * to have held for at least the flap grace — one sample proves nothing, since
+     * a live agent reads unseen during its own WS reconnect and during the window
+     * after a peer reconnect in which that peer advertises no agents yet.
+     *
+     * An adopted row means the agent reached some coordinator at least once, so
+     * its disappearance is a lost heartbeat rather than a spawn that never
+     * landed — both adopted shapes emit the same reason, and which coordinator
+     * the agent happened to reach stays invisible to the customer's teardown
+     * workflow. They differ only in how much sustained absence they need: a dead
+     * adopter is strong corroboration that the provision is orphaned (its agent's
+     * WS died with it), so the flap grace alone suffices; a live adopter is no
+     * evidence either way, so that arm waits out the full stranded window.
+     *
+     * A row with neither an owner nor an adopter predates migration 119 —
+     * `owner_instance_id IS NULL` reads as "unknown owner", never as "not mine",
+     * so it is left for a human rather than reaped during a rolling upgrade.
+     */
+    private verdict;
+}
+//# sourceMappingURL=event-provision-reaper.d.ts.map

package/dist/scaler/failure-tracker.d.ts

@@ -11,7 +11,7 @@
 export interface ScalerFailureRecord {
     /** Scaler instance name (the configured scaler `name`); the diagnose row key. */
     backendName: string;
-    /** Backend type: 'container' | 'bare-metal' | 'firecracker' | 'unknown'. */
+    /** Backend type: 'container' | 'bare-metal' | 'firecracker' | 'event' | 'unknown'. */
     backendType: string;
     /** True when the failed spawn was bound to a queued job (a run was affected). */
     bound: boolean;

package/dist/scaler/firecracker-backend.d.ts

@@ -20,7 +20,7 @@ import { type ToolRequirement } from '@kici-dev/shared';
 import type { IpAllocator, IpAllocationResult } from './ip-allocator.js';
 import type { AgentTokenStore } from '../agent/token-store.js';
 import { type FirecrackerBridgeConfig, type BridgeHealth, type ExecOptions } from '../firecracker/host-network.js';
-import type { ScalerBackend, ScalerEntry, ManagedAgent, LabelSetConfig, ScalerEventCallback, ValidationResult, EffectiveLimits, SpawnContext } from './types.js';
+import type { ScalerBackend, ScalerDestroyContext, ScalerEntry, ManagedAgent, LabelSetConfig, ScalerEventCallback, ValidationResult, EffectiveLimits, SpawnContext } from './types.js';
 /**
  * Infrastructure interfaces that share the `kici-` prefix but must NEVER be
  * deleted by the orphan sweep. Bridges (kici-br0/kici-br1) carry all VM
@@ -104,7 +104,7 @@ export interface FirecrackerScalerBackendOptions {
 export declare class FirecrackerScalerBackend implements ScalerBackend {
     readonly type: "firecracker";
     readonly spawnsOnLocalHost = true;
-    readonly maxAgents: number;
+    maxAgents: number;
     readonly logsSource = "firecracker-serial";
     /** AbortControllers for file tailing per managed VM (keyed by agent ID) */
     private readonly tailAbortControllers;
@@ -145,7 +145,7 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
     static getRequiredTools(entry: ScalerEntry): ToolRequirement[];
     get labelSets(): LabelSetConfig[];
     getActiveCount(): number;
-    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, _spawnContext?: SpawnContext, _signal?: AbortSignal): Promise<ManagedAgent>;
+    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext, _signal?: AbortSignal): Promise<ManagedAgent>;
     getScalerContext(agentId: string): Record<string, unknown> | undefined;
     /**
      * Bridge config for host-network diagnostics (read-only snapshot).
@@ -165,19 +165,46 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
      * ScalerManager.ensureHostsReady catches per-backend and degrades this scaler.
      */
     ensureHostReady(): Promise<void>;
-    destroy(managedId: string): Promise<void>;
+    destroy(managedId: string, _context?: ScalerDestroyContext): Promise<void>;
+    /**
+     * Force-reclaim a VM this backend no longer tracks in memory.
+     *
+     * `destroy()` opens with `if (!managed) return`, so after an orchestrator
+     * restart it silently no-ops while the VM keeps running, holding its RAM and
+     * its IP. `cleanupOrphans()` cannot recover that VM either: every one of its
+     * three passes skips an id whose jailer process is alive, deliberately, since
+     * a live process is the only thing that distinguishes a healthy VM from a
+     * leaked one when you are sweeping blind.
+     *
+     * This is the same reclaim addressed to ONE id whose orphan status the caller
+     * already established, so the liveness check is exactly what must not apply.
+     * See `ScalerBackend.reapUnowned` for the two properties the caller relies on;
+     * the host-local half is enforced here by refusing to act unless the jailer
+     * chroot for `managedId` exists on THIS host. That directory is the only
+     * host-local artifact a VM leaves: an `ip_allocations` row is not one, because
+     * an HA pair is "two identical orchestrators sharing the same PostgreSQL,
+     * scalers, and routing key" (`docs/operator/orchestrator/clustering.md`), so a
+     * peer's live VM has a row naming this very scaler. The row is read only to
+     * finish a reclaim the chroot already authorized.
+     */
+    reapUnowned(managedId: string): Promise<boolean>;
     shutdownAll(): Promise<void>;
     /**
      * Clear MMDS data for an agent after receiving config.ack via WS.
      * Belt-and-suspenders: even though MMDS only contains orchestrator URL,
      * clearing it reduces the attack surface to zero post-startup.
      *
-     * Note: Wiring from agent-handler is done in Plan 04.
+     * Called by `ScalerManager.onConfigAck`, which reaches it structurally
+     * (`'clearAgentMmds' in backend`) rather than through `ScalerBackend` — no
+     * other backend has an MMDS to clear.
      *
      * @param agentId - The agent ID whose VM MMDS should be cleared
      */
     clearAgentMmds(agentId: string): Promise<void>;
-    reload(labelSets: LabelSetConfig[]): ValidationResult;
+    reload(labelSets: LabelSetConfig[], opts?: {
+        maxAgents?: number;
+        entry?: ScalerEntry;
+    }): ValidationResult;
     /**
      * Clean up orphaned VMs.
      *

package/dist/scaler/index.d.ts

@@ -14,9 +14,19 @@ export { BareMetalScalerBackend } from './bare-metal-backend.js';
 export type { BareMetalScalerBackendOptions } from './bare-metal-backend.js';
 export { FirecrackerScalerBackend } from './firecracker-backend.js';
 export type { FirecrackerScalerBackendOptions, FirecrackerManagedAgent, } from './firecracker-backend.js';
+export { createScalerBackend, requiredToolsFor } from './backend-factory.js';
+export type { BackendFactoryContext, IpAllocatorParams } from './backend-factory.js';
+export { EventScalerBackend } from './event-backend.js';
+export type { EventScalerBackendOptions, ScalerEventEmitterLike } from './event-backend.js';
+export { ClaimStore, DEFAULT_CLAIM_TTL_SECONDS } from './claim-store.js';
+export type { ClaimSpec, ClaimedCredentials, ClaimStoreOptions, CreateEphemeralToken, } from './claim-store.js';
+export { SCALER_EVENT_NAMES, ScaleDownReason, ScalerScaleUpPayload, ScalerScaleDownPayload, KICI_EVENT_NAME_PREFIX, } from './scaler-events.js';
+export type { ScalerDestroyContext } from './types.js';
 export { FirecrackerApi, FirecrackerApiError } from './firecracker-api.js';
 export { ScalerManager } from './manager.js';
-export type { ScalerStatus } from './manager.js';
+export type { ScalerStatus, ScalerManagerDeps } from './manager.js';
+export { EventProvisionReaper } from './event-provision-reaper.js';
+export type { EventProvisionReaperOptions, ReaperWindows } from './event-provision-reaper.js';
 export { WarmPoolManager } from './warm-pool.js';
 export type { WarmPoolCallbacks } from './warm-pool.js';
 export { DbIpAllocator, InMemoryIpAllocator, parseCidr, ipToNumber, numberToIp, generateMac, generateTapName, } from './ip-allocator.js';

package/dist/scaler/label-matcher.d.ts

@@ -52,10 +52,10 @@ export declare function detectLabelSetOverlaps(scalers: Array<{
  * ['macos'] matches ['macos', 'darwin', 'bare-metal']. This is consistent
  * with peer routing (findPeersWithCapacity) which uses the same semantics.
  *
- * In addition, every label declared in a scaler's `mandatoryLabels` must be
+ * In addition, every label in the gate for the matched label set must be
  * present in the job's `runsOn`. This is the Kubernetes-taint-style opt-in
- * gate: a scaler with `mandatoryLabels: ['gpu']` only accepts jobs that
- * explicitly include `gpu` in `runsOn`.
+ * gate: a label set gated on `['gpu']` only accepts jobs that explicitly
+ * include `gpu` in `runsOn`.
  *
  * When multiple backends match, the one with the smallest label set wins
  * (most specific match).
@@ -66,6 +66,7 @@ export declare function findBackendForLabels(labels: string[], scalers: Array<{
         labels: string[];
     }>;
     mandatoryLabels?: string[];
+    labelSetMandatoryLabels?: string[][];
 }>, excludeLabels?: string[]): {
     scalerName: string;
     labelSetIndex: number;