@kici-dev/orchestrator 0.13.0 → 0.14.0

package/dist/scaler/bare-metal-launch.d.ts

@@ -0,0 +1,90 @@
+/**
+ * Starting and ending a bare-metal agent process.
+ *
+ * Each decision about the launch is a pure function, so a test computes the
+ * exact command a host runs without starting anything:
+ *   - Windows runs a `.cmd` / `.bat` launcher through cmd.exe. Node.js refuses
+ *     to start a batch file without a shell, and every launcher a KiCI package
+ *     ships on Windows is one.
+ *   - Linux with `enforceCgroups` wraps the agent in a transient systemd scope.
+ *   - Everything else runs the binary directly.
+ */
+import { type ChildProcess, type SpawnOptions } from 'node:child_process';
+import type { EffectiveLimits } from './types.js';
+export interface AgentLaunchInput {
+    binaryPath: string;
+    agentId: string;
+    effectiveLimits?: EffectiveLimits;
+    enforceCgroups: boolean;
+    platform: NodeJS.Platform;
+    /** The orchestrator's own environment; read for COMSPEC and SystemRoot on Windows. */
+    hostEnv: NodeJS.ProcessEnv;
+}
+/** A command and its arguments, as `spawn` / `execFile` take them. */
+export interface AgentLaunch {
+    command: string;
+    args: string[];
+}
+/**
+ * Why a Windows host cannot run `binaryPath` through cmd.exe, or null. Only a
+ * batch file on a Windows host goes through cmd.exe, so any other path or host
+ * is accepted.
+ */
+export declare function agentLaunchRefusal(binaryPath: string, platform: NodeJS.Platform): string | null;
+/**
+ * The command that starts the agent on this host.
+ *
+ * `CPUQuota` is a percent (1.0 cpus = 100%) and `MemoryMax` a raw byte count.
+ * The scope name embeds the agent id, so `systemctl --user status
+ * kici-agent-<agentId>.scope` finds it.
+ *
+ * @throws DeterministicSpawnError for a Windows batch path cmd.exe would read
+ *   as syntax. Config validation refuses such a path first.
+ */
+export declare function buildAgentLaunch(input: AgentLaunchInput): AgentLaunch;
+/**
+ * Spawn options for an agent process.
+ *
+ * On Linux and macOS the agent is detached: it leads its own process group,
+ * which `destroy()` signals.
+ *
+ * On Windows it is not. A detached process starts with no console, and the
+ * cmd.exe that runs a batch launcher then starts the agent's `node` in a new,
+ * hidden console of its own, so the agent's output never reaches the pipes the
+ * scaler reads. Not detached, cmd.exe gets a hidden console (`windowsHide`
+ * with no inherited stdio) that `node` shares, along with the pipes. Node.js
+ * puts a child that is not detached into a job object that ends it when the
+ * orchestrator exits; processes that child starts are not in that job, so the
+ * agent itself outlives an orchestrator restart, as it does on Linux and macOS.
+ */
+export declare function agentSpawnOptions(env: Record<string, string>, platform: NodeJS.Platform): SpawnOptions;
+/**
+ * Start the agent process.
+ *
+ * Node.js reports a missing or unreadable binary later, through the child's
+ * `'error'` event. Any other launch failure throws here, synchronously: the
+ * host refused this exact invocation (a Windows batch file without cmd.exe, an
+ * environment string longer than the kernel accepts), and it will refuse the
+ * next one the same way.
+ *
+ * @throws DeterministicSpawnError for that synchronous refusal.
+ */
+export declare function startAgentProcess(launch: AgentLaunch, env: Record<string, string>, platform: NodeJS.Platform): ChildProcess;
+/** The taskkill command that ends a Windows process and every process it started. */
+export declare function windowsTreeKill(pid: number, hostEnv: NodeJS.ProcessEnv): AgentLaunch;
+/** What a taskkill exit code means for the tree it was asked to end. */
+export declare enum TreeKillOutcome {
+    Ended = "ended",
+    /** No process had the pid: the tree exited on its own first. */
+    AlreadyGone = "already-gone",
+    Failed = "failed"
+}
+/** taskkill exits 0 when it ended the tree and 128 when no process had the pid. */
+export declare function treeKillOutcome(exitCode: number | null): TreeKillOutcome;
+/**
+ * End a Windows process tree with `taskkill /T /F`. Resolves when the tree was
+ * ended or was already gone; rejects with taskkill's exit code and stderr
+ * otherwise.
+ */
+export declare function killWindowsTree(pid: number, hostEnv: NodeJS.ProcessEnv, timeoutMs: number): Promise<void>;
+//# sourceMappingURL=bare-metal-launch.d.ts.map

package/dist/scaler/firecracker-backend.d.ts

@@ -16,11 +16,19 @@
  *
  * Follows the Docker backend pattern closely for consistency.
  */
+import { ScalerVmTracker, type ScalerVmStopResult } from '@kici-dev/engine';
 import { type ToolRequirement } from '@kici-dev/shared';
+import type { LiveVmBackend, LiveVmProbe } from './live-vms.js';
 import type { IpAllocator, IpAllocationResult } from './ip-allocator.js';
 import type { AgentTokenStore } from '../agent/token-store.js';
 import { type FirecrackerBridgeConfig, type BridgeHealth, type ExecOptions } from '../firecracker/host-network.js';
 import type { ScalerBackend, ScalerDestroyContext, ScalerEntry, ManagedAgent, LabelSetConfig, ScalerEventCallback, ValidationResult, EffectiveLimits, SpawnContext } from './types.js';
+/**
+ * How long a spawn waits for the VM's API socket when nothing configures it.
+ * Generous on purpose: under heavy disk write-back the jailer's copy of the
+ * firecracker binary into the chroot alone can take several seconds.
+ */
+export declare const DEFAULT_API_SOCKET_WAIT_MS = 30000;
 /** Extended ManagedAgent that includes the allocated IP for cleanup */
 export interface FirecrackerManagedAgent extends ManagedAgent {
     /** Allocated IP address for TAP/IP release during destroy */
@@ -35,6 +43,12 @@ export interface FirecrackerManagedAgent extends ManagedAgent {
      * default from.
      */
     hostAccess?: string[];
+    /**
+     * Set once the VM's agent registered with the orchestrator. Until then a VM
+     * that stops on its own is a failed spawn, and nothing else will tear it
+     * down; afterwards the agent's disconnect does.
+     */
+    registered?: boolean;
 }
 export interface FirecrackerScalerBackendOptions {
     /** Human-readable name for this scaler */
@@ -92,6 +106,17 @@ export interface FirecrackerScalerBackendOptions {
      * `cluster_settings.agent_token_ttl_ms` override; falls back to `tokenTtlMs`.
      */
     tokenTtlProvider?: () => Promise<number>;
+    /**
+     * How long a spawn waits for the VM's API socket after the jailer starts, in
+     * ms. Default {@link DEFAULT_API_SOCKET_WAIT_MS}.
+     */
+    apiSocketWaitMs?: number;
+    /**
+     * Live per-spawn resolver for the API-socket wait, so the fleet-wide
+     * `cluster_settings.firecracker_api_socket_wait_ms` applies without a
+     * restart. Falls back to `apiSocketWaitMs`.
+     */
+    apiSocketWaitMsProvider?: () => Promise<number>;
     /** Agent roles for this scaler. undefined = all, [] = execution only. */
     roles?: string[];
     /**
@@ -113,7 +138,7 @@ export interface FirecrackerScalerBackendOptions {
     /** Injectable bridge provisioner (tests). @default host-network provisionBridge */
     provisionBridgeFn?: (cfg: FirecrackerBridgeConfig, opts: ExecOptions) => Promise<void>;
 }
-export declare class FirecrackerScalerBackend implements ScalerBackend {
+export declare class FirecrackerScalerBackend implements ScalerBackend, LiveVmBackend {
     readonly type: "firecracker";
     readonly spawnsOnLocalHost = true;
     maxAgents: number;
@@ -129,6 +154,8 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
      * the failure instead of leaving the reservation held until restart.
      */
     private readonly spawnFailureHandlers;
+    /** Liveness probes of spawned VMs whose agent has not registered yet, by agent id. */
+    private readonly registrationWatches;
     private _labelSets;
     private readonly name;
     private readonly ipAllocator;
@@ -136,6 +163,8 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
     private readonly jailerPath;
     private readonly kernelPath;
     private readonly chrootBaseDir;
+    /** Pre-formatted overlay drive templates under {@link chrootBaseDir}. */
+    private readonly overlayTemplates;
     private readonly uid;
     private readonly gid;
     private readonly vcpuCount;
@@ -152,6 +181,8 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
     private readonly tokenStore?;
     private readonly tokenTtlMs;
     private readonly tokenTtlProvider?;
+    private readonly apiSocketWaitMs;
+    private readonly apiSocketWaitMsProvider?;
     private readonly roles;
     private readonly requireSudo;
     private readonly autoProvisionHost;
@@ -163,6 +194,8 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
     private orphanSweepTimer;
     /** Guard against re-entrant sweeps if a single run takes longer than the interval. */
     private orphanSweepInFlight;
+    /** Tail of the host reclaim chain; see {@link withHostReclaimLock}. */
+    private hostReclaimChain;
     constructor(options: FirecrackerScalerBackendOptions);
     /**
      * Declare required tools for a firecracker scaler entry.
@@ -238,6 +271,56 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
      * so a self-heal that fires twice does not duplicate the rules.
      */
     private reapplyIsolationForTrackedVms;
+    markRegistered(managedId: string): void;
+    /**
+     * Wait for the VM's API socket for the live setting (else the configured
+     * default), and throw when it does not appear in that time.
+     */
+    private waitForApiSocket;
+    /**
+     * Keep loosening a rootless VM's chroot permissions while firecracker starts.
+     *
+     * The jailer chmods the chroot root and every directory it creates (`/run`)
+     * to 0700 owned by its `--uid`, which locks out an orchestrator running as
+     * another user from `/run/firecracker.socket`. A `chmod -R go+rX` every
+     * 100 ms reopens each one as soon as it appears; it only changes the
+     * host-side view. One chmod runs at a time: on a slow disk a `sudo chmod -R`
+     * can outlast the tick, and overlapping ones would pile up for the whole
+     * wait.
+     *
+     * @returns a function that stops the loosening
+     */
+    private loosenChrootPermissions;
+    /** Record a spawn that resolved, and watch its VM until its agent registers. */
+    private markSpawned;
+    /**
+     * Tear the VM down if its process stops before its agent registers.
+     *
+     * A guest whose init exits — the agent's fatal startup error, a kernel panic
+     * — reboots, and firecracker exits. Its agent will never register, and
+     * nothing else releases the VM's `maxAgents` slot, TAP, nft rules, IP and
+     * chroot until the manager's stale-spawn prune. The jailer child's own exit
+     * cannot tell this: under `--new-pid-ns` it exits 0 right after the start.
+     * So the VM's process is probed through its PID file, and the probe stops
+     * once the agent registers (its disconnect owns the teardown from then on),
+     * the VM is being destroyed, or it is no longer tracked.
+     */
+    private watchUntilRegistered;
+    /**
+     * Whether there is positive evidence that a VM's firecracker process is
+     * gone: its PID file does not exist, the PID it names no longer exists, or
+     * that PID now belongs to a process that started after `sinceUptimeS`
+     * (seconds since boot, taken once firecracker was already running).
+     *
+     * Any other failure — an unreadable PID file, a `/proc` this user cannot
+     * see — is not evidence, so the answer is "not gone". The caller tears the
+     * VM down on a "gone", and tearing down a live VM leaves firecracker running
+     * without its TAP, IP and chroot.
+     */
+    private isVmProcessGone;
+    /** Stop forwarding a VM's serial console and VMM logs. */
+    private stopLogTailing;
+    private stopRegistrationWatch;
     destroy(managedId: string, _context?: ScalerDestroyContext): Promise<void>;
     /**
      * Force-reclaim a VM this backend no longer tracks in memory.
@@ -261,6 +344,84 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
      * finish a reclaim the chroot already authorized.
      */
     reapUnowned(managedId: string): Promise<boolean>;
+    private reapUnownedLocked;
+    /** Whether the jailer chroot for `vmId` exists on this host. */
+    private hasChrootOnHost;
+    /**
+     * The `ip_allocations` row for `vmId`, or null. Best effort: a failed read
+     * only costs the IP release, which the next sweep redoes.
+     */
+    private lookupAllocation;
+    /** Whether this scaler's allocator holds the address of `vmId`. */
+    ownsAllocationFor(vmId: string): Promise<boolean>;
+    /**
+     * Reclaim the host resources of a VM nothing tracks, once its process is
+     * dead: forward its remaining logs, remove its isolation rules, TAP and IP,
+     * and delete its chroot.
+     *
+     * A second firecracker scaler on this host has its own IP space and its own
+     * teardown, so only a row naming THIS scaler is ours to release. With no such
+     * row (a worker's in-memory allocator forgets every row on restart) the TAP
+     * named after the VM's id is deleted: the name derives from this VM's id
+     * alone, so it cannot be another VM's.
+     */
+    private teardownUntrackedVm;
+    /** Whether a live VM is one this backend spawned and still tracks. */
+    isTrackingVm(vmId: string): boolean;
+    /**
+     * Every VM under this scaler's chroot base whose firecracker process is
+     * running right now. A VM whose process is gone, or whose PID number was
+     * recycled, is dead and belongs to the orphan sweep, so it is not listed.
+     */
+    listLiveVms(): Promise<LiveVmProbe[]>;
+    /**
+     * Stop one live VM nothing on this node tracks, on an operator's request.
+     *
+     * The tracked check that authorises the kill and the first `kill(2)` run in
+     * one synchronous tick: tracking only changes between ticks (a registration,
+     * a rehydrated spawn row, a new spawn), so nothing can claim the VM between
+     * the check and the signal. The kernel can still recycle the PID number in
+     * that window; `destroy` and `reapUnowned` carry the same window, and Node
+     * has no pidfd to close it.
+     *
+     * Runs under the host reclaim lock, so a sweep cannot release this VM's
+     * address to a new VM while the stop is still removing its rules.
+     *
+     * @param trackersOf what else on this node tracks the VM, read in the
+     *   kill's tick; it must not await.
+     */
+    stopUntrackedVm(vmId: string, trackersOf: () => ScalerVmTracker[]): Promise<ScalerVmStopResult>;
+    private stopUntrackedVmLocked;
+    /**
+     * SIGKILL a VM's firecracker process.
+     *
+     * The jailer drops firecracker to `--uid`, so an orchestrator running as
+     * another unprivileged user gets EPERM from `kill(2)`. On a rootless host
+     * (`requireSudo`) the signal goes through `sudo -n -u #<uid> kill`; anywhere
+     * else EPERM means the process lacks CAP_KILL. ESRCH means the process is
+     * already gone, which is the outcome wanted.
+     *
+     * The first statement is the synchronous `kill(2)`: `stopUntrackedVm` relies
+     * on the signal leaving in its caller's tick.
+     */
+    private signalVmProcess;
+    /** Whether `/proc/<pid>` names a process right now. */
+    private procEntryExists;
+    /**
+     * {@link signalVmProcess} for the teardown paths: a failure is logged, never
+     * swallowed, and reported so the caller keeps the chroot of a process that
+     * still runs.
+     *
+     * @returns whether the process was signalled or was already gone
+     */
+    private signalVmProcessOrLog;
+    /**
+     * Serialise host reclaims: the orphan sweep, the unowned-VM reclaim and the
+     * operator stop. Each frees addresses and deletes rules and TAPs by name, so
+     * two interleaved can hand an address one is still tearing down to a new VM
+     * and then delete the new VM's rules.
+     */
+    private withHostReclaimLock;
     shutdownAll(): Promise<void>;
     /**
      * Clear MMDS data for an agent after receiving config.ack via WS.
@@ -295,6 +456,7 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
      * Returns the count of cleaned orphans.
      */
     cleanupOrphans(): Promise<number>;
+    private cleanupOrphansLocked;
     /**
      * Delete isolation rules for every identifier no live VM owns.
      *
@@ -360,13 +522,24 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
      */
     buildVmConfig(alloc: IpAllocationResult, labelSet: LabelSetConfig, effectiveLimits?: EffectiveLimits): Record<string, unknown>;
     /**
-     * Create a sparse ext4 overlay drive for per-VM writable layer.
-     * The file is sparse (allocates no disk blocks until written) and
-     * pre-formatted as ext4 so the guest can mount it immediately.
+     * Create the per-VM writable overlay drive: a sparse copy of this host's
+     * pre-formatted ext4 template for the size, so the guest can mount it
+     * immediately. See `overlay-template.ts` for why a spawn never formats.
      */
     private createOverlayDrive;
     /**
-     * Promisified execFile wrapper with 30s timeout.
+     * Build the overlay template of every configured drive size in the
+     * background, so the first spawn after startup copies instead of formatting.
+     * A failure is logged; the first spawn of that size builds it instead.
+     */
+    private prewarmOverlayTemplates;
+    /**
+     * Run a host command with a timeout (30s unless the caller passes one).
+     *
+     * A failure rejects with a `CommandError` whose message names the command,
+     * its exit code or signal, whether the timeout killed it, how long it ran,
+     * and the tail of its stderr. Spawn steps throw it as-is, so the same text
+     * reaches the `scaler.failed` event detail and the spawn-failure logs.
      */
     private execAsync;
     /**
@@ -406,8 +579,9 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
      *     naming it was written;
      *  4. field 2 (`comm`) is `firecracker` (11 characters, so it is not
      *     truncated by the 15-character `comm` limit) and `/proc/<pid>/root` —
-     *     where the jailer pivot_roots — is this VM's chroot when the link is
-     *     readable at all. Those carry `identityConfirmed`, not liveness.
+     *     where the jailer pivot_roots — resolves to this VM's chroot (same
+     *     device and inode) when it is readable at all. Those carry
+     *     `identityConfirmed`, not liveness.
      */
     private probeVmPid;
     /**

package/dist/scaler/live-vms.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Live Firecracker VMs on this host, and how the orchestrator relates to each.
+ *
+ * A Firecracker backend inventories the VMs whose firecracker process is still
+ * running under its chroot base (`listLiveVms`); the manager classifies each
+ * one against everything on this node that tracks a VM.
+ */
+import { type ScalerLiveVm, type ScalerVmStopResult, type ScalerVmTracker } from '@kici-dev/engine';
+/** One running VM a Firecracker backend found under its chroot base. */
+export interface LiveVmProbe {
+    vmId: string;
+    /** The scaler whose chroot base holds the VM. */
+    scaler: string;
+    pid: number;
+    /** When the process started, from `/proc/<pid>/stat`. */
+    startedAtMs: number;
+    chrootDir: string;
+    /** Whether the process is provably this VM's own firecracker. */
+    identityConfirmed: boolean;
+    /** Why the identity is unconfirmed; set only when `identityConfirmed` is false. */
+    detail?: string;
+}
+/** Why an orphaned VM is listed as one. */
+export declare const ORPHANED_REASON = "no in-memory VM, spawning entry, registered agent or active job binding";
+/** The Firecracker backend surface the live-VM listing and stop rely on. */
+export interface LiveVmBackend {
+    listLiveVms(): Promise<LiveVmProbe[]>;
+    /** Synchronous: the stop reads it in the same tick as its kill. */
+    isTrackingVm(vmId: string): boolean;
+    stopUntrackedVm(vmId: string, trackersOf: () => ScalerVmTracker[]): Promise<ScalerVmStopResult>;
+    /** Whether this scaler's allocator holds the VM's address. */
+    ownsAllocationFor(vmId: string): Promise<boolean>;
+}
+/**
+ * Classify one live VM: tracked by anything wins, then a confirmed identity is
+ * an orphan, and a running process that is not provably the VM's firecracker
+ * is unverified — listed, but never stopped.
+ */
+export declare function classifyLiveVm(probe: LiveVmProbe, trackers: ScalerVmTracker[], nowMs?: number): ScalerLiveVm;
+/** A local Firecracker backend under its scaler name. */
+export type NamedLiveVmBackend = readonly [name: string, backend: LiveVmBackend];
+/**
+ * Every live VM across `backends`, classified against `trackersOf`. Two
+ * scalers sharing a chroot base both list the same VM; it is reported once.
+ */
+export declare function listClassifiedLiveVms(backends: readonly NamedLiveVmBackend[], trackersOf: (vmId: string) => ScalerVmTracker[]): Promise<ScalerLiveVm[]>;
+/**
+ * Stop each VM in turn through the backend whose chroot holds it.
+ *
+ * The backend is the one whose scan lists the VM, preferring the one whose
+ * allocator holds its address when two scalers share a chroot base. A VM no
+ * scan lists is not running: each backend is asked in turn, so the answer
+ * names why (`not-live` for a dead chroot), and `not-found` when no backend
+ * holds a chroot for it.
+ */
+export declare function stopLiveVms(backends: readonly NamedLiveVmBackend[], vmIds: readonly string[], trackersOf: (vmId: string) => ScalerVmTracker[]): Promise<ScalerVmStopResult[]>;
+//# sourceMappingURL=live-vms.d.ts.map

package/dist/scaler/manager.d.ts

@@ -6,7 +6,8 @@
  * enforces global limits, tracks spawning agents to prevent over-provisioning,
  * and manages the agent lifecycle from spawn to destroy.
  */
-import type { ResourceRequest, ScalerCapacitySummary } from '@kici-dev/engine';
+import { ScalerVmTracker } from '@kici-dev/engine';
+import type { ResourceRequest, ScalerCapacitySummary, ScalerLiveVm, ScalerVmStopResult } from '@kici-dev/engine';
 import { ScaleDownReason } from './scaler-events.js';
 import { type ScalerAgentView } from './agent-fit.js';
 import type { ScalerEventEmitterLike } from './event-backend.js';
@@ -179,6 +180,11 @@ export interface ProvisionBackoffSettings {
     /** Consecutive failures past which a refusal names repeated failure. */
     maxConsecutiveFailures: number;
 }
+/** A job a pending provision was requested for. */
+export interface BoundJobRef {
+    runId: string;
+    jobId: string;
+}
 /** Everything the {@link ScalerManager} is constructed from. */
 export interface ScalerManagerDeps {
     config: ScalerConfig;
@@ -273,6 +279,14 @@ export interface ScalerManagerDeps {
      * {@link DEFAULT_PROVISION_BACKOFF}.
      */
     resolveProvisionBackoff?: () => Promise<ProvisionBackoffSettings>;
+    /**
+     * Current `execution_jobs.status` of the jobs bound to pending event
+     * provisions, keyed by job id. A job with no row is absent from the map.
+     * Each job comes with its run so the read stays on the `(run_id, job_id)`
+     * index. Wired in orchestrator-core (has DB); omitted by the worker and most
+     * unit tests, which then withdraw nothing and report every pruned provision.
+     */
+    readJobStatuses?: (jobs: readonly BoundJobRef[]) => Promise<ReadonlyMap<string, string>>;
     /**
      * Constructs a backend for a scaler added by a config reload. Supplied by
      * both hosts from the shared backend factory, so a reload can only build
@@ -303,8 +317,10 @@ export declare class ScalerManager {
     /** Recent scaler spawn failures, surfaced by `kici-admin diagnose`. */
     private readonly failureTracker;
     /**
-     * Consecutive external-provision failures per scaler NAME, and the deferral
-     * they earned.
+     * Consecutive provisioning failures per scaler NAME, and the deferral they
+     * earned: an event scaler's external provision that never registered an
+     * agent, or a local launch that produced no agent (refused outright, or the
+     * agent stopped before it registered).
      *
      * Keyed by name rather than by backend type so one failing scaler never
      * defers spawns for an unrelated one — two event scalers routinely drive two
@@ -316,6 +332,7 @@ export declare class ScalerManager {
      */
     private readonly provisionFailures;
     private readonly resolveProvisionBackoff;
+    private readonly readJobStatuses?;
     private globalMaxAgents;
     /** Per-scaler resource caps (`{ maxCpu, maxMemoryBytes }`), keyed by scaler name. */
     private readonly resourceCaps;
@@ -524,6 +541,27 @@ export declare class ScalerManager {
      * Whether this scaler pre-spawned (warm-filled) the agent.
      */
     isPrespawnedAgent(agentId: string): boolean;
+    /** Every local Firecracker backend, under its scaler name. */
+    private firecrackerBackends;
+    /** The names of this node's Firecracker scalers. */
+    firecrackerScalerNames(): string[];
+    /**
+     * What on this node tracks `vmId`. Synchronous on purpose: the Firecracker
+     * stop calls it in the same tick as its kill, so it must not await.
+     *
+     * Every local Firecracker backend is asked, so two scalers sharing a chroot
+     * base never see each other's VMs as orphans. `spawningAgents` includes the
+     * entries `recoverState` rehydrated after a restart, which no backend tracks.
+     */
+    trackersOf(vmId: string): ScalerVmTracker[];
+    /** Every live Firecracker VM on this host, classified against what tracks it. */
+    listLiveVms(): Promise<ScalerLiveVm[]>;
+    /**
+     * Stop the approved VMs one at a time. Each stop re-checks {@link trackersOf}
+     * in the tick of its kill, so a VM that became tracked since the listing is
+     * refused, never signalled.
+     */
+    stopUntrackedVms(vmIds: readonly string[]): Promise<ScalerVmStopResult[]>;
     /**
      * What this scaler knows about an agent it started, for `canAgentRunJob`,
      * or undefined for an agent it did not start.
@@ -1054,6 +1092,47 @@ export declare class ScalerManager {
      * immediately.
      */
     private reportProvisionFailure;
+    /**
+     * Defer a scaler whose host refused to start an agent, then free the spawn's
+     * reservation.
+     *
+     * Arming comes first: the release notifies freed capacity, and the re-drive
+     * that follows 250 ms later must meet the deferral rather than spawn straight
+     * into the same refusal. The release still notifies, because a job waiting on
+     * another scaler, or on the global cap, is legitimately unblocked by it.
+     * Two failures come here: a `spawn()` that rejected with a
+     * `DeterministicSpawnError`, and a settled spawn whose compute stopped before
+     * its agent registered (`dropFailedSettledSpawn`). Every other spawn
+     * rejection is released at once and retried.
+     */
+    private deferRefusedLaunch;
+    /**
+     * Whether a bound job has ended. A read error, a missing row, an unknown
+     * run, or no reader wired all read as "not ended", so the caller keeps
+     * reporting the provision as it would without the check.
+     */
+    private boundJobEnded;
+    /**
+     * Withdraw every settled, job-bound event provision on this coordinator
+     * whose job already ended — cancelled, timed out, or run on another agent.
+     *
+     * The job's row is the shared truth: a cancel reaches whichever coordinator
+     * the Platform picks, while the spawning entry lives on the one that asked.
+     * A withdrawal is not a provisioning failure, so nothing is reported and no
+     * backoff is armed. Returns how many provisions were withdrawn.
+     */
+    withdrawEndedBoundProvisions(): Promise<number>;
+    /**
+     * Forget one pending provision and tear it down as `job-complete` — unless a
+     * peer adopted its agent, in which case the adopter owns the teardown.
+     *
+     * The teardown goes through `emitScaleDownForSpec`, which drops the spawn
+     * row only once the scale-down is out: a failed emit keeps the row for the
+     * reaper, whose spawn-timeout verdict re-checks the job and sends
+     * `job-complete` too. `backend.destroy` is not used — a rehydrated entry is
+     * not tracked by the backend, so it would emit nothing.
+     */
+    private withdrawProvision;
     /**
      * Report a pruned event spawn as a dead provision — unless a peer adopted it.
      *
@@ -1085,6 +1164,8 @@ export declare class ScalerManager {
      * existed, and they age out within one stale-prune window of the deploy.
      */
     private reportPrunedProvisionFailure;
+    /** The adoption half of {@link reportPrunedProvisionFailure}. */
+    private reportPrunedUnlessAdopted;
     /**
      * Clear a scaler's consecutive-failure state after a successful provision.
      *
@@ -1397,6 +1478,34 @@ export declare class ScalerManager {
      * other instance already claimed is left exactly as it is.
      */
     private markSpawningAgentAdopted;
+    /**
+     * Record that `backend.spawn()` resolved for a spawn this instance started.
+     *
+     * Returns false when the stale-spawn prune dropped the entry while
+     * `spawn()` was still running (a per-org spawn deadline can outlast the
+     * prune window). Nothing else would ever tear that compute down, so a local
+     * backend's is destroyed here. An event backend's provision is left to the
+     * reaper, as the prune leaves it.
+     */
+    private onSpawnSettled;
+    /**
+     * Drop a settled local spawn whose backend reported it failed.
+     *
+     * After `spawn()` resolved, a backend that finds its compute gone before the
+     * agent registered (a Firecracker guest that exited, a bare-metal agent
+     * process that exited) tears the compute down itself and emits
+     * `scaler.failed`. The agent will never register, so the spawning entry, its
+     * durable row and its reservation go now — waiting for the stale-spawn prune
+     * would hold the scaler's slot for minutes.
+     *
+     * A spawn that produced no agent is a refused launch, the same rule an event
+     * scaler's dead provision follows: the scaler is deferred before the
+     * reservation is freed, so an agent that dies at startup every time is not
+     * respawned at its own startup cadence. Only a spawn that has not registered
+     * reaches here — registration deletes the spawning entry — so an agent that
+     * registered and exits later never arms the backoff.
+     */
+    private dropFailedSettledSpawn;
     /** Store-side half of the stale prune: drops the row only if nobody adopted it. */
     private deleteUnadoptedSpawningAgentFromStore;
     private persistReservation;

package/dist/scaler/orphan-requests.d.ts

@@ -0,0 +1,23 @@
+import { ScalerOrphansAction, type PeerScalerOrphansResponse } from '@kici-dev/engine';
+import type { ScalerManager } from './manager.js';
+/**
+ * The longest a scaler orphan request waits for its node. The `kici-admin`
+ * HTTP client stops waiting for response headers after 300 s, so a longer
+ * wait could never reach the operator.
+ */
+export declare const MAX_SCALER_ORPHANS_TIMEOUT_MS = 240000;
+/** A scaler orphan answer without its peer-message envelope. */
+export type ScalerOrphansAnswer = Omit<PeerScalerOrphansResponse, 'type' | 'messageId'>;
+/** What one scaler orphan request asks of the node. */
+export interface ScalerOrphansRequest {
+    action: ScalerOrphansAction;
+    /** `stop` only: the VM ids the operator approved. */
+    vmIds?: string[];
+}
+/**
+ * Answer a scaler orphan request on this node. A node with no scaler manager
+ * runs no Firecracker scaler: it lists nothing and holds none of the VMs a stop
+ * names. A failure is answered `ok: false` with its message, never thrown.
+ */
+export declare function answerScalerOrphansRequest(manager: Pick<ScalerManager, 'listLiveVms' | 'stopUntrackedVms' | 'firecrackerScalerNames'> | null, req: ScalerOrphansRequest): Promise<ScalerOrphansAnswer>;
+//# sourceMappingURL=orphan-requests.d.ts.map

package/dist/scaler/overlay-template.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Directory that holds the templates, relative to the jailer chroot base dir.
+ * It sits beside the per-VM chroots because the orchestrator must already be
+ * able to create directories there; the leading dot keeps it out of the
+ * orphan sweep, which reads every other entry as a VM id.
+ */
+export declare const OVERLAY_TEMPLATE_DIR: string;
+/**
+ * Timeout for formatting a template. It runs once per size per host, mostly
+ * off a spawn's critical path, and its `fsync` may wait behind a disk backlog,
+ * so it gets far more room than a per-spawn step.
+ */
+export declare const OVERLAY_TEMPLATE_MKFS_TIMEOUT_MS = 300000;
+/** Timeout for the per-spawn copy. */
+export declare const OVERLAY_COPY_TIMEOUT_MS = 30000;
+/** Runs a host command, rejecting on failure. The backend passes its `execAsync`. */
+export type OverlayCommandRunner = (cmd: string, args: string[], timeoutMs: number) => Promise<unknown>;
+/** Where the template for a drive size lives. */
+export declare function overlayTemplatePath(chrootBaseDir: string, sizeMib: number): string;
+/**
+ * Whether `path` is a usable template for `sizeMib`: it exists, its size is
+ * exactly `sizeMib` MiB, and it carries the ext4 superblock magic.
+ */
+export declare function isOverlayTemplate(path: string, sizeMib: number): Promise<boolean>;
+/** Builds and hands out overlay templates for one jailer chroot base dir. */
+export declare class OverlayTemplates {
+    private readonly chrootBaseDir;
+    private readonly run;
+    /** In-flight builds by size, so concurrent first uses share one build. */
+    private readonly builds;
+    constructor(chrootBaseDir: string, run: OverlayCommandRunner);
+    /** Return the path of a valid template for `sizeMib`, building it if needed. */
+    ensure(sizeMib: number): Promise<string>;
+    /** Create the overlay drive at `dest` as a sparse copy of the template. */
+    createOverlay(dest: string, sizeMib: number): Promise<void>;
+    private build;
+}
+//# sourceMappingURL=overlay-template.d.ts.map