@kici-dev/orchestrator 0.6.1 → 0.8.0

package/dist/scaler/container-backend.d.ts

@@ -6,9 +6,14 @@
  * Handles full container lifecycle: pull, create, start, stop, remove.
  * Supports container socket sharing, resource limits, and orphan cleanup.
  */
+import Docker from 'dockerode';
 import { type ToolRequirement } from '@kici-dev/shared';
 import type { AgentTokenStore } from '../agent/token-store.js';
 import type { ScalerBackend, ScalerDestroyContext, ManagedAgent, LabelSetConfig, LogCapture, ResourceRequest, EffectiveLimits, SpawnContext, ScalerEventCallback, ValidationResult, ScalerEntry } from './types.js';
+/** Name of the isolated bridge network for agent containers. */
+export declare const ISOLATED_NETWORK_NAME = "kici-agent-net";
+/** Gateway IP for the isolated agent network (host-side). */
+export declare const ISOLATED_NETWORK_GATEWAY = "172.30.0.1";
 /**
  * Result of runtime detection.
  */
@@ -40,6 +45,12 @@ export interface ContainerScalerBackendOptions {
     extraHosts?: string[];
     /** Disable nftables-based network isolation (default: true). Set to false when nft is unavailable (e.g. rootless containers without NET_ADMIN). */
     networkIsolation?: boolean;
+    /**
+     * Extra `hostAccess` entries the orchestrator itself directed agents at —
+     * today its object-storage endpoint. Folded into the agent default so the
+     * narrowing does not cut off a host-local object store.
+     */
+    hostServices?: string[];
     /** Token store for creating ephemeral agent auth tokens. Optional -- when undefined, no token is injected. */
     tokenStore?: AgentTokenStore;
     /** TTL for ephemeral agent tokens in ms. Default: 1 hour. */
@@ -52,7 +63,27 @@ export interface ContainerScalerBackendOptions {
     tokenTtlProvider?: () => Promise<number>;
     /** Agent roles for this scaler. undefined = all, [] = execution only. */
     roles?: string[];
+    /**
+     * Whether an agent id is currently registered with this orchestrator.
+     *
+     * Used by `cleanupOrphans` to distinguish a running container whose agent is
+     * doing work from one whose agent never registered (or has gone). Defaults
+     * to "assume registered" when unwired, so an un-plumbed caller never reaps a
+     * live agent.
+     */
+    isRegistered?: (agentId: string) => boolean;
 }
+/**
+ * Create or find the isolated agent bridge network, and return its id.
+ *
+ * Module-level rather than a method because the bare-metal backend's container
+ * mode attaches its agent containers to the same network: both agent-container
+ * spawn paths must land on it, or one of them runs on the runtime's default
+ * bridge with full LAN, RFC1918 and cloud-metadata reach.
+ *
+ * Idempotent, including against a concurrent creator (409).
+ */
+export declare function ensureIsolatedNetwork(docker: Docker): Promise<string>;
 export declare class ContainerScalerBackend implements ScalerBackend {
     readonly type: "container";
     readonly spawnsOnLocalHost: boolean;
@@ -69,6 +100,8 @@ export declare class ContainerScalerBackend implements ScalerBackend {
     private readonly extraHosts?;
     /** Whether nftables-based network isolation is enabled */
     private readonly networkIsolation;
+    /** Host services the orchestrator directed agents at, as `hostAccess` entries. */
+    private readonly hostServices?;
     /** Token store for creating ephemeral agent auth tokens */
     private readonly tokenStore?;
     /** TTL for ephemeral agent tokens in ms */
@@ -77,6 +110,8 @@ export declare class ContainerScalerBackend implements ScalerBackend {
     private readonly tokenTtlProvider?;
     /** Agent roles for this scaler. undefined = all, [] = execution only. */
     private readonly roles;
+    /** Registration probe for the orphan sweep; fail-safe default assumes registered. */
+    private readonly isRegistered;
     /** ID of the isolated bridge network (set after creation/discovery) */
     private isolatedNetworkId;
     /** Host bridge interface name for the isolated network (used by nftables) */
@@ -132,6 +167,15 @@ export declare class ContainerScalerBackend implements ScalerBackend {
      * `spawnContext` rather than letting the agent claim them.
      */
     private mintAgentIdentity;
+    /**
+     * Write one agent container's nftables rules, on both hooks.
+     *
+     * `forward` governs what the container reaches THROUGH the host; `input`
+     * governs what it reaches ON the host. A packet to one of the host's own
+     * addresses is delivered on input and never traverses forward, so the two
+     * are not interchangeable and both are written here.
+     */
+    private applyIsolationRules;
     spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext, signal?: AbortSignal): Promise<ManagedAgent>;
     getScalerContext(agentId: string): Record<string, unknown> | undefined;
     destroy(managedId: string, _context?: ScalerDestroyContext): Promise<void>;
@@ -145,10 +189,65 @@ export declare class ContainerScalerBackend implements ScalerBackend {
         entry?: ScalerEntry;
     }): ValidationResult;
     /**
-     * Clean up orphaned kici-managed containers on startup.
-     * Finds containers with the `kici-managed=true` label and removes them.
-     * Returns the count of cleaned containers.
+     * Clean up this scaler's orphaned containers and stale isolation rules.
+     *
+     * Every filter below exists because the sweep used to list every
+     * `kici-managed=true` container on the host and force-remove it, running at
+     * orchestrator boot, at worker-peer start, on a live config reload that adds
+     * a scaler, and from `kici-admin scaler reap-orphans`. Two orchestrators
+     * sharing one docker host therefore killed each other's running agents on
+     * every boot, and adding a second container scaler removed the first one's
+     * agents mid-job — every in-flight job failing as "agent disconnected" on a
+     * healthy system.
+     *
+     * A container is removed only when all of these hold:
+     *  - it carries this scaler's own `kici-scaler-name`, so a backend can only
+     *    ever reap what it stamped;
+     *  - its `kici-agent-id` is not one this backend is currently tracking
+     *    (spawning, running, or destroying);
+     *  - it is not running, or it is running with an agent id that is not
+     *    registered — a running, registered agent is doing work.
+     *
+     * Returns the count of cleaned containers plus reaped rules.
      */
     cleanupOrphans(): Promise<number>;
+    /**
+     * Remove containers this backend spawned for one agent id.
+     *
+     * Implements the optional `ScalerBackend.reapUnowned` hook. It is what
+     * reclaims a container whose registration the orchestrator refused — the
+     * case `cleanupOrphans` used to cover by removing every `kici-managed`
+     * container on the host, which is exactly the breadth that killed other
+     * orchestrators' live agents.
+     *
+     * @returns true when a container was found and removed
+     */
+    reapUnowned(managedId: string): Promise<boolean>;
+    /**
+     * Delete isolation rules for container IPs on this scaler's isolated network
+     * that no tracked container holds.
+     *
+     * Rules are removed only on the teardown paths this process drives, so a
+     * crash or a `kill -9` strands them. The network's address pool is recycled,
+     * so the next container to take that IP inherits the dead job's allowlist.
+     *
+     * Scoped to addresses no live container holds: `kici-agent-net` is one fixed
+     * network shared by every container-spawning backend on the host, so this
+     * backend's own tracking map is not enough to tell an abandoned address from
+     * a neighbour's live one.
+     */
+    private reapUnownedIsolationRules;
+    /**
+     * Every address held on the isolated agent network by a kici-managed
+     * container, across every scaler and every orchestrator on this runtime.
+     *
+     * Deliberately NOT filtered by `kici-scaler-name`: the point is to see the
+     * neighbours, and a neighbour is by definition stamped with a name this
+     * backend does not know.
+     *
+     * @returns the addresses, or `null` when the runtime could not be listed — in
+     * which case the caller reaps nothing rather than reaping blind.
+     */
+    private isolatedNetworkAddressesInUse;
 }
 //# sourceMappingURL=container-backend.d.ts.map

package/dist/scaler/container-hostconfig.d.ts

@@ -0,0 +1,60 @@
+/**
+ * The `HostConfig` every KiCI agent container is created with.
+ *
+ * Two backends spawn an agent container — `ContainerScalerBackend` and the
+ * bare-metal backend's container mode — and they had diverged into two
+ * postures for one threat: the first applied resource limits, the second
+ * applied none and dropped the limits the operator configured on the floor.
+ * One builder is what keeps them from diverging again the next time either is
+ * touched.
+ */
+import type Docker from 'dockerode';
+import type { EffectiveLimits } from './types.js';
+/**
+ * Maximum processes an agent container may create.
+ *
+ * High enough that no realistic build reaches it, low enough to bound a fork
+ * bomb. Not operator-configurable: a `limits.pids` key is a new surface with
+ * its own docs and validation, and nothing yet says the constant is wrong.
+ */
+export declare const AGENT_CONTAINER_PIDS_LIMIT = 4096;
+/**
+ * Why this container is NOT capability-dropped, though the job containers the
+ * agent starts are.
+ *
+ * Both spawn paths set `KICI_EXECUTION_MODE=bare-metal`, and bare-metal
+ * container mode also sets `KICI_JOB_IMAGE_AGENT=1` — so unless the job
+ * declares its own image, the workflow's steps run as host processes INSIDE
+ * this container. It is a build execution environment, not a supervisor.
+ * `CapDrop: ['ALL']` here would therefore land on customer build steps:
+ * `apt-get` and `dpkg` need CHOWN, DAC_OVERRIDE, FOWNER, SETUID and SETGID,
+ * and `no-new-privileges` breaks `sudo` and every setuid binary. There is no
+ * opt-out, so the break would be silent and total for `type: container`
+ * scalers.
+ *
+ * The hardening the operator docs promise already exists, one boundary in:
+ * the agent applies `CapDrop: ['ALL']`, `no-new-privileges` and a tighter
+ * `PidsLimit` to each nested JOB container
+ * (`packages/agent/src/execution/sandbox/container-hardening.ts`). That is the
+ * boundary that confines customer code; this one hosts it.
+ *
+ * `PidsLimit` is the exception and stays: a fork-bomb ceiling costs a build
+ * nothing at 4096.
+ */
+export interface AgentContainerHostConfigInput {
+    /** Resolved cpu / memory ceiling for this spawn, when the operator set one. */
+    limits?: EffectiveLimits;
+    /** Volume and path bind mounts. */
+    binds?: string[];
+    /** Extra `host:ip` entries (e.g. a private registry alias). */
+    extraHosts?: string[];
+}
+/**
+ * Build the agent container's `HostConfig`.
+ *
+ * Carries the operator's resource limits, a fork-bomb ceiling, and nothing
+ * that would restrict the build steps this container runs — see the note on
+ * {@link AGENT_CONTAINER_PIDS_LIMIT}.
+ */
+export declare function buildAgentContainerHostConfig(input: AgentContainerHostConfigInput): Docker.HostConfig;
+//# sourceMappingURL=container-hostconfig.d.ts.map

package/dist/scaler/event-backend.d.ts

@@ -108,12 +108,13 @@ export declare class EventScalerBackend implements ScalerBackend {
      */
     shutdownAll(): Promise<void>;
     /**
-     * Apply the new config. The backend reads `roles`, `mandatoryLabels`,
-     * `agentTokenTtlSeconds` and `provisioningTargets` off its entry at every
-     * spawn, so the entry is replaced here — otherwise a reload that retargets
-     * the provisioning workflow, or changes the scaler's roles, would keep
-     * emitting scale-up events to the old workflow refs and minting agent labels
-     * from the old roles, while the manager's own routing gate used the new ones.
+     * Apply the new config. The backend reads `roles`, `agentTokenTtlSeconds`
+     * and `provisioningTargets` off its entry at every spawn (and
+     * `mandatoryLabels` only when a spawn reaches it without a context), so the
+     * entry is replaced here — otherwise a reload that retargets the
+     * provisioning workflow, or changes the scaler's roles, would keep emitting
+     * scale-up events to the old workflow refs and minting agent labels from the
+     * old roles, while the manager's own routing gate used the new ones.
      */
     reload(labelSets: LabelSetConfig[], opts?: {
         maxAgents?: number;

package/dist/scaler/file-tail.d.ts

@@ -2,12 +2,35 @@
  * File tailing utility for append-only log files.
  *
  * Provides an async generator that yields complete lines as they are appended
- * to a file, using `fs.watch()` for change notifications and partial line
+ * to a file, using `fs.watchFile()` for change notifications and partial line
  * buffering for split writes.
  *
  * Used by the Firecracker backend to tail serial console and VMM log files
- * from the jailer chroot directory.
+ * from the jailer chroot directory. The guest writes to `/dev/console`
+ * unprivileged, so everything this module buffers is tenant-controlled: every
+ * buffer here is bounded, and an overflow is reported in-band rather than
+ * growing the orchestrator's heap.
  */
+/**
+ * Largest logical line kept intact. A guest that writes without ever emitting a
+ * newline (`yes | tr -d '\n' > /dev/console`) would otherwise grow the partial
+ * buffer at disk-write speed until the Node heap limit takes the orchestrator —
+ * and every in-flight job on it — down.
+ */
+export declare const MAX_LINE_BYTES: number;
+/**
+ * Largest slice read from the file per poll. A burst is drained across several
+ * ticks instead of materialising the whole delta in one string.
+ */
+export declare const MAX_READ_BYTES: number;
+/**
+ * Largest number of complete lines held for the consumer. `forwardLine` →
+ * Winston → Loki is orders of magnitude slower than a guest can write to the
+ * serial console, so the queue is the second place unbounded growth appears.
+ * Matches the agent-side log buffer's cap, which uses gap markers for the same
+ * situation.
+ */
+export declare const MAX_QUEUED_LINES = 10000;
 /**
  * Tail an append-only file, yielding complete lines as they are written.
  *

package/dist/scaler/firecracker-api.d.ts

@@ -35,7 +35,15 @@ export declare class FirecrackerApiError extends Error {
  */
 export declare class FirecrackerApi {
     private readonly socketPath;
-    constructor(socketPath: string);
+    /**
+     * Aborts every in-flight and subsequent request on this client.
+     *
+     * Set by the spawn path so a spawn deadline actually stops the VM
+     * provisioning calls, rather than leaving them running against a VM the
+     * manager has already given up on.
+     */
+    private signal;
+    constructor(socketPath: string, signal?: AbortSignal);
     /**
      * Send an HTTP request to the Firecracker API.
      *

package/dist/scaler/firecracker-backend.d.ts

@@ -34,6 +34,14 @@ export interface FirecrackerManagedAgent extends ManagedAgent {
     ip: string;
     /** TAP device name for cleanup */
     tapDevice: string;
+    /**
+     * The `hostAccess` policy resolved for this VM at provision time.
+     *
+     * Carried on the record because a bridge self-heal re-applies isolation for
+     * every live VM, and that path has no `orchestratorUrl` to re-derive the
+     * default from.
+     */
+    hostAccess?: string[];
 }
 export interface FirecrackerScalerBackendOptions {
     /** Human-readable name for this scaler */
@@ -44,6 +52,12 @@ export interface FirecrackerScalerBackendOptions {
     maxAgents: number;
     /** Shared IP allocator instance */
     ipAllocator: IpAllocator;
+    /**
+     * Extra `hostAccess` entries the orchestrator itself directed agents at —
+     * today its object-storage endpoint. Folded into the agent default so the
+     * narrowing does not cut off a host-local object store.
+     */
+    hostServices?: string[];
     /** Path to firecracker binary */
     firecrackerPath: string;
     /** Path to jailer binary */
@@ -108,6 +122,15 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
     readonly logsSource = "firecracker-serial";
     /** AbortControllers for file tailing per managed VM (keyed by agent ID) */
     private readonly tailAbortControllers;
+    /**
+     * Per-spawn `scaler.failed` emitters, keyed by agent id.
+     *
+     * The jailer's `exit` handler and the spawn-deadline abort both fire outside
+     * `spawn`'s own scope, and the manager releases a spawn reservation only on a
+     * failure event. Holding the emitter here is what lets those two paths report
+     * the failure instead of leaving the reservation held until restart.
+     */
+    private readonly spawnFailureHandlers;
     private _labelSets;
     private readonly name;
     private readonly ipAllocator;
@@ -122,6 +145,8 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
     private readonly bridgeName;
     private readonly cidr;
     private readonly gateway;
+    /** Host services the orchestrator directed agents at, as `hostAccess` entries. */
+    private readonly hostServices?;
     private readonly netmask;
     private readonly table;
     private readonly tokenStore?;
@@ -145,7 +170,16 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
     static getRequiredTools(entry: ScalerEntry): ToolRequirement[];
     get labelSets(): LabelSetConfig[];
     getActiveCount(): number;
-    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext, _signal?: AbortSignal): Promise<ManagedAgent>;
+    spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext, signal?: AbortSignal): Promise<ManagedAgent>;
+    /**
+     * Tear down a spawn that will never produce a usable VM, from a path that is
+     * not `spawn`'s own `catch` — a deadline abort, or the jailer exiting during
+     * boot.
+     *
+     * Idempotent: whichever of the two fires first wins and the other is a no-op,
+     * because `cleanupFailedSpawn` removes the tracking entry.
+     */
+    private abandonSpawn;
     getScalerContext(agentId: string): Record<string, unknown> | undefined;
     /**
      * Bridge config for host-network diagnostics (read-only snapshot).
@@ -164,7 +198,41 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
      * No-op unless autoProvisionHost is set. Throws on a real provision failure;
      * ScalerManager.ensureHostsReady catches per-backend and degrades this scaler.
      */
+    /**
+     * nft options for this backend's own rules.
+     *
+     * `table` is the operator-configured one, not the literal `kici`: with two
+     * coordinators on one host, coordinator B's baseline goes to `kici_b` while
+     * its per-VM rules used to go to `kici` — coordinator A's table. A's next
+     * re-provision then swept B's live VMs' isolation along with its own, and B's
+     * rules were evaluated against A's subnet-scoped baseline.
+     */
+    private nftOpts;
     ensureHostReady(): Promise<void>;
+    /**
+     * Fail loudly on a kernel that will not honour bridge port isolation.
+     *
+     * Without the flag every concurrent tenant's VM can reach every other one at
+     * L2, and no nft rule sees that traffic — so the isolation the operator docs
+     * promise silently does not exist. The same fail-closed posture
+     * `validateNftablesAvailability` takes for a missing `nft`: better to refuse
+     * to run VMs than to run them unisolated.
+     *
+     * The probe is an end-to-end one — create a TAP, enslave it isolated, delete
+     * it — because the flag can be rejected by the kernel, by iproute2, or by the
+     * bridge itself, and only the real command exercises all three.
+     */
+    private assertBridgePortIsolationSupported;
+    /**
+     * Re-apply per-VM isolation rules for every VM this backend still tracks.
+     *
+     * Provisioning no longer deletes the table, but it does sweep the `forward`
+     * chain of everything no live VM owns, and an old-shaped host carries its
+     * baseline there too. Re-applying is what guarantees a self-heal cannot leave
+     * a running VM fail-open. Each identifier is removed before it is re-added,
+     * so a self-heal that fires twice does not duplicate the rules.
+     */
+    private reapplyIsolationForTrackedVms;
     destroy(managedId: string, _context?: ScalerDestroyContext): Promise<void>;
     /**
      * Force-reclaim a VM this backend no longer tracks in memory.
@@ -222,6 +290,18 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
      * Returns the count of cleaned orphans.
      */
     cleanupOrphans(): Promise<number>;
+    /**
+     * Delete isolation rules for every identifier no live VM owns.
+     *
+     * The live set is the VM ids the passes above established, mapped to the IPs
+     * this scaler allocated for them. An identifier the DB does not attribute to
+     * a live VM of this scaler is left alone: a second scaler on the same host
+     * writes to the same chain, and reaping its rules would cut its running jobs
+     * off the network — the mistake this whole sweep exists to stop making.
+     *
+     * @returns the number of rules deleted
+     */
+    private reapUnownedIsolationRules;
     /**
      * Default interval for periodic orphan sweeps (15 minutes).
      *
@@ -309,7 +389,60 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
      */
     getChrootDir(agentId: string): string;
     /**
-     * Check if a VM process is still alive by reading its PID file and sending signal 0.
+     * Probe the process a VM's PID file names.
+     *
+     * Checks, in order:
+     *  1. the file parses to a plausible PID;
+     *  2. `/proc/<pid>/stat` exists at all — the file is world-readable
+     *     regardless of the jailer's `--uid` drop, so an unreadable one means no
+     *     such process;
+     *  3. field 22 (`starttime`) is not later than the PID file's mtime — a
+     *     recycled number belongs to a process that started after the file
+     *     naming it was written;
+     *  4. field 2 (`comm`) is `firecracker` (11 characters, so it is not
+     *     truncated by the 15-character `comm` limit) and `/proc/<pid>/root` —
+     *     where the jailer pivot_roots — is this VM's chroot when the link is
+     *     readable at all. Those carry `identityConfirmed`, not liveness.
+     */
+    private probeVmPid;
+    /**
+     * Read a VM's PID file and return the PID only when the process it names is
+     * provably still *this* VM's firecracker process.
+     *
+     * This is the guard for acting ON the process — `destroy` and `reapUnowned`
+     * SIGKILL through it — so it fails closed: anything short of a confirmed
+     * identity yields `undefined` and nothing is signalled.
+     *
+     * @returns the PID when it is provably this VM's, otherwise `undefined`
+     */
+    private readVmPid;
+    /**
+     * Whether a chroot's PID file names a process that is running right now.
+     *
+     * This is the guard for the orphan reaper, which asks the opposite question
+     * from {@link readVmPid} and therefore fails closed in the opposite
+     * direction. `readVmPid` refuses to signal a PID it cannot claim; the reaper
+     * must refuse to DELETE THE ROOTFS under one. A process whose `comm` or
+     * `/proc/<pid>/root` does not match is still a process, and deleting the
+     * files it is running out of corrupts whatever it is — while sparing it costs
+     * only the disk one chroot occupies.
+     *
+     * The leak the identity checks exist to prevent is unaffected: a recycled
+     * number is detected by start time, which this guard honours, so a chroot
+     * whose VM is genuinely over is still reclaimed.
+     */
+    private isChrootPidRunning;
+    /**
+     * Whether *this* VM's own firecracker process is still up.
+     *
+     * Liveness is `readVmPid` plus signal 0, so it answers the identity-bearing
+     * question — "is the VM I started still running?" — that
+     * {@link waitForProcessExit} needs. Without the identity checks a recycled
+     * PID answers signal 0 successfully and the VM is spared forever.
+     *
+     * The orphan reaper asks a different question and uses
+     * {@link isChrootPidRunning} instead: a PID it cannot claim is still a live
+     * process, and its rootfs must not be deleted underneath it.
      */
     private isVmProcessAlive;
     /**

package/dist/scaler/host-access.d.ts

@@ -0,0 +1,59 @@
+/**
+ * What a scaled agent may reach on the orchestrator's own host.
+ *
+ * An agent container or Firecracker VM sends its packets to one of the host's
+ * addresses — a bridge gateway, the host's LAN address — and those are
+ * delivered on the netfilter input hook. The `forward` rules the backends
+ * already write never see them, so until the `input` chain existed a sandbox
+ * reached every port on the host. These helpers resolve the narrow set it
+ * actually needs, expressed in the same `hostAccess` vocabulary an operator
+ * writes in `scalers.yaml`.
+ */
+import type { NetworkPolicy } from '@kici-dev/shared/net';
+import type { AppConfig } from '../config.js';
+/**
+ * Host-access entries for the agent-facing storage endpoint, when it is a
+ * service on this host rather than a remote one.
+ *
+ * A scaled agent connects to storage directly — that is why
+ * `assertAgentReachableStorage` refuses to start on a loopback endpoint — so an
+ * orchestrator-port-only default would cut off every deployment running its own
+ * object store beside the orchestrator.
+ *
+ * Two shapes are recognised, and nothing else:
+ *
+ *  - **An IP literal.** The entry is scoped to that exact address and port, the
+ *    narrowest rule that can work.
+ *  - **A name with an explicit port.** No address rule can be built for a name
+ *    that may move, so the entry names the port on any host address. It is
+ *    still one port rather than the whole host.
+ *
+ * A name with no port is a public endpoint (`https://…`), which the agent
+ * reaches through the host over the `forward` hook and which no input rule
+ * governs. Emitting `*:443` for it would open a host port for nothing.
+ */
+export declare function storageHostAccessEntries(config: AppConfig): string[];
+/**
+ * Resolve the `hostAccess` policy for one agent sandbox.
+ *
+ * An explicit `hostAccess` on the label set wins outright: host reachability is
+ * operator-configurable policy, and an operator who names it has said exactly
+ * what this class of agent may reach.
+ *
+ * The default is the set an agent cannot work without:
+ *
+ *  - **DNS on the bridge gateway.** Rootful podman resolves there, so a sandbox
+ *    with no such rule cannot resolve the orchestrator's own hostname. Docker
+ *    resolves inside the container's netns and is unaffected either way.
+ *  - **The orchestrator's port**, on any host address, since the address the
+ *    agent was told to dial is not knowable from here.
+ *  - **Whatever host services the orchestrator itself directed the agent at**,
+ *    which today means its object storage.
+ */
+export declare function resolveAgentHostAccess(input: {
+    policy: NetworkPolicy | undefined;
+    orchestratorUrl: string;
+    gateway: string;
+    hostServices?: string[];
+}): string[];
+//# sourceMappingURL=host-access.d.ts.map

package/dist/scaler/index.d.ts

@@ -16,6 +16,7 @@ export { FirecrackerScalerBackend } from './firecracker-backend.js';
 export type { FirecrackerScalerBackendOptions, FirecrackerManagedAgent, } from './firecracker-backend.js';
 export { createScalerBackend, requiredToolsFor } from './backend-factory.js';
 export type { BackendFactoryContext, IpAllocatorParams } from './backend-factory.js';
+export { resolveAgentHostAccess, storageHostAccessEntries } from './host-access.js';
 export { EventScalerBackend } from './event-backend.js';
 export type { EventScalerBackendOptions, ScalerEventEmitterLike } from './event-backend.js';
 export { ClaimStore, DEFAULT_CLAIM_TTL_SECONDS } from './claim-store.js';

package/dist/scaler/manager.d.ts

@@ -198,6 +198,12 @@ export interface ScalerManagerDeps {
             activeJobs: number;
             registeredAt: number;
         }>;
+        /** Whether one agent id is currently registered on this host. */
+        isRegistered(agentId: string): boolean;
+        /** Condemn an agent before tearing it down; returns its active-job count. */
+        markDraining(agentId: string): number | undefined;
+        /** Undo `markDraining` when the teardown did not happen. */
+        clearDraining(agentId: string): void;
     };
     /**
      * Id of the orchestrator instance this manager runs on. Rows this
@@ -778,6 +784,33 @@ export declare class ScalerManager {
      * which is always `releaseAll(agentId)`.
      */
     private reserveForSpawn;
+    /**
+     * How many reservations one scaler currently holds.
+     *
+     * Derived from `this.reservations` rather than kept as a counter beside it:
+     * the map IS the admitted set, so a derived count cannot drift out of step
+     * with it — including on the boot/leader-switch rehydrate path, which
+     * repopulates the map directly.
+     */
+    /**
+     * Record a refusal taken on one of the two agent-count caps.
+     *
+     * The refusal itself is returned to the dispatcher as `at-capacity`, which
+     * leaves the job queued and logs nothing — correct while agents are genuinely
+     * busy, and indistinguishable from a scaler that has stopped spawning
+     * altogether. So the counter the metric's own help text promises
+     * (`maxAgents`, `globalMaxAgents`) is incremented here, beside the resource
+     * caps that already increment it.
+     *
+     * The warning is narrower than the counter on purpose. Ordinary saturation
+     * refuses with live agents holding the cap, and warning on that would emit a
+     * line per queued job for as long as the cluster is busy. A refusal where the
+     * reservation ledger outruns every backend's active count is the other shape:
+     * capacity billed to agents that are not running, which does not resolve on
+     * its own.
+     */
+    private reportCountCapRefusal;
+    private reservedCountForScaler;
     /**
      * Run a critical section under the reservation lock. Serializes concurrent
      * `requestScale()` calls so the check+reserve sequence is atomic.
@@ -1248,10 +1281,25 @@ export declare class ScalerManager {
      */
     getScalerContextForAgent(agentId: string): Record<string, unknown> | undefined;
     /**
-     * Start the warm pool idle check interval, the machine-pool ledger reaper,
-     * and the retirement sweep that tears down drained retiring backends.
+     * Start the machine-pool ledger reaper and the retirement sweep that tears
+     * down drained retiring backends, and publish the warm-pool gauges so each
+     * pool's target is visible before its first fill.
+     *
+     * The warm pools themselves are NOT started here — see
+     * {@link startWarmPools}, which the host calls once every dependency a fill
+     * can reach is wired.
      */
     start(): void;
+    /**
+     * Run the warm pools' first deficit pass and arm their periodic tick.
+     *
+     * Separate from {@link start} because the first pass spawns immediately,
+     * and on an event backend a spawn emits through the event emitter — which
+     * the coordinator assigns after the scaler is built. The coordinator calls
+     * this once the emitter exists; the worker, which hosts no event backend,
+     * calls it right after `start()`.
+     */
+    startWarmPools(): void;
     /**
      * Provision/heal every backend's host prerequisites before spawning starts.
      * Awaits each backend's optional ensureHostReady, catching per-backend so one
@@ -1352,6 +1400,7 @@ export declare class ScalerManager {
      * coord boot or Raft leader switch. Reconstructs:
      *
      *   - `spawningAgents` (with `boundJobId` preserved for eager-dispatch on register)
+     *     — event-backend rows only; a local-backend row is dropped, see below
      *   - `agentJobCorrelation` (so scaler-lifecycle events route correctly)
      *   - `reservations` + `perScalerUsage` (so the cap-check critical
      *     section reflects the cluster-wide truth, not the local empty

package/dist/scaler/scaler-state-store.d.ts

@@ -374,8 +374,21 @@ export declare class ScalerStateStore {
  */
 export interface ScalerStateRecovery {
     spawningAgentsRehydrated: number;
+    /**
+     * Spawn rows released during recovery because their backend is local
+     * (container / firecracker / bare-metal): such a spawn cannot outlive the
+     * process that started it, so nothing could ever register against it, and
+     * rehydrating it would hold its reservation until the stale prune.
+     */
+    spawningDropped: number;
     agentJobsRehydrated: number;
     reservationsRehydrated: number;
+    /**
+     * Reservation rows discarded during recovery because no rehydrated spawn
+     * backs them. Counted separately from `reservationsRehydrated` so a
+     * deployment carrying orphaned rows says so on every boot until it is clean.
+     */
+    reservationsDropped: number;
     bufferedEventsLost: number;
 }
 /**