@kici-dev/orchestrator 0.11.0 → 0.13.0

package/dist/routes/health.d.ts

@@ -1,5 +1,14 @@
+import { type BuildFingerprint } from '@kici-dev/shared';
 import type { Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
+/**
+ * The fields the orchestrator adds to its `/health` body. The route below
+ * returns this type, and `kici-admin orchestrator status` renders it.
+ */
+export interface OrchestratorLivenessInfo extends BuildFingerprint {
+    /** ISO-8601 time the orchestrator bundle was built. */
+    buildDate: string;
+}
 export interface HealthRoutesDeps {
     /** Optional DB instance for readiness checks */
     db?: Kysely<Database>;

package/dist/scaler/container-backend.d.ts

@@ -41,7 +41,7 @@ export interface ContainerScalerBackendOptions {
     runtime?: 'docker' | 'podman' | 'auto';
     /** Default resource limits applied when label set has none */
     defaultResources?: ResourceRequest;
-    /** Extra host:IP mappings injected into spawned containers (e.g. ["verdaccio.local:host-gateway"]) */
+    /** Extra host:IP mappings injected into spawned containers (e.g. ["registry.local:host-gateway"]) */
     extraHosts?: string[];
     /** Disable nftables-based network isolation (default: true). Set to false when nft is unavailable (e.g. rootless containers without NET_ADMIN). */
     networkIsolation?: boolean;
@@ -211,6 +211,8 @@ export declare class ContainerScalerBackend implements ScalerBackend {
      * Returns the count of cleaned containers plus reaped rules.
      */
     cleanupOrphans(): Promise<number>;
+    /** Stop and remove one orphaned agent container; true when it was removed. */
+    private removeOrphan;
     /**
      * Remove containers this backend spawned for one agent id.
      *

package/dist/scaler/container-runtime.d.ts

@@ -0,0 +1,9 @@
+/**
+ * The container runtime behind a configured socket path.
+ *
+ * A path containing `podman` is Podman; any other path is Docker. The container
+ * backend reads a configured `socketPath` this way, and the runtime it picks
+ * only labels the agents' log source and the scaler diagnostics.
+ */
+export declare function containerRuntimeForSocketPath(socketPath: string): 'docker' | 'podman';
+//# sourceMappingURL=container-runtime.d.ts.map

package/dist/scaler/firecracker-backend.d.ts

@@ -21,13 +21,6 @@ import type { IpAllocator, IpAllocationResult } from './ip-allocator.js';
 import type { AgentTokenStore } from '../agent/token-store.js';
 import { type FirecrackerBridgeConfig, type BridgeHealth, type ExecOptions } from '../firecracker/host-network.js';
 import type { ScalerBackend, ScalerDestroyContext, ScalerEntry, ManagedAgent, LabelSetConfig, ScalerEventCallback, ValidationResult, EffectiveLimits, SpawnContext } from './types.js';
-/**
- * Infrastructure interfaces that share the `kici-` prefix but must NEVER be
- * deleted by the orphan sweep. Bridges (kici-br0/kici-br1) carry all VM
- * traffic; kici-m01 is the staging metadata interface. Losing any of these
- * kills all running VMs.
- */
-export declare const PROTECTED_INTERFACES: readonly string[];
 /** Extended ManagedAgent that includes the allocated IP for cleanup */
 export interface FirecrackerManagedAgent extends ManagedAgent {
     /** Allocated IP address for TAP/IP release during destroy */
@@ -80,6 +73,11 @@ export interface FirecrackerScalerBackendOptions {
     cidr?: string;
     /** Gateway IP for guest networking */
     gateway: string;
+    /**
+     * Extra `host:address` mappings the rootfs `/init` appends to each guest's
+     * `/etc/hosts`; `host-gateway` resolves to {@link gateway}. None by default.
+     */
+    extraHosts?: string[];
     /** Netmask for guest networking */
     netmask: string;
     /** nft table name for host-network diagnostics. @default 'kici' */
@@ -145,6 +143,8 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
     private readonly bridgeName;
     private readonly cidr;
     private readonly gateway;
+    /** The MMDS `kici-extra-hosts` value, or undefined when the scaler maps no host. */
+    private readonly guestExtraHosts;
     /** Host services the orchestrator directed agents at, as `hostAccess` entries. */
     private readonly hostServices?;
     private readonly netmask;
@@ -180,6 +180,11 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
      * because `cleanupFailedSpawn` removes the tracking entry.
      */
     private abandonSpawn;
+    /**
+     * {@link abandonSpawn} from a path that cannot await it (an abort listener, a
+     * process exit handler). A failure is logged.
+     */
+    private abandonSpawnDetached;
     getScalerContext(agentId: string): Record<string, unknown> | undefined;
     /**
      * Bridge config for host-network diagnostics (read-only snapshot).
@@ -305,9 +310,9 @@ export declare class FirecrackerScalerBackend implements ScalerBackend {
     /**
      * Default interval for periodic orphan sweeps (15 minutes).
      *
-     * Long-running orchestrators need in-process orphan sweeps because the
-     * external kici-leak-sweep timer skips TAP cleanup while a staging
-     * orchestrator is active (to avoid racing spawns). Without this, leaked
+     * Long-running orchestrators need in-process orphan sweeps because a
+     * host-level cleanup job skips TAP cleanup while an orchestrator is active
+     * (to avoid racing spawns). Without this, leaked
      * TAPs accumulate until the orchestrator restarts — which can be weeks.
      */
     private static readonly ORPHAN_SWEEP_INTERVAL_MS;

package/dist/scaler/firecracker-extra-hosts.d.ts

@@ -0,0 +1,20 @@
+/** The address keyword for the bridge gateway, as the container runtimes spell it. */
+export declare const HOST_GATEWAY = "host-gateway";
+export interface ExtraHost {
+    host: string;
+    /** An IPv4 or IPv6 address, or {@link HOST_GATEWAY}. */
+    address: string;
+}
+/**
+ * Parse one `host:address` entry. The host ends at the first colon, so an
+ * IPv6 address keeps its own colons. Throws on a malformed entry.
+ */
+export declare function parseExtraHost(entry: string): ExtraHost;
+/**
+ * The MMDS value the rootfs `/init` reads: `host:address` pairs joined by
+ * commas, with `host-gateway` replaced by `gateway`. `undefined` when the
+ * scaler configures no mapping, so the guest gets none. Throws on a malformed
+ * entry, and on `host-gateway` when `gateway` is not an IP address.
+ */
+export declare function renderGuestExtraHosts(entries: readonly string[] | undefined, gateway: string): string | undefined;
+//# sourceMappingURL=firecracker-extra-hosts.d.ts.map

package/dist/scaler/machine-ledger.d.ts

@@ -13,10 +13,9 @@
  * `mkdir` succeeds for exactly one caller when the directory does not exist,
  * so it works as a portable `flock` substitute without any native dependency.
  *
- * The plan considered `proper-lockfile` and `fs-ext` first; neither is a
- * transitive dep today (`pnpm why proper-lockfile` and `pnpm why fs-ext`
- * both empty), so we use the documented `mkdir` fallback. This avoids
- * adding a native binding for the sake of a single locking primitive.
+ * Neither `proper-lockfile` nor `fs-ext` is a dependency, so this uses the
+ * `mkdir` lock instead. This avoids adding a native binding for the sake of
+ * a single locking primitive.
  *
  * ## Crash safety
  *

package/dist/scaler/manager.d.ts

@@ -11,6 +11,21 @@ import { ScaleDownReason } from './scaler-events.js';
 import { type ScalerAgentView } from './agent-fit.js';
 import type { ScalerEventEmitterLike } from './event-backend.js';
 import type { ClaimStore, ClaimedCredentials } from './claim-store.js';
+/**
+ * How long a graceful shutdown waits for the scaler backends to stop their
+ * agents. It is the first window inside the coordinator's 30 s shutdown limit,
+ * and every later step (the HTTP stop grace, the database drain) still has to
+ * fit after it.
+ *
+ * A container stop normally takes at most its own 10 s stop timeout, but the
+ * container runtime can take far longer: a Docker daemon that misses a
+ * container's exit waits out the stop timeout and then another 10 s for its
+ * kill, and one such container held the whole step for 29.7 s. Past this
+ * window the orchestrator logs each backend still stopping and continues. The
+ * runtime finishes the stop on its own, and the next start's orphan cleanup
+ * removes any agent container that is still there.
+ */
+export declare const SCALER_SHUTDOWN_GRACE_MS = 15000;
 import type { BackendFailureSummary } from './failure-tracker.js';
 import type { ScalerBackend, ScalerConfig, ScalerEntry, ScaleResult, ScalerEvent, ResourceCap, ValidationResult, ResolvedContainerSpawn } from './types.js';
 import type { ScalerStateStore, ScalerStateRecovery, ReapCandidate } from './scaler-state-store.js';
@@ -948,9 +963,12 @@ export declare class ScalerManager {
      */
     private completeLocalRegistration;
     /**
-     * Called from agent-handler.ts when an agent disconnects.
+     * Called from agent-handler.ts when an agent disconnects. `reason` is what an
+     * event scaler's scale-down carries: `shutdown` for an ordinary disconnect,
+     * `heartbeat-timeout` for an agent the orchestrator dropped without waiting
+     * for its socket.
      */
-    onAgentDisconnected(agentId: string): void;
+    onAgentDisconnected(agentId: string, reason?: ScaleDownReason): void;
     /**
      * Tear down a stranded provision the `EventProvisionReaper` found.
      *
@@ -1145,6 +1163,12 @@ export declare class ScalerManager {
      * Sum of all backends' active count + spawning agents.
      */
     getGlobalActiveCount(): number;
+    /**
+     * Shut every backend down in parallel, waiting at most
+     * {@link SCALER_SHUTDOWN_GRACE_MS} for the slowest. A backend still stopping
+     * when the window closes is logged by name and left to finish on its own.
+     */
+    private shutdownBackendsWithinGrace;
     /**
      * Stop warm pool, shutdown all backends, clear tracking maps.
      */

package/dist/scaler/scaler-state-store.d.ts

@@ -346,7 +346,7 @@ export declare class ScalerStateStore {
      * the prune's verdict does not depend on it. It is written because the row
      * is the only surviving account of what became of a provision once the spawn
      * row is gone — which is what an operator investigating a torn-down
-     * provision, and the E2E that pins this behaviour, read it for.
+     * provision reads it for.
      */
     recordProvisionCondemned(agentId: string, scalerName: string, reason: string): Promise<void>;
     /**

package/dist/scaler/types.d.ts

@@ -476,7 +476,10 @@ export interface ScalerEntry {
     runtime?: 'docker' | 'podman' | 'auto';
     /** Orchestrator URL for spawned agents to connect back to */
     orchestratorUrl?: string;
-    /** Extra host:IP mappings injected into spawned containers (e.g. ["verdaccio.local:host-gateway"]) */
+    /**
+     * Extra host:IP mappings for spawned agents (e.g. ["registry.local:host-gateway"]): the
+     * container runtime's ExtraHosts, or each Firecracker guest's /etc/hosts.
+     */
     extraHosts?: string[];
     /** Disable nftables-based network isolation for container backend (default: true). Set to false when nft is unavailable. */
     networkIsolation?: boolean;

package/dist/secrets/audit-logger.d.ts

@@ -4,7 +4,7 @@
  * Records all secret access/denial events to the secret_audit_log table
  * for compliance and security monitoring.
  *
- * Phase D: query() supports cold-store read-through via the optional
+ * query() supports cold-store read-through via the optional
  * `coldStore` constructor argument and an `includeArchived` query
  * filter. Hot-only behavior is unchanged when neither is supplied.
  */
@@ -54,7 +54,7 @@ export declare class AuditLogger {
         limit?: number;
         offset?: number;
         /**
-         * Phase D opt-in. When true, falls through to cold-store via
+         * Opt-in. When true, falls through to cold-store via
          * `loadSecretAuditLogRange` for any portion of the requested
          * window past the warm cutoff. Default false preserves the
          * original semantics.

package/dist/security/reduced-privilege-note.d.ts

@@ -56,9 +56,8 @@ export declare const REDUCED_PRIVILEGE_MARKER = "**Reduced privileges (untrusted
  *   event, and an untrusted tier also reaches an internal-event child run that
  *   inherited it.
  *
- * Kept in step with the dashboard's reduced-privilege banner
- * (`packages/dashboard/src/components/run-detail/degraded-run-banner.tsx`),
- * which describes the same run and may be read beside these checks.
+ * Kept in step with the dashboard's reduced-privilege banner, which
+ * describes the same run and may be read beside these checks.
  */
 export declare function buildReducedPrivilegeNote(tier: string | null | undefined, lockFileSource?: string | null): string | null;
 //# sourceMappingURL=reduced-privilege-note.d.ts.map

package/dist/security/trust-policy-store.d.ts

@@ -67,9 +67,9 @@ export declare class TrustPolicyStore {
 }
 /**
  * Build a TrustPolicyStore backed by its own connection pool to the given
- * orchestrator database URL. Mirrors `createHeldRunStoreFromUrl`; consumed by
- * E2E tests that assert the pushed policy is persisted in the real deployed
- * orchestrator DB rather than held in process memory.
+ * orchestrator database URL. Mirrors `createHeldRunStoreFromUrl`; exported for
+ * tests that assert the pushed policy is persisted in a real orchestrator DB
+ * rather than held in process memory.
  */
 export declare function createTrustPolicyStoreFromUrl(databaseUrl: string, opts?: {
     maxConnections?: number;

package/dist/server.d.ts

@@ -7,12 +7,13 @@
  *
  * Both modes connect a PlatformClient to the Platform relay and run an agent WS server.
  *
- * Startup sequence follows packages/platform/src/server.ts pattern:
+ * Startup sequence (the same one the Platform follows):
  * config -> DB -> migrations -> provider registry -> dispatcher -> PlatformClient -> app -> HTTP -> heartbeat
  *
  * Graceful shutdown in reverse order:
  * Platform client -> agent WS -> heartbeat -> HTTP -> DB
  */
+import '@kici-dev/shared/load-service-env-file';
 import type { OrchestratorFaultInjection } from './fault-injection-types.js';
 /**
  * Boot the Platform-connected orchestrator. Factored out of the module entry so