@kici-dev/orchestrator 0.16.0 → 0.18.0

package/dist/scaler/config.d.ts

@@ -1,33 +1,23 @@
 /**
- * YAML configuration loading and Zod validation for the scaler module.
- *
- * Handles:
- * - Main YAML config file parsing
- * - scalers.d/ directory merging (alphabetically sorted)
- * - Zod schema validation with type-specific refinements
- * - Memory string parsing (e.g., "2g" -> bytes)
+ * Scaler config loading: the main YAML file plus `scalers.d/` (merged in
+ * alphabetical order), validated by Zod with type-specific refinements.
  */
 import { z } from 'zod';
 import type { ScalerConfig } from './types.js';
 /**
- * Default cap on concurrent `backend.spawn` operations per backend when a
- * scaler entry does not set `maxConcurrentSpawns`. Bounds the provisioning rate
- * (image pull + create + start) so a burst of queued jobs cannot storm the
- * container socket / registry.
+ * Default per-backend `maxConcurrentSpawns`. Bounds the provisioning rate (image
+ * pull + create + start) so a burst of queued jobs cannot storm the container
+ * socket / registry.
  */
 export declare const DEFAULT_MAX_CONCURRENT_SPAWNS = 8;
 /**
- * Zod schema for network policy configuration on a label set.
- * Controls RFC1918 and internet access for spawned agents.
+ * A label set's RFC1918 and internet access for spawned agents.
  */
 export declare const networkPolicySchema: z.ZodOptional<z.ZodObject<{
     allowlist: z.ZodOptional<z.ZodArray<z.ZodString>>;
     denyAll: z.ZodDefault<z.ZodBoolean>;
     hostAccess: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strip>>;
-/**
- * Zod schema for a label-set configuration entry.
- */
 export declare const labelSetConfigSchema: z.ZodObject<{
     labels: z.ZodPipe<z.ZodArray<z.ZodString>, z.ZodTransform<import("@kici-dev/engine").CanonicalLabel[], string[]>>;
     image: z.ZodOptional<z.ZodString>;
@@ -66,7 +56,6 @@ export declare const labelSetConfigSchema: z.ZodObject<{
     overlayDriveSizeMib: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strict>;
 /**
- * Zod schema for Firecracker network configuration.
  * Global across all Firecracker scalers on the orchestrator.
  */
 export declare const firecrackerNetworkSchema: z.ZodObject<{
@@ -77,10 +66,6 @@ export declare const firecrackerNetworkSchema: z.ZodObject<{
     table: z.ZodDefault<z.ZodString>;
     autoProvisionHost: z.ZodDefault<z.ZodBoolean>;
 }, z.core.$strip>;
-/**
- * Zod schema for the complete scaler configuration file.
- * Validates version, global limits, defaults, and scaler entries.
- */
 export declare const scalerFileSchema: z.ZodObject<{
     version: z.ZodLiteral<1>;
     globalMaxAgents: z.ZodDefault<z.ZodNumber>;
@@ -233,28 +218,16 @@ export declare const scalerFileSchema: z.ZodObject<{
     }, z.core.$strip>>;
 }, z.core.$strip>;
 /**
- * Parse a memory string with unit suffix to bytes.
- *
- * Supported formats:
- * - "512m" or "512M" -> 536870912 (512 * 1024^2)
- * - "2g" or "2G" -> 2147483648 (2 * 1024^3)
- * - "1024k" or "1024K" -> 1048576 (1024 * 1024)
+ * Parse a memory string to bytes, with binary units: "512m" -> 512 * 1024^2,
+ * "2G" -> 2 * 1024^3, "1024k" -> 1024 * 1024 (suffix case-insensitive).
  *
  * @throws Error if the format is invalid
  */
 export declare function parseMemoryString(memory: string): number;
 /**
- * Load and validate the scaler configuration from YAML files.
- *
- * 1. Reads the main config file
- * 2. If scalersDirPath provided, reads all *.yaml and *.yml files
- *    from the directory (sorted alphabetically) and merges their
- *    scalers arrays into the main config
- * 3. Validates the merged config through Zod schema
+ * Load the main YAML config, append the `scalers` arrays of every `*.yaml` /
+ * `*.yml` in `scalersDirPath` (alphabetical order), and validate the result.
  *
- * @param configPath - Path to the main scaler YAML config file
- * @param scalersDirPath - Optional path to scalers.d/ directory for additional scaler definitions
- * @returns Validated ScalerConfig
  * @throws ZodError if validation fails
  */
 export declare function loadScalerConfig(configPath: string, scalersDirPath?: string): Promise<ScalerConfig>;

package/dist/scaler/container-backend.d.ts

@@ -1,10 +1,7 @@
 /**
- * Container scaler backend implementation.
- *
- * Manages ephemeral agent containers using Docker or Podman via dockerode.
- * Supports auto-detection of container runtime socket.
- * Handles full container lifecycle: pull, create, start, stop, remove.
- * Supports container socket sharing, resource limits, and orphan cleanup.
+ * Container scaler backend: ephemeral agent containers on Docker or Podman via
+ * dockerode, with runtime-socket auto-detection, socket sharing, resource
+ * limits and orphan cleanup.
  */
 import Docker from 'dockerode';
 import { type ToolRequirement } from '@kici-dev/shared';
@@ -14,9 +11,6 @@ import type { ScalerBackend, ScalerDestroyContext, ManagedAgent, LabelSetConfig,
 export declare const ISOLATED_NETWORK_NAME = "kici-agent-net";
 /** Gateway IP for the isolated agent network (host-side). */
 export declare const ISOLATED_NETWORK_GATEWAY = "172.30.0.1";
-/**
- * Result of runtime detection.
- */
 export interface DetectedRuntime {
     socketPath: string;
     runtime: 'docker' | 'podman';
@@ -41,9 +35,12 @@ export interface ContainerScalerBackendOptions {
     runtime?: 'docker' | 'podman' | 'auto';
     /** Default resource limits applied when label set has none */
     defaultResources?: ResourceRequest;
-    /** Extra host:IP mappings injected into spawned containers (e.g. ["registry.local:host-gateway"]) */
+    /** Extra host:IP mappings for spawned containers (e.g. ["registry.local:host-gateway"]). */
     extraHosts?: string[];
-    /** Disable nftables-based network isolation (default: true). Set to false when nft is unavailable (e.g. rootless containers without NET_ADMIN). */
+    /**
+     * Enables nftables-based network isolation (default: true). Set to false when nft is
+     * unavailable (e.g. rootless containers without NET_ADMIN).
+     */
     networkIsolation?: boolean;
     /**
      * Extra `hostAccess` entries the orchestrator itself directed agents at —
@@ -51,7 +48,7 @@ export interface ContainerScalerBackendOptions {
      * narrowing does not cut off a host-local object store.
      */
     hostServices?: string[];
-    /** Token store for creating ephemeral agent auth tokens. Optional -- when undefined, no token is injected. */
+    /** Token store for ephemeral agent auth tokens; when undefined, no token is injected. */
     tokenStore?: AgentTokenStore;
     /** TTL for ephemeral agent tokens in ms. Default: 1 hour. */
     tokenTtlMs?: number;
@@ -138,13 +135,10 @@ export declare class ContainerScalerBackend implements ScalerBackend {
     /**
      * Declare required tools for a container scaler entry.
      *
-     * For the auto-detect case (no explicit socketPath / remote host) the
-     * orchestrator must have a local container runtime — docker OR podman — on
-     * PATH, otherwise the scaler cannot spawn agent containers. Declaring it
-     * here lets the startup tool-validation gate fail fast with a clear error
-     * instead of the first job hanging. When a socketPath or remote host is
-     * configured the binary need not be on PATH (the runtime may be remote), so
-     * reachability is validated later in create().
+     * Auto-detect (no socketPath / remote host) needs docker OR podman on PATH;
+     * declaring it lets the startup tool-validation gate fail fast instead of the
+     * first job hanging. With a socketPath or remote host the runtime may be
+     * remote, so create() validates reachability instead.
      */
     static getRequiredTools(entry: ScalerEntry): ToolRequirement[];
     /**
@@ -179,9 +173,7 @@ export declare class ContainerScalerBackend implements ScalerBackend {
     spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext, signal?: AbortSignal): Promise<ManagedAgent>;
     getScalerContext(agentId: string): Record<string, unknown> | undefined;
     destroy(managedId: string, _context?: ScalerDestroyContext): Promise<void>;
-    /**
-     * Get the LogCapture for a managed agent (used by ScalerManager for log forwarding).
-     */
+    /** Used by ScalerManager for log forwarding. */
     getLogCapture(managedId: string): LogCapture | undefined;
     shutdownAll(): Promise<void>;
     reload(labelSets: LabelSetConfig[], opts?: {
@@ -191,14 +183,11 @@ export declare class ContainerScalerBackend implements ScalerBackend {
     /**
      * Clean up this scaler's orphaned containers and stale isolation rules.
      *
-     * Every filter below exists because the sweep used to list every
-     * `kici-managed=true` container on the host and force-remove it, running at
-     * orchestrator boot, at worker-peer start, on a live config reload that adds
-     * a scaler, and from `kici-admin scaler reap-orphans`. Two orchestrators
-     * sharing one docker host therefore killed each other's running agents on
-     * every boot, and adding a second container scaler removed the first one's
-     * agents mid-job — every in-flight job failing as "agent disconnected" on a
-     * healthy system.
+     * Runs at orchestrator boot, at worker-peer start, on a live config reload
+     * that adds a scaler, and from `kici-admin scaler reap-orphans`. Without the
+     * filters below, two orchestrators sharing one docker host kill each other's
+     * running agents on every boot, and adding a second container scaler removes
+     * the first one's agents mid-job ("agent disconnected" on a healthy system).
      *
      * A container is removed only when all of these hold:
      *  - it carries this scaler's own `kici-scaler-name`, so a backend can only
@@ -216,11 +205,9 @@ export declare class ContainerScalerBackend implements ScalerBackend {
     /**
      * Remove containers this backend spawned for one agent id.
      *
-     * Implements the optional `ScalerBackend.reapUnowned` hook. It is what
-     * reclaims a container whose registration the orchestrator refused — the
-     * case `cleanupOrphans` used to cover by removing every `kici-managed`
-     * container on the host, which is exactly the breadth that killed other
-     * orchestrators' live agents.
+     * Implements the optional `ScalerBackend.reapUnowned` hook: it reclaims a
+     * container whose registration the orchestrator refused, so `cleanupOrphans`
+     * need not reap host-wide and kill other orchestrators' live agents.
      *
      * @returns true when a container was found and removed
      */

package/dist/scaler/container-hostconfig.d.ts

@@ -1,12 +1,9 @@
 /**
  * The `HostConfig` every KiCI agent container is created with.
  *
- * Two backends spawn an agent container — `ContainerScalerBackend` and the
- * bare-metal backend's container mode — and they had diverged into two
- * postures for one threat: the first applied resource limits, the second
- * applied none and dropped the limits the operator configured on the floor.
- * One builder is what keeps them from diverging again the next time either is
- * touched.
+ * Shared by both agent-container spawners — `ContainerScalerBackend` and the
+ * bare-metal backend's container mode — so they cannot diverge into two
+ * postures (one silently dropping the operator's resource limits).
  */
 import type Docker from 'dockerode';
 import type { EffectiveLimits } from './types.js';
@@ -25,18 +22,15 @@ export declare const AGENT_CONTAINER_PIDS_LIMIT = 4096;
  * Both spawn paths set `KICI_EXECUTION_MODE=bare-metal`, and bare-metal
  * container mode also sets `KICI_JOB_IMAGE_AGENT=1` — so unless the job
  * declares its own image, the workflow's steps run as host processes INSIDE
- * this container. It is a build execution environment, not a supervisor.
- * `CapDrop: ['ALL']` here would therefore land on customer build steps:
+ * this container. `CapDrop: ['ALL']` here would land on customer build steps:
  * `apt-get` and `dpkg` need CHOWN, DAC_OVERRIDE, FOWNER, SETUID and SETGID,
- * and `no-new-privileges` breaks `sudo` and every setuid binary. There is no
- * opt-out, so the break would be silent and total for `type: container`
- * scalers.
+ * and `no-new-privileges` breaks `sudo` and every setuid binary. With no
+ * opt-out, the break would be silent and total for `type: container` scalers.
  *
- * The hardening the operator docs promise already exists, one boundary in:
- * the agent applies `CapDrop: ['ALL']`, `no-new-privileges` and a tighter
- * `PidsLimit` to each nested JOB container
- * (`packages/agent/src/execution/sandbox/container-hardening.ts`). That is the
- * boundary that confines customer code; this one hosts it.
+ * The promised hardening lives one boundary in: the agent applies
+ * `CapDrop: ['ALL']`, `no-new-privileges` and a tighter `PidsLimit` to each
+ * nested JOB container (`packages/agent/src/execution/sandbox/container-hardening.ts`).
+ * That boundary confines customer code; this one hosts it.
  *
  * `PidsLimit` is the exception and stays: a fork-bomb ceiling costs a build
  * nothing at 4096.

package/dist/scaler/container-routing.d.ts

@@ -1,20 +1,17 @@
 /**
  * Label for hosts that can run containers.
  *
- * The agent reports it itself at registration when it finds a Docker socket,
- * alongside `kici:runtime:podman` for a Podman one and the `kici:os:*` /
- * `kici:arch:*` facts: whether a machine has a container runtime is that
- * machine's fact, not the orchestrator's. Probing the orchestrator's own
- * filesystem answers a different question — it was tried, and it stranded
- * container jobs that had been running fine, because the probe and the job ran
- * in different places.
+ * The agent reports it at registration when it finds a Docker socket (and
+ * `kici:runtime:podman` for a Podman one): a container runtime is the agent
+ * machine's fact. Probing the orchestrator's own filesystem was tried and
+ * stranded working container jobs, because the probe and the job ran in
+ * different places.
  *
- * Routing gates on the reported label (`canAgentRunJob`): a `container:` job
- * reaches an agent only when that agent was started for it, or reports
- * `kici:runtime:docker` or `kici:runtime:podman`. For an operator's own agent
- * the gate applies from the release that reports runtime labels at all; an
- * older agent reports none, so its silence proves nothing and a job it cannot
- * run fails on it with a message that names the missing runtime.
+ * Routing (`canAgentRunJob`): a `container:` job reaches an agent only when that
+ * agent was started for it, or reports `kici:runtime:docker` or
+ * `kici:runtime:podman`. An older agent reports no runtime labels, so its
+ * silence proves nothing; a job it cannot run fails with a message that names
+ * the missing runtime.
  *
  * An operator may still put the label on a pool's label sets and on a container
  * job's `runsOn` to steer container work at hosts with a runtime.

package/dist/scaler/disk-guard.d.ts

@@ -18,9 +18,8 @@ interface DiskGuardOpts {
     reapFn?: (cfg: ScalerConfig) => Promise<ReapCounts>;
 }
 /**
- * Run the disk guard. Returns reaped/recovered status; the caller decides
- * whether to proceed or exit with backoff. No-ops (recovered=true) when there
- * is no FC scaler or free space is already above threshold.
+ * The caller decides whether to proceed or exit with backoff. No-ops
+ * (recovered=true) when there is no FC scaler or free space is above threshold.
  */
 export declare function runDiskGuard(opts: DiskGuardOpts): Promise<DiskGuardResult>;
 export {};

package/dist/scaler/event-backend.d.ts

@@ -1,33 +1,25 @@
 /**
  * Event scaler backend — workflow-driven autoscaling.
  *
- * Unlike the local backends (container / bare-metal / firecracker), this backend
- * performs NO local compute. Its `spawn()` emits a `kici.scaler.scale-up` event
- * and its `destroy()` emits a `kici.scaler.scale-down` event; a customer-authored
- * provisioning / teardown workflow (subscribed via `kiciEvent()`) consumes those
- * events to boot and tear down an ephemeral cloud instance. The provisioned
- * instance's agent registers back with the scaler-chosen `agentId` and the
- * existing bound-job dispatch runs the pending job — reusing the whole scaling
- * engine (demand detection, caps, reservations, spawn timeout, teardown).
+ * No local compute: `spawn()` emits `kici.scaler.scale-up` and `destroy()` emits
+ * `kici.scaler.scale-down`, which a customer workflow (via `kiciEvent()`) consumes
+ * to boot and tear down an ephemeral cloud instance. Its agent registers with the
+ * scaler-chosen `agentId` and bound-job dispatch runs the pending job, so the
+ * whole scaling engine (caps, reservations, spawn timeout, teardown) applies.
  */
 import type { EffectiveLimits, LabelSetConfig, ManagedAgent, ScalerBackend, ScalerDestroyContext, ScalerEntry, ScalerEventCallback, SpawnContext, ValidationResult } from './types.js';
 import type { ClaimStore } from './claim-store.js';
 import type { ScalerScaleUpPayload, ScalerScaleDownPayload } from './scaler-events.js';
-/**
- * Subset of `EventEmitter` the backend needs. Injected so the backend is unit
- * testable without a real event router.
- */
+/** Subset of `EventEmitter` the backend needs; injected for unit tests. */
 export interface ScalerEventEmitterLike {
     emitScalerScaleUp(payload: ScalerScaleUpPayload, targets: string[]): Promise<string>;
     emitScalerScaleDown(payload: ScalerScaleDownPayload, targets: string[]): Promise<string>;
 }
 export interface EventScalerBackendOptions {
-    /** The parsed scaler entry (`type: 'event'`). */
+    /** `type: 'event'`. */
     entry: ScalerEntry;
-    /** Emits the reserved scale-up / scale-down events. */
     emitter: ScalerEventEmitterLike;
-    /** Mints ephemeral credentials when a claim code is redeemed — normally by
-     * the provisioned agent self-bootstrapping, or by a provisioning workflow. */
+    /** Redeemed by the self-bootstrapping agent, or by a provisioning workflow. */
     claimStore: ClaimStore;
     /** Correlation-id generator for scale-up / scale-down requests. */
     requestId: () => string;
@@ -48,40 +40,29 @@ export declare class EventScalerBackend implements ScalerBackend {
     /** The entry this backend is serving; the reload rollback restores it. */
     get currentEntry(): ScalerEntry;
     getActiveCount(): number;
-    /**
-     * Register a pending claim, emit a scale-up event to the provisioning targets,
-     * and track the agent as provisioning. Performs no local compute.
-     */
     spawn(labelSet: string[], agentId: string, orchestratorUrl: string, _onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext, signal?: AbortSignal): Promise<ManagedAgent>;
     /**
-     * Drop the tracked agent, emit a scale-down event (carrying the teardown
-     * reason), then invalidate the pending claim. Idempotent: a repeat destroy of
-     * an already-torn-down agent is a no-op (no duplicate scale-down).
+     * Drop the tracked agent, emit a scale-down with the reason, then invalidate
+     * the pending claim. Idempotent: a repeat destroy emits nothing.
      *
-     * The tracking entry is dropped before the first `await` so a concurrent
-     * destroy of the same agent returns at the guard above and cannot emit a
-     * second scale-down.
+     * The entry is dropped before the first `await`, so a concurrent destroy
+     * returns at the guard and cannot emit a second scale-down.
      *
-     * The claim invalidation is last and fails open. It is a DB delete, and the
-     * scale-down is the only thing that tears down the customer's cloud instance:
-     * running the delete first lets a DB error suppress the teardown for good,
-     * because the agent is already out of `this.agents` and a retried destroy
-     * returns at the guard. A claim left behind instead is inert — the row is
-     * single-use and TTL-bounded, so it can mint at most one token for an agent
-     * id that is being torn down.
+     * The claim invalidation is last and fails open. The scale-down is the only
+     * teardown of the customer's instance, so a DB error before it would suppress
+     * the teardown for good (a retried destroy returns at the guard). A leftover
+     * claim is single-use and TTL-bounded: at most one token for a dying agent id.
      */
     destroy(managedId: string, context?: ScalerDestroyContext): Promise<void>;
     /**
      * Drop the tracked agent without emitting anything.
      *
-     * The counterpart to `adopt()` on the other side of a cross-instance
-     * registration: the spawning coordinator is never told that a peer adopted
-     * its provision, so its entry would otherwise live until the process exits —
-     * inflating `getActiveCount()` against the global cap forever, and handing
-     * `shutdownAll()` a peer's live agent to tear down. Forgetting says "this
-     * coordinator no longer tracks the provision", which is exactly true and says
-     * nothing about whether the customer's instance is still running; the adopter
-     * emits the teardown, and the leader-gated reaper is the backstop.
+     * The spawning side of a cross-instance `adopt()`. The spawner is never told
+     * a peer adopted its provision, so its entry would otherwise inflate
+     * `getActiveCount()` against the global cap until exit, and hand
+     * `shutdownAll()` a peer's live agent. It says nothing about whether the
+     * instance runs: the adopter emits the teardown, the leader-gated reaper is
+     * the backstop.
      */
     forget(agentId: string): void;
     /**
@@ -92,29 +73,23 @@ export declare class EventScalerBackend implements ScalerBackend {
     adopt(agentId: string, labelSet: string[]): void;
     /** Mark a provisioned agent active once its instance registers over WS. */
     markActive(agentId: string): void;
-    /** Delegate a credential claim to the backend's claim store. */
     claim(code: string): Promise<import("./claim-store.js").ClaimedCredentials>;
     /**
-     * Emits nothing. An event provision is a customer cloud instance whose
-     * lifetime spans the cluster, not this process: the coordinator holding the
-     * agent emits its teardown from the WS-close path, and the leader-gated
-     * reaper is the backstop for one that reaches nobody. This map also carries
-     * provisions a peer has already adopted — the spawning coordinator is never
-     * told — so emitting here would make a routine restart tear down a peer's
-     * running agents.
+     * Emits nothing. A provision's lifetime spans the cluster, not this process:
+     * the coordinator holding the agent emits teardown on WS close, and the
+     * leader-gated reaper is the backstop. The map may hold provisions a peer
+     * adopted, so emitting here would make a routine restart tear down a peer's
+     * agents.
      *
-     * The map is deliberately left intact rather than cleared: a WS close landing
-     * during the remaining shutdown steps then still emits its own teardown.
+     * The map is left intact so a WS close during the remaining shutdown steps
+     * still emits its own teardown.
      */
     shutdownAll(): Promise<void>;
     /**
-     * Apply the new config. The backend reads `roles`, `agentTokenTtlSeconds`
-     * and `provisioningTargets` off its entry at every spawn (and
-     * `mandatoryLabels` only when a spawn reaches it without a context), so the
-     * entry is replaced here — otherwise a reload that retargets the
-     * provisioning workflow, or changes the scaler's roles, would keep emitting
-     * scale-up events to the old workflow refs and minting agent labels from the
-     * old roles, while the manager's own routing gate used the new ones.
+     * Replace the entry: each spawn reads `roles`, `agentTokenTtlSeconds`,
+     * `provisioningTargets` (and, without a context, `mandatoryLabels`) from it.
+     * Otherwise a reload would keep emitting to the old workflow refs and minting
+     * labels from the old roles while the manager's routing gate uses the new ones.
      */
     reload(labelSets: LabelSetConfig[], opts?: {
         maxAgents?: number;