@kici-dev/orchestrator 0.16.0 → 0.18.0

package/dist/scaler/event-provision-reaper.d.ts

@@ -3,19 +3,16 @@ import type { ReapCandidate } from './scaler-state-store.js';
 /**
  * How long a provision outcome is kept once its spawn row is gone.
  *
- * Deliberately a constant rather than a `cluster_settings` knob: unlike the
- * claim retention beside it, there is no operational decision here. The only
- * reader is the stale-spawn prune, and the interval the row must survive is
- * derived from windows the reaper already resolves — so a knob would be surface
- * with nothing behind it. The purge floors this against twice the spawn
- * deadline, so raising `KICI_SCALER_SPAWN_TIMEOUT_MS` for a slow cloud widens
- * the retention with it.
+ * A constant, not a `cluster_settings` knob: unlike the claim retention, it
+ * carries no operational decision. Its only reader is the stale-spawn prune,
+ * and the interval the row must survive derives from windows the reaper already
+ * resolves. The purge floors it at twice the spawn deadline, so raising
+ * `KICI_SCALER_SPAWN_TIMEOUT_MS` for a slow cloud widens the retention with it.
  */
 export declare const PROVISION_OUTCOME_RETENTION_MS = 86400000;
 /**
- * The live values one sweep runs against. Read together at the top of each tick
- * so a sweep is internally consistent even if an operator changes a knob
- * halfway through.
+ * The live values one sweep runs against, read together at the top of each tick
+ * so a knob changed mid-sweep cannot make the sweep inconsistent.
  */
 export interface ReaperWindows {
     /** How often the leader sweeps. A change reschedules the timer after this tick. */
@@ -39,21 +36,17 @@ export interface ReaperWindows {
 /**
  * The flap grace the reaper actually runs on.
  *
- * `reroute_flap_grace_ms` is shared with the rerouted-job guard, and the two
- * consumers do not carry the same risk: the reroute guard force-fails a run it
- * could have waited for, while this one deletes a customer's running instance.
- * An operator lowering the knob for reroute reasons must not silently collapse
- * that protection to two consecutive sweeps.
+ * `reroute_flap_grace_ms` is shared with the rerouted-job guard, which only
+ * force-fails a run it could have waited for; this consumer deletes a
+ * customer's running instance. Lowering the knob for reroute reasons must not
+ * silently collapse that protection to two consecutive sweeps.
  *
- * The floor is two peer-stale-timeout windows — the very derivation the knob's
- * own default (`DEFAULT_REROUTE_FLAP_GRACE_MS`) comes from, applied as a floor
- * rather than a default. It has to cover the worst honest gap between an agent
- * being alive and being visible here: an agent reconnect backing off to its cap
- * (60s), plus the peer heartbeat interval (30s) during which a freshly
- * reconnected peer advertises no agents at all. Two stale windows (120s at the
- * defaults) covers that 90s; one (60s) would not.
- *
- * A raised knob is honoured as-is: the floor only ever raises.
+ * The floor is two peer-stale-timeout windows, the derivation of the knob's own
+ * default (`DEFAULT_REROUTE_FLAP_GRACE_MS`) applied as a floor. It covers the
+ * worst honest gap between an agent being alive and visible here: a reconnect
+ * backing off to its cap (60s) plus the peer heartbeat interval (30s) in which a
+ * freshly reconnected peer advertises no agents. Two windows (120s at the
+ * defaults) cover that 90s; one (60s) would not. The floor only ever raises.
  */
 export declare function reaperFlapGraceMs(flapGraceMs: number, peerStaleTimeoutMs: number): number;
 /** The slice of the peer registry the cluster-view guard reads. */
@@ -67,28 +60,25 @@ export interface ReaperPeerView {
  * The reaper's `canReap` predicate, over static config plus the live peer
  * registry.
  *
- * Extracted and exported so the wiring itself is testable. The bug this closes
- * lived in exactly this expression — `clusterViewSufficient` was unit-tested
- * throughout, while the call site that fed it the wrong count was not, so the
- * guard read permissive on a Platform-mode HA pair through two review rounds.
- * A component whose false positive deletes a customer's running instances
- * cannot have an untested load-bearing line.
+ * Exported so the wiring itself is tested, not only `clusterViewSufficient`: a
+ * call site feeding it the wrong count reads permissive on a Platform-mode HA
+ * pair, and a component whose false positive deletes a customer's running
+ * instances cannot have an untested load-bearing line.
  *
- * The peer count is the max of the two sources because neither alone covers
- * both modes: `KICI_CLUSTER_PEERS` is set only in independent mode, while a
- * Platform-mode cluster learns its peers from the matchmaker and leaves that
- * array empty. `getCoordinatorPeers()` filters on role, not on connectivity, so
- * a peer lost to a partition stays counted and the guard stays armed.
+ * The peer count is the max of two sources because neither covers both modes:
+ * `KICI_CLUSTER_PEERS` is set only in independent mode, while a Platform-mode
+ * cluster learns its peers from the matchmaker and leaves that array empty.
+ * `getCoordinatorPeers()` filters on role, not connectivity, so a peer lost to
+ * a partition stays counted and the guard stays armed.
  */
 export declare function canReapForCluster(configuredPeerCount: number, peers: ReaperPeerView): boolean;
 /**
  * Whether an adopting coordinator should still be treated as live.
  *
- * Mirrors `shouldDeferReroutedJob`, and for the same reason: `PeerRegistry`
- * flips `connected` the instant a peer WS closes, with no grace of its own, so
- * sampling it alone cannot tell a link blip between two healthy processes from
- * a dead coordinator — while the agent is still attached to that peer running a
- * customer job. A peer we have never heard of is genuinely gone.
+ * Mirrors `shouldDeferReroutedJob`: `PeerRegistry` flips `connected` the
+ * instant a peer WS closes, with no grace, so sampling it alone cannot tell a
+ * link blip between two healthy processes (whose agent still runs a customer
+ * job) from a dead coordinator. A peer never heard of is genuinely gone.
  */
 export declare function adopterIsLive(peer: {
     connected: boolean;
@@ -106,13 +96,11 @@ export declare function adopterIsLive(peer: {
  * backstop; not standing down costs the customer their fleet.
  *
  * `knownCoordinatorPeerCount` must count peers this coordinator has *ever*
- * handshaken with, not the ones named in static config. `KICI_CLUSTER_PEERS` is
- * set only in multi-orchestrator independent mode: the documented HA-pair recipe
- * runs Platform mode and joins by token, so peers arrive through the Platform
- * matchmaker and the static array stays empty. Keying on it left the guard inert
- * on precisely the topology it exists to protect. A known peer survives a
- * partition because `evictStalePeers` only marks it disconnected, so the guard
- * stays armed for as long as the partition lasts.
+ * handshaken with, not the ones in static config: `KICI_CLUSTER_PEERS` is set
+ * only in independent mode, and the documented HA-pair recipe runs Platform mode
+ * with peers from the matchmaker, so keying on it leaves the guard inert on the
+ * topology it exists to protect. `evictStalePeers` only marks a known peer
+ * disconnected, so the guard stays armed for the whole partition.
  *
  * A coordinator that has never met a peer is a single-coordinator deployment,
  * not a partitioned one, so it is never blocked.
@@ -123,21 +111,17 @@ export interface EventProvisionReaperOptions {
     listCandidates: (spawnCutoff: Date) => Promise<ReapCandidate[]>;
     /**
      * Whether an instance id is still a live cluster member (self counts as live).
-     * `flapGraceMs` is passed in so the implementation can treat a peer that
-     * disconnected moments ago as live — `peer.connected` alone flips on WS close
-     * with no grace at all.
+     * `flapGraceMs` lets a peer that disconnected moments ago count as live:
+     * `peer.connected` flips on WS close with no grace.
      */
     isPeerLive: (instanceId: string, flapGraceMs: number) => boolean;
     /** Whether an agent id is registered anywhere in the cluster we can see. */
     isAgentRegistered: (agentId: string) => boolean;
     /**
-     * Whether this coordinator can see enough of the cluster to act on its
-     * behalf. False on a node that *knows* coordinator peers but is connected to
-     * none of them, which is exactly the shape a network partition produces.
-     *
-     * "Knows" spans both modes: peers named in static config, plus peers this
-     * instance has handshaken with. Keying on static config alone is what left
-     * this guard inert on a Platform-mode HA pair, which configures none.
+     * Whether this coordinator sees enough of the cluster to act for it: false on
+     * a node that *knows* coordinator peers (static config plus every peer it
+     * handshook with) but is connected to none, the shape a partition produces.
+     * Static config alone leaves the guard inert on a Platform-mode HA pair.
      */
     canReap: () => boolean;
     /** Emit the teardown and drop the row. */
@@ -146,21 +130,19 @@ export interface EventProvisionReaperOptions {
     purgeExpiredClaims: (cutoff: Date) => Promise<number>;
     /**
      * Delete provision outcomes older than the cutoff whose spawn row is gone;
-     * returns rows deleted. Runs on the same leader-gated sweep as the claim
-     * purge, which is the orchestrator's one precedent for a timer-driven purge
-     * of scaler bookkeeping.
+     * returns rows deleted. Runs on the leader-gated sweep beside the claim purge,
+     * the orchestrator's one precedent for a timer-driven scaler-bookkeeping purge.
      */
     purgeProvisionOutcomes: (cutoff: Date) => Promise<number>;
     /**
-     * Read the live windows for one sweep. Backed by `cluster_settings`, so an
-     * operator retunes the reaper on a running cluster; each read falls back to
-     * the orchestrator's configured default when the column is NULL.
+     * The live windows for one sweep, from `cluster_settings` so an operator
+     * retunes a running cluster; a NULL column falls back to the configured default.
      */
     resolveWindows: () => Promise<ReaperWindows>;
     /**
-     * Interval used for the very first schedule, before any knob has been read.
-     * `onBecomeLeader` is synchronous (it is a Raft callback), so the timer starts
-     * at the configured default and the first tick reconciles it to the live value.
+     * The first schedule's interval. `onBecomeLeader` is a synchronous Raft
+     * callback, so the timer starts at the configured default and the first tick
+     * reconciles it to the live value.
      */
     bootIntervalMs: number;
 }
@@ -229,25 +211,21 @@ export declare class EventProvisionReaper {
     /**
      * agentId → when this reaper first observed the agent registered nowhere.
      *
-     * This is the debounce that makes "unseen" mean sustained absence rather than
-     * one unlucky sample. It is leader-local observation, deliberately not a
-     * database column: `adopted_at` is refreshed by a coordinator re-adopting its
-     * own agent, so a restart loop on the adopter would keep resetting the clock
-     * on its own stranded provision.
+     * The debounce that makes "unseen" mean sustained absence, not one unlucky
+     * sample. Leader-local, deliberately not a database column: a coordinator
+     * re-adopting its own agent refreshes `adopted_at`, so a restart loop on the
+     * adopter would keep resetting the clock on its own stranded provision.
      */
     private readonly strandedSince;
     /**
      * How many entries of `strandedSince` the last sweep found absent for at
      * least the flap grace.
      *
-     * The gauge publishes this rather than `strandedSince.size`, because the two
-     * answer different questions. `size` counts every agent currently observed
-     * absent, and an agent transits that set on every ordinary reconnect — a 30s
-     * peer heartbeat against a 60s sweep, and `PeerRegistry` emptying a peer's
-     * agent list on any disconnect, mean one flap enrols every peer-adopted agent
-     * at once. So `size` is routinely non-zero on a healthy cluster, while
-     * `verdict()` refuses to act on anything below the flap grace. Publishing the
-     * past-grace count makes the gauge measure the same thing the reaper does.
+     * The gauge publishes this, not `strandedSince.size`: an agent transits that
+     * set on every ordinary reconnect (a 30s peer heartbeat against a 60s sweep,
+     * and `PeerRegistry` empties a peer's agent list on any disconnect, so one
+     * flap enrols every peer-adopted agent), so `size` is routinely non-zero on a
+     * healthy cluster. The past-grace count measures what `verdict()` acts on.
      */
     private unseenPastGrace;
     /**
@@ -306,11 +284,10 @@ export declare class EventProvisionReaper {
     /**
      * Adopt a changed sweep interval by rebuilding the timer.
      *
-     * Re-read every sweep rather than only at a leadership transition: a knob an
-     * operator can set but that is silently ignored until the next election is
-     * worse than no knob at all. A value that is not a usable interval is ignored
-     * — a zero or negative `setInterval` period would spin the sweep as fast as
-     * the event loop allows.
+     * Re-read every sweep, not only at a leadership transition: a knob silently
+     * ignored until the next election is worse than none. An unusable value is
+     * ignored, since a zero or negative `setInterval` period spins the sweep as
+     * fast as the event loop allows.
      */
     private reconcileInterval;
     /** Replace the timer with one running at `intervalMs`. */

package/dist/scaler/failure-tracker.d.ts

@@ -1,33 +1,27 @@
 /**
  * In-process bounded record of recent scaler spawn failures.
  *
- * The scaler manager records every `scaler.failed` event here at the same site
- * it increments the fleet-wide Prometheus counter. The diagnose scaler check
- * reads recent failures grouped per backend instance to produce its rows. This
- * is memory-only and bounded — the recent-failures window resets on restart,
+ * The scaler manager records every `scaler.failed` event here where it
+ * increments the fleet-wide Prometheus counter; the diagnose scaler check reads
+ * it grouped per backend instance. Memory-only: the window resets on restart,
  * which is acceptable for an on-demand operator view over a short window.
  */
-/** A single recorded scaler spawn failure. */
 export interface ScalerFailureRecord {
-    /** Scaler instance name (the configured scaler `name`); the diagnose row key. */
+    /** Configured scaler `name`; the diagnose row key. */
     backendName: string;
-    /** Backend type: 'container' | 'bare-metal' | 'firecracker' | 'event' | 'unknown'. */
+    /** 'container' | 'bare-metal' | 'firecracker' | 'event' | 'unknown'. */
     backendType: string;
     /** True when the failed spawn was bound to a queued job (a run was affected). */
     bound: boolean;
-    /** Captured error string from the scaler event detail. */
     detail: string;
-    /** Event timestamp in epoch milliseconds. */
     timestampMs: number;
 }
-/** Per-backend summary of recent failures within a window. */
 export interface BackendFailureSummary {
     backendType: string;
     boundCount: number;
     unboundCount: number;
     /** Detail of the most recent failure in the window. */
     lastError: string;
-    /** Timestamp of the most recent failure in the window. */
     lastAtMs: number;
 }
 export declare class ScalerFailureTracker {
@@ -38,8 +32,7 @@ export declare class ScalerFailureTracker {
     record(rec: ScalerFailureRecord): void;
     /**
      * Group failures newer than `nowMs - windowMs` by backend instance name.
-     * `nowMs` is injected so callers control the clock (and tests are
-     * deterministic).
+     * `nowMs` is injected so callers control the clock (and tests are deterministic).
      */
     recentByBackend(windowMs: number, nowMs: number): Map<string, BackendFailureSummary>;
 }

package/dist/scaler/file-tail.d.ts

@@ -1,12 +1,6 @@
 /**
- * File tailing utility for append-only log files.
- *
- * Provides an async generator that yields complete lines as they are appended
- * to a file, using `fs.watchFile()` for change notifications and partial line
- * buffering for split writes.
- *
- * Used by the Firecracker backend to tail serial console and VMM log files
- * from the jailer chroot directory. The guest writes to `/dev/console`
+ * Tails append-only log files: the Firecracker backend tails the serial console
+ * and VMM log files in the jailer chroot. The guest writes to `/dev/console`
  * unprivileged, so everything this module buffers is tenant-controlled: every
  * buffer here is bounded, and an overflow is reported in-band rather than
  * growing the orchestrator's heap.
@@ -19,8 +13,8 @@
  */
 export declare const MAX_LINE_BYTES: number;
 /**
- * Largest slice read from the file per poll. A burst is drained across several
- * ticks instead of materialising the whole delta in one string.
+ * Largest slice read per poll, so a burst drains across several reads instead
+ * of materialising the whole delta in one string.
  */
 export declare const MAX_READ_BYTES: number;
 /**
@@ -32,15 +26,8 @@ export declare const MAX_READ_BYTES: number;
  */
 export declare const MAX_QUEUED_LINES = 10000;
 /**
- * Tail an append-only file, yielding complete lines as they are written.
- *
- * Pre-creates the file if it does not exist to avoid watch errors.
- * Tracks byte offset and partial (incomplete) trailing lines across reads.
- * Stops when the provided AbortSignal is triggered.
- *
- * @param filePath - Path to the file to tail
- * @param signal - AbortSignal to stop tailing
- * @yields Complete lines (non-empty) as they are appended to the file
+ * Tail an append-only file, yielding complete non-empty lines as they are
+ * written, until `signal` aborts. Pre-creates the file to avoid watch errors.
  */
 export declare function tailFile(filePath: string, signal: AbortSignal): AsyncGenerator<string>;
 //# sourceMappingURL=file-tail.d.ts.map

package/dist/scaler/firecracker-api.d.ts

@@ -1,10 +1,7 @@
 /**
- * Firecracker REST API client over Unix domain sockets.
+ * Firecracker REST API client over a Unix domain socket (`http` with `socketPath`).
  *
- * Provides a thin wrapper around Node.js built-in `http` module for communicating
- * with Firecracker's REST API. Uses the `socketPath` option for HTTP-over-Unix-socket.
- *
- * Only used for post-boot operations:
+ * Post-boot operations only:
  * - MMDS data injection (passing agent config to the guest VM)
  * - Graceful shutdown via SendCtrlAltDel (x86_64 only)
  * - Socket readiness polling after jailer invocation
@@ -14,94 +11,53 @@
  * @see https://github.com/firecracker-microvm/firecracker/blob/main/docs/getting-started.md
  * @see https://github.com/firecracker-microvm/firecracker/blob/main/docs/mmds/mmds-user-guide.md
  */
-/**
- * Error thrown when the Firecracker API returns a non-2xx status code
- * or a network-level error occurs.
- */
+/** A non-2xx Firecracker API response, or a network-level error. */
 export declare class FirecrackerApiError extends Error {
-    /** HTTP status code from the Firecracker API (undefined for network errors) */
+    /** Undefined for a network error. */
     readonly statusCode: number | undefined;
-    /** The API path that was requested */
     readonly path: string;
-    /** The response body from the Firecracker API */
     readonly responseBody: string;
     constructor(message: string, path: string, statusCode?: number, responseBody?: string);
 }
-/**
- * Thin REST client for the Firecracker API over Unix domain sockets.
- *
- * No external dependencies -- uses Node.js built-in `http` module with
- * the `socketPath` option for HTTP-over-Unix-socket communication.
- */
 export declare class FirecrackerApi {
     private readonly socketPath;
     /**
-     * Aborts every in-flight and subsequent request on this client.
-     *
-     * Set by the spawn path so a spawn deadline actually stops the VM
-     * provisioning calls, rather than leaving them running against a VM the
-     * manager has already given up on.
+     * Aborts every in-flight and later request. The spawn path sets it so a spawn
+     * deadline stops the provisioning calls against a VM the manager gave up on.
      */
     private signal;
     constructor(socketPath: string, signal?: AbortSignal);
-    /**
-     * Send an HTTP request to the Firecracker API.
-     *
-     * @param method - HTTP method (GET, PUT, PATCH)
-     * @param path - API path (e.g., '/mmds', '/actions', '/')
-     * @param body - Optional JSON body for PUT/PATCH requests
-     * @returns Response with statusCode and body
-     * @throws FirecrackerApiError on non-2xx status or network error
-     */
+    /** @throws FirecrackerApiError on a non-2xx status or a network error */
     private request;
     /**
-     * Inject MMDS (Micro VM Metadata Service) data into a running VM.
-     *
-     * Uses MMDS v2 data injection endpoint. The data should follow the structure:
-     * `{ latest: { 'meta-data': { ... } } }`
+     * Inject MMDS (Micro VM Metadata Service) v2 data into a running VM, shaped
+     * `{ latest: { 'meta-data': { ... } } }`. The guest reads it from
+     * http://169.254.169.254/latest/meta-data/ with a token-based session.
      *
-     * The guest VM reads this data from http://169.254.169.254/latest/meta-data/
-     * using a token-based session (MMDS v2).
-     *
-     * @param data - Metadata object to inject
      * @see https://github.com/firecracker-microvm/firecracker/blob/main/docs/mmds/mmds-user-guide.md
      */
     putMmds(data: Record<string, unknown>): Promise<void>;
     /**
-     * Clear MMDS data after agent has received config via WS.
-     * Replaces the MMDS data with an empty meta-data object.
-     * Called after receiving config.ack from the agent.
+     * Replace the MMDS data with an empty meta-data object once the agent's
+     * config.ack confirms it received its config over WS.
      */
     clearMmds(): Promise<void>;
     /**
-     * Send a SendCtrlAltDel action for graceful x86_64 shutdown.
-     *
-     * This simulates pressing Ctrl+Alt+Del on the virtual keyboard,
-     * which triggers a clean shutdown in most Linux distributions.
-     *
-     * Note: Only works on x86_64. On arm64, the i8042 keyboard controller
-     * is not available, so this action is not supported. Use process kill
-     * as fallback on arm64.
+     * Graceful shutdown: Ctrl+Alt+Del on the virtual keyboard, which most Linux
+     * distributions answer with a clean shutdown. x86_64 only: arm64 has no i8042
+     * keyboard controller, so kill the process there instead.
      *
      * @see https://github.com/firecracker-microvm/firecracker/blob/main/docs/getting-started.md
      */
     sendCtrlAltDel(): Promise<void>;
     /**
-     * Get instance info from the Firecracker API root endpoint.
-     *
-     * Used to check if the API socket is reachable and the VM is running.
-     * Returns the parsed JSON response with instance information.
+     * Instance info from the API root, parsed. Doubles as the check that the API
+     * socket is reachable and the VM is running.
      */
     getInstanceInfo(): Promise<Record<string, unknown>>;
     /**
-     * Poll the Firecracker API socket until it becomes reachable or timeout expires.
-     *
-     * Used after jailer invocation to wait for Firecracker to create the API socket
-     * and become ready to accept requests.
-     *
-     * @param timeoutMs - Maximum time to wait in milliseconds (default: 5000)
-     * @param intervalMs - Polling interval in milliseconds (default: 100)
-     * @returns true if socket became reachable, false on timeout
+     * Poll the API socket after the jailer starts, until Firecracker accepts
+     * requests (true) or `timeoutMs` passes (false). Both arguments are in ms.
      */
     waitForSocket(timeoutMs?: number, intervalMs?: number): Promise<boolean>;
 }