@kici-dev/orchestrator 0.12.0 → 0.14.0

package/dist/scaler/orphan-requests.d.ts

@@ -0,0 +1,23 @@
+import { ScalerOrphansAction, type PeerScalerOrphansResponse } from '@kici-dev/engine';
+import type { ScalerManager } from './manager.js';
+/**
+ * The longest a scaler orphan request waits for its node. The `kici-admin`
+ * HTTP client stops waiting for response headers after 300 s, so a longer
+ * wait could never reach the operator.
+ */
+export declare const MAX_SCALER_ORPHANS_TIMEOUT_MS = 240000;
+/** A scaler orphan answer without its peer-message envelope. */
+export type ScalerOrphansAnswer = Omit<PeerScalerOrphansResponse, 'type' | 'messageId'>;
+/** What one scaler orphan request asks of the node. */
+export interface ScalerOrphansRequest {
+    action: ScalerOrphansAction;
+    /** `stop` only: the VM ids the operator approved. */
+    vmIds?: string[];
+}
+/**
+ * Answer a scaler orphan request on this node. A node with no scaler manager
+ * runs no Firecracker scaler: it lists nothing and holds none of the VMs a stop
+ * names. A failure is answered `ok: false` with its message, never thrown.
+ */
+export declare function answerScalerOrphansRequest(manager: Pick<ScalerManager, 'listLiveVms' | 'stopUntrackedVms' | 'firecrackerScalerNames'> | null, req: ScalerOrphansRequest): Promise<ScalerOrphansAnswer>;
+//# sourceMappingURL=orphan-requests.d.ts.map

package/dist/scaler/overlay-template.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Directory that holds the templates, relative to the jailer chroot base dir.
+ * It sits beside the per-VM chroots because the orchestrator must already be
+ * able to create directories there; the leading dot keeps it out of the
+ * orphan sweep, which reads every other entry as a VM id.
+ */
+export declare const OVERLAY_TEMPLATE_DIR: string;
+/**
+ * Timeout for formatting a template. It runs once per size per host, mostly
+ * off a spawn's critical path, and its `fsync` may wait behind a disk backlog,
+ * so it gets far more room than a per-spawn step.
+ */
+export declare const OVERLAY_TEMPLATE_MKFS_TIMEOUT_MS = 300000;
+/** Timeout for the per-spawn copy. */
+export declare const OVERLAY_COPY_TIMEOUT_MS = 30000;
+/** Runs a host command, rejecting on failure. The backend passes its `execAsync`. */
+export type OverlayCommandRunner = (cmd: string, args: string[], timeoutMs: number) => Promise<unknown>;
+/** Where the template for a drive size lives. */
+export declare function overlayTemplatePath(chrootBaseDir: string, sizeMib: number): string;
+/**
+ * Whether `path` is a usable template for `sizeMib`: it exists, its size is
+ * exactly `sizeMib` MiB, and it carries the ext4 superblock magic.
+ */
+export declare function isOverlayTemplate(path: string, sizeMib: number): Promise<boolean>;
+/** Builds and hands out overlay templates for one jailer chroot base dir. */
+export declare class OverlayTemplates {
+    private readonly chrootBaseDir;
+    private readonly run;
+    /** In-flight builds by size, so concurrent first uses share one build. */
+    private readonly builds;
+    constructor(chrootBaseDir: string, run: OverlayCommandRunner);
+    /** Return the path of a valid template for `sizeMib`, building it if needed. */
+    ensure(sizeMib: number): Promise<string>;
+    /** Create the overlay drive at `dest` as a sparse copy of the template. */
+    createOverlay(dest: string, sizeMib: number): Promise<void>;
+    private build;
+}
+//# sourceMappingURL=overlay-template.d.ts.map

package/dist/scaler/scaler-state-store.d.ts

@@ -270,6 +270,13 @@ export declare class ScalerStateStore {
      * has an unknown owner and is deliberately not returned to anybody.
      */
     listSpawningAgentsForOwner(instanceId: string): Promise<SpawningAgentSnapshot[]>;
+    /**
+     * Whether `instanceId` adopted an event provision whose spawn row is still
+     * live: one of the rows `listReapCandidates` returns that the reaper judges
+     * by its adopter's liveness rather than by its own deadline. `peer forget`
+     * reads it, so forgetting such an adopter never cuts the reaper's grace short.
+     */
+    hasAdoptedProvisions(instanceId: string): Promise<boolean>;
     /**
      * Every event row that is either past its spawn deadline without ever being
      * adopted, or adopted by some instance. The reaper decides which to tear down;

package/dist/scaler/spawn-errors.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Errors a scaler backend's `spawn()` throws to tell the manager how to treat
+ * the failure.
+ */
+/**
+ * A spawn the host refused outright. Retrying cannot succeed until an operator
+ * changes something — the binary path, the label set's env — so the manager
+ * defers the scaler instead of freeing the capacity for an immediate retry.
+ * The backend emits `scaler.failed` before it throws this.
+ */
+export declare class DeterministicSpawnError extends Error {
+    constructor(message: string, options?: {
+        cause?: unknown;
+    });
+}
+export declare function isDeterministicSpawnError(err: unknown): err is DeterministicSpawnError;
+//# sourceMappingURL=spawn-errors.d.ts.map

package/dist/scaler/startup-orphan-sweep.d.ts

@@ -0,0 +1,15 @@
+import type { ScalerBackend } from './types.js';
+/**
+ * Reap each container and Firecracker backend's orphans once, and start the
+ * Firecracker periodic sweep. A failure is logged and never stops startup.
+ *
+ * The periodic sweep is what reclaims a VM chroot, TAP or IP that a failed
+ * spawn could not remove (the jailer had already taken the chroot, or the
+ * removal raced the jailer). Startup-only cleanup leaves those to accumulate
+ * for as long as the process stays up.
+ */
+export declare function reapOrphansAtStartup(backends: ReadonlyArray<{
+    name: string;
+    backend: ScalerBackend;
+}>): Promise<void>;
+//# sourceMappingURL=startup-orphan-sweep.d.ts.map

package/dist/scaler/types.d.ts

@@ -200,9 +200,12 @@ export interface ManagedAgent {
  * Result of a scaling decision.
  * Discriminated union on 'action' for exhaustive pattern matching.
  */
-export type ScaleResult = {
+export type ScaleResult = 
+/** A spawn started; `agentId` is the scaler-managed id the new agent will register with. */
+{
     action: 'spawning';
     backendType: string;
+    agentId: string;
 } | {
     action: 'at-capacity';
 } | {
@@ -212,10 +215,15 @@ export type ScaleResult = {
     action: 'failed';
     error: string;
 }
-/** Scale-up declined without attempting a spawn (e.g. coordinator draining). */
+/**
+ * Scale-up declined without attempting a spawn (e.g. coordinator draining).
+ * `retryAfterMs` is set when the scaler is deferring after failed launches:
+ * a request before then is declined the same way.
+ */
  | {
     action: 'skipped';
     reason: string;
+    retryAfterMs?: number;
 };
 /**
  * What triggered a pending-scale re-drive (`Dispatcher.retryPendingScaleRequests`):
@@ -297,6 +305,9 @@ export interface ScalerBackend {
      *   the provisioned resource for operator inspection.
      * @returns The managed agent tracking object
      * @throws If the label set is not supported by this backend
+     * @throws DeterministicSpawnError, after emitting `scaler.failed`, when the
+     *   host refused the launch itself. The manager defers the scaler instead of
+     *   freeing the capacity for an immediate retry.
      */
     spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext, 
     /**
@@ -316,6 +327,14 @@ export interface ScalerBackend {
      *   teardown or a spawn timeout. Local backends accept and ignore it.
      */
     destroy(managedId: string, context?: ScalerDestroyContext): Promise<void>;
+    /**
+     * Record that a managed agent's process registered with the orchestrator.
+     * Called by ScalerManager when a spawn it started registers locally.
+     * Optional: a backend implements it when it must tell a compute that stops
+     * before registering (a failed spawn it tears down itself) from one that
+     * stops after (an agent whose disconnect triggers `destroy`).
+     */
+    markRegistered?(managedId: string): void;
     /**
      * Reclaim the HOST-LOCAL compute of a managed agent this backend no longer
      * tracks in memory, from whatever durable host state survived the loss.

package/dist/scaler/vm-job-bindings.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Durable job-to-agent bindings, read from the coordinator's dispatch queue.
+ *
+ * A worker has no database, so this is the coordinator's half of the
+ * live-VM tracking: a VM whose agent holds a non-terminal job is tracked, even
+ * when the node that runs the VM has lost every in-memory trace of it.
+ */
+import type { Kysely } from 'kysely';
+import type { Database } from '../db/types.js';
+/**
+ * The non-terminal job each of `agentIds` holds, keyed by agent id. An agent
+ * holds a job it was dispatched (`agent_id`) and a job waiting for it to
+ * reconnect (`recovery_agent_id`). The value is the job id the queue
+ * dispatches by, `dispatch_queue.id`.
+ */
+export declare function findActiveJobBindings(db: Kysely<Database>, agentIds: readonly string[]): Promise<Map<string, string>>;
+//# sourceMappingURL=vm-job-bindings.d.ts.map

package/dist/secrets/rbac.d.ts

@@ -17,7 +17,13 @@ export type Permission = 'context.create' | 'context.read' | 'context.update' |
 /** Read the org-wide CI trust policy that gates fork / unknown / workflow-change PRs. */
  | 'ci_trust.read'
 /** Modify org-wide trust policies (independent mode only — Platform-attached PATCH refuses). */
- | 'ci_trust.admin';
+ | 'ci_trust.admin'
+/** List the live Firecracker VMs on a node and how each is tracked. */
+ | 'scaler.read'
+/** Stop live Firecracker VMs the node's orchestrator does not track. */
+ | 'scaler.manage'
+/** Drop a departed peer from the coordinators' live peer registries. */
+ | 'peer.manage';
 /**
  * Error thrown when a role lacks the required permission.
  */