@kici-dev/orchestrator 0.1.14 → 0.1.16

package/dist/diagnostics/checks/index.d.ts

@@ -12,7 +12,8 @@ import { checkDiskSpace } from './disk.js';
 import { checkConfigValidity } from './config.js';
 import { checkCertificateExpiry } from './certs.js';
 import { checkScalerProvisioning } from './scaler.js';
+import { checkFirecrackerNetwork } from './firecracker-network.js';
 /** All diagnostic checks in display order. */
 export declare const defaultChecks: DiagnosticCheck[];
-export { checkDbConnectivity, checkWsToPlatform, checkAgentConnectivity, checkDiskSpace, checkConfigValidity, checkCertificateExpiry, checkScalerProvisioning, };
+export { checkDbConnectivity, checkWsToPlatform, checkAgentConnectivity, checkDiskSpace, checkConfigValidity, checkCertificateExpiry, checkScalerProvisioning, checkFirecrackerNetwork, };
 //# sourceMappingURL=index.d.ts.map

package/dist/diagnostics/fleet-collector.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Recursive fleet subtree assembler.
+ *
+ * Every orchestrator runs this to produce a self-similar subtree ZIP:
+ * local/ + nested agents/<id>.zip + workers/<id>.zip + peers/<id>.zip, plus
+ * fleet-manifest.json recording per-node status. Dead branches are recorded,
+ * never fatal (Promise.allSettled at each level). The loop guard
+ * (includeCoordinatorMesh) is true only on the root call; every downstream peer
+ * request sets it false so the coordinator mesh never echoes.
+ */
+import { z } from 'zod';
+/** Per-node collection outcome recorded in the fleet manifest. */
+export declare const FleetNodeStatus: z.ZodEnum<{
+    error: "error";
+    ok: "ok";
+    timeout: "timeout";
+    unreachable: "unreachable";
+}>;
+export type FleetNodeStatus = z.infer<typeof FleetNodeStatus>;
+export interface FleetCollectorDeps {
+    /** This orchestrator's instanceId (the manifest owner). */
+    instanceId: string;
+    /** Build this node's own debug bundle as a Buffer (createDebugBundle output). */
+    buildLocalBundle: () => Promise<Buffer>;
+    /** List this node's directly-connected agents. */
+    listAgents: () => {
+        agentId: string;
+    }[];
+    /** Request a connected agent's mini-bundle over the agent WS channel. */
+    requestAgentBundle: (agentId: string) => Promise<Buffer>;
+    /** List this node's downstream peers (coordinator-mesh peers + its workers). */
+    listPeers: () => {
+        instanceId: string;
+        role: 'coordinator' | 'worker';
+        kind: 'peer' | 'worker';
+    }[];
+    /** Request a peer's subtree bundle over the peer WS channel (loop-guarded false). */
+    requestPeerSubtree: (instanceId: string, includeCoordinatorMesh: boolean) => Promise<Buffer>;
+}
+export interface FleetCollectOptions {
+    logWindowHours: number;
+    selection: {
+        all: boolean;
+        agentIds: string[];
+        workerInstanceIds: string[];
+    };
+    /** Loop guard. True only on the root call; downstream requests force false. */
+    includeCoordinatorMesh: boolean;
+    timeoutMs: number;
+}
+export declare function collectFleetSubtree(opts: FleetCollectOptions, deps: FleetCollectorDeps): Promise<Buffer>;
+//# sourceMappingURL=fleet-collector.d.ts.map

package/dist/diagnostics/fleet-constants.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Shared constants for fleet log collection.
+ */
+/** Default per-node deadline for a fleet bundle request (overridable via --fleet-timeout). */
+export declare const FLEET_NODE_TIMEOUT_MS = 60000;
+/** Per-node cap on raw log bytes an agent includes in its mini-bundle (50 MiB). */
+export declare const FLEET_MAX_LOG_BYTES: number;
+//# sourceMappingURL=fleet-constants.d.ts.map

package/dist/diagnostics/fleet-selection.d.ts

@@ -0,0 +1,15 @@
+import type { FleetSelection } from '@kici-dev/engine';
+import type { FleetTopology } from './fleet-topology.js';
+/** A per-orchestrator selection, keyed by orchestrator instanceId. Absent = prune. */
+export type ResolvedSelectionMap = Map<string, FleetSelection>;
+/**
+ * Resolve `--pick` selectors into a per-orchestrator FleetSelection map.
+ *
+ * No selectors -> every orchestrator gets `{ all: true }`. With selectors, an
+ * orchestrator is included only if at least one of its agents/workers (or the
+ * orchestrator itself) matches; its FleetSelection then carries exactly the
+ * matched agent ids and worker instanceIds. Orchestrators with no matches are
+ * omitted from the map (their branch is pruned).
+ */
+export declare function resolveSelection(topology: FleetTopology, selectors: string[]): ResolvedSelectionMap;
+//# sourceMappingURL=fleet-selection.d.ts.map

package/dist/diagnostics/fleet-topology.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Fleet topology enumeration.
+ *
+ * Builds the cluster tree for `debug-bundle --fleet --list` / `--pick` from the
+ * local agent registry plus the heartbeat-cached peer inventory — no fan-out.
+ * Each peer's cached agent list is already maintained by ~30s heartbeats, so
+ * enumeration is a cheap in-memory read.
+ */
+/** A node in the enumerated fleet topology. */
+export interface FleetTopologyNode {
+    kind: 'orchestrator' | 'agent';
+    id: string;
+    /** Cluster role for orchestrator nodes (coordinator | worker); undefined for agents. */
+    role?: 'coordinator' | 'worker';
+    hostname?: string;
+    labels: Record<string, string>;
+    /** Parent orchestrator instanceId; null for the collector (root) node. */
+    parentId: string | null;
+}
+export interface FleetTopology {
+    nodes: FleetTopologyNode[];
+}
+export interface FleetTopologyDeps {
+    /** This collector's own instanceId (the root orchestrator). */
+    instanceId: string;
+    /** This collector's role. */
+    role: 'coordinator' | 'worker';
+    /** This collector's hostname, if known. */
+    hostname?: string;
+    /** This node's directly-connected agents. */
+    listLocalAgents: () => {
+        agentId: string;
+        labels: string[];
+    }[];
+    /** The heartbeat-cached peer inventory (each peer + its cached agents). */
+    listPeers: () => {
+        instanceId: string;
+        role: 'coordinator' | 'worker';
+        hostname?: string;
+        agents: {
+            agentId: string;
+            labels: string[];
+        }[];
+    }[];
+}
+export declare function buildFleetTopology(deps: FleetTopologyDeps): FleetTopology;
+//# sourceMappingURL=fleet-topology.d.ts.map

package/dist/diagnostics/fleet-wiring.d.ts

@@ -0,0 +1,60 @@
+import type { PeerLogsCollectRequest, PeerToPeerMessage, FleetSelection } from '@kici-dev/engine';
+import type { AgentRegistry } from '../agent/registry.js';
+import type { PeerRegistry } from '../cluster/peer-registry.js';
+import type { PeerClient } from '../cluster/peer-client.js';
+import type { DiagnosticDeps } from './types.js';
+import { type FleetCollectorDeps } from './fleet-collector.js';
+import { type FleetTopology, type FleetTopologyDeps } from './fleet-topology.js';
+/** A peer-handler-like object exposing the dual-direction collect send. */
+export interface FleetPeerHandlerLike {
+    sendLogsCollectAndWait: (targetInstanceId: string, msg: PeerLogsCollectRequest, timeoutMs: number) => Promise<Buffer>;
+}
+/** A collector that issues a fleet.logs.request to an agent and awaits its bundle. */
+export interface FleetAgentRequester {
+    request: (requestId: string, agentId: string, send: () => void) => Promise<Buffer>;
+}
+export interface FleetRuntime {
+    instanceId: string;
+    role: 'coordinator' | 'worker';
+    /** Loop window for log files (hours). */
+    logWindowHours: number;
+    /** Per-node deadline (ms). */
+    timeoutMs: number;
+    /** Local agent log directory (KICI_LOG_DIR), if configured. */
+    logDir?: string;
+    agentRegistry: AgentRegistry;
+    peerRegistry: PeerRegistry;
+    fleetAgentCollector: FleetAgentRequester;
+    /** Outgoing peer clients, keyed by instanceId. */
+    peerClients: Map<string, PeerClient>;
+    /** Incoming peer-handler with the dual-direction collect send. */
+    peerHandler: FleetPeerHandlerLike;
+    /** Deps for the local createDebugBundle. */
+    diagnosticDeps: DiagnosticDeps;
+    /** Raw orchestrator config (redacted by createDebugBundle). */
+    config: Record<string, unknown>;
+    /** Cluster health endpoint for the local bundle's cluster/health.json. */
+    clusterHealthUrl?: string;
+}
+/**
+ * Build the FleetCollectorDeps for a given selection of this node's downstream
+ * agents/workers. The collector passes per-branch selection in when it knows
+ * which downstream subset each peer should gather.
+ */
+export declare function buildFleetCollectorDeps(runtime: FleetRuntime, perBranchSelection: (instanceId: string) => FleetSelection): FleetCollectorDeps;
+/** Build the FleetTopologyDeps for `--list` / `--pick` enumeration. */
+export declare function buildFleetTopologyDeps(runtime: FleetRuntime): FleetTopologyDeps;
+/** Enumerate this node's fleet topology (no fan-out). */
+export declare function getFleetTopology(runtime: FleetRuntime): FleetTopology;
+/**
+ * Collect this node's full subtree (root call: includeCoordinatorMesh=true) with
+ * an optional per-orchestrator selection map. Absent map entry => collect all.
+ */
+export declare function collectFleet(runtime: FleetRuntime, selectionByOrch: Map<string, FleetSelection> | null): Promise<Buffer>;
+/**
+ * Peer-side responder: on an inbound peer.logs.collect.request, assemble this
+ * node's subtree (with the request's loop guard + selection) and stream it back
+ * as peer.logs.collect.chunk frames, or a peer.logs.collect.error on failure.
+ */
+export declare function makeFleetCollectResponder(runtime: FleetRuntime): (msg: PeerLogsCollectRequest, send: (out: PeerToPeerMessage) => boolean) => Promise<void>;
+//# sourceMappingURL=fleet-wiring.d.ts.map

package/dist/environments/environment-store.d.ts

@@ -1,6 +1,11 @@
 import { type Kysely } from 'kysely';
 import type { Environment as EngineEnvironment } from '@kici-dev/engine';
 import type { Database, Environment } from '../db/types.js';
+/** Thrown by `delete` when pending held runs reference the environment. */
+export declare class EnvironmentDeleteBlockedError extends Error {
+    readonly pendingCount: number;
+    constructor(pendingCount: number);
+}
 /**
  * Map a DB environment row (snake_case) to the engine Environment type (camelCase).
  *
@@ -60,7 +65,15 @@ export declare class EnvironmentStore {
     create(orgId: string, data: EnvironmentCreateInput): Promise<Environment>;
     /** Update an environment. Returns the updated row, or null if not found. */
     update(orgId: string, id: string, updates: EnvironmentUpdateInput): Promise<Environment | null>;
-    /** Delete an environment. Cascade deletes bindings, variables, and overrides. */
+    /**
+     * Delete an environment.
+     *
+     * Bindings, variables, and source overrides cascade away via their FK.
+     * Terminal held-run history survives with a null `environment_id` (the FK
+     * uses ON DELETE SET NULL). Pending held runs still reference the
+     * environment, so deletion is blocked with `EnvironmentDeleteBlockedError`
+     * until they are approved or rejected.
+     */
     delete(orgId: string, id: string): Promise<boolean>;
     /**
      * Match an environment name against org environments.

package/dist/environments/held-runs.d.ts

@@ -4,7 +4,8 @@
  * Manages the lifecycle: pending -> approved/rejected/expired.
  */
 import { type Kysely } from 'kysely';
-import type { Database, HeldRun } from '../db/types.js';
+import { type ApprovalRequirement, type ApproverClause, ApprovalDecision, HoldScope, TriggerSource } from '@kici-dev/engine';
+import type { Database, HeldRun, HeldRunApproval } from '../db/types.js';
 /** Status values for held runs (held_runs table). */
 export declare enum HeldRunStatus {
     Pending = "pending",
@@ -24,6 +25,47 @@ export interface CreateHeldRunData {
     /** Queue type: 'environment' (default) or 'security'. */
     queueType?: 'environment' | 'security';
 }
+/**
+ * Data required to create a generalized approval hold. Unlike the legacy
+ * environment-only `create()`, this carries the hold scope, trigger source,
+ * optional step index, and the normalized approval requirement.
+ */
+export interface CreateHoldData {
+    runId: string;
+    jobId: string;
+    /** Granularity of the held element. */
+    scope: HoldScope;
+    /** Step index within the job for step-scoped holds; omit otherwise. */
+    stepIndex?: number;
+    /** What triggered the hold (environment policy vs SDK requireApproval). */
+    triggerSource: TriggerSource;
+    /** The normalized requirement the hold must satisfy. */
+    requirement: ApprovalRequirement;
+    /** Environment id, when the hold originates from an environment policy. */
+    environmentId?: string | null;
+    /** Queue type: 'environment' (default) or 'security'. */
+    queueType?: 'environment' | 'security';
+}
+/** A single decision to record against a hold. */
+export interface RecordDecisionData {
+    approverSub: string;
+    decision: ApprovalDecision;
+    /** Which requirement clauses this decision satisfied (for attribution). */
+    clausesSatisfied?: ApproverClause[];
+}
+/**
+ * The outcome of `release()` — describes how the held element must be resumed.
+ * The store only writes the terminal DB state; the caller performs the actual
+ * re-dispatch (job/workflow) or agent notification (step) using this signal.
+ */
+export interface ReleaseSignal {
+    holdId: string;
+    runId: string;
+    jobId: string;
+    scope: HoldScope;
+    /** Set only for step-scoped holds. */
+    stepIndex: number | null;
+}
 /** Options for listing held runs. */
 export interface ListHeldRunsOptions {
     status?: string;
@@ -35,6 +77,26 @@ export declare class HeldRunStore {
     constructor(db: Kysely<Database>);
     /** Create a new held run with pending status. */
     create(orgId: string, data: CreateHeldRunData): Promise<HeldRun>;
+    /**
+     * Create a generalized approval hold (workflow/job/step scope, explicit or
+     * environment trigger) carrying a normalized `ApprovalRequirement`. Returns
+     * the created row.
+     */
+    createHold(orgId: string, data: CreateHoldData): Promise<HeldRun>;
+    /** Record one approve/reject decision against a hold. */
+    recordDecision(heldRunId: string, data: RecordDecisionData): Promise<HeldRunApproval>;
+    /** List the recorded decisions for a hold, oldest first. */
+    listDecisions(heldRunId: string): Promise<HeldRunApproval[]>;
+    /** Get a single held run by id (org-scoped). Returns null if absent. */
+    getById(orgId: string, heldRunId: string): Promise<HeldRun | null>;
+    /**
+     * Release a hold whose approval requirement is satisfied. Flips the row to
+     * 'approved' and returns a `ReleaseSignal` describing how the caller must
+     * resume the element (re-dispatch for job/workflow, agent notification for
+     * step). Throws if the hold is not found or not pending. Approver attribution
+     * lives in `held_run_approvals`, not on the row.
+     */
+    release(orgId: string, heldRunId: string): Promise<ReleaseSignal>;
     /** Approve a pending held run. Throws if not found or not pending. */
     approve(orgId: string, heldRunId: string, approvedBy: string): Promise<HeldRun>;
     /** Reject a pending held run. Throws if not found or not pending. */
@@ -53,6 +115,12 @@ export declare class HeldRunStore {
     approveByQueueType(orgId: string, heldRunId: string, approvedBy: string, queueType: 'environment' | 'security'): Promise<HeldRun>;
     /** Get a held run by run ID and job ID. Returns null if not found. */
     getByRunAndJob(orgId: string, runId: string, jobId: string): Promise<HeldRun | null>;
+    /**
+     * List pending holds past their `expires_at`. Called by the stale detector
+     * BEFORE `expireOverdue()` so it can route each overdue hold by scope (step
+     * holds notify the waiting agent; job/workflow holds fail the run).
+     */
+    listOverdue(): Promise<HeldRun[]>;
     /**
      * Expire overdue pending runs. Called by the stale detector.
      * Sets status to 'expired' and resolved_at to now() for all

package/dist/firecracker/host-network.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Firecracker host-network provisioning.
+ *
+ * Creates the per-coordinator bridge (kici-brN), assigns its gateway IP,
+ * marks kici-* interfaces unmanaged by NetworkManager, enables IP forwarding,
+ * and builds a disjoint, source-scoped nftables table for NAT + egress
+ * isolation. One pure command-builder drives live provisioning, the rendered
+ * boot script, and (read-only) verification.
+ *
+ * This is HOST setup, distinct from the runtime per-VM isolation in
+ * scaler/nftables.ts (added at spawn / removed at destroy). The two share the
+ * nft table name but have separate lifecycles; this module never touches the
+ * per-VM rules.
+ */
+/** A single subprocess invocation (no shell). */
+export interface CommandSpec {
+    bin: string;
+    args: string[];
+    /** Optional stdin payload (used for `nft -f -`). */
+    stdin?: string;
+}
+/** Host-bridge configuration for one Firecracker coordinator. */
+export interface FirecrackerBridgeConfig {
+    /** Bridge interface name, e.g. 'kici-br0'. */
+    bridgeName: string;
+    /** Gateway IP + prefix, e.g. '10.0.0.1/24'. */
+    bridgeCidr: string;
+    /** nft table name, e.g. 'kici' or 'kici_b'. */
+    table: string;
+    /** NAT egress interface; auto-detected from the default route when omitted. */
+    hostIface?: string;
+}
+/**
+ * Derive the network address (CIDR) from a gateway CIDR by masking host bits.
+ * '10.0.0.1/24' -> '10.0.0.0/24'.
+ */
+export declare function cidrToNetwork(cidr: string): string;
+/**
+ * Build the ordered command list that provisions one Firecracker host bridge.
+ * Pure — performs no I/O. `provisionBridge` executes these; `renderBootScript`
+ * serializes them.
+ *
+ * The nft `delete table`/`add table` here only ever touches `cfg.table`, so a
+ * coord-B provision never wipes coord A's table (and vice versa). Every
+ * forward/postrouting/MSS rule is source-scoped to the bridge subnet so two
+ * tables on the shared hooks do not cross-drop each other's traffic.
+ */
+export declare function buildBridgeCommands(cfg: FirecrackerBridgeConfig): CommandSpec[];
+export declare const NM_CONF_PATH = "/etc/NetworkManager/conf.d/90-kici-unmanaged.conf";
+export declare const NM_CONF_CONTENT: string;
+export type CommandRunner = (spec: CommandSpec) => Promise<{
+    stdout: string;
+}>;
+/** Writes the host-scoped NetworkManager conf. Injectable for tests. */
+export type FileWriter = (path: string, content: string) => Promise<void>;
+export interface ExecOptions {
+    /** Inject a runner for tests. */
+    runner?: CommandRunner;
+    /** Inject the NM-conf file writer for tests. */
+    writeNmConf?: FileWriter;
+    /** Wrap privileged bins with `sudo -n` (non-root orchestrator hosts). */
+    requireSudo?: boolean;
+}
+/** Resolve the default-route egress interface. */
+export declare function resolveHostIface(opts?: ExecOptions): Promise<string>;
+export interface BridgeHealth {
+    bridgeName: string;
+    bridgeExists: boolean;
+    bridgeUp: boolean;
+    addrPresent: boolean;
+    tablePresent: boolean;
+    healthy: boolean;
+    detail: string;
+}
+/** Provision (or heal) one Firecracker host bridge. Throws on any failure. */
+export declare function provisionBridge(cfg: FirecrackerBridgeConfig, opts?: ExecOptions): Promise<void>;
+/** Read-only health probe for one bridge. Never throws on a missing resource. */
+export declare function verifyBridge(cfg: FirecrackerBridgeConfig, opts?: ExecOptions): Promise<BridgeHealth>;
+/** Remove the bridge + its nft table. Leaves the host-scoped NM conf in place. */
+export declare function teardownBridge(cfg: FirecrackerBridgeConfig, opts?: ExecOptions): Promise<void>;
+/** Serialize the provisioning command list into a dependency-free boot script. */
+export declare function renderBootScript(cfg: FirecrackerBridgeConfig): string;
+//# sourceMappingURL=host-network.d.ts.map

package/dist/firecracker/persist.d.ts

@@ -0,0 +1,14 @@
+import { type FirecrackerBridgeConfig } from './host-network.js';
+export declare const BOOT_SCRIPT_DIR = "/usr/local/lib/kici-firecracker";
+export declare const SYSTEMD_DIR = "/etc/systemd/system";
+export declare function persistUnitName(bridgeName: string): string;
+export declare function renderPersistUnit(bridgeName: string): string;
+/**
+ * Install the boot script + unit and `systemctl enable` it (enable only — the
+ * deploy/CLI provisions the bridge live in the same run, so the unit fires on
+ * the NEXT boot rather than re-churning nft now).
+ */
+export declare function installPersist(cfg: FirecrackerBridgeConfig, opts?: {
+    requireSudo?: boolean;
+}): Promise<void>;
+//# sourceMappingURL=persist.d.ts.map

package/dist/index.d.ts

@@ -3,5 +3,6 @@ export { S3CacheStorage, type S3CacheStorageOptions } from './storage/s3.js';
 export { createCacheStorage } from './storage/index.js';
 export { SourceCache } from './cache/source-cache.js';
 export { DepCache } from './cache/dep-cache.js';
+export { UserCache, DEFAULT_USER_CACHE_QUOTA_BYTES, DEFAULT_USER_CACHE_TTL_MS, type UserCacheRef, type UserCacheRestoreResult, type UserCacheBeginSaveResult, } from './cache/user-cache.js';
 export { PeerCredentialStore, createPeerCredentialStoreFromUrl, type PeerCredential, } from './cluster/peer-credentials.js';
 //# sourceMappingURL=index.d.ts.map