@kici-dev/orchestrator 0.1.15 → 0.1.16

package/dist/cluster/peer-client.d.ts

@@ -10,7 +10,7 @@
  * Authentication uses ECDH key exchange followed by join token (first connect)
  * or HMAC credential proof (reconnection).
  */
-import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
+import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
 import type { PeerRegistry } from './peer-registry.js';
 type PeerConnectionState = 'disconnected' | 'connecting' | 'handshaking' | 'authenticating' | 'connected';
 export interface PeerClientOptions {
@@ -78,7 +78,20 @@ export interface PeerClientOptions {
      * Platform-mediated discovery dedupes against the same client.
      */
     onAuthenticated?: (targetInstanceId: string) => void;
+    /**
+     * Callback when a peer.logs.collect.request is received from the peer. Builds
+     * this node's subtree bundle and streams it back via the supplied `send`
+     * (peer.logs.collect.chunk frames, or a peer.logs.collect.error on failure).
+     * If undefined, incoming collect requests are ignored.
+     */
+    onLogsCollectRequest?: PeerLogsCollectResponder;
 }
+/**
+ * Builds a node's subtree bundle in response to a peer.logs.collect.request and
+ * streams it back through `send`. Shared by PeerClient (outgoing-dialed peers)
+ * and the peer handler (incoming-dialed peers).
+ */
+export type PeerLogsCollectResponder = (msg: PeerLogsCollectRequest, send: (out: PeerToPeerMessage) => boolean) => Promise<void>;
 export declare class PeerClient {
     private ws;
     private _state;
@@ -91,6 +104,8 @@ export declare class PeerClient {
     private readonly ackWaiters;
     private readonly cacheWaiters;
     private readonly configReloadWaiters;
+    /** Correlates peer.logs.collect.request with the peer's chunked subtree response. */
+    private readonly logsCollectWaiters;
     private readonly url;
     private readonly joinToken?;
     private readonly credentialFile;
@@ -112,6 +127,7 @@ export declare class PeerClient {
     private readonly onAgentTokenRevoke?;
     private readonly onPeerConfigReload?;
     private readonly onAuthenticated?;
+    private readonly onLogsCollectRequest?;
     constructor(options: PeerClientOptions);
     /** Current connection state. */
     get state(): PeerConnectionState;
@@ -150,6 +166,12 @@ export declare class PeerClient {
      *   if the peer doesn't reply within the timeout.
      */
     sendConfigReloadAndWait(msg: PeerConfigReload, timeoutMs?: number): Promise<PeerConfigReloadResponse | null>;
+    /**
+     * Send a peer.logs.collect.request to the connected peer and await its
+     * reassembled subtree-bundle ZIP. Rejects on timeout, an error frame, or
+     * peer disconnect.
+     */
+    sendLogsCollectAndWait(msg: PeerLogsCollectRequest, timeoutMs: number): Promise<Buffer>;
     /**
      * Send a cache upload request to the coordinator and wait for a response
      * with a pre-signed URL.

package/dist/cluster/peer-handler.d.ts

@@ -6,7 +6,7 @@
  * proof (reconnection), registers the peer in PeerRegistry, and routes
  * messages bidirectionally. Sends periodic heartbeats to the connecting peer.
  */
-import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type PeerScalerEvent, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
+import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type PeerScalerEvent, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
 import type { PeerRegistry } from './peer-registry.js';
 import type { PeerCredentialStore } from './peer-credentials.js';
 import { type JoinTokenManager } from './join-token.js';
@@ -78,6 +78,13 @@ export interface PeerHandlerDeps {
         restartRequired?: string[];
         fieldsChanged?: string[];
     }>;
+    /**
+     * Callback when a peer.logs.collect.request arrives from an incoming-dialed
+     * peer. Builds this node's subtree bundle and streams it back through `send`
+     * (peer.logs.collect.chunk frames, or a peer.logs.collect.error on failure).
+     * If undefined, incoming collect requests are ignored.
+     */
+    onLogsCollectRequest?: (msg: PeerLogsCollectRequest, send: (out: PeerToPeerMessage) => boolean) => Promise<void>;
 }
 /**
  * Create a handler function for incoming peer WebSocket connections.
@@ -90,6 +97,7 @@ export declare function createPeerHandler(deps: PeerHandlerDeps): {
     sendToPeer: (targetInstanceId: string, msg: PeerToPeerMessage) => boolean;
     sendAndWaitAck: (targetInstanceId: string, msg: JobReroute, timeoutMs?: number) => Promise<boolean>;
     sendConfigReloadAndWait: (targetInstanceId: string, msg: PeerConfigReload, timeoutMs?: number) => Promise<PeerConfigReloadResponse | null>;
+    sendLogsCollectAndWait: (targetInstanceId: string, msg: PeerLogsCollectRequest, timeoutMs: number) => Promise<Buffer>;
     getConnectionCount: () => number;
     broadcastHeartbeat: (inventory: Omit<PeerHeartbeat, "type">) => void;
     broadcastAgentTokenRevoke: (msg: PeerAgentTokenRevoke) => void;

package/dist/db/migrations/033_org_settings_approval.d.ts

@@ -0,0 +1,19 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add the two approval-policy columns to `org_settings`:
+ *
+ * - `approval_expiry_seconds INTEGER NOT NULL DEFAULT 86400` — how long a held
+ *   approval element waits before it expires (and its run/job/step is
+ *   rejected). One day by default. An SDK `requireApproval` `timeout` overrides
+ *   this per element; otherwise this per-org value applies.
+ * - `allow_self_approval BOOLEAN NOT NULL DEFAULT true` — whether the user who
+ *   triggered a run may also approve its held elements. Operators turn it off
+ *   to enforce four-eyes review.
+ *
+ * Both are cluster-configurable per org via `kici-admin org-settings approval`
+ * and the orchestrator admin route. Idempotent: a re-run on a DB that already
+ * has the columns is a no-op (each column is guarded independently).
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=033_org_settings_approval.d.ts.map

package/dist/db/migrations/034_held_runs_generalize.d.ts

@@ -0,0 +1,24 @@
+import { type Kysely } from 'kysely';
+/**
+ * Generalize `held_runs` from an environment-only hold into the unified
+ * "held element" model that backs per-element approvals, and add the
+ * `held_run_approvals` table that records each approver's decision.
+ *
+ * New `held_runs` columns (all idempotent, column-exists guarded):
+ * - `hold_scope text NOT NULL DEFAULT 'job'` — 'workflow' | 'job' | 'step'
+ *   (engine `HoldScope`). Existing rows held a single job, so they default to
+ *   'job'.
+ * - `step_index integer` — nullable; set only for step-scoped holds.
+ * - `trigger_source text NOT NULL DEFAULT 'environment'` — 'environment' |
+ *   'explicit' (engine `TriggerSource`). Existing holds came from environment
+ *   protection, so they default to 'environment'.
+ * - `approval_requirement jsonb` — the normalized `ApprovalRequirement`
+ *   (clauses + expiresAt + reason) the hold must satisfy. Nullable for legacy
+ *   rows that predate the approval model.
+ *
+ * New `held_run_approvals` table: one row per approver decision, FK to
+ * `held_runs.id` (uuid) with ON DELETE CASCADE.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=034_held_runs_generalize.d.ts.map

package/dist/db/types.d.ts

@@ -1,5 +1,5 @@
 import type { ColumnType, Generated, Insertable, Selectable, Updateable } from 'kysely';
-import type { InitFailure } from '@kici-dev/engine';
+import type { ApprovalRequirement, ApproverClause, InitFailure } from '@kici-dev/engine';
 /**
  * PostgreSQL-only database types.
  * Column names use snake_case matching the actual database column names.
@@ -19,6 +19,7 @@ export interface Database {
     environment_variables: EnvironmentVariablesTable;
     environment_source_overrides: EnvironmentSourceOverridesTable;
     held_runs: HeldRunsTable;
+    held_run_approvals: HeldRunApprovalsTable;
     admin_tokens: AdminTokenTable;
     agent_tokens: AgentTokenTable;
     config_versions: ConfigVersionTable;
@@ -630,10 +631,49 @@ export interface HeldRunsTable {
     expires_at: Date;
     /** When this hold was resolved */
     resolved_at: Date | null;
+    /**
+     * Hold granularity: 'workflow' | 'job' | 'step' (engine `HoldScope`).
+     * Existing environment holds are job-scoped, hence the 'job' default.
+     */
+    hold_scope: Generated<string>;
+    /** Step index within the job for step-scoped holds; null otherwise. */
+    step_index: number | null;
+    /**
+     * What created the hold: 'environment' (mandatory env policy) | 'explicit'
+     * (SDK `requireApproval`). Engine `TriggerSource`.
+     */
+    trigger_source: Generated<string>;
+    /**
+     * Normalized `ApprovalRequirement` (clauses + expiresAt + reason) the hold
+     * must satisfy. Null for legacy rows that predate the approval model.
+     */
+    approval_requirement: ColumnType<ApprovalRequirement | null, ApprovalRequirement | string | null | undefined, ApprovalRequirement | string | null>;
 }
 export type HeldRun = Selectable<HeldRunsTable>;
 export type NewHeldRun = Insertable<HeldRunsTable>;
 export type HeldRunUpdate = Updateable<HeldRunsTable>;
+/**
+ * One approver's recorded decision on a held element. Multiple rows accumulate
+ * until the hold's `ApprovalRequirement` clauses are all satisfied (approve) or
+ * any single reject lands.
+ */
+export interface HeldRunApprovalsTable {
+    /** UUID primary key */
+    id: Generated<string>;
+    /** FK to held_runs.id (ON DELETE CASCADE) */
+    held_run_id: string;
+    /** The approver's user id (Keycloak sub) */
+    approver_user_id: string;
+    /** 'approve' | 'reject' (engine `ApprovalDecision`) */
+    decision: string;
+    /** Which requirement clauses this decision satisfied (for attribution). */
+    clauses_satisfied: ColumnType<ApproverClause[] | null, ApproverClause[] | string | null | undefined, ApproverClause[] | string | null>;
+    /** When the decision was recorded */
+    created_at: Generated<Date>;
+}
+export type HeldRunApproval = Selectable<HeldRunApprovalsTable>;
+export type NewHeldRunApproval = Insertable<HeldRunApprovalsTable>;
+export type HeldRunApprovalUpdate = Updateable<HeldRunApprovalsTable>;
 /**
  * Secret audit log table
  * Immutable log of secret access and denial events.
@@ -1192,6 +1232,17 @@ export interface OrgSettingsTable {
      * BIGINT — pg returns a string on select; accept a number on insert/update.
      */
     dispatch_ack_timeout_ms: ColumnType<string | null, number | null | undefined, number | null>;
+    /**
+     * Per-org expiry (seconds) for a held approval element before it is rejected
+     * and its run/job/step fails. NOT NULL, default 86400 (one day). An SDK
+     * `requireApproval` `timeout` overrides this per element.
+     */
+    approval_expiry_seconds: ColumnType<number, number | undefined, number>;
+    /**
+     * Whether the user who triggered a run may also approve its held elements.
+     * NOT NULL, default true. Operators turn it off to enforce four-eyes review.
+     */
+    allow_self_approval: ColumnType<boolean, boolean | undefined, boolean>;
     /** When this setting was created */
     created_at: Generated<Date>;
     /** When this setting was last updated */

package/dist/diagnostics/bundle-writer.d.ts

@@ -6,8 +6,8 @@
  * This is the primary support tool -- operators run debug-bundle and share
  * the ZIP for troubleshooting.
  */
-import archiver from 'archiver';
 import type { DiagnosticDeps } from './types.js';
+export { redactConfig, addLogsToArchive } from '@kici-dev/shared';
 export interface BundleOptions {
     /** Where to write the ZIP. */
     outputPath: string;
@@ -26,11 +26,6 @@ export interface BundleOptions {
     /** URL for recent runs endpoint. */
     recentRunsUrl?: string;
 }
-/**
- * Redact config values using allowlist approach.
- * Only known-safe fields are preserved; everything else becomes "****".
- */
-export declare function redactConfig(obj: unknown, parentKey?: string): unknown;
 /**
  * Create a debug bundle ZIP file.
  *
@@ -43,15 +38,4 @@ export declare function redactConfig(obj: unknown, parentKey?: string): unknown;
  * - logs/summary.json: log statistics
  */
 export declare function createDebugBundle(options: BundleOptions): Promise<string>;
-/**
- * Add log files from logDir to the archive, respecting MAX_LOG_BYTES cap
- * and the logWindow time filter. Matches any `*.log` file in the directory,
- * so the per-instance filename pattern produced by
- * `buildLogFilename()` is picked up without additional configuration.
- *
- * Exported for reuse by the `kici-admin debug-bundle` CLI command, which
- * runs outside the orchestrator process but still needs to include the
- * same log files in its locally-assembled bundle.
- */
-export declare function addLogsToArchive(archive: archiver.Archiver, logDir: string, logWindowHours: number): Promise<void>;
 //# sourceMappingURL=bundle-writer.d.ts.map

package/dist/diagnostics/checks/firecracker-network.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Firecracker host-bridge diagnostic. Emits one row per configured Firecracker
+ * scaler backend: the bridge interface + its gateway addr + its nft table must
+ * all be present, else the scaler cannot spawn microVMs.
+ */
+import type { DiagnosticDeps, DiagnosticResult } from '../types.js';
+import { type BridgeHealth, type FirecrackerBridgeConfig } from '../../firecracker/host-network.js';
+interface CheckOpts {
+    verify?: (cfg: FirecrackerBridgeConfig) => Promise<BridgeHealth>;
+}
+export declare function checkFirecrackerNetwork(deps: DiagnosticDeps, opts?: CheckOpts): Promise<DiagnosticResult[]>;
+export {};
+//# sourceMappingURL=firecracker-network.d.ts.map

package/dist/diagnostics/checks/index.d.ts

@@ -12,7 +12,8 @@ import { checkDiskSpace } from './disk.js';
 import { checkConfigValidity } from './config.js';
 import { checkCertificateExpiry } from './certs.js';
 import { checkScalerProvisioning } from './scaler.js';
+import { checkFirecrackerNetwork } from './firecracker-network.js';
 /** All diagnostic checks in display order. */
 export declare const defaultChecks: DiagnosticCheck[];
-export { checkDbConnectivity, checkWsToPlatform, checkAgentConnectivity, checkDiskSpace, checkConfigValidity, checkCertificateExpiry, checkScalerProvisioning, };
+export { checkDbConnectivity, checkWsToPlatform, checkAgentConnectivity, checkDiskSpace, checkConfigValidity, checkCertificateExpiry, checkScalerProvisioning, checkFirecrackerNetwork, };
 //# sourceMappingURL=index.d.ts.map

package/dist/diagnostics/fleet-collector.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Recursive fleet subtree assembler.
+ *
+ * Every orchestrator runs this to produce a self-similar subtree ZIP:
+ * local/ + nested agents/<id>.zip + workers/<id>.zip + peers/<id>.zip, plus
+ * fleet-manifest.json recording per-node status. Dead branches are recorded,
+ * never fatal (Promise.allSettled at each level). The loop guard
+ * (includeCoordinatorMesh) is true only on the root call; every downstream peer
+ * request sets it false so the coordinator mesh never echoes.
+ */
+import { z } from 'zod';
+/** Per-node collection outcome recorded in the fleet manifest. */
+export declare const FleetNodeStatus: z.ZodEnum<{
+    error: "error";
+    ok: "ok";
+    timeout: "timeout";
+    unreachable: "unreachable";
+}>;
+export type FleetNodeStatus = z.infer<typeof FleetNodeStatus>;
+export interface FleetCollectorDeps {
+    /** This orchestrator's instanceId (the manifest owner). */
+    instanceId: string;
+    /** Build this node's own debug bundle as a Buffer (createDebugBundle output). */
+    buildLocalBundle: () => Promise<Buffer>;
+    /** List this node's directly-connected agents. */
+    listAgents: () => {
+        agentId: string;
+    }[];
+    /** Request a connected agent's mini-bundle over the agent WS channel. */
+    requestAgentBundle: (agentId: string) => Promise<Buffer>;
+    /** List this node's downstream peers (coordinator-mesh peers + its workers). */
+    listPeers: () => {
+        instanceId: string;
+        role: 'coordinator' | 'worker';
+        kind: 'peer' | 'worker';
+    }[];
+    /** Request a peer's subtree bundle over the peer WS channel (loop-guarded false). */
+    requestPeerSubtree: (instanceId: string, includeCoordinatorMesh: boolean) => Promise<Buffer>;
+}
+export interface FleetCollectOptions {
+    logWindowHours: number;
+    selection: {
+        all: boolean;
+        agentIds: string[];
+        workerInstanceIds: string[];
+    };
+    /** Loop guard. True only on the root call; downstream requests force false. */
+    includeCoordinatorMesh: boolean;
+    timeoutMs: number;
+}
+export declare function collectFleetSubtree(opts: FleetCollectOptions, deps: FleetCollectorDeps): Promise<Buffer>;
+//# sourceMappingURL=fleet-collector.d.ts.map

package/dist/diagnostics/fleet-constants.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Shared constants for fleet log collection.
+ */
+/** Default per-node deadline for a fleet bundle request (overridable via --fleet-timeout). */
+export declare const FLEET_NODE_TIMEOUT_MS = 60000;
+/** Per-node cap on raw log bytes an agent includes in its mini-bundle (50 MiB). */
+export declare const FLEET_MAX_LOG_BYTES: number;
+//# sourceMappingURL=fleet-constants.d.ts.map

package/dist/diagnostics/fleet-selection.d.ts

@@ -0,0 +1,15 @@
+import type { FleetSelection } from '@kici-dev/engine';
+import type { FleetTopology } from './fleet-topology.js';
+/** A per-orchestrator selection, keyed by orchestrator instanceId. Absent = prune. */
+export type ResolvedSelectionMap = Map<string, FleetSelection>;
+/**
+ * Resolve `--pick` selectors into a per-orchestrator FleetSelection map.
+ *
+ * No selectors -> every orchestrator gets `{ all: true }`. With selectors, an
+ * orchestrator is included only if at least one of its agents/workers (or the
+ * orchestrator itself) matches; its FleetSelection then carries exactly the
+ * matched agent ids and worker instanceIds. Orchestrators with no matches are
+ * omitted from the map (their branch is pruned).
+ */
+export declare function resolveSelection(topology: FleetTopology, selectors: string[]): ResolvedSelectionMap;
+//# sourceMappingURL=fleet-selection.d.ts.map

package/dist/diagnostics/fleet-topology.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Fleet topology enumeration.
+ *
+ * Builds the cluster tree for `debug-bundle --fleet --list` / `--pick` from the
+ * local agent registry plus the heartbeat-cached peer inventory — no fan-out.
+ * Each peer's cached agent list is already maintained by ~30s heartbeats, so
+ * enumeration is a cheap in-memory read.
+ */
+/** A node in the enumerated fleet topology. */
+export interface FleetTopologyNode {
+    kind: 'orchestrator' | 'agent';
+    id: string;
+    /** Cluster role for orchestrator nodes (coordinator | worker); undefined for agents. */
+    role?: 'coordinator' | 'worker';
+    hostname?: string;
+    labels: Record<string, string>;
+    /** Parent orchestrator instanceId; null for the collector (root) node. */
+    parentId: string | null;
+}
+export interface FleetTopology {
+    nodes: FleetTopologyNode[];
+}
+export interface FleetTopologyDeps {
+    /** This collector's own instanceId (the root orchestrator). */
+    instanceId: string;
+    /** This collector's role. */
+    role: 'coordinator' | 'worker';
+    /** This collector's hostname, if known. */
+    hostname?: string;
+    /** This node's directly-connected agents. */
+    listLocalAgents: () => {
+        agentId: string;
+        labels: string[];
+    }[];
+    /** The heartbeat-cached peer inventory (each peer + its cached agents). */
+    listPeers: () => {
+        instanceId: string;
+        role: 'coordinator' | 'worker';
+        hostname?: string;
+        agents: {
+            agentId: string;
+            labels: string[];
+        }[];
+    }[];
+}
+export declare function buildFleetTopology(deps: FleetTopologyDeps): FleetTopology;
+//# sourceMappingURL=fleet-topology.d.ts.map

package/dist/diagnostics/fleet-wiring.d.ts

@@ -0,0 +1,60 @@
+import type { PeerLogsCollectRequest, PeerToPeerMessage, FleetSelection } from '@kici-dev/engine';
+import type { AgentRegistry } from '../agent/registry.js';
+import type { PeerRegistry } from '../cluster/peer-registry.js';
+import type { PeerClient } from '../cluster/peer-client.js';
+import type { DiagnosticDeps } from './types.js';
+import { type FleetCollectorDeps } from './fleet-collector.js';
+import { type FleetTopology, type FleetTopologyDeps } from './fleet-topology.js';
+/** A peer-handler-like object exposing the dual-direction collect send. */
+export interface FleetPeerHandlerLike {
+    sendLogsCollectAndWait: (targetInstanceId: string, msg: PeerLogsCollectRequest, timeoutMs: number) => Promise<Buffer>;
+}
+/** A collector that issues a fleet.logs.request to an agent and awaits its bundle. */
+export interface FleetAgentRequester {
+    request: (requestId: string, agentId: string, send: () => void) => Promise<Buffer>;
+}
+export interface FleetRuntime {
+    instanceId: string;
+    role: 'coordinator' | 'worker';
+    /** Loop window for log files (hours). */
+    logWindowHours: number;
+    /** Per-node deadline (ms). */
+    timeoutMs: number;
+    /** Local agent log directory (KICI_LOG_DIR), if configured. */
+    logDir?: string;
+    agentRegistry: AgentRegistry;
+    peerRegistry: PeerRegistry;
+    fleetAgentCollector: FleetAgentRequester;
+    /** Outgoing peer clients, keyed by instanceId. */
+    peerClients: Map<string, PeerClient>;
+    /** Incoming peer-handler with the dual-direction collect send. */
+    peerHandler: FleetPeerHandlerLike;
+    /** Deps for the local createDebugBundle. */
+    diagnosticDeps: DiagnosticDeps;
+    /** Raw orchestrator config (redacted by createDebugBundle). */
+    config: Record<string, unknown>;
+    /** Cluster health endpoint for the local bundle's cluster/health.json. */
+    clusterHealthUrl?: string;
+}
+/**
+ * Build the FleetCollectorDeps for a given selection of this node's downstream
+ * agents/workers. The collector passes per-branch selection in when it knows
+ * which downstream subset each peer should gather.
+ */
+export declare function buildFleetCollectorDeps(runtime: FleetRuntime, perBranchSelection: (instanceId: string) => FleetSelection): FleetCollectorDeps;
+/** Build the FleetTopologyDeps for `--list` / `--pick` enumeration. */
+export declare function buildFleetTopologyDeps(runtime: FleetRuntime): FleetTopologyDeps;
+/** Enumerate this node's fleet topology (no fan-out). */
+export declare function getFleetTopology(runtime: FleetRuntime): FleetTopology;
+/**
+ * Collect this node's full subtree (root call: includeCoordinatorMesh=true) with
+ * an optional per-orchestrator selection map. Absent map entry => collect all.
+ */
+export declare function collectFleet(runtime: FleetRuntime, selectionByOrch: Map<string, FleetSelection> | null): Promise<Buffer>;
+/**
+ * Peer-side responder: on an inbound peer.logs.collect.request, assemble this
+ * node's subtree (with the request's loop guard + selection) and stream it back
+ * as peer.logs.collect.chunk frames, or a peer.logs.collect.error on failure.
+ */
+export declare function makeFleetCollectResponder(runtime: FleetRuntime): (msg: PeerLogsCollectRequest, send: (out: PeerToPeerMessage) => boolean) => Promise<void>;
+//# sourceMappingURL=fleet-wiring.d.ts.map

package/dist/environments/held-runs.d.ts

@@ -4,7 +4,8 @@
  * Manages the lifecycle: pending -> approved/rejected/expired.
  */
 import { type Kysely } from 'kysely';
-import type { Database, HeldRun } from '../db/types.js';
+import { type ApprovalRequirement, type ApproverClause, ApprovalDecision, HoldScope, TriggerSource } from '@kici-dev/engine';
+import type { Database, HeldRun, HeldRunApproval } from '../db/types.js';
 /** Status values for held runs (held_runs table). */
 export declare enum HeldRunStatus {
     Pending = "pending",
@@ -24,6 +25,47 @@ export interface CreateHeldRunData {
     /** Queue type: 'environment' (default) or 'security'. */
     queueType?: 'environment' | 'security';
 }
+/**
+ * Data required to create a generalized approval hold. Unlike the legacy
+ * environment-only `create()`, this carries the hold scope, trigger source,
+ * optional step index, and the normalized approval requirement.
+ */
+export interface CreateHoldData {
+    runId: string;
+    jobId: string;
+    /** Granularity of the held element. */
+    scope: HoldScope;
+    /** Step index within the job for step-scoped holds; omit otherwise. */
+    stepIndex?: number;
+    /** What triggered the hold (environment policy vs SDK requireApproval). */
+    triggerSource: TriggerSource;
+    /** The normalized requirement the hold must satisfy. */
+    requirement: ApprovalRequirement;
+    /** Environment id, when the hold originates from an environment policy. */
+    environmentId?: string | null;
+    /** Queue type: 'environment' (default) or 'security'. */
+    queueType?: 'environment' | 'security';
+}
+/** A single decision to record against a hold. */
+export interface RecordDecisionData {
+    approverSub: string;
+    decision: ApprovalDecision;
+    /** Which requirement clauses this decision satisfied (for attribution). */
+    clausesSatisfied?: ApproverClause[];
+}
+/**
+ * The outcome of `release()` — describes how the held element must be resumed.
+ * The store only writes the terminal DB state; the caller performs the actual
+ * re-dispatch (job/workflow) or agent notification (step) using this signal.
+ */
+export interface ReleaseSignal {
+    holdId: string;
+    runId: string;
+    jobId: string;
+    scope: HoldScope;
+    /** Set only for step-scoped holds. */
+    stepIndex: number | null;
+}
 /** Options for listing held runs. */
 export interface ListHeldRunsOptions {
     status?: string;
@@ -35,6 +77,26 @@ export declare class HeldRunStore {
     constructor(db: Kysely<Database>);
     /** Create a new held run with pending status. */
     create(orgId: string, data: CreateHeldRunData): Promise<HeldRun>;
+    /**
+     * Create a generalized approval hold (workflow/job/step scope, explicit or
+     * environment trigger) carrying a normalized `ApprovalRequirement`. Returns
+     * the created row.
+     */
+    createHold(orgId: string, data: CreateHoldData): Promise<HeldRun>;
+    /** Record one approve/reject decision against a hold. */
+    recordDecision(heldRunId: string, data: RecordDecisionData): Promise<HeldRunApproval>;
+    /** List the recorded decisions for a hold, oldest first. */
+    listDecisions(heldRunId: string): Promise<HeldRunApproval[]>;
+    /** Get a single held run by id (org-scoped). Returns null if absent. */
+    getById(orgId: string, heldRunId: string): Promise<HeldRun | null>;
+    /**
+     * Release a hold whose approval requirement is satisfied. Flips the row to
+     * 'approved' and returns a `ReleaseSignal` describing how the caller must
+     * resume the element (re-dispatch for job/workflow, agent notification for
+     * step). Throws if the hold is not found or not pending. Approver attribution
+     * lives in `held_run_approvals`, not on the row.
+     */
+    release(orgId: string, heldRunId: string): Promise<ReleaseSignal>;
     /** Approve a pending held run. Throws if not found or not pending. */
     approve(orgId: string, heldRunId: string, approvedBy: string): Promise<HeldRun>;
     /** Reject a pending held run. Throws if not found or not pending. */
@@ -53,6 +115,12 @@ export declare class HeldRunStore {
     approveByQueueType(orgId: string, heldRunId: string, approvedBy: string, queueType: 'environment' | 'security'): Promise<HeldRun>;
     /** Get a held run by run ID and job ID. Returns null if not found. */
     getByRunAndJob(orgId: string, runId: string, jobId: string): Promise<HeldRun | null>;
+    /**
+     * List pending holds past their `expires_at`. Called by the stale detector
+     * BEFORE `expireOverdue()` so it can route each overdue hold by scope (step
+     * holds notify the waiting agent; job/workflow holds fail the run).
+     */
+    listOverdue(): Promise<HeldRun[]>;
     /**
      * Expire overdue pending runs. Called by the stale detector.
      * Sets status to 'expired' and resolved_at to now() for all

package/dist/firecracker/host-network.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Firecracker host-network provisioning.
+ *
+ * Creates the per-coordinator bridge (kici-brN), assigns its gateway IP,
+ * marks kici-* interfaces unmanaged by NetworkManager, enables IP forwarding,
+ * and builds a disjoint, source-scoped nftables table for NAT + egress
+ * isolation. One pure command-builder drives live provisioning, the rendered
+ * boot script, and (read-only) verification.
+ *
+ * This is HOST setup, distinct from the runtime per-VM isolation in
+ * scaler/nftables.ts (added at spawn / removed at destroy). The two share the
+ * nft table name but have separate lifecycles; this module never touches the
+ * per-VM rules.
+ */
+/** A single subprocess invocation (no shell). */
+export interface CommandSpec {
+    bin: string;
+    args: string[];
+    /** Optional stdin payload (used for `nft -f -`). */
+    stdin?: string;
+}
+/** Host-bridge configuration for one Firecracker coordinator. */
+export interface FirecrackerBridgeConfig {
+    /** Bridge interface name, e.g. 'kici-br0'. */
+    bridgeName: string;
+    /** Gateway IP + prefix, e.g. '10.0.0.1/24'. */
+    bridgeCidr: string;
+    /** nft table name, e.g. 'kici' or 'kici_b'. */
+    table: string;
+    /** NAT egress interface; auto-detected from the default route when omitted. */
+    hostIface?: string;
+}
+/**
+ * Derive the network address (CIDR) from a gateway CIDR by masking host bits.
+ * '10.0.0.1/24' -> '10.0.0.0/24'.
+ */
+export declare function cidrToNetwork(cidr: string): string;
+/**
+ * Build the ordered command list that provisions one Firecracker host bridge.
+ * Pure — performs no I/O. `provisionBridge` executes these; `renderBootScript`
+ * serializes them.
+ *
+ * The nft `delete table`/`add table` here only ever touches `cfg.table`, so a
+ * coord-B provision never wipes coord A's table (and vice versa). Every
+ * forward/postrouting/MSS rule is source-scoped to the bridge subnet so two
+ * tables on the shared hooks do not cross-drop each other's traffic.
+ */
+export declare function buildBridgeCommands(cfg: FirecrackerBridgeConfig): CommandSpec[];
+export declare const NM_CONF_PATH = "/etc/NetworkManager/conf.d/90-kici-unmanaged.conf";
+export declare const NM_CONF_CONTENT: string;
+export type CommandRunner = (spec: CommandSpec) => Promise<{
+    stdout: string;
+}>;
+/** Writes the host-scoped NetworkManager conf. Injectable for tests. */
+export type FileWriter = (path: string, content: string) => Promise<void>;
+export interface ExecOptions {
+    /** Inject a runner for tests. */
+    runner?: CommandRunner;
+    /** Inject the NM-conf file writer for tests. */
+    writeNmConf?: FileWriter;
+    /** Wrap privileged bins with `sudo -n` (non-root orchestrator hosts). */
+    requireSudo?: boolean;
+}
+/** Resolve the default-route egress interface. */
+export declare function resolveHostIface(opts?: ExecOptions): Promise<string>;
+export interface BridgeHealth {
+    bridgeName: string;
+    bridgeExists: boolean;
+    bridgeUp: boolean;
+    addrPresent: boolean;
+    tablePresent: boolean;
+    healthy: boolean;
+    detail: string;
+}
+/** Provision (or heal) one Firecracker host bridge. Throws on any failure. */
+export declare function provisionBridge(cfg: FirecrackerBridgeConfig, opts?: ExecOptions): Promise<void>;
+/** Read-only health probe for one bridge. Never throws on a missing resource. */
+export declare function verifyBridge(cfg: FirecrackerBridgeConfig, opts?: ExecOptions): Promise<BridgeHealth>;
+/** Remove the bridge + its nft table. Leaves the host-scoped NM conf in place. */
+export declare function teardownBridge(cfg: FirecrackerBridgeConfig, opts?: ExecOptions): Promise<void>;
+/** Serialize the provisioning command list into a dependency-free boot script. */
+export declare function renderBootScript(cfg: FirecrackerBridgeConfig): string;
+//# sourceMappingURL=host-network.d.ts.map