@kici-dev/orchestrator 0.1.15 → 0.1.16

package/dist/app.d.ts

@@ -48,6 +48,8 @@ import type { PeerToPeerMessage } from '@kici-dev/engine';
 import type { AgentTokenStore } from './agent/token-store.js';
 import type { OwnershipTracker } from './agent/ownership-tracker.js';
 import type { ObserverRegistry } from './ws/observer-registry.js';
+import type { FleetAgentCollector } from './ws/fleet-agent-collector.js';
+import type { FleetTopology } from './diagnostics/fleet-topology.js';
 import type { TokenManager } from './secrets/token-manager.js';
 import type { SecretResolver } from './secrets/secret-resolver.js';
 import { type AdminRouteDeps } from './routes/admin.js';
@@ -68,6 +70,7 @@ import type { TrustStore } from './events/trust-store.js';
 import type { EnvironmentStore } from './environments/environment-store.js';
 import type { VariableStore } from './environments/variable-store.js';
 import type { HeldRunStore } from './environments/held-runs.js';
+import type { StepApprovalBridge } from './approvals/step-approval-bridge.js';
 import type { ContributorCache } from './security/contributor-cache.js';
 import { AgentMetricsAggregator } from './metrics/agent-metrics-aggregator.js';
 /**
@@ -132,6 +135,11 @@ export interface AppDependencies {
     tokenStore?: AgentTokenStore;
     /** Job ownership tracker for validating agent messages. Optional — requires database. */
     ownershipTracker?: OwnershipTracker;
+    /**
+     * Orchestrator-scoped collector correlating fleet.logs.request with agents'
+     * chunked bundle responses. Shared with the fleet fan-out in orchestrator-core.
+     */
+    fleetAgentCollector?: FleetAgentCollector;
     /** Admin API route dependencies. Optional -- only mounted when secrets management is configured. */
     adminDeps?: AdminRouteDeps;
     /**
@@ -170,6 +178,8 @@ export interface AppDependencies {
     variableStore?: VariableStore;
     /** Held run store for persisting protection rule holds. Optional -- if not set, holds are not persisted. */
     heldRunStore?: HeldRunStore;
+    /** Step-approval bridge — opens step-scoped holds and relays their resolution back to the waiting agent. Optional. */
+    stepApprovalBridge?: StepApprovalBridge;
     /** Global workflow policy for org-level permission enforcement. Optional -- if not set, global workflows are dispatched without permission checks. */
     globalWorkflowPolicy?: GlobalWorkflowPolicy;
     /** Inbound webhook delivery log writer. Optional -- if not set, deliveries are not persisted to event_log. */
@@ -215,6 +225,21 @@ export interface AppDependencies {
      * can read the same store and Mimir gets agent metrics per-org.
      */
     agentMetricsAggregator?: AgentMetricsAggregator;
+    /**
+     * Fleet log-collection route backing. `getTopology` enumerates the cluster
+     * (no fan-out); `collectBundle` drives the recursive fan-out and returns the
+     * assembled ZIP. Mounted at /admin/fleet-topology + /admin/fleet-bundle.
+     */
+    fleetRoutes?: FleetRoutesDeps;
+}
+/** Backing for the /admin/fleet-* routes. */
+export interface FleetRoutesDeps {
+    getTopology: () => FleetTopology;
+    collectBundle: (opts: {
+        selectors: string[];
+        logWindowHours?: number;
+        timeoutSeconds?: number;
+    }) => Promise<Buffer>;
 }
 /**
  * In-memory cache for step source locations extracted from lock files.

package/dist/approvals/apply-decision.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Shared approve/reject applier — the single code path behind both the
+ * dashboard WS handler and the `kici` CLI HTTP route, so the authorization +
+ * resume story is identical regardless of the surface.
+ *
+ * Flow for an approve:
+ *   1. Load the hold + its recorded decisions.
+ *   2. Build the team-membership lookup and run `canApprove` (eligibility +
+ *      self-approval gate). Ineligible actors are rejected with a clear reason.
+ *   3. Record the decision in `held_run_approvals`.
+ *   4. Re-evaluate. If all clauses are satisfied, `release()` the hold and
+ *      resume the element (re-dispatch for job/workflow; the agent bridge for
+ *      step scope — wired by the caller via `onStepRelease`).
+ *
+ * A reject records the decision and `reject()`s the hold (failing the element).
+ */
+import { ApprovalDecision } from '@kici-dev/engine';
+import type { HeldRunStore, ReleaseSignal } from '../environments/held-runs.js';
+import { type TeamMembershipLookup } from './approval-resolver.js';
+/** Outcome of applying a decision. */
+export interface ApplyDecisionResult {
+    /** Whether the decision was accepted (recorded). */
+    accepted: boolean;
+    /** Human-readable status: 'released' | 'rejected' | 'pending' | 'ineligible' | 'not-found'. */
+    status: 'released' | 'rejected' | 'pending' | 'ineligible' | 'not-found';
+    /** When still pending, how many clauses remain unsatisfied. */
+    remainingClauses?: number;
+    /** When ineligible/not-found, a reason for the caller to surface. */
+    reason?: string;
+    /** The release signal, when the element was released (for the caller to resume). */
+    release?: ReleaseSignal;
+}
+/** Dependencies injected into `applyDecision` (testable in isolation). */
+export interface ApplyDecisionDeps {
+    orgId: string;
+    store: HeldRunStore;
+    /** Team name → member user ids (from the Plan-1 trust-policy cache). */
+    teamMembershipLookup: TeamMembershipLookup;
+    /** Whether the run triggerer may self-approve (org_settings.allow_self_approval). */
+    allowSelfApproval: boolean;
+    /** Resolve the Keycloak sub of the user who triggered a run (for the self-approval gate). */
+    resolveTriggererSub: (runId: string) => Promise<string | undefined>;
+    /** Called when a job/workflow hold is released — re-dispatch the element. */
+    onJobRelease: (signal: ReleaseSignal) => Promise<void>;
+    /** Called when a step hold is released — notify the waiting agent (approved). */
+    onStepRelease?: (signal: ReleaseSignal) => Promise<void>;
+    /**
+     * Called when a step-scoped hold is rejected — notify the waiting agent so it
+     * fails the step instead of blocking until expiry. Carries the holdId.
+     */
+    onStepReject?: (heldRunId: string, reason?: string) => Promise<void> | void;
+}
+/** Apply a single approve/reject decision to a hold. */
+export declare function applyDecision(deps: ApplyDecisionDeps, args: {
+    heldRunId: string;
+    actorSub: string;
+    decision: ApprovalDecision;
+    reason?: string;
+}): Promise<ApplyDecisionResult>;
+//# sourceMappingURL=apply-decision.d.ts.map

package/dist/approvals/approval-resolver.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Pure approval-clause evaluator. No DB access, no I/O — given a requirement,
+ * a team-membership lookup, the recorded decisions, and (for eligibility) the
+ * run's triggerer + the self-approval policy, it answers:
+ *
+ * - `evaluate` — is the requirement satisfied (all clauses ANDed), and was it
+ *   rejected (any single reject decision)?
+ * - `canApprove` — may a given actor cast an approve decision right now?
+ *
+ * Both the dashboard handler and the CLI HTTP route funnel through these so the
+ * authorization story is identical regardless of the surface.
+ */
+import type { ApprovalRequirement, ApproverClause } from '@kici-dev/engine';
+/** A team name → set of member user ids (Keycloak subs). */
+export type TeamMembershipLookup = (team: string) => Set<string>;
+/** One recorded decision row (subset of `held_run_approvals`). */
+export interface RecordedDecision {
+    approver_user_id: string;
+    decision: 'approve' | 'reject';
+}
+/** Per-clause satisfaction detail for attribution + dashboard progress. */
+export interface PerClauseResult {
+    clause: ApproverClause;
+    satisfied: boolean;
+    /** The approver who satisfied this clause, if any. */
+    by?: string;
+}
+export interface EvaluationResult {
+    /** All clauses satisfied (AND). For empty clauses: any single approve. */
+    satisfied: boolean;
+    /** Any reject decision present → the element is rejected. */
+    rejected: boolean;
+    perClause: PerClauseResult[];
+}
+/**
+ * Is a single clause satisfied by any approve decision whose approver is
+ * eligible for the clause? Reject decisions never satisfy.
+ */
+export declare function isClauseSatisfied(clause: ApproverClause, decisions: RecordedDecision[], lookup: TeamMembershipLookup): boolean;
+/** Does an actor satisfy/qualify for a clause (team membership or exact user)? */
+export declare function isActorEligibleForClause(actorSub: string, clause: ApproverClause, lookup: TeamMembershipLookup): boolean;
+/**
+ * Evaluate a requirement against the recorded decisions.
+ *
+ * - A single reject decision sets `rejected` (the caller fails the element).
+ * - Empty clauses ⇒ satisfied by ANY single approve decision.
+ * - Non-empty clauses ⇒ satisfied iff every clause is satisfied (AND).
+ */
+export declare function evaluate(requirement: ApprovalRequirement, decisions: RecordedDecision[], lookup: TeamMembershipLookup): EvaluationResult;
+export interface CanApproveContext {
+    /** The Keycloak sub of the user who triggered the run. */
+    triggererSub: string;
+    /** Whether the triggerer may self-approve their own held elements. */
+    allowSelfApproval: boolean;
+}
+/**
+ * May `actorSub` cast an approve decision on this requirement right now?
+ *
+ * - Self-approval gate: when `allowSelfApproval` is false and the actor is the
+ *   run's triggerer, they may never approve.
+ * - Eligibility: the actor must qualify for at least one still-UNSATISFIED
+ *   clause. (Approving a clause that is already satisfied adds nothing.) For an
+ *   empty-clause requirement, any non-triggerer-blocked actor qualifies.
+ */
+export declare function canApprove(actorSub: string, requirement: ApprovalRequirement, decisions: RecordedDecision[], lookup: TeamMembershipLookup, ctx: CanApproveContext): boolean;
+//# sourceMappingURL=approval-resolver.d.ts.map

package/dist/approvals/step-approval-bridge.d.ts

@@ -0,0 +1,72 @@
+import type { HeldRunStore } from '../environments/held-runs.js';
+import type { AccessLogWriter } from '../audit/access-log.js';
+/** Outcome relayed back to the waiting agent. */
+export type StepApprovalOutcome = 'approved' | 'rejected' | 'expired';
+/** Arguments to open a step-scoped approval hold. */
+export interface StepApprovalRequest {
+    agentId: string;
+    runId: string;
+    jobId: string;
+    stepIndex: number;
+    stepName: string;
+    clauses: Array<{
+        team: string;
+    } | {
+        user: string;
+    }>;
+    reason: string;
+    /** Per-gate timeout (seconds); falls back to the org default. */
+    timeoutSeconds?: number;
+}
+/** Dependencies injected into the bridge. */
+export interface StepApprovalBridgeDeps {
+    store: HeldRunStore;
+    /**
+     * Resolve the orchestrator's tenant org id (same value the dashboard handler
+     * uses to read held_runs). A function because the org is resolved post-auth
+     * for sourceless orchestrators.
+     */
+    resolveOrgId: () => string;
+    /** Resolve the authoritative approval expiry (seconds) for an org. */
+    resolveExpirySeconds: (orgId: string) => Promise<number>;
+    /**
+     * Access-log writer for the orchestrator audit stream. Optional -- if not
+     * set, the step-hold creation audit row (`held_run.request`) is skipped.
+     */
+    accessLogWriter?: AccessLogWriter;
+    /** Routing key recorded on the audit row. Optional. */
+    routingKey?: string | null;
+}
+/**
+ * Owns the step-scoped held_runs rows and the map from holdId → the waiting
+ * agent's resolver. The only place that opens a step hold and the only place
+ * that settles a step-scoped wait.
+ */
+export declare class StepApprovalBridge {
+    private readonly deps;
+    private readonly pending;
+    constructor(deps: StepApprovalBridgeDeps);
+    /**
+     * Create a step-scoped hold and return a promise that settles when the hold
+     * is approved / rejected / expired (via {@link resolve}).
+     */
+    request(req: StepApprovalRequest): Promise<{
+        outcome: StepApprovalOutcome;
+        reason?: string;
+    }>;
+    /**
+     * Settle a step-scoped wait. Called by the approve/reject applier
+     * (`onStepRelease`) and by the stale detector (`expired`). A no-op when no
+     * agent is waiting on the hold (e.g. the agent already disconnected).
+     */
+    resolve(holdId: string, outcome: StepApprovalOutcome, reason?: string): boolean;
+    /**
+     * Reject every pending wait for a disconnected agent. The held_runs rows are
+     * left as-is (the stale detector will expire them) — only the in-memory
+     * resolver is dropped so the handler's relayed `ws.send` is skipped.
+     */
+    failAgent(agentId: string): void;
+    /** Number of in-flight step-approval waits (test/diagnostics). */
+    size(): number;
+}
+//# sourceMappingURL=step-approval-bridge.d.ts.map

package/dist/approvals/team-membership-lookup.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Read-only lookup over the team memberships pushed from the Platform via
+ * `trust_policy.update`. The orchestrator has no identity store, so this is the
+ * only source of "who is in team X" — the approval resolver uses it to satisfy
+ * `{team}` approver clauses.
+ */
+export interface TeamMembershipLookup {
+    /** Returns the set of member user ids for a team name (empty if unknown). */
+    getTeamMembers(name: string): Set<string>;
+}
+/** A lookup backed by no teams — the default before any push arrives. */
+export declare const EMPTY_TEAM_MEMBERSHIP_LOOKUP: TeamMembershipLookup;
+//# sourceMappingURL=team-membership-lookup.d.ts.map

package/dist/cache/user-cache.d.ts

@@ -110,7 +110,7 @@ export declare class UserCache {
         tempKey?: string;
     }): Promise<void>;
     private readHash;
-    /** Evict oldest entries for the org until total tarball size <= the per-org quota. */
+    /** Evict least-recently-used entries for the org until total tarball size <= the per-org quota. */
     private enforceQuota;
 }
 //# sourceMappingURL=user-cache.d.ts.map

package/dist/cli/api-client.d.ts

@@ -5,6 +5,19 @@
  * All methods are thin wrappers around fetch() that handle JSON serialization,
  * error formatting, and URL construction.
  */
+/** A node in the fleet topology returned by GET /admin/fleet-topology. */
+export interface FleetTopologyNodeResponse {
+    kind: 'orchestrator' | 'agent';
+    id: string;
+    role?: 'coordinator' | 'worker';
+    hostname?: string;
+    labels: Record<string, string>;
+    parentId: string | null;
+}
+/** Response shape for GET /admin/fleet-topology. */
+export interface FleetTopologyResponse {
+    nodes: FleetTopologyNodeResponse[];
+}
 /**
  * Universal-git configuration shape carried in `git_config` on
  * generic_webhook_sources. Matches `UniversalGitConfigSchema` in
@@ -71,6 +84,17 @@ export declare class AdminApiClient {
      * Public GET request returning raw response text.
      */
     getText(path: string): Promise<string>;
+    /** Enumerate the cluster topology for `debug-bundle --fleet --list` / `--pick`. */
+    getFleetTopology(): Promise<FleetTopologyResponse>;
+    /**
+     * Drive the fleet fan-out and write the assembled ZIP to `outPath`. The
+     * response is an octet-stream, so it is read as bytes rather than parsed JSON.
+     */
+    downloadFleetBundle(body: {
+        selectors: string[];
+        logWindowHours?: number;
+        timeoutSeconds?: number;
+    }, outPath: string): Promise<void>;
     listScopes(orgId: string): Promise<{
         scopes: string[];
     }>;

package/dist/cli/commands/firecracker/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * `kici-admin firecracker` command group.
+ *
+ * Local host-networking ops for the orchestrator's Firecracker scaler:
+ * provision/verify/teardown the per-coordinator bridge + nft table, with
+ * --persist for reboot survival. These run directly on the host (privileged
+ * `ip`/`nft`/`systemctl`) and do NOT use the admin HTTP client.
+ */
+import type { Command } from 'commander';
+export declare function registerFirecrackerCommands(program: Command): void;
+//# sourceMappingURL=index.d.ts.map

package/dist/cli/commands/firecracker/provision.d.ts

@@ -0,0 +1,13 @@
+import type { Command } from 'commander';
+import { type FirecrackerBridgeConfig } from '../../../firecracker/host-network.js';
+export interface ProvisionOptions {
+    bridge: string;
+    cidr?: string;
+    table?: string;
+    hostIface?: string;
+    persist?: boolean;
+    sudo?: boolean;
+}
+export declare function optionsToConfig(opts: ProvisionOptions): FirecrackerBridgeConfig;
+export declare function registerProvision(group: Command): void;
+//# sourceMappingURL=provision.d.ts.map

package/dist/cli/commands/firecracker/teardown.d.ts

@@ -0,0 +1,3 @@
+import type { Command } from 'commander';
+export declare function registerTeardown(group: Command): void;
+//# sourceMappingURL=teardown.d.ts.map

package/dist/cli/commands/firecracker/verify.d.ts

@@ -0,0 +1,3 @@
+import type { Command } from 'commander';
+export declare function registerVerify(group: Command): void;
+//# sourceMappingURL=verify.d.ts.map

package/dist/cli/commands/scaler.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Standalone scaler maintenance commands for kici-admin.
+ *
+ *   scaler reap-orphans   Free leaked Firecracker/container resources WITHOUT a
+ *                         running orchestrator (recovery for a wedged node whose
+ *                         data disk is full).
+ *
+ * This command loads the orchestrator's LOCAL config (no HTTP admin API, no DB)
+ * and runs the liveness-driven reaper. It is the sanctioned operator path for
+ * the ENOSPC bootstrap deadlock: when the data disk is 100% full the
+ * orchestrator crash-loops before its in-process orphan sweep can run, so a
+ * tool that frees disk WITHOUT the orchestrator is required.
+ */
+import type { Command } from 'commander';
+/** Probe the local orchestrator /health endpoint. Healthy => it reaps itself. */
+export declare function isOrchestratorHealthy(port: number, basePath: string): Promise<boolean>;
+export declare function registerScalerCommands(program: Command): void;
+//# sourceMappingURL=scaler.d.ts.map

package/dist/cli/kici-admin.d.ts

@@ -9,5 +9,14 @@
  *   --url / -u    Orchestrator URL (env: KICI_ADMIN_URL, default: http://localhost:8080)
  *   --token / -t  Admin API token (env: KICI_ADMIN_TOKEN, required)
  */
-export {};
+import { Command } from 'commander';
+/**
+ * Build the kici-admin Commander program with every command group registered.
+ * Exported so the surface registry can walk the real command tree without
+ * parsing argv (no action runs during a tree walk, so `getClient` is
+ * constructed but never invoked).
+ */
+export declare function buildProgram(): Command;
+/** Build the program and parse argv — the bin-shim entry point. */
+export declare function runCli(argv?: string[]): void;
 //# sourceMappingURL=kici-admin.d.ts.map

package/dist/cli/service/image-digests.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Resolves the customer installer's image reference from the digest record
+ * shipped with this package (installer-image-digests.json at the package root).
+ *
+ * Emits a manifest-list-digest-pinned ref so a `docker`/`podman pull` verifies
+ * the image hash — a registry that serves substituted bits fails the pull. When
+ * the record is missing (a dev tree that never ran a release) or lacks the
+ * image, falls back to the moving `:latest` tag and warns.
+ *
+ * The record resolves relative to this module's directory and works for both
+ * layouts: the source tree (`src/cli/service/` → the JSON is three dirs up at
+ * the package root) and the single-file bundle (`dist/cli.js` → one dir up).
+ */
+/**
+ * Resolve `quay.io/kici-dev/<name>:<version>@sha256:<digest>` for an installer
+ * image. Falls back to `:latest` (with a warning) when no recorded digest exists.
+ */
+export declare function resolveImageRef(name: string, opts?: {
+    filePath?: string;
+}): string;
+//# sourceMappingURL=image-digests.d.ts.map