@kici-dev/orchestrator 0.13.0 → 0.14.0

package/dist/agent/dispatcher.d.ts

@@ -300,6 +300,23 @@ export declare class Dispatcher {
      * @returns the number of jobs that entered `spawning`.
      */
     retryPendingScaleRequests(maxJobs?: number, trigger?: ScalerRedispatchTrigger): Promise<number>;
+    /**
+     * Re-offer ONE pending job to the scaler: a worker's spawn-retry backoff ending.
+     *
+     * Outside the batch's single-flight on purpose: a wake-up that lands during a
+     * batch pass would otherwise be dropped, and a worker runs no pending-scale
+     * sweep to pick it up. The worker's per-job attempt gate is what keeps this
+     * from starting a second spawn for the same job.
+     *
+     * @returns true when a spawn started.
+     */
+    redrivePendingJob(jobId: string): Promise<boolean>;
+    /**
+     * Offer one pending job to the scaler through the same `onNoMatchingAgent` →
+     * `requestScale` path a dispatch uses, or settle it when its sealed secrets
+     * cannot be opened here.
+     */
+    private offerPendingToScaler;
     /**
      * Release the slot claimed for a direct dispatch the queue refused because
      * the job's run has stopped. Nothing is sent to the agent. The queue already

package/dist/cancel/peer-job-cancel.d.ts

@@ -8,6 +8,10 @@
  * only to `dispatch_queue.agent_id`. So the agent is resolved from the
  * dispatcher first and from the dispatch record second, and a cancel that
  * cannot be delivered is logged with the reason.
+ *
+ * On a worker the receiver also removes the jobs of the cancel that are still
+ * queued (waiting for an agent spawn), and stop-marks the dispatched ones so a
+ * requeue cannot bring them back. A removed job is reported `dequeued`.
  */
 import type { Kysely } from 'kysely';
 import { z } from 'zod';
@@ -18,6 +22,7 @@ import type { Dispatcher } from '../agent/dispatcher.js';
 export declare const PeerCancelOutcome: z.ZodEnum<{
     "agent-not-connected": "agent-not-connected";
     delivered: "delivered";
+    dequeued: "dequeued";
     "lookup-failed": "lookup-failed";
     "not-tracked": "not-tracked";
 }>;
@@ -35,6 +40,13 @@ export interface PeerJobCancelDeps {
     registry: Pick<AgentRegistry, 'get'>;
     /** The durable dispatch record. Absent on a worker, which has no database. */
     lookupDispatched?: (runId: string, jobId: string | undefined) => Promise<DispatchedAgent[]>;
+    /**
+     * Remove this process's queued (not yet dispatched) jobs of the cancel, and
+     * stop-mark its dispatched ones so a requeue cannot bring them back. Wired only
+     * on a worker, whose queue lives in memory; a coordinator's queue rows are
+     * settled by its own run-cancel path.
+     */
+    releaseQueued?: (runId: string, jobId: string | undefined) => Promise<string[]>;
     logger: {
         info(message: string, meta?: object): unknown;
         warn(message: string, meta?: object): unknown;

package/dist/cli/api-client.d.ts

@@ -5,6 +5,42 @@
  * All methods are thin wrappers around fetch() that handle JSON serialization,
  * error formatting, and URL construction.
  */
+import type { PeerForgetOutcome, ScalerLiveVm, ScalerVmStopResult } from '@kici-dev/engine';
+/** `POST /api/v1/admin/peers/forget`: one result per coordinator, this one first. */
+export interface PeerForgetResponseBody {
+    instanceId: string;
+    results: Array<{
+        coordinator: string;
+        outcome: PeerForgetOutcome;
+        detail: string;
+    }>;
+    /** Set when the coordinator kept the peer only for want of the backstop acknowledgement. */
+    acknowledgementRequired?: boolean;
+    error?: string;
+}
+/** The node a scaler orphans request answered for. */
+export interface ScalerOrphansNode {
+    instanceId: string;
+    /** The node's cluster role; `unknown` for a peer the coordinator has no role for. */
+    role: 'coordinator' | 'worker' | 'unknown';
+    /** The node's Firecracker scaler names; empty when it runs none. */
+    firecrackerScalers: string[];
+    /**
+     * Coordinator peers the cluster expects but the answering coordinator has no
+     * live link to. While any is listed, no VM is `orphaned`.
+     */
+    disconnectedCoordinators?: string[];
+}
+/** `GET /api/v1/admin/scaler/orphans`: every live Firecracker VM on the node. */
+export interface ScalerOrphansListResponse {
+    node: ScalerOrphansNode;
+    vms: ScalerLiveVm[];
+}
+/** `POST /api/v1/admin/scaler/orphans/stop`: one result per requested VM. */
+export interface ScalerOrphansStopResponse {
+    node: ScalerOrphansNode;
+    results: ScalerVmStopResult[];
+}
 /** A node in the fleet topology returned by GET /admin/fleet-topology. */
 export interface FleetTopologyNodeResponse {
     kind: 'orchestrator' | 'agent';
@@ -384,6 +420,29 @@ export declare class AdminApiClient {
         draining: boolean;
         jobsRunning: number;
     }>;
+    /** Forget a departed peer on this coordinator and every connected sibling. */
+    /**
+     * Forget a departed peer on this coordinator and every connected sibling. A
+     * refusal that only needs the backstop acknowledgement comes back as a body
+     * with `acknowledgementRequired`, so the caller can ask and retry; every
+     * other failure throws.
+     */
+    forgetPeer(body: {
+        instanceId: string;
+        timeoutMs?: number;
+        acknowledgeBackstop?: boolean;
+    }): Promise<PeerForgetResponseBody>;
+    /** List the live Firecracker VMs on this coordinator, or on the `target` peer. */
+    listScalerOrphans(opts?: {
+        target?: string;
+        timeoutMs?: number;
+    }): Promise<ScalerOrphansListResponse>;
+    /** Stop the named VMs on this coordinator, or on the `target` peer. */
+    stopScalerOrphans(body: {
+        target?: string;
+        vmIds: string[];
+        timeoutMs?: number;
+    }): Promise<ScalerOrphansStopResponse>;
     createApiKey(opts: {
         label?: string;
         routingKeys?: string[];
@@ -547,6 +606,20 @@ export declare class AdminApiClient {
         includePayload?: boolean;
         routingKey?: string;
     }): Promise<Record<string, unknown>>;
+    /** List internal events, newest first (`GET /api/v1/admin/events`). */
+    listEvents(opts?: {
+        name?: string;
+        outcome?: string;
+        since?: string;
+        before?: string;
+        limit?: number;
+    }): Promise<{
+        events: Array<Record<string, unknown>>;
+        limit: number;
+        nextCursor: string | null;
+    }>;
+    /** One internal event with its redacted payload and dispatched runs. */
+    getEvent(id: string): Promise<Record<string, unknown>>;
     listAccessLog(opts?: {
         orgId?: string;
         actorType?: string;

package/dist/cli/commands/cluster-settings.d.ts

@@ -5,13 +5,15 @@
  *
  * Talks to the orchestrator admin API directly (not the Platform dashboard
  * proxy), so the CLI stays operable even when Platform is unavailable. Backed
- * by `packages/orchestrator/src/routes/admin-cluster-settings.ts`.
+ * by `packages/orchestrator/src/routes/admin-cluster-settings.ts`. `show` also
+ * lists which settings version each coordinator and worker applied.
  *
  * These knobs are fleet-wide (a single `cluster_settings` row), not per-tenant.
  * For per-tenant knobs use `kici-admin org-settings`.
  */
 import type { Command } from 'commander';
 import type { AdminApiClient } from '../api-client.js';
+import { type SettingsPropagationReport } from '../../cluster/settings-propagation.js';
 /**
  * One text knob. Kept separate from {@link KnobSpec} because the value parser,
  * the PATCH body type, and the `show` formatter all differ — a string knob has
@@ -36,8 +38,36 @@ interface BooleanKnobSpec {
     flag: string;
     label: string;
 }
+type ClusterSettings = Record<string, number | string | boolean | null>;
 /** camelCase → value (null clears the override). */
 type PatchBody = Record<string, number | string | boolean | null>;
+/** The outcome of the propagation request `show` makes after the settings read. */
+export type PropagationFetch = {
+    kind: 'ok';
+    report: SettingsPropagationReport;
+} | {
+    kind: 'unsupported';
+} | {
+    kind: 'error';
+    reason: string;
+};
+export declare const PROPAGATION_UNSUPPORTED_NOTE: string;
+/**
+ * Fetch the propagation report. A 404 means the orchestrator predates the
+ * route; `show` then prints the settings alone, because this CLI may be newer
+ * than the orchestrator it talks to.
+ */
+export declare function fetchPropagation(client: AdminApiClient): Promise<PropagationFetch>;
+/** The table section: one row per orchestrator, lagging ones in upper case. */
+export declare function formatPropagation(report: SettingsPropagationReport): string;
+/**
+ * What `show` prints. The JSON keeps the flat settings map exactly as before
+ * and adds one `propagation` key, so scripts that read knob keys keep working.
+ */
+export declare function renderShow(settings: ClusterSettings, propagation: PropagationFetch, format: string): {
+    stdout: string;
+    stderr: string | null;
+};
 /**
  * Parse an integer CLI flag against its knob's bounds, exiting on failure.
  *

package/dist/cli/commands/peer.d.ts

@@ -5,9 +5,35 @@
  *   peer create-token   Create a join token for a new peer
  *   peer list           List active peer credentials
  *   peer revoke         Revoke a specific peer's credential
+ *   peer forget         Drop a departed peer from every coordinator's live
+ *                       peer registry (admin HTTP API; credential untouched)
  *   peer revoke-all     Revoke all peer credentials
  */
 import type { Command } from 'commander';
 import type { AdminApiClient } from '../api-client.js';
-export declare function registerPeerCommands(program: Command, _getClient: () => AdminApiClient): void;
+/** Where `peer forget` writes its results. */
+export interface PeerForgetIo {
+    out: (line: string) => void;
+    /** Asks the operator; resolves true on yes. */
+    confirm: (prompt: string) => Promise<boolean>;
+    /** Whether an operator can answer a prompt (stdin is a terminal). */
+    interactive: boolean;
+}
+/**
+ * `kici-admin peer forget`: drop a departed peer from this coordinator's live
+ * peer registry and every connected sibling's. Exits 1 when any coordinator
+ * kept it (connected, heard from recently, or no answer).
+ *
+ * When the forget would switch a coordinator's event-provision backstop back
+ * on, the operator confirms that consequence first; `--yes` acknowledges it up
+ * front, and without a terminal the command refuses instead of asking.
+ *
+ * @returns the process exit code
+ */
+export declare function runPeerForget(client: Pick<AdminApiClient, 'forgetPeer'>, instanceId: string, opts: {
+    json: boolean;
+    timeout: string;
+    yes: boolean;
+}, io: PeerForgetIo): Promise<number>;
+export declare function registerPeerCommands(program: Command, getClient: () => AdminApiClient): void;
 //# sourceMappingURL=peer.d.ts.map

package/dist/cli/commands/scaler.d.ts

@@ -1,18 +1,54 @@
 /**
- * Standalone scaler maintenance commands for kici-admin.
+ * Scaler maintenance commands for kici-admin.
  *
+ *   scaler orphans        List the live Firecracker VMs a node's running
+ *                         orchestrator does not track and, with --stop, stop
+ *                         them. Goes through the admin HTTP API; `--target`
+ *                         reaches a worker through the coordinator.
  *   scaler reap-orphans   Free leaked Firecracker/container resources WITHOUT a
  *                         running orchestrator (recovery for a wedged node whose
  *                         data disk is full).
  *
- * This command loads the orchestrator's LOCAL config (no HTTP admin API, no DB)
- * and runs the liveness-driven reaper. It is the sanctioned operator path for
+ * `reap-orphans` loads the orchestrator's LOCAL config (no HTTP admin API, no
+ * DB) and runs the liveness-driven reaper, which spares every live VM. It is the sanctioned operator path for
  * the ENOSPC bootstrap deadlock: when the data disk is 100% full the
  * orchestrator crash-loops before its in-process orphan sweep can run, so a
  * tool that frees disk WITHOUT the orchestrator is required.
  */
 import type { Command } from 'commander';
+import type { AdminApiClient } from '../api-client.js';
 /** Probe the local orchestrator /health endpoint. Healthy => it reaps itself. */
 export declare function isOrchestratorHealthy(port: number, basePath: string): Promise<boolean>;
-export declare function registerScalerCommands(program: Command): void;
+/** The `scaler orphans` options as Commander parses them. */
+export interface ScalerOrphansOptions {
+    target?: string;
+    all: boolean;
+    stop: boolean;
+    vm: string[];
+    yes: boolean;
+    dryRun: boolean;
+    /** Seconds to wait for a `--target` node. */
+    timeout: string;
+    json: boolean;
+}
+/** Where `scaler orphans` reads its confirmation and writes its output. */
+export interface ScalerOrphansIo {
+    confirm: (prompt: string) => Promise<boolean>;
+    /** The listing, the results, or the JSON body. */
+    out: (line: string) => void;
+    /** Progress and the confirmation context; stderr, so `--json` output stays parseable. */
+    err: (line: string) => void;
+}
+type OrphansClient = Pick<AdminApiClient, 'listScalerOrphans' | 'stopScalerOrphans'>;
+/** `45s`, `12m3s`, `1h2m`, `3d4h`. */
+export declare function formatAge(seconds: number): string;
+/**
+ * `kici-admin scaler orphans`: list the live Firecracker VMs a node's
+ * orchestrator does not track, and with `--stop` stop the orphaned ones.
+ *
+ * @returns the process exit code
+ */
+export declare function runScalerOrphans(client: OrphansClient, opts: ScalerOrphansOptions, io: ScalerOrphansIo): Promise<number>;
+export declare function registerScalerCommands(program: Command, getClient: () => AdminApiClient): void;
+export {};
 //# sourceMappingURL=scaler.d.ts.map