@kici-dev/orchestrator 0.13.0 → 0.14.0

package/dist/db/types.d.ts

@@ -1506,6 +1506,10 @@ export interface KiciEventTable {
     dlq_at: Date | null;
     /** Short DLQ reason: 'exhausted_retries' | 'non_retryable' */
     dlq_reason: string | null;
+    /** How the router resolved the event (an `EventMatchOutcome`); NULL until processed */
+    match_outcome: string | null;
+    /** Registrations the event was dispatched to; NULL until processed */
+    matched_count: number | null;
 }
 export type KiciEvent = Selectable<KiciEventTable>;
 export type NewKiciEvent = Insertable<KiciEventTable>;
@@ -1989,6 +1993,16 @@ export interface OrgSettingsTable {
      * (config.rerouteMaxHops). Plain INTEGER — pg returns a number on select.
      */
     reroute_max_hops: ColumnType<number | null, number | null | undefined, number | null>;
+    /**
+     * Per-org agent spawns a worker attempts for one rerouted job. Null = cluster
+     * default (config.rerouteSpawnMaxAttempts). Plain INTEGER.
+     */
+    reroute_spawn_max_attempts: ColumnType<number | null, number | null | undefined, number | null>;
+    /**
+     * Per-org wait (ms) after a failed spawn of a rerouted job. Null = cluster
+     * default (config.rerouteSpawnRetryBackoffMs). BIGINT.
+     */
+    reroute_spawn_retry_backoff_ms: ColumnType<string | null, number | null | undefined, number | null>;
     /**
      * Per-org staleness threshold (hours) for the DB-backup freshness diagnostic.
      * Null = cluster default (config.backupStalenessWarnHours). Plain INTEGER.
@@ -2197,6 +2211,8 @@ export interface ClusterSettingsTable {
      * per round.
      */
     global_eval_wait_timeout_ms: ColumnType<string | null, number | null | undefined, number | null>;
+    /** How long a Firecracker spawn waits for the VM's API socket. NULL: config default. */
+    firecracker_api_socket_wait_ms: ColumnType<string | null, number | null | undefined, number | null>;
     /**
      * Retention window in days for `check_run_tracking` rows. The hourly cleanup
      * sweep deletes rows untouched for longer than this; 0 disables the sweep.

package/dist/events/event-router.d.ts

@@ -6,7 +6,7 @@ import type { EventStore } from './event-store.js';
 import type { EventCircuitBreaker } from './circuit-breaker.js';
 import type { TrustStore } from './trust-store.js';
 import type { RegisteredWorkflow, RegistrationIndex } from '../registration/registration-index.js';
-import type { EventRouterConfig, StoredEvent } from './types.js';
+import { type EventRouterConfig, type StoredEvent } from './types.js';
 import type { ClusterSettingsReader } from '../cluster/cluster-settings-reader.js';
 /**
  * System events are exempt from the per-source event-storm rate limiter: they
@@ -213,6 +213,10 @@ export declare class EventRouter {
      * schedule a retry or move the event to the DLQ. Swallowing the error
      * here would silently lose the event — exactly the failure mode the
      * lease pattern fixes.
+     *
+     * Returns how the event resolved, which `dispatchAndRecord` persists on the
+     * processed row: `matched` when it dispatched at least one workflow, else
+     * the reason nothing ran.
      */
     private processSubscriptions;
     /**
@@ -225,6 +229,8 @@ export declare class EventRouter {
      * itself dispatched by a failure-lifecycle trigger is neither buffered nor
      * dispatched — a broken notifier must not re-trigger the batch on its own
      * failure. The chain-depth breaker remains the backstop for anything missed.
+     *
+     * `bufferedCount` counts the decisions appended to a batch window.
      */
     private bufferBatchDecisions;
     /** Open (or reuse) the registration's batch window and append the failed run. */

package/dist/events/event-store.d.ts

@@ -1,6 +1,6 @@
 import { type Kysely, type Transaction } from 'kysely';
 import type { Database } from '../db/types.js';
-import type { DlqReason, EventRouterConfig, StoredEvent } from './types.js';
+import { EventMatchOutcome, type DlqReason, type EventMatchResult, type EventRouterConfig, type StoredEvent } from './types.js';
 /** Either a Kysely DB handle or an active transaction. */
 export type DbExecutor = Kysely<Database> | Transaction<Database>;
 /**
@@ -81,11 +81,28 @@ export declare class EventStore {
      * re-queries until a page returns fewer than `limit` rows.
      */
     getUnprocessedSince(sinceId: string | null, limit?: number): Promise<StoredEvent[]>;
+    /**
+     * List events newest first, for `kici-admin event list`. Every filter is
+     * optional except the limit. `before` keeps events created before a
+     * timestamp. `afterEventId` is the next-page cursor: the events that sort
+     * after that event in this order, compared on the stored `(created_at, id)`
+     * so events created in the same millisecond are neither skipped nor repeated.
+     */
+    list(opts: {
+        name?: string;
+        outcome?: EventMatchOutcome;
+        since?: Date;
+        before?: Date;
+        afterEventId?: string;
+        limit: number;
+        sourceRoutingKey?: string;
+    }): Promise<StoredEvent[]>;
     /**
      * Mark an event as fully processed. Called after a successful dispatch.
-     * Clears any prior lease so the row is unambiguously terminal.
+     * Clears any prior lease so the row is unambiguously terminal, and records
+     * how the router resolved it when `result` is given.
      */
-    markProcessed(id: string): Promise<void>;
+    markProcessed(id: string, result?: EventMatchResult): Promise<void>;
     /**
      * Atomically take a dispatch lease on an event. Replaces the previous
      * "flip processed=true upfront" pattern that silently lost events when

package/dist/events/types.d.ts

@@ -1,6 +1,7 @@
 /**
  * Event routing configuration and types.
  */
+import { z } from 'zod';
 /**
  * Configuration for the event routing system.
  */
@@ -45,6 +46,22 @@ export interface EventRouterConfig {
  * Reason an event landed in the DLQ.
  */
 export type DlqReason = 'exhausted_retries' | 'non_retryable';
+/** How the router resolved one event against the registered workflows. */
+export declare const EventMatchOutcome: z.ZodEnum<{
+    buffered: "buffered";
+    matched: "matched";
+    "no-registration": "no-registration";
+    "no-target-repo": "no-target-repo";
+    "no-trigger-match": "no-trigger-match";
+    "trust-blocked": "trust-blocked";
+}>;
+export type EventMatchOutcome = z.infer<typeof EventMatchOutcome>;
+/** What the router resolved for one event, persisted on its processed row. */
+export interface EventMatchResult {
+    outcome: EventMatchOutcome;
+    /** Registrations the event was dispatched to. */
+    matchedCount: number;
+}
 /**
  * A stored internal event (matches DB row shape but with JS types).
  */
@@ -70,7 +87,35 @@ export interface StoredEvent {
     nextRetryAt: Date | null;
     dlqAt: Date | null;
     dlqReason: DlqReason | null;
+    /**
+     * How the router resolved the event. Set when the event is processed; null
+     * before, and for rows processed before the column existed.
+     */
+    matchOutcome: EventMatchOutcome | null;
+    /** Registrations the event was dispatched to; null when `matchOutcome` is. */
+    matchedCount: number | null;
 }
+/**
+ * An event payload safe to show an operator. An event-scaler scale-up carries
+ * a single-use claim code an agent redeems for credentials, so every read
+ * surface (event show, the dead-letter queue) replaces it with `[redacted]`.
+ */
+export declare function redactEventPayload(payload: Record<string, unknown>): Record<string, unknown>;
+/** Where an internal event is in its dispatch lifecycle. */
+export declare const EventProcessingState: z.ZodEnum<{
+    dlq: "dlq";
+    leased: "leased";
+    pending: "pending";
+    processed: "processed";
+    retrying: "retrying";
+}>;
+export type EventProcessingState = z.infer<typeof EventProcessingState>;
+/**
+ * Derive an event's dispatch state from its row: DLQ wins, then processed;
+ * an unprocessed row is leased while a node holds it, retrying once a failed
+ * dispatch scheduled another attempt, and pending before its first lease.
+ */
+export declare function eventProcessingState(e: StoredEvent): EventProcessingState;
 /**
  * Default event router configuration.
  */

package/dist/helpers/windows-batch.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Running a Windows batch file (`.cmd` / `.bat`) through cmd.exe.
+ *
+ * Node.js refuses to start a batch file without a shell (it throws `EINVAL`),
+ * and every launcher a KiCI package ships on Windows is a batch file. Two
+ * callers run one: the service installer, which registers the command with
+ * shawl, and the bare-metal scaler, which spawns it.
+ *
+ * Both quote a token on cmd.exe's command line only when it contains a space
+ * or a tab — shawl does, and so does libuv when Node.js builds a Windows
+ * command line — so one safety rule fits both.
+ */
+/**
+ * The cmd.exe arguments that run a batch-file launcher, ahead of its path.
+ *
+ * shawl stops the service with a ctrl-C to its console, and cmd.exe gets it
+ * too: when the batch file's node process exits, cmd.exe asks "Terminate batch
+ * job (Y/N)?" and waits for an answer. shawl sees the service exit only when its
+ * stop timeout kills cmd.exe. With stdin from NUL ({@link BATCH_STDIN}), cmd.exe
+ * reads end-of-input at that prompt and exits with node.
+ *
+ * `call` keeps the launcher path out of the first position after `/c`, where
+ * cmd.exe strips the quotes from a path that contains `(`, `)` or `&`. `/e:on`
+ * and `/v:off` pin command extensions (the launchers use `%~dp0`) and delayed
+ * expansion whatever the host's registry defaults are.
+ */
+export declare const BATCH_WRAPPER_ARGS: readonly ['/d', '/e:on', '/v:off', '/c', 'call'];
+/** The redirect that closes {@link BATCH_WRAPPER_ARGS}: the launcher's stdin is NUL. */
+export declare const BATCH_STDIN = "<NUL";
+/** A `.cmd` or `.bat` file, which Windows runs through cmd.exe. */
+export declare function isBatchFile(executablePath: string): boolean;
+/**
+ * The cmd.exe that COMSPEC names in `env`, as Node itself uses to run a shell on
+ * Windows. A COMSPEC that names another shell is not used: the wrapper
+ * arguments are cmd.exe's.
+ */
+export declare function cmdExePath(env?: NodeJS.ProcessEnv): string;
+/**
+ * Why cmd.exe would read part of `token` as syntax, or null when it would not.
+ *
+ * A token is quoted on cmd.exe's command line only when it contains a space or
+ * a tab. Inside quotes cmd.exe reads `&`, `|`, `<`, `>`, `(`, `)` and the `,`
+ * `;` `=` delimiters as text; outside quotes it reads them as syntax. `%` and
+ * `^` change even inside quotes (`call` expands `%` again and doubles `^`), and
+ * a `"` ends the quoting.
+ *
+ * @param use - what the token runs as, completing "cannot run <token> …".
+ * @param remedy - the sentence that tells the operator what to change.
+ */
+export declare function cmdSafetyRefusal(token: string, use: string, remedy: string): string | null;
+/** Throw the {@link cmdSafetyRefusal} message for `token`, if it has one. */
+export declare function assertCmdSafe(token: string, use: string, remedy: string): void;
+/**
+ * The cmd.exe command that runs `command` — a batch file and its arguments —
+ * with stdin from NUL. The caller checks each token with
+ * {@link assertCmdSafe} first.
+ */
+export declare function batchFileCommand(command: readonly string[], env?: NodeJS.ProcessEnv): string[];
+//# sourceMappingURL=windows-batch.d.ts.map

package/dist/index.js

@@ -2860,6 +2860,51 @@ var AdminApiClient = class {
 	async drainStatus() {
 		return this.request("GET", "/api/v1/admin/orchestrator/drain");
 	}
+	/** Forget a departed peer on this coordinator and every connected sibling. */
+	/**
+	* Forget a departed peer on this coordinator and every connected sibling. A
+	* refusal that only needs the backstop acknowledgement comes back as a body
+	* with `acknowledgementRequired`, so the caller can ask and retry; every
+	* other failure throws.
+	*/
+	async forgetPeer(body) {
+		const res = await fetchAdminApi(`${this.baseUrl}/api/v1/admin/peers/forget`, {
+			method: "POST",
+			headers: {
+				Authorization: `Bearer ${this.token}`,
+				"Content-Type": "application/json"
+			},
+			body: JSON.stringify(body)
+		}, this.baseUrl);
+		const text = await res.text();
+		let parsed;
+		try {
+			parsed = JSON.parse(text);
+		} catch {
+			parsed = void 0;
+		}
+		const json = parsed ?? {};
+		if (res.ok) return json;
+		if (res.status === 409 && json.acknowledgementRequired === true) return {
+			instanceId: body.instanceId,
+			results: json.results ?? [],
+			acknowledgementRequired: true,
+			...json.error ? { error: json.error } : {}
+		};
+		throw new Error(`HTTP ${res.status}: ${json.error ?? text}`);
+	}
+	/** List the live Firecracker VMs on this coordinator, or on the `target` peer. */
+	async listScalerOrphans(opts = {}) {
+		const params = new URLSearchParams();
+		if (opts.target !== void 0) params.set("target", opts.target);
+		if (opts.timeoutMs !== void 0) params.set("timeoutMs", String(opts.timeoutMs));
+		const query = params.toString();
+		return this.request("GET", `/api/v1/admin/scaler/orphans${query ? `?${query}` : ""}`);
+	}
+	/** Stop the named VMs on this coordinator, or on the `target` peer. */
+	async stopScalerOrphans(body) {
+		return this.request("POST", "/api/v1/admin/scaler/orphans/stop", body);
+	}
 	async createApiKey(opts) {
 		return this.request("POST", "/api/v1/api-keys", opts);
 	}
@@ -2986,6 +3031,21 @@ var AdminApiClient = class {
 		if (opts.routingKey) params.set("routingKey", opts.routingKey);
 		return this.request("GET", `/api/v1/admin/event-log/${encodeURIComponent(deliveryId)}?${params}`);
 	}
+	/** List internal events, newest first (`GET /api/v1/admin/events`). */
+	async listEvents(opts) {
+		const params = new URLSearchParams();
+		if (opts?.name) params.set("name", opts.name);
+		if (opts?.outcome) params.set("outcome", opts.outcome);
+		if (opts?.since) params.set("since", opts.since);
+		if (opts?.before) params.set("before", opts.before);
+		if (opts?.limit !== void 0) params.set("limit", String(opts.limit));
+		const qs = params.toString();
+		return this.request("GET", `/api/v1/admin/events${qs ? `?${qs}` : ""}`);
+	}
+	/** One internal event with its redacted payload and dispatched runs. */
+	async getEvent(id) {
+		return this.request("GET", `/api/v1/admin/events/${encodeURIComponent(id)}`);
+	}
 	async listAccessLog(opts) {
 		const params = new URLSearchParams();
 		if (opts?.orgId) params.set("orgId", opts.orgId);

package/dist/metrics/prometheus.d.ts

@@ -539,6 +539,8 @@ export declare const eventRetryTotal: import("@opentelemetry/api").Counter<impor
  * - reason: why the event was DLQ'd (exhausted_retries)
  */
 export declare const eventDlqTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
+/** Record a reserved internal event that matched no subscriber. */
+export declare function incEventUnmatched(eventName: string, reason: string): void;
 /**
  * Startup catch-up scans that threw. Emits no labels.
  *

package/dist/orchestrator-core.d.ts

@@ -41,6 +41,7 @@ import { ArtifactStore } from './artifacts/artifact-store.js';
 import { CheckRunReporter } from './reporting/check-run-reporter.js';
 import type { ProcessingDeps } from './pipeline/processor.js';
 import type { InvokeGateDeps, SummonArgs, SummonedRun } from './pipeline/invoke-gate.js';
+import { type PeerForgetRequestHandler } from './cluster/peer-forget.js';
 import { StepLogBuffer } from './reporting/step-log-buffer.js';
 import { type LogStorage } from './reporting/log-storage.js';
 import { ExecutionTracker, type ExecutionTrackerDeps } from './reporting/execution-tracker.js';
@@ -167,6 +168,12 @@ export interface OrchestratorSubsystems {
     orphanRecovery: OrphanRecovery;
     peerHandler: ReturnType<typeof createPeerHandler>;
     peerClients: Map<string, PeerClient>;
+    /**
+     * Answers a `kici-admin peer forget` a sibling coordinator fanned out. Wired
+     * into every outgoing PeerClient (the incoming peer-handler is wired directly
+     * in setupCluster).
+     */
+    answerPeerForgetRequest: PeerForgetRequestHandler;
     getLocalInventory: () => Omit<PeerHeartbeat, 'type'>;
     broadcastHeartbeatToAllPeers: () => void;
     broadcastAgentTokenRevoke: (tokenId: string) => void;

package/dist/registration/registration-store.d.ts

@@ -74,6 +74,16 @@ interface ReplaceAllOptions {
     depCacheKey?: DepCacheKey;
     /** Set of workflow names that should be marked as global */
     globalWorkflowNames?: Set<string>;
+    /**
+     * Delete this routing key's rows for the repo whose workflow the incoming set
+     * does not name. `true` (the default) is the default-branch push sync: the
+     * lock file is the whole truth for the repo. The manual registration route
+     * passes `false`: it upserts the workflows it names and leaves every other
+     * registration of the repo alone, which is what
+     * `kici-admin workflow register-manual` documents and what its direct-DB
+     * mode does.
+     */
+    prune?: boolean;
 }
 /**
  * DB CRUD for workflow_registrations table with atomic upsert-based replace-all.

package/dist/routes/admin-cluster-settings.d.ts

@@ -4,7 +4,9 @@
  * Exposes GET / PATCH `/api/v1/admin/cluster-settings` so `kici-admin
  * cluster-settings` can read and write the single `cluster_settings` row
  * (id='default') without going through the Platform dashboard proxy — the CLI
- * stays operable even when Platform is unavailable.
+ * stays operable even when Platform is unavailable. `GET
+ * /api/v1/admin/cluster-settings/propagation` reports which settings version
+ * each orchestrator this coordinator knows has applied.
  *
  * These knobs are fleet-wide, not per-tenant, so the route is deliberately NOT
  * under `/orgs/:customerId/*`. Mutations require the `secret.write` RBAC
@@ -15,9 +17,19 @@ import { Hono } from 'hono';
 import { type Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
 import type { RbacEnforcer, Role } from '../secrets/rbac.js';
+import type { PeerRegistry } from '../cluster/peer-registry.js';
+/** The live view behind `GET /cluster-settings/propagation`. */
+export interface ClusterSettingsPropagationSource {
+    instanceId: string;
+    peerRegistry: Pick<PeerRegistry, 'getAllPeers'>;
+    /** The cluster-settings version this coordinator's reads currently serve. */
+    localVersion: () => number;
+}
 interface ClusterSettingsRouteDeps {
     db: Kysely<Database>;
     rbac: RbacEnforcer;
+    /** Mounts `GET /cluster-settings/propagation` when set. */
+    propagation?: ClusterSettingsPropagationSource;
 }
 type AdminEnv = {
     Variables: {

package/dist/routes/admin-event-inspect.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Admin API routes for inspecting internal events (`kici_events`).
+ *
+ *   GET /api/v1/admin/events      — list events, newest first (name, outcome, time filters)
+ *   GET /api/v1/admin/events/:id  — one event: its payload and the runs it dispatched
+ *
+ * Each row carries its dispatch state (`pending`, `leased`, `retrying`,
+ * `processed`, `dlq`) and, once processed, how the router resolved it
+ * (`matched`, `no-target-repo`, …) with the number of workflows it reached.
+ * Both routes need `event_dlq.read`, the permission that already gates reading
+ * `kici_events` rows; the payload on `show` also needs
+ * `event_log.read_payload`. A routing-key-scoped token sees only events
+ * emitted under its own key.
+ */
+import { Hono } from 'hono';
+import type { Kysely } from 'kysely';
+import type { Database } from '../db/types.js';
+import type { EventStore } from '../events/event-store.js';
+import type { TokenManager } from '../secrets/token-manager.js';
+import type { RbacEnforcer, Role } from '../secrets/rbac.js';
+export interface AdminEventInspectRoutesDeps {
+    db: Kysely<Database>;
+    eventStore: EventStore;
+    tokenManager: TokenManager;
+    rbac: RbacEnforcer;
+}
+type AdminEnv = {
+    Variables: {
+        role: Role;
+        userId: string;
+        routingKey: string | null;
+    };
+};
+export declare function createAdminEventInspectRoutes(deps: AdminEventInspectRoutesDeps): Hono<AdminEnv>;
+export {};
+//# sourceMappingURL=admin-event-inspect.d.ts.map

package/dist/routes/admin-peer-forget.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Forget a departed peer, behind `kici-admin peer forget <instance-id>`.
+ *
+ * Mounted inside `createAdminRoutes` at `/api/v1/admin`, so the Bearer-token
+ * auth middleware has already resolved the caller's role. Needs an unscoped
+ * token and the `peer.manage` permission (owner, admin).
+ *
+ * - `POST /api/v1/admin/peers/forget { instanceId, timeoutMs? }` — drop the
+ *   peer from this coordinator's live peer registry, then from every connected
+ *   sibling coordinator's, and answer with one result per coordinator.
+ *
+ * Forgetting never touches the peer's credential (`kici-admin peer revoke`
+ * refuses its next connection). A peer that connects again later is
+ * registered again as usual.
+ */
+import { Hono } from 'hono';
+import type { AccessLogWriter } from '../audit/access-log.js';
+import { type PeerForgetResult } from '../cluster/peer-forget.js';
+import type { RbacEnforcer, Role } from '../secrets/rbac.js';
+type AdminEnv = {
+    Variables: {
+        role: Role;
+        userId: string;
+        routingKey: string | null;
+    };
+};
+/** What the route needs from the coordinator it runs on. */
+export interface PeerForgetRouteDeps {
+    /**
+     * Forget `instanceId` here, then on every connected sibling coordinator
+     * (skipped when this coordinator keeps the peer). The first result is this
+     * coordinator's. `acknowledgeBackstop` lets a forget that switches an
+     * event-provision backstop back on go ahead.
+     */
+    forget: (instanceId: string, timeoutMs: number, acknowledgeBackstop: boolean) => Promise<PeerForgetResult[]>;
+}
+export declare function createPeerForgetRoutes(deps: {
+    peerForget: PeerForgetRouteDeps;
+    rbac: RbacEnforcer;
+    accessLog?: AccessLogWriter;
+}): Hono<AdminEnv>;
+export {};
+//# sourceMappingURL=admin-peer-forget.d.ts.map

package/dist/routes/admin-scaler-orphans.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Live orphaned Firecracker VM admin routes, behind `kici-admin scaler orphans`.
+ *
+ * Mounted inside `createAdminRoutes` at `/api/v1/admin`, so the Bearer-token
+ * auth middleware has already resolved the caller's role. Both routes need an
+ * unscoped token: a VM belongs to a host, not to one routing key.
+ *
+ * - `GET  /api/v1/admin/scaler/orphans?target=<instance-id>&timeoutMs=<n>` —
+ *   every live Firecracker VM on the node, with how it is tracked
+ *   (`scaler.read`).
+ * - `POST /api/v1/admin/scaler/orphans/stop { target?, vmIds, timeoutMs? }` —
+ *   stop the approved VMs nothing tracks (`scaler.manage`).
+ *
+ * The node is this coordinator, or the peer `target` names, reached over the
+ * authenticated peer channel. The coordinator adds the one tracker only it
+ * can answer — a non-terminal job in its dispatch queue bound to the VM's
+ * agent — and never forwards a bound VM to the node's stop.
+ */
+import { Hono } from 'hono';
+import { type PeerScalerOrphansResponse } from '@kici-dev/engine';
+import type { AccessLogWriter } from '../audit/access-log.js';
+import type { RbacEnforcer, Role } from '../secrets/rbac.js';
+import { type ScalerOrphansAnswer, type ScalerOrphansRequest } from '../scaler/orphan-requests.js';
+import { ScalerOrphansForwardFailure } from '../cluster/scaler-orphans-peer.js';
+type AdminEnv = {
+    Variables: {
+        role: Role;
+        userId: string;
+        routingKey: string | null;
+    };
+};
+/** What the routes need from the coordinator they run on. */
+export interface ScalerOrphansRouteDeps {
+    /** This coordinator's instance id. */
+    instanceId: string;
+    role: 'coordinator' | 'worker';
+    /** The cluster role a connected peer registered with, if known. */
+    peerRole: (instanceId: string) => 'coordinator' | 'worker' | undefined;
+    /** Answer from this coordinator's own host and tracking. */
+    answerLocal: (req: ScalerOrphansRequest) => Promise<ScalerOrphansAnswer>;
+    /** Forward to a peer over the peer channel. */
+    forward: (target: string, req: ScalerOrphansRequest, timeoutMs: number) => Promise<PeerScalerOrphansResponse | ScalerOrphansForwardFailure>;
+    /** Non-terminal dispatch-queue jobs bound to these agent ids, keyed by agent id. */
+    findBindings: (agentIds: string[]) => Promise<Map<string, string>>;
+    /**
+     * The cluster instance each of these agent ids is registered with — this
+     * coordinator or a peer whose heartbeat lists it — keyed by agent id. An
+     * agent can register with an HA sibling of the node that runs its VM.
+     */
+    findRegistrations: (agentIds: string[]) => Map<string, string>;
+    /**
+     * The coordinator peers this cluster expects but has no live link to. Their
+     * registered agents are unknown, so while any is missing no VM can be shown
+     * to be untracked.
+     */
+    disconnectedCoordinators: () => string[];
+}
+export declare function createScalerOrphansRoutes(deps: {
+    orphans: ScalerOrphansRouteDeps;
+    rbac: RbacEnforcer;
+    accessLog?: AccessLogWriter;
+}): Hono<AdminEnv>;
+export {};
+//# sourceMappingURL=admin-scaler-orphans.d.ts.map

package/dist/routes/admin.d.ts

@@ -18,6 +18,9 @@ import type { AgentRegistry } from '../agent/registry.js';
 import type { SharedConfigStore } from '../config/shared-store.js';
 import type { ResolvedMasterKeys } from '../secrets/config.js';
 import { type HeldRunReleaseWiring } from './admin-held-runs.js';
+import { type ClusterSettingsPropagationSource } from './admin-cluster-settings.js';
+import { type ScalerOrphansRouteDeps } from './admin-scaler-orphans.js';
+import { type PeerForgetRouteDeps } from './admin-peer-forget.js';
 import type { DrainController } from '../drain/drain-controller.js';
 import type { SourceStore } from '../sources/source-store.js';
 import type { JoinTokenManager } from '../cluster/join-token.js';
@@ -159,6 +162,26 @@ export interface AdminRouteDeps {
      * write is best-effort, never gating.
      */
     accessLog?: AccessLogWriter;
+    /**
+     * Optional -- the coordinator's live-VM listing and stop, for this node or a
+     * `--target` peer. Backs `GET /api/v1/admin/scaler/orphans` and
+     * `POST /api/v1/admin/scaler/orphans/stop` (`kici-admin scaler orphans`).
+     * Bound by orchestrator-core once the cluster is up.
+     */
+    scalerOrphans?: ScalerOrphansRouteDeps;
+    /**
+     * Optional -- forget a departed peer here and on every connected sibling
+     * coordinator. Backs `POST /api/v1/admin/peers/forget` (`kici-admin peer
+     * forget`). Bound by orchestrator-core once the cluster is up.
+     */
+    peerForget?: PeerForgetRouteDeps;
+    /**
+     * Optional -- the peer view behind
+     * `GET /api/v1/admin/cluster-settings/propagation` (the propagation section of
+     * `kici-admin cluster-settings show`). Bound by orchestrator-core once the
+     * cluster is up; the route is not mounted without it.
+     */
+    clusterSettingsPropagation?: ClusterSettingsPropagationSource;
     /**
      * Optional -- fulfil deferred attestations on demand (mints in the running
      * orchestrator process, which owns the Platform WS). Backs

package/dist/scaler/backend-factory.d.ts

@@ -38,7 +38,12 @@ export interface BackendFactoryContext {
     tokenTtlMs?: number;
     /** Live per-spawn resolver for the fleet-wide agent-token TTL. */
     tokenTtlProvider: () => Promise<number>;
-    /** DB-backed on the leader, in-memory on the worker. */
+    /**
+     * Live per-spawn resolver for the fleet-wide Firecracker API-socket wait.
+     * A coordinator reads `cluster_settings`; a worker reads its pulled snapshot.
+     */
+    firecrackerApiSocketWaitMsProvider: () => Promise<number>;
+    /** DB-backed on a coordinator, in-memory on a worker. */
     ipAllocator: (params: IpAllocatorParams) => IpAllocator;
     /**
      * The already-constructed scaler state store. The event backend's claim

package/dist/scaler/bare-metal-backend.d.ts

@@ -2,7 +2,8 @@
  * Bare-metal scaler backend implementation.
  *
  * Manages ephemeral agent processes using child_process.spawn.
- * Uses detached process groups for clean killing of entire process trees.
+ * On Linux and macOS each agent leads its own process group, which teardown
+ * signals; on Windows teardown ends the agent's process tree with taskkill.
  * All agents are single-use: destroyed after job completion.
  */
 import { type ToolRequirement } from '@kici-dev/shared';
@@ -98,6 +99,11 @@ export declare class BareMetalScalerBackend implements ScalerBackend {
     get labelSets(): LabelSetConfig[];
     getActiveCount(): number;
     spawn(labelSet: string[], agentId: string, orchestratorUrl: string, onEvent?: ScalerEventCallback, effectiveLimits?: EffectiveLimits, spawnContext?: SpawnContext, signal?: AbortSignal): Promise<ManagedAgent>;
+    /**
+     * Record that the agent registered. From then on its exit is a disconnect
+     * the manager handles, not a failed spawn.
+     */
+    markRegistered(managedId: string): void;
     /**
      * Launch the agent inside the job's own container image.
      *
@@ -145,23 +151,19 @@ export declare class BareMetalScalerBackend implements ScalerBackend {
         entry?: ScalerEntry;
     }): ValidationResult;
     /**
-     * Build the (command, args) tuple used to spawn an agent process.
+     * End a Windows agent's whole process tree.
      *
-     * Two modes:
-     *   - Direct: returns (binaryPath, []) — the historical bare-metal flow.
-     *     Used when enforceCgroups is false, the host is non-Linux, or the
-     *     resolved limits carry no positive cpus/memBytes.
-     *   - systemd-run scope: returns ('systemd-run', [...]) wrapping the
-     *     binary in a transient `--user --scope --slice=kici-scaler` cgroup
-     *     with `CPUQuota=` / `MemoryMax=` properties translated from the
-     *     effective limits. The scope name embeds `agentId` so each agent is
-     *     a separately-named transient unit (and `systemctl --user status
-     *     kici-agent-<agentId>.scope` works).
+     * Windows has no process groups to signal and no SIGTERM, and Node.js cannot
+     * send a ctrl-C there (`process.kill` terminates the target unconditionally),
+     * so the tree is ended at once with `taskkill /T /F`. The pid is cmd.exe's, which runs the agent's `node` and
+     * everything it started.
      *
-     * `CPUQuota` is expressed as a percent (1.0 cpus = 100%). `MemoryMax`
-     * accepts a raw byte count.
+     * Nothing runs once Node.js has seen the child exit: libuv then releases its
+     * handle on the process, Windows may give the pid to an unrelated process,
+     * and `/T` would end that process's tree. While the child runs, the open
+     * handle keeps the pid ours.
      */
-    private buildSpawnInvocation;
+    private endWindowsTree;
     /**
      * Wait for a process to exit within a timeout.
      * Returns true if the process exited, false if timed out.