@kici-dev/orchestrator 0.13.0 → 0.14.1

package/dist/cluster/cluster-settings-reader.d.ts

@@ -1,7 +1,7 @@
 import type { Kysely } from 'kysely';
-import type { Database } from '../db/types.js';
+import type { Database, ClusterSettings } from '../db/types.js';
 /** Numeric columns on cluster_settings readable via {@link ClusterSettingsReader}. */
-export type ClusterNumberColumn = 'max_github_payload_bytes' | 'event_log_max_payload_bytes' | 'lock_file_max_bytes' | 'webhook_dedup_ttl_ms' | 'event_router_event_ttl_seconds' | 'event_router_max_dispatch_attempts' | 'queue_max_depth' | 'reroute_flap_grace_ms' | 'max_fanout_hosts' | 'event_router_rate_limit_per_workflow_per_minute' | 'cache_max_tarball_bytes' | 'cache_ttl_days' | 'lockfile_cache_max' | 'lockfile_cache_max_bytes' | 'lockfile_cache_ttl_ms' | 'content_cache_max' | 'content_cache_max_bytes' | 'content_cache_ttl_ms' | 'global_eval_round_timeout_ms' | 'global_eval_candidate_timeout_ms' | 'global_eval_cache_max' | 'global_eval_wait_timeout_ms' | 'check_run_tracking_ttl_days' | 'run_retention_days' | 'audit_retention_days' | 'provenance_retention_days' | 'held_run_retention_days' | 'concurrency_wait_timeout_ms' | 'agent_token_ttl_ms' | 'ownership_db_check_timeout_ms' | 'sealed_secrets_retry_backoff_ms' | 'unroutable_grace_ms' | 'ingest_overflow_claim_timeout_ms' | 'scaler_reap_interval_ms' | 'scaler_reap_stranded_timeout_ms' | 'scaler_reap_reattempt_interval_ms' | 'scaler_claim_retention_ms' | 'scaler_provision_backoff_base_ms' | 'scaler_provision_backoff_max_ms' | 'scaler_provision_max_consecutive_failures';
+export type ClusterNumberColumn = 'max_github_payload_bytes' | 'event_log_max_payload_bytes' | 'lock_file_max_bytes' | 'webhook_dedup_ttl_ms' | 'event_router_event_ttl_seconds' | 'event_router_max_dispatch_attempts' | 'queue_max_depth' | 'reroute_flap_grace_ms' | 'max_fanout_hosts' | 'event_router_rate_limit_per_workflow_per_minute' | 'cache_max_tarball_bytes' | 'cache_ttl_days' | 'lockfile_cache_max' | 'lockfile_cache_max_bytes' | 'lockfile_cache_ttl_ms' | 'content_cache_max' | 'content_cache_max_bytes' | 'content_cache_ttl_ms' | 'global_eval_round_timeout_ms' | 'global_eval_candidate_timeout_ms' | 'global_eval_cache_max' | 'global_eval_wait_timeout_ms' | 'firecracker_api_socket_wait_ms' | 'check_run_tracking_ttl_days' | 'run_retention_days' | 'audit_retention_days' | 'provenance_retention_days' | 'held_run_retention_days' | 'concurrency_wait_timeout_ms' | 'agent_token_ttl_ms' | 'ownership_db_check_timeout_ms' | 'sealed_secrets_retry_backoff_ms' | 'unroutable_grace_ms' | 'ingest_overflow_claim_timeout_ms' | 'scaler_reap_interval_ms' | 'scaler_reap_stranded_timeout_ms' | 'scaler_reap_reattempt_interval_ms' | 'scaler_claim_retention_ms' | 'scaler_provision_backoff_base_ms' | 'scaler_provision_backoff_max_ms' | 'scaler_provision_max_consecutive_failures';
 /** Text columns on cluster_settings readable via {@link ClusterSettingsReader}. */
 export type ClusterStringColumn = 'dashboard_verified_issuer';
 /** Boolean columns on cluster_settings readable via {@link ClusterSettingsReader}. */
@@ -88,6 +88,12 @@ export declare class ClusterSettingsReader {
      */
     private loadSnapshot;
     private loadRow;
+    /**
+     * The whole row from one read (undefined when there is no row or the read
+     * failed). For a caller that needs several columns that agree with each
+     * other, such as a value and the row version it belongs to.
+     */
+    getRow(): Promise<ClusterSettings | undefined>;
     /** Resolve a fleet-wide numeric knob; `fallback` is the config.ts cluster default. */
     getNumber(column: ClusterNumberColumn, fallback: number): Promise<number>;
     /**

package/dist/cluster/coordinator.d.ts

@@ -10,7 +10,7 @@
  * - "Peers report step-by-step progress back to coordinator"
  * - "Cancel mode: graceful -- finish current step, cancel remaining"
  */
-import type { JobReroute, JobProgress, JobProgressAck, PeerScalerEvent, PeerToPeerMessage, ResourceRequest, LabelMatcher } from '@kici-dev/engine';
+import type { JobReroute, JobProgress, JobProgressAck, PeerScalerEvent, PeerToPeerMessage, RerouteSpawnRetry, ResourceRequest, LabelMatcher } from '@kici-dev/engine';
 import type { PeerRegistry } from './peer-registry.js';
 import type { PeerClient } from './peer-client.js';
 import type { Dispatcher } from '../agent/dispatcher.js';
@@ -114,6 +114,14 @@ export interface RunCoordinatorDeps {
      * {@link DEFAULT_MAX_HOPS}.
      */
     getRerouteMaxHops?: (job: JobToRoute) => Promise<number>;
+    /**
+     * Per-job spawn-retry budget sent to the receiving peer on `job.reroute` and
+     * enforced here on the peer's relayed spawn failures. Resolves the job's org
+     * overrides from `org_settings.reroute_spawn_max_attempts` /
+     * `reroute_spawn_retry_backoff_ms`, else the cluster defaults. When absent,
+     * {@link DEFAULT_REROUTE_SPAWN_RETRY} is used.
+     */
+    getRerouteSpawnRetry?: (job: JobToRoute) => Promise<RerouteSpawnRetry>;
 }
 export declare class RunCoordinator {
     private readonly instanceId;
@@ -128,6 +136,7 @@ export declare class RunCoordinator {
     private readonly getRerouteSpawnWindowMs;
     private readonly getRerouteAckTimeoutMs;
     private readonly getRerouteMaxHops;
+    private readonly getRerouteSpawnRetry;
     /**
      * Tracks which jobs have been rerouted to which peers, keyed by runId.
      * Used for cancel propagation, progress tracking, and the post-ACK spawn
@@ -193,12 +202,18 @@ export declare class RunCoordinator {
      * line + dispatch-queue last-error column) just as it would for a
      * locally-emitted scaler event.
      *
-     * Layer B fast path: a `scaler.failed` for a tracked rerouted job whose spawn
-     * window is still armed is the worker's NAK-after-accept — the peer accepted
-     * the reroute but its agent spawn failed asynchronously before reporting any
-     * progress. Re-dispatch immediately (the same routine the spawn-window timer
-     * runs) instead of waiting the window out. Idempotent with the Layer A timer:
-     * whichever fires first removes the tracking entry.
+     * Layer B: a `scaler.failed` for a tracked rerouted job whose spawn window is
+     * still armed is the worker reporting that an agent spawn failed after it
+     * accepted the reroute. The worker retries within the spawn-retry budget sent on
+     * `job.reroute`, and the relay carries its verdict:
+     *
+     * - `final: true` — the worker gave the job back. Re-dispatch immediately (the
+     *   same routine the spawn-window timer runs). Idempotent with the Layer A
+     *   timer: whichever fires first removes the tracking entry.
+     * - `final: false` — the worker is retrying. Re-arm the window for one more
+     *   attempt plus the backoff; treat the `maxAttempts`-th such relay as final.
+     * - no verdict — an older worker that retries without a bound. The window
+     *   armed at the reroute keeps running unchanged.
      *
      * Two guards make the re-dispatch safe. First, source provenance: the failure
      * must come from `fromPeerId === tracked.peerId` (the authenticated connection
@@ -211,6 +226,16 @@ export declare class RunCoordinator {
      * correct regardless of which peer relayed the event.
      */
     onPeerScalerEvent(msg: PeerScalerEvent, fromPeerId: string): void;
+    /**
+     * A worker's verdict on a failed spawn for a job it holds. `final: true` means it
+     * gave the job back, so re-dispatch now. `final: false` means it is retrying, so
+     * restart the window for one more attempt plus the backoff this coordinator told it
+     * to wait. No verdict means a worker that retries without a bound, so leave the
+     * window as it is. The count caps a worker that never says `final`.
+     */
+    private onRelayedSpawnFailure;
+    /** Restart a rerouted job's spawn window for one more worker attempt plus its backoff. */
+    private rearmAfterSpawnRetry;
     /**
      * Handle peer job completion. Updates ExecutionTracker.
      * The execution tracker's onJobStatus fires the needs-aware scheduler hook,
@@ -317,7 +342,8 @@ export declare class RunCoordinator {
     /**
      * Post-ACK spawn-window backstop. Fires when a rerouted job's peer accepted
      * but produced no progress within the window (Layer A), or immediately when a
-     * worker relays a `scaler.failed` for the job (Layer B, via onPeerScalerEvent).
+     * worker relays its final spawn-failure verdict for the job (Layer B, via
+     * onPeerScalerEvent).
      * Best-effort cancels the original peer (double-execution guard against a
      * slow-but-healthy spawn), then re-runs routing to another peer, then a local
      * fallback, and finally records the job failed rather than leaving it pending.

package/dist/cluster/peer-client.d.ts

@@ -10,8 +10,10 @@
  * Authentication uses ECDH key exchange followed by join token (first connect)
  * or HMAC credential proof (reconnection).
  */
-import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type JobProgressAck, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerClusterSettingsRequest, type PeerClusterSettingsResponse, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
+import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type JobProgressAck, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerScalerOrphansRequest, type PeerForgetRequest, type PeerForgetResponse, type PeerScalerOrphansResponse, type PeerClusterSettingsRequest, type PeerClusterSettingsResponse, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
 import type { PeerRegistry } from './peer-registry.js';
+import { type PEER_FORGET_TIMEOUT, type PeerForgetRequestHandler } from './peer-forget.js';
+import { type SCALER_ORPHANS_TIMEOUT, type ScalerOrphansRequestHandler } from './scaler-orphans-peer.js';
 import type { PeerAuthCoordinator } from './peer-auth-coordinator.js';
 /** Logged when this instance has neither a credential file nor a join token. */
 export declare const NO_AUTH_METHOD_MESSAGE = "No peer auth method: this instance has no credential file and no join token";
@@ -97,6 +99,17 @@ export interface PeerClientOptions {
      * If undefined, incoming collect requests are ignored.
      */
     onLogsCollectRequest?: PeerLogsCollectResponder;
+    /**
+     * Answers a peer.scaler.orphans.request the connected coordinator forwarded
+     * to this node (`kici-admin scaler orphans --target`). If undefined,
+     * requests are answered with ok=false.
+     */
+    onScalerOrphansRequest?: ScalerOrphansRequestHandler;
+    /**
+     * Forgets a departed peer the connected coordinator forgot (`kici-admin peer
+     * forget`). If undefined, requests are answered with outcome `error`.
+     */
+    onPeerForgetRequest?: PeerForgetRequestHandler;
 }
 /**
  * Builds a node's subtree bundle in response to a peer.logs.collect.request and
@@ -118,6 +131,8 @@ export declare class PeerClient {
     private readonly ackWaiters;
     private readonly cacheWaiters;
     private readonly configReloadWaiters;
+    private readonly scalerOrphansWaiters;
+    private readonly peerForgetWaiters;
     private readonly clusterSettingsWaiters;
     /** Correlates peer.logs.collect.request with the peer's chunked subtree response. */
     private readonly logsCollectWaiters;
@@ -150,6 +165,8 @@ export declare class PeerClient {
     private readonly onPeerConfigReload?;
     private readonly onAuthenticated?;
     private readonly onLogsCollectRequest?;
+    private readonly onScalerOrphansRequest?;
+    private readonly onPeerForgetRequest?;
     constructor(options: PeerClientOptions);
     /** Current connection state. */
     get state(): PeerConnectionState;
@@ -189,10 +206,26 @@ export declare class PeerClient {
      */
     sendConfigReloadAndWait(msg: PeerConfigReload, timeoutMs?: number): Promise<PeerConfigReloadResponse | null>;
     /**
-     * Send a peer.clusterSettings.request to the connected leader and wait for the
-     * matching peer.clusterSettings.response carrying the worker-settings snapshot.
+     * Send a peer.forget.request to the connected peer and wait for the matching
+     * response.
+     *
+     * @returns the response; null when not connected; `'timeout'` when no
+     *   response arrived within `timeoutMs`.
+     */
+    sendPeerForgetAndWait(msg: PeerForgetRequest, timeoutMs: number): Promise<PeerForgetResponse | null | typeof PEER_FORGET_TIMEOUT>;
+    /**
+     * Send a peer.scaler.orphans.request to the connected peer and wait for the
+     * matching response.
+     *
+     * @returns the response; null when not connected; `'timeout'` when no
+     *   response arrived within `timeoutMs`.
+     */
+    sendScalerOrphansAndWait(msg: PeerScalerOrphansRequest, timeoutMs: number): Promise<PeerScalerOrphansResponse | null | typeof SCALER_ORPHANS_TIMEOUT>;
+    /**
+     * Send a peer.clusterSettings.request to the connected coordinator and wait for
+     * the matching peer.clusterSettings.response carrying the worker-settings snapshot.
      *
-     * @returns The response, or null if not connected or the leader doesn't reply
+     * @returns The response, or null if not connected or the coordinator doesn't reply
      *   within the timeout — the caller keeps its current (config-default) settings.
      */
     sendClusterSettingsRequestAndWait(msg: PeerClusterSettingsRequest, timeoutMs?: number): Promise<PeerClusterSettingsResponse | null>;

package/dist/cluster/peer-forget.d.ts

@@ -0,0 +1,138 @@
+import { PeerForgetOutcome, type PeerForgetRequest, type PeerForgetResponse } from '@kici-dev/engine';
+import { z } from 'zod';
+import type { PeerRegistry } from './peer-registry.js';
+/** What forgetting did on one coordinator. */
+export interface PeerForgetResult {
+    /** The coordinator that answered. */
+    coordinator: string;
+    outcome: PeerForgetOutcome;
+    detail: string;
+}
+/** Local outcomes that stop a forget before it reaches any sibling. */
+export declare const LOCAL_REFUSALS: ReadonlySet<PeerForgetOutcome>;
+/** A sibling's answer that never arrived, or a sibling no connection reached. */
+export declare const PEER_FORGET_TIMEOUT = "timeout";
+/** The answer to a forget request a peer that is not a coordinator sent. */
+export declare const PEER_FORGET_COORDINATORS_ONLY = "peer forget requests are accepted from coordinators only";
+/** The answer of a peer with no forget handler wired. */
+export declare const PEER_FORGET_UNHANDLED = "peer forget requests are not handled by this peer";
+/** Which window a coordinator keeps treating a silent peer as alive for. */
+export declare const PeerLivenessWindowKind: z.ZodEnum<{
+    "adopter-grace": "adopter-grace";
+    "reroute-grace": "reroute-grace";
+    "stale-window": "stale-window";
+}>;
+export type PeerLivenessWindowKind = z.infer<typeof PeerLivenessWindowKind>;
+/** How long after its last heartbeat a coordinator still treats a peer as alive. */
+export interface PeerLivenessWindow {
+    kind: PeerLivenessWindowKind;
+    windowMs: number;
+}
+/** What a coordinator reads to choose a peer's liveness window. */
+export interface PeerLivenessSource {
+    /** The peer stale window (`KICI_CLUSTER_PEER_STALE_TIMEOUT_MS`). */
+    staleTimeoutMs: number;
+    /** Reads the live `reroute_flap_grace_ms` setting. */
+    rerouteFlapGraceMs: () => Promise<number>;
+    /**
+     * This coordinator's event-provision backstop, or null when it runs none.
+     * `holdsAdoptedProvisions` reads whether a peer adopted provisions the
+     * backstop judges by that peer's liveness.
+     */
+    backstop: {
+        holdsAdoptedProvisions: (instanceId: string) => Promise<boolean>;
+    } | null;
+}
+/**
+ * The longest window in which this coordinator still treats `instanceId` as
+ * alive after its last heartbeat. Forgetting the peer inside it would cut that
+ * grace short: every reader of the registry treats a peer it no longer knows
+ * as gone at once.
+ *
+ * - A peer that adopted provisions gets the event-provision backstop's own
+ *   grace (`reaperFlapGraceMs`), or the backstop tears its instances down.
+ * - Any other peer gets the longer of the stale window and the reroute flap
+ *   grace, which the rerouted-job guard and the master-key requeue give every
+ *   disconnected peer whether or not this coordinator runs a backstop.
+ */
+export declare function peerLivenessWindow(instanceId: string, source: PeerLivenessSource): Promise<PeerLivenessWindow>;
+/** The checks a coordinator runs before it forgets a peer. */
+export interface PeerForgetGuards {
+    /**
+     * How long after its last heartbeat the peer is still treated as alive. A
+     * peer heard from inside it may be partitioned rather than gone, so it is kept.
+     */
+    liveness: PeerLivenessWindow;
+    /**
+     * This coordinator's event-provision backstop, or null when it runs none.
+     * `configuredPeerCount` is the `KICI_CLUSTER_PEERS` count the backstop's
+     * cluster-view check reads beside the live registry.
+     */
+    backstop: {
+        configuredPeerCount: number;
+    } | null;
+    /** The operator acknowledged that the forget may switch the backstop back on. */
+    acknowledgeBackstop: boolean;
+    /** The current time; tests pin it. */
+    nowMs?: number;
+}
+/**
+ * The guards for forgetting `instanceId` on one coordinator, built from its
+ * cluster config, the live reroute flap grace and its event-provision
+ * backstop. The backstop runs on every coordinator that runs a scaler manager.
+ */
+export declare function peerForgetGuards(instanceId: string, cluster: {
+    peerStaleTimeoutMs: number;
+    peers: readonly unknown[];
+}, liveness: Omit<PeerLivenessSource, 'staleTimeoutMs'>, acknowledgeBackstop: boolean): Promise<PeerForgetGuards>;
+/** What forgetting a peer that would switch the backstop back on does. */
+export declare function backstopConsequence(instanceId: string): string;
+/**
+ * Drop `instanceId` from `registry` when it names a peer that left the
+ * cluster. Kept, with the reason: a connected peer (its next heartbeat would
+ * register it again), a peer heard from inside its liveness window (it may be
+ * partitioned rather than gone; see `peerLivenessWindow`), and a peer whose removal would switch the
+ * event-provision backstop back on, unless the request acknowledged that.
+ */
+export declare function forgetDepartedPeer(registry: PeerRegistry, selfInstanceId: string, instanceId: string, guards: PeerForgetGuards): Omit<PeerForgetResult, 'coordinator'>;
+/** Answers a forget request a sibling coordinator fanned out. */
+export type PeerForgetRequestHandler = (msg: PeerForgetRequest) => Promise<{
+    outcome: PeerForgetOutcome;
+    detail: string;
+}>;
+/**
+ * Answer one forget request with `handler`. A coordinator with no handler
+ * answers {@link PEER_FORGET_UNHANDLED}, and a handler that fails (its guards
+ * read the database) answers `error`, so the peer is kept and the operator
+ * sees why.
+ */
+export declare function replyToPeerForgetRequest(msg: PeerForgetRequest, handler: PeerForgetRequestHandler | undefined, send: (response: PeerForgetResponse) => void, logFields?: Record<string, unknown>): void;
+/**
+ * Sends one forget request to a sibling coordinator and waits for its answer.
+ * Resolves null when no connection reached the sibling, and
+ * {@link PEER_FORGET_TIMEOUT} when it did not answer within `timeoutMs`.
+ */
+export type SendPeerForget = (siblingInstanceId: string, msg: PeerForgetRequest, timeoutMs: number) => Promise<PeerForgetResponse | null | typeof PEER_FORGET_TIMEOUT>;
+/**
+ * Forget `instanceId` on this coordinator, then on every connected sibling
+ * coordinator, each of which holds its own registry. A peer this coordinator
+ * still has connected is refused and not fanned out. The first result is this
+ * coordinator's.
+ */
+export declare function forgetAcrossCoordinators(registry: PeerRegistry, selfInstanceId: string, instanceId: string, timeoutMs: number, guards: PeerForgetGuards, send: SendPeerForget): Promise<PeerForgetResult[]>;
+/**
+ * The coordinator peers the cluster expects but this coordinator has no live
+ * link to: registered coordinators whose connection is down.
+ */
+export declare function disconnectedCoordinatorIds(registry: PeerRegistry, selfInstanceId: string): string[];
+/** Pending forget requests sent to siblings, keyed by message id. */
+export declare class PeerForgetWaiters {
+    private readonly waiters;
+    /** Wait for the response to `messageId`, or {@link PEER_FORGET_TIMEOUT}. */
+    wait(messageId: string, timeoutMs: number): Promise<PeerForgetResponse | typeof PEER_FORGET_TIMEOUT>;
+    /** Hand a response to the request waiting for it. An unknown id is dropped. */
+    resolve(response: PeerForgetResponse): void;
+    /** End every pending wait with an `error` response naming `reason`. */
+    rejectAll(reason: string): void;
+}
+//# sourceMappingURL=peer-forget.d.ts.map

package/dist/cluster/peer-handler.d.ts

@@ -6,8 +6,10 @@
  * proof (reconnection), registers the peer in PeerRegistry, and routes
  * messages bidirectionally. Sends periodic heartbeats to the connecting peer.
  */
-import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type JobProgressAck, type PeerScalerEvent, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerClusterSettingsRequest, type WorkerClusterSettings, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
+import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type JobProgressAck, type PeerScalerEvent, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerScalerOrphansRequest, type PeerForgetRequest, type PeerForgetResponse, type PeerScalerOrphansResponse, type PeerClusterSettingsRequest, type WorkerClusterSettings, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
 import type { PeerRegistry } from './peer-registry.js';
+import { type PEER_FORGET_TIMEOUT, type PeerForgetRequestHandler } from './peer-forget.js';
+import { type SCALER_ORPHANS_TIMEOUT, type ScalerOrphansRequestHandler } from './scaler-orphans-peer.js';
 import type { PeerCredentialStore } from './peer-credentials.js';
 import { type JoinTokenManager } from './join-token.js';
 /**
@@ -104,6 +106,19 @@ export interface PeerHandlerDeps {
      * If undefined, incoming collect requests are ignored.
      */
     onLogsCollectRequest?: (msg: PeerLogsCollectRequest, send: (out: PeerToPeerMessage) => boolean) => Promise<void>;
+    /**
+     * Answers a peer.scaler.orphans.request a coordinator forwarded to this node
+     * (`kici-admin scaler orphans --target`). A request from a peer that is not
+     * a coordinator is refused before it reaches the handler. If undefined,
+     * requests are answered with ok=false.
+     */
+    onScalerOrphansRequest?: ScalerOrphansRequestHandler;
+    /**
+     * Forgets a departed peer a sibling coordinator forgot (`kici-admin peer
+     * forget`). A request from a peer that is not a coordinator is refused. If
+     * undefined, requests are answered with outcome `error`.
+     */
+    onPeerForgetRequest?: PeerForgetRequestHandler;
 }
 /**
  * Create a handler function for incoming peer WebSocket connections.
@@ -127,6 +142,8 @@ export declare function createPeerHandler(deps: PeerHandlerDeps): {
     closePeer: (targetInstanceId: string, code: number, reason: string) => void;
     sendAndWaitAck: (targetInstanceId: string, msg: JobReroute, timeoutMs?: number) => Promise<boolean>;
     sendConfigReloadAndWait: (targetInstanceId: string, msg: PeerConfigReload, timeoutMs?: number) => Promise<PeerConfigReloadResponse | null>;
+    sendScalerOrphansAndWait: (targetInstanceId: string, msg: PeerScalerOrphansRequest, timeoutMs: number) => Promise<PeerScalerOrphansResponse | null | typeof SCALER_ORPHANS_TIMEOUT>;
+    sendPeerForgetAndWait: (targetInstanceId: string, msg: PeerForgetRequest, timeoutMs: number) => Promise<PeerForgetResponse | null | typeof PEER_FORGET_TIMEOUT>;
     sendLogsCollectAndWait: (targetInstanceId: string, msg: PeerLogsCollectRequest, timeoutMs: number) => Promise<Buffer>;
     getConnectionCount: () => number;
     broadcastHeartbeat: (inventory: Omit<PeerHeartbeat, 'type'>) => void;

package/dist/cluster/scaler-orphans-peer.d.ts

@@ -0,0 +1,40 @@
+/**
+ * The peer-channel half of `kici-admin scaler orphans --target`: answering a
+ * forwarded peer.scaler.orphans.request, and correlating the response with
+ * the request that is waiting for it. Shared by the outgoing `PeerClient` and
+ * the incoming peer handler, which carry the same message pair.
+ */
+import { z } from 'zod';
+import type { PeerScalerOrphansRequest, PeerScalerOrphansResponse } from '@kici-dev/engine';
+import type { ScalerOrphansAnswer } from '../scaler/orphan-requests.js';
+/** Answers a forwarded request from this node's own host and tracking. */
+export type ScalerOrphansRequestHandler = (msg: PeerScalerOrphansRequest) => Promise<ScalerOrphansAnswer>;
+/** The answer of a peer that has no handler wired. */
+export declare const SCALER_ORPHANS_UNHANDLED_ERROR = "scaler orphan requests are not handled by this peer";
+/** The answer to a request a worker sent: a worker never stops another node's VMs. */
+export declare const SCALER_ORPHANS_COORDINATORS_ONLY_ERROR = "scaler orphan requests are accepted from coordinators only";
+/** A forward that reached no peer, or a peer that did not answer in time. */
+export declare const ScalerOrphansForwardFailure: z.ZodEnum<{
+    "not-connected": "not-connected";
+    timeout: "timeout";
+}>;
+export type ScalerOrphansForwardFailure = z.infer<typeof ScalerOrphansForwardFailure>;
+/** A wait that ended without a response. */
+export declare const SCALER_ORPHANS_TIMEOUT: "timeout";
+/**
+ * Answer one forwarded request through `send`. A missing handler and a
+ * handler failure are both answered `ok: false`, so the requester never waits
+ * out its timeout for a peer that cannot answer.
+ */
+export declare function replyToScalerOrphansRequest(msg: PeerScalerOrphansRequest, handler: ScalerOrphansRequestHandler | undefined, send: (response: PeerScalerOrphansResponse) => void, logFields?: Record<string, unknown>): void;
+/** Pending scaler orphan requests, keyed by message id. */
+export declare class ScalerOrphansWaiters {
+    private readonly waiters;
+    /** Wait for the response to `messageId`, or {@link SCALER_ORPHANS_TIMEOUT}. */
+    wait(messageId: string, timeoutMs: number): Promise<PeerScalerOrphansResponse | typeof SCALER_ORPHANS_TIMEOUT>;
+    /** Hand a response to the request waiting for it. An unknown id is dropped. */
+    resolve(response: PeerScalerOrphansResponse): void;
+    /** End every pending wait with an `ok: false` response naming `reason`. */
+    rejectAll(reason: string): void;
+}
+//# sourceMappingURL=scaler-orphans-peer.d.ts.map

package/dist/cluster/settings-propagation.d.ts

@@ -0,0 +1,85 @@
+/**
+ * Which cluster-settings version each orchestrator in a cluster has applied,
+ * compared with the version of the shared `cluster_settings` row.
+ *
+ * A worker reports the version of the snapshot it pulled, which is the one its
+ * next spawn reads. A coordinator reports the version its settings reader
+ * currently serves. Pure, because `kici-admin cluster-settings show` imports
+ * it to validate and render the report the admin route builds.
+ */
+import { z } from 'zod';
+import type { PeerInfo } from './peer-registry.js';
+export declare const settingsPropagationStatusSchema: z.ZodEnum<{
+    ahead: "ahead";
+    behind: "behind";
+    "in-sync": "in-sync";
+}>;
+export type SettingsPropagationStatus = z.infer<typeof settingsPropagationStatusSchema>;
+export declare const SettingsPropagationStatus: {
+    ahead: "ahead";
+    behind: "behind";
+    "in-sync": "in-sync";
+};
+export declare const settingsPropagationRoleSchema: z.ZodEnum<{
+    coordinator: "coordinator";
+    worker: "worker";
+}>;
+export declare const settingsPropagationEntrySchema: z.ZodObject<{
+    instanceId: z.ZodString;
+    role: z.ZodEnum<{
+        coordinator: "coordinator";
+        worker: "worker";
+    }>;
+    self: z.ZodBoolean;
+    connected: z.ZodBoolean;
+    appliedVersion: z.ZodNumber;
+    status: z.ZodEnum<{
+        ahead: "ahead";
+        behind: "behind";
+        "in-sync": "in-sync";
+    }>;
+    lastHeartbeatAt: z.ZodNullable<z.ZodString>;
+    lastHeartbeatAgeMs: z.ZodNullable<z.ZodNumber>;
+}, z.core.$loose>;
+export type SettingsPropagationEntry = z.infer<typeof settingsPropagationEntrySchema>;
+export declare const settingsPropagationReportSchema: z.ZodObject<{
+    currentVersion: z.ZodNumber;
+    reportedBy: z.ZodString;
+    generatedAt: z.ZodString;
+    orchestrators: z.ZodArray<z.ZodObject<{
+        instanceId: z.ZodString;
+        role: z.ZodEnum<{
+            coordinator: "coordinator";
+            worker: "worker";
+        }>;
+        self: z.ZodBoolean;
+        connected: z.ZodBoolean;
+        appliedVersion: z.ZodNumber;
+        status: z.ZodEnum<{
+            ahead: "ahead";
+            behind: "behind";
+            "in-sync": "in-sync";
+        }>;
+        lastHeartbeatAt: z.ZodNullable<z.ZodString>;
+        lastHeartbeatAgeMs: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$loose>>;
+}, z.core.$loose>;
+export type SettingsPropagationReport = z.infer<typeof settingsPropagationReportSchema>;
+/** The peer-registry fields a report reads. */
+export type PropagationPeer = Pick<PeerInfo, 'instanceId' | 'role' | 'connected' | 'lastHeartbeatAt' | 'clusterSettingsVersion'>;
+/**
+ * `ahead` means the row's version went backwards. A worker in that state never
+ * pulls again, because it pulls only when a coordinator advertises a higher
+ * version than its own.
+ */
+export declare function classifySettingsVersion(applied: number, current: number): SettingsPropagationStatus;
+export declare function buildSettingsPropagationReport(input: {
+    currentVersion: number;
+    self: {
+        instanceId: string;
+        appliedVersion: number;
+    };
+    peers: readonly PropagationPeer[];
+    now: number;
+}): SettingsPropagationReport;
+//# sourceMappingURL=settings-propagation.d.ts.map

package/dist/cluster/worker-cluster-settings.d.ts

@@ -0,0 +1,29 @@
+/**
+ * The cluster-settings snapshot a coordinator serves to DB-less workers.
+ *
+ * A worker cannot read `cluster_settings`, so every knob it applies at spawn
+ * reaches it through this snapshot, pulled over the peer channel whenever a
+ * coordinator advertises a newer settings version. Each value is the stored
+ * override, or the serving coordinator's configured default when the column
+ * is NULL.
+ */
+import type { WorkerClusterSettings } from '@kici-dev/engine';
+import type { ClusterSettingsReader } from './cluster-settings-reader.js';
+/** The configured defaults behind each snapshot field. */
+export interface WorkerClusterSettingDefaults {
+    agentTokenTtlMs: number;
+    firecrackerApiSocketWaitMs: number;
+}
+/** A worker snapshot and the `cluster_settings.version` its values belong to. */
+export interface WorkerClusterSettingsSnapshot {
+    version: number;
+    settings: WorkerClusterSettings;
+}
+/**
+ * Resolve the worker snapshot and its version from ONE read of the row. A
+ * worker stores the version with the values and pulls again only when a
+ * coordinator advertises a higher version, so a version read separately from
+ * the values could pair a newer version with older values for good.
+ */
+export declare function resolveWorkerClusterSettingsSnapshot(reader: Pick<ClusterSettingsReader, 'getRow'>, defaults: WorkerClusterSettingDefaults): Promise<WorkerClusterSettingsSnapshot>;
+//# sourceMappingURL=worker-cluster-settings.d.ts.map

package/dist/config.d.ts

@@ -72,6 +72,7 @@ declare const configSchema: z.ZodObject<{
     globalEvalCandidateTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     globalEvalCacheMax: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     globalEvalWaitTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    firecrackerApiSocketWaitMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     queueMaxDepth: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     queueTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     unroutableGraceMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
@@ -104,6 +105,8 @@ declare const configSchema: z.ZodObject<{
     rerouteSpawnWindowMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     rerouteAckTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     rerouteMaxHops: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    rerouteSpawnMaxAttempts: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    rerouteSpawnRetryBackoffMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     rerouteFlapGraceMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     scalerSpawnTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     backupStalenessWarnHours: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
@@ -283,6 +286,7 @@ export declare const packagingConfigSchema: z.ZodObject<{
     globalEvalCandidateTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     globalEvalCacheMax: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     globalEvalWaitTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    firecrackerApiSocketWaitMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     queueMaxDepth: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     queueTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     unroutableGraceMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
@@ -315,6 +319,8 @@ export declare const packagingConfigSchema: z.ZodObject<{
     rerouteSpawnWindowMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     rerouteAckTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     rerouteMaxHops: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    rerouteSpawnMaxAttempts: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    rerouteSpawnRetryBackoffMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     rerouteFlapGraceMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     scalerSpawnTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     backupStalenessWarnHours: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
@@ -527,6 +533,7 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     globalEvalCandidateTimeoutMs: number;
     globalEvalCacheMax: number;
     globalEvalWaitTimeoutMs: number;
+    firecrackerApiSocketWaitMs: number;
     queueMaxDepth: number;
     queueTimeoutMs: number;
     unroutableGraceMs: number;
@@ -556,6 +563,8 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     rerouteSpawnWindowMs: number;
     rerouteAckTimeoutMs: number;
     rerouteMaxHops: number;
+    rerouteSpawnMaxAttempts: number;
+    rerouteSpawnRetryBackoffMs: number;
     rerouteFlapGraceMs: number;
     scalerSpawnTimeoutMs: number;
     backupStalenessWarnHours: number;

package/dist/db/migrations/154_cluster_settings_firecracker_api_socket_wait.d.ts

@@ -0,0 +1,4 @@
+import { type Kysely } from 'kysely';
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=154_cluster_settings_firecracker_api_socket_wait.d.ts.map

package/dist/db/migrations/155_org_settings_reroute_spawn_retry.d.ts

@@ -0,0 +1,17 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add the two per-org spawn-retry columns a worker applies to a rerouted job (both nullable):
+ *
+ * - `reroute_spawn_max_attempts INTEGER` — agent spawns a worker attempts for one rerouted
+ *   job before it gives the job back to its coordinator. NULL falls back to the cluster
+ *   default (`rerouteSpawnMaxAttempts`, 3).
+ * - `reroute_spawn_retry_backoff_ms BIGINT` — wait after a failed spawn before the next
+ *   attempt. NULL falls back to `rerouteSpawnRetryBackoffMs` (5 s).
+ *
+ * Operators tune them per org via `kici-admin org-settings reroute set`.
+ *
+ * Idempotent: each add is guarded on column existence.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=155_org_settings_reroute_spawn_retry.d.ts.map

package/dist/db/migrations/156_kici_events_match_outcome.d.ts

@@ -0,0 +1,17 @@
+import { type Kysely } from 'kysely';
+/**
+ * Record how the event router resolved each internal event:
+ *
+ * - `match_outcome TEXT` — `matched`, `buffered`, `no-registration`,
+ *   `no-target-repo`, `trust-blocked` or `no-trigger-match`.
+ * - `matched_count INTEGER` — how many registrations the event was dispatched to.
+ *
+ * Both are written when the event is processed and are NULL before that, and
+ * for rows processed before these columns existed. Nullable with no default,
+ * so the ALTER is catalog-only on a large table.
+ *
+ * Idempotent: `ADD COLUMN IF NOT EXISTS`.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=156_kici_events_match_outcome.d.ts.map