@kici-dev/orchestrator 0.12.0 → 0.14.0

package/dist/cluster/cluster-settings-reader.d.ts

@@ -1,7 +1,7 @@
 import type { Kysely } from 'kysely';
-import type { Database } from '../db/types.js';
+import type { Database, ClusterSettings } from '../db/types.js';
 /** Numeric columns on cluster_settings readable via {@link ClusterSettingsReader}. */
-export type ClusterNumberColumn = 'max_github_payload_bytes' | 'event_log_max_payload_bytes' | 'lock_file_max_bytes' | 'webhook_dedup_ttl_ms' | 'event_router_event_ttl_seconds' | 'event_router_max_dispatch_attempts' | 'queue_max_depth' | 'reroute_flap_grace_ms' | 'max_fanout_hosts' | 'event_router_rate_limit_per_workflow_per_minute' | 'cache_max_tarball_bytes' | 'cache_ttl_days' | 'lockfile_cache_max' | 'lockfile_cache_max_bytes' | 'lockfile_cache_ttl_ms' | 'content_cache_max' | 'content_cache_max_bytes' | 'content_cache_ttl_ms' | 'global_eval_round_timeout_ms' | 'global_eval_candidate_timeout_ms' | 'global_eval_cache_max' | 'global_eval_wait_timeout_ms' | 'check_run_tracking_ttl_days' | 'run_retention_days' | 'audit_retention_days' | 'provenance_retention_days' | 'held_run_retention_days' | 'concurrency_wait_timeout_ms' | 'agent_token_ttl_ms' | 'ownership_db_check_timeout_ms' | 'sealed_secrets_retry_backoff_ms' | 'unroutable_grace_ms' | 'ingest_overflow_claim_timeout_ms' | 'scaler_reap_interval_ms' | 'scaler_reap_stranded_timeout_ms' | 'scaler_reap_reattempt_interval_ms' | 'scaler_claim_retention_ms' | 'scaler_provision_backoff_base_ms' | 'scaler_provision_backoff_max_ms' | 'scaler_provision_max_consecutive_failures';
+export type ClusterNumberColumn = 'max_github_payload_bytes' | 'event_log_max_payload_bytes' | 'lock_file_max_bytes' | 'webhook_dedup_ttl_ms' | 'event_router_event_ttl_seconds' | 'event_router_max_dispatch_attempts' | 'queue_max_depth' | 'reroute_flap_grace_ms' | 'max_fanout_hosts' | 'event_router_rate_limit_per_workflow_per_minute' | 'cache_max_tarball_bytes' | 'cache_ttl_days' | 'lockfile_cache_max' | 'lockfile_cache_max_bytes' | 'lockfile_cache_ttl_ms' | 'content_cache_max' | 'content_cache_max_bytes' | 'content_cache_ttl_ms' | 'global_eval_round_timeout_ms' | 'global_eval_candidate_timeout_ms' | 'global_eval_cache_max' | 'global_eval_wait_timeout_ms' | 'firecracker_api_socket_wait_ms' | 'check_run_tracking_ttl_days' | 'run_retention_days' | 'audit_retention_days' | 'provenance_retention_days' | 'held_run_retention_days' | 'concurrency_wait_timeout_ms' | 'agent_token_ttl_ms' | 'ownership_db_check_timeout_ms' | 'sealed_secrets_retry_backoff_ms' | 'unroutable_grace_ms' | 'ingest_overflow_claim_timeout_ms' | 'scaler_reap_interval_ms' | 'scaler_reap_stranded_timeout_ms' | 'scaler_reap_reattempt_interval_ms' | 'scaler_claim_retention_ms' | 'scaler_provision_backoff_base_ms' | 'scaler_provision_backoff_max_ms' | 'scaler_provision_max_consecutive_failures';
 /** Text columns on cluster_settings readable via {@link ClusterSettingsReader}. */
 export type ClusterStringColumn = 'dashboard_verified_issuer';
 /** Boolean columns on cluster_settings readable via {@link ClusterSettingsReader}. */
@@ -88,6 +88,12 @@ export declare class ClusterSettingsReader {
      */
     private loadSnapshot;
     private loadRow;
+    /**
+     * The whole row from one read (undefined when there is no row or the read
+     * failed). For a caller that needs several columns that agree with each
+     * other, such as a value and the row version it belongs to.
+     */
+    getRow(): Promise<ClusterSettings | undefined>;
     /** Resolve a fleet-wide numeric knob; `fallback` is the config.ts cluster default. */
     getNumber(column: ClusterNumberColumn, fallback: number): Promise<number>;
     /**

package/dist/cluster/coordinator-credential.d.ts

@@ -0,0 +1,45 @@
+import type { Kysely } from 'kysely';
+import { type Logger } from '@kici-dev/shared';
+import type { Database } from '../db/types.js';
+import { PeerCredentialStore } from './peer-credentials.js';
+/** What one reconciliation did. */
+export declare enum CoordinatorCredentialOutcome {
+    Valid = "valid",
+    Issued = "issued",
+    Replaced = "replaced",
+    Revoked = "revoked",
+    Failed = "failed"
+}
+/** Why a new credential was issued, recorded on the issue log line. */
+export declare enum CredentialIssueReason {
+    NoCredential = "no-credential",
+    Expired = "expired",
+    FileMissing = "credential-file-missing",
+    FileMismatch = "credential-file-mismatch"
+}
+export declare const COORDINATOR_CREDENTIAL_REVOKED_MESSAGE = "Peer credential for this coordinator was revoked; not issuing a new one";
+export type CoordinatorCredentialStore = Pick<PeerCredentialStore, 'save' | 'findUnrevokedByInstanceId' | 'findLatestRevokedByInstanceId' | 'retireSelfIssued'>;
+export interface EnsureCoordinatorCredentialOptions {
+    store: CoordinatorCredentialStore;
+    /** Absolute path of the credential file, with `~` already expanded. */
+    credentialFile: string;
+    instanceId: string;
+    /** Whether an instance has a live `cluster_instances` heartbeat. */
+    isInstanceLive: (instanceId: string) => Promise<boolean>;
+    logger?: Logger;
+    now?: () => Date;
+}
+/**
+ * Make sure this coordinator holds a credential its peers accept, issuing one
+ * when it has none. Never throws: a failure is logged and reported as `Failed`.
+ */
+export declare function ensureCoordinatorCredential(opts: EnsureCoordinatorCredentialOptions): Promise<CoordinatorCredentialOutcome>;
+/** The `selfIssue` callback the coordinator's `PeerAuthCoordinator` runs. */
+export declare function coordinatorSelfIssuer(opts: {
+    db: Kysely<Database>;
+    credentialFile: string;
+    instanceId: string;
+    agentMaxReconnectDelayMs: number;
+    clusterInstanceHeartbeatMs: number;
+}): () => Promise<CoordinatorCredentialOutcome>;
+//# sourceMappingURL=coordinator-credential.d.ts.map

package/dist/cluster/coordinator.d.ts

@@ -10,7 +10,7 @@
  * - "Peers report step-by-step progress back to coordinator"
  * - "Cancel mode: graceful -- finish current step, cancel remaining"
  */
-import type { JobReroute, JobProgress, JobProgressAck, PeerScalerEvent, PeerToPeerMessage, ResourceRequest, LabelMatcher } from '@kici-dev/engine';
+import type { JobReroute, JobProgress, JobProgressAck, PeerScalerEvent, PeerToPeerMessage, RerouteSpawnRetry, ResourceRequest, LabelMatcher } from '@kici-dev/engine';
 import type { PeerRegistry } from './peer-registry.js';
 import type { PeerClient } from './peer-client.js';
 import type { Dispatcher } from '../agent/dispatcher.js';
@@ -114,6 +114,14 @@ export interface RunCoordinatorDeps {
      * {@link DEFAULT_MAX_HOPS}.
      */
     getRerouteMaxHops?: (job: JobToRoute) => Promise<number>;
+    /**
+     * Per-job spawn-retry budget sent to the receiving peer on `job.reroute` and
+     * enforced here on the peer's relayed spawn failures. Resolves the job's org
+     * overrides from `org_settings.reroute_spawn_max_attempts` /
+     * `reroute_spawn_retry_backoff_ms`, else the cluster defaults. When absent,
+     * {@link DEFAULT_REROUTE_SPAWN_RETRY} is used.
+     */
+    getRerouteSpawnRetry?: (job: JobToRoute) => Promise<RerouteSpawnRetry>;
 }
 export declare class RunCoordinator {
     private readonly instanceId;
@@ -128,6 +136,7 @@ export declare class RunCoordinator {
     private readonly getRerouteSpawnWindowMs;
     private readonly getRerouteAckTimeoutMs;
     private readonly getRerouteMaxHops;
+    private readonly getRerouteSpawnRetry;
     /**
      * Tracks which jobs have been rerouted to which peers, keyed by runId.
      * Used for cancel propagation, progress tracking, and the post-ACK spawn
@@ -193,12 +202,18 @@ export declare class RunCoordinator {
      * line + dispatch-queue last-error column) just as it would for a
      * locally-emitted scaler event.
      *
-     * Layer B fast path: a `scaler.failed` for a tracked rerouted job whose spawn
-     * window is still armed is the worker's NAK-after-accept — the peer accepted
-     * the reroute but its agent spawn failed asynchronously before reporting any
-     * progress. Re-dispatch immediately (the same routine the spawn-window timer
-     * runs) instead of waiting the window out. Idempotent with the Layer A timer:
-     * whichever fires first removes the tracking entry.
+     * Layer B: a `scaler.failed` for a tracked rerouted job whose spawn window is
+     * still armed is the worker reporting that an agent spawn failed after it
+     * accepted the reroute. The worker retries within the spawn-retry budget sent on
+     * `job.reroute`, and the relay carries its verdict:
+     *
+     * - `final: true` — the worker gave the job back. Re-dispatch immediately (the
+     *   same routine the spawn-window timer runs). Idempotent with the Layer A
+     *   timer: whichever fires first removes the tracking entry.
+     * - `final: false` — the worker is retrying. Re-arm the window for one more
+     *   attempt plus the backoff; treat the `maxAttempts`-th such relay as final.
+     * - no verdict — an older worker that retries without a bound. The window
+     *   armed at the reroute keeps running unchanged.
      *
      * Two guards make the re-dispatch safe. First, source provenance: the failure
      * must come from `fromPeerId === tracked.peerId` (the authenticated connection
@@ -211,6 +226,16 @@ export declare class RunCoordinator {
      * correct regardless of which peer relayed the event.
      */
     onPeerScalerEvent(msg: PeerScalerEvent, fromPeerId: string): void;
+    /**
+     * A worker's verdict on a failed spawn for a job it holds. `final: true` means it
+     * gave the job back, so re-dispatch now. `final: false` means it is retrying, so
+     * restart the window for one more attempt plus the backoff this coordinator told it
+     * to wait. No verdict means a worker that retries without a bound, so leave the
+     * window as it is. The count caps a worker that never says `final`.
+     */
+    private onRelayedSpawnFailure;
+    /** Restart a rerouted job's spawn window for one more worker attempt plus its backoff. */
+    private rearmAfterSpawnRetry;
     /**
      * Handle peer job completion. Updates ExecutionTracker.
      * The execution tracker's onJobStatus fires the needs-aware scheduler hook,
@@ -317,7 +342,8 @@ export declare class RunCoordinator {
     /**
      * Post-ACK spawn-window backstop. Fires when a rerouted job's peer accepted
      * but produced no progress within the window (Layer A), or immediately when a
-     * worker relays a `scaler.failed` for the job (Layer B, via onPeerScalerEvent).
+     * worker relays its final spawn-failure verdict for the job (Layer B, via
+     * onPeerScalerEvent).
      * Best-effort cancels the original peer (double-execution guard against a
      * slow-but-healthy spawn), then re-runs routing to another peer, then a local
      * fallback, and finally records the job failed rather than leaving it pending.
@@ -334,9 +360,25 @@ export declare class RunCoordinator {
      * The run-cancel path needs this for a job this coordinator never routed:
      * the Platform picks any pool member to receive `run.cancel.request`, and only
      * the coordinator holding the agent's socket can put a cancel frame on it.
-     * {@link RunCoordinator.cancelRun} covers rerouted jobs and is unchanged.
+     * `force` rides on the message, so a force cancel stays forced on the peer.
+     * {@link RunCoordinator.cancelRun} covers rerouted jobs.
+     */
+    cancelJobOnPeer(peerId: string, runId: string, jobId: string, reason: string, force?: boolean): boolean;
+    /**
+     * Send a fire-and-forget message to a peer. The outbound client carries it
+     * when connected; otherwise the peer's own connection to this process does.
+     * The outbound client is down while it reconnects, and for good on a
+     * coordinator whose credential an operator revoked.
+     * `PeerClient.send` returns false only when it sent nothing, so the fallback
+     * never delivers a message twice.
+     */
+    private sendToPeer;
+    /**
+     * The link a reroute uses, chosen before sending. A reroute is never retried
+     * over the second link: `false` from a connected client is a NAK or an ack
+     * timeout, and resending would deliver the job twice.
      */
-    cancelJobOnPeer(peerId: string, runId: string, jobId: string, reason: string): boolean;
+    private rerouteTransport;
     /**
      * Local fallback for a re-dispatch: dispatch the job on this coordinator under
      * the existing jobId. Returns true when the local dispatcher accepts it.

package/dist/cluster/index.d.ts

@@ -9,6 +9,7 @@ export type { PeerInfo, PeerAgentInfo, PeerRegistryOptions } from './peer-regist
 export { PeerClient } from './peer-client.js';
 export type { PeerClientOptions } from './peer-client.js';
 export { PeerAuthCoordinator } from './peer-auth-coordinator.js';
+export { coordinatorSelfIssuer } from './coordinator-credential.js';
 export type { AuthDecision, RejectionAction } from './peer-auth-coordinator.js';
 export { createPeerHandler } from './peer-handler.js';
 export type { PeerHandlerDeps } from './peer-handler.js';

package/dist/cluster/instance-heartbeat.d.ts

@@ -1,4 +1,4 @@
-import type { Kysely } from 'kysely';
+import { type Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
 /** Default interval between `cluster_instances` heartbeat writes. */
 export declare const DEFAULT_INSTANCE_HEARTBEAT_MS = 10000;
@@ -23,6 +23,11 @@ export declare const DEFAULT_RECOVERY_GRACE_MS = 120000;
  * recovery machinery would have given it to reconnect.
  */
 export declare function instanceLivenessGraceMs(gracePeriodMs: number, heartbeatMs?: number): number;
+/**
+ * Whether `instanceId` has a `cluster_instances` heartbeat inside `graceMs`:
+ * the same liveness the dispatch plane's ownership predicate reads.
+ */
+export declare function isInstanceLive(db: Kysely<Database>, instanceId: string, graceMs: number): Promise<boolean>;
 export interface InstanceHeartbeatOptions {
     db: Kysely<Database>;
     instanceId: string;

package/dist/cluster/join-token.d.ts

@@ -18,6 +18,8 @@ import { Kysely } from 'kysely';
  * and a stray log line would break the contract).
  */
 export declare function silenceJoinTokenLogger(): void;
+export declare const TOKEN_ALREADY_USED_MESSAGE = "Join token has already been used. Create a new token with: kici-admin peer create-token";
+export declare const TOKEN_EXPIRED_MESSAGE = "Join token has expired. Create a new token with: kici-admin peer create-token";
 interface TokenRouting {
     orgId: string;
     routingKey: string;

package/dist/cluster/peer-auth-coordinator.d.ts

@@ -1,4 +1,5 @@
 import { type CredentialFileData } from './peer-credentials.js';
+import { CoordinatorCredentialOutcome } from './coordinator-credential.js';
 export type AuthDecision = {
     mode: 'credential';
     credential: CredentialFileData;
@@ -15,6 +16,11 @@ export declare class PeerAuthCoordinator {
     private readonly instanceId;
     private readonly joinToken?;
     private readonly joinWaitTimeoutMs;
+    private readonly selfIssue?;
+    /** The issuance in progress; concurrent sibling clients share it. */
+    private selfIssueInFlight;
+    /** An operator revoked this coordinator's credential: never issue again in this process. */
+    private selfIssueRevoked;
     /** Promise-chain mutex tail; every file op awaits the prior one. */
     private lock;
     /** Set while one peer-client is mid token-join; siblings await it. */
@@ -24,11 +30,18 @@ export declare class PeerAuthCoordinator {
         instanceId: string;
         joinToken?: string;
         joinWaitTimeoutMs?: number;
+        /**
+         * Issues this coordinator its own credential when a decision finds no
+         * credential and no join token. Only a coordinator wires it.
+         */
+        selfIssue?: () => Promise<CoordinatorCredentialOutcome>;
     });
     /** Run `fn` exclusively against the credential file. */
     private withLock;
     private readValidCredential;
     decideAuth(): Promise<AuthDecision>;
+    /** Run the issuer under the mutex, once for every concurrent caller. */
+    private runSelfIssue;
     private makeComplete;
     private awaitInFlightJoin;
     reportRejection(provedCredential: string | null, reason: string): Promise<RejectionAction>;

package/dist/cluster/peer-client.d.ts

@@ -10,9 +10,13 @@
  * Authentication uses ECDH key exchange followed by join token (first connect)
  * or HMAC credential proof (reconnection).
  */
-import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type JobProgressAck, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerClusterSettingsRequest, type PeerClusterSettingsResponse, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
+import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type JobProgressAck, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerScalerOrphansRequest, type PeerForgetRequest, type PeerForgetResponse, type PeerScalerOrphansResponse, type PeerClusterSettingsRequest, type PeerClusterSettingsResponse, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
 import type { PeerRegistry } from './peer-registry.js';
+import { type PEER_FORGET_TIMEOUT, type PeerForgetRequestHandler } from './peer-forget.js';
+import { type SCALER_ORPHANS_TIMEOUT, type ScalerOrphansRequestHandler } from './scaler-orphans-peer.js';
 import type { PeerAuthCoordinator } from './peer-auth-coordinator.js';
+/** Logged when this instance has neither a credential file nor a join token. */
+export declare const NO_AUTH_METHOD_MESSAGE = "No peer auth method: this instance has no credential file and no join token";
 type PeerConnectionState = 'disconnected' | 'connecting' | 'handshaking' | 'authenticating' | 'connected';
 export interface PeerClientOptions {
     /** WebSocket URL of the remote peer orchestrator. */
@@ -95,6 +99,17 @@ export interface PeerClientOptions {
      * If undefined, incoming collect requests are ignored.
      */
     onLogsCollectRequest?: PeerLogsCollectResponder;
+    /**
+     * Answers a peer.scaler.orphans.request the connected coordinator forwarded
+     * to this node (`kici-admin scaler orphans --target`). If undefined,
+     * requests are answered with ok=false.
+     */
+    onScalerOrphansRequest?: ScalerOrphansRequestHandler;
+    /**
+     * Forgets a departed peer the connected coordinator forgot (`kici-admin peer
+     * forget`). If undefined, requests are answered with outcome `error`.
+     */
+    onPeerForgetRequest?: PeerForgetRequestHandler;
 }
 /**
  * Builds a node's subtree bundle in response to a peer.logs.collect.request and
@@ -108,12 +123,16 @@ export declare class PeerClient {
     private heartbeatTimer;
     private reconnectTimer;
     private reconnectAttempts;
+    /** Whether this client already warned that it has no auth method; re-armed by an accepted auth. */
+    private noAuthWarned;
     private intentionalDisconnect;
     private _targetInstanceId;
     private sessionKey;
     private readonly ackWaiters;
     private readonly cacheWaiters;
     private readonly configReloadWaiters;
+    private readonly scalerOrphansWaiters;
+    private readonly peerForgetWaiters;
     private readonly clusterSettingsWaiters;
     /** Correlates peer.logs.collect.request with the peer's chunked subtree response. */
     private readonly logsCollectWaiters;
@@ -146,6 +165,8 @@ export declare class PeerClient {
     private readonly onPeerConfigReload?;
     private readonly onAuthenticated?;
     private readonly onLogsCollectRequest?;
+    private readonly onScalerOrphansRequest?;
+    private readonly onPeerForgetRequest?;
     constructor(options: PeerClientOptions);
     /** Current connection state. */
     get state(): PeerConnectionState;
@@ -185,10 +206,26 @@ export declare class PeerClient {
      */
     sendConfigReloadAndWait(msg: PeerConfigReload, timeoutMs?: number): Promise<PeerConfigReloadResponse | null>;
     /**
-     * Send a peer.clusterSettings.request to the connected leader and wait for the
-     * matching peer.clusterSettings.response carrying the worker-settings snapshot.
+     * Send a peer.forget.request to the connected peer and wait for the matching
+     * response.
+     *
+     * @returns the response; null when not connected; `'timeout'` when no
+     *   response arrived within `timeoutMs`.
+     */
+    sendPeerForgetAndWait(msg: PeerForgetRequest, timeoutMs: number): Promise<PeerForgetResponse | null | typeof PEER_FORGET_TIMEOUT>;
+    /**
+     * Send a peer.scaler.orphans.request to the connected peer and wait for the
+     * matching response.
+     *
+     * @returns the response; null when not connected; `'timeout'` when no
+     *   response arrived within `timeoutMs`.
+     */
+    sendScalerOrphansAndWait(msg: PeerScalerOrphansRequest, timeoutMs: number): Promise<PeerScalerOrphansResponse | null | typeof SCALER_ORPHANS_TIMEOUT>;
+    /**
+     * Send a peer.clusterSettings.request to the connected coordinator and wait for
+     * the matching peer.clusterSettings.response carrying the worker-settings snapshot.
      *
-     * @returns The response, or null if not connected or the leader doesn't reply
+     * @returns The response, or null if not connected or the coordinator doesn't reply
      *   within the timeout — the caller keeps its current (config-default) settings.
      */
     sendClusterSettingsRequestAndWait(msg: PeerClusterSettingsRequest, timeoutMs?: number): Promise<PeerClusterSettingsResponse | null>;

package/dist/cluster/peer-credentials.d.ts

@@ -1,4 +1,15 @@
 import { Kysely } from 'kysely';
+/** How a credential row was issued, recorded in `peer_credentials.metadata.issuance`. */
+export declare enum PeerCredentialIssuance {
+    /** A coordinator issued the credential to itself (`coordinator-credential.ts`). */
+    Self = "self"
+}
+/**
+ * `metadata` key set when a coordinator retires the self-issued credential its
+ * previous run left behind. An operator revoke never sets it, which is how a
+ * retirement is told apart from a revoke.
+ */
+export declare const RETIRED_BY_INSTANCE_KEY = "retiredByInstance";
 /**
  * A peer credential record stored in the coordinator's database.
  */
@@ -14,6 +25,7 @@ export interface PeerCredential {
     lastValidatedBy: string | null;
     expiresAt: Date;
     revokedAt: Date | null;
+    metadata: Record<string, unknown>;
 }
 /**
  * Data persisted in this orchestrator's local credential file
@@ -55,6 +67,7 @@ export declare class PeerCredentialStore {
         routingKeys: string[];
         sourceTokenHash?: string;
         expiryDays?: number;
+        metadata?: Record<string, unknown>;
     }): Promise<{
         revokedCount: number;
     }>;
@@ -66,6 +79,24 @@ export declare class PeerCredentialStore {
      * Find a non-revoked, non-expired credential by instance ID.
      */
     findByInstanceId(instanceId: string): Promise<PeerCredential | null>;
+    /**
+     * The credential row for `instanceId` that nobody revoked, expired or not.
+     * `peer_credentials_active_uniq` allows at most one.
+     */
+    findUnrevokedByInstanceId(instanceId: string): Promise<PeerCredential | null>;
+    /** The most recently revoked credential row for `instanceId`, or null. */
+    findLatestRevokedByInstanceId(instanceId: string): Promise<PeerCredential | null>;
+    /**
+     * Revoke `instanceId`'s unrevoked, self-issued row whose hash is
+     * `credentialHash`, recording who retired it. Every condition sits in the
+     * WHERE clause, so a row that changed meanwhile is left alone.
+     * Returns whether a row was retired.
+     */
+    retireSelfIssued(opts: {
+        instanceId: string;
+        credentialHash: string;
+        retiredByInstance: string;
+    }): Promise<boolean>;
     /**
      * Update the last_seen_at timestamp for a credential.
      * Optionally records which coordinator validated it.

package/dist/cluster/peer-forget.d.ts

@@ -0,0 +1,138 @@
+import { PeerForgetOutcome, type PeerForgetRequest, type PeerForgetResponse } from '@kici-dev/engine';
+import { z } from 'zod';
+import type { PeerRegistry } from './peer-registry.js';
+/** What forgetting did on one coordinator. */
+export interface PeerForgetResult {
+    /** The coordinator that answered. */
+    coordinator: string;
+    outcome: PeerForgetOutcome;
+    detail: string;
+}
+/** Local outcomes that stop a forget before it reaches any sibling. */
+export declare const LOCAL_REFUSALS: ReadonlySet<PeerForgetOutcome>;
+/** A sibling's answer that never arrived, or a sibling no connection reached. */
+export declare const PEER_FORGET_TIMEOUT = "timeout";
+/** The answer to a forget request a peer that is not a coordinator sent. */
+export declare const PEER_FORGET_COORDINATORS_ONLY = "peer forget requests are accepted from coordinators only";
+/** The answer of a peer with no forget handler wired. */
+export declare const PEER_FORGET_UNHANDLED = "peer forget requests are not handled by this peer";
+/** Which window a coordinator keeps treating a silent peer as alive for. */
+export declare const PeerLivenessWindowKind: z.ZodEnum<{
+    "adopter-grace": "adopter-grace";
+    "reroute-grace": "reroute-grace";
+    "stale-window": "stale-window";
+}>;
+export type PeerLivenessWindowKind = z.infer<typeof PeerLivenessWindowKind>;
+/** How long after its last heartbeat a coordinator still treats a peer as alive. */
+export interface PeerLivenessWindow {
+    kind: PeerLivenessWindowKind;
+    windowMs: number;
+}
+/** What a coordinator reads to choose a peer's liveness window. */
+export interface PeerLivenessSource {
+    /** The peer stale window (`KICI_CLUSTER_PEER_STALE_TIMEOUT_MS`). */
+    staleTimeoutMs: number;
+    /** Reads the live `reroute_flap_grace_ms` setting. */
+    rerouteFlapGraceMs: () => Promise<number>;
+    /**
+     * This coordinator's event-provision backstop, or null when it runs none.
+     * `holdsAdoptedProvisions` reads whether a peer adopted provisions the
+     * backstop judges by that peer's liveness.
+     */
+    backstop: {
+        holdsAdoptedProvisions: (instanceId: string) => Promise<boolean>;
+    } | null;
+}
+/**
+ * The longest window in which this coordinator still treats `instanceId` as
+ * alive after its last heartbeat. Forgetting the peer inside it would cut that
+ * grace short: every reader of the registry treats a peer it no longer knows
+ * as gone at once.
+ *
+ * - A peer that adopted provisions gets the event-provision backstop's own
+ *   grace (`reaperFlapGraceMs`), or the backstop tears its instances down.
+ * - Any other peer gets the longer of the stale window and the reroute flap
+ *   grace, which the rerouted-job guard and the master-key requeue give every
+ *   disconnected peer whether or not this coordinator runs a backstop.
+ */
+export declare function peerLivenessWindow(instanceId: string, source: PeerLivenessSource): Promise<PeerLivenessWindow>;
+/** The checks a coordinator runs before it forgets a peer. */
+export interface PeerForgetGuards {
+    /**
+     * How long after its last heartbeat the peer is still treated as alive. A
+     * peer heard from inside it may be partitioned rather than gone, so it is kept.
+     */
+    liveness: PeerLivenessWindow;
+    /**
+     * This coordinator's event-provision backstop, or null when it runs none.
+     * `configuredPeerCount` is the `KICI_CLUSTER_PEERS` count the backstop's
+     * cluster-view check reads beside the live registry.
+     */
+    backstop: {
+        configuredPeerCount: number;
+    } | null;
+    /** The operator acknowledged that the forget may switch the backstop back on. */
+    acknowledgeBackstop: boolean;
+    /** The current time; tests pin it. */
+    nowMs?: number;
+}
+/**
+ * The guards for forgetting `instanceId` on one coordinator, built from its
+ * cluster config, the live reroute flap grace and its event-provision
+ * backstop. The backstop runs on every coordinator that runs a scaler manager.
+ */
+export declare function peerForgetGuards(instanceId: string, cluster: {
+    peerStaleTimeoutMs: number;
+    peers: readonly unknown[];
+}, liveness: Omit<PeerLivenessSource, 'staleTimeoutMs'>, acknowledgeBackstop: boolean): Promise<PeerForgetGuards>;
+/** What forgetting a peer that would switch the backstop back on does. */
+export declare function backstopConsequence(instanceId: string): string;
+/**
+ * Drop `instanceId` from `registry` when it names a peer that left the
+ * cluster. Kept, with the reason: a connected peer (its next heartbeat would
+ * register it again), a peer heard from inside its liveness window (it may be
+ * partitioned rather than gone; see `peerLivenessWindow`), and a peer whose removal would switch the
+ * event-provision backstop back on, unless the request acknowledged that.
+ */
+export declare function forgetDepartedPeer(registry: PeerRegistry, selfInstanceId: string, instanceId: string, guards: PeerForgetGuards): Omit<PeerForgetResult, 'coordinator'>;
+/** Answers a forget request a sibling coordinator fanned out. */
+export type PeerForgetRequestHandler = (msg: PeerForgetRequest) => Promise<{
+    outcome: PeerForgetOutcome;
+    detail: string;
+}>;
+/**
+ * Answer one forget request with `handler`. A coordinator with no handler
+ * answers {@link PEER_FORGET_UNHANDLED}, and a handler that fails (its guards
+ * read the database) answers `error`, so the peer is kept and the operator
+ * sees why.
+ */
+export declare function replyToPeerForgetRequest(msg: PeerForgetRequest, handler: PeerForgetRequestHandler | undefined, send: (response: PeerForgetResponse) => void, logFields?: Record<string, unknown>): void;
+/**
+ * Sends one forget request to a sibling coordinator and waits for its answer.
+ * Resolves null when no connection reached the sibling, and
+ * {@link PEER_FORGET_TIMEOUT} when it did not answer within `timeoutMs`.
+ */
+export type SendPeerForget = (siblingInstanceId: string, msg: PeerForgetRequest, timeoutMs: number) => Promise<PeerForgetResponse | null | typeof PEER_FORGET_TIMEOUT>;
+/**
+ * Forget `instanceId` on this coordinator, then on every connected sibling
+ * coordinator, each of which holds its own registry. A peer this coordinator
+ * still has connected is refused and not fanned out. The first result is this
+ * coordinator's.
+ */
+export declare function forgetAcrossCoordinators(registry: PeerRegistry, selfInstanceId: string, instanceId: string, timeoutMs: number, guards: PeerForgetGuards, send: SendPeerForget): Promise<PeerForgetResult[]>;
+/**
+ * The coordinator peers the cluster expects but this coordinator has no live
+ * link to: registered coordinators whose connection is down.
+ */
+export declare function disconnectedCoordinatorIds(registry: PeerRegistry, selfInstanceId: string): string[];
+/** Pending forget requests sent to siblings, keyed by message id. */
+export declare class PeerForgetWaiters {
+    private readonly waiters;
+    /** Wait for the response to `messageId`, or {@link PEER_FORGET_TIMEOUT}. */
+    wait(messageId: string, timeoutMs: number): Promise<PeerForgetResponse | typeof PEER_FORGET_TIMEOUT>;
+    /** Hand a response to the request waiting for it. An unknown id is dropped. */
+    resolve(response: PeerForgetResponse): void;
+    /** End every pending wait with an `error` response naming `reason`. */
+    rejectAll(reason: string): void;
+}
+//# sourceMappingURL=peer-forget.d.ts.map

package/dist/cluster/peer-handler.d.ts

@@ -6,8 +6,10 @@
  * proof (reconnection), registers the peer in PeerRegistry, and routes
  * messages bidirectionally. Sends periodic heartbeats to the connecting peer.
  */
-import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type JobProgressAck, type PeerScalerEvent, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerClusterSettingsRequest, type WorkerClusterSettings, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
+import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type JobProgressAck, type PeerScalerEvent, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerScalerOrphansRequest, type PeerForgetRequest, type PeerForgetResponse, type PeerScalerOrphansResponse, type PeerClusterSettingsRequest, type WorkerClusterSettings, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
 import type { PeerRegistry } from './peer-registry.js';
+import { type PEER_FORGET_TIMEOUT, type PeerForgetRequestHandler } from './peer-forget.js';
+import { type SCALER_ORPHANS_TIMEOUT, type ScalerOrphansRequestHandler } from './scaler-orphans-peer.js';
 import type { PeerCredentialStore } from './peer-credentials.js';
 import { type JoinTokenManager } from './join-token.js';
 /**
@@ -104,6 +106,19 @@ export interface PeerHandlerDeps {
      * If undefined, incoming collect requests are ignored.
      */
     onLogsCollectRequest?: (msg: PeerLogsCollectRequest, send: (out: PeerToPeerMessage) => boolean) => Promise<void>;
+    /**
+     * Answers a peer.scaler.orphans.request a coordinator forwarded to this node
+     * (`kici-admin scaler orphans --target`). A request from a peer that is not
+     * a coordinator is refused before it reaches the handler. If undefined,
+     * requests are answered with ok=false.
+     */
+    onScalerOrphansRequest?: ScalerOrphansRequestHandler;
+    /**
+     * Forgets a departed peer a sibling coordinator forgot (`kici-admin peer
+     * forget`). A request from a peer that is not a coordinator is refused. If
+     * undefined, requests are answered with outcome `error`.
+     */
+    onPeerForgetRequest?: PeerForgetRequestHandler;
 }
 /**
  * Create a handler function for incoming peer WebSocket connections.
@@ -127,6 +142,8 @@ export declare function createPeerHandler(deps: PeerHandlerDeps): {
     closePeer: (targetInstanceId: string, code: number, reason: string) => void;
     sendAndWaitAck: (targetInstanceId: string, msg: JobReroute, timeoutMs?: number) => Promise<boolean>;
     sendConfigReloadAndWait: (targetInstanceId: string, msg: PeerConfigReload, timeoutMs?: number) => Promise<PeerConfigReloadResponse | null>;
+    sendScalerOrphansAndWait: (targetInstanceId: string, msg: PeerScalerOrphansRequest, timeoutMs: number) => Promise<PeerScalerOrphansResponse | null | typeof SCALER_ORPHANS_TIMEOUT>;
+    sendPeerForgetAndWait: (targetInstanceId: string, msg: PeerForgetRequest, timeoutMs: number) => Promise<PeerForgetResponse | null | typeof PEER_FORGET_TIMEOUT>;
     sendLogsCollectAndWait: (targetInstanceId: string, msg: PeerLogsCollectRequest, timeoutMs: number) => Promise<Buffer>;
     getConnectionCount: () => number;
     broadcastHeartbeat: (inventory: Omit<PeerHeartbeat, 'type'>) => void;

package/dist/cluster/scaler-orphans-peer.d.ts

@@ -0,0 +1,40 @@
+/**
+ * The peer-channel half of `kici-admin scaler orphans --target`: answering a
+ * forwarded peer.scaler.orphans.request, and correlating the response with
+ * the request that is waiting for it. Shared by the outgoing `PeerClient` and
+ * the incoming peer handler, which carry the same message pair.
+ */
+import { z } from 'zod';
+import type { PeerScalerOrphansRequest, PeerScalerOrphansResponse } from '@kici-dev/engine';
+import type { ScalerOrphansAnswer } from '../scaler/orphan-requests.js';
+/** Answers a forwarded request from this node's own host and tracking. */
+export type ScalerOrphansRequestHandler = (msg: PeerScalerOrphansRequest) => Promise<ScalerOrphansAnswer>;
+/** The answer of a peer that has no handler wired. */
+export declare const SCALER_ORPHANS_UNHANDLED_ERROR = "scaler orphan requests are not handled by this peer";
+/** The answer to a request a worker sent: a worker never stops another node's VMs. */
+export declare const SCALER_ORPHANS_COORDINATORS_ONLY_ERROR = "scaler orphan requests are accepted from coordinators only";
+/** A forward that reached no peer, or a peer that did not answer in time. */
+export declare const ScalerOrphansForwardFailure: z.ZodEnum<{
+    "not-connected": "not-connected";
+    timeout: "timeout";
+}>;
+export type ScalerOrphansForwardFailure = z.infer<typeof ScalerOrphansForwardFailure>;
+/** A wait that ended without a response. */
+export declare const SCALER_ORPHANS_TIMEOUT: "timeout";
+/**
+ * Answer one forwarded request through `send`. A missing handler and a
+ * handler failure are both answered `ok: false`, so the requester never waits
+ * out its timeout for a peer that cannot answer.
+ */
+export declare function replyToScalerOrphansRequest(msg: PeerScalerOrphansRequest, handler: ScalerOrphansRequestHandler | undefined, send: (response: PeerScalerOrphansResponse) => void, logFields?: Record<string, unknown>): void;
+/** Pending scaler orphan requests, keyed by message id. */
+export declare class ScalerOrphansWaiters {
+    private readonly waiters;
+    /** Wait for the response to `messageId`, or {@link SCALER_ORPHANS_TIMEOUT}. */
+    wait(messageId: string, timeoutMs: number): Promise<PeerScalerOrphansResponse | typeof SCALER_ORPHANS_TIMEOUT>;
+    /** Hand a response to the request waiting for it. An unknown id is dropped. */
+    resolve(response: PeerScalerOrphansResponse): void;
+    /** End every pending wait with an `ok: false` response naming `reason`. */
+    rejectAll(reason: string): void;
+}
+//# sourceMappingURL=scaler-orphans-peer.d.ts.map