@kici-dev/orchestrator 0.14.2 → 0.16.0

package/dist/cluster/peer-client.d.ts

@@ -7,16 +7,38 @@
  * - Periodic heartbeat (30s default)
  * - Intentional disconnect flag to prevent reconnection
  *
- * Authentication uses ECDH key exchange followed by join token (first connect)
- * or HMAC credential proof (reconnection).
+ * Authentication is mutual-v2: after the ECDH key exchange the client proves
+ * its credential (or, on first join, its join token) with an HMAC over the
+ * handshake transcript, and accepts nothing until the server proves it holds
+ * the same credential or token. The join token never leaves this host. Every
+ * frame after acceptance uses an application key that needs the shared secret.
+ * A server that does not offer mutual-v2 gets no proof at all.
  */
-import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type JobProgressAck, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerScalerOrphansRequest, type PeerForgetRequest, type PeerForgetResponse, type PeerScalerOrphansResponse, type PeerClusterSettingsRequest, type PeerClusterSettingsResponse, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
-import type { PeerRegistry } from './peer-registry.js';
+import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type JobProgressAck, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerScalerOrphansRequest, type PeerForgetRequest, type PeerForgetResponse, type PeerScalerOrphansResponse, type PeerClusterSettingsRequest, type PeerClusterSettingsResponse, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries, type PeerScalerReloadRequest, type PeerScalerReloadResponse } from '@kici-dev/engine';
+import { type PeerRegistry } from './peer-registry.js';
 import { type PEER_FORGET_TIMEOUT, type PeerForgetRequestHandler } from './peer-forget.js';
 import { type SCALER_ORPHANS_TIMEOUT, type ScalerOrphansRequestHandler } from './scaler-orphans-peer.js';
+import type { PEER_REQUEST_TIMEOUT } from './peer-request-waiters.js';
+import type { ScalerFileReload } from '../scaler/file-reload.js';
 import type { PeerAuthCoordinator } from './peer-auth-coordinator.js';
 /** Logged when this instance has neither a credential file nor a join token. */
 export declare const NO_AUTH_METHOD_MESSAGE = "No peer auth method: this instance has no credential file and no join token";
+/** Logged when a dialled server does not offer mutual authentication. */
+export declare const PEER_MUTUAL_AUTH_UNSUPPORTED_MESSAGE = "Peer does not support mutual authentication; upgrade every orchestrator in the cluster";
+/** Where the address this client dials came from. */
+export declare enum PeerDialOrigin {
+    /** KICI_CLUSTER_PEERS or a worker's coordinator URLs: reconnects with backoff. */
+    Static = "static",
+    /** A Platform announcement: a hint, dropped for good when mutual authentication fails. */
+    Discovered = "discovered"
+}
+/** Why a server failed mutual authentication. */
+export declare enum PeerMutualAuthFailure {
+    SchemeMissing = "scheme-missing",
+    ServerProofInvalid = "server-proof-invalid",
+    InstanceIdMissing = "instance-id-missing",
+    InstanceIdMismatch = "instance-id-mismatch"
+}
 type PeerConnectionState = 'disconnected' | 'connecting' | 'handshaking' | 'authenticating' | 'connected';
 export interface PeerClientOptions {
     /** WebSocket URL of the remote peer orchestrator. */
@@ -31,6 +53,21 @@ export interface PeerClientOptions {
     instanceId: string;
     /** Peer registry to update on heartbeats from remote peer. */
     peerRegistry: PeerRegistry;
+    /**
+     * Where the dialled address came from. A static target reconnects with
+     * backoff after any failure; a discovered target that fails mutual
+     * authentication is dropped. Default: static.
+     */
+    origin?: PeerDialOrigin;
+    /** The instance id a discovered target was announced with; the server must report it. */
+    expectedInstanceId?: string;
+    /**
+     * Closes a connection that has not completed authentication within this
+     * many ms, after which the normal reconnect policy applies. Default: 15000.
+     */
+    handshakeTimeoutMs?: number;
+    /** Called when a discovered target fails mutual authentication; the client then stays down. */
+    onMutualAuthFailed?: (failure: PeerMutualAuthFailure) => void;
     /** Callback to get this orchestrator's local agent inventory for heartbeats. */
     getLocalInventory: () => Omit<PeerHeartbeat, 'type'>;
     /** Heartbeat interval in ms. Default: 30000 (30s). */
@@ -105,6 +142,12 @@ export interface PeerClientOptions {
      * requests are answered with ok=false.
      */
     onScalerOrphansRequest?: ScalerOrphansRequestHandler;
+    /**
+     * Reloads this node's scaler config for a peer.scaler.reload.request the
+     * connected coordinator sent (`kici-admin scaler reload`). If undefined,
+     * requests are answered `rejected`.
+     */
+    onScalerReloadRequest?: ScalerFileReload;
     /**
      * Forgets a departed peer the connected coordinator forgot (`kici-admin peer
      * forget`). If undefined, requests are answered with outcome `error`.
@@ -127,11 +170,18 @@ export declare class PeerClient {
     private noAuthWarned;
     private intentionalDisconnect;
     private _targetInstanceId;
+    /** The key the current frames use: K_hs while authenticating, K_app once connected. */
     private sessionKey;
+    /** The handshake in flight; cleared on acceptance or close. */
+    private handshake;
+    /** The proof this connection sent, until the server's proof is checked. */
+    private pendingProof;
+    private handshakeTimer;
     private readonly ackWaiters;
     private readonly cacheWaiters;
     private readonly configReloadWaiters;
     private readonly scalerOrphansWaiters;
+    private readonly scalerReloadWaiters;
     private readonly peerForgetWaiters;
     private readonly clusterSettingsWaiters;
     /** Correlates peer.logs.collect.request with the peer's chunked subtree response. */
@@ -142,11 +192,15 @@ export declare class PeerClient {
     private isJoiner;
     /** complete() callback for the in-flight token-join, if this client is joiner. */
     private joinComplete;
-    /** The credential string this client last built an HMAC proof with. */
+    /** The credential string this client last built a proof with. */
     private lastProvedCredential;
     private readonly instanceId;
     private readonly role;
     private readonly peerRegistry;
+    private readonly origin;
+    private readonly expectedInstanceId?;
+    private readonly handshakeTimeoutMs;
+    private readonly onMutualAuthFailed?;
     private readonly getLocalInventory;
     private readonly heartbeatIntervalMs;
     private readonly maxReconnectDelayMs;
@@ -166,6 +220,7 @@ export declare class PeerClient {
     private readonly onAuthenticated?;
     private readonly onLogsCollectRequest?;
     private readonly onScalerOrphansRequest?;
+    private readonly onScalerReloadRequest?;
     private readonly onPeerForgetRequest?;
     constructor(options: PeerClientOptions);
     /** Current connection state. */
@@ -221,6 +276,14 @@ export declare class PeerClient {
      *   response arrived within `timeoutMs`.
      */
     sendScalerOrphansAndWait(msg: PeerScalerOrphansRequest, timeoutMs: number): Promise<PeerScalerOrphansResponse | null | typeof SCALER_ORPHANS_TIMEOUT>;
+    /**
+     * Send a peer.scaler.reload.request to the connected peer and wait for the
+     * matching response.
+     *
+     * @returns the response; null when not connected; `'timeout'` when no
+     *   response arrived within `timeoutMs`.
+     */
+    sendScalerReloadAndWait(msg: PeerScalerReloadRequest, timeoutMs: number): Promise<PeerScalerReloadResponse | null | typeof PEER_REQUEST_TIMEOUT>;
     /**
      * Send a peer.clusterSettings.request to the connected coordinator and wait for
      * the matching peer.clusterSettings.response carrying the worker-settings snapshot.
@@ -248,6 +311,29 @@ export declare class PeerClient {
     getReconnectDelay(): number;
     private doConnect;
     private handleMessage;
+    private armHandshakeTimer;
+    private clearHandshakeTimer;
+    /** A token join in flight must be released so sibling clients stop waiting on it. */
+    private releaseJoin;
+    /** Forget the handshake of a connection that closed or is being torn down. */
+    private resetHandshake;
+    /** Close a connection whose handshake broke the protocol. */
+    private abortHandshake;
+    /** Close a connection whose server failed mutual authentication. */
+    private failMutualAuth;
+    /** The server's plaintext peer.hello: refuse it unless it offers mutual-v2. */
+    private handleHello;
+    /**
+     * The encrypted peer.auth.response. Any frame that is not one closes the
+     * socket. An acceptance is checked synchronously, before any state change,
+     * registry write or callback, so a frame pipelined behind an unproven
+     * acceptance meets a client that is not connected.
+     */
+    private handleAuthResponse;
+    /** Returns why the acceptance fails mutual authentication, or null when it passes. */
+    private checkAcceptance;
+    /** The server proved itself: register the peer and start the connected session. */
+    private completeAcceptance;
     /**
      * Handle a rejected peer.auth.response. For the three credential-divergence
      * reasons, delegate to the coordinator (which deletes the shared file only if
@@ -274,6 +360,14 @@ export declare class PeerClient {
      * client never reads or writes the credential file directly.
      */
     private sendAuthRequest;
+    /**
+     * Route the operator-relay pairs (`scaler orphans --target`, `peer forget`,
+     * `scaler reload`): answer a request from the coordinator this client
+     * dialled, or hand a response to the request waiting for it.
+     *
+     * @returns whether `msg` was one of them.
+     */
+    private routeOperatorRelay;
     private routeMessage;
     private startHeartbeat;
     private stopHeartbeat;

package/dist/cluster/peer-crypto.d.ts

@@ -1,9 +1,18 @@
 /**
- * ECDH key exchange and session encryption for peer-to-peer channels.
+ * ECDH key exchange, mutual-v2 authentication and session encryption for
+ * peer-to-peer channels.
  *
  * Uses X25519 for key agreement and AES-256-GCM for message encryption.
- * Session keys are derived via HKDF-SHA256 from the ECDH shared secret
- * and a caller-provided nonce (to ensure per-session uniqueness).
+ * The handshake key (K_hs) is derived via HKDF-SHA256 from the ECDH shared
+ * secret and the server nonce; it protects the authentication exchange only.
+ *
+ * mutual-v2: both sides hash the handshake transcript (TH: both ephemeral keys,
+ * the nonce and the advertised schemes). The client proves the shared key (PSK:
+ * sha256 of the peer credential, or the join token hash) with an HMAC over TH,
+ * and the server answers with its own HMAC over TH and the client proof. Every
+ * frame after acceptance uses the application key (K_app), derived from K_hs
+ * and the PSK, so a side that skipped the proof check cannot read or write them.
+ * Every field is length-prefixed, so field boundaries are unambiguous.
  *
  * Wire format for encrypted messages (base64 encoded):
  *   IV (12 bytes) || AuthTag (16 bytes) || Ciphertext
@@ -54,4 +63,52 @@ export declare function encryptMessage(plaintext: string, sessionKey: Buffer): s
  * @throws If the session key is wrong or the ciphertext has been tampered with
  */
 export declare function decryptMessage(encrypted: string, sessionKey: Buffer): string;
+/** Length of the server nonce in `peer.hello`. */
+export declare const HANDSHAKE_NONCE_BYTES = 32;
+/** Length of a mutual-v2 proof (HMAC-SHA256). */
+export declare const PEER_PROOF_BYTES = 32;
+/** Encode each field as a 4-byte big-endian length followed by its bytes. */
+export declare function lengthPrefixed(...fields: Array<Buffer | string>): Buffer;
+export interface PeerTranscriptInput {
+    serverEphemeralPublicKey: Buffer;
+    serverNonce: Buffer;
+    /** The schemes exactly as the server advertised them, unknown ones included. */
+    authSchemes: readonly string[];
+    clientEphemeralPublicKey: Buffer;
+}
+/** TH: binds both ephemeral keys, the nonce and the advertised schemes. */
+export declare function peerTranscriptHash(input: PeerTranscriptInput): Buffer;
+export interface ClientProofInput {
+    psk: Buffer;
+    transcriptHash: Buffer;
+    mode: string;
+    clientInstanceId: string;
+    role: string;
+    protocolVersion: number;
+    /** The join token's routing segment in token mode; empty in credential mode. */
+    tokenRouting: string;
+}
+export declare function computeClientProof(input: ClientProofInput): Buffer;
+export interface ServerProofInput {
+    psk: Buffer;
+    transcriptHash: Buffer;
+    clientProof: Buffer;
+    serverInstanceId: string;
+    grantedRole: string;
+    /** The credential the server issues in this response, or null when it issues none. */
+    sessionCredential: string | null;
+}
+export declare function computeServerProof(input: ServerProofInput): Buffer;
+/** A proof as sent on the wire (64 hex characters), or null when malformed. */
+export declare function decodeProof(hex: string): Buffer | null;
+/** Constant-time comparison; a length mismatch is a mismatch, never a throw. */
+export declare function proofMatches(expected: Buffer, presented: Buffer): boolean;
+/** K_app: protects every frame after acceptance, and needs the PSK as well as K_hs. */
+export declare function deriveAppKey(input: {
+    handshakeKey: Buffer;
+    psk: Buffer;
+    transcriptHash: Buffer;
+    clientProof: Buffer;
+    serverProof: Buffer;
+}): Buffer;
 //# sourceMappingURL=peer-crypto.d.ts.map

package/dist/cluster/peer-discovery.d.ts

@@ -0,0 +1,67 @@
+import { OrchRole } from '@kici-dev/engine';
+import { PeerDiscoveryMode } from '../config/schema.js';
+/** Discovered clients plus pending candidates, far above any real coordinator count. */
+export declare const MAX_DISCOVERED_PEER_CLIENTS = 64;
+/** A discovered client not connected for this long is dropped; a returning peer is announced again. */
+export declare const DISCOVERED_PEER_IDLE_PRUNE_MS: number;
+/** What discovery needs from a peer client. */
+export interface DiscoveryPeerClient {
+    readonly state: string;
+    connect(): void;
+    disconnect(): void;
+}
+/** Hooks a discovered client calls back. */
+export interface DiscoveredClientHooks {
+    /** Mutual authentication succeeded with the announced instance id. */
+    onAuthenticated: () => void;
+    /** Mutual authentication failed; the client closed and will not reconnect. */
+    onMutualAuthFailed: () => void;
+}
+/** One peer the Platform announced. */
+export interface PeerAnnouncement {
+    instanceId?: string;
+    address?: string | null;
+    orchRole?: OrchRole;
+}
+export interface PeerDiscoveryDeps {
+    mode: PeerDiscoveryMode;
+    selfInstanceId: string;
+    /** Every outbound peer client. Static clients sit under their URL until they authenticate. */
+    peerClients: Map<string, DiscoveryPeerClient>;
+    createClient: (address: string, expectedInstanceId: string, hooks: DiscoveredClientHooks) => DiscoveryPeerClient;
+    /** Minimum time between dials of an address whose last discovered dial failed mutual authentication. */
+    redialAfterFailureMs: number;
+    maxClients?: number;
+    idlePruneMs?: number;
+    now?: () => number;
+}
+export declare class PeerDiscovery {
+    private readonly deps;
+    private readonly discovered;
+    private readonly candidates;
+    /** Address -> when its last discovered dial failed mutual authentication. */
+    private readonly failedAt;
+    private readonly maxClients;
+    private readonly idlePruneMs;
+    private readonly now;
+    private timer;
+    constructor(deps: PeerDiscoveryDeps);
+    announce(peer: PeerAnnouncement): void;
+    /** A static client authenticated as this peer: discovery lets go of it. */
+    releaseToStatic(instanceId: string): void;
+    /** Drop discovered clients and candidates that stayed unconnected for the idle window. */
+    pruneIdle(): void;
+    start(): void;
+    stop(): void;
+    private size;
+    private dialDiscovered;
+    private dialCandidate;
+    /**
+     * The candidate authenticated. It runs before the candidate registers the
+     * peer, so the replaced client disconnects first.
+     */
+    private promote;
+    private drop;
+    private dropCandidate;
+}
+//# sourceMappingURL=peer-discovery.d.ts.map

package/dist/cluster/peer-forget.d.ts

@@ -1,6 +1,7 @@
 import { PeerForgetOutcome, type PeerForgetRequest, type PeerForgetResponse } from '@kici-dev/engine';
 import { z } from 'zod';
 import type { PeerRegistry } from './peer-registry.js';
+import { PeerRequestWaiters } from './peer-request-waiters.js';
 /** What forgetting did on one coordinator. */
 export interface PeerForgetResult {
     /** The coordinator that answered. */
@@ -125,14 +126,8 @@ export declare function forgetAcrossCoordinators(registry: PeerRegistry, selfIns
  * link to: registered coordinators whose connection is down.
  */
 export declare function disconnectedCoordinatorIds(registry: PeerRegistry, selfInstanceId: string): string[];
-/** Pending forget requests sent to siblings, keyed by message id. */
-export declare class PeerForgetWaiters {
-    private readonly waiters;
-    /** Wait for the response to `messageId`, or {@link PEER_FORGET_TIMEOUT}. */
-    wait(messageId: string, timeoutMs: number): Promise<PeerForgetResponse | typeof PEER_FORGET_TIMEOUT>;
-    /** Hand a response to the request waiting for it. An unknown id is dropped. */
-    resolve(response: PeerForgetResponse): void;
-    /** End every pending wait with an `error` response naming `reason`. */
-    rejectAll(reason: string): void;
+/** Pending forget requests sent to siblings; a close answers each `error` naming the reason. */
+export declare class PeerForgetWaiters extends PeerRequestWaiters<PeerForgetResponse> {
+    constructor();
 }
 //# sourceMappingURL=peer-forget.d.ts.map

package/dist/cluster/peer-handler.d.ts

@@ -1,15 +1,22 @@
 /**
  * WebSocket handler for incoming peer connections from other orchestrators.
  *
- * Accepts WS upgrade, performs ECDH key exchange for encrypted channel,
- * validates authentication via join token (first connect) or HMAC credential
- * proof (reconnection), registers the peer in PeerRegistry, and routes
- * messages bidirectionally. Sends periodic heartbeats to the connecting peer.
+ * Accepts WS upgrade, performs ECDH key exchange, and authenticates the peer
+ * with mutual-v2: `peer.hello` advertises the scheme, the peer proves its
+ * credential (or, on first join, its join token) with an HMAC over the
+ * handshake transcript, and this server answers with its own proof and
+ * switches to the application key. A join token never arrives here: the peer
+ * sends the token's routing segment, and the role and routing key come from
+ * the join_tokens row. A request in the scheme earlier releases used is refused
+ * with PEER_MUTUAL_AUTH_REQUIRED_REASON. Registers the peer in PeerRegistry and
+ * routes messages bidirectionally. Sends periodic heartbeats to the peer.
  */
-import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type JobProgressAck, type PeerScalerEvent, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerScalerOrphansRequest, type PeerForgetRequest, type PeerForgetResponse, type PeerScalerOrphansResponse, type PeerClusterSettingsRequest, type WorkerClusterSettings, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
-import type { PeerRegistry } from './peer-registry.js';
+import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type JobProgressAck, type PeerScalerEvent, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerScalerOrphansRequest, type PeerForgetRequest, type PeerForgetResponse, type PeerScalerOrphansResponse, type PeerClusterSettingsRequest, type WorkerClusterSettings, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries, type PeerScalerReloadRequest, type PeerScalerReloadResponse } from '@kici-dev/engine';
+import { type PeerRegistry } from './peer-registry.js';
 import { type PEER_FORGET_TIMEOUT, type PeerForgetRequestHandler } from './peer-forget.js';
 import { type SCALER_ORPHANS_TIMEOUT, type ScalerOrphansRequestHandler } from './scaler-orphans-peer.js';
+import type { PEER_REQUEST_TIMEOUT } from './peer-request-waiters.js';
+import type { ScalerFileReload } from '../scaler/file-reload.js';
 import type { PeerCredentialStore } from './peer-credentials.js';
 import { type JoinTokenManager } from './join-token.js';
 /**
@@ -113,6 +120,13 @@ export interface PeerHandlerDeps {
      * requests are answered with ok=false.
      */
     onScalerOrphansRequest?: ScalerOrphansRequestHandler;
+    /**
+     * Reloads this node's scaler config for a peer.scaler.reload.request
+     * (`kici-admin scaler reload`). A request from a peer that is not a
+     * coordinator is refused before it reaches the handler. If undefined,
+     * requests are answered `rejected`.
+     */
+    onScalerReloadRequest?: ScalerFileReload;
     /**
      * Forgets a departed peer a sibling coordinator forgot (`kici-admin peer
      * forget`). A request from a peer that is not a coordinator is refused. If
@@ -120,6 +134,13 @@ export interface PeerHandlerDeps {
      */
     onPeerForgetRequest?: PeerForgetRequestHandler;
 }
+/** The schemes this server lists in peer.hello, in order. */
+export declare const ADVERTISED_PEER_AUTH_SCHEMES: readonly string[];
+/**
+ * A peer.auth.request from a release before mutual authentication: it carries
+ * a proof or a token, or names no scheme.
+ */
+export declare function isLegacyPeerAuthRequest(raw: unknown): boolean;
 /**
  * Create a handler function for incoming peer WebSocket connections.
  *
@@ -143,6 +164,7 @@ export declare function createPeerHandler(deps: PeerHandlerDeps): {
     sendAndWaitAck: (targetInstanceId: string, msg: JobReroute, timeoutMs?: number) => Promise<boolean>;
     sendConfigReloadAndWait: (targetInstanceId: string, msg: PeerConfigReload, timeoutMs?: number) => Promise<PeerConfigReloadResponse | null>;
     sendScalerOrphansAndWait: (targetInstanceId: string, msg: PeerScalerOrphansRequest, timeoutMs: number) => Promise<PeerScalerOrphansResponse | null | typeof SCALER_ORPHANS_TIMEOUT>;
+    sendScalerReloadAndWait: (targetInstanceId: string, msg: PeerScalerReloadRequest, timeoutMs: number) => Promise<PeerScalerReloadResponse | null | typeof PEER_REQUEST_TIMEOUT>;
     sendPeerForgetAndWait: (targetInstanceId: string, msg: PeerForgetRequest, timeoutMs: number) => Promise<PeerForgetResponse | null | typeof PEER_FORGET_TIMEOUT>;
     sendLogsCollectAndWait: (targetInstanceId: string, msg: PeerLogsCollectRequest, timeoutMs: number) => Promise<Buffer>;
     getConnectionCount: () => number;

package/dist/cluster/peer-registry.d.ts

@@ -5,7 +5,17 @@
  * enabling routing decisions without real-time queries. Follows the
  * same label-matching semantics as agent/registry.ts for consistency.
  */
-import type { PeerHeartbeat, PeerCapabilities, ScalerCapacitySummary } from '@kici-dev/engine';
+import type { PeerAuthScheme, PeerHeartbeat, PeerCapabilities, ScalerCapacitySummary } from '@kici-dev/engine';
+/** Which side of a peer link authenticated: the peer dialled us, or we dialled it. */
+export declare enum PeerLinkDirection {
+    Inbound = "inbound",
+    Outbound = "outbound"
+}
+/** The scheme each direction of a peer link authenticated with; null while that direction is down. */
+export interface PeerAuthSchemes {
+    inbound: PeerAuthScheme | null;
+    outbound: PeerAuthScheme | null;
+}
 export interface PeerAgentInfo {
     agentId: string;
     labels: string[];
@@ -52,6 +62,8 @@ export interface PeerInfo {
     clusterSettingsVersion: number;
     /** Cluster role of the peer. */
     role: 'coordinator' | 'worker';
+    /** The authentication scheme of each link direction. */
+    authScheme: PeerAuthSchemes;
     hostname?: string;
     osRelease?: string;
     totalMemoryMb?: number;
@@ -119,6 +131,8 @@ export declare class PeerRegistry {
         routingKeys: string[];
         role?: 'coordinator' | 'worker';
     }): void;
+    /** Record the scheme one direction of the link authenticated with, or null when it closed. */
+    setAuthScheme(instanceId: string, direction: PeerLinkDirection, scheme: PeerAuthScheme | null): void;
     /**
      * Remove a peer entirely from the registry.
      */

package/dist/cluster/peer-request-waiters.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Correlates a peer request with its response by message id. Shared by every
+ * request/response pair the outgoing `PeerClient` and the incoming peer handler
+ * carry, so each pair needs only its response type and its close-time answer.
+ */
+/** A wait that ended without a response. */
+export declare const PEER_REQUEST_TIMEOUT = "timeout";
+/** Pending peer requests of one message pair, keyed by message id. */
+export declare class PeerRequestWaiters<TResponse extends {
+    messageId: string;
+}> {
+    private readonly closedResponse;
+    private readonly waiters;
+    /**
+     * @param closedResponse the answer each pending wait gets when the connection
+     *   closes, built from its message id and the close reason.
+     */
+    constructor(closedResponse: (messageId: string, reason: string) => TResponse);
+    /** Wait for the response to `messageId`, or {@link PEER_REQUEST_TIMEOUT}. */
+    wait(messageId: string, timeoutMs: number): Promise<TResponse | typeof PEER_REQUEST_TIMEOUT>;
+    /** Hand a response to the request waiting for it. An unknown id is dropped. */
+    resolve(response: TResponse): void;
+    /** End every pending wait with the close-time answer naming `reason`. */
+    rejectAll(reason: string): void;
+}
+//# sourceMappingURL=peer-request-waiters.d.ts.map

package/dist/cluster/scaler-orphans-peer.d.ts

@@ -7,6 +7,7 @@
 import { z } from 'zod';
 import type { PeerScalerOrphansRequest, PeerScalerOrphansResponse } from '@kici-dev/engine';
 import type { ScalerOrphansAnswer } from '../scaler/orphan-requests.js';
+import { PeerRequestWaiters } from './peer-request-waiters.js';
 /** Answers a forwarded request from this node's own host and tracking. */
 export type ScalerOrphansRequestHandler = (msg: PeerScalerOrphansRequest) => Promise<ScalerOrphansAnswer>;
 /** The answer of a peer that has no handler wired. */
@@ -27,14 +28,8 @@ export declare const SCALER_ORPHANS_TIMEOUT: "timeout";
  * out its timeout for a peer that cannot answer.
  */
 export declare function replyToScalerOrphansRequest(msg: PeerScalerOrphansRequest, handler: ScalerOrphansRequestHandler | undefined, send: (response: PeerScalerOrphansResponse) => void, logFields?: Record<string, unknown>): void;
-/** Pending scaler orphan requests, keyed by message id. */
-export declare class ScalerOrphansWaiters {
-    private readonly waiters;
-    /** Wait for the response to `messageId`, or {@link SCALER_ORPHANS_TIMEOUT}. */
-    wait(messageId: string, timeoutMs: number): Promise<PeerScalerOrphansResponse | typeof SCALER_ORPHANS_TIMEOUT>;
-    /** Hand a response to the request waiting for it. An unknown id is dropped. */
-    resolve(response: PeerScalerOrphansResponse): void;
-    /** End every pending wait with an `ok: false` response naming `reason`. */
-    rejectAll(reason: string): void;
+/** Pending scaler orphan requests; a close answers each `ok: false` naming the reason. */
+export declare class ScalerOrphansWaiters extends PeerRequestWaiters<PeerScalerOrphansResponse> {
+    constructor();
 }
 //# sourceMappingURL=scaler-orphans-peer.d.ts.map

package/dist/cluster/scaler-reload-fanout.d.ts

@@ -0,0 +1,29 @@
+import { type PeerScalerReloadRequest, type PeerScalerReloadResponse, type ScalerReloadInstanceResult } from '@kici-dev/engine';
+import type { ScalerFileReload } from '../scaler/file-reload.js';
+import type { PeerRegistry } from './peer-registry.js';
+import { PEER_REQUEST_TIMEOUT } from './peer-request-waiters.js';
+/**
+ * Sends one scaler reload request to a peer and waits for its answer. Resolves
+ * null when no connection reached the peer, and {@link PEER_REQUEST_TIMEOUT}
+ * when it did not answer within `timeoutMs`.
+ */
+export type SendScalerReload = (instanceId: string, msg: PeerScalerReloadRequest, timeoutMs: number) => Promise<PeerScalerReloadResponse | null | typeof PEER_REQUEST_TIMEOUT>;
+export interface ScalerReloadAcrossClusterOptions {
+    selfInstanceId: string;
+    selfRole: 'coordinator' | 'worker';
+    registry: Pick<PeerRegistry, 'getAllPeers'>;
+    local: ScalerFileReload;
+    send: SendScalerReload;
+    /** Reload this orchestrator only. */
+    single: boolean;
+    /** How long to wait for each peer's answer. */
+    timeoutMs: number;
+}
+/**
+ * Reload this orchestrator and every peer in its registry. A connected peer
+ * (coordinator or worker) gets the request; a known peer that is not connected
+ * is reported `unreachable` without one. The local reload and the peer
+ * requests run at the same time; the first result is this orchestrator's.
+ */
+export declare function reloadScalersAcrossCluster(opts: ScalerReloadAcrossClusterOptions): Promise<ScalerReloadInstanceResult[]>;
+//# sourceMappingURL=scaler-reload-fanout.d.ts.map

package/dist/cluster/scaler-reload-peer.d.ts

@@ -0,0 +1,16 @@
+import { type PeerScalerReloadRequest, type PeerScalerReloadResponse } from '@kici-dev/engine';
+import type { ScalerFileReload } from '../scaler/file-reload.js';
+import { PeerRequestWaiters } from './peer-request-waiters.js';
+/** The answer to a request a worker sent: a worker never directs another node's scalers. */
+export declare const SCALER_RELOAD_COORDINATORS_ONLY_DETAIL = "scaler reload requests are accepted from coordinators only";
+/**
+ * Answer one request through `send`. A missing handler and a handler failure
+ * are both answered `rejected`, so the requester never waits out its timeout
+ * for a peer that cannot answer.
+ */
+export declare function replyToScalerReloadRequest(msg: PeerScalerReloadRequest, handler: ScalerFileReload | undefined, send: (response: PeerScalerReloadResponse) => void, logFields?: Record<string, unknown>): void;
+/** Pending scaler reload requests; a close answers each `unreachable` naming the reason. */
+export declare class ScalerReloadWaiters extends PeerRequestWaiters<PeerScalerReloadResponse> {
+    constructor();
+}
+//# sourceMappingURL=scaler-reload-peer.d.ts.map

package/dist/cluster/worker-cluster-settings.d.ts

@@ -13,6 +13,7 @@ import type { ClusterSettingsReader } from './cluster-settings-reader.js';
 export interface WorkerClusterSettingDefaults {
     agentTokenTtlMs: number;
     firecrackerApiSocketWaitMs: number;
+    concurrencyWaitTimeoutMs: number;
 }
 /** A worker snapshot and the `cluster_settings.version` its values belong to. */
 export interface WorkerClusterSettingsSnapshot {

package/dist/cold-store/load-access-log-range.d.ts

@@ -14,9 +14,9 @@
  * the hot tuple comparison `(created_at, id) < (cursor.createdAt, cursor.id)`.
  * The cursor also carries a `source: 'hot' | 'cold'` discriminator so a
  * cursor minted while the page boundary was inside cold can resume in
- * the cold stream on the next request. Cursors without a `source` field
- * default to `'hot'` — backwards-compatible with any tab the user had
- * open across a deploy boundary.
+ * the cold stream on the next request. A cursor without a valid `source` is
+ * invalid, like any other unparsable cursor, and the read starts from the
+ * first page.
  */
 import { type Kysely, type Selectable } from 'kysely';
 import { type ColdStore } from '@kici-dev/shared';

package/dist/cold-store/tables/access-log.d.ts

@@ -1,15 +1,13 @@
 /**
  * `access_log` cold-store adapter (Orchestrator side).
  *
- *
  * Contract:
  *   - tenant column: `org_id` (NULL → synthetic `__orchestrator__`)
  *   - partition column: `created_at`
  *   - warm TTL: 30 days
  *
- * Replaces the previous 90-day `expires_at`-based hard delete (removed
- * by migration 007). Rows older than 30 days now live in S3
- * indefinitely.
+ * Rows older than 30 days live in S3 indefinitely; there is no
+ * `expires_at`-based hard delete.
  *
  * Synthetic tenant: orchestrator-level access events (e.g. scheduler
  * tick failures, admin-triggered scheduled jobs) carry `org_id IS
@@ -21,20 +19,18 @@
  * audit row per ~50K-row chunk.
  */
 import { type Kysely, type Selectable } from 'kysely';
-import { type ChunkCommitMetadata, type ColdRetention, type ColdStoreTableConfig, type EligiblePartition, type TableAdapter } from '@kici-dev/shared';
+import { type ChunkCommitMetadata, type ColdRetention, type ColdStoreTableConfig, type EligiblePartition, type TableAdapter, PgTableAdapterBase } from '@kici-dev/shared';
 import type { AccessLogTable, Database } from '../../db/types.js';
 export type AccessLogColdStoreRow = Selectable<AccessLogTable>;
 export interface AccessLogAdapterOptions {
     overrides?: Partial<ColdStoreTableConfig>;
 }
-export declare class AccessLogAdapter implements TableAdapter<AccessLogColdStoreRow> {
-    private readonly kdb;
+export declare class AccessLogAdapter extends PgTableAdapterBase<Database> implements TableAdapter<AccessLogColdStoreRow> {
     private readonly instanceId;
     readonly db: 'orchestrator';
     readonly table = "access_log";
     readonly tenantColumn = "org_id";
     readonly partitionColumn = "created_at";
-    readonly config: ColdStoreTableConfig;
     constructor(kdb: Kysely<Database>, instanceId: string, opts?: AccessLogAdapterOptions);
     listEligiblePartitions(args: {
         warmCutoff: Date;
@@ -43,10 +39,6 @@ export declare class AccessLogAdapter implements TableAdapter<AccessLogColdStore
         tenantId: string;
         warmCutoff: Date;
     }): Promise<number>;
-    withPartitionLock<T>(args: {
-        tenantId: string;
-        partitionDate: string;
-    }, fn: () => Promise<T>): Promise<T | null>;
     selectEligible(args: {
         tenantId: string;
         partitionDate: string;