@kici-dev/orchestrator 0.1.19 → 0.1.21

package/dist/cluster/coordinator.d.ts

@@ -10,7 +10,7 @@
  * - "Peers report step-by-step progress back to coordinator"
  * - "Cancel mode: graceful -- finish current step, cancel remaining"
  */
-import type { JobReroute, JobProgress, PeerScalerEvent, PeerToPeerMessage, ResourceRequest, LabelMatcher } from '@kici-dev/engine';
+import type { JobReroute, JobProgress, JobProgressAck, PeerScalerEvent, PeerToPeerMessage, ResourceRequest, LabelMatcher } from '@kici-dev/engine';
 import type { PeerRegistry } from './peer-registry.js';
 import type { PeerClient } from './peer-client.js';
 import type { Dispatcher } from '../agent/dispatcher.js';
@@ -142,7 +142,7 @@ export declare class RunCoordinator {
      * this split, every job-level event was silently funnelled into
      * `onStepStatus` and the run stayed in `running` forever.
      */
-    onPeerJobProgress(msg: JobProgress): void;
+    onPeerJobProgress(msg: JobProgress, reply?: (m: JobProgressAck) => void): void;
     /**
      * Handle a scaler provisioning event forwarded by a worker peer.
      *
@@ -211,7 +211,9 @@ export declare class RunCoordinator {
      */
     private computeAvailableCapacity;
     /**
-     * Track a rerouted job for cancel propagation.
+     * Track a rerouted job for cancel propagation, and durably tag the projected
+     * `execution_jobs` row with the owning worker peer so run-recovery sweepers
+     * do not force-fail the job while its worker is connected.
      */
     private trackReroutedJob;
 }

package/dist/cluster/index.d.ts

@@ -8,6 +8,8 @@ export { PeerRegistry } from './peer-registry.js';
 export type { PeerInfo, PeerAgentInfo, PeerRegistryOptions } from './peer-registry.js';
 export { PeerClient } from './peer-client.js';
 export type { PeerClientOptions } from './peer-client.js';
+export { PeerAuthCoordinator } from './peer-auth-coordinator.js';
+export type { AuthDecision, RejectionAction } from './peer-auth-coordinator.js';
 export { createPeerHandler } from './peer-handler.js';
 export type { PeerHandlerDeps } from './peer-handler.js';
 export { PeerCredentialStore, readCredentialFile, writeCredentialFile, } from './peer-credentials.js';

package/dist/cluster/join-token.d.ts

@@ -12,7 +12,7 @@
  * Config bundle encryption: AES-256-GCM with random 12-byte IV.
  * Wire format: <12-byte IV><16-byte auth tag><ciphertext>
  */
-import type { Kysely } from 'kysely';
+import { Kysely } from 'kysely';
 /**
  * Silence the module-level logger (e.g. when a CLI is emitting JSON on stdout
  * and a stray log line would break the contract).
@@ -57,15 +57,37 @@ export declare class JoinTokenManager {
      * `peer_credentials_active_uniq` partial unique index).
      *
      * On a 0-row claim, a follow-up SELECT disambiguates not-found / expired
-     * / already-used so callers can branch on the specific reason — the
-     * recovery path in peer-handler.ts keys on the "already used" string
-     * specifically. The follow-up only fires on the unhappy path.
+     * / already-used / reusable-by-same-instance so callers can branch on the
+     * specific reason — the recovery path in peer-handler.ts keys on the
+     * "already used" string specifically. The follow-up only fires on the
+     * unhappy path.
+     *
+     * Self-healing reuse: a join token is re-consumable by the same joining
+     * peer (`peerInstanceId`) until its `expires_at`. A peer that lost its
+     * credential (transient outage / deleted credential file) re-presents the
+     * still-valid join token already in its env; the coordinator accepts the
+     * reuse and issues a fresh credential — no operator action, no cluster
+     * redeploy. Reuse is bounded by BOTH `expires_at` AND the consuming
+     * instanceId, so it never widens a leaked token's usefulness beyond the
+     * instance that first consumed it.
      */
-    validateAndConsumeToken(token: string, consumedBy: string): Promise<{
+    validateAndConsumeToken(token: string, consumedBy: string, peerInstanceId: string): Promise<{
         routing: TokenRouting;
         keys: DerivedKeys;
     }>;
 }
+/**
+ * Build a JoinTokenManager backed by its own connection pool to the given
+ * orchestrator database URL. Mirrors `createPeerCredentialStoreFromUrl`;
+ * consumed by E2E tests that need to exercise token validation/reuse against
+ * the real cluster DB.
+ */
+export declare function createJoinTokenManagerFromUrl(databaseUrl: string, opts?: {
+    maxConnections?: number;
+}): {
+    manager: JoinTokenManager;
+    dispose: () => Promise<void>;
+};
 /**
  * Narrow detector for the "already been used" error thrown by
  * validateAndConsumeToken(). Used by peer-handler.ts to branch into the

package/dist/cluster/peer-auth-coordinator.d.ts

@@ -0,0 +1,36 @@
+import { type CredentialFileData } from './peer-credentials.js';
+export type AuthDecision = {
+    mode: 'credential';
+    credential: CredentialFileData;
+} | {
+    mode: 'token-join';
+    token: string;
+    complete: (issued: CredentialFileData | null) => void;
+} | {
+    mode: 'no-auth';
+};
+export type RejectionAction = 'retry-credential' | 'rejoin';
+export declare class PeerAuthCoordinator {
+    private readonly credentialFile;
+    private readonly instanceId;
+    private readonly joinToken?;
+    private readonly joinWaitTimeoutMs;
+    /** Promise-chain mutex tail; every file op awaits the prior one. */
+    private lock;
+    /** Set while one peer-client is mid token-join; siblings await it. */
+    private inFlightJoin;
+    constructor(opts: {
+        credentialFile: string;
+        instanceId: string;
+        joinToken?: string;
+        joinWaitTimeoutMs?: number;
+    });
+    /** Run `fn` exclusively against the credential file. */
+    private withLock;
+    private readValidCredential;
+    decideAuth(): Promise<AuthDecision>;
+    private makeComplete;
+    private awaitInFlightJoin;
+    reportRejection(provedCredential: string | null, reason: string): Promise<RejectionAction>;
+}
+//# sourceMappingURL=peer-auth-coordinator.d.ts.map

package/dist/cluster/peer-client.d.ts

@@ -10,8 +10,9 @@
  * Authentication uses ECDH key exchange followed by join token (first connect)
  * or HMAC credential proof (reconnection).
  */
-import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
+import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type JobProgressAck, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
 import type { PeerRegistry } from './peer-registry.js';
+import type { PeerAuthCoordinator } from './peer-auth-coordinator.js';
 type PeerConnectionState = 'disconnected' | 'connecting' | 'handshaking' | 'authenticating' | 'connected';
 export interface PeerClientOptions {
     /** WebSocket URL of the remote peer orchestrator. */
@@ -20,6 +21,8 @@ export interface PeerClientOptions {
     joinToken?: string;
     /** Path to the credential file for reconnection. */
     credentialFile: string;
+    /** Shared per-orchestrator coordinator that owns the credential file. */
+    authCoordinator: PeerAuthCoordinator;
     /** This orchestrator's instance ID. */
     instanceId: string;
     /** Peer registry to update on heartbeats from remote peer. */
@@ -33,7 +36,14 @@ export interface PeerClientOptions {
     /** Callback when a job reroute request is received from peer. */
     onJobReroute: (msg: JobReroute) => Promise<void>;
     /** Callback when a job progress update is received from peer. */
-    onJobProgress: (msg: JobProgress) => void;
+    onJobProgress: (msg: JobProgress, reply: (m: JobProgressAck) => void) => void;
+    /**
+     * Callback invoked once this client reaches the `connected` state (initial
+     * connect AND every reconnect). Carries this client's peer URL.
+     */
+    onConnected?: (url: string) => void;
+    /** Callback when a coordinator ACKs a terminal job.progress this worker sent. */
+    onJobProgressAck?: (msg: JobProgressAck) => void;
     /** This orchestrator's role. Default: 'coordinator'. */
     role?: 'coordinator' | 'worker';
     /** Callback when a job cancel request is received from peer. */
@@ -107,8 +117,13 @@ export declare class PeerClient {
     /** Correlates peer.logs.collect.request with the peer's chunked subtree response. */
     private readonly logsCollectWaiters;
     private readonly url;
-    private readonly joinToken?;
-    private readonly credentialFile;
+    private readonly authCoordinator;
+    /** Set true between deciding token-join and receiving the auth response. */
+    private isJoiner;
+    /** complete() callback for the in-flight token-join, if this client is joiner. */
+    private joinComplete;
+    /** The credential string this client last built an HMAC proof with. */
+    private lastProvedCredential;
     private readonly instanceId;
     private readonly role;
     private readonly peerRegistry;
@@ -117,6 +132,8 @@ export declare class PeerClient {
     private readonly maxReconnectDelayMs;
     private readonly onJobReroute;
     private readonly onJobProgress;
+    private readonly onConnected?;
+    private readonly onJobProgressAck?;
     private readonly onJobCancel;
     private readonly onPeerLogChunk?;
     private readonly onPeerCacheUploadRequest?;
@@ -185,6 +202,15 @@ export declare class PeerClient {
     getReconnectDelay(): number;
     private doConnect;
     private handleMessage;
+    /**
+     * Handle a rejected peer.auth.response. For the three credential-divergence
+     * reasons, delegate to the coordinator (which deletes the shared file only if
+     * no sibling has refreshed it, preventing a revocation cascade). A failed
+     * token-join releases the in-flight join so siblings can retry. Config-error
+     * reasons (role mismatch / missing auth method / protocol version) skip the
+     * coordinator entirely — deletion would not help.
+     */
+    private handleAuthRejected;
     /**
      * Determine auth method and send encrypted auth request.
      *
@@ -196,16 +222,10 @@ export declare class PeerClient {
      * connecting to any peer can use the same credential, as long as the
      * `instanceId` on disk matches ours.
      *
-     * (4-coordinator mesh) bug history: the previous gate was
-     * `cred.coordinatorUrl === this.coordinatorUrl`, which broke the moment a
-     * second peer-client on the same orchestrator tried to authenticate to a
-     * different peer URL — it would see the first peer-client's credential,
-     * reject it (URL mismatch), fall through to the join token, and get
-     * permanently rejected because the token had already been consumed by the
-     * first peer-client. The Option B fix (commit 79f93da4) keyed the match on
-     * `instanceId`; this follow-up (Plan 04 Task 1 cleanup) drops the
-     * `coordinatorUrl` field from the credential file schema entirely because
-     * it was dead weight once the match switched to identity scope.
+     * The auth-method decision is delegated to the shared `PeerAuthCoordinator`,
+     * which serializes sibling peer-clients so exactly one token-joins per
+     * reconnect storm (the rest reuse the freshly-written credential). This
+     * client never reads or writes the credential file directly.
      */
     private sendAuthRequest;
     private routeMessage;

package/dist/cluster/peer-credentials.d.ts

@@ -36,7 +36,17 @@ export declare class PeerCredentialStore {
     private readonly db;
     constructor(db: Kysely<any>);
     /**
-     * Save a new peer credential.
+     * Save a new peer credential, revoking any prior active credential for the
+     * same instanceId (the `peer_credentials_active_uniq` partial unique index
+     * permits exactly one active credential per instanceId).
+     *
+     * Returns `{ revokedCount }` — how many previously-active credentials this
+     * save revoked. Because a coordinator's peer-clients share one
+     * identity-scoped credential, a `revokedCount > 0` here means every sibling
+     * peer-client of `instanceId` that was authenticating with the old
+     * credential is now invalidated and will be rejected on its next proof. The
+     * caller logs this so a revoke that cascades sibling rejections is visible in
+     * the orchestrator logs.
      */
     save(opts: {
         instanceId: string;
@@ -45,7 +55,9 @@ export declare class PeerCredentialStore {
         routingKeys: string[];
         sourceTokenHash?: string;
         expiryDays?: number;
-    }): Promise<void>;
+    }): Promise<{
+        revokedCount: number;
+    }>;
     /**
      * Find a non-revoked, non-expired credential by its hash.
      */

package/dist/cluster/peer-handler.d.ts

@@ -6,7 +6,7 @@
  * proof (reconnection), registers the peer in PeerRegistry, and routes
  * messages bidirectionally. Sends periodic heartbeats to the connecting peer.
  */
-import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type PeerScalerEvent, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
+import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type JobProgressAck, type PeerScalerEvent, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
 import type { PeerRegistry } from './peer-registry.js';
 import type { PeerCredentialStore } from './peer-credentials.js';
 import { type JoinTokenManager } from './join-token.js';
@@ -40,7 +40,7 @@ export interface PeerHandlerDeps {
     /** Callback when a job reroute request is received from peer. */
     onJobReroute: (msg: JobReroute) => Promise<void>;
     /** Callback when a job progress update is received from peer. */
-    onJobProgress: (msg: JobProgress) => void;
+    onJobProgress: (msg: JobProgress, reply: (m: JobProgressAck) => void) => void;
     /** Callback when a scaler provisioning event is forwarded by a worker peer. */
     onPeerScalerEvent?: (msg: PeerScalerEvent) => void;
     /** Callback when a job cancel request is received from peer. */

package/dist/cluster/rerouted-job-guard.d.ts

@@ -0,0 +1,39 @@
+import type { PeerRegistry } from './peer-registry.js';
+/**
+ * Grace window (ms) during which a rerouted job stays deferred from the
+ * recovery sweepers even though its worker peer is momentarily disconnected.
+ *
+ * A worker that completes a job durably buffers the terminal status in its
+ * PeerOutbox and replays it on the next reconnect. A coordinator restart or a
+ * leadership transition makes the worker's peer-WS flap (an abnormal 1006
+ * close) for a few seconds before it reconnects and replays — so sampling
+ * `peer.connected` at the exact instant a sweeper runs can catch the peer
+ * mid-flap and wrongly force-fail a run whose job already succeeded.
+ *
+ * The grace is two peer-stale-timeout windows (peerStaleTimeoutMs default
+ * 60_000): a peer that flaps and reconnects inside one stale window is always
+ * deferred, while a peer absent for two full windows is treated as dead and
+ * its job is failed — so a genuinely dead worker can never hang a run forever.
+ */
+export declare const DEFAULT_REROUTE_FLAP_GRACE_MS = 120000;
+/**
+ * A non-terminal job that was rerouted to a remote worker peer must NOT be
+ * force-failed by the run-recovery sweepers (OrphanRecovery, StaleRunDetector)
+ * while that worker peer can still replay the job's terminal status from its
+ * durable outbox.
+ *
+ * Returns `true` (defer the sweeper's force-fail) when the job carries a
+ * `rerouted_to_peer` marker AND its peer is either currently connected OR was
+ * last seen within the flap-grace window (a transient peer-WS reconnect during
+ * a coordinator restart / leadership transition). A rerouted job whose peer has
+ * been gone longer than the grace window — or is no longer tracked at all — is
+ * NOT deferred, so a dead worker's job is still failed and cannot hang forever.
+ * Local (non-rerouted) jobs — `rerouted_to_peer === null` — are never deferred.
+ */
+export declare function shouldDeferReroutedJob(job: {
+    rerouted_to_peer: string | null;
+}, peerRegistry: Pick<PeerRegistry, 'getPeer'>, opts?: {
+    nowMs?: number;
+    flapGraceMs?: number;
+}): boolean;
+//# sourceMappingURL=rerouted-job-guard.d.ts.map

package/dist/db/migrations/043_rerouted_to_peer.d.ts

@@ -0,0 +1,4 @@
+import { type Kysely } from 'kysely';
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=043_rerouted_to_peer.d.ts.map

package/dist/db/migrations/044_check_mode.d.ts

@@ -0,0 +1,4 @@
+import { type Kysely } from 'kysely';
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=044_check_mode.d.ts.map

package/dist/db/migrations/045_host_properties.d.ts

@@ -0,0 +1,16 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add the typed host-vars dimension to the host roster:
+ *
+ * - `host_roster.host_properties jsonb NOT NULL DEFAULT '{}'` — the typed
+ *   property bag (`string | number | boolean` values) reported by the agent at
+ *   registration and/or declared by the operator (`kici-admin host declare
+ *   --prop`). Labels stay the flat-string grouping dimension; properties are
+ *   the separate queryable host-vars dimension.
+ *
+ * Idempotent (`ADD COLUMN IF NOT EXISTS`): re-running on a DB that already has
+ * the column is a no-op. Staging data is preserved (additive column).
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=045_host_properties.d.ts.map

package/dist/db/migrations/046_join_token_consumed_by_instance.d.ts

@@ -0,0 +1,17 @@
+import { type Kysely } from 'kysely';
+/**
+ * Record the joining peer's instanceId at first consumption of a join token:
+ *
+ * - `join_tokens.consumed_by_instance text` — the `instanceId` of the peer that
+ *   consumed the token. This is the *joining peer*, distinct from `consumed_by`
+ *   (the coordinator that processed the claim). It lets a token be reused by the
+ *   same peer instance until `expires_at`, so a peer that lost its credential
+ *   (transient outage / deleted credential file) self-heals by re-presenting the
+ *   join token already in its env — no operator action, no cluster redeploy.
+ *
+ * Idempotent (`ADD COLUMN IF NOT EXISTS`): re-running on a DB that already has
+ * the column is a no-op. Staging data is preserved (additive nullable column).
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=046_join_token_consumed_by_instance.d.ts.map

package/dist/db/types.d.ts

@@ -86,8 +86,14 @@ export interface JoinTokenTable {
     expires_at: Date;
     /** When this token was consumed (null = unused) */
     consumed_at: Date | null;
-    /** Who consumed this token (null = unused) */
+    /** Who consumed this token (null = unused) — the coordinator that processed the claim */
     consumed_by: string | null;
+    /**
+     * The instanceId of the joining peer that consumed this token (null = unused).
+     * Distinct from `consumed_by` (the coordinator). Lets the same peer instance
+     * reuse the token until `expires_at` to self-heal after a transient outage.
+     */
+    consumed_by_instance: string | null;
 }
 export type JoinToken = Selectable<JoinTokenTable>;
 export type NewJoinToken = Insertable<JoinTokenTable>;
@@ -300,6 +306,12 @@ export interface ExecutionRunTable {
     workflow_timeout_ms: ColumnType<number | null, number | null | undefined, number | null>;
     /** When this record was created */
     created_at: Generated<Date>;
+    /**
+     * Run mode for idempotent steps (`apply` | `check` | `check-fail-on-drift`,
+     * the `CheckMode` enum). NULL means a legacy/apply run. A non-apply value
+     * labels the run a check-mode preview in the dashboard.
+     */
+    check_mode: string | null;
     /**
      * Set inside the cold-store archive transaction before the row is
      * DELETEd. Survivors carry NULL. Exists so a future
@@ -382,6 +394,8 @@ export interface ExecutionJobTable {
     wave_max_parallel: number | null;
     /** The fan-out base's `failFast` policy, stamped on every child. NULL = no bounded wave. */
     wave_fail_fast: boolean | null;
+    /** Instance id of the worker peer this job was rerouted to, or null if local. */
+    rerouted_to_peer: string | null;
     /** When this record was created */
     created_at: Generated<Date>;
     /**
@@ -429,6 +443,16 @@ export interface ExecutionStepTable {
     step_type: Generated<string>;
     /** JSON array of secret context names accessed by this step. NULL = tracking not available (old runs). */
     secrets_accessed: string | null;
+    /**
+     * Idempotent per-step outcome (`CheckStepOutcome`: skipped | applied |
+     * declined | dry-run | no_check). NULL when the step ran without a check
+     * mode. Orthogonal to `status`.
+     */
+    check_outcome: string | null;
+    /** Human-readable drift summary (`summarize(drift)`). NULL when no drift. */
+    drift_summary: string | null;
+    /** Structured drift value returned by `check()` (JSONB). NULL when no drift. */
+    drift: ColumnType<unknown | null, unknown, unknown>;
     /** When this record was created */
     created_at: Generated<Date>;
     /**
@@ -1583,6 +1607,14 @@ export interface HostRosterTable {
     arch: string | null;
     /** Which orchestrator instance holds the live WS; null = disconnected. */
     connected_instance_id: string | null;
+    /**
+     * Typed host-vars dimension (jsonb). A `{ [key]: string | number | boolean }`
+     * bag reported by the agent at registration and/or declared by the operator
+     * (`kici-admin host declare --prop`), shallow-merged on upsert. NOT NULL,
+     * defaults to `{}`. pg returns the parsed object on select; accept a
+     * JSON-stringified value on insert/update.
+     */
+    host_properties: ColumnType<Record<string, string | number | boolean>, Record<string, string | number | boolean> | string | undefined, Record<string, string | number | boolean> | string>;
     last_seen: ColumnType<Date, Date | string | undefined, Date | string>;
     created_at: Generated<Date>;
     updated_at: ColumnType<Date, Date | string | undefined, Date | string>;

package/dist/index.d.ts

@@ -4,5 +4,9 @@ export { createCacheStorage } from './storage/index.js';
 export { SourceCache } from './cache/source-cache.js';
 export { DepCache } from './cache/dep-cache.js';
 export { UserCache, DEFAULT_USER_CACHE_QUOTA_BYTES, DEFAULT_USER_CACHE_TTL_MS, type UserCacheRef, type UserCacheRestoreResult, type UserCacheBeginSaveResult, } from './cache/user-cache.js';
-export { PeerCredentialStore, createPeerCredentialStoreFromUrl, type PeerCredential, } from './cluster/peer-credentials.js';
+export { PeerCredentialStore, createPeerCredentialStoreFromUrl, type PeerCredential, type CredentialFileData, } from './cluster/peer-credentials.js';
+export { PeerAuthCoordinator, type AuthDecision, type RejectionAction, } from './cluster/peer-auth-coordinator.js';
+export { JoinTokenManager, createJoinTokenManagerFromUrl } from './cluster/join-token.js';
+export { AdminApiClient } from './cli/api-client.js';
+export { runGithubManifestSetup, type ManifestSetupOptions, type ManifestSetupDeps, } from './cli/commands/source-manifest.js';
 //# sourceMappingURL=index.d.ts.map