@kici-dev/orchestrator 0.12.0 → 0.13.0

package/dist/cluster/coordinator-credential.d.ts

@@ -0,0 +1,45 @@
+import type { Kysely } from 'kysely';
+import { type Logger } from '@kici-dev/shared';
+import type { Database } from '../db/types.js';
+import { PeerCredentialStore } from './peer-credentials.js';
+/** What one reconciliation did. */
+export declare enum CoordinatorCredentialOutcome {
+    Valid = "valid",
+    Issued = "issued",
+    Replaced = "replaced",
+    Revoked = "revoked",
+    Failed = "failed"
+}
+/** Why a new credential was issued, recorded on the issue log line. */
+export declare enum CredentialIssueReason {
+    NoCredential = "no-credential",
+    Expired = "expired",
+    FileMissing = "credential-file-missing",
+    FileMismatch = "credential-file-mismatch"
+}
+export declare const COORDINATOR_CREDENTIAL_REVOKED_MESSAGE = "Peer credential for this coordinator was revoked; not issuing a new one";
+export type CoordinatorCredentialStore = Pick<PeerCredentialStore, 'save' | 'findUnrevokedByInstanceId' | 'findLatestRevokedByInstanceId' | 'retireSelfIssued'>;
+export interface EnsureCoordinatorCredentialOptions {
+    store: CoordinatorCredentialStore;
+    /** Absolute path of the credential file, with `~` already expanded. */
+    credentialFile: string;
+    instanceId: string;
+    /** Whether an instance has a live `cluster_instances` heartbeat. */
+    isInstanceLive: (instanceId: string) => Promise<boolean>;
+    logger?: Logger;
+    now?: () => Date;
+}
+/**
+ * Make sure this coordinator holds a credential its peers accept, issuing one
+ * when it has none. Never throws: a failure is logged and reported as `Failed`.
+ */
+export declare function ensureCoordinatorCredential(opts: EnsureCoordinatorCredentialOptions): Promise<CoordinatorCredentialOutcome>;
+/** The `selfIssue` callback the coordinator's `PeerAuthCoordinator` runs. */
+export declare function coordinatorSelfIssuer(opts: {
+    db: Kysely<Database>;
+    credentialFile: string;
+    instanceId: string;
+    agentMaxReconnectDelayMs: number;
+    clusterInstanceHeartbeatMs: number;
+}): () => Promise<CoordinatorCredentialOutcome>;
+//# sourceMappingURL=coordinator-credential.d.ts.map

package/dist/cluster/coordinator.d.ts

@@ -334,9 +334,25 @@ export declare class RunCoordinator {
      * The run-cancel path needs this for a job this coordinator never routed:
      * the Platform picks any pool member to receive `run.cancel.request`, and only
      * the coordinator holding the agent's socket can put a cancel frame on it.
-     * {@link RunCoordinator.cancelRun} covers rerouted jobs and is unchanged.
+     * `force` rides on the message, so a force cancel stays forced on the peer.
+     * {@link RunCoordinator.cancelRun} covers rerouted jobs.
      */
-    cancelJobOnPeer(peerId: string, runId: string, jobId: string, reason: string): boolean;
+    cancelJobOnPeer(peerId: string, runId: string, jobId: string, reason: string, force?: boolean): boolean;
+    /**
+     * Send a fire-and-forget message to a peer. The outbound client carries it
+     * when connected; otherwise the peer's own connection to this process does.
+     * The outbound client is down while it reconnects, and for good on a
+     * coordinator whose credential an operator revoked.
+     * `PeerClient.send` returns false only when it sent nothing, so the fallback
+     * never delivers a message twice.
+     */
+    private sendToPeer;
+    /**
+     * The link a reroute uses, chosen before sending. A reroute is never retried
+     * over the second link: `false` from a connected client is a NAK or an ack
+     * timeout, and resending would deliver the job twice.
+     */
+    private rerouteTransport;
     /**
      * Local fallback for a re-dispatch: dispatch the job on this coordinator under
      * the existing jobId. Returns true when the local dispatcher accepts it.

package/dist/cluster/index.d.ts

@@ -9,6 +9,7 @@ export type { PeerInfo, PeerAgentInfo, PeerRegistryOptions } from './peer-regist
 export { PeerClient } from './peer-client.js';
 export type { PeerClientOptions } from './peer-client.js';
 export { PeerAuthCoordinator } from './peer-auth-coordinator.js';
+export { coordinatorSelfIssuer } from './coordinator-credential.js';
 export type { AuthDecision, RejectionAction } from './peer-auth-coordinator.js';
 export { createPeerHandler } from './peer-handler.js';
 export type { PeerHandlerDeps } from './peer-handler.js';

package/dist/cluster/instance-heartbeat.d.ts

@@ -1,4 +1,4 @@
-import type { Kysely } from 'kysely';
+import { type Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
 /** Default interval between `cluster_instances` heartbeat writes. */
 export declare const DEFAULT_INSTANCE_HEARTBEAT_MS = 10000;
@@ -23,6 +23,11 @@ export declare const DEFAULT_RECOVERY_GRACE_MS = 120000;
  * recovery machinery would have given it to reconnect.
  */
 export declare function instanceLivenessGraceMs(gracePeriodMs: number, heartbeatMs?: number): number;
+/**
+ * Whether `instanceId` has a `cluster_instances` heartbeat inside `graceMs`:
+ * the same liveness the dispatch plane's ownership predicate reads.
+ */
+export declare function isInstanceLive(db: Kysely<Database>, instanceId: string, graceMs: number): Promise<boolean>;
 export interface InstanceHeartbeatOptions {
     db: Kysely<Database>;
     instanceId: string;

package/dist/cluster/join-token.d.ts

@@ -18,6 +18,8 @@ import { Kysely } from 'kysely';
  * and a stray log line would break the contract).
  */
 export declare function silenceJoinTokenLogger(): void;
+export declare const TOKEN_ALREADY_USED_MESSAGE = "Join token has already been used. Create a new token with: kici-admin peer create-token";
+export declare const TOKEN_EXPIRED_MESSAGE = "Join token has expired. Create a new token with: kici-admin peer create-token";
 interface TokenRouting {
     orgId: string;
     routingKey: string;

package/dist/cluster/peer-auth-coordinator.d.ts

@@ -1,4 +1,5 @@
 import { type CredentialFileData } from './peer-credentials.js';
+import { CoordinatorCredentialOutcome } from './coordinator-credential.js';
 export type AuthDecision = {
     mode: 'credential';
     credential: CredentialFileData;
@@ -15,6 +16,11 @@ export declare class PeerAuthCoordinator {
     private readonly instanceId;
     private readonly joinToken?;
     private readonly joinWaitTimeoutMs;
+    private readonly selfIssue?;
+    /** The issuance in progress; concurrent sibling clients share it. */
+    private selfIssueInFlight;
+    /** An operator revoked this coordinator's credential: never issue again in this process. */
+    private selfIssueRevoked;
     /** Promise-chain mutex tail; every file op awaits the prior one. */
     private lock;
     /** Set while one peer-client is mid token-join; siblings await it. */
@@ -24,11 +30,18 @@ export declare class PeerAuthCoordinator {
         instanceId: string;
         joinToken?: string;
         joinWaitTimeoutMs?: number;
+        /**
+         * Issues this coordinator its own credential when a decision finds no
+         * credential and no join token. Only a coordinator wires it.
+         */
+        selfIssue?: () => Promise<CoordinatorCredentialOutcome>;
     });
     /** Run `fn` exclusively against the credential file. */
     private withLock;
     private readValidCredential;
     decideAuth(): Promise<AuthDecision>;
+    /** Run the issuer under the mutex, once for every concurrent caller. */
+    private runSelfIssue;
     private makeComplete;
     private awaitInFlightJoin;
     reportRejection(provedCredential: string | null, reason: string): Promise<RejectionAction>;

package/dist/cluster/peer-client.d.ts

@@ -13,6 +13,8 @@
 import { type PeerHeartbeat, type PeerToPeerMessage, type JobReroute, type JobProgress, type JobProgressAck, type PeerJobCancel, type PeerLogChunk, type PeerCacheUploadRequest, type PeerCacheUploadResponse, type PeerConfigReload, type PeerConfigReloadResponse, type PeerClusterSettingsRequest, type PeerClusterSettingsResponse, type PeerLogsCollectRequest, type PeerLeaving, type PeerAgentTokenRevoke, type RaftVoteRequest, type RaftVoteResponse, type RaftAppendEntries } from '@kici-dev/engine';
 import type { PeerRegistry } from './peer-registry.js';
 import type { PeerAuthCoordinator } from './peer-auth-coordinator.js';
+/** Logged when this instance has neither a credential file nor a join token. */
+export declare const NO_AUTH_METHOD_MESSAGE = "No peer auth method: this instance has no credential file and no join token";
 type PeerConnectionState = 'disconnected' | 'connecting' | 'handshaking' | 'authenticating' | 'connected';
 export interface PeerClientOptions {
     /** WebSocket URL of the remote peer orchestrator. */
@@ -108,6 +110,8 @@ export declare class PeerClient {
     private heartbeatTimer;
     private reconnectTimer;
     private reconnectAttempts;
+    /** Whether this client already warned that it has no auth method; re-armed by an accepted auth. */
+    private noAuthWarned;
     private intentionalDisconnect;
     private _targetInstanceId;
     private sessionKey;

package/dist/cluster/peer-credentials.d.ts

@@ -1,4 +1,15 @@
 import { Kysely } from 'kysely';
+/** How a credential row was issued, recorded in `peer_credentials.metadata.issuance`. */
+export declare enum PeerCredentialIssuance {
+    /** A coordinator issued the credential to itself (`coordinator-credential.ts`). */
+    Self = "self"
+}
+/**
+ * `metadata` key set when a coordinator retires the self-issued credential its
+ * previous run left behind. An operator revoke never sets it, which is how a
+ * retirement is told apart from a revoke.
+ */
+export declare const RETIRED_BY_INSTANCE_KEY = "retiredByInstance";
 /**
  * A peer credential record stored in the coordinator's database.
  */
@@ -14,6 +25,7 @@ export interface PeerCredential {
     lastValidatedBy: string | null;
     expiresAt: Date;
     revokedAt: Date | null;
+    metadata: Record<string, unknown>;
 }
 /**
  * Data persisted in this orchestrator's local credential file
@@ -55,6 +67,7 @@ export declare class PeerCredentialStore {
         routingKeys: string[];
         sourceTokenHash?: string;
         expiryDays?: number;
+        metadata?: Record<string, unknown>;
     }): Promise<{
         revokedCount: number;
     }>;
@@ -66,6 +79,24 @@ export declare class PeerCredentialStore {
      * Find a non-revoked, non-expired credential by instance ID.
      */
     findByInstanceId(instanceId: string): Promise<PeerCredential | null>;
+    /**
+     * The credential row for `instanceId` that nobody revoked, expired or not.
+     * `peer_credentials_active_uniq` allows at most one.
+     */
+    findUnrevokedByInstanceId(instanceId: string): Promise<PeerCredential | null>;
+    /** The most recently revoked credential row for `instanceId`, or null. */
+    findLatestRevokedByInstanceId(instanceId: string): Promise<PeerCredential | null>;
+    /**
+     * Revoke `instanceId`'s unrevoked, self-issued row whose hash is
+     * `credentialHash`, recording who retired it. Every condition sits in the
+     * WHERE clause, so a row that changed meanwhile is left alone.
+     * Returns whether a row was retired.
+     */
+    retireSelfIssued(opts: {
+        instanceId: string;
+        credentialHash: string;
+        retiredByInstance: string;
+    }): Promise<boolean>;
     /**
      * Update the last_seen_at timestamp for a credential.
      * Optionally records which coordinator validated it.

package/dist/config.d.ts

@@ -147,6 +147,7 @@ declare const configSchema: z.ZodObject<{
     reconnectReplayWindowHours: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheBuildTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheMaxTarballBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    cacheUploadSettleTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     userCacheQuotaBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     userCacheTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     artifactQuotaBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
@@ -357,6 +358,7 @@ export declare const packagingConfigSchema: z.ZodObject<{
     reconnectReplayWindowHours: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheBuildTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     cacheMaxTarballBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    cacheUploadSettleTimeoutMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     userCacheQuotaBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     userCacheTtlMs: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
     artifactQuotaBytes: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
@@ -591,6 +593,7 @@ export declare const envDef: import("@kici-dev/shared/env").DefineEnvResult<{
     reconnectReplayWindowHours: number;
     cacheBuildTimeoutMs: number;
     cacheMaxTarballBytes: number;
+    cacheUploadSettleTimeoutMs: number;
     userCacheQuotaBytes: number;
     userCacheTtlMs: number;
     artifactQuotaBytes: number;

package/dist/db/migrations/152_org_settings_cache_upload_settle_timeout.d.ts

@@ -0,0 +1,15 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add `org_settings.cache_upload_settle_timeout_ms BIGINT` (nullable).
+ *
+ * Per-org bound on how long a build job's `job.status success` waits for the
+ * cache publish (object metadata plus pointer) its agent reported just before
+ * it. NULL falls back to the cluster default
+ * (`KICI_CACHE_UPLOAD_SETTLE_TIMEOUT_MS`, 10s); 0 turns the wait off. Operators
+ * tune it via `kici-admin org-settings cache-upload-settle`.
+ *
+ * Idempotent: a re-run on a DB that already has the column is a no-op.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=152_org_settings_cache_upload_settle_timeout.d.ts.map

package/dist/db/migrations/153_execution_runs_cancel_force.d.ts

@@ -0,0 +1,16 @@
+import { type Kysely } from 'kysely';
+/**
+ * Add `execution_runs.cancel_force BOOLEAN` (nullable, no default).
+ *
+ * TRUE once a forced cancel reached the run. The leader's stuck-cancelling
+ * sweep reads it to re-send a cancel that did not complete with `force` set,
+ * so a forced cancel whose first forward to a sibling coordinator was lost is
+ * not re-sent as a graceful one. NULL (every existing row, and every run no
+ * forced cancel reached) re-sends gracefully. No backfill: nothing recorded
+ * whether an earlier cancel was forced.
+ *
+ * Idempotent: a re-run on a DB that already has the column is a no-op.
+ */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;
+//# sourceMappingURL=153_execution_runs_cancel_force.d.ts.map

package/dist/db/types.d.ts

@@ -514,6 +514,13 @@ export interface ExecutionRunTable {
      * cancelled and for one that entered `cancelling` before the column existed.
      */
     cancelling_at: ColumnType<Date | null, Date | null | undefined, Date | null>;
+    /**
+     * TRUE once a forced cancel reached the run (migration 153). The
+     * stuck-cancelling re-drive reads it so a cancel that did not complete is
+     * re-sent with `force`. Never reset to FALSE: a later graceful cancel does
+     * not undo a forced one. NULL re-drives gracefully.
+     */
+    cancel_force: ColumnType<boolean | null, boolean | null | undefined, boolean | null>;
     /**
      * The run's status generation (migration 150). Starts at 0 and rises by one
      * each time the run leaves a terminal status to continue. Forwarded on every
@@ -1994,6 +2001,14 @@ export interface OrgSettingsTable {
      * returns a string on select; accept a number on insert/update.
      */
     queue_timeout_ms: ColumnType<string | null, number | null | undefined, number | null>;
+    /**
+     * Per-org bound (ms) on how long a build job's success waits for the cache
+     * publish its agent reported just before it. 0 turns the wait off. Null =
+     * cluster default (config.cacheUploadSettleTimeoutMs /
+     * KICI_CACHE_UPLOAD_SETTLE_TIMEOUT_MS). Postgres BIGINT — pg returns a string
+     * on select; accept a number on insert/update.
+     */
+    cache_upload_settle_timeout_ms: ColumnType<string | null, number | null | undefined, number | null>;
     /**
      * Per-org expiry (seconds) for a held approval element before it is rejected
      * and its run/job/step fails. NOT NULL, default 86400 (one day). An SDK