@kici-dev/orchestrator 0.11.0 → 0.13.0

package/dist/index.js

@@ -2,7 +2,7 @@ import "node:module";
 import { CopyObjectCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3";
 import { Upload } from "@aws-sdk/lib-storage";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
-import { createLogger, createS3Client, sha256 } from "@kici-dev/shared";
+import { createLogger, createS3Client, sha256, toErrorMessage } from "@kici-dev/shared";
 import * as fs from "node:fs";
 import { mkdtempSync, promises, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
@@ -33,6 +33,7 @@ import { exec } from "node:child_process";
 * TTL is enforced lazily: expired items are deleted on access.
 * Pre-signed URLs are generated for getUrl() with 15-minute expiry.
 */
+const logger$5 = createLogger({ prefix: "s3-cache" });
 /** Pre-signed download URL expiry in seconds (15 minutes) */
 const PRESIGNED_URL_EXPIRY_SECONDS = 900;
 /** Pre-signed upload URL expiry in seconds (30 minutes -- uploads take longer) */
@@ -46,6 +47,11 @@ var S3CacheStorage = class {
 	bucket;
 	prefix;
 	ttlMs;
+	/**
+	* False once the backend answered a conditional self-copy with
+	* `501 NotImplemented`. From then on `updateMeta` copies unconditionally.
+	*/
+	conditionalCopySupported = true;
 	constructor(options) {
 		this.bucket = options.bucket;
 		this.prefix = options.prefix;
@@ -77,6 +83,13 @@ var S3CacheStorage = class {
 	* Returns null if the object doesn't exist (NoSuchKey/NotFound).
 	*/
 	async readMeta(key) {
+		return (await this.readHead(key))?.meta ?? null;
+	}
+	/**
+	* Read an object's TTL metadata together with the ETag of the version it
+	* belongs to. Returns null if the object doesn't exist (NoSuchKey/NotFound).
+	*/
+	async readHead(key) {
 		try {
 			const head = await this.client.send(new HeadObjectCommand({
 				Bucket: this.bucket,
@@ -86,8 +99,11 @@ var S3CacheStorage = class {
 			const lastAccessedAt = head.Metadata?.["last-accessed-at"] ?? "";
 			if (!createdAt || !lastAccessedAt) return null;
 			return {
-				createdAt,
-				lastAccessedAt
+				meta: {
+					createdAt,
+					lastAccessedAt
+				},
+				...head.ETag && { etag: head.ETag }
 			};
 		} catch (err) {
 			if (this.isNotFoundError(err)) return null;
@@ -97,8 +113,41 @@ var S3CacheStorage = class {
 	/**
 	* Update metadata on an S3 object by copying it to itself.
 	* This is the standard S3 pattern for updating metadata without re-uploading data.
+	*
+	* The copy is conditional on `etag`, the version whose metadata the caller
+	* read. A self-copy is a whole-object write: without the condition, one that
+	* races a `put()` of the same key can land after it and write the older body
+	* back. Cache pointers are rewritten in place while builds publish them, so a
+	* touch-on-read racing a publish reverted the pointer to a tarball that no
+	* longer exists. `CopySourceIfMatch` pins the bytes copied to that version and
+	* `IfMatch` refuses the write unless the key still holds it.
+	*
+	* Returns false when the object was replaced since the read (`412`), or is
+	* being replaced right now (`409 ConditionalRequestConflict`): the newer
+	* write stamps metadata of its own, so there is nothing left to refresh.
+	*
+	* An S3-compatible backend that does not implement conditional copies
+	* answers `501 NotImplemented`. The copy then runs again without the
+	* condition, and later copies skip it, as on a backend that ignores the
+	* headers.
 	*/
-	async updateMeta(key, meta) {
+	async updateMeta(key, meta, etag) {
+		const condition = this.conditionalCopySupported ? etag : void 0;
+		try {
+			await this.copyOntoItself(key, meta, condition);
+			return true;
+		} catch (err) {
+			if (!condition) throw err;
+			if (this.isPreconditionFailedError(err) || this.isConditionalConflictError(err)) return false;
+			if (!this.isNotImplementedError(err)) throw err;
+			this.conditionalCopySupported = false;
+			logger$5.warn("The object store does not implement conditional copies; refreshing cache metadata unconditionally", { bucket: this.bucket });
+			await this.copyOntoItself(key, meta, void 0);
+			return true;
+		}
+	}
+	/** Copy an object onto itself with new TTL metadata, conditional on `etag` when given. */
+	async copyOntoItself(key, meta, etag) {
 		const objectKey = this.objectKey(key);
 		await this.client.send(new CopyObjectCommand({
 			Bucket: this.bucket,
@@ -108,9 +157,30 @@ var S3CacheStorage = class {
 			Metadata: {
 				"created-at": meta.createdAt,
 				"last-accessed-at": meta.lastAccessedAt
+			},
+			...etag && {
+				CopySourceIfMatch: etag,
+				IfMatch: etag
 			}
 		}));
 	}
+	/** Check if an error is a failed conditional request (PreconditionFailed, 412). */
+	isPreconditionFailedError(err) {
+		return this.hasErrorShape(err, "PreconditionFailed", 412);
+	}
+	/** Check if an error is a conditional write that raced another write (ConditionalRequestConflict, 409). */
+	isConditionalConflictError(err) {
+		return this.hasErrorShape(err, "ConditionalRequestConflict", 409);
+	}
+	/** Check if an error is a request the backend does not implement (NotImplemented, 501). */
+	isNotImplementedError(err) {
+		return this.hasErrorShape(err, "NotImplemented", 501);
+	}
+	hasErrorShape(err, name, status) {
+		if (typeof err !== "object" || err === null) return false;
+		if (err.name === name) return true;
+		return err.$metadata?.httpStatusCode === status;
+	}
 	/** Check if an error is a "not found" error (NoSuchKey, NotFound, 404). */
 	isNotFoundError(err) {
 		if (typeof err !== "object" || err === null) return false;
@@ -135,8 +205,9 @@ var S3CacheStorage = class {
 		}).done();
 	}
 	async get(key, ttlMsOverride) {
-		const meta = await this.readMeta(key);
-		if (!meta) return null;
+		const head = await this.readHead(key);
+		if (!head) return null;
+		const { meta, etag } = head;
 		if (this.isExpired(meta, ttlMsOverride)) {
 			await this.deleteObject(key);
 			return null;
@@ -155,7 +226,7 @@ var S3CacheStorage = class {
 		}
 		try {
 			meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
-			await this.updateMeta(key, meta);
+			await this.updateMeta(key, meta, etag);
 		} catch {}
 		return data;
 	}
@@ -174,10 +245,10 @@ var S3CacheStorage = class {
 		return existed;
 	}
 	async touch(key) {
-		const meta = await this.readMeta(key);
-		if (!meta) return;
-		meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
-		await this.updateMeta(key, meta);
+		const head = await this.readHead(key);
+		if (!head) return;
+		head.meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
+		await this.updateMeta(key, head.meta, head.etag);
 	}
 	async getUrl(key, ttlMsOverride) {
 		const meta = await this.readMeta(key);
@@ -299,7 +370,8 @@ var S3CacheStorage = class {
 * Signature input: `${method}:${key}:${expiresUnixMs}` with HMAC-SHA256 keyed
 * by a per-orchestrator-process secret (random 32-byte hex at boot, persists
 * for the lifetime of the process). Tokens become invalid on orchestrator
-* restart — fine for the filesystem backend's E2E / single-host use case.
+* restart — fine for the filesystem backend's single-host / test-sandbox use
+* case.
 */
 /** Default URL lifetime: 1 hour (matches S3 backend's PUT-URL lifetime). */
 const DEFAULT_SIGN_URL_TTL_MS = 36e5;
@@ -328,7 +400,7 @@ function signToken(secret, method, key, ttlMs = DEFAULT_SIGN_URL_TTL_MS) {
 * are replaced by HMAC-signed `http://<orchUrl>/api/v1/cache/blob/...` URLs
 * — see `sign-url.ts` for the token mechanics and `app.ts` for the route.
 *
-* Intended for single-host deployments and E2E sandboxes where standing up
+* Intended for single-host deployments and test sandboxes where standing up
 * an S3-compatible service is overkill. Production should still use S3.
 *
 * Metadata is co-located as a sibling JSON file (`<key>.meta.json`) so the
@@ -632,7 +704,7 @@ function createCacheStorage(config) {
 * Old objects are orphaned. The cache TTL is enforced lazily on access, so
 * nothing here removes them; `kici-admin cache purge-legacy` does.
 */
-const logger$3 = createLogger({ prefix: "source-cache" });
+const logger$4 = createLogger({ prefix: "source-cache" });
 /**
 * Cache key for a source tarball, addressed by the tarball's OWN content hash:
 * `source/v2/{orgId}/{sourceTarDigest}.tar.gz`.
@@ -675,25 +747,25 @@ var SourceCache = class SourceCache {
 	async has(orgId, contentHash) {
 		const digest = await this.resolvePointer(orgId, contentHash);
 		if (!digest) {
-			logger$3.debug(`has(${contentHash}): false (no pointer)`, { orgId });
+			logger$4.debug(`has(${contentHash}): false (no pointer)`, { orgId });
 			return false;
 		}
 		const exists = await this.storage.has(sourceTarballKey(orgId, digest));
-		logger$3.debug(`has(${contentHash}): ${exists}`, { orgId });
+		logger$4.debug(`has(${contentHash}): ${exists}`, { orgId });
 		return exists;
 	}
 	async get(orgId, contentHash) {
 		const digest = await this.resolvePointer(orgId, contentHash);
 		if (!digest) {
-			logger$3.debug(`get(${contentHash}): miss (no pointer)`, { orgId });
+			logger$4.debug(`get(${contentHash}): miss (no pointer)`, { orgId });
 			return null;
 		}
 		const key = sourceTarballKey(orgId, digest);
 		const data = await this.storage.get(key);
 		if (data) {
 			await this.storage.touch(key);
-			logger$3.debug(`get(${contentHash}): hit (${data.length} bytes)`, { orgId });
-		} else logger$3.debug(`get(${contentHash}): miss (dangling pointer)`, { orgId });
+			logger$4.debug(`get(${contentHash}): hit (${data.length} bytes)`, { orgId });
+		} else logger$4.debug(`get(${contentHash}): miss (dangling pointer)`, { orgId });
 		return data;
 	}
 	/**
@@ -709,18 +781,18 @@ var SourceCache = class SourceCache {
 		const pointerKey = sourcePointerKey(orgId, contentHash);
 		const digest = (await this.storage.get(pointerKey))?.toString("utf-8").trim() || null;
 		if (!digest) {
-			logger$3.debug(`getUrlAndDigest(${contentHash}): miss (no pointer)`, { orgId });
+			logger$4.debug(`getUrlAndDigest(${contentHash}): miss (no pointer)`, { orgId });
 			return null;
 		}
 		const key = sourceTarballKey(orgId, digest);
 		const url = await this.storage.getUrl(key);
 		if (!url) {
-			logger$3.debug(`getUrlAndDigest(${contentHash}): miss (dangling pointer)`, { orgId });
+			logger$4.debug(`getUrlAndDigest(${contentHash}): miss (dangling pointer)`, { orgId });
 			return null;
 		}
 		await this.storage.touch(key);
 		await this.storage.touch(pointerKey);
-		logger$3.debug(`getUrlAndDigest(${contentHash}): hit`, { orgId });
+		logger$4.debug(`getUrlAndDigest(${contentHash}): hit`, { orgId });
 		return {
 			url,
 			digest
@@ -746,7 +818,7 @@ var SourceCache = class SourceCache {
 	*/
 	async publishPointer(orgId, contentHash, sourceTarDigest) {
 		await this.storage.put(sourcePointerKey(orgId, contentHash), sourceTarDigest);
-		logger$3.info(`publishPointer(${contentHash}) -> ${sourceTarDigest.slice(0, 12)}`, { orgId });
+		logger$4.info(`publishPointer(${contentHash}) -> ${sourceTarDigest.slice(0, 12)}`, { orgId });
 	}
 	/** Store a tarball and the pointer that names it. */
 	async store(orgId, contentHash, tarball) {
@@ -754,7 +826,7 @@ var SourceCache = class SourceCache {
 		const digest = SourceCache.computeDigest(data);
 		await this.storage.put(sourceTarballKey(orgId, digest), data);
 		await this.storage.put(sourcePointerKey(orgId, contentHash), digest);
-		logger$3.info(`store(${contentHash}): stored (${data.length} bytes)`, {
+		logger$4.info(`store(${contentHash}): stored (${data.length} bytes)`, {
 			orgId,
 			digest: digest.slice(0, 12)
 		});
@@ -768,7 +840,7 @@ var SourceCache = class SourceCache {
 		const digest = await this.resolvePointer(orgId, contentHash);
 		await this.storage.delete(sourcePointerKey(orgId, contentHash));
 		const removed = digest ? await this.storage.delete(sourceTarballKey(orgId, digest)) : false;
-		logger$3.info(`remove(${contentHash}): ${removed ? "removed" : "not found"}`, { orgId });
+		logger$4.info(`remove(${contentHash}): ${removed ? "removed" : "not found"}`, { orgId });
 		return removed;
 	}
 };
@@ -785,7 +857,7 @@ var SourceCache = class SourceCache {
 * Keys: `deps/{platform}-{arch}/{depsHash}.tar.gz` (immutable) and
 * `deps/{platform}-{arch}/{lockfileHash}.hash` (the pointer).
 */
-const logger$2 = createLogger({ prefix: "dep-cache" });
+const logger$3 = createLogger({ prefix: "dep-cache" });
 /** Default max tarball size: 500MB */
 const DEFAULT_MAX_TARBALL_BYTES = 524288e3;
 /** Default dependency-cache entry TTL: 30 days. */
@@ -866,14 +938,14 @@ var DepCache = class DepCache {
 		const ttlMsOverride = await this.resolveTtlMsOverride();
 		const depsHash = await this.resolvePointer(lockfileHash, platform, arch, ttlMsOverride, siblingsDigest);
 		if (!depsHash) {
-			logger$2.debug(`has(${lockfileHash}): false (no pointer)`, {
+			logger$3.debug(`has(${lockfileHash}): false (no pointer)`, {
 				platform,
 				arch
 			});
 			return false;
 		}
 		const exists = await this.storage.has(depTarballKey(depsHash, platform, arch), ttlMsOverride);
-		logger$2.debug(`has(${lockfileHash}): ${exists}`, {
+		logger$3.debug(`has(${lockfileHash}): ${exists}`, {
 			platform,
 			arch
 		});
@@ -897,7 +969,7 @@ var DepCache = class DepCache {
 		const pointerKey = depPointerKey(lockfileHash, platform, arch, siblingsDigest);
 		const hash = (await this.storage.get(pointerKey, ttlMsOverride))?.toString("utf-8").trim() || void 0;
 		if (!hash) {
-			logger$2.debug(`getUrlAndHash(${lockfileHash}): miss (no pointer)`, {
+			logger$3.debug(`getUrlAndHash(${lockfileHash}): miss (no pointer)`, {
 				platform,
 				arch
 			});
@@ -906,7 +978,7 @@ var DepCache = class DepCache {
 		const key = depTarballKey(hash, platform, arch);
 		const url = await this.storage.getUrl(key, ttlMsOverride);
 		if (!url) {
-			logger$2.debug(`getUrlAndHash(${lockfileHash}): miss (dangling pointer)`, {
+			logger$3.debug(`getUrlAndHash(${lockfileHash}): miss (dangling pointer)`, {
 				platform,
 				arch
 			});
@@ -914,7 +986,7 @@ var DepCache = class DepCache {
 		}
 		await this.storage.touch(key);
 		await this.storage.touch(pointerKey);
-		logger$2.debug(`getUrlAndHash(${lockfileHash}): hit`, {
+		logger$3.debug(`getUrlAndHash(${lockfileHash}): hit`, {
 			platform,
 			arch,
 			hasHash: true
@@ -938,7 +1010,7 @@ var DepCache = class DepCache {
 	*/
 	async publishPointer(lockfileHash, platform, arch, depsHash, siblingsDigest) {
 		await this.storage.put(depPointerKey(lockfileHash, platform, arch, siblingsDigest), depsHash);
-		logger$2.info(`publishPointer(${lockfileHash}) -> ${depsHash.slice(0, 12)}`, {
+		logger$3.info(`publishPointer(${lockfileHash}) -> ${depsHash.slice(0, 12)}`, {
 			platform,
 			arch
 		});
@@ -953,7 +1025,7 @@ var DepCache = class DepCache {
 		const depsHash = DepCache.computeHash(tarballData);
 		await this.storage.put(depTarballKey(depsHash, platform, arch), tarballData);
 		await this.storage.put(depPointerKey(lockfileHash, platform, arch), depsHash);
-		logger$2.info(`store: ${tarballData.length} bytes`, {
+		logger$3.info(`store: ${tarballData.length} bytes`, {
 			lockfileHash,
 			platform,
 			arch,
@@ -972,7 +1044,7 @@ var DepCache = class DepCache {
 		const depsHash = await this.resolvePointer(lockfileHash, platform, arch, await this.resolveTtlMsOverride());
 		await this.storage.delete(depPointerKey(lockfileHash, platform, arch));
 		const removed = depsHash ? await this.storage.delete(depTarballKey(depsHash, platform, arch)) : false;
-		logger$2.info(`remove(${lockfileHash}): ${removed ? "removed" : "not found"}`, {
+		logger$3.info(`remove(${lockfileHash}): ${removed ? "removed" : "not found"}`, {
 			platform,
 			arch
 		});
@@ -1016,7 +1088,7 @@ function keyDiscriminator(exact) {
 * hash and size accounting outside the tarball's own (presigned, metadata-less)
 * upload.
 */
-const logger$1 = createLogger({ prefix: "user-cache" });
+const logger$2 = createLogger({ prefix: "user-cache" });
 /**
 * Cluster-wide default quota: 5 GiB. Serves as the fallback when an org has no
 * per-org override in `org_settings.user_cache_quota_bytes`. The cluster-wide
@@ -1164,7 +1236,7 @@ var UserCache = class {
 		try {
 			limits = await this.orgLimitsReader(org);
 		} catch (err) {
-			logger$1.warn("user-cache org-limits lookup failed — using cluster defaults", {
+			logger$2.warn("user-cache org-limits lookup failed — using cluster defaults", {
 				org,
 				error: err instanceof Error ? err.message : String(err)
 			});
@@ -1221,7 +1293,7 @@ var UserCache = class {
 	async beginSave(ref) {
 		const final = userCacheEntryKey(ref, ref.key);
 		if (await this.storage.has(final)) {
-			logger$1.info("user-cache save skipped (immutable key exists)", { key: ref.key });
+			logger$2.info("user-cache save skipped (immutable key exists)", { key: ref.key });
 			return { skip: true };
 		}
 		const tempKey = `${writePrefix(ref)}${TEMP_UPLOAD_STEM}${randomUUID()}${TAR_SUFFIX}`;
@@ -1242,7 +1314,7 @@ var UserCache = class {
 		await this.storage.initMeta(final);
 		await this.storage.put(`${final}.hash`, ref.tarHash);
 		await this.storage.put(`${final}.size`, String(ref.sizeBytes));
-		logger$1.info("user-cache entry committed", {
+		logger$2.info("user-cache entry committed", {
 			key: ref.key,
 			sizeBytes: ref.sizeBytes
 		});
@@ -1292,7 +1364,7 @@ var UserCache = class {
 			await this.storage.delete(`${key}.hash`);
 			await this.storage.delete(`${key}.size`);
 			total -= size;
-			logger$1.info("user-cache eviction (org over quota)", {
+			logger$2.info("user-cache eviction (org over quota)", {
 				org: ref.org,
 				key,
 				freedBytes: size
@@ -1300,18 +1372,12 @@ var UserCache = class {
 		}
 	}
 };
-//#endregion
-//#region src/cluster/peer-credentials.ts
 /**
-* Peer credential management for cluster authentication.
-*
-* Provides two persistence layers:
-* 1. DB CRUD (PeerCredentialStore) — coordinator stores credentials for all peers
-* 2. File I/O (readCredentialFile/writeCredentialFile) — workers persist their
-*    credential locally for reconnection after restart
-*
-* Credential files are written with 0600 permissions to protect the secret.
+* `metadata` key set when a coordinator retires the self-issued credential its
+* previous run left behind. An operator revoke never sets it, which is how a
+* retirement is told apart from a revoke.
 */
+const RETIRED_BY_INSTANCE_KEY = "retiredByInstance";
 /** Default credential expiry: 90 days. */
 const DEFAULT_EXPIRY_DAYS = 90;
 /**
@@ -1348,7 +1414,8 @@ var PeerCredentialStore = class {
 					role: opts.role,
 					routing_keys: sql`${sql.val(opts.routingKeys)}`,
 					source_token_hash: opts.sourceTokenHash ?? null,
-					expires_at: expiresAt
+					expires_at: expiresAt,
+					...opts.metadata && { metadata: sql`${JSON.stringify(opts.metadata)}::jsonb` }
 				}).execute();
 				return Number(revokeResult?.numUpdatedRows ?? 0n);
 			});
@@ -1375,6 +1442,33 @@ var PeerCredentialStore = class {
 		return row ? mapRow(row) : null;
 	}
 	/**
+	* The credential row for `instanceId` that nobody revoked, expired or not.
+	* `peer_credentials_active_uniq` allows at most one.
+	*/
+	async findUnrevokedByInstanceId(instanceId) {
+		const row = await this.db.selectFrom("peer_credentials").selectAll().where("instance_id", "=", instanceId).where("revoked_at", "is", null).executeTakeFirst();
+		return row ? mapRow(row) : null;
+	}
+	/** The most recently revoked credential row for `instanceId`, or null. */
+	async findLatestRevokedByInstanceId(instanceId) {
+		const row = await this.db.selectFrom("peer_credentials").selectAll().where("instance_id", "=", instanceId).where("revoked_at", "is not", null).orderBy("revoked_at", "desc").executeTakeFirst();
+		return row ? mapRow(row) : null;
+	}
+	/**
+	* Revoke `instanceId`'s unrevoked, self-issued row whose hash is
+	* `credentialHash`, recording who retired it. Every condition sits in the
+	* WHERE clause, so a row that changed meanwhile is left alone.
+	* Returns whether a row was retired.
+	*/
+	async retireSelfIssued(opts) {
+		const marker = JSON.stringify({ [RETIRED_BY_INSTANCE_KEY]: opts.retiredByInstance });
+		const result = await this.db.updateTable("peer_credentials").set({
+			revoked_at: /* @__PURE__ */ new Date(),
+			metadata: sql`COALESCE(metadata, '{}'::jsonb) || ${marker}::jsonb`
+		}).where("instance_id", "=", opts.instanceId).where("credential_hash", "=", opts.credentialHash).where("revoked_at", "is", null).where(sql`metadata->>'issuance'`, "=", "self").executeTakeFirst();
+		return Number(result?.numUpdatedRows ?? 0n) > 0;
+	}
+	/**
 	* Update the last_seen_at timestamp for a credential.
 	* Optionally records which coordinator validated it.
 	*/
@@ -1419,7 +1513,8 @@ function mapRow(row) {
 		lastSeenAt: row.last_seen_at ? new Date(row.last_seen_at) : null,
 		lastValidatedBy: row.last_validated_by ?? null,
 		expiresAt: new Date(row.expires_at),
-		revokedAt: row.revoked_at ? new Date(row.revoked_at) : null
+		revokedAt: row.revoked_at ? new Date(row.revoked_at) : null,
+		metadata: row.metadata ?? {}
 	};
 }
 /**
@@ -1461,7 +1556,7 @@ async function readCredentialFile(filePath) {
 * store plus a disposer that closes the underlying pool. Lets tests
 * exercise the real Kysely-backed store without importing pg/kysely.
 *
-* Intended for e2e coverage of the store's own CRUD methods — production
+* Intended for tests of the store's own CRUD methods — production
 * code constructs the store with an already-allocated Kysely instance
 * shared with the orchestrator.
 */
@@ -1479,6 +1574,43 @@ async function createPeerCredentialStoreFromUrl(databaseUrl, opts) {
 	};
 }
 //#endregion
+//#region src/helpers/run-detached.ts
+/**
+* Running work that nothing awaits.
+*
+* A promise that nobody awaits and nobody catches becomes an unhandled
+* rejection when it fails, and the orchestrator's shutdown hook treats an
+* unhandled rejection as fatal: one failed query in a timer tick or a message
+* handler would stop the whole process. `runDetached` runs such work and logs
+* its failure instead. ESLint's `no-floating-promises` (with `ignoreVoid:
+* false`) refuses a bare or `void`-ed promise anywhere in this package, so
+* detached work runs through here or carries its own `.catch`.
+*/
+/**
+* Run `work` without waiting for it. A throw, or a rejection of the promise it
+* returns, is logged at error level as `<task> failed` with `context` and the
+* error message, and goes no further. `context` is read when the failure is
+* logged, so the work may fill it in while it runs.
+*/
+function runDetached(logger, task, work, context = {}) {
+	const fail = (err) => {
+		logger.error(`${task} failed`, {
+			...context,
+			error: toErrorMessage(err)
+		});
+	};
+	let outcome;
+	try {
+		outcome = work();
+	} catch (err) {
+		fail(err);
+		return;
+	}
+	Promise.resolve(outcome).catch(fail);
+}
+createLogger({ prefix: "instance-heartbeat" });
+createLogger({ prefix: "coordinator-credential" });
+//#endregion
 //#region src/cluster/peer-auth-coordinator.ts
 /**
 * Per-orchestrator coordinator that owns the shared peer credential file.
@@ -1490,6 +1622,10 @@ async function createPeerCredentialStoreFromUrl(databaseUrl, opts) {
 * rejection — a credential revocation cascade. This coordinator serializes all
 * file access through one in-process mutex so only one peer-client token-joins
 * per storm, and it never deletes a credential file a sibling has refreshed.
+*
+* A coordinator without a join token also wires a `selfIssue` callback. When a
+* decision finds no credential and no token, the callback issues the
+* coordinator its own credential, once per storm, under the same mutex.
 */
 const logger = createLogger({ prefix: "peer-auth-coordinator" });
 /** How long a waiting peer-client awaits an in-flight sibling token-join. */
@@ -1510,6 +1646,11 @@ var PeerAuthCoordinator = class {
 	instanceId;
 	joinToken;
 	joinWaitTimeoutMs;
+	selfIssue;
+	/** The issuance in progress; concurrent sibling clients share it. */
+	selfIssueInFlight = null;
+	/** An operator revoked this coordinator's credential: never issue again in this process. */
+	selfIssueRevoked = false;
 	/** Promise-chain mutex tail; every file op awaits the prior one. */
 	lock = Promise.resolve();
 	/** Set while one peer-client is mid token-join; siblings await it. */
@@ -1519,6 +1660,7 @@ var PeerAuthCoordinator = class {
 		this.instanceId = opts.instanceId;
 		this.joinToken = opts.joinToken;
 		this.joinWaitTimeoutMs = opts.joinWaitTimeoutMs ?? DEFAULT_JOIN_WAIT_TIMEOUT_MS;
+		this.selfIssue = opts.selfIssue;
 	}
 	/** Run `fn` exclusively against the credential file. */
 	async withLock(fn) {
@@ -1527,10 +1669,21 @@ var PeerAuthCoordinator = class {
 		return run;
 	}
 	async readValidCredential() {
-		const cred = await readCredentialFile(this.credentialFile);
+		let cred;
+		try {
+			cred = await readCredentialFile(this.credentialFile);
+		} catch (err) {
+			logger.warn("Ignoring unreadable peer credential file", {
+				instanceId: this.instanceId,
+				path: this.credentialFile,
+				error: toErrorMessage(err)
+			});
+			return null;
+		}
 		return cred && cred.instanceId === this.instanceId ? cred : null;
 	}
 	async decideAuth() {
+		let selfIssueTried = false;
 		for (let i = 0; i < MAX_DECIDE_ITERATIONS; i++) {
 			const decision = await this.withLock(async () => {
 				const cred = await this.readValidCredential();
@@ -1548,10 +1701,19 @@ var PeerAuthCoordinator = class {
 						complete: this.makeComplete(join)
 					};
 				}
+				if (this.selfIssue && !selfIssueTried && !this.selfIssueRevoked) return "self-issue";
 				return { mode: "no-auth" };
 			});
-			if (decision !== "await-join") return decision;
-			await this.awaitInFlightJoin();
+			if (decision === "await-join") {
+				await this.awaitInFlightJoin();
+				continue;
+			}
+			if (decision === "self-issue") {
+				selfIssueTried = true;
+				await this.runSelfIssue();
+				continue;
+			}
+			return decision;
 		}
 		if (this.joinToken) {
 			const join = deferred();
@@ -1564,13 +1726,25 @@ var PeerAuthCoordinator = class {
 		}
 		return { mode: "no-auth" };
 	}
+	/** Run the issuer under the mutex, once for every concurrent caller. */
+	runSelfIssue() {
+		const selfIssue = this.selfIssue;
+		if (!selfIssue) return Promise.resolve("failed");
+		this.selfIssueInFlight ??= this.withLock(selfIssue).then((outcome) => {
+			if (outcome === "revoked") this.selfIssueRevoked = true;
+			return outcome;
+		}).finally(() => {
+			this.selfIssueInFlight = null;
+		});
+		return this.selfIssueInFlight;
+	}
 	makeComplete(join) {
 		return (issued) => {
-			this.withLock(async () => {
+			runDetached(logger, "Peer token-join completion", () => this.withLock(async () => {
 				if (issued) await writeCredentialFile(this.credentialFile, issued);
 				if (this.inFlightJoin === join) this.inFlightJoin = null;
 				join.resolve(issued);
-			});
+			}), { instanceId: this.instanceId });
 		};
 	}
 	async awaitInFlightJoin() {
@@ -1632,7 +1806,8 @@ var PeerAuthCoordinator = class {
 const joinTokenLogger = createLogger({ prefix: "join-token" });
 const TOKEN_PREFIX = "kici_join_v1";
 const DEFAULT_EXPIRY_MS = 36e5;
-const TOKEN_ALREADY_USED_MESSAGE = "Join token has already been used. Generate a new token with: kici admin create-join-token";
+const TOKEN_ALREADY_USED_MESSAGE = "Join token has already been used. Create a new token with: kici-admin peer create-token";
+const TOKEN_EXPIRED_MESSAGE = "Join token has expired. Create a new token with: kici-admin peer create-token";
 var JoinTokenManager = class {
 	deps;
 	constructor(deps) {
@@ -1717,7 +1892,7 @@ var JoinTokenManager = class {
 		}
 		const row = await this.deps.db.selectFrom("join_tokens").selectAll().where("token_hash", "=", keys.validationHash).executeTakeFirst();
 		if (!row) throw new Error("Invalid join token");
-		if (new Date(row.expires_at).getTime() <= Date.now()) throw new Error("Join token has expired. Generate a new token with: kici admin create-join-token");
+		if (new Date(row.expires_at).getTime() <= Date.now()) throw new Error(TOKEN_EXPIRED_MESSAGE);
 		if (row.consumed_at) {
 			if (row.consumed_by_instance === peerInstanceId) {
 				joinTokenLogger.info("Re-validated join token for returning instance", {
@@ -1737,8 +1912,8 @@ var JoinTokenManager = class {
 /**
 * Build a JoinTokenManager backed by its own connection pool to the given
 * orchestrator database URL. Mirrors `createPeerCredentialStoreFromUrl`;
-* consumed by E2E tests that need to exercise token validation/reuse against
-* the real cluster DB.
+* used by tests that need to exercise token validation/reuse against a real
+* cluster DB.
 */
 function createJoinTokenManagerFromUrl(databaseUrl, opts) {
 	const pool = new pg.Pool({
@@ -2137,8 +2312,8 @@ var HeldRunStore = class {
 /**
 * Build a HeldRunStore backed by its own connection pool to the given
 * orchestrator database URL. Mirrors `createPeerCredentialStoreFromUrl` /
-* `createJoinTokenManagerFromUrl`; consumed by E2E tests that exercise the
-* PR-scoped hold selection against the real deployed orchestrator DB.
+* `createJoinTokenManagerFromUrl`; used by tests that exercise the PR-scoped
+* hold selection against a real orchestrator DB.
 */
 function createHeldRunStoreFromUrl(databaseUrl, opts) {
 	const pool = new pg.Pool({
@@ -2303,9 +2478,9 @@ var TrustPolicyStore = class {
 };
 /**
 * Build a TrustPolicyStore backed by its own connection pool to the given
-* orchestrator database URL. Mirrors `createHeldRunStoreFromUrl`; consumed by
-* E2E tests that assert the pushed policy is persisted in the real deployed
-* orchestrator DB rather than held in process memory.
+* orchestrator database URL. Mirrors `createHeldRunStoreFromUrl`; exported for
+* tests that assert the pushed policy is persisted in a real orchestrator DB
+* rather than held in process memory.
 */
 function createTrustPolicyStoreFromUrl(databaseUrl, opts) {
 	const pool = new pg.Pool({
@@ -3107,7 +3282,7 @@ function startManifestLoopback(opts) {
 * blocked launcher) just falls back to copy-paste.
 *
 * `KICI_BROWSER_CMD` overrides the launcher: `none` suppresses it entirely
-* (E2E / headless capture), any other value is run with `{url}` substituted —
+* (automated tests / headless capture), any other value is run with `{url}` substituted —
 * mirroring the `kici login` convention.
 */
 function openBrowserBestEffort(url) {