@kici-dev/orchestrator 0.12.0 → 0.14.0

package/dist/index.js

@@ -33,6 +33,7 @@ import { exec } from "node:child_process";
 * TTL is enforced lazily: expired items are deleted on access.
 * Pre-signed URLs are generated for getUrl() with 15-minute expiry.
 */
+const logger$5 = createLogger({ prefix: "s3-cache" });
 /** Pre-signed download URL expiry in seconds (15 minutes) */
 const PRESIGNED_URL_EXPIRY_SECONDS = 900;
 /** Pre-signed upload URL expiry in seconds (30 minutes -- uploads take longer) */
@@ -46,6 +47,11 @@ var S3CacheStorage = class {
 	bucket;
 	prefix;
 	ttlMs;
+	/**
+	* False once the backend answered a conditional self-copy with
+	* `501 NotImplemented`. From then on `updateMeta` copies unconditionally.
+	*/
+	conditionalCopySupported = true;
 	constructor(options) {
 		this.bucket = options.bucket;
 		this.prefix = options.prefix;
@@ -77,6 +83,13 @@ var S3CacheStorage = class {
 	* Returns null if the object doesn't exist (NoSuchKey/NotFound).
 	*/
 	async readMeta(key) {
+		return (await this.readHead(key))?.meta ?? null;
+	}
+	/**
+	* Read an object's TTL metadata together with the ETag of the version it
+	* belongs to. Returns null if the object doesn't exist (NoSuchKey/NotFound).
+	*/
+	async readHead(key) {
 		try {
 			const head = await this.client.send(new HeadObjectCommand({
 				Bucket: this.bucket,
@@ -86,8 +99,11 @@ var S3CacheStorage = class {
 			const lastAccessedAt = head.Metadata?.["last-accessed-at"] ?? "";
 			if (!createdAt || !lastAccessedAt) return null;
 			return {
-				createdAt,
-				lastAccessedAt
+				meta: {
+					createdAt,
+					lastAccessedAt
+				},
+				...head.ETag && { etag: head.ETag }
 			};
 		} catch (err) {
 			if (this.isNotFoundError(err)) return null;
@@ -97,8 +113,41 @@ var S3CacheStorage = class {
 	/**
 	* Update metadata on an S3 object by copying it to itself.
 	* This is the standard S3 pattern for updating metadata without re-uploading data.
+	*
+	* The copy is conditional on `etag`, the version whose metadata the caller
+	* read. A self-copy is a whole-object write: without the condition, one that
+	* races a `put()` of the same key can land after it and write the older body
+	* back. Cache pointers are rewritten in place while builds publish them, so a
+	* touch-on-read racing a publish reverted the pointer to a tarball that no
+	* longer exists. `CopySourceIfMatch` pins the bytes copied to that version and
+	* `IfMatch` refuses the write unless the key still holds it.
+	*
+	* Returns false when the object was replaced since the read (`412`), or is
+	* being replaced right now (`409 ConditionalRequestConflict`): the newer
+	* write stamps metadata of its own, so there is nothing left to refresh.
+	*
+	* An S3-compatible backend that does not implement conditional copies
+	* answers `501 NotImplemented`. The copy then runs again without the
+	* condition, and later copies skip it, as on a backend that ignores the
+	* headers.
 	*/
-	async updateMeta(key, meta) {
+	async updateMeta(key, meta, etag) {
+		const condition = this.conditionalCopySupported ? etag : void 0;
+		try {
+			await this.copyOntoItself(key, meta, condition);
+			return true;
+		} catch (err) {
+			if (!condition) throw err;
+			if (this.isPreconditionFailedError(err) || this.isConditionalConflictError(err)) return false;
+			if (!this.isNotImplementedError(err)) throw err;
+			this.conditionalCopySupported = false;
+			logger$5.warn("The object store does not implement conditional copies; refreshing cache metadata unconditionally", { bucket: this.bucket });
+			await this.copyOntoItself(key, meta, void 0);
+			return true;
+		}
+	}
+	/** Copy an object onto itself with new TTL metadata, conditional on `etag` when given. */
+	async copyOntoItself(key, meta, etag) {
 		const objectKey = this.objectKey(key);
 		await this.client.send(new CopyObjectCommand({
 			Bucket: this.bucket,
@@ -108,9 +157,30 @@ var S3CacheStorage = class {
 			Metadata: {
 				"created-at": meta.createdAt,
 				"last-accessed-at": meta.lastAccessedAt
+			},
+			...etag && {
+				CopySourceIfMatch: etag,
+				IfMatch: etag
 			}
 		}));
 	}
+	/** Check if an error is a failed conditional request (PreconditionFailed, 412). */
+	isPreconditionFailedError(err) {
+		return this.hasErrorShape(err, "PreconditionFailed", 412);
+	}
+	/** Check if an error is a conditional write that raced another write (ConditionalRequestConflict, 409). */
+	isConditionalConflictError(err) {
+		return this.hasErrorShape(err, "ConditionalRequestConflict", 409);
+	}
+	/** Check if an error is a request the backend does not implement (NotImplemented, 501). */
+	isNotImplementedError(err) {
+		return this.hasErrorShape(err, "NotImplemented", 501);
+	}
+	hasErrorShape(err, name, status) {
+		if (typeof err !== "object" || err === null) return false;
+		if (err.name === name) return true;
+		return err.$metadata?.httpStatusCode === status;
+	}
 	/** Check if an error is a "not found" error (NoSuchKey, NotFound, 404). */
 	isNotFoundError(err) {
 		if (typeof err !== "object" || err === null) return false;
@@ -135,8 +205,9 @@ var S3CacheStorage = class {
 		}).done();
 	}
 	async get(key, ttlMsOverride) {
-		const meta = await this.readMeta(key);
-		if (!meta) return null;
+		const head = await this.readHead(key);
+		if (!head) return null;
+		const { meta, etag } = head;
 		if (this.isExpired(meta, ttlMsOverride)) {
 			await this.deleteObject(key);
 			return null;
@@ -155,7 +226,7 @@ var S3CacheStorage = class {
 		}
 		try {
 			meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
-			await this.updateMeta(key, meta);
+			await this.updateMeta(key, meta, etag);
 		} catch {}
 		return data;
 	}
@@ -174,10 +245,10 @@ var S3CacheStorage = class {
 		return existed;
 	}
 	async touch(key) {
-		const meta = await this.readMeta(key);
-		if (!meta) return;
-		meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
-		await this.updateMeta(key, meta);
+		const head = await this.readHead(key);
+		if (!head) return;
+		head.meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
+		await this.updateMeta(key, head.meta, head.etag);
 	}
 	async getUrl(key, ttlMsOverride) {
 		const meta = await this.readMeta(key);
@@ -633,7 +704,7 @@ function createCacheStorage(config) {
 * Old objects are orphaned. The cache TTL is enforced lazily on access, so
 * nothing here removes them; `kici-admin cache purge-legacy` does.
 */
-const logger$3 = createLogger({ prefix: "source-cache" });
+const logger$4 = createLogger({ prefix: "source-cache" });
 /**
 * Cache key for a source tarball, addressed by the tarball's OWN content hash:
 * `source/v2/{orgId}/{sourceTarDigest}.tar.gz`.
@@ -676,25 +747,25 @@ var SourceCache = class SourceCache {
 	async has(orgId, contentHash) {
 		const digest = await this.resolvePointer(orgId, contentHash);
 		if (!digest) {
-			logger$3.debug(`has(${contentHash}): false (no pointer)`, { orgId });
+			logger$4.debug(`has(${contentHash}): false (no pointer)`, { orgId });
 			return false;
 		}
 		const exists = await this.storage.has(sourceTarballKey(orgId, digest));
-		logger$3.debug(`has(${contentHash}): ${exists}`, { orgId });
+		logger$4.debug(`has(${contentHash}): ${exists}`, { orgId });
 		return exists;
 	}
 	async get(orgId, contentHash) {
 		const digest = await this.resolvePointer(orgId, contentHash);
 		if (!digest) {
-			logger$3.debug(`get(${contentHash}): miss (no pointer)`, { orgId });
+			logger$4.debug(`get(${contentHash}): miss (no pointer)`, { orgId });
 			return null;
 		}
 		const key = sourceTarballKey(orgId, digest);
 		const data = await this.storage.get(key);
 		if (data) {
 			await this.storage.touch(key);
-			logger$3.debug(`get(${contentHash}): hit (${data.length} bytes)`, { orgId });
-		} else logger$3.debug(`get(${contentHash}): miss (dangling pointer)`, { orgId });
+			logger$4.debug(`get(${contentHash}): hit (${data.length} bytes)`, { orgId });
+		} else logger$4.debug(`get(${contentHash}): miss (dangling pointer)`, { orgId });
 		return data;
 	}
 	/**
@@ -710,18 +781,18 @@ var SourceCache = class SourceCache {
 		const pointerKey = sourcePointerKey(orgId, contentHash);
 		const digest = (await this.storage.get(pointerKey))?.toString("utf-8").trim() || null;
 		if (!digest) {
-			logger$3.debug(`getUrlAndDigest(${contentHash}): miss (no pointer)`, { orgId });
+			logger$4.debug(`getUrlAndDigest(${contentHash}): miss (no pointer)`, { orgId });
 			return null;
 		}
 		const key = sourceTarballKey(orgId, digest);
 		const url = await this.storage.getUrl(key);
 		if (!url) {
-			logger$3.debug(`getUrlAndDigest(${contentHash}): miss (dangling pointer)`, { orgId });
+			logger$4.debug(`getUrlAndDigest(${contentHash}): miss (dangling pointer)`, { orgId });
 			return null;
 		}
 		await this.storage.touch(key);
 		await this.storage.touch(pointerKey);
-		logger$3.debug(`getUrlAndDigest(${contentHash}): hit`, { orgId });
+		logger$4.debug(`getUrlAndDigest(${contentHash}): hit`, { orgId });
 		return {
 			url,
 			digest
@@ -747,7 +818,7 @@ var SourceCache = class SourceCache {
 	*/
 	async publishPointer(orgId, contentHash, sourceTarDigest) {
 		await this.storage.put(sourcePointerKey(orgId, contentHash), sourceTarDigest);
-		logger$3.info(`publishPointer(${contentHash}) -> ${sourceTarDigest.slice(0, 12)}`, { orgId });
+		logger$4.info(`publishPointer(${contentHash}) -> ${sourceTarDigest.slice(0, 12)}`, { orgId });
 	}
 	/** Store a tarball and the pointer that names it. */
 	async store(orgId, contentHash, tarball) {
@@ -755,7 +826,7 @@ var SourceCache = class SourceCache {
 		const digest = SourceCache.computeDigest(data);
 		await this.storage.put(sourceTarballKey(orgId, digest), data);
 		await this.storage.put(sourcePointerKey(orgId, contentHash), digest);
-		logger$3.info(`store(${contentHash}): stored (${data.length} bytes)`, {
+		logger$4.info(`store(${contentHash}): stored (${data.length} bytes)`, {
 			orgId,
 			digest: digest.slice(0, 12)
 		});
@@ -769,7 +840,7 @@ var SourceCache = class SourceCache {
 		const digest = await this.resolvePointer(orgId, contentHash);
 		await this.storage.delete(sourcePointerKey(orgId, contentHash));
 		const removed = digest ? await this.storage.delete(sourceTarballKey(orgId, digest)) : false;
-		logger$3.info(`remove(${contentHash}): ${removed ? "removed" : "not found"}`, { orgId });
+		logger$4.info(`remove(${contentHash}): ${removed ? "removed" : "not found"}`, { orgId });
 		return removed;
 	}
 };
@@ -786,7 +857,7 @@ var SourceCache = class SourceCache {
 * Keys: `deps/{platform}-{arch}/{depsHash}.tar.gz` (immutable) and
 * `deps/{platform}-{arch}/{lockfileHash}.hash` (the pointer).
 */
-const logger$2 = createLogger({ prefix: "dep-cache" });
+const logger$3 = createLogger({ prefix: "dep-cache" });
 /** Default max tarball size: 500MB */
 const DEFAULT_MAX_TARBALL_BYTES = 524288e3;
 /** Default dependency-cache entry TTL: 30 days. */
@@ -867,14 +938,14 @@ var DepCache = class DepCache {
 		const ttlMsOverride = await this.resolveTtlMsOverride();
 		const depsHash = await this.resolvePointer(lockfileHash, platform, arch, ttlMsOverride, siblingsDigest);
 		if (!depsHash) {
-			logger$2.debug(`has(${lockfileHash}): false (no pointer)`, {
+			logger$3.debug(`has(${lockfileHash}): false (no pointer)`, {
 				platform,
 				arch
 			});
 			return false;
 		}
 		const exists = await this.storage.has(depTarballKey(depsHash, platform, arch), ttlMsOverride);
-		logger$2.debug(`has(${lockfileHash}): ${exists}`, {
+		logger$3.debug(`has(${lockfileHash}): ${exists}`, {
 			platform,
 			arch
 		});
@@ -898,7 +969,7 @@ var DepCache = class DepCache {
 		const pointerKey = depPointerKey(lockfileHash, platform, arch, siblingsDigest);
 		const hash = (await this.storage.get(pointerKey, ttlMsOverride))?.toString("utf-8").trim() || void 0;
 		if (!hash) {
-			logger$2.debug(`getUrlAndHash(${lockfileHash}): miss (no pointer)`, {
+			logger$3.debug(`getUrlAndHash(${lockfileHash}): miss (no pointer)`, {
 				platform,
 				arch
 			});
@@ -907,7 +978,7 @@ var DepCache = class DepCache {
 		const key = depTarballKey(hash, platform, arch);
 		const url = await this.storage.getUrl(key, ttlMsOverride);
 		if (!url) {
-			logger$2.debug(`getUrlAndHash(${lockfileHash}): miss (dangling pointer)`, {
+			logger$3.debug(`getUrlAndHash(${lockfileHash}): miss (dangling pointer)`, {
 				platform,
 				arch
 			});
@@ -915,7 +986,7 @@ var DepCache = class DepCache {
 		}
 		await this.storage.touch(key);
 		await this.storage.touch(pointerKey);
-		logger$2.debug(`getUrlAndHash(${lockfileHash}): hit`, {
+		logger$3.debug(`getUrlAndHash(${lockfileHash}): hit`, {
 			platform,
 			arch,
 			hasHash: true
@@ -939,7 +1010,7 @@ var DepCache = class DepCache {
 	*/
 	async publishPointer(lockfileHash, platform, arch, depsHash, siblingsDigest) {
 		await this.storage.put(depPointerKey(lockfileHash, platform, arch, siblingsDigest), depsHash);
-		logger$2.info(`publishPointer(${lockfileHash}) -> ${depsHash.slice(0, 12)}`, {
+		logger$3.info(`publishPointer(${lockfileHash}) -> ${depsHash.slice(0, 12)}`, {
 			platform,
 			arch
 		});
@@ -954,7 +1025,7 @@ var DepCache = class DepCache {
 		const depsHash = DepCache.computeHash(tarballData);
 		await this.storage.put(depTarballKey(depsHash, platform, arch), tarballData);
 		await this.storage.put(depPointerKey(lockfileHash, platform, arch), depsHash);
-		logger$2.info(`store: ${tarballData.length} bytes`, {
+		logger$3.info(`store: ${tarballData.length} bytes`, {
 			lockfileHash,
 			platform,
 			arch,
@@ -973,7 +1044,7 @@ var DepCache = class DepCache {
 		const depsHash = await this.resolvePointer(lockfileHash, platform, arch, await this.resolveTtlMsOverride());
 		await this.storage.delete(depPointerKey(lockfileHash, platform, arch));
 		const removed = depsHash ? await this.storage.delete(depTarballKey(depsHash, platform, arch)) : false;
-		logger$2.info(`remove(${lockfileHash}): ${removed ? "removed" : "not found"}`, {
+		logger$3.info(`remove(${lockfileHash}): ${removed ? "removed" : "not found"}`, {
 			platform,
 			arch
 		});
@@ -1017,7 +1088,7 @@ function keyDiscriminator(exact) {
 * hash and size accounting outside the tarball's own (presigned, metadata-less)
 * upload.
 */
-const logger$1 = createLogger({ prefix: "user-cache" });
+const logger$2 = createLogger({ prefix: "user-cache" });
 /**
 * Cluster-wide default quota: 5 GiB. Serves as the fallback when an org has no
 * per-org override in `org_settings.user_cache_quota_bytes`. The cluster-wide
@@ -1165,7 +1236,7 @@ var UserCache = class {
 		try {
 			limits = await this.orgLimitsReader(org);
 		} catch (err) {
-			logger$1.warn("user-cache org-limits lookup failed — using cluster defaults", {
+			logger$2.warn("user-cache org-limits lookup failed — using cluster defaults", {
 				org,
 				error: err instanceof Error ? err.message : String(err)
 			});
@@ -1222,7 +1293,7 @@ var UserCache = class {
 	async beginSave(ref) {
 		const final = userCacheEntryKey(ref, ref.key);
 		if (await this.storage.has(final)) {
-			logger$1.info("user-cache save skipped (immutable key exists)", { key: ref.key });
+			logger$2.info("user-cache save skipped (immutable key exists)", { key: ref.key });
 			return { skip: true };
 		}
 		const tempKey = `${writePrefix(ref)}${TEMP_UPLOAD_STEM}${randomUUID()}${TAR_SUFFIX}`;
@@ -1243,7 +1314,7 @@ var UserCache = class {
 		await this.storage.initMeta(final);
 		await this.storage.put(`${final}.hash`, ref.tarHash);
 		await this.storage.put(`${final}.size`, String(ref.sizeBytes));
-		logger$1.info("user-cache entry committed", {
+		logger$2.info("user-cache entry committed", {
 			key: ref.key,
 			sizeBytes: ref.sizeBytes
 		});
@@ -1293,7 +1364,7 @@ var UserCache = class {
 			await this.storage.delete(`${key}.hash`);
 			await this.storage.delete(`${key}.size`);
 			total -= size;
-			logger$1.info("user-cache eviction (org over quota)", {
+			logger$2.info("user-cache eviction (org over quota)", {
 				org: ref.org,
 				key,
 				freedBytes: size
@@ -1301,18 +1372,12 @@ var UserCache = class {
 		}
 	}
 };
-//#endregion
-//#region src/cluster/peer-credentials.ts
 /**
-* Peer credential management for cluster authentication.
-*
-* Provides two persistence layers:
-* 1. DB CRUD (PeerCredentialStore) — coordinator stores credentials for all peers
-* 2. File I/O (readCredentialFile/writeCredentialFile) — workers persist their
-*    credential locally for reconnection after restart
-*
-* Credential files are written with 0600 permissions to protect the secret.
+* `metadata` key set when a coordinator retires the self-issued credential its
+* previous run left behind. An operator revoke never sets it, which is how a
+* retirement is told apart from a revoke.
 */
+const RETIRED_BY_INSTANCE_KEY = "retiredByInstance";
 /** Default credential expiry: 90 days. */
 const DEFAULT_EXPIRY_DAYS = 90;
 /**
@@ -1349,7 +1414,8 @@ var PeerCredentialStore = class {
 					role: opts.role,
 					routing_keys: sql`${sql.val(opts.routingKeys)}`,
 					source_token_hash: opts.sourceTokenHash ?? null,
-					expires_at: expiresAt
+					expires_at: expiresAt,
+					...opts.metadata && { metadata: sql`${JSON.stringify(opts.metadata)}::jsonb` }
 				}).execute();
 				return Number(revokeResult?.numUpdatedRows ?? 0n);
 			});
@@ -1376,6 +1442,33 @@ var PeerCredentialStore = class {
 		return row ? mapRow(row) : null;
 	}
 	/**
+	* The credential row for `instanceId` that nobody revoked, expired or not.
+	* `peer_credentials_active_uniq` allows at most one.
+	*/
+	async findUnrevokedByInstanceId(instanceId) {
+		const row = await this.db.selectFrom("peer_credentials").selectAll().where("instance_id", "=", instanceId).where("revoked_at", "is", null).executeTakeFirst();
+		return row ? mapRow(row) : null;
+	}
+	/** The most recently revoked credential row for `instanceId`, or null. */
+	async findLatestRevokedByInstanceId(instanceId) {
+		const row = await this.db.selectFrom("peer_credentials").selectAll().where("instance_id", "=", instanceId).where("revoked_at", "is not", null).orderBy("revoked_at", "desc").executeTakeFirst();
+		return row ? mapRow(row) : null;
+	}
+	/**
+	* Revoke `instanceId`'s unrevoked, self-issued row whose hash is
+	* `credentialHash`, recording who retired it. Every condition sits in the
+	* WHERE clause, so a row that changed meanwhile is left alone.
+	* Returns whether a row was retired.
+	*/
+	async retireSelfIssued(opts) {
+		const marker = JSON.stringify({ [RETIRED_BY_INSTANCE_KEY]: opts.retiredByInstance });
+		const result = await this.db.updateTable("peer_credentials").set({
+			revoked_at: /* @__PURE__ */ new Date(),
+			metadata: sql`COALESCE(metadata, '{}'::jsonb) || ${marker}::jsonb`
+		}).where("instance_id", "=", opts.instanceId).where("credential_hash", "=", opts.credentialHash).where("revoked_at", "is", null).where(sql`metadata->>'issuance'`, "=", "self").executeTakeFirst();
+		return Number(result?.numUpdatedRows ?? 0n) > 0;
+	}
+	/**
 	* Update the last_seen_at timestamp for a credential.
 	* Optionally records which coordinator validated it.
 	*/
@@ -1420,7 +1513,8 @@ function mapRow(row) {
 		lastSeenAt: row.last_seen_at ? new Date(row.last_seen_at) : null,
 		lastValidatedBy: row.last_validated_by ?? null,
 		expiresAt: new Date(row.expires_at),
-		revokedAt: row.revoked_at ? new Date(row.revoked_at) : null
+		revokedAt: row.revoked_at ? new Date(row.revoked_at) : null,
+		metadata: row.metadata ?? {}
 	};
 }
 /**
@@ -1514,6 +1608,8 @@ function runDetached(logger, task, work, context = {}) {
 	}
 	Promise.resolve(outcome).catch(fail);
 }
+createLogger({ prefix: "instance-heartbeat" });
+createLogger({ prefix: "coordinator-credential" });
 //#endregion
 //#region src/cluster/peer-auth-coordinator.ts
 /**
@@ -1526,6 +1622,10 @@ function runDetached(logger, task, work, context = {}) {
 * rejection — a credential revocation cascade. This coordinator serializes all
 * file access through one in-process mutex so only one peer-client token-joins
 * per storm, and it never deletes a credential file a sibling has refreshed.
+*
+* A coordinator without a join token also wires a `selfIssue` callback. When a
+* decision finds no credential and no token, the callback issues the
+* coordinator its own credential, once per storm, under the same mutex.
 */
 const logger = createLogger({ prefix: "peer-auth-coordinator" });
 /** How long a waiting peer-client awaits an in-flight sibling token-join. */
@@ -1546,6 +1646,11 @@ var PeerAuthCoordinator = class {
 	instanceId;
 	joinToken;
 	joinWaitTimeoutMs;
+	selfIssue;
+	/** The issuance in progress; concurrent sibling clients share it. */
+	selfIssueInFlight = null;
+	/** An operator revoked this coordinator's credential: never issue again in this process. */
+	selfIssueRevoked = false;
 	/** Promise-chain mutex tail; every file op awaits the prior one. */
 	lock = Promise.resolve();
 	/** Set while one peer-client is mid token-join; siblings await it. */
@@ -1555,6 +1660,7 @@ var PeerAuthCoordinator = class {
 		this.instanceId = opts.instanceId;
 		this.joinToken = opts.joinToken;
 		this.joinWaitTimeoutMs = opts.joinWaitTimeoutMs ?? DEFAULT_JOIN_WAIT_TIMEOUT_MS;
+		this.selfIssue = opts.selfIssue;
 	}
 	/** Run `fn` exclusively against the credential file. */
 	async withLock(fn) {
@@ -1563,10 +1669,21 @@ var PeerAuthCoordinator = class {
 		return run;
 	}
 	async readValidCredential() {
-		const cred = await readCredentialFile(this.credentialFile);
+		let cred;
+		try {
+			cred = await readCredentialFile(this.credentialFile);
+		} catch (err) {
+			logger.warn("Ignoring unreadable peer credential file", {
+				instanceId: this.instanceId,
+				path: this.credentialFile,
+				error: toErrorMessage(err)
+			});
+			return null;
+		}
 		return cred && cred.instanceId === this.instanceId ? cred : null;
 	}
 	async decideAuth() {
+		let selfIssueTried = false;
 		for (let i = 0; i < MAX_DECIDE_ITERATIONS; i++) {
 			const decision = await this.withLock(async () => {
 				const cred = await this.readValidCredential();
@@ -1584,10 +1701,19 @@ var PeerAuthCoordinator = class {
 						complete: this.makeComplete(join)
 					};
 				}
+				if (this.selfIssue && !selfIssueTried && !this.selfIssueRevoked) return "self-issue";
 				return { mode: "no-auth" };
 			});
-			if (decision !== "await-join") return decision;
-			await this.awaitInFlightJoin();
+			if (decision === "await-join") {
+				await this.awaitInFlightJoin();
+				continue;
+			}
+			if (decision === "self-issue") {
+				selfIssueTried = true;
+				await this.runSelfIssue();
+				continue;
+			}
+			return decision;
 		}
 		if (this.joinToken) {
 			const join = deferred();
@@ -1600,6 +1726,18 @@ var PeerAuthCoordinator = class {
 		}
 		return { mode: "no-auth" };
 	}
+	/** Run the issuer under the mutex, once for every concurrent caller. */
+	runSelfIssue() {
+		const selfIssue = this.selfIssue;
+		if (!selfIssue) return Promise.resolve("failed");
+		this.selfIssueInFlight ??= this.withLock(selfIssue).then((outcome) => {
+			if (outcome === "revoked") this.selfIssueRevoked = true;
+			return outcome;
+		}).finally(() => {
+			this.selfIssueInFlight = null;
+		});
+		return this.selfIssueInFlight;
+	}
 	makeComplete(join) {
 		return (issued) => {
 			runDetached(logger, "Peer token-join completion", () => this.withLock(async () => {
@@ -1668,7 +1806,8 @@ var PeerAuthCoordinator = class {
 const joinTokenLogger = createLogger({ prefix: "join-token" });
 const TOKEN_PREFIX = "kici_join_v1";
 const DEFAULT_EXPIRY_MS = 36e5;
-const TOKEN_ALREADY_USED_MESSAGE = "Join token has already been used. Generate a new token with: kici admin create-join-token";
+const TOKEN_ALREADY_USED_MESSAGE = "Join token has already been used. Create a new token with: kici-admin peer create-token";
+const TOKEN_EXPIRED_MESSAGE = "Join token has expired. Create a new token with: kici-admin peer create-token";
 var JoinTokenManager = class {
 	deps;
 	constructor(deps) {
@@ -1753,7 +1892,7 @@ var JoinTokenManager = class {
 		}
 		const row = await this.deps.db.selectFrom("join_tokens").selectAll().where("token_hash", "=", keys.validationHash).executeTakeFirst();
 		if (!row) throw new Error("Invalid join token");
-		if (new Date(row.expires_at).getTime() <= Date.now()) throw new Error("Join token has expired. Generate a new token with: kici admin create-join-token");
+		if (new Date(row.expires_at).getTime() <= Date.now()) throw new Error(TOKEN_EXPIRED_MESSAGE);
 		if (row.consumed_at) {
 			if (row.consumed_by_instance === peerInstanceId) {
 				joinTokenLogger.info("Re-validated join token for returning instance", {
@@ -2721,6 +2860,51 @@ var AdminApiClient = class {
 	async drainStatus() {
 		return this.request("GET", "/api/v1/admin/orchestrator/drain");
 	}
+	/** Forget a departed peer on this coordinator and every connected sibling. */
+	/**
+	* Forget a departed peer on this coordinator and every connected sibling. A
+	* refusal that only needs the backstop acknowledgement comes back as a body
+	* with `acknowledgementRequired`, so the caller can ask and retry; every
+	* other failure throws.
+	*/
+	async forgetPeer(body) {
+		const res = await fetchAdminApi(`${this.baseUrl}/api/v1/admin/peers/forget`, {
+			method: "POST",
+			headers: {
+				Authorization: `Bearer ${this.token}`,
+				"Content-Type": "application/json"
+			},
+			body: JSON.stringify(body)
+		}, this.baseUrl);
+		const text = await res.text();
+		let parsed;
+		try {
+			parsed = JSON.parse(text);
+		} catch {
+			parsed = void 0;
+		}
+		const json = parsed ?? {};
+		if (res.ok) return json;
+		if (res.status === 409 && json.acknowledgementRequired === true) return {
+			instanceId: body.instanceId,
+			results: json.results ?? [],
+			acknowledgementRequired: true,
+			...json.error ? { error: json.error } : {}
+		};
+		throw new Error(`HTTP ${res.status}: ${json.error ?? text}`);
+	}
+	/** List the live Firecracker VMs on this coordinator, or on the `target` peer. */
+	async listScalerOrphans(opts = {}) {
+		const params = new URLSearchParams();
+		if (opts.target !== void 0) params.set("target", opts.target);
+		if (opts.timeoutMs !== void 0) params.set("timeoutMs", String(opts.timeoutMs));
+		const query = params.toString();
+		return this.request("GET", `/api/v1/admin/scaler/orphans${query ? `?${query}` : ""}`);
+	}
+	/** Stop the named VMs on this coordinator, or on the `target` peer. */
+	async stopScalerOrphans(body) {
+		return this.request("POST", "/api/v1/admin/scaler/orphans/stop", body);
+	}
 	async createApiKey(opts) {
 		return this.request("POST", "/api/v1/api-keys", opts);
 	}
@@ -2847,6 +3031,21 @@ var AdminApiClient = class {
 		if (opts.routingKey) params.set("routingKey", opts.routingKey);
 		return this.request("GET", `/api/v1/admin/event-log/${encodeURIComponent(deliveryId)}?${params}`);
 	}
+	/** List internal events, newest first (`GET /api/v1/admin/events`). */
+	async listEvents(opts) {
+		const params = new URLSearchParams();
+		if (opts?.name) params.set("name", opts.name);
+		if (opts?.outcome) params.set("outcome", opts.outcome);
+		if (opts?.since) params.set("since", opts.since);
+		if (opts?.before) params.set("before", opts.before);
+		if (opts?.limit !== void 0) params.set("limit", String(opts.limit));
+		const qs = params.toString();
+		return this.request("GET", `/api/v1/admin/events${qs ? `?${qs}` : ""}`);
+	}
+	/** One internal event with its redacted payload and dispatched runs. */
+	async getEvent(id) {
+		return this.request("GET", `/api/v1/admin/events/${encodeURIComponent(id)}`);
+	}
 	async listAccessLog(opts) {
 		const params = new URLSearchParams();
 		if (opts?.orgId) params.set("orgId", opts.orgId);

package/dist/metrics/prometheus.d.ts

@@ -539,6 +539,8 @@ export declare const eventRetryTotal: import("@opentelemetry/api").Counter<impor
  * - reason: why the event was DLQ'd (exhausted_retries)
  */
 export declare const eventDlqTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
+/** Record a reserved internal event that matched no subscriber. */
+export declare function incEventUnmatched(eventName: string, reason: string): void;
 /**
  * Startup catch-up scans that threw. Emits no labels.
  *

package/dist/orchestrator-core.d.ts

@@ -41,6 +41,7 @@ import { ArtifactStore } from './artifacts/artifact-store.js';
 import { CheckRunReporter } from './reporting/check-run-reporter.js';
 import type { ProcessingDeps } from './pipeline/processor.js';
 import type { InvokeGateDeps, SummonArgs, SummonedRun } from './pipeline/invoke-gate.js';
+import { type PeerForgetRequestHandler } from './cluster/peer-forget.js';
 import { StepLogBuffer } from './reporting/step-log-buffer.js';
 import { type LogStorage } from './reporting/log-storage.js';
 import { ExecutionTracker, type ExecutionTrackerDeps } from './reporting/execution-tracker.js';
@@ -167,6 +168,12 @@ export interface OrchestratorSubsystems {
     orphanRecovery: OrphanRecovery;
     peerHandler: ReturnType<typeof createPeerHandler>;
     peerClients: Map<string, PeerClient>;
+    /**
+     * Answers a `kici-admin peer forget` a sibling coordinator fanned out. Wired
+     * into every outgoing PeerClient (the incoming peer-handler is wired directly
+     * in setupCluster).
+     */
+    answerPeerForgetRequest: PeerForgetRequestHandler;
     getLocalInventory: () => Omit<PeerHeartbeat, 'type'>;
     broadcastHeartbeatToAllPeers: () => void;
     broadcastAgentTokenRevoke: (tokenId: string) => void;