@kici-dev/orchestrator 0.12.0 → 0.13.0

package/dist/index.js

@@ -33,6 +33,7 @@ import { exec } from "node:child_process";
 * TTL is enforced lazily: expired items are deleted on access.
 * Pre-signed URLs are generated for getUrl() with 15-minute expiry.
 */
+const logger$5 = createLogger({ prefix: "s3-cache" });
 /** Pre-signed download URL expiry in seconds (15 minutes) */
 const PRESIGNED_URL_EXPIRY_SECONDS = 900;
 /** Pre-signed upload URL expiry in seconds (30 minutes -- uploads take longer) */
@@ -46,6 +47,11 @@ var S3CacheStorage = class {
 	bucket;
 	prefix;
 	ttlMs;
+	/**
+	* False once the backend answered a conditional self-copy with
+	* `501 NotImplemented`. From then on `updateMeta` copies unconditionally.
+	*/
+	conditionalCopySupported = true;
 	constructor(options) {
 		this.bucket = options.bucket;
 		this.prefix = options.prefix;
@@ -77,6 +83,13 @@ var S3CacheStorage = class {
 	* Returns null if the object doesn't exist (NoSuchKey/NotFound).
 	*/
 	async readMeta(key) {
+		return (await this.readHead(key))?.meta ?? null;
+	}
+	/**
+	* Read an object's TTL metadata together with the ETag of the version it
+	* belongs to. Returns null if the object doesn't exist (NoSuchKey/NotFound).
+	*/
+	async readHead(key) {
 		try {
 			const head = await this.client.send(new HeadObjectCommand({
 				Bucket: this.bucket,
@@ -86,8 +99,11 @@ var S3CacheStorage = class {
 			const lastAccessedAt = head.Metadata?.["last-accessed-at"] ?? "";
 			if (!createdAt || !lastAccessedAt) return null;
 			return {
-				createdAt,
-				lastAccessedAt
+				meta: {
+					createdAt,
+					lastAccessedAt
+				},
+				...head.ETag && { etag: head.ETag }
 			};
 		} catch (err) {
 			if (this.isNotFoundError(err)) return null;
@@ -97,8 +113,41 @@ var S3CacheStorage = class {
 	/**
 	* Update metadata on an S3 object by copying it to itself.
 	* This is the standard S3 pattern for updating metadata without re-uploading data.
+	*
+	* The copy is conditional on `etag`, the version whose metadata the caller
+	* read. A self-copy is a whole-object write: without the condition, one that
+	* races a `put()` of the same key can land after it and write the older body
+	* back. Cache pointers are rewritten in place while builds publish them, so a
+	* touch-on-read racing a publish reverted the pointer to a tarball that no
+	* longer exists. `CopySourceIfMatch` pins the bytes copied to that version and
+	* `IfMatch` refuses the write unless the key still holds it.
+	*
+	* Returns false when the object was replaced since the read (`412`), or is
+	* being replaced right now (`409 ConditionalRequestConflict`): the newer
+	* write stamps metadata of its own, so there is nothing left to refresh.
+	*
+	* An S3-compatible backend that does not implement conditional copies
+	* answers `501 NotImplemented`. The copy then runs again without the
+	* condition, and later copies skip it, as on a backend that ignores the
+	* headers.
 	*/
-	async updateMeta(key, meta) {
+	async updateMeta(key, meta, etag) {
+		const condition = this.conditionalCopySupported ? etag : void 0;
+		try {
+			await this.copyOntoItself(key, meta, condition);
+			return true;
+		} catch (err) {
+			if (!condition) throw err;
+			if (this.isPreconditionFailedError(err) || this.isConditionalConflictError(err)) return false;
+			if (!this.isNotImplementedError(err)) throw err;
+			this.conditionalCopySupported = false;
+			logger$5.warn("The object store does not implement conditional copies; refreshing cache metadata unconditionally", { bucket: this.bucket });
+			await this.copyOntoItself(key, meta, void 0);
+			return true;
+		}
+	}
+	/** Copy an object onto itself with new TTL metadata, conditional on `etag` when given. */
+	async copyOntoItself(key, meta, etag) {
 		const objectKey = this.objectKey(key);
 		await this.client.send(new CopyObjectCommand({
 			Bucket: this.bucket,
@@ -108,9 +157,30 @@ var S3CacheStorage = class {
 			Metadata: {
 				"created-at": meta.createdAt,
 				"last-accessed-at": meta.lastAccessedAt
+			},
+			...etag && {
+				CopySourceIfMatch: etag,
+				IfMatch: etag
 			}
 		}));
 	}
+	/** Check if an error is a failed conditional request (PreconditionFailed, 412). */
+	isPreconditionFailedError(err) {
+		return this.hasErrorShape(err, "PreconditionFailed", 412);
+	}
+	/** Check if an error is a conditional write that raced another write (ConditionalRequestConflict, 409). */
+	isConditionalConflictError(err) {
+		return this.hasErrorShape(err, "ConditionalRequestConflict", 409);
+	}
+	/** Check if an error is a request the backend does not implement (NotImplemented, 501). */
+	isNotImplementedError(err) {
+		return this.hasErrorShape(err, "NotImplemented", 501);
+	}
+	hasErrorShape(err, name, status) {
+		if (typeof err !== "object" || err === null) return false;
+		if (err.name === name) return true;
+		return err.$metadata?.httpStatusCode === status;
+	}
 	/** Check if an error is a "not found" error (NoSuchKey, NotFound, 404). */
 	isNotFoundError(err) {
 		if (typeof err !== "object" || err === null) return false;
@@ -135,8 +205,9 @@ var S3CacheStorage = class {
 		}).done();
 	}
 	async get(key, ttlMsOverride) {
-		const meta = await this.readMeta(key);
-		if (!meta) return null;
+		const head = await this.readHead(key);
+		if (!head) return null;
+		const { meta, etag } = head;
 		if (this.isExpired(meta, ttlMsOverride)) {
 			await this.deleteObject(key);
 			return null;
@@ -155,7 +226,7 @@ var S3CacheStorage = class {
 		}
 		try {
 			meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
-			await this.updateMeta(key, meta);
+			await this.updateMeta(key, meta, etag);
 		} catch {}
 		return data;
 	}
@@ -174,10 +245,10 @@ var S3CacheStorage = class {
 		return existed;
 	}
 	async touch(key) {
-		const meta = await this.readMeta(key);
-		if (!meta) return;
-		meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
-		await this.updateMeta(key, meta);
+		const head = await this.readHead(key);
+		if (!head) return;
+		head.meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
+		await this.updateMeta(key, head.meta, head.etag);
 	}
 	async getUrl(key, ttlMsOverride) {
 		const meta = await this.readMeta(key);
@@ -633,7 +704,7 @@ function createCacheStorage(config) {
 * Old objects are orphaned. The cache TTL is enforced lazily on access, so
 * nothing here removes them; `kici-admin cache purge-legacy` does.
 */
-const logger$3 = createLogger({ prefix: "source-cache" });
+const logger$4 = createLogger({ prefix: "source-cache" });
 /**
 * Cache key for a source tarball, addressed by the tarball's OWN content hash:
 * `source/v2/{orgId}/{sourceTarDigest}.tar.gz`.
@@ -676,25 +747,25 @@ var SourceCache = class SourceCache {
 	async has(orgId, contentHash) {
 		const digest = await this.resolvePointer(orgId, contentHash);
 		if (!digest) {
-			logger$3.debug(`has(${contentHash}): false (no pointer)`, { orgId });
+			logger$4.debug(`has(${contentHash}): false (no pointer)`, { orgId });
 			return false;
 		}
 		const exists = await this.storage.has(sourceTarballKey(orgId, digest));
-		logger$3.debug(`has(${contentHash}): ${exists}`, { orgId });
+		logger$4.debug(`has(${contentHash}): ${exists}`, { orgId });
 		return exists;
 	}
 	async get(orgId, contentHash) {
 		const digest = await this.resolvePointer(orgId, contentHash);
 		if (!digest) {
-			logger$3.debug(`get(${contentHash}): miss (no pointer)`, { orgId });
+			logger$4.debug(`get(${contentHash}): miss (no pointer)`, { orgId });
 			return null;
 		}
 		const key = sourceTarballKey(orgId, digest);
 		const data = await this.storage.get(key);
 		if (data) {
 			await this.storage.touch(key);
-			logger$3.debug(`get(${contentHash}): hit (${data.length} bytes)`, { orgId });
-		} else logger$3.debug(`get(${contentHash}): miss (dangling pointer)`, { orgId });
+			logger$4.debug(`get(${contentHash}): hit (${data.length} bytes)`, { orgId });
+		} else logger$4.debug(`get(${contentHash}): miss (dangling pointer)`, { orgId });
 		return data;
 	}
 	/**
@@ -710,18 +781,18 @@ var SourceCache = class SourceCache {
 		const pointerKey = sourcePointerKey(orgId, contentHash);
 		const digest = (await this.storage.get(pointerKey))?.toString("utf-8").trim() || null;
 		if (!digest) {
-			logger$3.debug(`getUrlAndDigest(${contentHash}): miss (no pointer)`, { orgId });
+			logger$4.debug(`getUrlAndDigest(${contentHash}): miss (no pointer)`, { orgId });
 			return null;
 		}
 		const key = sourceTarballKey(orgId, digest);
 		const url = await this.storage.getUrl(key);
 		if (!url) {
-			logger$3.debug(`getUrlAndDigest(${contentHash}): miss (dangling pointer)`, { orgId });
+			logger$4.debug(`getUrlAndDigest(${contentHash}): miss (dangling pointer)`, { orgId });
 			return null;
 		}
 		await this.storage.touch(key);
 		await this.storage.touch(pointerKey);
-		logger$3.debug(`getUrlAndDigest(${contentHash}): hit`, { orgId });
+		logger$4.debug(`getUrlAndDigest(${contentHash}): hit`, { orgId });
 		return {
 			url,
 			digest
@@ -747,7 +818,7 @@ var SourceCache = class SourceCache {
 	*/
 	async publishPointer(orgId, contentHash, sourceTarDigest) {
 		await this.storage.put(sourcePointerKey(orgId, contentHash), sourceTarDigest);
-		logger$3.info(`publishPointer(${contentHash}) -> ${sourceTarDigest.slice(0, 12)}`, { orgId });
+		logger$4.info(`publishPointer(${contentHash}) -> ${sourceTarDigest.slice(0, 12)}`, { orgId });
 	}
 	/** Store a tarball and the pointer that names it. */
 	async store(orgId, contentHash, tarball) {
@@ -755,7 +826,7 @@ var SourceCache = class SourceCache {
 		const digest = SourceCache.computeDigest(data);
 		await this.storage.put(sourceTarballKey(orgId, digest), data);
 		await this.storage.put(sourcePointerKey(orgId, contentHash), digest);
-		logger$3.info(`store(${contentHash}): stored (${data.length} bytes)`, {
+		logger$4.info(`store(${contentHash}): stored (${data.length} bytes)`, {
 			orgId,
 			digest: digest.slice(0, 12)
 		});
@@ -769,7 +840,7 @@ var SourceCache = class SourceCache {
 		const digest = await this.resolvePointer(orgId, contentHash);
 		await this.storage.delete(sourcePointerKey(orgId, contentHash));
 		const removed = digest ? await this.storage.delete(sourceTarballKey(orgId, digest)) : false;
-		logger$3.info(`remove(${contentHash}): ${removed ? "removed" : "not found"}`, { orgId });
+		logger$4.info(`remove(${contentHash}): ${removed ? "removed" : "not found"}`, { orgId });
 		return removed;
 	}
 };
@@ -786,7 +857,7 @@ var SourceCache = class SourceCache {
 * Keys: `deps/{platform}-{arch}/{depsHash}.tar.gz` (immutable) and
 * `deps/{platform}-{arch}/{lockfileHash}.hash` (the pointer).
 */
-const logger$2 = createLogger({ prefix: "dep-cache" });
+const logger$3 = createLogger({ prefix: "dep-cache" });
 /** Default max tarball size: 500MB */
 const DEFAULT_MAX_TARBALL_BYTES = 524288e3;
 /** Default dependency-cache entry TTL: 30 days. */
@@ -867,14 +938,14 @@ var DepCache = class DepCache {
 		const ttlMsOverride = await this.resolveTtlMsOverride();
 		const depsHash = await this.resolvePointer(lockfileHash, platform, arch, ttlMsOverride, siblingsDigest);
 		if (!depsHash) {
-			logger$2.debug(`has(${lockfileHash}): false (no pointer)`, {
+			logger$3.debug(`has(${lockfileHash}): false (no pointer)`, {
 				platform,
 				arch
 			});
 			return false;
 		}
 		const exists = await this.storage.has(depTarballKey(depsHash, platform, arch), ttlMsOverride);
-		logger$2.debug(`has(${lockfileHash}): ${exists}`, {
+		logger$3.debug(`has(${lockfileHash}): ${exists}`, {
 			platform,
 			arch
 		});
@@ -898,7 +969,7 @@ var DepCache = class DepCache {
 		const pointerKey = depPointerKey(lockfileHash, platform, arch, siblingsDigest);
 		const hash = (await this.storage.get(pointerKey, ttlMsOverride))?.toString("utf-8").trim() || void 0;
 		if (!hash) {
-			logger$2.debug(`getUrlAndHash(${lockfileHash}): miss (no pointer)`, {
+			logger$3.debug(`getUrlAndHash(${lockfileHash}): miss (no pointer)`, {
 				platform,
 				arch
 			});
@@ -907,7 +978,7 @@ var DepCache = class DepCache {
 		const key = depTarballKey(hash, platform, arch);
 		const url = await this.storage.getUrl(key, ttlMsOverride);
 		if (!url) {
-			logger$2.debug(`getUrlAndHash(${lockfileHash}): miss (dangling pointer)`, {
+			logger$3.debug(`getUrlAndHash(${lockfileHash}): miss (dangling pointer)`, {
 				platform,
 				arch
 			});
@@ -915,7 +986,7 @@ var DepCache = class DepCache {
 		}
 		await this.storage.touch(key);
 		await this.storage.touch(pointerKey);
-		logger$2.debug(`getUrlAndHash(${lockfileHash}): hit`, {
+		logger$3.debug(`getUrlAndHash(${lockfileHash}): hit`, {
 			platform,
 			arch,
 			hasHash: true
@@ -939,7 +1010,7 @@ var DepCache = class DepCache {
 	*/
 	async publishPointer(lockfileHash, platform, arch, depsHash, siblingsDigest) {
 		await this.storage.put(depPointerKey(lockfileHash, platform, arch, siblingsDigest), depsHash);
-		logger$2.info(`publishPointer(${lockfileHash}) -> ${depsHash.slice(0, 12)}`, {
+		logger$3.info(`publishPointer(${lockfileHash}) -> ${depsHash.slice(0, 12)}`, {
 			platform,
 			arch
 		});
@@ -954,7 +1025,7 @@ var DepCache = class DepCache {
 		const depsHash = DepCache.computeHash(tarballData);
 		await this.storage.put(depTarballKey(depsHash, platform, arch), tarballData);
 		await this.storage.put(depPointerKey(lockfileHash, platform, arch), depsHash);
-		logger$2.info(`store: ${tarballData.length} bytes`, {
+		logger$3.info(`store: ${tarballData.length} bytes`, {
 			lockfileHash,
 			platform,
 			arch,
@@ -973,7 +1044,7 @@ var DepCache = class DepCache {
 		const depsHash = await this.resolvePointer(lockfileHash, platform, arch, await this.resolveTtlMsOverride());
 		await this.storage.delete(depPointerKey(lockfileHash, platform, arch));
 		const removed = depsHash ? await this.storage.delete(depTarballKey(depsHash, platform, arch)) : false;
-		logger$2.info(`remove(${lockfileHash}): ${removed ? "removed" : "not found"}`, {
+		logger$3.info(`remove(${lockfileHash}): ${removed ? "removed" : "not found"}`, {
 			platform,
 			arch
 		});
@@ -1017,7 +1088,7 @@ function keyDiscriminator(exact) {
 * hash and size accounting outside the tarball's own (presigned, metadata-less)
 * upload.
 */
-const logger$1 = createLogger({ prefix: "user-cache" });
+const logger$2 = createLogger({ prefix: "user-cache" });
 /**
 * Cluster-wide default quota: 5 GiB. Serves as the fallback when an org has no
 * per-org override in `org_settings.user_cache_quota_bytes`. The cluster-wide
@@ -1165,7 +1236,7 @@ var UserCache = class {
 		try {
 			limits = await this.orgLimitsReader(org);
 		} catch (err) {
-			logger$1.warn("user-cache org-limits lookup failed — using cluster defaults", {
+			logger$2.warn("user-cache org-limits lookup failed — using cluster defaults", {
 				org,
 				error: err instanceof Error ? err.message : String(err)
 			});
@@ -1222,7 +1293,7 @@ var UserCache = class {
 	async beginSave(ref) {
 		const final = userCacheEntryKey(ref, ref.key);
 		if (await this.storage.has(final)) {
-			logger$1.info("user-cache save skipped (immutable key exists)", { key: ref.key });
+			logger$2.info("user-cache save skipped (immutable key exists)", { key: ref.key });
 			return { skip: true };
 		}
 		const tempKey = `${writePrefix(ref)}${TEMP_UPLOAD_STEM}${randomUUID()}${TAR_SUFFIX}`;
@@ -1243,7 +1314,7 @@ var UserCache = class {
 		await this.storage.initMeta(final);
 		await this.storage.put(`${final}.hash`, ref.tarHash);
 		await this.storage.put(`${final}.size`, String(ref.sizeBytes));
-		logger$1.info("user-cache entry committed", {
+		logger$2.info("user-cache entry committed", {
 			key: ref.key,
 			sizeBytes: ref.sizeBytes
 		});
@@ -1293,7 +1364,7 @@ var UserCache = class {
 			await this.storage.delete(`${key}.hash`);
 			await this.storage.delete(`${key}.size`);
 			total -= size;
-			logger$1.info("user-cache eviction (org over quota)", {
+			logger$2.info("user-cache eviction (org over quota)", {
 				org: ref.org,
 				key,
 				freedBytes: size
@@ -1301,18 +1372,12 @@ var UserCache = class {
 		}
 	}
 };
-//#endregion
-//#region src/cluster/peer-credentials.ts
 /**
-* Peer credential management for cluster authentication.
-*
-* Provides two persistence layers:
-* 1. DB CRUD (PeerCredentialStore) — coordinator stores credentials for all peers
-* 2. File I/O (readCredentialFile/writeCredentialFile) — workers persist their
-*    credential locally for reconnection after restart
-*
-* Credential files are written with 0600 permissions to protect the secret.
+* `metadata` key set when a coordinator retires the self-issued credential its
+* previous run left behind. An operator revoke never sets it, which is how a
+* retirement is told apart from a revoke.
 */
+const RETIRED_BY_INSTANCE_KEY = "retiredByInstance";
 /** Default credential expiry: 90 days. */
 const DEFAULT_EXPIRY_DAYS = 90;
 /**
@@ -1349,7 +1414,8 @@ var PeerCredentialStore = class {
 					role: opts.role,
 					routing_keys: sql`${sql.val(opts.routingKeys)}`,
 					source_token_hash: opts.sourceTokenHash ?? null,
-					expires_at: expiresAt
+					expires_at: expiresAt,
+					...opts.metadata && { metadata: sql`${JSON.stringify(opts.metadata)}::jsonb` }
 				}).execute();
 				return Number(revokeResult?.numUpdatedRows ?? 0n);
 			});
@@ -1376,6 +1442,33 @@ var PeerCredentialStore = class {
 		return row ? mapRow(row) : null;
 	}
 	/**
+	* The credential row for `instanceId` that nobody revoked, expired or not.
+	* `peer_credentials_active_uniq` allows at most one.
+	*/
+	async findUnrevokedByInstanceId(instanceId) {
+		const row = await this.db.selectFrom("peer_credentials").selectAll().where("instance_id", "=", instanceId).where("revoked_at", "is", null).executeTakeFirst();
+		return row ? mapRow(row) : null;
+	}
+	/** The most recently revoked credential row for `instanceId`, or null. */
+	async findLatestRevokedByInstanceId(instanceId) {
+		const row = await this.db.selectFrom("peer_credentials").selectAll().where("instance_id", "=", instanceId).where("revoked_at", "is not", null).orderBy("revoked_at", "desc").executeTakeFirst();
+		return row ? mapRow(row) : null;
+	}
+	/**
+	* Revoke `instanceId`'s unrevoked, self-issued row whose hash is
+	* `credentialHash`, recording who retired it. Every condition sits in the
+	* WHERE clause, so a row that changed meanwhile is left alone.
+	* Returns whether a row was retired.
+	*/
+	async retireSelfIssued(opts) {
+		const marker = JSON.stringify({ [RETIRED_BY_INSTANCE_KEY]: opts.retiredByInstance });
+		const result = await this.db.updateTable("peer_credentials").set({
+			revoked_at: /* @__PURE__ */ new Date(),
+			metadata: sql`COALESCE(metadata, '{}'::jsonb) || ${marker}::jsonb`
+		}).where("instance_id", "=", opts.instanceId).where("credential_hash", "=", opts.credentialHash).where("revoked_at", "is", null).where(sql`metadata->>'issuance'`, "=", "self").executeTakeFirst();
+		return Number(result?.numUpdatedRows ?? 0n) > 0;
+	}
+	/**
 	* Update the last_seen_at timestamp for a credential.
 	* Optionally records which coordinator validated it.
 	*/
@@ -1420,7 +1513,8 @@ function mapRow(row) {
 		lastSeenAt: row.last_seen_at ? new Date(row.last_seen_at) : null,
 		lastValidatedBy: row.last_validated_by ?? null,
 		expiresAt: new Date(row.expires_at),
-		revokedAt: row.revoked_at ? new Date(row.revoked_at) : null
+		revokedAt: row.revoked_at ? new Date(row.revoked_at) : null,
+		metadata: row.metadata ?? {}
 	};
 }
 /**
@@ -1514,6 +1608,8 @@ function runDetached(logger, task, work, context = {}) {
 	}
 	Promise.resolve(outcome).catch(fail);
 }
+createLogger({ prefix: "instance-heartbeat" });
+createLogger({ prefix: "coordinator-credential" });
 //#endregion
 //#region src/cluster/peer-auth-coordinator.ts
 /**
@@ -1526,6 +1622,10 @@ function runDetached(logger, task, work, context = {}) {
 * rejection — a credential revocation cascade. This coordinator serializes all
 * file access through one in-process mutex so only one peer-client token-joins
 * per storm, and it never deletes a credential file a sibling has refreshed.
+*
+* A coordinator without a join token also wires a `selfIssue` callback. When a
+* decision finds no credential and no token, the callback issues the
+* coordinator its own credential, once per storm, under the same mutex.
 */
 const logger = createLogger({ prefix: "peer-auth-coordinator" });
 /** How long a waiting peer-client awaits an in-flight sibling token-join. */
@@ -1546,6 +1646,11 @@ var PeerAuthCoordinator = class {
 	instanceId;
 	joinToken;
 	joinWaitTimeoutMs;
+	selfIssue;
+	/** The issuance in progress; concurrent sibling clients share it. */
+	selfIssueInFlight = null;
+	/** An operator revoked this coordinator's credential: never issue again in this process. */
+	selfIssueRevoked = false;
 	/** Promise-chain mutex tail; every file op awaits the prior one. */
 	lock = Promise.resolve();
 	/** Set while one peer-client is mid token-join; siblings await it. */
@@ -1555,6 +1660,7 @@ var PeerAuthCoordinator = class {
 		this.instanceId = opts.instanceId;
 		this.joinToken = opts.joinToken;
 		this.joinWaitTimeoutMs = opts.joinWaitTimeoutMs ?? DEFAULT_JOIN_WAIT_TIMEOUT_MS;
+		this.selfIssue = opts.selfIssue;
 	}
 	/** Run `fn` exclusively against the credential file. */
 	async withLock(fn) {
@@ -1563,10 +1669,21 @@ var PeerAuthCoordinator = class {
 		return run;
 	}
 	async readValidCredential() {
-		const cred = await readCredentialFile(this.credentialFile);
+		let cred;
+		try {
+			cred = await readCredentialFile(this.credentialFile);
+		} catch (err) {
+			logger.warn("Ignoring unreadable peer credential file", {
+				instanceId: this.instanceId,
+				path: this.credentialFile,
+				error: toErrorMessage(err)
+			});
+			return null;
+		}
 		return cred && cred.instanceId === this.instanceId ? cred : null;
 	}
 	async decideAuth() {
+		let selfIssueTried = false;
 		for (let i = 0; i < MAX_DECIDE_ITERATIONS; i++) {
 			const decision = await this.withLock(async () => {
 				const cred = await this.readValidCredential();
@@ -1584,10 +1701,19 @@ var PeerAuthCoordinator = class {
 						complete: this.makeComplete(join)
 					};
 				}
+				if (this.selfIssue && !selfIssueTried && !this.selfIssueRevoked) return "self-issue";
 				return { mode: "no-auth" };
 			});
-			if (decision !== "await-join") return decision;
-			await this.awaitInFlightJoin();
+			if (decision === "await-join") {
+				await this.awaitInFlightJoin();
+				continue;
+			}
+			if (decision === "self-issue") {
+				selfIssueTried = true;
+				await this.runSelfIssue();
+				continue;
+			}
+			return decision;
 		}
 		if (this.joinToken) {
 			const join = deferred();
@@ -1600,6 +1726,18 @@ var PeerAuthCoordinator = class {
 		}
 		return { mode: "no-auth" };
 	}
+	/** Run the issuer under the mutex, once for every concurrent caller. */
+	runSelfIssue() {
+		const selfIssue = this.selfIssue;
+		if (!selfIssue) return Promise.resolve("failed");
+		this.selfIssueInFlight ??= this.withLock(selfIssue).then((outcome) => {
+			if (outcome === "revoked") this.selfIssueRevoked = true;
+			return outcome;
+		}).finally(() => {
+			this.selfIssueInFlight = null;
+		});
+		return this.selfIssueInFlight;
+	}
 	makeComplete(join) {
 		return (issued) => {
 			runDetached(logger, "Peer token-join completion", () => this.withLock(async () => {
@@ -1668,7 +1806,8 @@ var PeerAuthCoordinator = class {
 const joinTokenLogger = createLogger({ prefix: "join-token" });
 const TOKEN_PREFIX = "kici_join_v1";
 const DEFAULT_EXPIRY_MS = 36e5;
-const TOKEN_ALREADY_USED_MESSAGE = "Join token has already been used. Generate a new token with: kici admin create-join-token";
+const TOKEN_ALREADY_USED_MESSAGE = "Join token has already been used. Create a new token with: kici-admin peer create-token";
+const TOKEN_EXPIRED_MESSAGE = "Join token has expired. Create a new token with: kici-admin peer create-token";
 var JoinTokenManager = class {
 	deps;
 	constructor(deps) {
@@ -1753,7 +1892,7 @@ var JoinTokenManager = class {
 		}
 		const row = await this.deps.db.selectFrom("join_tokens").selectAll().where("token_hash", "=", keys.validationHash).executeTakeFirst();
 		if (!row) throw new Error("Invalid join token");
-		if (new Date(row.expires_at).getTime() <= Date.now()) throw new Error("Join token has expired. Generate a new token with: kici admin create-join-token");
+		if (new Date(row.expires_at).getTime() <= Date.now()) throw new Error(TOKEN_EXPIRED_MESSAGE);
 		if (row.consumed_at) {
 			if (row.consumed_by_instance === peerInstanceId) {
 				joinTokenLogger.info("Re-validated join token for returning instance", {

package/dist/pipeline/changed-files.d.ts

@@ -0,0 +1,51 @@
+/**
+ * The one changed-files resolver every dispatch path calls: same-source, the
+ * no-lock-file organization-wide pass, cross-source repo mode, and rerun.
+ *
+ * `unavailable` means the orchestrator holds no authoritative list; whether the
+ * agent can compute one is the range's question, answered by `resolveDiffRange`.
+ */
+import { type ChangedFilesStatus, type DiffRange, type LockWorkflow, type SimulatedEvent, type WebhookNormalizer, type WorkflowDecision } from '@kici-dev/engine';
+import type { ProviderBundle } from '../provider-registry.js';
+export interface ResolvedChangedFiles {
+    files: string[];
+    status: ChangedFilesStatus;
+    range: DiffRange;
+}
+/**
+ * Resolve the files an event changed through a bundle's fetcher.
+ *
+ * - no workflow in `workflows` has a `paths` trigger → `skipped`;
+ * - a deleted branch → `fetched` + `[]`, whichever provider delivered it;
+ * - no fetcher, or a fetcher that throws → `unavailable`, never a failed delivery.
+ */
+export declare function resolveEventChangedFiles(args: {
+    bundle: Pick<ProviderBundle, 'changedFilesFetcher'>;
+    credentials: unknown;
+    repoIdentifier: string;
+    eventName: string;
+    payload: unknown;
+    event: SimulatedEvent;
+    /** Omitted: always resolve — a re-evaluation has no lock file to consult. */
+    workflows?: readonly LockWorkflow[];
+}): Promise<ResolvedChangedFiles>;
+/**
+ * Stamp the repository's default branch on a normalized event, so the
+ * new-branch range (`resolveDiffRange`) and the agent that receives this event
+ * read the same value. `extractDefaultBranch` prefers the normalizer's hook,
+ * which covers GitLab's `project.default_branch`.
+ */
+export declare function withDefaultBranch(event: SimulatedEvent, payload: unknown, normalizer: WebhookNormalizer): SimulatedEvent;
+/** Copy a resolved list and its status onto the event the matcher reads. */
+export declare function stampChangedFiles(event: SimulatedEvent, resolved: ResolvedChangedFiles): SimulatedEvent;
+/**
+ * Downgrade a cross-source pull-request deferral to a conservative match.
+ *
+ * A cross-source job checks out the registration's commit, never the pull
+ * request's head, so no agent can diff the pull request for it. A path
+ * decision the orchestrator could not make therefore runs the workflow, as a
+ * range-less push does, instead of handing the agent a diff of the wrong
+ * commits. Push ranges diff explicit commits and keep their deferral.
+ */
+export declare function withoutCrossSourcePrDeferral(decision: WorkflowDecision, event: SimulatedEvent): WorkflowDecision;
+//# sourceMappingURL=changed-files.d.ts.map

package/dist/pipeline/cross-source-tally.d.ts

@@ -0,0 +1,21 @@
+/**
+ * A generic delivery can run both halves — cross-source for other sources'
+ * registrations, and the inbound source's own lock when its normalizer reads a
+ * repository. The delivery still gets one event-log row: the same-source path
+ * writes it, and this wrapper folds the cross-source half into it.
+ */
+import type { ProcessingDeps } from './processor.js';
+export interface CrossSourceTally {
+    candidatesConsidered: number;
+    jobsDispatched: number;
+    /** A cross-source candidate evaluated `paths` against an unavailable diff. */
+    degraded: boolean;
+}
+/**
+ * Wrap `deps.eventLog` so whichever row the same-source path records — corrupt
+ * lock, missing lock, skip or processed — also counts the cross-source half's
+ * dispatched jobs and carries its degraded reason. A reason the same-source
+ * path already set wins.
+ */
+export declare function withCrossSourceTally(deps: ProcessingDeps, tally: CrossSourceTally): ProcessingDeps;
+//# sourceMappingURL=cross-source-tally.d.ts.map

package/dist/pipeline/degraded-reason.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Annotation written to a `processed` event-log row when path filters were
+ * evaluated against an unavailable changed-files diff. Reuses the existing
+ * free-text field so the outcome is never a silent `processed / matched 0` —
+ * no new event-log status enum value is added.
+ */
+export declare const DEGRADED_CHANGED_FILES_REASON = "trigger evaluation degraded: changed files unavailable \u2014 path filters matched conservatively";
+//# sourceMappingURL=degraded-reason.d.ts.map

package/dist/pipeline/dispatch-matched-workflow.d.ts

@@ -704,6 +704,8 @@ export declare function materializeStaticJobsSafe(staticJobs: readonly LockJob[]
     expansionMap: Map<string, readonly string[]>;
     matrixFailures: RejectedJob[];
 }>;
+/** The `paths` lists this workflow's matching left to the agent, or undefined. */
+export declare function deferredPathsOf(ctx: Pick<WorkflowDispatchContext, 'decision'>): string[][] | undefined;
 /**
  * Scan a lock workflow's always-mode approval timeouts (workflow-level +
  * each static job) for a non-positive / non-finite `timeoutSeconds`. Returns the
@@ -879,6 +881,8 @@ export declare function catchUpNeedsGatedJobs(args: {
 }): Promise<void>;
 /** Why an init result must not lead to a dispatch. */
 export declare enum InitDispatchSuppression {
+    /** The agent's clone diff matched none of the deferred `paths`. */
+    Paths = "paths",
     /** The workflow's own `filter` decided the workflow does not apply. */
     Filter = "filter",
     /** The job is already rejected by a context rule, or held for approval. */
@@ -887,6 +891,11 @@ export declare enum InitDispatchSuppression {
 /**
  * Decide whether an arrived init result may dispatch its job.
  *
+ * `Paths` is checked first: a workflow whose deferred `paths` did not match
+ * never triggered, whatever its filter says. Only an explicit `false`
+ * suppresses — an agent that predates deferred paths reports no verdict, and
+ * that absence dispatches.
+ *
  * `Filter` requires the workflow to actually declare a filter as well as the
  * agent to have reported `false`: a buggy or rogue agent must not be able to
  * suppress a filter-less workflow by inventing the field, and an agent that
@@ -917,6 +926,7 @@ export declare enum InitDispatchSuppression {
  */
 export declare function initDispatchSuppression(workflow: Pick<LockWorkflow, 'hasFilter'>, initResult: {
     filterPassed?: boolean;
+    pathsPassed?: boolean;
 }, jobEnvData: Pick<JobEnvData, 'rejected' | 'held'>): InitDispatchSuppression | null;
 export interface GeneratedJobConfig {
     /**