@kici-dev/orchestrator 0.1.14 → 0.1.16

package/dist/index.js

@@ -1,14 +1,11 @@
-import { fileURLToPath as __cjs_fileURLToPath } from "node:url";
-import { dirname as __cjs_dirname } from "node:path";
-__cjs_dirname(__cjs_fileURLToPath(import.meta.url));
 import "node:module";
-import { CopyObjectCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
+import { CopyObjectCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3";
 import { Upload } from "@aws-sdk/lib-storage";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 import { createLogger, createS3Client, sha256 } from "@kici-dev/shared";
 import { promises } from "node:fs";
 import { dirname, join } from "node:path";
-import { createHmac, randomBytes } from "node:crypto";
+import { createHmac, randomBytes, randomUUID } from "node:crypto";
 import "node:fs/promises";
 import { Kysely, PostgresDialect, sql } from "kysely";
 import pg from "pg";
@@ -36,6 +33,8 @@ var S3CacheStorage = class {
 	client;
 	/** Separate client for pre-signed URL generation (uses externalEndpoint if configured). */
 	presignClient;
+	/** Separate client for the host CLI's pre-signed upload URL (uses uploadEndpoint if configured). */
+	uploadPresignClient;
 	bucket;
 	prefix;
 	ttlMs;
@@ -49,15 +48,21 @@ var S3CacheStorage = class {
 			endpoint: options.externalEndpoint
 		});
 		else this.presignClient = this.client;
+		if (options.uploadEndpoint) this.uploadPresignClient = createS3Client({
+			...options,
+			endpoint: options.uploadEndpoint
+		});
+		else this.uploadPresignClient = this.client;
 	}
 	/** Build the full S3 object key from a cache key. */
 	objectKey(key) {
 		return `${this.prefix}${key}`;
 	}
-	/** Check if metadata indicates an expired item. */
-	isExpired(meta) {
+	/** Check if metadata indicates an expired item, honoring a per-op TTL override. */
+	isExpired(meta, ttlMsOverride) {
+		const ttl = ttlMsOverride ?? this.ttlMs;
 		const lastAccessed = new Date(meta.lastAccessedAt).getTime();
-		return Date.now() - lastAccessed > this.ttlMs;
+		return Date.now() - lastAccessed > ttl;
 	}
 	/**
 	* Read metadata from an S3 object's custom headers.
@@ -121,10 +126,10 @@ var S3CacheStorage = class {
 			}
 		}).done();
 	}
-	async get(key) {
+	async get(key, ttlMsOverride) {
 		const meta = await this.readMeta(key);
 		if (!meta) return null;
-		if (this.isExpired(meta)) {
+		if (this.isExpired(meta, ttlMsOverride)) {
 			await this.deleteObject(key);
 			return null;
 		}
@@ -146,10 +151,10 @@ var S3CacheStorage = class {
 		} catch {}
 		return data;
 	}
-	async has(key) {
+	async has(key, ttlMsOverride) {
 		const meta = await this.readMeta(key);
 		if (!meta) return false;
-		if (this.isExpired(meta)) {
+		if (this.isExpired(meta, ttlMsOverride)) {
 			await this.deleteObject(key);
 			return false;
 		}
@@ -166,10 +171,10 @@ var S3CacheStorage = class {
 		meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
 		await this.updateMeta(key, meta);
 	}
-	async getUrl(key) {
+	async getUrl(key, ttlMsOverride) {
 		const meta = await this.readMeta(key);
 		if (!meta) return null;
-		if (this.isExpired(meta)) {
+		if (this.isExpired(meta, ttlMsOverride)) {
 			await this.deleteObject(key);
 			return null;
 		}
@@ -187,7 +192,7 @@ var S3CacheStorage = class {
 	}
 	async getInternalUploadUrl(key) {
 		const objectKey = this.objectKey(key);
-		return getSignedUrl(this.client, new PutObjectCommand({
+		return getSignedUrl(this.uploadPresignClient, new PutObjectCommand({
 			Bucket: this.bucket,
 			Key: objectKey
 		}), { expiresIn: UPLOAD_URL_EXPIRY_SECONDS });
@@ -206,6 +211,46 @@ var S3CacheStorage = class {
 			}
 		}));
 	}
+	async list(subPrefix) {
+		const fullPrefix = this.objectKey(subPrefix);
+		const items = [];
+		let token;
+		do {
+			const resp = await this.client.send(new ListObjectsV2Command({
+				Bucket: this.bucket,
+				Prefix: fullPrefix,
+				ContinuationToken: token
+			}));
+			for (const obj of resp.Contents ?? []) {
+				if (!obj.Key) continue;
+				items.push({
+					key: obj.Key.slice(this.prefix.length),
+					created: obj.LastModified ? obj.LastModified.getTime() : 0
+				});
+			}
+			token = resp.IsTruncated ? resp.NextContinuationToken : void 0;
+		} while (token);
+		items.sort((a, b) => b.created - a.created);
+		return items.map((i) => i.key);
+	}
+	async getMetadata(key) {
+		return this.readMeta(key);
+	}
+	async copy(srcKey, destKey) {
+		const srcObj = this.objectKey(srcKey);
+		const destObj = this.objectKey(destKey);
+		const now = (/* @__PURE__ */ new Date()).toISOString();
+		await this.client.send(new CopyObjectCommand({
+			Bucket: this.bucket,
+			Key: destObj,
+			CopySource: `${this.bucket}/${srcObj}`,
+			MetadataDirective: "REPLACE",
+			Metadata: {
+				"created-at": now,
+				"last-accessed-at": now
+			}
+		}));
+	}
 	/** Delete an S3 object. Idempotent (doesn't error if missing). */
 	async deleteObject(key) {
 		try {
@@ -298,9 +343,10 @@ var FilesystemCacheStorage = class {
 	metaPath(key) {
 		return `${this.resolvePath(key)}.meta.json`;
 	}
-	isExpired(meta) {
+	isExpired(meta, ttlMsOverride) {
+		const ttl = ttlMsOverride ?? this.ttlMs;
 		const lastAccessed = new Date(meta.lastAccessedAt).getTime();
-		return Date.now() - lastAccessed > this.ttlMs;
+		return Date.now() - lastAccessed > ttl;
 	}
 	async readMeta(key) {
 		try {
@@ -328,10 +374,10 @@ var FilesystemCacheStorage = class {
 			lastAccessedAt: now
 		});
 	}
-	async get(key) {
+	async get(key, ttlMsOverride) {
 		const meta = await this.readMeta(key);
 		if (!meta) return null;
-		if (this.isExpired(meta)) {
+		if (this.isExpired(meta, ttlMsOverride)) {
 			await this.deleteFiles(key);
 			return null;
 		}
@@ -348,10 +394,10 @@ var FilesystemCacheStorage = class {
 		} catch {}
 		return data;
 	}
-	async has(key) {
+	async has(key, ttlMsOverride) {
 		const meta = await this.readMeta(key);
 		if (!meta) return false;
-		if (this.isExpired(meta)) {
+		if (this.isExpired(meta, ttlMsOverride)) {
 			await this.deleteFiles(key);
 			return false;
 		}
@@ -368,10 +414,10 @@ var FilesystemCacheStorage = class {
 		meta.lastAccessedAt = (/* @__PURE__ */ new Date()).toISOString();
 		await this.writeMeta(key, meta);
 	}
-	async getUrl(key) {
+	async getUrl(key, ttlMsOverride) {
 		const meta = await this.readMeta(key);
 		if (!meta) return null;
-		if (this.isExpired(meta)) {
+		if (this.isExpired(meta, ttlMsOverride)) {
 			await this.deleteFiles(key);
 			return null;
 		}
@@ -390,6 +436,57 @@ var FilesystemCacheStorage = class {
 			lastAccessedAt: now
 		});
 	}
+	async list(subPrefix) {
+		const root = this.resolvePath(subPrefix.replace(/\/$/, ""));
+		let entries;
+		try {
+			entries = await this.walkDataFiles(root, subPrefix.replace(/\/$/, ""));
+		} catch (err) {
+			if (isNotFoundFsError(err)) return [];
+			throw err;
+		}
+		entries.sort((a, b) => b.mtime - a.mtime);
+		return entries.map((e) => e.key);
+	}
+	async getMetadata(key) {
+		return this.readMeta(key);
+	}
+	async copy(srcKey, destKey) {
+		const srcPath = this.resolvePath(srcKey);
+		const destPath = this.resolvePath(destKey);
+		await promises.mkdir(dirname(destPath), { recursive: true });
+		await promises.copyFile(srcPath, destPath);
+		const now = (/* @__PURE__ */ new Date()).toISOString();
+		await this.writeMeta(destKey, {
+			createdAt: now,
+			lastAccessedAt: now
+		});
+	}
+	/**
+	* Recursively collect data files under `dir`, returning each as a cache key
+	* relative to `basePath` paired with its mtime. Metadata sidecars
+	* (`*.meta.json`) and the atomic-write temp files (`*.tmp-*`) are skipped so
+	* `list()` only surfaces real cache objects.
+	*/
+	async walkDataFiles(dir, keyPrefix) {
+		const out = [];
+		const dirents = await promises.readdir(dir, { withFileTypes: true });
+		for (const dirent of dirents) {
+			const childPath = join(dir, dirent.name);
+			const childKey = keyPrefix ? `${keyPrefix}/${dirent.name}` : dirent.name;
+			if (dirent.isDirectory()) {
+				out.push(...await this.walkDataFiles(childPath, childKey));
+				continue;
+			}
+			if (dirent.name.endsWith(".meta.json") || /\.tmp-[0-9a-f]+$/.test(dirent.name)) continue;
+			const stat = await promises.stat(childPath);
+			out.push({
+				key: childKey,
+				mtime: stat.mtimeMs
+			});
+		}
+		return out;
+	}
 	signedUrl(method, key) {
 		const { token } = signToken(this.signingSecret, method, key);
 		const encodedKey = key.split("/").map((seg) => encodeURIComponent(seg)).join("/");
@@ -432,6 +529,7 @@ function createCacheStorage(config) {
 		region: config.region,
 		endpoint: config.endpoint,
 		externalEndpoint: config.externalEndpoint,
+		uploadEndpoint: config.uploadEndpoint,
 		forcePathStyle: config.forcePathStyle
 	});
 	if (config.type === "filesystem") return new FilesystemCacheStorage({
@@ -460,7 +558,7 @@ function createCacheStorage(config) {
 * downloads the source tarball, extracts it, and imports the workflow entry
 * via the shared oxc-transform ESM loader hook.
 */
-const logger$1 = createLogger({ prefix: "source-cache" });
+const logger$2 = createLogger({ prefix: "source-cache" });
 /** Cache key format: source/{contentHash}.tar.gz */
 function sourceKey(contentHash) {
 	return `source/${contentHash}.tar.gz`;
@@ -473,7 +571,7 @@ var SourceCache = class {
 	async has(contentHash) {
 		const key = sourceKey(contentHash);
 		const exists = await this.storage.has(key);
-		logger$1.debug(`has(${contentHash}): ${exists}`);
+		logger$2.debug(`has(${contentHash}): ${exists}`);
 		return exists;
 	}
 	async get(contentHash) {
@@ -481,8 +579,8 @@ var SourceCache = class {
 		const data = await this.storage.get(key);
 		if (data) {
 			await this.storage.touch(key);
-			logger$1.debug(`get(${contentHash}): hit (${data.length} bytes)`);
-		} else logger$1.debug(`get(${contentHash}): miss`);
+			logger$2.debug(`get(${contentHash}): hit (${data.length} bytes)`);
+		} else logger$2.debug(`get(${contentHash}): miss`);
 		return data;
 	}
 	async getUrl(contentHash) {
@@ -490,8 +588,8 @@ var SourceCache = class {
 		const url = await this.storage.getUrl(key);
 		if (url) {
 			await this.storage.touch(key);
-			logger$1.debug(`getUrl(${contentHash}): hit`);
-		} else logger$1.debug(`getUrl(${contentHash}): miss`);
+			logger$2.debug(`getUrl(${contentHash}): hit`);
+		} else logger$2.debug(`getUrl(${contentHash}): miss`);
 		return url;
 	}
 	async getUploadUrl(contentHash) {
@@ -502,12 +600,12 @@ var SourceCache = class {
 		const key = sourceKey(contentHash);
 		await this.storage.put(key, tarball);
 		const size = typeof tarball === "string" ? Buffer.byteLength(tarball) : tarball.length;
-		logger$1.info(`store(${contentHash}): stored (${size} bytes)`);
+		logger$2.info(`store(${contentHash}): stored (${size} bytes)`);
 	}
 	async remove(contentHash) {
 		const key = sourceKey(contentHash);
 		const removed = await this.storage.delete(key);
-		logger$1.info(`remove(${contentHash}): ${removed ? "removed" : "not found"}`);
+		logger$2.info(`remove(${contentHash}): ${removed ? "removed" : "not found"}`);
 		return removed;
 	}
 };
@@ -522,7 +620,7 @@ var SourceCache = class {
 *
 * Cache key format: deps/{platform}-{arch}/{lockfileHash}.tar.gz
 */
-const logger = createLogger({ prefix: "dep-cache" });
+const logger$1 = createLogger({ prefix: "dep-cache" });
 /** Default max tarball size: 500MB */
 const DEFAULT_MAX_TARBALL_BYTES = 524288e3;
 /** Build cache key for dependency tarball: deps/{platform}-{arch}/{lockfileHash}.tar.gz */
@@ -540,7 +638,7 @@ var DepCache = class {
 	async has(lockfileHash, platform, arch) {
 		const key = depKey(lockfileHash, platform, arch);
 		const exists = await this.storage.has(key);
-		logger.debug(`has(${lockfileHash}): ${exists}`, {
+		logger$1.debug(`has(${lockfileHash}): ${exists}`, {
 			platform,
 			arch
 		});
@@ -555,11 +653,11 @@ var DepCache = class {
 		const url = await this.storage.getUrl(key);
 		if (url) {
 			await this.storage.touch(key);
-			logger.debug(`getUrl(${lockfileHash}): hit`, {
+			logger$1.debug(`getUrl(${lockfileHash}): hit`, {
 				platform,
 				arch
 			});
-		} else logger.debug(`getUrl(${lockfileHash}): miss`, {
+		} else logger$1.debug(`getUrl(${lockfileHash}): miss`, {
 			platform,
 			arch
 		});
@@ -574,7 +672,7 @@ var DepCache = class {
 		const key = depKey(lockfileHash, platform, arch);
 		const url = await this.storage.getUrl(key);
 		if (!url) {
-			logger.debug(`getUrlAndHash(${lockfileHash}): miss`, {
+			logger$1.debug(`getUrlAndHash(${lockfileHash}): miss`, {
 				platform,
 				arch
 			});
@@ -583,7 +681,7 @@ var DepCache = class {
 		await this.storage.touch(key);
 		const hashKey = `deps/${platform}-${arch}/${lockfileHash}.hash`;
 		const hash = (await this.storage.get(hashKey))?.toString("utf-8") || void 0;
-		logger.debug(`getUrlAndHash(${lockfileHash}): hit`, {
+		logger$1.debug(`getUrlAndHash(${lockfileHash}): hit`, {
 			platform,
 			arch,
 			hasHash: !!hash
@@ -608,7 +706,7 @@ var DepCache = class {
 		if (tarballData.length > this.maxTarballBytes) throw new Error(`Dep tarball exceeds max size: ${tarballData.length} bytes > ${this.maxTarballBytes} bytes limit`);
 		const key = depKey(lockfileHash, platform, arch);
 		await this.storage.put(key, tarballData);
-		logger.info(`store: ${tarballData.length} bytes`, {
+		logger$1.info(`store: ${tarballData.length} bytes`, {
 			lockfileHash,
 			platform,
 			arch
@@ -625,7 +723,7 @@ var DepCache = class {
 	async remove(lockfileHash, platform, arch) {
 		const key = depKey(lockfileHash, platform, arch);
 		const removed = await this.storage.delete(key);
-		logger.info(`remove(${lockfileHash}): ${removed ? "removed" : "not found"}`, {
+		logger$1.info(`remove(${lockfileHash}): ${removed ? "removed" : "not found"}`, {
 			platform,
 			arch
 		});
@@ -633,6 +731,248 @@ var DepCache = class {
 	}
 };
 //#endregion
+//#region src/cache/user-cache.ts
+/**
+* User-facing cache layer wrapping CacheStorage.
+*
+* Namespacing: `cache/<orgId>/<repoId>/<refScope>/<key>.tar.gz`, where
+* `refScope` is `shared` for trusted refs (org-shared, default-branch cache)
+* or `iso/<runId>` for untrusted refs (per-run isolated scope). Restores from
+* an untrusted ref read the shared scope as a fallback but writes can NEVER
+* land in the shared scope — the GitHub Actions cache-isolation model. Keyed
+* strictly per org so no tenant can read another tenant's cache.
+*
+* Saves are immutable (first save under an exact key wins) and atomic
+* (upload to a `.tmp-<uuid>` key, then server-side copy to the final key and
+* delete the temp), so a crashed save never leaves a corrupt final entry.
+*
+* Eviction: per-org byte quota plus the TTL the backing CacheStorage already
+* enforces lazily on access. On a save that pushes the org over quota,
+* least-recently-used entries (oldest `lastAccessedAt`, read from the storage
+* backend's metadata) are evicted until under quota; each eviction is logged.
+* The companion `.hash` / `.size` sidecar objects carry the integrity
+* hash and size accounting outside the tarball's own (presigned, metadata-less)
+* upload.
+*/
+const logger = createLogger({ prefix: "user-cache" });
+/**
+* Cluster-wide default quota: 5 GiB. Serves as the fallback when an org has no
+* per-org override in `org_settings.user_cache_quota_bytes`. The cluster-wide
+* value is itself operator-configurable via KICI_USER_CACHE_QUOTA_BYTES.
+*/
+const DEFAULT_USER_CACHE_QUOTA_BYTES = 5 * 1024 * 1024 * 1024;
+/**
+* Cluster-wide default entry TTL: 7 days. Fallback when an org has no per-org
+* override in `org_settings.user_cache_ttl_ms`. The cluster-wide value is
+* operator-configurable via KICI_USER_CACHE_TTL_MS.
+*/
+const DEFAULT_USER_CACHE_TTL_MS = 10080 * 60 * 1e3;
+/** Tarball suffix for committed cache entries. */
+const TAR_SUFFIX = ".tar.gz";
+var UserCache = class {
+	storage;
+	/** Cluster-wide default quota (the `KICI_USER_CACHE_QUOTA_BYTES` value). */
+	defaultQuotaBytes;
+	/** Cluster-wide default TTL (the `KICI_USER_CACHE_TTL_MS` value). */
+	defaultTtlMs;
+	/** Optional per-org override reader; absent = always use the cluster defaults. */
+	orgLimitsReader;
+	constructor(opts) {
+		this.storage = opts.storage;
+		this.defaultQuotaBytes = opts.quotaBytes ?? 5368709120;
+		this.defaultTtlMs = opts.ttlMs ?? 6048e5;
+		this.orgLimitsReader = opts.orgLimitsReader;
+	}
+	/**
+	* Resolve the effective quota + TTL for an org: the per-org override from
+	* `org_settings` when present, otherwise the cluster-wide default. A reader
+	* failure falls back to the defaults (logged) — the cache must never fail a
+	* restore/save because the settings lookup hiccupped.
+	*/
+	async resolveLimits(org) {
+		if (!this.orgLimitsReader) return {
+			quotaBytes: this.defaultQuotaBytes,
+			ttlMs: this.defaultTtlMs
+		};
+		let limits = {};
+		try {
+			limits = await this.orgLimitsReader(org);
+		} catch (err) {
+			logger.warn("user-cache org-limits lookup failed — using cluster defaults", {
+				org,
+				error: err instanceof Error ? err.message : String(err)
+			});
+		}
+		return {
+			quotaBytes: limits.quotaBytes ?? this.defaultQuotaBytes,
+			ttlMs: limits.ttlMs ?? this.defaultTtlMs
+		};
+	}
+	/**
+	* Sanitize a path segment so a key can never escape its org/repo/scope
+	* namespace. Beyond stripping disallowed characters, a segment consisting
+	* only of dots (`.`, `..`, …) is replaced wholesale: such a segment is a
+	* dot-segment that HTTP/S3 path canonicalization collapses (`a/./b` → `a/b`,
+	* `a/../b` → `b`), which both corrupts the namespace and breaks the SigV4
+	* signature on a pre-signed PUT/GET. Repo identifiers like `.` (the internal
+	* provider's repo id) hit exactly this case, so the all-dots guard keeps the
+	* object key canonical and the namespace boundary intact.
+	*/
+	seg(s) {
+		const cleaned = s.replace(/[^A-Za-z0-9._-]/g, "_");
+		return /^\.+$/.test(cleaned) ? `_${cleaned}` : cleaned;
+	}
+	/** Org-level prefix: the per-tenant isolation boundary and quota scope. */
+	orgPrefix(ref) {
+		return `cache/${this.seg(ref.org)}/`;
+	}
+	/** Org + repo prefix shared by every scope of a repo. */
+	repoPrefix(ref) {
+		return `${this.orgPrefix(ref)}${this.seg(ref.repo)}`;
+	}
+	/** Namespace prefix for the WRITE scope of a ref (shared OR per-run isolated). */
+	writePrefix(ref) {
+		const base = this.repoPrefix(ref);
+		if (ref.scope === "isolated") {
+			if (!ref.runId) throw new Error("isolated cache scope requires a runId");
+			return `${base}/iso/${this.seg(ref.runId)}/`;
+		}
+		return `${base}/shared/`;
+	}
+	/** Namespace prefixes the ref may READ, in priority order. Isolated reads its own run scope, then shared. */
+	readPrefixes(ref) {
+		const base = this.repoPrefix(ref);
+		if (ref.scope === "isolated") {
+			if (!ref.runId) throw new Error("isolated cache scope requires a runId");
+			return [`${base}/iso/${this.seg(ref.runId)}/`, `${base}/shared/`];
+		}
+		return [`${base}/shared/`];
+	}
+	finalKey(prefix, key) {
+		return `${prefix}${this.seg(key)}${TAR_SUFFIX}`;
+	}
+	/** Restore: try the exact key across read prefixes, then restoreKeys prefix scan (newest wins). */
+	async restore(ref) {
+		const { ttlMs } = await this.resolveLimits(ref.org);
+		const prefixes = this.readPrefixes(ref);
+		const exact = await this.restoreExact(ref, prefixes, ttlMs);
+		if (exact) return exact;
+		return await this.restoreByPrefix(ref, prefixes, ttlMs) ?? { hit: false };
+	}
+	/** Try the exact key in read-prefix priority order. */
+	async restoreExact(ref, prefixes, ttlMs) {
+		for (const prefix of prefixes) {
+			const key = this.finalKey(prefix, ref.key);
+			const url = await this.storage.getUrl(key, ttlMs);
+			if (url) {
+				await this.storage.touch(key);
+				return {
+					hit: true,
+					matchedKey: ref.key,
+					downloadUrl: url,
+					tarHash: await this.readHash(key)
+				};
+			}
+		}
+		return null;
+	}
+	/** restoreKeys prefix fallback (ordered); within a prefix, list() returns newest-first. */
+	async restoreByPrefix(ref, prefixes, ttlMs) {
+		for (const rk of ref.restoreKeys ?? []) for (const prefix of prefixes) {
+			const matches = (await this.storage.list(`${prefix}${this.seg(rk)}`)).filter((k) => k.endsWith(TAR_SUFFIX));
+			if (matches.length === 0) continue;
+			const winner = matches[0];
+			const url = await this.storage.getUrl(winner, ttlMs);
+			if (!url) continue;
+			await this.storage.touch(winner);
+			return {
+				hit: true,
+				matchedKey: winner.slice(prefix.length, -7),
+				downloadUrl: url,
+				tarHash: await this.readHash(winner)
+			};
+		}
+		return null;
+	}
+	/** Begin a save: presigned PUT to a temp key, or skip=true when the immutable key exists. */
+	async beginSave(ref) {
+		const prefix = this.writePrefix(ref);
+		const final = this.finalKey(prefix, ref.key);
+		if (await this.storage.has(final)) {
+			logger.info("user-cache save skipped (immutable key exists)", { key: ref.key });
+			return { skip: true };
+		}
+		const tempKey = `${prefix}.tmp-${randomUUID()}${TAR_SUFFIX}`;
+		return {
+			skip: false,
+			uploadUrl: await this.storage.getUploadUrl(tempKey),
+			tempKey
+		};
+	}
+	/** Commit a save: copy temp -> final, init metadata, store companion hash/size, delete temp, enforce quota. */
+	async commitSave(ref) {
+		const prefix = this.writePrefix(ref);
+		const final = this.finalKey(prefix, ref.key);
+		if (await this.storage.has(final)) return;
+		if (ref.tempKey) {
+			await this.storage.copy(ref.tempKey, final);
+			await this.storage.delete(ref.tempKey);
+		}
+		await this.storage.initMeta(final);
+		await this.storage.put(`${final}.hash`, ref.tarHash);
+		await this.storage.put(`${final}.size`, String(ref.sizeBytes));
+		logger.info("user-cache entry committed", {
+			key: ref.key,
+			sizeBytes: ref.sizeBytes
+		});
+		await this.enforceQuota(ref);
+	}
+	async readHash(key) {
+		return (await this.storage.get(`${key}.hash`))?.toString("utf-8") || void 0;
+	}
+	/** Evict least-recently-used entries for the org until total tarball size <= the per-org quota. */
+	async enforceQuota(ref) {
+		const { quotaBytes } = await this.resolveLimits(ref.org);
+		const orgPrefix = this.orgPrefix(ref);
+		const keys = (await this.storage.list(orgPrefix)).filter((k) => k.endsWith(TAR_SUFFIX));
+		const sized = [];
+		let total = 0;
+		for (const k of keys) {
+			const sizeData = await this.storage.get(`${k}.size`);
+			const size = sizeData ? Number(sizeData.toString("utf-8")) : 0;
+			total += size;
+			sized.push({
+				key: k,
+				size
+			});
+		}
+		if (total <= quotaBytes) return;
+		const candidates = [];
+		for (const e of sized) {
+			const meta = await this.storage.getMetadata(e.key);
+			const lastAccessed = meta ? new Date(meta.lastAccessedAt).getTime() : 0;
+			candidates.push({
+				key: e.key,
+				size: e.size,
+				lastAccessed
+			});
+		}
+		candidates.sort((a, b) => a.lastAccessed - b.lastAccessed);
+		for (const { key, size } of candidates) {
+			if (total <= quotaBytes) break;
+			await this.storage.delete(key);
+			await this.storage.delete(`${key}.hash`);
+			await this.storage.delete(`${key}.size`);
+			total -= size;
+			logger.info("user-cache eviction (org over quota)", {
+				org: ref.org,
+				key,
+				freedBytes: size
+			});
+		}
+	}
+};
+//#endregion
 //#region src/cluster/peer-credentials.ts
 /**
 * Peer credential management for cluster authentication.
@@ -767,6 +1107,6 @@ async function createPeerCredentialStoreFromUrl(databaseUrl, opts) {
 	};
 }
 //#endregion
-export { DepCache, PeerCredentialStore, S3CacheStorage, SourceCache, createCacheStorage, createPeerCredentialStoreFromUrl };
+export { DEFAULT_USER_CACHE_QUOTA_BYTES, DEFAULT_USER_CACHE_TTL_MS, DepCache, PeerCredentialStore, S3CacheStorage, SourceCache, UserCache, createCacheStorage, createPeerCredentialStoreFromUrl };
 
 //# sourceMappingURL=index.js.map

package/dist/lockfile-cache.d.ts

@@ -7,7 +7,7 @@
  *
  * Replaces the old GitHub-specific LockFileCache in github/lockfile.ts.
  */
-import type { LockFileFetcher, LockFile } from '@kici-dev/engine';
+import { type LockFileFetcher, type LockFile } from '@kici-dev/engine';
 export declare class LockFileCache {
     private readonly cache;
     private hits;

package/dist/metrics/prometheus.d.ts

@@ -64,6 +64,14 @@ export declare const triggerMatchDurationSeconds: import("@opentelemetry/api").H
  * Total deduplication cache hits (duplicate webhooks rejected).
  */
 export declare const dedupHitsTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
+/**
+ * pg connection errors absorbed by the pool error handlers instead of
+ * crashing the process (e.g. a database failover terminating idle pooled
+ * backends).
+ * Labels:
+ * - source: idle-pool | client
+ */
+export declare const pgPoolClientErrorsTotal: import("@opentelemetry/api").Counter<import("@opentelemetry/api").Attributes>;
 /**
  * Total source cache hits (`.kici/` source tarball found for content hash).
  */

package/dist/orchestrator-core.d.ts

@@ -22,11 +22,11 @@ import { DedupCache } from './webhook/dedup.js';
 import { ObserverRegistry } from './ws/observer-registry.js';
 import { AgentMetricsAggregator } from './metrics/agent-metrics-aggregator.js';
 import { SourceLocationStore } from './app.js';
-import { type PeerHeartbeat } from '@kici-dev/engine';
+import { type PeerHeartbeat, type PeerLogsCollectRequest, type PeerToPeerMessage } from '@kici-dev/engine';
 import { ScalerManager } from './scaler/index.js';
 import type { ScalerConfig } from './scaler/index.js';
 import type { CacheStorage } from './storage/index.js';
-import { SourceCache, BuildCoordinator, DepCache, PendingBuildTracker, PendingInitTracker, PendingDynamicTracker } from './cache/index.js';
+import { SourceCache, BuildCoordinator, DepCache, UserCache, DispatchCacheRefTracker, PendingBuildTracker, PendingInitTracker, PendingDynamicTracker } from './cache/index.js';
 import { CheckRunReporter } from './reporting/check-run-reporter.js';
 import { StepLogBuffer } from './reporting/step-log-buffer.js';
 import { type LogStorage } from './reporting/log-storage.js';
@@ -70,6 +70,9 @@ export interface OrchestratorSubsystems {
     cacheStorage: CacheStorage | undefined;
     sourceCache: SourceCache | undefined;
     depCache: DepCache | undefined;
+    userCache: UserCache | undefined;
+    /** Server-side jobId -> user-cache-namespace store, written at dispatch, read by the agent-WS handler. */
+    dispatchCacheRefs: DispatchCacheRefTracker;
     buildCoordinator: BuildCoordinator | undefined;
     pendingBuilds: PendingBuildTracker | undefined;
     pendingInits: PendingInitTracker;
@@ -143,6 +146,14 @@ export interface OrchestratorSubsystems {
      * store. the metrics enforcement plan.
      */
     agentMetricsAggregator: AgentMetricsAggregator;
+    /**
+     * Peer-side fleet collect responder. On an inbound peer.logs.collect.request,
+     * assembles this node's subtree and streams it back. Wired into every outgoing
+     * PeerClient (the incoming peer-handler is wired directly in setupCluster).
+     * Armed after the fleet runtime is built, so calls before that resolve to a
+     * no-op.
+     */
+    fleetCollectResponder: (msg: PeerLogsCollectRequest, send: (out: PeerToPeerMessage) => boolean) => Promise<void>;
 }
 /**
  * Mode-specific hooks injected by each entry point.