@kici-dev/orchestrator 0.1.26 → 0.2.0

package/dist/index.js

@@ -6,10 +6,12 @@ import { createLogger, createS3Client, sha256 } from "@kici-dev/shared";
 import * as fs from "node:fs";
 import { mkdtempSync, promises, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
-import { createHmac, hkdfSync, randomBytes, randomUUID } from "node:crypto";
+import { createHash, createHmac, hkdfSync, randomBytes, randomUUID } from "node:crypto";
 import { chmod, mkdir, readFile, unlink, writeFile } from "node:fs/promises";
 import { Kysely, PostgresDialect, sql } from "kysely";
 import pg from "pg";
+import { z } from "zod";
+import { DEFAULT_APPROVAL_EXPIRY_HOURS, HoldScope, HoldType, TriggerSource, persistedHoldTypeSpellings } from "@kici-dev/engine";
 import { platform, tmpdir } from "node:os";
 import { createInterface } from "node:readline";
 import { Octokit } from "@octokit/rest";
@@ -189,6 +191,9 @@ var S3CacheStorage = class {
 			Key: this.objectKey(key)
 		}), { expiresIn: PRESIGNED_URL_EXPIRY_SECONDS });
 	}
+	presignedGetTtlSeconds() {
+		return PRESIGNED_URL_EXPIRY_SECONDS;
+	}
 	async getUploadUrl(key) {
 		const objectKey = this.objectKey(key);
 		return getSignedUrl(this.presignClient, new PutObjectCommand({
@@ -242,6 +247,17 @@ var S3CacheStorage = class {
 	async getMetadata(key) {
 		return this.readMeta(key);
 	}
+	async getObjectSize(key) {
+		try {
+			return (await this.client.send(new HeadObjectCommand({
+				Bucket: this.bucket,
+				Key: this.objectKey(key)
+			}))).ContentLength ?? null;
+		} catch (err) {
+			if (this.isNotFoundError(err)) return null;
+			throw err;
+		}
+	}
 	async copy(srcKey, destKey) {
 		const srcObj = this.objectKey(srcKey);
 		const destObj = this.objectKey(destKey);
@@ -429,6 +445,9 @@ var FilesystemCacheStorage = class {
 		}
 		return this.signedUrl("GET", key);
 	}
+	presignedGetTtlSeconds() {
+		return Math.floor(DEFAULT_SIGN_URL_TTL_MS / 1e3);
+	}
 	async getUploadUrl(key) {
 		return this.signedUrl("PUT", key);
 	}
@@ -442,21 +461,48 @@ var FilesystemCacheStorage = class {
 			lastAccessedAt: now
 		});
 	}
+	/**
+	* List cache keys under a prefix.
+	*
+	* The prefix is a **key prefix**, not necessarily a directory — `UserCache`'s
+	* `restoreKeys` fallback lists on `${prefix}${seg(restoreKey)}`, a partial
+	* object name. Two shapes, and both must work the way the S3 backend does:
+	*
+	* - trailing `/` — a directory. Walk it directly; this is the hot path.
+	* - no trailing `/` — a partial name. Walk the parent directory and filter by
+	*   the prefix, since the prefix itself may name a file (or nothing at all).
+	*
+	* Reading the prefix as a directory in the second case is what made the
+	* `restoreKeys` fallback silently never hit here (`ENOENT` → `[]`) or throw
+	* (`ENOTDIR`, when a real object carried that exact name).
+	*/
 	async list(subPrefix) {
-		const root = this.resolvePath(subPrefix.replace(/\/$/, ""));
+		const isDirPrefix = subPrefix.endsWith("/");
+		const trimmed = subPrefix.replace(/\/$/, "");
+		const walkKey = isDirPrefix ? trimmed : trimmed.slice(0, trimmed.lastIndexOf("/") + 1);
+		const walkRoot = walkKey === "" ? this.basePath : this.resolvePath(walkKey.replace(/\/$/, ""));
 		let entries;
 		try {
-			entries = await this.walkDataFiles(root, subPrefix.replace(/\/$/, ""));
+			entries = await this.walkDataFiles(walkRoot, walkKey.replace(/\/$/, ""));
 		} catch (err) {
 			if (isNotFoundFsError(err)) return [];
 			throw err;
 		}
+		if (!isDirPrefix) entries = entries.filter((e) => e.key.startsWith(trimmed));
 		entries.sort((a, b) => b.mtime - a.mtime);
 		return entries.map((e) => e.key);
 	}
 	async getMetadata(key) {
 		return this.readMeta(key);
 	}
+	async getObjectSize(key) {
+		try {
+			return (await promises.stat(this.resolvePath(key))).size;
+		} catch (err) {
+			if (isNotFoundFsError(err)) return null;
+			throw err;
+		}
+	}
 	async copy(srcKey, destKey) {
 		const srcPath = this.resolvePath(srcKey);
 		const destPath = this.resolvePath(destKey);
@@ -504,7 +550,9 @@ var FilesystemCacheStorage = class {
 	}
 };
 function isNotFoundFsError(err) {
-	return typeof err === "object" && err !== null && err.code === "ENOENT";
+	if (typeof err !== "object" || err === null) return false;
+	const code = err.code;
+	return code === "ENOENT" || code === "ENOTDIR";
 }
 /**
 * Write a file atomically: write to a temp sibling, fsync, rename. Avoids
@@ -629,6 +677,9 @@ var SourceCache = class {
 const logger$2 = createLogger({ prefix: "dep-cache" });
 /** Default max tarball size: 500MB */
 const DEFAULT_MAX_TARBALL_BYTES = 524288e3;
+/** Default dependency-cache entry TTL: 30 days. */
+const DEFAULT_CACHE_TTL_DAYS = 30;
+const MS_PER_DAY = 864e5;
 /** Build cache key for dependency tarball: deps/{platform}-{arch}/{lockfileHash}.tar.gz */
 function depKey(lockfileHash, platform, arch) {
 	return `deps/${platform}-${arch}/${lockfileHash}.tar.gz`;
@@ -636,14 +687,29 @@ function depKey(lockfileHash, platform, arch) {
 var DepCache = class {
 	storage;
 	maxTarballBytes;
+	cacheTtlDaysFallback;
+	clusterSettings;
 	constructor(options) {
 		this.storage = options.storage;
 		this.maxTarballBytes = options.maxTarballBytes ?? DEFAULT_MAX_TARBALL_BYTES;
+		this.cacheTtlDaysFallback = options.cacheTtlDaysFallback ?? DEFAULT_CACHE_TTL_DAYS;
+		this.clusterSettings = options.clusterSettings;
+	}
+	/**
+	* Resolve the live dependency-cache entry TTL as a per-operation override
+	* (ms) from `cluster_settings.cache_ttl_days`, falling back to the cluster
+	* default. `undefined` when no reader is wired, so the storage backend uses
+	* its own configured TTL. Passed to read/expiry operations so an operator's
+	* `cache_ttl_days` change takes effect on the next lookup.
+	*/
+	async resolveTtlMsOverride() {
+		if (!this.clusterSettings) return void 0;
+		return await this.clusterSettings.getNumber("cache_ttl_days", this.cacheTtlDaysFallback) * MS_PER_DAY;
 	}
 	/** Check if a dep tarball exists in cache. */
 	async has(lockfileHash, platform, arch) {
 		const key = depKey(lockfileHash, platform, arch);
-		const exists = await this.storage.has(key);
+		const exists = await this.storage.has(key, await this.resolveTtlMsOverride());
 		logger$2.debug(`has(${lockfileHash}): ${exists}`, {
 			platform,
 			arch
@@ -656,7 +722,7 @@ var DepCache = class {
 	*/
 	async getUrl(lockfileHash, platform, arch) {
 		const key = depKey(lockfileHash, platform, arch);
-		const url = await this.storage.getUrl(key);
+		const url = await this.storage.getUrl(key, await this.resolveTtlMsOverride());
 		if (url) {
 			await this.storage.touch(key);
 			logger$2.debug(`getUrl(${lockfileHash}): hit`, {
@@ -676,7 +742,7 @@ var DepCache = class {
 	*/
 	async getUrlAndHash(lockfileHash, platform, arch) {
 		const key = depKey(lockfileHash, platform, arch);
-		const url = await this.storage.getUrl(key);
+		const url = await this.storage.getUrl(key, await this.resolveTtlMsOverride());
 		if (!url) {
 			logger$2.debug(`getUrlAndHash(${lockfileHash}): miss`, {
 				platform,
@@ -709,7 +775,8 @@ var DepCache = class {
 	* Throws if tarball exceeds max size (per user decision).
 	*/
 	async store(lockfileHash, platform, arch, tarballData) {
-		if (tarballData.length > this.maxTarballBytes) throw new Error(`Dep tarball exceeds max size: ${tarballData.length} bytes > ${this.maxTarballBytes} bytes limit`);
+		const maxTarballBytes = this.clusterSettings ? await this.clusterSettings.getNumber("cache_max_tarball_bytes", this.maxTarballBytes) : this.maxTarballBytes;
+		if (tarballData.length > maxTarballBytes) throw new Error(`Dep tarball exceeds max size: ${tarballData.length} bytes > ${maxTarballBytes} bytes limit`);
 		const key = depKey(lockfileHash, platform, arch);
 		await this.storage.put(key, tarballData);
 		logger$2.info(`store: ${tarballData.length} bytes`, {
@@ -736,13 +803,26 @@ var DepCache = class {
 		return removed;
 	}
 };
+/**
+* A collision-resistant discriminator for a user-supplied object-key segment.
+*
+* Hashes the **exact** name, before any sanitization. Sanitizing is many-to-one
+* — `seg()` maps both `/` and `_` to `_`, so `a/b` and `a_b` collapse to one
+* segment — and a case-insensitive backend folds `build` onto `Build`. Hashing
+* the pre-sanitization name is what keeps each of those pairs two objects.
+*/
+function keyDiscriminator(exact) {
+	return createHash("sha256").update(exact, "utf8").digest("hex").slice(0, 32);
+}
 //#endregion
 //#region src/cache/user-cache.ts
 /**
 * User-facing cache layer wrapping CacheStorage.
 *
-* Namespacing: `cache/<orgId>/<repoId>/<refScope>/<key>.tar.gz`, where
-* `refScope` is `shared` for trusted refs (org-shared, default-branch cache)
+* Namespacing: `cache/<orgId>/<repoId>/<refScope>/<key>-<discriminator>.tar.gz`.
+* The discriminator is a hash of the exact cache key, so two keys differing
+* only by case stay two objects even on a case-insensitive store. `refScope`
+* is `shared` for trusted refs (org-shared, default-branch cache)
 * or `iso/<runId>` for untrusted refs (per-run isolated scope). Restores from
 * an untrusted ref read the shared scope as a fallback but writes can NEVER
 * land in the shared scope — the GitHub Actions cache-isolation model. Keyed
@@ -775,6 +855,42 @@ const DEFAULT_USER_CACHE_QUOTA_BYTES = 5 * 1024 * 1024 * 1024;
 const DEFAULT_USER_CACHE_TTL_MS = 10080 * 60 * 1e3;
 /** Tarball suffix for committed cache entries. */
 const TAR_SUFFIX = ".tar.gz";
+/** Matches a stem ending in the fixed-width discriminator this module appends. */
+const KEY_DISCRIMINATOR_TAIL = new RegExp(`^(.*)-([0-9a-f]{32})$`);
+/**
+* Recover the reported cache key from a stored object's stem.
+*
+* The value on this path is already the sanitized, lossy form, so this only
+* needs to undo the discriminator the key format appends. A stem that itself
+* ends in a discriminator-shaped tail loses it — that affects only the string
+* reported back as `matchedKey`, never which object is served.
+*/
+function stripKeyDiscriminator(stem) {
+	return KEY_DISCRIMINATOR_TAIL.exec(stem)?.[1] ?? stem;
+}
+/** Device names NTFS reserves; a file cannot actually be created under one. */
+const WINDOWS_RESERVED_NAME = /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(\.|$)/;
+/** Keys that cannot have been part of a case collision in the old key format. */
+const UNAMBIGUOUS_LEGACY_KEY = /^[a-z0-9_][a-z0-9._-]*$/;
+/**
+* Whether a cache key is safe to look up in the pre-discriminator key format.
+*
+* Every clause rules out a way the old format could have folded two distinct
+* keys onto one object:
+*
+* - the character class is a subset of what `seg()` preserves, so a match
+*   implies the key survives sanitization unchanged (no many-to-one rewrite)
+*   and is pure ASCII, which rules out HFS+ Unicode normalisation;
+* - requiring lowercase rules out case folding, the collision this change fixes;
+* - a trailing `.` is stripped by NTFS, and a reserved device name cannot be
+*   created there at all.
+*
+* A key failing any clause simply gets no legacy fallback — it is re-fetched
+* and re-saved under the discriminated key, which is always correct.
+*/
+function unambiguousLegacyKey(key) {
+	return UNAMBIGUOUS_LEGACY_KEY.test(key) && !key.endsWith(".") && !WINDOWS_RESERVED_NAME.test(key);
+}
 var UserCache = class {
 	storage;
 	/** Cluster-wide default quota (the `KICI_USER_CACHE_QUOTA_BYTES` value). */
@@ -854,9 +970,61 @@ var UserCache = class {
 		}
 		return [`${base}/shared/`];
 	}
+	/**
+	* Object key for a committed cache entry.
+	*
+	* The trailing discriminator is a hash of the EXACT cache key, and it is what
+	* keeps `build` and `Build` apart on a case-insensitive namespace — the
+	* filesystem backend on a macOS or Windows host, where the two would
+	* otherwise resolve to one object and a restore could return the other key's
+	* tarball.
+	*
+	* It is deliberately a SUFFIX: `restoreByPrefix` matches `restoreKeys` by
+	* string prefix, so appending leaves those semantics untouched. The only
+	* parse that has to know about it is the `matchedKey` slice.
+	*/
 	finalKey(prefix, key) {
+		return `${prefix}${this.seg(key)}-${keyDiscriminator(key)}${TAR_SUFFIX}`;
+	}
+	/**
+	* The pre-discriminator object key.
+	*
+	* @deprecated Read-only compatibility path for entries written before the key
+	* carried a discriminator. Nothing writes this format; it is removed at the
+	* next major, by which point every such entry has aged out of the cache TTL.
+	*/
+	legacyFinalKey(prefix, key) {
 		return `${prefix}${this.seg(key)}${TAR_SUFFIX}`;
 	}
+	/**
+	* Restore an entry still stored in the pre-discriminator key format.
+	*
+	* Two gates, and BOTH are load-bearing — dropping either re-creates the
+	* wrong-cache-hit this change fixes:
+	*
+	* 1. `unambiguousLegacyKey` statically rules out any key that could have
+	*    collided with a case variant in the old format.
+	* 2. The listed name must match byte-exactly. A case-insensitive backend
+	*    resolves `getUrl('build')` to an object created as `Build`, but its
+	*    *listing* reports the real created name — so comparing against the
+	*    listing is what detects that the object is not really ours.
+	*
+	* @deprecated Removed at the next major; see `legacyFinalKey`.
+	*/
+	async restoreLegacyExact(ref, prefix, ttlMs) {
+		if (!unambiguousLegacyKey(ref.key)) return null;
+		const legacyKey = this.legacyFinalKey(prefix, ref.key);
+		if (!(await this.storage.list(prefix)).includes(legacyKey)) return null;
+		const url = await this.storage.getUrl(legacyKey, ttlMs);
+		if (!url) return null;
+		await this.storage.touch(legacyKey);
+		return {
+			hit: true,
+			matchedKey: ref.key,
+			downloadUrl: url,
+			tarHash: await this.readHash(legacyKey)
+		};
+	}
 	/** Restore: try the exact key across read prefixes, then restoreKeys prefix scan (newest wins). */
 	async restore(ref) {
 		const { ttlMs } = await this.resolveLimits(ref.org);
@@ -879,6 +1047,8 @@ var UserCache = class {
 					tarHash: await this.readHash(key)
 				};
 			}
+			const legacy = await this.restoreLegacyExact(ref, prefix, ttlMs);
+			if (legacy) return legacy;
 		}
 		return null;
 	}
@@ -893,7 +1063,7 @@ var UserCache = class {
 			await this.storage.touch(winner);
 			return {
 				hit: true,
-				matchedKey: winner.slice(prefix.length, -7),
+				matchedKey: stripKeyDiscriminator(winner.slice(prefix.length, -7)),
 				downloadUrl: url,
 				tarHash: await this.readHash(winner)
 			};
@@ -1328,7 +1498,9 @@ var JoinTokenManager = class {
 			role
 		};
 		const secret = randomBytes(32);
-		const token = `${TOKEN_PREFIX}.${Buffer.from(JSON.stringify(routing)).toString("base64url")}.${secret.toString("hex")}`;
+		const routingB64 = Buffer.from(JSON.stringify(routing)).toString("base64url");
+		const secretHex = secret.toString("hex");
+		const token = `${TOKEN_PREFIX}.${routingB64}.${secretHex}`;
 		const { validationHash } = deriveKeys(secret);
 		await this.deps.db.insertInto("join_tokens").values({
 			id: randomUUID(),
@@ -1452,6 +1624,404 @@ function deriveKeys(secret) {
 		validationHash
 	};
 }
+/**
+* Reason a run was held in the security queue. Persisted verbatim in
+* `held_runs.reason` and switched on by `buildSecurityHoldSummary`.
+*/
+const SecurityHoldReason = z.enum([
+	"workflow_modification",
+	"unknown_contributor",
+	"fork_pr",
+	"context_trust"
+]);
+/** Manages held run lifecycle (pending -> approved/rejected/expired). */
+var HeldRunStore = class {
+	db;
+	constructor(db) {
+		this.db = db;
+	}
+	/** Create a new held run with pending status. */
+	async create(orgId, data) {
+		return this.db.insertInto("held_runs").values({
+			org_id: orgId,
+			run_id: data.runId,
+			job_id: data.jobId,
+			context_id: data.contextId,
+			hold_type: data.holdType,
+			queue_type: data.queueType ?? "context",
+			reason: data.reason,
+			expires_at: data.expiresAt
+		}).returningAll().executeTakeFirstOrThrow();
+	}
+	/**
+	* Create a generalized approval hold (workflow/job/step scope, explicit or
+	* context trigger) carrying a normalized `ApprovalRequirement`. Returns
+	* the created row.
+	*/
+	async createHold(orgId, data) {
+		return this.db.insertInto("held_runs").values({
+			org_id: orgId,
+			run_id: data.runId,
+			job_id: data.jobId,
+			context_id: data.contextId ?? null,
+			hold_type: data.holdType ?? HoldType.enum.reviewer,
+			queue_type: data.queueType ?? "context",
+			reason: data.requirement.reason,
+			expires_at: new Date(data.requirement.expiresAt),
+			hold_scope: data.scope,
+			step_index: data.stepIndex ?? null,
+			trigger_source: data.triggerSource,
+			approval_requirement: data.requirement,
+			...data.payload !== void 0 && { payload: JSON.stringify(data.payload) }
+		}).returningAll().executeTakeFirstOrThrow();
+	}
+	/** INSERT one decision row using the given executor (root or transaction). */
+	insertDecisionRow(exec, heldRunId, data) {
+		const clausesSatisfied = data.clausesSatisfied != null ? JSON.stringify(data.clausesSatisfied) : null;
+		return exec.insertInto("held_run_approvals").values({
+			held_run_id: heldRunId,
+			approver_user_id: data.approverSub,
+			decision: data.decision,
+			clauses_satisfied: clausesSatisfied
+		}).returningAll().executeTakeFirstOrThrow();
+	}
+	/** Flip a pending hold to 'approved' using the given executor. Undefined if not pending. */
+	flipToApproved(exec, orgId, heldRunId) {
+		return exec.updateTable("held_runs").set({
+			status: "approved",
+			resolved_at: sql`now()`
+		}).where("id", "=", heldRunId).where("org_id", "=", orgId).where("status", "=", "pending").returningAll().executeTakeFirst();
+	}
+	/** Flip a pending hold to 'rejected' using the given executor. Undefined if not pending. */
+	flipToRejected(exec, orgId, heldRunId, reason) {
+		const set = {
+			status: "rejected",
+			resolved_at: sql`now()`
+		};
+		if (reason !== void 0) set.reason = reason;
+		return exec.updateTable("held_runs").set(set).where("id", "=", heldRunId).where("org_id", "=", orgId).where("status", "=", "pending").returningAll().executeTakeFirst();
+	}
+	/** Map a released held_runs row to the resume ReleaseSignal. */
+	toReleaseSignal(row) {
+		return {
+			holdId: row.id,
+			runId: row.run_id,
+			jobId: row.job_id,
+			scope: row.hold_scope ?? HoldScope.enum.job,
+			stepIndex: row.step_index,
+			triggerSource: row.trigger_source ?? TriggerSource.enum.context
+		};
+	}
+	/** Record one approve/reject decision against a hold. */
+	async recordDecision(heldRunId, data) {
+		return this.insertDecisionRow(this.db, heldRunId, data);
+	}
+	/**
+	* Atomically record an approve decision and release the (now-satisfied) hold.
+	* The INSERT into `held_run_approvals` and the `held_runs` → approved UPDATE
+	* run in a single transaction, so a crash between them cannot strand the hold
+	* `pending` with a recorded approve. Throws if the hold is not found or no
+	* longer pending (the whole transaction rolls back).
+	*/
+	async recordAndRelease(orgId, heldRunId, data) {
+		return this.db.transaction().execute(async (tx) => {
+			await this.insertDecisionRow(tx, heldRunId, data);
+			const row = await this.flipToApproved(tx, orgId, heldRunId);
+			if (!row) throw new Error(`Held run '${heldRunId}' not found or not pending`);
+			return this.toReleaseSignal(row);
+		});
+	}
+	/**
+	* Atomically record a reject decision and reject the hold. The INSERT and the
+	* `held_runs` → rejected UPDATE run in a single transaction, so a crash
+	* between them cannot strand the hold `pending` with a recorded reject (which
+	* would poison `evaluate()` forever). Throws if the hold is not found or no
+	* longer pending (the whole transaction rolls back).
+	*/
+	async recordAndReject(orgId, heldRunId, data, reason) {
+		return this.db.transaction().execute(async (tx) => {
+			await this.insertDecisionRow(tx, heldRunId, data);
+			const row = await this.flipToRejected(tx, orgId, heldRunId, reason);
+			if (!row) throw new Error(`Held run '${heldRunId}' not found or not pending`);
+			return row;
+		});
+	}
+	/** List the recorded decisions for a hold, oldest first. */
+	async listDecisions(heldRunId) {
+		return this.db.selectFrom("held_run_approvals").selectAll().where("held_run_id", "=", heldRunId).orderBy("created_at", "asc").execute();
+	}
+	/** Get a single held run by id (org-scoped). Returns null if absent. */
+	async getById(orgId, heldRunId) {
+		return await this.db.selectFrom("held_runs").selectAll().where("id", "=", heldRunId).where("org_id", "=", orgId).executeTakeFirst() ?? null;
+	}
+	/**
+	* Release a hold whose approval requirement is satisfied. Flips the row to
+	* 'approved' and returns a `ReleaseSignal` describing how the caller must
+	* resume the element (re-dispatch for job/workflow, agent notification for
+	* step). Throws if the hold is not found or not pending. Approver attribution
+	* lives in `held_run_approvals`, not on the row.
+	*/
+	async release(orgId, heldRunId) {
+		const row = await this.flipToApproved(this.db, orgId, heldRunId);
+		if (!row) throw new Error(`Held run '${heldRunId}' not found or not pending`);
+		return this.toReleaseSignal(row);
+	}
+	/** Approve a pending held run. Throws if not found or not pending. */
+	async approve(orgId, heldRunId, approvedBy) {
+		const row = await this.db.updateTable("held_runs").set({
+			status: "approved",
+			approved_by: approvedBy,
+			resolved_at: sql`now()`
+		}).where("id", "=", heldRunId).where("org_id", "=", orgId).where("status", "=", "pending").returningAll().executeTakeFirst();
+		if (!row) throw new Error(`Held run '${heldRunId}' not found or not pending`);
+		return row;
+	}
+	/** Reject a pending held run. Throws if not found or not pending. */
+	async reject(orgId, heldRunId, reason) {
+		const row = await this.flipToRejected(this.db, orgId, heldRunId, reason);
+		if (!row) throw new Error(`Held run '${heldRunId}' not found or not pending`);
+		return row;
+	}
+	/** List pending held runs for an org, ordered by creation time. */
+	async listPending(orgId) {
+		return this.db.selectFrom("held_runs").selectAll().where("org_id", "=", orgId).where("status", "=", "pending").orderBy("created_at", "asc").execute();
+	}
+	/** List held runs for an org with optional filters. */
+	async listAll(orgId, options) {
+		let query = this.db.selectFrom("held_runs").selectAll().where("org_id", "=", orgId).orderBy("created_at", "desc");
+		if (options?.status) query = query.where("status", "=", options.status);
+		if (options?.limit) query = query.limit(options.limit);
+		return query.execute();
+	}
+	/** List held runs for an org filtered by queue type with optional filters. */
+	async listByQueueType(orgId, queueType, options) {
+		let query = this.db.selectFrom("held_runs").selectAll().where("org_id", "=", orgId).where("queue_type", "=", queueType).orderBy("created_at", "desc");
+		if (options?.status) query = query.where("status", "=", options.status);
+		if (options?.limit) query = query.limit(options.limit);
+		return query.execute();
+	}
+	/**
+	* List pending `security` holds whose run belongs to a specific PR (and repo).
+	*
+	* Joins `execution_runs` on `run_id` and filters by `repo_identifier` +
+	* `pr_number` so `/kici approve|reject` only affects the commented PR's holds
+	* rather than every pending security hold in the org. A run with a NULL
+	* `pr_number` (non-PR run, or a legacy run predating the column) never matches
+	* the equality predicate, so such holds are never released here — fail-closed.
+	*/
+	async listPendingSecurityHoldsForPr(orgId, repoIdentifier, prNumber) {
+		return this.db.selectFrom("held_runs").innerJoin("execution_runs", "execution_runs.run_id", "held_runs.run_id").selectAll("held_runs").where("held_runs.org_id", "=", orgId).where("held_runs.queue_type", "=", "security").where("held_runs.status", "=", "pending").where("execution_runs.repo_identifier", "=", repoIdentifier).where("execution_runs.pr_number", "=", prNumber).orderBy("held_runs.created_at", "desc").execute();
+	}
+	/**
+	* Approve a pending held run, enforcing queue_type boundary.
+	* Prevents context approvals from approving security holds and vice versa.
+	* Throws if not found, not pending, or queue_type mismatch.
+	*/
+	async approveByQueueType(orgId, heldRunId, approvedBy, queueType) {
+		const row = await this.db.updateTable("held_runs").set({
+			status: "approved",
+			approved_by: approvedBy,
+			resolved_at: sql`now()`
+		}).where("id", "=", heldRunId).where("org_id", "=", orgId).where("status", "=", "pending").where("queue_type", "=", queueType).returningAll().executeTakeFirst();
+		if (!row) throw new Error(`Held run '${heldRunId}' not found, not pending, or queue type mismatch (expected '${queueType}')`);
+		return row;
+	}
+	/** Get a held run by run ID and job ID. Returns null if not found. */
+	async getByRunAndJob(orgId, runId, jobId) {
+		return await this.db.selectFrom("held_runs").selectAll().where("org_id", "=", orgId).where("run_id", "=", runId).where("job_id", "=", jobId).executeTakeFirst() ?? null;
+	}
+	/**
+	* List pending holds past their `expires_at`. Called by the stale detector
+	* BEFORE `expireOverdue()` so it can route each overdue hold by scope (step
+	* holds notify the waiting agent; job/workflow holds fail the run).
+	*/
+	async listOverdue() {
+		return this.db.selectFrom("held_runs").selectAll().where("status", "=", "pending").where("expires_at", "<", sql`now()`).execute();
+	}
+	/**
+	* Release overdue workflow timer holds. The install-gate wait action pauses
+	* the workflow as a held run; on timer expiry it must RESUME (not fail like
+	* a reviewer-hold expiry). Flips each overdue pending `hold_type` timer,
+	* `hold_scope='workflow'` row to `released` and returns a `ReleaseSignal`
+	* per row so the caller can resume the workflow. Runs BEFORE
+	* `expireOverdue()` so these rows leave the pending pool before the
+	* expire-and-fail sweep sees them.
+	*
+	* The filter matches every persisted spelling of the timer hold type, so a
+	* row an un-upgraded orchestrator wrote as `wait_timer` still resumes rather
+	* than falling through to the expire-and-fail sweep. `hold_scope` is what
+	* keeps job-scoped dispatch-gate timer holds out of this sweep.
+	*/
+	async releaseDueWaitHolds() {
+		return (await this.db.updateTable("held_runs").set({
+			status: "released",
+			resolved_at: sql`now()`
+		}).where("status", "=", "pending").where("hold_type", "in", persistedHoldTypeSpellings(HoldType.enum.timer)).where("hold_scope", "=", HoldScope.enum.workflow).where("expires_at", "<", sql`now()`).returningAll().execute()).map((row) => ({
+			holdId: row.id,
+			runId: row.run_id,
+			jobId: row.job_id,
+			scope: row.hold_scope ?? HoldScope.enum.workflow,
+			stepIndex: row.step_index,
+			triggerSource: row.trigger_source ?? TriggerSource.enum.context
+		}));
+	}
+	/**
+	* Expire overdue pending runs. Called by the stale detector.
+	* Sets status to 'expired' and resolved_at to now() for all
+	* pending runs past their expires_at.
+	* Returns the number of expired runs.
+	*/
+	async expireOverdue() {
+		const result = await this.db.updateTable("held_runs").set({
+			status: "expired",
+			resolved_at: sql`now()`
+		}).where("status", "=", "pending").where("expires_at", "<", sql`now()`).execute();
+		const updateResult = Array.isArray(result) ? result[0] : result;
+		return Number(updateResult?.numUpdatedRows ?? 0n);
+	}
+};
+/**
+* Build a HeldRunStore backed by its own connection pool to the given
+* orchestrator database URL. Mirrors `createPeerCredentialStoreFromUrl` /
+* `createJoinTokenManagerFromUrl`; consumed by E2E tests that exercise the
+* PR-scoped hold selection against the real deployed orchestrator DB.
+*/
+function createHeldRunStoreFromUrl(databaseUrl, opts) {
+	const db = new Kysely({ dialect: new PostgresDialect({ pool: new pg.Pool({
+		connectionString: databaseUrl,
+		max: opts?.maxConnections ?? 3
+	}) }) });
+	return {
+		store: new HeldRunStore(db),
+		dispose: async () => {
+			await db.destroy();
+		}
+	};
+}
+//#endregion
+//#region src/security/trust-policy-store.ts
+/**
+* Cache of the Platform-owned org trust policy.
+*
+* The Platform is the authority for a Platform-attached org and pushes the
+* policy on `trust_policy.update`; this store persists it so the policy survives
+* a restart instead of being unknown until the next push. In independent mode
+* there is no Platform, so the operator writes the same row via
+* `kici-admin trust-policy set` and `source` records which wrote it.
+*/
+/** Who wrote the cached policy row. */
+const TrustPolicySource = z.enum(["platform", "local"]);
+/**
+* The strictest documented defaults. Used to fill gaps on a local merge; the
+* fail-closed policy an unknown org resolves to lives in `trust-policy-gate.ts`.
+*/
+const DEFAULT_TRUST_POLICY = {
+	forkPolicy: "hold",
+	unknownContributorPolicy: "hold",
+	workflowChangePolicy: "hold",
+	approvalExpiryHours: DEFAULT_APPROVAL_EXPIRY_HOURS
+};
+/**
+* Namespace for the per-org advisory lock taken by `upsertLocal`, so the key
+* cannot collide with another feature's advisory lock in the same database.
+*/
+const UPSERT_LOCK_NAMESPACE = "org-trust-policy-upsert";
+var TrustPolicyStore = class {
+	db;
+	constructor(db) {
+		this.db = db;
+	}
+	/** `executor` lets a caller read inside its own transaction. */
+	async get(orgId, executor = this.db) {
+		const row = await executor.selectFrom("org_trust_policy").selectAll().where("customer_id", "=", orgId).executeTakeFirst();
+		if (!row) return null;
+		return {
+			forkPolicy: row.fork_policy,
+			unknownContributorPolicy: row.unknown_contributor_policy,
+			workflowChangePolicy: row.workflow_change_policy,
+			approvalExpiryHours: Number(row.approval_expiry_hours),
+			source: row.source,
+			updatedAt: row.updated_at
+		};
+	}
+	/**
+	* Needs no lock, unlike `upsertLocal`: a push carries every field, so there
+	* is no read-merge to lose and concurrent pushes correctly settle on the
+	* last one. The two writers also never overlap — `upsertLocal` is reachable
+	* only in independent mode, where no Platform pushes.
+	*/
+	async upsertFromPlatform(orgId, policy) {
+		await this.write(orgId, policy, TrustPolicySource.enum.platform);
+	}
+	/**
+	* Write an operator-supplied policy. Only reachable in independent mode — the
+	* admin route refuses on a Platform-attached orchestrator, because the next
+	* push would clobber the write.
+	*
+	* The read-merge-write is serialised per org by a transaction-scoped
+	* advisory lock. The transaction alone would NOT give that: under READ
+	* COMMITTED two concurrent PATCHes both read the pre-existing row, and the
+	* second `ON CONFLICT DO UPDATE` then overwrites every column from its own
+	* stale merge — silently dropping the first operator's change (a tightened
+	* `unknownContributorPolicy` reverting to whatever the second caller last
+	* saw). The lock releases on commit or rollback, so there is no unlock to
+	* leak.
+	*
+	* `onWrite` receives the same transaction and the merged result, so an audit
+	* row written there commits or rolls back with the policy itself — a
+	* loosened `forkPolicy` can never land unattributed.
+	*
+	* Returns the merged policy so the caller does not need a second read (which
+	* would be outside the transaction and could observe a later write).
+	*/
+	async upsertLocal(orgId, patch, onWrite) {
+		return await this.db.transaction().execute(async (trx) => {
+			await sql`SELECT pg_advisory_xact_lock(hashtext(${`${UPSERT_LOCK_NAMESPACE}|${orgId}`}))`.execute(trx);
+			const existing = await this.get(orgId, trx);
+			const merged = {
+				forkPolicy: patch.forkPolicy ?? existing?.forkPolicy ?? DEFAULT_TRUST_POLICY.forkPolicy,
+				unknownContributorPolicy: patch.unknownContributorPolicy ?? existing?.unknownContributorPolicy ?? DEFAULT_TRUST_POLICY.unknownContributorPolicy,
+				workflowChangePolicy: patch.workflowChangePolicy ?? existing?.workflowChangePolicy ?? DEFAULT_TRUST_POLICY.workflowChangePolicy,
+				approvalExpiryHours: patch.approvalExpiryHours ?? existing?.approvalExpiryHours ?? DEFAULT_TRUST_POLICY.approvalExpiryHours
+			};
+			await this.write(orgId, merged, TrustPolicySource.enum.local, trx);
+			await onWrite?.(trx, merged);
+			return merged;
+		});
+	}
+	async write(orgId, policy, source, executor = this.db) {
+		const columns = {
+			fork_policy: policy.forkPolicy,
+			unknown_contributor_policy: policy.unknownContributorPolicy,
+			workflow_change_policy: policy.workflowChangePolicy,
+			approval_expiry_hours: policy.approvalExpiryHours,
+			source,
+			updated_at: sql`now()`
+		};
+		await executor.insertInto("org_trust_policy").values({
+			customer_id: orgId,
+			...columns
+		}).onConflict((oc) => oc.column("customer_id").doUpdateSet(columns)).execute();
+	}
+};
+/**
+* Build a TrustPolicyStore backed by its own connection pool to the given
+* orchestrator database URL. Mirrors `createHeldRunStoreFromUrl`; consumed by
+* E2E tests that assert the pushed policy is persisted in the real deployed
+* orchestrator DB rather than held in process memory.
+*/
+function createTrustPolicyStoreFromUrl(databaseUrl, opts) {
+	const db = new Kysely({ dialect: new PostgresDialect({ pool: new pg.Pool({
+		connectionString: databaseUrl,
+		max: opts?.maxConnections ?? 3
+	}) }) });
+	return {
+		store: new TrustPolicyStore(db),
+		dispose: async () => {
+			await db.destroy();
+		}
+	};
+}
 //#endregion
 //#region src/cli/api-client.ts
 /**
@@ -1558,8 +2128,17 @@ var AdminApiClient = class {
 		const buf = Buffer.from(await res.arrayBuffer());
 		fs.writeFileSync(outPath, buf);
 	}
-	async listScopes(orgId) {
-		return this.request("GET", `/api/v1/admin/secrets/scopes?orgId=${encodeURIComponent(orgId)}`);
+	/**
+	* List secret scopes for an org.
+	*
+	* @param allBackends - Aggregate every registered backend and return scopes
+	*   in qualified `<backend>:<path>` form. Default (false) returns the bare,
+	*   pg-only listing; that default flips at v1.0.0.
+	*/
+	async listScopes(orgId, allBackends = false) {
+		const params = new URLSearchParams({ orgId });
+		if (allBackends) params.set("allBackends", "true");
+		return this.request("GET", `/api/v1/admin/secrets/scopes?${params}`);
 	}
 	async listKeys(orgId, scope) {
 		const params = new URLSearchParams({
@@ -1574,20 +2153,35 @@ var AdminApiClient = class {
 	async deleteSecret(orgId, scope, key) {
 		return this.request("DELETE", `/api/v1/admin/secrets/${encodeURIComponent(orgId)}/${encodeURIComponent(scope)}/${encodeURIComponent(key)}`);
 	}
-	async listVariables(orgId, environment) {
+	/**
+	* Create (or upsert) a deployment context. `allowLocalExecution` lets
+	* CLI-initiated / local runs resolve secrets through this context.
+	*/
+	async createContext(data) {
+		return this.request("POST", `/api/v1/admin/contexts`, data);
+	}
+	/**
+	* Bind a scope pattern to a context so its scoped secrets resolve at dispatch.
+	* `hostPattern` defaults to `**` (every host) server-side.
+	*/
+	async bindContext(data) {
+		const { name, ...body } = data;
+		return this.request("POST", `/api/v1/admin/contexts/${encodeURIComponent(name)}/bind`, body);
+	}
+	async listVariables(orgId, context) {
 		const params = new URLSearchParams({ orgId });
-		return this.request("GET", `/api/v1/admin/environments/${encodeURIComponent(environment)}/variables?${params}`);
+		return this.request("GET", `/api/v1/admin/contexts/${encodeURIComponent(context)}/variables?${params}`);
 	}
-	async setVariable(orgId, environment, key, value, locked) {
+	async setVariable(orgId, context, key, value, locked) {
 		const params = new URLSearchParams({ orgId });
-		return this.request("PUT", `/api/v1/admin/environments/${encodeURIComponent(environment)}/variables/${encodeURIComponent(key)}?${params}`, {
+		return this.request("PUT", `/api/v1/admin/contexts/${encodeURIComponent(context)}/variables/${encodeURIComponent(key)}?${params}`, {
 			value,
 			locked
 		});
 	}
-	async deleteVariable(orgId, environment, key) {
+	async deleteVariable(orgId, context, key) {
 		const params = new URLSearchParams({ orgId });
-		return this.request("DELETE", `/api/v1/admin/environments/${encodeURIComponent(environment)}/variables/${encodeURIComponent(key)}?${params}`);
+		return this.request("DELETE", `/api/v1/admin/contexts/${encodeURIComponent(context)}/variables/${encodeURIComponent(key)}?${params}`);
 	}
 	async rotateKey() {
 		return this.request("POST", "/api/v1/admin/rotate-key");
@@ -1702,6 +2296,14 @@ var AdminApiClient = class {
 	async configReload(opts) {
 		return this.request("POST", "/admin/config/reload", opts);
 	}
+	/** Flip the coordinator drain flag on ('drain') or off ('resume'). */
+	async drain(action) {
+		return this.request("POST", "/api/v1/admin/orchestrator/drain", { action });
+	}
+	/** Report the coordinator's current drain state without changing it. */
+	async drainStatus() {
+		return this.request("GET", "/api/v1/admin/orchestrator/drain");
+	}
 	async createApiKey(opts) {
 		return this.request("POST", "/api/v1/api-keys", opts);
 	}
@@ -2187,7 +2789,8 @@ async function resolveWebhookUrl(client) {
 */
 async function captureCode(opts, deps, manifestJson, state, redirectUrl, createUrl, loopback) {
 	if (opts.noBrowser) {
-		const url = `${STATIC_CALLBACK_URL}#m=${Buffer.from(manifestJson, "utf-8").toString("base64url")}&state=${encodeURIComponent(state)}&createUrl=${encodeURIComponent(createUrl)}`;
+		const m = Buffer.from(manifestJson, "utf-8").toString("base64url");
+		const url = `${STATIC_CALLBACK_URL}#m=${m}&state=${encodeURIComponent(state)}&createUrl=${encodeURIComponent(createUrl)}`;
 		console.log("Open this URL in a browser to create your App:");
 		console.log(`  ${url}`);
 		console.log("After clicking \"Create\", copy the setup code shown on the page.");
@@ -2272,6 +2875,6 @@ async function runGithubManifestSetup(opts, client, deps = realManifestSetupDeps
 	}
 }
 //#endregion
-export { AdminApiClient, DEFAULT_USER_CACHE_QUOTA_BYTES, DEFAULT_USER_CACHE_TTL_MS, DepCache, JoinTokenManager, PeerAuthCoordinator, PeerCredentialStore, S3CacheStorage, SourceCache, UserCache, createCacheStorage, createJoinTokenManagerFromUrl, createPeerCredentialStoreFromUrl, runGithubManifestSetup };
+export { AdminApiClient, DEFAULT_TRUST_POLICY, DEFAULT_USER_CACHE_QUOTA_BYTES, DEFAULT_USER_CACHE_TTL_MS, DepCache, HeldRunStore, JoinTokenManager, PeerAuthCoordinator, PeerCredentialStore, S3CacheStorage, SecurityHoldReason, SourceCache, TrustPolicySource, TrustPolicyStore, UserCache, createCacheStorage, createHeldRunStoreFromUrl, createJoinTokenManagerFromUrl, createPeerCredentialStoreFromUrl, createTrustPolicyStoreFromUrl, runGithubManifestSetup };
 
 //# sourceMappingURL=index.js.map