@cmmd-center/forge 0.13.73 → 0.13.74

package/dist/bin.cjs

@@ -66275,7 +66275,7 @@ function normalizeNumberish(value) {
 }
 //#endregion
 //#region package.json
-var version$1 = "0.13.73";
+var version$1 = "0.13.74";
 //#endregion
 //#region src/sentry.ts
 const SERVER_APP_NAME = "forge-server";
@@ -241692,17 +241692,30 @@ function isPromiseLike(value) {
 *    outlives the delete re-acquires the hold on its next tick; that was
 *    previously measured leaking 37 renewals over 18 minutes, pinning a sprite
 *    running — and billing — 33 minutes after its work had ended.
-* 2. Stopping the interval only cancels FUTURE ticks. A PUT already on the wire
-*    must be JOINED before the DELETE, or it lands afterwards and recreates the
-*    hold for another full expiry window. fly.io's reference worker cancels and
-*    awaits its heartbeat for the same reason.
+* 2. Cancellation must be CONFIRMED before the final DELETE is trusted. A PUT
+*    already on the wire has to settle first, or the DELETE can land mid-PUT
+*    and the PUT recreate the hold for another full expiry window; fly.io's
+*    reference worker cancels and awaits its heartbeat for the same reason.
+*    The join is bounded on the shutdown path
+*    (SPRITE_TASK_HOLD_RELEASE_JOIN_TIMEOUT_MS) and the bound decides the
+*    shape: every PUT settled within it, with no earlier PUT rejection left
+*    unanswered by a clean refresh, confirms cancellation and one DELETE
+*    finishes the hold; the bound firing first means cancellation cannot be
+*    confirmed, so the release sweeps, DELETE, verify with a bounded GET, and
+*    re-DELETE while the server still shows the hold, capped at
+*    SPRITE_TASK_HOLD_SWEEP_ROUNDS rounds. Every hold gets at least one
+*    DELETE, a resurrected hold is cleaned and swept again, and only a hold
+*    positively present after the cap is reported unreleased at warn instead
+*    of hanging SIGTERM open.
 * 3. `release` can arrive while `acquire` is still awaiting its own PUT —
 *    `releaseAll` runs from runtime drain and the shutdown finalizer,
 *    concurrently with the provider event stream. The hold is registered before
-*    that await (so a duplicate turn.started cannot double-acquire), so release
-*    can remove the map entry first; `acquire` must then NOT install a
-*    refresher. One attached to an unmapped hold is unreachable by release,
-*    releaseThread and releaseAll alike, and re-PUTs every 60 s for the life of
+*    that await (so a duplicate turn.started cannot double-acquire), and while
+*    a release runs the hold stays mapped (joining, then deleting), so the
+*    shutdown finalizer's bounded `releaseAll` can take a parked hold over;
+*    `acquire` must then NOT install a refresher on the old hold, and a fresh
+*    acquire replaces a released hold in the map. One refresher attached to a
+*    hold no release path can reach would re-PUT every 60 s for the life of
 *    the process — an unbounded pin.
 *
 * All three are pinned by tests.
@@ -241726,6 +241739,30 @@ const SPRITE_TASK_HOLD_REFRESH_MS = 6e4;
 */
 const SPRITE_TASK_HOLD_REQUEST_TIMEOUT_MS = 1e4;
 /**
+* Shutdown bounds for the release the server finalizer issues. Both are one
+* request deadline. With a transport that honors its abort, every PUT still on
+* the wire settles within its own 10 s deadline, so a join sized at one
+* deadline always completes, cancellation is confirmed, and a single DELETE is
+* issued; the DELETE answers within its own deadline too.
+*
+* When the join bound fires, cancellation is unconfirmed and the release
+* sweeps: DELETE, verify with a bounded GET, and re-DELETE if the hold
+* survived, capped at SPRITE_TASK_HOLD_SWEEP_ROUNDS rounds. Worst case a
+* shutdown release costs join + 2 x (delete + verify), about 50 s, after
+* which the process exits and any hold left behind expires on its own within
+* 5 minutes. Drain passes no bounds. It keeps the full join, and its tick is
+* already capped by the heartbeat timeout.
+*/
+const SPRITE_TASK_HOLD_RELEASE_JOIN_TIMEOUT_MS = SPRITE_TASK_HOLD_REQUEST_TIMEOUT_MS;
+const SPRITE_TASK_HOLD_RELEASE_DELETE_TIMEOUT_MS = SPRITE_TASK_HOLD_REQUEST_TIMEOUT_MS;
+/**
+* How many delete-plus-verify rounds an unconfirmed release sweeps before it
+* gives up and reports the hold unreleased. Two is one retry past the first
+* delete: enough to clean a timed-out PUT that commits late, cheap enough
+* that a truly wedged socket cannot hold SIGTERM open much past a minute.
+*/
+const SPRITE_TASK_HOLD_SWEEP_ROUNDS = 2;
+/**
 * Resolve the hold configuration from the process environment.
 *
 * Returns null on every runtime that is not a project sprite — desktop-local and
@@ -241747,6 +241784,20 @@ function resolveSpriteTaskHoldConfig(env = process.env) {
 function spriteTaskHoldName(namePrefix, turnId) {
 	return `${namePrefix}${turnId}`;
 }
+const noTimer = () => {};
+/**
+* Resolve with `promise`, but stop waiting after `timeoutMs`. Rejections pass
+* through untouched, and the caller decides what a timeout means. The
+* underlying operation keeps running either way.
+*/
+function boundedAwait(promise, timeoutMs) {
+	if (timeoutMs === void 0) return promise;
+	let clearTimer = noTimer;
+	return Promise.race([promise, new Promise((resolve) => {
+		const timer = setTimeout(resolve, timeoutMs);
+		clearTimer = () => clearTimeout(timer);
+	})]).finally(() => clearTimer());
+}
 const DEFAULT_TIMER = {
 	setInterval: (handler, ms) => setInterval(handler, ms),
 	clearInterval: (handle) => {
@@ -241782,7 +241833,8 @@ var SpriteTaskHoldManager = class {
 	* whose turn may never start. When the budget is spent the hold releases itself.
 	*/
 	async acquire(turnId, threadId = null, options = {}) {
-		if (this.#holds.has(turnId)) return;
+		const existing = this.#holds.get(turnId);
+		if (existing !== void 0 && !existing.released) return;
 		let refreshesLeft = options.maxRefreshes ?? Number.POSITIVE_INFINITY;
 		const name = spriteTaskHoldName(this.#config.namePrefix, turnId);
 		const hold = {
@@ -241790,15 +241842,21 @@ var SpriteTaskHoldManager = class {
 			threadId,
 			refresher: null,
 			inFlight: /* @__PURE__ */ new Set(),
-			released: false
+			released: false,
+			phase: "idle",
+			releaseAttempt: null,
+			deletion: null,
+			uncertainPut: false
 		};
 		this.#holds.set(turnId, hold);
 		const put = this.#transport.put(name, 300);
-		const tracked = put.then(() => {}, () => {});
+		const tracked = put.then(() => "succeeded", () => "failed");
 		hold.inFlight.add(tracked);
 		try {
 			await put;
+			hold.uncertainPut = false;
 		} catch (cause) {
+			hold.uncertainPut = true;
 			hold.inFlight.delete(tracked);
 			this.#onError(`failed to acquire sprite task hold for turn ${turnId}`, cause);
 			if (hold.released || this.#holds.get(turnId) !== hold) return;
@@ -241812,28 +241870,189 @@ var SpriteTaskHoldManager = class {
 				return;
 			}
 			refreshesLeft -= 1;
-			const refresh = this.#transport.put(name, 300).catch((cause) => this.#onError(`failed to refresh sprite task hold for turn ${turnId}`, cause));
+			const refresh = this.#transport.put(name, 300).then(() => {
+				hold.uncertainPut = false;
+				return "succeeded";
+			}, (cause) => {
+				hold.uncertainPut = true;
+				this.#onError(`failed to refresh sprite task hold for turn ${turnId}`, cause);
+				return "failed";
+			});
 			hold.inFlight.add(refresh);
 			refresh.finally(() => hold.inFlight.delete(refresh));
 		}, SPRITE_TASK_HOLD_REFRESH_MS);
 	}
-	async release(turnId) {
+	/**
+	* Release one hold.
+	*
+	* Cancellation must be confirmed before a single DELETE is trusted: every
+	* PUT on the wire has to settle successfully, or the DELETE can land mid-PUT
+	* and the PUT re-create the hold behind it. With a join bound (shutdown),
+	* PUTs that do not settle within the bound, or settle as rejections, make
+	* the attempt sweep instead: DELETE, verify, re-DELETE while the server
+	* still shows the hold, capped at two rounds. The fate outlives the PUT's
+	* settlement: a PUT that rejected before the release began, with no
+	* successful refresh after it, sweeps too. Without a bound (drain,
+	* per-turn release) the join is patient and never gives up, though a PUT
+	* that rejects still routes the release through the sweep.
+	*
+	* While an attempt runs, the hold stays in the map in its joining or
+	* deleting phase, so shutdown can see it. A bounded (shutdown) caller takes
+	* a parked hold over with its own bounded pass; a hold already in its
+	* deleting phase is skipped, because its single deletion is already in
+	* flight. Unbounded duplicate callers join the running attempt instead.
+	*/
+	async release(turnId, options = {}) {
 		const hold = this.#holds.get(turnId);
-		if (!hold) return;
+		if (!hold) return "released";
+		if (hold.phase === "deleting") return "released";
+		const bounded = options.joinTimeoutMs !== void 0;
+		if (hold.releaseAttempt !== null) {
+			if (!bounded) return hold.releaseAttempt;
+			return this.#concludeRelease(hold, turnId, options);
+		}
 		hold.released = true;
 		if (hold.refresher !== null) {
 			this.#timer.clearInterval(hold.refresher);
 			hold.refresher = null;
 		}
-		this.#holds.delete(turnId);
-		if (hold.inFlight.size > 0) await Promise.all(hold.inFlight);
+		hold.phase = "joining";
+		const attempt = (async () => {
+			const outcome = await this.#concludeRelease(hold, turnId, options);
+			if (outcome === "unreleased") {
+				hold.phase = "idle";
+				this.#removeIfCurrent(turnId, hold);
+			}
+			return outcome;
+		})();
+		hold.releaseAttempt = attempt;
 		try {
-			await this.#transport.delete(hold.name);
+			return await attempt;
+		} finally {
+			if (hold.releaseAttempt === attempt) hold.releaseAttempt = null;
+		}
+	}
+	/**
+	* Drive one release attempt to its outcome. On the confirmed path (every
+	* PUT on the wire settled within the join bound) a single DELETE finishes
+	* the hold. On the unconfirmed path cancellation is unknown, so the hold
+	* is swept: DELETE, verify, and re-DELETE while the server still shows it,
+	* capped. Shared by the first attempt and a shutdown takeover: whichever
+	* caller first reaches its verdict installs the hold's single deletion
+	* promise and the rest join it, so the DELETE or sweep is issued once and
+	* concurrent callers report the same outcome.
+	*/
+	async #concludeRelease(hold, turnId, options) {
+		const verdict = await this.#confirmCancellation(hold, options.joinTimeoutMs);
+		if (hold.deletion !== null) return hold.deletion;
+		hold.phase = "deleting";
+		const deletion = verdict === "confirmed" ? this.#deleteConfirmed(hold, turnId, options) : this.#sweepUnconfirmed(hold, turnId, options);
+		hold.deletion = deletion;
+		return deletion;
+	}
+	/**
+	* The confirmed-path deletion: every PUT on the wire settled cleanly, so
+	* one DELETE finishes the hold. A rejected DELETE is logged, never thrown;
+	* the hold still leaves the map and expires on its own server-side.
+	*/
+	async #deleteConfirmed(hold, turnId, options) {
+		await this.#issueDelete(hold, options.deleteTimeoutMs, turnId);
+		this.#removeIfCurrent(turnId, hold);
+		return "released";
+	}
+	/**
+	* Best-effort cleanup for a hold whose cancellation is uncertain: the join
+	* bound fired with a PUT unsettled, or a PUT rejected after possibly being
+	* processed server-side. The hold is owed its delete either way. Each round
+	* is a bounded DELETE followed by a bounded GET; a verify that still shows
+	* the hold (the timed-out PUT committed after our delete, or the delete
+	* never landed) triggers the next round. Verification is best-effort too:
+	* "absent" proves cleanup, "unknown" proves nothing and cannot by itself
+	* report the hold unreleased, so after the cap an unverifiable hold is
+	* reported released-with-its-delete while a positively present one is
+	* reported unreleased at warn. Either way the hold expires on its own
+	* within 300 s. Only `#concludeRelease` starts this, after it has already
+	* moved the hold into its deleting phase.
+	*/
+	async #sweepUnconfirmed(hold, turnId, options) {
+		for (let round = 1; round <= SPRITE_TASK_HOLD_SWEEP_ROUNDS; round += 1) {
+			await this.#issueDelete(hold, options.deleteTimeoutMs, turnId);
+			const state = await this.#verifyHold(hold, options.deleteTimeoutMs, turnId);
+			const last = round === SPRITE_TASK_HOLD_SWEEP_ROUNDS;
+			if (state === "absent") {
+				this.#removeIfCurrent(turnId, hold);
+				return "released";
+			}
+			if (state === "present") {
+				this.#onError(`sprite task hold for turn ${turnId} was still present after delete (sweep ${round}/${SPRITE_TASK_HOLD_SWEEP_ROUNDS})`, "a refresh PUT that ignored its abort appears to have committed after the delete");
+				if (last) {
+					this.#onError(`sprite task hold for turn ${turnId} was left unreleased: still present after ${SPRITE_TASK_HOLD_SWEEP_ROUNDS} delete+verify sweeps, expires on its own`, "hold survived the full shutdown sweep");
+					this.#removeIfCurrent(turnId, hold);
+					return "unreleased";
+				}
+				continue;
+			}
+			this.#onError(`sprite task hold for turn ${turnId} could not be verified after delete (sweep ${round}/${SPRITE_TASK_HOLD_SWEEP_ROUNDS})`, "hold state unverifiable");
+			if (last) {
+				this.#removeIfCurrent(turnId, hold);
+				return "released";
+			}
+		}
+		return "released";
+	}
+	/**
+	* Ask the transport whether the hold still exists. No transport `get`, a
+	* rejection, or a bound that fires all come back "unknown": the sweep has
+	* no evidence, and never invents it.
+	*/
+	async #verifyHold(hold, timeoutMs, turnId) {
+		const get = this.#transport.get;
+		if (!get) return "unknown";
+		try {
+			return await boundedAwait(get(hold.name), timeoutMs);
+		} catch (cause) {
+			this.#onError(`failed to verify sprite task hold for turn ${turnId}`, cause);
+			return "unknown";
+		}
+	}
+	/**
+	* Wait for every PUT on the wire to settle and report whether cancellation
+	* is confirmed. Only a clean success confirms: a PUT that REJECTED is
+	* uncertain, because the server may have processed the request before the
+	* failure surfaced, and a DELETE trusted after it could be raced by the
+	* hold it re-created. `inFlight` forgets each PUT once it settles, so a
+	* rejection that settled before the release even started is read from
+	* `uncertainPut` instead, and stays uncertain until some later PUT
+	* succeeds. With no bound the wait is patient; unbounded callers
+	* (drain, per-turn release) trade shutdown speed for the certainty.
+	*/
+	async #confirmCancellation(hold, timeoutMs) {
+		if (hold.inFlight.size === 0) return hold.uncertainPut ? "uncertain" : "confirmed";
+		const settled = Promise.all(hold.inFlight).then((fates) => fates.every((fate) => fate === "succeeded") && !hold.uncertainPut ? "confirmed" : "uncertain");
+		if (timeoutMs === void 0) return settled;
+		let clearTimer = noTimer;
+		return Promise.race([settled, new Promise((resolve) => {
+			const timer = setTimeout(() => resolve("uncertain"), timeoutMs);
+			clearTimer = () => clearTimeout(timer);
+		})]).finally(() => clearTimer());
+	}
+	/** Issue the DELETE, reporting (never throwing) a failure. */
+	async #issueDelete(hold, deleteTimeoutMs, turnId) {
+		try {
+			await boundedAwait(this.#transport.delete(hold.name), deleteTimeoutMs);
 		} catch (cause) {
 			this.#onError(`failed to release sprite task hold for turn ${turnId}`, cause);
 		}
 	}
 	/**
+	* Drop the map entry only if this attempt's hold still owns the id: a newer
+	* acquire may have replaced it while the release was awaiting, and deleting
+	* by key alone would evict the replacement.
+	*/
+	#removeIfCurrent(turnId, hold) {
+		if (this.#holds.get(turnId) === hold) this.#holds.delete(turnId);
+	}
+	/**
 	* Release every hold belonging to one thread.
 	*
 	* `session.exited` names only the thread whose provider process died, and a
@@ -241848,9 +242067,22 @@ var SpriteTaskHoldManager = class {
 	/**
 	* Release everything. Runtime drain and server shutdown must call this, or a
 	* drained sprite keeps billing until each hold expires.
+	*
+	* Shutdown passes `SPRITE_TASK_HOLD_RELEASE_JOIN_TIMEOUT_MS` /
+	* `SPRITE_TASK_HOLD_RELEASE_DELETE_TIMEOUT_MS` so a socket that never
+	* settles cannot hang the process open past its own termination, and gets a
+	* per-hold report back: every hold gets its DELETE, and only holds the
+	* sweep still saw present after the full delete+verify cap come back
+	* unreleased, left to their own 300 s expiry. Drain passes no bounds and
+	* keeps the patient join.
 	*/
-	async releaseAll() {
-		await Promise.all([...this.#holds.keys()].map((turnId) => this.release(turnId)));
+	async releaseAll(options = {}) {
+		const turnIds = [...this.#holds.keys()];
+		const outcomes = await Promise.all(turnIds.map(async (turnId) => [turnId, await this.release(turnId, options)]));
+		return {
+			released: outcomes.filter(([, outcome]) => outcome === "released").map(([turnId]) => turnId),
+			unreleased: outcomes.filter(([, outcome]) => outcome === "unreleased").map(([turnId]) => turnId)
+		};
 	}
 };
 /**
@@ -241881,7 +242113,25 @@ function makeSpriteTaskHoldTransport(config, fetchImpl = fetch) {
 			headers: { "content-type": "application/json" },
 			body: JSON.stringify({ expire: `${expireSeconds}s` })
 		}),
-		delete: (name) => call(`/${encodeURIComponent(name)}`, { method: "DELETE" })
+		delete: (name) => call(`/${encodeURIComponent(name)}`, { method: "DELETE" }),
+		get: async (name) => {
+			const controller = new AbortController();
+			const deadline = setTimeout(() => controller.abort(), SPRITE_TASK_HOLD_REQUEST_TIMEOUT_MS);
+			try {
+				const response = await fetchImpl(`http://sprite/v1/tasks/${encodeURIComponent(name)}`, {
+					method: "GET",
+					signal: controller.signal,
+					unix: config.socketPath
+				});
+				if (response.ok) return "present";
+				if (response.status === 404) return "absent";
+				return "unknown";
+			} catch {
+				return "unknown";
+			} finally {
+				clearTimeout(deadline);
+			}
+		}
 	};
 }
 /**
@@ -286476,10 +286726,10 @@ const AgentBrowserManagerLive = require_Schema$1.effect(BrowserManager, require_
 		mediaSourceFactory: displayCaptureRuntime.mediaSourceFactory,
 		taskHold: displayTaskHoldManager ? {
 			acquire: (holdId) => displayTaskHoldManager.acquire(holdId),
-			release: (holdId) => displayTaskHoldManager.release(holdId)
+			release: (holdId) => displayTaskHoldManager.release(holdId).then(() => {})
 		} : void 0
 	});
-	yield* require_Schema$1.addFinalizer(() => require_Schema$1.promise(() => displayTaskHoldManager?.releaseAll() ?? Promise.resolve()));
+	yield* require_Schema$1.addFinalizer(() => require_Schema$1.promise(() => displayTaskHoldManager?.releaseAll().then(() => {}) ?? Promise.resolve()));
 	const baseBackendOptions = displayCaptureRuntime.backendOptions;
 	const sessionInitialUrls = /* @__PURE__ */ new Map();
 	let runtimeBuild = null;
@@ -294456,6 +294706,19 @@ const make$6 = require_Schema$1.gen(function* () {
 	const manager = makeSpriteTaskHoldManager(process.env, (message, cause) => {
 		require_Schema$1.runFork(require_Schema$1.logWarning(message, { cause: cause instanceof Error ? cause.message : cause }));
 	});
+	/**
+	* Run a manager call, or nothing on runtimes that are not sprites. One
+	* null-guard for every caller: the releaseAll service and the shutdown
+	* finalizer alike.
+	*/
+	const onManager = (run) => require_Schema$1.promise(async () => {
+		if (manager === null) return;
+		await run(manager);
+	});
+	yield* require_Schema$1.addFinalizer(() => onManager((taskHolds) => taskHolds.releaseAll({
+		joinTimeoutMs: SPRITE_TASK_HOLD_RELEASE_JOIN_TIMEOUT_MS,
+		deleteTimeoutMs: SPRITE_TASK_HOLD_RELEASE_DELETE_TIMEOUT_MS
+	})));
 	const launchCallers = /* @__PURE__ */ new Map();
 	const start = require_Schema$1.fn("start")(function* () {
 		if (manager === null) return;
@@ -294472,12 +294735,12 @@ const make$6 = require_Schema$1.gen(function* () {
 			if (event.type === "turn.completed") {
 				const turnId = event.turnId;
 				if (turnId === void 0) return require_Schema$1.void_;
-				return require_Schema$1.promise(() => manager.release(String(turnId)));
+				return require_Schema$1.promise(() => manager.release(String(turnId)).then(() => {}));
 			}
 			if (event.type === "turn.aborted") {
 				const turnId = event.turnId;
 				if (turnId === void 0) return require_Schema$1.void_;
-				return require_Schema$1.promise(() => manager.release(String(turnId)));
+				return require_Schema$1.promise(() => manager.release(String(turnId)).then(() => {}));
 			}
 			if (event.type === "session.exited") {
 				launchCallers.delete(String(event.threadId));
@@ -294504,7 +294767,7 @@ const make$6 = require_Schema$1.gen(function* () {
 		start,
 		holdLaunch,
 		releaseLaunch,
-		releaseAll: require_Schema$1.promise(() => manager === null ? Promise.resolve() : manager.releaseAll())
+		releaseAll: onManager((taskHolds) => taskHolds.releaseAll())
 	};
 });
 const SpriteTaskHoldReactorLive = require_Schema$1.effect(SpriteTaskHoldReactor, make$6);
@@ -298649,7 +298912,7 @@ function resolveBuildCommitFromEnv(env) {
 * environment descriptor down instead of reporting an honest "unknown".
 */
 function readBakedBuildCommit() {
-	return "9beb25327a2591f7938669b9014396e64c2bb674";
+	return "d1c5c1c87c650d1d49b596cbbe4bcc3ff14c7bfe";
 }
 async function resolveServerBuildCommit(input) {
 	if (isFullCommitSha(input.baked)) return input.baked;