@tangle-network/agent-runtime 0.342.2 → 0.343.3

package/dist/durable.js

@@ -1,23 +1,22 @@
-import { i as publishExclusiveDurableFile, o as writeAtomicDurableFile } from "./durable-file-gNI9ILAn.js";
-import { Hr as authoredProfileDigest, S as FileSpawnJournal, Si as contentAddress, ar as isFiniteNumber, at as loadSpawnForest, ir as failedItems, xi as detachedSnapshot } from "./supervisor-C_03z693.js";
+import { a as syncDurableDirectory, i as publishExclusiveDurableFile, o as writeAtomicDurableFile } from "./durable-file-DLcrzrjL.js";
+import { Bi as assertValidBudget, C as FileSpawnJournal, Mi as contentAddress, Qr as authoredProfileDigest, _t as readCommittedJsonLines, ct as loadSpawnForest, hr as isFiniteNumber, ji as detachedSnapshot, mr as failedItems, mt as isNoEntError, pt as appendCommittedJsonLine, r as assertResumeLimits } from "./supervisor-D3M9ab1v.js";
 import { f as RuntimeRunStateError, m as ValidationError } from "./errors-DodWX-cb.js";
-import { t as isRecord } from "./record-value-B6bwJxLa.js";
-import { W as addSpend, X as errorMessage, ft as zeroSpend, q as cloneSpend } from "./sandbox-profile-compat-FN2kab2-.js";
+import { W as addSpend, X as errorMessage, ft as zeroSpend, q as cloneSpend } from "./sandbox-profile-compat-_fEtHPCj.js";
 import { a as withPursuitContext, t as composeRuntimeHooks } from "./runtime-hooks-tXpAarhW.js";
-import { a as readCommittedJsonLines, n as isNoEntError, t as appendCommittedJsonLine } from "./jsonl-file-C9Kde_0Z.js";
-import { J as readRootStreamReceipt, a as superviseRootProfile, q as readRootStream, t as supervise } from "./supervise-C_GAgiJU.js";
-import { a as applyExactAgentProfileDiff } from "./profile-CbMw5hAn.js";
-import { g as supervisorRunDir } from "./run-cancellation-C9oL62sW.js";
-import { i as superviseAgentGraph } from "./graph-BCghJFlS.js";
+import { a as applyExactAgentProfileDiff } from "./profile-6OLpbjvV.js";
+import { a as superviseRootProfile, ot as readRootStream, st as readRootStreamReceipt, t as supervise } from "./supervise-BuVVC7Bl.js";
+import { a as runCancelRequestFile, o as runCancellationFile, v as supervisorRunDir, x as workerCancellationsDir } from "./run-cancellation-CEDiqfhC.js";
+import { i as superviseAgentGraph } from "./graph-kgzTaNMA.js";
 import { agentProfileSchema, canonicalAgentProfileDigest, canonicalCandidateDigest, canonicalCandidateJson, sha256Bytes } from "@tangle-network/agent-interface";
+import { isRecord } from "@tangle-network/agent-core";
 import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync } from "node:fs";
 import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
-import { execFile } from "node:child_process";
-import { access, mkdir, readFile, realpath, rmdir, unlink } from "node:fs/promises";
-import { promisify } from "node:util";
+import { access, mkdir, open, readFile, realpath, rm, rmdir, unlink } from "node:fs/promises";
 import { SearchRecorder, developmentClaim, incumbent, openSearchLedger, parseSearchLedgerLine, runSearch, uniform } from "@tangle-network/agent-eval/campaign";
 import { pathToFileURL } from "node:url";
 import { setTimeout as setTimeout$1 } from "node:timers/promises";
+import { lockHolderVerdict, lockIdentity, parseLockHolderRecord, readLockHolderIdentity } from "@tangle-network/agent-eval/lock-holder";
 //#region src/durable/chat-engine.ts
 const encoder = new TextEncoder();
 function encodeLine(event) {
@@ -897,7 +896,7 @@ var SettledRunDirectoryError = class extends Error {
 	/** The root of the recorded run, which is its `runId`. */
 	recordedRunId;
 	constructor(path, recordedRunId, requestedRunId) {
-		super(recordedRunId === requestedRunId ? `supervisePursuit: ${path} records that run '${recordedRunId}' already settled; a settled run is not re-entered` : `supervisePursuit: ${path} records that run '${recordedRunId}' already settled in this directory; a directory holds one settle record, so run '${requestedRunId}' needs its own runDir`);
+		super(recordedRunId === requestedRunId ? `supervisePursuit: ${path} records that run '${recordedRunId}' already settled; a settled run is not re-entered unless the call passes reopen: { operator, reason, operationId }` : `supervisePursuit: ${path} records that run '${recordedRunId}' already settled in this directory; a directory holds one settle record, so run '${requestedRunId}' needs its own runDir`);
 		this.name = "SettledRunDirectoryError";
 		this.path = path;
 		this.recordedRunId = recordedRunId;
@@ -949,6 +948,10 @@ function jsonValue(result) {
 function settleRecordJson(result) {
 	return canonicalCandidateJson(jsonValue(result));
 }
+/** The digest the settle record's bytes hash to: the candidate digest of the result's JSON value. */
+function settleRecordDigest(result) {
+	return canonicalCandidateDigest(jsonValue(result));
+}
 /** Publish `result.json` in the run directory once. Throws when a record is already there. */
 async function writeSettleRecord(runDir, result) {
 	const path = resolve(runDir, SETTLE_RECORD_FILE);
@@ -2040,28 +2043,37 @@ var RunDirectoryLockedError = class extends Error {
 * Take `runDir/supervise.lock`, or refuse.
 *
 * The file is published with its full content or not at all, so a contender never reads a
-* half-written holder. A lock whose holder is gone (its pid no longer exists, or the pid now
-* belongs to a process with a different start token) is stale and is removed under the mutation guard;
-* a pid this process may not signal (`EPERM`) is alive and refuses. An empty file names no
-* holder and is reclaimed. A file with unreadable content is left in place and refused:
+* half-written holder. A lock whose holder is proved dead (its pid no longer exists, the pid now
+* belongs to a process with a different start token, or the host rebooted) is stale and is removed
+* under the mutation guard; a holder that is alive or cannot be proved dead refuses. An empty file
+* names no holder and is reclaimed. A file with unreadable content is left in place and refused:
 * reclaiming it could evict a live holder written by something other than this module.
 */
 async function acquireRunDirectoryLock(runDir, runId, now = Date.now) {
 	const dir = resolve(runDir);
 	const path = resolve(dir, RUN_DIRECTORY_LOCK_FILE);
 	await mkdir(dir, { recursive: true });
-	const processStart = await readProcessStart(process.pid);
-	const holder = Object.freeze({
-		pid: process.pid,
+	const { pid, ...identity } = readLockHolderIdentity(process.pid);
+	const owner = {
+		pid,
 		startedAt: new Date(now()).toISOString(),
 		runId,
-		...processStart === void 0 ? {} : { processStart }
-	});
+		...identity
+	};
 	return withMutationGuard(path, async () => {
 		const existing = await readLockFile(path);
-		if (existing.state === "holder" && await holderIsLive(existing.holder)) throw new RunDirectoryLockedError(path, existing.holder);
+		if (existing.state === "holder" && !holderIsDead(existing)) throw new RunDirectoryLockedError(path, existing.holder);
 		if (existing.state !== "missing") await removeIfPresent(path);
-		if (!publishExclusiveDurableFile(path, `${JSON.stringify(holder)}\n`)) throw new Error(`supervisePursuit: could not take ${path}`);
+		const written = {};
+		const published = publishExclusiveDurableFile(path, (file) => {
+			written.holder = Object.freeze({
+				...owner,
+				lock: lockIdentity(file)
+			});
+			return `${JSON.stringify(written.holder)}\n`;
+		});
+		const holder = written.holder;
+		if (!published || holder === void 0) throw new Error(`supervisePursuit: could not take ${path}`);
 		let released = false;
 		return Object.freeze({
 			...holder,
@@ -2098,47 +2110,21 @@ async function runDirectoryHolderIsLive(runDir) {
 	const existing = await readLockFile(resolve(runDir, RUN_DIRECTORY_LOCK_FILE));
 	if (existing.state !== "holder") return { live: false };
 	return {
-		live: await holderIsLive(existing.holder),
+		live: !holderIsDead(existing),
 		holder: existing.holder
 	};
 }
-const execFileAsync = promisify(execFile);
-/**
-* The OS's start token for a live pid: on Linux the `starttime` field of `/proc/<pid>/stat`
-* (clock ticks since boot), elsewhere the `ps` `lstart` column. The same process reports the
-* same token for its whole life and a later holder of the pid reports a different one.
-* `undefined` when the host cannot report one: the pid is gone, there is no `/proc` and no
-* `ps`, or the platform is Windows.
-*/
-async function readProcessStart(pid) {
-	if (!Number.isInteger(pid) || pid <= 0 || process.platform === "win32") return void 0;
-	if (process.platform === "linux") {
-		let stat;
-		try {
-			stat = await readFile(`/proc/${pid}/stat`, "utf8");
-		} catch {
-			return;
-		}
-		const startTime = stat.slice(stat.lastIndexOf(")") + 1).trim().split(/\s+/u)[19];
-		return startTime !== void 0 && /^\d+$/u.test(startTime) ? startTime : void 0;
-	}
-	try {
-		const { stdout } = await execFileAsync("ps", [
-			"-o",
-			"lstart=",
-			"-p",
-			String(pid)
-		]);
-		const token = stdout.trim();
-		return token.length === 0 ? void 0 : token;
-	} catch {
-		return;
-	}
-}
 async function readLockFile(path) {
 	let text;
+	let current;
 	try {
-		text = await readFile(path, "utf8");
+		const file = await open(path, "r");
+		try {
+			current = lockIdentity(await file.stat({ bigint: true }));
+			text = await file.readFile("utf8");
+		} finally {
+			await file.close();
+		}
 	} catch (error) {
 		if (isNoEntError(error)) return { state: "missing" };
 		throw error;
@@ -2153,30 +2139,24 @@ async function readLockFile(path) {
 	const record = parsed;
 	if (typeof record !== "object" || record === null || typeof record.pid !== "number" || !Number.isInteger(record.pid) || record.pid <= 0 || typeof record.startedAt !== "string" || typeof record.runId !== "string" || record.processStart !== void 0 && typeof record.processStart !== "string") throw new Error(`supervisePursuit: ${path} is not a readable lock file; remove it by hand`);
 	const { pid, startedAt, runId, processStart } = record;
+	const verdictRecord = parseLockHolderRecord(parsed);
+	const { pid: _pid, ...identity } = verdictRecord ?? { pid };
 	return {
 		state: "holder",
 		holder: Object.freeze({
 			pid,
 			startedAt,
 			runId,
-			...processStart === void 0 ? {} : { processStart }
-		})
+			...processStart === void 0 ? {} : { processStart },
+			...identity
+		}),
+		record: verdictRecord,
+		current
 	};
 }
-async function holderIsLive(holder) {
-	if (!processExists(holder.pid)) return false;
-	if (holder.processStart === void 0) return true;
-	const current = await readProcessStart(holder.pid);
-	return current === void 0 || current === holder.processStart;
-}
-function processExists(pid) {
-	if (!Number.isInteger(pid) || pid <= 0) return false;
-	try {
-		process.kill(pid, 0);
-		return true;
-	} catch (error) {
-		return error.code !== "ESRCH";
-	}
+/** agent-eval's dead-owner rule: only a holder proved dead is stale; alive or unknown stays held. */
+function holderIsDead(existing) {
+	return lockHolderVerdict(existing.record, existing.current).liveness === "dead";
 }
 /** Remove the lock only while it still names this holder; a reclaimed lock belongs to its new owner. */
 async function releaseHolder(path, holder) {
@@ -2322,6 +2302,119 @@ function startPursuitObserverDelivery(delivery, read, warn = (message) => consol
 	};
 }
 //#endregion
+//#region src/durable/run-reopen.ts
+/** The directory that holds a run's archived settle records. */
+const SETTLE_ARCHIVE_DIR = "settles";
+/** Refuse a reopen that names no operator or reason, or that is combined with a fork or versions. */
+function assertRunReopen(reopen, opts) {
+	if (reopen.operator.trim().length === 0) throw new TypeError("supervisePursuit: reopen.operator must be non-empty");
+	if (reopen.reason.trim().length === 0) throw new TypeError("supervisePursuit: reopen.reason must be non-empty");
+	if (reopen.operationId.trim().length === 0) throw new TypeError("supervisePursuit: reopen.operationId must be non-empty");
+	if (reopen.budget !== void 0) assertValidBudget(reopen.budget, "reopen.budget");
+	if (opts.fork !== void 0 || opts.versions !== void 0) throw new TypeError("supervisePursuit: reopen continues this run under its own id; fork and versions start new runs");
+}
+/**
+* Archive the settled segment of the run at `runDir` and clear it for a resume. Runs under the
+* directory lock. Throws when there is nothing to reopen, or when `operationId` already reopened a
+* segment that has since settled again.
+*/
+async function reopenSettledRun(input) {
+	const { runDir, runId, reopen, observer, pursuitId, now } = input;
+	const operationId = reopen.operationId.trim();
+	const settlePath = resolve(runDir, SETTLE_RECORD_FILE);
+	const journal = input.journal ?? new FileSpawnJournal(resolve(runDir, "spawn-journal.jsonl"));
+	const events = await journal.loadTree(runId) ?? [];
+	const reopened = events.filter((event) => event.kind === "reopened" && event.id === runId);
+	const mine = reopened.find((event) => event.operationId === operationId);
+	const settled = await readBytes(settlePath);
+	if (settled === void 0) {
+		if (mine !== void 0) return;
+		throw new Error(`supervisePursuit: ${runDir} holds no ${SETTLE_RECORD_FILE}; nothing to reopen; resume it`);
+	}
+	assertResumeLimits(events, {
+		runId,
+		budget: reopen.budget ?? input.limits.budget,
+		...input.limits.reservationPolicy === void 0 ? {} : { reservationPolicy: input.limits.reservationPolicy }
+	}, mine === void 0 ? reopen.budget : void 0);
+	const result = JSON.parse(settled);
+	const digest = settleRecordDigest(result);
+	if (mine !== void 0 && mine.prior.digest !== digest) throw new Error(`supervisePursuit: reopen '${operationId}' already reopened run '${runId}', which has settled again; reopen it under a new operationId`);
+	const archivedAs = mine?.prior.archivedAs ?? await archiveName(runDir, settled);
+	const archivePath = resolve(runDir, archivedAs);
+	if (!publishExclusiveDurableFile(archivePath, settled) && await readBytes(archivePath) !== settled) throw new Error(`supervisePursuit: ${archivePath} already holds a different settle record; reopen refused`);
+	const at = new Date(now()).toISOString();
+	if (mine === void 0) await journal.appendEvent(runId, {
+		kind: "reopened",
+		id: runId,
+		operator: reopen.operator,
+		reason: reopen.reason,
+		operationId,
+		prior: {
+			kind: String(result.kind),
+			...typeof result.reason === "string" ? { reason: result.reason } : {},
+			digest,
+			archivedAs
+		},
+		...reopen.budget === void 0 ? {} : { budget: reopen.budget },
+		seq: reopened.length,
+		at
+	});
+	const eventId = `${runId}:pursuit:reopened:${operationId}`;
+	if (!(await observer.read()).some((record) => record.event?.id === eventId)) await observer.appendEvent({
+		id: eventId,
+		pursuitId,
+		runId,
+		target: "agent.run",
+		phase: "event",
+		timestamp: now(),
+		payload: { reopened: {
+			operator: reopen.operator,
+			reason: reopen.reason,
+			archivedAs
+		} }
+	});
+	setCancellationAside(runDir, operationId);
+	await rm(settlePath);
+	syncDurableDirectory(runDir);
+}
+/** The next free `settles/<n>.json`, or the archive that already holds these bytes (a reopen that
+*  crashed after archiving and before its journal row). */
+async function archiveName(runDir, settled) {
+	const dir = resolve(runDir, SETTLE_ARCHIVE_DIR);
+	mkdirSync(dir, { recursive: true });
+	const numbers = readdirSync(dir).map((name) => /^(\d+)\.json$/u.exec(name)?.[1]).filter((n) => n !== void 0).map(Number).sort((a, b) => a - b);
+	for (const n of numbers) if (await readBytes(join(dir, `${n}.json`)) === settled) return `${SETTLE_ARCHIVE_DIR}/${n}.json`;
+	return `${SETTLE_ARCHIVE_DIR}/${(numbers.at(-1) ?? 0) + 1}.json`;
+}
+/** Move the settled run's cancel request and acknowledgement aside under the reopen's and the
+*  request's operation ids, so the reopened run accepts a new `cancelRun`. Idempotent: an absent
+*  file is skipped. A reopen's operation id names one segment, so a later segment cancelled under
+*  the same cancel operation id lands beside this one instead of over it. */
+function setCancellationAside(runDir, reopenOperationId) {
+	const requestFile = runCancelRequestFile(runDir);
+	const ackFile = runCancellationFile(runDir);
+	if (!existsSync(requestFile) && !existsSync(ackFile)) return;
+	const source = existsSync(requestFile) ? requestFile : ackFile;
+	const { operationId } = JSON.parse(readFileSync(source, "utf8"));
+	const reopenedDir = join(workerCancellationsDir(runDir), "reopened");
+	const aside = join(reopenedDir, encodeURIComponent(reopenOperationId), encodeURIComponent(operationId));
+	mkdirSync(aside, { recursive: true });
+	if (existsSync(ackFile)) renameSync(ackFile, join(aside, "run.json"));
+	if (existsSync(requestFile)) renameSync(requestFile, join(aside, "run.request.json"));
+	syncDurableDirectory(aside);
+	syncDurableDirectory(join(reopenedDir, encodeURIComponent(reopenOperationId)));
+	syncDurableDirectory(reopenedDir);
+	syncDurableDirectory(workerCancellationsDir(runDir));
+}
+async function readBytes(path) {
+	try {
+		return await readFile(path, "utf8");
+	} catch (error) {
+		if (error.code === "ENOENT") return void 0;
+		throw error;
+	}
+}
+//#endregion
 //#region src/durable/supervise-pursuit.ts
 /** A failed Runtime execution whose complete third-person projection was retained. */
 var SupervisePursuitError = class extends Error {
@@ -2359,16 +2452,33 @@ var SupervisePursuitError = class extends Error {
 */
 async function supervisePursuit(profile, task, opts) {
 	assertPursuitDirectory(opts, "supervisePursuit");
+	if (opts.reopen !== void 0) assertRunReopen(opts.reopen, opts);
 	if (opts.versions !== void 0) return runPursuitVersions(profile, task, opts, supervisePursuit);
-	const { pursuitId: _pursuitId, hooks: _hooks, fork, versions: _versions, observerDelivery: _observerDelivery, ...superviseOptions } = opts;
+	const { pursuitId: _pursuitId, hooks: _hooks, fork, versions: _versions, observerDelivery: _observerDelivery, reopen, ...superviseOptions } = opts;
 	const forked = fork === void 0 ? void 0 : await prepareRunFork(profile, task, superviseOptions, fork);
 	return runDurablePursuit(opts, ({ hooks, steerDir }) => supervise(forked?.profile ?? profile, task, {
 		...superviseOptions,
 		...forked === void 0 ? {} : { execution: forked.execution },
+		...reopen === void 0 ? {} : {
+			resume: true,
+			...reopen.budget === void 0 ? {} : { budget: reopen.budget }
+		},
 		steerDir,
 		retainedAtSettlement: "release",
 		hooks
-	}), (result) => result);
+	}), (result) => result, reopen === void 0 ? void 0 : (observer) => reopenSettledRun({
+		runDir: opts.runDir.trim(),
+		runId: opts.runId ?? "supervise",
+		reopen,
+		journal: opts.journal,
+		limits: {
+			budget: opts.budget,
+			...opts.reservationPolicy === void 0 ? {} : { reservationPolicy: opts.reservationPolicy }
+		},
+		observer,
+		pursuitId: opts.pursuitId.trim(),
+		now: opts.now ?? Date.now
+	}));
 }
 /**
 * One-call durable execution of an `AgentGraph`: the graph twin of {@link supervisePursuit}.
@@ -2404,7 +2514,7 @@ function assertPursuitDirectory(opts, entry) {
 * journal, failure record and settle record. `execute` receives the observer-composed hooks and
 * the steer directory the run's control plane uses; `settled` names the value `result.json` holds.
 */
-async function runDurablePursuit(opts, execute, settled) {
+async function runDurablePursuit(opts, execute, settled, reopen) {
 	const pursuitId = opts.pursuitId.trim();
 	const runDir = opts.runDir.trim();
 	const observerPath = resolve(runDir, "observer.jsonl");
@@ -2416,9 +2526,12 @@ async function runDurablePursuit(opts, execute, settled) {
 	let delivery;
 	let terminalState = "failed";
 	try {
-		const prior = await readSettleRecord(runDir);
-		if (prior !== void 0) throw new SettledRunDirectoryError(settlePath, prior.tree.root, runId);
 		const observer = createFileObserverHooks(observerPath, pursuitId);
+		if (reopen !== void 0) await reopen(observer.journal);
+		else {
+			const prior = await readSettleRecord(runDir);
+			if (prior !== void 0) throw new SettledRunDirectoryError(settlePath, prior.tree.root, runId);
+		}
 		await observer.journal.appendEvent(rootEvent(pursuitId, runId, "before", now()));
 		delivery = opts.observerDelivery === void 0 ? void 0 : startPursuitObserverDelivery(opts.observerDelivery, async () => projectPursuit(await observer.journal.read()));
 		let result;