@themoltnet/agent-daemon 0.24.0 → 0.25.1

package/README.md

@@ -179,14 +179,13 @@ This matters for evals in particular. `run_eval` tasks declare their intended
 workspace shape in `input.execution.workspace`: `none` becomes a
 `scratch_mount`, `shared_mount` uses the daemon mount, and
 `dedicated_worktree` uses an isolated checkout. Downstream
-`judge_eval_attempt` tasks only resolve against a still-live producer
-session/workspace slot. If the producer slot or its local session/workspace
-files cannot be resolved, the judge fails with `producer_context_missing`.
-When the judge does claim in time, it immediately forks the producer session and copies the
-producer workspace into judge-owned scratch state so the running judge no
-longer depends on the producer slot after claim time. Repo-specific
-`resumeCommands` that should not run in scratch mode must still be guarded
-with `when.workspaceMode`.
+`judge_eval_attempt` tasks can hydrate the producer Pi session from durable
+runtime-session storage when producer slot/workspace metadata is available but
+the local session file is unavailable. Workspace copying still depends on
+producer slot/workspace metadata; if the daemon cannot resolve the required
+producer context, the judge fails with `producer_context_missing`.
+Repo-specific `resumeCommands` that should not run in scratch mode must still
+be guarded with `when.workspaceMode`.
 
 ### 1. Start the local stack
 

package/dist/main.js

@@ -12,7 +12,7 @@ import { parseArgs, promisify } from "node:util";
 import { AgentRuntime, ApiTaskReporter, ApiTaskSource, PollingApiTaskSource } from "@themoltnet/agent-runtime";
 import { createPiTaskExecutor, findMainWorktree } from "@themoltnet/pi-extension";
 import { execFile, execFileSync } from "node:child_process";
-import { accessSync, constants, existsSync, mkdirSync, readdirSync } from "node:fs";
+import { accessSync, constants, createReadStream, createWriteStream, existsSync, mkdirSync, readdirSync } from "node:fs";
 import { MoltNetError, connect } from "@themoltnet/sdk";
 import { once } from "node:events";
 import { pino, transport } from "pino";
@@ -22,6 +22,8 @@ import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base";
 import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
 import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
 import { AsyncLocalStorage } from "node:async_hooks";
+import { mkdir } from "node:fs/promises";
+import { pipeline } from "node:stream/promises";
 //#region ../../libs/observability/src/instrumentation.ts
 /**
 * Register OTel auto-instrumentation for common Node.js modules.
@@ -4569,6 +4571,64 @@ _Object_({
 	additionalProperties: false
 });
 //#endregion
+//#region ../../libs/tasks/src/runtime-sessions.ts
+var RuntimeSessionKind = Union([
+	Literal("root"),
+	Literal("extend"),
+	Literal("fork")
+]);
+var RuntimeSessionCheckpointKind = Union([Literal("attempt_final")]);
+_Object_({
+	id: String$1({ format: "uuid" }),
+	teamId: String$1({ format: "uuid" }),
+	taskId: String$1({ format: "uuid" }),
+	attemptN: Integer({ minimum: 1 }),
+	sourceSlotId: Union([String$1({ format: "uuid" }), Null()]),
+	sourceRuntimeProfileId: Union([String$1({ format: "uuid" }), Null()]),
+	sessionKind: RuntimeSessionKind,
+	parentSessionId: Union([String$1({ format: "uuid" }), Null()]),
+	contentType: String$1({
+		minLength: 1,
+		maxLength: 200
+	}),
+	contentEncoding: Union([String$1({
+		minLength: 1,
+		maxLength: 100
+	}), Null()]),
+	sizeBytes: Integer({ minimum: 0 }),
+	sha256: String$1({
+		minLength: 64,
+		maxLength: 64
+	}),
+	storageClass: String$1({
+		minLength: 1,
+		maxLength: 100
+	}),
+	checkpointKind: RuntimeSessionCheckpointKind,
+	uploadedAt: String$1({ format: "date-time" })
+}, { $id: "RuntimeSession" });
+_Object_({
+	sourceSlotId: Optional(String$1({ format: "uuid" })),
+	sourceRuntimeProfileId: Optional(String$1({ format: "uuid" })),
+	sessionKind: RuntimeSessionKind,
+	parentSessionId: Optional(String$1({ format: "uuid" }))
+}, {
+	$id: "UploadRuntimeSessionQuery",
+	additionalProperties: false
+});
+String$1({
+	$id: "RuntimeSessionContent",
+	description: "Runtime session content stream.",
+	format: "binary"
+});
+_Object_({
+	taskId: String$1({ format: "uuid" }),
+	attemptN: Integer({ minimum: 1 })
+}, {
+	$id: "RuntimeSessionAttemptParams",
+	additionalProperties: false
+});
+//#endregion
 //#region ../../libs/tasks/src/runtime-slots.ts
 var RuntimeWorkspaceKind = Union([
 	Literal("origin"),
@@ -9572,12 +9632,13 @@ var ProducerContextResolutionError = class extends Error {
 };
 function createExecutionPlanCache(args) {
 	const cache = /* @__PURE__ */ new Map();
+	const runtimeSessionStore = args.runtimeSessionStore ?? createNullRuntimeSessionStore();
 	return {
 		async getOrCreate(claimedTask) {
 			const key = buildClaimedTaskKey(claimedTask);
 			const existing = cache.get(key);
 			if (existing) return existing;
-			const plan = await maybeAttachWarmSlotContext(claimedTask, buildDaemonTaskExecutionPlan(claimedTask.task, args.stateDirs, args.slotIdentity, args.warmSessionTtlSec, args.workspacePolicy), args.stateDirs, args.slotRegistry);
+			const plan = await maybeAttachWarmSlotContext(claimedTask, buildDaemonTaskExecutionPlan(claimedTask.task, args.stateDirs, args.slotIdentity, args.warmSessionTtlSec, args.workspacePolicy), args.stateDirs, args.slotRegistry, runtimeSessionStore);
 			assertPlanAllowedByWorkspacePolicy(plan, args.workspacePolicy);
 			cache.set(key, plan);
 			return plan;
@@ -9587,6 +9648,17 @@ function createExecutionPlanCache(args) {
 		}
 	};
 }
+function createNullRuntimeSessionStore() {
+	return {
+		async findRuntimeSessionByTaskAttempt() {
+			return null;
+		},
+		async hydrateSession() {
+			throw new ProducerContextResolutionError("Cannot hydrate runtime session: no runtime session store configured");
+		},
+		async uploadAttemptFinal() {}
+	};
+}
 function assertPlanAllowedByWorkspacePolicy(plan, policy) {
 	const allowed = new Set(policy?.allowedWorkspaceModes && policy.allowedWorkspaceModes.length > 0 ? policy.allowedWorkspaceModes : [
 		"none",
@@ -9604,10 +9676,17 @@ function planToRuntimeProfileWorkspaceMode(plan) {
 function buildClaimedTaskKey(task) {
 	return `${task.task.id}:${task.attemptN}`;
 }
-async function resolveWarmSlot(slotRegistry, teamId, sourceTaskId, sourceAttemptN, stateDirs) {
+async function resolveWarmSlot(slotRegistry, runtimeSessionStore, teamId, sourceTaskId, sourceAttemptN, stateDirs) {
 	const producerContext = await slotRegistry.findLatestSlotByTaskAttempt(teamId, sourceTaskId, sourceAttemptN);
 	if (!producerContext) return { kind: "missing" };
-	const sourceSessionPath = resolveProducerSessionPath(producerContext);
+	const localSessionPath = resolveProducerSessionPath(producerContext);
+	const remoteSession = localSessionPath ? null : await runtimeSessionStore.findRuntimeSessionByTaskAttempt(teamId, sourceTaskId, sourceAttemptN);
+	const sourceSessionPath = localSessionPath ? localSessionPath : remoteSession ? await runtimeSessionStore.hydrateSession({
+		attemptN: sourceAttemptN,
+		destinationDir: `${stateDirs.piSessionsDir}/remote-${sourceTaskId}-attempt-${sourceAttemptN}`,
+		taskId: sourceTaskId,
+		teamId
+	}) : null;
 	if (!sourceSessionPath) return { kind: "no-session-path" };
 	return {
 		kind: "found",
@@ -9616,11 +9695,11 @@ async function resolveWarmSlot(slotRegistry, teamId, sourceTaskId, sourceAttempt
 		workspacePath: resolveProducerWorkspaceCopySource(producerContext, stateDirs)
 	};
 }
-async function maybeAttachWarmSlotContext(claimedTask, basePlan, stateDirs, slotRegistry) {
+async function maybeAttachWarmSlotContext(claimedTask, basePlan, stateDirs, slotRegistry, runtimeSessionStore) {
 	if (claimedTask.task.taskType === "freeform") {
 		const continueFrom = claimedTask.task.input.continueFrom;
 		if (!continueFrom) return basePlan;
-		const resolution = await resolveWarmSlot(slotRegistry, claimedTask.task.teamId, continueFrom.taskId, continueFrom.attemptN, stateDirs);
+		const resolution = await resolveWarmSlot(slotRegistry, runtimeSessionStore, claimedTask.task.teamId, continueFrom.taskId, continueFrom.attemptN, stateDirs);
 		if (resolution.kind === "missing") throw new ProducerContextResolutionError(`Continuation source task ${continueFrom.taskId} attempt ${continueFrom.attemptN} has no live runtime slot on this daemon — claim affinity filter should have prevented this claim`);
 		if (resolution.kind === "no-session-path") throw new ProducerContextResolutionError(`Continuation source attempt ${continueFrom.taskId}/${continueFrom.attemptN} has no persisted Pi session path`);
 		const sessionDir = `${stateDirs.piSessionsDir}/continue-${claimedTask.task.id}-attempt-${claimedTask.attemptN}`;
@@ -9657,7 +9736,7 @@ async function maybeAttachWarmSlotContext(claimedTask, basePlan, stateDirs, slot
 	const targetTaskId = typeof claimedTask.task.input.targetTaskId === "string" ? claimedTask.task.input.targetTaskId : null;
 	const targetAttemptN = typeof claimedTask.task.input.targetAttemptN === "number" ? claimedTask.task.input.targetAttemptN : null;
 	if (!targetTaskId || !targetAttemptN) throw new ProducerContextResolutionError("judge_eval_attempt is missing targetTaskId/targetAttemptN");
-	const resolution = await resolveWarmSlot(slotRegistry, claimedTask.task.teamId, targetTaskId, targetAttemptN, stateDirs);
+	const resolution = await resolveWarmSlot(slotRegistry, runtimeSessionStore, claimedTask.task.teamId, targetTaskId, targetAttemptN, stateDirs);
 	if (resolution.kind === "missing") throw new ProducerContextResolutionError(`No live producer runtime slot found for task ${targetTaskId} attempt ${targetAttemptN}`);
 	if (resolution.kind === "no-session-path") throw new ProducerContextResolutionError(`Producer task ${targetTaskId} attempt ${targetAttemptN} has no persisted Pi session path`);
 	return {
@@ -10067,6 +10146,70 @@ function isExecutable(path) {
 	}
 }
 //#endregion
+//#region src/lib/runtime-sessions.ts
+function resolveRuntimeSessionKind(claimedTask) {
+	const continueFrom = resolveContinueFrom(claimedTask);
+	if (!continueFrom) return "root";
+	return continueFrom.mode === "fork" ? "fork" : "extend";
+}
+async function resolveParentRuntimeSession(runtimeSessionStore, claimedTask) {
+	const continueFrom = resolveContinueFrom(claimedTask);
+	if (!continueFrom) return null;
+	return runtimeSessionStore.findRuntimeSessionByTaskAttempt(claimedTask.task.teamId, continueFrom.taskId, continueFrom.attemptN);
+}
+function applyRuntimeSessionUploadFailure(output, err) {
+	if (output.status !== "completed") return output;
+	return {
+		...output,
+		contentSignature: void 0,
+		error: {
+			code: "runtime_session_upload_failed",
+			message: "Task completed, but durable runtime session checkpoint upload failed: " + (err instanceof Error ? err.message : String(err)),
+			retryable: true
+		},
+		output: null,
+		outputCid: null,
+		status: "failed"
+	};
+}
+function createApiRuntimeSessionStore(args) {
+	const { agent } = args;
+	return {
+		async findRuntimeSessionByTaskAttempt(teamId, taskId, attemptN) {
+			return agent.runtimeSessions.getForAttempt({
+				attemptN,
+				taskId
+			}, { teamId });
+		},
+		async hydrateSession(input) {
+			const downloaded = await agent.runtimeSessions.download({
+				attemptN: input.attemptN,
+				taskId: input.taskId
+			}, { teamId: input.teamId });
+			await mkdir(input.destinationDir, { recursive: true });
+			const sessionPath = join(input.destinationDir, `remote-${input.taskId}-attempt-${input.attemptN}.jsonl`);
+			await pipeline(downloaded, createWriteStream(sessionPath));
+			return sessionPath;
+		},
+		async uploadAttemptFinal(input) {
+			const sessionPath = resolveLatestPiSessionPath(input.sessionDir);
+			if (!sessionPath) throw new Error(`Cannot upload runtime session for ${input.taskId}/${input.attemptN}: no local session file in ${input.sessionDir}`);
+			await agent.runtimeSessions.upload({
+				attemptN: input.attemptN,
+				taskId: input.taskId
+			}, createReadStream(sessionPath), {
+				parentSessionId: input.parentSessionId ?? void 0,
+				sessionKind: input.sessionKind,
+				sourceRuntimeProfileId: input.sourceRuntimeProfileId ?? void 0,
+				sourceSlotId: input.sourceSlotId ?? void 0
+			}, { teamId: input.teamId });
+		}
+	};
+}
+function resolveContinueFrom(claimedTask) {
+	return claimedTask.task.input.continueFrom;
+}
+//#endregion
 //#region src/lib/runtime-slots.ts
 function createApiRuntimeSlotStore(args) {
 	const { agent } = args;
@@ -10108,7 +10251,11 @@ function createApiRuntimeSlotStore(args) {
 			}, { teamId });
 			if (!resolved) return null;
 			return {
-				slot: { expiresAtMs: resolved.slot.expiresAtMs },
+				slot: {
+					expiresAtMs: resolved.slot.expiresAtMs,
+					id: resolved.slot.id,
+					runtimeProfileId: resolved.slot.runtimeProfileId
+				},
 				session: resolved.slot.sessionDir ? {
 					sessionDir: resolved.slot.sessionDir,
 					sessionPath: resolved.slot.sessionPath
@@ -10263,6 +10410,7 @@ async function runPolling(opts) {
 	});
 	for (const profile of profiles) validateRuntimeProfilePrerequisites(profile, cfg.profilePrerequisiteEnv, cfg.profilePrerequisitePath);
 	const slotRegistry = createApiRuntimeSlotStore({ agent: ctx.agent });
+	const runtimeSessionStore = createApiRuntimeSessionStore({ agent: ctx.agent });
 	const runtimes = /* @__PURE__ */ new Map();
 	for (const profile of profiles) {
 		const common = parseCommonOptions(values, { runtimeDefaults: {
@@ -10292,7 +10440,8 @@ async function runPolling(opts) {
 				defaultWorkspaceMode: profile.defaultWorkspaceMode,
 				allowedWorkspaceModes: profile.allowedWorkspaceModes
 			},
-			slotRegistry
+			slotRegistry,
+			runtimeSessionStore
 		});
 		runtimes.set(profile.id, {
 			common,
@@ -10401,7 +10550,8 @@ async function runPolling(opts) {
 				stopWhenEmpty: opts.stopWhenEmpty,
 				debug: baseCommon.debug,
 				logger: rootLogger,
-				slotRegistry
+				slotRegistry,
+				sessionRegistry: runtimeSessionStore
 			}),
 			makeReporter: (claimedTask) => {
 				const selected = runtimeForClaimedTask(runtimes, claimedTask);
@@ -10416,7 +10566,28 @@ async function runPolling(opts) {
 			onTaskFinished: async (output, claimedTask) => {
 				const selected = runtimeForClaimedTask(runtimes, claimedTask);
 				const resolved = await slotRegistry.findLatestSlotByTaskAttempt(claimedTask.task.teamId, claimedTask.task.id, claimedTask.attemptN);
-				return finalizeTask(ctx.agent, output, {
+				let terminalOutput = output;
+				if (resolved?.session?.sessionDir) try {
+					const parentSession = await resolveParentRuntimeSession(runtimeSessionStore, claimedTask);
+					await runtimeSessionStore.uploadAttemptFinal({
+						attemptN: claimedTask.attemptN,
+						parentSessionId: parentSession?.id ?? null,
+						sessionDir: resolved.session.sessionDir,
+						sessionKind: resolveRuntimeSessionKind(claimedTask),
+						sourceRuntimeProfileId: resolved.slot.runtimeProfileId,
+						sourceSlotId: resolved.slot.id,
+						taskId: claimedTask.task.id,
+						teamId: claimedTask.task.teamId
+					});
+				} catch (err) {
+					rootLogger.error({
+						err,
+						taskId: claimedTask.task.id,
+						attemptN: claimedTask.attemptN
+					}, "agent-daemon.runtime_session_upload_failed");
+					terminalOutput = applyRuntimeSessionUploadFailure(output, err);
+				}
+				return finalizeTask(ctx.agent, terminalOutput, {
 					task: claimedTask.task,
 					slot: resolved ? { expiresAtMs: resolved.slot.expiresAtMs } : null,
 					writeCorrelationAnchors: makePrBodyAnchorWriter({
@@ -10681,6 +10852,7 @@ async function runOnce(argv) {
 	activatePiCodingAgentDir(piAgentDir.path);
 	const stateDirs = ensureDaemonStateDirs(sandbox.rootDir);
 	const slotRegistry = createApiRuntimeSlotStore({ agent: ctx.agent });
+	const runtimeSessionStore = createApiRuntimeSessionStore({ agent: ctx.agent });
 	const slotIdentity = {
 		agentName: opts.agent,
 		runtimeProfileId: profile.id
@@ -10693,7 +10865,8 @@ async function runOnce(argv) {
 			defaultWorkspaceMode: profile.defaultWorkspaceMode,
 			allowedWorkspaceModes: profile.allowedWorkspaceModes
 		},
-		slotRegistry
+		slotRegistry,
+		runtimeSessionStore
 	});
 	const otelShutdown = await initWorkerOtel({
 		serviceName: "moltnet.agent-daemon.once",
@@ -10853,7 +11026,28 @@ async function runOnce(argv) {
 			}),
 			onTaskFinished: async (output, claimedTask) => {
 				const resolved = await slotRegistry.findLatestSlotByTaskAttempt(claimedTask.task.teamId, claimedTask.task.id, claimedTask.attemptN);
-				return finalizeTask(ctx.agent, output, {
+				let terminalOutput = output;
+				if (resolved?.session?.sessionDir) try {
+					const parentSession = await resolveParentRuntimeSession(runtimeSessionStore, claimedTask);
+					await runtimeSessionStore.uploadAttemptFinal({
+						attemptN: claimedTask.attemptN,
+						parentSessionId: parentSession?.id ?? null,
+						sessionDir: resolved.session.sessionDir,
+						sessionKind: resolveRuntimeSessionKind(claimedTask),
+						sourceRuntimeProfileId: resolved.slot.runtimeProfileId,
+						sourceSlotId: resolved.slot.id,
+						taskId: claimedTask.task.id,
+						teamId: claimedTask.task.teamId
+					});
+				} catch (err) {
+					rootLogger.error({
+						err,
+						taskId: claimedTask.task.id,
+						attemptN: claimedTask.attemptN
+					}, "agent-daemon.runtime_session_upload_failed");
+					terminalOutput = applyRuntimeSessionUploadFailure(output, err);
+				}
+				return finalizeTask(ctx.agent, terminalOutput, {
 					task: claimedTask.task,
 					slot: resolved ? { expiresAtMs: resolved.slot.expiresAtMs } : null,
 					writeCorrelationAnchors,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@themoltnet/agent-daemon",
-  "version": "0.24.0",
+  "version": "0.25.1",
   "license": "AGPL-3.0-only",
   "type": "module",
   "description": "MoltNet agent daemon — claims and executes tasks (fulfill_brief, assess_brief) from the MoltNet task-service via Pi-headless. CLI: moltnet-agent.",
@@ -45,18 +45,18 @@
     "@opentelemetry/semantic-conventions": "^1.39.0",
     "pino": "^10.3.1",
     "pino-pretty": "^13.1.3",
-    "@themoltnet/agent-runtime": "0.29.0",
-    "@themoltnet/pi-extension": "0.27.0",
-    "@themoltnet/sdk": "0.112.0"
+    "@themoltnet/pi-extension": "0.27.2",
+    "@themoltnet/sdk": "0.113.1",
+    "@themoltnet/agent-runtime": "0.30.1"
   },
   "devDependencies": {
     "tsx": "^4.7.0",
     "typescript": "~5.9.2",
     "vite": "^8.0.0",
     "vitest": "^3.0.0",
-    "@moltnet/observability": "0.1.0",
     "@moltnet/bootstrap": "0.1.0",
     "@moltnet/crypto-service": "0.1.0",
+    "@moltnet/observability": "0.1.0",
     "@moltnet/tasks": "0.1.0"
   },
   "nx": {