@themoltnet/agent-daemon 0.28.0 → 0.29.0

package/dist/main.js

@@ -10,7 +10,7 @@ import crypto from "crypto";
 import { delimiter, dirname, isAbsolute, join, relative, resolve } from "node:path";
 import { parseArgs, promisify } from "node:util";
 import { AgentRuntime, ApiTaskReporter, ApiTaskSource, PollingApiTaskSource } from "@themoltnet/agent-runtime";
-import { createPiTaskExecutor, findMainWorktree } from "@themoltnet/pi-extension";
+import { createPiRetryTriage, createPiTaskExecutor, findMainWorktree, normalizeRetryTriageResult, redactRetryTriageSecrets } from "@themoltnet/pi-extension";
 import { execFile, execFileSync } from "node:child_process";
 import { accessSync, constants, createReadStream, createWriteStream, existsSync, mkdirSync, readdirSync } from "node:fs";
 import { MoltNetError, connect } from "@themoltnet/sdk";
@@ -22,6 +22,7 @@ import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base";
 import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
 import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
 import { AsyncLocalStorage } from "node:async_hooks";
+import { getModel } from "@earendil-works/pi-ai";
 import { mkdir, realpath, stat } from "node:fs/promises";
 import { pipeline } from "node:stream/promises";
 import { createHash } from "node:crypto";
@@ -9217,6 +9218,27 @@ var TaskUsage = _Object_({
 	$id: "TaskUsage",
 	additionalProperties: false
 });
+var TaskRetryDecision = Union([Literal("retry"), Literal("do_not_retry")]);
+var TaskRetryConfidence = Union([
+	Literal("low"),
+	Literal("medium"),
+	Literal("high")
+]);
+var TaskRetryInfo = _Object_({
+	source: Union([
+		Literal("explicit"),
+		Literal("deterministic"),
+		Literal("attempts_exhausted"),
+		Literal("triage"),
+		Literal("triage_failed")
+	]),
+	decision: Optional(TaskRetryDecision),
+	confidence: Optional(TaskRetryConfidence),
+	reason: Optional(String$1())
+}, {
+	$id: "TaskRetryInfo",
+	additionalProperties: false
+});
 /**
 * Structured error returned from a failed attempt.
 */
@@ -9224,7 +9246,8 @@ var TaskError = _Object_({
 	code: String$1(),
 	message: String$1(),
 	stack: Optional(String$1()),
-	retryable: Optional(Boolean$1())
+	retryable: Optional(Boolean$1()),
+	retry: Optional(TaskRetryInfo)
 }, {
 	$id: "TaskError",
 	additionalProperties: false
@@ -10010,6 +10033,157 @@ function recoverScratchWorkspacePath(producer, stateDirs) {
 	return existsSync(fallback) ? fallback : null;
 }
 //#endregion
+//#region src/lib/retry-triage.ts
+var RETRYABLE_CODES = new Set([
+	"checkpoint_upload_failed",
+	"complete_call_failed",
+	"daemon_abort",
+	"dispatch_expired",
+	"lease_expired",
+	"llm_api_error",
+	"runtime_session_checkpoint_failed",
+	"session_prompt_failed"
+]);
+var NON_RETRYABLE_CODES = new Set([
+	"bad_api_key",
+	"invalid_api_key",
+	"invalid_model",
+	"output_rejected_by_server",
+	"output_validation_failed",
+	"producer_context_missing",
+	"running_max_bash_timeouts_exceeded",
+	"running_max_turns_exceeded",
+	"task_cancelled",
+	"unknown_task_type"
+]);
+var RETRYABLE_MESSAGE_PATTERNS = [
+	/\b429\b/i,
+	/\b5(?:02|03|04)\b/i,
+	/\btimeout\b/i,
+	/\btimed out\b/i,
+	/\brate limit/i,
+	/\btemporar(?:y|ily)\b/i,
+	/\bunavailable\b/i,
+	/\bECONNRESET\b/i,
+	/\bECONNREFUSED\b/i,
+	/\bETIMEDOUT\b/i,
+	/\bENOTFOUND\b/i,
+	/\bEAI_AGAIN\b/i,
+	/\bDNS\b/i
+];
+var NON_RETRYABLE_MESSAGE_PATTERNS = [
+	/\b401\b/i,
+	/\b403\b/i,
+	/\bunauthori[sz]ed\b/i,
+	/\bforbidden\b/i,
+	/\binvalid (?:api )?key\b/i,
+	/\bmissing credentials?\b/i,
+	/\bmodel .*not (?:found|registered|available)\b/i,
+	/\bunknown task type\b/i,
+	/\bvalidation failed\b/i,
+	/\bcancelled\b/i,
+	/\bmax (?:turn|bash)/i
+];
+async function classifyAttemptFailure(input) {
+	if (input.remainingAttempts !== null && input.remainingAttempts !== void 0) {
+		if (input.remainingAttempts <= 0) return {
+			error: withRetryInfo(input.error, {
+				retryable: false,
+				source: "attempts_exhausted",
+				reason: `Attempt budget exhausted at attempt ${input.attemptN}${input.maxAttempts ? ` of ${input.maxAttempts}` : ""}.`
+			}),
+			source: "attempts_exhausted"
+		};
+	}
+	const deterministic = classifyDeterministically(input.error);
+	if (deterministic !== "ambiguous") {
+		const retryable = deterministic === "retryable";
+		const source = input.error.retryable === retryable ? "explicit" : "deterministic";
+		return {
+			error: withRetryInfo(input.error, {
+				retryable,
+				source,
+				decision: retryable ? "retry" : "do_not_retry",
+				confidence: "high",
+				reason: retryable ? "Matched deterministic retry policy." : "Matched deterministic no-retry policy."
+			}),
+			source
+		};
+	}
+	if (!input.triage) return {
+		error: withRetryInfo(input.error, {
+			retryable: false,
+			source: "triage_failed",
+			reason: "Failure was ambiguous and no retry triage agent was configured; defaulted to no retry."
+		}),
+		source: "triage_failed"
+	};
+	try {
+		const triage = normalizeRetryTriageResult(await input.triage(input));
+		const retryable = triage.decision === "retry" && (triage.confidence === "medium" || triage.confidence === "high");
+		return {
+			error: withRetryInfo({
+				...input.error,
+				message: appendTriageReason(input.error.message, triage)
+			}, {
+				retryable,
+				source: "triage",
+				decision: triage.decision,
+				confidence: triage.confidence,
+				reason: triage.reason
+			}),
+			source: "triage",
+			triage
+		};
+	} catch (err) {
+		return {
+			error: withRetryInfo({
+				...input.error,
+				message: appendTriageFailure(input.error.message, err)
+			}, {
+				retryable: false,
+				source: "triage_failed",
+				reason: `Retry triage failed: ${sanitizeReason(err)}`
+			}),
+			source: "triage_failed"
+		};
+	}
+}
+function classifyDeterministically(error) {
+	const code = error.code.toLowerCase();
+	const message = error.message;
+	if (NON_RETRYABLE_CODES.has(code)) return "non_retryable";
+	if (NON_RETRYABLE_MESSAGE_PATTERNS.some((pattern) => pattern.test(message))) return "non_retryable";
+	if (RETRYABLE_CODES.has(code)) return "retryable";
+	if (RETRYABLE_MESSAGE_PATTERNS.some((pattern) => pattern.test(message))) return "retryable";
+	if (error.retryable === true) return "retryable";
+	return "ambiguous";
+}
+function appendTriageReason(message, triage) {
+	const suffix = ` Retry triage: ${triage.decision}/${triage.confidence}: ${triage.reason}`;
+	if (message.includes("Retry triage:")) return message;
+	return `${message}${suffix}`.slice(0, 4e3);
+}
+function appendTriageFailure(message, err) {
+	if (message.includes("Retry triage failed:")) return message;
+	return `${message} Retry triage failed: ${sanitizeReason(err)}`.slice(0, 4e3);
+}
+function sanitizeReason(value) {
+	return redactRetryTriageSecrets(value instanceof Error ? value.message : String(value)).slice(0, 500);
+}
+function withRetryInfo(error, info) {
+	return {
+		...error,
+		retryable: info.retryable,
+		retry: {
+			source: info.source,
+			...info.decision ? { decision: info.decision } : {},
+			...info.confidence ? { confidence: info.confidence } : {},
+			...info.reason ? { reason: info.reason.slice(0, 500) } : {}
+		}
+	};
+}
+//#endregion
 //#region src/lib/finalize.ts
 /**
 * Build the `daemonState` payload for a `/complete` call. Freeform
@@ -10045,9 +10219,10 @@ async function finalizeTask(agent, output, ctx = {}) {
 				...daemonState ? { daemonState } : {}
 			});
 		} catch (err) {
-			const reason = errorToFailReason(err);
+			const classified = await prepareAttemptFailure(agent, output, errorToFailReason(err), ctx);
 			ctx.log?.("complete-rejected-falling-back-to-fail", err);
-			await agent.tasks.fail(output.taskId, output.attemptN, { error: reason });
+			ctx.log?.(`attempt-failure-classified:${classified.source}`);
+			await agent.tasks.failAttempt(output.taskId, output.attemptN, { error: classified.error });
 			return;
 		}
 		await maybeWriteAnchors(output, ctx);
@@ -10059,10 +10234,16 @@ async function finalizeTask(agent, output, ctx = {}) {
 		retryable: false
 	};
 	if ((await agent.tasks.heartbeat(output.taskId, output.attemptN, {})).cancelled) return;
-	await agent.tasks.fail(output.taskId, output.attemptN, { error });
+	const classified = await prepareAttemptFailure(agent, output, error, ctx);
+	ctx.log?.(`attempt-failure-classified:${classified.source}`);
+	await agent.tasks.failAttempt(output.taskId, output.attemptN, { error: classified.error });
 }
 function errorToFailReason(err) {
 	if (err instanceof MoltNetError) {
+		if (err.code !== "VALIDATION_FAILED" && err.statusCode !== 400) return {
+			code: "complete_call_failed",
+			message: `Failed to report task completion (${err.code}, status ${err.statusCode ?? "?"}): ${err.detail ?? err.message}`
+		};
 		const fields = err.validationErrors?.length ? "; " + err.validationErrors.map((e) => `${e.field}: ${e.message}`).join(" | ") : "";
 		return {
 			code: "output_rejected_by_server",
@@ -10076,6 +10257,28 @@ function errorToFailReason(err) {
 		retryable: false
 	};
 }
+async function prepareAttemptFailure(agent, output, error, ctx) {
+	const task = ctx.task ?? {
+		id: output.taskId,
+		taskType: "unknown",
+		teamId: "unknown",
+		input: {},
+		maxAttempts: null
+	};
+	const recentMessages = classifyDeterministically(error) === "ambiguous" && ctx.retryTriage && ctx.task ? await agent.tasks.listMessages(output.taskId, output.attemptN).then((messages) => messages.slice(-12)).catch((err) => {
+		ctx.log?.("attempt-failure-message-fetch-failed", err);
+		return [];
+	}) : [];
+	return classifyAttemptFailure({
+		task,
+		attemptN: output.attemptN,
+		maxAttempts: ctx.task?.maxAttempts ?? null,
+		remainingAttempts: ctx.task?.maxAttempts === void 0 ? null : Math.max(0, ctx.task.maxAttempts - output.attemptN),
+		error,
+		recentMessages,
+		triage: ctx.retryTriage
+	});
+}
 async function maybeWriteAnchors(output, ctx) {
 	const { task, writeCorrelationAnchors, log } = ctx;
 	if (!task || task.taskType !== "fulfill_brief") return;
@@ -10387,6 +10590,17 @@ function isExecutable(path) {
 	}
 }
 //#endregion
+//#region src/lib/runtime-profile-retry-triage.ts
+function createRuntimeProfileRetryTriage(options) {
+	return createPiRetryTriage({
+		model: getModel(options.runtimeProfile.provider, options.runtimeProfile.model),
+		thinkingLevel: options.runtimeProfile.thinkingLevel,
+		piAgentDir: options.piAgentDir,
+		timeoutMs: options.timeoutMs,
+		cwd: options.cwd
+	});
+}
+//#endregion
 //#region src/lib/runtime-sessions.ts
 function resolveRuntimeSessionKind(claimedTask) {
 	const continueFrom = resolveContinueFrom(claimedTask);
@@ -10882,6 +11096,11 @@ async function runPolling(opts) {
 				return finalizeTask(ctx.agent, terminalOutput, {
 					task: claimedTask.task,
 					slot: resolved ? { expiresAtMs: resolved.slot.expiresAtMs } : null,
+					retryTriage: createRuntimeProfileRetryTriage({
+						runtimeProfile: selected.profile,
+						piAgentDir: selected.piAgentDir.path,
+						cwd: ctx.agentRootDir
+					}),
 					writeCorrelationAnchors: makePrBodyAnchorWriter({
 						gh: createGhCliClient(),
 						logger: rootLogger.child({
@@ -11381,6 +11600,11 @@ async function runOnce(argv) {
 				return finalizeTask(ctx.agent, terminalOutput, {
 					task: claimedTask.task,
 					slot: resolved ? { expiresAtMs: resolved.slot.expiresAtMs } : null,
+					retryTriage: createRuntimeProfileRetryTriage({
+						runtimeProfile: profile,
+						piAgentDir: piAgentDir.path,
+						cwd: ctx.agentRootDir
+					}),
 					writeCorrelationAnchors,
 					log: (msg, err) => rootLogger.warn({ err }, msg)
 				});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@themoltnet/agent-daemon",
-  "version": "0.28.0",
+  "version": "0.29.0",
   "license": "AGPL-3.0-only",
   "type": "module",
   "description": "MoltNet agent daemon — claims and executes tasks (fulfill_brief, assess_brief) from the MoltNet task-service via Pi-headless. CLI: moltnet-agent.",
@@ -45,21 +45,22 @@
     "@opentelemetry/semantic-conventions": "^1.39.0",
     "pino": "^10.3.1",
     "pino-pretty": "^13.1.3",
-    "@themoltnet/agent-runtime": "0.33.0",
-    "@themoltnet/pi-extension": "0.30.0",
-    "@themoltnet/sdk": "0.116.0"
+    "@themoltnet/agent-runtime": "0.33.1",
+    "@themoltnet/pi-extension": "0.31.0",
+    "@themoltnet/sdk": "0.117.0"
   },
   "devDependencies": {
     "tsx": "^4.7.0",
     "typescript": "~5.9.2",
     "vite": "^8.0.0",
     "vitest": "^3.0.0",
-    "@moltnet/bootstrap": "0.1.0",
     "@moltnet/crypto-service": "0.1.0",
     "@moltnet/tasks": "0.1.0",
+    "@moltnet/bootstrap": "0.1.0",
     "@moltnet/observability": "0.1.0"
   },
   "nx": {
+    "projectType": "application",
     "tags": [
       "type:app",
       "scope:agent",