@themoltnet/pi-extension 0.31.1 → 0.32.0

package/dist/index.d.ts

@@ -382,6 +382,22 @@ export declare interface ExecutePiTaskOptions {
      * the attempt fails with output_validation_failed.
      */
     maxSubmitValidationRetries?: number;
+    /**
+     * Number of same-session re-prompts when the model ends its turn WITHOUT
+     * calling the submit-output tool at all (no captured payload and no
+     * exhausted validation budget). Distinct from
+     * `maxSubmitValidationRetries`, which recovers *invalid-args* submit calls.
+     * Each re-prompt names the submit tool and forbids a prose reply. When the
+     * budget is spent the attempt still fails with `submit_output_missing`.
+     * Only applies to task types that register a submit tool. Default `3`. Set
+     * to `0` to disable. See #1528.
+     */
+    maxSubmitMissingReprompts?: number;
+    /**
+     * Continuation prompt sent when a turn ends without a submit call. Defaults
+     * to `buildSubmitMissingPrompt(<tool name>)`.
+     */
+    submitMissingPrompt?: string;
     /**
      * Cap provider-error retries inside the same Pi session. A retry is attempted
      * only after a Pi assistant turn ends with `stopReason: "error"` and the

package/dist/index.js

@@ -14226,6 +14226,86 @@ var PRODUCER_TASK_TYPES = new Set([
 	"render_pack",
 	"run_eval"
 ]);
+function isNonEmptyString(value) {
+	return typeof value === "string" && value.length > 0;
+}
+function criterionWeight(criterion, index) {
+	if (typeof criterion.weight === "number") return criterion.weight;
+	if (typeof criterion.max_score === "number") return criterion.max_score / 100;
+	if (typeof criterion.maxScore === "number") return criterion.maxScore / 100;
+	throw new TaskBuildError([{
+		field: `successCriteria/rubric/criteria/${index}/weight`,
+		message: "criterion is missing weight or max_score"
+	}]);
+}
+/**
+* Normalize authoring-time rubric criteria to canonical MoltNet rubric
+* criteria. Accepts `{id,title,description,weight}` and
+* `{name,description,max_score}` style inputs, strips authoring-only fields,
+* and fills a default scoring mode.
+*/
+function normalizeRubricCriteria(criteria, options) {
+	const errors = [];
+	const normalized = criteria.map((criterion, index) => {
+		const id = criterion.id ?? criterion.name;
+		const description = criterion.description ?? criterion.title;
+		if (!isNonEmptyString(id)) errors.push({
+			field: `successCriteria/rubric/criteria/${index}/id`,
+			message: "criterion is missing id or name"
+		});
+		if (!isNonEmptyString(description)) errors.push({
+			field: `successCriteria/rubric/criteria/${index}/description`,
+			message: "criterion is missing description or title"
+		});
+		return {
+			id: id ?? "",
+			description: description ?? "",
+			weight: criterionWeight(criterion, index),
+			scoring: criterion.scoring ?? options?.scoring ?? "llm_score"
+		};
+	});
+	if (errors.length > 0) throw new TaskBuildError(errors);
+	return normalized;
+}
+/**
+* Build a canonical `SuccessCriteria` envelope from rubric/checklist-style
+* criteria. This keeps rubrics readable at the authoring boundary while
+* preserving the strict task schema on the wire.
+*/
+function buildRubricSuccessCriteria(options) {
+	const rubric = {
+		rubricId: options.rubricId,
+		version: options.version ?? "v1",
+		criteria: normalizeRubricCriteria(options.criteria, { scoring: options.scoring }),
+		...options.contentHash ? { contentHash: options.contentHash } : {},
+		...options.preamble ? { preamble: options.preamble } : {},
+		...options.scope ? { scope: options.scope } : {}
+	};
+	const weightError = validateRubricWeights(rubric);
+	if (weightError) throw new TaskBuildError([{
+		field: "successCriteria/rubric/criteria",
+		message: weightError
+	}]);
+	return {
+		version: 1,
+		rubric
+	};
+}
+function resolveJudgeEvalAttemptTarget(target) {
+	if ("judgeEvalTarget" in target && typeof target.judgeEvalTarget === "function") return target.judgeEvalTarget();
+	if ("targetTaskId" in target) return {
+		targetTaskId: target.targetTaskId,
+		targetAttemptN: target.targetAttemptN
+	};
+	if ("taskId" in target) return {
+		targetTaskId: target.taskId,
+		targetAttemptN: target.accepted?.attemptN ?? target.attemptN ?? 1
+	};
+	throw new TaskBuildError([{
+		field: "target",
+		message: "judge_eval_attempt target is missing task id"
+	}]);
+}
 /**
 * Fluent, network-free builder for a `tasks.create` body. Encodes the
 * non-obvious task schema (context arrays, success-criteria gates,
@@ -14676,6 +14756,20 @@ function buildJudgeEvalAttempt(input) {
 	return buildTask("judge_eval_attempt", input);
 }
 /**
+* Build a `judge_eval_attempt` task from an accepted `run_eval` result (or a
+* small target tuple) plus human-friendly rubric criteria.
+*
+* @param target - A `TaskResultReader` or `{targetTaskId,targetAttemptN}` tuple.
+* @param options - Rubric metadata and eval/checklist-style criteria.
+* @returns A typed {@link TaskBuilder}.
+*/
+function buildJudgeEvalAttemptForRunEval(target, options) {
+	return buildJudgeEvalAttempt({
+		...resolveJudgeEvalAttemptTarget(target),
+		successCriteria: buildRubricSuccessCriteria(options)
+	});
+}
+/**
 * Build a `pr_review` task. Requires `subject` + `successCriteria`. Note the
 * rubric criteria must use `boolean` scoring for this task type.
 *
@@ -14713,7 +14807,9 @@ var TaskResultReader = class {
 	accepted;
 	/** Token / cost usage for the accepted attempt, if reported. */
 	usage;
+	/** Task id for the task whose accepted attempt is being read. */
 	taskId;
+	/** CID of the accepted attempt output. */
 	outputCid;
 	constructor(task, attempt) {
 		const errors = [];
@@ -14806,6 +14902,19 @@ var TaskResultReader = class {
 		};
 	}
 	/**
+	* Return the target tuple required by `judge_eval_attempt`.
+	*
+	* This intentionally uses the accepted attempt number, not merely the
+	* attempt object passed to the reader, so a downstream judge is pinned to
+	* the producer output that the task accepted.
+	*/
+	judgeEvalTarget() {
+		return {
+			targetTaskId: this.taskId,
+			targetAttemptN: this.accepted.attemptN
+		};
+	}
+	/**
 	* Build a `TaskRef` that anchors a downstream task to this accepted output
 	* and points at one persistent task artifact by CID.
 	*
@@ -14943,6 +15052,7 @@ function createTasksNamespace(context) {
 		buildAssessBrief,
 		buildJudgePack,
 		buildJudgeEvalAttempt,
+		buildJudgeEvalAttemptForRunEval,
 		buildPrReview,
 		async readResult(taskOrId) {
 			const task = typeof taskOrId === "string" ? unwrapResult(await getTask({
@@ -25621,6 +25731,7 @@ function createSubmitOutputTool(taskType, opts = {}) {
 				};
 			}
 		}),
+		toolName: contract.toolName,
 		getCaptured: () => captured,
 		getCallCount: () => callCount,
 		getInvalidCallCount: () => invalidCallCount,
@@ -26316,7 +26427,8 @@ async function executePiTask(claimedTask, reporter, opts) {
 					const message = err instanceof Error ? err.message : String(err);
 					reporterError = {
 						code: "reporter_failed",
-						message
+						message,
+						retryable: true
 					};
 					process.stderr.write(`[reporter] ${message}\n`);
 				}
@@ -26383,9 +26495,9 @@ async function executePiTask(claimedTask, reporter, opts) {
 			}
 		});
 		let runError = null;
-		runError = (await promptWithProviderErrorRetries({
-			session,
-			initialPrompt: taskPrompt,
+		const runPrompt = (promptText) => promptWithProviderErrorRetries({
+			session: liveSession,
+			initialPrompt: promptText,
 			cancelSignal: reporter.cancelSignal,
 			isCapAborted: () => capAbort !== null,
 			getProviderErrorState: () => ({
@@ -26404,7 +26516,33 @@ async function executePiTask(claimedTask, reporter, opts) {
 				message,
 				phase: "session_prompt"
 			})
-		})).runError;
+		});
+		const submitMissingConfig = resolveSubmitMissingConfig({
+			submitToolHandle,
+			maxSubmitMissingReprompts: opts.maxSubmitMissingReprompts,
+			submitMissingPrompt: opts.submitMissingPrompt
+		});
+		const promptResult = await promptUntilSubmitted({
+			runPrompt,
+			initialPrompt: taskPrompt,
+			submitMissingPrompt: submitMissingConfig.submitMissingPrompt,
+			maxSubmitMissingReprompts: submitMissingConfig.maxSubmitMissingReprompts,
+			getSubmitState: submitMissingConfig.getSubmitState,
+			isStopped: () => submitRepromptStopped({
+				cancelled: reporter.cancelSignal.aborted,
+				capAborted: capAbort !== null,
+				llmAbort
+			}),
+			onSubmitReprompt: async (event) => {
+				await emit("info", event);
+			}
+		});
+		runError = promptResult.runError;
+		if (promptResult.submitReprompts > 0) await emit("info", {
+			event: "submit_missing_summary",
+			submitReprompts: promptResult.submitReprompts,
+			captured: submitToolHandle ? submitToolHandle.getCaptured() !== null : false
+		});
 		if (subagentHandle && subagentHandle.getCallCount() > 0) await emit("info", {
 			event: "subagent_summary",
 			callCount: subagentHandle.getCallCount()
@@ -26443,10 +26581,16 @@ async function executePiTask(claimedTask, reporter, opts) {
 				});
 			}
 			else if (submitToolHandle) {
-				parseError = submitToolHandle.getExhaustedValidationFailure() ?? {
+				const exhausted = submitToolHandle.getExhaustedValidationFailure();
+				parseError = exhausted ?? {
 					code: "submit_output_missing",
 					message: "Agent did not satisfy the promised submit-output criterion: no valid task submit tool call was captured before the session ended."
 				};
+				if (!exhausted) recordTaskOutputParseResult({
+					taskType: task.taskType,
+					model: opts.model,
+					code: "output_missing"
+				});
 				await emit("error", {
 					message: parseError.message,
 					phase: "output_validation"
@@ -26494,9 +26638,11 @@ async function executePiTask(claimedTask, reporter, opts) {
 				retryable: false
 			}
 		};
-		const status = runError || llmAbort || parseError || reporterError ? "failed" : "completed";
-		const errorCode = runError?.code ?? parseError?.code ?? reporterError?.code ?? (llmAbort ? "llm_api_error" : void 0);
-		const errorMessage = runError?.message ?? parseError?.message ?? reporterError?.message ?? (llmAbort ? llmErrorMessage ?? "LLM API error during turn" : void 0);
+		const reporterErrorSnapshot = reporterError;
+		const status = runError || llmAbort || parseError || reporterErrorSnapshot ? "failed" : "completed";
+		const errorCode = runError?.code ?? parseError?.code ?? reporterErrorSnapshot?.code ?? (llmAbort ? "llm_api_error" : void 0);
+		const errorMessage = runError?.message ?? parseError?.message ?? reporterErrorSnapshot?.message ?? (llmAbort ? llmErrorMessage ?? "LLM API error during turn" : void 0);
+		const errorRetryable = reporterErrorSnapshot && errorCode === reporterErrorSnapshot.code && errorMessage === reporterErrorSnapshot.message ? reporterErrorSnapshot.retryable ?? false : false;
 		return {
 			taskId: task.id,
 			attemptN,
@@ -26508,7 +26654,7 @@ async function executePiTask(claimedTask, reporter, opts) {
 			...errorCode && errorMessage ? { error: {
 				code: errorCode,
 				message: errorMessage,
-				retryable: false
+				retryable: errorRetryable
 			} } : {}
 		};
 	} catch (err) {
@@ -26723,6 +26869,102 @@ async function promptWithProviderErrorRetries(args) {
 		promptText = args.retryPrompt;
 	}
 }
+/**
+* Continuation prompt used to recover a session that ended without calling
+* the submit-output tool. Names the exact tool and forbids a prose reply so a
+* model that "answered" in text is pushed to actually emit the tool call.
+*/
+function buildSubmitMissingPrompt(toolName) {
+	return `You ended your turn but did not call the required \`${toolName}\` tool, so no output was captured and the task is not yet complete. Call \`${toolName}\` now with the final structured output exactly as described in the task prompt. Do not reply with prose, a summary, or an apology — the only way to finish is to call the tool.`;
+}
+/**
+* Whether the submit-missing re-prompt loop must stop before the next nudge.
+*
+* `llmAbort` matters as much as cancel/cap: when a turn ended with
+* `stopReason: 'error'` and the provider-error retry budget is spent (or the
+* error is non-retryable), `promptWithProviderErrorRetries` returns
+* `runError: null` yet leaves `llmAbort` set. Re-prompting then would nudge a
+* dead provider N more times (extra prompts + backoff) and emit misleading
+* `submit_missing_reprompt` events. We only re-prompt after a genuinely clean
+* `end_turn`.
+*/
+function submitRepromptStopped(state) {
+	return state.cancelled || state.capAborted || state.llmAbort;
+}
+/**
+* Resolve the submit-missing recovery config from the registered submit tool
+* (if any) plus caller overrides. Extracted as a pure function so the
+* default-budget / disable-when-no-tool / gate-mapping logic is unit-tested —
+* `executePiTask` itself needs a booted VM and can't cover this seam.
+*
+* The default budget (3) is deliberately one higher than the invalid-args
+* correction budget (`maxSubmitValidationRetries`, default 2): a model that
+* never called the tool just needs a clear nudge, which converts more cheaply
+* and more often than fixing a malformed payload, so the extra attempt is
+* worth it.
+*/
+function resolveSubmitMissingConfig(args) {
+	const handle = args.submitToolHandle;
+	if (!handle) return {
+		maxSubmitMissingReprompts: 0,
+		submitMissingPrompt: "",
+		getSubmitState: () => null
+	};
+	return {
+		maxSubmitMissingReprompts: args.maxSubmitMissingReprompts ?? 3,
+		submitMissingPrompt: args.submitMissingPrompt ?? buildSubmitMissingPrompt(handle.toolName),
+		getSubmitState: () => ({
+			captured: handle.getCaptured() !== null,
+			exhausted: handle.getExhaustedValidationFailure() !== null
+		})
+	};
+}
+/**
+* Drive a Pi session until it either captures a valid submit-output call or
+* exhausts the submit-missing re-prompt budget.
+*
+* This is the third same-session recovery path, complementing the two that
+* already existed:
+*   1. Invalid submit args → the submit tool returns `isError`, the model
+*      re-calls within the same turn (see `submit-output-tool.ts`).
+*   2. Provider/LLM API errors → `promptWithProviderErrorRetries` re-prompts.
+*
+* The gap this closes: a model (typically a weaker one) that ends its turn
+* cleanly with a prose answer and *never calls the submit tool at all*. With
+* neither a captured payload nor an exhausted validation budget, the executor
+* would otherwise fail straight to `submit_output_missing` with no chance to
+* recover. Here we nudge the model — up to `maxSubmitMissingReprompts` times —
+* to call the submit tool. Pi cannot force `toolChoice`, so this re-prompt is
+* the only in-session lever short of patching Pi. See issue #1528.
+*/
+async function promptUntilSubmitted(args) {
+	const first = await args.runPrompt(args.initialPrompt);
+	if (first.runError) return {
+		runError: first.runError,
+		submitReprompts: 0
+	};
+	let submitReprompts = 0;
+	while (submitReprompts < args.maxSubmitMissingReprompts) {
+		if (args.isStopped()) break;
+		const state = args.getSubmitState();
+		if (!state || state.captured || state.exhausted) break;
+		submitReprompts += 1;
+		await args.onSubmitReprompt?.({
+			event: "submit_missing_reprompt",
+			retry: submitReprompts,
+			maxReprompts: args.maxSubmitMissingReprompts
+		});
+		const pass = await args.runPrompt(args.submitMissingPrompt);
+		if (pass.runError) return {
+			runError: pass.runError,
+			submitReprompts
+		};
+	}
+	return {
+		runError: null,
+		submitReprompts
+	};
+}
 function sanitizeProviderErrorRetryReason(value) {
 	return redactRetryTriageSecrets(value ?? "Pi turn ended with stopReason=error").slice(0, 500);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@themoltnet/pi-extension",
-  "version": "0.31.1",
+  "version": "0.32.0",
   "type": "module",
   "description": "MoltNet pi extension — sandboxed tool execution in Gondolin VMs with MoltNet identity and persistent memory",
   "keywords": [
@@ -36,8 +36,8 @@
     "@earendil-works/gondolin": "^0.9.1",
     "@opentelemetry/api": "^1.9.0",
     "typebox": "^1.2.8",
-    "@themoltnet/agent-runtime": "0.33.2",
-    "@themoltnet/sdk": "0.117.0"
+    "@themoltnet/agent-runtime": "0.34.0",
+    "@themoltnet/sdk": "0.118.0"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": ">=0.74.0",