@granular-software/sdk 0.4.46 → 0.4.47

package/dist/agent-evals.mjs

@@ -19862,6 +19862,202 @@ async function writeJson(filePath, value) {
   await writeFile(filePath, `${JSON.stringify(value, null, 2)}
 `);
 }
+function safePathSegment(value, fallback) {
+  const normalized = (value || "").trim().toLowerCase();
+  const sanitized = normalized.replace(/[^a-z0-9._-]+/g, "-").replace(/-+/g, "-").replace(/^-+|-+$/g, "").slice(0, 128);
+  return sanitized || fallback;
+}
+function extractToolCalls(rawGeneration) {
+  const raw = asRecord6(rawGeneration);
+  if (!raw) return null;
+  const choices = asArray3(raw.choices);
+  const firstChoiceMessage = asRecord6(choices[0]);
+  const message = asRecord6(firstChoiceMessage?.message);
+  const toolCalls = asArray3(message?.tool_calls);
+  if (toolCalls.length > 0) return toolCalls;
+  return asArray3(raw.tool_calls).length > 0 ? asArray3(raw.tool_calls) : null;
+}
+function buildTurnMdxReport(input) {
+  const {
+    status,
+    conversation,
+    request,
+    requestTimestamp,
+    turnLog,
+    responseText,
+    terminalKind,
+    actionSummary = [],
+    promptInteractions = [],
+    result,
+    prompts = [],
+    verification,
+    error
+  } = input;
+  const iterationLines = [];
+  for (const iteration of turnLog.iterations) {
+    iterationLines.push(`### Iteration ${iteration.iteration}`);
+    iterationLines.push(
+      `- Request: ${iteration.request}`,
+      `- Generation duration: ${iteration.generationDurationMs !== void 0 ? `${iteration.generationDurationMs}ms` : "unknown"}`
+    );
+    if (iteration.templateId) {
+      iterationLines.push(
+        `- Template: ${iteration.templateId}@${iteration.templateVersion || "unknown"}`
+      );
+    }
+    if (iteration.templateHash) {
+      iterationLines.push(`- Template hash: ${iteration.templateHash}`);
+    }
+    if (iteration.promptInstanceHash) {
+      iterationLines.push(`- Prompt hash: ${iteration.promptInstanceHash}`);
+    }
+    iterationLines.push("", "#### Full prompt sent to LLM", "");
+    iterationLines.push(fenced(iteration.systemPrompt, "text"));
+    if (iteration.generationReply?.trim()) {
+      iterationLines.push(
+        "",
+        "#### LLM reply text",
+        iteration.generationReply.trim(),
+        ""
+      );
+    }
+    if (iteration.generatedCode?.trim()) {
+      iterationLines.push("#### Generated code", "", fenced(iteration.generatedCode.trim(), "ts"));
+    }
+    const toolCalls = extractToolCalls(iteration.rawGeneration);
+    iterationLines.push("#### Tool calls / raw generation", "");
+    if (toolCalls) {
+      iterationLines.push(fenced(JSON.stringify(toolCalls, null, 2), "json"));
+    } else if (iteration.rawGeneration) {
+      iterationLines.push(fenced(JSON.stringify(iteration.rawGeneration, null, 2), "json"));
+    } else {
+      iterationLines.push("_No tool call information._");
+    }
+    if (iteration.tokenUsage) {
+      iterationLines.push("", "#### Token usage", ...formatTokenUsage(iteration.tokenUsage));
+    }
+    if (iteration.responseText?.trim()) {
+      iterationLines.push("", "#### Runtime/prompt outcome", iteration.responseText);
+    }
+    if (iteration.actionSummary?.length) {
+      iterationLines.push("", "#### Action summary", "");
+      iterationLines.push(...iteration.actionSummary.map((line) => `- ${line}`));
+    }
+    if (iteration.continuation) {
+      iterationLines.push("", "#### Continuation", "", jsonBlock(iteration.continuation));
+    }
+    if (iteration.result !== void 0) {
+      iterationLines.push("", "#### Result", "", jsonBlock(iteration.result));
+    }
+    if (iteration.error) {
+      iterationLines.push("", `#### Error`, "", iteration.error);
+    }
+    iterationLines.push("", "---", "");
+  }
+  const promptEventLines = conversation.promptEvents.filter((event) => event.receivedAt >= requestTimestamp).map((event, index) => {
+    const prompt = event.prompt;
+    return `${index + 1}. ${prompt.type} ${prompt.title || ""} ${prompt.message ? `\u2014 ${prompt.message}` : ""}`;
+  });
+  const historyLines = conversation.history.map((entry, index) => {
+    const label = `${index + 1}. ${entry.role}`;
+    const detail = entry.content || (entry.code ? "(code)" : "");
+    return `${label}: ${detail || "(no text)"}`;
+  });
+  const lines = [
+    "# Local Agent Query Report",
+    "",
+    `- timestamp: ${new Date(requestTimestamp).toISOString()}`,
+    `- turn: ${turnLog.turnNumber} (${turnLog.turnId})`,
+    `- status: ${status}`,
+    "",
+    "## Table of contents",
+    "- [Metadata](#metadata)",
+    "- [User query and context](#user-query-and-context)",
+    "- [Conversation history](#conversation-history)",
+    "- [Harness loop iterations](#harness-loop-iterations)",
+    "- [Result and interactions](#result-and-interactions)",
+    "",
+    "## Metadata",
+    `- Ontology: ${conversation.environment.ontologyId}`,
+    `- Subject: ${conversation.environment.subjectId}`,
+    `- Session: ${conversation.environment.sessionId}`,
+    `- Environment: ${conversation.environment.environmentId}`,
+    `- Sandbox: ${conversation.environment.sandboxId}`,
+    `- Permission profile: ${conversation.environment.permissionProfileId}`,
+    `- Conversation artifacts: ${turnLog.turnDir}`,
+    "",
+    "## User query and context",
+    "",
+    "### Request",
+    fenced(request, "text"),
+    "",
+    "### Open prompts at/after request time"
+  ];
+  if (promptEventLines.length) {
+    lines.push(...promptEventLines.map((line) => `- ${line}`));
+  } else {
+    lines.push("- _None");
+  }
+  lines.push(
+    "",
+    "## Conversation history",
+    ...historyLines.map((line) => `- ${line}`),
+    "",
+    "## Harness loop iterations",
+    "",
+    ...iterationLines,
+    "## Result and interactions",
+    ""
+  );
+  if (responseText) {
+    lines.push("### Final response", responseText, "");
+  }
+  if (terminalKind) {
+    lines.push(`### Terminal kind`, terminalKind, "");
+  }
+  lines.push("### Action summary");
+  if (actionSummary.length) {
+    lines.push(...actionSummary.map((line) => `- ${line}`));
+  } else {
+    lines.push("- None");
+  }
+  lines.push("", "### User interactions");
+  if (promptInteractions.length) {
+    for (const interaction of promptInteractions) {
+      lines.push(
+        `- ${interaction.type} ${interaction.message || interaction.title} -> ${JSON.stringify(interaction.answer)}`
+      );
+    }
+  } else {
+    lines.push("- None");
+  }
+  lines.push("", "### Pending prompts");
+  if (prompts.length) {
+    for (const prompt of prompts) {
+      lines.push(`- ${prompt.type} ${prompt.title || ""} ${prompt.message || ""}`);
+    }
+  } else {
+    lines.push("- None");
+  }
+  if (result !== void 0) {
+    lines.push("", "### Result payload", "", jsonBlock(result));
+  }
+  if (verification !== void 0) {
+    lines.push("", "### Verification", "", jsonBlock(verification));
+  }
+  if (error) {
+    lines.push("", `### Error`, "", error);
+  }
+  if (turnLog.error) {
+    lines.push("", "### Turn log error", "", turnLog.error);
+  }
+  lines.push("");
+  if (iterationLines.length === 0) {
+    lines.splice(lines.indexOf("## Harness loop iterations") + 1, 0, "- _No iterations recorded._");
+  }
+  return `${lines.join("\n")}
+`;
+}
 function describeScenarioBehavior(result) {
   if (result.scenario.description?.trim()) {
     return result.scenario.description.trim();
@@ -20642,6 +20838,16 @@ function createAgentEvalHarness(options) {
   const chatTimeoutMs = options.chatTimeoutMs ?? 12e4;
   const jobTimeoutMs = options.jobTimeoutMs ?? 9e4;
   const pollIntervalMs = options.pollIntervalMs ?? 250;
+  const isTruthyEnv = (value) => {
+    return value?.trim().toLowerCase() === "1" || value?.trim().toLowerCase() === "true" || value?.trim().toLowerCase() === "yes" || value?.trim().toLowerCase() === "on";
+  };
+  const localTurnReportsEnabled = (() => {
+    if (typeof options.local === "boolean") return options.local;
+    return process.env.NODE_ENV === "development" || isTruthyEnv(process.env.GRANULAR_LOCAL) || isTruthyEnv(process.env.GRANULAR_USE_LOCAL_ENDPOINTS) || isTruthyEnv(process.env.GRANULAR_USE_LOCAL) || process.env.GRANULAR_ENDPOINT_MODE?.toLowerCase() === "local";
+  })();
+  const localTurnReportBaseDir = path.resolve(
+    options.localTurnReportBaseDir || process.cwd()
+  );
   const resolvedTemplate = resolveHarnessTemplate(
     options.harnessTemplateId || process.env.GRANULAR_AGENT_HARNESS_TEMPLATE || "stable");
   const promptRenderer = options.promptRenderer || resolvedTemplate.renderPrompt;
@@ -20690,6 +20896,41 @@ function createAgentEvalHarness(options) {
     } catch {
     }
   }
+  async function writeLocalTurnReport(input) {
+    if (!localTurnReportsEnabled) return;
+    const { status, conversation, turnLog, request, requestTimestamp, error } = input;
+    const requestId = safePathSegment(
+      new Date(requestTimestamp).toISOString().replace(/[:.]/g, "-"),
+      "turn"
+    );
+    const reportDir = path.join(
+      localTurnReportBaseDir,
+      safePathSegment(conversation.environment.ontologyId, "ontology"),
+      safePathSegment(conversation.environment.subjectId, "subject"),
+      safePathSegment(conversation.environment.sessionId, "session")
+    );
+    await ensureDir(reportDir);
+    const completed = input.completed;
+    const pending = input.pending;
+    const report = buildTurnMdxReport({
+      status,
+      conversation,
+      request,
+      requestTimestamp,
+      turnLog,
+      responseText: completed?.responseText,
+      terminalKind: completed?.terminalKind,
+      actionSummary: completed?.actionSummary,
+      promptInteractions: completed?.promptInteractions || pending?.promptInteractions,
+      result: completed?.result,
+      prompts: pending?.prompts,
+      verification: completed?.verification,
+      error
+    });
+    const reportPath = path.join(reportDir, `${requestId}.mdx`);
+    await writeFile(reportPath, report);
+    console.log(`[agent][local] saved query report: ${reportPath}`);
+  }
   async function runCheckJob(code, session) {
     const job = await session.submitJob(code);
     return withTimeout2(job.result, jobTimeoutMs, `check job ${job.id}`);
@@ -20949,99 +21190,96 @@ function createAgentEvalHarness(options) {
     const baselineClosureId = getCurrentClosureId(
       cloneJson(conversation.environment.document)
     );
-    while (iteration < maxIterations) {
-      const liveDoc = cloneJson(conversation.environment.document);
-      const promptHeap = normalizeHeapSnapshot2(asRecord6(liveDoc?.heap));
-      const pendingPrompts = filterPromptsByBoundary(
-        liveDoc,
-        getOpenPromptsFromDoc(liveDoc),
-        boundaryTimestamp
-      );
-      const workflowFocus = projectWorkflowFocus(liveDoc, pendingPrompts, {
-        boundaryTimestamp
+    const writeTurnReport = async (status) => {
+      await writeLocalTurnReport({
+        status: status.type,
+        conversation,
+        turnLog,
+        request: input.request,
+        requestTimestamp: boundaryTimestamp,
+        completed: status.completed,
+        pending: status.pending,
+        error: status.error
       });
-      const referentFocus = projectConversationReferentFocus(liveDoc);
-      const heapFocus = {
-        variableNames: [
-          ...workflowFocus.variableNames,
-          ...referentFocus.variableNames
-        ],
-        listNames: [...workflowFocus.listNames, ...referentFocus.listNames],
-        entryPaths: [
-          ...workflowFocus.entryPaths,
-          ...referentFocus.entryPaths,
-          ...focusedHeapEntryPathsFromUiContext(promptHeap, input.uiContext)
-        ]
-      };
-      const tools = conversation.environment.getEffects().map((tool) => ({
-        name: tool.name,
-        description: tool.description,
-        className: tool.className,
-        static: tool.static,
-        ready: tool.ready,
-        inputSchema: tool.inputSchema,
-        outputSchema: tool.outputSchema
-      }));
-      const renderedPrompt = promptRenderer({
-        domainDocumentation: await conversation.environment.getDomainDocumentation(),
-        sessionContext: {
-          sandboxId: conversation.environment.sandboxId,
-          environmentId: conversation.environment.environmentId,
-          domainRevision: conversation.environment.domainRevision,
-          uiContext: input.uiContext || null
-        },
-        heapSummary: projectHeapSummary(asRecord6(liveDoc?.heap), {
-          focus: heapFocus
-        }),
-        fileSummary: projectSessionFileSummary(liveDoc),
-        referentSummary: projectConversationReferentSummary(liveDoc),
-        loopSummary: projectLoopSummary(liveDoc, pendingPrompts, {
+    };
+    try {
+      while (iteration < maxIterations) {
+        const liveDoc = cloneJson(conversation.environment.document);
+        const promptHeap = normalizeHeapSnapshot2(asRecord6(liveDoc?.heap));
+        const pendingPrompts = filterPromptsByBoundary(
+          liveDoc,
+          getOpenPromptsFromDoc(liveDoc),
           boundaryTimestamp
-        }),
-        workflowSummary: projectWorkflowSummary(liveDoc, pendingPrompts, {
+        );
+        const workflowFocus = projectWorkflowFocus(liveDoc, pendingPrompts, {
           boundaryTimestamp
-        }),
-        tools,
-        checkpoint: latestCheckpoint
-      });
-      const systemPrompt = renderedPrompt.prompt;
-      await emitProgress(onProgress, {
-        phase: "prompt",
-        status: "passed",
-        scenarioId: conversation.label,
-        stepId: turnId,
-        iteration: iteration + 1,
-        templateId: renderedPrompt.templateId,
-        templateVersion: renderedPrompt.templateVersion,
-        title: "Rendered harness prompt",
-        message: `${systemPrompt.split("\n").length} lines`,
-        data: {
+        });
+        const referentFocus = projectConversationReferentFocus(liveDoc);
+        const heapFocus = {
+          variableNames: [
+            ...workflowFocus.variableNames,
+            ...referentFocus.variableNames
+          ],
+          listNames: [...workflowFocus.listNames, ...referentFocus.listNames],
+          entryPaths: [
+            ...workflowFocus.entryPaths,
+            ...referentFocus.entryPaths,
+            ...focusedHeapEntryPathsFromUiContext(promptHeap, input.uiContext)
+          ]
+        };
+        const tools = conversation.environment.getEffects().map((tool) => ({
+          name: tool.name,
+          description: tool.description,
+          className: tool.className,
+          static: tool.static,
+          ready: tool.ready,
+          inputSchema: tool.inputSchema,
+          outputSchema: tool.outputSchema
+        }));
+        const renderedPrompt = promptRenderer({
+          domainDocumentation: await conversation.environment.getDomainDocumentation(),
+          sessionContext: {
+            sandboxId: conversation.environment.sandboxId,
+            environmentId: conversation.environment.environmentId,
+            domainRevision: conversation.environment.domainRevision,
+            uiContext: input.uiContext || null
+          },
+          heapSummary: projectHeapSummary(asRecord6(liveDoc?.heap), {
+            focus: heapFocus
+          }),
+          fileSummary: projectSessionFileSummary(liveDoc),
+          referentSummary: projectConversationReferentSummary(liveDoc),
+          loopSummary: projectLoopSummary(liveDoc, pendingPrompts, {
+            boundaryTimestamp
+          }),
+          workflowSummary: projectWorkflowSummary(liveDoc, pendingPrompts, {
+            boundaryTimestamp
+          }),
+          tools,
+          checkpoint: latestCheckpoint
+        });
+        const systemPrompt = renderedPrompt.prompt;
+        await emitProgress(onProgress, {
+          phase: "prompt",
+          status: "passed",
+          scenarioId: conversation.label,
+          stepId: turnId,
+          iteration: iteration + 1,
           templateId: renderedPrompt.templateId,
           templateVersion: renderedPrompt.templateVersion,
-          templateHash: renderedPrompt.templateHash,
-          promptInstanceHash: renderedPrompt.promptInstanceHash,
-          prompt: systemPrompt
-        }
-      });
-      const request = iteration === 0 ? input.request : continuationRenderer(
-        buildContinuationPreview(latestCheckpoint, noProgressCount)
-      ).instruction;
-      await emitProgress(onProgress, {
-        phase: "generation",
-        status: "running",
-        scenarioId: conversation.label,
-        stepId: turnId,
-        iteration: iteration + 1,
-        templateId: renderedPrompt.templateId,
-        templateVersion: renderedPrompt.templateVersion,
-        title: iteration === 0 ? "Generating agent response" : "Generating continuation",
-        message: request
-      });
-      const emittedGeneratedReasoningLines = /* @__PURE__ */ new Set();
-      let generatedReasoningBuffer = "";
-      const emitGeneratedReasoningLine = async (line) => {
-        if (emittedGeneratedReasoningLines.has(line)) return;
-        emittedGeneratedReasoningLines.add(line);
+          title: "Rendered harness prompt",
+          message: `${systemPrompt.split("\n").length} lines`,
+          data: {
+            templateId: renderedPrompt.templateId,
+            templateVersion: renderedPrompt.templateVersion,
+            templateHash: renderedPrompt.templateHash,
+            promptInstanceHash: renderedPrompt.promptInstanceHash,
+            prompt: systemPrompt
+          }
+        });
+        const request = iteration === 0 ? input.request : continuationRenderer(
+          buildContinuationPreview(latestCheckpoint, noProgressCount)
+        ).instruction;
         await emitProgress(onProgress, {
           phase: "generation",
           status: "running",
@@ -21050,190 +21288,231 @@ function createAgentEvalHarness(options) {
           iteration: iteration + 1,
           templateId: renderedPrompt.templateId,
           templateVersion: renderedPrompt.templateVersion,
-          title: "Generated reasoning comment",
-          message: line,
-          data: { source: "generated_code_comment" }
+          title: iteration === 0 ? "Generating agent response" : "Generating continuation",
+          message: request
         });
-      };
-      const generation = await withTimeout2(
-        generateTurnWithRepair(options.generator, {
-          systemPrompt,
-          history: buildHistory(conversation.history),
-          request,
-          attempt: 1,
-          tools,
-          onReplyDelta: async (delta) => {
-            await emitProgress(onProgress, {
-              phase: "generation",
-              status: "running",
-              scenarioId: conversation.label,
-              stepId: turnId,
-              iteration: iteration + 1,
-              templateId: renderedPrompt.templateId,
-              templateVersion: renderedPrompt.templateVersion,
-              title: "Generated text reply delta",
-              message: delta,
-              data: { delta }
-            });
-          },
-          onCodeDelta: async (delta) => {
-            const parsed = consumeGranularReasoningOnlyChunk(
-              generatedReasoningBuffer,
-              delta
-            );
-            generatedReasoningBuffer = parsed.buffer;
-            for (const line of parsed.reasoningLines) {
-              await emitGeneratedReasoningLine(line);
-            }
-          },
-          usageContext: {
-            sandboxId: conversation.environment.sandboxId,
-            environmentId: conversation.environment.environmentId,
-            sessionId: conversation.environment.sessionId,
-            subjectId: conversation.environment.subjectId,
-            permissionProfileId: conversation.environment.permissionProfileId
-          }
-        }),
-        chatTimeoutMs,
-        `chat generation for ${conversation.label} iteration ${iteration + 1}`
-      );
-      await emitProgress(onProgress, {
-        phase: "generation",
-        status: "passed",
-        scenarioId: conversation.label,
-        stepId: turnId,
-        iteration: iteration + 1,
-        templateId: renderedPrompt.templateId,
-        templateVersion: renderedPrompt.templateVersion,
-        title: generation.code ? "Generated job code" : "Generated text reply",
-        message: generation.code || generation.reply || "",
-        data: {
-          reply: generation.reply,
-          code: generation.code,
-          attempts: generation.generationAttempts,
-          usage: tokenUsageForGenerationOutput(generation)
-        }
-      });
-      const generatedReasoningLines = generation.code ? consumeGranularReasoningOnlyChunk("", generation.code, {
-        final: true
-      }).reasoningLines : [];
-      for (const line of generatedReasoningLines) {
-        await emitGeneratedReasoningLine(line);
-      }
-      const iterationLog = {
-        iteration: iteration + 1,
-        request,
-        systemPrompt,
-        templateId: renderedPrompt.templateId,
-        templateVersion: renderedPrompt.templateVersion,
-        templateHash: renderedPrompt.templateHash,
-        promptInstanceHash: renderedPrompt.promptInstanceHash,
-        generationReply: generation.reply,
-        generatedCode: generation.code,
-        rawGeneration: generation.raw,
-        generationAttempts: generation.generationAttempts,
-        tokenUsage: tokenUsageForGenerationOutput(generation)
-      };
-      turnLog.iterations.push(iterationLog);
-      await writeJson(
-        path.join(turnDir, `iteration-${iteration + 1}-generation.json`),
-        generation
-      );
-      if (!generation.code) {
-        const responseText2 = generation.reply?.trim() || "Done.";
-        conversation.history.push({ role: "assistant", content: responseText2 });
-        const completed = {
-          conversation,
-          request: input.request,
-          turnDir,
-          responseText: responseText2,
-          terminalKind: "reply",
-          actionSummary: [],
-          promptInteractions: [],
-          verification: null,
-          result: generation.reply?.trim() || responseText2
-        };
-        if (input.verification) {
-          completed.verification = await runInspection(
-            conversation,
-            input.verification,
-            completed,
-            turnDir
-          );
-        }
-        iterationLog.responseText = responseText2;
-        iterationLog.terminalKind = "reply";
-        iterationLog.actionSummary = [];
-        iterationLog.promptInteractions = [];
-        iterationLog.result = completed.result;
-        turnLog.completed = {
-          responseText: responseText2,
-          terminalKind: "reply",
-          actionSummary: [],
-          promptInteractions: [],
-          result: completed.result
+        const emittedGeneratedReasoningLines = /* @__PURE__ */ new Set();
+        let generatedReasoningBuffer = "";
+        const emitGeneratedReasoningLine = async (line) => {
+          if (emittedGeneratedReasoningLines.has(line)) return;
+          emittedGeneratedReasoningLines.add(line);
+          await emitProgress(onProgress, {
+            phase: "generation",
+            status: "running",
+            scenarioId: conversation.label,
+            stepId: turnId,
+            iteration: iteration + 1,
+            templateId: renderedPrompt.templateId,
+            templateVersion: renderedPrompt.templateVersion,
+            title: "Generated reasoning comment",
+            message: line,
+            data: { source: "generated_code_comment" }
+          });
         };
-        await writeJson(path.join(turnDir, "result.json"), completed);
+        const generationStartedAt = Date.now();
+        const generation = await withTimeout2(
+          generateTurnWithRepair(options.generator, {
+            systemPrompt,
+            history: buildHistory(conversation.history),
+            request,
+            attempt: 1,
+            tools,
+            onReplyDelta: async (delta) => {
+              await emitProgress(onProgress, {
+                phase: "generation",
+                status: "running",
+                scenarioId: conversation.label,
+                stepId: turnId,
+                iteration: iteration + 1,
+                templateId: renderedPrompt.templateId,
+                templateVersion: renderedPrompt.templateVersion,
+                title: "Generated text reply delta",
+                message: delta,
+                data: { delta }
+              });
+            },
+            onCodeDelta: async (delta) => {
+              const parsed = consumeGranularReasoningOnlyChunk(
+                generatedReasoningBuffer,
+                delta
+              );
+              generatedReasoningBuffer = parsed.buffer;
+              for (const line of parsed.reasoningLines) {
+                await emitGeneratedReasoningLine(line);
+              }
+            },
+            usageContext: {
+              sandboxId: conversation.environment.sandboxId,
+              environmentId: conversation.environment.environmentId,
+              sessionId: conversation.environment.sessionId,
+              subjectId: conversation.environment.subjectId,
+              permissionProfileId: conversation.environment.permissionProfileId
+            }
+          }),
+          chatTimeoutMs,
+          `chat generation for ${conversation.label} iteration ${iteration + 1}`
+        );
+        const generationDurationMs = Date.now() - generationStartedAt;
         await emitProgress(onProgress, {
-          phase: "step",
+          phase: "generation",
           status: "passed",
           scenarioId: conversation.label,
           stepId: turnId,
-          title: "Step completed with text reply",
-          message: responseText2,
-          data: completed
+          iteration: iteration + 1,
+          templateId: renderedPrompt.templateId,
+          templateVersion: renderedPrompt.templateVersion,
+          title: generation.code ? "Generated job code" : "Generated text reply",
+          message: generation.code || generation.reply || "",
+          data: {
+            reply: generation.reply,
+            code: generation.code,
+            attempts: generation.generationAttempts,
+            usage: tokenUsageForGenerationOutput(generation)
+          }
         });
-        return completed;
-      }
-      const session = conversation.environment;
-      const job = await session.submitJob(generation.code, {
-        agent: {
-          userRequest: input.request,
-          generationRequest: request,
-          systemPrompt,
-          history: buildHistory(conversation.history),
-          scenarioLabel: conversation.label,
-          turnId,
+        const generatedReasoningLines = generation.code ? consumeGranularReasoningOnlyChunk("", generation.code, {
+          final: true
+        }).reasoningLines : [];
+        for (const line of generatedReasoningLines) {
+          await emitGeneratedReasoningLine(line);
+        }
+        const iterationLog = {
           iteration: iteration + 1,
-          tools,
+          request,
+          generationDurationMs,
+          systemPrompt,
+          templateId: renderedPrompt.templateId,
+          templateVersion: renderedPrompt.templateVersion,
+          templateHash: renderedPrompt.templateHash,
+          promptInstanceHash: renderedPrompt.promptInstanceHash,
           generationReply: generation.reply,
+          generatedCode: generation.code,
           rawGeneration: generation.raw,
-          repairIssues: generation.generationAttempts?.flatMap(
-            (attempt) => attempt.repairIssues || []
-          )
-        }
-      });
-      await emitProgress(onProgress, {
-        phase: "job",
-        status: "running",
-        scenarioId: conversation.label,
-        stepId: turnId,
-        iteration: iteration + 1,
-        jobId: job.id,
-        templateId: renderedPrompt.templateId,
-        templateVersion: renderedPrompt.templateVersion,
-        title: "Submitted Granular job",
-        message: job.id,
-        data: { code: generation.code }
-      });
-      const outcome = await waitForJobOutcome({
-        environment: conversation.environment,
-        job,
-        boundaryTimestamp,
-        timeoutMs: jobTimeoutMs,
-        pollIntervalMs,
-        onProgress: (event) => void onProgress?.(event),
-        progressContext: {
+          generationAttempts: generation.generationAttempts,
+          tokenUsage: tokenUsageForGenerationOutput(generation)
+        };
+        turnLog.iterations.push(iterationLog);
+        await writeJson(
+          path.join(turnDir, `iteration-${iteration + 1}-generation.json`),
+          generation
+        );
+        if (!generation.code) {
+          const responseText2 = generation.reply?.trim() || "Done.";
+          conversation.history.push({ role: "assistant", content: responseText2 });
+          const completed = {
+            conversation,
+            request: input.request,
+            turnDir,
+            responseText: responseText2,
+            terminalKind: "reply",
+            actionSummary: [],
+            promptInteractions: [],
+            verification: null,
+            result: generation.reply?.trim() || responseText2
+          };
+          if (input.verification) {
+            completed.verification = await runInspection(
+              conversation,
+              input.verification,
+              completed,
+              turnDir
+            );
+          }
+          iterationLog.responseText = responseText2;
+          iterationLog.terminalKind = "reply";
+          iterationLog.actionSummary = [];
+          iterationLog.promptInteractions = [];
+          iterationLog.result = completed.result;
+          turnLog.completed = {
+            responseText: responseText2,
+            terminalKind: "reply",
+            actionSummary: [],
+            promptInteractions: [],
+            result: completed.result
+          };
+          await writeJson(path.join(turnDir, "result.json"), completed);
+          await writeTurnReport({ type: "completed", completed });
+          await emitProgress(onProgress, {
+            phase: "step",
+            status: "passed",
+            scenarioId: conversation.label,
+            stepId: turnId,
+            title: "Step completed with text reply",
+            message: responseText2,
+            data: completed
+          });
+          return completed;
+        }
+        const session = conversation.environment;
+        const job = await session.submitJob(generation.code, {
+          agent: {
+            userRequest: input.request,
+            generationRequest: request,
+            systemPrompt,
+            history: buildHistory(conversation.history),
+            scenarioLabel: conversation.label,
+            turnId,
+            iteration: iteration + 1,
+            tools,
+            generationReply: generation.reply,
+            rawGeneration: generation.raw,
+            repairIssues: generation.generationAttempts?.flatMap(
+              (attempt) => attempt.repairIssues || []
+            )
+          }
+        });
+        await emitProgress(onProgress, {
+          phase: "job",
+          status: "running",
           scenarioId: conversation.label,
           stepId: turnId,
           iteration: iteration + 1,
+          jobId: job.id,
           templateId: renderedPrompt.templateId,
-          templateVersion: renderedPrompt.templateVersion
-        }
-      });
-      if (outcome.kind === "prompt") {
-        if (!autoAnswerPrompts) {
-          return {
+          templateVersion: renderedPrompt.templateVersion,
+          title: "Submitted Granular job",
+          message: job.id,
+          data: { code: generation.code }
+        });
+        const outcome = await waitForJobOutcome({
+          environment: conversation.environment,
+          job,
+          boundaryTimestamp,
+          timeoutMs: jobTimeoutMs,
+          pollIntervalMs,
+          onProgress: (event) => void onProgress?.(event),
+          progressContext: {
+            scenarioId: conversation.label,
+            stepId: turnId,
+            iteration: iteration + 1,
+            templateId: renderedPrompt.templateId,
+            templateVersion: renderedPrompt.templateVersion
+          }
+        });
+        if (outcome.kind === "prompt") {
+          if (!autoAnswerPrompts) {
+            const pending2 = {
+              conversation,
+              request: input.request,
+              turnDir,
+              boundaryTimestamp,
+              finalCode: generation.code,
+              finalReply: generation.reply?.trim() || "",
+              stdout: outcome.stdout,
+              stderr: outcome.stderr,
+              prompts: outcome.prompts,
+              promptInteractions: [],
+              job
+            };
+            await writeTurnReport({ type: "pending", pending: pending2 });
+            return pending2;
+          }
+          if (!input.human) {
+            throw new Error(
+              "This turn reached a human prompt but no responder was provided"
+            );
+          }
+          let pending = {
             conversation,
             request: input.request,
             turnDir,
@@ -21246,187 +21525,180 @@ function createAgentEvalHarness(options) {
             promptInteractions: [],
             job
           };
+          while ("prompts" in pending) {
+            const resumed = await resumePendingTurn(pending, input.human);
+            if ("prompts" in resumed) {
+              pending = resumed;
+              continue;
+            }
+            if (input.verification) {
+              resumed.verification = await runInspection(
+                conversation,
+                input.verification,
+                resumed,
+                turnDir
+              );
+            }
+            turnLog.completed = {
+              responseText: resumed.responseText,
+              terminalKind: resumed.terminalKind,
+              actionSummary: resumed.actionSummary,
+              promptInteractions: resumed.promptInteractions,
+              result: resumed.result
+            };
+            await writeTurnReport({ type: "completed", completed: resumed });
+            return resumed;
+          }
         }
-        if (!input.human) {
+        if (outcome.kind !== "completed") {
           throw new Error(
-            "This turn reached a human prompt but no responder was provided"
+            "Unexpected non-completed outcome after prompt handling"
           );
         }
-        let pending = {
-          conversation,
-          request: input.request,
-          turnDir,
-          boundaryTimestamp,
-          finalCode: generation.code,
-          finalReply: generation.reply?.trim() || "",
+        await sleep2(350);
+        const settledLiveDoc = cloneJson(
+          conversation.environment.document
+        );
+        const sessionHeap = normalizeHeapSnapshot2(asRecord6(settledLiveDoc?.heap));
+        const presentation = resolveJobPresentation({
+          jobId: job.id,
+          result: outcome.result,
           stdout: outcome.stdout,
-          stderr: outcome.stderr,
-          prompts: outcome.prompts,
-          promptInteractions: [],
-          job
+          agentMessages: getJobAgentMessages(settledLiveDoc, job.id),
+          sessionHeap
+        });
+        const responseText = presentation.responseText || generation.reply?.trim() || "Done.";
+        const verifierSnapshot = createHarnessVerifierSnapshot({
+          finalCode: generation.code,
+          resultPreview: JSON.stringify(outcome.result, null, 2),
+          liveDoc: settledLiveDoc,
+          projectionOptions: { boundaryTimestamp }
+        });
+        const continuation = evaluateContinuation({
+          iteration,
+          budgets: controllerBudgets,
+          baselineClosureId,
+          currentClosureId: getCurrentClosureId(settledLiveDoc),
+          liveDoc: settledLiveDoc,
+          pendingPrompts: filterPromptsByBoundary(
+            settledLiveDoc,
+            getOpenPromptsFromDoc(settledLiveDoc),
+            boundaryTimestamp
+          ),
+          projectionOptions: { boundaryTimestamp },
+          latestResponseText: responseText,
+          previousSnapshot,
+          currentSnapshot: verifierSnapshot,
+          previousNoProgressCount: noProgressCount
+        });
+        latestCheckpoint = {
+          iteration: iteration + 1,
+          latestJobStatus: "succeeded",
+          latestJobResult: JSON.stringify(outcome.result, null, 2),
+          latestActionSummary: getActionSummary(settledLiveDoc, job.id),
+          controllerOutcome: continuation.outcome,
+          controllerReason: continuation.reason,
+          noProgressCount: continuation.nextNoProgressCount
         };
-        while ("prompts" in pending) {
-          const resumed = await resumePendingTurn(pending, input.human);
-          if ("prompts" in resumed) {
-            pending = resumed;
-            continue;
+        await emitProgress(onProgress, {
+          phase: "continuation",
+          status: continuation.shouldContinue ? "running" : "passed",
+          scenarioId: conversation.label,
+          stepId: turnId,
+          iteration: iteration + 1,
+          jobId: job.id,
+          templateId: renderedPrompt.templateId,
+          templateVersion: renderedPrompt.templateVersion,
+          title: continuation.shouldContinue ? "Harness requested another loop" : "Harness accepted completion",
+          message: `${continuation.reason}; ${continuation.outcome}`,
+          data: {
+            continuation,
+            checkpoint: latestCheckpoint,
+            verifierSnapshot
+          }
+        });
+        previousSnapshot = verifierSnapshot;
+        noProgressCount = continuation.nextNoProgressCount;
+        conversation.history.push({
+          role: "assistant",
+          content: responseText,
+          code: generation.code,
+          jobStatus: "succeeded",
+          jobResultPreview: JSON.stringify(outcome.result, null, 2)
+        });
+        await writeJson(
+          path.join(turnDir, `iteration-${iteration + 1}-result.json`),
+          {
+            responseText,
+            continuation,
+            actionSummary: latestCheckpoint.latestActionSummary,
+            result: outcome.result
           }
+        );
+        iterationLog.responseText = responseText;
+        iterationLog.terminalKind = getCurrentClosureId(settledLiveDoc) ? "closure" : "reply";
+        iterationLog.actionSummary = latestCheckpoint.latestActionSummary || [];
+        iterationLog.promptInteractions = [];
+        iterationLog.continuation = continuation;
+        iterationLog.result = outcome.result;
+        if (!continuation.shouldContinue) {
+          const completed = {
+            conversation,
+            request: input.request,
+            turnDir,
+            responseText,
+            terminalKind: getCurrentClosureId(settledLiveDoc) ? "closure" : "reply",
+            finalCode: generation.code,
+            actionSummary: latestCheckpoint.latestActionSummary || [],
+            promptInteractions: [],
+            verification: null,
+            result: outcome.result
+          };
           if (input.verification) {
-            resumed.verification = await runInspection(
+            completed.verification = await runInspection(
               conversation,
               input.verification,
-              resumed,
+              completed,
               turnDir
             );
           }
           turnLog.completed = {
-            responseText: resumed.responseText,
-            terminalKind: resumed.terminalKind,
-            actionSummary: resumed.actionSummary,
-            promptInteractions: resumed.promptInteractions,
-            result: resumed.result
+            responseText,
+            terminalKind: completed.terminalKind,
+            actionSummary: completed.actionSummary,
+            promptInteractions: [],
+            result: outcome.result
           };
-          return resumed;
+          await writeJson(path.join(turnDir, "result.json"), completed);
+          await writeTurnReport({ type: "completed", completed });
+          await emitProgress(onProgress, {
+            phase: "step",
+            status: "passed",
+            scenarioId: conversation.label,
+            stepId: turnId,
+            iteration: iteration + 1,
+            jobId: job.id,
+            title: "Step completed",
+            message: responseText,
+            data: completed
+          });
+          return completed;
         }
+        iteration += 1;
       }
-      if (outcome.kind !== "completed") {
-        throw new Error(
-          "Unexpected non-completed outcome after prompt handling"
-        );
-      }
-      await sleep2(350);
-      const settledLiveDoc = cloneJson(
-        conversation.environment.document
-      );
-      const sessionHeap = normalizeHeapSnapshot2(asRecord6(settledLiveDoc?.heap));
-      const presentation = resolveJobPresentation({
-        jobId: job.id,
-        result: outcome.result,
-        stdout: outcome.stdout,
-        agentMessages: getJobAgentMessages(settledLiveDoc, job.id),
-        sessionHeap
-      });
-      const responseText = presentation.responseText || generation.reply?.trim() || "Done.";
-      const verifierSnapshot = createHarnessVerifierSnapshot({
-        finalCode: generation.code,
-        resultPreview: JSON.stringify(outcome.result, null, 2),
-        liveDoc: settledLiveDoc,
-        projectionOptions: { boundaryTimestamp }
-      });
-      const continuation = evaluateContinuation({
-        iteration,
-        budgets: controllerBudgets,
-        baselineClosureId,
-        currentClosureId: getCurrentClosureId(settledLiveDoc),
-        liveDoc: settledLiveDoc,
-        pendingPrompts: filterPromptsByBoundary(
-          settledLiveDoc,
-          getOpenPromptsFromDoc(settledLiveDoc),
-          boundaryTimestamp
-        ),
-        projectionOptions: { boundaryTimestamp },
-        latestResponseText: responseText,
-        previousSnapshot,
-        currentSnapshot: verifierSnapshot,
-        previousNoProgressCount: noProgressCount
-      });
-      latestCheckpoint = {
-        iteration: iteration + 1,
-        latestJobStatus: "succeeded",
-        latestJobResult: JSON.stringify(outcome.result, null, 2),
-        latestActionSummary: getActionSummary(settledLiveDoc, job.id),
-        controllerOutcome: continuation.outcome,
-        controllerReason: continuation.reason,
-        noProgressCount: continuation.nextNoProgressCount
-      };
-      await emitProgress(onProgress, {
-        phase: "continuation",
-        status: continuation.shouldContinue ? "running" : "passed",
-        scenarioId: conversation.label,
-        stepId: turnId,
-        iteration: iteration + 1,
-        jobId: job.id,
-        templateId: renderedPrompt.templateId,
-        templateVersion: renderedPrompt.templateVersion,
-        title: continuation.shouldContinue ? "Harness requested another loop" : "Harness accepted completion",
-        message: `${continuation.reason}; ${continuation.outcome}`,
-        data: {
-          continuation,
-          checkpoint: latestCheckpoint,
-          verifierSnapshot
-        }
-      });
-      previousSnapshot = verifierSnapshot;
-      noProgressCount = continuation.nextNoProgressCount;
-      conversation.history.push({
-        role: "assistant",
-        content: responseText,
-        code: generation.code,
-        jobStatus: "succeeded",
-        jobResultPreview: JSON.stringify(outcome.result, null, 2)
-      });
-      await writeJson(
-        path.join(turnDir, `iteration-${iteration + 1}-result.json`),
-        {
-          responseText,
-          continuation,
-          actionSummary: latestCheckpoint.latestActionSummary,
-          result: outcome.result
-        }
+      throw new Error(
+        `Turn exceeded iteration budget (${maxIterations}) for ${conversation.label}`
       );
-      iterationLog.responseText = responseText;
-      iterationLog.terminalKind = getCurrentClosureId(settledLiveDoc) ? "closure" : "reply";
-      iterationLog.actionSummary = latestCheckpoint.latestActionSummary || [];
-      iterationLog.promptInteractions = [];
-      iterationLog.continuation = continuation;
-      iterationLog.result = outcome.result;
-      if (!continuation.shouldContinue) {
-        const completed = {
-          conversation,
-          request: input.request,
-          turnDir,
-          responseText,
-          terminalKind: getCurrentClosureId(settledLiveDoc) ? "closure" : "reply",
-          finalCode: generation.code,
-          actionSummary: latestCheckpoint.latestActionSummary || [],
-          promptInteractions: [],
-          verification: null,
-          result: outcome.result
-        };
-        if (input.verification) {
-          completed.verification = await runInspection(
-            conversation,
-            input.verification,
-            completed,
-            turnDir
-          );
-        }
-        turnLog.completed = {
-          responseText,
-          terminalKind: completed.terminalKind,
-          actionSummary: completed.actionSummary,
-          promptInteractions: [],
-          result: outcome.result
-        };
-        await writeJson(path.join(turnDir, "result.json"), completed);
-        await emitProgress(onProgress, {
-          phase: "step",
-          status: "passed",
-          scenarioId: conversation.label,
-          stepId: turnId,
-          iteration: iteration + 1,
-          jobId: job.id,
-          title: "Step completed",
-          message: responseText,
-          data: completed
-        });
-        return completed;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      const latestIteration = latestIterationLog(turnLog);
+      if (latestIteration) {
+        latestIteration.error = message;
       }
-      iteration += 1;
+      turnLog.error = message;
+      await writeTurnReport({ type: "failed", error: message });
+      throw error;
     }
-    throw new Error(
-      `Turn exceeded iteration budget (${maxIterations}) for ${conversation.label}`
-    );
   }
   return {
     artifactDir,
@@ -21461,6 +21733,7 @@ function createAgentTester(options) {
   const harness = createAgentEvalHarness({
     granular,
     environmentId: resolvedEnvironmentId || void 0,
+    local: options.local,
     openEnvironment: async ({ clientId }) => {
       if (resolvedEnvironmentId) {
         return granular.createSession({