@foldrun/core 2.4.0 → 2.6.0

package/src/step-exec.ts

@@ -8,13 +8,14 @@
 // results. That is what lets the same function run somewhere the vault, the
 // library and the account do not exist.
 
+import { judgeVerdict, JUDGE_VERDICT_INSTRUCTION } from "./judge-verdict.ts";
 import { runHook, runToolCallHooks, matchesTool, hookLine, resolveHooks, type HookDef, type HookRecord } from "./hooks.ts";
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
 import { query } from "@anthropic-ai/claude-agent-sdk";
 import type { HookInput, McpServerConfig, SDKMessage } from "@anthropic-ai/claude-agent-sdk";
-import type { Effort } from "./store.ts";
+import { CAPTURE_CAP, type Effort, type ModelCall } from "./store.ts";
 import { costOf, type TokenPrice } from "./catalog.ts";
 import type { TestEffect } from "./test-mode.ts";
 import type { OperatorEvent } from "./operator.ts";
@@ -23,7 +24,7 @@ import { checkPaths, checkBash, isFilesystemTool, linkWorkspace, workspaceLinkEn
 import { DELEGATE_TOOLS, subagentGuard, toAgentDefinitions, type SubagentSpec } from "./subagents.ts";
 import { hostSafeEnv } from "./host-env.ts";
 import { validateSchema, describeSchemaErrors, looksLikeSchema } from "./json-schema.ts";
-import { CallCounter, limitKeysFor, type Limits } from "./limits.ts";
+import { CallCounter, callKeysFor, type Limits } from "./limits.ts";
 import { classifyCall, podAt, readPodEvents, type BrowserPodOutcome, type CallKind, type ClassifyContext } from "./browser-pod.ts";
 
 export interface ExecUsage {
@@ -86,6 +87,11 @@ export interface ExecOutcome {
 }
 
 export interface ExecOptions {
+  /** What the step reports beyond its events (FOL-54 step 4): `metadata`
+   *  sends each model call (`turn`); `full` also sends each tool call's input
+   *  and output (`io`). Unset sends neither — the consult, the probe and any
+   *  caller that does not store them. */
+  capture?: "metadata" | "full";
   /** `hooks:` resolved host-side (hooks.ts): run at this step's own dispatch,
    *  inside its sandbox. `hookBase` is what every hook event carries. */
   hooks?: HookDef[];
@@ -128,6 +134,11 @@ export interface ExecOptions {
    *  `matches: re`, `file: path`, `judge: sentence`) — see checkVerify. */
   verify?: string;
   verifyEnv?: Record<string, string>;
+  /** The model `verify: judge:` grades with — the workspace's `judge:` from
+   *  AGENTS.md, resolved; absent, the fast tier. */
+  judgeModel?: string;
+  /** And how long it thinks — `judge: {effort}`; absent, the SDK's default. */
+  judgeEffort?: Effort;
   /** `output: json` — the reply must carry one JSON value; extracting it is
    *  part of finishing the step, and failing to is failing the step. */
   output?: "json";
@@ -195,7 +206,50 @@ export type QueryLike = AsyncIterable<unknown> & { interrupt(): Promise<unknown>
 export type QueryFn = (args: { prompt: string; options: Record<string, unknown> }) => QueryLike;
 
 /** The pairing fields on a tool event — see RunEvent in store.ts. */
-export type EventExtra = { call?: string; ms?: number; err?: boolean; bytes?: number; effect?: TestEffect; operator?: OperatorEvent; subagent?: string; check?: boolean; hook?: HookRecord };
+export type EventExtra = {
+  call?: string;
+  ms?: number;
+  err?: boolean;
+  bytes?: number;
+  effect?: TestEffect;
+  operator?: OperatorEvent;
+  subagent?: string;
+  check?: boolean;
+  hook?: HookRecord;
+  /** A tool call's content, on its way to the runner (FOL-54 step 4): the
+   *  runner masks it, stores it beside the run and keeps only `captured` on
+   *  the event. Sent only under `capture: "full"`. */
+  io?: CallIo;
+  /** One model call, on its way to the runner's `step.calls` — not an event. */
+  turn?: ModelCall;
+  /** The model's first response of the step has begun streaming — the
+   *  moment the prompt cache entry its request wrote becomes readable. Sent
+   *  once; the runner's fan-out gate reads it. Not an event. */
+  responding?: true;
+  /** Set by the runner once it stored `io` (RunEvent.captured). */
+  captured?: boolean;
+  /** On a tool call: the model call (its message id) that asked for it. */
+  inCall?: string;
+};
+
+/** A tool call's input (on the call) or output (on its completion), capped. */
+export interface CallIo {
+  input?: string;
+  output?: string;
+  /** The whole input's / output's length, in characters, before any cut. */
+  inputChars?: number;
+  outputChars?: number;
+  truncated?: boolean;
+}
+
+// The cap plus slack: the runner masks, then cuts to the cap (capture.ts), so
+// a secret on the edge is masked whole.
+const SEND_CAP = CAPTURE_CAP + 4096;
+
+function capped(text: string): { text: string; chars: number; truncated: boolean } {
+  if (text.length <= SEND_CAP) return { text, chars: text.length, truncated: false };
+  return { text: text.slice(0, SEND_CAP), chars: text.length, truncated: true };
+}
 
 /** The tag on every event a failed check writes — `verify:`, `output: json`,
  *  a schema — headline and detail alike. The runner's no-retry rule for
@@ -391,6 +445,10 @@ async function executeStepInner(opts: ExecOptions, runQuery: QueryFn): Promise<E
       ...(opts.effort ? { effort: opts.effort } : {}),
       systemPrompt: opts.systemPrompt,
       ...(opts.maxTurns ? { maxTurns: opts.maxTurns } : {}),
+      // The raw stream, only when model calls are recorded: an assistant
+      // message carries usage from the call's start (output a few tokens) and
+      // no stop reason; the call's message_delta has both (FOL-72).
+      ...(opts.capture ? { includePartialMessages: true } : {}),
       // Sub-agents: the SDK's own delegation. Every definition carries an
       // explicit tool list (an absent one inherits all of ours).
       ...(subagents ? { agents: toAgentDefinitions(subagents) } : {}),
@@ -474,7 +532,7 @@ async function executeStepInner(opts: ExecOptions, runQuery: QueryFn): Promise<E
             // refused before it runs: the tool, and any paid API behind it,
             // never sees it.
             if (counter.active) {
-              const refusal = counter.take(limitKeysFor(hookInput.tool_name, hookInput.tool_input as Record<string, unknown>, opts.toolOwners));
+              const refusal = counter.take(callKeysFor(hookInput.tool_name, hookInput.tool_input as Record<string, unknown>, opts.toolOwners));
               if (refusal) {
                 emit("info", `${refusal.message} (${hookInput.tool_name} refused)`, via ? { subagent: via } : undefined);
                 return { hookSpecificOutput: { hookEventName: "PreToolUse" as const, permissionDecision: "deny" as const, permissionDecisionReason: refusal.message } };
@@ -489,7 +547,7 @@ async function executeStepInner(opts: ExecOptions, runQuery: QueryFn): Promise<E
               // model differently from its delegates can tell them apart.
               const v = await runToolCallHooks(toolHooks, { tool: hookInput.tool_name, input, base: via ? { ...hookBase, subagent: via } : hookBase }, callHook);
               for (const r of v.records) record(r, via);
-              const refundCall = () => { if (counter.active) counter.refund(limitKeysFor(hookInput.tool_name, hookInput.tool_input as Record<string, unknown>, opts.toolOwners)); };
+              const refundCall = () => { if (counter.active) counter.refund(callKeysFor(hookInput.tool_name, hookInput.tool_input as Record<string, unknown>, opts.toolOwners)); };
               if (v.decision === "deny") {
                 refundCall();
                 return { hookSpecificOutput: { hookEventName: "PreToolUse" as const, permissionDecision: "deny" as const, permissionDecisionReason: v.reason! } };
@@ -558,7 +616,7 @@ async function executeStepInner(opts: ExecOptions, runQuery: QueryFn): Promise<E
         // anything outside it is a denial with a reason the model can act on.
         const fromGrantedServer = opts.mcpNames.some((n) => toolName.startsWith(`mcp__${n}__`));
         // A call the hook counted and this refuses never ran: give it back.
-        const refund = () => { if (counter.active) counter.refund(limitKeysFor(toolName, input, opts.toolOwners)); };
+        const refund = () => { if (counter.active) counter.refund(callKeysFor(toolName, input, opts.toolOwners)); };
         if (!opts.allowed.includes(toolName) && !fromGrantedServer) {
           refund();
           return {
@@ -630,6 +688,29 @@ async function executeStepInner(opts: ExecOptions, runQuery: QueryFn): Promise<E
   // ceiling and on the bill.
   const turns = new Map<string, { u: Record<string, number | undefined>; price: TokenPrice | null }>();
   let anonymousTurns = 0;
+  // Each model call as a span (FOL-54 step 4), keyed like `turns`: a turn's
+  // blocks arrive as several messages under one id, so it is updated in place
+  // and sent again; the runner keeps the latest by id.
+  const spans = new Map<string, ModelCall>();
+  // From the raw stream: which call each agent is in (message_start), and each
+  // call's final output tokens and stop reason (message_delta).
+  const streaming = new Map<string, string>();
+  let saidResponding = false;
+  const finals = new Map<string, { output?: number; stop?: string }>();
+  const withFinal = (span: ModelCall): ModelCall => {
+    const f = finals.get(span.id);
+    return {
+      ...span,
+      ...(f?.output !== undefined ? { tokens: { ...span.tokens, output: f.output } } : {}),
+      // Without the stream (an older CLI), a call that used tools stopped to use them.
+      ...(f?.stop ? { stop: f.stop } : !span.stop && span.tools > 0 ? { stop: "tool_use" } : {}),
+    };
+  };
+  // When each agent's next request went out — the step's own, and each
+  // sub-agent's apart, so one's tool results do not restart another's clock.
+  const lastInputAt = new Map<string, number>();
+  const sentAt = (via: string | null | undefined) => lastInputAt.get(via ?? "") ?? loopStartedAt;
+  let loopStartedAt = Date.now();
   const specByName = new Map((subagents ?? []).map((s) => [s.name, s]));
   const priceOf = (message: unknown): TokenPrice | null => {
     const parent = (message as { parent_tool_use_id?: string | null }).parent_tool_use_id;
@@ -705,11 +786,62 @@ async function executeStepInner(opts: ExecOptions, runQuery: QueryFn): Promise<E
 
   try {
   for await (const message of q as AsyncIterable<SDKMessage>) {
+    // The session is up: the first model call is timed from here, not from
+    // before the SDK and its MCP servers started.
+    if (message.type === "system" && (message as { subtype?: string }).subtype === "init") loopStartedAt = Date.now();
+    if (message.type === "stream_event") {
+      const ev = (message as { event?: { type?: string; message?: { id?: string }; usage?: { output_tokens?: number }; delta?: { stop_reason?: string | null } } }).event;
+      const agentKey = (message as { parent_tool_use_id?: string | null }).parent_tool_use_id ?? "";
+      if (ev?.type === "message_start" && ev.message?.id) {
+        streaming.set(agentKey, ev.message.id);
+        // The step's own first response has begun: its prompt is in the
+        // cache now, so siblings held back to read it may start.
+        if (!agentKey && !saidResponding) {
+          saidResponding = true;
+          emit("info", "", { responding: true });
+        }
+      }
+      if (ev?.type === "message_delta") {
+        const id = streaming.get(agentKey);
+        if (id) {
+          finals.set(id, {
+            ...(typeof ev.usage?.output_tokens === "number" ? { output: ev.usage.output_tokens } : {}),
+            ...(ev.delta?.stop_reason ? { stop: ev.delta.stop_reason } : {}),
+          });
+          const span = spans.get(id);
+          if (span) {
+            const final = withFinal(span);
+            spans.set(id, final);
+            emit("info", "", { turn: final });
+          }
+        }
+      }
+      continue;
+    }
     if (ended) break;
     if (message.type === "assistant") {
-      const m = message.message as unknown as { id?: string; usage?: Record<string, number | undefined> };
+      const m = message.message as unknown as { id?: string; model?: string; stop_reason?: string | null; usage?: Record<string, number | undefined>; content?: { type?: string }[] };
       const u = m.usage;
-      if (u) turns.set(m.id ?? `turn-${anonymousTurns++}`, { u, price: priceOf(message) });
+      const turnKey = m.id ?? `turn-${anonymousTurns++}`;
+      if (u) turns.set(turnKey, { u, price: priceOf(message) });
+      if (opts.capture && u) {
+        const t = usageFromSdk(u);
+        const prior = spans.get(turnKey);
+        const span: ModelCall = {
+          id: turnKey,
+          n: prior?.n ?? spans.size + 1,
+          at: prior?.at ?? new Date().toISOString(),
+          ms: prior?.ms ?? Date.now() - sentAt(subagentOf(message)),
+          ...(m.model ? { model: m.model } : prior?.model ? { model: prior.model } : {}),
+          ...(m.stop_reason ? { stop: m.stop_reason } : prior?.stop ? { stop: prior.stop } : {}),
+          tokens: { input: t.inputTokens, output: t.outputTokens, cacheRead: t.cacheReadTokens, cacheWrite: t.cacheWriteTokens },
+          tools: (prior?.tools ?? 0) + (m.content ?? []).filter((b) => b?.type === "tool_use").length,
+          ...(subagentOf(message) ? { subagent: subagentOf(message)! } : {}),
+        };
+        const final = withFinal(span);
+        spans.set(turnKey, final);
+        emit("info", "", { turn: final });
+      }
       if (ceiling && u) {
         spentUsd = turnsTotal();
         if (spentUsd >= ceiling) {
@@ -731,13 +863,21 @@ async function executeStepInner(opts: ExecOptions, runQuery: QueryFn): Promise<E
             const input = block.input as { subagent_type?: unknown; description?: unknown; prompt?: unknown } | undefined;
             const to = String(input?.subagent_type ?? "subagent");
             delegations.set(block.id, to);
+            // That sub-agent's first model call is timed from its job going out.
+            lastInputAt.set(to, Date.now());
             // One line a person can read: who got the job, and what it was.
             const job = String(input?.description ?? input?.prompt ?? "").replace(/\s+/g, " ").trim().slice(0, 160);
             emit("info", `delegating to ${to}${job ? `: ${job}` : ""}`, { subagent: to });
           }
           openCalls.set(block.id, { name: block.name, at: Date.now(), ...(via ? { subagent: via } : {}) });
           if (pod) ledger.set(block.id, classifyCall(block.name, block.input as Record<string, unknown> | undefined, pod));
-          emit("tool", block.name, { call: block.id, ...(via ? { subagent: via } : {}) });
+          const input = opts.capture === "full" ? capped(JSON.stringify(block.input ?? {})) : null;
+          emit("tool", block.name, {
+            call: block.id,
+            ...(opts.capture ? { inCall: turnKey } : {}),
+            ...(via ? { subagent: via } : {}),
+            ...(input ? { io: { input: input.text, inputChars: input.chars, ...(input.truncated ? { truncated: true } : {}) } } : {}),
+          });
         }
       }
     } else if (message.type === "user") {
@@ -750,6 +890,7 @@ async function executeStepInner(opts: ExecOptions, runQuery: QueryFn): Promise<E
           const open = openCalls.get(block.tool_use_id);
           if (!open) continue;
           openCalls.delete(block.tool_use_id);
+          const output = opts.capture === "full" ? capped(toolResultText(block.content)) : null;
           emit("tool", open.name, {
             call: block.tool_use_id,
             ms: Date.now() - open.at,
@@ -758,6 +899,7 @@ async function executeStepInner(opts: ExecOptions, runQuery: QueryFn): Promise<E
             bytes: toolResultText(block.content).length,
             ...(block.is_error ? { err: true } : {}),
             ...(open.subagent ? { subagent: open.subagent } : {}),
+            ...(output ? { io: { output: output.text, outputChars: output.chars, ...(output.truncated ? { truncated: true } : {}) } } : {}),
           });
           // Why it failed, on the trail. The span above carried only `err`,
           // so a refused call left the agent's own guess as the only account
@@ -769,6 +911,8 @@ async function executeStepInner(opts: ExecOptions, runQuery: QueryFn): Promise<E
           }
           readPod(block.tool_use_id);
         }
+        // That agent's next model call is timed from here: the results went back.
+        lastInputAt.set(subagentOf(message) ?? "", Date.now());
       }
     } else if (message.type === "result") {
       status = message.subtype === "success" ? "completed" : "failed";
@@ -928,6 +1072,8 @@ async function executeStepInner(opts: ExecOptions, runQuery: QueryFn): Promise<E
         data,
         modelEnv: opts.env,
         signal: halt.signal,
+        ...(opts.judgeModel ? { judgeModel: opts.judgeModel } : {}),
+        ...(opts.judgeEffort ? { judgeEffort: opts.judgeEffort } : {}),
       });
     } finally {
       for (const t of clocks) clearTimeout(t);
@@ -1032,6 +1178,9 @@ export async function checkVerify(
     /** The step's own model environment, for `judge:` — it grades on the
      *  fast tier through the same credential the step rode. */
     modelEnv?: Record<string, string | undefined>;
+    /** The judge's model and effort (the workspace's `judge:`). */
+    judgeModel?: string;
+    judgeEffort?: Effort;
     /** Ends a shell check or a judge mid-way: the step's clock ran out, or a
      *  person asked the run to stop. */
     signal?: AbortSignal;
@@ -1095,9 +1244,9 @@ export async function checkVerify(
       // GOOD` on a reply that did (fix-desk, run-muwevg72-7mza): the joined
       // text opened with the narration before it.
       const reply = ctx.conclusion?.trim() ? ctx.conclusion : output;
-      const verdict = await judgeReply(value, reply, output, ctx.modelEnv ?? {}, ctx.signal);
-      const ok = /^\s*PASS\b/i.test(verdict);
-      return { ok, headline: ok ? "PASS" : "FAIL", detail: verdict.slice(0, 300) };
+      const said = await judgeReply(value, reply, output, ctx.modelEnv ?? {}, ctx.signal, ctx.judgeModel, ctx.judgeEffort);
+      const { passed: ok, detail } = judgeVerdict(said);
+      return { ok, headline: ok ? "PASS" : "FAIL", detail };
     }
   }
   return { ok: false, headline: `unknown check "${kind}"`, detail: "" };
@@ -1105,8 +1254,8 @@ export async function checkVerify(
 
 /**
  * A toolless, fast-tier grading call: does the reply satisfy the sentence?
- * Same shape as the eval judge, and answered with one word first so the
- * verdict is a prefix test rather than a reading.
+ * Same shape as the eval judge: reasoning first, then the verdict alone on
+ * the last line (judge-verdict.ts says why).
  */
 export function judgePrompt(rubric: string, reply: string, work: string): string {
   const shown = reply.slice(0, 20_000);
@@ -1115,8 +1264,7 @@ export function judgePrompt(rubric: string, reply: string, work: string): string
       `never this:\n<work>\n${work.slice(-20_000)}\n</work>\n\n`
     : "";
   return (
-    `You are grading a reply against one requirement. Answer with PASS or FAIL as the ` +
-    `first word, then one sentence of reason.\n\nRequirement: ${rubric}\n\n` +
+    `You are grading a reply against one requirement. ${JUDGE_VERDICT_INSTRUCTION}\n\nRequirement: ${rubric}\n\n` +
     context +
     `<reply>\n${shown}\n</reply>`
   );
@@ -1128,6 +1276,8 @@ async function judgeReply(
   work: string,
   env: Record<string, string | undefined>,
   signal?: AbortSignal,
+  model = "haiku",
+  effort?: Effort,
 ): Promise<string> {
   const texts: string[] = [];
   const abortController = new AbortController();
@@ -1137,7 +1287,8 @@ async function judgeReply(
     const q = query({
       prompt: judgePrompt(rubric, reply, work),
       options: {
-        model: "haiku",
+        model,
+        ...(effort ? { effort } : {}),
         systemPrompt: "You grade text against a stated requirement. Be strict and literal.",
         tools: [],
         settingSources: [],