@apo-ai/sdk 0.8.0 → 0.9.0

package/dist/agent-task/public.js

@@ -8,11 +8,11 @@ import {
   resetApoTracing,
   withApoRun,
   withApoRunSync
-} from "../chunk-IPQ6IIEG.js";
+} from "../chunk-KLZLE3VQ.js";
 import {
   createApoTracer
-} from "../chunk-CKB3SUDO.js";
-import "../chunk-HLRXQAZ5.js";
+} from "../chunk-IK4GJX7H.js";
+import "../chunk-HRYXLRXJ.js";
 import {
   createApoOpenAI
 } from "../chunk-3B3L2H4Y.js";
@@ -22,6 +22,7 @@ import {
 import "../chunk-3OQ3JBFJ.js";
 
 // src/agent-task/trace-projection/view.ts
+var TASK_TURN_SPAN_NAME = "task.turn";
 function invocationOrderKey(obs) {
   const hasTs = obs.startedAt != null ? 0 : 1;
   return [hasTs, obs.startedAt ?? "", obs.spanId];
@@ -32,11 +33,11 @@ var TraceView = class {
   }
   /** Evidence availability for a capability. */
   requireCapability(capability) {
-    return this.snapshot.capabilities[capability];
+    return this.snapshot.capabilities[capability] ?? "unavailable";
   }
   /** Whether a capability is reported as available (not partial/unavailable). */
   isAvailable(capability) {
-    return this.snapshot.capabilities[capability] === "available";
+    return this.requireCapability(capability) === "available";
   }
   /** All chat messages flattened across generation observations, in order. */
   get messages() {
@@ -115,6 +116,135 @@ var TraceView = class {
     }
     return void 0;
   }
+  /**
+   * Task Turns in invocation order, or `undefined` when timing evidence is
+   * not available. Each is one `task.turn` span: the adapter's whole
+   * `sendUserTurn` call for that turn.
+   */
+  get turns() {
+    if (!this.isAvailable("timing")) return void 0;
+    return this.turnSpans.map((o, i) => ({
+      turnNumber: i + 1,
+      spanId: o.spanId,
+      ...o.durationMs !== void 0 ? { durationMs: o.durationMs } : {},
+      status: o.status
+    }));
+  }
+  /**
+   * Tokens the agent under test spent: usage on observations inside
+   * `task.turn` spans (every turn, or only turn `turn`). Work outside the
+   * turns — the evaluation phase's judges, adapter setup — is not the
+   * agent's and is excluded. `undefined` when usage evidence is not
+   * available or the requested turn does not exist.
+   */
+  /** How many Task Turns ran — independent of timing evidence. */
+  get turnCountFromSpans() {
+    return this.turnSpans.length;
+  }
+  tokens(kind, opts) {
+    if (!this.isAvailable("usage")) return void 0;
+    const turnSpans = this.turnSpans;
+    let roots;
+    if (opts?.turn !== void 0) {
+      const one = turnSpans[opts.turn - 1];
+      if (!one) return void 0;
+      roots = [one];
+    } else {
+      roots = turnSpans;
+    }
+    const tally = { tokens: 0, reported: 0, unreported: 0 };
+    const children = this.childrenByParent;
+    const isLlmCall = (o) => o.type === "GENERATION" || o.usage != null;
+    const isStep = (o) => isLlmCall(o) && o.type !== "TOOL" && o.type !== "AGENT";
+    const order = [];
+    const visited = new Set(roots.map((r) => r.spanId));
+    const topLevel = roots.flatMap((r) => children.get(r.spanId) ?? []);
+    const stack = [...topLevel];
+    const scopeChildren = /* @__PURE__ */ new Map();
+    while (stack.length > 0) {
+      const obs = stack.pop();
+      if (visited.has(obs.spanId)) continue;
+      visited.add(obs.spanId);
+      order.push(obs);
+      const kids = (children.get(obs.spanId) ?? []).filter((c) => !visited.has(c.spanId));
+      scopeChildren.set(obs.spanId, kids);
+      for (const kid of kids) stack.push(kid);
+    }
+    const effective = /* @__PURE__ */ new Map();
+    for (let i = order.length - 1; i >= 0; i--) {
+      const obs = order[i];
+      const { count, complete } = tokenCount(obs, kind);
+      if (count !== void 0) tally.reported += 1;
+      const ownKnown = complete && obs.status !== "error";
+      let steps;
+      let stepsUnreported = 0;
+      let separate;
+      let separateUnreported = 0;
+      for (const child of scopeChildren.get(obs.spanId) ?? []) {
+        const c = effective.get(child.spanId);
+        if (!c) continue;
+        if (isStep(child)) {
+          if (c.tokens !== void 0) steps = (steps ?? 0) + c.tokens;
+          stepsUnreported += c.unreported;
+        } else {
+          if (c.tokens !== void 0) separate = (separate ?? 0) + c.tokens;
+          separateUnreported += c.unreported;
+        }
+      }
+      const own = count === void 0 ? steps : steps === void 0 ? count : Math.max(count, steps);
+      const unreported = (isLlmCall(obs) && !ownKnown ? 1 : 0) + (isLlmCall(obs) && ownKnown ? 0 : stepsUnreported) + separateUnreported;
+      effective.set(obs.spanId, {
+        tokens: own === void 0 && separate === void 0 ? void 0 : (own ?? 0) + (separate ?? 0),
+        unreported
+      });
+    }
+    for (const obs of topLevel) {
+      const e = effective.get(obs.spanId);
+      if (!e) continue;
+      tally.tokens += e.tokens ?? 0;
+      tally.unreported += e.unreported;
+      effective.delete(obs.spanId);
+    }
+    return tally;
+  }
+  /**
+   * The run's own Task Turns: `task.turn` observations with no `task.turn`
+   * ancestor (a nested run or a user span of the same name inside a turn is
+   * part of that turn, not a turn of its own). Turns run sequentially, so
+   * invocation order is turn order.
+   */
+  get turnSpans() {
+    if (this._turnSpans !== void 0) return this._turnSpans;
+    const byId = new Map(this.snapshot.observations.map((o) => [o.spanId, o]));
+    const insideAnotherTurn = (obs) => {
+      const visited = /* @__PURE__ */ new Set([obs.spanId]);
+      let parentId = obs.parentSpanId;
+      while (parentId !== void 0 && !visited.has(parentId)) {
+        visited.add(parentId);
+        const parent = byId.get(parentId);
+        if (!parent) return false;
+        if (parent.name === TASK_TURN_SPAN_NAME) return true;
+        parentId = parent.parentSpanId;
+      }
+      return false;
+    };
+    this._turnSpans = this.sortedObservations.filter(
+      (o) => o.name === TASK_TURN_SPAN_NAME && !insideAnotherTurn(o)
+    );
+    return this._turnSpans;
+  }
+  get childrenByParent() {
+    if (this._children !== void 0) return this._children;
+    const children = /* @__PURE__ */ new Map();
+    for (const obs of this.snapshot.observations) {
+      if (obs.parentSpanId == null) continue;
+      const list = children.get(obs.parentSpanId);
+      if (list) list.push(obs);
+      else children.set(obs.parentSpanId, [obs]);
+    }
+    this._children = children;
+    return children;
+  }
   /**
    * Observations sorted deterministically by invocation time then span ID.
    * Memoized per TraceView instance since the snapshot is immutable.
@@ -130,6 +260,18 @@ var TraceView = class {
     return sorted;
   }
 };
+function tokenCount(obs, kind) {
+  const known = (v) => typeof v === "number" && Number.isFinite(v) ? v : void 0;
+  const input = known(obs.usage?.inputTokens);
+  const output = known(obs.usage?.outputTokens);
+  if (kind === "input") return { count: input, complete: input !== void 0 };
+  if (kind === "output") return { count: output, complete: output !== void 0 };
+  if (input === void 0 && output === void 0) return { count: void 0, complete: false };
+  return {
+    count: (input ?? 0) + (output ?? 0),
+    complete: input !== void 0 && output !== void 0
+  };
+}
 
 // src/agent-task/checks/matchers.ts
 function asString(value) {
@@ -253,9 +395,24 @@ function decisionsEndpoint(chatBaseURL) {
   const base = chatBaseURL.replace(/\/v1\/?$/, "");
   return `${base}/alpha/decisions`;
 }
+var INPUT_TOO_LARGE_MARKERS = [
+  "max_tokens_exceeded",
+  "context_length_exceeded",
+  "maximum context length"
+];
+function isInputTooLargeRejection(status, body) {
+  if (status !== 400 && status !== 413) return false;
+  return INPUT_TOO_LARGE_MARKERS.some((marker2) => body.includes(marker2));
+}
+function estimateTokens(text) {
+  return Math.round(text.length / 4 / 1e3) * 1e3;
+}
 async function callSecondJudge(args) {
   const startedAt = Date.now();
-  const evidence = { model: args.model };
+  const evidence = {
+    model: args.model,
+    ...args.projected ? { projected: true } : {}
+  };
   try {
     const response = await fetch(decisionsEndpoint(args.baseURL), {
       method: "POST",
@@ -281,6 +438,10 @@ async function callSecondJudge(args) {
     });
     if (!response.ok) {
       const body = await response.text().catch(() => "");
+      if (isInputTooLargeRejection(response.status, body)) {
+        evidence.skipped = `input ~${estimateTokens(args.state).toLocaleString("en-US")} tokens exceeds ${args.model}'s context limit \u2014 project a smaller view with t.judge(value, instruction, { secondJudgeValue })`;
+        return evidence;
+      }
       evidence.error = `Second judge API ${response.status}: ${body.slice(0, 200)}`;
       return evidence;
     }
@@ -504,13 +665,20 @@ ${formatJudgeValues(args.values)}`;
 
 ${deliverableText}`;
   const secondJudgeModel = resolveSecondJudgeModel();
+  const secondJudgeProjected = args.secondJudgeValue !== void 0;
+  const secondJudgeValues = args.secondJudgeValue ?? args.values;
+  const secondJudgeDeliverableText = `Values to evaluate:
+${formatJudgeValues(secondJudgeValues)}`;
   const secondJudgePromise = secondJudgeModel ? callSecondJudge({
-    state: `${systemPromptText}
+    state: `${briefingText}
+
+${secondJudgeDeliverableText}
 
 ${instructionText}`,
     model: secondJudgeModel,
     baseURL: resolveSecondJudgeBaseURL(baseURL),
-    apiKey: resolveSecondJudgeAPIKey(apiKey)
+    apiKey: resolveSecondJudgeAPIKey(apiKey),
+    ...secondJudgeProjected ? { projected: true } : {}
   }) : void 0;
   const cacheKey = `${args.model}\0${briefingText}\0${deliverableText}`;
   const requestBody = (stream) => JSON.stringify({
@@ -1163,7 +1331,7 @@ function createAgentMethod(rec, judgeConfig, judgeScope, evidence, judgeTracer)
         label,
         false,
         "No judge model configured for t.agent. Set one of:\n\u2022 OPENROUTER_MODEL + OPENROUTER_API_KEY (OpenRouter \u2014 needs a tool-calling-capable model)\n\u2022 OPENAI_MODEL + OPENAI_API_KEY (OpenAI direct)\nOr pass { judge } to runTask() programmatically.",
-        { evaluator_type: "agent", location }
+        { evaluator_type: "agent", location, outcome: "error" }
       );
       return;
     }
@@ -1271,6 +1439,8 @@ var TEST_METHOD_NAMES = [
   "messageIncludes",
   "maxTurns",
   "maxDurationMs",
+  "maxTokens",
+  "minTokens",
   "assert",
   "check",
   "judge",
@@ -1304,11 +1474,13 @@ function createJudgeMethod(rec, judgeConfig, judgeScope, judgeTracer) {
         label,
         false,
         "No judge model configured. Set one of:\n\u2022 OPENROUTER_MODEL + OPENROUTER_API_KEY (OpenRouter \u2014 works with 200+ models, one account)\n\u2022 OPENAI_MODEL + OPENAI_API_KEY (OpenAI direct)\nOr pass { judge } to runTask() programmatically.",
-        { evaluator_type: "llm", location }
+        // No judge means no verdict — the same bucket as an unreachable judge.
+        { evaluator_type: "llm", location, outcome: "error" }
       );
       return;
     }
     const context2 = judgeScopeToContext(judgeScope);
+    const secondJudgeValue = opts?.secondJudgeValue;
     const call = () => callJudge({
       values: valueArray,
       instruction,
@@ -1316,7 +1488,10 @@ function createJudgeMethod(rec, judgeConfig, judgeScope, judgeTracer) {
       baseURL: effective.baseURL,
       apiKey: effective.apiKey,
       prompt: effective.prompt,
-      ...context2 ? { context: context2 } : {}
+      ...context2 ? { context: context2 } : {},
+      ...secondJudgeValue !== void 0 ? {
+        secondJudgeValue: Array.isArray(secondJudgeValue) ? secondJudgeValue : [secondJudgeValue]
+      } : {}
     });
     let judgeSpanId;
     const traced = judgeTracer ? judgeTracer.step(
@@ -1395,7 +1570,8 @@ var CAPABILITY_LABELS = {
   errors: "error",
   timing: "timing",
   skills: "skill",
-  subagents: "subagent"
+  subagents: "subagent",
+  usage: "token usage"
 };
 function createTraceTestContext(view, rec, judgeConfig, judgeScope, agentEvidence, judgeTracer) {
   const unsupported = (id, capability) => {
@@ -1550,7 +1726,11 @@ function createTraceTestContext(view, rec, judgeConfig, judgeScope, agentEvidenc
         received: `${turns}`
       });
     },
-    maxDurationMs(n) {
+    maxDurationMs(n, opts) {
+      if (opts?.turn !== void 0) {
+        recordTurnDuration(view, rec, n, opts.turn, unsupported);
+        return;
+      }
       if (!isAvailable("timing")) {
         unsupported(`maxDurationMs(${n})`, "timing");
         return;
@@ -1565,6 +1745,12 @@ function createTraceTestContext(view, rec, judgeConfig, judgeScope, agentEvidenc
         received: `${ms}ms`
       });
     },
+    maxTokens(n, opts) {
+      recordTokenBudget(view, rec, "max", n, opts, unsupported);
+    },
+    minTokens(n, opts) {
+      recordTokenBudget(view, rec, "min", n, opts, unsupported);
+    },
     assert(label, predicate) {
       const pass = predicate(view);
       rec.record(label, pass, pass ? "" : `assertion "${label}" failed`, {
@@ -1587,6 +1773,95 @@ function createTraceTestContext(view, rec, judgeConfig, judgeScope, agentEvidenc
     agent: trackPending(rec, createAgentMethod(rec, judgeConfig, judgeScope, agentEvidence ?? { deliverables: {}, view }, judgeTracer))
   };
 }
+function requireBudget(method, n) {
+  if (!Number.isFinite(n) || n < 0) {
+    throw new TypeError(`${method}: the budget must be a finite, non-negative number, got ${n}`);
+  }
+}
+var TOKEN_KINDS = ["input", "output", "total"];
+function requireTurnNumber(method, turn2) {
+  if (!Number.isInteger(turn2) || turn2 < 1) {
+    throw new TypeError(`${method}: turn must be a positive integer (1-based), got ${turn2}`);
+  }
+}
+function recordTurnDuration(view, rec, n, turn2, unsupported) {
+  requireBudget("maxDurationMs", n);
+  requireTurnNumber("maxDurationMs", turn2);
+  const id = `maxDurationMs(${n}, turn ${turn2})`;
+  const turns = view.turns;
+  if (turns === void 0) {
+    unsupported(id, "timing");
+    return;
+  }
+  const record = turns[turn2 - 1];
+  if (!record) {
+    rec.record(id, false, `turn ${turn2} did not run (the run had ${turns.length} turn(s))`, {
+      expected: `turn ${turn2} \u2264 ${n}ms`,
+      received: `${turns.length} turn(s)`
+    });
+    return;
+  }
+  const ms = record.durationMs;
+  if (ms === void 0) {
+    unsupported(id, "timing");
+    return;
+  }
+  rec.record(id, ms <= n, ms <= n ? "" : `expected turn ${turn2} \u2264 ${n}ms, took ${ms}ms`, {
+    expected: `turn ${turn2} \u2264 ${n}ms`,
+    received: `${ms}ms`
+  });
+}
+function recordTokenBudget(view, rec, bound, n, opts, unsupported) {
+  const method = bound === "max" ? "maxTokens" : "minTokens";
+  const kind = opts?.kind ?? "total";
+  const turn2 = opts?.turn;
+  requireBudget(method, n);
+  if (!TOKEN_KINDS.includes(kind)) {
+    throw new TypeError(`${method}: kind must be one of ${TOKEN_KINDS.join(", ")}, got ${String(kind)}`);
+  }
+  if (turn2 !== void 0) requireTurnNumber(method, turn2);
+  const scope = turn2 === void 0 ? "" : `, turn ${turn2}`;
+  const id = `${method}(${n}${kind === "total" ? "" : `, ${kind}`}${scope})`;
+  const op = bound === "max" ? "\u2264" : "\u2265";
+  const expected = `${op} ${n} ${kind} tokens${turn2 === void 0 ? "" : ` in turn ${turn2}`}`;
+  if (view.requireCapability("usage") !== "available") {
+    unsupported(id, "usage");
+    return;
+  }
+  if (turn2 !== void 0) {
+    const turnCount = view.turnCountFromSpans;
+    if (turn2 > turnCount) {
+      rec.record(id, false, `turn ${turn2} did not run (the run had ${turnCount} turn(s))`, {
+        expected,
+        received: `${turnCount} turn(s)`
+      });
+      return;
+    }
+  }
+  const tally = view.tokens(kind, turn2 === void 0 ? void 0 : { turn: turn2 });
+  if (tally === void 0 || tally.reported === 0 && tally.unreported === 0) {
+    rec.record(id, false, `no LLM call${turn2 === void 0 ? " inside a turn" : ` in turn ${turn2}`} reported ${kind} token usage`, {
+      outcome: "unsupported",
+      expected,
+      received: "no usage-bearing LLM calls in scope"
+    });
+    return;
+  }
+  const received = tally.unreported > 0 ? `\u2265 ${tally.tokens} tokens (${tally.unreported} LLM call(s) with unknown or errored usage)` : `${tally.tokens} tokens`;
+  const within = bound === "max" ? tally.tokens <= n : tally.tokens >= n;
+  if (tally.unreported > 0 && !(bound === "min" && within)) {
+    rec.record(id, false, `usage is incomplete: ${tally.unreported} LLM call(s) in scope have unknown or errored ${kind} usage`, {
+      outcome: "unsupported",
+      expected,
+      received
+    });
+    return;
+  }
+  rec.record(id, within, within ? "" : `expected ${expected}, got ${tally.tokens}`, {
+    expected,
+    received
+  });
+}
 function matchesTraceToolCall(call, name, opts) {
   if (!matchName(call.name, name)) return false;
   if (opts?.input !== void 0 && !matchValue(call.input, opts.input)) {
@@ -2688,6 +2963,7 @@ function resolveDeliverableSchema(definition) {
 }
 
 // src/agent-task/trace-projection/projection-tee.ts
+import { AsyncLocalStorage } from "async_hooks";
 function monotonicNowMs() {
   if (typeof performance !== "undefined" && typeof performance.now === "function") {
     return performance.now();
@@ -2727,7 +3003,7 @@ function extractText(output) {
   }
   return "";
 }
-function deriveCapabilities(observations) {
+function deriveCapabilities(observations, hierarchyAmbiguous) {
   const hasType = (t) => observations.some((o) => o.type === t);
   const hasMessages = observations.some(
     (o) => o.type === "GENERATION" && o.messages && o.messages.length > 0
@@ -2740,13 +3016,29 @@ function deriveCapabilities(observations) {
     errors: "available",
     timing: "available",
     skills: hasType("SKILL") ? "available" : "unavailable",
-    subagents: hasType("AGENT") ? "available" : "unavailable"
+    subagents: hasType("AGENT") ? "available" : "unavailable",
+    // Usage is attributed to turns through parentSpanId; with an ambiguous
+    // step id the chain is broken, so a per-turn or per-run sum could
+    // silently drop turns.
+    usage: !observations.some((o) => o.usage !== void 0) ? "unavailable" : hierarchyAmbiguous ? "partial" : "available"
+  };
+}
+function usageFrom(params) {
+  if (params?.prompt_tokens === void 0 && params?.completion_tokens === void 0) {
+    return void 0;
+  }
+  return {
+    ...params.prompt_tokens !== void 0 ? { inputTokens: params.prompt_tokens } : {},
+    ...params.completion_tokens !== void 0 ? { outputTokens: params.completion_tokens } : {}
   };
 }
 function createProjectionTee(real) {
   const observations = [];
   const pending = /* @__PURE__ */ new Map();
   const realIdByTeeId = /* @__PURE__ */ new Map();
+  const teeIdByRealStepId = /* @__PURE__ */ new Map();
+  const ambiguousRealStepIds = /* @__PURE__ */ new Set();
+  const activeStep = new AsyncLocalStorage();
   let spanCounter = 0;
   const rootStartMs = monotonicNowMs();
   const rootStartIso = new Date(rootStartMs).toISOString();
@@ -2768,6 +3060,8 @@ function createProjectionTee(real) {
     };
     if (p.parentSpanId) obs.parentSpanId = p.parentSpanId;
     if (p.model) obs.model = p.model;
+    const usage = usageFrom(params);
+    if (usage) obs.usage = usage;
     if (isError && params?.status_message) obs.errorMessage = params.status_message;
     const meta = params?.metadata;
     if (p.type === "TOOL") {
@@ -2790,9 +3084,10 @@ function createProjectionTee(real) {
   function recordStart(opts) {
     spanCounter += 1;
     const spanId = `${real.rootSpanId}-${String(spanCounter).padStart(6, "0")}`;
+    const parent = opts.parent_call_id ?? activeStep.getStore() ?? real.rootSpanId;
     pending.set(spanId, {
       spanId,
-      parentSpanId: opts.parent_call_id ?? real.rootSpanId,
+      parentSpanId: teeIdByRealStepId.get(parent) ?? parent,
       type: observationTypeFor(opts.observation_type),
       name: opts.step_name ?? "step",
       toolName: opts.tool_name ?? void 0,
@@ -2810,11 +3105,22 @@ function createProjectionTee(real) {
       const obsSpanId = recordStart(opts);
       const start = monotonicNowMs();
       try {
-        const result = await real.step(opts, fn);
+        const result = await real.step(opts, (realSpanId) => {
+          if (realSpanId && !ambiguousRealStepIds.has(realSpanId)) {
+            if (teeIdByRealStepId.has(realSpanId)) {
+              teeIdByRealStepId.delete(realSpanId);
+              ambiguousRealStepIds.add(realSpanId);
+            } else {
+              teeIdByRealStepId.set(realSpanId, obsSpanId);
+            }
+          }
+          return activeStep.run(obsSpanId, () => fn(realSpanId));
+        });
         complete(obsSpanId, {
           latency_ms: round3(monotonicNowMs() - start),
           output: opts.summarize?.(result),
-          metadata: opts.metadata
+          metadata: opts.metadata,
+          ...opts.usage?.(result)
         });
         return result;
       } catch (error) {
@@ -2927,7 +3233,7 @@ function createProjectionTee(real) {
           durationMs: round3(endMs - rootStartMs),
           complete: true
         },
-        capabilities: deriveCapabilities(observations),
+        capabilities: deriveCapabilities(observations, ambiguousRealStepIds.size > 0),
         observations: [...observations]
       };
     }
@@ -3079,7 +3385,22 @@ function aggregateResult(checksResults) {
     return { checks: checksResults, pass: false };
   }
   const pass = checksResults.every((r) => r.pass);
-  return { checks: checksResults, pass };
+  return isNoVerdict(checksResults) ? { checks: checksResults, pass, noVerdict: true } : { checks: checksResults, pass };
+}
+function isNoVerdict(checksResults) {
+  const failing = checksResults.filter((r) => !r.pass);
+  return failing.length > 0 && failing.every((r) => checkOutcome(r) === "error");
+}
+function checkOutcome(check) {
+  if (check.pass) return void 0;
+  const assertions = check.assertions;
+  if (assertions && assertions.length > 0) {
+    const failed = assertions.filter((a) => !a.pass).map((a) => a.outcome);
+    if (failed.length === 0) return void 0;
+    if (!failed.every((o) => o === "error" || o === "unsupported")) return void 0;
+    return failed.includes("error") ? "error" : "unsupported";
+  }
+  return check.outcome;
 }
 
 // src/agent-task/run/runTask.ts
@@ -3447,11 +3768,18 @@ async function runTurnLoop(loaded, options, session, trace2, rawTrace) {
         metadata: { turnNumber: turnNum },
         summarize: (r) => ({ response: r.response })
       },
-      async (spanId) => session.sendUserTurn(userTurn, {
-        trace: trace2,
-        turnNumber: turnNum,
-        parentSpanId: spanId
-      })
+      // Re-scope the run context to this turn so GenAI spans the OTel
+      // processor captures parent under the turn span, not the run root —
+      // turn-scoped assertions (t.maxTokens({ turn })) walk that chain.
+      async (spanId) => {
+        const run = getActiveApoRun();
+        const send = () => session.sendUserTurn(userTurn, {
+          trace: trace2,
+          turnNumber: turnNum,
+          parentSpanId: spanId
+        });
+        return run ? withApoRun({ ...run, parentSpanId: spanId, turnNumber: turnNum }, send) : send();
+      }
     );
     turnTranscript.push({
       turnNumber: turnNum,
@@ -3579,12 +3907,20 @@ var RejudgeError = class extends Error {
 };
 var TERMINAL_VERDICT_STATUSES = /* @__PURE__ */ new Set(["passed", "failed"]);
 var MAX_SAMPLES = 50;
+var NO_VERDICT_MESSAGE_PREFIX = "No verdict: ";
+function isJudgeNoVerdictRun(detail) {
+  if (detail.status !== "error" || detail.pass_result != null) return false;
+  if (detail.no_verdict_reason !== void 0) return detail.no_verdict_reason === "judge";
+  const generations = detail.generation_execution;
+  const generationsDominate = generations != null && generations.total > 0 && generations.errored * 2 > generations.total;
+  return (detail.error_message ?? "").startsWith(NO_VERDICT_MESSAGE_PREFIX) && detail.failed_checks === 0 && (detail.errored_checks ?? 0) > 0 && !generationsDominate;
+}
 async function rejudgeTaskRun(runId, endpoints, options = {}) {
   const api = createApiClient(endpoints);
   const progress = options.onProgress ?? (() => {
   });
   const detail = await api.get(`/v1/agent-task-runs/${runId}`);
-  if (!TERMINAL_VERDICT_STATUSES.has(detail.status)) {
+  if (!TERMINAL_VERDICT_STATUSES.has(detail.status) && !isJudgeNoVerdictRun(detail)) {
     throw new RejudgeError(
       `Run ${runId} is '${detail.status}', not a completed run with a verdict \u2014 there is nothing to replay against yet`
     );
@@ -3656,27 +3992,14 @@ async function rejudgeTaskRun(runId, endpoints, options = {}) {
       loaded.adapter.deliverables
     );
     const proxied = proxyBrokenDeliverables(deliverables, validation.brokenDeliverables);
-    const passCounts = /* @__PURE__ */ new Map();
-    const errorCounts = /* @__PURE__ */ new Map();
-    let primary = [];
+    const sampled = [];
     const judgeConfig = resolveJudgeConfig(options.judge, loaded.task.judge);
     for (let sample = 0; sample < samples; sample++) {
       if (samples > 1) progress(`judging sample ${sample + 1}/${samples}`);
-      const results = await runChecksOnce(loaded, proxied, snapshot, judgeConfig);
-      if (sample === 0) primary = results;
-      for (const result of results) {
-        passCounts.set(result.id, (passCounts.get(result.id) ?? 0) + (result.pass ? 1 : 0));
-        if (!result.pass && result.outcome === "error") {
-          errorCounts.set(result.id, (errorCounts.get(result.id) ?? 0) + 1);
-        }
-      }
+      sampled.push(await runChecksOnce(loaded, proxied, snapshot, judgeConfig));
     }
-    const stability = primary.map((result) => ({
-      check_id: result.id,
-      passes: passCounts.get(result.id) ?? 0,
-      samples,
-      ...errorCounts.get(result.id) ? { errored: errorCounts.get(result.id) } : {}
-    }));
+    const primary = sampled[0] ?? [];
+    const stability = checkStability(sampled);
     const aggregate = aggregateResult(primary);
     return {
       runId,
@@ -3690,6 +4013,7 @@ async function rejudgeTaskRun(runId, endpoints, options = {}) {
       samples,
       checks: aggregate.checks,
       pass: aggregate.pass,
+      ...aggregate.noVerdict ? { noVerdict: true } : {},
       stability,
       traceSnapshotAvailable: snapshotResult.available,
       taskDirUsed: taskDir
@@ -3699,6 +4023,24 @@ async function rejudgeTaskRun(runId, endpoints, options = {}) {
     if (scaffoldDir) rmSync(scaffoldDir, { recursive: true, force: true });
   }
 }
+function checkStability(sampled) {
+  const passCounts = /* @__PURE__ */ new Map();
+  const errorCounts = /* @__PURE__ */ new Map();
+  for (const results of sampled) {
+    for (const result of results) {
+      passCounts.set(result.id, (passCounts.get(result.id) ?? 0) + (result.pass ? 1 : 0));
+      if (checkOutcome(result) === "error") {
+        errorCounts.set(result.id, (errorCounts.get(result.id) ?? 0) + 1);
+      }
+    }
+  }
+  return (sampled[0] ?? []).map((result) => ({
+    check_id: result.id,
+    passes: passCounts.get(result.id) ?? 0,
+    samples: sampled.length,
+    ...errorCounts.get(result.id) ? { errored: errorCounts.get(result.id) } : {}
+  }));
+}
 async function runChecksOnce(loaded, deliverables, snapshot, judgeConfig) {
   if (loaded.inlineChecks) {
     return runTraceChecks({
@@ -3771,7 +4113,8 @@ function degenerateSnapshot(runId) {
       errors: "unavailable",
       timing: "unavailable",
       skills: "unavailable",
-      subagents: "unavailable"
+      subagents: "unavailable",
+      usage: "unavailable"
     },
     observations: []
   };
@@ -4303,6 +4646,7 @@ async function runTaskDir(taskDir) {
     taskDir: loaded.taskDir,
     taskId: loaded.task.id,
     pass: result.result.pass,
+    ...result.result.noVerdict ? { noVerdict: true } : {},
     checks: result.result.checks,
     adapterName: loaded.adapter.name,
     traceRunId: result.traceRunId,
@@ -4379,7 +4723,8 @@ async function runAgentTaskCli(argv, cwd = process.cwd()) {
     if (result.status === "fulfilled") {
       summaries.push(result.value);
       printTaskSummary(result.value, cwd);
-      hasFailures ||= !result.value.pass;
+      if (result.value.noVerdict) hasErrors = true;
+      else hasFailures ||= !result.value.pass;
     } else {
       hasErrors = true;
       printTaskError(taskDir, cwd, result.reason);
@@ -4427,7 +4772,8 @@ function printHelp() {
 function printTaskSummary(summary, cwd) {
   const relativeTaskDir = relative(cwd, summary.taskDir);
   console.log("");
-  console.log(`${summary.pass ? "PASS" : "FAIL"} ${summary.taskId}`);
+  const verdict = summary.noVerdict ? "NO VERDICT" : summary.pass ? "PASS" : "FAIL";
+  console.log(`${verdict} ${summary.taskId}`);
   console.log(`Path: ${relativeTaskDir}`);
   printResultGroup("Checks", summary.checks);
   if (summary.checks.length === 0 && !summary.pass) {
@@ -4454,12 +4800,14 @@ function printTaskError(taskDir, cwd, error) {
 }
 function printOverallSummary(summaries, hasErrors) {
   const passed = summaries.filter((summary) => summary.pass).length;
-  const failed = summaries.length - passed;
+  const noVerdict = summaries.filter((summary) => summary.noVerdict).length;
+  const failed = summaries.length - passed - noVerdict;
   console.log("");
   console.log("Summary");
   console.log(`- total: ${summaries.length}`);
   console.log(`- passed: ${passed}`);
   console.log(`- failed: ${failed}`);
+  if (noVerdict > 0) console.log(`- no verdict: ${noVerdict}`);
   console.log(`- errors: ${hasErrors ? "yes" : "no"}`);
 }