@apo-ai/sdk 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,18 +8,18 @@ import {
8
8
  resetApoTracing,
9
9
  withApoRun,
10
10
  withApoRunSync
11
- } from "../chunk-GETY34VI.js";
11
+ } from "../chunk-IPQ6IIEG.js";
12
12
  import {
13
13
  createApoTracer
14
- } from "../chunk-QHRWPVPZ.js";
15
- import "../chunk-HWZVL6TS.js";
14
+ } from "../chunk-CKB3SUDO.js";
15
+ import "../chunk-HLRXQAZ5.js";
16
16
  import {
17
17
  createApoOpenAI
18
- } from "../chunk-6ZRW5IW6.js";
18
+ } from "../chunk-3B3L2H4Y.js";
19
19
  import {
20
20
  createApoAnthropic
21
- } from "../chunk-SZDSIPXC.js";
22
- import "../chunk-GDB6SDGN.js";
21
+ } from "../chunk-43OI6GP7.js";
22
+ import "../chunk-3OQ3JBFJ.js";
23
23
 
24
24
  // src/agent-task/trace-projection/view.ts
25
25
  function invocationOrderKey(obs) {
@@ -284,7 +284,18 @@ async function callSecondJudge(args) {
284
284
  evidence.error = `Second judge API ${response.status}: ${body.slice(0, 200)}`;
285
285
  return evidence;
286
286
  }
287
- const data = await response.json();
287
+ const contentType = response.headers.get("content-type") ?? "";
288
+ if (contentType.includes("text/html")) {
289
+ evidence.error = "Second judge endpoint returned HTML, not the decisions API \u2014 APO_SECOND_JUDGE_BASE_URL likely points at the site root. Use the API root, e.g. https://openrouter.ai/api/v1";
290
+ return evidence;
291
+ }
292
+ let data;
293
+ try {
294
+ data = JSON.parse(await response.text());
295
+ } catch {
296
+ evidence.error = "Second judge endpoint returned a non-JSON body \u2014 check APO_SECOND_JUDGE_BASE_URL points at the decisions API root (e.g. https://openrouter.ai/api/v1)";
297
+ return evidence;
298
+ }
288
299
  const verdict = data.answers?.verdict;
289
300
  if (verdict?.choice !== "pass" && verdict?.choice !== "fail") {
290
301
  evidence.error = "Second judge returned no verdict choice";
@@ -303,6 +314,12 @@ async function callSecondJudge(args) {
303
314
  }
304
315
 
305
316
  // src/agent-task/checks/judge.ts
317
+ var JudgeUnavailableError = class extends Error {
318
+ constructor(message) {
319
+ super(message);
320
+ this.name = "JudgeUnavailableError";
321
+ }
322
+ };
306
323
  var VERDICT_FIRST_CONTRACT = 'Respond with ONLY a JSON object: {"pass": true/false, "reasoning": "your reasoning"}';
307
324
  var REASONING_FIRST_CONTRACT = 'Respond with ONLY a JSON object: {"reasoning": "your reasoning", "pass": true/false}';
308
325
  function isJudgeVerdictFirstOverrideEnabled() {
@@ -378,7 +395,92 @@ function parseJudgeJson(raw) {
378
395
  reasoning: "Judge response could not be parsed as JSON \u2014 the verdict is unknown, so this check is treated as a failure. The model's raw response is available in the judge metadata."
379
396
  };
380
397
  }
381
- var JUDGE_TIMEOUT_MS = 12e4;
398
+ var JUDGE_TIMEOUT_MS = 3e5;
399
+ var JUDGE_IDLE_TIMEOUT_MS = 9e4;
400
+ var JUDGE_RETRY_DELAY_MS = 1e3;
401
+ var JUDGE_RETRY_AFTER_CAP_MS = 2e4;
402
+ var JUDGE_MIN_RETRY_BUDGET_MS = 6e4;
403
+ function isRetryableStatus(status) {
404
+ return status === 429 || status >= 500;
405
+ }
406
+ function retryDelayMs(retryAfter) {
407
+ if (!retryAfter) return JUDGE_RETRY_DELAY_MS;
408
+ const seconds = Number(retryAfter);
409
+ const ms = Number.isFinite(seconds) ? seconds * 1e3 : Date.parse(retryAfter) - Date.now();
410
+ if (!Number.isFinite(ms)) return JUDGE_RETRY_DELAY_MS;
411
+ return Math.min(Math.max(ms, JUDGE_RETRY_DELAY_MS), JUDGE_RETRY_AFTER_CAP_MS);
412
+ }
413
+ async function readCompletion(response, onData) {
414
+ const contentType = response.headers.get("content-type") ?? "";
415
+ if (!contentType.includes("text/event-stream")) {
416
+ const data = await response.json();
417
+ return {
418
+ text: data.choices?.[0]?.message?.content ?? "",
419
+ usage: data.usage,
420
+ finishReason: data.choices?.[0]?.finish_reason ?? void 0
421
+ };
422
+ }
423
+ let text = "";
424
+ let usage;
425
+ let finishReason;
426
+ let eventType = "message";
427
+ let dataLines = [];
428
+ let finished = false;
429
+ const dispatch = () => {
430
+ const payload = dataLines.join("\n").trim();
431
+ const type = eventType;
432
+ eventType = "message";
433
+ dataLines = [];
434
+ if (payload === "[DONE]") finished = true;
435
+ if (!payload || finished) return;
436
+ onData();
437
+ let chunk;
438
+ try {
439
+ chunk = JSON.parse(payload);
440
+ } catch {
441
+ chunk = void 0;
442
+ }
443
+ if (type === "error" || chunk?.error) {
444
+ const error = chunk?.error;
445
+ const message = typeof error === "string" ? error : error?.message ?? (chunk ? void 0 : payload);
446
+ throw new JudgeUnavailableError(`Judge stream error: ${message ?? "unknown"}`);
447
+ }
448
+ if (!chunk) return;
449
+ const choice = chunk.choices?.[0];
450
+ text += choice?.delta?.content ?? "";
451
+ if (choice?.finish_reason) finishReason = choice.finish_reason;
452
+ if (chunk.usage) usage = chunk.usage;
453
+ };
454
+ const handleLine = (line) => {
455
+ if (line === "") return dispatch();
456
+ if (line.startsWith(":")) return;
457
+ const colon = line.indexOf(":");
458
+ const field = colon === -1 ? line : line.slice(0, colon);
459
+ let value = colon === -1 ? "" : line.slice(colon + 1);
460
+ if (value.startsWith(" ")) value = value.slice(1);
461
+ if (field === "data") dataLines.push(value);
462
+ else if (field === "event") eventType = value;
463
+ };
464
+ const reader = response.body?.getReader();
465
+ if (!reader) return { text, usage, finishReason };
466
+ const decoder = new TextDecoder();
467
+ let buffer = "";
468
+ for (; ; ) {
469
+ const { done, value } = await reader.read();
470
+ if (done) break;
471
+ buffer += decoder.decode(value, { stream: true });
472
+ const lines = buffer.split(/\r\n|\r(?!$)|\n/);
473
+ buffer = lines.pop() ?? "";
474
+ for (const line of lines) {
475
+ handleLine(line);
476
+ if (finished) return { text, usage, finishReason };
477
+ }
478
+ }
479
+ buffer += decoder.decode();
480
+ for (const line of buffer.split(/\r\n|\r|\n/)) if (line) handleLine(line);
481
+ dispatch();
482
+ return { text, usage, finishReason };
483
+ }
382
484
  var prefixQueues = /* @__PURE__ */ new Map();
383
485
  function runWithSharedPrefix(key, task2) {
384
486
  const prev = prefixQueues.get(key) ?? Promise.resolve();
@@ -411,75 +513,155 @@ ${instructionText}`,
411
513
  apiKey: resolveSecondJudgeAPIKey(apiKey)
412
514
  }) : void 0;
413
515
  const cacheKey = `${args.model}\0${briefingText}\0${deliverableText}`;
414
- return runWithSharedPrefix(cacheKey, async () => {
415
- const startedAt = Date.now();
416
- const response = await fetch(`${baseURL}/chat/completions`, {
417
- method: "POST",
418
- signal: AbortSignal.timeout(JUDGE_TIMEOUT_MS),
419
- headers: {
420
- "Content-Type": "application/json",
421
- ...apiKey ? { Authorization: `Bearer ${apiKey}` } : {}
422
- },
423
- body: JSON.stringify({
424
- model: args.model,
425
- messages: [
516
+ const requestBody = (stream) => JSON.stringify({
517
+ model: args.model,
518
+ messages: [
519
+ {
520
+ role: "system",
521
+ content: [
522
+ { type: "text", text: briefingText },
426
523
  {
427
- role: "system",
428
- content: [
429
- { type: "text", text: briefingText },
430
- {
431
- type: "text",
432
- text: deliverableText,
433
- cache_control: { type: "ephemeral" }
434
- }
435
- ]
524
+ type: "text",
525
+ text: deliverableText,
526
+ cache_control: { type: "ephemeral" }
527
+ }
528
+ ]
529
+ },
530
+ { role: "user", content: instructionText }
531
+ ],
532
+ temperature: 0,
533
+ response_format: { type: "json_object" },
534
+ ...stream ? { stream: true, stream_options: { include_usage: true } } : {}
535
+ });
536
+ const attempt = async (stream, deadline) => {
537
+ const controller = new AbortController();
538
+ const expire = (reason, name) => () => controller.abort(new DOMException(reason, name));
539
+ const total = setTimeout(
540
+ expire(`no complete response within ${JUDGE_TIMEOUT_MS / 1e3}s`, "TimeoutError"),
541
+ Math.max(deadline - Date.now(), 0)
542
+ );
543
+ const stalled = expire(`no data for ${JUDGE_IDLE_TIMEOUT_MS / 1e3}s`, "IdleTimeoutError");
544
+ let idle;
545
+ const onData = () => {
546
+ clearTimeout(idle);
547
+ idle = setTimeout(stalled, JUDGE_IDLE_TIMEOUT_MS);
548
+ };
549
+ try {
550
+ let response;
551
+ try {
552
+ response = await fetch(`${baseURL}/chat/completions`, {
553
+ method: "POST",
554
+ signal: controller.signal,
555
+ headers: {
556
+ "Content-Type": "application/json",
557
+ ...apiKey ? { Authorization: `Bearer ${apiKey}` } : {}
436
558
  },
437
- { role: "user", content: instructionText }
438
- ],
439
- temperature: 0,
440
- response_format: { type: "json_object" }
441
- })
442
- });
443
- if (!response.ok) {
444
- const body = await response.text().catch(() => "");
445
- throw new Error(`Judge API ${response.status}: ${body.slice(0, 200)}`);
559
+ body: requestBody(stream)
560
+ });
561
+ } catch (error) {
562
+ return transportFailure("Judge request failed", controller.signal.reason ?? error);
563
+ }
564
+ if (!response.ok) {
565
+ const body = await response.text().catch(() => "");
566
+ return {
567
+ kind: "unavailable",
568
+ error: new JudgeUnavailableError(`Judge API ${response.status}: ${body.slice(0, 200)}`),
569
+ retryable: isRetryableStatus(response.status),
570
+ ...isRetryableStatus(response.status) ? { retryDelayMs: retryDelayMs(response.headers.get("retry-after")) } : {},
571
+ // Name the field, not the word: gateways echo request params in
572
+ // unrelated 400 bodies.
573
+ ...stream && response.status === 400 && /stream_options|["'`]stream["'`]/.test(body) ? { streamRejected: true } : {}
574
+ };
575
+ }
576
+ try {
577
+ return { kind: "completion", completion: await readCompletion(response, onData) };
578
+ } catch (error) {
579
+ return transportFailure("Judge response failed", controller.signal.reason ?? error);
580
+ }
581
+ } finally {
582
+ clearTimeout(total);
583
+ clearTimeout(idle);
584
+ controller.abort();
446
585
  }
447
- const data = await response.json();
448
- const text = data.choices?.[0]?.message?.content ?? "";
449
- const outputTokens = data.usage?.completion_tokens;
450
- const isEmpty = !text.trim();
451
- const reportedZeroTokens = data.usage !== void 0 && outputTokens === 0;
452
- if (isEmpty || reportedZeroTokens) {
586
+ };
587
+ const isTruncated = ({ text, usage, finishReason }) => {
588
+ if (!text.trim()) return true;
589
+ if (usage !== void 0 && usage.completion_tokens === 0) return true;
590
+ if (finishReason !== "length" && finishReason !== "content_filter" && finishReason !== "error") {
591
+ return false;
592
+ }
593
+ try {
594
+ JSON.parse(text);
595
+ return false;
596
+ } catch {
597
+ return true;
598
+ }
599
+ };
600
+ return runWithSharedPrefix(cacheKey, async () => {
601
+ const startedAt = Date.now();
602
+ const deadline = startedAt + JUDGE_TIMEOUT_MS;
603
+ let stream = true;
604
+ let result = await attempt(stream, deadline);
605
+ const truncated = result.kind === "completion" && isTruncated(result.completion);
606
+ const streamRejected = result.kind === "unavailable" && result.streamRejected === true;
607
+ const retryable = result.kind === "unavailable" && result.retryable;
608
+ if (truncated || streamRejected || retryable) {
609
+ const delay = result.kind === "unavailable" && result.retryDelayMs !== void 0 ? result.retryDelayMs : streamRejected ? 0 : JUDGE_RETRY_DELAY_MS;
610
+ if (deadline - Date.now() - delay >= JUDGE_MIN_RETRY_BUDGET_MS) {
611
+ if (streamRejected) stream = false;
612
+ await new Promise((resolve4) => setTimeout(resolve4, delay));
613
+ const first = result;
614
+ result = await attempt(stream, deadline);
615
+ if (result.kind === "unavailable" && first.kind === "unavailable") {
616
+ result = {
617
+ ...result,
618
+ error: new JudgeUnavailableError(
619
+ `${first.error.message} (retry: ${result.error.message})`
620
+ )
621
+ };
622
+ }
623
+ }
624
+ }
625
+ if (result.kind === "unavailable") throw result.error;
626
+ const { text, usage } = result.completion;
627
+ const judge = {
628
+ model: args.model,
629
+ contract: judgeContractInUse(),
630
+ prompt: { system: systemPromptText, user: instructionText },
631
+ response: text,
632
+ tokens: parseJudgeUsage(usage),
633
+ latency_ms: Date.now() - startedAt,
634
+ secondJudge: await secondJudgePromise
635
+ };
636
+ if (isTruncated(result.completion)) {
453
637
  return {
454
638
  pass: false,
455
- reasoning: "Judge returned an empty or truncated response \u2014 likely a transient provider failure. The verdict is unknown, so this check is treated as a failure.",
456
- judge: {
457
- model: args.model,
458
- contract: judgeContractInUse(),
459
- prompt: { system: systemPromptText, user: instructionText },
460
- response: text,
461
- tokens: parseJudgeUsage(data.usage),
462
- latency_ms: Date.now() - startedAt,
463
- secondJudge: await secondJudgePromise
464
- }
639
+ reasoning: "Judge returned an empty or truncated response \u2014 likely a transient provider failure. The verdict is unknown, so this check is recorded as a judge error, not a verdict.",
640
+ judge,
641
+ unavailable: true
465
642
  };
466
643
  }
467
644
  const parsed = parseJudgeJson(text);
468
645
  return {
469
646
  pass: parsed.pass === true,
470
647
  reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : "",
471
- judge: {
472
- model: args.model,
473
- contract: judgeContractInUse(),
474
- prompt: { system: systemPromptText, user: instructionText },
475
- response: text,
476
- tokens: parseJudgeUsage(data.usage),
477
- latency_ms: Date.now() - startedAt,
478
- secondJudge: await secondJudgePromise
479
- }
648
+ judge
480
649
  };
481
650
  });
482
651
  }
652
+ function transportFailure(prefix, error) {
653
+ if (error instanceof JudgeUnavailableError) {
654
+ return { kind: "unavailable", error, retryable: true };
655
+ }
656
+ const timedOut = error instanceof Error && error.name === "TimeoutError";
657
+ return {
658
+ kind: "unavailable",
659
+ error: new JudgeUnavailableError(
660
+ `${prefix}: ${error instanceof Error ? error.message : String(error)}`
661
+ ),
662
+ retryable: !timedOut
663
+ };
664
+ }
483
665
  function assembleBriefing(args) {
484
666
  if (!args.prompt) {
485
667
  return {
@@ -510,6 +692,53 @@ ${args.instruction}`
510
692
 
511
693
  // src/agent-task/checks/agent-session.ts
512
694
  import { createHash } from "crypto";
695
+
696
+ // src/agent-task/tracing.ts
697
+ function createNoopAgentTaskTraceContext() {
698
+ const noopFn = async (fn) => fn();
699
+ return {
700
+ runId: "agent-task-untraced",
701
+ rootSpanId: "agent-task-untraced-root",
702
+ async step(options, fn) {
703
+ return fn(options.id ?? "agent-task-untraced-step");
704
+ },
705
+ recordEvent(options) {
706
+ return options.id ?? "agent-task-untraced-event";
707
+ },
708
+ endRoot() {
709
+ },
710
+ traceTool(_name, _params, fn) {
711
+ return noopFn(fn);
712
+ },
713
+ traceRetriever(_query, fn) {
714
+ return noopFn(fn);
715
+ },
716
+ traceChain(_name, fn) {
717
+ return noopFn(fn);
718
+ },
719
+ traceAgent(_name, fn) {
720
+ return noopFn(fn);
721
+ },
722
+ traceGuardrail(_name, fn) {
723
+ return noopFn(fn);
724
+ },
725
+ traceEmbedding(_model, _input, fn) {
726
+ return noopFn(fn);
727
+ },
728
+ async score(_params) {
729
+ },
730
+ createSpan(_options) {
731
+ return "agent-task-untraced-span";
732
+ },
733
+ endSpan(_spanId, _params) {
734
+ }
735
+ };
736
+ }
737
+ function isTraceableSpanId(spanId) {
738
+ return spanId !== void 0 && !spanId.startsWith("agent-task-untraced");
739
+ }
740
+
741
+ // src/agent-task/checks/agent-session.ts
513
742
  var DEFAULT_BUDGET = {
514
743
  maxTurns: 12,
515
744
  maxToolCalls: 24,
@@ -570,7 +799,7 @@ function extractVerdict(steps) {
570
799
  return void 0;
571
800
  }
572
801
  function buildEvidenceTools(args) {
573
- const { z, tool, evidence, scope, budget, ledger, stepIndexOf, includeTrace } = args;
802
+ const { z, tool, evidence, scope, budget, ledger, stepIndexOf, includeTrace, includeHistory, tracer } = args;
574
803
  const budgetGuard = () => {
575
804
  ledger.toolCallsUsed += 1;
576
805
  if (ledger.toolCallsUsed > budget.maxToolCalls) {
@@ -589,6 +818,7 @@ function buildEvidenceTools(args) {
589
818
  });
590
819
  return ledger.readBytesUsed > budget.maxReadBytes ? overflow() : null;
591
820
  };
821
+ const span = (name, input, fn) => tracer ? tracer.traceTool(name, input, fn) : fn();
592
822
  const tools = {
593
823
  read_deliverable: tool({
594
824
  description: `Read one of this run's deliverables by name, with optional offset/limit for pagination (max ${MAX_READ_WINDOW} bytes per call).`,
@@ -599,21 +829,23 @@ function buildEvidenceTools(args) {
599
829
  }),
600
830
  execute: async (input) => {
601
831
  const { name, offset, limit } = input;
602
- const guard = budgetGuard();
603
- if (guard) return { error: guard };
604
- if (!Object.prototype.hasOwnProperty.call(evidence.deliverables, name)) {
605
- return { error: `unknown deliverable: ${name}` };
606
- }
607
- const content = renderValue(evidence.deliverables[name]);
608
- const slice = content.slice(offset, offset + limit);
609
- const overflow = accountRead(
610
- "read_deliverable",
611
- JSON.stringify({ name, offset, limit }),
612
- slice,
613
- () => `read budget exhausted (${budget.maxReadBytes} bytes); call finish_verdict now`
614
- );
615
- if (overflow) return { error: overflow };
616
- return { name, total_bytes: content.length, offset, returned: slice.length, content: slice };
832
+ return span("read_deliverable", input, async () => {
833
+ const guard = budgetGuard();
834
+ if (guard) return { error: guard };
835
+ if (!Object.prototype.hasOwnProperty.call(evidence.deliverables, name)) {
836
+ return { error: `unknown deliverable: ${name}` };
837
+ }
838
+ const content = renderValue(evidence.deliverables[name]);
839
+ const slice = content.slice(offset, offset + limit);
840
+ const overflow = accountRead(
841
+ "read_deliverable",
842
+ JSON.stringify({ name, offset, limit }),
843
+ slice,
844
+ () => `read budget exhausted (${budget.maxReadBytes} bytes); call finish_verdict now`
845
+ );
846
+ if (overflow) return { error: overflow };
847
+ return { name, total_bytes: content.length, offset, returned: slice.length, content: slice };
848
+ });
617
849
  }
618
850
  }),
619
851
  search_deliverable: tool({
@@ -621,49 +853,53 @@ function buildEvidenceTools(args) {
621
853
  inputSchema: z.object({ name: z.string(), pattern: z.string() }),
622
854
  execute: async (input) => {
623
855
  const { name, pattern } = input;
624
- const guard = budgetGuard();
625
- if (guard) return { error: guard };
626
- if (!Object.prototype.hasOwnProperty.call(evidence.deliverables, name)) {
627
- return { error: `unknown deliverable: ${name}` };
628
- }
629
- if (pattern.length > SEARCH_MAX_PATTERN) {
630
- return { error: `pattern too long (max ${SEARCH_MAX_PATTERN} chars)` };
631
- }
632
- let re;
633
- try {
634
- re = new RegExp(pattern, "gi");
635
- } catch (error) {
636
- return { error: `invalid regex: ${error instanceof Error ? error.message : String(error)}` };
637
- }
638
- const content = renderValue(evidence.deliverables[name]).slice(0, SEARCH_SCAN_LIMIT);
639
- const hits = [];
640
- let m;
641
- let execs = 0;
642
- while ((m = re.exec(content)) && hits.length < SEARCH_HITS) {
643
- if (++execs > SEARCH_MAX_EXEC) break;
644
- const start = Math.max(0, m.index - SEARCH_CONTEXT);
645
- hits.push({ at_byte: m.index, context: content.slice(start, m.index + m[0].length + SEARCH_CONTEXT) });
646
- if (m.index === re.lastIndex) re.lastIndex++;
647
- }
648
- const served = JSON.stringify({ total_bytes: content.length, match_count: hits.length, matches: hits });
649
- const overflow = accountRead(
650
- "search_deliverable",
651
- JSON.stringify({ name, pattern }),
652
- served,
653
- () => `read budget exhausted (${budget.maxReadBytes} bytes); call finish_verdict now`
654
- );
655
- if (overflow) return { error: overflow };
656
- return { total_bytes: content.length, match_count: hits.length, matches: hits };
856
+ return span("search_deliverable", input, async () => {
857
+ const guard = budgetGuard();
858
+ if (guard) return { error: guard };
859
+ if (!Object.prototype.hasOwnProperty.call(evidence.deliverables, name)) {
860
+ return { error: `unknown deliverable: ${name}` };
861
+ }
862
+ if (pattern.length > SEARCH_MAX_PATTERN) {
863
+ return { error: `pattern too long (max ${SEARCH_MAX_PATTERN} chars)` };
864
+ }
865
+ let re;
866
+ try {
867
+ re = new RegExp(pattern, "gi");
868
+ } catch (error) {
869
+ return { error: `invalid regex: ${error instanceof Error ? error.message : String(error)}` };
870
+ }
871
+ const content = renderValue(evidence.deliverables[name]).slice(0, SEARCH_SCAN_LIMIT);
872
+ const hits = [];
873
+ let m;
874
+ let execs = 0;
875
+ while ((m = re.exec(content)) && hits.length < SEARCH_HITS) {
876
+ if (++execs > SEARCH_MAX_EXEC) break;
877
+ const start = Math.max(0, m.index - SEARCH_CONTEXT);
878
+ hits.push({ at_byte: m.index, context: content.slice(start, m.index + m[0].length + SEARCH_CONTEXT) });
879
+ if (m.index === re.lastIndex) re.lastIndex++;
880
+ }
881
+ const served = JSON.stringify({ total_bytes: content.length, match_count: hits.length, matches: hits });
882
+ const overflow = accountRead(
883
+ "search_deliverable",
884
+ JSON.stringify({ name, pattern }),
885
+ served,
886
+ () => `read budget exhausted (${budget.maxReadBytes} bytes); call finish_verdict now`
887
+ );
888
+ if (overflow) return { error: overflow };
889
+ return { total_bytes: content.length, match_count: hits.length, matches: hits };
890
+ });
657
891
  }
658
892
  }),
659
- ...evidence.history ? {
893
+ ...includeHistory && evidence.history ? {
660
894
  list_runs: tool({
661
895
  description: "List runs of this task \u2014 id, status, pass/fail, model, check counts. The run under judgment is flagged; use get_run for a prior run's full check report.",
662
896
  inputSchema: z.object({}),
663
897
  execute: async () => {
664
- const guard = budgetGuard();
665
- if (guard) return { error: guard };
666
- return evidence.history.runs;
898
+ return span("list_runs", {}, async () => {
899
+ const guard = budgetGuard();
900
+ if (guard) return { error: guard };
901
+ return evidence.history.runs;
902
+ });
667
903
  }
668
904
  }),
669
905
  get_run: tool({
@@ -671,18 +907,20 @@ function buildEvidenceTools(args) {
671
907
  inputSchema: z.object({ run_id: z.string() }),
672
908
  execute: async (input) => {
673
909
  const { run_id } = input;
674
- const guard = budgetGuard();
675
- if (guard) return { error: guard };
676
- const detail = await evidence.history.getRun(run_id);
677
- const served = JSON.stringify(detail);
678
- const overflow = accountRead(
679
- "get_run",
680
- JSON.stringify({ run_id }),
681
- served,
682
- () => `read budget exhausted (${budget.maxReadBytes} bytes); call finish_verdict now`
683
- );
684
- if (overflow) return { error: overflow };
685
- return detail;
910
+ return span("get_run", input, async () => {
911
+ const guard = budgetGuard();
912
+ if (guard) return { error: guard };
913
+ const detail = await evidence.history.getRun(run_id);
914
+ const served = JSON.stringify(detail);
915
+ const overflow = accountRead(
916
+ "get_run",
917
+ JSON.stringify({ run_id }),
918
+ served,
919
+ () => `read budget exhausted (${budget.maxReadBytes} bytes); call finish_verdict now`
920
+ );
921
+ if (overflow) return { error: overflow };
922
+ return detail;
923
+ });
686
924
  }
687
925
  })
688
926
  } : {},
@@ -690,13 +928,15 @@ function buildEvidenceTools(args) {
690
928
  description: "The task this run executed: id, description, deliverable names.",
691
929
  inputSchema: z.object({}),
692
930
  execute: async () => {
693
- const guard = budgetGuard();
694
- if (guard) return { error: guard };
695
- return {
696
- task_id: scope?.taskId,
697
- description: scope?.taskDescription ?? "(unavailable)",
698
- deliverables: Object.keys(evidence.deliverables)
699
- };
931
+ return span("get_task_definition", {}, async () => {
932
+ const guard = budgetGuard();
933
+ if (guard) return { error: guard };
934
+ return {
935
+ task_id: scope?.taskId,
936
+ description: scope?.taskDescription ?? "(unavailable)",
937
+ deliverables: Object.keys(evidence.deliverables)
938
+ };
939
+ });
700
940
  }
701
941
  })
702
942
  };
@@ -706,21 +946,23 @@ function buildEvidenceTools(args) {
706
946
  inputSchema: z.object({ query: z.string().describe("substring filter on tool names; empty = all") }),
707
947
  execute: async (input) => {
708
948
  const { query } = input;
709
- const guard = budgetGuard();
710
- if (guard) return { error: guard };
711
- const view = evidence.view;
712
- if (!view) {
713
- return {
714
- status: "unsupported",
715
- detail: "no trace projection exists for this run \u2014 judge on deliverables instead"
716
- };
717
- }
718
- if (view.requireCapability("tools") !== "available") {
719
- return { status: "unsupported", detail: "tool-call evidence is unavailable in this trace projection" };
720
- }
721
- const calls = view.toolCalls.filter((c) => query ? c.name.includes(query) : true).slice(0, 30).map((c) => ({ name: c.name, input: renderValue(c.input).slice(0, 200) }));
722
- const reply = view.requireCapability("messages") === "available" && view.reply ? view.reply.slice(0, 1500) : void 0;
723
- return { turns: view.turnCount, tool_calls: calls, ...reply !== void 0 ? { final_reply: reply } : {} };
949
+ return span("get_trace", input, async () => {
950
+ const guard = budgetGuard();
951
+ if (guard) return { error: guard };
952
+ const view = evidence.view;
953
+ if (!view) {
954
+ return {
955
+ status: "unsupported",
956
+ detail: "no trace projection exists for this run \u2014 judge on deliverables instead"
957
+ };
958
+ }
959
+ if (view.requireCapability("tools") !== "available") {
960
+ return { status: "unsupported", detail: "tool-call evidence is unavailable in this trace projection" };
961
+ }
962
+ const calls = view.toolCalls.filter((c) => query ? c.name.includes(query) : true).slice(0, 30).map((c) => ({ name: c.name, input: renderValue(c.input).slice(0, 200) }));
963
+ const reply = view.requireCapability("messages") === "available" && view.reply ? view.reply.slice(0, 1500) : void 0;
964
+ return { turns: view.turnCount, tool_calls: calls, ...reply !== void 0 ? { final_reply: reply } : {} };
965
+ });
724
966
  }
725
967
  });
726
968
  }
@@ -847,7 +1089,9 @@ async function runAgentSession(spec) {
847
1089
  budget,
848
1090
  ledger,
849
1091
  stepIndexOf: () => liveSteps.length,
850
- includeTrace: spec.tools?.trace !== false
1092
+ includeTrace: spec.tools?.trace !== false,
1093
+ includeHistory: spec.tools?.history !== false,
1094
+ tracer: spec.tracer
851
1095
  });
852
1096
  const system = buildBriefing(spec.scope, spec.evidence, budget);
853
1097
  const exhibitsBlock = spec.exhibits?.length ? "\n\nValues submitted for judgment:\n" + spec.exhibits.map((v, i) => ` [${i + 1}] ${renderValue(v).slice(0, RECORD_RESULT_LIMIT)}`).join("\n") : "";
@@ -907,7 +1151,7 @@ async function runAgentSession(spec) {
907
1151
  latency_ms: Date.now() - started
908
1152
  };
909
1153
  }
910
- function createAgentMethod(rec, judgeConfig, judgeScope, evidence) {
1154
+ function createAgentMethod(rec, judgeConfig, judgeScope, evidence, judgeTracer) {
911
1155
  return async (instruction, opts) => {
912
1156
  const label = opts?.label ?? "agent";
913
1157
  const location = rec.captureLocation();
@@ -923,18 +1167,54 @@ function createAgentMethod(rec, judgeConfig, judgeScope, evidence) {
923
1167
  );
924
1168
  return;
925
1169
  }
1170
+ const run = () => runAgentSession({
1171
+ instruction,
1172
+ model: effective.model,
1173
+ baseURL: effective.baseURL,
1174
+ apiKey: effective.apiKey,
1175
+ evidence: evidence ?? { deliverables: {} },
1176
+ scope: judgeScope,
1177
+ exhibits,
1178
+ tools: opts?.tools,
1179
+ budget: opts?.budget,
1180
+ tracer: judgeTracer
1181
+ });
926
1182
  try {
927
- const result = await runAgentSession({
928
- instruction,
929
- model: effective.model,
930
- baseURL: effective.baseURL,
931
- apiKey: effective.apiKey,
932
- evidence: evidence ?? { deliverables: {} },
933
- scope: judgeScope,
934
- exhibits,
935
- tools: opts?.tools,
936
- budget: opts?.budget
937
- });
1183
+ let sessionSpanId;
1184
+ const result = await (judgeTracer ? judgeTracer.step(
1185
+ {
1186
+ step_name: `t.agent:${judgeScope?.checkName ?? label}`,
1187
+ observation_type: "AGENT",
1188
+ // The span carries the model + tokens so the backend prices the
1189
+ // judge session and trace rollups count its spend (issue #288).
1190
+ model: effective.model,
1191
+ input: { model: effective.model, instruction },
1192
+ usage: (r) => {
1193
+ const usage = r?.usage;
1194
+ return usage ? {
1195
+ prompt_tokens: usage.input_tokens,
1196
+ completion_tokens: usage.output_tokens
1197
+ } : void 0;
1198
+ },
1199
+ summarize: (r) => {
1200
+ const res = r;
1201
+ const reasoning = res?.verdict?.reasoning?.slice(0, 2e3) ?? `session ended without a verdict (${res?.outcome ?? "unknown"})`;
1202
+ return {
1203
+ text: reasoning,
1204
+ // reasoning-first, matching the judge response
1205
+ // contract's default order (#163).
1206
+ verdict: {
1207
+ reasoning,
1208
+ pass: res?.verdict?.pass ?? null
1209
+ }
1210
+ };
1211
+ }
1212
+ },
1213
+ async (spanId) => {
1214
+ sessionSpanId = spanId;
1215
+ return run();
1216
+ }
1217
+ ) : run());
938
1218
  const judge = {
939
1219
  model: effective.model,
940
1220
  temperature: 0,
@@ -945,7 +1225,8 @@ function createAgentMethod(rec, judgeConfig, judgeScope, evidence) {
945
1225
  },
946
1226
  ...result.cost !== void 0 ? { cost: result.cost } : {},
947
1227
  latency_ms: result.latency_ms,
948
- session: result.session
1228
+ session: result.session,
1229
+ ...isTraceableSpanId(sessionSpanId) ? { span_id: sessionSpanId } : {}
949
1230
  };
950
1231
  const detail = result.outcome === "error" ? `agentic judge session errored before verdicting: ${result.error ?? "unknown error"}` : `agentic judge session ended without a verdict (budget exhausted after ${result.session.usage?.steps ?? "?"} steps; last tool: ${result.session.steps?.at(-1)?.tool_calls?.map((c) => c.name).join(", ") ?? "none"})`;
951
1232
  rec.record(label, result.verdict?.pass ?? false, result.verdict?.reasoning ?? detail, {
@@ -1005,7 +1286,14 @@ function matchName(name, m) {
1005
1286
  if (m instanceof RegExp) return m.test(name);
1006
1287
  return m(name);
1007
1288
  }
1008
- function createJudgeMethod(rec, judgeConfig, judgeScope) {
1289
+ function trackPending(rec, method) {
1290
+ return ((...args) => {
1291
+ const promise = method(...args);
1292
+ rec.track(promise);
1293
+ return promise;
1294
+ });
1295
+ }
1296
+ function createJudgeMethod(rec, judgeConfig, judgeScope, judgeTracer) {
1009
1297
  return async (values, instruction, opts) => {
1010
1298
  const label = opts?.label ?? "judge";
1011
1299
  const location = rec.captureLocation();
@@ -1021,16 +1309,49 @@ function createJudgeMethod(rec, judgeConfig, judgeScope) {
1021
1309
  return;
1022
1310
  }
1023
1311
  const context2 = judgeScopeToContext(judgeScope);
1024
- try {
1025
- const { pass, reasoning, judge } = await callJudge({
1026
- values: valueArray,
1027
- instruction,
1312
+ const call = () => callJudge({
1313
+ values: valueArray,
1314
+ instruction,
1315
+ model: effective.model,
1316
+ baseURL: effective.baseURL,
1317
+ apiKey: effective.apiKey,
1318
+ prompt: effective.prompt,
1319
+ ...context2 ? { context: context2 } : {}
1320
+ });
1321
+ let judgeSpanId;
1322
+ const traced = judgeTracer ? judgeTracer.step(
1323
+ {
1324
+ step_name: `judge:${judgeScope?.checkName ?? label}`,
1325
+ // A single-shot judge call is an LLM generation — project it
1326
+ // with the same observation semantics as the main agent's.
1327
+ observation_type: "GENERATION",
1328
+ // The span carries the model + tokens so the backend prices the
1329
+ // judge call and trace rollups count its spend (issue #288).
1028
1330
  model: effective.model,
1029
- baseURL: effective.baseURL,
1030
- apiKey: effective.apiKey,
1031
- prompt: effective.prompt,
1032
- ...context2 ? { context: context2 } : {}
1033
- });
1331
+ input: { model: effective.model, instruction },
1332
+ summarize: (r) => {
1333
+ const res = r;
1334
+ return {
1335
+ text: res?.reasoning?.slice(0, 2e3) ?? "",
1336
+ verdict: {
1337
+ reasoning: res?.reasoning?.slice(0, 2e3) ?? null,
1338
+ pass: res?.pass ?? null
1339
+ }
1340
+ };
1341
+ },
1342
+ usage: (r) => {
1343
+ const tokens = r?.judge?.tokens;
1344
+ return tokens ? { prompt_tokens: tokens.input, completion_tokens: tokens.output } : void 0;
1345
+ }
1346
+ },
1347
+ async (spanId) => {
1348
+ judgeSpanId = spanId;
1349
+ return call();
1350
+ }
1351
+ ) : call();
1352
+ try {
1353
+ const { pass, reasoning, judge, unavailable } = await traced;
1354
+ if (isTraceableSpanId(judgeSpanId)) judge.span_id = judgeSpanId;
1034
1355
  rec.record(label, pass, reasoning, {
1035
1356
  evaluator_type: "llm",
1036
1357
  judge,
@@ -1039,7 +1360,8 @@ function createJudgeMethod(rec, judgeConfig, judgeScope) {
1039
1360
  // dashboard can render it with the right viewer — Markdown for
1040
1361
  // prose, a JSON tree for structured objects.
1041
1362
  received: valueArray.length === 1 ? valueArray[0] : valueArray,
1042
- location
1363
+ location,
1364
+ ...unavailable ? { outcome: "error" } : {}
1043
1365
  });
1044
1366
  } catch (error) {
1045
1367
  rec.record(
@@ -1050,7 +1372,8 @@ function createJudgeMethod(rec, judgeConfig, judgeScope) {
1050
1372
  evaluator_type: "llm",
1051
1373
  expected: instruction,
1052
1374
  received: valueArray.length === 1 ? valueArray[0] : valueArray,
1053
- location
1375
+ location,
1376
+ outcome: "error"
1054
1377
  }
1055
1378
  );
1056
1379
  }
@@ -1074,7 +1397,7 @@ var CAPABILITY_LABELS = {
1074
1397
  skills: "skill",
1075
1398
  subagents: "subagent"
1076
1399
  };
1077
- function createTraceTestContext(view, rec, judgeConfig, judgeScope, agentEvidence) {
1400
+ function createTraceTestContext(view, rec, judgeConfig, judgeScope, agentEvidence, judgeTracer) {
1078
1401
  const unsupported = (id, capability) => {
1079
1402
  const label = CAPABILITY_LABELS[capability];
1080
1403
  rec.record(id, false, `${label} evidence is unavailable in this trace projection`, {
@@ -1260,8 +1583,8 @@ function createTraceTestContext(view, rec, judgeConfig, judgeScope, agentEvidenc
1260
1583
  },
1261
1584
  // judge does not consult trace capabilities; the scope carries the
1262
1585
  // task/check frame for prompt builders (#161).
1263
- judge: createJudgeMethod(rec, judgeConfig, judgeScope),
1264
- agent: createAgentMethod(rec, judgeConfig, judgeScope, agentEvidence ?? { deliverables: {}, view })
1586
+ judge: trackPending(rec, createJudgeMethod(rec, judgeConfig, judgeScope, judgeTracer)),
1587
+ agent: trackPending(rec, createAgentMethod(rec, judgeConfig, judgeScope, agentEvidence ?? { deliverables: {}, view }, judgeTracer))
1265
1588
  };
1266
1589
  }
1267
1590
  function matchesTraceToolCall(call, name, opts) {
@@ -1282,16 +1605,9 @@ function matchesTraceToolCall(call, name, opts) {
1282
1605
  var Recorder = class {
1283
1606
  constructor(locate) {
1284
1607
  this.records = [];
1608
+ this.pending = [];
1285
1609
  this.locate = locate;
1286
1610
  }
1287
- /**
1288
- * Record an assertion.
1289
- *
1290
- * - ``extra.location`` overrides auto-capture (used by the runner for thrown
1291
- * errors, whose relevant stack is the error's own, not the call site).
1292
- * - ``extra.expected`` / ``extra.received`` carry the structured values for
1293
- * testing-framework-style display.
1294
- */
1295
1611
  /**
1296
1612
  * Capture the call site location synchronously. Use this when a record will
1297
1613
  * happen AFTER an `await` (e.g. `t.judge`): once an async function resumes,
@@ -1303,6 +1619,28 @@ var Recorder = class {
1303
1619
  captureLocation() {
1304
1620
  return this.locate ? this.locate(new Error().stack ?? "") : void 0;
1305
1621
  }
1622
+ /**
1623
+ * Register an async evaluation (``t.judge`` / ``t.agent``) started by this
1624
+ * check. If the check returns without awaiting it, the runner still waits
1625
+ * for its record via ``settlePending`` — a dropped await must not let the
1626
+ * check vacuously pass ("no assertions recorded") while the verdict is
1627
+ * still in flight.
1628
+ */
1629
+ track(promise) {
1630
+ this.pending.push(promise);
1631
+ }
1632
+ /** Wait until every tracked evaluation has settled (and recorded). */
1633
+ async settlePending() {
1634
+ await Promise.allSettled(this.pending);
1635
+ }
1636
+ /**
1637
+ * Record an assertion.
1638
+ *
1639
+ * - ``extra.location`` overrides auto-capture (used by the runner for thrown
1640
+ * errors, whose relevant stack is the error's own, not the call site).
1641
+ * - ``extra.expected`` / ``extra.received`` carry the structured values for
1642
+ * testing-framework-style display.
1643
+ */
1306
1644
  record(id, pass, reasoning, extra) {
1307
1645
  const location = extra?.location ?? (this.locate ? this.locate(new Error().stack ?? "") : void 0);
1308
1646
  this.records.push({
@@ -1414,8 +1752,10 @@ async function runTraceChecks(args) {
1414
1752
  {
1415
1753
  deliverables: args.deliverables,
1416
1754
  view,
1417
- ...args.historyPlane ? { history: args.historyPlane } : {}
1418
- }
1755
+ ...args.historyPlane ? { history: args.historyPlane } : {},
1756
+ ...args.historyUnavailableReason ? { historyUnavailable: args.historyUnavailableReason } : {}
1757
+ },
1758
+ args.judgeTracer
1419
1759
  );
1420
1760
  let thrownLocation;
1421
1761
  try {
@@ -1430,8 +1770,10 @@ async function runTraceChecks(args) {
1430
1770
  location: thrownLocation
1431
1771
  });
1432
1772
  }
1773
+ await rec.settlePending();
1433
1774
  const failed = rec.all.filter((r) => !r.pass);
1434
1775
  const pass = failed.length === 0;
1776
+ const outcome = rollUpCheckOutcome(failed);
1435
1777
  const reasoning = failed.length > 0 ? failed.map((r) => r.reasoning || r.id).join("; ") : rec.all.length > 0 ? "passed" : "no assertions recorded";
1436
1778
  const location = failed.find((r) => r.location)?.location;
1437
1779
  const judge = rec.all.find((r) => r.judge)?.judge;
@@ -1440,6 +1782,7 @@ async function runTraceChecks(args) {
1440
1782
  pass,
1441
1783
  reasoning,
1442
1784
  evaluator_type: "code",
1785
+ ...outcome ? { outcome } : {},
1443
1786
  ...judge ? { judge } : {},
1444
1787
  ...location ? { location } : {},
1445
1788
  ...args.displayFile ? { source_file: args.displayFile } : {},
@@ -1454,6 +1797,14 @@ async function runTraceChecks(args) {
1454
1797
  );
1455
1798
  return results;
1456
1799
  }
1800
+ function rollUpCheckOutcome(failed) {
1801
+ if (failed.length === 0) return void 0;
1802
+ const verdictless = failed.filter(
1803
+ (r) => r.outcome === "error" || r.outcome === "unsupported"
1804
+ );
1805
+ if (verdictless.length < failed.length) return void 0;
1806
+ return verdictless.some((r) => r.outcome === "error") ? "error" : "unsupported";
1807
+ }
1457
1808
  async function loadAndRunFlowChecks(checksPath, args, brokenDeliverables = {}) {
1458
1809
  if (!checksPath) return [];
1459
1810
  resetFlowChecks();
@@ -1466,6 +1817,7 @@ async function loadAndRunFlowChecks(checksPath, args, brokenDeliverables = {}) {
1466
1817
  task: args.task,
1467
1818
  judgeConfig: args.judgeConfig,
1468
1819
  ...args.historyPlane ? { historyPlane: args.historyPlane } : {},
1820
+ ...args.judgeTracer ? { judgeTracer: args.judgeTracer } : {},
1469
1821
  moduleUrl,
1470
1822
  displayFile: basename(checksPath)
1471
1823
  });
@@ -2517,7 +2869,29 @@ function createProjectionTee(real) {
2517
2869
  }
2518
2870
  },
2519
2871
  async traceTool(name, params, fn) {
2520
- return real.traceTool(name, params, fn);
2872
+ const obsSpanId = recordStart({
2873
+ step_name: `tool ${name}`,
2874
+ observation_type: "TOOL",
2875
+ tool_name: name,
2876
+ input: params
2877
+ });
2878
+ const start = monotonicNowMs();
2879
+ try {
2880
+ const result = await real.traceTool(name, params, fn);
2881
+ complete(obsSpanId, {
2882
+ latency_ms: round3(monotonicNowMs() - start),
2883
+ output: result
2884
+ });
2885
+ return result;
2886
+ } catch (error) {
2887
+ const message = error instanceof Error ? error.message : String(error);
2888
+ complete(obsSpanId, {
2889
+ latency_ms: round3(monotonicNowMs() - start),
2890
+ status_message: message,
2891
+ level: "ERROR"
2892
+ });
2893
+ throw error;
2894
+ }
2521
2895
  },
2522
2896
  async traceRetriever(query, fn) {
2523
2897
  return real.traceRetriever(query, fn);
@@ -2708,48 +3082,6 @@ function aggregateResult(checksResults) {
2708
3082
  return { checks: checksResults, pass };
2709
3083
  }
2710
3084
 
2711
- // src/agent-task/tracing.ts
2712
- function createNoopAgentTaskTraceContext() {
2713
- const noopFn = async (fn) => fn();
2714
- return {
2715
- runId: "agent-task-untraced",
2716
- rootSpanId: "agent-task-untraced-root",
2717
- async step(options, fn) {
2718
- return fn(options.id ?? "agent-task-untraced-step");
2719
- },
2720
- recordEvent(options) {
2721
- return options.id ?? "agent-task-untraced-event";
2722
- },
2723
- endRoot() {
2724
- },
2725
- traceTool(_name, _params, fn) {
2726
- return noopFn(fn);
2727
- },
2728
- traceRetriever(_query, fn) {
2729
- return noopFn(fn);
2730
- },
2731
- traceChain(_name, fn) {
2732
- return noopFn(fn);
2733
- },
2734
- traceAgent(_name, fn) {
2735
- return noopFn(fn);
2736
- },
2737
- traceGuardrail(_name, fn) {
2738
- return noopFn(fn);
2739
- },
2740
- traceEmbedding(_model, _input, fn) {
2741
- return noopFn(fn);
2742
- },
2743
- async score(_params) {
2744
- },
2745
- createSpan(_options) {
2746
- return "agent-task-untraced-span";
2747
- },
2748
- endSpan(_spanId, _params) {
2749
- }
2750
- };
2751
- }
2752
-
2753
3085
  // src/agent-task/run/runTask.ts
2754
3086
  var AgentTaskRunError = class extends Error {
2755
3087
  constructor(message, options) {
@@ -2776,13 +3108,15 @@ async function runTask(taskDir, options) {
2776
3108
  void 0
2777
3109
  );
2778
3110
  }
2779
- const phase1 = await trace2.client.traceRun(
3111
+ return trace2.client.traceRun(
2780
3112
  buildTraceRunOptions(loaded, trace2),
2781
- async (traceContext) => captureExecution(loaded, options, traceContext)
3113
+ async (traceContext) => {
3114
+ const phase1 = await captureExecution(loaded, options, traceContext);
3115
+ const canonical = await readCanonicalSnapshot(trace2);
3116
+ if (canonical) phase1.snapshot = canonical;
3117
+ return evaluate(loaded, options, phase1, traceContext);
3118
+ }
2782
3119
  );
2783
- const canonical = await readCanonicalSnapshot(trace2);
2784
- if (canonical) phase1.snapshot = canonical;
2785
- return evaluate(loaded, options, phase1);
2786
3120
  }
2787
3121
  async function readCanonicalSnapshot(trace2) {
2788
3122
  if (!trace2.taskRunId) return null;
@@ -2980,6 +3314,7 @@ async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
2980
3314
  files,
2981
3315
  task: task2,
2982
3316
  ...judgeConfig ? { judgeConfig } : {},
3317
+ judgeTracer: trace2,
2983
3318
  ...historyPlane ? { historyPlane } : {}
2984
3319
  },
2985
3320
  validationResults.brokenDeliverables
@@ -2994,6 +3329,7 @@ async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
2994
3329
  files,
2995
3330
  task: task2,
2996
3331
  ...judgeConfig ? { judgeConfig } : {},
3332
+ judgeTracer: trace2,
2997
3333
  ...historyPlane ? { historyPlane } : {},
2998
3334
  moduleUrl,
2999
3335
  displayFile: evalFileName
@@ -3022,7 +3358,7 @@ async function captureExecution(loaded, options, rawTrace) {
3022
3358
  snapshot: outcome.getSnapshot()
3023
3359
  }));
3024
3360
  }
3025
- async function evaluate(loaded, options, phase1) {
3361
+ async function evaluate(loaded, options, phase1, judgeTracer) {
3026
3362
  const {
3027
3363
  task: task2,
3028
3364
  adapter,
@@ -3040,12 +3376,13 @@ async function evaluate(loaded, options, phase1) {
3040
3376
  );
3041
3377
  const judgeConfig = resolveJudgeConfig(options?.judge, task2.judge);
3042
3378
  const historyPlane = await freezeHistoryPlaneFromEnv(task2.id);
3043
- const checksResults = await (inlineChecks ? runTraceChecks({
3379
+ const runChecks = async () => inlineChecks ? runTraceChecks({
3044
3380
  snapshot: phase1.snapshot,
3045
3381
  deliverables,
3046
3382
  files,
3047
3383
  task: task2,
3048
3384
  ...judgeConfig ? { judgeConfig } : {},
3385
+ ...judgeTracer ? { judgeTracer } : {},
3049
3386
  ...historyPlane ? { historyPlane } : {},
3050
3387
  moduleUrl,
3051
3388
  displayFile: evalFileName
@@ -3057,10 +3394,22 @@ async function evaluate(loaded, options, phase1) {
3057
3394
  files,
3058
3395
  task: task2,
3059
3396
  ...judgeConfig ? { judgeConfig } : {},
3397
+ ...judgeTracer ? { judgeTracer } : {},
3060
3398
  ...historyPlane ? { historyPlane } : {}
3061
3399
  },
3062
3400
  validationResults.brokenDeliverables
3063
- ));
3401
+ );
3402
+ const hasChecksModule = inlineChecks || checksPath !== null;
3403
+ const checksResults = await (judgeTracer && hasChecksModule ? judgeTracer.step(
3404
+ {
3405
+ step_name: "checks.run",
3406
+ observation_type: "CHAIN",
3407
+ input: { sourceFile: inlineChecks ? evalFileName : checksPath },
3408
+ metadata: { sourceFile: inlineChecks ? evalFileName : checksPath },
3409
+ summarize: (result2) => summarizeEvaluationResults(result2)
3410
+ },
3411
+ runChecks
3412
+ ) : runChecks());
3064
3413
  const result = aggregateResult(checksResults);
3065
3414
  return {
3066
3415
  task: task2,
@@ -3158,17 +3507,29 @@ async function cleanupAdapter(loaded, trace2, state, session) {
3158
3507
  }
3159
3508
  function summarizeEvaluationResults(results) {
3160
3509
  const passCount = results.filter((r) => r.pass).length;
3510
+ const noVerdictCount = results.filter(
3511
+ (r) => !r.pass && (r.outcome === "error" || r.outcome === "unsupported")
3512
+ ).length;
3513
+ const failCount = results.length - passCount - noVerdictCount;
3514
+ const suffix = noVerdictCount > 0 ? ` (${noVerdictCount} no-verdict)` : "";
3161
3515
  return {
3162
- total: results.length,
3163
- passCount,
3164
- failCount: results.length - passCount,
3165
- results: results.map((r) => ({
3166
- id: r.id,
3167
- pass: r.pass,
3168
- reasoning: r.reasoning,
3169
- evaluator_type: r.evaluator_type,
3170
- judge_model: r.judge?.model
3171
- }))
3516
+ text: `${passCount}/${results.length} checks passed${suffix}`,
3517
+ verdict: {
3518
+ total: results.length,
3519
+ passCount,
3520
+ failCount,
3521
+ noVerdictCount,
3522
+ results: results.map((r) => ({
3523
+ id: r.id,
3524
+ pass: r.pass,
3525
+ reasoning: r.reasoning?.slice(0, 2e3),
3526
+ evaluator_type: r.evaluator_type,
3527
+ judge_model: r.judge?.model,
3528
+ ...r.outcome ? { outcome: r.outcome } : {},
3529
+ ...r.group_id ? { group_id: r.group_id } : {},
3530
+ ...r.group_name ? { group_name: r.group_name } : {}
3531
+ }))
3532
+ }
3172
3533
  };
3173
3534
  }
3174
3535
  function buildTraceRunOptions(loaded, tracing) {
@@ -3296,6 +3657,7 @@ async function rejudgeTaskRun(runId, endpoints, options = {}) {
3296
3657
  );
3297
3658
  const proxied = proxyBrokenDeliverables(deliverables, validation.brokenDeliverables);
3298
3659
  const passCounts = /* @__PURE__ */ new Map();
3660
+ const errorCounts = /* @__PURE__ */ new Map();
3299
3661
  let primary = [];
3300
3662
  const judgeConfig = resolveJudgeConfig(options.judge, loaded.task.judge);
3301
3663
  for (let sample = 0; sample < samples; sample++) {
@@ -3304,12 +3666,16 @@ async function rejudgeTaskRun(runId, endpoints, options = {}) {
3304
3666
  if (sample === 0) primary = results;
3305
3667
  for (const result of results) {
3306
3668
  passCounts.set(result.id, (passCounts.get(result.id) ?? 0) + (result.pass ? 1 : 0));
3669
+ if (!result.pass && result.outcome === "error") {
3670
+ errorCounts.set(result.id, (errorCounts.get(result.id) ?? 0) + 1);
3671
+ }
3307
3672
  }
3308
3673
  }
3309
3674
  const stability = primary.map((result) => ({
3310
3675
  check_id: result.id,
3311
3676
  passes: passCounts.get(result.id) ?? 0,
3312
- samples
3677
+ samples,
3678
+ ...errorCounts.get(result.id) ? { errored: errorCounts.get(result.id) } : {}
3313
3679
  }));
3314
3680
  const aggregate = aggregateResult(primary);
3315
3681
  return {
@@ -3533,6 +3899,7 @@ function createOtelAgentTaskTraceClient(config) {
3533
3899
  activeSpans.set(spanId, {
3534
3900
  span,
3535
3901
  startedAt: monotonicNowMs2(),
3902
+ startWallMs: Date.now(),
3536
3903
  model: params.model || "unknown",
3537
3904
  observationType: params.observation_type
3538
3905
  });
@@ -3550,6 +3917,13 @@ function createOtelAgentTaskTraceClient(config) {
3550
3917
  const payload = params.output && typeof params.output === "object" ? params.output : { value: params.output };
3551
3918
  active.span.setAttribute("gen_ai.tool.call.result", JSON.stringify(payload));
3552
3919
  } else if (params.output.text) {
3920
+ active.span.setAttribute("gen_ai.response.text", String(params.output.text));
3921
+ if (params.output.verdict !== void 0) {
3922
+ active.span.setAttribute(
3923
+ "gen_ai.tool.call.result",
3924
+ JSON.stringify(params.output.verdict)
3925
+ );
3926
+ }
3553
3927
  active.span.setAttribute("gen_ai.output.messages", JSON.stringify([{
3554
3928
  role: "assistant",
3555
3929
  parts: [{ type: "text", content: String(params.output.text) }]
@@ -3574,6 +3948,12 @@ function createOtelAgentTaskTraceClient(config) {
3574
3948
  if (params.completion_tokens !== void 0) {
3575
3949
  active.span.setAttribute("gen_ai.usage.output_tokens", params.completion_tokens);
3576
3950
  }
3951
+ if (params.reasoning_tokens !== void 0) {
3952
+ active.span.setAttribute(
3953
+ "gen_ai.usage.reasoning.output_tokens",
3954
+ params.reasoning_tokens
3955
+ );
3956
+ }
3577
3957
  if (params.status_message) {
3578
3958
  active.span.setAttribute("apo.status_message", params.status_message);
3579
3959
  }
@@ -3583,7 +3963,7 @@ function createOtelAgentTaskTraceClient(config) {
3583
3963
  if (params.metadata) {
3584
3964
  active.span.setAttribute("apo.metadata", JSON.stringify(params.metadata));
3585
3965
  }
3586
- active.span.end();
3966
+ active.span.end(new Date(active.startWallMs + latency));
3587
3967
  activeSpans.delete(params.id);
3588
3968
  }
3589
3969
  return {
@@ -3617,6 +3997,7 @@ function createOtelAgentTaskTraceClient(config) {
3617
3997
  activeSpans.set(rootSpanId, {
3618
3998
  span: rootSpan,
3619
3999
  startedAt: monotonicNowMs2(),
4000
+ startWallMs: Date.now(),
3620
4001
  model: "trace"
3621
4002
  });
3622
4003
  let callCount = 1;
@@ -3642,7 +4023,8 @@ function createOtelAgentTaskTraceClient(config) {
3642
4023
  id: spanId,
3643
4024
  latency_ms: round32(monotonicNowMs2() - startedAt),
3644
4025
  output: options.summarize?.(result),
3645
- metadata: options.metadata
4026
+ metadata: options.metadata,
4027
+ ...options.usage?.(result)
3646
4028
  });
3647
4029
  callCount += 1;
3648
4030
  return result;