@apo-ai/sdk 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-task/integrations/ai-sdk.d.ts +1 -1
- package/dist/agent-task/integrations/ai-sdk.js +2 -2
- package/dist/agent-task/integrations/anthropic.d.ts +1 -1
- package/dist/agent-task/integrations/anthropic.js +2 -2
- package/dist/agent-task/integrations/openai.d.ts +2 -2
- package/dist/agent-task/integrations/openai.js +2 -2
- package/dist/agent-task/public.d.ts +35 -5
- package/dist/agent-task/public.js +635 -253
- package/dist/agent-task/public.js.map +1 -1
- package/dist/{chunk-6ZRW5IW6.js → chunk-3B3L2H4Y.js} +5 -2
- package/dist/chunk-3B3L2H4Y.js.map +1 -0
- package/dist/{chunk-GDB6SDGN.js → chunk-3OQ3JBFJ.js} +2 -1
- package/dist/chunk-3OQ3JBFJ.js.map +1 -0
- package/dist/{chunk-SZDSIPXC.js → chunk-43OI6GP7.js} +2 -2
- package/dist/{chunk-QHRWPVPZ.js → chunk-CKB3SUDO.js} +2 -2
- package/dist/{chunk-HWZVL6TS.js → chunk-HLRXQAZ5.js} +11 -6
- package/dist/chunk-HLRXQAZ5.js.map +1 -0
- package/dist/{chunk-GETY34VI.js → chunk-IPQ6IIEG.js} +3 -2
- package/dist/{chunk-GETY34VI.js.map → chunk-IPQ6IIEG.js.map} +1 -1
- package/dist/otel/index.js +2 -2
- package/dist/{tracing-DVLUqD-i.d.ts → tracing-CevT0hQg.d.ts} +13 -0
- package/package.json +1 -1
- package/dist/chunk-6ZRW5IW6.js.map +0 -1
- package/dist/chunk-GDB6SDGN.js.map +0 -1
- package/dist/chunk-HWZVL6TS.js.map +0 -1
- /package/dist/{chunk-SZDSIPXC.js.map → chunk-43OI6GP7.js.map} +0 -0
- /package/dist/{chunk-QHRWPVPZ.js.map → chunk-CKB3SUDO.js.map} +0 -0
|
@@ -8,18 +8,18 @@ import {
|
|
|
8
8
|
resetApoTracing,
|
|
9
9
|
withApoRun,
|
|
10
10
|
withApoRunSync
|
|
11
|
-
} from "../chunk-
|
|
11
|
+
} from "../chunk-IPQ6IIEG.js";
|
|
12
12
|
import {
|
|
13
13
|
createApoTracer
|
|
14
|
-
} from "../chunk-
|
|
15
|
-
import "../chunk-
|
|
14
|
+
} from "../chunk-CKB3SUDO.js";
|
|
15
|
+
import "../chunk-HLRXQAZ5.js";
|
|
16
16
|
import {
|
|
17
17
|
createApoOpenAI
|
|
18
|
-
} from "../chunk-
|
|
18
|
+
} from "../chunk-3B3L2H4Y.js";
|
|
19
19
|
import {
|
|
20
20
|
createApoAnthropic
|
|
21
|
-
} from "../chunk-
|
|
22
|
-
import "../chunk-
|
|
21
|
+
} from "../chunk-43OI6GP7.js";
|
|
22
|
+
import "../chunk-3OQ3JBFJ.js";
|
|
23
23
|
|
|
24
24
|
// src/agent-task/trace-projection/view.ts
|
|
25
25
|
function invocationOrderKey(obs) {
|
|
@@ -284,7 +284,18 @@ async function callSecondJudge(args) {
|
|
|
284
284
|
evidence.error = `Second judge API ${response.status}: ${body.slice(0, 200)}`;
|
|
285
285
|
return evidence;
|
|
286
286
|
}
|
|
287
|
-
const
|
|
287
|
+
const contentType = response.headers.get("content-type") ?? "";
|
|
288
|
+
if (contentType.includes("text/html")) {
|
|
289
|
+
evidence.error = "Second judge endpoint returned HTML, not the decisions API \u2014 APO_SECOND_JUDGE_BASE_URL likely points at the site root. Use the API root, e.g. https://openrouter.ai/api/v1";
|
|
290
|
+
return evidence;
|
|
291
|
+
}
|
|
292
|
+
let data;
|
|
293
|
+
try {
|
|
294
|
+
data = JSON.parse(await response.text());
|
|
295
|
+
} catch {
|
|
296
|
+
evidence.error = "Second judge endpoint returned a non-JSON body \u2014 check APO_SECOND_JUDGE_BASE_URL points at the decisions API root (e.g. https://openrouter.ai/api/v1)";
|
|
297
|
+
return evidence;
|
|
298
|
+
}
|
|
288
299
|
const verdict = data.answers?.verdict;
|
|
289
300
|
if (verdict?.choice !== "pass" && verdict?.choice !== "fail") {
|
|
290
301
|
evidence.error = "Second judge returned no verdict choice";
|
|
@@ -303,6 +314,12 @@ async function callSecondJudge(args) {
|
|
|
303
314
|
}
|
|
304
315
|
|
|
305
316
|
// src/agent-task/checks/judge.ts
|
|
317
|
+
var JudgeUnavailableError = class extends Error {
|
|
318
|
+
constructor(message) {
|
|
319
|
+
super(message);
|
|
320
|
+
this.name = "JudgeUnavailableError";
|
|
321
|
+
}
|
|
322
|
+
};
|
|
306
323
|
var VERDICT_FIRST_CONTRACT = 'Respond with ONLY a JSON object: {"pass": true/false, "reasoning": "your reasoning"}';
|
|
307
324
|
var REASONING_FIRST_CONTRACT = 'Respond with ONLY a JSON object: {"reasoning": "your reasoning", "pass": true/false}';
|
|
308
325
|
function isJudgeVerdictFirstOverrideEnabled() {
|
|
@@ -378,7 +395,92 @@ function parseJudgeJson(raw) {
|
|
|
378
395
|
reasoning: "Judge response could not be parsed as JSON \u2014 the verdict is unknown, so this check is treated as a failure. The model's raw response is available in the judge metadata."
|
|
379
396
|
};
|
|
380
397
|
}
|
|
381
|
-
var JUDGE_TIMEOUT_MS =
|
|
398
|
+
var JUDGE_TIMEOUT_MS = 3e5;
|
|
399
|
+
var JUDGE_IDLE_TIMEOUT_MS = 9e4;
|
|
400
|
+
var JUDGE_RETRY_DELAY_MS = 1e3;
|
|
401
|
+
var JUDGE_RETRY_AFTER_CAP_MS = 2e4;
|
|
402
|
+
var JUDGE_MIN_RETRY_BUDGET_MS = 6e4;
|
|
403
|
+
function isRetryableStatus(status) {
|
|
404
|
+
return status === 429 || status >= 500;
|
|
405
|
+
}
|
|
406
|
+
function retryDelayMs(retryAfter) {
|
|
407
|
+
if (!retryAfter) return JUDGE_RETRY_DELAY_MS;
|
|
408
|
+
const seconds = Number(retryAfter);
|
|
409
|
+
const ms = Number.isFinite(seconds) ? seconds * 1e3 : Date.parse(retryAfter) - Date.now();
|
|
410
|
+
if (!Number.isFinite(ms)) return JUDGE_RETRY_DELAY_MS;
|
|
411
|
+
return Math.min(Math.max(ms, JUDGE_RETRY_DELAY_MS), JUDGE_RETRY_AFTER_CAP_MS);
|
|
412
|
+
}
|
|
413
|
+
async function readCompletion(response, onData) {
|
|
414
|
+
const contentType = response.headers.get("content-type") ?? "";
|
|
415
|
+
if (!contentType.includes("text/event-stream")) {
|
|
416
|
+
const data = await response.json();
|
|
417
|
+
return {
|
|
418
|
+
text: data.choices?.[0]?.message?.content ?? "",
|
|
419
|
+
usage: data.usage,
|
|
420
|
+
finishReason: data.choices?.[0]?.finish_reason ?? void 0
|
|
421
|
+
};
|
|
422
|
+
}
|
|
423
|
+
let text = "";
|
|
424
|
+
let usage;
|
|
425
|
+
let finishReason;
|
|
426
|
+
let eventType = "message";
|
|
427
|
+
let dataLines = [];
|
|
428
|
+
let finished = false;
|
|
429
|
+
const dispatch = () => {
|
|
430
|
+
const payload = dataLines.join("\n").trim();
|
|
431
|
+
const type = eventType;
|
|
432
|
+
eventType = "message";
|
|
433
|
+
dataLines = [];
|
|
434
|
+
if (payload === "[DONE]") finished = true;
|
|
435
|
+
if (!payload || finished) return;
|
|
436
|
+
onData();
|
|
437
|
+
let chunk;
|
|
438
|
+
try {
|
|
439
|
+
chunk = JSON.parse(payload);
|
|
440
|
+
} catch {
|
|
441
|
+
chunk = void 0;
|
|
442
|
+
}
|
|
443
|
+
if (type === "error" || chunk?.error) {
|
|
444
|
+
const error = chunk?.error;
|
|
445
|
+
const message = typeof error === "string" ? error : error?.message ?? (chunk ? void 0 : payload);
|
|
446
|
+
throw new JudgeUnavailableError(`Judge stream error: ${message ?? "unknown"}`);
|
|
447
|
+
}
|
|
448
|
+
if (!chunk) return;
|
|
449
|
+
const choice = chunk.choices?.[0];
|
|
450
|
+
text += choice?.delta?.content ?? "";
|
|
451
|
+
if (choice?.finish_reason) finishReason = choice.finish_reason;
|
|
452
|
+
if (chunk.usage) usage = chunk.usage;
|
|
453
|
+
};
|
|
454
|
+
const handleLine = (line) => {
|
|
455
|
+
if (line === "") return dispatch();
|
|
456
|
+
if (line.startsWith(":")) return;
|
|
457
|
+
const colon = line.indexOf(":");
|
|
458
|
+
const field = colon === -1 ? line : line.slice(0, colon);
|
|
459
|
+
let value = colon === -1 ? "" : line.slice(colon + 1);
|
|
460
|
+
if (value.startsWith(" ")) value = value.slice(1);
|
|
461
|
+
if (field === "data") dataLines.push(value);
|
|
462
|
+
else if (field === "event") eventType = value;
|
|
463
|
+
};
|
|
464
|
+
const reader = response.body?.getReader();
|
|
465
|
+
if (!reader) return { text, usage, finishReason };
|
|
466
|
+
const decoder = new TextDecoder();
|
|
467
|
+
let buffer = "";
|
|
468
|
+
for (; ; ) {
|
|
469
|
+
const { done, value } = await reader.read();
|
|
470
|
+
if (done) break;
|
|
471
|
+
buffer += decoder.decode(value, { stream: true });
|
|
472
|
+
const lines = buffer.split(/\r\n|\r(?!$)|\n/);
|
|
473
|
+
buffer = lines.pop() ?? "";
|
|
474
|
+
for (const line of lines) {
|
|
475
|
+
handleLine(line);
|
|
476
|
+
if (finished) return { text, usage, finishReason };
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
buffer += decoder.decode();
|
|
480
|
+
for (const line of buffer.split(/\r\n|\r|\n/)) if (line) handleLine(line);
|
|
481
|
+
dispatch();
|
|
482
|
+
return { text, usage, finishReason };
|
|
483
|
+
}
|
|
382
484
|
var prefixQueues = /* @__PURE__ */ new Map();
|
|
383
485
|
function runWithSharedPrefix(key, task2) {
|
|
384
486
|
const prev = prefixQueues.get(key) ?? Promise.resolve();
|
|
@@ -411,75 +513,155 @@ ${instructionText}`,
|
|
|
411
513
|
apiKey: resolveSecondJudgeAPIKey(apiKey)
|
|
412
514
|
}) : void 0;
|
|
413
515
|
const cacheKey = `${args.model}\0${briefingText}\0${deliverableText}`;
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
...apiKey ? { Authorization: `Bearer ${apiKey}` } : {}
|
|
422
|
-
},
|
|
423
|
-
body: JSON.stringify({
|
|
424
|
-
model: args.model,
|
|
425
|
-
messages: [
|
|
516
|
+
const requestBody = (stream) => JSON.stringify({
|
|
517
|
+
model: args.model,
|
|
518
|
+
messages: [
|
|
519
|
+
{
|
|
520
|
+
role: "system",
|
|
521
|
+
content: [
|
|
522
|
+
{ type: "text", text: briefingText },
|
|
426
523
|
{
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
524
|
+
type: "text",
|
|
525
|
+
text: deliverableText,
|
|
526
|
+
cache_control: { type: "ephemeral" }
|
|
527
|
+
}
|
|
528
|
+
]
|
|
529
|
+
},
|
|
530
|
+
{ role: "user", content: instructionText }
|
|
531
|
+
],
|
|
532
|
+
temperature: 0,
|
|
533
|
+
response_format: { type: "json_object" },
|
|
534
|
+
...stream ? { stream: true, stream_options: { include_usage: true } } : {}
|
|
535
|
+
});
|
|
536
|
+
const attempt = async (stream, deadline) => {
|
|
537
|
+
const controller = new AbortController();
|
|
538
|
+
const expire = (reason, name) => () => controller.abort(new DOMException(reason, name));
|
|
539
|
+
const total = setTimeout(
|
|
540
|
+
expire(`no complete response within ${JUDGE_TIMEOUT_MS / 1e3}s`, "TimeoutError"),
|
|
541
|
+
Math.max(deadline - Date.now(), 0)
|
|
542
|
+
);
|
|
543
|
+
const stalled = expire(`no data for ${JUDGE_IDLE_TIMEOUT_MS / 1e3}s`, "IdleTimeoutError");
|
|
544
|
+
let idle;
|
|
545
|
+
const onData = () => {
|
|
546
|
+
clearTimeout(idle);
|
|
547
|
+
idle = setTimeout(stalled, JUDGE_IDLE_TIMEOUT_MS);
|
|
548
|
+
};
|
|
549
|
+
try {
|
|
550
|
+
let response;
|
|
551
|
+
try {
|
|
552
|
+
response = await fetch(`${baseURL}/chat/completions`, {
|
|
553
|
+
method: "POST",
|
|
554
|
+
signal: controller.signal,
|
|
555
|
+
headers: {
|
|
556
|
+
"Content-Type": "application/json",
|
|
557
|
+
...apiKey ? { Authorization: `Bearer ${apiKey}` } : {}
|
|
436
558
|
},
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
}
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
559
|
+
body: requestBody(stream)
|
|
560
|
+
});
|
|
561
|
+
} catch (error) {
|
|
562
|
+
return transportFailure("Judge request failed", controller.signal.reason ?? error);
|
|
563
|
+
}
|
|
564
|
+
if (!response.ok) {
|
|
565
|
+
const body = await response.text().catch(() => "");
|
|
566
|
+
return {
|
|
567
|
+
kind: "unavailable",
|
|
568
|
+
error: new JudgeUnavailableError(`Judge API ${response.status}: ${body.slice(0, 200)}`),
|
|
569
|
+
retryable: isRetryableStatus(response.status),
|
|
570
|
+
...isRetryableStatus(response.status) ? { retryDelayMs: retryDelayMs(response.headers.get("retry-after")) } : {},
|
|
571
|
+
// Name the field, not the word: gateways echo request params in
|
|
572
|
+
// unrelated 400 bodies.
|
|
573
|
+
...stream && response.status === 400 && /stream_options|["'`]stream["'`]/.test(body) ? { streamRejected: true } : {}
|
|
574
|
+
};
|
|
575
|
+
}
|
|
576
|
+
try {
|
|
577
|
+
return { kind: "completion", completion: await readCompletion(response, onData) };
|
|
578
|
+
} catch (error) {
|
|
579
|
+
return transportFailure("Judge response failed", controller.signal.reason ?? error);
|
|
580
|
+
}
|
|
581
|
+
} finally {
|
|
582
|
+
clearTimeout(total);
|
|
583
|
+
clearTimeout(idle);
|
|
584
|
+
controller.abort();
|
|
446
585
|
}
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
586
|
+
};
|
|
587
|
+
const isTruncated = ({ text, usage, finishReason }) => {
|
|
588
|
+
if (!text.trim()) return true;
|
|
589
|
+
if (usage !== void 0 && usage.completion_tokens === 0) return true;
|
|
590
|
+
if (finishReason !== "length" && finishReason !== "content_filter" && finishReason !== "error") {
|
|
591
|
+
return false;
|
|
592
|
+
}
|
|
593
|
+
try {
|
|
594
|
+
JSON.parse(text);
|
|
595
|
+
return false;
|
|
596
|
+
} catch {
|
|
597
|
+
return true;
|
|
598
|
+
}
|
|
599
|
+
};
|
|
600
|
+
return runWithSharedPrefix(cacheKey, async () => {
|
|
601
|
+
const startedAt = Date.now();
|
|
602
|
+
const deadline = startedAt + JUDGE_TIMEOUT_MS;
|
|
603
|
+
let stream = true;
|
|
604
|
+
let result = await attempt(stream, deadline);
|
|
605
|
+
const truncated = result.kind === "completion" && isTruncated(result.completion);
|
|
606
|
+
const streamRejected = result.kind === "unavailable" && result.streamRejected === true;
|
|
607
|
+
const retryable = result.kind === "unavailable" && result.retryable;
|
|
608
|
+
if (truncated || streamRejected || retryable) {
|
|
609
|
+
const delay = result.kind === "unavailable" && result.retryDelayMs !== void 0 ? result.retryDelayMs : streamRejected ? 0 : JUDGE_RETRY_DELAY_MS;
|
|
610
|
+
if (deadline - Date.now() - delay >= JUDGE_MIN_RETRY_BUDGET_MS) {
|
|
611
|
+
if (streamRejected) stream = false;
|
|
612
|
+
await new Promise((resolve4) => setTimeout(resolve4, delay));
|
|
613
|
+
const first = result;
|
|
614
|
+
result = await attempt(stream, deadline);
|
|
615
|
+
if (result.kind === "unavailable" && first.kind === "unavailable") {
|
|
616
|
+
result = {
|
|
617
|
+
...result,
|
|
618
|
+
error: new JudgeUnavailableError(
|
|
619
|
+
`${first.error.message} (retry: ${result.error.message})`
|
|
620
|
+
)
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
if (result.kind === "unavailable") throw result.error;
|
|
626
|
+
const { text, usage } = result.completion;
|
|
627
|
+
const judge = {
|
|
628
|
+
model: args.model,
|
|
629
|
+
contract: judgeContractInUse(),
|
|
630
|
+
prompt: { system: systemPromptText, user: instructionText },
|
|
631
|
+
response: text,
|
|
632
|
+
tokens: parseJudgeUsage(usage),
|
|
633
|
+
latency_ms: Date.now() - startedAt,
|
|
634
|
+
secondJudge: await secondJudgePromise
|
|
635
|
+
};
|
|
636
|
+
if (isTruncated(result.completion)) {
|
|
453
637
|
return {
|
|
454
638
|
pass: false,
|
|
455
|
-
reasoning: "Judge returned an empty or truncated response \u2014 likely a transient provider failure. The verdict is unknown, so this check is
|
|
456
|
-
judge
|
|
457
|
-
|
|
458
|
-
contract: judgeContractInUse(),
|
|
459
|
-
prompt: { system: systemPromptText, user: instructionText },
|
|
460
|
-
response: text,
|
|
461
|
-
tokens: parseJudgeUsage(data.usage),
|
|
462
|
-
latency_ms: Date.now() - startedAt,
|
|
463
|
-
secondJudge: await secondJudgePromise
|
|
464
|
-
}
|
|
639
|
+
reasoning: "Judge returned an empty or truncated response \u2014 likely a transient provider failure. The verdict is unknown, so this check is recorded as a judge error, not a verdict.",
|
|
640
|
+
judge,
|
|
641
|
+
unavailable: true
|
|
465
642
|
};
|
|
466
643
|
}
|
|
467
644
|
const parsed = parseJudgeJson(text);
|
|
468
645
|
return {
|
|
469
646
|
pass: parsed.pass === true,
|
|
470
647
|
reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : "",
|
|
471
|
-
judge
|
|
472
|
-
model: args.model,
|
|
473
|
-
contract: judgeContractInUse(),
|
|
474
|
-
prompt: { system: systemPromptText, user: instructionText },
|
|
475
|
-
response: text,
|
|
476
|
-
tokens: parseJudgeUsage(data.usage),
|
|
477
|
-
latency_ms: Date.now() - startedAt,
|
|
478
|
-
secondJudge: await secondJudgePromise
|
|
479
|
-
}
|
|
648
|
+
judge
|
|
480
649
|
};
|
|
481
650
|
});
|
|
482
651
|
}
|
|
652
|
+
function transportFailure(prefix, error) {
|
|
653
|
+
if (error instanceof JudgeUnavailableError) {
|
|
654
|
+
return { kind: "unavailable", error, retryable: true };
|
|
655
|
+
}
|
|
656
|
+
const timedOut = error instanceof Error && error.name === "TimeoutError";
|
|
657
|
+
return {
|
|
658
|
+
kind: "unavailable",
|
|
659
|
+
error: new JudgeUnavailableError(
|
|
660
|
+
`${prefix}: ${error instanceof Error ? error.message : String(error)}`
|
|
661
|
+
),
|
|
662
|
+
retryable: !timedOut
|
|
663
|
+
};
|
|
664
|
+
}
|
|
483
665
|
function assembleBriefing(args) {
|
|
484
666
|
if (!args.prompt) {
|
|
485
667
|
return {
|
|
@@ -510,6 +692,53 @@ ${args.instruction}`
|
|
|
510
692
|
|
|
511
693
|
// src/agent-task/checks/agent-session.ts
|
|
512
694
|
import { createHash } from "crypto";
|
|
695
|
+
|
|
696
|
+
// src/agent-task/tracing.ts
|
|
697
|
+
function createNoopAgentTaskTraceContext() {
|
|
698
|
+
const noopFn = async (fn) => fn();
|
|
699
|
+
return {
|
|
700
|
+
runId: "agent-task-untraced",
|
|
701
|
+
rootSpanId: "agent-task-untraced-root",
|
|
702
|
+
async step(options, fn) {
|
|
703
|
+
return fn(options.id ?? "agent-task-untraced-step");
|
|
704
|
+
},
|
|
705
|
+
recordEvent(options) {
|
|
706
|
+
return options.id ?? "agent-task-untraced-event";
|
|
707
|
+
},
|
|
708
|
+
endRoot() {
|
|
709
|
+
},
|
|
710
|
+
traceTool(_name, _params, fn) {
|
|
711
|
+
return noopFn(fn);
|
|
712
|
+
},
|
|
713
|
+
traceRetriever(_query, fn) {
|
|
714
|
+
return noopFn(fn);
|
|
715
|
+
},
|
|
716
|
+
traceChain(_name, fn) {
|
|
717
|
+
return noopFn(fn);
|
|
718
|
+
},
|
|
719
|
+
traceAgent(_name, fn) {
|
|
720
|
+
return noopFn(fn);
|
|
721
|
+
},
|
|
722
|
+
traceGuardrail(_name, fn) {
|
|
723
|
+
return noopFn(fn);
|
|
724
|
+
},
|
|
725
|
+
traceEmbedding(_model, _input, fn) {
|
|
726
|
+
return noopFn(fn);
|
|
727
|
+
},
|
|
728
|
+
async score(_params) {
|
|
729
|
+
},
|
|
730
|
+
createSpan(_options) {
|
|
731
|
+
return "agent-task-untraced-span";
|
|
732
|
+
},
|
|
733
|
+
endSpan(_spanId, _params) {
|
|
734
|
+
}
|
|
735
|
+
};
|
|
736
|
+
}
|
|
737
|
+
function isTraceableSpanId(spanId) {
|
|
738
|
+
return spanId !== void 0 && !spanId.startsWith("agent-task-untraced");
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// src/agent-task/checks/agent-session.ts
|
|
513
742
|
var DEFAULT_BUDGET = {
|
|
514
743
|
maxTurns: 12,
|
|
515
744
|
maxToolCalls: 24,
|
|
@@ -570,7 +799,7 @@ function extractVerdict(steps) {
|
|
|
570
799
|
return void 0;
|
|
571
800
|
}
|
|
572
801
|
function buildEvidenceTools(args) {
|
|
573
|
-
const { z, tool, evidence, scope, budget, ledger, stepIndexOf, includeTrace } = args;
|
|
802
|
+
const { z, tool, evidence, scope, budget, ledger, stepIndexOf, includeTrace, includeHistory, tracer } = args;
|
|
574
803
|
const budgetGuard = () => {
|
|
575
804
|
ledger.toolCallsUsed += 1;
|
|
576
805
|
if (ledger.toolCallsUsed > budget.maxToolCalls) {
|
|
@@ -589,6 +818,7 @@ function buildEvidenceTools(args) {
|
|
|
589
818
|
});
|
|
590
819
|
return ledger.readBytesUsed > budget.maxReadBytes ? overflow() : null;
|
|
591
820
|
};
|
|
821
|
+
const span = (name, input, fn) => tracer ? tracer.traceTool(name, input, fn) : fn();
|
|
592
822
|
const tools = {
|
|
593
823
|
read_deliverable: tool({
|
|
594
824
|
description: `Read one of this run's deliverables by name, with optional offset/limit for pagination (max ${MAX_READ_WINDOW} bytes per call).`,
|
|
@@ -599,21 +829,23 @@ function buildEvidenceTools(args) {
|
|
|
599
829
|
}),
|
|
600
830
|
execute: async (input) => {
|
|
601
831
|
const { name, offset, limit } = input;
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
832
|
+
return span("read_deliverable", input, async () => {
|
|
833
|
+
const guard = budgetGuard();
|
|
834
|
+
if (guard) return { error: guard };
|
|
835
|
+
if (!Object.prototype.hasOwnProperty.call(evidence.deliverables, name)) {
|
|
836
|
+
return { error: `unknown deliverable: ${name}` };
|
|
837
|
+
}
|
|
838
|
+
const content = renderValue(evidence.deliverables[name]);
|
|
839
|
+
const slice = content.slice(offset, offset + limit);
|
|
840
|
+
const overflow = accountRead(
|
|
841
|
+
"read_deliverable",
|
|
842
|
+
JSON.stringify({ name, offset, limit }),
|
|
843
|
+
slice,
|
|
844
|
+
() => `read budget exhausted (${budget.maxReadBytes} bytes); call finish_verdict now`
|
|
845
|
+
);
|
|
846
|
+
if (overflow) return { error: overflow };
|
|
847
|
+
return { name, total_bytes: content.length, offset, returned: slice.length, content: slice };
|
|
848
|
+
});
|
|
617
849
|
}
|
|
618
850
|
}),
|
|
619
851
|
search_deliverable: tool({
|
|
@@ -621,49 +853,53 @@ function buildEvidenceTools(args) {
|
|
|
621
853
|
inputSchema: z.object({ name: z.string(), pattern: z.string() }),
|
|
622
854
|
execute: async (input) => {
|
|
623
855
|
const { name, pattern } = input;
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
856
|
+
return span("search_deliverable", input, async () => {
|
|
857
|
+
const guard = budgetGuard();
|
|
858
|
+
if (guard) return { error: guard };
|
|
859
|
+
if (!Object.prototype.hasOwnProperty.call(evidence.deliverables, name)) {
|
|
860
|
+
return { error: `unknown deliverable: ${name}` };
|
|
861
|
+
}
|
|
862
|
+
if (pattern.length > SEARCH_MAX_PATTERN) {
|
|
863
|
+
return { error: `pattern too long (max ${SEARCH_MAX_PATTERN} chars)` };
|
|
864
|
+
}
|
|
865
|
+
let re;
|
|
866
|
+
try {
|
|
867
|
+
re = new RegExp(pattern, "gi");
|
|
868
|
+
} catch (error) {
|
|
869
|
+
return { error: `invalid regex: ${error instanceof Error ? error.message : String(error)}` };
|
|
870
|
+
}
|
|
871
|
+
const content = renderValue(evidence.deliverables[name]).slice(0, SEARCH_SCAN_LIMIT);
|
|
872
|
+
const hits = [];
|
|
873
|
+
let m;
|
|
874
|
+
let execs = 0;
|
|
875
|
+
while ((m = re.exec(content)) && hits.length < SEARCH_HITS) {
|
|
876
|
+
if (++execs > SEARCH_MAX_EXEC) break;
|
|
877
|
+
const start = Math.max(0, m.index - SEARCH_CONTEXT);
|
|
878
|
+
hits.push({ at_byte: m.index, context: content.slice(start, m.index + m[0].length + SEARCH_CONTEXT) });
|
|
879
|
+
if (m.index === re.lastIndex) re.lastIndex++;
|
|
880
|
+
}
|
|
881
|
+
const served = JSON.stringify({ total_bytes: content.length, match_count: hits.length, matches: hits });
|
|
882
|
+
const overflow = accountRead(
|
|
883
|
+
"search_deliverable",
|
|
884
|
+
JSON.stringify({ name, pattern }),
|
|
885
|
+
served,
|
|
886
|
+
() => `read budget exhausted (${budget.maxReadBytes} bytes); call finish_verdict now`
|
|
887
|
+
);
|
|
888
|
+
if (overflow) return { error: overflow };
|
|
889
|
+
return { total_bytes: content.length, match_count: hits.length, matches: hits };
|
|
890
|
+
});
|
|
657
891
|
}
|
|
658
892
|
}),
|
|
659
|
-
...evidence.history ? {
|
|
893
|
+
...includeHistory && evidence.history ? {
|
|
660
894
|
list_runs: tool({
|
|
661
895
|
description: "List runs of this task \u2014 id, status, pass/fail, model, check counts. The run under judgment is flagged; use get_run for a prior run's full check report.",
|
|
662
896
|
inputSchema: z.object({}),
|
|
663
897
|
execute: async () => {
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
898
|
+
return span("list_runs", {}, async () => {
|
|
899
|
+
const guard = budgetGuard();
|
|
900
|
+
if (guard) return { error: guard };
|
|
901
|
+
return evidence.history.runs;
|
|
902
|
+
});
|
|
667
903
|
}
|
|
668
904
|
}),
|
|
669
905
|
get_run: tool({
|
|
@@ -671,18 +907,20 @@ function buildEvidenceTools(args) {
|
|
|
671
907
|
inputSchema: z.object({ run_id: z.string() }),
|
|
672
908
|
execute: async (input) => {
|
|
673
909
|
const { run_id } = input;
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
910
|
+
return span("get_run", input, async () => {
|
|
911
|
+
const guard = budgetGuard();
|
|
912
|
+
if (guard) return { error: guard };
|
|
913
|
+
const detail = await evidence.history.getRun(run_id);
|
|
914
|
+
const served = JSON.stringify(detail);
|
|
915
|
+
const overflow = accountRead(
|
|
916
|
+
"get_run",
|
|
917
|
+
JSON.stringify({ run_id }),
|
|
918
|
+
served,
|
|
919
|
+
() => `read budget exhausted (${budget.maxReadBytes} bytes); call finish_verdict now`
|
|
920
|
+
);
|
|
921
|
+
if (overflow) return { error: overflow };
|
|
922
|
+
return detail;
|
|
923
|
+
});
|
|
686
924
|
}
|
|
687
925
|
})
|
|
688
926
|
} : {},
|
|
@@ -690,13 +928,15 @@ function buildEvidenceTools(args) {
|
|
|
690
928
|
description: "The task this run executed: id, description, deliverable names.",
|
|
691
929
|
inputSchema: z.object({}),
|
|
692
930
|
execute: async () => {
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
931
|
+
return span("get_task_definition", {}, async () => {
|
|
932
|
+
const guard = budgetGuard();
|
|
933
|
+
if (guard) return { error: guard };
|
|
934
|
+
return {
|
|
935
|
+
task_id: scope?.taskId,
|
|
936
|
+
description: scope?.taskDescription ?? "(unavailable)",
|
|
937
|
+
deliverables: Object.keys(evidence.deliverables)
|
|
938
|
+
};
|
|
939
|
+
});
|
|
700
940
|
}
|
|
701
941
|
})
|
|
702
942
|
};
|
|
@@ -706,21 +946,23 @@ function buildEvidenceTools(args) {
|
|
|
706
946
|
inputSchema: z.object({ query: z.string().describe("substring filter on tool names; empty = all") }),
|
|
707
947
|
execute: async (input) => {
|
|
708
948
|
const { query } = input;
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
949
|
+
return span("get_trace", input, async () => {
|
|
950
|
+
const guard = budgetGuard();
|
|
951
|
+
if (guard) return { error: guard };
|
|
952
|
+
const view = evidence.view;
|
|
953
|
+
if (!view) {
|
|
954
|
+
return {
|
|
955
|
+
status: "unsupported",
|
|
956
|
+
detail: "no trace projection exists for this run \u2014 judge on deliverables instead"
|
|
957
|
+
};
|
|
958
|
+
}
|
|
959
|
+
if (view.requireCapability("tools") !== "available") {
|
|
960
|
+
return { status: "unsupported", detail: "tool-call evidence is unavailable in this trace projection" };
|
|
961
|
+
}
|
|
962
|
+
const calls = view.toolCalls.filter((c) => query ? c.name.includes(query) : true).slice(0, 30).map((c) => ({ name: c.name, input: renderValue(c.input).slice(0, 200) }));
|
|
963
|
+
const reply = view.requireCapability("messages") === "available" && view.reply ? view.reply.slice(0, 1500) : void 0;
|
|
964
|
+
return { turns: view.turnCount, tool_calls: calls, ...reply !== void 0 ? { final_reply: reply } : {} };
|
|
965
|
+
});
|
|
724
966
|
}
|
|
725
967
|
});
|
|
726
968
|
}
|
|
@@ -847,7 +1089,9 @@ async function runAgentSession(spec) {
|
|
|
847
1089
|
budget,
|
|
848
1090
|
ledger,
|
|
849
1091
|
stepIndexOf: () => liveSteps.length,
|
|
850
|
-
includeTrace: spec.tools?.trace !== false
|
|
1092
|
+
includeTrace: spec.tools?.trace !== false,
|
|
1093
|
+
includeHistory: spec.tools?.history !== false,
|
|
1094
|
+
tracer: spec.tracer
|
|
851
1095
|
});
|
|
852
1096
|
const system = buildBriefing(spec.scope, spec.evidence, budget);
|
|
853
1097
|
const exhibitsBlock = spec.exhibits?.length ? "\n\nValues submitted for judgment:\n" + spec.exhibits.map((v, i) => ` [${i + 1}] ${renderValue(v).slice(0, RECORD_RESULT_LIMIT)}`).join("\n") : "";
|
|
@@ -907,7 +1151,7 @@ async function runAgentSession(spec) {
|
|
|
907
1151
|
latency_ms: Date.now() - started
|
|
908
1152
|
};
|
|
909
1153
|
}
|
|
910
|
-
function createAgentMethod(rec, judgeConfig, judgeScope, evidence) {
|
|
1154
|
+
function createAgentMethod(rec, judgeConfig, judgeScope, evidence, judgeTracer) {
|
|
911
1155
|
return async (instruction, opts) => {
|
|
912
1156
|
const label = opts?.label ?? "agent";
|
|
913
1157
|
const location = rec.captureLocation();
|
|
@@ -923,18 +1167,54 @@ function createAgentMethod(rec, judgeConfig, judgeScope, evidence) {
|
|
|
923
1167
|
);
|
|
924
1168
|
return;
|
|
925
1169
|
}
|
|
1170
|
+
const run = () => runAgentSession({
|
|
1171
|
+
instruction,
|
|
1172
|
+
model: effective.model,
|
|
1173
|
+
baseURL: effective.baseURL,
|
|
1174
|
+
apiKey: effective.apiKey,
|
|
1175
|
+
evidence: evidence ?? { deliverables: {} },
|
|
1176
|
+
scope: judgeScope,
|
|
1177
|
+
exhibits,
|
|
1178
|
+
tools: opts?.tools,
|
|
1179
|
+
budget: opts?.budget,
|
|
1180
|
+
tracer: judgeTracer
|
|
1181
|
+
});
|
|
926
1182
|
try {
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
1183
|
+
let sessionSpanId;
|
|
1184
|
+
const result = await (judgeTracer ? judgeTracer.step(
|
|
1185
|
+
{
|
|
1186
|
+
step_name: `t.agent:${judgeScope?.checkName ?? label}`,
|
|
1187
|
+
observation_type: "AGENT",
|
|
1188
|
+
// The span carries the model + tokens so the backend prices the
|
|
1189
|
+
// judge session and trace rollups count its spend (issue #288).
|
|
1190
|
+
model: effective.model,
|
|
1191
|
+
input: { model: effective.model, instruction },
|
|
1192
|
+
usage: (r) => {
|
|
1193
|
+
const usage = r?.usage;
|
|
1194
|
+
return usage ? {
|
|
1195
|
+
prompt_tokens: usage.input_tokens,
|
|
1196
|
+
completion_tokens: usage.output_tokens
|
|
1197
|
+
} : void 0;
|
|
1198
|
+
},
|
|
1199
|
+
summarize: (r) => {
|
|
1200
|
+
const res = r;
|
|
1201
|
+
const reasoning = res?.verdict?.reasoning?.slice(0, 2e3) ?? `session ended without a verdict (${res?.outcome ?? "unknown"})`;
|
|
1202
|
+
return {
|
|
1203
|
+
text: reasoning,
|
|
1204
|
+
// reasoning-first, matching the judge response
|
|
1205
|
+
// contract's default order (#163).
|
|
1206
|
+
verdict: {
|
|
1207
|
+
reasoning,
|
|
1208
|
+
pass: res?.verdict?.pass ?? null
|
|
1209
|
+
}
|
|
1210
|
+
};
|
|
1211
|
+
}
|
|
1212
|
+
},
|
|
1213
|
+
async (spanId) => {
|
|
1214
|
+
sessionSpanId = spanId;
|
|
1215
|
+
return run();
|
|
1216
|
+
}
|
|
1217
|
+
) : run());
|
|
938
1218
|
const judge = {
|
|
939
1219
|
model: effective.model,
|
|
940
1220
|
temperature: 0,
|
|
@@ -945,7 +1225,8 @@ function createAgentMethod(rec, judgeConfig, judgeScope, evidence) {
|
|
|
945
1225
|
},
|
|
946
1226
|
...result.cost !== void 0 ? { cost: result.cost } : {},
|
|
947
1227
|
latency_ms: result.latency_ms,
|
|
948
|
-
session: result.session
|
|
1228
|
+
session: result.session,
|
|
1229
|
+
...isTraceableSpanId(sessionSpanId) ? { span_id: sessionSpanId } : {}
|
|
949
1230
|
};
|
|
950
1231
|
const detail = result.outcome === "error" ? `agentic judge session errored before verdicting: ${result.error ?? "unknown error"}` : `agentic judge session ended without a verdict (budget exhausted after ${result.session.usage?.steps ?? "?"} steps; last tool: ${result.session.steps?.at(-1)?.tool_calls?.map((c) => c.name).join(", ") ?? "none"})`;
|
|
951
1232
|
rec.record(label, result.verdict?.pass ?? false, result.verdict?.reasoning ?? detail, {
|
|
@@ -1005,7 +1286,14 @@ function matchName(name, m) {
|
|
|
1005
1286
|
if (m instanceof RegExp) return m.test(name);
|
|
1006
1287
|
return m(name);
|
|
1007
1288
|
}
|
|
1008
|
-
function
|
|
1289
|
+
function trackPending(rec, method) {
|
|
1290
|
+
return ((...args) => {
|
|
1291
|
+
const promise = method(...args);
|
|
1292
|
+
rec.track(promise);
|
|
1293
|
+
return promise;
|
|
1294
|
+
});
|
|
1295
|
+
}
|
|
1296
|
+
function createJudgeMethod(rec, judgeConfig, judgeScope, judgeTracer) {
|
|
1009
1297
|
return async (values, instruction, opts) => {
|
|
1010
1298
|
const label = opts?.label ?? "judge";
|
|
1011
1299
|
const location = rec.captureLocation();
|
|
@@ -1021,16 +1309,49 @@ function createJudgeMethod(rec, judgeConfig, judgeScope) {
|
|
|
1021
1309
|
return;
|
|
1022
1310
|
}
|
|
1023
1311
|
const context2 = judgeScopeToContext(judgeScope);
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1312
|
+
const call = () => callJudge({
|
|
1313
|
+
values: valueArray,
|
|
1314
|
+
instruction,
|
|
1315
|
+
model: effective.model,
|
|
1316
|
+
baseURL: effective.baseURL,
|
|
1317
|
+
apiKey: effective.apiKey,
|
|
1318
|
+
prompt: effective.prompt,
|
|
1319
|
+
...context2 ? { context: context2 } : {}
|
|
1320
|
+
});
|
|
1321
|
+
let judgeSpanId;
|
|
1322
|
+
const traced = judgeTracer ? judgeTracer.step(
|
|
1323
|
+
{
|
|
1324
|
+
step_name: `judge:${judgeScope?.checkName ?? label}`,
|
|
1325
|
+
// A single-shot judge call is an LLM generation — project it
|
|
1326
|
+
// with the same observation semantics as the main agent's.
|
|
1327
|
+
observation_type: "GENERATION",
|
|
1328
|
+
// The span carries the model + tokens so the backend prices the
|
|
1329
|
+
// judge call and trace rollups count its spend (issue #288).
|
|
1028
1330
|
model: effective.model,
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1331
|
+
input: { model: effective.model, instruction },
|
|
1332
|
+
summarize: (r) => {
|
|
1333
|
+
const res = r;
|
|
1334
|
+
return {
|
|
1335
|
+
text: res?.reasoning?.slice(0, 2e3) ?? "",
|
|
1336
|
+
verdict: {
|
|
1337
|
+
reasoning: res?.reasoning?.slice(0, 2e3) ?? null,
|
|
1338
|
+
pass: res?.pass ?? null
|
|
1339
|
+
}
|
|
1340
|
+
};
|
|
1341
|
+
},
|
|
1342
|
+
usage: (r) => {
|
|
1343
|
+
const tokens = r?.judge?.tokens;
|
|
1344
|
+
return tokens ? { prompt_tokens: tokens.input, completion_tokens: tokens.output } : void 0;
|
|
1345
|
+
}
|
|
1346
|
+
},
|
|
1347
|
+
async (spanId) => {
|
|
1348
|
+
judgeSpanId = spanId;
|
|
1349
|
+
return call();
|
|
1350
|
+
}
|
|
1351
|
+
) : call();
|
|
1352
|
+
try {
|
|
1353
|
+
const { pass, reasoning, judge, unavailable } = await traced;
|
|
1354
|
+
if (isTraceableSpanId(judgeSpanId)) judge.span_id = judgeSpanId;
|
|
1034
1355
|
rec.record(label, pass, reasoning, {
|
|
1035
1356
|
evaluator_type: "llm",
|
|
1036
1357
|
judge,
|
|
@@ -1039,7 +1360,8 @@ function createJudgeMethod(rec, judgeConfig, judgeScope) {
|
|
|
1039
1360
|
// dashboard can render it with the right viewer — Markdown for
|
|
1040
1361
|
// prose, a JSON tree for structured objects.
|
|
1041
1362
|
received: valueArray.length === 1 ? valueArray[0] : valueArray,
|
|
1042
|
-
location
|
|
1363
|
+
location,
|
|
1364
|
+
...unavailable ? { outcome: "error" } : {}
|
|
1043
1365
|
});
|
|
1044
1366
|
} catch (error) {
|
|
1045
1367
|
rec.record(
|
|
@@ -1050,7 +1372,8 @@ function createJudgeMethod(rec, judgeConfig, judgeScope) {
|
|
|
1050
1372
|
evaluator_type: "llm",
|
|
1051
1373
|
expected: instruction,
|
|
1052
1374
|
received: valueArray.length === 1 ? valueArray[0] : valueArray,
|
|
1053
|
-
location
|
|
1375
|
+
location,
|
|
1376
|
+
outcome: "error"
|
|
1054
1377
|
}
|
|
1055
1378
|
);
|
|
1056
1379
|
}
|
|
@@ -1074,7 +1397,7 @@ var CAPABILITY_LABELS = {
|
|
|
1074
1397
|
skills: "skill",
|
|
1075
1398
|
subagents: "subagent"
|
|
1076
1399
|
};
|
|
1077
|
-
function createTraceTestContext(view, rec, judgeConfig, judgeScope, agentEvidence) {
|
|
1400
|
+
function createTraceTestContext(view, rec, judgeConfig, judgeScope, agentEvidence, judgeTracer) {
|
|
1078
1401
|
const unsupported = (id, capability) => {
|
|
1079
1402
|
const label = CAPABILITY_LABELS[capability];
|
|
1080
1403
|
rec.record(id, false, `${label} evidence is unavailable in this trace projection`, {
|
|
@@ -1260,8 +1583,8 @@ function createTraceTestContext(view, rec, judgeConfig, judgeScope, agentEvidenc
|
|
|
1260
1583
|
},
|
|
1261
1584
|
// judge does not consult trace capabilities; the scope carries the
|
|
1262
1585
|
// task/check frame for prompt builders (#161).
|
|
1263
|
-
judge: createJudgeMethod(rec, judgeConfig, judgeScope),
|
|
1264
|
-
agent: createAgentMethod(rec, judgeConfig, judgeScope, agentEvidence ?? { deliverables: {}, view })
|
|
1586
|
+
judge: trackPending(rec, createJudgeMethod(rec, judgeConfig, judgeScope, judgeTracer)),
|
|
1587
|
+
agent: trackPending(rec, createAgentMethod(rec, judgeConfig, judgeScope, agentEvidence ?? { deliverables: {}, view }, judgeTracer))
|
|
1265
1588
|
};
|
|
1266
1589
|
}
|
|
1267
1590
|
function matchesTraceToolCall(call, name, opts) {
|
|
@@ -1282,16 +1605,9 @@ function matchesTraceToolCall(call, name, opts) {
|
|
|
1282
1605
|
var Recorder = class {
|
|
1283
1606
|
constructor(locate) {
|
|
1284
1607
|
this.records = [];
|
|
1608
|
+
this.pending = [];
|
|
1285
1609
|
this.locate = locate;
|
|
1286
1610
|
}
|
|
1287
|
-
/**
|
|
1288
|
-
* Record an assertion.
|
|
1289
|
-
*
|
|
1290
|
-
* - ``extra.location`` overrides auto-capture (used by the runner for thrown
|
|
1291
|
-
* errors, whose relevant stack is the error's own, not the call site).
|
|
1292
|
-
* - ``extra.expected`` / ``extra.received`` carry the structured values for
|
|
1293
|
-
* testing-framework-style display.
|
|
1294
|
-
*/
|
|
1295
1611
|
/**
|
|
1296
1612
|
* Capture the call site location synchronously. Use this when a record will
|
|
1297
1613
|
* happen AFTER an `await` (e.g. `t.judge`): once an async function resumes,
|
|
@@ -1303,6 +1619,28 @@ var Recorder = class {
|
|
|
1303
1619
|
captureLocation() {
|
|
1304
1620
|
return this.locate ? this.locate(new Error().stack ?? "") : void 0;
|
|
1305
1621
|
}
|
|
1622
|
+
/**
|
|
1623
|
+
* Register an async evaluation (``t.judge`` / ``t.agent``) started by this
|
|
1624
|
+
* check. If the check returns without awaiting it, the runner still waits
|
|
1625
|
+
* for its record via ``settlePending`` — a dropped await must not let the
|
|
1626
|
+
* check vacuously pass ("no assertions recorded") while the verdict is
|
|
1627
|
+
* still in flight.
|
|
1628
|
+
*/
|
|
1629
|
+
track(promise) {
|
|
1630
|
+
this.pending.push(promise);
|
|
1631
|
+
}
|
|
1632
|
+
/** Wait until every tracked evaluation has settled (and recorded). */
|
|
1633
|
+
async settlePending() {
|
|
1634
|
+
await Promise.allSettled(this.pending);
|
|
1635
|
+
}
|
|
1636
|
+
/**
|
|
1637
|
+
* Record an assertion.
|
|
1638
|
+
*
|
|
1639
|
+
* - ``extra.location`` overrides auto-capture (used by the runner for thrown
|
|
1640
|
+
* errors, whose relevant stack is the error's own, not the call site).
|
|
1641
|
+
* - ``extra.expected`` / ``extra.received`` carry the structured values for
|
|
1642
|
+
* testing-framework-style display.
|
|
1643
|
+
*/
|
|
1306
1644
|
record(id, pass, reasoning, extra) {
|
|
1307
1645
|
const location = extra?.location ?? (this.locate ? this.locate(new Error().stack ?? "") : void 0);
|
|
1308
1646
|
this.records.push({
|
|
@@ -1414,8 +1752,10 @@ async function runTraceChecks(args) {
|
|
|
1414
1752
|
{
|
|
1415
1753
|
deliverables: args.deliverables,
|
|
1416
1754
|
view,
|
|
1417
|
-
...args.historyPlane ? { history: args.historyPlane } : {}
|
|
1418
|
-
|
|
1755
|
+
...args.historyPlane ? { history: args.historyPlane } : {},
|
|
1756
|
+
...args.historyUnavailableReason ? { historyUnavailable: args.historyUnavailableReason } : {}
|
|
1757
|
+
},
|
|
1758
|
+
args.judgeTracer
|
|
1419
1759
|
);
|
|
1420
1760
|
let thrownLocation;
|
|
1421
1761
|
try {
|
|
@@ -1430,8 +1770,10 @@ async function runTraceChecks(args) {
|
|
|
1430
1770
|
location: thrownLocation
|
|
1431
1771
|
});
|
|
1432
1772
|
}
|
|
1773
|
+
await rec.settlePending();
|
|
1433
1774
|
const failed = rec.all.filter((r) => !r.pass);
|
|
1434
1775
|
const pass = failed.length === 0;
|
|
1776
|
+
const outcome = rollUpCheckOutcome(failed);
|
|
1435
1777
|
const reasoning = failed.length > 0 ? failed.map((r) => r.reasoning || r.id).join("; ") : rec.all.length > 0 ? "passed" : "no assertions recorded";
|
|
1436
1778
|
const location = failed.find((r) => r.location)?.location;
|
|
1437
1779
|
const judge = rec.all.find((r) => r.judge)?.judge;
|
|
@@ -1440,6 +1782,7 @@ async function runTraceChecks(args) {
|
|
|
1440
1782
|
pass,
|
|
1441
1783
|
reasoning,
|
|
1442
1784
|
evaluator_type: "code",
|
|
1785
|
+
...outcome ? { outcome } : {},
|
|
1443
1786
|
...judge ? { judge } : {},
|
|
1444
1787
|
...location ? { location } : {},
|
|
1445
1788
|
...args.displayFile ? { source_file: args.displayFile } : {},
|
|
@@ -1454,6 +1797,14 @@ async function runTraceChecks(args) {
|
|
|
1454
1797
|
);
|
|
1455
1798
|
return results;
|
|
1456
1799
|
}
|
|
1800
|
+
function rollUpCheckOutcome(failed) {
|
|
1801
|
+
if (failed.length === 0) return void 0;
|
|
1802
|
+
const verdictless = failed.filter(
|
|
1803
|
+
(r) => r.outcome === "error" || r.outcome === "unsupported"
|
|
1804
|
+
);
|
|
1805
|
+
if (verdictless.length < failed.length) return void 0;
|
|
1806
|
+
return verdictless.some((r) => r.outcome === "error") ? "error" : "unsupported";
|
|
1807
|
+
}
|
|
1457
1808
|
async function loadAndRunFlowChecks(checksPath, args, brokenDeliverables = {}) {
|
|
1458
1809
|
if (!checksPath) return [];
|
|
1459
1810
|
resetFlowChecks();
|
|
@@ -1466,6 +1817,7 @@ async function loadAndRunFlowChecks(checksPath, args, brokenDeliverables = {}) {
|
|
|
1466
1817
|
task: args.task,
|
|
1467
1818
|
judgeConfig: args.judgeConfig,
|
|
1468
1819
|
...args.historyPlane ? { historyPlane: args.historyPlane } : {},
|
|
1820
|
+
...args.judgeTracer ? { judgeTracer: args.judgeTracer } : {},
|
|
1469
1821
|
moduleUrl,
|
|
1470
1822
|
displayFile: basename(checksPath)
|
|
1471
1823
|
});
|
|
@@ -2517,7 +2869,29 @@ function createProjectionTee(real) {
|
|
|
2517
2869
|
}
|
|
2518
2870
|
},
|
|
2519
2871
|
async traceTool(name, params, fn) {
|
|
2520
|
-
|
|
2872
|
+
const obsSpanId = recordStart({
|
|
2873
|
+
step_name: `tool ${name}`,
|
|
2874
|
+
observation_type: "TOOL",
|
|
2875
|
+
tool_name: name,
|
|
2876
|
+
input: params
|
|
2877
|
+
});
|
|
2878
|
+
const start = monotonicNowMs();
|
|
2879
|
+
try {
|
|
2880
|
+
const result = await real.traceTool(name, params, fn);
|
|
2881
|
+
complete(obsSpanId, {
|
|
2882
|
+
latency_ms: round3(monotonicNowMs() - start),
|
|
2883
|
+
output: result
|
|
2884
|
+
});
|
|
2885
|
+
return result;
|
|
2886
|
+
} catch (error) {
|
|
2887
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2888
|
+
complete(obsSpanId, {
|
|
2889
|
+
latency_ms: round3(monotonicNowMs() - start),
|
|
2890
|
+
status_message: message,
|
|
2891
|
+
level: "ERROR"
|
|
2892
|
+
});
|
|
2893
|
+
throw error;
|
|
2894
|
+
}
|
|
2521
2895
|
},
|
|
2522
2896
|
async traceRetriever(query, fn) {
|
|
2523
2897
|
return real.traceRetriever(query, fn);
|
|
@@ -2708,48 +3082,6 @@ function aggregateResult(checksResults) {
|
|
|
2708
3082
|
return { checks: checksResults, pass };
|
|
2709
3083
|
}
|
|
2710
3084
|
|
|
2711
|
-
// src/agent-task/tracing.ts
|
|
2712
|
-
function createNoopAgentTaskTraceContext() {
|
|
2713
|
-
const noopFn = async (fn) => fn();
|
|
2714
|
-
return {
|
|
2715
|
-
runId: "agent-task-untraced",
|
|
2716
|
-
rootSpanId: "agent-task-untraced-root",
|
|
2717
|
-
async step(options, fn) {
|
|
2718
|
-
return fn(options.id ?? "agent-task-untraced-step");
|
|
2719
|
-
},
|
|
2720
|
-
recordEvent(options) {
|
|
2721
|
-
return options.id ?? "agent-task-untraced-event";
|
|
2722
|
-
},
|
|
2723
|
-
endRoot() {
|
|
2724
|
-
},
|
|
2725
|
-
traceTool(_name, _params, fn) {
|
|
2726
|
-
return noopFn(fn);
|
|
2727
|
-
},
|
|
2728
|
-
traceRetriever(_query, fn) {
|
|
2729
|
-
return noopFn(fn);
|
|
2730
|
-
},
|
|
2731
|
-
traceChain(_name, fn) {
|
|
2732
|
-
return noopFn(fn);
|
|
2733
|
-
},
|
|
2734
|
-
traceAgent(_name, fn) {
|
|
2735
|
-
return noopFn(fn);
|
|
2736
|
-
},
|
|
2737
|
-
traceGuardrail(_name, fn) {
|
|
2738
|
-
return noopFn(fn);
|
|
2739
|
-
},
|
|
2740
|
-
traceEmbedding(_model, _input, fn) {
|
|
2741
|
-
return noopFn(fn);
|
|
2742
|
-
},
|
|
2743
|
-
async score(_params) {
|
|
2744
|
-
},
|
|
2745
|
-
createSpan(_options) {
|
|
2746
|
-
return "agent-task-untraced-span";
|
|
2747
|
-
},
|
|
2748
|
-
endSpan(_spanId, _params) {
|
|
2749
|
-
}
|
|
2750
|
-
};
|
|
2751
|
-
}
|
|
2752
|
-
|
|
2753
3085
|
// src/agent-task/run/runTask.ts
|
|
2754
3086
|
var AgentTaskRunError = class extends Error {
|
|
2755
3087
|
constructor(message, options) {
|
|
@@ -2776,13 +3108,15 @@ async function runTask(taskDir, options) {
|
|
|
2776
3108
|
void 0
|
|
2777
3109
|
);
|
|
2778
3110
|
}
|
|
2779
|
-
|
|
3111
|
+
return trace2.client.traceRun(
|
|
2780
3112
|
buildTraceRunOptions(loaded, trace2),
|
|
2781
|
-
async (traceContext) =>
|
|
3113
|
+
async (traceContext) => {
|
|
3114
|
+
const phase1 = await captureExecution(loaded, options, traceContext);
|
|
3115
|
+
const canonical = await readCanonicalSnapshot(trace2);
|
|
3116
|
+
if (canonical) phase1.snapshot = canonical;
|
|
3117
|
+
return evaluate(loaded, options, phase1, traceContext);
|
|
3118
|
+
}
|
|
2782
3119
|
);
|
|
2783
|
-
const canonical = await readCanonicalSnapshot(trace2);
|
|
2784
|
-
if (canonical) phase1.snapshot = canonical;
|
|
2785
|
-
return evaluate(loaded, options, phase1);
|
|
2786
3120
|
}
|
|
2787
3121
|
async function readCanonicalSnapshot(trace2) {
|
|
2788
3122
|
if (!trace2.taskRunId) return null;
|
|
@@ -2980,6 +3314,7 @@ async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
|
|
|
2980
3314
|
files,
|
|
2981
3315
|
task: task2,
|
|
2982
3316
|
...judgeConfig ? { judgeConfig } : {},
|
|
3317
|
+
judgeTracer: trace2,
|
|
2983
3318
|
...historyPlane ? { historyPlane } : {}
|
|
2984
3319
|
},
|
|
2985
3320
|
validationResults.brokenDeliverables
|
|
@@ -2994,6 +3329,7 @@ async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
|
|
|
2994
3329
|
files,
|
|
2995
3330
|
task: task2,
|
|
2996
3331
|
...judgeConfig ? { judgeConfig } : {},
|
|
3332
|
+
judgeTracer: trace2,
|
|
2997
3333
|
...historyPlane ? { historyPlane } : {},
|
|
2998
3334
|
moduleUrl,
|
|
2999
3335
|
displayFile: evalFileName
|
|
@@ -3022,7 +3358,7 @@ async function captureExecution(loaded, options, rawTrace) {
|
|
|
3022
3358
|
snapshot: outcome.getSnapshot()
|
|
3023
3359
|
}));
|
|
3024
3360
|
}
|
|
3025
|
-
async function evaluate(loaded, options, phase1) {
|
|
3361
|
+
async function evaluate(loaded, options, phase1, judgeTracer) {
|
|
3026
3362
|
const {
|
|
3027
3363
|
task: task2,
|
|
3028
3364
|
adapter,
|
|
@@ -3040,12 +3376,13 @@ async function evaluate(loaded, options, phase1) {
|
|
|
3040
3376
|
);
|
|
3041
3377
|
const judgeConfig = resolveJudgeConfig(options?.judge, task2.judge);
|
|
3042
3378
|
const historyPlane = await freezeHistoryPlaneFromEnv(task2.id);
|
|
3043
|
-
const
|
|
3379
|
+
const runChecks = async () => inlineChecks ? runTraceChecks({
|
|
3044
3380
|
snapshot: phase1.snapshot,
|
|
3045
3381
|
deliverables,
|
|
3046
3382
|
files,
|
|
3047
3383
|
task: task2,
|
|
3048
3384
|
...judgeConfig ? { judgeConfig } : {},
|
|
3385
|
+
...judgeTracer ? { judgeTracer } : {},
|
|
3049
3386
|
...historyPlane ? { historyPlane } : {},
|
|
3050
3387
|
moduleUrl,
|
|
3051
3388
|
displayFile: evalFileName
|
|
@@ -3057,10 +3394,22 @@ async function evaluate(loaded, options, phase1) {
|
|
|
3057
3394
|
files,
|
|
3058
3395
|
task: task2,
|
|
3059
3396
|
...judgeConfig ? { judgeConfig } : {},
|
|
3397
|
+
...judgeTracer ? { judgeTracer } : {},
|
|
3060
3398
|
...historyPlane ? { historyPlane } : {}
|
|
3061
3399
|
},
|
|
3062
3400
|
validationResults.brokenDeliverables
|
|
3063
|
-
)
|
|
3401
|
+
);
|
|
3402
|
+
const hasChecksModule = inlineChecks || checksPath !== null;
|
|
3403
|
+
const checksResults = await (judgeTracer && hasChecksModule ? judgeTracer.step(
|
|
3404
|
+
{
|
|
3405
|
+
step_name: "checks.run",
|
|
3406
|
+
observation_type: "CHAIN",
|
|
3407
|
+
input: { sourceFile: inlineChecks ? evalFileName : checksPath },
|
|
3408
|
+
metadata: { sourceFile: inlineChecks ? evalFileName : checksPath },
|
|
3409
|
+
summarize: (result2) => summarizeEvaluationResults(result2)
|
|
3410
|
+
},
|
|
3411
|
+
runChecks
|
|
3412
|
+
) : runChecks());
|
|
3064
3413
|
const result = aggregateResult(checksResults);
|
|
3065
3414
|
return {
|
|
3066
3415
|
task: task2,
|
|
@@ -3158,17 +3507,29 @@ async function cleanupAdapter(loaded, trace2, state, session) {
|
|
|
3158
3507
|
}
|
|
3159
3508
|
function summarizeEvaluationResults(results) {
|
|
3160
3509
|
const passCount = results.filter((r) => r.pass).length;
|
|
3510
|
+
const noVerdictCount = results.filter(
|
|
3511
|
+
(r) => !r.pass && (r.outcome === "error" || r.outcome === "unsupported")
|
|
3512
|
+
).length;
|
|
3513
|
+
const failCount = results.length - passCount - noVerdictCount;
|
|
3514
|
+
const suffix = noVerdictCount > 0 ? ` (${noVerdictCount} no-verdict)` : "";
|
|
3161
3515
|
return {
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
|
|
3167
|
-
|
|
3168
|
-
|
|
3169
|
-
|
|
3170
|
-
|
|
3171
|
-
|
|
3516
|
+
text: `${passCount}/${results.length} checks passed${suffix}`,
|
|
3517
|
+
verdict: {
|
|
3518
|
+
total: results.length,
|
|
3519
|
+
passCount,
|
|
3520
|
+
failCount,
|
|
3521
|
+
noVerdictCount,
|
|
3522
|
+
results: results.map((r) => ({
|
|
3523
|
+
id: r.id,
|
|
3524
|
+
pass: r.pass,
|
|
3525
|
+
reasoning: r.reasoning?.slice(0, 2e3),
|
|
3526
|
+
evaluator_type: r.evaluator_type,
|
|
3527
|
+
judge_model: r.judge?.model,
|
|
3528
|
+
...r.outcome ? { outcome: r.outcome } : {},
|
|
3529
|
+
...r.group_id ? { group_id: r.group_id } : {},
|
|
3530
|
+
...r.group_name ? { group_name: r.group_name } : {}
|
|
3531
|
+
}))
|
|
3532
|
+
}
|
|
3172
3533
|
};
|
|
3173
3534
|
}
|
|
3174
3535
|
function buildTraceRunOptions(loaded, tracing) {
|
|
@@ -3296,6 +3657,7 @@ async function rejudgeTaskRun(runId, endpoints, options = {}) {
|
|
|
3296
3657
|
);
|
|
3297
3658
|
const proxied = proxyBrokenDeliverables(deliverables, validation.brokenDeliverables);
|
|
3298
3659
|
const passCounts = /* @__PURE__ */ new Map();
|
|
3660
|
+
const errorCounts = /* @__PURE__ */ new Map();
|
|
3299
3661
|
let primary = [];
|
|
3300
3662
|
const judgeConfig = resolveJudgeConfig(options.judge, loaded.task.judge);
|
|
3301
3663
|
for (let sample = 0; sample < samples; sample++) {
|
|
@@ -3304,12 +3666,16 @@ async function rejudgeTaskRun(runId, endpoints, options = {}) {
|
|
|
3304
3666
|
if (sample === 0) primary = results;
|
|
3305
3667
|
for (const result of results) {
|
|
3306
3668
|
passCounts.set(result.id, (passCounts.get(result.id) ?? 0) + (result.pass ? 1 : 0));
|
|
3669
|
+
if (!result.pass && result.outcome === "error") {
|
|
3670
|
+
errorCounts.set(result.id, (errorCounts.get(result.id) ?? 0) + 1);
|
|
3671
|
+
}
|
|
3307
3672
|
}
|
|
3308
3673
|
}
|
|
3309
3674
|
const stability = primary.map((result) => ({
|
|
3310
3675
|
check_id: result.id,
|
|
3311
3676
|
passes: passCounts.get(result.id) ?? 0,
|
|
3312
|
-
samples
|
|
3677
|
+
samples,
|
|
3678
|
+
...errorCounts.get(result.id) ? { errored: errorCounts.get(result.id) } : {}
|
|
3313
3679
|
}));
|
|
3314
3680
|
const aggregate = aggregateResult(primary);
|
|
3315
3681
|
return {
|
|
@@ -3533,6 +3899,7 @@ function createOtelAgentTaskTraceClient(config) {
|
|
|
3533
3899
|
activeSpans.set(spanId, {
|
|
3534
3900
|
span,
|
|
3535
3901
|
startedAt: monotonicNowMs2(),
|
|
3902
|
+
startWallMs: Date.now(),
|
|
3536
3903
|
model: params.model || "unknown",
|
|
3537
3904
|
observationType: params.observation_type
|
|
3538
3905
|
});
|
|
@@ -3550,6 +3917,13 @@ function createOtelAgentTaskTraceClient(config) {
|
|
|
3550
3917
|
const payload = params.output && typeof params.output === "object" ? params.output : { value: params.output };
|
|
3551
3918
|
active.span.setAttribute("gen_ai.tool.call.result", JSON.stringify(payload));
|
|
3552
3919
|
} else if (params.output.text) {
|
|
3920
|
+
active.span.setAttribute("gen_ai.response.text", String(params.output.text));
|
|
3921
|
+
if (params.output.verdict !== void 0) {
|
|
3922
|
+
active.span.setAttribute(
|
|
3923
|
+
"gen_ai.tool.call.result",
|
|
3924
|
+
JSON.stringify(params.output.verdict)
|
|
3925
|
+
);
|
|
3926
|
+
}
|
|
3553
3927
|
active.span.setAttribute("gen_ai.output.messages", JSON.stringify([{
|
|
3554
3928
|
role: "assistant",
|
|
3555
3929
|
parts: [{ type: "text", content: String(params.output.text) }]
|
|
@@ -3574,6 +3948,12 @@ function createOtelAgentTaskTraceClient(config) {
|
|
|
3574
3948
|
if (params.completion_tokens !== void 0) {
|
|
3575
3949
|
active.span.setAttribute("gen_ai.usage.output_tokens", params.completion_tokens);
|
|
3576
3950
|
}
|
|
3951
|
+
if (params.reasoning_tokens !== void 0) {
|
|
3952
|
+
active.span.setAttribute(
|
|
3953
|
+
"gen_ai.usage.reasoning.output_tokens",
|
|
3954
|
+
params.reasoning_tokens
|
|
3955
|
+
);
|
|
3956
|
+
}
|
|
3577
3957
|
if (params.status_message) {
|
|
3578
3958
|
active.span.setAttribute("apo.status_message", params.status_message);
|
|
3579
3959
|
}
|
|
@@ -3583,7 +3963,7 @@ function createOtelAgentTaskTraceClient(config) {
|
|
|
3583
3963
|
if (params.metadata) {
|
|
3584
3964
|
active.span.setAttribute("apo.metadata", JSON.stringify(params.metadata));
|
|
3585
3965
|
}
|
|
3586
|
-
active.span.end();
|
|
3966
|
+
active.span.end(new Date(active.startWallMs + latency));
|
|
3587
3967
|
activeSpans.delete(params.id);
|
|
3588
3968
|
}
|
|
3589
3969
|
return {
|
|
@@ -3617,6 +3997,7 @@ function createOtelAgentTaskTraceClient(config) {
|
|
|
3617
3997
|
activeSpans.set(rootSpanId, {
|
|
3618
3998
|
span: rootSpan,
|
|
3619
3999
|
startedAt: monotonicNowMs2(),
|
|
4000
|
+
startWallMs: Date.now(),
|
|
3620
4001
|
model: "trace"
|
|
3621
4002
|
});
|
|
3622
4003
|
let callCount = 1;
|
|
@@ -3642,7 +4023,8 @@ function createOtelAgentTaskTraceClient(config) {
|
|
|
3642
4023
|
id: spanId,
|
|
3643
4024
|
latency_ms: round32(monotonicNowMs2() - startedAt),
|
|
3644
4025
|
output: options.summarize?.(result),
|
|
3645
|
-
metadata: options.metadata
|
|
4026
|
+
metadata: options.metadata,
|
|
4027
|
+
...options.usage?.(result)
|
|
3646
4028
|
});
|
|
3647
4029
|
callCount += 1;
|
|
3648
4030
|
return result;
|