@apo-ai/sdk 0.2.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/agent-task/public.d.ts +336 -195
- package/dist/agent-task/public.js +418 -60
- package/dist/agent-task/public.js.map +1 -1
- package/package.json +1 -1
|
@@ -235,7 +235,8 @@ function levenshtein(a, b) {
|
|
|
235
235
|
}
|
|
236
236
|
|
|
237
237
|
// src/agent-task/checks/judge.ts
|
|
238
|
-
var
|
|
238
|
+
var JUDGE_RESPONSE_CONTRACT = 'Respond with ONLY a JSON object: {"pass": true/false, "reasoning": "your reasoning"}';
|
|
239
|
+
var JUDGE_SYSTEM_PROMPT = `You are an evaluation judge. Evaluate the given value(s) against the instruction. ${JUDGE_RESPONSE_CONTRACT}`;
|
|
239
240
|
function formatValue(value, depth = 0) {
|
|
240
241
|
const indent = " ".repeat(depth);
|
|
241
242
|
if (value === null || value === void 0) return String(value);
|
|
@@ -314,12 +315,11 @@ async function callJudge(args) {
|
|
|
314
315
|
const apiKey = args.apiKey ?? process.env.OPENROUTER_API_KEY ?? process.env.OPENAI_API_KEY;
|
|
315
316
|
const deliverableText = `Values to evaluate:
|
|
316
317
|
${formatJudgeValues(args.values)}`;
|
|
317
|
-
const instructionText =
|
|
318
|
-
|
|
319
|
-
const systemPromptText = `${JUDGE_SYSTEM_PROMPT}
|
|
318
|
+
const { briefingText, instructionText } = assembleBriefing(args);
|
|
319
|
+
const systemPromptText = `${briefingText}
|
|
320
320
|
|
|
321
321
|
${deliverableText}`;
|
|
322
|
-
const cacheKey = `${args.model}\0${deliverableText}`;
|
|
322
|
+
const cacheKey = `${args.model}\0${briefingText}\0${deliverableText}`;
|
|
323
323
|
return runWithSharedPrefix(cacheKey, async () => {
|
|
324
324
|
const startedAt = Date.now();
|
|
325
325
|
const response = await fetch(`${baseURL}/chat/completions`, {
|
|
@@ -334,7 +334,7 @@ ${deliverableText}`;
|
|
|
334
334
|
{
|
|
335
335
|
role: "system",
|
|
336
336
|
content: [
|
|
337
|
-
{ type: "text", text:
|
|
337
|
+
{ type: "text", text: briefingText },
|
|
338
338
|
{
|
|
339
339
|
type: "text",
|
|
340
340
|
text: deliverableText,
|
|
@@ -384,6 +384,33 @@ ${deliverableText}`;
|
|
|
384
384
|
};
|
|
385
385
|
});
|
|
386
386
|
}
|
|
387
|
+
function assembleBriefing(args) {
|
|
388
|
+
if (!args.prompt) {
|
|
389
|
+
return {
|
|
390
|
+
briefingText: JUDGE_SYSTEM_PROMPT,
|
|
391
|
+
instructionText: `Instruction:
|
|
392
|
+
${args.instruction}`
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
const ctx = {
|
|
396
|
+
taskId: args.context?.taskId ?? "",
|
|
397
|
+
checkName: args.context?.checkName ?? "",
|
|
398
|
+
...args.context?.taskDescription !== void 0 ? { taskDescription: args.context.taskDescription } : {},
|
|
399
|
+
...args.context?.deliverableNames !== void 0 && args.context.deliverableNames.length > 0 ? { deliverableNames: args.context.deliverableNames } : {},
|
|
400
|
+
instruction: args.instruction
|
|
401
|
+
};
|
|
402
|
+
const built = args.prompt(ctx);
|
|
403
|
+
const system = built.system?.trim();
|
|
404
|
+
const user = built.user?.trim();
|
|
405
|
+
return {
|
|
406
|
+
// The SDK appends its own response contract to any custom briefing.
|
|
407
|
+
briefingText: system ? `${system}
|
|
408
|
+
|
|
409
|
+
${JUDGE_RESPONSE_CONTRACT}` : JUDGE_SYSTEM_PROMPT,
|
|
410
|
+
instructionText: user ?? `Instruction:
|
|
411
|
+
${args.instruction}`
|
|
412
|
+
};
|
|
413
|
+
}
|
|
387
414
|
|
|
388
415
|
// src/agent-task/checks/t.ts
|
|
389
416
|
function resolveJudgeConfig(judgeConfig, override) {
|
|
@@ -392,7 +419,8 @@ function resolveJudgeConfig(judgeConfig, override) {
|
|
|
392
419
|
return {
|
|
393
420
|
model,
|
|
394
421
|
baseURL: override?.baseURL ?? judgeConfig?.baseURL,
|
|
395
|
-
apiKey: override?.apiKey ?? judgeConfig?.apiKey
|
|
422
|
+
apiKey: override?.apiKey ?? judgeConfig?.apiKey,
|
|
423
|
+
prompt: override?.prompt ?? judgeConfig?.prompt
|
|
396
424
|
};
|
|
397
425
|
}
|
|
398
426
|
var TEST_METHOD_NAMES = [
|
|
@@ -421,6 +449,67 @@ function matchName(name, m) {
|
|
|
421
449
|
if (m instanceof RegExp) return m.test(name);
|
|
422
450
|
return m(name);
|
|
423
451
|
}
|
|
452
|
+
function createJudgeMethod(rec, judgeConfig, judgeScope) {
|
|
453
|
+
return async (values, instruction, opts) => {
|
|
454
|
+
const label = opts?.label ?? "judge";
|
|
455
|
+
const location = rec.captureLocation();
|
|
456
|
+
const valueArray = Array.isArray(values) ? values : [values];
|
|
457
|
+
const effective = resolveJudgeConfig(judgeConfig, opts?.judge);
|
|
458
|
+
if (!effective) {
|
|
459
|
+
rec.record(
|
|
460
|
+
label,
|
|
461
|
+
false,
|
|
462
|
+
"No judge model configured. Set one of:\n\u2022 OPENROUTER_MODEL + OPENROUTER_API_KEY (OpenRouter \u2014 works with 200+ models, one account)\n\u2022 OPENAI_MODEL + OPENAI_API_KEY (OpenAI direct)\nOr pass { judge } to runTask() programmatically.",
|
|
463
|
+
{ evaluator_type: "llm", location }
|
|
464
|
+
);
|
|
465
|
+
return;
|
|
466
|
+
}
|
|
467
|
+
const context2 = judgeScopeToContext(judgeScope);
|
|
468
|
+
try {
|
|
469
|
+
const { pass, reasoning, judge } = await callJudge({
|
|
470
|
+
values: valueArray,
|
|
471
|
+
instruction,
|
|
472
|
+
model: effective.model,
|
|
473
|
+
baseURL: effective.baseURL,
|
|
474
|
+
apiKey: effective.apiKey,
|
|
475
|
+
prompt: effective.prompt,
|
|
476
|
+
...context2 ? { context: context2 } : {}
|
|
477
|
+
});
|
|
478
|
+
rec.record(label, pass, reasoning, {
|
|
479
|
+
evaluator_type: "llm",
|
|
480
|
+
judge,
|
|
481
|
+
expected: instruction,
|
|
482
|
+
// Store the raw evaluated value (not a truncated string) so the
|
|
483
|
+
// dashboard can render it with the right viewer — Markdown for
|
|
484
|
+
// prose, a JSON tree for structured objects.
|
|
485
|
+
received: valueArray.length === 1 ? valueArray[0] : valueArray,
|
|
486
|
+
location
|
|
487
|
+
});
|
|
488
|
+
} catch (error) {
|
|
489
|
+
rec.record(
|
|
490
|
+
label,
|
|
491
|
+
false,
|
|
492
|
+
`judge failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
493
|
+
{
|
|
494
|
+
evaluator_type: "llm",
|
|
495
|
+
expected: instruction,
|
|
496
|
+
received: valueArray.length === 1 ? valueArray[0] : valueArray,
|
|
497
|
+
location
|
|
498
|
+
}
|
|
499
|
+
);
|
|
500
|
+
}
|
|
501
|
+
};
|
|
502
|
+
}
|
|
503
|
+
function judgeScopeToContext(scope) {
|
|
504
|
+
if (!scope) return void 0;
|
|
505
|
+
const deliverableNames = scope.readDeliverableNames?.();
|
|
506
|
+
return {
|
|
507
|
+
taskId: scope.taskId,
|
|
508
|
+
checkName: scope.checkName,
|
|
509
|
+
...scope.taskDescription !== void 0 ? { taskDescription: scope.taskDescription } : {},
|
|
510
|
+
...deliverableNames && deliverableNames.length > 0 ? { deliverableNames } : {}
|
|
511
|
+
};
|
|
512
|
+
}
|
|
424
513
|
var CAPABILITY_LABELS = {
|
|
425
514
|
messages: "message",
|
|
426
515
|
tools: "tool",
|
|
@@ -429,7 +518,7 @@ var CAPABILITY_LABELS = {
|
|
|
429
518
|
skills: "skill",
|
|
430
519
|
subagents: "subagent"
|
|
431
520
|
};
|
|
432
|
-
function createTraceTestContext(view, rec, judgeConfig) {
|
|
521
|
+
function createTraceTestContext(view, rec, judgeConfig, judgeScope) {
|
|
433
522
|
const unsupported = (id, capability) => {
|
|
434
523
|
const label = CAPABILITY_LABELS[capability];
|
|
435
524
|
rec.record(id, false, `${label} evidence is unavailable in this trace projection`, {
|
|
@@ -613,49 +702,9 @@ function createTraceTestContext(view, rec, judgeConfig) {
|
|
|
613
702
|
{ expected: matcher.label, received: describeValue(value) }
|
|
614
703
|
);
|
|
615
704
|
},
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
const valueArray = Array.isArray(values) ? values : [values];
|
|
620
|
-
const effective = resolveJudgeConfig(judgeConfig, opts?.judge);
|
|
621
|
-
if (!effective) {
|
|
622
|
-
rec.record(
|
|
623
|
-
label,
|
|
624
|
-
false,
|
|
625
|
-
"No judge model configured. Set one of:\n\u2022 OPENROUTER_MODEL + OPENROUTER_API_KEY (OpenRouter \u2014 works with 200+ models, one account)\n\u2022 OPENAI_MODEL + OPENAI_API_KEY (OpenAI direct)\nOr pass { judge } to runTask() programmatically.",
|
|
626
|
-
{ evaluator_type: "llm", location }
|
|
627
|
-
);
|
|
628
|
-
return;
|
|
629
|
-
}
|
|
630
|
-
try {
|
|
631
|
-
const { pass, reasoning, judge } = await callJudge({
|
|
632
|
-
values: valueArray,
|
|
633
|
-
instruction,
|
|
634
|
-
model: effective.model,
|
|
635
|
-
baseURL: effective.baseURL,
|
|
636
|
-
apiKey: effective.apiKey
|
|
637
|
-
});
|
|
638
|
-
rec.record(label, pass, reasoning, {
|
|
639
|
-
evaluator_type: "llm",
|
|
640
|
-
judge,
|
|
641
|
-
expected: instruction,
|
|
642
|
-
received: valueArray.length === 1 ? valueArray[0] : valueArray,
|
|
643
|
-
location
|
|
644
|
-
});
|
|
645
|
-
} catch (error) {
|
|
646
|
-
rec.record(
|
|
647
|
-
label,
|
|
648
|
-
false,
|
|
649
|
-
`judge failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
650
|
-
{
|
|
651
|
-
evaluator_type: "llm",
|
|
652
|
-
expected: instruction,
|
|
653
|
-
received: valueArray.length === 1 ? valueArray[0] : valueArray,
|
|
654
|
-
location
|
|
655
|
-
}
|
|
656
|
-
);
|
|
657
|
-
}
|
|
658
|
-
}
|
|
705
|
+
// judge does not consult trace capabilities; the scope carries the
|
|
706
|
+
// task/check frame for prompt builders (#161).
|
|
707
|
+
judge: createJudgeMethod(rec, judgeConfig, judgeScope)
|
|
659
708
|
};
|
|
660
709
|
}
|
|
661
710
|
function matchesTraceToolCall(call, name, opts) {
|
|
@@ -787,14 +836,21 @@ function describe(id, nameOrFn, fn) {
|
|
|
787
836
|
async function runTraceChecks(args) {
|
|
788
837
|
const view = new TraceView(args.snapshot);
|
|
789
838
|
const locate = args.moduleUrl && args.displayFile ? (stack) => parseCheckLocation(stack, args.moduleUrl, args.displayFile) : void 0;
|
|
839
|
+
const taskMeta = readTaskMeta(args.task);
|
|
790
840
|
const results = await Promise.all(
|
|
791
841
|
registry.map(async (check) => {
|
|
792
842
|
const rec = new Recorder(locate);
|
|
793
|
-
const
|
|
843
|
+
const reads = trackDeliverableReads(args.deliverables);
|
|
844
|
+
const t = createTraceTestContext(view, rec, args.judgeConfig, {
|
|
845
|
+
taskId: taskMeta.id ?? "",
|
|
846
|
+
...taskMeta.description !== void 0 ? { taskDescription: taskMeta.description } : {},
|
|
847
|
+
checkName: check.id,
|
|
848
|
+
readDeliverableNames: reads.names
|
|
849
|
+
});
|
|
794
850
|
let thrownLocation;
|
|
795
851
|
try {
|
|
796
852
|
await check.fn(t, {
|
|
797
|
-
deliverables:
|
|
853
|
+
deliverables: reads.proxied,
|
|
798
854
|
files: args.files,
|
|
799
855
|
task: args.task
|
|
800
856
|
});
|
|
@@ -866,6 +922,26 @@ function proxyBrokenDeliverables(deliverables, brokenDeliverables) {
|
|
|
866
922
|
}
|
|
867
923
|
});
|
|
868
924
|
}
|
|
925
|
+
function readTaskMeta(task2) {
|
|
926
|
+
if (!task2 || typeof task2 !== "object") return {};
|
|
927
|
+
const record = task2;
|
|
928
|
+
return {
|
|
929
|
+
...typeof record.id === "string" ? { id: record.id } : {},
|
|
930
|
+
...typeof record.description === "string" ? { description: record.description } : {}
|
|
931
|
+
};
|
|
932
|
+
}
|
|
933
|
+
function trackDeliverableReads(deliverables) {
|
|
934
|
+
const read = /* @__PURE__ */ new Set();
|
|
935
|
+
const proxied = new Proxy(deliverables, {
|
|
936
|
+
get(target, property, receiver) {
|
|
937
|
+
if (typeof property === "string" && Object.prototype.hasOwnProperty.call(target, property)) {
|
|
938
|
+
read.add(property);
|
|
939
|
+
}
|
|
940
|
+
return Reflect.get(target, property, receiver);
|
|
941
|
+
}
|
|
942
|
+
});
|
|
943
|
+
return { proxied, names: () => [...read] };
|
|
944
|
+
}
|
|
869
945
|
|
|
870
946
|
// src/agent-task/task/defineTask.ts
|
|
871
947
|
var TASK_ADAPTER_SYMBOL = /* @__PURE__ */ Symbol.for("agent-task.adapter-definition");
|
|
@@ -919,7 +995,7 @@ function attachAdapter(taskDefinition, adapter) {
|
|
|
919
995
|
}
|
|
920
996
|
|
|
921
997
|
// src/agent-task/task/loadTask.ts
|
|
922
|
-
import { copyFileSync as copyFileSync2, readdirSync, statSync, existsSync as existsSync2, unlinkSync as unlinkSync2 } from "fs";
|
|
998
|
+
import { copyFileSync as copyFileSync2, readdirSync, statSync, existsSync as existsSync2, unlinkSync as unlinkSync2, writeFileSync } from "fs";
|
|
923
999
|
import { join, resolve, basename as basename2 } from "path";
|
|
924
1000
|
import { pathToFileURL as pathToFileURL2 } from "url";
|
|
925
1001
|
|
|
@@ -966,6 +1042,48 @@ async function loadTask(taskDir) {
|
|
|
966
1042
|
evalFileName: basename2(taskFilePath)
|
|
967
1043
|
};
|
|
968
1044
|
}
|
|
1045
|
+
async function loadEvalSource(fileName, content, opts = {}) {
|
|
1046
|
+
const dir = opts.siblingDir ?? opts.isolatedDir;
|
|
1047
|
+
if (!dir) {
|
|
1048
|
+
throw new Error("loadEvalSource requires a siblingDir or isolatedDir to materialize next to");
|
|
1049
|
+
}
|
|
1050
|
+
const tempPath = join(
|
|
1051
|
+
dir,
|
|
1052
|
+
`${fileName}.rejudge-${Date.now()}-${Math.random().toString(36).slice(2)}.ts`
|
|
1053
|
+
);
|
|
1054
|
+
writeFileSync(tempPath, content);
|
|
1055
|
+
try {
|
|
1056
|
+
const { task: task2, adapter, inlineChecks, moduleUrl } = await loadTaskDefinition(tempPath);
|
|
1057
|
+
validateTaskDefinition(task2, dir);
|
|
1058
|
+
if (!inlineChecks && !existsSync2(join(dir, "checks.ts"))) {
|
|
1059
|
+
throw new Error(
|
|
1060
|
+
`The pinned eval did not register checks: legacy two-file tasks need their checks module, which the stored definition revision does not include`
|
|
1061
|
+
);
|
|
1062
|
+
}
|
|
1063
|
+
return {
|
|
1064
|
+
task: task2,
|
|
1065
|
+
adapter,
|
|
1066
|
+
taskDir: dir,
|
|
1067
|
+
files: loadFilesDirectory(dir),
|
|
1068
|
+
checksPath: resolveChecksPath(task2, dir),
|
|
1069
|
+
inlineChecks,
|
|
1070
|
+
moduleUrl,
|
|
1071
|
+
evalFileName: fileName
|
|
1072
|
+
};
|
|
1073
|
+
} catch (error) {
|
|
1074
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1075
|
+
if (!opts.siblingDir && /Failed to load task definition/.test(message)) {
|
|
1076
|
+
throw new Error(
|
|
1077
|
+
`Replay needs the task directory: the pinned eval imports sibling files that are not part of the stored definition revision. Point the command at a checkout of the task (--task-dir) and retry. Underlying error: ${message}`
|
|
1078
|
+
);
|
|
1079
|
+
}
|
|
1080
|
+
throw error;
|
|
1081
|
+
} finally {
|
|
1082
|
+
if (existsSync2(tempPath)) {
|
|
1083
|
+
unlinkSync2(tempPath);
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
969
1087
|
async function loadTaskDefinition(taskFilePath) {
|
|
970
1088
|
const tempModulePath = `${taskFilePath}.${Date.now()}-${Math.random().toString(36).slice(2)}.ts`;
|
|
971
1089
|
try {
|
|
@@ -2034,6 +2152,7 @@ async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
|
|
|
2034
2152
|
summarize: (result2) => summarizeEvaluationResults(result2)
|
|
2035
2153
|
},
|
|
2036
2154
|
async () => {
|
|
2155
|
+
const judgeConfig = resolveJudgeConfig(options?.judge, task2.judge);
|
|
2037
2156
|
if (!inlineChecks) {
|
|
2038
2157
|
return loadAndRunFlowChecks(
|
|
2039
2158
|
checksPath,
|
|
@@ -2042,7 +2161,7 @@ async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
|
|
|
2042
2161
|
deliverables: collected,
|
|
2043
2162
|
files,
|
|
2044
2163
|
task: task2,
|
|
2045
|
-
judgeConfig:
|
|
2164
|
+
...judgeConfig ? { judgeConfig } : {}
|
|
2046
2165
|
},
|
|
2047
2166
|
validationResults.brokenDeliverables
|
|
2048
2167
|
);
|
|
@@ -2055,7 +2174,7 @@ async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
|
|
|
2055
2174
|
),
|
|
2056
2175
|
files,
|
|
2057
2176
|
task: task2,
|
|
2058
|
-
judgeConfig:
|
|
2177
|
+
...judgeConfig ? { judgeConfig } : {},
|
|
2059
2178
|
moduleUrl,
|
|
2060
2179
|
displayFile: evalFileName
|
|
2061
2180
|
});
|
|
@@ -2235,12 +2354,13 @@ async function evaluate(loaded, options, phase1) {
|
|
|
2235
2354
|
phase1.collected,
|
|
2236
2355
|
validationResults.brokenDeliverables
|
|
2237
2356
|
);
|
|
2357
|
+
const judgeConfig = resolveJudgeConfig(options?.judge, task2.judge);
|
|
2238
2358
|
const checksResults = await (inlineChecks ? runTraceChecks({
|
|
2239
2359
|
snapshot: phase1.snapshot,
|
|
2240
2360
|
deliverables,
|
|
2241
2361
|
files,
|
|
2242
2362
|
task: task2,
|
|
2243
|
-
judgeConfig:
|
|
2363
|
+
...judgeConfig ? { judgeConfig } : {},
|
|
2244
2364
|
moduleUrl,
|
|
2245
2365
|
displayFile: evalFileName
|
|
2246
2366
|
}) : loadAndRunFlowChecks(
|
|
@@ -2250,7 +2370,7 @@ async function evaluate(loaded, options, phase1) {
|
|
|
2250
2370
|
deliverables,
|
|
2251
2371
|
files,
|
|
2252
2372
|
task: task2,
|
|
2253
|
-
judgeConfig:
|
|
2373
|
+
...judgeConfig ? { judgeConfig } : {}
|
|
2254
2374
|
},
|
|
2255
2375
|
validationResults.brokenDeliverables
|
|
2256
2376
|
));
|
|
@@ -2315,12 +2435,248 @@ function buildTraceRunOptions(loaded, tracing) {
|
|
|
2315
2435
|
};
|
|
2316
2436
|
}
|
|
2317
2437
|
|
|
2438
|
+
// src/agent-task/run/rejudge.ts
|
|
2439
|
+
import { mkdtempSync, rmSync, writeFileSync as writeFileSync2 } from "fs";
|
|
2440
|
+
import { tmpdir } from "os";
|
|
2441
|
+
import { basename as basename4, dirname, join as join2 } from "path";
|
|
2442
|
+
import { fileURLToPath } from "url";
|
|
2443
|
+
var RejudgeError = class extends Error {
|
|
2444
|
+
constructor(message) {
|
|
2445
|
+
super(message);
|
|
2446
|
+
this.name = "RejudgeError";
|
|
2447
|
+
}
|
|
2448
|
+
};
|
|
2449
|
+
var TERMINAL_VERDICT_STATUSES = /* @__PURE__ */ new Set(["passed", "failed"]);
|
|
2450
|
+
var MAX_SAMPLES = 50;
|
|
2451
|
+
async function rejudgeTaskRun(runId, endpoints, options = {}) {
|
|
2452
|
+
const api = createApiClient(endpoints);
|
|
2453
|
+
const progress = options.onProgress ?? (() => {
|
|
2454
|
+
});
|
|
2455
|
+
const detail = await api.get(`/v1/agent-task-runs/${runId}`);
|
|
2456
|
+
if (!TERMINAL_VERDICT_STATUSES.has(detail.status)) {
|
|
2457
|
+
throw new RejudgeError(
|
|
2458
|
+
`Run ${runId} is '${detail.status}', not a completed run with a verdict \u2014 there is nothing to replay against yet`
|
|
2459
|
+
);
|
|
2460
|
+
}
|
|
2461
|
+
const manifest = await api.get(
|
|
2462
|
+
`/v1/agent-task-runs/${runId}/deliverables`
|
|
2463
|
+
);
|
|
2464
|
+
const items = manifest.items ?? [];
|
|
2465
|
+
const notReady = items.filter((item) => item.status !== "ready");
|
|
2466
|
+
if (notReady.length > 0) {
|
|
2467
|
+
const names = notReady.map((item) => `${item.name} (${item.status})`).join(", ");
|
|
2468
|
+
throw new RejudgeError(
|
|
2469
|
+
`Refusing to replay run ${runId}: deliverables are not ready: ${names}. A missing deliverable must not render as a low score \u2014 re-run the agent instead.`
|
|
2470
|
+
);
|
|
2471
|
+
}
|
|
2472
|
+
const requestedRevision = options.definitionRevisionId ?? detail.task_definition?.id ?? null;
|
|
2473
|
+
if (!requestedRevision) {
|
|
2474
|
+
throw new RejudgeError(
|
|
2475
|
+
`Run ${runId} has no pinned Task Definition Revision \u2014 its checks cannot be replayed`
|
|
2476
|
+
);
|
|
2477
|
+
}
|
|
2478
|
+
const sourceQuery = options.definitionRevisionId && options.definitionRevisionId !== detail.task_definition?.id ? `?revision=${encodeURIComponent(options.definitionRevisionId)}` : "";
|
|
2479
|
+
progress(`fetching definition source (${requestedRevision})`);
|
|
2480
|
+
const source = await api.get(
|
|
2481
|
+
`/v1/agent-task-runs/${runId}/definition-source${sourceQuery}`
|
|
2482
|
+
);
|
|
2483
|
+
const revisionId = source.task_definition_revision_id;
|
|
2484
|
+
const revisionIsPinned = revisionId === (detail.task_definition?.id ?? null);
|
|
2485
|
+
const evalFile = source.files[0];
|
|
2486
|
+
if (!evalFile) {
|
|
2487
|
+
throw new RejudgeError(`Definition revision ${revisionId} carries no source files`);
|
|
2488
|
+
}
|
|
2489
|
+
progress("fetching stored deliverables");
|
|
2490
|
+
const artifactDir = mkdtempSync(join2(tmpdir(), "apo-rejudge-artifacts-"));
|
|
2491
|
+
let scaffoldDir = null;
|
|
2492
|
+
try {
|
|
2493
|
+
const taskDir = options.taskDir ?? null;
|
|
2494
|
+
scaffoldDir = taskDir ? null : isolatedScaffoldDir();
|
|
2495
|
+
const deliverables = await fetchDeliverables(api, runId, items, artifactDir);
|
|
2496
|
+
progress("reading canonical trace projection");
|
|
2497
|
+
const snapshotResult = await readCanonicalSnapshot2(
|
|
2498
|
+
endpoints,
|
|
2499
|
+
runId,
|
|
2500
|
+
options.projectionDeadlineMs
|
|
2501
|
+
);
|
|
2502
|
+
if (!snapshotResult.available) {
|
|
2503
|
+
progress(
|
|
2504
|
+
"trace projection unavailable \u2014 trajectory assertions (t.calledTool, \u2026) will be recorded as unsupported rather than replayed"
|
|
2505
|
+
);
|
|
2506
|
+
}
|
|
2507
|
+
const snapshot = snapshotResult.snapshot;
|
|
2508
|
+
const loaded = await loadEvalSource(evalFile.path, evalFile.content, {
|
|
2509
|
+
siblingDir: taskDir,
|
|
2510
|
+
isolatedDir: scaffoldDir
|
|
2511
|
+
});
|
|
2512
|
+
progress(`loaded eval ${evalFile.path} from revision ${revisionId}`);
|
|
2513
|
+
const missingDeliverables = loaded.task.deliverables.filter(
|
|
2514
|
+
(name) => !Object.prototype.hasOwnProperty.call(deliverables, name)
|
|
2515
|
+
);
|
|
2516
|
+
if (missingDeliverables.length > 0) {
|
|
2517
|
+
throw new RejudgeError(
|
|
2518
|
+
`Refusing to replay run ${runId}: stored deliverables are incomplete; missing ${missingDeliverables.join(", ")}. Re-run the agent instead.`
|
|
2519
|
+
);
|
|
2520
|
+
}
|
|
2521
|
+
const samples = Math.max(1, Math.min(options.samples ?? 1, MAX_SAMPLES));
|
|
2522
|
+
const validation = validateDeliverables(
|
|
2523
|
+
loaded.task,
|
|
2524
|
+
deliverables,
|
|
2525
|
+
loaded.adapter.deliverables
|
|
2526
|
+
);
|
|
2527
|
+
const proxied = proxyBrokenDeliverables(deliverables, validation.brokenDeliverables);
|
|
2528
|
+
const passCounts = /* @__PURE__ */ new Map();
|
|
2529
|
+
let primary = [];
|
|
2530
|
+
const judgeConfig = resolveJudgeConfig(options.judge, loaded.task.judge);
|
|
2531
|
+
for (let sample = 0; sample < samples; sample++) {
|
|
2532
|
+
if (samples > 1) progress(`judging sample ${sample + 1}/${samples}`);
|
|
2533
|
+
const results = await runChecksOnce(loaded, proxied, snapshot, judgeConfig);
|
|
2534
|
+
if (sample === 0) primary = results;
|
|
2535
|
+
for (const result of results) {
|
|
2536
|
+
passCounts.set(result.id, (passCounts.get(result.id) ?? 0) + (result.pass ? 1 : 0));
|
|
2537
|
+
}
|
|
2538
|
+
}
|
|
2539
|
+
const stability = primary.map((result) => ({
|
|
2540
|
+
check_id: result.id,
|
|
2541
|
+
passes: passCounts.get(result.id) ?? 0,
|
|
2542
|
+
samples
|
|
2543
|
+
}));
|
|
2544
|
+
const aggregate = aggregateResult(primary);
|
|
2545
|
+
return {
|
|
2546
|
+
runId,
|
|
2547
|
+
taskId: detail.task_id,
|
|
2548
|
+
definitionRevisionId: revisionId,
|
|
2549
|
+
definitionRevisionIsPinned: revisionIsPinned,
|
|
2550
|
+
judge: judgeConfig ? {
|
|
2551
|
+
model: judgeConfig.model,
|
|
2552
|
+
baseURL: judgeConfig.baseURL ?? process.env.OPENROUTER_BASE_URL
|
|
2553
|
+
} : null,
|
|
2554
|
+
samples,
|
|
2555
|
+
checks: aggregate.checks,
|
|
2556
|
+
pass: aggregate.pass,
|
|
2557
|
+
stability,
|
|
2558
|
+
traceSnapshotAvailable: snapshotResult.available,
|
|
2559
|
+
taskDirUsed: taskDir
|
|
2560
|
+
};
|
|
2561
|
+
} finally {
|
|
2562
|
+
rmSync(artifactDir, { recursive: true, force: true });
|
|
2563
|
+
if (scaffoldDir) rmSync(scaffoldDir, { recursive: true, force: true });
|
|
2564
|
+
}
|
|
2565
|
+
}
|
|
2566
|
+
async function runChecksOnce(loaded, deliverables, snapshot, judgeConfig) {
|
|
2567
|
+
if (loaded.inlineChecks) {
|
|
2568
|
+
return runTraceChecks({
|
|
2569
|
+
snapshot,
|
|
2570
|
+
deliverables,
|
|
2571
|
+
files: loaded.files,
|
|
2572
|
+
task: loaded.task,
|
|
2573
|
+
judgeConfig,
|
|
2574
|
+
moduleUrl: loaded.moduleUrl,
|
|
2575
|
+
displayFile: loaded.evalFileName
|
|
2576
|
+
});
|
|
2577
|
+
}
|
|
2578
|
+
return loadAndRunFlowChecks(
|
|
2579
|
+
loaded.checksPath,
|
|
2580
|
+
{
|
|
2581
|
+
snapshot,
|
|
2582
|
+
deliverables,
|
|
2583
|
+
files: loaded.files,
|
|
2584
|
+
task: loaded.task,
|
|
2585
|
+
judgeConfig
|
|
2586
|
+
},
|
|
2587
|
+
{}
|
|
2588
|
+
);
|
|
2589
|
+
}
|
|
2590
|
+
async function fetchDeliverables(api, runId, items, artifactDir) {
|
|
2591
|
+
const deliverables = {};
|
|
2592
|
+
for (const [index, item] of items.entries()) {
|
|
2593
|
+
if (item.kind === "artifact") {
|
|
2594
|
+
const bytes = await api.getBytes(`/v1/agent-task-runs/${runId}/deliverables/${item.id}`);
|
|
2595
|
+
const displayFilename = item.display_filename || item.name;
|
|
2596
|
+
const safeFilename = basename4(displayFilename) || "artifact";
|
|
2597
|
+
const path = join2(artifactDir, `artifact-${index}-${safeFilename}`);
|
|
2598
|
+
writeFileSync2(path, bytes);
|
|
2599
|
+
deliverables[item.name] = Object.freeze({
|
|
2600
|
+
kind: "apo.file-artifact",
|
|
2601
|
+
path,
|
|
2602
|
+
mediaType: item.media_type ?? "application/octet-stream",
|
|
2603
|
+
displayFilename
|
|
2604
|
+
});
|
|
2605
|
+
} else {
|
|
2606
|
+
deliverables[item.name] = await api.getJson(
|
|
2607
|
+
`/v1/agent-task-runs/${runId}/deliverables/${item.id}`
|
|
2608
|
+
);
|
|
2609
|
+
}
|
|
2610
|
+
}
|
|
2611
|
+
return deliverables;
|
|
2612
|
+
}
|
|
2613
|
+
async function readCanonicalSnapshot2(endpoints, runId, deadlineMs) {
|
|
2614
|
+
try {
|
|
2615
|
+
const snapshot = await readTaskRunProjection({
|
|
2616
|
+
endpoint: endpoints.backendUrl,
|
|
2617
|
+
authToken: endpoints.authToken,
|
|
2618
|
+
taskRunId: runId,
|
|
2619
|
+
deadlineMs: deadlineMs ?? 15e3
|
|
2620
|
+
});
|
|
2621
|
+
return { available: true, snapshot };
|
|
2622
|
+
} catch {
|
|
2623
|
+
return { available: false, snapshot: degenerateSnapshot(runId) };
|
|
2624
|
+
}
|
|
2625
|
+
}
|
|
2626
|
+
function degenerateSnapshot(runId) {
|
|
2627
|
+
return {
|
|
2628
|
+
schemaVersion: 1,
|
|
2629
|
+
projectionVersion: 1,
|
|
2630
|
+
source: "local",
|
|
2631
|
+
trace: { traceId: `rejudge-${runId}`, complete: true },
|
|
2632
|
+
capabilities: {
|
|
2633
|
+
messages: "unavailable",
|
|
2634
|
+
tools: "unavailable",
|
|
2635
|
+
errors: "unavailable",
|
|
2636
|
+
timing: "unavailable",
|
|
2637
|
+
skills: "unavailable",
|
|
2638
|
+
subagents: "unavailable"
|
|
2639
|
+
},
|
|
2640
|
+
observations: []
|
|
2641
|
+
};
|
|
2642
|
+
}
|
|
2643
|
+
function isolatedScaffoldDir() {
|
|
2644
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
2645
|
+
return mkdtempSync(join2(here, ".rejudge-scaffold-"));
|
|
2646
|
+
}
|
|
2647
|
+
function createApiClient(endpoints) {
|
|
2648
|
+
const base = endpoints.backendUrl.replace(/\/$/, "");
|
|
2649
|
+
const headers = { Authorization: `Bearer ${endpoints.authToken}` };
|
|
2650
|
+
async function request(path) {
|
|
2651
|
+
const response = await fetch(`${base}${path}`, { headers });
|
|
2652
|
+
if (!response.ok) {
|
|
2653
|
+
const body = await response.text().catch(() => "");
|
|
2654
|
+
throw new RejudgeError(
|
|
2655
|
+
`${path} failed (${response.status}): ${body.slice(0, 200) || response.statusText}`
|
|
2656
|
+
);
|
|
2657
|
+
}
|
|
2658
|
+
return response;
|
|
2659
|
+
}
|
|
2660
|
+
return {
|
|
2661
|
+
async get(path) {
|
|
2662
|
+
return await request(path).then((r) => r.json());
|
|
2663
|
+
},
|
|
2664
|
+
async getJson(path) {
|
|
2665
|
+
return request(path).then((r) => r.json());
|
|
2666
|
+
},
|
|
2667
|
+
async getBytes(path) {
|
|
2668
|
+
const buffer = await request(path).then((r) => r.arrayBuffer());
|
|
2669
|
+
return Buffer.from(new Uint8Array(buffer));
|
|
2670
|
+
}
|
|
2671
|
+
};
|
|
2672
|
+
}
|
|
2673
|
+
|
|
2318
2674
|
// src/agent-task/cli.ts
|
|
2319
2675
|
import { relative, resolve as resolve3 } from "path";
|
|
2320
2676
|
|
|
2321
2677
|
// src/agent-task/discovery.ts
|
|
2322
2678
|
import { existsSync as existsSync3, readdirSync as readdirSync2, statSync as statSync3 } from "fs";
|
|
2323
|
-
import { join as
|
|
2679
|
+
import { join as join3, resolve as resolve2 } from "path";
|
|
2324
2680
|
function discoverAgentTaskDirs(rootDir) {
|
|
2325
2681
|
const absoluteRoot = resolve2(rootDir);
|
|
2326
2682
|
if (!existsSync3(absoluteRoot) || !statSync3(absoluteRoot).isDirectory()) {
|
|
@@ -2336,7 +2692,7 @@ function walkForTaskDirs(currentDir, discovered) {
|
|
|
2336
2692
|
return;
|
|
2337
2693
|
}
|
|
2338
2694
|
for (const entry of readdirSync2(currentDir)) {
|
|
2339
|
-
const absolutePath =
|
|
2695
|
+
const absolutePath = join3(currentDir, entry);
|
|
2340
2696
|
if (!statSync3(absolutePath).isDirectory()) {
|
|
2341
2697
|
continue;
|
|
2342
2698
|
}
|
|
@@ -2998,6 +3354,7 @@ export {
|
|
|
2998
3354
|
AgentTaskRunError,
|
|
2999
3355
|
ApoSpanProcessor,
|
|
3000
3356
|
FlowView,
|
|
3357
|
+
RejudgeError,
|
|
3001
3358
|
TEST_METHOD_NAMES,
|
|
3002
3359
|
TaskFiles,
|
|
3003
3360
|
buildManifest,
|
|
@@ -3028,6 +3385,7 @@ export {
|
|
|
3028
3385
|
parseAgentTaskCliArgs,
|
|
3029
3386
|
persistFileArtifacts,
|
|
3030
3387
|
registerApoTracing,
|
|
3388
|
+
rejudgeTaskRun,
|
|
3031
3389
|
resetApoTracing,
|
|
3032
3390
|
resetFlowChecks,
|
|
3033
3391
|
resetTaskRegistry,
|