@apo-ai/sdk 0.2.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -235,7 +235,8 @@ function levenshtein(a, b) {
235
235
  }
236
236
 
237
237
  // src/agent-task/checks/judge.ts
238
- var JUDGE_SYSTEM_PROMPT = 'You are an evaluation judge. Evaluate the given value(s) against the instruction. Respond with ONLY a JSON object: {"pass": true/false, "reasoning": "your reasoning"}';
238
+ var JUDGE_RESPONSE_CONTRACT = 'Respond with ONLY a JSON object: {"pass": true/false, "reasoning": "your reasoning"}';
239
+ var JUDGE_SYSTEM_PROMPT = `You are an evaluation judge. Evaluate the given value(s) against the instruction. ${JUDGE_RESPONSE_CONTRACT}`;
239
240
  function formatValue(value, depth = 0) {
240
241
  const indent = " ".repeat(depth);
241
242
  if (value === null || value === void 0) return String(value);
@@ -314,12 +315,11 @@ async function callJudge(args) {
314
315
  const apiKey = args.apiKey ?? process.env.OPENROUTER_API_KEY ?? process.env.OPENAI_API_KEY;
315
316
  const deliverableText = `Values to evaluate:
316
317
  ${formatJudgeValues(args.values)}`;
317
- const instructionText = `Instruction:
318
- ${args.instruction}`;
319
- const systemPromptText = `${JUDGE_SYSTEM_PROMPT}
318
+ const { briefingText, instructionText } = assembleBriefing(args);
319
+ const systemPromptText = `${briefingText}
320
320
 
321
321
  ${deliverableText}`;
322
- const cacheKey = `${args.model}\0${deliverableText}`;
322
+ const cacheKey = `${args.model}\0${briefingText}\0${deliverableText}`;
323
323
  return runWithSharedPrefix(cacheKey, async () => {
324
324
  const startedAt = Date.now();
325
325
  const response = await fetch(`${baseURL}/chat/completions`, {
@@ -334,7 +334,7 @@ ${deliverableText}`;
334
334
  {
335
335
  role: "system",
336
336
  content: [
337
- { type: "text", text: JUDGE_SYSTEM_PROMPT },
337
+ { type: "text", text: briefingText },
338
338
  {
339
339
  type: "text",
340
340
  text: deliverableText,
@@ -384,6 +384,33 @@ ${deliverableText}`;
384
384
  };
385
385
  });
386
386
  }
387
+ function assembleBriefing(args) {
388
+ if (!args.prompt) {
389
+ return {
390
+ briefingText: JUDGE_SYSTEM_PROMPT,
391
+ instructionText: `Instruction:
392
+ ${args.instruction}`
393
+ };
394
+ }
395
+ const ctx = {
396
+ taskId: args.context?.taskId ?? "",
397
+ checkName: args.context?.checkName ?? "",
398
+ ...args.context?.taskDescription !== void 0 ? { taskDescription: args.context.taskDescription } : {},
399
+ ...args.context?.deliverableNames !== void 0 && args.context.deliverableNames.length > 0 ? { deliverableNames: args.context.deliverableNames } : {},
400
+ instruction: args.instruction
401
+ };
402
+ const built = args.prompt(ctx);
403
+ const system = built.system?.trim();
404
+ const user = built.user?.trim();
405
+ return {
406
+ // The SDK appends its own response contract to any custom briefing.
407
+ briefingText: system ? `${system}
408
+
409
+ ${JUDGE_RESPONSE_CONTRACT}` : JUDGE_SYSTEM_PROMPT,
410
+ instructionText: user ?? `Instruction:
411
+ ${args.instruction}`
412
+ };
413
+ }
387
414
 
388
415
  // src/agent-task/checks/t.ts
389
416
  function resolveJudgeConfig(judgeConfig, override) {
@@ -392,7 +419,8 @@ function resolveJudgeConfig(judgeConfig, override) {
392
419
  return {
393
420
  model,
394
421
  baseURL: override?.baseURL ?? judgeConfig?.baseURL,
395
- apiKey: override?.apiKey ?? judgeConfig?.apiKey
422
+ apiKey: override?.apiKey ?? judgeConfig?.apiKey,
423
+ prompt: override?.prompt ?? judgeConfig?.prompt
396
424
  };
397
425
  }
398
426
  var TEST_METHOD_NAMES = [
@@ -421,6 +449,67 @@ function matchName(name, m) {
421
449
  if (m instanceof RegExp) return m.test(name);
422
450
  return m(name);
423
451
  }
452
+ function createJudgeMethod(rec, judgeConfig, judgeScope) {
453
+ return async (values, instruction, opts) => {
454
+ const label = opts?.label ?? "judge";
455
+ const location = rec.captureLocation();
456
+ const valueArray = Array.isArray(values) ? values : [values];
457
+ const effective = resolveJudgeConfig(judgeConfig, opts?.judge);
458
+ if (!effective) {
459
+ rec.record(
460
+ label,
461
+ false,
462
+ "No judge model configured. Set one of:\n\u2022 OPENROUTER_MODEL + OPENROUTER_API_KEY (OpenRouter \u2014 works with 200+ models, one account)\n\u2022 OPENAI_MODEL + OPENAI_API_KEY (OpenAI direct)\nOr pass { judge } to runTask() programmatically.",
463
+ { evaluator_type: "llm", location }
464
+ );
465
+ return;
466
+ }
467
+ const context2 = judgeScopeToContext(judgeScope);
468
+ try {
469
+ const { pass, reasoning, judge } = await callJudge({
470
+ values: valueArray,
471
+ instruction,
472
+ model: effective.model,
473
+ baseURL: effective.baseURL,
474
+ apiKey: effective.apiKey,
475
+ prompt: effective.prompt,
476
+ ...context2 ? { context: context2 } : {}
477
+ });
478
+ rec.record(label, pass, reasoning, {
479
+ evaluator_type: "llm",
480
+ judge,
481
+ expected: instruction,
482
+ // Store the raw evaluated value (not a truncated string) so the
483
+ // dashboard can render it with the right viewer — Markdown for
484
+ // prose, a JSON tree for structured objects.
485
+ received: valueArray.length === 1 ? valueArray[0] : valueArray,
486
+ location
487
+ });
488
+ } catch (error) {
489
+ rec.record(
490
+ label,
491
+ false,
492
+ `judge failed: ${error instanceof Error ? error.message : String(error)}`,
493
+ {
494
+ evaluator_type: "llm",
495
+ expected: instruction,
496
+ received: valueArray.length === 1 ? valueArray[0] : valueArray,
497
+ location
498
+ }
499
+ );
500
+ }
501
+ };
502
+ }
503
+ function judgeScopeToContext(scope) {
504
+ if (!scope) return void 0;
505
+ const deliverableNames = scope.readDeliverableNames?.();
506
+ return {
507
+ taskId: scope.taskId,
508
+ checkName: scope.checkName,
509
+ ...scope.taskDescription !== void 0 ? { taskDescription: scope.taskDescription } : {},
510
+ ...deliverableNames && deliverableNames.length > 0 ? { deliverableNames } : {}
511
+ };
512
+ }
424
513
  var CAPABILITY_LABELS = {
425
514
  messages: "message",
426
515
  tools: "tool",
@@ -429,7 +518,7 @@ var CAPABILITY_LABELS = {
429
518
  skills: "skill",
430
519
  subagents: "subagent"
431
520
  };
432
- function createTraceTestContext(view, rec, judgeConfig) {
521
+ function createTraceTestContext(view, rec, judgeConfig, judgeScope) {
433
522
  const unsupported = (id, capability) => {
434
523
  const label = CAPABILITY_LABELS[capability];
435
524
  rec.record(id, false, `${label} evidence is unavailable in this trace projection`, {
@@ -613,49 +702,9 @@ function createTraceTestContext(view, rec, judgeConfig) {
613
702
  { expected: matcher.label, received: describeValue(value) }
614
703
  );
615
704
  },
616
- async judge(values, instruction, opts) {
617
- const label = opts?.label ?? "judge";
618
- const location = rec.captureLocation();
619
- const valueArray = Array.isArray(values) ? values : [values];
620
- const effective = resolveJudgeConfig(judgeConfig, opts?.judge);
621
- if (!effective) {
622
- rec.record(
623
- label,
624
- false,
625
- "No judge model configured. Set one of:\n\u2022 OPENROUTER_MODEL + OPENROUTER_API_KEY (OpenRouter \u2014 works with 200+ models, one account)\n\u2022 OPENAI_MODEL + OPENAI_API_KEY (OpenAI direct)\nOr pass { judge } to runTask() programmatically.",
626
- { evaluator_type: "llm", location }
627
- );
628
- return;
629
- }
630
- try {
631
- const { pass, reasoning, judge } = await callJudge({
632
- values: valueArray,
633
- instruction,
634
- model: effective.model,
635
- baseURL: effective.baseURL,
636
- apiKey: effective.apiKey
637
- });
638
- rec.record(label, pass, reasoning, {
639
- evaluator_type: "llm",
640
- judge,
641
- expected: instruction,
642
- received: valueArray.length === 1 ? valueArray[0] : valueArray,
643
- location
644
- });
645
- } catch (error) {
646
- rec.record(
647
- label,
648
- false,
649
- `judge failed: ${error instanceof Error ? error.message : String(error)}`,
650
- {
651
- evaluator_type: "llm",
652
- expected: instruction,
653
- received: valueArray.length === 1 ? valueArray[0] : valueArray,
654
- location
655
- }
656
- );
657
- }
658
- }
705
+ // judge does not consult trace capabilities; the scope carries the
706
+ // task/check frame for prompt builders (#161).
707
+ judge: createJudgeMethod(rec, judgeConfig, judgeScope)
659
708
  };
660
709
  }
661
710
  function matchesTraceToolCall(call, name, opts) {
@@ -787,14 +836,21 @@ function describe(id, nameOrFn, fn) {
787
836
  async function runTraceChecks(args) {
788
837
  const view = new TraceView(args.snapshot);
789
838
  const locate = args.moduleUrl && args.displayFile ? (stack) => parseCheckLocation(stack, args.moduleUrl, args.displayFile) : void 0;
839
+ const taskMeta = readTaskMeta(args.task);
790
840
  const results = await Promise.all(
791
841
  registry.map(async (check) => {
792
842
  const rec = new Recorder(locate);
793
- const t = createTraceTestContext(view, rec, args.judgeConfig);
843
+ const reads = trackDeliverableReads(args.deliverables);
844
+ const t = createTraceTestContext(view, rec, args.judgeConfig, {
845
+ taskId: taskMeta.id ?? "",
846
+ ...taskMeta.description !== void 0 ? { taskDescription: taskMeta.description } : {},
847
+ checkName: check.id,
848
+ readDeliverableNames: reads.names
849
+ });
794
850
  let thrownLocation;
795
851
  try {
796
852
  await check.fn(t, {
797
- deliverables: args.deliverables,
853
+ deliverables: reads.proxied,
798
854
  files: args.files,
799
855
  task: args.task
800
856
  });
@@ -866,6 +922,26 @@ function proxyBrokenDeliverables(deliverables, brokenDeliverables) {
866
922
  }
867
923
  });
868
924
  }
925
+ function readTaskMeta(task2) {
926
+ if (!task2 || typeof task2 !== "object") return {};
927
+ const record = task2;
928
+ return {
929
+ ...typeof record.id === "string" ? { id: record.id } : {},
930
+ ...typeof record.description === "string" ? { description: record.description } : {}
931
+ };
932
+ }
933
+ function trackDeliverableReads(deliverables) {
934
+ const read = /* @__PURE__ */ new Set();
935
+ const proxied = new Proxy(deliverables, {
936
+ get(target, property, receiver) {
937
+ if (typeof property === "string" && Object.prototype.hasOwnProperty.call(target, property)) {
938
+ read.add(property);
939
+ }
940
+ return Reflect.get(target, property, receiver);
941
+ }
942
+ });
943
+ return { proxied, names: () => [...read] };
944
+ }
869
945
 
870
946
  // src/agent-task/task/defineTask.ts
871
947
  var TASK_ADAPTER_SYMBOL = /* @__PURE__ */ Symbol.for("agent-task.adapter-definition");
@@ -919,7 +995,7 @@ function attachAdapter(taskDefinition, adapter) {
919
995
  }
920
996
 
921
997
  // src/agent-task/task/loadTask.ts
922
- import { copyFileSync as copyFileSync2, readdirSync, statSync, existsSync as existsSync2, unlinkSync as unlinkSync2 } from "fs";
998
+ import { copyFileSync as copyFileSync2, readdirSync, statSync, existsSync as existsSync2, unlinkSync as unlinkSync2, writeFileSync } from "fs";
923
999
  import { join, resolve, basename as basename2 } from "path";
924
1000
  import { pathToFileURL as pathToFileURL2 } from "url";
925
1001
 
@@ -966,6 +1042,48 @@ async function loadTask(taskDir) {
966
1042
  evalFileName: basename2(taskFilePath)
967
1043
  };
968
1044
  }
1045
+ async function loadEvalSource(fileName, content, opts = {}) {
1046
+ const dir = opts.siblingDir ?? opts.isolatedDir;
1047
+ if (!dir) {
1048
+ throw new Error("loadEvalSource requires a siblingDir or isolatedDir to materialize next to");
1049
+ }
1050
+ const tempPath = join(
1051
+ dir,
1052
+ `${fileName}.rejudge-${Date.now()}-${Math.random().toString(36).slice(2)}.ts`
1053
+ );
1054
+ writeFileSync(tempPath, content);
1055
+ try {
1056
+ const { task: task2, adapter, inlineChecks, moduleUrl } = await loadTaskDefinition(tempPath);
1057
+ validateTaskDefinition(task2, dir);
1058
+ if (!inlineChecks && !existsSync2(join(dir, "checks.ts"))) {
1059
+ throw new Error(
1060
+ `The pinned eval did not register checks: legacy two-file tasks need their checks module, which the stored definition revision does not include`
1061
+ );
1062
+ }
1063
+ return {
1064
+ task: task2,
1065
+ adapter,
1066
+ taskDir: dir,
1067
+ files: loadFilesDirectory(dir),
1068
+ checksPath: resolveChecksPath(task2, dir),
1069
+ inlineChecks,
1070
+ moduleUrl,
1071
+ evalFileName: fileName
1072
+ };
1073
+ } catch (error) {
1074
+ const message = error instanceof Error ? error.message : String(error);
1075
+ if (!opts.siblingDir && /Failed to load task definition/.test(message)) {
1076
+ throw new Error(
1077
+ `Replay needs the task directory: the pinned eval imports sibling files that are not part of the stored definition revision. Point the command at a checkout of the task (--task-dir) and retry. Underlying error: ${message}`
1078
+ );
1079
+ }
1080
+ throw error;
1081
+ } finally {
1082
+ if (existsSync2(tempPath)) {
1083
+ unlinkSync2(tempPath);
1084
+ }
1085
+ }
1086
+ }
969
1087
  async function loadTaskDefinition(taskFilePath) {
970
1088
  const tempModulePath = `${taskFilePath}.${Date.now()}-${Math.random().toString(36).slice(2)}.ts`;
971
1089
  try {
@@ -2034,6 +2152,7 @@ async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
2034
2152
  summarize: (result2) => summarizeEvaluationResults(result2)
2035
2153
  },
2036
2154
  async () => {
2155
+ const judgeConfig = resolveJudgeConfig(options?.judge, task2.judge);
2037
2156
  if (!inlineChecks) {
2038
2157
  return loadAndRunFlowChecks(
2039
2158
  checksPath,
@@ -2042,7 +2161,7 @@ async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
2042
2161
  deliverables: collected,
2043
2162
  files,
2044
2163
  task: task2,
2045
- judgeConfig: options?.judge
2164
+ ...judgeConfig ? { judgeConfig } : {}
2046
2165
  },
2047
2166
  validationResults.brokenDeliverables
2048
2167
  );
@@ -2055,7 +2174,7 @@ async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
2055
2174
  ),
2056
2175
  files,
2057
2176
  task: task2,
2058
- judgeConfig: options?.judge,
2177
+ ...judgeConfig ? { judgeConfig } : {},
2059
2178
  moduleUrl,
2060
2179
  displayFile: evalFileName
2061
2180
  });
@@ -2235,12 +2354,13 @@ async function evaluate(loaded, options, phase1) {
2235
2354
  phase1.collected,
2236
2355
  validationResults.brokenDeliverables
2237
2356
  );
2357
+ const judgeConfig = resolveJudgeConfig(options?.judge, task2.judge);
2238
2358
  const checksResults = await (inlineChecks ? runTraceChecks({
2239
2359
  snapshot: phase1.snapshot,
2240
2360
  deliverables,
2241
2361
  files,
2242
2362
  task: task2,
2243
- judgeConfig: options?.judge,
2363
+ ...judgeConfig ? { judgeConfig } : {},
2244
2364
  moduleUrl,
2245
2365
  displayFile: evalFileName
2246
2366
  }) : loadAndRunFlowChecks(
@@ -2250,7 +2370,7 @@ async function evaluate(loaded, options, phase1) {
2250
2370
  deliverables,
2251
2371
  files,
2252
2372
  task: task2,
2253
- judgeConfig: options?.judge
2373
+ ...judgeConfig ? { judgeConfig } : {}
2254
2374
  },
2255
2375
  validationResults.brokenDeliverables
2256
2376
  ));
@@ -2315,12 +2435,248 @@ function buildTraceRunOptions(loaded, tracing) {
2315
2435
  };
2316
2436
  }
2317
2437
 
2438
+ // src/agent-task/run/rejudge.ts
2439
+ import { mkdtempSync, rmSync, writeFileSync as writeFileSync2 } from "fs";
2440
+ import { tmpdir } from "os";
2441
+ import { basename as basename4, dirname, join as join2 } from "path";
2442
+ import { fileURLToPath } from "url";
2443
+ var RejudgeError = class extends Error {
2444
+ constructor(message) {
2445
+ super(message);
2446
+ this.name = "RejudgeError";
2447
+ }
2448
+ };
2449
+ var TERMINAL_VERDICT_STATUSES = /* @__PURE__ */ new Set(["passed", "failed"]);
2450
+ var MAX_SAMPLES = 50;
2451
+ async function rejudgeTaskRun(runId, endpoints, options = {}) {
2452
+ const api = createApiClient(endpoints);
2453
+ const progress = options.onProgress ?? (() => {
2454
+ });
2455
+ const detail = await api.get(`/v1/agent-task-runs/${runId}`);
2456
+ if (!TERMINAL_VERDICT_STATUSES.has(detail.status)) {
2457
+ throw new RejudgeError(
2458
+ `Run ${runId} is '${detail.status}', not a completed run with a verdict \u2014 there is nothing to replay against yet`
2459
+ );
2460
+ }
2461
+ const manifest = await api.get(
2462
+ `/v1/agent-task-runs/${runId}/deliverables`
2463
+ );
2464
+ const items = manifest.items ?? [];
2465
+ const notReady = items.filter((item) => item.status !== "ready");
2466
+ if (notReady.length > 0) {
2467
+ const names = notReady.map((item) => `${item.name} (${item.status})`).join(", ");
2468
+ throw new RejudgeError(
2469
+ `Refusing to replay run ${runId}: deliverables are not ready: ${names}. A missing deliverable must not render as a low score \u2014 re-run the agent instead.`
2470
+ );
2471
+ }
2472
+ const requestedRevision = options.definitionRevisionId ?? detail.task_definition?.id ?? null;
2473
+ if (!requestedRevision) {
2474
+ throw new RejudgeError(
2475
+ `Run ${runId} has no pinned Task Definition Revision \u2014 its checks cannot be replayed`
2476
+ );
2477
+ }
2478
+ const sourceQuery = options.definitionRevisionId && options.definitionRevisionId !== detail.task_definition?.id ? `?revision=${encodeURIComponent(options.definitionRevisionId)}` : "";
2479
+ progress(`fetching definition source (${requestedRevision})`);
2480
+ const source = await api.get(
2481
+ `/v1/agent-task-runs/${runId}/definition-source${sourceQuery}`
2482
+ );
2483
+ const revisionId = source.task_definition_revision_id;
2484
+ const revisionIsPinned = revisionId === (detail.task_definition?.id ?? null);
2485
+ const evalFile = source.files[0];
2486
+ if (!evalFile) {
2487
+ throw new RejudgeError(`Definition revision ${revisionId} carries no source files`);
2488
+ }
2489
+ progress("fetching stored deliverables");
2490
+ const artifactDir = mkdtempSync(join2(tmpdir(), "apo-rejudge-artifacts-"));
2491
+ let scaffoldDir = null;
2492
+ try {
2493
+ const taskDir = options.taskDir ?? null;
2494
+ scaffoldDir = taskDir ? null : isolatedScaffoldDir();
2495
+ const deliverables = await fetchDeliverables(api, runId, items, artifactDir);
2496
+ progress("reading canonical trace projection");
2497
+ const snapshotResult = await readCanonicalSnapshot2(
2498
+ endpoints,
2499
+ runId,
2500
+ options.projectionDeadlineMs
2501
+ );
2502
+ if (!snapshotResult.available) {
2503
+ progress(
2504
+ "trace projection unavailable \u2014 trajectory assertions (t.calledTool, \u2026) will be recorded as unsupported rather than replayed"
2505
+ );
2506
+ }
2507
+ const snapshot = snapshotResult.snapshot;
2508
+ const loaded = await loadEvalSource(evalFile.path, evalFile.content, {
2509
+ siblingDir: taskDir,
2510
+ isolatedDir: scaffoldDir
2511
+ });
2512
+ progress(`loaded eval ${evalFile.path} from revision ${revisionId}`);
2513
+ const missingDeliverables = loaded.task.deliverables.filter(
2514
+ (name) => !Object.prototype.hasOwnProperty.call(deliverables, name)
2515
+ );
2516
+ if (missingDeliverables.length > 0) {
2517
+ throw new RejudgeError(
2518
+ `Refusing to replay run ${runId}: stored deliverables are incomplete; missing ${missingDeliverables.join(", ")}. Re-run the agent instead.`
2519
+ );
2520
+ }
2521
+ const samples = Math.max(1, Math.min(options.samples ?? 1, MAX_SAMPLES));
2522
+ const validation = validateDeliverables(
2523
+ loaded.task,
2524
+ deliverables,
2525
+ loaded.adapter.deliverables
2526
+ );
2527
+ const proxied = proxyBrokenDeliverables(deliverables, validation.brokenDeliverables);
2528
+ const passCounts = /* @__PURE__ */ new Map();
2529
+ let primary = [];
2530
+ const judgeConfig = resolveJudgeConfig(options.judge, loaded.task.judge);
2531
+ for (let sample = 0; sample < samples; sample++) {
2532
+ if (samples > 1) progress(`judging sample ${sample + 1}/${samples}`);
2533
+ const results = await runChecksOnce(loaded, proxied, snapshot, judgeConfig);
2534
+ if (sample === 0) primary = results;
2535
+ for (const result of results) {
2536
+ passCounts.set(result.id, (passCounts.get(result.id) ?? 0) + (result.pass ? 1 : 0));
2537
+ }
2538
+ }
2539
+ const stability = primary.map((result) => ({
2540
+ check_id: result.id,
2541
+ passes: passCounts.get(result.id) ?? 0,
2542
+ samples
2543
+ }));
2544
+ const aggregate = aggregateResult(primary);
2545
+ return {
2546
+ runId,
2547
+ taskId: detail.task_id,
2548
+ definitionRevisionId: revisionId,
2549
+ definitionRevisionIsPinned: revisionIsPinned,
2550
+ judge: judgeConfig ? {
2551
+ model: judgeConfig.model,
2552
+ baseURL: judgeConfig.baseURL ?? process.env.OPENROUTER_BASE_URL
2553
+ } : null,
2554
+ samples,
2555
+ checks: aggregate.checks,
2556
+ pass: aggregate.pass,
2557
+ stability,
2558
+ traceSnapshotAvailable: snapshotResult.available,
2559
+ taskDirUsed: taskDir
2560
+ };
2561
+ } finally {
2562
+ rmSync(artifactDir, { recursive: true, force: true });
2563
+ if (scaffoldDir) rmSync(scaffoldDir, { recursive: true, force: true });
2564
+ }
2565
+ }
2566
+ async function runChecksOnce(loaded, deliverables, snapshot, judgeConfig) {
2567
+ if (loaded.inlineChecks) {
2568
+ return runTraceChecks({
2569
+ snapshot,
2570
+ deliverables,
2571
+ files: loaded.files,
2572
+ task: loaded.task,
2573
+ judgeConfig,
2574
+ moduleUrl: loaded.moduleUrl,
2575
+ displayFile: loaded.evalFileName
2576
+ });
2577
+ }
2578
+ return loadAndRunFlowChecks(
2579
+ loaded.checksPath,
2580
+ {
2581
+ snapshot,
2582
+ deliverables,
2583
+ files: loaded.files,
2584
+ task: loaded.task,
2585
+ judgeConfig
2586
+ },
2587
+ {}
2588
+ );
2589
+ }
2590
+ async function fetchDeliverables(api, runId, items, artifactDir) {
2591
+ const deliverables = {};
2592
+ for (const [index, item] of items.entries()) {
2593
+ if (item.kind === "artifact") {
2594
+ const bytes = await api.getBytes(`/v1/agent-task-runs/${runId}/deliverables/${item.id}`);
2595
+ const displayFilename = item.display_filename || item.name;
2596
+ const safeFilename = basename4(displayFilename) || "artifact";
2597
+ const path = join2(artifactDir, `artifact-${index}-${safeFilename}`);
2598
+ writeFileSync2(path, bytes);
2599
+ deliverables[item.name] = Object.freeze({
2600
+ kind: "apo.file-artifact",
2601
+ path,
2602
+ mediaType: item.media_type ?? "application/octet-stream",
2603
+ displayFilename
2604
+ });
2605
+ } else {
2606
+ deliverables[item.name] = await api.getJson(
2607
+ `/v1/agent-task-runs/${runId}/deliverables/${item.id}`
2608
+ );
2609
+ }
2610
+ }
2611
+ return deliverables;
2612
+ }
2613
+ async function readCanonicalSnapshot2(endpoints, runId, deadlineMs) {
2614
+ try {
2615
+ const snapshot = await readTaskRunProjection({
2616
+ endpoint: endpoints.backendUrl,
2617
+ authToken: endpoints.authToken,
2618
+ taskRunId: runId,
2619
+ deadlineMs: deadlineMs ?? 15e3
2620
+ });
2621
+ return { available: true, snapshot };
2622
+ } catch {
2623
+ return { available: false, snapshot: degenerateSnapshot(runId) };
2624
+ }
2625
+ }
2626
+ function degenerateSnapshot(runId) {
2627
+ return {
2628
+ schemaVersion: 1,
2629
+ projectionVersion: 1,
2630
+ source: "local",
2631
+ trace: { traceId: `rejudge-${runId}`, complete: true },
2632
+ capabilities: {
2633
+ messages: "unavailable",
2634
+ tools: "unavailable",
2635
+ errors: "unavailable",
2636
+ timing: "unavailable",
2637
+ skills: "unavailable",
2638
+ subagents: "unavailable"
2639
+ },
2640
+ observations: []
2641
+ };
2642
+ }
2643
+ function isolatedScaffoldDir() {
2644
+ const here = dirname(fileURLToPath(import.meta.url));
2645
+ return mkdtempSync(join2(here, ".rejudge-scaffold-"));
2646
+ }
2647
+ function createApiClient(endpoints) {
2648
+ const base = endpoints.backendUrl.replace(/\/$/, "");
2649
+ const headers = { Authorization: `Bearer ${endpoints.authToken}` };
2650
+ async function request(path) {
2651
+ const response = await fetch(`${base}${path}`, { headers });
2652
+ if (!response.ok) {
2653
+ const body = await response.text().catch(() => "");
2654
+ throw new RejudgeError(
2655
+ `${path} failed (${response.status}): ${body.slice(0, 200) || response.statusText}`
2656
+ );
2657
+ }
2658
+ return response;
2659
+ }
2660
+ return {
2661
+ async get(path) {
2662
+ return await request(path).then((r) => r.json());
2663
+ },
2664
+ async getJson(path) {
2665
+ return request(path).then((r) => r.json());
2666
+ },
2667
+ async getBytes(path) {
2668
+ const buffer = await request(path).then((r) => r.arrayBuffer());
2669
+ return Buffer.from(new Uint8Array(buffer));
2670
+ }
2671
+ };
2672
+ }
2673
+
2318
2674
  // src/agent-task/cli.ts
2319
2675
  import { relative, resolve as resolve3 } from "path";
2320
2676
 
2321
2677
  // src/agent-task/discovery.ts
2322
2678
  import { existsSync as existsSync3, readdirSync as readdirSync2, statSync as statSync3 } from "fs";
2323
- import { join as join2, resolve as resolve2 } from "path";
2679
+ import { join as join3, resolve as resolve2 } from "path";
2324
2680
  function discoverAgentTaskDirs(rootDir) {
2325
2681
  const absoluteRoot = resolve2(rootDir);
2326
2682
  if (!existsSync3(absoluteRoot) || !statSync3(absoluteRoot).isDirectory()) {
@@ -2336,7 +2692,7 @@ function walkForTaskDirs(currentDir, discovered) {
2336
2692
  return;
2337
2693
  }
2338
2694
  for (const entry of readdirSync2(currentDir)) {
2339
- const absolutePath = join2(currentDir, entry);
2695
+ const absolutePath = join3(currentDir, entry);
2340
2696
  if (!statSync3(absolutePath).isDirectory()) {
2341
2697
  continue;
2342
2698
  }
@@ -2998,6 +3354,7 @@ export {
2998
3354
  AgentTaskRunError,
2999
3355
  ApoSpanProcessor,
3000
3356
  FlowView,
3357
+ RejudgeError,
3001
3358
  TEST_METHOD_NAMES,
3002
3359
  TaskFiles,
3003
3360
  buildManifest,
@@ -3028,6 +3385,7 @@ export {
3028
3385
  parseAgentTaskCliArgs,
3029
3386
  persistFileArtifacts,
3030
3387
  registerApoTracing,
3388
+ rejudgeTaskRun,
3031
3389
  resetApoTracing,
3032
3390
  resetFlowChecks,
3033
3391
  resetTaskRegistry,