@apo-ai/sdk 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,14 +8,14 @@ import {
8
8
  resetApoTracing,
9
9
  withApoRun,
10
10
  withApoRunSync
11
- } from "../chunk-PMYLL2H4.js";
11
+ } from "../chunk-GETY34VI.js";
12
12
  import {
13
13
  createApoTracer
14
- } from "../chunk-HKLBPIB4.js";
14
+ } from "../chunk-H7ITLBWU.js";
15
15
  import "../chunk-HWZVL6TS.js";
16
16
  import {
17
17
  createApoOpenAI
18
- } from "../chunk-DT2LMB7O.js";
18
+ } from "../chunk-6ZRW5IW6.js";
19
19
  import {
20
20
  createApoAnthropic
21
21
  } from "../chunk-SZDSIPXC.js";
@@ -235,8 +235,21 @@ function levenshtein(a, b) {
235
235
  }
236
236
 
237
237
  // src/agent-task/checks/judge.ts
238
- var JUDGE_RESPONSE_CONTRACT = 'Respond with ONLY a JSON object: {"pass": true/false, "reasoning": "your reasoning"}';
239
- var JUDGE_SYSTEM_PROMPT = `You are an evaluation judge. Evaluate the given value(s) against the instruction. ${JUDGE_RESPONSE_CONTRACT}`;
238
+ var VERDICT_FIRST_CONTRACT = 'Respond with ONLY a JSON object: {"pass": true/false, "reasoning": "your reasoning"}';
239
+ var REASONING_FIRST_CONTRACT = 'Respond with ONLY a JSON object: {"reasoning": "your reasoning", "pass": true/false}';
240
+ function isJudgeVerdictFirstOverrideEnabled() {
241
+ const value = process.env.APO_JUDGE_VERDICT_FIRST?.trim().toLowerCase();
242
+ return value === "1" || value === "true";
243
+ }
244
+ function judgeResponseContract() {
245
+ return isJudgeVerdictFirstOverrideEnabled() ? VERDICT_FIRST_CONTRACT : REASONING_FIRST_CONTRACT;
246
+ }
247
+ function judgeContractInUse() {
248
+ return isJudgeVerdictFirstOverrideEnabled() ? "verdict-first" : "reasoning-first";
249
+ }
250
+ function judgeSystemPrompt() {
251
+ return `You are an evaluation judge. Evaluate the given value(s) against the instruction. ${judgeResponseContract()}`;
252
+ }
240
253
  function formatValue(value, depth = 0) {
241
254
  const indent = " ".repeat(depth);
242
255
  if (value === null || value === void 0) return String(value);
@@ -297,6 +310,7 @@ function parseJudgeJson(raw) {
297
310
  reasoning: "Judge response could not be parsed as JSON \u2014 the verdict is unknown, so this check is treated as a failure. The model's raw response is available in the judge metadata."
298
311
  };
299
312
  }
313
+ var JUDGE_TIMEOUT_MS = 12e4;
300
314
  var prefixQueues = /* @__PURE__ */ new Map();
301
315
  function runWithSharedPrefix(key, task2) {
302
316
  const prev = prefixQueues.get(key) ?? Promise.resolve();
@@ -324,6 +338,7 @@ ${deliverableText}`;
324
338
  const startedAt = Date.now();
325
339
  const response = await fetch(`${baseURL}/chat/completions`, {
326
340
  method: "POST",
341
+ signal: AbortSignal.timeout(JUDGE_TIMEOUT_MS),
327
342
  headers: {
328
343
  "Content-Type": "application/json",
329
344
  ...apiKey ? { Authorization: `Bearer ${apiKey}` } : {}
@@ -363,6 +378,7 @@ ${deliverableText}`;
363
378
  reasoning: "Judge returned an empty or truncated response \u2014 likely a transient provider failure. The verdict is unknown, so this check is treated as a failure.",
364
379
  judge: {
365
380
  model: args.model,
381
+ contract: judgeContractInUse(),
366
382
  prompt: { system: systemPromptText, user: instructionText },
367
383
  response: text,
368
384
  tokens: parseJudgeUsage(data.usage),
@@ -376,6 +392,7 @@ ${deliverableText}`;
376
392
  reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : "",
377
393
  judge: {
378
394
  model: args.model,
395
+ contract: judgeContractInUse(),
379
396
  prompt: { system: systemPromptText, user: instructionText },
380
397
  response: text,
381
398
  tokens: parseJudgeUsage(data.usage),
@@ -387,7 +404,7 @@ ${deliverableText}`;
387
404
  function assembleBriefing(args) {
388
405
  if (!args.prompt) {
389
406
  return {
390
- briefingText: JUDGE_SYSTEM_PROMPT,
407
+ briefingText: judgeSystemPrompt(),
391
408
  instructionText: `Instruction:
392
409
  ${args.instruction}`
393
410
  };
@@ -406,7 +423,7 @@ ${args.instruction}`
406
423
  // The SDK appends its own response contract to any custom briefing.
407
424
  briefingText: system ? `${system}
408
425
 
409
- ${JUDGE_RESPONSE_CONTRACT}` : JUDGE_SYSTEM_PROMPT,
426
+ ${judgeResponseContract()}` : judgeSystemPrompt(),
410
427
  instructionText: user ?? `Instruction:
411
428
  ${args.instruction}`
412
429
  };
@@ -876,8 +893,8 @@ async function runTraceChecks(args) {
876
893
  ...rec.all.length > 0 ? { assertions: rec.all.map((a) => ({ ...a })) } : {},
877
894
  ...check.group_id ? { group_id: check.group_id } : {},
878
895
  ...check.group_name ? { group_name: check.group_name } : {},
879
- // stamp the snapshot source so consumers can detect
880
- // the deprecated legacy-flow compatibility path (source="legacy-flow").
896
+ // stamp the snapshot source so consumers can tell projection-first
897
+ // results (canonical) from locally recorded snapshots.
881
898
  ...args.snapshot.source !== "canonical" ? { source: args.snapshot.source } : {}
882
899
  };
883
900
  })
@@ -1380,6 +1397,115 @@ async function safeDetail(response) {
1380
1397
  }
1381
1398
  }
1382
1399
 
1400
+ // src/agent-task/checks/compact.ts
1401
+ import { createHash as createHash2 } from "crypto";
1402
+ var RECEIVED_VALUE_LIMIT = 4 * 1024;
1403
+ var JUDGE_SEGMENT_LIMIT = 16 * 1024;
1404
+ var PREVIEW_CHARS = 256;
1405
+ function compactChecksForSubmission(checks) {
1406
+ let truncatedValues = 0;
1407
+ let truncatedSegments = 0;
1408
+ const compacted = checks.map((check) => {
1409
+ const next = { ...check };
1410
+ if ("received" in next && next.received !== void 0) {
1411
+ const replaced = truncateValue(next.received);
1412
+ if (replaced !== next.received) {
1413
+ next.received = replaced;
1414
+ truncatedValues += 1;
1415
+ }
1416
+ }
1417
+ if (next.judge !== void 0) {
1418
+ const { value, truncated } = truncateJudge(next.judge);
1419
+ if (value !== next.judge) {
1420
+ next.judge = value;
1421
+ truncatedSegments += truncated;
1422
+ }
1423
+ }
1424
+ const sourceAssertions = check.assertions;
1425
+ if (sourceAssertions) {
1426
+ const assertions = sourceAssertions.map((assertion) => {
1427
+ const nextAssertion = { ...assertion };
1428
+ if (nextAssertion.received !== void 0) {
1429
+ const replaced = truncateValue(nextAssertion.received);
1430
+ if (replaced !== nextAssertion.received) {
1431
+ nextAssertion.received = replaced;
1432
+ truncatedValues += 1;
1433
+ }
1434
+ }
1435
+ if (nextAssertion.judge !== void 0) {
1436
+ const { value, truncated } = truncateJudge(nextAssertion.judge);
1437
+ if (value !== nextAssertion.judge) {
1438
+ nextAssertion.judge = value;
1439
+ truncatedSegments += truncated;
1440
+ }
1441
+ }
1442
+ return nextAssertion;
1443
+ });
1444
+ if (assertions.some((a, i) => a !== sourceAssertions[i])) {
1445
+ next.assertions = assertions;
1446
+ }
1447
+ }
1448
+ return next;
1449
+ });
1450
+ return { checks: compacted, truncatedValues, truncatedSegments };
1451
+ }
1452
+ function truncateJudge(judge) {
1453
+ if (typeof judge !== "object" || judge === null) {
1454
+ return { value: judge, truncated: 0 };
1455
+ }
1456
+ let truncated = 0;
1457
+ const next = { ...judge };
1458
+ const prompt = next.prompt;
1459
+ if (typeof prompt === "object" && prompt !== null) {
1460
+ const nextPrompt = { ...prompt };
1461
+ for (const key of ["system", "user"]) {
1462
+ const value = nextPrompt[key];
1463
+ if (typeof value === "string") {
1464
+ const replaced = truncateText(value);
1465
+ if (replaced !== value) {
1466
+ nextPrompt[key] = replaced;
1467
+ truncated += 1;
1468
+ }
1469
+ }
1470
+ }
1471
+ next.prompt = nextPrompt;
1472
+ }
1473
+ if (typeof next.response === "string") {
1474
+ const replaced = truncateText(next.response);
1475
+ if (replaced !== next.response) {
1476
+ next.response = replaced;
1477
+ truncated += 1;
1478
+ }
1479
+ }
1480
+ return { value: next, truncated };
1481
+ }
1482
+ function truncateValue(value) {
1483
+ if (value === void 0) return value;
1484
+ let bytes;
1485
+ try {
1486
+ bytes = new TextEncoder().encode(JSON.stringify(value));
1487
+ } catch {
1488
+ bytes = new TextEncoder().encode(JSON.stringify(String(value)));
1489
+ }
1490
+ if (bytes.length <= RECEIVED_VALUE_LIMIT) return value;
1491
+ return marker(bytes);
1492
+ }
1493
+ function truncateText(value) {
1494
+ const bytes = new TextEncoder().encode(value);
1495
+ if (bytes.length <= JUDGE_SEGMENT_LIMIT) return value;
1496
+ return marker(bytes);
1497
+ }
1498
+ function marker(bytes) {
1499
+ const previewBytes = bytes.slice(0, PREVIEW_CHARS);
1500
+ const preview = Buffer.from(previewBytes).toString("utf8");
1501
+ return {
1502
+ kind: "truncated",
1503
+ preview,
1504
+ size_bytes: bytes.length,
1505
+ sha256: createHash2("sha256").update(bytes).digest("hex")
1506
+ };
1507
+ }
1508
+
1383
1509
  // src/agent-task/flow/view.ts
1384
1510
  var FlowView = class {
1385
1511
  constructor(flow) {
@@ -1629,6 +1755,8 @@ function observationTypeFor(t) {
1629
1755
  return "EMBEDDING";
1630
1756
  case "GUARDRAIL":
1631
1757
  return "GUARDRAIL";
1758
+ case "SKILL":
1759
+ return "SKILL";
1632
1760
  default:
1633
1761
  return "SPAN";
1634
1762
  }
@@ -1975,369 +2103,222 @@ async function readCanonicalSnapshot(trace2) {
1975
2103
  return null;
1976
2104
  }
1977
2105
  }
1978
- async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
2106
+ async function executeAdapterRun(loaded, options, rawTrace, continueWith) {
1979
2107
  const {
1980
2108
  task: task2,
1981
2109
  adapter,
1982
2110
  taskDir: absoluteDir,
1983
2111
  files,
1984
2112
  checksPath,
1985
- inlineChecks,
1986
- moduleUrl,
1987
- evalFileName
2113
+ inlineChecks
1988
2114
  } = loaded;
1989
- const taskFiles = new TaskFiles(files);
1990
2115
  const tee = createProjectionTee(rawTrace);
1991
- const { trace: trace2, getSnapshot } = { trace: tee.trace, getSnapshot: () => tee.getSnapshot() };
2116
+ const trace2 = tee.trace;
1992
2117
  return withApoRun(
1993
2118
  { trace: trace2, parentSpanId: rawTrace.rootSpanId, taskId: task2.id },
1994
2119
  async () => {
1995
- return executeRunBody();
1996
- }
1997
- );
1998
- async function executeRunBody() {
1999
- let state;
2000
- let session;
2001
- const transcriptTurns = [];
2002
- try {
2003
- await trace2.step(
2004
- {
2005
- step_name: "task.load",
2006
- input: { taskId: task2.id, taskDir: absoluteDir },
2007
- metadata: {
2008
- taskId: task2.id,
2009
- taskDir: absoluteDir,
2010
- fileCount: files.length,
2011
- hasChecks: inlineChecks || checksPath !== null
2012
- },
2013
- summarize: () => ({
2014
- taskId: task2.id,
2015
- adapterName: adapter.name
2016
- })
2017
- },
2018
- async () => loaded
2019
- );
2020
- if (adapter.initialize) {
2021
- const initState = await trace2.step(
2022
- {
2023
- step_name: "adapter.initialize",
2024
- input: { adapterName: adapter.name },
2025
- metadata: { adapterName: adapter.name },
2026
- summarize: (result) => ({
2027
- initialized: true,
2028
- stateKeys: result && typeof result === "object" ? Object.keys(result) : []
2029
- })
2030
- },
2031
- async () => adapter.initialize?.({
2032
- task: task2,
2033
- taskDir: absoluteDir,
2034
- files,
2035
- trace: trace2
2036
- })
2037
- );
2038
- state = initState ?? void 0;
2039
- }
2040
- session = await trace2.step(
2041
- {
2042
- step_name: "adapter.open-session",
2043
- input: { adapterName: adapter.name, hasState: state !== void 0 },
2044
- metadata: { adapterName: adapter.name },
2045
- summarize: () => ({ sessionOpened: true })
2046
- },
2047
- async () => adapter.startSession({
2048
- task: task2,
2049
- taskDir: absoluteDir,
2050
- files,
2051
- state,
2052
- trace: trace2
2053
- })
2054
- );
2055
- const runConfiguration = normalizeRunConfiguration(session.runConfiguration);
2120
+ let state;
2121
+ let session;
2056
2122
  try {
2057
- if (!inlineChecks && checksPath) {
2058
- resetTaskTurn();
2059
- resetFlowChecks();
2060
- await loadChecksModule(checksPath);
2061
- }
2062
- const taskTurn = getTaskTurn();
2063
- resetTaskTurn();
2064
- const turnFn = resolveTurn(adapter.turn, taskTurn);
2065
- if (turnFn) {
2066
- const turnTranscript = [];
2067
- let lastTurnResponse;
2068
- const maxTurns = options?.maxTurnsOverride ?? task2.maxTurns ?? 10;
2069
- for (let turnNum = 1; turnNum <= maxTurns; turnNum++) {
2070
- const userTurn = await turnFn({ files: taskFiles, transcript: turnTranscript });
2071
- if (userTurn === null || userTurn === void 0) break;
2072
- const result2 = await trace2.step(
2073
- {
2074
- step_name: "task.turn",
2075
- metadata: { turnNumber: turnNum },
2076
- summarize: (r) => ({ response: r.response })
2077
- },
2078
- async (spanId) => {
2079
- if (!session) throw new Error("Session not created");
2080
- return session.sendUserTurn(userTurn, {
2081
- trace: trace2,
2082
- turnNumber: turnNum,
2083
- parentSpanId: spanId
2084
- });
2085
- }
2086
- );
2087
- turnTranscript.push({
2088
- turnNumber: turnNum,
2089
- input: userTurn,
2090
- output: result2.response
2091
- });
2092
- transcriptTurns.push({
2093
- turnNumber: turnNum,
2094
- userAction: userTurn,
2095
- agentResponse: result2.response
2096
- });
2097
- options?.onTurn?.(turnNum, userTurn, result2.response);
2098
- lastTurnResponse = result2.response;
2099
- }
2100
- if (lastTurnResponse !== void 0) {
2101
- rawTrace.endRoot({ output: { response: lastTurnResponse } });
2102
- }
2103
- }
2104
- const collected = await trace2.step(
2123
+ await trace2.step(
2105
2124
  {
2106
- step_name: "adapter.collect-deliverables",
2107
- input: { adapterName: adapter.name, expectedDeliverables: task2.deliverables },
2108
- metadata: { adapterName: adapter.name },
2109
- summarize: (result2) => {
2110
- const keys = result2 && typeof result2 === "object" ? Object.keys(result2) : [];
2111
- return { deliverableCount: keys.length, deliverableNames: keys };
2112
- }
2113
- },
2114
- async () => {
2115
- if (!session) {
2116
- throw new Error("Adapter session was not created");
2117
- }
2118
- return adapter.collectDeliverables({
2119
- task: task2,
2125
+ step_name: "task.load",
2126
+ input: { taskId: task2.id, taskDir: absoluteDir },
2127
+ metadata: {
2128
+ taskId: task2.id,
2120
2129
  taskDir: absoluteDir,
2121
- files,
2122
- state,
2123
- session,
2124
- trace: trace2
2125
- });
2126
- }
2127
- );
2128
- const validationResults = await trace2.step(
2129
- {
2130
- step_name: "deliverables.validate",
2131
- input: { deliverableNames: task2.deliverables },
2132
- summarize: (result2) => {
2133
- const r = result2;
2134
- const passCount = r.results.filter((x) => x.pass).length;
2135
- const broken = Object.keys(r.brokenDeliverables);
2136
- return {
2137
- total: r.results.length,
2138
- passCount,
2139
- failCount: r.results.length - passCount,
2140
- brokenDeliverableCount: broken.length,
2141
- brokenDeliverables: broken
2142
- };
2143
- }
2144
- },
2145
- async () => validateDeliverables(task2, collected, adapter.deliverables)
2146
- );
2147
- const checksResults = await trace2.step(
2148
- {
2149
- step_name: "checks.run",
2150
- input: { sourceFile: inlineChecks ? evalFileName : checksPath },
2151
- metadata: { sourceFile: inlineChecks ? evalFileName : checksPath },
2152
- summarize: (result2) => summarizeEvaluationResults(result2)
2130
+ fileCount: files.length,
2131
+ hasChecks: inlineChecks || checksPath !== null
2132
+ },
2133
+ summarize: () => ({
2134
+ taskId: task2.id,
2135
+ adapterName: adapter.name
2136
+ })
2153
2137
  },
2154
- async () => {
2155
- const judgeConfig = resolveJudgeConfig(options?.judge, task2.judge);
2156
- if (!inlineChecks) {
2157
- return loadAndRunFlowChecks(
2158
- checksPath,
2159
- {
2160
- snapshot: getSnapshot(),
2161
- deliverables: collected,
2162
- files,
2163
- task: task2,
2164
- ...judgeConfig ? { judgeConfig } : {}
2165
- },
2166
- validationResults.brokenDeliverables
2167
- );
2168
- }
2169
- return runTraceChecks({
2170
- snapshot: getSnapshot(),
2171
- deliverables: proxyBrokenDeliverables(
2172
- collected,
2173
- validationResults.brokenDeliverables
2174
- ),
2175
- files,
2176
- task: task2,
2177
- ...judgeConfig ? { judgeConfig } : {},
2178
- moduleUrl,
2179
- displayFile: evalFileName
2180
- });
2181
- }
2138
+ async () => loaded
2182
2139
  );
2183
- const result = aggregateResult(checksResults);
2184
- return {
2185
- task: task2,
2186
- taskDir: absoluteDir,
2187
- files,
2188
- traceRunId,
2189
- result,
2190
- deliverables: collected,
2191
- transcript: { turns: transcriptTurns },
2192
- runConfiguration
2193
- };
2194
- } catch (error) {
2195
- throw withRunConfiguration(error, runConfiguration);
2196
- }
2197
- } finally {
2198
- if (adapter.cleanup) {
2199
- try {
2200
- await trace2.step(
2140
+ if (adapter.initialize) {
2141
+ const initState = await trace2.step(
2201
2142
  {
2202
- step_name: "adapter.cleanup",
2143
+ step_name: "adapter.initialize",
2203
2144
  input: { adapterName: adapter.name },
2204
2145
  metadata: { adapterName: adapter.name },
2205
- summarize: () => ({ cleaned: true })
2146
+ summarize: (result) => ({
2147
+ initialized: true,
2148
+ stateKeys: result && typeof result === "object" ? Object.keys(result) : []
2149
+ })
2206
2150
  },
2207
- async () => adapter.cleanup?.({
2151
+ async () => adapter.initialize?.({
2208
2152
  task: task2,
2209
2153
  taskDir: absoluteDir,
2210
2154
  files,
2211
- state,
2212
- session,
2213
2155
  trace: trace2
2214
2156
  })
2215
2157
  );
2216
- } catch (error) {
2217
- console.error(
2218
- "[AgentTask] Cleanup failed:",
2219
- error instanceof Error ? error.message : String(error)
2220
- );
2221
- }
2222
- }
2223
- if (session?.close) {
2224
- try {
2225
- await session.close();
2226
- } catch {
2227
- }
2228
- }
2229
- }
2230
- }
2231
- }
2232
- async function captureExecution(loaded, options, rawTrace) {
2233
- const {
2234
- task: task2,
2235
- adapter,
2236
- taskDir: absoluteDir,
2237
- files,
2238
- checksPath,
2239
- inlineChecks
2240
- } = loaded;
2241
- const taskFiles = new TaskFiles(files);
2242
- const tee = createProjectionTee(rawTrace);
2243
- const trace2 = tee.trace;
2244
- return withApoRun(
2245
- { trace: trace2, parentSpanId: rawTrace.rootSpanId, taskId: task2.id },
2246
- async () => {
2247
- let state;
2248
- let session;
2249
- const transcriptTurns = [];
2250
- try {
2251
- await trace2.step(
2252
- { step_name: "task.load", input: { taskId: task2.id, taskDir: absoluteDir }, metadata: { taskId: task2.id, taskDir: absoluteDir, fileCount: files.length, hasChecks: inlineChecks || checksPath !== null }, summarize: () => ({ taskId: task2.id, adapterName: adapter.name }) },
2253
- async () => loaded
2254
- );
2255
- if (adapter.initialize) {
2256
- const initState = await trace2.step(
2257
- { step_name: "adapter.initialize", input: { adapterName: adapter.name }, metadata: { adapterName: adapter.name }, summarize: (result) => ({ initialized: true, stateKeys: result && typeof result === "object" ? Object.keys(result) : [] }) },
2258
- async () => adapter.initialize?.({ task: task2, taskDir: absoluteDir, files, trace: trace2 })
2259
- );
2260
2158
  state = initState ?? void 0;
2261
2159
  }
2262
2160
  session = await trace2.step(
2263
- { step_name: "adapter.open-session", input: { adapterName: adapter.name, hasState: state !== void 0 }, metadata: { adapterName: adapter.name }, summarize: () => ({ sessionOpened: true }) },
2264
- async () => adapter.startSession({ task: task2, taskDir: absoluteDir, files, state, trace: trace2 })
2161
+ {
2162
+ step_name: "adapter.open-session",
2163
+ input: { adapterName: adapter.name, hasState: state !== void 0 },
2164
+ metadata: { adapterName: adapter.name },
2165
+ summarize: () => ({ sessionOpened: true })
2166
+ },
2167
+ async () => adapter.startSession({
2168
+ task: task2,
2169
+ taskDir: absoluteDir,
2170
+ files,
2171
+ state,
2172
+ trace: trace2
2173
+ })
2174
+ );
2175
+ const runConfiguration = normalizeRunConfiguration(
2176
+ session.runConfiguration
2265
2177
  );
2266
- const runConfiguration = normalizeRunConfiguration(session.runConfiguration);
2267
2178
  try {
2268
- if (!inlineChecks && checksPath) {
2269
- resetTaskTurn();
2270
- resetFlowChecks();
2271
- await loadChecksModule(checksPath);
2272
- }
2273
- const taskTurn = getTaskTurn();
2274
- resetTaskTurn();
2275
- const turnFn = resolveTurn(adapter.turn, taskTurn);
2276
- if (turnFn) {
2277
- const turnTranscript = [];
2278
- let lastTurnResponse;
2279
- const maxTurns = options?.maxTurnsOverride ?? task2.maxTurns ?? 10;
2280
- for (let turnNum = 1; turnNum <= maxTurns; turnNum++) {
2281
- const userTurn = await turnFn({ files: taskFiles, transcript: turnTranscript });
2282
- if (userTurn === null || userTurn === void 0) break;
2283
- const result = await trace2.step(
2284
- { step_name: "task.turn", metadata: { turnNumber: turnNum }, summarize: (r) => ({ response: r.response }) },
2285
- async (spanId) => {
2286
- if (!session) throw new Error("Session not created");
2287
- return session.sendUserTurn(userTurn, { trace: trace2, turnNumber: turnNum, parentSpanId: spanId });
2288
- }
2289
- );
2290
- turnTranscript.push({ turnNumber: turnNum, input: userTurn, output: result.response });
2291
- transcriptTurns.push({ turnNumber: turnNum, userAction: userTurn, agentResponse: result.response });
2292
- options?.onTurn?.(turnNum, userTurn, result.response);
2293
- lastTurnResponse = result.response;
2294
- }
2295
- if (lastTurnResponse !== void 0) {
2296
- rawTrace.endRoot({ output: { response: lastTurnResponse } });
2297
- }
2298
- }
2179
+ const transcriptTurns = await runTurnLoop(
2180
+ loaded,
2181
+ options,
2182
+ session,
2183
+ trace2,
2184
+ rawTrace
2185
+ );
2299
2186
  const collected = await trace2.step(
2300
- { step_name: "adapter.collect-deliverables", input: { adapterName: adapter.name, expectedDeliverables: task2.deliverables }, metadata: { adapterName: adapter.name }, summarize: (result) => {
2301
- const keys = result && typeof result === "object" ? Object.keys(result) : [];
2302
- return { deliverableCount: keys.length, deliverableNames: keys };
2303
- } },
2187
+ {
2188
+ step_name: "adapter.collect-deliverables",
2189
+ input: {
2190
+ adapterName: adapter.name,
2191
+ expectedDeliverables: task2.deliverables
2192
+ },
2193
+ metadata: { adapterName: adapter.name },
2194
+ summarize: (result) => {
2195
+ const keys = result && typeof result === "object" ? Object.keys(result) : [];
2196
+ return {
2197
+ deliverableCount: keys.length,
2198
+ deliverableNames: keys
2199
+ };
2200
+ }
2201
+ },
2304
2202
  async () => {
2305
- if (!session) throw new Error("Adapter session was not created");
2306
- return adapter.collectDeliverables({ task: task2, taskDir: absoluteDir, files, state, session, trace: trace2 });
2203
+ if (!session) {
2204
+ throw new Error("Adapter session was not created");
2205
+ }
2206
+ return adapter.collectDeliverables({
2207
+ task: task2,
2208
+ taskDir: absoluteDir,
2209
+ files,
2210
+ state,
2211
+ session,
2212
+ trace: trace2
2213
+ });
2307
2214
  }
2308
2215
  );
2309
- const snapshot = tee.getSnapshot();
2310
- return {
2311
- traceRunId: rawTrace.runId,
2216
+ return await continueWith({
2312
2217
  collected,
2313
2218
  transcriptTurns,
2314
2219
  runConfiguration,
2315
- snapshot
2316
- };
2220
+ trace: trace2,
2221
+ getSnapshot: () => tee.getSnapshot()
2222
+ });
2317
2223
  } catch (error) {
2318
2224
  throw withRunConfiguration(error, runConfiguration);
2319
2225
  }
2320
2226
  } finally {
2321
- if (adapter.cleanup) {
2322
- try {
2323
- await trace2.step(
2324
- { step_name: "adapter.cleanup", input: { adapterName: adapter.name }, metadata: { adapterName: adapter.name }, summarize: () => ({ cleaned: true }) },
2325
- async () => adapter.cleanup?.({ task: task2, taskDir: absoluteDir, files, state, session, trace: trace2 })
2326
- );
2327
- } catch (error) {
2328
- console.error("[AgentTask] Cleanup failed:", error instanceof Error ? error.message : String(error));
2329
- }
2330
- }
2331
- if (session?.close) {
2332
- try {
2333
- await session.close();
2334
- } catch {
2335
- }
2336
- }
2227
+ await cleanupAdapter(loaded, trace2, state, session);
2337
2228
  }
2338
2229
  }
2339
2230
  );
2340
2231
  }
2232
+ async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
2233
+ return executeAdapterRun(loaded, options, rawTrace, async (outcome) => {
2234
+ const {
2235
+ task: task2,
2236
+ adapter,
2237
+ taskDir: absoluteDir,
2238
+ files,
2239
+ checksPath,
2240
+ inlineChecks,
2241
+ moduleUrl,
2242
+ evalFileName
2243
+ } = loaded;
2244
+ const { trace: trace2, collected, transcriptTurns, runConfiguration } = outcome;
2245
+ const validationResults = await trace2.step(
2246
+ {
2247
+ step_name: "deliverables.validate",
2248
+ input: { deliverableNames: task2.deliverables },
2249
+ summarize: (result) => {
2250
+ const r = result;
2251
+ const passCount = r.results.filter((x) => x.pass).length;
2252
+ const broken = Object.keys(r.brokenDeliverables);
2253
+ return {
2254
+ total: r.results.length,
2255
+ passCount,
2256
+ failCount: r.results.length - passCount,
2257
+ brokenDeliverableCount: broken.length,
2258
+ brokenDeliverables: broken
2259
+ };
2260
+ }
2261
+ },
2262
+ async () => validateDeliverables(task2, collected, adapter.deliverables)
2263
+ );
2264
+ const checksResults = await trace2.step(
2265
+ {
2266
+ step_name: "checks.run",
2267
+ input: { sourceFile: inlineChecks ? evalFileName : checksPath },
2268
+ metadata: { sourceFile: inlineChecks ? evalFileName : checksPath },
2269
+ summarize: (result) => summarizeEvaluationResults(result)
2270
+ },
2271
+ async () => {
2272
+ const judgeConfig = resolveJudgeConfig(options?.judge, task2.judge);
2273
+ if (!inlineChecks) {
2274
+ return loadAndRunFlowChecks(
2275
+ checksPath,
2276
+ {
2277
+ snapshot: outcome.getSnapshot(),
2278
+ deliverables: collected,
2279
+ files,
2280
+ task: task2,
2281
+ ...judgeConfig ? { judgeConfig } : {}
2282
+ },
2283
+ validationResults.brokenDeliverables
2284
+ );
2285
+ }
2286
+ return runTraceChecks({
2287
+ snapshot: outcome.getSnapshot(),
2288
+ deliverables: proxyBrokenDeliverables(
2289
+ collected,
2290
+ validationResults.brokenDeliverables
2291
+ ),
2292
+ files,
2293
+ task: task2,
2294
+ ...judgeConfig ? { judgeConfig } : {},
2295
+ moduleUrl,
2296
+ displayFile: evalFileName
2297
+ });
2298
+ }
2299
+ );
2300
+ return {
2301
+ task: task2,
2302
+ taskDir: absoluteDir,
2303
+ files,
2304
+ traceRunId,
2305
+ result: aggregateResult(checksResults),
2306
+ deliverables: collected,
2307
+ transcript: { turns: transcriptTurns },
2308
+ runConfiguration
2309
+ };
2310
+ });
2311
+ }
2312
+ async function captureExecution(loaded, options, rawTrace) {
2313
+ return executeAdapterRun(loaded, options, rawTrace, async (outcome) => ({
2314
+ traceRunId: rawTrace.runId,
2315
+ collected: outcome.collected,
2316
+ transcriptTurns: outcome.transcriptTurns,
2317
+ runConfiguration: outcome.runConfiguration,
2318
+ // Freeze the snapshot before the root span ends — Phase 2 reads it.
2319
+ snapshot: outcome.getSnapshot()
2320
+ }));
2321
+ }
2341
2322
  async function evaluate(loaded, options, phase1) {
2342
2323
  const {
2343
2324
  task: task2,
@@ -2386,6 +2367,89 @@ async function evaluate(loaded, options, phase1) {
2386
2367
  runConfiguration: phase1.runConfiguration
2387
2368
  };
2388
2369
  }
2370
+ async function runTurnLoop(loaded, options, session, trace2, rawTrace) {
2371
+ const { task: task2, adapter, files, checksPath, inlineChecks } = loaded;
2372
+ if (!inlineChecks && checksPath) {
2373
+ resetTaskTurn();
2374
+ resetFlowChecks();
2375
+ await loadChecksModule(checksPath);
2376
+ }
2377
+ const taskTurn = getTaskTurn();
2378
+ resetTaskTurn();
2379
+ const turnFn = resolveTurn(adapter.turn, taskTurn);
2380
+ if (!turnFn) return [];
2381
+ const taskFiles = new TaskFiles(files);
2382
+ const turnTranscript = [];
2383
+ const transcriptTurns = [];
2384
+ let lastTurnResponse;
2385
+ const maxTurns = options?.maxTurnsOverride ?? task2.maxTurns ?? 10;
2386
+ for (let turnNum = 1; turnNum <= maxTurns; turnNum++) {
2387
+ const userTurn = await turnFn({ files: taskFiles, transcript: turnTranscript });
2388
+ if (userTurn === null || userTurn === void 0) break;
2389
+ const result = await trace2.step(
2390
+ {
2391
+ step_name: "task.turn",
2392
+ metadata: { turnNumber: turnNum },
2393
+ summarize: (r) => ({ response: r.response })
2394
+ },
2395
+ async (spanId) => session.sendUserTurn(userTurn, {
2396
+ trace: trace2,
2397
+ turnNumber: turnNum,
2398
+ parentSpanId: spanId
2399
+ })
2400
+ );
2401
+ turnTranscript.push({
2402
+ turnNumber: turnNum,
2403
+ input: userTurn,
2404
+ output: result.response
2405
+ });
2406
+ transcriptTurns.push({
2407
+ turnNumber: turnNum,
2408
+ userAction: userTurn,
2409
+ agentResponse: result.response
2410
+ });
2411
+ options?.onTurn?.(turnNum, userTurn, result.response);
2412
+ lastTurnResponse = result.response;
2413
+ }
2414
+ if (lastTurnResponse !== void 0) {
2415
+ rawTrace.endRoot({ output: { response: lastTurnResponse } });
2416
+ }
2417
+ return transcriptTurns;
2418
+ }
2419
+ async function cleanupAdapter(loaded, trace2, state, session) {
2420
+ const { task: task2, adapter, taskDir: absoluteDir, files } = loaded;
2421
+ if (adapter.cleanup) {
2422
+ try {
2423
+ await trace2.step(
2424
+ {
2425
+ step_name: "adapter.cleanup",
2426
+ input: { adapterName: adapter.name },
2427
+ metadata: { adapterName: adapter.name },
2428
+ summarize: () => ({ cleaned: true })
2429
+ },
2430
+ async () => adapter.cleanup?.({
2431
+ task: task2,
2432
+ taskDir: absoluteDir,
2433
+ files,
2434
+ state,
2435
+ session,
2436
+ trace: trace2
2437
+ })
2438
+ );
2439
+ } catch (error) {
2440
+ console.error(
2441
+ "[AgentTask] Cleanup failed:",
2442
+ error instanceof Error ? error.message : String(error)
2443
+ );
2444
+ }
2445
+ }
2446
+ if (session?.close) {
2447
+ try {
2448
+ await session.close();
2449
+ } catch {
2450
+ }
2451
+ }
2452
+ }
2389
2453
  function summarizeEvaluationResults(results) {
2390
2454
  const passCount = results.filter((r) => r.pass).length;
2391
2455
  return {
@@ -3301,12 +3365,12 @@ function printOverallSummary(summaries, hasErrors) {
3301
3365
  }
3302
3366
 
3303
3367
  // src/agent-task/task-revision-manifest.ts
3304
- import { createHash as createHash2 } from "crypto";
3368
+ import { createHash as createHash3 } from "crypto";
3305
3369
  function normalizeManifestPath(path) {
3306
3370
  return path.split("\\").join("/").normalize("NFC");
3307
3371
  }
3308
3372
  function sha256Hex(bytes) {
3309
- return createHash2("sha256").update(bytes).digest("hex");
3373
+ return createHash3("sha256").update(bytes).digest("hex");
3310
3374
  }
3311
3375
  function compareBytewiseUtf8(a, b) {
3312
3376
  const ba = Buffer.from(a, "utf8");
@@ -3354,11 +3418,14 @@ export {
3354
3418
  AgentTaskRunError,
3355
3419
  ApoSpanProcessor,
3356
3420
  FlowView,
3421
+ JUDGE_SEGMENT_LIMIT,
3422
+ RECEIVED_VALUE_LIMIT,
3357
3423
  RejudgeError,
3358
3424
  TEST_METHOD_NAMES,
3359
3425
  TaskFiles,
3360
3426
  buildManifest,
3361
3427
  canonicalManifestJson,
3428
+ compactChecksForSubmission,
3362
3429
  contentSha256,
3363
3430
  createApoAnthropic,
3364
3431
  createApoOpenAI,