@apo-ai/sdk 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,14 +8,14 @@ import {
8
8
  resetApoTracing,
9
9
  withApoRun,
10
10
  withApoRunSync
11
- } from "../chunk-PMYLL2H4.js";
11
+ } from "../chunk-GETY34VI.js";
12
12
  import {
13
13
  createApoTracer
14
- } from "../chunk-HKLBPIB4.js";
14
+ } from "../chunk-H7ITLBWU.js";
15
15
  import "../chunk-HWZVL6TS.js";
16
16
  import {
17
17
  createApoOpenAI
18
- } from "../chunk-DT2LMB7O.js";
18
+ } from "../chunk-6ZRW5IW6.js";
19
19
  import {
20
20
  createApoAnthropic
21
21
  } from "../chunk-SZDSIPXC.js";
@@ -235,8 +235,21 @@ function levenshtein(a, b) {
235
235
  }
236
236
 
237
237
  // src/agent-task/checks/judge.ts
238
- var JUDGE_RESPONSE_CONTRACT = 'Respond with ONLY a JSON object: {"pass": true/false, "reasoning": "your reasoning"}';
239
- var JUDGE_SYSTEM_PROMPT = `You are an evaluation judge. Evaluate the given value(s) against the instruction. ${JUDGE_RESPONSE_CONTRACT}`;
238
+ var VERDICT_FIRST_CONTRACT = 'Respond with ONLY a JSON object: {"pass": true/false, "reasoning": "your reasoning"}';
239
+ var REASONING_FIRST_CONTRACT = 'Respond with ONLY a JSON object: {"reasoning": "your reasoning", "pass": true/false}';
240
+ function isJudgeVerdictFirstOverrideEnabled() {
241
+ const value = process.env.APO_JUDGE_VERDICT_FIRST?.trim().toLowerCase();
242
+ return value === "1" || value === "true";
243
+ }
244
+ function judgeResponseContract() {
245
+ return isJudgeVerdictFirstOverrideEnabled() ? VERDICT_FIRST_CONTRACT : REASONING_FIRST_CONTRACT;
246
+ }
247
+ function judgeContractInUse() {
248
+ return isJudgeVerdictFirstOverrideEnabled() ? "verdict-first" : "reasoning-first";
249
+ }
250
+ function judgeSystemPrompt() {
251
+ return `You are an evaluation judge. Evaluate the given value(s) against the instruction. ${judgeResponseContract()}`;
252
+ }
240
253
  function formatValue(value, depth = 0) {
241
254
  const indent = " ".repeat(depth);
242
255
  if (value === null || value === void 0) return String(value);
@@ -297,6 +310,7 @@ function parseJudgeJson(raw) {
297
310
  reasoning: "Judge response could not be parsed as JSON \u2014 the verdict is unknown, so this check is treated as a failure. The model's raw response is available in the judge metadata."
298
311
  };
299
312
  }
313
+ var JUDGE_TIMEOUT_MS = 12e4;
300
314
  var prefixQueues = /* @__PURE__ */ new Map();
301
315
  function runWithSharedPrefix(key, task2) {
302
316
  const prev = prefixQueues.get(key) ?? Promise.resolve();
@@ -324,6 +338,7 @@ ${deliverableText}`;
324
338
  const startedAt = Date.now();
325
339
  const response = await fetch(`${baseURL}/chat/completions`, {
326
340
  method: "POST",
341
+ signal: AbortSignal.timeout(JUDGE_TIMEOUT_MS),
327
342
  headers: {
328
343
  "Content-Type": "application/json",
329
344
  ...apiKey ? { Authorization: `Bearer ${apiKey}` } : {}
@@ -363,6 +378,7 @@ ${deliverableText}`;
363
378
  reasoning: "Judge returned an empty or truncated response \u2014 likely a transient provider failure. The verdict is unknown, so this check is treated as a failure.",
364
379
  judge: {
365
380
  model: args.model,
381
+ contract: judgeContractInUse(),
366
382
  prompt: { system: systemPromptText, user: instructionText },
367
383
  response: text,
368
384
  tokens: parseJudgeUsage(data.usage),
@@ -376,6 +392,7 @@ ${deliverableText}`;
376
392
  reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : "",
377
393
  judge: {
378
394
  model: args.model,
395
+ contract: judgeContractInUse(),
379
396
  prompt: { system: systemPromptText, user: instructionText },
380
397
  response: text,
381
398
  tokens: parseJudgeUsage(data.usage),
@@ -387,7 +404,7 @@ ${deliverableText}`;
387
404
  function assembleBriefing(args) {
388
405
  if (!args.prompt) {
389
406
  return {
390
- briefingText: JUDGE_SYSTEM_PROMPT,
407
+ briefingText: judgeSystemPrompt(),
391
408
  instructionText: `Instruction:
392
409
  ${args.instruction}`
393
410
  };
@@ -406,7 +423,7 @@ ${args.instruction}`
406
423
  // The SDK appends its own response contract to any custom briefing.
407
424
  briefingText: system ? `${system}
408
425
 
409
- ${JUDGE_RESPONSE_CONTRACT}` : JUDGE_SYSTEM_PROMPT,
426
+ ${judgeResponseContract()}` : judgeSystemPrompt(),
410
427
  instructionText: user ?? `Instruction:
411
428
  ${args.instruction}`
412
429
  };
@@ -876,8 +893,8 @@ async function runTraceChecks(args) {
876
893
  ...rec.all.length > 0 ? { assertions: rec.all.map((a) => ({ ...a })) } : {},
877
894
  ...check.group_id ? { group_id: check.group_id } : {},
878
895
  ...check.group_name ? { group_name: check.group_name } : {},
879
- // stamp the snapshot source so consumers can detect
880
- // the deprecated legacy-flow compatibility path (source="legacy-flow").
896
+ // stamp the snapshot source so consumers can tell projection-first
897
+ // results (canonical) from locally recorded snapshots.
881
898
  ...args.snapshot.source !== "canonical" ? { source: args.snapshot.source } : {}
882
899
  };
883
900
  })
@@ -1380,6 +1397,115 @@ async function safeDetail(response) {
1380
1397
  }
1381
1398
  }
1382
1399
 
1400
+ // src/agent-task/checks/compact.ts
1401
+ import { createHash as createHash2 } from "crypto";
1402
+ var RECEIVED_VALUE_LIMIT = 4 * 1024;
1403
+ var JUDGE_SEGMENT_LIMIT = 16 * 1024;
1404
+ var PREVIEW_CHARS = 256;
1405
+ function compactChecksForSubmission(checks) {
1406
+ let truncatedValues = 0;
1407
+ let truncatedSegments = 0;
1408
+ const compacted = checks.map((check) => {
1409
+ const next = { ...check };
1410
+ if ("received" in next && next.received !== void 0) {
1411
+ const replaced = truncateValue(next.received);
1412
+ if (replaced !== next.received) {
1413
+ next.received = replaced;
1414
+ truncatedValues += 1;
1415
+ }
1416
+ }
1417
+ if (next.judge !== void 0) {
1418
+ const { value, truncated } = truncateJudge(next.judge);
1419
+ if (value !== next.judge) {
1420
+ next.judge = value;
1421
+ truncatedSegments += truncated;
1422
+ }
1423
+ }
1424
+ const sourceAssertions = check.assertions;
1425
+ if (sourceAssertions) {
1426
+ const assertions = sourceAssertions.map((assertion) => {
1427
+ const nextAssertion = { ...assertion };
1428
+ if (nextAssertion.received !== void 0) {
1429
+ const replaced = truncateValue(nextAssertion.received);
1430
+ if (replaced !== nextAssertion.received) {
1431
+ nextAssertion.received = replaced;
1432
+ truncatedValues += 1;
1433
+ }
1434
+ }
1435
+ if (nextAssertion.judge !== void 0) {
1436
+ const { value, truncated } = truncateJudge(nextAssertion.judge);
1437
+ if (value !== nextAssertion.judge) {
1438
+ nextAssertion.judge = value;
1439
+ truncatedSegments += truncated;
1440
+ }
1441
+ }
1442
+ return nextAssertion;
1443
+ });
1444
+ if (assertions.some((a, i) => a !== sourceAssertions[i])) {
1445
+ next.assertions = assertions;
1446
+ }
1447
+ }
1448
+ return next;
1449
+ });
1450
+ return { checks: compacted, truncatedValues, truncatedSegments };
1451
+ }
1452
+ function truncateJudge(judge) {
1453
+ if (typeof judge !== "object" || judge === null) {
1454
+ return { value: judge, truncated: 0 };
1455
+ }
1456
+ let truncated = 0;
1457
+ const next = { ...judge };
1458
+ const prompt = next.prompt;
1459
+ if (typeof prompt === "object" && prompt !== null) {
1460
+ const nextPrompt = { ...prompt };
1461
+ for (const key of ["system", "user"]) {
1462
+ const value = nextPrompt[key];
1463
+ if (typeof value === "string") {
1464
+ const replaced = truncateText(value);
1465
+ if (replaced !== value) {
1466
+ nextPrompt[key] = replaced;
1467
+ truncated += 1;
1468
+ }
1469
+ }
1470
+ }
1471
+ next.prompt = nextPrompt;
1472
+ }
1473
+ if (typeof next.response === "string") {
1474
+ const replaced = truncateText(next.response);
1475
+ if (replaced !== next.response) {
1476
+ next.response = replaced;
1477
+ truncated += 1;
1478
+ }
1479
+ }
1480
+ return { value: next, truncated };
1481
+ }
1482
+ function truncateValue(value) {
1483
+ if (value === void 0) return value;
1484
+ let bytes;
1485
+ try {
1486
+ bytes = new TextEncoder().encode(JSON.stringify(value));
1487
+ } catch {
1488
+ bytes = new TextEncoder().encode(JSON.stringify(String(value)));
1489
+ }
1490
+ if (bytes.length <= RECEIVED_VALUE_LIMIT) return value;
1491
+ return marker(bytes);
1492
+ }
1493
+ function truncateText(value) {
1494
+ const bytes = new TextEncoder().encode(value);
1495
+ if (bytes.length <= JUDGE_SEGMENT_LIMIT) return value;
1496
+ return marker(bytes);
1497
+ }
1498
+ function marker(bytes) {
1499
+ const previewBytes = bytes.slice(0, PREVIEW_CHARS);
1500
+ const preview = Buffer.from(previewBytes).toString("utf8");
1501
+ return {
1502
+ kind: "truncated",
1503
+ preview,
1504
+ size_bytes: bytes.length,
1505
+ sha256: createHash2("sha256").update(bytes).digest("hex")
1506
+ };
1507
+ }
1508
+
1383
1509
  // src/agent-task/flow/view.ts
1384
1510
  var FlowView = class {
1385
1511
  constructor(flow) {
@@ -1977,369 +2103,222 @@ async function readCanonicalSnapshot(trace2) {
1977
2103
  return null;
1978
2104
  }
1979
2105
  }
1980
- async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
2106
+ async function executeAdapterRun(loaded, options, rawTrace, continueWith) {
1981
2107
  const {
1982
2108
  task: task2,
1983
2109
  adapter,
1984
2110
  taskDir: absoluteDir,
1985
2111
  files,
1986
2112
  checksPath,
1987
- inlineChecks,
1988
- moduleUrl,
1989
- evalFileName
2113
+ inlineChecks
1990
2114
  } = loaded;
1991
- const taskFiles = new TaskFiles(files);
1992
2115
  const tee = createProjectionTee(rawTrace);
1993
- const { trace: trace2, getSnapshot } = { trace: tee.trace, getSnapshot: () => tee.getSnapshot() };
2116
+ const trace2 = tee.trace;
1994
2117
  return withApoRun(
1995
2118
  { trace: trace2, parentSpanId: rawTrace.rootSpanId, taskId: task2.id },
1996
2119
  async () => {
1997
- return executeRunBody();
1998
- }
1999
- );
2000
- async function executeRunBody() {
2001
- let state;
2002
- let session;
2003
- const transcriptTurns = [];
2004
- try {
2005
- await trace2.step(
2006
- {
2007
- step_name: "task.load",
2008
- input: { taskId: task2.id, taskDir: absoluteDir },
2009
- metadata: {
2010
- taskId: task2.id,
2011
- taskDir: absoluteDir,
2012
- fileCount: files.length,
2013
- hasChecks: inlineChecks || checksPath !== null
2014
- },
2015
- summarize: () => ({
2016
- taskId: task2.id,
2017
- adapterName: adapter.name
2018
- })
2019
- },
2020
- async () => loaded
2021
- );
2022
- if (adapter.initialize) {
2023
- const initState = await trace2.step(
2024
- {
2025
- step_name: "adapter.initialize",
2026
- input: { adapterName: adapter.name },
2027
- metadata: { adapterName: adapter.name },
2028
- summarize: (result) => ({
2029
- initialized: true,
2030
- stateKeys: result && typeof result === "object" ? Object.keys(result) : []
2031
- })
2032
- },
2033
- async () => adapter.initialize?.({
2034
- task: task2,
2035
- taskDir: absoluteDir,
2036
- files,
2037
- trace: trace2
2038
- })
2039
- );
2040
- state = initState ?? void 0;
2041
- }
2042
- session = await trace2.step(
2043
- {
2044
- step_name: "adapter.open-session",
2045
- input: { adapterName: adapter.name, hasState: state !== void 0 },
2046
- metadata: { adapterName: adapter.name },
2047
- summarize: () => ({ sessionOpened: true })
2048
- },
2049
- async () => adapter.startSession({
2050
- task: task2,
2051
- taskDir: absoluteDir,
2052
- files,
2053
- state,
2054
- trace: trace2
2055
- })
2056
- );
2057
- const runConfiguration = normalizeRunConfiguration(session.runConfiguration);
2120
+ let state;
2121
+ let session;
2058
2122
  try {
2059
- if (!inlineChecks && checksPath) {
2060
- resetTaskTurn();
2061
- resetFlowChecks();
2062
- await loadChecksModule(checksPath);
2063
- }
2064
- const taskTurn = getTaskTurn();
2065
- resetTaskTurn();
2066
- const turnFn = resolveTurn(adapter.turn, taskTurn);
2067
- if (turnFn) {
2068
- const turnTranscript = [];
2069
- let lastTurnResponse;
2070
- const maxTurns = options?.maxTurnsOverride ?? task2.maxTurns ?? 10;
2071
- for (let turnNum = 1; turnNum <= maxTurns; turnNum++) {
2072
- const userTurn = await turnFn({ files: taskFiles, transcript: turnTranscript });
2073
- if (userTurn === null || userTurn === void 0) break;
2074
- const result2 = await trace2.step(
2075
- {
2076
- step_name: "task.turn",
2077
- metadata: { turnNumber: turnNum },
2078
- summarize: (r) => ({ response: r.response })
2079
- },
2080
- async (spanId) => {
2081
- if (!session) throw new Error("Session not created");
2082
- return session.sendUserTurn(userTurn, {
2083
- trace: trace2,
2084
- turnNumber: turnNum,
2085
- parentSpanId: spanId
2086
- });
2087
- }
2088
- );
2089
- turnTranscript.push({
2090
- turnNumber: turnNum,
2091
- input: userTurn,
2092
- output: result2.response
2093
- });
2094
- transcriptTurns.push({
2095
- turnNumber: turnNum,
2096
- userAction: userTurn,
2097
- agentResponse: result2.response
2098
- });
2099
- options?.onTurn?.(turnNum, userTurn, result2.response);
2100
- lastTurnResponse = result2.response;
2101
- }
2102
- if (lastTurnResponse !== void 0) {
2103
- rawTrace.endRoot({ output: { response: lastTurnResponse } });
2104
- }
2105
- }
2106
- const collected = await trace2.step(
2123
+ await trace2.step(
2107
2124
  {
2108
- step_name: "adapter.collect-deliverables",
2109
- input: { adapterName: adapter.name, expectedDeliverables: task2.deliverables },
2110
- metadata: { adapterName: adapter.name },
2111
- summarize: (result2) => {
2112
- const keys = result2 && typeof result2 === "object" ? Object.keys(result2) : [];
2113
- return { deliverableCount: keys.length, deliverableNames: keys };
2114
- }
2115
- },
2116
- async () => {
2117
- if (!session) {
2118
- throw new Error("Adapter session was not created");
2119
- }
2120
- return adapter.collectDeliverables({
2121
- task: task2,
2125
+ step_name: "task.load",
2126
+ input: { taskId: task2.id, taskDir: absoluteDir },
2127
+ metadata: {
2128
+ taskId: task2.id,
2122
2129
  taskDir: absoluteDir,
2123
- files,
2124
- state,
2125
- session,
2126
- trace: trace2
2127
- });
2128
- }
2129
- );
2130
- const validationResults = await trace2.step(
2131
- {
2132
- step_name: "deliverables.validate",
2133
- input: { deliverableNames: task2.deliverables },
2134
- summarize: (result2) => {
2135
- const r = result2;
2136
- const passCount = r.results.filter((x) => x.pass).length;
2137
- const broken = Object.keys(r.brokenDeliverables);
2138
- return {
2139
- total: r.results.length,
2140
- passCount,
2141
- failCount: r.results.length - passCount,
2142
- brokenDeliverableCount: broken.length,
2143
- brokenDeliverables: broken
2144
- };
2145
- }
2146
- },
2147
- async () => validateDeliverables(task2, collected, adapter.deliverables)
2148
- );
2149
- const checksResults = await trace2.step(
2150
- {
2151
- step_name: "checks.run",
2152
- input: { sourceFile: inlineChecks ? evalFileName : checksPath },
2153
- metadata: { sourceFile: inlineChecks ? evalFileName : checksPath },
2154
- summarize: (result2) => summarizeEvaluationResults(result2)
2130
+ fileCount: files.length,
2131
+ hasChecks: inlineChecks || checksPath !== null
2132
+ },
2133
+ summarize: () => ({
2134
+ taskId: task2.id,
2135
+ adapterName: adapter.name
2136
+ })
2155
2137
  },
2156
- async () => {
2157
- const judgeConfig = resolveJudgeConfig(options?.judge, task2.judge);
2158
- if (!inlineChecks) {
2159
- return loadAndRunFlowChecks(
2160
- checksPath,
2161
- {
2162
- snapshot: getSnapshot(),
2163
- deliverables: collected,
2164
- files,
2165
- task: task2,
2166
- ...judgeConfig ? { judgeConfig } : {}
2167
- },
2168
- validationResults.brokenDeliverables
2169
- );
2170
- }
2171
- return runTraceChecks({
2172
- snapshot: getSnapshot(),
2173
- deliverables: proxyBrokenDeliverables(
2174
- collected,
2175
- validationResults.brokenDeliverables
2176
- ),
2177
- files,
2178
- task: task2,
2179
- ...judgeConfig ? { judgeConfig } : {},
2180
- moduleUrl,
2181
- displayFile: evalFileName
2182
- });
2183
- }
2138
+ async () => loaded
2184
2139
  );
2185
- const result = aggregateResult(checksResults);
2186
- return {
2187
- task: task2,
2188
- taskDir: absoluteDir,
2189
- files,
2190
- traceRunId,
2191
- result,
2192
- deliverables: collected,
2193
- transcript: { turns: transcriptTurns },
2194
- runConfiguration
2195
- };
2196
- } catch (error) {
2197
- throw withRunConfiguration(error, runConfiguration);
2198
- }
2199
- } finally {
2200
- if (adapter.cleanup) {
2201
- try {
2202
- await trace2.step(
2140
+ if (adapter.initialize) {
2141
+ const initState = await trace2.step(
2203
2142
  {
2204
- step_name: "adapter.cleanup",
2143
+ step_name: "adapter.initialize",
2205
2144
  input: { adapterName: adapter.name },
2206
2145
  metadata: { adapterName: adapter.name },
2207
- summarize: () => ({ cleaned: true })
2146
+ summarize: (result) => ({
2147
+ initialized: true,
2148
+ stateKeys: result && typeof result === "object" ? Object.keys(result) : []
2149
+ })
2208
2150
  },
2209
- async () => adapter.cleanup?.({
2151
+ async () => adapter.initialize?.({
2210
2152
  task: task2,
2211
2153
  taskDir: absoluteDir,
2212
2154
  files,
2213
- state,
2214
- session,
2215
2155
  trace: trace2
2216
2156
  })
2217
2157
  );
2218
- } catch (error) {
2219
- console.error(
2220
- "[AgentTask] Cleanup failed:",
2221
- error instanceof Error ? error.message : String(error)
2222
- );
2223
- }
2224
- }
2225
- if (session?.close) {
2226
- try {
2227
- await session.close();
2228
- } catch {
2229
- }
2230
- }
2231
- }
2232
- }
2233
- }
2234
- async function captureExecution(loaded, options, rawTrace) {
2235
- const {
2236
- task: task2,
2237
- adapter,
2238
- taskDir: absoluteDir,
2239
- files,
2240
- checksPath,
2241
- inlineChecks
2242
- } = loaded;
2243
- const taskFiles = new TaskFiles(files);
2244
- const tee = createProjectionTee(rawTrace);
2245
- const trace2 = tee.trace;
2246
- return withApoRun(
2247
- { trace: trace2, parentSpanId: rawTrace.rootSpanId, taskId: task2.id },
2248
- async () => {
2249
- let state;
2250
- let session;
2251
- const transcriptTurns = [];
2252
- try {
2253
- await trace2.step(
2254
- { step_name: "task.load", input: { taskId: task2.id, taskDir: absoluteDir }, metadata: { taskId: task2.id, taskDir: absoluteDir, fileCount: files.length, hasChecks: inlineChecks || checksPath !== null }, summarize: () => ({ taskId: task2.id, adapterName: adapter.name }) },
2255
- async () => loaded
2256
- );
2257
- if (adapter.initialize) {
2258
- const initState = await trace2.step(
2259
- { step_name: "adapter.initialize", input: { adapterName: adapter.name }, metadata: { adapterName: adapter.name }, summarize: (result) => ({ initialized: true, stateKeys: result && typeof result === "object" ? Object.keys(result) : [] }) },
2260
- async () => adapter.initialize?.({ task: task2, taskDir: absoluteDir, files, trace: trace2 })
2261
- );
2262
2158
  state = initState ?? void 0;
2263
2159
  }
2264
2160
  session = await trace2.step(
2265
- { step_name: "adapter.open-session", input: { adapterName: adapter.name, hasState: state !== void 0 }, metadata: { adapterName: adapter.name }, summarize: () => ({ sessionOpened: true }) },
2266
- async () => adapter.startSession({ task: task2, taskDir: absoluteDir, files, state, trace: trace2 })
2161
+ {
2162
+ step_name: "adapter.open-session",
2163
+ input: { adapterName: adapter.name, hasState: state !== void 0 },
2164
+ metadata: { adapterName: adapter.name },
2165
+ summarize: () => ({ sessionOpened: true })
2166
+ },
2167
+ async () => adapter.startSession({
2168
+ task: task2,
2169
+ taskDir: absoluteDir,
2170
+ files,
2171
+ state,
2172
+ trace: trace2
2173
+ })
2174
+ );
2175
+ const runConfiguration = normalizeRunConfiguration(
2176
+ session.runConfiguration
2267
2177
  );
2268
- const runConfiguration = normalizeRunConfiguration(session.runConfiguration);
2269
2178
  try {
2270
- if (!inlineChecks && checksPath) {
2271
- resetTaskTurn();
2272
- resetFlowChecks();
2273
- await loadChecksModule(checksPath);
2274
- }
2275
- const taskTurn = getTaskTurn();
2276
- resetTaskTurn();
2277
- const turnFn = resolveTurn(adapter.turn, taskTurn);
2278
- if (turnFn) {
2279
- const turnTranscript = [];
2280
- let lastTurnResponse;
2281
- const maxTurns = options?.maxTurnsOverride ?? task2.maxTurns ?? 10;
2282
- for (let turnNum = 1; turnNum <= maxTurns; turnNum++) {
2283
- const userTurn = await turnFn({ files: taskFiles, transcript: turnTranscript });
2284
- if (userTurn === null || userTurn === void 0) break;
2285
- const result = await trace2.step(
2286
- { step_name: "task.turn", metadata: { turnNumber: turnNum }, summarize: (r) => ({ response: r.response }) },
2287
- async (spanId) => {
2288
- if (!session) throw new Error("Session not created");
2289
- return session.sendUserTurn(userTurn, { trace: trace2, turnNumber: turnNum, parentSpanId: spanId });
2290
- }
2291
- );
2292
- turnTranscript.push({ turnNumber: turnNum, input: userTurn, output: result.response });
2293
- transcriptTurns.push({ turnNumber: turnNum, userAction: userTurn, agentResponse: result.response });
2294
- options?.onTurn?.(turnNum, userTurn, result.response);
2295
- lastTurnResponse = result.response;
2296
- }
2297
- if (lastTurnResponse !== void 0) {
2298
- rawTrace.endRoot({ output: { response: lastTurnResponse } });
2299
- }
2300
- }
2179
+ const transcriptTurns = await runTurnLoop(
2180
+ loaded,
2181
+ options,
2182
+ session,
2183
+ trace2,
2184
+ rawTrace
2185
+ );
2301
2186
  const collected = await trace2.step(
2302
- { step_name: "adapter.collect-deliverables", input: { adapterName: adapter.name, expectedDeliverables: task2.deliverables }, metadata: { adapterName: adapter.name }, summarize: (result) => {
2303
- const keys = result && typeof result === "object" ? Object.keys(result) : [];
2304
- return { deliverableCount: keys.length, deliverableNames: keys };
2305
- } },
2187
+ {
2188
+ step_name: "adapter.collect-deliverables",
2189
+ input: {
2190
+ adapterName: adapter.name,
2191
+ expectedDeliverables: task2.deliverables
2192
+ },
2193
+ metadata: { adapterName: adapter.name },
2194
+ summarize: (result) => {
2195
+ const keys = result && typeof result === "object" ? Object.keys(result) : [];
2196
+ return {
2197
+ deliverableCount: keys.length,
2198
+ deliverableNames: keys
2199
+ };
2200
+ }
2201
+ },
2306
2202
  async () => {
2307
- if (!session) throw new Error("Adapter session was not created");
2308
- return adapter.collectDeliverables({ task: task2, taskDir: absoluteDir, files, state, session, trace: trace2 });
2203
+ if (!session) {
2204
+ throw new Error("Adapter session was not created");
2205
+ }
2206
+ return adapter.collectDeliverables({
2207
+ task: task2,
2208
+ taskDir: absoluteDir,
2209
+ files,
2210
+ state,
2211
+ session,
2212
+ trace: trace2
2213
+ });
2309
2214
  }
2310
2215
  );
2311
- const snapshot = tee.getSnapshot();
2312
- return {
2313
- traceRunId: rawTrace.runId,
2216
+ return await continueWith({
2314
2217
  collected,
2315
2218
  transcriptTurns,
2316
2219
  runConfiguration,
2317
- snapshot
2318
- };
2220
+ trace: trace2,
2221
+ getSnapshot: () => tee.getSnapshot()
2222
+ });
2319
2223
  } catch (error) {
2320
2224
  throw withRunConfiguration(error, runConfiguration);
2321
2225
  }
2322
2226
  } finally {
2323
- if (adapter.cleanup) {
2324
- try {
2325
- await trace2.step(
2326
- { step_name: "adapter.cleanup", input: { adapterName: adapter.name }, metadata: { adapterName: adapter.name }, summarize: () => ({ cleaned: true }) },
2327
- async () => adapter.cleanup?.({ task: task2, taskDir: absoluteDir, files, state, session, trace: trace2 })
2328
- );
2329
- } catch (error) {
2330
- console.error("[AgentTask] Cleanup failed:", error instanceof Error ? error.message : String(error));
2331
- }
2332
- }
2333
- if (session?.close) {
2334
- try {
2335
- await session.close();
2336
- } catch {
2337
- }
2338
- }
2227
+ await cleanupAdapter(loaded, trace2, state, session);
2339
2228
  }
2340
2229
  }
2341
2230
  );
2342
2231
  }
2232
+ async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
2233
+ return executeAdapterRun(loaded, options, rawTrace, async (outcome) => {
2234
+ const {
2235
+ task: task2,
2236
+ adapter,
2237
+ taskDir: absoluteDir,
2238
+ files,
2239
+ checksPath,
2240
+ inlineChecks,
2241
+ moduleUrl,
2242
+ evalFileName
2243
+ } = loaded;
2244
+ const { trace: trace2, collected, transcriptTurns, runConfiguration } = outcome;
2245
+ const validationResults = await trace2.step(
2246
+ {
2247
+ step_name: "deliverables.validate",
2248
+ input: { deliverableNames: task2.deliverables },
2249
+ summarize: (result) => {
2250
+ const r = result;
2251
+ const passCount = r.results.filter((x) => x.pass).length;
2252
+ const broken = Object.keys(r.brokenDeliverables);
2253
+ return {
2254
+ total: r.results.length,
2255
+ passCount,
2256
+ failCount: r.results.length - passCount,
2257
+ brokenDeliverableCount: broken.length,
2258
+ brokenDeliverables: broken
2259
+ };
2260
+ }
2261
+ },
2262
+ async () => validateDeliverables(task2, collected, adapter.deliverables)
2263
+ );
2264
+ const checksResults = await trace2.step(
2265
+ {
2266
+ step_name: "checks.run",
2267
+ input: { sourceFile: inlineChecks ? evalFileName : checksPath },
2268
+ metadata: { sourceFile: inlineChecks ? evalFileName : checksPath },
2269
+ summarize: (result) => summarizeEvaluationResults(result)
2270
+ },
2271
+ async () => {
2272
+ const judgeConfig = resolveJudgeConfig(options?.judge, task2.judge);
2273
+ if (!inlineChecks) {
2274
+ return loadAndRunFlowChecks(
2275
+ checksPath,
2276
+ {
2277
+ snapshot: outcome.getSnapshot(),
2278
+ deliverables: collected,
2279
+ files,
2280
+ task: task2,
2281
+ ...judgeConfig ? { judgeConfig } : {}
2282
+ },
2283
+ validationResults.brokenDeliverables
2284
+ );
2285
+ }
2286
+ return runTraceChecks({
2287
+ snapshot: outcome.getSnapshot(),
2288
+ deliverables: proxyBrokenDeliverables(
2289
+ collected,
2290
+ validationResults.brokenDeliverables
2291
+ ),
2292
+ files,
2293
+ task: task2,
2294
+ ...judgeConfig ? { judgeConfig } : {},
2295
+ moduleUrl,
2296
+ displayFile: evalFileName
2297
+ });
2298
+ }
2299
+ );
2300
+ return {
2301
+ task: task2,
2302
+ taskDir: absoluteDir,
2303
+ files,
2304
+ traceRunId,
2305
+ result: aggregateResult(checksResults),
2306
+ deliverables: collected,
2307
+ transcript: { turns: transcriptTurns },
2308
+ runConfiguration
2309
+ };
2310
+ });
2311
+ }
2312
+ async function captureExecution(loaded, options, rawTrace) {
2313
+ return executeAdapterRun(loaded, options, rawTrace, async (outcome) => ({
2314
+ traceRunId: rawTrace.runId,
2315
+ collected: outcome.collected,
2316
+ transcriptTurns: outcome.transcriptTurns,
2317
+ runConfiguration: outcome.runConfiguration,
2318
+ // Freeze the snapshot before the root span ends — Phase 2 reads it.
2319
+ snapshot: outcome.getSnapshot()
2320
+ }));
2321
+ }
2343
2322
  async function evaluate(loaded, options, phase1) {
2344
2323
  const {
2345
2324
  task: task2,
@@ -2388,6 +2367,89 @@ async function evaluate(loaded, options, phase1) {
2388
2367
  runConfiguration: phase1.runConfiguration
2389
2368
  };
2390
2369
  }
2370
+ async function runTurnLoop(loaded, options, session, trace2, rawTrace) {
2371
+ const { task: task2, adapter, files, checksPath, inlineChecks } = loaded;
2372
+ if (!inlineChecks && checksPath) {
2373
+ resetTaskTurn();
2374
+ resetFlowChecks();
2375
+ await loadChecksModule(checksPath);
2376
+ }
2377
+ const taskTurn = getTaskTurn();
2378
+ resetTaskTurn();
2379
+ const turnFn = resolveTurn(adapter.turn, taskTurn);
2380
+ if (!turnFn) return [];
2381
+ const taskFiles = new TaskFiles(files);
2382
+ const turnTranscript = [];
2383
+ const transcriptTurns = [];
2384
+ let lastTurnResponse;
2385
+ const maxTurns = options?.maxTurnsOverride ?? task2.maxTurns ?? 10;
2386
+ for (let turnNum = 1; turnNum <= maxTurns; turnNum++) {
2387
+ const userTurn = await turnFn({ files: taskFiles, transcript: turnTranscript });
2388
+ if (userTurn === null || userTurn === void 0) break;
2389
+ const result = await trace2.step(
2390
+ {
2391
+ step_name: "task.turn",
2392
+ metadata: { turnNumber: turnNum },
2393
+ summarize: (r) => ({ response: r.response })
2394
+ },
2395
+ async (spanId) => session.sendUserTurn(userTurn, {
2396
+ trace: trace2,
2397
+ turnNumber: turnNum,
2398
+ parentSpanId: spanId
2399
+ })
2400
+ );
2401
+ turnTranscript.push({
2402
+ turnNumber: turnNum,
2403
+ input: userTurn,
2404
+ output: result.response
2405
+ });
2406
+ transcriptTurns.push({
2407
+ turnNumber: turnNum,
2408
+ userAction: userTurn,
2409
+ agentResponse: result.response
2410
+ });
2411
+ options?.onTurn?.(turnNum, userTurn, result.response);
2412
+ lastTurnResponse = result.response;
2413
+ }
2414
+ if (lastTurnResponse !== void 0) {
2415
+ rawTrace.endRoot({ output: { response: lastTurnResponse } });
2416
+ }
2417
+ return transcriptTurns;
2418
+ }
2419
+ async function cleanupAdapter(loaded, trace2, state, session) {
2420
+ const { task: task2, adapter, taskDir: absoluteDir, files } = loaded;
2421
+ if (adapter.cleanup) {
2422
+ try {
2423
+ await trace2.step(
2424
+ {
2425
+ step_name: "adapter.cleanup",
2426
+ input: { adapterName: adapter.name },
2427
+ metadata: { adapterName: adapter.name },
2428
+ summarize: () => ({ cleaned: true })
2429
+ },
2430
+ async () => adapter.cleanup?.({
2431
+ task: task2,
2432
+ taskDir: absoluteDir,
2433
+ files,
2434
+ state,
2435
+ session,
2436
+ trace: trace2
2437
+ })
2438
+ );
2439
+ } catch (error) {
2440
+ console.error(
2441
+ "[AgentTask] Cleanup failed:",
2442
+ error instanceof Error ? error.message : String(error)
2443
+ );
2444
+ }
2445
+ }
2446
+ if (session?.close) {
2447
+ try {
2448
+ await session.close();
2449
+ } catch {
2450
+ }
2451
+ }
2452
+ }
2391
2453
  function summarizeEvaluationResults(results) {
2392
2454
  const passCount = results.filter((r) => r.pass).length;
2393
2455
  return {
@@ -3303,12 +3365,12 @@ function printOverallSummary(summaries, hasErrors) {
3303
3365
  }
3304
3366
 
3305
3367
  // src/agent-task/task-revision-manifest.ts
3306
- import { createHash as createHash2 } from "crypto";
3368
+ import { createHash as createHash3 } from "crypto";
3307
3369
  function normalizeManifestPath(path) {
3308
3370
  return path.split("\\").join("/").normalize("NFC");
3309
3371
  }
3310
3372
  function sha256Hex(bytes) {
3311
- return createHash2("sha256").update(bytes).digest("hex");
3373
+ return createHash3("sha256").update(bytes).digest("hex");
3312
3374
  }
3313
3375
  function compareBytewiseUtf8(a, b) {
3314
3376
  const ba = Buffer.from(a, "utf8");
@@ -3356,11 +3418,14 @@ export {
3356
3418
  AgentTaskRunError,
3357
3419
  ApoSpanProcessor,
3358
3420
  FlowView,
3421
+ JUDGE_SEGMENT_LIMIT,
3422
+ RECEIVED_VALUE_LIMIT,
3359
3423
  RejudgeError,
3360
3424
  TEST_METHOD_NAMES,
3361
3425
  TaskFiles,
3362
3426
  buildManifest,
3363
3427
  canonicalManifestJson,
3428
+ compactChecksForSubmission,
3364
3429
  contentSha256,
3365
3430
  createApoAnthropic,
3366
3431
  createApoOpenAI,