@apo-ai/sdk 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -69
- package/dist/agent-task/integrations/ai-sdk.d.ts +6 -6
- package/dist/agent-task/integrations/ai-sdk.js +1 -1
- package/dist/agent-task/integrations/anthropic.d.ts +1 -1
- package/dist/agent-task/integrations/openai.d.ts +3 -3
- package/dist/agent-task/integrations/openai.js +1 -1
- package/dist/agent-task/public.d.ts +49 -9
- package/dist/agent-task/public.js +393 -328
- package/dist/agent-task/public.js.map +1 -1
- package/dist/{chunk-DT2LMB7O.js → chunk-6ZRW5IW6.js} +1 -1
- package/dist/chunk-6ZRW5IW6.js.map +1 -0
- package/dist/{chunk-PMYLL2H4.js → chunk-GETY34VI.js} +1 -1
- package/dist/chunk-GETY34VI.js.map +1 -0
- package/dist/{chunk-HKLBPIB4.js → chunk-H7ITLBWU.js} +1 -1
- package/dist/chunk-H7ITLBWU.js.map +1 -0
- package/dist/otel/index.d.ts +3 -4
- package/dist/otel/index.js +1 -1
- package/dist/{tracing-D5Yhzq8e.d.ts → tracing-CTmq4PSw.d.ts} +1 -1
- package/package.json +20 -3
- package/dist/chunk-DT2LMB7O.js.map +0 -1
- package/dist/chunk-HKLBPIB4.js.map +0 -1
- package/dist/chunk-PMYLL2H4.js.map +0 -1
|
@@ -8,14 +8,14 @@ import {
|
|
|
8
8
|
resetApoTracing,
|
|
9
9
|
withApoRun,
|
|
10
10
|
withApoRunSync
|
|
11
|
-
} from "../chunk-
|
|
11
|
+
} from "../chunk-GETY34VI.js";
|
|
12
12
|
import {
|
|
13
13
|
createApoTracer
|
|
14
|
-
} from "../chunk-
|
|
14
|
+
} from "../chunk-H7ITLBWU.js";
|
|
15
15
|
import "../chunk-HWZVL6TS.js";
|
|
16
16
|
import {
|
|
17
17
|
createApoOpenAI
|
|
18
|
-
} from "../chunk-
|
|
18
|
+
} from "../chunk-6ZRW5IW6.js";
|
|
19
19
|
import {
|
|
20
20
|
createApoAnthropic
|
|
21
21
|
} from "../chunk-SZDSIPXC.js";
|
|
@@ -235,8 +235,21 @@ function levenshtein(a, b) {
|
|
|
235
235
|
}
|
|
236
236
|
|
|
237
237
|
// src/agent-task/checks/judge.ts
|
|
238
|
-
var
|
|
239
|
-
var
|
|
238
|
+
var VERDICT_FIRST_CONTRACT = 'Respond with ONLY a JSON object: {"pass": true/false, "reasoning": "your reasoning"}';
|
|
239
|
+
var REASONING_FIRST_CONTRACT = 'Respond with ONLY a JSON object: {"reasoning": "your reasoning", "pass": true/false}';
|
|
240
|
+
function isJudgeVerdictFirstOverrideEnabled() {
|
|
241
|
+
const value = process.env.APO_JUDGE_VERDICT_FIRST?.trim().toLowerCase();
|
|
242
|
+
return value === "1" || value === "true";
|
|
243
|
+
}
|
|
244
|
+
function judgeResponseContract() {
|
|
245
|
+
return isJudgeVerdictFirstOverrideEnabled() ? VERDICT_FIRST_CONTRACT : REASONING_FIRST_CONTRACT;
|
|
246
|
+
}
|
|
247
|
+
function judgeContractInUse() {
|
|
248
|
+
return isJudgeVerdictFirstOverrideEnabled() ? "verdict-first" : "reasoning-first";
|
|
249
|
+
}
|
|
250
|
+
function judgeSystemPrompt() {
|
|
251
|
+
return `You are an evaluation judge. Evaluate the given value(s) against the instruction. ${judgeResponseContract()}`;
|
|
252
|
+
}
|
|
240
253
|
function formatValue(value, depth = 0) {
|
|
241
254
|
const indent = " ".repeat(depth);
|
|
242
255
|
if (value === null || value === void 0) return String(value);
|
|
@@ -297,6 +310,7 @@ function parseJudgeJson(raw) {
|
|
|
297
310
|
reasoning: "Judge response could not be parsed as JSON \u2014 the verdict is unknown, so this check is treated as a failure. The model's raw response is available in the judge metadata."
|
|
298
311
|
};
|
|
299
312
|
}
|
|
313
|
+
var JUDGE_TIMEOUT_MS = 12e4;
|
|
300
314
|
var prefixQueues = /* @__PURE__ */ new Map();
|
|
301
315
|
function runWithSharedPrefix(key, task2) {
|
|
302
316
|
const prev = prefixQueues.get(key) ?? Promise.resolve();
|
|
@@ -324,6 +338,7 @@ ${deliverableText}`;
|
|
|
324
338
|
const startedAt = Date.now();
|
|
325
339
|
const response = await fetch(`${baseURL}/chat/completions`, {
|
|
326
340
|
method: "POST",
|
|
341
|
+
signal: AbortSignal.timeout(JUDGE_TIMEOUT_MS),
|
|
327
342
|
headers: {
|
|
328
343
|
"Content-Type": "application/json",
|
|
329
344
|
...apiKey ? { Authorization: `Bearer ${apiKey}` } : {}
|
|
@@ -363,6 +378,7 @@ ${deliverableText}`;
|
|
|
363
378
|
reasoning: "Judge returned an empty or truncated response \u2014 likely a transient provider failure. The verdict is unknown, so this check is treated as a failure.",
|
|
364
379
|
judge: {
|
|
365
380
|
model: args.model,
|
|
381
|
+
contract: judgeContractInUse(),
|
|
366
382
|
prompt: { system: systemPromptText, user: instructionText },
|
|
367
383
|
response: text,
|
|
368
384
|
tokens: parseJudgeUsage(data.usage),
|
|
@@ -376,6 +392,7 @@ ${deliverableText}`;
|
|
|
376
392
|
reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : "",
|
|
377
393
|
judge: {
|
|
378
394
|
model: args.model,
|
|
395
|
+
contract: judgeContractInUse(),
|
|
379
396
|
prompt: { system: systemPromptText, user: instructionText },
|
|
380
397
|
response: text,
|
|
381
398
|
tokens: parseJudgeUsage(data.usage),
|
|
@@ -387,7 +404,7 @@ ${deliverableText}`;
|
|
|
387
404
|
function assembleBriefing(args) {
|
|
388
405
|
if (!args.prompt) {
|
|
389
406
|
return {
|
|
390
|
-
briefingText:
|
|
407
|
+
briefingText: judgeSystemPrompt(),
|
|
391
408
|
instructionText: `Instruction:
|
|
392
409
|
${args.instruction}`
|
|
393
410
|
};
|
|
@@ -406,7 +423,7 @@ ${args.instruction}`
|
|
|
406
423
|
// The SDK appends its own response contract to any custom briefing.
|
|
407
424
|
briefingText: system ? `${system}
|
|
408
425
|
|
|
409
|
-
${
|
|
426
|
+
${judgeResponseContract()}` : judgeSystemPrompt(),
|
|
410
427
|
instructionText: user ?? `Instruction:
|
|
411
428
|
${args.instruction}`
|
|
412
429
|
};
|
|
@@ -876,8 +893,8 @@ async function runTraceChecks(args) {
|
|
|
876
893
|
...rec.all.length > 0 ? { assertions: rec.all.map((a) => ({ ...a })) } : {},
|
|
877
894
|
...check.group_id ? { group_id: check.group_id } : {},
|
|
878
895
|
...check.group_name ? { group_name: check.group_name } : {},
|
|
879
|
-
// stamp the snapshot source so consumers can
|
|
880
|
-
//
|
|
896
|
+
// stamp the snapshot source so consumers can tell projection-first
|
|
897
|
+
// results (canonical) from locally recorded snapshots.
|
|
881
898
|
...args.snapshot.source !== "canonical" ? { source: args.snapshot.source } : {}
|
|
882
899
|
};
|
|
883
900
|
})
|
|
@@ -1380,6 +1397,115 @@ async function safeDetail(response) {
|
|
|
1380
1397
|
}
|
|
1381
1398
|
}
|
|
1382
1399
|
|
|
1400
|
+
// src/agent-task/checks/compact.ts
|
|
1401
|
+
import { createHash as createHash2 } from "crypto";
|
|
1402
|
+
var RECEIVED_VALUE_LIMIT = 4 * 1024;
|
|
1403
|
+
var JUDGE_SEGMENT_LIMIT = 16 * 1024;
|
|
1404
|
+
var PREVIEW_CHARS = 256;
|
|
1405
|
+
function compactChecksForSubmission(checks) {
|
|
1406
|
+
let truncatedValues = 0;
|
|
1407
|
+
let truncatedSegments = 0;
|
|
1408
|
+
const compacted = checks.map((check) => {
|
|
1409
|
+
const next = { ...check };
|
|
1410
|
+
if ("received" in next && next.received !== void 0) {
|
|
1411
|
+
const replaced = truncateValue(next.received);
|
|
1412
|
+
if (replaced !== next.received) {
|
|
1413
|
+
next.received = replaced;
|
|
1414
|
+
truncatedValues += 1;
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
if (next.judge !== void 0) {
|
|
1418
|
+
const { value, truncated } = truncateJudge(next.judge);
|
|
1419
|
+
if (value !== next.judge) {
|
|
1420
|
+
next.judge = value;
|
|
1421
|
+
truncatedSegments += truncated;
|
|
1422
|
+
}
|
|
1423
|
+
}
|
|
1424
|
+
const sourceAssertions = check.assertions;
|
|
1425
|
+
if (sourceAssertions) {
|
|
1426
|
+
const assertions = sourceAssertions.map((assertion) => {
|
|
1427
|
+
const nextAssertion = { ...assertion };
|
|
1428
|
+
if (nextAssertion.received !== void 0) {
|
|
1429
|
+
const replaced = truncateValue(nextAssertion.received);
|
|
1430
|
+
if (replaced !== nextAssertion.received) {
|
|
1431
|
+
nextAssertion.received = replaced;
|
|
1432
|
+
truncatedValues += 1;
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
if (nextAssertion.judge !== void 0) {
|
|
1436
|
+
const { value, truncated } = truncateJudge(nextAssertion.judge);
|
|
1437
|
+
if (value !== nextAssertion.judge) {
|
|
1438
|
+
nextAssertion.judge = value;
|
|
1439
|
+
truncatedSegments += truncated;
|
|
1440
|
+
}
|
|
1441
|
+
}
|
|
1442
|
+
return nextAssertion;
|
|
1443
|
+
});
|
|
1444
|
+
if (assertions.some((a, i) => a !== sourceAssertions[i])) {
|
|
1445
|
+
next.assertions = assertions;
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
return next;
|
|
1449
|
+
});
|
|
1450
|
+
return { checks: compacted, truncatedValues, truncatedSegments };
|
|
1451
|
+
}
|
|
1452
|
+
function truncateJudge(judge) {
|
|
1453
|
+
if (typeof judge !== "object" || judge === null) {
|
|
1454
|
+
return { value: judge, truncated: 0 };
|
|
1455
|
+
}
|
|
1456
|
+
let truncated = 0;
|
|
1457
|
+
const next = { ...judge };
|
|
1458
|
+
const prompt = next.prompt;
|
|
1459
|
+
if (typeof prompt === "object" && prompt !== null) {
|
|
1460
|
+
const nextPrompt = { ...prompt };
|
|
1461
|
+
for (const key of ["system", "user"]) {
|
|
1462
|
+
const value = nextPrompt[key];
|
|
1463
|
+
if (typeof value === "string") {
|
|
1464
|
+
const replaced = truncateText(value);
|
|
1465
|
+
if (replaced !== value) {
|
|
1466
|
+
nextPrompt[key] = replaced;
|
|
1467
|
+
truncated += 1;
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
next.prompt = nextPrompt;
|
|
1472
|
+
}
|
|
1473
|
+
if (typeof next.response === "string") {
|
|
1474
|
+
const replaced = truncateText(next.response);
|
|
1475
|
+
if (replaced !== next.response) {
|
|
1476
|
+
next.response = replaced;
|
|
1477
|
+
truncated += 1;
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1480
|
+
return { value: next, truncated };
|
|
1481
|
+
}
|
|
1482
|
+
function truncateValue(value) {
|
|
1483
|
+
if (value === void 0) return value;
|
|
1484
|
+
let bytes;
|
|
1485
|
+
try {
|
|
1486
|
+
bytes = new TextEncoder().encode(JSON.stringify(value));
|
|
1487
|
+
} catch {
|
|
1488
|
+
bytes = new TextEncoder().encode(JSON.stringify(String(value)));
|
|
1489
|
+
}
|
|
1490
|
+
if (bytes.length <= RECEIVED_VALUE_LIMIT) return value;
|
|
1491
|
+
return marker(bytes);
|
|
1492
|
+
}
|
|
1493
|
+
function truncateText(value) {
|
|
1494
|
+
const bytes = new TextEncoder().encode(value);
|
|
1495
|
+
if (bytes.length <= JUDGE_SEGMENT_LIMIT) return value;
|
|
1496
|
+
return marker(bytes);
|
|
1497
|
+
}
|
|
1498
|
+
function marker(bytes) {
|
|
1499
|
+
const previewBytes = bytes.slice(0, PREVIEW_CHARS);
|
|
1500
|
+
const preview = Buffer.from(previewBytes).toString("utf8");
|
|
1501
|
+
return {
|
|
1502
|
+
kind: "truncated",
|
|
1503
|
+
preview,
|
|
1504
|
+
size_bytes: bytes.length,
|
|
1505
|
+
sha256: createHash2("sha256").update(bytes).digest("hex")
|
|
1506
|
+
};
|
|
1507
|
+
}
|
|
1508
|
+
|
|
1383
1509
|
// src/agent-task/flow/view.ts
|
|
1384
1510
|
var FlowView = class {
|
|
1385
1511
|
constructor(flow) {
|
|
@@ -1977,369 +2103,222 @@ async function readCanonicalSnapshot(trace2) {
|
|
|
1977
2103
|
return null;
|
|
1978
2104
|
}
|
|
1979
2105
|
}
|
|
1980
|
-
async function
|
|
2106
|
+
async function executeAdapterRun(loaded, options, rawTrace, continueWith) {
|
|
1981
2107
|
const {
|
|
1982
2108
|
task: task2,
|
|
1983
2109
|
adapter,
|
|
1984
2110
|
taskDir: absoluteDir,
|
|
1985
2111
|
files,
|
|
1986
2112
|
checksPath,
|
|
1987
|
-
inlineChecks
|
|
1988
|
-
moduleUrl,
|
|
1989
|
-
evalFileName
|
|
2113
|
+
inlineChecks
|
|
1990
2114
|
} = loaded;
|
|
1991
|
-
const taskFiles = new TaskFiles(files);
|
|
1992
2115
|
const tee = createProjectionTee(rawTrace);
|
|
1993
|
-
const
|
|
2116
|
+
const trace2 = tee.trace;
|
|
1994
2117
|
return withApoRun(
|
|
1995
2118
|
{ trace: trace2, parentSpanId: rawTrace.rootSpanId, taskId: task2.id },
|
|
1996
2119
|
async () => {
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
);
|
|
2000
|
-
async function executeRunBody() {
|
|
2001
|
-
let state;
|
|
2002
|
-
let session;
|
|
2003
|
-
const transcriptTurns = [];
|
|
2004
|
-
try {
|
|
2005
|
-
await trace2.step(
|
|
2006
|
-
{
|
|
2007
|
-
step_name: "task.load",
|
|
2008
|
-
input: { taskId: task2.id, taskDir: absoluteDir },
|
|
2009
|
-
metadata: {
|
|
2010
|
-
taskId: task2.id,
|
|
2011
|
-
taskDir: absoluteDir,
|
|
2012
|
-
fileCount: files.length,
|
|
2013
|
-
hasChecks: inlineChecks || checksPath !== null
|
|
2014
|
-
},
|
|
2015
|
-
summarize: () => ({
|
|
2016
|
-
taskId: task2.id,
|
|
2017
|
-
adapterName: adapter.name
|
|
2018
|
-
})
|
|
2019
|
-
},
|
|
2020
|
-
async () => loaded
|
|
2021
|
-
);
|
|
2022
|
-
if (adapter.initialize) {
|
|
2023
|
-
const initState = await trace2.step(
|
|
2024
|
-
{
|
|
2025
|
-
step_name: "adapter.initialize",
|
|
2026
|
-
input: { adapterName: adapter.name },
|
|
2027
|
-
metadata: { adapterName: adapter.name },
|
|
2028
|
-
summarize: (result) => ({
|
|
2029
|
-
initialized: true,
|
|
2030
|
-
stateKeys: result && typeof result === "object" ? Object.keys(result) : []
|
|
2031
|
-
})
|
|
2032
|
-
},
|
|
2033
|
-
async () => adapter.initialize?.({
|
|
2034
|
-
task: task2,
|
|
2035
|
-
taskDir: absoluteDir,
|
|
2036
|
-
files,
|
|
2037
|
-
trace: trace2
|
|
2038
|
-
})
|
|
2039
|
-
);
|
|
2040
|
-
state = initState ?? void 0;
|
|
2041
|
-
}
|
|
2042
|
-
session = await trace2.step(
|
|
2043
|
-
{
|
|
2044
|
-
step_name: "adapter.open-session",
|
|
2045
|
-
input: { adapterName: adapter.name, hasState: state !== void 0 },
|
|
2046
|
-
metadata: { adapterName: adapter.name },
|
|
2047
|
-
summarize: () => ({ sessionOpened: true })
|
|
2048
|
-
},
|
|
2049
|
-
async () => adapter.startSession({
|
|
2050
|
-
task: task2,
|
|
2051
|
-
taskDir: absoluteDir,
|
|
2052
|
-
files,
|
|
2053
|
-
state,
|
|
2054
|
-
trace: trace2
|
|
2055
|
-
})
|
|
2056
|
-
);
|
|
2057
|
-
const runConfiguration = normalizeRunConfiguration(session.runConfiguration);
|
|
2120
|
+
let state;
|
|
2121
|
+
let session;
|
|
2058
2122
|
try {
|
|
2059
|
-
|
|
2060
|
-
resetTaskTurn();
|
|
2061
|
-
resetFlowChecks();
|
|
2062
|
-
await loadChecksModule(checksPath);
|
|
2063
|
-
}
|
|
2064
|
-
const taskTurn = getTaskTurn();
|
|
2065
|
-
resetTaskTurn();
|
|
2066
|
-
const turnFn = resolveTurn(adapter.turn, taskTurn);
|
|
2067
|
-
if (turnFn) {
|
|
2068
|
-
const turnTranscript = [];
|
|
2069
|
-
let lastTurnResponse;
|
|
2070
|
-
const maxTurns = options?.maxTurnsOverride ?? task2.maxTurns ?? 10;
|
|
2071
|
-
for (let turnNum = 1; turnNum <= maxTurns; turnNum++) {
|
|
2072
|
-
const userTurn = await turnFn({ files: taskFiles, transcript: turnTranscript });
|
|
2073
|
-
if (userTurn === null || userTurn === void 0) break;
|
|
2074
|
-
const result2 = await trace2.step(
|
|
2075
|
-
{
|
|
2076
|
-
step_name: "task.turn",
|
|
2077
|
-
metadata: { turnNumber: turnNum },
|
|
2078
|
-
summarize: (r) => ({ response: r.response })
|
|
2079
|
-
},
|
|
2080
|
-
async (spanId) => {
|
|
2081
|
-
if (!session) throw new Error("Session not created");
|
|
2082
|
-
return session.sendUserTurn(userTurn, {
|
|
2083
|
-
trace: trace2,
|
|
2084
|
-
turnNumber: turnNum,
|
|
2085
|
-
parentSpanId: spanId
|
|
2086
|
-
});
|
|
2087
|
-
}
|
|
2088
|
-
);
|
|
2089
|
-
turnTranscript.push({
|
|
2090
|
-
turnNumber: turnNum,
|
|
2091
|
-
input: userTurn,
|
|
2092
|
-
output: result2.response
|
|
2093
|
-
});
|
|
2094
|
-
transcriptTurns.push({
|
|
2095
|
-
turnNumber: turnNum,
|
|
2096
|
-
userAction: userTurn,
|
|
2097
|
-
agentResponse: result2.response
|
|
2098
|
-
});
|
|
2099
|
-
options?.onTurn?.(turnNum, userTurn, result2.response);
|
|
2100
|
-
lastTurnResponse = result2.response;
|
|
2101
|
-
}
|
|
2102
|
-
if (lastTurnResponse !== void 0) {
|
|
2103
|
-
rawTrace.endRoot({ output: { response: lastTurnResponse } });
|
|
2104
|
-
}
|
|
2105
|
-
}
|
|
2106
|
-
const collected = await trace2.step(
|
|
2123
|
+
await trace2.step(
|
|
2107
2124
|
{
|
|
2108
|
-
step_name: "
|
|
2109
|
-
input: {
|
|
2110
|
-
metadata: {
|
|
2111
|
-
|
|
2112
|
-
const keys = result2 && typeof result2 === "object" ? Object.keys(result2) : [];
|
|
2113
|
-
return { deliverableCount: keys.length, deliverableNames: keys };
|
|
2114
|
-
}
|
|
2115
|
-
},
|
|
2116
|
-
async () => {
|
|
2117
|
-
if (!session) {
|
|
2118
|
-
throw new Error("Adapter session was not created");
|
|
2119
|
-
}
|
|
2120
|
-
return adapter.collectDeliverables({
|
|
2121
|
-
task: task2,
|
|
2125
|
+
step_name: "task.load",
|
|
2126
|
+
input: { taskId: task2.id, taskDir: absoluteDir },
|
|
2127
|
+
metadata: {
|
|
2128
|
+
taskId: task2.id,
|
|
2122
2129
|
taskDir: absoluteDir,
|
|
2123
|
-
files,
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
const validationResults = await trace2.step(
|
|
2131
|
-
{
|
|
2132
|
-
step_name: "deliverables.validate",
|
|
2133
|
-
input: { deliverableNames: task2.deliverables },
|
|
2134
|
-
summarize: (result2) => {
|
|
2135
|
-
const r = result2;
|
|
2136
|
-
const passCount = r.results.filter((x) => x.pass).length;
|
|
2137
|
-
const broken = Object.keys(r.brokenDeliverables);
|
|
2138
|
-
return {
|
|
2139
|
-
total: r.results.length,
|
|
2140
|
-
passCount,
|
|
2141
|
-
failCount: r.results.length - passCount,
|
|
2142
|
-
brokenDeliverableCount: broken.length,
|
|
2143
|
-
brokenDeliverables: broken
|
|
2144
|
-
};
|
|
2145
|
-
}
|
|
2146
|
-
},
|
|
2147
|
-
async () => validateDeliverables(task2, collected, adapter.deliverables)
|
|
2148
|
-
);
|
|
2149
|
-
const checksResults = await trace2.step(
|
|
2150
|
-
{
|
|
2151
|
-
step_name: "checks.run",
|
|
2152
|
-
input: { sourceFile: inlineChecks ? evalFileName : checksPath },
|
|
2153
|
-
metadata: { sourceFile: inlineChecks ? evalFileName : checksPath },
|
|
2154
|
-
summarize: (result2) => summarizeEvaluationResults(result2)
|
|
2130
|
+
fileCount: files.length,
|
|
2131
|
+
hasChecks: inlineChecks || checksPath !== null
|
|
2132
|
+
},
|
|
2133
|
+
summarize: () => ({
|
|
2134
|
+
taskId: task2.id,
|
|
2135
|
+
adapterName: adapter.name
|
|
2136
|
+
})
|
|
2155
2137
|
},
|
|
2156
|
-
async () =>
|
|
2157
|
-
const judgeConfig = resolveJudgeConfig(options?.judge, task2.judge);
|
|
2158
|
-
if (!inlineChecks) {
|
|
2159
|
-
return loadAndRunFlowChecks(
|
|
2160
|
-
checksPath,
|
|
2161
|
-
{
|
|
2162
|
-
snapshot: getSnapshot(),
|
|
2163
|
-
deliverables: collected,
|
|
2164
|
-
files,
|
|
2165
|
-
task: task2,
|
|
2166
|
-
...judgeConfig ? { judgeConfig } : {}
|
|
2167
|
-
},
|
|
2168
|
-
validationResults.brokenDeliverables
|
|
2169
|
-
);
|
|
2170
|
-
}
|
|
2171
|
-
return runTraceChecks({
|
|
2172
|
-
snapshot: getSnapshot(),
|
|
2173
|
-
deliverables: proxyBrokenDeliverables(
|
|
2174
|
-
collected,
|
|
2175
|
-
validationResults.brokenDeliverables
|
|
2176
|
-
),
|
|
2177
|
-
files,
|
|
2178
|
-
task: task2,
|
|
2179
|
-
...judgeConfig ? { judgeConfig } : {},
|
|
2180
|
-
moduleUrl,
|
|
2181
|
-
displayFile: evalFileName
|
|
2182
|
-
});
|
|
2183
|
-
}
|
|
2138
|
+
async () => loaded
|
|
2184
2139
|
);
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
task: task2,
|
|
2188
|
-
taskDir: absoluteDir,
|
|
2189
|
-
files,
|
|
2190
|
-
traceRunId,
|
|
2191
|
-
result,
|
|
2192
|
-
deliverables: collected,
|
|
2193
|
-
transcript: { turns: transcriptTurns },
|
|
2194
|
-
runConfiguration
|
|
2195
|
-
};
|
|
2196
|
-
} catch (error) {
|
|
2197
|
-
throw withRunConfiguration(error, runConfiguration);
|
|
2198
|
-
}
|
|
2199
|
-
} finally {
|
|
2200
|
-
if (adapter.cleanup) {
|
|
2201
|
-
try {
|
|
2202
|
-
await trace2.step(
|
|
2140
|
+
if (adapter.initialize) {
|
|
2141
|
+
const initState = await trace2.step(
|
|
2203
2142
|
{
|
|
2204
|
-
step_name: "adapter.
|
|
2143
|
+
step_name: "adapter.initialize",
|
|
2205
2144
|
input: { adapterName: adapter.name },
|
|
2206
2145
|
metadata: { adapterName: adapter.name },
|
|
2207
|
-
summarize: () => ({
|
|
2146
|
+
summarize: (result) => ({
|
|
2147
|
+
initialized: true,
|
|
2148
|
+
stateKeys: result && typeof result === "object" ? Object.keys(result) : []
|
|
2149
|
+
})
|
|
2208
2150
|
},
|
|
2209
|
-
async () => adapter.
|
|
2151
|
+
async () => adapter.initialize?.({
|
|
2210
2152
|
task: task2,
|
|
2211
2153
|
taskDir: absoluteDir,
|
|
2212
2154
|
files,
|
|
2213
|
-
state,
|
|
2214
|
-
session,
|
|
2215
2155
|
trace: trace2
|
|
2216
2156
|
})
|
|
2217
2157
|
);
|
|
2218
|
-
} catch (error) {
|
|
2219
|
-
console.error(
|
|
2220
|
-
"[AgentTask] Cleanup failed:",
|
|
2221
|
-
error instanceof Error ? error.message : String(error)
|
|
2222
|
-
);
|
|
2223
|
-
}
|
|
2224
|
-
}
|
|
2225
|
-
if (session?.close) {
|
|
2226
|
-
try {
|
|
2227
|
-
await session.close();
|
|
2228
|
-
} catch {
|
|
2229
|
-
}
|
|
2230
|
-
}
|
|
2231
|
-
}
|
|
2232
|
-
}
|
|
2233
|
-
}
|
|
2234
|
-
async function captureExecution(loaded, options, rawTrace) {
|
|
2235
|
-
const {
|
|
2236
|
-
task: task2,
|
|
2237
|
-
adapter,
|
|
2238
|
-
taskDir: absoluteDir,
|
|
2239
|
-
files,
|
|
2240
|
-
checksPath,
|
|
2241
|
-
inlineChecks
|
|
2242
|
-
} = loaded;
|
|
2243
|
-
const taskFiles = new TaskFiles(files);
|
|
2244
|
-
const tee = createProjectionTee(rawTrace);
|
|
2245
|
-
const trace2 = tee.trace;
|
|
2246
|
-
return withApoRun(
|
|
2247
|
-
{ trace: trace2, parentSpanId: rawTrace.rootSpanId, taskId: task2.id },
|
|
2248
|
-
async () => {
|
|
2249
|
-
let state;
|
|
2250
|
-
let session;
|
|
2251
|
-
const transcriptTurns = [];
|
|
2252
|
-
try {
|
|
2253
|
-
await trace2.step(
|
|
2254
|
-
{ step_name: "task.load", input: { taskId: task2.id, taskDir: absoluteDir }, metadata: { taskId: task2.id, taskDir: absoluteDir, fileCount: files.length, hasChecks: inlineChecks || checksPath !== null }, summarize: () => ({ taskId: task2.id, adapterName: adapter.name }) },
|
|
2255
|
-
async () => loaded
|
|
2256
|
-
);
|
|
2257
|
-
if (adapter.initialize) {
|
|
2258
|
-
const initState = await trace2.step(
|
|
2259
|
-
{ step_name: "adapter.initialize", input: { adapterName: adapter.name }, metadata: { adapterName: adapter.name }, summarize: (result) => ({ initialized: true, stateKeys: result && typeof result === "object" ? Object.keys(result) : [] }) },
|
|
2260
|
-
async () => adapter.initialize?.({ task: task2, taskDir: absoluteDir, files, trace: trace2 })
|
|
2261
|
-
);
|
|
2262
2158
|
state = initState ?? void 0;
|
|
2263
2159
|
}
|
|
2264
2160
|
session = await trace2.step(
|
|
2265
|
-
{
|
|
2266
|
-
|
|
2161
|
+
{
|
|
2162
|
+
step_name: "adapter.open-session",
|
|
2163
|
+
input: { adapterName: adapter.name, hasState: state !== void 0 },
|
|
2164
|
+
metadata: { adapterName: adapter.name },
|
|
2165
|
+
summarize: () => ({ sessionOpened: true })
|
|
2166
|
+
},
|
|
2167
|
+
async () => adapter.startSession({
|
|
2168
|
+
task: task2,
|
|
2169
|
+
taskDir: absoluteDir,
|
|
2170
|
+
files,
|
|
2171
|
+
state,
|
|
2172
|
+
trace: trace2
|
|
2173
|
+
})
|
|
2174
|
+
);
|
|
2175
|
+
const runConfiguration = normalizeRunConfiguration(
|
|
2176
|
+
session.runConfiguration
|
|
2267
2177
|
);
|
|
2268
|
-
const runConfiguration = normalizeRunConfiguration(session.runConfiguration);
|
|
2269
2178
|
try {
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
const turnFn = resolveTurn(adapter.turn, taskTurn);
|
|
2278
|
-
if (turnFn) {
|
|
2279
|
-
const turnTranscript = [];
|
|
2280
|
-
let lastTurnResponse;
|
|
2281
|
-
const maxTurns = options?.maxTurnsOverride ?? task2.maxTurns ?? 10;
|
|
2282
|
-
for (let turnNum = 1; turnNum <= maxTurns; turnNum++) {
|
|
2283
|
-
const userTurn = await turnFn({ files: taskFiles, transcript: turnTranscript });
|
|
2284
|
-
if (userTurn === null || userTurn === void 0) break;
|
|
2285
|
-
const result = await trace2.step(
|
|
2286
|
-
{ step_name: "task.turn", metadata: { turnNumber: turnNum }, summarize: (r) => ({ response: r.response }) },
|
|
2287
|
-
async (spanId) => {
|
|
2288
|
-
if (!session) throw new Error("Session not created");
|
|
2289
|
-
return session.sendUserTurn(userTurn, { trace: trace2, turnNumber: turnNum, parentSpanId: spanId });
|
|
2290
|
-
}
|
|
2291
|
-
);
|
|
2292
|
-
turnTranscript.push({ turnNumber: turnNum, input: userTurn, output: result.response });
|
|
2293
|
-
transcriptTurns.push({ turnNumber: turnNum, userAction: userTurn, agentResponse: result.response });
|
|
2294
|
-
options?.onTurn?.(turnNum, userTurn, result.response);
|
|
2295
|
-
lastTurnResponse = result.response;
|
|
2296
|
-
}
|
|
2297
|
-
if (lastTurnResponse !== void 0) {
|
|
2298
|
-
rawTrace.endRoot({ output: { response: lastTurnResponse } });
|
|
2299
|
-
}
|
|
2300
|
-
}
|
|
2179
|
+
const transcriptTurns = await runTurnLoop(
|
|
2180
|
+
loaded,
|
|
2181
|
+
options,
|
|
2182
|
+
session,
|
|
2183
|
+
trace2,
|
|
2184
|
+
rawTrace
|
|
2185
|
+
);
|
|
2301
2186
|
const collected = await trace2.step(
|
|
2302
|
-
{
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2187
|
+
{
|
|
2188
|
+
step_name: "adapter.collect-deliverables",
|
|
2189
|
+
input: {
|
|
2190
|
+
adapterName: adapter.name,
|
|
2191
|
+
expectedDeliverables: task2.deliverables
|
|
2192
|
+
},
|
|
2193
|
+
metadata: { adapterName: adapter.name },
|
|
2194
|
+
summarize: (result) => {
|
|
2195
|
+
const keys = result && typeof result === "object" ? Object.keys(result) : [];
|
|
2196
|
+
return {
|
|
2197
|
+
deliverableCount: keys.length,
|
|
2198
|
+
deliverableNames: keys
|
|
2199
|
+
};
|
|
2200
|
+
}
|
|
2201
|
+
},
|
|
2306
2202
|
async () => {
|
|
2307
|
-
if (!session)
|
|
2308
|
-
|
|
2203
|
+
if (!session) {
|
|
2204
|
+
throw new Error("Adapter session was not created");
|
|
2205
|
+
}
|
|
2206
|
+
return adapter.collectDeliverables({
|
|
2207
|
+
task: task2,
|
|
2208
|
+
taskDir: absoluteDir,
|
|
2209
|
+
files,
|
|
2210
|
+
state,
|
|
2211
|
+
session,
|
|
2212
|
+
trace: trace2
|
|
2213
|
+
});
|
|
2309
2214
|
}
|
|
2310
2215
|
);
|
|
2311
|
-
|
|
2312
|
-
return {
|
|
2313
|
-
traceRunId: rawTrace.runId,
|
|
2216
|
+
return await continueWith({
|
|
2314
2217
|
collected,
|
|
2315
2218
|
transcriptTurns,
|
|
2316
2219
|
runConfiguration,
|
|
2317
|
-
|
|
2318
|
-
|
|
2220
|
+
trace: trace2,
|
|
2221
|
+
getSnapshot: () => tee.getSnapshot()
|
|
2222
|
+
});
|
|
2319
2223
|
} catch (error) {
|
|
2320
2224
|
throw withRunConfiguration(error, runConfiguration);
|
|
2321
2225
|
}
|
|
2322
2226
|
} finally {
|
|
2323
|
-
|
|
2324
|
-
try {
|
|
2325
|
-
await trace2.step(
|
|
2326
|
-
{ step_name: "adapter.cleanup", input: { adapterName: adapter.name }, metadata: { adapterName: adapter.name }, summarize: () => ({ cleaned: true }) },
|
|
2327
|
-
async () => adapter.cleanup?.({ task: task2, taskDir: absoluteDir, files, state, session, trace: trace2 })
|
|
2328
|
-
);
|
|
2329
|
-
} catch (error) {
|
|
2330
|
-
console.error("[AgentTask] Cleanup failed:", error instanceof Error ? error.message : String(error));
|
|
2331
|
-
}
|
|
2332
|
-
}
|
|
2333
|
-
if (session?.close) {
|
|
2334
|
-
try {
|
|
2335
|
-
await session.close();
|
|
2336
|
-
} catch {
|
|
2337
|
-
}
|
|
2338
|
-
}
|
|
2227
|
+
await cleanupAdapter(loaded, trace2, state, session);
|
|
2339
2228
|
}
|
|
2340
2229
|
}
|
|
2341
2230
|
);
|
|
2342
2231
|
}
|
|
2232
|
+
async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
|
|
2233
|
+
return executeAdapterRun(loaded, options, rawTrace, async (outcome) => {
|
|
2234
|
+
const {
|
|
2235
|
+
task: task2,
|
|
2236
|
+
adapter,
|
|
2237
|
+
taskDir: absoluteDir,
|
|
2238
|
+
files,
|
|
2239
|
+
checksPath,
|
|
2240
|
+
inlineChecks,
|
|
2241
|
+
moduleUrl,
|
|
2242
|
+
evalFileName
|
|
2243
|
+
} = loaded;
|
|
2244
|
+
const { trace: trace2, collected, transcriptTurns, runConfiguration } = outcome;
|
|
2245
|
+
const validationResults = await trace2.step(
|
|
2246
|
+
{
|
|
2247
|
+
step_name: "deliverables.validate",
|
|
2248
|
+
input: { deliverableNames: task2.deliverables },
|
|
2249
|
+
summarize: (result) => {
|
|
2250
|
+
const r = result;
|
|
2251
|
+
const passCount = r.results.filter((x) => x.pass).length;
|
|
2252
|
+
const broken = Object.keys(r.brokenDeliverables);
|
|
2253
|
+
return {
|
|
2254
|
+
total: r.results.length,
|
|
2255
|
+
passCount,
|
|
2256
|
+
failCount: r.results.length - passCount,
|
|
2257
|
+
brokenDeliverableCount: broken.length,
|
|
2258
|
+
brokenDeliverables: broken
|
|
2259
|
+
};
|
|
2260
|
+
}
|
|
2261
|
+
},
|
|
2262
|
+
async () => validateDeliverables(task2, collected, adapter.deliverables)
|
|
2263
|
+
);
|
|
2264
|
+
const checksResults = await trace2.step(
|
|
2265
|
+
{
|
|
2266
|
+
step_name: "checks.run",
|
|
2267
|
+
input: { sourceFile: inlineChecks ? evalFileName : checksPath },
|
|
2268
|
+
metadata: { sourceFile: inlineChecks ? evalFileName : checksPath },
|
|
2269
|
+
summarize: (result) => summarizeEvaluationResults(result)
|
|
2270
|
+
},
|
|
2271
|
+
async () => {
|
|
2272
|
+
const judgeConfig = resolveJudgeConfig(options?.judge, task2.judge);
|
|
2273
|
+
if (!inlineChecks) {
|
|
2274
|
+
return loadAndRunFlowChecks(
|
|
2275
|
+
checksPath,
|
|
2276
|
+
{
|
|
2277
|
+
snapshot: outcome.getSnapshot(),
|
|
2278
|
+
deliverables: collected,
|
|
2279
|
+
files,
|
|
2280
|
+
task: task2,
|
|
2281
|
+
...judgeConfig ? { judgeConfig } : {}
|
|
2282
|
+
},
|
|
2283
|
+
validationResults.brokenDeliverables
|
|
2284
|
+
);
|
|
2285
|
+
}
|
|
2286
|
+
return runTraceChecks({
|
|
2287
|
+
snapshot: outcome.getSnapshot(),
|
|
2288
|
+
deliverables: proxyBrokenDeliverables(
|
|
2289
|
+
collected,
|
|
2290
|
+
validationResults.brokenDeliverables
|
|
2291
|
+
),
|
|
2292
|
+
files,
|
|
2293
|
+
task: task2,
|
|
2294
|
+
...judgeConfig ? { judgeConfig } : {},
|
|
2295
|
+
moduleUrl,
|
|
2296
|
+
displayFile: evalFileName
|
|
2297
|
+
});
|
|
2298
|
+
}
|
|
2299
|
+
);
|
|
2300
|
+
return {
|
|
2301
|
+
task: task2,
|
|
2302
|
+
taskDir: absoluteDir,
|
|
2303
|
+
files,
|
|
2304
|
+
traceRunId,
|
|
2305
|
+
result: aggregateResult(checksResults),
|
|
2306
|
+
deliverables: collected,
|
|
2307
|
+
transcript: { turns: transcriptTurns },
|
|
2308
|
+
runConfiguration
|
|
2309
|
+
};
|
|
2310
|
+
});
|
|
2311
|
+
}
|
|
2312
|
+
async function captureExecution(loaded, options, rawTrace) {
|
|
2313
|
+
return executeAdapterRun(loaded, options, rawTrace, async (outcome) => ({
|
|
2314
|
+
traceRunId: rawTrace.runId,
|
|
2315
|
+
collected: outcome.collected,
|
|
2316
|
+
transcriptTurns: outcome.transcriptTurns,
|
|
2317
|
+
runConfiguration: outcome.runConfiguration,
|
|
2318
|
+
// Freeze the snapshot before the root span ends — Phase 2 reads it.
|
|
2319
|
+
snapshot: outcome.getSnapshot()
|
|
2320
|
+
}));
|
|
2321
|
+
}
|
|
2343
2322
|
async function evaluate(loaded, options, phase1) {
|
|
2344
2323
|
const {
|
|
2345
2324
|
task: task2,
|
|
@@ -2388,6 +2367,89 @@ async function evaluate(loaded, options, phase1) {
|
|
|
2388
2367
|
runConfiguration: phase1.runConfiguration
|
|
2389
2368
|
};
|
|
2390
2369
|
}
|
|
2370
|
+
async function runTurnLoop(loaded, options, session, trace2, rawTrace) {
|
|
2371
|
+
const { task: task2, adapter, files, checksPath, inlineChecks } = loaded;
|
|
2372
|
+
if (!inlineChecks && checksPath) {
|
|
2373
|
+
resetTaskTurn();
|
|
2374
|
+
resetFlowChecks();
|
|
2375
|
+
await loadChecksModule(checksPath);
|
|
2376
|
+
}
|
|
2377
|
+
const taskTurn = getTaskTurn();
|
|
2378
|
+
resetTaskTurn();
|
|
2379
|
+
const turnFn = resolveTurn(adapter.turn, taskTurn);
|
|
2380
|
+
if (!turnFn) return [];
|
|
2381
|
+
const taskFiles = new TaskFiles(files);
|
|
2382
|
+
const turnTranscript = [];
|
|
2383
|
+
const transcriptTurns = [];
|
|
2384
|
+
let lastTurnResponse;
|
|
2385
|
+
const maxTurns = options?.maxTurnsOverride ?? task2.maxTurns ?? 10;
|
|
2386
|
+
for (let turnNum = 1; turnNum <= maxTurns; turnNum++) {
|
|
2387
|
+
const userTurn = await turnFn({ files: taskFiles, transcript: turnTranscript });
|
|
2388
|
+
if (userTurn === null || userTurn === void 0) break;
|
|
2389
|
+
const result = await trace2.step(
|
|
2390
|
+
{
|
|
2391
|
+
step_name: "task.turn",
|
|
2392
|
+
metadata: { turnNumber: turnNum },
|
|
2393
|
+
summarize: (r) => ({ response: r.response })
|
|
2394
|
+
},
|
|
2395
|
+
async (spanId) => session.sendUserTurn(userTurn, {
|
|
2396
|
+
trace: trace2,
|
|
2397
|
+
turnNumber: turnNum,
|
|
2398
|
+
parentSpanId: spanId
|
|
2399
|
+
})
|
|
2400
|
+
);
|
|
2401
|
+
turnTranscript.push({
|
|
2402
|
+
turnNumber: turnNum,
|
|
2403
|
+
input: userTurn,
|
|
2404
|
+
output: result.response
|
|
2405
|
+
});
|
|
2406
|
+
transcriptTurns.push({
|
|
2407
|
+
turnNumber: turnNum,
|
|
2408
|
+
userAction: userTurn,
|
|
2409
|
+
agentResponse: result.response
|
|
2410
|
+
});
|
|
2411
|
+
options?.onTurn?.(turnNum, userTurn, result.response);
|
|
2412
|
+
lastTurnResponse = result.response;
|
|
2413
|
+
}
|
|
2414
|
+
if (lastTurnResponse !== void 0) {
|
|
2415
|
+
rawTrace.endRoot({ output: { response: lastTurnResponse } });
|
|
2416
|
+
}
|
|
2417
|
+
return transcriptTurns;
|
|
2418
|
+
}
|
|
2419
|
+
async function cleanupAdapter(loaded, trace2, state, session) {
|
|
2420
|
+
const { task: task2, adapter, taskDir: absoluteDir, files } = loaded;
|
|
2421
|
+
if (adapter.cleanup) {
|
|
2422
|
+
try {
|
|
2423
|
+
await trace2.step(
|
|
2424
|
+
{
|
|
2425
|
+
step_name: "adapter.cleanup",
|
|
2426
|
+
input: { adapterName: adapter.name },
|
|
2427
|
+
metadata: { adapterName: adapter.name },
|
|
2428
|
+
summarize: () => ({ cleaned: true })
|
|
2429
|
+
},
|
|
2430
|
+
async () => adapter.cleanup?.({
|
|
2431
|
+
task: task2,
|
|
2432
|
+
taskDir: absoluteDir,
|
|
2433
|
+
files,
|
|
2434
|
+
state,
|
|
2435
|
+
session,
|
|
2436
|
+
trace: trace2
|
|
2437
|
+
})
|
|
2438
|
+
);
|
|
2439
|
+
} catch (error) {
|
|
2440
|
+
console.error(
|
|
2441
|
+
"[AgentTask] Cleanup failed:",
|
|
2442
|
+
error instanceof Error ? error.message : String(error)
|
|
2443
|
+
);
|
|
2444
|
+
}
|
|
2445
|
+
}
|
|
2446
|
+
if (session?.close) {
|
|
2447
|
+
try {
|
|
2448
|
+
await session.close();
|
|
2449
|
+
} catch {
|
|
2450
|
+
}
|
|
2451
|
+
}
|
|
2452
|
+
}
|
|
2391
2453
|
function summarizeEvaluationResults(results) {
|
|
2392
2454
|
const passCount = results.filter((r) => r.pass).length;
|
|
2393
2455
|
return {
|
|
@@ -3303,12 +3365,12 @@ function printOverallSummary(summaries, hasErrors) {
|
|
|
3303
3365
|
}
|
|
3304
3366
|
|
|
3305
3367
|
// src/agent-task/task-revision-manifest.ts
|
|
3306
|
-
import { createHash as
|
|
3368
|
+
import { createHash as createHash3 } from "crypto";
|
|
3307
3369
|
function normalizeManifestPath(path) {
|
|
3308
3370
|
return path.split("\\").join("/").normalize("NFC");
|
|
3309
3371
|
}
|
|
3310
3372
|
function sha256Hex(bytes) {
|
|
3311
|
-
return
|
|
3373
|
+
return createHash3("sha256").update(bytes).digest("hex");
|
|
3312
3374
|
}
|
|
3313
3375
|
function compareBytewiseUtf8(a, b) {
|
|
3314
3376
|
const ba = Buffer.from(a, "utf8");
|
|
@@ -3356,11 +3418,14 @@ export {
|
|
|
3356
3418
|
AgentTaskRunError,
|
|
3357
3419
|
ApoSpanProcessor,
|
|
3358
3420
|
FlowView,
|
|
3421
|
+
JUDGE_SEGMENT_LIMIT,
|
|
3422
|
+
RECEIVED_VALUE_LIMIT,
|
|
3359
3423
|
RejudgeError,
|
|
3360
3424
|
TEST_METHOD_NAMES,
|
|
3361
3425
|
TaskFiles,
|
|
3362
3426
|
buildManifest,
|
|
3363
3427
|
canonicalManifestJson,
|
|
3428
|
+
compactChecksForSubmission,
|
|
3364
3429
|
contentSha256,
|
|
3365
3430
|
createApoAnthropic,
|
|
3366
3431
|
createApoOpenAI,
|