@apo-ai/sdk 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +314 -0
- package/dist/agent-task/integrations/ai-sdk.d.ts +121 -0
- package/dist/agent-task/integrations/ai-sdk.js +10 -0
- package/dist/agent-task/integrations/ai-sdk.js.map +1 -0
- package/dist/agent-task/integrations/anthropic.d.ts +67 -0
- package/dist/agent-task/integrations/anthropic.js +10 -0
- package/dist/agent-task/integrations/anthropic.js.map +1 -0
- package/dist/agent-task/integrations/openai.d.ts +77 -0
- package/dist/agent-task/integrations/openai.js +10 -0
- package/dist/agent-task/integrations/openai.js.map +1 -0
- package/dist/agent-task/public.d.ts +1326 -0
- package/dist/agent-task/public.js +3026 -0
- package/dist/agent-task/public.js.map +1 -0
- package/dist/chunk-DT2LMB7O.js +80 -0
- package/dist/chunk-DT2LMB7O.js.map +1 -0
- package/dist/chunk-GDB6SDGN.js +75 -0
- package/dist/chunk-GDB6SDGN.js.map +1 -0
- package/dist/chunk-HKLBPIB4.js +232 -0
- package/dist/chunk-HKLBPIB4.js.map +1 -0
- package/dist/chunk-HWZVL6TS.js +37 -0
- package/dist/chunk-HWZVL6TS.js.map +1 -0
- package/dist/chunk-PMYLL2H4.js +495 -0
- package/dist/chunk-PMYLL2H4.js.map +1 -0
- package/dist/chunk-SZDSIPXC.js +74 -0
- package/dist/chunk-SZDSIPXC.js.map +1 -0
- package/dist/index.d.ts +95 -0
- package/dist/index.js +39 -0
- package/dist/index.js.map +1 -0
- package/dist/otel/index.d.ts +270 -0
- package/dist/otel/index.js +32 -0
- package/dist/otel/index.js.map +1 -0
- package/dist/tracing-DtbhYQV6.d.ts +26 -0
- package/dist/types-C-nrDEVf.d.ts +140 -0
- package/package.json +92 -0
|
@@ -0,0 +1,3026 @@
|
|
|
1
|
+
// Auto-generated by `pnpm --filter @apo-ai/sdk build` (tsup.config.ts).
|
|
2
|
+
// Do not edit by hand — edit the src/ originals and rebuild.
|
|
3
|
+
import {
|
|
4
|
+
ApoSpanProcessor,
|
|
5
|
+
configureApoTelemetry,
|
|
6
|
+
getActiveApoRun,
|
|
7
|
+
registerApoTracing,
|
|
8
|
+
resetApoTracing,
|
|
9
|
+
withApoRun,
|
|
10
|
+
withApoRunSync
|
|
11
|
+
} from "../chunk-PMYLL2H4.js";
|
|
12
|
+
import {
|
|
13
|
+
createApoTracer
|
|
14
|
+
} from "../chunk-HKLBPIB4.js";
|
|
15
|
+
import "../chunk-HWZVL6TS.js";
|
|
16
|
+
import {
|
|
17
|
+
createApoOpenAI
|
|
18
|
+
} from "../chunk-DT2LMB7O.js";
|
|
19
|
+
import {
|
|
20
|
+
createApoAnthropic
|
|
21
|
+
} from "../chunk-SZDSIPXC.js";
|
|
22
|
+
import "../chunk-GDB6SDGN.js";
|
|
23
|
+
|
|
24
|
+
// src/agent-task/trace-projection/view.ts
|
|
25
|
+
function invocationOrderKey(obs) {
|
|
26
|
+
const hasTs = obs.startedAt != null ? 0 : 1;
|
|
27
|
+
return [hasTs, obs.startedAt ?? "", obs.spanId];
|
|
28
|
+
}
|
|
29
|
+
var TraceView = class {
|
|
30
|
+
constructor(snapshot) {
|
|
31
|
+
this.snapshot = snapshot;
|
|
32
|
+
}
|
|
33
|
+
/** Evidence availability for a capability. */
|
|
34
|
+
requireCapability(capability) {
|
|
35
|
+
return this.snapshot.capabilities[capability];
|
|
36
|
+
}
|
|
37
|
+
/** Whether a capability is reported as available (not partial/unavailable). */
|
|
38
|
+
isAvailable(capability) {
|
|
39
|
+
return this.snapshot.capabilities[capability] === "available";
|
|
40
|
+
}
|
|
41
|
+
/** All chat messages flattened across generation observations, in order. */
|
|
42
|
+
get messages() {
|
|
43
|
+
const out = [];
|
|
44
|
+
for (const obs of this.sortedObservations) {
|
|
45
|
+
if (obs.messages) out.push(...obs.messages);
|
|
46
|
+
}
|
|
47
|
+
return out;
|
|
48
|
+
}
|
|
49
|
+
/** Tool calls derived from `TOOL` observations, in invocation order. */
|
|
50
|
+
get toolCalls() {
|
|
51
|
+
return this.sortedObservations.filter((o) => o.type === "TOOL").map((o) => ({
|
|
52
|
+
spanId: o.spanId,
|
|
53
|
+
name: o.toolName ?? o.name,
|
|
54
|
+
input: o.toolParameters ?? o.input,
|
|
55
|
+
output: o.toolResult ?? o.output,
|
|
56
|
+
status: o.status,
|
|
57
|
+
startedAt: o.startedAt
|
|
58
|
+
}));
|
|
59
|
+
}
|
|
60
|
+
/** Tool names in invocation order with deterministic span-ID tie-breaking. */
|
|
61
|
+
get toolNamesInOrder() {
|
|
62
|
+
return this.toolCalls.map((c) => c.name);
|
|
63
|
+
}
|
|
64
|
+
/** Skill loads derived from `SKILL` observations, in invocation order. */
|
|
65
|
+
get skillLoads() {
|
|
66
|
+
return this.sortedObservations.filter((o) => o.type === "SKILL").map((o) => ({ spanId: o.spanId, skill: o.name, startedAt: o.startedAt }));
|
|
67
|
+
}
|
|
68
|
+
/** Subagent delegations derived from `AGENT` observations, in invocation order. */
|
|
69
|
+
get subagentCalls() {
|
|
70
|
+
return this.sortedObservations.filter((o) => o.type === "AGENT").map((o) => ({
|
|
71
|
+
spanId: o.spanId,
|
|
72
|
+
agent: o.name,
|
|
73
|
+
output: o.output,
|
|
74
|
+
status: o.status,
|
|
75
|
+
startedAt: o.startedAt
|
|
76
|
+
}));
|
|
77
|
+
}
|
|
78
|
+
/** Last assistant message content — the agent's "reply". Empty if none. */
|
|
79
|
+
get reply() {
|
|
80
|
+
const msgs = this.messages;
|
|
81
|
+
for (let i = msgs.length - 1; i >= 0; i--) {
|
|
82
|
+
if (msgs[i].role === "assistant") return msgs[i].content;
|
|
83
|
+
}
|
|
84
|
+
return "";
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Number of completed assistant turns, or `undefined` when message evidence
|
|
88
|
+
* is unavailable (capability honesty).
|
|
89
|
+
*/
|
|
90
|
+
get turnCount() {
|
|
91
|
+
if (!this.isAvailable("messages")) return void 0;
|
|
92
|
+
return this.messages.filter((m) => m.role === "assistant").length;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Number of tool/subagent observations that reported an error, or `undefined`
|
|
96
|
+
* when error evidence is unavailable (capability honesty).
|
|
97
|
+
*/
|
|
98
|
+
get failedActions() {
|
|
99
|
+
if (!this.isAvailable("errors")) return void 0;
|
|
100
|
+
return this.sortedObservations.filter(
|
|
101
|
+
(o) => (o.type === "TOOL" || o.type === "AGENT") && o.status === "error"
|
|
102
|
+
).length;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Total execution duration in milliseconds, or `undefined` when timing
|
|
106
|
+
* evidence is unavailable (capability honesty). Uses trace-level
|
|
107
|
+
* startedAt/endedAt when present.
|
|
108
|
+
*/
|
|
109
|
+
get durationMs() {
|
|
110
|
+
if (!this.isAvailable("timing")) return void 0;
|
|
111
|
+
const { startedAt, endedAt } = this.snapshot.trace;
|
|
112
|
+
if (startedAt != null && endedAt != null) {
|
|
113
|
+
const ms = Date.parse(endedAt) - Date.parse(startedAt);
|
|
114
|
+
return Number.isNaN(ms) ? void 0 : Math.max(0, ms);
|
|
115
|
+
}
|
|
116
|
+
return void 0;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Observations sorted deterministically by invocation time then span ID.
|
|
120
|
+
* Memoized per TraceView instance since the snapshot is immutable.
|
|
121
|
+
*/
|
|
122
|
+
get sortedObservations() {
|
|
123
|
+
if (this._sorted !== void 0) return this._sorted;
|
|
124
|
+
const sorted = [...this.snapshot.observations].sort((a, b) => {
|
|
125
|
+
const ka = invocationOrderKey(a);
|
|
126
|
+
const kb = invocationOrderKey(b);
|
|
127
|
+
return ka[0] - kb[0] || ka[1].localeCompare(kb[1]) || ka[2].localeCompare(kb[2]);
|
|
128
|
+
});
|
|
129
|
+
this._sorted = sorted;
|
|
130
|
+
return sorted;
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
// src/agent-task/checks/matchers.ts
|
|
135
|
+
function asString(value) {
|
|
136
|
+
if (typeof value === "string") return value;
|
|
137
|
+
if (value === void 0) return "undefined";
|
|
138
|
+
if (value === null) return "null";
|
|
139
|
+
try {
|
|
140
|
+
return JSON.stringify(value);
|
|
141
|
+
} catch {
|
|
142
|
+
return String(value);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
function describeValue(value) {
|
|
146
|
+
return asString(value);
|
|
147
|
+
}
|
|
148
|
+
function escapeRegExp(s) {
|
|
149
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
150
|
+
}
|
|
151
|
+
function deepEqual(a, b) {
|
|
152
|
+
if (a === b) return true;
|
|
153
|
+
if (a === null || b === null || typeof a !== "object" || typeof b !== "object") {
|
|
154
|
+
return false;
|
|
155
|
+
}
|
|
156
|
+
if (Array.isArray(a) !== Array.isArray(b)) return false;
|
|
157
|
+
if (Array.isArray(a) && Array.isArray(b)) {
|
|
158
|
+
if (a.length !== b.length) return false;
|
|
159
|
+
return a.every((v, i) => deepEqual(v, b[i]));
|
|
160
|
+
}
|
|
161
|
+
const ao = a;
|
|
162
|
+
const bo = b;
|
|
163
|
+
const ak = Object.keys(ao);
|
|
164
|
+
const bk = Object.keys(bo);
|
|
165
|
+
if (ak.length !== bk.length) return false;
|
|
166
|
+
return ak.every(
|
|
167
|
+
(k) => Object.prototype.hasOwnProperty.call(bo, k) && deepEqual(ao[k], bo[k])
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
function includes(needle) {
|
|
171
|
+
const re = typeof needle === "string" ? new RegExp(escapeRegExp(needle)) : needle;
|
|
172
|
+
return {
|
|
173
|
+
label: `includes ${needle instanceof RegExp ? needle.source : needle}`,
|
|
174
|
+
test: (v) => re.test(asString(v))
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
function equals(expected) {
|
|
178
|
+
return {
|
|
179
|
+
label: `equals ${asString(expected)}`,
|
|
180
|
+
test: (v) => deepEqual(v, expected)
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
function matches(schema) {
|
|
184
|
+
return {
|
|
185
|
+
label: "matches schema",
|
|
186
|
+
test: (v) => schema.safeParse(v).success === true
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
function satisfies(predicate, label) {
|
|
190
|
+
return { label, test: predicate };
|
|
191
|
+
}
|
|
192
|
+
function matchValue(actual, matcher) {
|
|
193
|
+
if (matcher instanceof RegExp) return matcher.test(asString(actual));
|
|
194
|
+
if (typeof matcher === "function") {
|
|
195
|
+
return matcher(actual);
|
|
196
|
+
}
|
|
197
|
+
if (matcher !== null && typeof matcher === "object") {
|
|
198
|
+
if (actual === null || typeof actual !== "object") return false;
|
|
199
|
+
const a = actual;
|
|
200
|
+
const m = matcher;
|
|
201
|
+
return Object.keys(m).every(
|
|
202
|
+
(k) => Object.prototype.hasOwnProperty.call(a, k) && matchValue(a[k], m[k])
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
return deepEqual(actual, matcher);
|
|
206
|
+
}
|
|
207
|
+
function similarity(expected, threshold = 0.8) {
|
|
208
|
+
return {
|
|
209
|
+
label: `similar to "${expected}" (\u2265 ${threshold})`,
|
|
210
|
+
test: (v) => normalizedSimilarity(asString(v), expected) >= threshold
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
function normalizedSimilarity(a, b) {
|
|
214
|
+
const maxLen = Math.max(a.length, b.length);
|
|
215
|
+
if (maxLen === 0) return 1;
|
|
216
|
+
return 1 - levenshtein(a, b) / maxLen;
|
|
217
|
+
}
|
|
218
|
+
function levenshtein(a, b) {
|
|
219
|
+
const m = a.length;
|
|
220
|
+
const n = b.length;
|
|
221
|
+
if (m === 0) return n;
|
|
222
|
+
if (n === 0) return m;
|
|
223
|
+
let prev = Array.from({ length: n + 1 }, () => 0);
|
|
224
|
+
let curr = Array.from({ length: n + 1 }, () => 0);
|
|
225
|
+
for (let j = 0; j <= n; j++) prev[j] = j;
|
|
226
|
+
for (let i = 1; i <= m; i++) {
|
|
227
|
+
curr[0] = i;
|
|
228
|
+
for (let j = 1; j <= n; j++) {
|
|
229
|
+
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
|
230
|
+
curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost);
|
|
231
|
+
}
|
|
232
|
+
[prev, curr] = [curr, prev];
|
|
233
|
+
}
|
|
234
|
+
return prev[n];
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// src/agent-task/checks/judge.ts
|
|
238
|
+
var JUDGE_SYSTEM_PROMPT = 'You are an evaluation judge. Evaluate the given value(s) against the instruction. Respond with ONLY a JSON object: {"pass": true/false, "reasoning": "your reasoning"}';
|
|
239
|
+
function formatValue(value, depth = 0) {
|
|
240
|
+
const indent = " ".repeat(depth);
|
|
241
|
+
if (value === null || value === void 0) return String(value);
|
|
242
|
+
if (typeof value === "string") return value;
|
|
243
|
+
if (typeof value === "number" || typeof value === "boolean") return String(value);
|
|
244
|
+
if (Array.isArray(value)) {
|
|
245
|
+
return value.map((item) => `${indent}${formatValue(item, depth + 1)}`).join("\n");
|
|
246
|
+
}
|
|
247
|
+
if (typeof value === "object") {
|
|
248
|
+
return Object.entries(value).map(([key, val]) => {
|
|
249
|
+
if (val && typeof val === "object") {
|
|
250
|
+
return `${indent}${key}:
|
|
251
|
+
${formatValue(val, depth + 1)}`;
|
|
252
|
+
}
|
|
253
|
+
return `${indent}${key}: ${formatValue(val, depth + 1)}`;
|
|
254
|
+
}).join("\n");
|
|
255
|
+
}
|
|
256
|
+
return String(value);
|
|
257
|
+
}
|
|
258
|
+
function formatJudgeValues(values) {
|
|
259
|
+
if (values.length === 1) return formatValue(values[0]);
|
|
260
|
+
return values.map((v, i) => `--- Value ${i + 1} ---
|
|
261
|
+
${formatValue(v)}`).join("\n\n");
|
|
262
|
+
}
|
|
263
|
+
function parseJudgeUsage(usage) {
|
|
264
|
+
if (!usage) return void 0;
|
|
265
|
+
const cacheCreation = usage.cache_creation_input_tokens ?? usage.prompt_tokens_details?.cache_write_tokens;
|
|
266
|
+
const cacheRead = usage.cache_read_input_tokens ?? usage.prompt_tokens_details?.cached_tokens;
|
|
267
|
+
const tokens = {
|
|
268
|
+
input: usage.prompt_tokens ?? 0,
|
|
269
|
+
output: usage.completion_tokens ?? 0
|
|
270
|
+
};
|
|
271
|
+
if (typeof cacheCreation === "number") tokens.cache_creation = cacheCreation;
|
|
272
|
+
if (typeof cacheRead === "number") tokens.cache_read = cacheRead;
|
|
273
|
+
return tokens;
|
|
274
|
+
}
|
|
275
|
+
function parseJudgeJson(raw) {
|
|
276
|
+
try {
|
|
277
|
+
return JSON.parse(raw);
|
|
278
|
+
} catch {
|
|
279
|
+
}
|
|
280
|
+
const fenced = raw.match(/```(?:json)?\s*([\s\S]*?)```/);
|
|
281
|
+
if (fenced) {
|
|
282
|
+
try {
|
|
283
|
+
return JSON.parse(fenced[1].trim());
|
|
284
|
+
} catch {
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
const block = raw.match(/\{[\s\S]*\}/);
|
|
288
|
+
if (block) {
|
|
289
|
+
try {
|
|
290
|
+
return JSON.parse(block[0]);
|
|
291
|
+
} catch {
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
return {
|
|
295
|
+
pass: false,
|
|
296
|
+
reasoning: "Judge response could not be parsed as JSON \u2014 the verdict is unknown, so this check is treated as a failure. The model's raw response is available in the judge metadata."
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
var prefixQueues = /* @__PURE__ */ new Map();
|
|
300
|
+
function runWithSharedPrefix(key, task2) {
|
|
301
|
+
const prev = prefixQueues.get(key) ?? Promise.resolve();
|
|
302
|
+
const next = prev.then(task2, task2);
|
|
303
|
+
prefixQueues.set(
|
|
304
|
+
key,
|
|
305
|
+
next.then(
|
|
306
|
+
() => void 0,
|
|
307
|
+
() => void 0
|
|
308
|
+
)
|
|
309
|
+
);
|
|
310
|
+
return next;
|
|
311
|
+
}
|
|
312
|
+
async function callJudge(args) {
|
|
313
|
+
const baseURL = args.baseURL ?? process.env.OPENROUTER_BASE_URL ?? "https://openrouter.ai/api/v1";
|
|
314
|
+
const apiKey = args.apiKey ?? process.env.OPENROUTER_API_KEY ?? process.env.OPENAI_API_KEY;
|
|
315
|
+
const deliverableText = `Values to evaluate:
|
|
316
|
+
${formatJudgeValues(args.values)}`;
|
|
317
|
+
const instructionText = `Instruction:
|
|
318
|
+
${args.instruction}`;
|
|
319
|
+
const systemPromptText = `${JUDGE_SYSTEM_PROMPT}
|
|
320
|
+
|
|
321
|
+
${deliverableText}`;
|
|
322
|
+
const cacheKey = `${args.model}\0${deliverableText}`;
|
|
323
|
+
return runWithSharedPrefix(cacheKey, async () => {
|
|
324
|
+
const startedAt = Date.now();
|
|
325
|
+
const response = await fetch(`${baseURL}/chat/completions`, {
|
|
326
|
+
method: "POST",
|
|
327
|
+
headers: {
|
|
328
|
+
"Content-Type": "application/json",
|
|
329
|
+
...apiKey ? { Authorization: `Bearer ${apiKey}` } : {}
|
|
330
|
+
},
|
|
331
|
+
body: JSON.stringify({
|
|
332
|
+
model: args.model,
|
|
333
|
+
messages: [
|
|
334
|
+
{
|
|
335
|
+
role: "system",
|
|
336
|
+
content: [
|
|
337
|
+
{ type: "text", text: JUDGE_SYSTEM_PROMPT },
|
|
338
|
+
{
|
|
339
|
+
type: "text",
|
|
340
|
+
text: deliverableText,
|
|
341
|
+
cache_control: { type: "ephemeral" }
|
|
342
|
+
}
|
|
343
|
+
]
|
|
344
|
+
},
|
|
345
|
+
{ role: "user", content: instructionText }
|
|
346
|
+
],
|
|
347
|
+
temperature: 0,
|
|
348
|
+
response_format: { type: "json_object" }
|
|
349
|
+
})
|
|
350
|
+
});
|
|
351
|
+
if (!response.ok) {
|
|
352
|
+
const body = await response.text().catch(() => "");
|
|
353
|
+
throw new Error(`Judge API ${response.status}: ${body.slice(0, 200)}`);
|
|
354
|
+
}
|
|
355
|
+
const data = await response.json();
|
|
356
|
+
const text = data.choices?.[0]?.message?.content ?? "";
|
|
357
|
+
const outputTokens = data.usage?.completion_tokens;
|
|
358
|
+
const isEmpty = !text.trim();
|
|
359
|
+
const reportedZeroTokens = data.usage !== void 0 && outputTokens === 0;
|
|
360
|
+
if (isEmpty || reportedZeroTokens) {
|
|
361
|
+
return {
|
|
362
|
+
pass: false,
|
|
363
|
+
reasoning: "Judge returned an empty or truncated response \u2014 likely a transient provider failure. The verdict is unknown, so this check is treated as a failure.",
|
|
364
|
+
judge: {
|
|
365
|
+
model: args.model,
|
|
366
|
+
prompt: { system: systemPromptText, user: instructionText },
|
|
367
|
+
response: text,
|
|
368
|
+
tokens: parseJudgeUsage(data.usage),
|
|
369
|
+
latency_ms: Date.now() - startedAt
|
|
370
|
+
}
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
const parsed = parseJudgeJson(text);
|
|
374
|
+
return {
|
|
375
|
+
pass: parsed.pass === true,
|
|
376
|
+
reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : "",
|
|
377
|
+
judge: {
|
|
378
|
+
model: args.model,
|
|
379
|
+
prompt: { system: systemPromptText, user: instructionText },
|
|
380
|
+
response: text,
|
|
381
|
+
tokens: parseJudgeUsage(data.usage),
|
|
382
|
+
latency_ms: Date.now() - startedAt
|
|
383
|
+
}
|
|
384
|
+
};
|
|
385
|
+
});
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
// src/agent-task/checks/t.ts
|
|
389
|
+
function resolveJudgeConfig(judgeConfig, override) {
|
|
390
|
+
const model = override?.model ?? judgeConfig?.model;
|
|
391
|
+
if (!model) return void 0;
|
|
392
|
+
return {
|
|
393
|
+
model,
|
|
394
|
+
baseURL: override?.baseURL ?? judgeConfig?.baseURL,
|
|
395
|
+
apiKey: override?.apiKey ?? judgeConfig?.apiKey
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
var TEST_METHOD_NAMES = [
|
|
399
|
+
"calledTool",
|
|
400
|
+
"notCalledTool",
|
|
401
|
+
"toolOrder",
|
|
402
|
+
"usedNoTools",
|
|
403
|
+
"maxToolCalls",
|
|
404
|
+
"noFailedActions",
|
|
405
|
+
"loadedSkill",
|
|
406
|
+
"calledSubagent",
|
|
407
|
+
"messageIncludes",
|
|
408
|
+
"maxTurns",
|
|
409
|
+
"maxDurationMs",
|
|
410
|
+
"assert",
|
|
411
|
+
"check",
|
|
412
|
+
"judge"
|
|
413
|
+
];
|
|
414
|
+
function describeName(m) {
|
|
415
|
+
if (typeof m === "string") return `"${m}"`;
|
|
416
|
+
if (m instanceof RegExp) return m.source;
|
|
417
|
+
return "<predicate>";
|
|
418
|
+
}
|
|
419
|
+
function matchName(name, m) {
|
|
420
|
+
if (typeof m === "string") return name === m;
|
|
421
|
+
if (m instanceof RegExp) return m.test(name);
|
|
422
|
+
return m(name);
|
|
423
|
+
}
|
|
424
|
+
var CAPABILITY_LABELS = {
|
|
425
|
+
messages: "message",
|
|
426
|
+
tools: "tool",
|
|
427
|
+
errors: "error",
|
|
428
|
+
timing: "timing",
|
|
429
|
+
skills: "skill",
|
|
430
|
+
subagents: "subagent"
|
|
431
|
+
};
|
|
432
|
+
function createTraceTestContext(view, rec, judgeConfig) {
|
|
433
|
+
const unsupported = (id, capability) => {
|
|
434
|
+
const label = CAPABILITY_LABELS[capability];
|
|
435
|
+
rec.record(id, false, `${label} evidence is unavailable in this trace projection`, {
|
|
436
|
+
outcome: "unsupported",
|
|
437
|
+
expected: `${label} evidence available`,
|
|
438
|
+
received: view.requireCapability(capability)
|
|
439
|
+
});
|
|
440
|
+
};
|
|
441
|
+
const isAvailable = (c) => view.requireCapability(c) === "available";
|
|
442
|
+
return {
|
|
443
|
+
calledTool(name, opts) {
|
|
444
|
+
if (!isAvailable("tools")) {
|
|
445
|
+
unsupported(`calledTool(${describeName(name)})`, "tools");
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
448
|
+
if (opts?.status !== void 0 && !isAvailable("errors")) {
|
|
449
|
+
unsupported(`calledTool(${describeName(name)}, { status })`, "errors");
|
|
450
|
+
return;
|
|
451
|
+
}
|
|
452
|
+
const count = view.toolCalls.filter(
|
|
453
|
+
(call) => matchesTraceToolCall(call, name, opts)
|
|
454
|
+
).length;
|
|
455
|
+
const want = opts?.count;
|
|
456
|
+
const pass = want === void 0 ? count > 0 : count === want;
|
|
457
|
+
rec.record(
|
|
458
|
+
`calledTool(${describeName(name)})`,
|
|
459
|
+
pass,
|
|
460
|
+
pass ? "" : want === void 0 ? `expected at least one ${describeName(name)} call, got ${count}` : `expected exactly ${want} ${describeName(name)} calls, got ${count}`,
|
|
461
|
+
{
|
|
462
|
+
expected: want === void 0 ? `\u22651 ${describeName(name)} call` : `exactly ${want} ${describeName(name)} calls`,
|
|
463
|
+
received: `${count}`
|
|
464
|
+
}
|
|
465
|
+
);
|
|
466
|
+
},
|
|
467
|
+
notCalledTool(name, opts) {
|
|
468
|
+
if (!isAvailable("tools")) {
|
|
469
|
+
unsupported(`notCalledTool(${describeName(name)})`, "tools");
|
|
470
|
+
return;
|
|
471
|
+
}
|
|
472
|
+
const count = view.toolCalls.filter(
|
|
473
|
+
(call) => matchesTraceToolCall(call, name, opts)
|
|
474
|
+
).length;
|
|
475
|
+
rec.record(
|
|
476
|
+
`notCalledTool(${describeName(name)})`,
|
|
477
|
+
count === 0,
|
|
478
|
+
count === 0 ? "" : `expected no ${describeName(name)} calls, got ${count}`,
|
|
479
|
+
{ expected: `0 ${describeName(name)} calls`, received: `${count}` }
|
|
480
|
+
);
|
|
481
|
+
},
|
|
482
|
+
toolOrder(names) {
|
|
483
|
+
if (!isAvailable("tools")) {
|
|
484
|
+
unsupported(`toolOrder(${names.join(" \u2192 ")})`, "tools");
|
|
485
|
+
return;
|
|
486
|
+
}
|
|
487
|
+
const order = view.toolNamesInOrder;
|
|
488
|
+
let cursor = 0;
|
|
489
|
+
let missingIdx = -1;
|
|
490
|
+
for (let i = 0; i < names.length; i++) {
|
|
491
|
+
const found = order.indexOf(names[i], cursor);
|
|
492
|
+
if (found === -1) {
|
|
493
|
+
missingIdx = i;
|
|
494
|
+
break;
|
|
495
|
+
}
|
|
496
|
+
cursor = found + 1;
|
|
497
|
+
}
|
|
498
|
+
const ok = missingIdx === -1;
|
|
499
|
+
const expectedStr = names.join(" \u2192 ");
|
|
500
|
+
const actualStr = order.length > 0 ? order.join(" \u2192 ") : "(no tools)";
|
|
501
|
+
const message = ok ? "" : missingIdx === 0 ? `expected "${names[0]}" first; actual order ${actualStr}` : `expected "${names[missingIdx]}" after "${names[missingIdx - 1]}"; actual order ${actualStr}`;
|
|
502
|
+
rec.record(`toolOrder(${expectedStr})`, ok, message, {
|
|
503
|
+
expected: expectedStr,
|
|
504
|
+
received: actualStr
|
|
505
|
+
});
|
|
506
|
+
},
|
|
507
|
+
usedNoTools() {
|
|
508
|
+
if (!isAvailable("tools")) {
|
|
509
|
+
unsupported("usedNoTools", "tools");
|
|
510
|
+
return;
|
|
511
|
+
}
|
|
512
|
+
const count = view.toolCalls.length;
|
|
513
|
+
rec.record("usedNoTools", count === 0, count === 0 ? "" : `expected no tool calls, got ${count}`, {
|
|
514
|
+
expected: "0 tool calls",
|
|
515
|
+
received: `${count}`
|
|
516
|
+
});
|
|
517
|
+
},
|
|
518
|
+
maxToolCalls(n) {
|
|
519
|
+
if (!isAvailable("tools")) {
|
|
520
|
+
unsupported(`maxToolCalls(${n})`, "tools");
|
|
521
|
+
return;
|
|
522
|
+
}
|
|
523
|
+
const count = view.toolCalls.length;
|
|
524
|
+
rec.record(`maxToolCalls(${n})`, count <= n, count <= n ? "" : `expected \u2264 ${n} tool calls, got ${count}`, {
|
|
525
|
+
expected: `\u2264 ${n} tool calls`,
|
|
526
|
+
received: `${count}`
|
|
527
|
+
});
|
|
528
|
+
},
|
|
529
|
+
noFailedActions() {
|
|
530
|
+
if (!isAvailable("errors")) {
|
|
531
|
+
unsupported("noFailedActions", "errors");
|
|
532
|
+
return;
|
|
533
|
+
}
|
|
534
|
+
const failed = view.failedActions ?? 0;
|
|
535
|
+
rec.record("noFailedActions", failed === 0, failed === 0 ? "" : `${failed} failed action(s)`, {
|
|
536
|
+
expected: "0 failed actions",
|
|
537
|
+
received: `${failed}`
|
|
538
|
+
});
|
|
539
|
+
},
|
|
540
|
+
loadedSkill(skill) {
|
|
541
|
+
if (!isAvailable("skills")) {
|
|
542
|
+
unsupported(`loadedSkill("${skill}")`, "skills");
|
|
543
|
+
return;
|
|
544
|
+
}
|
|
545
|
+
const ok = view.skillLoads.some((s) => s.skill === skill);
|
|
546
|
+
rec.record(`loadedSkill("${skill}")`, ok, ok ? "" : `skill "${skill}" was not loaded`, {
|
|
547
|
+
expected: `skill "${skill}" loaded`,
|
|
548
|
+
received: view.skillLoads.map((s) => s.skill).join(", ") || "none"
|
|
549
|
+
});
|
|
550
|
+
},
|
|
551
|
+
calledSubagent(agent) {
|
|
552
|
+
if (!isAvailable("subagents")) {
|
|
553
|
+
unsupported(`calledSubagent("${agent}")`, "subagents");
|
|
554
|
+
return;
|
|
555
|
+
}
|
|
556
|
+
const ok = view.subagentCalls.some((s) => s.agent === agent);
|
|
557
|
+
rec.record(`calledSubagent("${agent}")`, ok, ok ? "" : `subagent "${agent}" was not called`, {
|
|
558
|
+
expected: `subagent "${agent}" called`,
|
|
559
|
+
received: view.subagentCalls.map((s) => s.agent).join(", ") || "none"
|
|
560
|
+
});
|
|
561
|
+
},
|
|
562
|
+
messageIncludes(token) {
|
|
563
|
+
if (!isAvailable("messages")) {
|
|
564
|
+
unsupported(`messageIncludes(${token instanceof RegExp ? token.source : token})`, "messages");
|
|
565
|
+
return;
|
|
566
|
+
}
|
|
567
|
+
const re = typeof token === "string" ? new RegExp(token) : token;
|
|
568
|
+
const ok = re.test(view.reply);
|
|
569
|
+
rec.record(`messageIncludes(${token instanceof RegExp ? token.source : token})`, ok, ok ? "" : `reply did not include ${token}`, {
|
|
570
|
+
expected: `reply includes ${token instanceof RegExp ? token.source : token}`,
|
|
571
|
+
received: describeValue(view.reply).slice(0, 200)
|
|
572
|
+
});
|
|
573
|
+
},
|
|
574
|
+
maxTurns(n) {
|
|
575
|
+
if (!isAvailable("messages")) {
|
|
576
|
+
unsupported(`maxTurns(${n})`, "messages");
|
|
577
|
+
return;
|
|
578
|
+
}
|
|
579
|
+
const turns = view.turnCount ?? 0;
|
|
580
|
+
rec.record(`maxTurns(${n})`, turns <= n, turns <= n ? "" : `expected \u2264 ${n} turns, got ${turns}`, {
|
|
581
|
+
expected: `\u2264 ${n} turns`,
|
|
582
|
+
received: `${turns}`
|
|
583
|
+
});
|
|
584
|
+
},
|
|
585
|
+
maxDurationMs(n) {
|
|
586
|
+
if (!isAvailable("timing")) {
|
|
587
|
+
unsupported(`maxDurationMs(${n})`, "timing");
|
|
588
|
+
return;
|
|
589
|
+
}
|
|
590
|
+
const ms = view.durationMs;
|
|
591
|
+
if (ms === void 0) {
|
|
592
|
+
unsupported(`maxDurationMs(${n})`, "timing");
|
|
593
|
+
return;
|
|
594
|
+
}
|
|
595
|
+
rec.record(`maxDurationMs(${n})`, ms <= n, ms <= n ? "" : `expected \u2264 ${n}ms, took ${ms}ms`, {
|
|
596
|
+
expected: `\u2264 ${n}ms`,
|
|
597
|
+
received: `${ms}ms`
|
|
598
|
+
});
|
|
599
|
+
},
|
|
600
|
+
assert(label, predicate) {
|
|
601
|
+
const pass = predicate(view);
|
|
602
|
+
rec.record(label, pass, pass ? "" : `assertion "${label}" failed`, {
|
|
603
|
+
expected: label,
|
|
604
|
+
received: pass ? "true" : "false"
|
|
605
|
+
});
|
|
606
|
+
},
|
|
607
|
+
check(value, matcher, label) {
|
|
608
|
+
const pass = matcher.test(value);
|
|
609
|
+
rec.record(
|
|
610
|
+
label ?? "check",
|
|
611
|
+
pass,
|
|
612
|
+
pass ? "" : `expected ${label ?? "value"} ${matcher.label}`,
|
|
613
|
+
{ expected: matcher.label, received: describeValue(value) }
|
|
614
|
+
);
|
|
615
|
+
},
|
|
616
|
+
async judge(values, instruction, opts) {
|
|
617
|
+
const label = opts?.label ?? "judge";
|
|
618
|
+
const location = rec.captureLocation();
|
|
619
|
+
const valueArray = Array.isArray(values) ? values : [values];
|
|
620
|
+
const effective = resolveJudgeConfig(judgeConfig, opts?.judge);
|
|
621
|
+
if (!effective) {
|
|
622
|
+
rec.record(
|
|
623
|
+
label,
|
|
624
|
+
false,
|
|
625
|
+
"No judge model configured. Set one of:\n\u2022 OPENROUTER_MODEL + OPENROUTER_API_KEY (OpenRouter \u2014 works with 200+ models, one account)\n\u2022 OPENAI_MODEL + OPENAI_API_KEY (OpenAI direct)\nOr pass { judge } to runTask() programmatically.",
|
|
626
|
+
{ evaluator_type: "llm", location }
|
|
627
|
+
);
|
|
628
|
+
return;
|
|
629
|
+
}
|
|
630
|
+
try {
|
|
631
|
+
const { pass, reasoning, judge } = await callJudge({
|
|
632
|
+
values: valueArray,
|
|
633
|
+
instruction,
|
|
634
|
+
model: effective.model,
|
|
635
|
+
baseURL: effective.baseURL,
|
|
636
|
+
apiKey: effective.apiKey
|
|
637
|
+
});
|
|
638
|
+
rec.record(label, pass, reasoning, {
|
|
639
|
+
evaluator_type: "llm",
|
|
640
|
+
judge,
|
|
641
|
+
expected: instruction,
|
|
642
|
+
received: valueArray.length === 1 ? valueArray[0] : valueArray,
|
|
643
|
+
location
|
|
644
|
+
});
|
|
645
|
+
} catch (error) {
|
|
646
|
+
rec.record(
|
|
647
|
+
label,
|
|
648
|
+
false,
|
|
649
|
+
`judge failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
650
|
+
{
|
|
651
|
+
evaluator_type: "llm",
|
|
652
|
+
expected: instruction,
|
|
653
|
+
received: valueArray.length === 1 ? valueArray[0] : valueArray,
|
|
654
|
+
location
|
|
655
|
+
}
|
|
656
|
+
);
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
function matchesTraceToolCall(call, name, opts) {
|
|
662
|
+
if (!matchName(call.name, name)) return false;
|
|
663
|
+
if (opts?.input !== void 0 && !matchValue(call.input, opts.input)) {
|
|
664
|
+
return false;
|
|
665
|
+
}
|
|
666
|
+
if (opts?.output !== void 0 && !matchValue(call.output, opts.output)) {
|
|
667
|
+
return false;
|
|
668
|
+
}
|
|
669
|
+
if (opts?.status !== void 0 && call.status !== opts.status) {
|
|
670
|
+
return false;
|
|
671
|
+
}
|
|
672
|
+
return true;
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
// src/agent-task/checks/recorder.ts
|
|
676
|
+
var Recorder = class {
|
|
677
|
+
constructor(locate) {
|
|
678
|
+
this.records = [];
|
|
679
|
+
this.locate = locate;
|
|
680
|
+
}
|
|
681
|
+
/**
|
|
682
|
+
* Record an assertion.
|
|
683
|
+
*
|
|
684
|
+
* - ``extra.location`` overrides auto-capture (used by the runner for thrown
|
|
685
|
+
* errors, whose relevant stack is the error's own, not the call site).
|
|
686
|
+
* - ``extra.expected`` / ``extra.received`` carry the structured values for
|
|
687
|
+
* testing-framework-style display.
|
|
688
|
+
*/
|
|
689
|
+
/**
|
|
690
|
+
* Capture the call site location synchronously. Use this when a record will
|
|
691
|
+
* happen AFTER an `await` (e.g. `t.judge`): once an async function resumes,
|
|
692
|
+
* `new Error().stack` reports the caller's frame at an unreliable line
|
|
693
|
+
* (often the statement's closing brace). Capturing here, before the await,
|
|
694
|
+
* pins the location to the actual call line; pass it to ``record`` via
|
|
695
|
+
* ``extra.location``.
|
|
696
|
+
*/
|
|
697
|
+
captureLocation() {
|
|
698
|
+
return this.locate ? this.locate(new Error().stack ?? "") : void 0;
|
|
699
|
+
}
|
|
700
|
+
record(id, pass, reasoning, extra) {
|
|
701
|
+
const location = extra?.location ?? (this.locate ? this.locate(new Error().stack ?? "") : void 0);
|
|
702
|
+
this.records.push({
|
|
703
|
+
id,
|
|
704
|
+
pass,
|
|
705
|
+
reasoning,
|
|
706
|
+
location,
|
|
707
|
+
expected: extra?.expected,
|
|
708
|
+
received: extra?.received,
|
|
709
|
+
evaluator_type: extra?.evaluator_type,
|
|
710
|
+
judge: extra?.judge,
|
|
711
|
+
outcome: extra?.outcome
|
|
712
|
+
});
|
|
713
|
+
}
|
|
714
|
+
get all() {
|
|
715
|
+
return this.records;
|
|
716
|
+
}
|
|
717
|
+
};
|
|
718
|
+
|
|
719
|
+
// src/agent-task/checks/location.ts
|
|
720
|
+
var FRAME_TAIL = /:(\d+):(\d+)\)?\s*$/;
|
|
721
|
+
function parseCheckLocation(stack, moduleUrl, displayFile) {
|
|
722
|
+
if (!stack || !moduleUrl) return void 0;
|
|
723
|
+
const needle = moduleUrl.replace(/^file:\/\//, "");
|
|
724
|
+
for (const line of stack.split("\n")) {
|
|
725
|
+
if (!line.includes(moduleUrl) && !line.includes(needle)) continue;
|
|
726
|
+
const m = line.match(FRAME_TAIL);
|
|
727
|
+
if (m) {
|
|
728
|
+
return { file: displayFile, line: Number(m[1]), column: Number(m[2]) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
return void 0;
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
// src/agent-task/checks/flow-runner.ts
|
|
735
|
+
import { copyFileSync, existsSync, unlinkSync } from "fs";
|
|
736
|
+
import { basename } from "path";
|
|
737
|
+
import { pathToFileURL } from "url";
|
|
738
|
+
function filePaths(files) {
|
|
739
|
+
if (!Array.isArray(files)) return [];
|
|
740
|
+
return files.map((f) => f.relativePath);
|
|
741
|
+
}
|
|
742
|
+
var REGISTRY_KEY = /* @__PURE__ */ Symbol.for("@apo-ai/sdk/agent-task/check-registry");
|
|
743
|
+
var registryStore = globalThis;
|
|
744
|
+
var registry = registryStore[REGISTRY_KEY] ??= [];
|
|
745
|
+
var GROUP_REGISTRY_KEY = /* @__PURE__ */ Symbol.for("@apo-ai/sdk/agent-task/group-registry");
|
|
746
|
+
var groupRegistry = registryStore[GROUP_REGISTRY_KEY] ??= [];
|
|
747
|
+
var currentGroupId = null;
|
|
748
|
+
var currentGroupName = null;
|
|
749
|
+
function defineCheck(id, fn) {
|
|
750
|
+
if (registry.some((c) => c.id === id)) {
|
|
751
|
+
throw new Error(`Duplicate check id '${id}'`);
|
|
752
|
+
}
|
|
753
|
+
registry.push({
|
|
754
|
+
id,
|
|
755
|
+
fn,
|
|
756
|
+
...currentGroupId ? { group_id: currentGroupId } : {},
|
|
757
|
+
...currentGroupName ? { group_name: currentGroupName } : {}
|
|
758
|
+
});
|
|
759
|
+
}
|
|
760
|
+
function resetFlowChecks() {
|
|
761
|
+
registry.length = 0;
|
|
762
|
+
groupRegistry.length = 0;
|
|
763
|
+
currentGroupId = null;
|
|
764
|
+
currentGroupName = null;
|
|
765
|
+
}
|
|
766
|
+
function describe(id, nameOrFn, fn) {
|
|
767
|
+
const name = typeof nameOrFn === "string" ? nameOrFn : id;
|
|
768
|
+
const body = typeof nameOrFn === "string" ? fn : nameOrFn;
|
|
769
|
+
if (currentGroupId !== null) {
|
|
770
|
+
throw new Error(
|
|
771
|
+
`describe("${id}") cannot nest inside describe("${currentGroupId}"); groups are single-level only.`
|
|
772
|
+
);
|
|
773
|
+
}
|
|
774
|
+
if (groupRegistry.some((g) => g.id === id)) {
|
|
775
|
+
throw new Error(`Duplicate describe id '${id}'`);
|
|
776
|
+
}
|
|
777
|
+
groupRegistry.push({ id, name });
|
|
778
|
+
currentGroupId = id;
|
|
779
|
+
currentGroupName = name;
|
|
780
|
+
try {
|
|
781
|
+
body();
|
|
782
|
+
} finally {
|
|
783
|
+
currentGroupId = null;
|
|
784
|
+
currentGroupName = null;
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
async function runTraceChecks(args) {
|
|
788
|
+
const view = new TraceView(args.snapshot);
|
|
789
|
+
const locate = args.moduleUrl && args.displayFile ? (stack) => parseCheckLocation(stack, args.moduleUrl, args.displayFile) : void 0;
|
|
790
|
+
const results = await Promise.all(
|
|
791
|
+
registry.map(async (check) => {
|
|
792
|
+
const rec = new Recorder(locate);
|
|
793
|
+
const t = createTraceTestContext(view, rec, args.judgeConfig);
|
|
794
|
+
let thrownLocation;
|
|
795
|
+
try {
|
|
796
|
+
await check.fn(t, {
|
|
797
|
+
deliverables: args.deliverables,
|
|
798
|
+
files: args.files,
|
|
799
|
+
task: args.task
|
|
800
|
+
});
|
|
801
|
+
} catch (error) {
|
|
802
|
+
thrownLocation = locate ? locate(error instanceof Error ? error.stack ?? "" : "") : void 0;
|
|
803
|
+
rec.record("check-error", false, error instanceof Error ? error.message : String(error), {
|
|
804
|
+
location: thrownLocation
|
|
805
|
+
});
|
|
806
|
+
}
|
|
807
|
+
const failed = rec.all.filter((r) => !r.pass);
|
|
808
|
+
const pass = failed.length === 0;
|
|
809
|
+
const reasoning = failed.length > 0 ? failed.map((r) => r.reasoning || r.id).join("; ") : rec.all.length > 0 ? "passed" : "no assertions recorded";
|
|
810
|
+
const location = failed.find((r) => r.location)?.location;
|
|
811
|
+
const judge = rec.all.find((r) => r.judge)?.judge;
|
|
812
|
+
return {
|
|
813
|
+
id: check.id,
|
|
814
|
+
pass,
|
|
815
|
+
reasoning,
|
|
816
|
+
evaluator_type: "code",
|
|
817
|
+
...judge ? { judge } : {},
|
|
818
|
+
...location ? { location } : {},
|
|
819
|
+
...args.displayFile ? { source_file: args.displayFile } : {},
|
|
820
|
+
...rec.all.length > 0 ? { assertions: rec.all.map((a) => ({ ...a })) } : {},
|
|
821
|
+
...check.group_id ? { group_id: check.group_id } : {},
|
|
822
|
+
...check.group_name ? { group_name: check.group_name } : {},
|
|
823
|
+
// stamp the snapshot source so consumers can detect
|
|
824
|
+
// the deprecated legacy-flow compatibility path (source="legacy-flow").
|
|
825
|
+
...args.snapshot.source !== "canonical" ? { source: args.snapshot.source } : {}
|
|
826
|
+
};
|
|
827
|
+
})
|
|
828
|
+
);
|
|
829
|
+
return results;
|
|
830
|
+
}
|
|
831
|
+
async function loadAndRunFlowChecks(checksPath, args, brokenDeliverables = {}) {
|
|
832
|
+
if (!checksPath) return [];
|
|
833
|
+
resetFlowChecks();
|
|
834
|
+
const moduleUrl = await loadChecksModule(checksPath);
|
|
835
|
+
if (moduleUrl === null || registry.length === 0) return [];
|
|
836
|
+
return runTraceChecks({
|
|
837
|
+
snapshot: args.snapshot,
|
|
838
|
+
deliverables: proxyBrokenDeliverables(args.deliverables, brokenDeliverables),
|
|
839
|
+
files: args.files,
|
|
840
|
+
task: args.task,
|
|
841
|
+
judgeConfig: args.judgeConfig,
|
|
842
|
+
moduleUrl,
|
|
843
|
+
displayFile: basename(checksPath)
|
|
844
|
+
});
|
|
845
|
+
}
|
|
846
|
+
async function loadChecksModule(checksPath) {
|
|
847
|
+
const tempModulePath = `${checksPath}.${Date.now()}-${Math.random().toString(36).slice(2)}.ts`;
|
|
848
|
+
try {
|
|
849
|
+
copyFileSync(checksPath, tempModulePath);
|
|
850
|
+
const moduleUrl = pathToFileURL(tempModulePath).href;
|
|
851
|
+
await import(moduleUrl);
|
|
852
|
+
if (registry.length === 0) return null;
|
|
853
|
+
return moduleUrl;
|
|
854
|
+
} finally {
|
|
855
|
+
if (existsSync(tempModulePath)) unlinkSync(tempModulePath);
|
|
856
|
+
}
|
|
857
|
+
}
|
|
858
|
+
function proxyBrokenDeliverables(deliverables, brokenDeliverables) {
|
|
859
|
+
if (Object.keys(brokenDeliverables).length === 0) return deliverables;
|
|
860
|
+
return new Proxy(deliverables, {
|
|
861
|
+
get(target, property, receiver) {
|
|
862
|
+
if (typeof property === "string" && property in brokenDeliverables) {
|
|
863
|
+
throw new Error(brokenDeliverables[property]);
|
|
864
|
+
}
|
|
865
|
+
return Reflect.get(target, property, receiver);
|
|
866
|
+
}
|
|
867
|
+
});
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
// src/agent-task/task/defineTask.ts
|
|
871
|
+
var TASK_ADAPTER_SYMBOL = /* @__PURE__ */ Symbol.for("agent-task.adapter-definition");
|
|
872
|
+
var TASK_REGISTRY_KEY = /* @__PURE__ */ Symbol.for("@apo-ai/sdk/agent-task/task-registry");
|
|
873
|
+
var taskRegistryStore = globalThis;
|
|
874
|
+
var taskRegistry = taskRegistryStore[TASK_REGISTRY_KEY] ??= [];
|
|
875
|
+
function resetTaskRegistry() {
|
|
876
|
+
taskRegistry.length = 0;
|
|
877
|
+
}
|
|
878
|
+
function getRegisteredTask() {
|
|
879
|
+
return taskRegistry[0];
|
|
880
|
+
}
|
|
881
|
+
function defineTask(adapter, config) {
|
|
882
|
+
const definedTask = {
|
|
883
|
+
...config,
|
|
884
|
+
adapter: adapter.name
|
|
885
|
+
};
|
|
886
|
+
attachAdapter(definedTask, adapter);
|
|
887
|
+
return definedTask;
|
|
888
|
+
}
|
|
889
|
+
function task(name, config) {
|
|
890
|
+
const adapter = config.adapter;
|
|
891
|
+
const {
|
|
892
|
+
adapter: _adapter,
|
|
893
|
+
deliverables,
|
|
894
|
+
...rest
|
|
895
|
+
} = config;
|
|
896
|
+
const definedTask = {
|
|
897
|
+
...rest,
|
|
898
|
+
id: name,
|
|
899
|
+
adapter: adapter.name,
|
|
900
|
+
deliverables: [...deliverables]
|
|
901
|
+
};
|
|
902
|
+
attachAdapter(definedTask, adapter);
|
|
903
|
+
taskRegistry.push(definedTask);
|
|
904
|
+
return {
|
|
905
|
+
test: defineCheck,
|
|
906
|
+
describe
|
|
907
|
+
};
|
|
908
|
+
}
|
|
909
|
+
function getTaskAdapterDefinition(task2) {
|
|
910
|
+
return task2[TASK_ADAPTER_SYMBOL] ?? null;
|
|
911
|
+
}
|
|
912
|
+
function attachAdapter(taskDefinition, adapter) {
|
|
913
|
+
Object.defineProperty(taskDefinition, TASK_ADAPTER_SYMBOL, {
|
|
914
|
+
value: adapter,
|
|
915
|
+
enumerable: false,
|
|
916
|
+
configurable: false,
|
|
917
|
+
writable: false
|
|
918
|
+
});
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
// src/agent-task/task/loadTask.ts
|
|
922
|
+
import { copyFileSync as copyFileSync2, readdirSync, statSync, existsSync as existsSync2, unlinkSync as unlinkSync2 } from "fs";
|
|
923
|
+
import { join, resolve, basename as basename2 } from "path";
|
|
924
|
+
import { pathToFileURL as pathToFileURL2 } from "url";
|
|
925
|
+
|
|
926
|
+
// src/agent-task/turn.ts
|
|
927
|
+
var TURN_KEY = /* @__PURE__ */ Symbol.for("@apo-ai/sdk/agent-task/task-turn");
|
|
928
|
+
function turn(fn) {
|
|
929
|
+
globalThis[TURN_KEY] = fn;
|
|
930
|
+
}
|
|
931
|
+
function getTaskTurn() {
|
|
932
|
+
return globalThis[TURN_KEY];
|
|
933
|
+
}
|
|
934
|
+
function resetTaskTurn() {
|
|
935
|
+
delete globalThis[TURN_KEY];
|
|
936
|
+
}
|
|
937
|
+
function resolveTurn(adapterTurn, taskTurn) {
|
|
938
|
+
const resolved = taskTurn ?? adapterTurn;
|
|
939
|
+
if (!resolved) {
|
|
940
|
+
throw new Error(
|
|
941
|
+
"No turn behavior defined. Add a turn() call to your task file or a turn field to your adapter."
|
|
942
|
+
);
|
|
943
|
+
}
|
|
944
|
+
return resolved;
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
// src/agent-task/task/loadTask.ts
|
|
948
|
+
async function loadTask(taskDir) {
|
|
949
|
+
const absoluteDir = resolve(taskDir);
|
|
950
|
+
const taskFilePath = findEvalFile(absoluteDir);
|
|
951
|
+
if (!taskFilePath) {
|
|
952
|
+
throw new Error(`No .eval.ts file found in: ${absoluteDir}`);
|
|
953
|
+
}
|
|
954
|
+
const { task: task2, adapter, inlineChecks, moduleUrl } = await loadTaskDefinition(taskFilePath);
|
|
955
|
+
validateTaskDefinition(task2, absoluteDir);
|
|
956
|
+
const files = loadFilesDirectory(absoluteDir);
|
|
957
|
+
const checksPath = resolveChecksPath(task2, absoluteDir);
|
|
958
|
+
return {
|
|
959
|
+
task: task2,
|
|
960
|
+
adapter,
|
|
961
|
+
taskDir: absoluteDir,
|
|
962
|
+
files,
|
|
963
|
+
checksPath,
|
|
964
|
+
inlineChecks,
|
|
965
|
+
moduleUrl,
|
|
966
|
+
evalFileName: basename2(taskFilePath)
|
|
967
|
+
};
|
|
968
|
+
}
|
|
969
|
+
async function loadTaskDefinition(taskFilePath) {
|
|
970
|
+
const tempModulePath = `${taskFilePath}.${Date.now()}-${Math.random().toString(36).slice(2)}.ts`;
|
|
971
|
+
try {
|
|
972
|
+
copyFileSync2(taskFilePath, tempModulePath);
|
|
973
|
+
const moduleUrl = pathToFileURL2(tempModulePath).href;
|
|
974
|
+
resetTaskRegistry();
|
|
975
|
+
resetFlowChecks();
|
|
976
|
+
resetTaskTurn();
|
|
977
|
+
const loaded = await import(moduleUrl);
|
|
978
|
+
const registeredTask = getRegisteredTask();
|
|
979
|
+
const task2 = registeredTask ?? loaded.default;
|
|
980
|
+
if (!task2 || typeof task2 !== "object") {
|
|
981
|
+
throw new Error(
|
|
982
|
+
"No task definition found \u2014 call task(name, config) or default export defineTask(adapter, config)"
|
|
983
|
+
);
|
|
984
|
+
}
|
|
985
|
+
const adapter = getTaskAdapterDefinition(task2);
|
|
986
|
+
if (!adapter) {
|
|
987
|
+
throw new Error(
|
|
988
|
+
registeredTask ? "task() config must include an adapter" : "Task module must default export defineTask(adapter, {...})"
|
|
989
|
+
);
|
|
990
|
+
}
|
|
991
|
+
return {
|
|
992
|
+
task: task2,
|
|
993
|
+
adapter,
|
|
994
|
+
inlineChecks: registeredTask !== void 0,
|
|
995
|
+
moduleUrl
|
|
996
|
+
};
|
|
997
|
+
} catch (error) {
|
|
998
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
999
|
+
throw new Error(`Failed to load task definition: ${taskFilePath} (${message})`);
|
|
1000
|
+
} finally {
|
|
1001
|
+
if (existsSync2(tempModulePath)) {
|
|
1002
|
+
unlinkSync2(tempModulePath);
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
function validateTaskDefinition(task2, taskDir) {
|
|
1007
|
+
if (!task2.id || typeof task2.id !== "string") {
|
|
1008
|
+
throw new Error(`Task must have a string 'id': ${taskDir}`);
|
|
1009
|
+
}
|
|
1010
|
+
if (!task2.adapter || typeof task2.adapter !== "string") {
|
|
1011
|
+
throw new Error(`Task must have a derived string 'adapter': ${taskDir}`);
|
|
1012
|
+
}
|
|
1013
|
+
if (!Array.isArray(task2.deliverables)) {
|
|
1014
|
+
throw new Error(`Task must have a 'deliverables' array: ${taskDir}`);
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
function loadFilesDirectory(taskDir) {
|
|
1018
|
+
const filesDir = join(taskDir, "files");
|
|
1019
|
+
if (!existsSync2(filesDir) || !statSync(filesDir).isDirectory()) {
|
|
1020
|
+
return [];
|
|
1021
|
+
}
|
|
1022
|
+
return walkDirectory(filesDir, filesDir);
|
|
1023
|
+
}
|
|
1024
|
+
function walkDirectory(dir, root) {
|
|
1025
|
+
const entries = [];
|
|
1026
|
+
for (const name of readdirSync(dir)) {
|
|
1027
|
+
const fullPath = join(dir, name);
|
|
1028
|
+
const stat = statSync(fullPath);
|
|
1029
|
+
if (stat.isDirectory()) {
|
|
1030
|
+
entries.push(...walkDirectory(fullPath, root));
|
|
1031
|
+
} else {
|
|
1032
|
+
entries.push({
|
|
1033
|
+
relativePath: fullPath.slice(root.length + 1),
|
|
1034
|
+
absolutePath: fullPath
|
|
1035
|
+
});
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
return entries;
|
|
1039
|
+
}
|
|
1040
|
+
function resolveChecksPath(task2, taskDir) {
|
|
1041
|
+
if (task2.checks === false) {
|
|
1042
|
+
return null;
|
|
1043
|
+
}
|
|
1044
|
+
if (typeof task2.checks === "string") {
|
|
1045
|
+
const customPath = resolve(taskDir, task2.checks);
|
|
1046
|
+
if (!existsSync2(customPath)) {
|
|
1047
|
+
throw new Error(`Custom checks file not found: ${customPath}`);
|
|
1048
|
+
}
|
|
1049
|
+
return customPath;
|
|
1050
|
+
}
|
|
1051
|
+
const conventionPath = join(taskDir, "checks.ts");
|
|
1052
|
+
return existsSync2(conventionPath) ? conventionPath : null;
|
|
1053
|
+
}
|
|
1054
|
+
function findEvalFile(dir) {
|
|
1055
|
+
try {
|
|
1056
|
+
const files = readdirSync(dir);
|
|
1057
|
+
const evalFile = files.find((f) => f.endsWith(".eval.ts"));
|
|
1058
|
+
return evalFile ? join(dir, evalFile) : null;
|
|
1059
|
+
} catch {
|
|
1060
|
+
return null;
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
// src/agent-task/task/TaskFiles.ts
|
|
1065
|
+
import { readFile } from "fs/promises";
|
|
1066
|
+
var TaskFiles = class {
|
|
1067
|
+
constructor(entries) {
|
|
1068
|
+
this.entries = entries;
|
|
1069
|
+
}
|
|
1070
|
+
async read(relativePath) {
|
|
1071
|
+
const entry = this.find(relativePath);
|
|
1072
|
+
if (!entry) {
|
|
1073
|
+
throw new Error(`File not found: ${relativePath}`);
|
|
1074
|
+
}
|
|
1075
|
+
return readFile(entry.absolutePath, "utf-8");
|
|
1076
|
+
}
|
|
1077
|
+
async readBuffer(relativePath) {
|
|
1078
|
+
const entry = this.find(relativePath);
|
|
1079
|
+
if (!entry) {
|
|
1080
|
+
throw new Error(`File not found: ${relativePath}`);
|
|
1081
|
+
}
|
|
1082
|
+
return readFile(entry.absolutePath);
|
|
1083
|
+
}
|
|
1084
|
+
find(relativePath) {
|
|
1085
|
+
return this.entries.find((entry) => entry.relativePath === relativePath);
|
|
1086
|
+
}
|
|
1087
|
+
list() {
|
|
1088
|
+
return [...this.entries];
|
|
1089
|
+
}
|
|
1090
|
+
has(relativePath) {
|
|
1091
|
+
return this.entries.some((entry) => entry.relativePath === relativePath);
|
|
1092
|
+
}
|
|
1093
|
+
};
|
|
1094
|
+
|
|
1095
|
+
// src/agent-task/adapter/defineAdapter.ts
|
|
1096
|
+
function defineAdapter(adapter) {
|
|
1097
|
+
return adapter;
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
// src/agent-task/deliverables/artifact.ts
|
|
1101
|
+
import { basename as basename3 } from "path";
|
|
1102
|
+
import { lstatSync, statSync as statSync2 } from "fs";
|
|
1103
|
+
var DEFAULT_MEDIA_TYPE = "application/octet-stream";
|
|
1104
|
+
function fileArtifact(path, options = {}) {
|
|
1105
|
+
if (!path) {
|
|
1106
|
+
throw new Error("fileArtifact: path must not be empty");
|
|
1107
|
+
}
|
|
1108
|
+
let stat;
|
|
1109
|
+
try {
|
|
1110
|
+
const lstat = lstatSync(path);
|
|
1111
|
+
if (lstat.isSymbolicLink()) {
|
|
1112
|
+
throw new Error(`fileArtifact: '${path}' is a symbolic link, not a regular file`);
|
|
1113
|
+
}
|
|
1114
|
+
stat = statSync2(path);
|
|
1115
|
+
} catch (error) {
|
|
1116
|
+
if (!existsLike(error)) throw error;
|
|
1117
|
+
throw new Error(`fileArtifact: '${path}' is not a regular file`);
|
|
1118
|
+
}
|
|
1119
|
+
if (!stat.isFile()) {
|
|
1120
|
+
throw new Error(`fileArtifact: '${path}' is not a regular file`);
|
|
1121
|
+
}
|
|
1122
|
+
const displayFilename = options.displayFilename ?? basename3(path);
|
|
1123
|
+
if (!displayFilename) {
|
|
1124
|
+
throw new Error("fileArtifact: displayFilename must not be empty");
|
|
1125
|
+
}
|
|
1126
|
+
return Object.freeze({
|
|
1127
|
+
kind: "apo.file-artifact",
|
|
1128
|
+
path,
|
|
1129
|
+
mediaType: options.mediaType ?? DEFAULT_MEDIA_TYPE,
|
|
1130
|
+
displayFilename
|
|
1131
|
+
});
|
|
1132
|
+
}
|
|
1133
|
+
function isFileArtifact(value) {
|
|
1134
|
+
return typeof value === "object" && value !== null && value.kind === "apo.file-artifact" && typeof value.path === "string";
|
|
1135
|
+
}
|
|
1136
|
+
function existsLike(error) {
|
|
1137
|
+
if (!(error instanceof Error)) return false;
|
|
1138
|
+
const code = error.code;
|
|
1139
|
+
return code === "ENOENT";
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
// src/agent-task/deliverables/upload.ts
|
|
1143
|
+
import { createHash } from "crypto";
|
|
1144
|
+
import { createReadStream } from "fs";
|
|
1145
|
+
import { basename as basename4 } from "path";
|
|
1146
|
+
import { Readable } from "stream";
|
|
1147
|
+
var DEFAULT_FETCH = typeof fetch === "function" ? fetch : void 0;
|
|
1148
|
+
async function persistFileArtifacts(deliverables, config) {
|
|
1149
|
+
const jsonDeliverables = {};
|
|
1150
|
+
const artifactUploads = [];
|
|
1151
|
+
for (const [name, value] of Object.entries(deliverables)) {
|
|
1152
|
+
if (isFileArtifact(value)) {
|
|
1153
|
+
const summary = await uploadOne(name, value, config);
|
|
1154
|
+
artifactUploads.push(summary);
|
|
1155
|
+
} else {
|
|
1156
|
+
jsonDeliverables[name] = value;
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
return { jsonDeliverables, artifactUploads };
|
|
1160
|
+
}
|
|
1161
|
+
async function uploadOne(name, artifact, config) {
|
|
1162
|
+
const { size, sha256 } = await hashFile(artifact.path);
|
|
1163
|
+
const intent = await createIntent(name, artifact, size, sha256, config);
|
|
1164
|
+
const body = Readable.toWeb(createReadStream(artifact.path));
|
|
1165
|
+
const summary = await putBytes(intent.upload_url, body, size, config);
|
|
1166
|
+
void basename4;
|
|
1167
|
+
return summary;
|
|
1168
|
+
}
|
|
1169
|
+
async function hashFile(path) {
|
|
1170
|
+
return new Promise((resolve4, reject) => {
|
|
1171
|
+
const hash = createHash("sha256");
|
|
1172
|
+
let size = 0;
|
|
1173
|
+
const stream = createReadStream(path);
|
|
1174
|
+
stream.on("data", (chunk) => {
|
|
1175
|
+
hash.update(chunk);
|
|
1176
|
+
size += chunk.length;
|
|
1177
|
+
});
|
|
1178
|
+
stream.on("error", reject);
|
|
1179
|
+
stream.on("end", () => resolve4({ size, sha256: hash.digest("hex") }));
|
|
1180
|
+
});
|
|
1181
|
+
}
|
|
1182
|
+
async function createIntent(name, artifact, sizeBytes, sha256, config) {
|
|
1183
|
+
const url = `${config.baseUrl}/v1/agent-task-runs/${config.taskRunId}/artifact-uploads`;
|
|
1184
|
+
const response = await doFetch(config, url, {
|
|
1185
|
+
method: "POST",
|
|
1186
|
+
headers: {
|
|
1187
|
+
Authorization: `Bearer ${config.authToken}`,
|
|
1188
|
+
"Content-Type": "application/json"
|
|
1189
|
+
},
|
|
1190
|
+
body: JSON.stringify({
|
|
1191
|
+
name,
|
|
1192
|
+
display_filename: artifact.displayFilename,
|
|
1193
|
+
media_type: artifact.mediaType,
|
|
1194
|
+
size_bytes: sizeBytes,
|
|
1195
|
+
sha256
|
|
1196
|
+
})
|
|
1197
|
+
});
|
|
1198
|
+
if (response.status === 201) {
|
|
1199
|
+
const body = await response.json();
|
|
1200
|
+
return body;
|
|
1201
|
+
}
|
|
1202
|
+
const detail = await safeDetail(response);
|
|
1203
|
+
throw new Error(
|
|
1204
|
+
`artifact upload intent for '${name}' failed (${response.status}): ${detail}`
|
|
1205
|
+
);
|
|
1206
|
+
}
|
|
1207
|
+
async function putBytes(uploadUrl, body, sizeBytes, config) {
|
|
1208
|
+
const url = uploadUrl.startsWith("http") ? uploadUrl : `${config.baseUrl}${uploadUrl}`;
|
|
1209
|
+
const response = await doFetch(config, url, {
|
|
1210
|
+
method: "PUT",
|
|
1211
|
+
headers: {
|
|
1212
|
+
Authorization: `Bearer ${config.authToken}`,
|
|
1213
|
+
"Content-Type": "application/octet-stream",
|
|
1214
|
+
"Content-Length": String(sizeBytes)
|
|
1215
|
+
},
|
|
1216
|
+
body,
|
|
1217
|
+
duplex: "half"
|
|
1218
|
+
});
|
|
1219
|
+
if (response.status === 200) {
|
|
1220
|
+
return await response.json();
|
|
1221
|
+
}
|
|
1222
|
+
const detail = await safeDetail(response);
|
|
1223
|
+
throw new Error(
|
|
1224
|
+
`artifact upload of ${sizeBytes} bytes failed (${response.status}): ${detail}`
|
|
1225
|
+
);
|
|
1226
|
+
}
|
|
1227
|
+
async function doFetch(config, input, init) {
|
|
1228
|
+
const fetchImpl = config.fetch ?? DEFAULT_FETCH;
|
|
1229
|
+
if (!fetchImpl) {
|
|
1230
|
+
throw new Error("persistFileArtifacts: fetch is not available in this environment");
|
|
1231
|
+
}
|
|
1232
|
+
return fetchImpl(input, init);
|
|
1233
|
+
}
|
|
1234
|
+
async function safeDetail(response) {
|
|
1235
|
+
try {
|
|
1236
|
+
const text = await response.text();
|
|
1237
|
+
return text.slice(0, 500);
|
|
1238
|
+
} catch {
|
|
1239
|
+
return "<no body>";
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1243
|
+
// src/agent-task/flow/view.ts
|
|
1244
|
+
var FlowView = class {
|
|
1245
|
+
constructor(flow) {
|
|
1246
|
+
this.events = flow.events;
|
|
1247
|
+
this.startedAt = flow.startedAt;
|
|
1248
|
+
this.endedAt = flow.endedAt;
|
|
1249
|
+
}
|
|
1250
|
+
get messages() {
|
|
1251
|
+
return this.events.filter((e) => e.kind === "message");
|
|
1252
|
+
}
|
|
1253
|
+
get toolCalls() {
|
|
1254
|
+
return this.events.filter((e) => e.kind === "tool_call");
|
|
1255
|
+
}
|
|
1256
|
+
get skillLoads() {
|
|
1257
|
+
return this.events.filter((e) => e.kind === "skill_load");
|
|
1258
|
+
}
|
|
1259
|
+
get subagentCalls() {
|
|
1260
|
+
return this.events.filter(
|
|
1261
|
+
(e) => e.kind === "subagent_call"
|
|
1262
|
+
);
|
|
1263
|
+
}
|
|
1264
|
+
get toolNamesInOrder() {
|
|
1265
|
+
return this.toolCalls.map((c) => c.name);
|
|
1266
|
+
}
|
|
1267
|
+
/** Last assistant message text — the agent's "reply". */
|
|
1268
|
+
get reply() {
|
|
1269
|
+
for (let i = this.messages.length - 1; i >= 0; i--) {
|
|
1270
|
+
if (this.messages[i].role === "assistant") return this.messages[i].text;
|
|
1271
|
+
}
|
|
1272
|
+
return "";
|
|
1273
|
+
}
|
|
1274
|
+
/** Number of completed assistant turns. */
|
|
1275
|
+
get turnCount() {
|
|
1276
|
+
return this.messages.filter((m) => m.role === "assistant").length;
|
|
1277
|
+
}
|
|
1278
|
+
/** Tool or subagent calls that reported an error. */
|
|
1279
|
+
get failedActions() {
|
|
1280
|
+
return this.events.filter(
|
|
1281
|
+
(e) => (e.kind === "tool_call" || e.kind === "subagent_call") && e.status === "error"
|
|
1282
|
+
).length;
|
|
1283
|
+
}
|
|
1284
|
+
get durationMs() {
|
|
1285
|
+
if (this.startedAt != null && this.endedAt != null) {
|
|
1286
|
+
return Math.max(0, this.endedAt - this.startedAt);
|
|
1287
|
+
}
|
|
1288
|
+
return this.events.reduce((sum, e) => {
|
|
1289
|
+
if ("latencyMs" in e && typeof e.latencyMs === "number") {
|
|
1290
|
+
return sum + e.latencyMs;
|
|
1291
|
+
}
|
|
1292
|
+
return sum;
|
|
1293
|
+
}, 0);
|
|
1294
|
+
}
|
|
1295
|
+
};
|
|
1296
|
+
|
|
1297
|
+
// src/agent-task/flow/sources.ts
|
|
1298
|
+
function safeParse(raw) {
|
|
1299
|
+
if (typeof raw !== "string") return raw;
|
|
1300
|
+
try {
|
|
1301
|
+
return JSON.parse(raw);
|
|
1302
|
+
} catch {
|
|
1303
|
+
return raw;
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
function fromOpenAIMessages(messages) {
|
|
1307
|
+
const events = [];
|
|
1308
|
+
const results = /* @__PURE__ */ new Map();
|
|
1309
|
+
for (const m of messages) {
|
|
1310
|
+
if (m.role === "tool" && m.tool_call_id) {
|
|
1311
|
+
results.set(m.tool_call_id, m.content);
|
|
1312
|
+
}
|
|
1313
|
+
}
|
|
1314
|
+
for (const m of messages) {
|
|
1315
|
+
if (m.role === "user") {
|
|
1316
|
+
events.push({ kind: "message", role: "user", text: m.content ?? "" });
|
|
1317
|
+
} else if (m.role === "assistant") {
|
|
1318
|
+
events.push({ kind: "message", role: "assistant", text: m.content ?? "" });
|
|
1319
|
+
for (const tc of m.tool_calls ?? []) {
|
|
1320
|
+
events.push({
|
|
1321
|
+
kind: "tool_call",
|
|
1322
|
+
name: tc.function?.name ?? "unknown",
|
|
1323
|
+
input: safeParse(tc.function?.arguments),
|
|
1324
|
+
output: tc.id ? results.get(tc.id) : void 0
|
|
1325
|
+
});
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
}
|
|
1329
|
+
return { events };
|
|
1330
|
+
}
|
|
1331
|
+
function fromAnthropicMessages(messages) {
|
|
1332
|
+
const events = [];
|
|
1333
|
+
const results = /* @__PURE__ */ new Map();
|
|
1334
|
+
for (const m of messages) {
|
|
1335
|
+
for (const block of m.content ?? []) {
|
|
1336
|
+
if (block.type === "tool_result") {
|
|
1337
|
+
results.set(block.tool_use_id, block.content);
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
for (const m of messages) {
|
|
1342
|
+
const role = m.role === "user" ? "user" : "assistant";
|
|
1343
|
+
for (const block of m.content ?? []) {
|
|
1344
|
+
if (block.type === "text") {
|
|
1345
|
+
events.push({ kind: "message", role, text: block.text });
|
|
1346
|
+
} else if (block.type === "tool_use") {
|
|
1347
|
+
events.push({
|
|
1348
|
+
kind: "tool_call",
|
|
1349
|
+
name: block.name,
|
|
1350
|
+
input: block.input,
|
|
1351
|
+
output: results.get(block.id)
|
|
1352
|
+
});
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1356
|
+
return { events };
|
|
1357
|
+
}
|
|
1358
|
+
function fromAISDK(result) {
|
|
1359
|
+
const events = [];
|
|
1360
|
+
for (const step of result.steps ?? []) {
|
|
1361
|
+
if (step.text) {
|
|
1362
|
+
events.push({ kind: "message", role: "assistant", text: step.text });
|
|
1363
|
+
}
|
|
1364
|
+
const outputs = step.toolResults ?? [];
|
|
1365
|
+
const toolCalls = step.toolCalls ?? [];
|
|
1366
|
+
toolCalls.forEach((tc, i) => {
|
|
1367
|
+
events.push({
|
|
1368
|
+
kind: "tool_call",
|
|
1369
|
+
name: tc.toolName,
|
|
1370
|
+
input: tc.input,
|
|
1371
|
+
output: outputs[i]?.output
|
|
1372
|
+
});
|
|
1373
|
+
});
|
|
1374
|
+
}
|
|
1375
|
+
return { events };
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1378
|
+
// src/agent-task/run/run-configuration.ts
|
|
1379
|
+
function normalizeRunConfiguration(raw) {
|
|
1380
|
+
if (!raw) return void 0;
|
|
1381
|
+
const model = normalizeScalar(raw.model, "run_configuration.model", MAX_MODEL_BYTES, true);
|
|
1382
|
+
const effort = normalizeScalar(
|
|
1383
|
+
raw.effort,
|
|
1384
|
+
"run_configuration.effort",
|
|
1385
|
+
MAX_EFFORT_BYTES,
|
|
1386
|
+
false
|
|
1387
|
+
);
|
|
1388
|
+
return effort === void 0 ? { model } : { model, effort };
|
|
1389
|
+
}
|
|
1390
|
+
var MAX_MODEL_BYTES = 255;
|
|
1391
|
+
var MAX_EFFORT_BYTES = 64;
|
|
1392
|
+
var CONTROL_CHARS = /[\x00-\x1f\x7f]/;
|
|
1393
|
+
var utf8Encoder = new TextEncoder();
|
|
1394
|
+
function utf8ByteLength(value) {
|
|
1395
|
+
return utf8Encoder.encode(value).length;
|
|
1396
|
+
}
|
|
1397
|
+
function normalizeScalar(value, field, maxBytes, required) {
|
|
1398
|
+
if (value === void 0) {
|
|
1399
|
+
if (required) throw new Error(`${field} is required`);
|
|
1400
|
+
return void 0;
|
|
1401
|
+
}
|
|
1402
|
+
if (CONTROL_CHARS.test(value)) {
|
|
1403
|
+
throw new Error(`${field} must not contain control characters`);
|
|
1404
|
+
}
|
|
1405
|
+
const trimmed = value.trim();
|
|
1406
|
+
if (trimmed === "") {
|
|
1407
|
+
if (required) throw new Error(`${field} is required and must be non-empty`);
|
|
1408
|
+
return void 0;
|
|
1409
|
+
}
|
|
1410
|
+
if (utf8ByteLength(trimmed) > maxBytes) {
|
|
1411
|
+
throw new Error(`${field} must be at most ${maxBytes} UTF-8 bytes`);
|
|
1412
|
+
}
|
|
1413
|
+
return trimmed;
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
// src/agent-task/deliverables/validate.ts
|
|
1417
|
+
function isZodSchema(value) {
|
|
1418
|
+
return typeof value === "object" && value !== null && "safeParse" in value && typeof value.safeParse === "function";
|
|
1419
|
+
}
|
|
1420
|
+
function validateDeliverables(task2, collected, deliverableDefs) {
|
|
1421
|
+
const results = [];
|
|
1422
|
+
const brokenDeliverables = {};
|
|
1423
|
+
for (const deliverableName of task2.deliverables) {
|
|
1424
|
+
const value = collected[deliverableName];
|
|
1425
|
+
if (value === void 0 || value === null) {
|
|
1426
|
+
const reason = `Deliverable '${deliverableName}' is missing`;
|
|
1427
|
+
results.push({
|
|
1428
|
+
id: `deliverable:${deliverableName}`,
|
|
1429
|
+
pass: false,
|
|
1430
|
+
reasoning: reason
|
|
1431
|
+
});
|
|
1432
|
+
brokenDeliverables[deliverableName] = reason;
|
|
1433
|
+
continue;
|
|
1434
|
+
}
|
|
1435
|
+
const schema = resolveDeliverableSchema(
|
|
1436
|
+
deliverableDefs?.[deliverableName]
|
|
1437
|
+
);
|
|
1438
|
+
if (schema && isZodSchema(schema)) {
|
|
1439
|
+
const result = schema.safeParse(value);
|
|
1440
|
+
if (!result.success) {
|
|
1441
|
+
const reason = `Deliverable '${deliverableName}' failed schema validation: ${result.error?.message ?? "Unknown error"}`;
|
|
1442
|
+
results.push({
|
|
1443
|
+
id: `deliverable:${deliverableName}`,
|
|
1444
|
+
pass: false,
|
|
1445
|
+
reasoning: reason
|
|
1446
|
+
});
|
|
1447
|
+
brokenDeliverables[deliverableName] = reason;
|
|
1448
|
+
}
|
|
1449
|
+
}
|
|
1450
|
+
}
|
|
1451
|
+
return { results, brokenDeliverables };
|
|
1452
|
+
}
|
|
1453
|
+
function resolveDeliverableSchema(definition) {
|
|
1454
|
+
if (!definition) {
|
|
1455
|
+
return void 0;
|
|
1456
|
+
}
|
|
1457
|
+
if (isZodSchema(definition)) {
|
|
1458
|
+
return definition;
|
|
1459
|
+
}
|
|
1460
|
+
if (typeof definition === "object" && definition !== null && "schema" in definition && isZodSchema(definition.schema)) {
|
|
1461
|
+
return definition.schema;
|
|
1462
|
+
}
|
|
1463
|
+
return void 0;
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
// src/agent-task/trace-projection/projection-tee.ts
|
|
1467
|
+
function monotonicNowMs() {
|
|
1468
|
+
if (typeof performance !== "undefined" && typeof performance.now === "function") {
|
|
1469
|
+
return performance.now();
|
|
1470
|
+
}
|
|
1471
|
+
return Date.now();
|
|
1472
|
+
}
|
|
1473
|
+
function round3(n) {
|
|
1474
|
+
return Math.round(n * 1e3) / 1e3;
|
|
1475
|
+
}
|
|
1476
|
+
function observationTypeFor(t) {
|
|
1477
|
+
switch (t) {
|
|
1478
|
+
case "GENERATION":
|
|
1479
|
+
return "GENERATION";
|
|
1480
|
+
case "TOOL":
|
|
1481
|
+
return "TOOL";
|
|
1482
|
+
case "AGENT":
|
|
1483
|
+
return "AGENT";
|
|
1484
|
+
case "CHAIN":
|
|
1485
|
+
return "CHAIN";
|
|
1486
|
+
case "RETRIEVER":
|
|
1487
|
+
return "RETRIEVER";
|
|
1488
|
+
case "EMBEDDING":
|
|
1489
|
+
return "EMBEDDING";
|
|
1490
|
+
case "GUARDRAIL":
|
|
1491
|
+
return "GUARDRAIL";
|
|
1492
|
+
default:
|
|
1493
|
+
return "SPAN";
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
function extractText(output) {
|
|
1497
|
+
if (typeof output === "string") return output;
|
|
1498
|
+
if (output != null && typeof output === "object" && "text" in output) {
|
|
1499
|
+
const text = output.text;
|
|
1500
|
+
if (typeof text === "string") return text;
|
|
1501
|
+
}
|
|
1502
|
+
return "";
|
|
1503
|
+
}
|
|
1504
|
+
function deriveCapabilities(observations) {
|
|
1505
|
+
const hasType = (t) => observations.some((o) => o.type === t);
|
|
1506
|
+
const hasMessages = observations.some(
|
|
1507
|
+
(o) => o.type === "GENERATION" && o.messages && o.messages.length > 0
|
|
1508
|
+
);
|
|
1509
|
+
return {
|
|
1510
|
+
messages: hasMessages ? "available" : "unavailable",
|
|
1511
|
+
tools: hasType("TOOL") ? "available" : "unavailable",
|
|
1512
|
+
// Local capture records every span's status, so error evidence is always
|
|
1513
|
+
// available — the absence of error observations IS proof of no errors.
|
|
1514
|
+
errors: "available",
|
|
1515
|
+
timing: "available",
|
|
1516
|
+
skills: hasType("SKILL") ? "available" : "unavailable",
|
|
1517
|
+
subagents: hasType("AGENT") ? "available" : "unavailable"
|
|
1518
|
+
};
|
|
1519
|
+
}
|
|
1520
|
+
function createProjectionTee(real) {
|
|
1521
|
+
const observations = [];
|
|
1522
|
+
const pending = /* @__PURE__ */ new Map();
|
|
1523
|
+
const realIdByTeeId = /* @__PURE__ */ new Map();
|
|
1524
|
+
let spanCounter = 0;
|
|
1525
|
+
const rootStartMs = monotonicNowMs();
|
|
1526
|
+
const rootStartIso = new Date(rootStartMs).toISOString();
|
|
1527
|
+
function complete(spanId, params) {
|
|
1528
|
+
const p = pending.get(spanId);
|
|
1529
|
+
if (!p) return;
|
|
1530
|
+
pending.delete(spanId);
|
|
1531
|
+
const endedAtMs = monotonicNowMs();
|
|
1532
|
+
const isError = params?.level === "ERROR";
|
|
1533
|
+
const status = isError ? "error" : "ok";
|
|
1534
|
+
const obs = {
|
|
1535
|
+
spanId: p.spanId,
|
|
1536
|
+
type: p.type,
|
|
1537
|
+
name: p.name,
|
|
1538
|
+
startedAt: p.startedAtIso,
|
|
1539
|
+
endedAt: new Date(endedAtMs).toISOString(),
|
|
1540
|
+
durationMs: round3(endedAtMs - p.startedAtMs),
|
|
1541
|
+
status
|
|
1542
|
+
};
|
|
1543
|
+
if (p.parentSpanId) obs.parentSpanId = p.parentSpanId;
|
|
1544
|
+
if (p.model) obs.model = p.model;
|
|
1545
|
+
if (isError && params?.status_message) obs.errorMessage = params.status_message;
|
|
1546
|
+
const meta = params?.metadata;
|
|
1547
|
+
if (p.type === "TOOL") {
|
|
1548
|
+
obs.toolName = meta?.tool_name ?? p.name;
|
|
1549
|
+
if (meta?.tool_parameters !== void 0) obs.toolParameters = meta.tool_parameters;
|
|
1550
|
+
else if (p.input !== void 0) obs.toolParameters = p.input;
|
|
1551
|
+
if (meta?.tool_result !== void 0) obs.toolResult = meta.tool_result;
|
|
1552
|
+
else if (params?.output !== void 0) obs.output = params.output;
|
|
1553
|
+
} else if (params?.output !== void 0) {
|
|
1554
|
+
obs.output = params.output;
|
|
1555
|
+
}
|
|
1556
|
+
if (p.type === "GENERATION" && params?.output != null) {
|
|
1557
|
+
const text = extractText(params.output);
|
|
1558
|
+
if (text !== "") {
|
|
1559
|
+
obs.messages = [{ role: "assistant", content: text }];
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1562
|
+
observations.push(obs);
|
|
1563
|
+
}
|
|
1564
|
+
function recordStart(opts) {
|
|
1565
|
+
spanCounter += 1;
|
|
1566
|
+
const spanId = `${real.rootSpanId}-${String(spanCounter).padStart(6, "0")}`;
|
|
1567
|
+
pending.set(spanId, {
|
|
1568
|
+
spanId,
|
|
1569
|
+
parentSpanId: opts.parent_call_id ?? real.rootSpanId,
|
|
1570
|
+
type: observationTypeFor(opts.observation_type),
|
|
1571
|
+
name: opts.step_name ?? "step",
|
|
1572
|
+
startedAtIso: new Date(monotonicNowMs()).toISOString(),
|
|
1573
|
+
startedAtMs: monotonicNowMs(),
|
|
1574
|
+
model: opts.model ?? void 0,
|
|
1575
|
+
input: opts.input
|
|
1576
|
+
});
|
|
1577
|
+
return spanId;
|
|
1578
|
+
}
|
|
1579
|
+
const trace2 = {
|
|
1580
|
+
runId: real.runId,
|
|
1581
|
+
rootSpanId: real.rootSpanId,
|
|
1582
|
+
async step(opts, fn) {
|
|
1583
|
+
const obsSpanId = recordStart(opts);
|
|
1584
|
+
const start = monotonicNowMs();
|
|
1585
|
+
try {
|
|
1586
|
+
const result = await real.step(opts, fn);
|
|
1587
|
+
complete(obsSpanId, {
|
|
1588
|
+
latency_ms: round3(monotonicNowMs() - start),
|
|
1589
|
+
output: opts.summarize?.(result),
|
|
1590
|
+
metadata: opts.metadata
|
|
1591
|
+
});
|
|
1592
|
+
return result;
|
|
1593
|
+
} catch (error) {
|
|
1594
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1595
|
+
complete(obsSpanId, {
|
|
1596
|
+
latency_ms: round3(monotonicNowMs() - start),
|
|
1597
|
+
status_message: message,
|
|
1598
|
+
level: "ERROR",
|
|
1599
|
+
metadata: { ...opts.metadata, failed: true }
|
|
1600
|
+
});
|
|
1601
|
+
throw error;
|
|
1602
|
+
}
|
|
1603
|
+
},
|
|
1604
|
+
recordEvent(evOptions) {
|
|
1605
|
+
const spanId = recordStart(evOptions);
|
|
1606
|
+
complete(spanId, {
|
|
1607
|
+
latency_ms: evOptions.latency_ms,
|
|
1608
|
+
output: evOptions.output,
|
|
1609
|
+
metadata: evOptions.metadata
|
|
1610
|
+
});
|
|
1611
|
+
try {
|
|
1612
|
+
real.recordEvent(evOptions);
|
|
1613
|
+
} catch {
|
|
1614
|
+
}
|
|
1615
|
+
return spanId;
|
|
1616
|
+
},
|
|
1617
|
+
endRoot(endParams) {
|
|
1618
|
+
try {
|
|
1619
|
+
real.endRoot(endParams);
|
|
1620
|
+
} catch {
|
|
1621
|
+
}
|
|
1622
|
+
},
|
|
1623
|
+
createSpan(spanOptions) {
|
|
1624
|
+
const spanId = recordStart(spanOptions);
|
|
1625
|
+
const isFromOtelProcessor = spanOptions.metadata?.source === "otel-processor";
|
|
1626
|
+
if (!isFromOtelProcessor) {
|
|
1627
|
+
try {
|
|
1628
|
+
const realId = real.createSpan(spanOptions);
|
|
1629
|
+
realIdByTeeId.set(spanId, realId);
|
|
1630
|
+
} catch {
|
|
1631
|
+
}
|
|
1632
|
+
}
|
|
1633
|
+
return spanId;
|
|
1634
|
+
},
|
|
1635
|
+
endSpan(spanId, endParams) {
|
|
1636
|
+
complete(spanId, endParams);
|
|
1637
|
+
const realId = realIdByTeeId.get(spanId) ?? spanId;
|
|
1638
|
+
realIdByTeeId.delete(spanId);
|
|
1639
|
+
try {
|
|
1640
|
+
real.endSpan(realId, endParams);
|
|
1641
|
+
} catch {
|
|
1642
|
+
}
|
|
1643
|
+
},
|
|
1644
|
+
async traceTool(name, params, fn) {
|
|
1645
|
+
return real.traceTool(name, params, fn);
|
|
1646
|
+
},
|
|
1647
|
+
async traceRetriever(query, fn) {
|
|
1648
|
+
return real.traceRetriever(query, fn);
|
|
1649
|
+
},
|
|
1650
|
+
async traceChain(name, fn) {
|
|
1651
|
+
return real.traceChain(name, fn);
|
|
1652
|
+
},
|
|
1653
|
+
async traceAgent(name, fn) {
|
|
1654
|
+
return real.traceAgent(name, fn);
|
|
1655
|
+
},
|
|
1656
|
+
async traceGuardrail(name, fn) {
|
|
1657
|
+
return real.traceGuardrail(name, fn);
|
|
1658
|
+
},
|
|
1659
|
+
async traceEmbedding(model, input, fn) {
|
|
1660
|
+
return real.traceEmbedding(model, input, fn);
|
|
1661
|
+
},
|
|
1662
|
+
async score(params) {
|
|
1663
|
+
return real.score(params);
|
|
1664
|
+
}
|
|
1665
|
+
};
|
|
1666
|
+
return {
|
|
1667
|
+
trace: trace2,
|
|
1668
|
+
getSnapshot() {
|
|
1669
|
+
const endMs = monotonicNowMs();
|
|
1670
|
+
return {
|
|
1671
|
+
schemaVersion: 1,
|
|
1672
|
+
projectionVersion: 1,
|
|
1673
|
+
source: "local",
|
|
1674
|
+
trace: {
|
|
1675
|
+
traceId: real.runId,
|
|
1676
|
+
startedAt: rootStartIso,
|
|
1677
|
+
endedAt: new Date(endMs).toISOString(),
|
|
1678
|
+
durationMs: round3(endMs - rootStartMs),
|
|
1679
|
+
complete: true
|
|
1680
|
+
},
|
|
1681
|
+
capabilities: deriveCapabilities(observations),
|
|
1682
|
+
observations: [...observations]
|
|
1683
|
+
};
|
|
1684
|
+
}
|
|
1685
|
+
};
|
|
1686
|
+
}
|
|
1687
|
+
|
|
1688
|
+
// src/agent-task/trace-projection/remote-capture.ts
|
|
1689
|
+
var ProjectionTimeoutError = class extends Error {
|
|
1690
|
+
constructor(taskRunId, deadlineMs) {
|
|
1691
|
+
super(
|
|
1692
|
+
`Trace projection for task run ${taskRunId} was not ready within ${deadlineMs}ms`
|
|
1693
|
+
);
|
|
1694
|
+
this.name = "ProjectionTimeoutError";
|
|
1695
|
+
}
|
|
1696
|
+
};
|
|
1697
|
+
async function readTaskRunProjection(options) {
|
|
1698
|
+
const deadlineMs = options.deadlineMs ?? 3e4;
|
|
1699
|
+
const initialIntervalMs = options.initialIntervalMs ?? 500;
|
|
1700
|
+
const maxIntervalMs = options.maxIntervalMs ?? 5e3;
|
|
1701
|
+
const url = `${options.endpoint.replace(/\/$/, "")}/v1/agent-task-runs/${encodeURIComponent(options.taskRunId)}/trace-projection`;
|
|
1702
|
+
const start = Date.now();
|
|
1703
|
+
let interval = initialIntervalMs;
|
|
1704
|
+
while (true) {
|
|
1705
|
+
if (Date.now() - start > deadlineMs) {
|
|
1706
|
+
throw new ProjectionTimeoutError(options.taskRunId, deadlineMs);
|
|
1707
|
+
}
|
|
1708
|
+
const response = await fetch(url, {
|
|
1709
|
+
headers: { Authorization: `Bearer ${options.authToken}` }
|
|
1710
|
+
});
|
|
1711
|
+
if (response.status === 200) {
|
|
1712
|
+
return await response.json();
|
|
1713
|
+
}
|
|
1714
|
+
if (response.status === 202) {
|
|
1715
|
+
await sleep(interval);
|
|
1716
|
+
interval = Math.min(interval * 1.5, maxIntervalMs);
|
|
1717
|
+
continue;
|
|
1718
|
+
}
|
|
1719
|
+
const detail = await response.json().catch(() => ({ detail: response.statusText }));
|
|
1720
|
+
throw new Error(
|
|
1721
|
+
`Projection read failed (${response.status}): ${detail.detail ?? response.statusText}`
|
|
1722
|
+
);
|
|
1723
|
+
}
|
|
1724
|
+
}
|
|
1725
|
+
function sleep(ms) {
|
|
1726
|
+
return new Promise((resolve4) => setTimeout(resolve4, ms));
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1729
|
+
// src/semconv.ts
|
|
1730
|
+
var APO_TASK_ID = "apo.task.id";
|
|
1731
|
+
var APO_TASK_RUN_ID = "apo.task.run.id";
|
|
1732
|
+
|
|
1733
|
+
// src/agent-task/run/aggregate.ts
|
|
1734
|
+
var NO_CHECKS_REGISTERED_MESSAGE = "No tests were registered by the eval module \u2014 a task must define at least one test().";
|
|
1735
|
+
function aggregateResult(checksResults) {
|
|
1736
|
+
if (checksResults.length === 0) {
|
|
1737
|
+
return { checks: checksResults, pass: false };
|
|
1738
|
+
}
|
|
1739
|
+
const pass = checksResults.every((r) => r.pass);
|
|
1740
|
+
return { checks: checksResults, pass };
|
|
1741
|
+
}
|
|
1742
|
+
|
|
1743
|
+
// src/agent-task/tracing.ts
|
|
1744
|
+
function createNoopAgentTaskTraceContext() {
|
|
1745
|
+
const noopFn = async (fn) => fn();
|
|
1746
|
+
return {
|
|
1747
|
+
runId: "agent-task-untraced",
|
|
1748
|
+
rootSpanId: "agent-task-untraced-root",
|
|
1749
|
+
async step(options, fn) {
|
|
1750
|
+
return fn(options.id ?? "agent-task-untraced-step");
|
|
1751
|
+
},
|
|
1752
|
+
recordEvent(options) {
|
|
1753
|
+
return options.id ?? "agent-task-untraced-event";
|
|
1754
|
+
},
|
|
1755
|
+
endRoot() {
|
|
1756
|
+
},
|
|
1757
|
+
traceTool(_name, _params, fn) {
|
|
1758
|
+
return noopFn(fn);
|
|
1759
|
+
},
|
|
1760
|
+
traceRetriever(_query, fn) {
|
|
1761
|
+
return noopFn(fn);
|
|
1762
|
+
},
|
|
1763
|
+
traceChain(_name, fn) {
|
|
1764
|
+
return noopFn(fn);
|
|
1765
|
+
},
|
|
1766
|
+
traceAgent(_name, fn) {
|
|
1767
|
+
return noopFn(fn);
|
|
1768
|
+
},
|
|
1769
|
+
traceGuardrail(_name, fn) {
|
|
1770
|
+
return noopFn(fn);
|
|
1771
|
+
},
|
|
1772
|
+
traceEmbedding(_model, _input, fn) {
|
|
1773
|
+
return noopFn(fn);
|
|
1774
|
+
},
|
|
1775
|
+
async score(_params) {
|
|
1776
|
+
},
|
|
1777
|
+
createSpan(_options) {
|
|
1778
|
+
return "agent-task-untraced-span";
|
|
1779
|
+
},
|
|
1780
|
+
endSpan(_spanId, _params) {
|
|
1781
|
+
}
|
|
1782
|
+
};
|
|
1783
|
+
}
|
|
1784
|
+
|
|
1785
|
+
// src/agent-task/run/runTask.ts
|
|
1786
|
+
var AgentTaskRunError = class extends Error {
|
|
1787
|
+
constructor(message, options) {
|
|
1788
|
+
super(message);
|
|
1789
|
+
this.name = "AgentTaskRunError";
|
|
1790
|
+
this.runConfiguration = options.runConfiguration;
|
|
1791
|
+
this.cause = options.cause;
|
|
1792
|
+
}
|
|
1793
|
+
};
|
|
1794
|
+
function withRunConfiguration(error, runConfiguration) {
|
|
1795
|
+
if (!runConfiguration) return error;
|
|
1796
|
+
if (error instanceof AgentTaskRunError) return error;
|
|
1797
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1798
|
+
return new AgentTaskRunError(message, { runConfiguration, cause: error });
|
|
1799
|
+
}
|
|
1800
|
+
async function runTask(taskDir, options) {
|
|
1801
|
+
const loaded = options?.loaded ?? await loadTask(taskDir);
|
|
1802
|
+
const trace2 = options?.tracing;
|
|
1803
|
+
if (!trace2) {
|
|
1804
|
+
return executeLoadedTask(
|
|
1805
|
+
loaded,
|
|
1806
|
+
options,
|
|
1807
|
+
createNoopAgentTaskTraceContext(),
|
|
1808
|
+
void 0
|
|
1809
|
+
);
|
|
1810
|
+
}
|
|
1811
|
+
const phase1 = await trace2.client.traceRun(
|
|
1812
|
+
buildTraceRunOptions(loaded, trace2),
|
|
1813
|
+
async (traceContext) => captureExecution(loaded, options, traceContext)
|
|
1814
|
+
);
|
|
1815
|
+
const canonical = await readCanonicalSnapshot(trace2);
|
|
1816
|
+
if (canonical) phase1.snapshot = canonical;
|
|
1817
|
+
return evaluate(loaded, options, phase1);
|
|
1818
|
+
}
|
|
1819
|
+
async function readCanonicalSnapshot(trace2) {
|
|
1820
|
+
if (!trace2.taskRunId) return null;
|
|
1821
|
+
const endpoint = process.env.AGENT_TASK_TRACE_ENDPOINT;
|
|
1822
|
+
const authToken = process.env.APO_AUTH_TOKEN;
|
|
1823
|
+
if (!endpoint || !authToken) return null;
|
|
1824
|
+
try {
|
|
1825
|
+
return await readTaskRunProjection({
|
|
1826
|
+
endpoint,
|
|
1827
|
+
authToken,
|
|
1828
|
+
taskRunId: trace2.taskRunId
|
|
1829
|
+
});
|
|
1830
|
+
} catch (error) {
|
|
1831
|
+
console.error(
|
|
1832
|
+
"[AgentTask] Backend projection read failed, using local snapshot:",
|
|
1833
|
+
error instanceof Error ? error.message : String(error)
|
|
1834
|
+
);
|
|
1835
|
+
return null;
|
|
1836
|
+
}
|
|
1837
|
+
}
|
|
1838
|
+
async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
|
|
1839
|
+
const {
|
|
1840
|
+
task: task2,
|
|
1841
|
+
adapter,
|
|
1842
|
+
taskDir: absoluteDir,
|
|
1843
|
+
files,
|
|
1844
|
+
checksPath,
|
|
1845
|
+
inlineChecks,
|
|
1846
|
+
moduleUrl,
|
|
1847
|
+
evalFileName
|
|
1848
|
+
} = loaded;
|
|
1849
|
+
const taskFiles = new TaskFiles(files);
|
|
1850
|
+
const tee = createProjectionTee(rawTrace);
|
|
1851
|
+
const { trace: trace2, getSnapshot } = { trace: tee.trace, getSnapshot: () => tee.getSnapshot() };
|
|
1852
|
+
return withApoRun(
|
|
1853
|
+
{ trace: trace2, parentSpanId: rawTrace.rootSpanId, taskId: task2.id },
|
|
1854
|
+
async () => {
|
|
1855
|
+
return executeRunBody();
|
|
1856
|
+
}
|
|
1857
|
+
);
|
|
1858
|
+
async function executeRunBody() {
|
|
1859
|
+
let state;
|
|
1860
|
+
let session;
|
|
1861
|
+
const transcriptTurns = [];
|
|
1862
|
+
try {
|
|
1863
|
+
await trace2.step(
|
|
1864
|
+
{
|
|
1865
|
+
step_name: "task.load",
|
|
1866
|
+
input: { taskId: task2.id, taskDir: absoluteDir },
|
|
1867
|
+
metadata: {
|
|
1868
|
+
taskId: task2.id,
|
|
1869
|
+
taskDir: absoluteDir,
|
|
1870
|
+
fileCount: files.length,
|
|
1871
|
+
hasChecks: inlineChecks || checksPath !== null
|
|
1872
|
+
},
|
|
1873
|
+
summarize: () => ({
|
|
1874
|
+
taskId: task2.id,
|
|
1875
|
+
adapterName: adapter.name
|
|
1876
|
+
})
|
|
1877
|
+
},
|
|
1878
|
+
async () => loaded
|
|
1879
|
+
);
|
|
1880
|
+
if (adapter.initialize) {
|
|
1881
|
+
const initState = await trace2.step(
|
|
1882
|
+
{
|
|
1883
|
+
step_name: "adapter.initialize",
|
|
1884
|
+
input: { adapterName: adapter.name },
|
|
1885
|
+
metadata: { adapterName: adapter.name },
|
|
1886
|
+
summarize: (result) => ({
|
|
1887
|
+
initialized: true,
|
|
1888
|
+
stateKeys: result && typeof result === "object" ? Object.keys(result) : []
|
|
1889
|
+
})
|
|
1890
|
+
},
|
|
1891
|
+
async () => adapter.initialize?.({
|
|
1892
|
+
task: task2,
|
|
1893
|
+
taskDir: absoluteDir,
|
|
1894
|
+
files,
|
|
1895
|
+
trace: trace2
|
|
1896
|
+
})
|
|
1897
|
+
);
|
|
1898
|
+
state = initState ?? void 0;
|
|
1899
|
+
}
|
|
1900
|
+
session = await trace2.step(
|
|
1901
|
+
{
|
|
1902
|
+
step_name: "adapter.open-session",
|
|
1903
|
+
input: { adapterName: adapter.name, hasState: state !== void 0 },
|
|
1904
|
+
metadata: { adapterName: adapter.name },
|
|
1905
|
+
summarize: () => ({ sessionOpened: true })
|
|
1906
|
+
},
|
|
1907
|
+
async () => adapter.startSession({
|
|
1908
|
+
task: task2,
|
|
1909
|
+
taskDir: absoluteDir,
|
|
1910
|
+
files,
|
|
1911
|
+
state,
|
|
1912
|
+
trace: trace2
|
|
1913
|
+
})
|
|
1914
|
+
);
|
|
1915
|
+
const runConfiguration = normalizeRunConfiguration(session.runConfiguration);
|
|
1916
|
+
try {
|
|
1917
|
+
if (!inlineChecks && checksPath) {
|
|
1918
|
+
resetTaskTurn();
|
|
1919
|
+
resetFlowChecks();
|
|
1920
|
+
await loadChecksModule(checksPath);
|
|
1921
|
+
}
|
|
1922
|
+
const taskTurn = getTaskTurn();
|
|
1923
|
+
resetTaskTurn();
|
|
1924
|
+
const turnFn = resolveTurn(adapter.turn, taskTurn);
|
|
1925
|
+
if (turnFn) {
|
|
1926
|
+
const turnTranscript = [];
|
|
1927
|
+
let lastTurnResponse;
|
|
1928
|
+
const maxTurns = options?.maxTurnsOverride ?? task2.maxTurns ?? 10;
|
|
1929
|
+
for (let turnNum = 1; turnNum <= maxTurns; turnNum++) {
|
|
1930
|
+
const userTurn = await turnFn({ files: taskFiles, transcript: turnTranscript });
|
|
1931
|
+
if (userTurn === null || userTurn === void 0) break;
|
|
1932
|
+
const result2 = await trace2.step(
|
|
1933
|
+
{
|
|
1934
|
+
step_name: "task.turn",
|
|
1935
|
+
metadata: { turnNumber: turnNum },
|
|
1936
|
+
summarize: (r) => ({ response: r.response })
|
|
1937
|
+
},
|
|
1938
|
+
async (spanId) => {
|
|
1939
|
+
if (!session) throw new Error("Session not created");
|
|
1940
|
+
return session.sendUserTurn(userTurn, {
|
|
1941
|
+
trace: trace2,
|
|
1942
|
+
turnNumber: turnNum,
|
|
1943
|
+
parentSpanId: spanId
|
|
1944
|
+
});
|
|
1945
|
+
}
|
|
1946
|
+
);
|
|
1947
|
+
turnTranscript.push({
|
|
1948
|
+
turnNumber: turnNum,
|
|
1949
|
+
input: userTurn,
|
|
1950
|
+
output: result2.response
|
|
1951
|
+
});
|
|
1952
|
+
transcriptTurns.push({
|
|
1953
|
+
turnNumber: turnNum,
|
|
1954
|
+
userAction: userTurn,
|
|
1955
|
+
agentResponse: result2.response
|
|
1956
|
+
});
|
|
1957
|
+
options?.onTurn?.(turnNum, userTurn, result2.response);
|
|
1958
|
+
lastTurnResponse = result2.response;
|
|
1959
|
+
}
|
|
1960
|
+
if (lastTurnResponse !== void 0) {
|
|
1961
|
+
rawTrace.endRoot({ output: { response: lastTurnResponse } });
|
|
1962
|
+
}
|
|
1963
|
+
}
|
|
1964
|
+
const collected = await trace2.step(
|
|
1965
|
+
{
|
|
1966
|
+
step_name: "adapter.collect-deliverables",
|
|
1967
|
+
input: { adapterName: adapter.name, expectedDeliverables: task2.deliverables },
|
|
1968
|
+
metadata: { adapterName: adapter.name },
|
|
1969
|
+
summarize: (result2) => {
|
|
1970
|
+
const keys = result2 && typeof result2 === "object" ? Object.keys(result2) : [];
|
|
1971
|
+
return { deliverableCount: keys.length, deliverableNames: keys };
|
|
1972
|
+
}
|
|
1973
|
+
},
|
|
1974
|
+
async () => {
|
|
1975
|
+
if (!session) {
|
|
1976
|
+
throw new Error("Adapter session was not created");
|
|
1977
|
+
}
|
|
1978
|
+
return adapter.collectDeliverables({
|
|
1979
|
+
task: task2,
|
|
1980
|
+
taskDir: absoluteDir,
|
|
1981
|
+
files,
|
|
1982
|
+
state,
|
|
1983
|
+
session,
|
|
1984
|
+
trace: trace2
|
|
1985
|
+
});
|
|
1986
|
+
}
|
|
1987
|
+
);
|
|
1988
|
+
const validationResults = await trace2.step(
|
|
1989
|
+
{
|
|
1990
|
+
step_name: "deliverables.validate",
|
|
1991
|
+
input: { deliverableNames: task2.deliverables },
|
|
1992
|
+
summarize: (result2) => {
|
|
1993
|
+
const r = result2;
|
|
1994
|
+
const passCount = r.results.filter((x) => x.pass).length;
|
|
1995
|
+
const broken = Object.keys(r.brokenDeliverables);
|
|
1996
|
+
return {
|
|
1997
|
+
total: r.results.length,
|
|
1998
|
+
passCount,
|
|
1999
|
+
failCount: r.results.length - passCount,
|
|
2000
|
+
brokenDeliverableCount: broken.length,
|
|
2001
|
+
brokenDeliverables: broken
|
|
2002
|
+
};
|
|
2003
|
+
}
|
|
2004
|
+
},
|
|
2005
|
+
async () => validateDeliverables(task2, collected, adapter.deliverables)
|
|
2006
|
+
);
|
|
2007
|
+
const checksResults = await trace2.step(
|
|
2008
|
+
{
|
|
2009
|
+
step_name: "checks.run",
|
|
2010
|
+
input: { sourceFile: inlineChecks ? evalFileName : checksPath },
|
|
2011
|
+
metadata: { sourceFile: inlineChecks ? evalFileName : checksPath },
|
|
2012
|
+
summarize: (result2) => summarizeEvaluationResults(result2)
|
|
2013
|
+
},
|
|
2014
|
+
async () => {
|
|
2015
|
+
if (!inlineChecks) {
|
|
2016
|
+
return loadAndRunFlowChecks(
|
|
2017
|
+
checksPath,
|
|
2018
|
+
{
|
|
2019
|
+
snapshot: getSnapshot(),
|
|
2020
|
+
deliverables: collected,
|
|
2021
|
+
files,
|
|
2022
|
+
task: task2,
|
|
2023
|
+
judgeConfig: options?.judge
|
|
2024
|
+
},
|
|
2025
|
+
validationResults.brokenDeliverables
|
|
2026
|
+
);
|
|
2027
|
+
}
|
|
2028
|
+
return runTraceChecks({
|
|
2029
|
+
snapshot: getSnapshot(),
|
|
2030
|
+
deliverables: proxyBrokenDeliverables(
|
|
2031
|
+
collected,
|
|
2032
|
+
validationResults.brokenDeliverables
|
|
2033
|
+
),
|
|
2034
|
+
files,
|
|
2035
|
+
task: task2,
|
|
2036
|
+
judgeConfig: options?.judge,
|
|
2037
|
+
moduleUrl,
|
|
2038
|
+
displayFile: evalFileName
|
|
2039
|
+
});
|
|
2040
|
+
}
|
|
2041
|
+
);
|
|
2042
|
+
const result = aggregateResult(checksResults);
|
|
2043
|
+
return {
|
|
2044
|
+
task: task2,
|
|
2045
|
+
taskDir: absoluteDir,
|
|
2046
|
+
files,
|
|
2047
|
+
traceRunId,
|
|
2048
|
+
result,
|
|
2049
|
+
deliverables: collected,
|
|
2050
|
+
transcript: { turns: transcriptTurns },
|
|
2051
|
+
runConfiguration
|
|
2052
|
+
};
|
|
2053
|
+
} catch (error) {
|
|
2054
|
+
throw withRunConfiguration(error, runConfiguration);
|
|
2055
|
+
}
|
|
2056
|
+
} finally {
|
|
2057
|
+
if (adapter.cleanup) {
|
|
2058
|
+
try {
|
|
2059
|
+
await trace2.step(
|
|
2060
|
+
{
|
|
2061
|
+
step_name: "adapter.cleanup",
|
|
2062
|
+
input: { adapterName: adapter.name },
|
|
2063
|
+
metadata: { adapterName: adapter.name },
|
|
2064
|
+
summarize: () => ({ cleaned: true })
|
|
2065
|
+
},
|
|
2066
|
+
async () => adapter.cleanup?.({
|
|
2067
|
+
task: task2,
|
|
2068
|
+
taskDir: absoluteDir,
|
|
2069
|
+
files,
|
|
2070
|
+
state,
|
|
2071
|
+
session,
|
|
2072
|
+
trace: trace2
|
|
2073
|
+
})
|
|
2074
|
+
);
|
|
2075
|
+
} catch (error) {
|
|
2076
|
+
console.error(
|
|
2077
|
+
"[AgentTask] Cleanup failed:",
|
|
2078
|
+
error instanceof Error ? error.message : String(error)
|
|
2079
|
+
);
|
|
2080
|
+
}
|
|
2081
|
+
}
|
|
2082
|
+
if (session?.close) {
|
|
2083
|
+
try {
|
|
2084
|
+
await session.close();
|
|
2085
|
+
} catch {
|
|
2086
|
+
}
|
|
2087
|
+
}
|
|
2088
|
+
}
|
|
2089
|
+
}
|
|
2090
|
+
}
|
|
2091
|
+
async function captureExecution(loaded, options, rawTrace) {
|
|
2092
|
+
const {
|
|
2093
|
+
task: task2,
|
|
2094
|
+
adapter,
|
|
2095
|
+
taskDir: absoluteDir,
|
|
2096
|
+
files,
|
|
2097
|
+
checksPath,
|
|
2098
|
+
inlineChecks
|
|
2099
|
+
} = loaded;
|
|
2100
|
+
const taskFiles = new TaskFiles(files);
|
|
2101
|
+
const tee = createProjectionTee(rawTrace);
|
|
2102
|
+
const trace2 = tee.trace;
|
|
2103
|
+
return withApoRun(
|
|
2104
|
+
{ trace: trace2, parentSpanId: rawTrace.rootSpanId, taskId: task2.id },
|
|
2105
|
+
async () => {
|
|
2106
|
+
let state;
|
|
2107
|
+
let session;
|
|
2108
|
+
const transcriptTurns = [];
|
|
2109
|
+
try {
|
|
2110
|
+
await trace2.step(
|
|
2111
|
+
{ step_name: "task.load", input: { taskId: task2.id, taskDir: absoluteDir }, metadata: { taskId: task2.id, taskDir: absoluteDir, fileCount: files.length, hasChecks: inlineChecks || checksPath !== null }, summarize: () => ({ taskId: task2.id, adapterName: adapter.name }) },
|
|
2112
|
+
async () => loaded
|
|
2113
|
+
);
|
|
2114
|
+
if (adapter.initialize) {
|
|
2115
|
+
const initState = await trace2.step(
|
|
2116
|
+
{ step_name: "adapter.initialize", input: { adapterName: adapter.name }, metadata: { adapterName: adapter.name }, summarize: (result) => ({ initialized: true, stateKeys: result && typeof result === "object" ? Object.keys(result) : [] }) },
|
|
2117
|
+
async () => adapter.initialize?.({ task: task2, taskDir: absoluteDir, files, trace: trace2 })
|
|
2118
|
+
);
|
|
2119
|
+
state = initState ?? void 0;
|
|
2120
|
+
}
|
|
2121
|
+
session = await trace2.step(
|
|
2122
|
+
{ step_name: "adapter.open-session", input: { adapterName: adapter.name, hasState: state !== void 0 }, metadata: { adapterName: adapter.name }, summarize: () => ({ sessionOpened: true }) },
|
|
2123
|
+
async () => adapter.startSession({ task: task2, taskDir: absoluteDir, files, state, trace: trace2 })
|
|
2124
|
+
);
|
|
2125
|
+
const runConfiguration = normalizeRunConfiguration(session.runConfiguration);
|
|
2126
|
+
try {
|
|
2127
|
+
if (!inlineChecks && checksPath) {
|
|
2128
|
+
resetTaskTurn();
|
|
2129
|
+
resetFlowChecks();
|
|
2130
|
+
await loadChecksModule(checksPath);
|
|
2131
|
+
}
|
|
2132
|
+
const taskTurn = getTaskTurn();
|
|
2133
|
+
resetTaskTurn();
|
|
2134
|
+
const turnFn = resolveTurn(adapter.turn, taskTurn);
|
|
2135
|
+
if (turnFn) {
|
|
2136
|
+
const turnTranscript = [];
|
|
2137
|
+
let lastTurnResponse;
|
|
2138
|
+
const maxTurns = options?.maxTurnsOverride ?? task2.maxTurns ?? 10;
|
|
2139
|
+
for (let turnNum = 1; turnNum <= maxTurns; turnNum++) {
|
|
2140
|
+
const userTurn = await turnFn({ files: taskFiles, transcript: turnTranscript });
|
|
2141
|
+
if (userTurn === null || userTurn === void 0) break;
|
|
2142
|
+
const result = await trace2.step(
|
|
2143
|
+
{ step_name: "task.turn", metadata: { turnNumber: turnNum }, summarize: (r) => ({ response: r.response }) },
|
|
2144
|
+
async (spanId) => {
|
|
2145
|
+
if (!session) throw new Error("Session not created");
|
|
2146
|
+
return session.sendUserTurn(userTurn, { trace: trace2, turnNumber: turnNum, parentSpanId: spanId });
|
|
2147
|
+
}
|
|
2148
|
+
);
|
|
2149
|
+
turnTranscript.push({ turnNumber: turnNum, input: userTurn, output: result.response });
|
|
2150
|
+
transcriptTurns.push({ turnNumber: turnNum, userAction: userTurn, agentResponse: result.response });
|
|
2151
|
+
options?.onTurn?.(turnNum, userTurn, result.response);
|
|
2152
|
+
lastTurnResponse = result.response;
|
|
2153
|
+
}
|
|
2154
|
+
if (lastTurnResponse !== void 0) {
|
|
2155
|
+
rawTrace.endRoot({ output: { response: lastTurnResponse } });
|
|
2156
|
+
}
|
|
2157
|
+
}
|
|
2158
|
+
const collected = await trace2.step(
|
|
2159
|
+
{ step_name: "adapter.collect-deliverables", input: { adapterName: adapter.name, expectedDeliverables: task2.deliverables }, metadata: { adapterName: adapter.name }, summarize: (result) => {
|
|
2160
|
+
const keys = result && typeof result === "object" ? Object.keys(result) : [];
|
|
2161
|
+
return { deliverableCount: keys.length, deliverableNames: keys };
|
|
2162
|
+
} },
|
|
2163
|
+
async () => {
|
|
2164
|
+
if (!session) throw new Error("Adapter session was not created");
|
|
2165
|
+
return adapter.collectDeliverables({ task: task2, taskDir: absoluteDir, files, state, session, trace: trace2 });
|
|
2166
|
+
}
|
|
2167
|
+
);
|
|
2168
|
+
const snapshot = tee.getSnapshot();
|
|
2169
|
+
return {
|
|
2170
|
+
traceRunId: rawTrace.runId,
|
|
2171
|
+
collected,
|
|
2172
|
+
transcriptTurns,
|
|
2173
|
+
runConfiguration,
|
|
2174
|
+
snapshot
|
|
2175
|
+
};
|
|
2176
|
+
} catch (error) {
|
|
2177
|
+
throw withRunConfiguration(error, runConfiguration);
|
|
2178
|
+
}
|
|
2179
|
+
} finally {
|
|
2180
|
+
if (adapter.cleanup) {
|
|
2181
|
+
try {
|
|
2182
|
+
await trace2.step(
|
|
2183
|
+
{ step_name: "adapter.cleanup", input: { adapterName: adapter.name }, metadata: { adapterName: adapter.name }, summarize: () => ({ cleaned: true }) },
|
|
2184
|
+
async () => adapter.cleanup?.({ task: task2, taskDir: absoluteDir, files, state, session, trace: trace2 })
|
|
2185
|
+
);
|
|
2186
|
+
} catch (error) {
|
|
2187
|
+
console.error("[AgentTask] Cleanup failed:", error instanceof Error ? error.message : String(error));
|
|
2188
|
+
}
|
|
2189
|
+
}
|
|
2190
|
+
if (session?.close) {
|
|
2191
|
+
try {
|
|
2192
|
+
await session.close();
|
|
2193
|
+
} catch {
|
|
2194
|
+
}
|
|
2195
|
+
}
|
|
2196
|
+
}
|
|
2197
|
+
}
|
|
2198
|
+
);
|
|
2199
|
+
}
|
|
2200
|
+
async function evaluate(loaded, options, phase1) {
|
|
2201
|
+
const {
|
|
2202
|
+
task: task2,
|
|
2203
|
+
adapter,
|
|
2204
|
+
taskDir: absoluteDir,
|
|
2205
|
+
files,
|
|
2206
|
+
checksPath,
|
|
2207
|
+
inlineChecks,
|
|
2208
|
+
moduleUrl,
|
|
2209
|
+
evalFileName
|
|
2210
|
+
} = loaded;
|
|
2211
|
+
const validationResults = validateDeliverables(task2, phase1.collected, adapter.deliverables);
|
|
2212
|
+
const deliverables = proxyBrokenDeliverables(
|
|
2213
|
+
phase1.collected,
|
|
2214
|
+
validationResults.brokenDeliverables
|
|
2215
|
+
);
|
|
2216
|
+
const checksResults = await (inlineChecks ? runTraceChecks({
|
|
2217
|
+
snapshot: phase1.snapshot,
|
|
2218
|
+
deliverables,
|
|
2219
|
+
files,
|
|
2220
|
+
task: task2,
|
|
2221
|
+
judgeConfig: options?.judge,
|
|
2222
|
+
moduleUrl,
|
|
2223
|
+
displayFile: evalFileName
|
|
2224
|
+
}) : loadAndRunFlowChecks(
|
|
2225
|
+
checksPath,
|
|
2226
|
+
{
|
|
2227
|
+
snapshot: phase1.snapshot,
|
|
2228
|
+
deliverables,
|
|
2229
|
+
files,
|
|
2230
|
+
task: task2,
|
|
2231
|
+
judgeConfig: options?.judge
|
|
2232
|
+
},
|
|
2233
|
+
validationResults.brokenDeliverables
|
|
2234
|
+
));
|
|
2235
|
+
const result = aggregateResult(checksResults);
|
|
2236
|
+
return {
|
|
2237
|
+
task: task2,
|
|
2238
|
+
taskDir: absoluteDir,
|
|
2239
|
+
files,
|
|
2240
|
+
traceRunId: phase1.traceRunId,
|
|
2241
|
+
result,
|
|
2242
|
+
deliverables: phase1.collected,
|
|
2243
|
+
transcript: { turns: phase1.transcriptTurns },
|
|
2244
|
+
runConfiguration: phase1.runConfiguration
|
|
2245
|
+
};
|
|
2246
|
+
}
|
|
2247
|
+
function summarizeEvaluationResults(results) {
|
|
2248
|
+
const passCount = results.filter((r) => r.pass).length;
|
|
2249
|
+
return {
|
|
2250
|
+
total: results.length,
|
|
2251
|
+
passCount,
|
|
2252
|
+
failCount: results.length - passCount,
|
|
2253
|
+
results: results.map((r) => ({
|
|
2254
|
+
id: r.id,
|
|
2255
|
+
pass: r.pass,
|
|
2256
|
+
reasoning: r.reasoning,
|
|
2257
|
+
evaluator_type: r.evaluator_type,
|
|
2258
|
+
judge_model: r.judge?.model
|
|
2259
|
+
}))
|
|
2260
|
+
};
|
|
2261
|
+
}
|
|
2262
|
+
function buildTraceRunOptions(loaded, tracing) {
|
|
2263
|
+
const tags = Array.from(/* @__PURE__ */ new Set(["agent-task", "e2e", ...tracing.tags ?? []]));
|
|
2264
|
+
const rootMetadata = {
|
|
2265
|
+
taskId: loaded.task.id,
|
|
2266
|
+
taskDir: loaded.taskDir,
|
|
2267
|
+
adapterName: loaded.adapter.name
|
|
2268
|
+
};
|
|
2269
|
+
if (tracing.taskRunId) {
|
|
2270
|
+
rootMetadata[APO_TASK_ID] = loaded.task.id;
|
|
2271
|
+
rootMetadata[APO_TASK_RUN_ID] = tracing.taskRunId;
|
|
2272
|
+
}
|
|
2273
|
+
return {
|
|
2274
|
+
project: tracing.project,
|
|
2275
|
+
task_id: loaded.task.id,
|
|
2276
|
+
flow_name: tracing.flowName ?? `agent-task.${loaded.task.id}`,
|
|
2277
|
+
version: tracing.version,
|
|
2278
|
+
environment: tracing.environment,
|
|
2279
|
+
tags,
|
|
2280
|
+
run_metadata: {
|
|
2281
|
+
taskDir: loaded.taskDir,
|
|
2282
|
+
adapterName: loaded.adapter.name,
|
|
2283
|
+
source: "agent-task-e2e",
|
|
2284
|
+
...tracing.runMetadata
|
|
2285
|
+
},
|
|
2286
|
+
rootSpan: {
|
|
2287
|
+
task_id: loaded.task.id,
|
|
2288
|
+
step_name: "agent-task.run",
|
|
2289
|
+
observation_type: "CHAIN",
|
|
2290
|
+
model: "agent-task",
|
|
2291
|
+
metadata: rootMetadata
|
|
2292
|
+
}
|
|
2293
|
+
};
|
|
2294
|
+
}
|
|
2295
|
+
|
|
2296
|
+
// src/agent-task/cli.ts
|
|
2297
|
+
import { relative, resolve as resolve3 } from "path";
|
|
2298
|
+
|
|
2299
|
+
// src/agent-task/discovery.ts
|
|
2300
|
+
import { existsSync as existsSync3, readdirSync as readdirSync2, statSync as statSync3 } from "fs";
|
|
2301
|
+
import { join as join2, resolve as resolve2 } from "path";
|
|
2302
|
+
function discoverAgentTaskDirs(rootDir) {
|
|
2303
|
+
const absoluteRoot = resolve2(rootDir);
|
|
2304
|
+
if (!existsSync3(absoluteRoot) || !statSync3(absoluteRoot).isDirectory()) {
|
|
2305
|
+
throw new Error(`Task directory not found: ${absoluteRoot}`);
|
|
2306
|
+
}
|
|
2307
|
+
const discovered = [];
|
|
2308
|
+
walkForTaskDirs(absoluteRoot, discovered);
|
|
2309
|
+
return discovered.sort();
|
|
2310
|
+
}
|
|
2311
|
+
function walkForTaskDirs(currentDir, discovered) {
|
|
2312
|
+
if (readdirSync2(currentDir).some((f) => f.endsWith(".eval.ts"))) {
|
|
2313
|
+
discovered.push(currentDir);
|
|
2314
|
+
return;
|
|
2315
|
+
}
|
|
2316
|
+
for (const entry of readdirSync2(currentDir)) {
|
|
2317
|
+
const absolutePath = join2(currentDir, entry);
|
|
2318
|
+
if (!statSync3(absolutePath).isDirectory()) {
|
|
2319
|
+
continue;
|
|
2320
|
+
}
|
|
2321
|
+
walkForTaskDirs(absolutePath, discovered);
|
|
2322
|
+
}
|
|
2323
|
+
}
|
|
2324
|
+
|
|
2325
|
+
// src/agent-task/otel-trace-client.ts
|
|
2326
|
+
import { trace, context } from "@opentelemetry/api";
|
|
2327
|
+
function monotonicNowMs2() {
|
|
2328
|
+
if (typeof performance !== "undefined" && typeof performance.now === "function") {
|
|
2329
|
+
return performance.now();
|
|
2330
|
+
}
|
|
2331
|
+
return Date.now();
|
|
2332
|
+
}
|
|
2333
|
+
function round32(n) {
|
|
2334
|
+
return Math.round(n * 1e3) / 1e3;
|
|
2335
|
+
}
|
|
2336
|
+
function createOtelAgentTaskTraceClient(config) {
|
|
2337
|
+
let myHandle = null;
|
|
2338
|
+
async function ensureProvider() {
|
|
2339
|
+
if (!myHandle) {
|
|
2340
|
+
const headers = config.headers ?? (config.authToken ? { Authorization: `Bearer ${config.authToken}` } : {});
|
|
2341
|
+
myHandle = await configureApoTelemetry({
|
|
2342
|
+
takeOwnership: true,
|
|
2343
|
+
endpoint: `${config.endpoint.replace(/\/$/, "")}/api/public/otel/v1/traces`,
|
|
2344
|
+
serviceName: "apo-agent-task",
|
|
2345
|
+
project: config.project,
|
|
2346
|
+
headers,
|
|
2347
|
+
processor: "simple",
|
|
2348
|
+
registerGlobal: true
|
|
2349
|
+
});
|
|
2350
|
+
}
|
|
2351
|
+
return myHandle.tracer;
|
|
2352
|
+
}
|
|
2353
|
+
const activeSpans = /* @__PURE__ */ new Map();
|
|
2354
|
+
let activeTracer = null;
|
|
2355
|
+
function createSpan(params) {
|
|
2356
|
+
if (!activeTracer) throw new Error("traceRun not started");
|
|
2357
|
+
const spanName = params.step_name || params.observation_type || "span";
|
|
2358
|
+
const span = activeTracer.startSpan(spanName);
|
|
2359
|
+
if (params.observation_type) {
|
|
2360
|
+
span.setAttribute("apo.observation.type", params.observation_type);
|
|
2361
|
+
}
|
|
2362
|
+
if (params.model) {
|
|
2363
|
+
span.setAttribute("gen_ai.request.model", params.model);
|
|
2364
|
+
}
|
|
2365
|
+
if (params.input) {
|
|
2366
|
+
if (params.input.messages) {
|
|
2367
|
+
span.setAttribute("gen_ai.input.messages", JSON.stringify(params.input.messages));
|
|
2368
|
+
} else {
|
|
2369
|
+
span.setAttribute("gen_ai.input.messages", JSON.stringify([{
|
|
2370
|
+
role: "system",
|
|
2371
|
+
parts: [{ type: "text", content: JSON.stringify(params.input) }]
|
|
2372
|
+
}]));
|
|
2373
|
+
}
|
|
2374
|
+
}
|
|
2375
|
+
if (params.metadata) {
|
|
2376
|
+
span.setAttribute("apo.metadata", JSON.stringify(params.metadata));
|
|
2377
|
+
}
|
|
2378
|
+
const spanId = span.spanContext().spanId;
|
|
2379
|
+
activeSpans.set(spanId, {
|
|
2380
|
+
span,
|
|
2381
|
+
startedAt: monotonicNowMs2(),
|
|
2382
|
+
model: params.model || "unknown"
|
|
2383
|
+
});
|
|
2384
|
+
return spanId;
|
|
2385
|
+
}
|
|
2386
|
+
function endSpan(params) {
|
|
2387
|
+
const active = activeSpans.get(params.id);
|
|
2388
|
+
if (!active) {
|
|
2389
|
+
return;
|
|
2390
|
+
}
|
|
2391
|
+
const latency = params.latency_ms ?? round32(monotonicNowMs2() - active.startedAt);
|
|
2392
|
+
active.span.setAttribute("latency_ms", latency);
|
|
2393
|
+
if (params.output !== void 0) {
|
|
2394
|
+
if (params.output.text) {
|
|
2395
|
+
active.span.setAttribute("gen_ai.output.messages", JSON.stringify([{
|
|
2396
|
+
role: "assistant",
|
|
2397
|
+
parts: [{ type: "text", content: String(params.output.text) }]
|
|
2398
|
+
}]));
|
|
2399
|
+
} else if (params.output.messages) {
|
|
2400
|
+
active.span.setAttribute("gen_ai.output.messages", JSON.stringify(params.output.messages));
|
|
2401
|
+
} else if (params.output.error) {
|
|
2402
|
+
active.span.setAttribute("gen_ai.output.messages", JSON.stringify([{
|
|
2403
|
+
role: "assistant",
|
|
2404
|
+
parts: [{ type: "text", content: String(params.output.error) }]
|
|
2405
|
+
}]));
|
|
2406
|
+
} else {
|
|
2407
|
+
active.span.setAttribute("gen_ai.output.messages", JSON.stringify([{
|
|
2408
|
+
role: "assistant",
|
|
2409
|
+
parts: [{ type: "text", content: JSON.stringify(params.output) }]
|
|
2410
|
+
}]));
|
|
2411
|
+
}
|
|
2412
|
+
}
|
|
2413
|
+
if (params.prompt_tokens !== void 0) {
|
|
2414
|
+
active.span.setAttribute("gen_ai.usage.input_tokens", params.prompt_tokens);
|
|
2415
|
+
}
|
|
2416
|
+
if (params.completion_tokens !== void 0) {
|
|
2417
|
+
active.span.setAttribute("gen_ai.usage.output_tokens", params.completion_tokens);
|
|
2418
|
+
}
|
|
2419
|
+
if (params.status_message) {
|
|
2420
|
+
active.span.setAttribute("apo.status_message", params.status_message);
|
|
2421
|
+
}
|
|
2422
|
+
if (params.level === "ERROR") {
|
|
2423
|
+
active.span.setStatus({ code: 2, message: params.status_message || "error" });
|
|
2424
|
+
}
|
|
2425
|
+
if (params.metadata) {
|
|
2426
|
+
active.span.setAttribute("apo.metadata", JSON.stringify(params.metadata));
|
|
2427
|
+
}
|
|
2428
|
+
active.span.end();
|
|
2429
|
+
activeSpans.delete(params.id);
|
|
2430
|
+
}
|
|
2431
|
+
return {
|
|
2432
|
+
async traceRun(params, fn) {
|
|
2433
|
+
const tracer = await ensureProvider();
|
|
2434
|
+
activeTracer = tracer;
|
|
2435
|
+
const project = params.project ?? config.project;
|
|
2436
|
+
const taskId = params.task_id ?? params.flow_name ?? "task";
|
|
2437
|
+
const flowName = params.flow_name ?? taskId;
|
|
2438
|
+
const rootSpan = tracer.startSpan("apo.task.run");
|
|
2439
|
+
rootSpan.setAttribute("apo.observation.type", "AGENT");
|
|
2440
|
+
rootSpan.setAttribute("apo.run.flow_name", flowName);
|
|
2441
|
+
rootSpan.setAttribute("apo.run.task_id", taskId);
|
|
2442
|
+
if (params.version) {
|
|
2443
|
+
rootSpan.setAttribute("apo.run.version", params.version);
|
|
2444
|
+
}
|
|
2445
|
+
if (params.run_metadata) {
|
|
2446
|
+
rootSpan.setAttribute("apo.run.metadata", JSON.stringify(params.run_metadata));
|
|
2447
|
+
}
|
|
2448
|
+
if (params.tags?.length) {
|
|
2449
|
+
rootSpan.setAttribute("apo.run.tags", JSON.stringify(params.tags));
|
|
2450
|
+
}
|
|
2451
|
+
const rootMeta = params.rootSpan?.metadata;
|
|
2452
|
+
const taskRunId = params.taskRunId ?? rootMeta?.["apo.task.run.id"] ?? params.run_metadata?.["agent_task_run_id"];
|
|
2453
|
+
if (taskRunId) {
|
|
2454
|
+
rootSpan.setAttribute("apo.task.id", taskId);
|
|
2455
|
+
rootSpan.setAttribute("apo.task.run.id", taskRunId);
|
|
2456
|
+
}
|
|
2457
|
+
const runId = rootSpan.spanContext().traceId;
|
|
2458
|
+
const rootSpanId = rootSpan.spanContext().spanId;
|
|
2459
|
+
activeSpans.set(rootSpanId, {
|
|
2460
|
+
span: rootSpan,
|
|
2461
|
+
startedAt: monotonicNowMs2(),
|
|
2462
|
+
model: "trace"
|
|
2463
|
+
});
|
|
2464
|
+
let callCount = 1;
|
|
2465
|
+
let rootEnded = false;
|
|
2466
|
+
const traceContext = {
|
|
2467
|
+
runId,
|
|
2468
|
+
rootSpanId,
|
|
2469
|
+
async step(options, stepFn) {
|
|
2470
|
+
const spanId = createSpan({
|
|
2471
|
+
...options,
|
|
2472
|
+
project,
|
|
2473
|
+
task_id: options.task_id ?? taskId,
|
|
2474
|
+
run_id: runId,
|
|
2475
|
+
parent_call_id: options.parent_call_id ?? rootSpanId,
|
|
2476
|
+
flow_name: options.flow_name ?? flowName
|
|
2477
|
+
});
|
|
2478
|
+
const startedAt = monotonicNowMs2();
|
|
2479
|
+
const activeSpan = activeSpans.get(spanId);
|
|
2480
|
+
const ctxWithSpan = activeSpan ? trace.setSpan(context.active(), activeSpan.span) : context.active();
|
|
2481
|
+
try {
|
|
2482
|
+
const result = await context.with(ctxWithSpan, () => stepFn(spanId));
|
|
2483
|
+
endSpan({
|
|
2484
|
+
id: spanId,
|
|
2485
|
+
latency_ms: round32(monotonicNowMs2() - startedAt),
|
|
2486
|
+
output: options.summarize?.(result),
|
|
2487
|
+
metadata: options.metadata
|
|
2488
|
+
});
|
|
2489
|
+
callCount += 1;
|
|
2490
|
+
return result;
|
|
2491
|
+
} catch (error) {
|
|
2492
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2493
|
+
endSpan({
|
|
2494
|
+
id: spanId,
|
|
2495
|
+
latency_ms: round32(monotonicNowMs2() - startedAt),
|
|
2496
|
+
output: { error: message },
|
|
2497
|
+
status_message: message,
|
|
2498
|
+
level: "ERROR",
|
|
2499
|
+
metadata: { ...options.metadata, failed: true }
|
|
2500
|
+
});
|
|
2501
|
+
throw error;
|
|
2502
|
+
}
|
|
2503
|
+
},
|
|
2504
|
+
recordEvent(options) {
|
|
2505
|
+
const spanId = createSpan({
|
|
2506
|
+
...options,
|
|
2507
|
+
project,
|
|
2508
|
+
task_id: options.task_id ?? taskId,
|
|
2509
|
+
run_id: runId,
|
|
2510
|
+
parent_call_id: options.parent_call_id ?? rootSpanId,
|
|
2511
|
+
flow_name: options.flow_name ?? flowName
|
|
2512
|
+
});
|
|
2513
|
+
endSpan({
|
|
2514
|
+
id: spanId,
|
|
2515
|
+
latency_ms: options.latency_ms,
|
|
2516
|
+
output: options.output,
|
|
2517
|
+
metadata: options.metadata
|
|
2518
|
+
});
|
|
2519
|
+
callCount += 1;
|
|
2520
|
+
return spanId;
|
|
2521
|
+
},
|
|
2522
|
+
endRoot(endParams) {
|
|
2523
|
+
if (rootEnded) return;
|
|
2524
|
+
endSpan({ id: rootSpanId, ...endParams });
|
|
2525
|
+
rootEnded = true;
|
|
2526
|
+
},
|
|
2527
|
+
// Non-LLM tracing helpers — these create spans through the same tracer
|
|
2528
|
+
async traceTool(name, params2, fn2) {
|
|
2529
|
+
const spanId = createSpan({
|
|
2530
|
+
project,
|
|
2531
|
+
task_id: taskId,
|
|
2532
|
+
run_id: runId,
|
|
2533
|
+
parent_call_id: rootSpanId,
|
|
2534
|
+
flow_name: flowName,
|
|
2535
|
+
step_name: `tool ${name}`,
|
|
2536
|
+
observation_type: "TOOL",
|
|
2537
|
+
model: "unknown",
|
|
2538
|
+
input: params2
|
|
2539
|
+
});
|
|
2540
|
+
try {
|
|
2541
|
+
const result = await fn2();
|
|
2542
|
+
endSpan({ id: spanId, output: result });
|
|
2543
|
+
callCount += 1;
|
|
2544
|
+
return result;
|
|
2545
|
+
} catch (error) {
|
|
2546
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
2547
|
+
endSpan({ id: spanId, output: { error: msg }, status_message: msg, level: "ERROR" });
|
|
2548
|
+
throw error;
|
|
2549
|
+
}
|
|
2550
|
+
},
|
|
2551
|
+
async traceRetriever(query, fn2) {
|
|
2552
|
+
const spanId = createSpan({
|
|
2553
|
+
project,
|
|
2554
|
+
task_id: taskId,
|
|
2555
|
+
run_id: runId,
|
|
2556
|
+
parent_call_id: rootSpanId,
|
|
2557
|
+
flow_name: flowName,
|
|
2558
|
+
step_name: "retrieve",
|
|
2559
|
+
observation_type: "RETRIEVER",
|
|
2560
|
+
model: "unknown",
|
|
2561
|
+
input: { query }
|
|
2562
|
+
});
|
|
2563
|
+
try {
|
|
2564
|
+
const r = await fn2();
|
|
2565
|
+
endSpan({ id: spanId, output: r });
|
|
2566
|
+
callCount += 1;
|
|
2567
|
+
return r;
|
|
2568
|
+
} catch (e) {
|
|
2569
|
+
const m = e instanceof Error ? e.message : String(e);
|
|
2570
|
+
endSpan({ id: spanId, output: { error: m }, level: "ERROR" });
|
|
2571
|
+
throw e;
|
|
2572
|
+
}
|
|
2573
|
+
},
|
|
2574
|
+
async traceChain(name, fn2) {
|
|
2575
|
+
const spanId = createSpan({
|
|
2576
|
+
project,
|
|
2577
|
+
task_id: taskId,
|
|
2578
|
+
run_id: runId,
|
|
2579
|
+
parent_call_id: rootSpanId,
|
|
2580
|
+
flow_name: flowName,
|
|
2581
|
+
step_name: name,
|
|
2582
|
+
observation_type: "CHAIN",
|
|
2583
|
+
model: "unknown"
|
|
2584
|
+
});
|
|
2585
|
+
try {
|
|
2586
|
+
const r = await fn2();
|
|
2587
|
+
endSpan({ id: spanId, output: r });
|
|
2588
|
+
callCount += 1;
|
|
2589
|
+
return r;
|
|
2590
|
+
} catch (e) {
|
|
2591
|
+
const m = e instanceof Error ? e.message : String(e);
|
|
2592
|
+
endSpan({ id: spanId, output: { error: m }, level: "ERROR" });
|
|
2593
|
+
throw e;
|
|
2594
|
+
}
|
|
2595
|
+
},
|
|
2596
|
+
async traceAgent(name, fn2) {
|
|
2597
|
+
const spanId = createSpan({
|
|
2598
|
+
project,
|
|
2599
|
+
task_id: taskId,
|
|
2600
|
+
run_id: runId,
|
|
2601
|
+
parent_call_id: rootSpanId,
|
|
2602
|
+
flow_name: flowName,
|
|
2603
|
+
step_name: name,
|
|
2604
|
+
observation_type: "AGENT",
|
|
2605
|
+
model: "unknown"
|
|
2606
|
+
});
|
|
2607
|
+
try {
|
|
2608
|
+
const r = await fn2();
|
|
2609
|
+
endSpan({ id: spanId, output: r });
|
|
2610
|
+
callCount += 1;
|
|
2611
|
+
return r;
|
|
2612
|
+
} catch (e) {
|
|
2613
|
+
const m = e instanceof Error ? e.message : String(e);
|
|
2614
|
+
endSpan({ id: spanId, output: { error: m }, level: "ERROR" });
|
|
2615
|
+
throw e;
|
|
2616
|
+
}
|
|
2617
|
+
},
|
|
2618
|
+
async traceGuardrail(name, fn2) {
|
|
2619
|
+
const spanId = createSpan({
|
|
2620
|
+
project,
|
|
2621
|
+
task_id: taskId,
|
|
2622
|
+
run_id: runId,
|
|
2623
|
+
parent_call_id: rootSpanId,
|
|
2624
|
+
flow_name: flowName,
|
|
2625
|
+
step_name: name,
|
|
2626
|
+
observation_type: "GUARDRAIL",
|
|
2627
|
+
model: "unknown"
|
|
2628
|
+
});
|
|
2629
|
+
try {
|
|
2630
|
+
const r = await fn2();
|
|
2631
|
+
endSpan({ id: spanId, output: r });
|
|
2632
|
+
callCount += 1;
|
|
2633
|
+
return r;
|
|
2634
|
+
} catch (e) {
|
|
2635
|
+
const m = e instanceof Error ? e.message : String(e);
|
|
2636
|
+
endSpan({ id: spanId, output: { error: m }, level: "ERROR" });
|
|
2637
|
+
throw e;
|
|
2638
|
+
}
|
|
2639
|
+
},
|
|
2640
|
+
async traceEmbedding(model, input, fn2) {
|
|
2641
|
+
const spanId = createSpan({
|
|
2642
|
+
project,
|
|
2643
|
+
task_id: taskId,
|
|
2644
|
+
run_id: runId,
|
|
2645
|
+
parent_call_id: rootSpanId,
|
|
2646
|
+
flow_name: flowName,
|
|
2647
|
+
step_name: "embedding",
|
|
2648
|
+
observation_type: "EMBEDDING",
|
|
2649
|
+
model,
|
|
2650
|
+
input: { value: input }
|
|
2651
|
+
});
|
|
2652
|
+
try {
|
|
2653
|
+
const r = await fn2();
|
|
2654
|
+
endSpan({ id: spanId, output: r });
|
|
2655
|
+
callCount += 1;
|
|
2656
|
+
return r;
|
|
2657
|
+
} catch (e) {
|
|
2658
|
+
const m = e instanceof Error ? e.message : String(e);
|
|
2659
|
+
endSpan({ id: spanId, output: { error: m }, level: "ERROR" });
|
|
2660
|
+
throw e;
|
|
2661
|
+
}
|
|
2662
|
+
},
|
|
2663
|
+
async score(scoreParams) {
|
|
2664
|
+
const { score: scoreFn } = await import("../otel/index.js");
|
|
2665
|
+
const scoreHeaders = config.headers ?? (config.authToken ? { Authorization: `Bearer ${config.authToken}` } : {});
|
|
2666
|
+
await scoreFn(
|
|
2667
|
+
{ traceId: runId, ...scoreParams },
|
|
2668
|
+
{ endpoint: config.endpoint, headers: scoreHeaders }
|
|
2669
|
+
);
|
|
2670
|
+
}
|
|
2671
|
+
};
|
|
2672
|
+
const agentTaskContext = {
|
|
2673
|
+
...traceContext,
|
|
2674
|
+
createSpan(options) {
|
|
2675
|
+
const spanId = createSpan({
|
|
2676
|
+
...options,
|
|
2677
|
+
project,
|
|
2678
|
+
run_id: runId,
|
|
2679
|
+
task_id: options.task_id ?? taskId,
|
|
2680
|
+
flow_name: options.flow_name ?? flowName
|
|
2681
|
+
});
|
|
2682
|
+
callCount += 1;
|
|
2683
|
+
return spanId;
|
|
2684
|
+
},
|
|
2685
|
+
endSpan(spanId, endParams) {
|
|
2686
|
+
endSpan({ id: spanId, ...endParams });
|
|
2687
|
+
}
|
|
2688
|
+
};
|
|
2689
|
+
return context.with(trace.setSpan(context.active(), rootSpan), async () => {
|
|
2690
|
+
let result;
|
|
2691
|
+
let runError;
|
|
2692
|
+
try {
|
|
2693
|
+
result = await fn(agentTaskContext);
|
|
2694
|
+
agentTaskContext.endRoot();
|
|
2695
|
+
} catch (error) {
|
|
2696
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2697
|
+
agentTaskContext.endRoot({
|
|
2698
|
+
output: { error: message },
|
|
2699
|
+
status_message: message,
|
|
2700
|
+
level: "ERROR",
|
|
2701
|
+
metadata: { failed: true }
|
|
2702
|
+
});
|
|
2703
|
+
runError = error;
|
|
2704
|
+
}
|
|
2705
|
+
let flushError;
|
|
2706
|
+
if (myHandle) {
|
|
2707
|
+
try {
|
|
2708
|
+
await myHandle.forceFlush();
|
|
2709
|
+
} catch (error) {
|
|
2710
|
+
flushError = error;
|
|
2711
|
+
}
|
|
2712
|
+
await myHandle.shutdown();
|
|
2713
|
+
myHandle = null;
|
|
2714
|
+
}
|
|
2715
|
+
if (runError !== void 0) throw runError;
|
|
2716
|
+
if (flushError !== void 0 && config.requirePersistence) {
|
|
2717
|
+
const detail = flushError instanceof Error ? flushError.message : String(flushError);
|
|
2718
|
+
throw new Error(`OTLP trace persistence failed: ${detail}`);
|
|
2719
|
+
}
|
|
2720
|
+
return result;
|
|
2721
|
+
});
|
|
2722
|
+
}
|
|
2723
|
+
};
|
|
2724
|
+
}
|
|
2725
|
+
|
|
2726
|
+
// src/agent-task/task-runtime.ts
|
|
2727
|
+
async function loadTaskRuntime(_taskDir) {
|
|
2728
|
+
return {
|
|
2729
|
+
judge: resolveJudgeFromEnv()
|
|
2730
|
+
};
|
|
2731
|
+
}
|
|
2732
|
+
function resolveJudgeFromEnv() {
|
|
2733
|
+
const openRouterModel = process.env.OPENROUTER_MODEL;
|
|
2734
|
+
const model = openRouterModel ?? process.env.OPENAI_MODEL;
|
|
2735
|
+
if (!model) return void 0;
|
|
2736
|
+
return {
|
|
2737
|
+
model,
|
|
2738
|
+
baseURL: openRouterModel ? process.env.OPENROUTER_BASE_URL : process.env.OPENAI_BASE_URL ?? "https://api.openai.com/v1",
|
|
2739
|
+
apiKey: openRouterModel ? process.env.OPENROUTER_API_KEY : process.env.OPENAI_API_KEY
|
|
2740
|
+
};
|
|
2741
|
+
}
|
|
2742
|
+
async function runTaskDir(taskDir) {
|
|
2743
|
+
const [loaded, runtime] = await Promise.all([
|
|
2744
|
+
loadTask(taskDir),
|
|
2745
|
+
loadTaskRuntime(taskDir)
|
|
2746
|
+
]);
|
|
2747
|
+
const endpoint = process.env.AGENT_TASK_TRACE_ENDPOINT;
|
|
2748
|
+
const hasTracing = endpoint && process.env.AGENT_TASK_PROJECT;
|
|
2749
|
+
const taskRunId = process.env.AGENT_TASK_RUN_ID;
|
|
2750
|
+
const tracing = hasTracing ? {
|
|
2751
|
+
client: createOtelAgentTaskTraceClient({
|
|
2752
|
+
endpoint,
|
|
2753
|
+
project: process.env.AGENT_TASK_PROJECT,
|
|
2754
|
+
headers: _buildAuthHeaders()
|
|
2755
|
+
}),
|
|
2756
|
+
project: process.env.AGENT_TASK_PROJECT,
|
|
2757
|
+
environment: process.env.AGENT_TASK_ENVIRONMENT ?? "default",
|
|
2758
|
+
...taskRunId ? { taskRunId } : {}
|
|
2759
|
+
} : void 0;
|
|
2760
|
+
const result = await runTask(taskDir, { ...runtime, tracing, loaded });
|
|
2761
|
+
return {
|
|
2762
|
+
taskDir: loaded.taskDir,
|
|
2763
|
+
taskId: loaded.task.id,
|
|
2764
|
+
pass: result.result.pass,
|
|
2765
|
+
checks: result.result.checks,
|
|
2766
|
+
adapterName: loaded.adapter.name,
|
|
2767
|
+
traceRunId: result.traceRunId,
|
|
2768
|
+
deliverables: result.deliverables,
|
|
2769
|
+
transcript: result.transcript,
|
|
2770
|
+
runConfiguration: result.runConfiguration
|
|
2771
|
+
};
|
|
2772
|
+
}
|
|
2773
|
+
function _buildAuthHeaders() {
|
|
2774
|
+
const pk = process.env.APO_PUBLIC_KEY;
|
|
2775
|
+
const sk = process.env.APO_SECRET_KEY;
|
|
2776
|
+
if (pk && sk) {
|
|
2777
|
+
const creds = typeof btoa === "function" ? btoa(`${pk}:${sk}`) : Buffer.from(`${pk}:${sk}`).toString("base64");
|
|
2778
|
+
return { Authorization: `Basic ${creds}` };
|
|
2779
|
+
}
|
|
2780
|
+
const token = process.env.APO_AUTH_TOKEN;
|
|
2781
|
+
if (token) {
|
|
2782
|
+
return { Authorization: `Bearer ${token}` };
|
|
2783
|
+
}
|
|
2784
|
+
return void 0;
|
|
2785
|
+
}
|
|
2786
|
+
|
|
2787
|
+
// src/agent-task/cli.ts
|
|
2788
|
+
function parseAgentTaskCliArgs(argv) {
|
|
2789
|
+
const options = {};
|
|
2790
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
2791
|
+
const arg = argv[index];
|
|
2792
|
+
if (arg === "--help" || arg === "-h") {
|
|
2793
|
+
options.help = true;
|
|
2794
|
+
continue;
|
|
2795
|
+
}
|
|
2796
|
+
if (arg === "--task") {
|
|
2797
|
+
options.task = requireNextValue(argv, index, "--task");
|
|
2798
|
+
index += 1;
|
|
2799
|
+
continue;
|
|
2800
|
+
}
|
|
2801
|
+
if (arg === "--dir") {
|
|
2802
|
+
options.dir = requireNextValue(argv, index, "--dir");
|
|
2803
|
+
index += 1;
|
|
2804
|
+
continue;
|
|
2805
|
+
}
|
|
2806
|
+
if (arg === "--grep") {
|
|
2807
|
+
options.grep = requireNextValue(argv, index, "--grep");
|
|
2808
|
+
index += 1;
|
|
2809
|
+
continue;
|
|
2810
|
+
}
|
|
2811
|
+
throw new Error(`Unknown argument: ${arg}`);
|
|
2812
|
+
}
|
|
2813
|
+
if (options.task && options.dir) {
|
|
2814
|
+
throw new Error("Use either --task or --dir, not both");
|
|
2815
|
+
}
|
|
2816
|
+
return options;
|
|
2817
|
+
}
|
|
2818
|
+
async function runAgentTaskCli(argv, cwd = process.cwd()) {
|
|
2819
|
+
const normalizedArgv = argv[0] === "--" ? argv.slice(1) : argv;
|
|
2820
|
+
const options = parseAgentTaskCliArgs(normalizedArgv);
|
|
2821
|
+
if (options.help) {
|
|
2822
|
+
printHelp();
|
|
2823
|
+
return 0;
|
|
2824
|
+
}
|
|
2825
|
+
const selectedTaskDirs = resolveSelectedTaskDirs(options, cwd);
|
|
2826
|
+
if (selectedTaskDirs.length === 0) {
|
|
2827
|
+
throw new Error("No task folders found for the selected input");
|
|
2828
|
+
}
|
|
2829
|
+
let hasFailures = false;
|
|
2830
|
+
let hasErrors = false;
|
|
2831
|
+
const results = await Promise.allSettled(
|
|
2832
|
+
selectedTaskDirs.map((taskDir) => runTaskDir(taskDir))
|
|
2833
|
+
);
|
|
2834
|
+
const summaries = [];
|
|
2835
|
+
for (let i = 0; i < results.length; i++) {
|
|
2836
|
+
const result = results[i];
|
|
2837
|
+
const taskDir = selectedTaskDirs[i];
|
|
2838
|
+
if (result.status === "fulfilled") {
|
|
2839
|
+
summaries.push(result.value);
|
|
2840
|
+
printTaskSummary(result.value, cwd);
|
|
2841
|
+
hasFailures ||= !result.value.pass;
|
|
2842
|
+
} else {
|
|
2843
|
+
hasErrors = true;
|
|
2844
|
+
printTaskError(taskDir, cwd, result.reason);
|
|
2845
|
+
}
|
|
2846
|
+
}
|
|
2847
|
+
printOverallSummary(summaries, hasErrors);
|
|
2848
|
+
if (hasErrors) {
|
|
2849
|
+
return 2;
|
|
2850
|
+
}
|
|
2851
|
+
return hasFailures ? 1 : 0;
|
|
2852
|
+
}
|
|
2853
|
+
function resolveSelectedTaskDirs(options, cwd) {
|
|
2854
|
+
if (options.task) {
|
|
2855
|
+
return [resolve3(cwd, options.task)];
|
|
2856
|
+
}
|
|
2857
|
+
const rootDir = resolve3(cwd, options.dir ?? "e2e");
|
|
2858
|
+
const discovered = discoverAgentTaskDirs(rootDir);
|
|
2859
|
+
if (!options.grep) {
|
|
2860
|
+
return discovered;
|
|
2861
|
+
}
|
|
2862
|
+
return discovered.filter(
|
|
2863
|
+
(taskDir) => (
|
|
2864
|
+
// String.prototype.includes (not Array.includes) — false positive
|
|
2865
|
+
// react-doctor-disable-next-line react-doctor/js-set-map-lookups
|
|
2866
|
+
relative(cwd, taskDir).includes(options.grep)
|
|
2867
|
+
)
|
|
2868
|
+
);
|
|
2869
|
+
}
|
|
2870
|
+
function requireNextValue(argv, index, flag) {
|
|
2871
|
+
const value = argv[index + 1];
|
|
2872
|
+
if (!value || value.startsWith("--")) {
|
|
2873
|
+
throw new Error(`Missing value for ${flag}`);
|
|
2874
|
+
}
|
|
2875
|
+
return value;
|
|
2876
|
+
}
|
|
2877
|
+
function printHelp() {
|
|
2878
|
+
console.log("Usage: agent-task-e2e [--task <task-folder> | --dir <tasks-root>] [--grep <substring>]");
|
|
2879
|
+
console.log("");
|
|
2880
|
+
console.log("Options");
|
|
2881
|
+
console.log("- --task <path> Run one task folder");
|
|
2882
|
+
console.log("- --dir <path> Run all task folders under a root directory");
|
|
2883
|
+
console.log("- --grep <text> Filter discovered task paths by substring");
|
|
2884
|
+
console.log("- --help Show this help");
|
|
2885
|
+
}
|
|
2886
|
+
function printTaskSummary(summary, cwd) {
|
|
2887
|
+
const relativeTaskDir = relative(cwd, summary.taskDir);
|
|
2888
|
+
console.log("");
|
|
2889
|
+
console.log(`${summary.pass ? "PASS" : "FAIL"} ${summary.taskId}`);
|
|
2890
|
+
console.log(`Path: ${relativeTaskDir}`);
|
|
2891
|
+
printResultGroup("Checks", summary.checks);
|
|
2892
|
+
if (summary.checks.length === 0 && !summary.pass) {
|
|
2893
|
+
console.log(NO_CHECKS_REGISTERED_MESSAGE);
|
|
2894
|
+
}
|
|
2895
|
+
}
|
|
2896
|
+
function printResultGroup(label, results) {
|
|
2897
|
+
if (results.length === 0) {
|
|
2898
|
+
return;
|
|
2899
|
+
}
|
|
2900
|
+
console.log(label);
|
|
2901
|
+
for (const result of results) {
|
|
2902
|
+
console.log(`- ${result.id}: ${result.pass ? "PASS" : "FAIL"}`);
|
|
2903
|
+
if (result.reasoning) {
|
|
2904
|
+
console.log(` ${result.reasoning}`);
|
|
2905
|
+
}
|
|
2906
|
+
}
|
|
2907
|
+
}
|
|
2908
|
+
function printTaskError(taskDir, cwd, error) {
|
|
2909
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2910
|
+
console.log("");
|
|
2911
|
+
console.log(`ERROR ${relative(cwd, taskDir)}`);
|
|
2912
|
+
console.log(message);
|
|
2913
|
+
}
|
|
2914
|
+
function printOverallSummary(summaries, hasErrors) {
|
|
2915
|
+
const passed = summaries.filter((summary) => summary.pass).length;
|
|
2916
|
+
const failed = summaries.length - passed;
|
|
2917
|
+
console.log("");
|
|
2918
|
+
console.log("Summary");
|
|
2919
|
+
console.log(`- total: ${summaries.length}`);
|
|
2920
|
+
console.log(`- passed: ${passed}`);
|
|
2921
|
+
console.log(`- failed: ${failed}`);
|
|
2922
|
+
console.log(`- errors: ${hasErrors ? "yes" : "no"}`);
|
|
2923
|
+
}
|
|
2924
|
+
|
|
2925
|
+
// src/agent-task/task-revision-manifest.ts
|
|
2926
|
+
import { createHash as createHash2 } from "crypto";
|
|
2927
|
+
function normalizeManifestPath(path) {
|
|
2928
|
+
return path.split("\\").join("/").normalize("NFC");
|
|
2929
|
+
}
|
|
2930
|
+
function sha256Hex(bytes) {
|
|
2931
|
+
return createHash2("sha256").update(bytes).digest("hex");
|
|
2932
|
+
}
|
|
2933
|
+
function compareBytewiseUtf8(a, b) {
|
|
2934
|
+
const ba = Buffer.from(a, "utf8");
|
|
2935
|
+
const bb = Buffer.from(b, "utf8");
|
|
2936
|
+
for (let i = 0; i < Math.min(ba.length, bb.length); i++) {
|
|
2937
|
+
if (ba[i] !== bb[i]) return ba[i] < bb[i] ? -1 : 1;
|
|
2938
|
+
}
|
|
2939
|
+
return ba.length - bb.length;
|
|
2940
|
+
}
|
|
2941
|
+
function buildManifest(files, summary) {
|
|
2942
|
+
const entries = files.map((f) => {
|
|
2943
|
+
const path = normalizeManifestPath(f.path);
|
|
2944
|
+
return {
|
|
2945
|
+
path,
|
|
2946
|
+
sizeBytes: f.content.byteLength,
|
|
2947
|
+
sha256: sha256Hex(f.content),
|
|
2948
|
+
modeClass: f.modeClass
|
|
2949
|
+
};
|
|
2950
|
+
});
|
|
2951
|
+
entries.sort((a, b) => compareBytewiseUtf8(a.path, b.path));
|
|
2952
|
+
return {
|
|
2953
|
+
schemaVersion: 1,
|
|
2954
|
+
files: entries,
|
|
2955
|
+
summary: {
|
|
2956
|
+
fileCount: entries.length,
|
|
2957
|
+
uncompressedSizeBytes: entries.reduce((n, f) => n + f.sizeBytes, 0),
|
|
2958
|
+
excludedFileCount: summary?.excludedFileCount ?? 0,
|
|
2959
|
+
excludedDirectoryCount: summary?.excludedDirectoryCount ?? 0
|
|
2960
|
+
}
|
|
2961
|
+
};
|
|
2962
|
+
}
|
|
2963
|
+
function canonicalManifestJson(manifest) {
|
|
2964
|
+
const filesPayload = manifest.files.map((f) => ({
|
|
2965
|
+
modeClass: f.modeClass,
|
|
2966
|
+
path: f.path,
|
|
2967
|
+
sha256: f.sha256,
|
|
2968
|
+
sizeBytes: f.sizeBytes
|
|
2969
|
+
}));
|
|
2970
|
+
return JSON.stringify({ files: filesPayload, schemaVersion: manifest.schemaVersion });
|
|
2971
|
+
}
|
|
2972
|
+
function contentSha256(manifest) {
|
|
2973
|
+
return sha256Hex(Buffer.from(canonicalManifestJson(manifest), "utf8"));
|
|
2974
|
+
}
|
|
2975
|
+
export {
|
|
2976
|
+
AgentTaskRunError,
|
|
2977
|
+
ApoSpanProcessor,
|
|
2978
|
+
FlowView,
|
|
2979
|
+
TEST_METHOD_NAMES,
|
|
2980
|
+
TaskFiles,
|
|
2981
|
+
buildManifest,
|
|
2982
|
+
canonicalManifestJson,
|
|
2983
|
+
contentSha256,
|
|
2984
|
+
createApoAnthropic,
|
|
2985
|
+
createApoOpenAI,
|
|
2986
|
+
createApoTracer,
|
|
2987
|
+
defineAdapter,
|
|
2988
|
+
defineTask,
|
|
2989
|
+
describe,
|
|
2990
|
+
discoverAgentTaskDirs,
|
|
2991
|
+
equals,
|
|
2992
|
+
fileArtifact,
|
|
2993
|
+
filePaths,
|
|
2994
|
+
fromAISDK,
|
|
2995
|
+
fromAnthropicMessages,
|
|
2996
|
+
fromOpenAIMessages,
|
|
2997
|
+
getActiveApoRun,
|
|
2998
|
+
getTaskTurn,
|
|
2999
|
+
includes,
|
|
3000
|
+
isFileArtifact,
|
|
3001
|
+
loadTask,
|
|
3002
|
+
loadTaskRuntime,
|
|
3003
|
+
matchValue,
|
|
3004
|
+
matches,
|
|
3005
|
+
normalizeManifestPath,
|
|
3006
|
+
parseAgentTaskCliArgs,
|
|
3007
|
+
persistFileArtifacts,
|
|
3008
|
+
registerApoTracing,
|
|
3009
|
+
resetApoTracing,
|
|
3010
|
+
resetFlowChecks,
|
|
3011
|
+
resetTaskRegistry,
|
|
3012
|
+
resetTaskTurn,
|
|
3013
|
+
resolveTurn,
|
|
3014
|
+
runAgentTaskCli,
|
|
3015
|
+
runTask,
|
|
3016
|
+
runTaskDir,
|
|
3017
|
+
satisfies,
|
|
3018
|
+
sha256Hex,
|
|
3019
|
+
similarity,
|
|
3020
|
+
task,
|
|
3021
|
+
defineCheck as test,
|
|
3022
|
+
turn,
|
|
3023
|
+
withApoRun,
|
|
3024
|
+
withApoRunSync
|
|
3025
|
+
};
|
|
3026
|
+
//# sourceMappingURL=public.js.map
|