@apo-ai/sdk 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +314 -0
- package/dist/agent-task/integrations/ai-sdk.d.ts +121 -0
- package/dist/agent-task/integrations/ai-sdk.js +10 -0
- package/dist/agent-task/integrations/ai-sdk.js.map +1 -0
- package/dist/agent-task/integrations/anthropic.d.ts +67 -0
- package/dist/agent-task/integrations/anthropic.js +10 -0
- package/dist/agent-task/integrations/anthropic.js.map +1 -0
- package/dist/agent-task/integrations/openai.d.ts +77 -0
- package/dist/agent-task/integrations/openai.js +10 -0
- package/dist/agent-task/integrations/openai.js.map +1 -0
- package/dist/agent-task/public.d.ts +1326 -0
- package/dist/agent-task/public.js +3026 -0
- package/dist/agent-task/public.js.map +1 -0
- package/dist/chunk-DT2LMB7O.js +80 -0
- package/dist/chunk-DT2LMB7O.js.map +1 -0
- package/dist/chunk-GDB6SDGN.js +75 -0
- package/dist/chunk-GDB6SDGN.js.map +1 -0
- package/dist/chunk-HKLBPIB4.js +232 -0
- package/dist/chunk-HKLBPIB4.js.map +1 -0
- package/dist/chunk-HWZVL6TS.js +37 -0
- package/dist/chunk-HWZVL6TS.js.map +1 -0
- package/dist/chunk-PMYLL2H4.js +495 -0
- package/dist/chunk-PMYLL2H4.js.map +1 -0
- package/dist/chunk-SZDSIPXC.js +74 -0
- package/dist/chunk-SZDSIPXC.js.map +1 -0
- package/dist/index.d.ts +95 -0
- package/dist/index.js +39 -0
- package/dist/index.js.map +1 -0
- package/dist/otel/index.d.ts +270 -0
- package/dist/otel/index.js +32 -0
- package/dist/otel/index.js.map +1 -0
- package/dist/tracing-DtbhYQV6.d.ts +26 -0
- package/dist/types-C-nrDEVf.d.ts +140 -0
- package/package.json +92 -0
|
@@ -0,0 +1,1326 @@
|
|
|
1
|
+
import { A as AgentTaskTraceContext, a as AgentTaskTraceOptions } from '../tracing-DtbhYQV6.js';
|
|
2
|
+
export { CreateApoTracerOptions, createApoTracer } from './integrations/ai-sdk.js';
|
|
3
|
+
export { CreateApoOpenAIOptions, createApoOpenAI } from './integrations/openai.js';
|
|
4
|
+
export { CreateApoAnthropicOptions, createApoAnthropic } from './integrations/anthropic.js';
|
|
5
|
+
import '../types-C-nrDEVf.js';
|
|
6
|
+
|
|
7
|
+
type TaskDefinition<TAdapterName extends string = string, TDeliverable extends string = string> = {
|
|
8
|
+
id: string;
|
|
9
|
+
adapter: TAdapterName;
|
|
10
|
+
description?: string;
|
|
11
|
+
deliverables: TDeliverable[];
|
|
12
|
+
maxTurns?: number;
|
|
13
|
+
metadata?: Record<string, unknown>;
|
|
14
|
+
checks?: string | false;
|
|
15
|
+
};
|
|
16
|
+
type TaskConfig<TDeliverable extends string = string> = Omit<TaskDefinition<string, TDeliverable>, "adapter">;
|
|
17
|
+
type FileEntry = {
|
|
18
|
+
relativePath: string;
|
|
19
|
+
absolutePath: string;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
declare class TaskFiles {
|
|
23
|
+
private entries;
|
|
24
|
+
constructor(entries: FileEntry[]);
|
|
25
|
+
read(relativePath: string): Promise<string>;
|
|
26
|
+
readBuffer(relativePath: string): Promise<Buffer>;
|
|
27
|
+
find(relativePath: string): FileEntry | undefined;
|
|
28
|
+
list(): FileEntry[];
|
|
29
|
+
has(relativePath: string): boolean;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
type TurnRecord = {
|
|
33
|
+
turnNumber: number;
|
|
34
|
+
input: unknown;
|
|
35
|
+
output: unknown;
|
|
36
|
+
};
|
|
37
|
+
type TurnContext = {
|
|
38
|
+
files: TaskFiles;
|
|
39
|
+
transcript: TurnRecord[];
|
|
40
|
+
};
|
|
41
|
+
type TurnFn<TUserTurn = unknown> = (ctx: TurnContext) => Promise<TUserTurn | null> | TUserTurn | null;
|
|
42
|
+
declare function turn<TUserTurn>(fn: TurnFn<TUserTurn>): void;
|
|
43
|
+
declare function getTaskTurn(): TurnFn | undefined;
|
|
44
|
+
declare function resetTaskTurn(): void;
|
|
45
|
+
declare function resolveTurn(adapterTurn: TurnFn | undefined, taskTurn: TurnFn | undefined): TurnFn;
|
|
46
|
+
|
|
47
|
+
type ValidatableSchemaLike = {
|
|
48
|
+
safeParse: (data: unknown) => {
|
|
49
|
+
success: boolean;
|
|
50
|
+
error?: {
|
|
51
|
+
message: string;
|
|
52
|
+
};
|
|
53
|
+
};
|
|
54
|
+
};
|
|
55
|
+
type DeliverableDefinition = ValidatableSchemaLike | {
|
|
56
|
+
schema?: ValidatableSchemaLike;
|
|
57
|
+
} | null;
|
|
58
|
+
type CollectedDeliverables = Record<string, unknown>;
|
|
59
|
+
type AgentTurnResult = {
|
|
60
|
+
response: unknown;
|
|
61
|
+
};
|
|
62
|
+
/**
|
|
63
|
+
* The adapter-reported identity of the agent under test for one Task Run
|
|
64
|
+
*. The adapter resolves its configuration (env vars, aliases,
|
|
65
|
+
* defaults), constructs the agent from that same resolved object, and reports
|
|
66
|
+
* the resolved values here.
|
|
67
|
+
*
|
|
68
|
+
* - `model` is the exact identifier the adapter passed to its runtime.
|
|
69
|
+
* - `effort` is the exact short value that runtime used (`low`/`medium`/...
|
|
70
|
+
* or `"default"`). Omitted means "the adapter did not report effort".
|
|
71
|
+
*
|
|
72
|
+
* The whole configuration is absent (`undefined`) when the adapter does not
|
|
73
|
+
* support reporting it. Values are descriptive — they never affect task
|
|
74
|
+
* selection, execution, scoring, retry, or deduplication.
|
|
75
|
+
*/
|
|
76
|
+
interface AgentTaskRunConfiguration {
|
|
77
|
+
model: string;
|
|
78
|
+
effort?: string;
|
|
79
|
+
}
|
|
80
|
+
type AdapterRuntimeState = Record<string, unknown>;
|
|
81
|
+
type InitializeContext = {
|
|
82
|
+
task: TaskDefinition;
|
|
83
|
+
taskDir: string;
|
|
84
|
+
files: FileEntry[];
|
|
85
|
+
trace: AgentTaskTraceContext;
|
|
86
|
+
};
|
|
87
|
+
type StartSessionContext = {
|
|
88
|
+
task: TaskDefinition;
|
|
89
|
+
taskDir: string;
|
|
90
|
+
files: FileEntry[];
|
|
91
|
+
state?: AdapterRuntimeState;
|
|
92
|
+
trace: AgentTaskTraceContext;
|
|
93
|
+
};
|
|
94
|
+
type CollectDeliverablesContext = {
|
|
95
|
+
task: TaskDefinition;
|
|
96
|
+
taskDir: string;
|
|
97
|
+
files: FileEntry[];
|
|
98
|
+
state?: AdapterRuntimeState;
|
|
99
|
+
session: AdapterSession;
|
|
100
|
+
trace: AgentTaskTraceContext;
|
|
101
|
+
};
|
|
102
|
+
type CleanupContext = {
|
|
103
|
+
task: TaskDefinition;
|
|
104
|
+
taskDir: string;
|
|
105
|
+
files: FileEntry[];
|
|
106
|
+
state?: AdapterRuntimeState;
|
|
107
|
+
session?: AdapterSession;
|
|
108
|
+
trace: AgentTaskTraceContext;
|
|
109
|
+
};
|
|
110
|
+
type AdapterSession = {
|
|
111
|
+
/**
|
|
112
|
+
* The adapter's resolved model/effort for this run. Read by
|
|
113
|
+
* `runTask()` immediately after `startSession()` returns, validated, and
|
|
114
|
+
* copied into {@link TaskRunResult.runConfiguration}. Absent for adapters
|
|
115
|
+
* that do not report configuration.
|
|
116
|
+
*/
|
|
117
|
+
runConfiguration?: AgentTaskRunConfiguration;
|
|
118
|
+
sendUserTurn: (turn: unknown, context: {
|
|
119
|
+
trace: AgentTaskTraceContext;
|
|
120
|
+
turnNumber: number;
|
|
121
|
+
parentSpanId?: string;
|
|
122
|
+
}) => Promise<AgentTurnResult>;
|
|
123
|
+
close?: () => Promise<void>;
|
|
124
|
+
};
|
|
125
|
+
type AdapterDefinition = {
|
|
126
|
+
name: string;
|
|
127
|
+
deliverables: Record<string, DeliverableDefinition>;
|
|
128
|
+
turn?: TurnFn;
|
|
129
|
+
initialize?: (ctx: InitializeContext) => Promise<AdapterRuntimeState | void>;
|
|
130
|
+
startSession: (ctx: StartSessionContext) => Promise<AdapterSession>;
|
|
131
|
+
collectDeliverables: (ctx: CollectDeliverablesContext) => Promise<CollectedDeliverables>;
|
|
132
|
+
cleanup?: (ctx: CleanupContext) => Promise<void>;
|
|
133
|
+
};
|
|
134
|
+
type TypedAdapterDefinition<TName extends string, TDeliverables extends Record<string, DeliverableDefinition>, TCollected extends CollectedDeliverables = CollectedDeliverables> = Omit<AdapterDefinition, "name" | "deliverables" | "collectDeliverables"> & {
|
|
135
|
+
name: TName;
|
|
136
|
+
deliverables: TDeliverables;
|
|
137
|
+
collectDeliverables: (ctx: CollectDeliverablesContext) => Promise<TCollected>;
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Trace Projection snapshot contract.
|
|
142
|
+
*
|
|
143
|
+
* This is the immutable transport/read snapshot of a Trace Projection — the
|
|
144
|
+
* query-optimized representation shared by the dashboard and agent-task
|
|
145
|
+
* assertions. It is NOT a new domain entity and never a source of truth; the
|
|
146
|
+
* canonical OpenTelemetry spans remain authoritative.
|
|
147
|
+
*
|
|
148
|
+
* Both the Python backend and the TypeScript SDK MUST serialize to this exact
|
|
149
|
+
* lower-camel-case JSON contract, so assertions read the same shape whether
|
|
150
|
+
* the snapshot came from the canonical repository, a local/offline recorder,
|
|
151
|
+
* or a legacy-Flow compatibility adapter.
|
|
152
|
+
*
|
|
153
|
+
* Rules:
|
|
154
|
+
* - `schemaVersion` versions the JSON contract.
|
|
155
|
+
* - `projectionVersion` identifies the normalizer/projector interpretation.
|
|
156
|
+
* - `source` is diagnostic; only `"canonical"` represents a durably projected
|
|
157
|
+
* Trace.
|
|
158
|
+
* - `complete` means the task execution root ended and every span in the
|
|
159
|
+
* acknowledged export was projected. It does not promise no unrelated
|
|
160
|
+
* service will ever export a late span.
|
|
161
|
+
* - Missing evidence is represented through `capabilities`, never converted to
|
|
162
|
+
* empty arrays, zero duration, or successful status.
|
|
163
|
+
* - Observations are immutable and sorted deterministically by invocation
|
|
164
|
+
* time, then span ID. Missing timestamps sort after timestamped ones.
|
|
165
|
+
*/
|
|
166
|
+
/** Whether a category of evidence is present in the projection. */
|
|
167
|
+
type EvidenceAvailability = "available" | "partial" | "unavailable";
|
|
168
|
+
/** Lifecycle status of an observation. `unset` = the source had no status. */
|
|
169
|
+
type ObservationStatus = "unset" | "ok" | "error";
|
|
170
|
+
/** What kind of Span an Observation was derived from. */
|
|
171
|
+
type ObservationType = "SPAN" | "GENERATION" | "TOOL" | "AGENT" | "SKILL" | "CHAIN" | "RETRIEVER" | "EMBEDDING" | "GUARDRAIL";
|
|
172
|
+
/**
|
|
173
|
+
* Per-category evidence availability. Every category must declare honestly
|
|
174
|
+
* what it carries so assertions can fail closed when evidence is missing
|
|
175
|
+
* instead of vacuously passing.
|
|
176
|
+
*/
|
|
177
|
+
interface TraceProjectionCapabilities {
|
|
178
|
+
messages: EvidenceAvailability;
|
|
179
|
+
tools: EvidenceAvailability;
|
|
180
|
+
errors: EvidenceAvailability;
|
|
181
|
+
timing: EvidenceAvailability;
|
|
182
|
+
skills: EvidenceAvailability;
|
|
183
|
+
subagents: EvidenceAvailability;
|
|
184
|
+
}
|
|
185
|
+
/** One chat message reconstructed from a generation observation. */
|
|
186
|
+
interface TraceProjectionMessage {
|
|
187
|
+
role: "system" | "user" | "assistant" | "tool";
|
|
188
|
+
content: string;
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* A derived interpretation of one Span, as shown by the dashboard and read by
|
|
192
|
+
* assertions. Unknown span kinds survive as `SPAN`; hierarchy is preserved via
|
|
193
|
+
* `parentSpanId`.
|
|
194
|
+
*/
|
|
195
|
+
interface TraceProjectionObservation {
|
|
196
|
+
spanId: string;
|
|
197
|
+
parentSpanId?: string;
|
|
198
|
+
type: ObservationType;
|
|
199
|
+
name: string;
|
|
200
|
+
startedAt?: string;
|
|
201
|
+
endedAt?: string;
|
|
202
|
+
durationMs?: number;
|
|
203
|
+
status: ObservationStatus;
|
|
204
|
+
errorMessage?: string;
|
|
205
|
+
input?: unknown;
|
|
206
|
+
output?: unknown;
|
|
207
|
+
model?: string;
|
|
208
|
+
toolName?: string;
|
|
209
|
+
toolParameters?: unknown;
|
|
210
|
+
toolResult?: unknown;
|
|
211
|
+
messages?: readonly TraceProjectionMessage[];
|
|
212
|
+
metadata?: Readonly<Record<string, unknown>>;
|
|
213
|
+
}
|
|
214
|
+
/** Trace-level facts. All timing is optional; absence is honest. */
|
|
215
|
+
interface TraceProjectionTrace {
|
|
216
|
+
traceId: string;
|
|
217
|
+
taskRunId?: string;
|
|
218
|
+
name?: string;
|
|
219
|
+
startedAt?: string;
|
|
220
|
+
endedAt?: string;
|
|
221
|
+
durationMs?: number;
|
|
222
|
+
complete: boolean;
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* The immutable read snapshot. Identical JSON shape across canonical, local,
|
|
226
|
+
* and legacy-flow sources — only `source` and `capabilities` differ.
|
|
227
|
+
*/
|
|
228
|
+
interface TraceProjectionSnapshot {
|
|
229
|
+
schemaVersion: 1;
|
|
230
|
+
projectionVersion: number;
|
|
231
|
+
source: "canonical" | "local" | "legacy-flow";
|
|
232
|
+
trace: TraceProjectionTrace;
|
|
233
|
+
capabilities: TraceProjectionCapabilities;
|
|
234
|
+
observations: readonly TraceProjectionObservation[];
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* The outcome of a single assertion.
|
|
239
|
+
*
|
|
240
|
+
* - ``"pass"`` → ``pass=true``
|
|
241
|
+
* - ``"fail"`` → ``pass=false``
|
|
242
|
+
* - ``"unsupported"`` → ``pass=false``. The trace projection lacked the
|
|
243
|
+
* evidence this assertion needed (e.g. timing, error status). An unsupported
|
|
244
|
+
* trace assertion fails closed and explains which evidence was unavailable;
|
|
245
|
+
* it must never silently pass. Value assertions (``t.check``) and LLM
|
|
246
|
+
* assertions (``t.judge``) do not consult trace capabilities and retain
|
|
247
|
+
* their existing pass/fail behavior.
|
|
248
|
+
*/
|
|
249
|
+
type AssertionOutcome = "pass" | "fail" | "unsupported";
|
|
250
|
+
/**
|
|
251
|
+
* Metadata about an LLM judge call. Populated by evaluators that use an
|
|
252
|
+
* LLM to make their pass/fail decision. All fields optional so code-only
|
|
253
|
+
* evaluators can leave this undefined.
|
|
254
|
+
*/
|
|
255
|
+
type JudgeMetadata = {
|
|
256
|
+
/** Model identifier, e.g. ``"google/gemini-2.5-flash-lite"``. */
|
|
257
|
+
model?: string;
|
|
258
|
+
/** The messages sent to the judge LLM. */
|
|
259
|
+
prompt?: {
|
|
260
|
+
system?: string;
|
|
261
|
+
user?: string;
|
|
262
|
+
};
|
|
263
|
+
/** Raw LLM response text before parsing into {pass, reasoning}. */
|
|
264
|
+
response?: string;
|
|
265
|
+
/** Token usage if available from the provider. */
|
|
266
|
+
tokens?: {
|
|
267
|
+
input: number;
|
|
268
|
+
output: number;
|
|
269
|
+
cache_creation?: number;
|
|
270
|
+
cache_read?: number;
|
|
271
|
+
};
|
|
272
|
+
/** Estimated cost in USD, if available. */
|
|
273
|
+
cost?: number;
|
|
274
|
+
/** Wall-clock latency of the judge call in milliseconds. */
|
|
275
|
+
latency_ms?: number;
|
|
276
|
+
/** Temperature or other sampling parameters, if relevant. */
|
|
277
|
+
temperature?: number;
|
|
278
|
+
};
|
|
279
|
+
/**
|
|
280
|
+
* A source location for a failed code check — lets the dashboard render the
|
|
281
|
+
* failure inline, editor-style. ``file`` is a display name (e.g. the checks
|
|
282
|
+
* filename); ``line``/``column`` are 1-indexed into that file.
|
|
283
|
+
*/
|
|
284
|
+
type CheckLocation = {
|
|
285
|
+
file: string;
|
|
286
|
+
line: number;
|
|
287
|
+
column?: number;
|
|
288
|
+
};
|
|
289
|
+
/**
|
|
290
|
+
* A single assertion within a code check (the recorder collects many per
|
|
291
|
+
* check). Carries structured Expected/Received so the dashboard can render
|
|
292
|
+
* testing-framework-style failures instead of a flattened prose string.
|
|
293
|
+
*
|
|
294
|
+
* - ``expected`` — what the assertion wanted (matcher label, or "≥1 read_file
|
|
295
|
+
* call" for trace asserts). Absent when not meaningful.
|
|
296
|
+
* - ``received`` — the actual value/count the run produced. Serialized scalar
|
|
297
|
+
* for code assertions; the raw evaluated value for LLM judges (see field).
|
|
298
|
+
* - ``location`` — where in the checks source this assertion lives.
|
|
299
|
+
*/
|
|
300
|
+
type AssertionResult = {
|
|
301
|
+
id: string;
|
|
302
|
+
pass: boolean;
|
|
303
|
+
reasoning: string;
|
|
304
|
+
/**
|
|
305
|
+
* The outcome category. ``"unsupported"`` means the trace
|
|
306
|
+
* projection lacked the evidence this assertion needed (e.g. timing,
|
|
307
|
+
* errors) — it fails closed (``pass=false``) with an explanatory reason
|
|
308
|
+
* rather than vacuously passing. Absent on legacy results.
|
|
309
|
+
*/
|
|
310
|
+
outcome?: AssertionOutcome;
|
|
311
|
+
expected?: string;
|
|
312
|
+
/**
|
|
313
|
+
* The actual value the assertion observed. For code assertions this is a
|
|
314
|
+
* short serialized scalar (`"3"`, `"read_file → write_file"`); for LLM
|
|
315
|
+
* judges it is the **raw value passed to `t.judge`** — an arbitrary JSON
|
|
316
|
+
* value (string, object, array, primitive), so the dashboard can render it
|
|
317
|
+
* with a structured viewer instead of a truncated string.
|
|
318
|
+
*/
|
|
319
|
+
received?: unknown;
|
|
320
|
+
location?: CheckLocation;
|
|
321
|
+
evaluator_type?: "llm" | "code";
|
|
322
|
+
judge?: JudgeMetadata;
|
|
323
|
+
};
|
|
324
|
+
/**
|
|
325
|
+
* Result of evaluating a single check.
|
|
326
|
+
*
|
|
327
|
+
* The three required fields ({@link id}, {@link pass}, {@link reasoning})
|
|
328
|
+
* have been here since the beginning. The optional fields below are
|
|
329
|
+
* enriched metadata that lets the dashboard show *what* was evaluated,
|
|
330
|
+
* *how* it was evaluated, and — for LLM judges — exactly what the judge
|
|
331
|
+
* was asked and what it answered.
|
|
332
|
+
*
|
|
333
|
+
* Evaluators that don't populate the optional fields still work; the
|
|
334
|
+
* dashboard gracefully falls back to the legacy three-field display.
|
|
335
|
+
*/
|
|
336
|
+
type EvaluationItemResult = {
|
|
337
|
+
id: string;
|
|
338
|
+
pass: boolean;
|
|
339
|
+
reasoning: string;
|
|
340
|
+
/** The rubric instruction from the task definition ("PASS if …"). */
|
|
341
|
+
instruction?: string;
|
|
342
|
+
/** Name of the deliverable this check was evaluated against. */
|
|
343
|
+
deliverable?: string;
|
|
344
|
+
/**
|
|
345
|
+
* What kind of evaluator produced this result.
|
|
346
|
+
* Registered checks use ``"code"``. LLM-backed assertions inside a check
|
|
347
|
+
* carry ``evaluator_type: "llm"`` in {@link assertions}.
|
|
348
|
+
* - ``"regex"`` — pattern matching
|
|
349
|
+
* Older persisted results may still use ``"llm"`` at this level.
|
|
350
|
+
*/
|
|
351
|
+
evaluator_type?: "llm" | "code" | "regex";
|
|
352
|
+
/**
|
|
353
|
+
* If this check was judged by an LLM, details about the judge call
|
|
354
|
+
* (model, prompt, response, tokens, cost, latency). Populated by the
|
|
355
|
+
* evaluator author; absent for code-only evaluators.
|
|
356
|
+
*/
|
|
357
|
+
judge?: JudgeMetadata;
|
|
358
|
+
/**
|
|
359
|
+
* For code checks: the source location of the failure (parsed from the
|
|
360
|
+
* thrown error / failed assertion stack). Lets the dashboard highlight the
|
|
361
|
+
* failing line. Absent when not a code check, when the check passed, or
|
|
362
|
+
* when no frame could be resolved to the checks module.
|
|
363
|
+
*/
|
|
364
|
+
location?: CheckLocation;
|
|
365
|
+
/**
|
|
366
|
+
* For code checks: the source filename the result came from (normally
|
|
367
|
+
* the ``*.eval.ts`` file; ``"checks.ts"`` for legacy tasks), so the dashboard can show it even
|
|
368
|
+
* when no line was resolved.
|
|
369
|
+
*/
|
|
370
|
+
source_file?: string;
|
|
371
|
+
/**
|
|
372
|
+
* For code checks: the per-assertion breakdown (Expected/Received/location
|
|
373
|
+
* each). Lets the dashboard mark every failing assertion at its own line,
|
|
374
|
+
* testing-framework-style. Absent for non-code evaluators and old runs.
|
|
375
|
+
*/
|
|
376
|
+
assertions?: AssertionResult[];
|
|
377
|
+
/**
|
|
378
|
+
* the snapshot source the checks ran against. Set to
|
|
379
|
+
* ``"legacy-flow"`` by the deprecated {@link runFlowChecks} wrapper so
|
|
380
|
+
* consumers can detect they're on the compatibility path. Absent on
|
|
381
|
+
* projection-first results (the default).
|
|
382
|
+
*/
|
|
383
|
+
source?: "canonical" | "local" | "legacy-flow";
|
|
384
|
+
/**
|
|
385
|
+
* the id of the `describe()` group this check was declared
|
|
386
|
+
* inside. Absent for checks declared at the top level (no enclosing
|
|
387
|
+
* describe) and for old results. The dashboard groups checks by this field.
|
|
388
|
+
*/
|
|
389
|
+
group_id?: string;
|
|
390
|
+
/**
|
|
391
|
+
* the display name of the enclosing `describe()` group.
|
|
392
|
+
* Defaults to the group id when the group was declared without a name.
|
|
393
|
+
* Absent when {@link group_id} is absent.
|
|
394
|
+
*/
|
|
395
|
+
group_name?: string;
|
|
396
|
+
};
|
|
397
|
+
type TaskEvaluationResult = {
|
|
398
|
+
checks: EvaluationItemResult[];
|
|
399
|
+
pass: boolean;
|
|
400
|
+
};
|
|
401
|
+
type TaskTranscript = {
|
|
402
|
+
turns: TaskTranscriptTurn[];
|
|
403
|
+
};
|
|
404
|
+
type TaskTranscriptTurn = {
|
|
405
|
+
turnNumber: number;
|
|
406
|
+
userAction: unknown;
|
|
407
|
+
agentResponse: unknown;
|
|
408
|
+
};
|
|
409
|
+
type TaskRunResult = {
|
|
410
|
+
task: TaskDefinition;
|
|
411
|
+
taskDir: string;
|
|
412
|
+
files: FileEntry[];
|
|
413
|
+
traceRunId?: string;
|
|
414
|
+
result: TaskEvaluationResult;
|
|
415
|
+
deliverables: Record<string, unknown>;
|
|
416
|
+
transcript: TaskTranscript;
|
|
417
|
+
/**
|
|
418
|
+
* The adapter's resolved model/effort for this run. Captured and
|
|
419
|
+
* normalized from `AdapterSession.runConfiguration` immediately after the
|
|
420
|
+
* session opens. Absent for adapters that do not report configuration.
|
|
421
|
+
*/
|
|
422
|
+
runConfiguration?: AgentTaskRunConfiguration;
|
|
423
|
+
};
|
|
424
|
+
|
|
425
|
+
/**
|
|
426
|
+
* TraceView — typed, capability-gated, derived access over a
|
|
427
|
+
* {@link TraceProjectionSnapshot}. This is the read model the projection-first
|
|
428
|
+
* assertion surface queries. It is the projection analogue of `FlowView`.
|
|
429
|
+
*
|
|
430
|
+
* The defining difference from `FlowView` is **capability honesty**: when the
|
|
431
|
+
* snapshot declares a category of evidence `unavailable`, the derived numeric
|
|
432
|
+
* facts (`durationMs`, `failedActions`, `turnCount`) return `undefined` rather
|
|
433
|
+
* than zero — so an assertion like `t.maxDurationMs` can record an explicit
|
|
434
|
+
* `unsupported` outcome instead of vacuously passing against a fabricated 0ms.
|
|
435
|
+
*
|
|
436
|
+
* `toolNamesInOrder` sorts by observation invocation time (`startedAt`) with a
|
|
437
|
+
* deterministic span-ID tie-breaker — NOT array/completion order. This fixes
|
|
438
|
+
* the drift where concurrent tools appeared in completion order.
|
|
439
|
+
*/
|
|
440
|
+
|
|
441
|
+
/** A tool call derived from a `TOOL` observation. */
|
|
442
|
+
interface TraceToolCall {
|
|
443
|
+
spanId: string;
|
|
444
|
+
name: string;
|
|
445
|
+
input?: unknown;
|
|
446
|
+
output?: unknown;
|
|
447
|
+
status: ObservationStatus;
|
|
448
|
+
startedAt?: string;
|
|
449
|
+
}
|
|
450
|
+
/** A skill load derived from a `SKILL` observation. */
|
|
451
|
+
interface TraceSkillLoad {
|
|
452
|
+
spanId: string;
|
|
453
|
+
skill: string;
|
|
454
|
+
startedAt?: string;
|
|
455
|
+
}
|
|
456
|
+
/** A subagent delegation derived from an `AGENT` observation. */
|
|
457
|
+
interface TraceSubagentCall {
|
|
458
|
+
spanId: string;
|
|
459
|
+
agent: string;
|
|
460
|
+
output?: unknown;
|
|
461
|
+
status: ObservationStatus;
|
|
462
|
+
startedAt?: string;
|
|
463
|
+
}
|
|
464
|
+
declare class TraceView {
|
|
465
|
+
readonly snapshot: TraceProjectionSnapshot;
|
|
466
|
+
constructor(snapshot: TraceProjectionSnapshot);
|
|
467
|
+
/** Evidence availability for a capability. */
|
|
468
|
+
requireCapability(capability: keyof TraceProjectionCapabilities): EvidenceAvailability;
|
|
469
|
+
/** Whether a capability is reported as available (not partial/unavailable). */
|
|
470
|
+
private isAvailable;
|
|
471
|
+
/** All chat messages flattened across generation observations, in order. */
|
|
472
|
+
get messages(): readonly TraceProjectionMessage[];
|
|
473
|
+
/** Tool calls derived from `TOOL` observations, in invocation order. */
|
|
474
|
+
get toolCalls(): readonly TraceToolCall[];
|
|
475
|
+
/** Tool names in invocation order with deterministic span-ID tie-breaking. */
|
|
476
|
+
get toolNamesInOrder(): readonly string[];
|
|
477
|
+
/** Skill loads derived from `SKILL` observations, in invocation order. */
|
|
478
|
+
get skillLoads(): readonly TraceSkillLoad[];
|
|
479
|
+
/** Subagent delegations derived from `AGENT` observations, in invocation order. */
|
|
480
|
+
get subagentCalls(): readonly TraceSubagentCall[];
|
|
481
|
+
/** Last assistant message content — the agent's "reply". Empty if none. */
|
|
482
|
+
get reply(): string;
|
|
483
|
+
/**
|
|
484
|
+
* Number of completed assistant turns, or `undefined` when message evidence
|
|
485
|
+
* is unavailable (capability honesty).
|
|
486
|
+
*/
|
|
487
|
+
get turnCount(): number | undefined;
|
|
488
|
+
/**
|
|
489
|
+
* Number of tool/subagent observations that reported an error, or `undefined`
|
|
490
|
+
* when error evidence is unavailable (capability honesty).
|
|
491
|
+
*/
|
|
492
|
+
get failedActions(): number | undefined;
|
|
493
|
+
/**
|
|
494
|
+
* Total execution duration in milliseconds, or `undefined` when timing
|
|
495
|
+
* evidence is unavailable (capability honesty). Uses trace-level
|
|
496
|
+
* startedAt/endedAt when present.
|
|
497
|
+
*/
|
|
498
|
+
get durationMs(): number | undefined;
|
|
499
|
+
/**
|
|
500
|
+
* Observations sorted deterministically by invocation time then span ID.
|
|
501
|
+
* Memoized per TraceView instance since the snapshot is immutable.
|
|
502
|
+
*/
|
|
503
|
+
private get sortedObservations();
|
|
504
|
+
private _sorted;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/**
|
|
508
|
+
* Value matchers for `t.check(value, matcher)`. Small, composable, and the
|
|
509
|
+
* single way to assert on deliverables/values (replaces the old `expect`).
|
|
510
|
+
*
|
|
511
|
+
* A matcher is just `{ label, test }` — easy to build custom ones inline with
|
|
512
|
+
* `satisfies(fn, label)`.
|
|
513
|
+
*/
|
|
514
|
+
type Matcher<T> = {
|
|
515
|
+
label: string;
|
|
516
|
+
test: (value: T) => boolean;
|
|
517
|
+
};
|
|
518
|
+
/** Coerces the value to a string and checks it contains a substring or RegExp. Gate. */
|
|
519
|
+
declare function includes(needle: string | RegExp): Matcher<unknown>;
|
|
520
|
+
/** Deep structural equality. Gate. */
|
|
521
|
+
declare function equals<T>(expected: T): Matcher<T>;
|
|
522
|
+
/**
|
|
523
|
+
* Validates against a Standard-Schema-shaped validator (Zod, Valibot, …) —
|
|
524
|
+
* anything exposing `safeParse`. Gate.
|
|
525
|
+
*/
|
|
526
|
+
declare function matches<T = unknown>(schema: {
|
|
527
|
+
safeParse: (value: unknown) => {
|
|
528
|
+
success: boolean;
|
|
529
|
+
};
|
|
530
|
+
}): Matcher<T>;
|
|
531
|
+
/** Custom boolean predicate. Gate. */
|
|
532
|
+
declare function satisfies<T>(predicate: (value: T) => boolean, label: string): Matcher<T>;
|
|
533
|
+
type ValueMatcher<T> = T | RegExp | ((value: T) => boolean);
|
|
534
|
+
declare function matchValue<T>(actual: T, matcher: ValueMatcher<T>): boolean;
|
|
535
|
+
/** Normalized Levenshtein similarity. Pass when score ≥ threshold (default 0.8). */
|
|
536
|
+
declare function similarity(expected: string, threshold?: number): Matcher<unknown>;
|
|
537
|
+
|
|
538
|
+
/**
|
|
539
|
+
* The assertion surface — `t`. Flat and eve-style: reads the trace (via
|
|
540
|
+
* {@link TraceView}) for behavioural checks and grades explicit values via
|
|
541
|
+
* `t.check`. Everything records (never throws), so a check reports all of its
|
|
542
|
+
* failures, not just the first.
|
|
543
|
+
*
|
|
544
|
+
* Every assertion records structured ``expected`` / ``received`` values so
|
|
545
|
+
* the dashboard can render testing-framework-style failures (Expected /
|
|
546
|
+
* Received) instead of only a prose message.
|
|
547
|
+
*/
|
|
548
|
+
|
|
549
|
+
/** A tool/agent name matcher: literal (exact), RegExp, or predicate. */
|
|
550
|
+
type NameMatcher = string | RegExp | ((name: string) => boolean);
|
|
551
|
+
/** Options for matching tool calls by count and recorded fields. */
|
|
552
|
+
type ToolCallOptions = {
|
|
553
|
+
count?: number;
|
|
554
|
+
input?: ValueMatcher<unknown>;
|
|
555
|
+
output?: ValueMatcher<unknown>;
|
|
556
|
+
status?: "ok" | "error";
|
|
557
|
+
};
|
|
558
|
+
/**
|
|
559
|
+
* LLM judge model config for `t.judge(...)` calls in checks.ts. Threaded
|
|
560
|
+
* through `runTask({ judge })` from the caller — the runner has no opinion
|
|
561
|
+
* about which model grades a deliverable.
|
|
562
|
+
*/
|
|
563
|
+
type JudgeConfig = {
|
|
564
|
+
/** Model id, e.g. ``"google/gemini-2.5-flash-lite"``. */
|
|
565
|
+
model: string;
|
|
566
|
+
/** Override the OpenRouter/OpenAI-compatible base URL. */
|
|
567
|
+
baseURL?: string;
|
|
568
|
+
/** Override the API key (defaults to env). */
|
|
569
|
+
apiKey?: string;
|
|
570
|
+
};
|
|
571
|
+
interface TestContext {
|
|
572
|
+
/** Asserts a matching tool was called. Pass `{ count }` for an exact count. */
|
|
573
|
+
calledTool(name: NameMatcher, opts?: ToolCallOptions): void;
|
|
574
|
+
/** Asserts no tool call matches the supplied name and field constraints. */
|
|
575
|
+
notCalledTool(name: NameMatcher, opts?: Omit<ToolCallOptions, "count">): void;
|
|
576
|
+
/** Asserts the named tools appear, in this order (as a subsequence). */
|
|
577
|
+
toolOrder(names: string[]): void;
|
|
578
|
+
/** Asserts no tool calls happened at all. */
|
|
579
|
+
usedNoTools(): void;
|
|
580
|
+
/** Asserts at most `n` tool calls — anti-flail. */
|
|
581
|
+
maxToolCalls(n: number): void;
|
|
582
|
+
/** Asserts no tool/subagent call reported an error — anti-flail. */
|
|
583
|
+
noFailedActions(): void;
|
|
584
|
+
/** Asserts a skill was loaded. */
|
|
585
|
+
loadedSkill(skill: string): void;
|
|
586
|
+
/** Asserts a subagent delegation happened. */
|
|
587
|
+
calledSubagent(agent: string): void;
|
|
588
|
+
/** Asserts the agent's reply contains a substring or RegExp. */
|
|
589
|
+
messageIncludes(token: string | RegExp): void;
|
|
590
|
+
/** Asserts the run took at most `n` turns — anti-flail. */
|
|
591
|
+
maxTurns(n: number): void;
|
|
592
|
+
/** Asserts the run took at most `n` milliseconds — anti-flail. */
|
|
593
|
+
maxDurationMs(n: number): void;
|
|
594
|
+
/** Escape hatch for a named predicate over the complete normalized run. */
|
|
595
|
+
assert(label: string, predicate: (flow: TraceView) => boolean): void;
|
|
596
|
+
/** Grades any value (a deliverable, parsed JSON, anything) with a matcher. */
|
|
597
|
+
check<T>(value: T, matcher: Matcher<T>, label?: string): void;
|
|
598
|
+
/**
|
|
599
|
+
* LLM-as-judge: asks the configured judge model to grade ``value`` against
|
|
600
|
+
* ``instruction`` (a natural-language rubric). Async — the check function
|
|
601
|
+
* must `await` it. Records a single assertion tagged
|
|
602
|
+
* ``evaluator_type: "llm"`` with judge metadata (model, prompt, response,
|
|
603
|
+
* tokens, latency) attached.
|
|
604
|
+
*
|
|
605
|
+
* ``opts.judge`` overrides the run's judge config **for this call only**,
|
|
606
|
+
* merging field-by-field onto the config threaded in via ``runTask({ judge })``
|
|
607
|
+
* — set ``{ model }`` to escalate a finicky criterion to a stronger model
|
|
608
|
+
* without switching the whole run. Absent fields inherit from the base config.
|
|
609
|
+
*/
|
|
610
|
+
judge(values: unknown | unknown[], instruction: string, opts?: {
|
|
611
|
+
label?: string;
|
|
612
|
+
judge?: Partial<JudgeConfig>;
|
|
613
|
+
}): Promise<void>;
|
|
614
|
+
}
|
|
615
|
+
/**
|
|
616
|
+
* Canonical names of every assertion method on {@link TestContext}.
|
|
617
|
+
*
|
|
618
|
+
* Single source of truth for "what is a ``t.<method>`` call?" — consumers that
|
|
619
|
+
* highlight, lint, or otherwise detect `t.*` usage in check source read this
|
|
620
|
+
* array instead of keeping their own copy (which historically drifted out of
|
|
621
|
+
* sync with the interface, leaving methods like ``t.maxTurns`` unstyled).
|
|
622
|
+
*
|
|
623
|
+
* The literal below is compile-time-checked against ``keyof TestContext``:
|
|
624
|
+
* adding a method to the interface without registering it here is a type
|
|
625
|
+
* error, and vice versa. See ``tests/test-context-methods.test.ts``.
|
|
626
|
+
*/
|
|
627
|
+
declare const TEST_METHOD_NAMES: readonly ["calledTool", "notCalledTool", "toolOrder", "usedNoTools", "maxToolCalls", "noFailedActions", "loadedSkill", "calledSubagent", "messageIncludes", "maxTurns", "maxDurationMs", "assert", "check", "judge"];
|
|
628
|
+
|
|
629
|
+
/**
|
|
630
|
+
* Check runner — builds a {@link TestContext} (`t`) per registered check from
|
|
631
|
+
* a {@link TraceProjectionSnapshot}, runs it, and aggregates every recorded
|
|
632
|
+
* assertion into one evaluation result per check.
|
|
633
|
+
*/
|
|
634
|
+
|
|
635
|
+
/**
|
|
636
|
+
* The check context — deliberately framework-agnostic. `deliverables` is the
|
|
637
|
+
* output; `files`/`task` are optional and untyped here so the core never
|
|
638
|
+
* depends on any agent-framework's types. apo passes its own typed values;
|
|
639
|
+
* other frameworks pass whatever they have (or omit).
|
|
640
|
+
*
|
|
641
|
+
* A task-scoped ``test`` gets typed deliverables from the adapter automatically:
|
|
642
|
+
*
|
|
643
|
+
* ```ts
|
|
644
|
+
* const { test } = task("review", {
|
|
645
|
+
* adapter: reviewAdapter,
|
|
646
|
+
* deliverables: ["result"],
|
|
647
|
+
* });
|
|
648
|
+
* test("id", (t, { deliverables }) => {
|
|
649
|
+
* deliverables.result // typed as ReviewResult
|
|
650
|
+
* });
|
|
651
|
+
* ```
|
|
652
|
+
*
|
|
653
|
+
* The generic overload of {@link defineCheck} remains available for
|
|
654
|
+
* framework-agnostic checks that have no task scope.
|
|
655
|
+
*/
|
|
656
|
+
type CheckContext<TDeliverables = Record<string, unknown>> = {
|
|
657
|
+
deliverables: TDeliverables;
|
|
658
|
+
files?: unknown;
|
|
659
|
+
task?: unknown;
|
|
660
|
+
};
|
|
661
|
+
declare function filePaths(files: unknown): string[];
|
|
662
|
+
type CheckFn<TDeliverables = Record<string, unknown>> = (t: TestContext, ctx: CheckContext<TDeliverables>) => Promise<void> | void;
|
|
663
|
+
type TestRegistration<TDeliverables = Record<string, unknown>> = (id: string, fn: CheckFn<TDeliverables>) => void;
|
|
664
|
+
/**
|
|
665
|
+
* A registered `describe()` group — single-level, organizational only. See
|
|
666
|
+
* {@link describe}.
|
|
667
|
+
*/
|
|
668
|
+
type CheckGroup = {
|
|
669
|
+
/** Stable, unique group id (kebab-case). Grouping key + React key. */
|
|
670
|
+
id: string;
|
|
671
|
+
/** Human-readable label shown in the dashboard. Defaults to the id. */
|
|
672
|
+
name: string;
|
|
673
|
+
};
|
|
674
|
+
/** Signature of the `describe` registration primitive. */
|
|
675
|
+
type DescribeRegistration = {
|
|
676
|
+
(id: string, fn: () => void): void;
|
|
677
|
+
(id: string, name: string, fn: () => void): void;
|
|
678
|
+
};
|
|
679
|
+
/** Register a check. The `fn` receives the assertion surface `t` and the output.
|
|
680
|
+
*
|
|
681
|
+
* Framework-agnostic callers can pass a deliverables type explicitly:
|
|
682
|
+
* ```ts
|
|
683
|
+
* type Deliverables = { result: ReviewResult; stats: Stats };
|
|
684
|
+
* defineCheck<Deliverables>("id", (t, { deliverables }) => { ... });
|
|
685
|
+
* ```
|
|
686
|
+
*/
|
|
687
|
+
declare function defineCheck<TDeliverables>(id: string, fn: CheckFn<TDeliverables>): void;
|
|
688
|
+
declare function defineCheck(id: string, fn: CheckFn): void;
|
|
689
|
+
declare function resetFlowChecks(): void;
|
|
690
|
+
/**
|
|
691
|
+
* Register a single-level group of checks. Runs `fn` synchronously;
|
|
692
|
+
* any `test()`/`defineCheck()` call inside it is stamped with the group's id
|
|
693
|
+
* and display name, so the dashboard can nest those checks under a collapsible
|
|
694
|
+
* header with a roll-up verdict.
|
|
695
|
+
*
|
|
696
|
+
* Groups are **organizational only** — they do not change execution order,
|
|
697
|
+
* concurrency, or the task verdict. Nesting is prohibited: calling `describe`
|
|
698
|
+
* inside a `describe` callback throws. Use sibling groups under one task
|
|
699
|
+
* instead.
|
|
700
|
+
*
|
|
701
|
+
* The `id` is stable (survives display-name edits); `name` is the human label
|
|
702
|
+
* shown in the dashboard and defaults to `id` when omitted.
|
|
703
|
+
*
|
|
704
|
+
* ```ts
|
|
705
|
+
* const { test, describe } = task("bind", { adapter, deliverables });
|
|
706
|
+
* describe("rules", "Rules — each comment becomes a rule", () => {
|
|
707
|
+
* RULE_GOLD.forEach((g) => test(`R-${g.id} — …`, async (t, { deliverables }) => { … }));
|
|
708
|
+
* });
|
|
709
|
+
* ```
|
|
710
|
+
*/
|
|
711
|
+
declare function describe(id: string, fn: () => void): void;
|
|
712
|
+
declare function describe(id: string, name: string, fn: () => void): void;
|
|
713
|
+
|
|
714
|
+
declare const TASK_ADAPTER_SYMBOL: unique symbol;
|
|
715
|
+
declare function resetTaskRegistry(): void;
|
|
716
|
+
type DefinedTask<TName extends string, TDeliverableDefs extends Record<string, DeliverableDefinition>> = TaskDefinition<TName, keyof TDeliverableDefs & string> & {
|
|
717
|
+
readonly [TASK_ADAPTER_SYMBOL]: TypedAdapterDefinition<TName, TDeliverableDefs>;
|
|
718
|
+
};
|
|
719
|
+
/** Legacy two-file task definition. New tasks should use {@link task}. */
|
|
720
|
+
declare function defineTask<const TName extends string, const TDeliverableDefs extends Record<string, DeliverableDefinition>>(adapter: TypedAdapterDefinition<TName, TDeliverableDefs>, config: TaskConfig<keyof TDeliverableDefs & string>): DefinedTask<TName, TDeliverableDefs>;
|
|
721
|
+
type TaskScope<TDeliverables> = {
|
|
722
|
+
test: TestRegistration<TDeliverables>;
|
|
723
|
+
/**
|
|
724
|
+
* Register a single-level group of checks. See
|
|
725
|
+
* {@link describe} in `checks/flow-runner.ts`. Returned from `task()` so the
|
|
726
|
+
* `test` inside the callback stays task-scoped (typed deliverables).
|
|
727
|
+
*/
|
|
728
|
+
describe: DescribeRegistration;
|
|
729
|
+
};
|
|
730
|
+
/**
|
|
731
|
+
* Register a task + its checks in ONE file. The `name` is the task id;
|
|
732
|
+
* `config` includes the adapter, deliverables, and other metadata. Checks
|
|
733
|
+
* are registered through the returned, adapter-typed `test` function.
|
|
734
|
+
*
|
|
735
|
+
* ```ts
|
|
736
|
+
* const { test } = task("code-review", {
|
|
737
|
+
* adapter: realAgentAdapter,
|
|
738
|
+
* deliverables: ["result", "tool_log", "stats"],
|
|
739
|
+
* maxTurns: 2,
|
|
740
|
+
* });
|
|
741
|
+
*
|
|
742
|
+
* test("reviewed-methodically", (t, { deliverables }) => {
|
|
743
|
+
* t.check(deliverables.result, includes("finding"));
|
|
744
|
+
* });
|
|
745
|
+
*
|
|
746
|
+
* // A task that needs dev-machine resources (cloud creds, VPC, stage) can
|
|
747
|
+
* // declare execution: "local" so `apo task run` runs it on the caller's
|
|
748
|
+
* // machine while still recording a backend run row.
|
|
749
|
+
* task("bind-e2e", {
|
|
750
|
+
* adapter: bindAdapter,
|
|
751
|
+
* deliverables: ["summary"],
|
|
752
|
+
* execution: "local",
|
|
753
|
+
* });
|
|
754
|
+
* ```
|
|
755
|
+
*/
|
|
756
|
+
declare function task<const TTaskId extends string, const TAdapterName extends string, const TDeliverableDefs extends Record<string, DeliverableDefinition>, TCollected extends CollectedDeliverables, const TSelected extends readonly (DeliverableKey<TCollected> & keyof TDeliverableDefs)[]>(name: TTaskId, config: Omit<TaskConfig<TSelected[number] & string>, "id" | "checks" | "deliverables"> & {
|
|
757
|
+
adapter: TypedAdapterDefinition<TAdapterName, TDeliverableDefs, TCollected>;
|
|
758
|
+
deliverables: TSelected;
|
|
759
|
+
}): TaskScope<SelectedDeliverables<TCollected, TSelected>>;
|
|
760
|
+
type DeliverableKey<TCollected> = TCollected extends unknown ? keyof TCollected & string : never;
|
|
761
|
+
type DeliverableValue<TCollected, TKey extends PropertyKey> = TCollected extends unknown ? TKey extends keyof TCollected ? TCollected[TKey] : never : never;
|
|
762
|
+
type SelectedDeliverables<TCollected, TSelected extends readonly PropertyKey[]> = {
|
|
763
|
+
[TKey in TSelected[number]]-?: Exclude<DeliverableValue<TCollected, TKey>, null | undefined>;
|
|
764
|
+
};
|
|
765
|
+
|
|
766
|
+
type LoadedTask = {
|
|
767
|
+
task: TaskDefinition;
|
|
768
|
+
adapter: AdapterDefinition;
|
|
769
|
+
taskDir: string;
|
|
770
|
+
files: FileEntry[];
|
|
771
|
+
/** Legacy external checks module, absent for single-file tasks. */
|
|
772
|
+
checksPath: string | null;
|
|
773
|
+
/** True when task(), test(), and turn() were registered by the eval file. */
|
|
774
|
+
inlineChecks: boolean;
|
|
775
|
+
/** The temp-module URL the task was imported from (used for location pinning). */
|
|
776
|
+
moduleUrl: string;
|
|
777
|
+
/** The original eval filename (e.g. "code-review.eval.ts") — used as source_file on check results. */
|
|
778
|
+
evalFileName: string;
|
|
779
|
+
};
|
|
780
|
+
declare function loadTask(taskDir: string): Promise<LoadedTask>;
|
|
781
|
+
|
|
782
|
+
declare function defineAdapter<const TName extends string, const TDeliverables extends Record<string, DeliverableDefinition>, TCollected extends CollectedDeliverables>(adapter: TypedAdapterDefinition<TName, TDeliverables, TCollected>): TypedAdapterDefinition<TName, TDeliverables, TCollected>;
|
|
783
|
+
|
|
784
|
+
/**
|
|
785
|
+
* file Artifact declaration helper.
|
|
786
|
+
*
|
|
787
|
+
* A ``FileArtifact`` is a path descriptor that identifies a regular local
|
|
788
|
+
* file at persistence time. The SDK streams it to the backend (two-phase
|
|
789
|
+
* upload) rather than buffering bytes in the result body.
|
|
790
|
+
*
|
|
791
|
+
* Rules:
|
|
792
|
+
* - ``path`` must identify a regular local file at persistence time;
|
|
793
|
+
* - directories, sockets, devices, and symbolic links are rejected;
|
|
794
|
+
* - the default display filename is ``basename(path)``;
|
|
795
|
+
* - the default media type is ``application/octet-stream``;
|
|
796
|
+
* - ``t.judge(fileArtifact(...))`` is rejected: a path descriptor is not file
|
|
797
|
+
* content. Authors must explicitly read/parse the content to judge it.
|
|
798
|
+
*/
|
|
799
|
+
interface FileArtifact {
|
|
800
|
+
readonly kind: "apo.file-artifact";
|
|
801
|
+
readonly path: string;
|
|
802
|
+
readonly mediaType: string;
|
|
803
|
+
readonly displayFilename: string;
|
|
804
|
+
}
|
|
805
|
+
interface FileArtifactOptions {
|
|
806
|
+
readonly mediaType?: string;
|
|
807
|
+
readonly displayFilename?: string;
|
|
808
|
+
}
|
|
809
|
+
/**
|
|
810
|
+
* Declare a file Deliverable from a local file path.
|
|
811
|
+
*
|
|
812
|
+
* Validates eagerly that ``path`` points to an existing regular file (not a
|
|
813
|
+
* directory, symlink, socket, or device) so authors get an actionable error
|
|
814
|
+
* before the backend creates an upload intent.
|
|
815
|
+
*/
|
|
816
|
+
declare function fileArtifact(path: string, options?: FileArtifactOptions): FileArtifact;
|
|
817
|
+
declare function isFileArtifact(value: unknown): value is FileArtifact;
|
|
818
|
+
|
|
819
|
+
/**
|
|
820
|
+
* file Artifact uploader.
|
|
821
|
+
*
|
|
822
|
+
* After checks run against the original in-memory Deliverables, this module
|
|
823
|
+
* partitions JSON values from file Artifacts, streams each file to the
|
|
824
|
+
* backend's two-phase upload endpoint, and returns only JSON Deliverables for
|
|
825
|
+
* the final result body. File bytes never travel through the result request
|
|
826
|
+
* or subprocess stdout.
|
|
827
|
+
*
|
|
828
|
+
* Storage failure does not flip a failed test into a pass: it raises, so the
|
|
829
|
+
* caller records the run as errored.
|
|
830
|
+
*/
|
|
831
|
+
interface DeliverableSummary {
|
|
832
|
+
id: string;
|
|
833
|
+
name: string;
|
|
834
|
+
kind: "json" | "artifact";
|
|
835
|
+
status: "pending" | "ready" | "failed";
|
|
836
|
+
media_type: string;
|
|
837
|
+
display_filename: string | null;
|
|
838
|
+
size_bytes: number;
|
|
839
|
+
sha256: string;
|
|
840
|
+
download_url: string | null;
|
|
841
|
+
}
|
|
842
|
+
interface ArtifactUploadConfig {
|
|
843
|
+
/** The Task Run id owning these Deliverables. */
|
|
844
|
+
taskRunId: string;
|
|
845
|
+
/** Bearer token (service token or API key) authorizing the upload. */
|
|
846
|
+
authToken: string;
|
|
847
|
+
/** Backend base URL, e.g. ``http://127.0.0.1:8000``. */
|
|
848
|
+
baseUrl: string;
|
|
849
|
+
/** Injectable fetch (tests pass a stub); defaults to global fetch. */
|
|
850
|
+
fetch?: typeof fetch;
|
|
851
|
+
}
|
|
852
|
+
interface PreparedTaskResult {
|
|
853
|
+
/** JSON-only deliverables safe to send in the final result body. */
|
|
854
|
+
jsonDeliverables: Record<string, unknown>;
|
|
855
|
+
/** Successfully uploaded file Artifacts (summaries only, no bodies). */
|
|
856
|
+
artifactUploads: DeliverableSummary[];
|
|
857
|
+
}
|
|
858
|
+
/**
|
|
859
|
+
* Partition and persist Deliverables after checks complete.
|
|
860
|
+
*
|
|
861
|
+
* - file Artifacts are streamed to the backend (hash, create intent, PUT);
|
|
862
|
+
* - JSON values are returned untouched for the result body;
|
|
863
|
+
* - the local path of any Artifact is never embedded in the result.
|
|
864
|
+
*/
|
|
865
|
+
declare function persistFileArtifacts(deliverables: Record<string, unknown>, config: ArtifactUploadConfig): Promise<PreparedTaskResult>;
|
|
866
|
+
|
|
867
|
+
/**
|
|
868
|
+
* Flow — the canonical, source-agnostic recording of "what happened" during
|
|
869
|
+
* an agent run. This is the single substrate the testing framework reads.
|
|
870
|
+
*
|
|
871
|
+
* It is deliberately neutral: it does not know about our trace, the AI SDK,
|
|
872
|
+
* OpenAI, Anthropic, etc. Concrete sources are converted into a {@link Flow}
|
|
873
|
+
* by a normalizer (see `fromApoTrace`, etc.). The framework depends only on
|
|
874
|
+
* this interface — that is what hides the source complexity.
|
|
875
|
+
*/
|
|
876
|
+
type ToolCallStatus = "ok" | "error";
|
|
877
|
+
type FlowEvent = {
|
|
878
|
+
kind: "message";
|
|
879
|
+
role: "user" | "assistant" | "system";
|
|
880
|
+
text: string;
|
|
881
|
+
ts?: number;
|
|
882
|
+
} | {
|
|
883
|
+
kind: "tool_call";
|
|
884
|
+
name: string;
|
|
885
|
+
input?: unknown;
|
|
886
|
+
output?: unknown;
|
|
887
|
+
status?: ToolCallStatus;
|
|
888
|
+
latencyMs?: number;
|
|
889
|
+
ts?: number;
|
|
890
|
+
} | {
|
|
891
|
+
kind: "skill_load";
|
|
892
|
+
skill: string;
|
|
893
|
+
ts?: number;
|
|
894
|
+
} | {
|
|
895
|
+
kind: "subagent_call";
|
|
896
|
+
agent: string;
|
|
897
|
+
output?: unknown;
|
|
898
|
+
status?: ToolCallStatus;
|
|
899
|
+
latencyMs?: number;
|
|
900
|
+
ts?: number;
|
|
901
|
+
};
|
|
902
|
+
type Flow = {
|
|
903
|
+
/** Ordered events — the timeline of the run. */
|
|
904
|
+
events: readonly FlowEvent[];
|
|
905
|
+
/** Wall-clock start/end, when known. Sources without clocks may omit them. */
|
|
906
|
+
startedAt?: number;
|
|
907
|
+
endedAt?: number;
|
|
908
|
+
};
|
|
909
|
+
|
|
910
|
+
/**
|
|
911
|
+
* FlowView — typed, derived access over a {@link Flow}. This is the read model
|
|
912
|
+
* the assertion surface (`t`) queries. It hides all the filtering/derivation
|
|
913
|
+
* so assertions stay one-liners.
|
|
914
|
+
*/
|
|
915
|
+
|
|
916
|
+
type ToolCall = Extract<FlowEvent, {
|
|
917
|
+
kind: "tool_call";
|
|
918
|
+
}>;
|
|
919
|
+
type SkillLoad = Extract<FlowEvent, {
|
|
920
|
+
kind: "skill_load";
|
|
921
|
+
}>;
|
|
922
|
+
type SubagentCall = Extract<FlowEvent, {
|
|
923
|
+
kind: "subagent_call";
|
|
924
|
+
}>;
|
|
925
|
+
type Message = Extract<FlowEvent, {
|
|
926
|
+
kind: "message";
|
|
927
|
+
}>;
|
|
928
|
+
declare class FlowView {
|
|
929
|
+
readonly events: readonly FlowEvent[];
|
|
930
|
+
readonly startedAt?: number;
|
|
931
|
+
readonly endedAt?: number;
|
|
932
|
+
constructor(flow: Flow);
|
|
933
|
+
get messages(): Message[];
|
|
934
|
+
get toolCalls(): ToolCall[];
|
|
935
|
+
get skillLoads(): SkillLoad[];
|
|
936
|
+
get subagentCalls(): SubagentCall[];
|
|
937
|
+
get toolNamesInOrder(): string[];
|
|
938
|
+
/** Last assistant message text — the agent's "reply". */
|
|
939
|
+
get reply(): string;
|
|
940
|
+
/** Number of completed assistant turns. */
|
|
941
|
+
get turnCount(): number;
|
|
942
|
+
/** Tool or subagent calls that reported an error. */
|
|
943
|
+
get failedActions(): number;
|
|
944
|
+
get durationMs(): number;
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
/**
|
|
948
|
+
* Cross-framework Flow normalizers — the "plugs" that let agents NOT built on
|
|
949
|
+
* apo's adapter produce a {@link Flow} from their own output, so the same
|
|
950
|
+
* `t.*` checks work for everyone.
|
|
951
|
+
*
|
|
952
|
+
* Each normalizer is `(source) => Flow`. Add new ones by following the shape.
|
|
953
|
+
*
|
|
954
|
+
* @deprecated These are **compatibility recording adapters**,
|
|
955
|
+
* not the primary testing architecture. The canonical path is now the Trace
|
|
956
|
+
* Projection snapshot (`TraceProjectionSnapshot` → `TraceView` →
|
|
957
|
+
* `runTraceChecks`). These normalizers remain available during the
|
|
958
|
+
* deprecation period; `snapshotFromFlow` bridges their output into the
|
|
959
|
+
* projection contract. New integrations should emit standard OTel spans
|
|
960
|
+
* instead of producing a Flow.
|
|
961
|
+
*/
|
|
962
|
+
|
|
963
|
+
interface OpenAIMessage {
|
|
964
|
+
role?: string;
|
|
965
|
+
content?: string | null;
|
|
966
|
+
tool_calls?: Array<{
|
|
967
|
+
id?: string;
|
|
968
|
+
function?: {
|
|
969
|
+
name?: string;
|
|
970
|
+
arguments?: string;
|
|
971
|
+
};
|
|
972
|
+
}>;
|
|
973
|
+
tool_call_id?: string;
|
|
974
|
+
}
|
|
975
|
+
declare function fromOpenAIMessages(messages: OpenAIMessage[]): Flow;
|
|
976
|
+
interface AnthropicMessage {
|
|
977
|
+
role?: string;
|
|
978
|
+
content?: Array<{
|
|
979
|
+
type: "text";
|
|
980
|
+
text: string;
|
|
981
|
+
} | {
|
|
982
|
+
type: "tool_use";
|
|
983
|
+
id: string;
|
|
984
|
+
name: string;
|
|
985
|
+
input?: unknown;
|
|
986
|
+
} | {
|
|
987
|
+
type: "tool_result";
|
|
988
|
+
tool_use_id: string;
|
|
989
|
+
content?: unknown;
|
|
990
|
+
}>;
|
|
991
|
+
}
|
|
992
|
+
declare function fromAnthropicMessages(messages: AnthropicMessage[]): Flow;
|
|
993
|
+
interface AISDKResult {
|
|
994
|
+
steps?: Array<{
|
|
995
|
+
text?: string;
|
|
996
|
+
toolCalls?: Array<{
|
|
997
|
+
toolName: string;
|
|
998
|
+
input?: unknown;
|
|
999
|
+
}>;
|
|
1000
|
+
toolResults?: Array<{
|
|
1001
|
+
toolName: string;
|
|
1002
|
+
output?: unknown;
|
|
1003
|
+
}>;
|
|
1004
|
+
}>;
|
|
1005
|
+
}
|
|
1006
|
+
declare function fromAISDK(result: AISDKResult): Flow;
|
|
1007
|
+
|
|
1008
|
+
/**
|
|
1009
|
+
* Error thrown when a Task Run fails after the adapter's Run Configuration
|
|
1010
|
+
* has been resolved. Carries the resolved {@link AgentTaskRunConfiguration} so
|
|
1011
|
+
* callers (e.g. the CLI error-report path) can forward model/effort to the
|
|
1012
|
+
* backend even when the run never produced a summary (issue #40).
|
|
1013
|
+
*
|
|
1014
|
+
* The original failure is preserved on `cause`. Only errors thrown after
|
|
1015
|
+
* `startSession` reported a *valid* configuration are wrapped — a
|
|
1016
|
+
* configuration-validation failure propagates unwrapped (there is nothing to
|
|
1017
|
+
* attach).
|
|
1018
|
+
*/
|
|
1019
|
+
declare class AgentTaskRunError extends Error {
|
|
1020
|
+
readonly runConfiguration?: AgentTaskRunConfiguration;
|
|
1021
|
+
/** The original error that caused the run to fail (preserved for diagnostics). */
|
|
1022
|
+
readonly cause?: unknown;
|
|
1023
|
+
constructor(message: string, options: {
|
|
1024
|
+
runConfiguration?: AgentTaskRunConfiguration;
|
|
1025
|
+
cause?: unknown;
|
|
1026
|
+
});
|
|
1027
|
+
}
|
|
1028
|
+
type RunTaskOptions = {
|
|
1029
|
+
maxTurnsOverride?: number;
|
|
1030
|
+
tracing?: AgentTaskTraceOptions;
|
|
1031
|
+
/** LLM judge model config for `t.judge(...)` calls in the task checks. */
|
|
1032
|
+
judge?: JudgeConfig;
|
|
1033
|
+
onTurn?: (turnNumber: number, userAction: TaskTranscriptTurn["userAction"], agentResponse: unknown) => void;
|
|
1034
|
+
/**
|
|
1035
|
+
* Skip the internal `loadTask(taskDir)` call and reuse an already-loaded
|
|
1036
|
+
* task definition.
|
|
1037
|
+
*
|
|
1038
|
+
* The eval module's top level runs once per `loadTask` (it is copied to a
|
|
1039
|
+
* temp file and imported with all registries reset), so loading twice per
|
|
1040
|
+
* run breaks evals whose load-time behavior is not idempotent across module
|
|
1041
|
+
* systems (e.g. a helper that re-exports CJS `require()`-cached registrations
|
|
1042
|
+
* — the second import hits the cache and registers zero checks). Callers
|
|
1043
|
+
* that already need a `LoadedTask` for their own purposes (`runTaskDir`,
|
|
1044
|
+
* `runner-entry.ts`) should always pass it through here to keep the eval
|
|
1045
|
+
* import count at exactly one per run.
|
|
1046
|
+
*
|
|
1047
|
+
* When omitted, `runTask` calls `loadTask(taskDir)` itself. The `taskDir`
|
|
1048
|
+
* argument is still required for that fallback and is otherwise unused.
|
|
1049
|
+
*/
|
|
1050
|
+
loaded?: LoadedTask;
|
|
1051
|
+
};
|
|
1052
|
+
declare function runTask(taskDir: string, options?: RunTaskOptions): Promise<TaskRunResult>;
|
|
1053
|
+
|
|
1054
|
+
type AgentTaskCliOptions = {
|
|
1055
|
+
task?: string;
|
|
1056
|
+
dir?: string;
|
|
1057
|
+
grep?: string;
|
|
1058
|
+
help?: boolean;
|
|
1059
|
+
};
|
|
1060
|
+
declare function parseAgentTaskCliArgs(argv: string[]): AgentTaskCliOptions;
|
|
1061
|
+
declare function runAgentTaskCli(argv: string[], cwd?: string): Promise<number>;
|
|
1062
|
+
|
|
1063
|
+
declare function discoverAgentTaskDirs(rootDir: string): string[];
|
|
1064
|
+
|
|
1065
|
+
type AgentTaskRuntime = {
|
|
1066
|
+
judge?: JudgeConfig;
|
|
1067
|
+
};
|
|
1068
|
+
type AgentTaskRunSummary = {
|
|
1069
|
+
taskDir: string;
|
|
1070
|
+
taskId: string;
|
|
1071
|
+
pass: boolean;
|
|
1072
|
+
checks: EvaluationItemResult[];
|
|
1073
|
+
/** Adapter that ran the task. Forwarded to the backend when recording locally. */
|
|
1074
|
+
adapterName?: string;
|
|
1075
|
+
/** Trace id this run claimed (when tracing was enabled). */
|
|
1076
|
+
traceRunId?: string;
|
|
1077
|
+
/** Deliverables the adapter produced. */
|
|
1078
|
+
deliverables?: Record<string, unknown>;
|
|
1079
|
+
/** Per-turn transcript of the run. */
|
|
1080
|
+
transcript?: Record<string, unknown>;
|
|
1081
|
+
/** adapter-reported model/effort, forwarded when recording locally. */
|
|
1082
|
+
runConfiguration?: AgentTaskRunConfiguration;
|
|
1083
|
+
};
|
|
1084
|
+
declare function loadTaskRuntime(_taskDir: string): Promise<AgentTaskRuntime>;
|
|
1085
|
+
declare function runTaskDir(taskDir: string): Promise<AgentTaskRunSummary>;
|
|
1086
|
+
|
|
1087
|
+
type DeliverableValidationResult = {
|
|
1088
|
+
results: EvaluationItemResult[];
|
|
1089
|
+
brokenDeliverables: Record<string, string>;
|
|
1090
|
+
};
|
|
1091
|
+
|
|
1092
|
+
/**
|
|
1093
|
+
* ApoSpanProcessor — an OpenTelemetry SpanProcessor that routes GenAI spans
|
|
1094
|
+
* into apo's Flow.
|
|
1095
|
+
*
|
|
1096
|
+
* This is the OTel-native tracing path. After calling {@link registerApoTracing},
|
|
1097
|
+
* any SDK that emits OTel spans (Vercel AI SDK, OpenAI Agents SDK, etc.)
|
|
1098
|
+
* will be traced automatically — no per-SDK wrapper needed.
|
|
1099
|
+
*
|
|
1100
|
+
* The processor is global, but run isolation is maintained via
|
|
1101
|
+
* {@link AsyncLocalStorage}: {@link getActiveApoRun} returns the current
|
|
1102
|
+
* run's trace context, so each span is routed to the correct run's FlowTee.
|
|
1103
|
+
*
|
|
1104
|
+
* **Design: eager creation, deferred completion.** The apo span is created
|
|
1105
|
+
* at `onStart` (when initial attributes like tool name and model are
|
|
1106
|
+
* available). The span is ended at `onEnd` with the final attributes
|
|
1107
|
+
* (results, tokens, text). This gives correct ordering: parent spans
|
|
1108
|
+
* are created before children, matching the real call hierarchy.
|
|
1109
|
+
*
|
|
1110
|
+
* @module
|
|
1111
|
+
*/
|
|
1112
|
+
interface OTelSpanContext {
|
|
1113
|
+
spanId: string;
|
|
1114
|
+
traceId: string;
|
|
1115
|
+
traceFlags: number;
|
|
1116
|
+
}
|
|
1117
|
+
interface OTelSpanStatus {
|
|
1118
|
+
code: number;
|
|
1119
|
+
message?: string;
|
|
1120
|
+
}
|
|
1121
|
+
interface OTelReadableSpan {
|
|
1122
|
+
name: string;
|
|
1123
|
+
spanContext(): OTelSpanContext;
|
|
1124
|
+
parentSpanContext?: OTelSpanContext;
|
|
1125
|
+
attributes: Record<string, unknown>;
|
|
1126
|
+
status: OTelSpanStatus;
|
|
1127
|
+
duration: [number, number];
|
|
1128
|
+
startTime: [number, number];
|
|
1129
|
+
ended: boolean;
|
|
1130
|
+
}
|
|
1131
|
+
interface OTelSpan extends OTelReadableSpan {
|
|
1132
|
+
setAttribute(key: string, value: unknown): this;
|
|
1133
|
+
setAttributes(attributes: Record<string, unknown>): this;
|
|
1134
|
+
setStatus(status: OTelSpanStatus): this;
|
|
1135
|
+
recordException(exception: unknown): void;
|
|
1136
|
+
updateName(name: string): this;
|
|
1137
|
+
end(endTime?: unknown): void;
|
|
1138
|
+
isRecording(): boolean;
|
|
1139
|
+
}
|
|
1140
|
+
interface OTelContext {
|
|
1141
|
+
getValue(key: symbol): unknown;
|
|
1142
|
+
setValue(key: symbol, value: unknown): OTelContext;
|
|
1143
|
+
deleteValue(key: symbol): OTelContext;
|
|
1144
|
+
}
|
|
1145
|
+
interface OTelSpanProcessorLifecycle {
|
|
1146
|
+
forceFlush(): Promise<void>;
|
|
1147
|
+
shutdown(): Promise<void>;
|
|
1148
|
+
}
|
|
1149
|
+
/**
|
|
1150
|
+
* A SpanProcessor that translates GenAI OTel spans into apo observations.
|
|
1151
|
+
*
|
|
1152
|
+
* Spans are created at `onStart` (eager) and completed at `onEnd`
|
|
1153
|
+
* (deferred). This gives correct parent/child ordering — parent spans
|
|
1154
|
+
* are created before their children, matching the real call hierarchy.
|
|
1155
|
+
*
|
|
1156
|
+
* If a span isn't load-bearing (not a GenAI tool/generation span), no
|
|
1157
|
+
* apo span is created.
|
|
1158
|
+
*/
|
|
1159
|
+
declare class ApoSpanProcessor implements OTelSpanProcessorLifecycle {
|
|
1160
|
+
private spanMap;
|
|
1161
|
+
onStart(span: OTelSpan, _parentContext: OTelContext): void;
|
|
1162
|
+
onEnd(span: OTelReadableSpan): void;
|
|
1163
|
+
forceFlush(): Promise<void>;
|
|
1164
|
+
shutdown(): Promise<void>;
|
|
1165
|
+
/** Clear all state — for testing. */
|
|
1166
|
+
reset(): void;
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1169
|
+
/**
|
|
1170
|
+
* Registration API for the OTel-native tracing integration.
|
|
1171
|
+
*
|
|
1172
|
+
* Call {@link registerApoTracing} once at startup (e.g., in `startSession`
|
|
1173
|
+
* or at module load). After this, any SDK that emits OTel spans during a run
|
|
1174
|
+
* will be traced automatically — no per-SDK wrapper needed.
|
|
1175
|
+
*
|
|
1176
|
+
* @example
|
|
1177
|
+
* ```ts
|
|
1178
|
+
* import { registerApoTracing } from "@apo-ai/sdk/agent-task";
|
|
1179
|
+
*
|
|
1180
|
+
* async startSession(ctx) {
|
|
1181
|
+
* registerApoTracing(); // one line, once
|
|
1182
|
+
* return { async sendUserTurn(turn) { ... } };
|
|
1183
|
+
* }
|
|
1184
|
+
* ```
|
|
1185
|
+
*
|
|
1186
|
+
* @module
|
|
1187
|
+
*/
|
|
1188
|
+
|
|
1189
|
+
interface RegisterApoTracingOptions {
|
|
1190
|
+
/**
|
|
1191
|
+
* Set as the global tracer provider (default: true).
|
|
1192
|
+
* Set to false if the host app has its own OTel setup and you want to
|
|
1193
|
+
* add apo's processor to an existing provider instead.
|
|
1194
|
+
*/
|
|
1195
|
+
setGlobal?: boolean;
|
|
1196
|
+
}
|
|
1197
|
+
/**
|
|
1198
|
+
* Register apo's OTel SpanProcessor. Idempotent — calling it more than
|
|
1199
|
+
* once is a no-op.
|
|
1200
|
+
*
|
|
1201
|
+
* After registration, any OTel span created during an apo run (inside
|
|
1202
|
+
* `withApoRun`) that matches the GenAI conventions (`ai.*` or `gen_ai.*`)
|
|
1203
|
+
* is automatically translated and fed into the run's Flow.
|
|
1204
|
+
*/
|
|
1205
|
+
declare function registerApoTracing(_options?: RegisterApoTracingOptions): Promise<void>;
|
|
1206
|
+
/**
|
|
1207
|
+
* Reset registration state — for testing only.
|
|
1208
|
+
*/
|
|
1209
|
+
declare function resetApoTracing(): void;
|
|
1210
|
+
|
|
1211
|
+
/**
|
|
1212
|
+
* Per-run context propagation via Node's AsyncLocalStorage.
|
|
1213
|
+
*
|
|
1214
|
+
* The OTel {@link ApoSpanProcessor} is global — it doesn't know which run a
|
|
1215
|
+
* span belongs to. This module solves that: {@link withApoRun} establishes
|
|
1216
|
+
* the run's trace context on AsyncLocalStorage, and any code called during
|
|
1217
|
+
* the run (including the processor's `onStart`/`onEnd`) can read it via
|
|
1218
|
+
* {@link getActiveApoRun}.
|
|
1219
|
+
*
|
|
1220
|
+
* This is the same propagation mechanism OTel's own
|
|
1221
|
+
* `AsyncLocalStorageContextManager` uses under the hood.
|
|
1222
|
+
*
|
|
1223
|
+
* @module
|
|
1224
|
+
*/
|
|
1225
|
+
|
|
1226
|
+
interface ApoRunContext {
|
|
1227
|
+
/** The tee'd trace context for this run. */
|
|
1228
|
+
trace: AgentTaskTraceContext;
|
|
1229
|
+
/** The parent span id (the run's root span or the current turn span). */
|
|
1230
|
+
parentSpanId?: string;
|
|
1231
|
+
/** The task id, surfaced in span metadata. */
|
|
1232
|
+
taskId?: string;
|
|
1233
|
+
/** The current turn number, surfaced in span metadata. */
|
|
1234
|
+
turnNumber?: number;
|
|
1235
|
+
}
|
|
1236
|
+
/**
|
|
1237
|
+
* Run `fn` with `ctx` as the active apo run context. Any code inside `fn`
|
|
1238
|
+
* (including across `await` boundaries) can call {@link getActiveApoRun}
|
|
1239
|
+
* to retrieve it.
|
|
1240
|
+
*/
|
|
1241
|
+
declare function withApoRun<T>(ctx: ApoRunContext, fn: () => Promise<T>): Promise<T>;
|
|
1242
|
+
/**
|
|
1243
|
+
* Run `fn` synchronously with `ctx` as the active apo run context.
|
|
1244
|
+
*/
|
|
1245
|
+
declare function withApoRunSync<T>(ctx: ApoRunContext, fn: () => T): T;
|
|
1246
|
+
/**
|
|
1247
|
+
* Get the active run context, or `undefined` if called outside a run.
|
|
1248
|
+
*
|
|
1249
|
+
* This is what the {@link ApoSpanProcessor} calls in `onStart`/`onEnd` to
|
|
1250
|
+
* know which run a span belongs to.
|
|
1251
|
+
*/
|
|
1252
|
+
declare function getActiveApoRun(): ApoRunContext | undefined;
|
|
1253
|
+
|
|
1254
|
+
/**
|
|
1255
|
+
* canonical Task Revision manifest.
|
|
1256
|
+
*
|
|
1257
|
+
* Pure, dependency-free canonicalizer. Both this module and its Python twin
|
|
1258
|
+
* (`backend/apo/execution/task_revision_manifest.py`) MUST produce
|
|
1259
|
+
* byte-identical canonical JSON and digests for every fixture under
|
|
1260
|
+
* `specs/contracts/task-revision/v1/cases/`.
|
|
1261
|
+
*
|
|
1262
|
+
* Canonicalization rules:
|
|
1263
|
+
* - `/` path separators (caller `\` normalized to `/`);
|
|
1264
|
+
* - Unicode NFC normalization of the path;
|
|
1265
|
+
* - bytewise lexical ordering of normalized UTF-8 paths (NOT UTF-16 —
|
|
1266
|
+
* diverges for codepoints > U+007F);
|
|
1267
|
+
* - exact file-byte SHA-256 (lowercase hex);
|
|
1268
|
+
* - mode reduced to `regular` | `executable`;
|
|
1269
|
+
* - ownership, timestamps, inode metadata, and absolute paths excluded.
|
|
1270
|
+
*
|
|
1271
|
+
* `contentSha256` is the SHA-256 of the compact canonical JSON built from
|
|
1272
|
+
* `schemaVersion` and the sorted `files` array. The `summary` block is NOT
|
|
1273
|
+
* part of source identity.
|
|
1274
|
+
*/
|
|
1275
|
+
type ModeClass = "regular" | "executable";
|
|
1276
|
+
interface ManifestFile {
|
|
1277
|
+
path: string;
|
|
1278
|
+
sizeBytes: number;
|
|
1279
|
+
sha256: string;
|
|
1280
|
+
modeClass: ModeClass;
|
|
1281
|
+
}
|
|
1282
|
+
interface TaskRevisionManifestV1 {
|
|
1283
|
+
schemaVersion: 1;
|
|
1284
|
+
files: ManifestFile[];
|
|
1285
|
+
summary: {
|
|
1286
|
+
fileCount: number;
|
|
1287
|
+
uncompressedSizeBytes: number;
|
|
1288
|
+
excludedFileCount: number;
|
|
1289
|
+
excludedDirectoryCount: number;
|
|
1290
|
+
};
|
|
1291
|
+
}
|
|
1292
|
+
/** Caller-reported file, before canonicalization. `content` is the exact bytes. */
|
|
1293
|
+
interface ManifestFileInput {
|
|
1294
|
+
path: string;
|
|
1295
|
+
modeClass: ModeClass;
|
|
1296
|
+
content: Uint8Array;
|
|
1297
|
+
}
|
|
1298
|
+
/**
|
|
1299
|
+
* Normalize a caller-reported path to canonical POSIX + NFC form.
|
|
1300
|
+
* Validation (no leading slash, no `.`/`..` segments) is the filesystem
|
|
1301
|
+
* walker's responsibility; this function only makes representation canonical.
|
|
1302
|
+
*/
|
|
1303
|
+
declare function normalizeManifestPath(path: string): string;
|
|
1304
|
+
/** SHA-256 of exact bytes, lowercase hex. */
|
|
1305
|
+
declare function sha256Hex(bytes: Uint8Array): string;
|
|
1306
|
+
/**
|
|
1307
|
+
* Build the canonical V1 manifest from already-included files. Exclusion
|
|
1308
|
+
* counts default to zero; the filesystem walker supplies real counts when it
|
|
1309
|
+
* wraps this via `withExclusionCounts`.
|
|
1310
|
+
*/
|
|
1311
|
+
declare function buildManifest(files: ManifestFileInput[], summary?: {
|
|
1312
|
+
excludedFileCount: number;
|
|
1313
|
+
excludedDirectoryCount: number;
|
|
1314
|
+
}): TaskRevisionManifestV1;
|
|
1315
|
+
/**
|
|
1316
|
+
* Compact canonical JSON over `schemaVersion` + the sorted `files` array only.
|
|
1317
|
+
* Object keys are alphabetical at every level; no whitespace; non-ASCII
|
|
1318
|
+
* emitted as UTF-8 (matches Python `json.dumps(sort_keys=True,
|
|
1319
|
+
* ensure_ascii=False, separators=(",", ":"))`). The `summary` block is
|
|
1320
|
+
* intentionally excluded from source identity.
|
|
1321
|
+
*/
|
|
1322
|
+
declare function canonicalManifestJson(manifest: TaskRevisionManifestV1): string;
|
|
1323
|
+
/** SHA-256 (lowercase hex) of the canonical manifest JSON's UTF-8 bytes. */
|
|
1324
|
+
declare function contentSha256(manifest: TaskRevisionManifestV1): string;
|
|
1325
|
+
|
|
1326
|
+
export { type AISDKResult, type AdapterDefinition, type AdapterRuntimeState, type AdapterSession, type AgentTaskCliOptions, type AgentTaskRunConfiguration, AgentTaskRunError, type AgentTaskRunSummary, type AgentTaskRuntime, AgentTaskTraceContext, AgentTaskTraceOptions, type AgentTurnResult, type AnthropicMessage, type ApoRunContext, ApoSpanProcessor, type ArtifactUploadConfig, type CheckContext, type CheckGroup, type CleanupContext, type CollectDeliverablesContext, type CollectedDeliverables, type DeliverableDefinition, type DeliverableSummary, type DeliverableValidationResult, type DescribeRegistration, type FileArtifact, type FileArtifactOptions, type FileEntry, type Flow, type FlowEvent, FlowView, type InitializeContext, type JudgeConfig, type LoadedTask, type ManifestFile, type ManifestFileInput, type Matcher, type ModeClass, type NameMatcher, type OpenAIMessage, type PreparedTaskResult, type RegisterApoTracingOptions, type RunTaskOptions, type StartSessionContext, TEST_METHOD_NAMES, type TaskConfig, type TaskDefinition, type TaskEvaluationResult, TaskFiles, type TaskRevisionManifestV1, type TaskRunResult, type TaskScope, type TaskTranscript, type TaskTranscriptTurn, type TestContext, type TestRegistration, type ToolCallOptions, type ToolCallStatus, type TurnContext, type TurnFn, type TurnRecord, type TypedAdapterDefinition, type ValueMatcher, buildManifest, canonicalManifestJson, contentSha256, defineAdapter, defineTask, describe, discoverAgentTaskDirs, equals, fileArtifact, filePaths, fromAISDK, fromAnthropicMessages, fromOpenAIMessages, getActiveApoRun, getTaskTurn, includes, isFileArtifact, loadTask, loadTaskRuntime, matchValue, matches, normalizeManifestPath, parseAgentTaskCliArgs, persistFileArtifacts, registerApoTracing, resetApoTracing, resetFlowChecks, resetTaskRegistry, resetTaskTurn, resolveTurn, runAgentTaskCli, runTask, runTaskDir, satisfies, sha256Hex, similarity, task, defineCheck as test, turn, withApoRun, withApoRunSync };
|