@apo-ai/sdk 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +314 -0
  3. package/dist/agent-task/integrations/ai-sdk.d.ts +121 -0
  4. package/dist/agent-task/integrations/ai-sdk.js +10 -0
  5. package/dist/agent-task/integrations/ai-sdk.js.map +1 -0
  6. package/dist/agent-task/integrations/anthropic.d.ts +67 -0
  7. package/dist/agent-task/integrations/anthropic.js +10 -0
  8. package/dist/agent-task/integrations/anthropic.js.map +1 -0
  9. package/dist/agent-task/integrations/openai.d.ts +77 -0
  10. package/dist/agent-task/integrations/openai.js +10 -0
  11. package/dist/agent-task/integrations/openai.js.map +1 -0
  12. package/dist/agent-task/public.d.ts +1326 -0
  13. package/dist/agent-task/public.js +3026 -0
  14. package/dist/agent-task/public.js.map +1 -0
  15. package/dist/chunk-DT2LMB7O.js +80 -0
  16. package/dist/chunk-DT2LMB7O.js.map +1 -0
  17. package/dist/chunk-GDB6SDGN.js +75 -0
  18. package/dist/chunk-GDB6SDGN.js.map +1 -0
  19. package/dist/chunk-HKLBPIB4.js +232 -0
  20. package/dist/chunk-HKLBPIB4.js.map +1 -0
  21. package/dist/chunk-HWZVL6TS.js +37 -0
  22. package/dist/chunk-HWZVL6TS.js.map +1 -0
  23. package/dist/chunk-PMYLL2H4.js +495 -0
  24. package/dist/chunk-PMYLL2H4.js.map +1 -0
  25. package/dist/chunk-SZDSIPXC.js +74 -0
  26. package/dist/chunk-SZDSIPXC.js.map +1 -0
  27. package/dist/index.d.ts +95 -0
  28. package/dist/index.js +39 -0
  29. package/dist/index.js.map +1 -0
  30. package/dist/otel/index.d.ts +270 -0
  31. package/dist/otel/index.js +32 -0
  32. package/dist/otel/index.js.map +1 -0
  33. package/dist/tracing-DtbhYQV6.d.ts +26 -0
  34. package/dist/types-C-nrDEVf.d.ts +140 -0
  35. package/package.json +92 -0
@@ -0,0 +1,1326 @@
1
+ import { A as AgentTaskTraceContext, a as AgentTaskTraceOptions } from '../tracing-DtbhYQV6.js';
2
+ export { CreateApoTracerOptions, createApoTracer } from './integrations/ai-sdk.js';
3
+ export { CreateApoOpenAIOptions, createApoOpenAI } from './integrations/openai.js';
4
+ export { CreateApoAnthropicOptions, createApoAnthropic } from './integrations/anthropic.js';
5
+ import '../types-C-nrDEVf.js';
6
+
7
+ type TaskDefinition<TAdapterName extends string = string, TDeliverable extends string = string> = {
8
+ id: string;
9
+ adapter: TAdapterName;
10
+ description?: string;
11
+ deliverables: TDeliverable[];
12
+ maxTurns?: number;
13
+ metadata?: Record<string, unknown>;
14
+ checks?: string | false;
15
+ };
16
+ type TaskConfig<TDeliverable extends string = string> = Omit<TaskDefinition<string, TDeliverable>, "adapter">;
17
+ type FileEntry = {
18
+ relativePath: string;
19
+ absolutePath: string;
20
+ };
21
+
22
+ declare class TaskFiles {
23
+ private entries;
24
+ constructor(entries: FileEntry[]);
25
+ read(relativePath: string): Promise<string>;
26
+ readBuffer(relativePath: string): Promise<Buffer>;
27
+ find(relativePath: string): FileEntry | undefined;
28
+ list(): FileEntry[];
29
+ has(relativePath: string): boolean;
30
+ }
31
+
32
+ type TurnRecord = {
33
+ turnNumber: number;
34
+ input: unknown;
35
+ output: unknown;
36
+ };
37
+ type TurnContext = {
38
+ files: TaskFiles;
39
+ transcript: TurnRecord[];
40
+ };
41
+ type TurnFn<TUserTurn = unknown> = (ctx: TurnContext) => Promise<TUserTurn | null> | TUserTurn | null;
42
+ declare function turn<TUserTurn>(fn: TurnFn<TUserTurn>): void;
43
+ declare function getTaskTurn(): TurnFn | undefined;
44
+ declare function resetTaskTurn(): void;
45
+ declare function resolveTurn(adapterTurn: TurnFn | undefined, taskTurn: TurnFn | undefined): TurnFn;
46
+
47
+ type ValidatableSchemaLike = {
48
+ safeParse: (data: unknown) => {
49
+ success: boolean;
50
+ error?: {
51
+ message: string;
52
+ };
53
+ };
54
+ };
55
+ type DeliverableDefinition = ValidatableSchemaLike | {
56
+ schema?: ValidatableSchemaLike;
57
+ } | null;
58
+ type CollectedDeliverables = Record<string, unknown>;
59
+ type AgentTurnResult = {
60
+ response: unknown;
61
+ };
62
+ /**
63
+ * The adapter-reported identity of the agent under test for one Task Run
64
+ *. The adapter resolves its configuration (env vars, aliases,
65
+ * defaults), constructs the agent from that same resolved object, and reports
66
+ * the resolved values here.
67
+ *
68
+ * - `model` is the exact identifier the adapter passed to its runtime.
69
+ * - `effort` is the exact short value that runtime used (`low`/`medium`/...
70
+ * or `"default"`). Omitted means "the adapter did not report effort".
71
+ *
72
+ * The whole configuration is absent (`undefined`) when the adapter does not
73
+ * support reporting it. Values are descriptive — they never affect task
74
+ * selection, execution, scoring, retry, or deduplication.
75
+ */
76
+ interface AgentTaskRunConfiguration {
77
+ model: string;
78
+ effort?: string;
79
+ }
80
+ type AdapterRuntimeState = Record<string, unknown>;
81
+ type InitializeContext = {
82
+ task: TaskDefinition;
83
+ taskDir: string;
84
+ files: FileEntry[];
85
+ trace: AgentTaskTraceContext;
86
+ };
87
+ type StartSessionContext = {
88
+ task: TaskDefinition;
89
+ taskDir: string;
90
+ files: FileEntry[];
91
+ state?: AdapterRuntimeState;
92
+ trace: AgentTaskTraceContext;
93
+ };
94
+ type CollectDeliverablesContext = {
95
+ task: TaskDefinition;
96
+ taskDir: string;
97
+ files: FileEntry[];
98
+ state?: AdapterRuntimeState;
99
+ session: AdapterSession;
100
+ trace: AgentTaskTraceContext;
101
+ };
102
+ type CleanupContext = {
103
+ task: TaskDefinition;
104
+ taskDir: string;
105
+ files: FileEntry[];
106
+ state?: AdapterRuntimeState;
107
+ session?: AdapterSession;
108
+ trace: AgentTaskTraceContext;
109
+ };
110
+ type AdapterSession = {
111
+ /**
112
+ * The adapter's resolved model/effort for this run. Read by
113
+ * `runTask()` immediately after `startSession()` returns, validated, and
114
+ * copied into {@link TaskRunResult.runConfiguration}. Absent for adapters
115
+ * that do not report configuration.
116
+ */
117
+ runConfiguration?: AgentTaskRunConfiguration;
118
+ sendUserTurn: (turn: unknown, context: {
119
+ trace: AgentTaskTraceContext;
120
+ turnNumber: number;
121
+ parentSpanId?: string;
122
+ }) => Promise<AgentTurnResult>;
123
+ close?: () => Promise<void>;
124
+ };
125
+ type AdapterDefinition = {
126
+ name: string;
127
+ deliverables: Record<string, DeliverableDefinition>;
128
+ turn?: TurnFn;
129
+ initialize?: (ctx: InitializeContext) => Promise<AdapterRuntimeState | void>;
130
+ startSession: (ctx: StartSessionContext) => Promise<AdapterSession>;
131
+ collectDeliverables: (ctx: CollectDeliverablesContext) => Promise<CollectedDeliverables>;
132
+ cleanup?: (ctx: CleanupContext) => Promise<void>;
133
+ };
134
+ type TypedAdapterDefinition<TName extends string, TDeliverables extends Record<string, DeliverableDefinition>, TCollected extends CollectedDeliverables = CollectedDeliverables> = Omit<AdapterDefinition, "name" | "deliverables" | "collectDeliverables"> & {
135
+ name: TName;
136
+ deliverables: TDeliverables;
137
+ collectDeliverables: (ctx: CollectDeliverablesContext) => Promise<TCollected>;
138
+ };
139
+
140
+ /**
141
+ * Trace Projection snapshot contract.
142
+ *
143
+ * This is the immutable transport/read snapshot of a Trace Projection — the
144
+ * query-optimized representation shared by the dashboard and agent-task
145
+ * assertions. It is NOT a new domain entity and never a source of truth; the
146
+ * canonical OpenTelemetry spans remain authoritative.
147
+ *
148
+ * Both the Python backend and the TypeScript SDK MUST serialize to this exact
149
+ * lower-camel-case JSON contract, so assertions read the same shape whether
150
+ * the snapshot came from the canonical repository, a local/offline recorder,
151
+ * or a legacy-Flow compatibility adapter.
152
+ *
153
+ * Rules:
154
+ * - `schemaVersion` versions the JSON contract.
155
+ * - `projectionVersion` identifies the normalizer/projector interpretation.
156
+ * - `source` is diagnostic; only `"canonical"` represents a durably projected
157
+ * Trace.
158
+ * - `complete` means the task execution root ended and every span in the
159
+ * acknowledged export was projected. It does not promise no unrelated
160
+ * service will ever export a late span.
161
+ * - Missing evidence is represented through `capabilities`, never converted to
162
+ * empty arrays, zero duration, or successful status.
163
+ * - Observations are immutable and sorted deterministically by invocation
164
+ * time, then span ID. Missing timestamps sort after timestamped ones.
165
+ */
166
+ /** Whether a category of evidence is present in the projection. */
167
+ type EvidenceAvailability = "available" | "partial" | "unavailable";
168
+ /** Lifecycle status of an observation. `unset` = the source had no status. */
169
+ type ObservationStatus = "unset" | "ok" | "error";
170
+ /** What kind of Span an Observation was derived from. */
171
+ type ObservationType = "SPAN" | "GENERATION" | "TOOL" | "AGENT" | "SKILL" | "CHAIN" | "RETRIEVER" | "EMBEDDING" | "GUARDRAIL";
172
+ /**
173
+ * Per-category evidence availability. Every category must declare honestly
174
+ * what it carries so assertions can fail closed when evidence is missing
175
+ * instead of vacuously passing.
176
+ */
177
+ interface TraceProjectionCapabilities {
178
+ messages: EvidenceAvailability;
179
+ tools: EvidenceAvailability;
180
+ errors: EvidenceAvailability;
181
+ timing: EvidenceAvailability;
182
+ skills: EvidenceAvailability;
183
+ subagents: EvidenceAvailability;
184
+ }
185
+ /** One chat message reconstructed from a generation observation. */
186
+ interface TraceProjectionMessage {
187
+ role: "system" | "user" | "assistant" | "tool";
188
+ content: string;
189
+ }
190
+ /**
191
+ * A derived interpretation of one Span, as shown by the dashboard and read by
192
+ * assertions. Unknown span kinds survive as `SPAN`; hierarchy is preserved via
193
+ * `parentSpanId`.
194
+ */
195
+ interface TraceProjectionObservation {
196
+ spanId: string;
197
+ parentSpanId?: string;
198
+ type: ObservationType;
199
+ name: string;
200
+ startedAt?: string;
201
+ endedAt?: string;
202
+ durationMs?: number;
203
+ status: ObservationStatus;
204
+ errorMessage?: string;
205
+ input?: unknown;
206
+ output?: unknown;
207
+ model?: string;
208
+ toolName?: string;
209
+ toolParameters?: unknown;
210
+ toolResult?: unknown;
211
+ messages?: readonly TraceProjectionMessage[];
212
+ metadata?: Readonly<Record<string, unknown>>;
213
+ }
214
+ /** Trace-level facts. All timing is optional; absence is honest. */
215
+ interface TraceProjectionTrace {
216
+ traceId: string;
217
+ taskRunId?: string;
218
+ name?: string;
219
+ startedAt?: string;
220
+ endedAt?: string;
221
+ durationMs?: number;
222
+ complete: boolean;
223
+ }
224
+ /**
225
+ * The immutable read snapshot. Identical JSON shape across canonical, local,
226
+ * and legacy-flow sources — only `source` and `capabilities` differ.
227
+ */
228
+ interface TraceProjectionSnapshot {
229
+ schemaVersion: 1;
230
+ projectionVersion: number;
231
+ source: "canonical" | "local" | "legacy-flow";
232
+ trace: TraceProjectionTrace;
233
+ capabilities: TraceProjectionCapabilities;
234
+ observations: readonly TraceProjectionObservation[];
235
+ }
236
+
237
+ /**
238
+ * The outcome of a single assertion.
239
+ *
240
+ * - ``"pass"`` → ``pass=true``
241
+ * - ``"fail"`` → ``pass=false``
242
+ * - ``"unsupported"`` → ``pass=false``. The trace projection lacked the
243
+ * evidence this assertion needed (e.g. timing, error status). An unsupported
244
+ * trace assertion fails closed and explains which evidence was unavailable;
245
+ * it must never silently pass. Value assertions (``t.check``) and LLM
246
+ * assertions (``t.judge``) do not consult trace capabilities and retain
247
+ * their existing pass/fail behavior.
248
+ */
249
+ type AssertionOutcome = "pass" | "fail" | "unsupported";
250
+ /**
251
+ * Metadata about an LLM judge call. Populated by evaluators that use an
252
+ * LLM to make their pass/fail decision. All fields optional so code-only
253
+ * evaluators can leave this undefined.
254
+ */
255
+ type JudgeMetadata = {
256
+ /** Model identifier, e.g. ``"google/gemini-2.5-flash-lite"``. */
257
+ model?: string;
258
+ /** The messages sent to the judge LLM. */
259
+ prompt?: {
260
+ system?: string;
261
+ user?: string;
262
+ };
263
+ /** Raw LLM response text before parsing into {pass, reasoning}. */
264
+ response?: string;
265
+ /** Token usage if available from the provider. */
266
+ tokens?: {
267
+ input: number;
268
+ output: number;
269
+ cache_creation?: number;
270
+ cache_read?: number;
271
+ };
272
+ /** Estimated cost in USD, if available. */
273
+ cost?: number;
274
+ /** Wall-clock latency of the judge call in milliseconds. */
275
+ latency_ms?: number;
276
+ /** Temperature or other sampling parameters, if relevant. */
277
+ temperature?: number;
278
+ };
279
+ /**
280
+ * A source location for a failed code check — lets the dashboard render the
281
+ * failure inline, editor-style. ``file`` is a display name (e.g. the checks
282
+ * filename); ``line``/``column`` are 1-indexed into that file.
283
+ */
284
+ type CheckLocation = {
285
+ file: string;
286
+ line: number;
287
+ column?: number;
288
+ };
289
+ /**
290
+ * A single assertion within a code check (the recorder collects many per
291
+ * check). Carries structured Expected/Received so the dashboard can render
292
+ * testing-framework-style failures instead of a flattened prose string.
293
+ *
294
+ * - ``expected`` — what the assertion wanted (matcher label, or "≥1 read_file
295
+ * call" for trace asserts). Absent when not meaningful.
296
+ * - ``received`` — the actual value/count the run produced. Serialized scalar
297
+ * for code assertions; the raw evaluated value for LLM judges (see field).
298
+ * - ``location`` — where in the checks source this assertion lives.
299
+ */
300
+ type AssertionResult = {
301
+ id: string;
302
+ pass: boolean;
303
+ reasoning: string;
304
+ /**
305
+ * The outcome category. ``"unsupported"`` means the trace
306
+ * projection lacked the evidence this assertion needed (e.g. timing,
307
+ * errors) — it fails closed (``pass=false``) with an explanatory reason
308
+ * rather than vacuously passing. Absent on legacy results.
309
+ */
310
+ outcome?: AssertionOutcome;
311
+ expected?: string;
312
+ /**
313
+ * The actual value the assertion observed. For code assertions this is a
314
+ * short serialized scalar (`"3"`, `"read_file → write_file"`); for LLM
315
+ * judges it is the **raw value passed to `t.judge`** — an arbitrary JSON
316
+ * value (string, object, array, primitive), so the dashboard can render it
317
+ * with a structured viewer instead of a truncated string.
318
+ */
319
+ received?: unknown;
320
+ location?: CheckLocation;
321
+ evaluator_type?: "llm" | "code";
322
+ judge?: JudgeMetadata;
323
+ };
324
+ /**
325
+ * Result of evaluating a single check.
326
+ *
327
+ * The three required fields ({@link id}, {@link pass}, {@link reasoning})
328
+ * have been here since the beginning. The optional fields below are
329
+ * enriched metadata that lets the dashboard show *what* was evaluated,
330
+ * *how* it was evaluated, and — for LLM judges — exactly what the judge
331
+ * was asked and what it answered.
332
+ *
333
+ * Evaluators that don't populate the optional fields still work; the
334
+ * dashboard gracefully falls back to the legacy three-field display.
335
+ */
336
+ type EvaluationItemResult = {
337
+ id: string;
338
+ pass: boolean;
339
+ reasoning: string;
340
+ /** The rubric instruction from the task definition ("PASS if …"). */
341
+ instruction?: string;
342
+ /** Name of the deliverable this check was evaluated against. */
343
+ deliverable?: string;
344
+ /**
345
+ * What kind of evaluator produced this result.
346
+ * Registered checks use ``"code"``. LLM-backed assertions inside a check
347
+ * carry ``evaluator_type: "llm"`` in {@link assertions}.
348
+ * - ``"regex"`` — pattern matching
349
+ * Older persisted results may still use ``"llm"`` at this level.
350
+ */
351
+ evaluator_type?: "llm" | "code" | "regex";
352
+ /**
353
+ * If this check was judged by an LLM, details about the judge call
354
+ * (model, prompt, response, tokens, cost, latency). Populated by the
355
+ * evaluator author; absent for code-only evaluators.
356
+ */
357
+ judge?: JudgeMetadata;
358
+ /**
359
+ * For code checks: the source location of the failure (parsed from the
360
+ * thrown error / failed assertion stack). Lets the dashboard highlight the
361
+ * failing line. Absent when not a code check, when the check passed, or
362
+ * when no frame could be resolved to the checks module.
363
+ */
364
+ location?: CheckLocation;
365
+ /**
366
+ * For code checks: the source filename the result came from (normally
367
+ * the ``*.eval.ts`` file; ``"checks.ts"`` for legacy tasks), so the dashboard can show it even
368
+ * when no line was resolved.
369
+ */
370
+ source_file?: string;
371
+ /**
372
+ * For code checks: the per-assertion breakdown (Expected/Received/location
373
+ * each). Lets the dashboard mark every failing assertion at its own line,
374
+ * testing-framework-style. Absent for non-code evaluators and old runs.
375
+ */
376
+ assertions?: AssertionResult[];
377
+ /**
378
+ * the snapshot source the checks ran against. Set to
379
+ * ``"legacy-flow"`` by the deprecated {@link runFlowChecks} wrapper so
380
+ * consumers can detect they're on the compatibility path. Absent on
381
+ * projection-first results (the default).
382
+ */
383
+ source?: "canonical" | "local" | "legacy-flow";
384
+ /**
385
+ * the id of the `describe()` group this check was declared
386
+ * inside. Absent for checks declared at the top level (no enclosing
387
+ * describe) and for old results. The dashboard groups checks by this field.
388
+ */
389
+ group_id?: string;
390
+ /**
391
+ * the display name of the enclosing `describe()` group.
392
+ * Defaults to the group id when the group was declared without a name.
393
+ * Absent when {@link group_id} is absent.
394
+ */
395
+ group_name?: string;
396
+ };
397
+ type TaskEvaluationResult = {
398
+ checks: EvaluationItemResult[];
399
+ pass: boolean;
400
+ };
401
+ type TaskTranscript = {
402
+ turns: TaskTranscriptTurn[];
403
+ };
404
+ type TaskTranscriptTurn = {
405
+ turnNumber: number;
406
+ userAction: unknown;
407
+ agentResponse: unknown;
408
+ };
409
+ type TaskRunResult = {
410
+ task: TaskDefinition;
411
+ taskDir: string;
412
+ files: FileEntry[];
413
+ traceRunId?: string;
414
+ result: TaskEvaluationResult;
415
+ deliverables: Record<string, unknown>;
416
+ transcript: TaskTranscript;
417
+ /**
418
+ * The adapter's resolved model/effort for this run. Captured and
419
+ * normalized from `AdapterSession.runConfiguration` immediately after the
420
+ * session opens. Absent for adapters that do not report configuration.
421
+ */
422
+ runConfiguration?: AgentTaskRunConfiguration;
423
+ };
424
+
425
+ /**
426
+ * TraceView — typed, capability-gated, derived access over a
427
+ * {@link TraceProjectionSnapshot}. This is the read model the projection-first
428
+ * assertion surface queries. It is the projection analogue of `FlowView`.
429
+ *
430
+ * The defining difference from `FlowView` is **capability honesty**: when the
431
+ * snapshot declares a category of evidence `unavailable`, the derived numeric
432
+ * facts (`durationMs`, `failedActions`, `turnCount`) return `undefined` rather
433
+ * than zero — so an assertion like `t.maxDurationMs` can record an explicit
434
+ * `unsupported` outcome instead of vacuously passing against a fabricated 0ms.
435
+ *
436
+ * `toolNamesInOrder` sorts by observation invocation time (`startedAt`) with a
437
+ * deterministic span-ID tie-breaker — NOT array/completion order. This fixes
438
+ * the drift where concurrent tools appeared in completion order.
439
+ */
440
+
441
+ /** A tool call derived from a `TOOL` observation. */
442
+ interface TraceToolCall {
443
+ spanId: string;
444
+ name: string;
445
+ input?: unknown;
446
+ output?: unknown;
447
+ status: ObservationStatus;
448
+ startedAt?: string;
449
+ }
450
+ /** A skill load derived from a `SKILL` observation. */
451
+ interface TraceSkillLoad {
452
+ spanId: string;
453
+ skill: string;
454
+ startedAt?: string;
455
+ }
456
+ /** A subagent delegation derived from an `AGENT` observation. */
457
+ interface TraceSubagentCall {
458
+ spanId: string;
459
+ agent: string;
460
+ output?: unknown;
461
+ status: ObservationStatus;
462
+ startedAt?: string;
463
+ }
464
+ declare class TraceView {
465
+ readonly snapshot: TraceProjectionSnapshot;
466
+ constructor(snapshot: TraceProjectionSnapshot);
467
+ /** Evidence availability for a capability. */
468
+ requireCapability(capability: keyof TraceProjectionCapabilities): EvidenceAvailability;
469
+ /** Whether a capability is reported as available (not partial/unavailable). */
470
+ private isAvailable;
471
+ /** All chat messages flattened across generation observations, in order. */
472
+ get messages(): readonly TraceProjectionMessage[];
473
+ /** Tool calls derived from `TOOL` observations, in invocation order. */
474
+ get toolCalls(): readonly TraceToolCall[];
475
+ /** Tool names in invocation order with deterministic span-ID tie-breaking. */
476
+ get toolNamesInOrder(): readonly string[];
477
+ /** Skill loads derived from `SKILL` observations, in invocation order. */
478
+ get skillLoads(): readonly TraceSkillLoad[];
479
+ /** Subagent delegations derived from `AGENT` observations, in invocation order. */
480
+ get subagentCalls(): readonly TraceSubagentCall[];
481
+ /** Last assistant message content — the agent's "reply". Empty if none. */
482
+ get reply(): string;
483
+ /**
484
+ * Number of completed assistant turns, or `undefined` when message evidence
485
+ * is unavailable (capability honesty).
486
+ */
487
+ get turnCount(): number | undefined;
488
+ /**
489
+ * Number of tool/subagent observations that reported an error, or `undefined`
490
+ * when error evidence is unavailable (capability honesty).
491
+ */
492
+ get failedActions(): number | undefined;
493
+ /**
494
+ * Total execution duration in milliseconds, or `undefined` when timing
495
+ * evidence is unavailable (capability honesty). Uses trace-level
496
+ * startedAt/endedAt when present.
497
+ */
498
+ get durationMs(): number | undefined;
499
+ /**
500
+ * Observations sorted deterministically by invocation time then span ID.
501
+ * Memoized per TraceView instance since the snapshot is immutable.
502
+ */
503
+ private get sortedObservations();
504
+ private _sorted;
505
+ }
506
+
507
+ /**
508
+ * Value matchers for `t.check(value, matcher)`. Small, composable, and the
509
+ * single way to assert on deliverables/values (replaces the old `expect`).
510
+ *
511
+ * A matcher is just `{ label, test }` — easy to build custom ones inline with
512
+ * `satisfies(fn, label)`.
513
+ */
514
+ type Matcher<T> = {
515
+ label: string;
516
+ test: (value: T) => boolean;
517
+ };
518
+ /** Coerces the value to a string and checks it contains a substring or RegExp. Gate. */
519
+ declare function includes(needle: string | RegExp): Matcher<unknown>;
520
+ /** Deep structural equality. Gate. */
521
+ declare function equals<T>(expected: T): Matcher<T>;
522
+ /**
523
+ * Validates against a Standard-Schema-shaped validator (Zod, Valibot, …) —
524
+ * anything exposing `safeParse`. Gate.
525
+ */
526
+ declare function matches<T = unknown>(schema: {
527
+ safeParse: (value: unknown) => {
528
+ success: boolean;
529
+ };
530
+ }): Matcher<T>;
531
+ /** Custom boolean predicate. Gate. */
532
+ declare function satisfies<T>(predicate: (value: T) => boolean, label: string): Matcher<T>;
533
+ type ValueMatcher<T> = T | RegExp | ((value: T) => boolean);
534
+ declare function matchValue<T>(actual: T, matcher: ValueMatcher<T>): boolean;
535
+ /** Normalized Levenshtein similarity. Pass when score ≥ threshold (default 0.8). */
536
+ declare function similarity(expected: string, threshold?: number): Matcher<unknown>;
537
+
538
+ /**
539
+ * The assertion surface — `t`. Flat and eve-style: reads the trace (via
540
+ * {@link TraceView}) for behavioural checks and grades explicit values via
541
+ * `t.check`. Everything records (never throws), so a check reports all of its
542
+ * failures, not just the first.
543
+ *
544
+ * Every assertion records structured ``expected`` / ``received`` values so
545
+ * the dashboard can render testing-framework-style failures (Expected /
546
+ * Received) instead of only a prose message.
547
+ */
548
+
549
+ /** A tool/agent name matcher: literal (exact), RegExp, or predicate. */
550
+ type NameMatcher = string | RegExp | ((name: string) => boolean);
551
+ /** Options for matching tool calls by count and recorded fields. */
552
+ type ToolCallOptions = {
553
+ count?: number;
554
+ input?: ValueMatcher<unknown>;
555
+ output?: ValueMatcher<unknown>;
556
+ status?: "ok" | "error";
557
+ };
558
+ /**
559
+ * LLM judge model config for `t.judge(...)` calls in checks.ts. Threaded
560
+ * through `runTask({ judge })` from the caller — the runner has no opinion
561
+ * about which model grades a deliverable.
562
+ */
563
+ type JudgeConfig = {
564
+ /** Model id, e.g. ``"google/gemini-2.5-flash-lite"``. */
565
+ model: string;
566
+ /** Override the OpenRouter/OpenAI-compatible base URL. */
567
+ baseURL?: string;
568
+ /** Override the API key (defaults to env). */
569
+ apiKey?: string;
570
+ };
571
+ interface TestContext {
572
+ /** Asserts a matching tool was called. Pass `{ count }` for an exact count. */
573
+ calledTool(name: NameMatcher, opts?: ToolCallOptions): void;
574
+ /** Asserts no tool call matches the supplied name and field constraints. */
575
+ notCalledTool(name: NameMatcher, opts?: Omit<ToolCallOptions, "count">): void;
576
+ /** Asserts the named tools appear, in this order (as a subsequence). */
577
+ toolOrder(names: string[]): void;
578
+ /** Asserts no tool calls happened at all. */
579
+ usedNoTools(): void;
580
+ /** Asserts at most `n` tool calls — anti-flail. */
581
+ maxToolCalls(n: number): void;
582
+ /** Asserts no tool/subagent call reported an error — anti-flail. */
583
+ noFailedActions(): void;
584
+ /** Asserts a skill was loaded. */
585
+ loadedSkill(skill: string): void;
586
+ /** Asserts a subagent delegation happened. */
587
+ calledSubagent(agent: string): void;
588
+ /** Asserts the agent's reply contains a substring or RegExp. */
589
+ messageIncludes(token: string | RegExp): void;
590
+ /** Asserts the run took at most `n` turns — anti-flail. */
591
+ maxTurns(n: number): void;
592
+ /** Asserts the run took at most `n` milliseconds — anti-flail. */
593
+ maxDurationMs(n: number): void;
594
+ /** Escape hatch for a named predicate over the complete normalized run. */
595
+ assert(label: string, predicate: (flow: TraceView) => boolean): void;
596
+ /** Grades any value (a deliverable, parsed JSON, anything) with a matcher. */
597
+ check<T>(value: T, matcher: Matcher<T>, label?: string): void;
598
+ /**
599
+ * LLM-as-judge: asks the configured judge model to grade ``value`` against
600
+ * ``instruction`` (a natural-language rubric). Async — the check function
601
+ * must `await` it. Records a single assertion tagged
602
+ * ``evaluator_type: "llm"`` with judge metadata (model, prompt, response,
603
+ * tokens, latency) attached.
604
+ *
605
+ * ``opts.judge`` overrides the run's judge config **for this call only**,
606
+ * merging field-by-field onto the config threaded in via ``runTask({ judge })``
607
+ * — set ``{ model }`` to escalate a finicky criterion to a stronger model
608
+ * without switching the whole run. Absent fields inherit from the base config.
609
+ */
610
+ judge(values: unknown | unknown[], instruction: string, opts?: {
611
+ label?: string;
612
+ judge?: Partial<JudgeConfig>;
613
+ }): Promise<void>;
614
+ }
615
+ /**
616
+ * Canonical names of every assertion method on {@link TestContext}.
617
+ *
618
+ * Single source of truth for "what is a ``t.<method>`` call?" — consumers that
619
+ * highlight, lint, or otherwise detect `t.*` usage in check source read this
620
+ * array instead of keeping their own copy (which historically drifted out of
621
+ * sync with the interface, leaving methods like ``t.maxTurns`` unstyled).
622
+ *
623
+ * The literal below is compile-time-checked against ``keyof TestContext``:
624
+ * adding a method to the interface without registering it here is a type
625
+ * error, and vice versa. See ``tests/test-context-methods.test.ts``.
626
+ */
627
+ declare const TEST_METHOD_NAMES: readonly ["calledTool", "notCalledTool", "toolOrder", "usedNoTools", "maxToolCalls", "noFailedActions", "loadedSkill", "calledSubagent", "messageIncludes", "maxTurns", "maxDurationMs", "assert", "check", "judge"];
628
+
629
+ /**
630
+ * Check runner — builds a {@link TestContext} (`t`) per registered check from
631
+ * a {@link TraceProjectionSnapshot}, runs it, and aggregates every recorded
632
+ * assertion into one evaluation result per check.
633
+ */
634
+
635
+ /**
636
+ * The check context — deliberately framework-agnostic. `deliverables` is the
637
+ * output; `files`/`task` are optional and untyped here so the core never
638
+ * depends on any agent-framework's types. apo passes its own typed values;
639
+ * other frameworks pass whatever they have (or omit).
640
+ *
641
+ * A task-scoped ``test`` gets typed deliverables from the adapter automatically:
642
+ *
643
+ * ```ts
644
+ * const { test } = task("review", {
645
+ * adapter: reviewAdapter,
646
+ * deliverables: ["result"],
647
+ * });
648
+ * test("id", (t, { deliverables }) => {
649
+ * deliverables.result // typed as ReviewResult
650
+ * });
651
+ * ```
652
+ *
653
+ * The generic overload of {@link defineCheck} remains available for
654
+ * framework-agnostic checks that have no task scope.
655
+ */
656
+ type CheckContext<TDeliverables = Record<string, unknown>> = {
657
+ deliverables: TDeliverables;
658
+ files?: unknown;
659
+ task?: unknown;
660
+ };
661
+ declare function filePaths(files: unknown): string[];
662
+ type CheckFn<TDeliverables = Record<string, unknown>> = (t: TestContext, ctx: CheckContext<TDeliverables>) => Promise<void> | void;
663
+ type TestRegistration<TDeliverables = Record<string, unknown>> = (id: string, fn: CheckFn<TDeliverables>) => void;
664
+ /**
665
+ * A registered `describe()` group — single-level, organizational only. See
666
+ * {@link describe}.
667
+ */
668
+ type CheckGroup = {
669
+ /** Stable, unique group id (kebab-case). Grouping key + React key. */
670
+ id: string;
671
+ /** Human-readable label shown in the dashboard. Defaults to the id. */
672
+ name: string;
673
+ };
674
+ /** Signature of the `describe` registration primitive. */
675
+ type DescribeRegistration = {
676
+ (id: string, fn: () => void): void;
677
+ (id: string, name: string, fn: () => void): void;
678
+ };
679
+ /** Register a check. The `fn` receives the assertion surface `t` and the output.
680
+ *
681
+ * Framework-agnostic callers can pass a deliverables type explicitly:
682
+ * ```ts
683
+ * type Deliverables = { result: ReviewResult; stats: Stats };
684
+ * defineCheck<Deliverables>("id", (t, { deliverables }) => { ... });
685
+ * ```
686
+ */
687
+ declare function defineCheck<TDeliverables>(id: string, fn: CheckFn<TDeliverables>): void;
688
+ declare function defineCheck(id: string, fn: CheckFn): void;
689
+ declare function resetFlowChecks(): void;
690
+ /**
691
+ * Register a single-level group of checks. Runs `fn` synchronously;
692
+ * any `test()`/`defineCheck()` call inside it is stamped with the group's id
693
+ * and display name, so the dashboard can nest those checks under a collapsible
694
+ * header with a roll-up verdict.
695
+ *
696
+ * Groups are **organizational only** — they do not change execution order,
697
+ * concurrency, or the task verdict. Nesting is prohibited: calling `describe`
698
+ * inside a `describe` callback throws. Use sibling groups under one task
699
+ * instead.
700
+ *
701
+ * The `id` is stable (survives display-name edits); `name` is the human label
702
+ * shown in the dashboard and defaults to `id` when omitted.
703
+ *
704
+ * ```ts
705
+ * const { test, describe } = task("bind", { adapter, deliverables });
706
+ * describe("rules", "Rules — each comment becomes a rule", () => {
707
+ * RULE_GOLD.forEach((g) => test(`R-${g.id} — …`, async (t, { deliverables }) => { … }));
708
+ * });
709
+ * ```
710
+ */
711
+ declare function describe(id: string, fn: () => void): void;
712
+ declare function describe(id: string, name: string, fn: () => void): void;
713
+
714
+ declare const TASK_ADAPTER_SYMBOL: unique symbol;
715
+ declare function resetTaskRegistry(): void;
716
+ type DefinedTask<TName extends string, TDeliverableDefs extends Record<string, DeliverableDefinition>> = TaskDefinition<TName, keyof TDeliverableDefs & string> & {
717
+ readonly [TASK_ADAPTER_SYMBOL]: TypedAdapterDefinition<TName, TDeliverableDefs>;
718
+ };
719
+ /** Legacy two-file task definition. New tasks should use {@link task}. */
720
+ declare function defineTask<const TName extends string, const TDeliverableDefs extends Record<string, DeliverableDefinition>>(adapter: TypedAdapterDefinition<TName, TDeliverableDefs>, config: TaskConfig<keyof TDeliverableDefs & string>): DefinedTask<TName, TDeliverableDefs>;
721
+ type TaskScope<TDeliverables> = {
722
+ test: TestRegistration<TDeliverables>;
723
+ /**
724
+ * Register a single-level group of checks. See
725
+ * {@link describe} in `checks/flow-runner.ts`. Returned from `task()` so the
726
+ * `test` inside the callback stays task-scoped (typed deliverables).
727
+ */
728
+ describe: DescribeRegistration;
729
+ };
730
+ /**
731
+ * Register a task + its checks in ONE file. The `name` is the task id;
732
+ * `config` includes the adapter, deliverables, and other metadata. Checks
733
+ * are registered through the returned, adapter-typed `test` function.
734
+ *
735
+ * ```ts
736
+ * const { test } = task("code-review", {
737
+ * adapter: realAgentAdapter,
738
+ * deliverables: ["result", "tool_log", "stats"],
739
+ * maxTurns: 2,
740
+ * });
741
+ *
742
+ * test("reviewed-methodically", (t, { deliverables }) => {
743
+ * t.check(deliverables.result, includes("finding"));
744
+ * });
745
+ *
746
+ * // A task that needs dev-machine resources (cloud creds, VPC, stage) can
747
+ * // declare execution: "local" so `apo task run` runs it on the caller's
748
+ * // machine while still recording a backend run row.
749
+ * task("bind-e2e", {
750
+ * adapter: bindAdapter,
751
+ * deliverables: ["summary"],
752
+ * execution: "local",
753
+ * });
754
+ * ```
755
+ */
756
+ declare function task<const TTaskId extends string, const TAdapterName extends string, const TDeliverableDefs extends Record<string, DeliverableDefinition>, TCollected extends CollectedDeliverables, const TSelected extends readonly (DeliverableKey<TCollected> & keyof TDeliverableDefs)[]>(name: TTaskId, config: Omit<TaskConfig<TSelected[number] & string>, "id" | "checks" | "deliverables"> & {
757
+ adapter: TypedAdapterDefinition<TAdapterName, TDeliverableDefs, TCollected>;
758
+ deliverables: TSelected;
759
+ }): TaskScope<SelectedDeliverables<TCollected, TSelected>>;
760
+ type DeliverableKey<TCollected> = TCollected extends unknown ? keyof TCollected & string : never;
761
+ type DeliverableValue<TCollected, TKey extends PropertyKey> = TCollected extends unknown ? TKey extends keyof TCollected ? TCollected[TKey] : never : never;
762
+ type SelectedDeliverables<TCollected, TSelected extends readonly PropertyKey[]> = {
763
+ [TKey in TSelected[number]]-?: Exclude<DeliverableValue<TCollected, TKey>, null | undefined>;
764
+ };
765
+
766
+ type LoadedTask = {
767
+ task: TaskDefinition;
768
+ adapter: AdapterDefinition;
769
+ taskDir: string;
770
+ files: FileEntry[];
771
+ /** Legacy external checks module, absent for single-file tasks. */
772
+ checksPath: string | null;
773
+ /** True when task(), test(), and turn() were registered by the eval file. */
774
+ inlineChecks: boolean;
775
+ /** The temp-module URL the task was imported from (used for location pinning). */
776
+ moduleUrl: string;
777
+ /** The original eval filename (e.g. "code-review.eval.ts") — used as source_file on check results. */
778
+ evalFileName: string;
779
+ };
780
+ declare function loadTask(taskDir: string): Promise<LoadedTask>;
781
+
782
+ declare function defineAdapter<const TName extends string, const TDeliverables extends Record<string, DeliverableDefinition>, TCollected extends CollectedDeliverables>(adapter: TypedAdapterDefinition<TName, TDeliverables, TCollected>): TypedAdapterDefinition<TName, TDeliverables, TCollected>;
783
+
784
+ /**
785
+ * file Artifact declaration helper.
786
+ *
787
+ * A ``FileArtifact`` is a path descriptor that identifies a regular local
788
+ * file at persistence time. The SDK streams it to the backend (two-phase
789
+ * upload) rather than buffering bytes in the result body.
790
+ *
791
+ * Rules:
792
+ * - ``path`` must identify a regular local file at persistence time;
793
+ * - directories, sockets, devices, and symbolic links are rejected;
794
+ * - the default display filename is ``basename(path)``;
795
+ * - the default media type is ``application/octet-stream``;
796
+ * - ``t.judge(fileArtifact(...))`` is rejected: a path descriptor is not file
797
+ * content. Authors must explicitly read/parse the content to judge it.
798
+ */
799
+ interface FileArtifact {
800
+ readonly kind: "apo.file-artifact";
801
+ readonly path: string;
802
+ readonly mediaType: string;
803
+ readonly displayFilename: string;
804
+ }
805
+ interface FileArtifactOptions {
806
+ readonly mediaType?: string;
807
+ readonly displayFilename?: string;
808
+ }
809
+ /**
810
+ * Declare a file Deliverable from a local file path.
811
+ *
812
+ * Validates eagerly that ``path`` points to an existing regular file (not a
813
+ * directory, symlink, socket, or device) so authors get an actionable error
814
+ * before the backend creates an upload intent.
815
+ */
816
+ declare function fileArtifact(path: string, options?: FileArtifactOptions): FileArtifact;
817
+ declare function isFileArtifact(value: unknown): value is FileArtifact;
818
+
819
+ /**
820
+ * file Artifact uploader.
821
+ *
822
+ * After checks run against the original in-memory Deliverables, this module
823
+ * partitions JSON values from file Artifacts, streams each file to the
824
+ * backend's two-phase upload endpoint, and returns only JSON Deliverables for
825
+ * the final result body. File bytes never travel through the result request
826
+ * or subprocess stdout.
827
+ *
828
+ * Storage failure does not flip a failed test into a pass: it raises, so the
829
+ * caller records the run as errored.
830
+ */
831
+ interface DeliverableSummary {
832
+ id: string;
833
+ name: string;
834
+ kind: "json" | "artifact";
835
+ status: "pending" | "ready" | "failed";
836
+ media_type: string;
837
+ display_filename: string | null;
838
+ size_bytes: number;
839
+ sha256: string;
840
+ download_url: string | null;
841
+ }
842
+ interface ArtifactUploadConfig {
843
+ /** The Task Run id owning these Deliverables. */
844
+ taskRunId: string;
845
+ /** Bearer token (service token or API key) authorizing the upload. */
846
+ authToken: string;
847
+ /** Backend base URL, e.g. ``http://127.0.0.1:8000``. */
848
+ baseUrl: string;
849
+ /** Injectable fetch (tests pass a stub); defaults to global fetch. */
850
+ fetch?: typeof fetch;
851
+ }
852
+ interface PreparedTaskResult {
853
+ /** JSON-only deliverables safe to send in the final result body. */
854
+ jsonDeliverables: Record<string, unknown>;
855
+ /** Successfully uploaded file Artifacts (summaries only, no bodies). */
856
+ artifactUploads: DeliverableSummary[];
857
+ }
858
+ /**
859
+ * Partition and persist Deliverables after checks complete.
860
+ *
861
+ * - file Artifacts are streamed to the backend (hash, create intent, PUT);
862
+ * - JSON values are returned untouched for the result body;
863
+ * - the local path of any Artifact is never embedded in the result.
864
+ */
865
+ declare function persistFileArtifacts(deliverables: Record<string, unknown>, config: ArtifactUploadConfig): Promise<PreparedTaskResult>;
866
+
867
+ /**
868
+ * Flow — the canonical, source-agnostic recording of "what happened" during
869
+ * an agent run. This is the single substrate the testing framework reads.
870
+ *
871
+ * It is deliberately neutral: it does not know about our trace, the AI SDK,
872
+ * OpenAI, Anthropic, etc. Concrete sources are converted into a {@link Flow}
873
+ * by a normalizer (see `fromApoTrace`, etc.). The framework depends only on
874
+ * this interface — that is what hides the source complexity.
875
+ */
876
+ type ToolCallStatus = "ok" | "error";
877
+ type FlowEvent = {
878
+ kind: "message";
879
+ role: "user" | "assistant" | "system";
880
+ text: string;
881
+ ts?: number;
882
+ } | {
883
+ kind: "tool_call";
884
+ name: string;
885
+ input?: unknown;
886
+ output?: unknown;
887
+ status?: ToolCallStatus;
888
+ latencyMs?: number;
889
+ ts?: number;
890
+ } | {
891
+ kind: "skill_load";
892
+ skill: string;
893
+ ts?: number;
894
+ } | {
895
+ kind: "subagent_call";
896
+ agent: string;
897
+ output?: unknown;
898
+ status?: ToolCallStatus;
899
+ latencyMs?: number;
900
+ ts?: number;
901
+ };
902
+ type Flow = {
903
+ /** Ordered events — the timeline of the run. */
904
+ events: readonly FlowEvent[];
905
+ /** Wall-clock start/end, when known. Sources without clocks may omit them. */
906
+ startedAt?: number;
907
+ endedAt?: number;
908
+ };
909
+
910
+ /**
911
+ * FlowView — typed, derived access over a {@link Flow}. This is the read model
912
+ * the assertion surface (`t`) queries. It hides all the filtering/derivation
913
+ * so assertions stay one-liners.
914
+ */
915
+
916
+ type ToolCall = Extract<FlowEvent, {
917
+ kind: "tool_call";
918
+ }>;
919
+ type SkillLoad = Extract<FlowEvent, {
920
+ kind: "skill_load";
921
+ }>;
922
+ type SubagentCall = Extract<FlowEvent, {
923
+ kind: "subagent_call";
924
+ }>;
925
+ type Message = Extract<FlowEvent, {
926
+ kind: "message";
927
+ }>;
928
+ declare class FlowView {
929
+ readonly events: readonly FlowEvent[];
930
+ readonly startedAt?: number;
931
+ readonly endedAt?: number;
932
+ constructor(flow: Flow);
933
+ get messages(): Message[];
934
+ get toolCalls(): ToolCall[];
935
+ get skillLoads(): SkillLoad[];
936
+ get subagentCalls(): SubagentCall[];
937
+ get toolNamesInOrder(): string[];
938
+ /** Last assistant message text — the agent's "reply". */
939
+ get reply(): string;
940
+ /** Number of completed assistant turns. */
941
+ get turnCount(): number;
942
+ /** Tool or subagent calls that reported an error. */
943
+ get failedActions(): number;
944
+ get durationMs(): number;
945
+ }
946
+
947
+ /**
948
+ * Cross-framework Flow normalizers — the "plugs" that let agents NOT built on
949
+ * apo's adapter produce a {@link Flow} from their own output, so the same
950
+ * `t.*` checks work for everyone.
951
+ *
952
+ * Each normalizer is `(source) => Flow`. Add new ones by following the shape.
953
+ *
954
+ * @deprecated These are **compatibility recording adapters**,
955
+ * not the primary testing architecture. The canonical path is now the Trace
956
+ * Projection snapshot (`TraceProjectionSnapshot` → `TraceView` →
957
+ * `runTraceChecks`). These normalizers remain available during the
958
+ * deprecation period; `snapshotFromFlow` bridges their output into the
959
+ * projection contract. New integrations should emit standard OTel spans
960
+ * instead of producing a Flow.
961
+ */
962
+
963
+ interface OpenAIMessage {
964
+ role?: string;
965
+ content?: string | null;
966
+ tool_calls?: Array<{
967
+ id?: string;
968
+ function?: {
969
+ name?: string;
970
+ arguments?: string;
971
+ };
972
+ }>;
973
+ tool_call_id?: string;
974
+ }
975
+ declare function fromOpenAIMessages(messages: OpenAIMessage[]): Flow;
976
+ interface AnthropicMessage {
977
+ role?: string;
978
+ content?: Array<{
979
+ type: "text";
980
+ text: string;
981
+ } | {
982
+ type: "tool_use";
983
+ id: string;
984
+ name: string;
985
+ input?: unknown;
986
+ } | {
987
+ type: "tool_result";
988
+ tool_use_id: string;
989
+ content?: unknown;
990
+ }>;
991
+ }
992
+ declare function fromAnthropicMessages(messages: AnthropicMessage[]): Flow;
993
+ interface AISDKResult {
994
+ steps?: Array<{
995
+ text?: string;
996
+ toolCalls?: Array<{
997
+ toolName: string;
998
+ input?: unknown;
999
+ }>;
1000
+ toolResults?: Array<{
1001
+ toolName: string;
1002
+ output?: unknown;
1003
+ }>;
1004
+ }>;
1005
+ }
1006
+ declare function fromAISDK(result: AISDKResult): Flow;
1007
+
1008
+ /**
1009
+ * Error thrown when a Task Run fails after the adapter's Run Configuration
1010
+ * has been resolved. Carries the resolved {@link AgentTaskRunConfiguration} so
1011
+ * callers (e.g. the CLI error-report path) can forward model/effort to the
1012
+ * backend even when the run never produced a summary (issue #40).
1013
+ *
1014
+ * The original failure is preserved on `cause`. Only errors thrown after
1015
+ * `startSession` reported a *valid* configuration are wrapped — a
1016
+ * configuration-validation failure propagates unwrapped (there is nothing to
1017
+ * attach).
1018
+ */
1019
+ declare class AgentTaskRunError extends Error {
1020
+ readonly runConfiguration?: AgentTaskRunConfiguration;
1021
+ /** The original error that caused the run to fail (preserved for diagnostics). */
1022
+ readonly cause?: unknown;
1023
+ constructor(message: string, options: {
1024
+ runConfiguration?: AgentTaskRunConfiguration;
1025
+ cause?: unknown;
1026
+ });
1027
+ }
1028
+ type RunTaskOptions = {
1029
+ maxTurnsOverride?: number;
1030
+ tracing?: AgentTaskTraceOptions;
1031
+ /** LLM judge model config for `t.judge(...)` calls in the task checks. */
1032
+ judge?: JudgeConfig;
1033
+ onTurn?: (turnNumber: number, userAction: TaskTranscriptTurn["userAction"], agentResponse: unknown) => void;
1034
+ /**
1035
+ * Skip the internal `loadTask(taskDir)` call and reuse an already-loaded
1036
+ * task definition.
1037
+ *
1038
+ * The eval module's top level runs once per `loadTask` (it is copied to a
1039
+ * temp file and imported with all registries reset), so loading twice per
1040
+ * run breaks evals whose load-time behavior is not idempotent across module
1041
+ * systems (e.g. a helper that re-exports CJS `require()`-cached registrations
1042
+ * — the second import hits the cache and registers zero checks). Callers
1043
+ * that already need a `LoadedTask` for their own purposes (`runTaskDir`,
1044
+ * `runner-entry.ts`) should always pass it through here to keep the eval
1045
+ * import count at exactly one per run.
1046
+ *
1047
+ * When omitted, `runTask` calls `loadTask(taskDir)` itself. The `taskDir`
1048
+ * argument is still required for that fallback and is otherwise unused.
1049
+ */
1050
+ loaded?: LoadedTask;
1051
+ };
1052
+ declare function runTask(taskDir: string, options?: RunTaskOptions): Promise<TaskRunResult>;
1053
+
1054
+ type AgentTaskCliOptions = {
1055
+ task?: string;
1056
+ dir?: string;
1057
+ grep?: string;
1058
+ help?: boolean;
1059
+ };
1060
+ declare function parseAgentTaskCliArgs(argv: string[]): AgentTaskCliOptions;
1061
+ declare function runAgentTaskCli(argv: string[], cwd?: string): Promise<number>;
1062
+
1063
+ declare function discoverAgentTaskDirs(rootDir: string): string[];
1064
+
1065
+ type AgentTaskRuntime = {
1066
+ judge?: JudgeConfig;
1067
+ };
1068
+ type AgentTaskRunSummary = {
1069
+ taskDir: string;
1070
+ taskId: string;
1071
+ pass: boolean;
1072
+ checks: EvaluationItemResult[];
1073
+ /** Adapter that ran the task. Forwarded to the backend when recording locally. */
1074
+ adapterName?: string;
1075
+ /** Trace id this run claimed (when tracing was enabled). */
1076
+ traceRunId?: string;
1077
+ /** Deliverables the adapter produced. */
1078
+ deliverables?: Record<string, unknown>;
1079
+ /** Per-turn transcript of the run. */
1080
+ transcript?: Record<string, unknown>;
1081
+ /** adapter-reported model/effort, forwarded when recording locally. */
1082
+ runConfiguration?: AgentTaskRunConfiguration;
1083
+ };
1084
+ declare function loadTaskRuntime(_taskDir: string): Promise<AgentTaskRuntime>;
1085
+ declare function runTaskDir(taskDir: string): Promise<AgentTaskRunSummary>;
1086
+
1087
+ type DeliverableValidationResult = {
1088
+ results: EvaluationItemResult[];
1089
+ brokenDeliverables: Record<string, string>;
1090
+ };
1091
+
1092
+ /**
1093
+ * ApoSpanProcessor — an OpenTelemetry SpanProcessor that routes GenAI spans
1094
+ * into apo's Flow.
1095
+ *
1096
+ * This is the OTel-native tracing path. After calling {@link registerApoTracing},
1097
+ * any SDK that emits OTel spans (Vercel AI SDK, OpenAI Agents SDK, etc.)
1098
+ * will be traced automatically — no per-SDK wrapper needed.
1099
+ *
1100
+ * The processor is global, but run isolation is maintained via
1101
+ * {@link AsyncLocalStorage}: {@link getActiveApoRun} returns the current
1102
+ * run's trace context, so each span is routed to the correct run's FlowTee.
1103
+ *
1104
+ * **Design: eager creation, deferred completion.** The apo span is created
1105
+ * at `onStart` (when initial attributes like tool name and model are
1106
+ * available). The span is ended at `onEnd` with the final attributes
1107
+ * (results, tokens, text). This gives correct ordering: parent spans
1108
+ * are created before children, matching the real call hierarchy.
1109
+ *
1110
+ * @module
1111
+ */
1112
+ interface OTelSpanContext {
1113
+ spanId: string;
1114
+ traceId: string;
1115
+ traceFlags: number;
1116
+ }
1117
+ interface OTelSpanStatus {
1118
+ code: number;
1119
+ message?: string;
1120
+ }
1121
+ interface OTelReadableSpan {
1122
+ name: string;
1123
+ spanContext(): OTelSpanContext;
1124
+ parentSpanContext?: OTelSpanContext;
1125
+ attributes: Record<string, unknown>;
1126
+ status: OTelSpanStatus;
1127
+ duration: [number, number];
1128
+ startTime: [number, number];
1129
+ ended: boolean;
1130
+ }
1131
+ interface OTelSpan extends OTelReadableSpan {
1132
+ setAttribute(key: string, value: unknown): this;
1133
+ setAttributes(attributes: Record<string, unknown>): this;
1134
+ setStatus(status: OTelSpanStatus): this;
1135
+ recordException(exception: unknown): void;
1136
+ updateName(name: string): this;
1137
+ end(endTime?: unknown): void;
1138
+ isRecording(): boolean;
1139
+ }
1140
+ interface OTelContext {
1141
+ getValue(key: symbol): unknown;
1142
+ setValue(key: symbol, value: unknown): OTelContext;
1143
+ deleteValue(key: symbol): OTelContext;
1144
+ }
1145
+ interface OTelSpanProcessorLifecycle {
1146
+ forceFlush(): Promise<void>;
1147
+ shutdown(): Promise<void>;
1148
+ }
1149
+ /**
1150
+ * A SpanProcessor that translates GenAI OTel spans into apo observations.
1151
+ *
1152
+ * Spans are created at `onStart` (eager) and completed at `onEnd`
1153
+ * (deferred). This gives correct parent/child ordering — parent spans
1154
+ * are created before their children, matching the real call hierarchy.
1155
+ *
1156
+ * If a span isn't load-bearing (not a GenAI tool/generation span), no
1157
+ * apo span is created.
1158
+ */
1159
+ declare class ApoSpanProcessor implements OTelSpanProcessorLifecycle {
1160
+ private spanMap;
1161
+ onStart(span: OTelSpan, _parentContext: OTelContext): void;
1162
+ onEnd(span: OTelReadableSpan): void;
1163
+ forceFlush(): Promise<void>;
1164
+ shutdown(): Promise<void>;
1165
+ /** Clear all state — for testing. */
1166
+ reset(): void;
1167
+ }
1168
+
1169
+ /**
1170
+ * Registration API for the OTel-native tracing integration.
1171
+ *
1172
+ * Call {@link registerApoTracing} once at startup (e.g., in `startSession`
1173
+ * or at module load). After this, any SDK that emits OTel spans during a run
1174
+ * will be traced automatically — no per-SDK wrapper needed.
1175
+ *
1176
+ * @example
1177
+ * ```ts
1178
+ * import { registerApoTracing } from "@apo-ai/sdk/agent-task";
1179
+ *
1180
+ * async startSession(ctx) {
1181
+ * registerApoTracing(); // one line, once
1182
+ * return { async sendUserTurn(turn) { ... } };
1183
+ * }
1184
+ * ```
1185
+ *
1186
+ * @module
1187
+ */
1188
+
1189
+ interface RegisterApoTracingOptions {
1190
+ /**
1191
+ * Set as the global tracer provider (default: true).
1192
+ * Set to false if the host app has its own OTel setup and you want to
1193
+ * add apo's processor to an existing provider instead.
1194
+ */
1195
+ setGlobal?: boolean;
1196
+ }
1197
+ /**
1198
+ * Register apo's OTel SpanProcessor. Idempotent — calling it more than
1199
+ * once is a no-op.
1200
+ *
1201
+ * After registration, any OTel span created during an apo run (inside
1202
+ * `withApoRun`) that matches the GenAI conventions (`ai.*` or `gen_ai.*`)
1203
+ * is automatically translated and fed into the run's Flow.
1204
+ */
1205
+ declare function registerApoTracing(_options?: RegisterApoTracingOptions): Promise<void>;
1206
+ /**
1207
+ * Reset registration state — for testing only.
1208
+ */
1209
+ declare function resetApoTracing(): void;
1210
+
1211
+ /**
1212
+ * Per-run context propagation via Node's AsyncLocalStorage.
1213
+ *
1214
+ * The OTel {@link ApoSpanProcessor} is global — it doesn't know which run a
1215
+ * span belongs to. This module solves that: {@link withApoRun} establishes
1216
+ * the run's trace context on AsyncLocalStorage, and any code called during
1217
+ * the run (including the processor's `onStart`/`onEnd`) can read it via
1218
+ * {@link getActiveApoRun}.
1219
+ *
1220
+ * This is the same propagation mechanism OTel's own
1221
+ * `AsyncLocalStorageContextManager` uses under the hood.
1222
+ *
1223
+ * @module
1224
+ */
1225
+
1226
+ interface ApoRunContext {
1227
+ /** The tee'd trace context for this run. */
1228
+ trace: AgentTaskTraceContext;
1229
+ /** The parent span id (the run's root span or the current turn span). */
1230
+ parentSpanId?: string;
1231
+ /** The task id, surfaced in span metadata. */
1232
+ taskId?: string;
1233
+ /** The current turn number, surfaced in span metadata. */
1234
+ turnNumber?: number;
1235
+ }
1236
+ /**
1237
+ * Run `fn` with `ctx` as the active apo run context. Any code inside `fn`
1238
+ * (including across `await` boundaries) can call {@link getActiveApoRun}
1239
+ * to retrieve it.
1240
+ */
1241
+ declare function withApoRun<T>(ctx: ApoRunContext, fn: () => Promise<T>): Promise<T>;
1242
+ /**
1243
+ * Run `fn` synchronously with `ctx` as the active apo run context.
1244
+ */
1245
+ declare function withApoRunSync<T>(ctx: ApoRunContext, fn: () => T): T;
1246
+ /**
1247
+ * Get the active run context, or `undefined` if called outside a run.
1248
+ *
1249
+ * This is what the {@link ApoSpanProcessor} calls in `onStart`/`onEnd` to
1250
+ * know which run a span belongs to.
1251
+ */
1252
+ declare function getActiveApoRun(): ApoRunContext | undefined;
1253
+
1254
+ /**
1255
+ * canonical Task Revision manifest.
1256
+ *
1257
+ * Pure, dependency-free canonicalizer. Both this module and its Python twin
1258
+ * (`backend/apo/execution/task_revision_manifest.py`) MUST produce
1259
+ * byte-identical canonical JSON and digests for every fixture under
1260
+ * `specs/contracts/task-revision/v1/cases/`.
1261
+ *
1262
+ * Canonicalization rules:
1263
+ * - `/` path separators (caller `\` normalized to `/`);
1264
+ * - Unicode NFC normalization of the path;
1265
+ * - bytewise lexical ordering of normalized UTF-8 paths (NOT UTF-16 —
1266
+ * diverges for codepoints > U+007F);
1267
+ * - exact file-byte SHA-256 (lowercase hex);
1268
+ * - mode reduced to `regular` | `executable`;
1269
+ * - ownership, timestamps, inode metadata, and absolute paths excluded.
1270
+ *
1271
+ * `contentSha256` is the SHA-256 of the compact canonical JSON built from
1272
+ * `schemaVersion` and the sorted `files` array. The `summary` block is NOT
1273
+ * part of source identity.
1274
+ */
1275
+ type ModeClass = "regular" | "executable";
1276
+ interface ManifestFile {
1277
+ path: string;
1278
+ sizeBytes: number;
1279
+ sha256: string;
1280
+ modeClass: ModeClass;
1281
+ }
1282
+ interface TaskRevisionManifestV1 {
1283
+ schemaVersion: 1;
1284
+ files: ManifestFile[];
1285
+ summary: {
1286
+ fileCount: number;
1287
+ uncompressedSizeBytes: number;
1288
+ excludedFileCount: number;
1289
+ excludedDirectoryCount: number;
1290
+ };
1291
+ }
1292
+ /** Caller-reported file, before canonicalization. `content` is the exact bytes. */
1293
+ interface ManifestFileInput {
1294
+ path: string;
1295
+ modeClass: ModeClass;
1296
+ content: Uint8Array;
1297
+ }
1298
+ /**
1299
+ * Normalize a caller-reported path to canonical POSIX + NFC form.
1300
+ * Validation (no leading slash, no `.`/`..` segments) is the filesystem
1301
+ * walker's responsibility; this function only makes representation canonical.
1302
+ */
1303
+ declare function normalizeManifestPath(path: string): string;
1304
+ /** SHA-256 of exact bytes, lowercase hex. */
1305
+ declare function sha256Hex(bytes: Uint8Array): string;
1306
+ /**
1307
+ * Build the canonical V1 manifest from already-included files. Exclusion
1308
+ * counts default to zero; the filesystem walker supplies real counts when it
1309
+ * wraps this via `withExclusionCounts`.
1310
+ */
1311
+ declare function buildManifest(files: ManifestFileInput[], summary?: {
1312
+ excludedFileCount: number;
1313
+ excludedDirectoryCount: number;
1314
+ }): TaskRevisionManifestV1;
1315
+ /**
1316
+ * Compact canonical JSON over `schemaVersion` + the sorted `files` array only.
1317
+ * Object keys are alphabetical at every level; no whitespace; non-ASCII
1318
+ * emitted as UTF-8 (matches Python `json.dumps(sort_keys=True,
1319
+ * ensure_ascii=False, separators=(",", ":"))`). The `summary` block is
1320
+ * intentionally excluded from source identity.
1321
+ */
1322
+ declare function canonicalManifestJson(manifest: TaskRevisionManifestV1): string;
1323
+ /** SHA-256 (lowercase hex) of the canonical manifest JSON's UTF-8 bytes. */
1324
+ declare function contentSha256(manifest: TaskRevisionManifestV1): string;
1325
+
1326
+ export { type AISDKResult, type AdapterDefinition, type AdapterRuntimeState, type AdapterSession, type AgentTaskCliOptions, type AgentTaskRunConfiguration, AgentTaskRunError, type AgentTaskRunSummary, type AgentTaskRuntime, AgentTaskTraceContext, AgentTaskTraceOptions, type AgentTurnResult, type AnthropicMessage, type ApoRunContext, ApoSpanProcessor, type ArtifactUploadConfig, type CheckContext, type CheckGroup, type CleanupContext, type CollectDeliverablesContext, type CollectedDeliverables, type DeliverableDefinition, type DeliverableSummary, type DeliverableValidationResult, type DescribeRegistration, type FileArtifact, type FileArtifactOptions, type FileEntry, type Flow, type FlowEvent, FlowView, type InitializeContext, type JudgeConfig, type LoadedTask, type ManifestFile, type ManifestFileInput, type Matcher, type ModeClass, type NameMatcher, type OpenAIMessage, type PreparedTaskResult, type RegisterApoTracingOptions, type RunTaskOptions, type StartSessionContext, TEST_METHOD_NAMES, type TaskConfig, type TaskDefinition, type TaskEvaluationResult, TaskFiles, type TaskRevisionManifestV1, type TaskRunResult, type TaskScope, type TaskTranscript, type TaskTranscriptTurn, type TestContext, type TestRegistration, type ToolCallOptions, type ToolCallStatus, type TurnContext, type TurnFn, type TurnRecord, type TypedAdapterDefinition, type ValueMatcher, buildManifest, canonicalManifestJson, contentSha256, defineAdapter, defineTask, describe, discoverAgentTaskDirs, equals, fileArtifact, filePaths, fromAISDK, fromAnthropicMessages, fromOpenAIMessages, getActiveApoRun, getTaskTurn, includes, isFileArtifact, loadTask, loadTaskRuntime, matchValue, matches, normalizeManifestPath, parseAgentTaskCliArgs, persistFileArtifacts, registerApoTracing, resetApoTracing, resetFlowChecks, resetTaskRegistry, resetTaskTurn, resolveTurn, runAgentTaskCli, runTask, runTaskDir, satisfies, sha256Hex, similarity, task, defineCheck as test, turn, withApoRun, withApoRunSync };