@apo-ai/sdk 0.9.4 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- import { A as AgentTaskTraceContext } from '../../tracing-Dqbuc7AK.js';
1
+ import { A as AgentTaskTraceContext } from '../../tracing-DoIGvJW1.js';
2
2
 
3
3
  /**
4
4
  * Vercel AI SDK tracing integration for apo.
@@ -2,7 +2,7 @@
2
2
  // Do not edit by hand — edit the src/ originals and rebuild.
3
3
  import {
4
4
  createApoTracer
5
- } from "../../chunk-IK4GJX7H.js";
5
+ } from "../../chunk-OZWOLCJ5.js";
6
6
  import "../../chunk-HRYXLRXJ.js";
7
7
  export {
8
8
  createApoTracer
@@ -1,4 +1,4 @@
1
- import { A as AgentTaskTraceContext } from '../../tracing-Dqbuc7AK.js';
1
+ import { A as AgentTaskTraceContext } from '../../tracing-DoIGvJW1.js';
2
2
 
3
3
  /**
4
4
  * Anthropic JS SDK tracing integration for apo.
@@ -2,8 +2,8 @@
2
2
  // Do not edit by hand — edit the src/ originals and rebuild.
3
3
  import {
4
4
  createApoAnthropic
5
- } from "../../chunk-43OI6GP7.js";
6
- import "../../chunk-3OQ3JBFJ.js";
5
+ } from "../../chunk-6H2GACBG.js";
6
+ import "../../chunk-Z64A566W.js";
7
7
  export {
8
8
  createApoAnthropic
9
9
  };
@@ -1,4 +1,4 @@
1
- import { A as AgentTaskTraceContext } from '../../tracing-Dqbuc7AK.js';
1
+ import { A as AgentTaskTraceContext } from '../../tracing-DoIGvJW1.js';
2
2
 
3
3
  /**
4
4
  * OpenAI JS SDK tracing integration for apo.
@@ -2,8 +2,8 @@
2
2
  // Do not edit by hand — edit the src/ originals and rebuild.
3
3
  import {
4
4
  createApoOpenAI
5
- } from "../../chunk-3B3L2H4Y.js";
6
- import "../../chunk-3OQ3JBFJ.js";
5
+ } from "../../chunk-P43KL5AR.js";
6
+ import "../../chunk-Z64A566W.js";
7
7
  export {
8
8
  createApoOpenAI
9
9
  };
@@ -1,4 +1,4 @@
1
- import { A as AgentTaskTraceContext, a as AgentTaskTraceOptions } from '../tracing-Dqbuc7AK.js';
1
+ import { A as AgentTaskTraceContext, a as AgentTaskTraceOptions } from '../tracing-DoIGvJW1.js';
2
2
  export { CreateApoTracerOptions, createApoTracer } from './integrations/ai-sdk.js';
3
3
  export { CreateApoOpenAIOptions, createApoOpenAI } from './integrations/openai.js';
4
4
  export { CreateApoAnthropicOptions, createApoAnthropic } from './integrations/anthropic.js';
@@ -131,6 +131,37 @@ interface TraceProjectionSnapshot {
131
131
  * the drift where concurrent tools appeared in completion order.
132
132
  */
133
133
 
134
+ /** One mid-run steer, derived from a `task.steer` observation. */
135
+ interface TraceSteer {
136
+ /** 1-based, registration order across the run. */
137
+ number: number;
138
+ /** Scripted turn the steer targeted. */
139
+ turn: number;
140
+ label?: string;
141
+ /**
142
+ * The serialized trigger (`{"toolResults":2}`, `"runStart"`). Records
143
+ * carry the trigger as an object — the local tee embeds the SteerRecord
144
+ * directly and the canonical path re-parses its JSON — while the event
145
+ * metadata carries the pre-stringified form; both render identically.
146
+ */
147
+ trigger: string;
148
+ message: unknown;
149
+ status: "delivered" | "undelivered" | "error";
150
+ /** ISO timestamp of successful delivery (wall clock, display only). */
151
+ deliveredAt?: string;
152
+ /**
153
+ * The steer observation's own `startedAt` — the snapshot-clock position of
154
+ * the delivery. Window/delivery comparisons use THIS, not `deliveredAt`:
155
+ * local tee snapshots timestamp with a monotonic clock (performance.now
156
+ * epoch), so a wall-clock deliveredAt is not comparable to sibling
157
+ * observations, while the steer observation's own timestamp always is.
158
+ */
159
+ spanStartedAt?: string;
160
+ /** The steer observation's span id — the deterministic tie-breaker. */
161
+ spanId: string;
162
+ boundary?: string;
163
+ reason?: string;
164
+ }
134
165
  /** A tool call derived from a `TOOL` observation. */
135
166
  interface TraceToolCall {
136
167
  spanId: string;
@@ -199,6 +230,29 @@ declare class TraceView {
199
230
  get skillLoads(): readonly TraceSkillLoad[];
200
231
  /** Subagent delegations derived from `AGENT` observations, in invocation order. */
201
232
  get subagentCalls(): readonly TraceSubagentCall[];
233
+ /**
234
+ * Mid-run steers, derived from `task.steer` observations, in number order.
235
+ * Empty when the trace carries none — steering assertions fail closed on
236
+ * the empty case instead of passing vacuously.
237
+ */
238
+ get steers(): readonly TraceSteer[];
239
+ /**
240
+ * Whether an observation started after the steer observation, using the
241
+ * same ordering key as everywhere else in this class (startedAt, then
242
+ * span id). The id tie-break matters on the local tee path, whose ISO
243
+ * timestamps truncate to milliseconds — a steer and the generation that
244
+ * consumes it can share one.
245
+ */
246
+ private static startedAfter;
247
+ /**
248
+ * A TraceView over the run after steer n was delivered: observations that
249
+ * started after the steer's delivery timestamp (the first generation that
250
+ * consumed it through run end). Missing timestamps sort after timestamped
251
+ * observations per the projection contract, so they belong to the window
252
+ * too. An undelivered steer yields an empty view — assertions inside it
253
+ * fail on missing evidence rather than passing vacuously.
254
+ */
255
+ windowAfterSteer(n: number): TraceView;
202
256
  /** Last assistant message content — the agent's "reply". Empty if none. */
203
257
  get reply(): string;
204
258
  /**
@@ -348,6 +402,38 @@ type CollectedDeliverables = Record<string, unknown>;
348
402
  type AgentTurnResult = {
349
403
  response: unknown;
350
404
  };
405
+ /**
406
+ * Live progress signal from the agent under test, fired while a turn runs.
407
+ * The adapter forwards these from its harness's event stream via the
408
+ * `notifyAgentEvent` callback on `sendUserTurn`'s context — the runner cannot
409
+ * observe subprocess spans live (the projection tee only sees spans created
410
+ * through the runner's trace context), so the adapter is the source of
411
+ * progress. `run_start` MUST be the first event a turn emits.
412
+ */
413
+ type AgentProgressEvent = {
414
+ kind: "run_start";
415
+ } | {
416
+ kind: "tool_result";
417
+ toolName?: string;
418
+ } | {
419
+ kind: "assistant_reply";
420
+ };
421
+ /** Where the harness actually queued/landed an injected message. */
422
+ type SteerDeliveryBoundary = "run_start" | "tool_results" | "assistant_reply";
423
+ type SteerResult = {
424
+ /**
425
+ * The boundary the harness reports for the injection. Harness truth wins
426
+ * over the scheduler's trigger intent — a `{ toolResults: 2 }` steer may
427
+ * land at `assistant_reply` on a harness that drains its queue there.
428
+ */
429
+ boundary: SteerDeliveryBoundary;
430
+ };
431
+ /** Context handed to `session.steer()`. */
432
+ type SteerContext = {
433
+ trace: AgentTaskTraceContext;
434
+ turnNumber: number;
435
+ steerNumber: number;
436
+ };
351
437
  /**
352
438
  * The adapter-reported identity of the agent under test for one Task Run
353
439
  * The adapter resolves its configuration (env vars, aliases,
@@ -431,7 +517,22 @@ type AdapterSession = {
431
517
  trace: AgentTaskTraceContext;
432
518
  turnNumber: number;
433
519
  parentSpanId?: string;
520
+ /**
521
+ * The adapter calls this for every agent progress event while the turn
522
+ * runs. Fire-and-forget (sync void); the runner serializes internally
523
+ * and never lets scheduler errors break the turn. MUST be called with
524
+ * `{ kind: "run_start" }` before the first model call.
525
+ */
526
+ notifyAgentEvent: (event: AgentProgressEvent) => void;
434
527
  }) => Promise<AgentTurnResult>;
528
+ /**
529
+ * Mid-run injection primitive. Optional = negotiated capability: a task
530
+ * with registered steers fails closed before turn 1 when this is absent.
531
+ * Resolve when the harness has accepted/queued the message, not when the
532
+ * agent has reacted to it — reaction is asserted afterward by
533
+ * `t.steerDelivered` / `t.afterSteer`.
534
+ */
535
+ steer?: (input: unknown, context: SteerContext) => Promise<SteerResult>;
435
536
  close?: () => Promise<void>;
436
537
  };
437
538
  type AdapterDefinition = {
@@ -449,6 +550,47 @@ type TypedAdapterDefinition<TName extends string, TDeliverables extends Record<s
449
550
  collectDeliverables: (ctx: CollectDeliverablesContext) => Promise<TCollected>;
450
551
  };
451
552
 
553
+ /**
554
+ * Mid-run steering registration — the task-plane half of steering.
555
+ *
556
+ * `steer({...})` is called at eval-file top level next to `turn()`: it
557
+ * declares that a user message will be injected into a turn that is already
558
+ * running, at a boundary described relative to that turn's progress (after
559
+ * its n-th tool result, its n-th assistant reply, or right at run start).
560
+ *
561
+ * Registration mechanics mirror `turn.ts`: a module-global array under a
562
+ * `Symbol.for` key so the registry survives the eval module being imported
563
+ * from a temp path, reset by `loadTask` before each import and read by the
564
+ * runner's turn loop afterwards.
565
+ */
566
+ type SteerTrigger =
567
+ /** After the n-th completed tool result of the target turn (1-based). */
568
+ {
569
+ toolResults: number;
570
+ }
571
+ /** After the n-th completed assistant message of the target turn (1-based). */
572
+ | {
573
+ assistantReply: number;
574
+ }
575
+ /** Immediately after the turn's user input is handed to the agent. */
576
+ | "runStart";
577
+ /** One scripted steer, registered from the eval file (or legacy checks.ts). */
578
+ type SteerSpec = {
579
+ when: SteerTrigger;
580
+ /** The injected user message — same shape a `turn()` return value has. */
581
+ message: unknown;
582
+ /** Human label shown in the trace, transcript, and check failures. */
583
+ label?: string;
584
+ /** 1-based scripted turn this steer belongs to. Default 1. */
585
+ turn?: number;
586
+ };
587
+ /** Register a mid-run steer. Call at eval-file top level, next to turn(). */
588
+ declare function steer(spec: SteerSpec): void;
589
+ /** All steers registered so far, in registration order (runner use). */
590
+ declare function getTaskSteers(): SteerSpec[];
591
+ /** Clear registrations (loadTask use — mirrors resetTaskTurn). */
592
+ declare function resetTaskSteers(): void;
593
+
452
594
  /**
453
595
  * The outcome of a single assertion.
454
596
  *
@@ -524,6 +666,13 @@ type JudgeMetadata = {
524
666
  * as extra signals per check, not a second vote.
525
667
  */
526
668
  secondJudge?: SecondJudgeEvidence;
669
+ /**
670
+ * Present only when the second judge's verdict decided this check (cascade
671
+ * mode, `judge.mode: "cascade"`): the primary LLM judge was not called.
672
+ * Absent = the primary judge decided. The `model` field above then names
673
+ * the decision model, and `response` carries its typed verdict.
674
+ */
675
+ verdict_by?: "second-judge";
527
676
  };
528
677
  /**
529
678
  * Evidence from the second grader. `error` is set when the decisions
@@ -766,10 +915,36 @@ type TaskEvaluationResult = {
766
915
  type TaskTranscript = {
767
916
  turns: TaskTranscriptTurn[];
768
917
  };
918
+ /**
919
+ * Outcome of one scripted steer. Steers never fail the run by themselves —
920
+ * a dropped or errored steer is recorded here and on the trace, and
921
+ * `t.steerDelivered` is what turns it red.
922
+ */
923
+ type SteerRecord = {
924
+ /** 1-based, registration order across the whole run. */
925
+ number: number;
926
+ /** Scripted turn the steer targeted. */
927
+ turn: number;
928
+ label?: string;
929
+ trigger: SteerTrigger;
930
+ message: unknown;
931
+ status: "delivered" | "undelivered" | "error";
932
+ /** ISO timestamp of successful delivery. */
933
+ deliveredAt?: string;
934
+ boundary?: SteerDeliveryBoundary;
935
+ /**
936
+ * Why undelivered ("turn ended before trigger fired", "no progress events
937
+ * observed for this turn", "target turn never ran"), or the injection
938
+ * error message when status === "error".
939
+ */
940
+ reason?: string;
941
+ };
769
942
  type TaskTranscriptTurn = {
770
943
  turnNumber: number;
771
944
  userAction: unknown;
772
945
  agentResponse: unknown;
946
+ /** Steers that targeted this turn, in steerNumber order. */
947
+ steers?: SteerRecord[];
773
948
  };
774
949
  type TaskRunResult = {
775
950
  task: TaskDefinition;
@@ -1044,6 +1219,17 @@ type AgentJudgeOptions = {
1044
1219
  };
1045
1220
  budget?: AgentBudget;
1046
1221
  label?: string;
1222
+ /**
1223
+ * The ground-truth verdict this check asserts: `"pass"` (default) or
1224
+ * `"fail"`. The session never sees it — the instruction is sent unchanged,
1225
+ * so the judge still decides independently. What changes is the recorded
1226
+ * outcome: the check passes only when the judge's verdict matches the
1227
+ * author's ground truth. `"fail"` is how a known-bad deliverable (a
1228
+ * fabricated figure, a claim the trace contradicts) becomes a passing
1229
+ * check when the judge catches it — judging the judge against cases whose
1230
+ * correct verdict is fixed by construction.
1231
+ */
1232
+ expect?: "pass" | "fail";
1047
1233
  };
1048
1234
  /** The local evidence plane a session can investigate. */
1049
1235
  type AgentEvidence = {
@@ -1107,6 +1293,13 @@ type JudgeConfig = {
1107
1293
  * {@link JudgePromptBuilder}.
1108
1294
  */
1109
1295
  prompt?: JudgePromptBuilder;
1296
+ /**
1297
+ * `"cascade"`: the second judge (decision model, `APO_SECOND_JUDGE_MODEL`)
1298
+ * answers first and a confident verdict (native confidence ≥ 0.95) stands
1299
+ * without calling the primary judge — the cost tier. Unset = the default
1300
+ * dual behavior (second judge as parallel, evidence-only shadow).
1301
+ */
1302
+ mode?: "cascade";
1110
1303
  };
1111
1304
  /**
1112
1305
  * What a check knows about its scope when it invokes the judge — the
@@ -1152,6 +1345,20 @@ interface TestContext {
1152
1345
  minTokens(n: number, opts?: TokenBudgetOptions): void;
1153
1346
  /** Escape hatch for a named predicate over the complete normalized run. */
1154
1347
  assert(label: string, predicate: (flow: TraceView) => boolean): void;
1348
+ /**
1349
+ * Asserts steer n was delivered AND consumed: at least one generation
1350
+ * observation started after the steer's delivery timestamp. Fails closed —
1351
+ * a trace with no steering evidence at all is a failure, never a pass.
1352
+ */
1353
+ steerDelivered(n: number): void;
1354
+ /**
1355
+ * Scopes trace-based assertions to what the agent did after steer n: the
1356
+ * fn receives a TestContext over `view.windowAfterSteer(n)`, and every
1357
+ * existing matcher works unchanged inside it. When the steer is missing
1358
+ * from the trace, records one failure and skips fn (inner assertions must
1359
+ * not run — they would pass vacuously against an empty window).
1360
+ */
1361
+ afterSteer(n: number, fn: (t: TestContext) => void): void;
1155
1362
  /** Grades any value (a deliverable, parsed JSON, anything) with a matcher. */
1156
1363
  check<T>(value: T, matcher: Matcher<T>, label?: string): void;
1157
1364
  /**
@@ -1170,6 +1377,11 @@ interface TestContext {
1170
1377
  * only, when the full value exceeds its context limit: the primary judge
1171
1378
  * still sees all of ``value``, so other criteria that need the complete
1172
1379
  * deliverable are unaffected.
1380
+ *
1381
+ * ``opts.expect`` pins the ground-truth verdict (``"pass"`` default, or
1382
+ * ``"fail"``): the judge still decides independently, but the check passes
1383
+ * only when its verdict matches — how a known-bad value yields a passing
1384
+ * check exactly when the judge catches it.
1173
1385
  */
1174
1386
  judge(values: unknown | unknown[], instruction: string, opts?: {
1175
1387
  label?: string;
@@ -1181,6 +1393,14 @@ interface TestContext {
1181
1393
  * the second judge sees exactly what the primary judge sees.
1182
1394
  */
1183
1395
  secondJudgeValue?: unknown | unknown[];
1396
+ /**
1397
+ * The ground-truth verdict this check asserts: `"pass"` (default) or
1398
+ * `"fail"`. The judge never sees it — the instruction is sent
1399
+ * unchanged. The check passes only when the judge's verdict matches
1400
+ * the author's ground truth, so a known-bad value becomes a passing
1401
+ * check exactly when the judge catches it.
1402
+ */
1403
+ expect?: "pass" | "fail";
1184
1404
  }): Promise<void>;
1185
1405
  /**
1186
1406
  * Agentic judge: a tool-using LLM session that investigates the run's own
@@ -1195,6 +1415,10 @@ interface TestContext {
1195
1415
  * 2 MiB read) and fail-closed: a session that ends without a verdict is a
1196
1416
  * recorded failure, never a silent pass. Costs more and runs minutes, not
1197
1417
  * seconds — reserve it for rubrics that need investigation.
1418
+ *
1419
+ * ``opts.expect`` pins the ground-truth verdict (``"pass"`` default, or
1420
+ * ``"fail"``): the judge still investigates and decides independently, but
1421
+ * the check passes only when its verdict matches.
1198
1422
  */
1199
1423
  agent(instruction: string, opts?: AgentJudgeOptions): Promise<void>;
1200
1424
  }
@@ -1210,7 +1434,7 @@ interface TestContext {
1210
1434
  * adding a method to the interface without registering it here is a type
1211
1435
  * error, and vice versa. See ``tests/test-context-methods.test.ts``.
1212
1436
  */
1213
- declare const TEST_METHOD_NAMES: readonly ["calledTool", "notCalledTool", "toolOrder", "usedNoTools", "maxToolCalls", "noFailedActions", "loadedSkill", "calledSubagent", "messageIncludes", "maxTurns", "maxDurationMs", "maxTokens", "minTokens", "assert", "check", "judge", "agent"];
1437
+ declare const TEST_METHOD_NAMES: readonly ["calledTool", "notCalledTool", "toolOrder", "usedNoTools", "maxToolCalls", "noFailedActions", "loadedSkill", "calledSubagent", "messageIncludes", "maxTurns", "maxDurationMs", "maxTokens", "minTokens", "steerDelivered", "afterSteer", "assert", "check", "judge", "agent"];
1214
1438
 
1215
1439
  type TaskDefinition<TAdapterName extends string = string, TDeliverable extends string = string> = {
1216
1440
  id: string;
@@ -1365,15 +1589,6 @@ type TaskScope<TDeliverables> = {
1365
1589
  * test("reviewed-methodically", (t, { deliverables }) => {
1366
1590
  * t.check(deliverables.result, includes("finding"));
1367
1591
  * });
1368
- *
1369
- * // A task that needs dev-machine resources (cloud creds, VPC, stage) can
1370
- * // declare execution: "local" so `apo task run` runs it on the caller's
1371
- * // machine while still recording a backend run row.
1372
- * task("bind-e2e", {
1373
- * adapter: bindAdapter,
1374
- * deliverables: ["summary"],
1375
- * execution: "local",
1376
- * });
1377
1592
  * ```
1378
1593
  */
1379
1594
  declare function task<const TTaskId extends string, const TAdapterName extends string, const TDeliverableDefs extends Record<string, DeliverableDefinition>, TCollected extends CollectedDeliverables, const TSelected extends readonly (DeliverableKey<TCollected> & keyof TDeliverableDefs)[]>(name: TTaskId, config: Omit<TaskConfig<TSelected[number] & string>, "id" | "checks" | "deliverables"> & {
@@ -1497,6 +1712,65 @@ interface PreparedTaskResult {
1497
1712
  */
1498
1713
  declare function persistFileArtifacts(deliverables: Record<string, unknown>, config: ArtifactUploadConfig): Promise<PreparedTaskResult>;
1499
1714
 
1715
+ /**
1716
+ * Second grader: a typed-decision model (System One / Jev) that grades the
1717
+ * same deliverable + instruction as the primary LLM judge, in parallel.
1718
+ *
1719
+ * Why a second grader at all: the primary judge's verdict is the product,
1720
+ * but its trustworthiness is invisible. A non-generative decision model
1721
+ * cannot fabricate quoted evidence the way text judges measurably do, and
1722
+ * its calibrated confidence acts as a difficulty meter — so agreement +
1723
+ * confidence turn every check into self-describing evidence (auto-verified /
1724
+ * borderline / disputed) at ~1/100th of a cent per call.
1725
+ *
1726
+ * The second opinion NEVER changes the verdict. It is opt-in
1727
+ * (`APO_SECOND_JUDGE_MODEL`), runs on a decisions endpoint (not
1728
+ * chat/completions — decision models reject it), and any failure is
1729
+ * recorded as `error` on the evidence instead of affecting the check.
1730
+ *
1731
+ * Providers: every decisions endpoint speaks its own request/response
1732
+ * dialect, selected via `APO_SECOND_JUDGE_PROVIDER` (or auto-detected from
1733
+ * the base URL). Adding a provider means adding one object to `PROVIDERS`
1734
+ * — nothing else in this file or the caller changes.
1735
+ */
1736
+
1737
+ /** How a recorded second-judge opinion relates to the check's verdict. */
1738
+ type SecondJudgeFacts = {
1739
+ kind: "none";
1740
+ } | {
1741
+ kind: "error";
1742
+ } | {
1743
+ kind: "skipped";
1744
+ } | {
1745
+ kind: "split";
1746
+ confidence: number;
1747
+ } | {
1748
+ kind: "agree";
1749
+ confidence: number;
1750
+ } | {
1751
+ kind: "unsure";
1752
+ confidence: number;
1753
+ };
1754
+ /**
1755
+ * Structural input for {@link secondJudgeFacts}: any check shape carrying a
1756
+ * verdict and optional judge evidence — the SDK's own results and the wire
1757
+ * types the CLI and dashboard read from the backend are all assignable.
1758
+ */
1759
+ type SecondJudgeFactsInput = {
1760
+ pass: boolean;
1761
+ judge?: {
1762
+ secondJudge?: {
1763
+ choice?: "pass" | "fail";
1764
+ confidence?: number;
1765
+ skipped?: string;
1766
+ error?: string;
1767
+ };
1768
+ };
1769
+ };
1770
+ declare function secondJudgeFacts(check: SecondJudgeFactsInput): SecondJudgeFacts;
1771
+ /** One honest sentence about the relation; null when corroborated. */
1772
+ declare function secondJudgeTakeaway(check: SecondJudgeFactsInput): string | null;
1773
+
1500
1774
  /**
1501
1775
  * Check submission compaction (issue #175).
1502
1776
  *
@@ -1702,6 +1976,8 @@ type RunTaskOptions = {
1702
1976
  */
1703
1977
  judgeTools?: JudgeToolsConfig;
1704
1978
  onTurn?: (turnNumber: number, userAction: TaskTranscriptTurn["userAction"], agentResponse: unknown) => void;
1979
+ /** Called for every steer outcome (delivered / undelivered / error). */
1980
+ onSteer?: (record: SteerRecord) => void;
1705
1981
  /**
1706
1982
  * Skip the internal `loadTask(taskDir)` call and reuse an already-loaded
1707
1983
  * task definition.
@@ -1722,6 +1998,20 @@ type RunTaskOptions = {
1722
1998
  };
1723
1999
  declare function runTask(taskDir: string, options?: RunTaskOptions): Promise<TaskRunResult>;
1724
2000
 
2001
+ /**
2002
+ * Issue #8: shown when a run ends with zero registered checks. A bare
2003
+ * `FAIL <task>` with no Checks section looked like a real failure but was
2004
+ * almost always a silent registration bug (e.g. a double-import that wiped
2005
+ * the check registry). Naming `test()` matches the documented registration
2006
+ * function — `apps/docs` reference/task.md.
2007
+ *
2008
+ * Lives next to {@link aggregateResult} because the empty-checks verdict is
2009
+ * the single source of truth for "why did this run fail with no checks?".
2010
+ * The CLI and the standalone e2e runner both import this so the wording
2011
+ * stays in one place.
2012
+ */
2013
+ declare const NO_CHECKS_REGISTERED_MESSAGE = "No tests were registered by the eval module \u2014 a task must define at least one test().";
2014
+
1725
2015
  /**
1726
2016
  * Issue #159: replay Phase 2 of a completed Run against its stored
1727
2017
  * Deliverables — without re-running the agent.
@@ -2309,4 +2599,4 @@ type TranscriptCaptureResult = {
2309
2599
  };
2310
2600
  declare function replayAdapterTranscript(capture: AdapterTranscriptCapture, options: ReplayAdapterTranscriptOptions): Promise<TranscriptCaptureResult>;
2311
2601
 
2312
- export { type AISDKResult, type AdapterDefinition, type AdapterRuntimeState, type AdapterSession, type AdapterTranscriptCapture, type AgentBudget, type AgentEvidence, type AgentJudgeOptions, type AgentJudgeOutcome, type AgentJudgeSession, type AgentJudgeStep, type AgentTaskCliOptions, type AgentTaskRunConfiguration, AgentTaskRunError, type AgentTaskRunSummary, type AgentTaskRuntime, AgentTaskTraceContext, AgentTaskTraceOptions, type AgentTurnResult, type AnthropicMessage, type ApoRunContext, ApoSpanProcessor, type ArtifactUploadConfig, type CheckCompaction, type CheckContext, type CheckGroup, type CleanupContext, type CollectDeliverablesContext, type CollectedDeliverables, type DeliverableDefinition, type DeliverableSummary, type DeliverableValidationResult, type DescribeRegistration, type DurationBudgetOptions, type EvidenceFingerprint, type ExportOtlpTracesOptions, type FileArtifact, type FileArtifactOptions, type FileEntry, type Flow, type FlowEvent, FlowView, type InitializeContext, JUDGE_SEGMENT_LIMIT, type JudgeConfig, type JudgeContext, type JudgePromptBuilder, type JudgeScope, type JudgeToolsConfig, type LoadedTask, type ManifestFile, type ManifestFileInput, type Matcher, type McpHttpTransport, type McpServerConfig, type McpStdioTransport, type ModeClass, type NameMatcher, type OpenAIMessage, type OtlpAttribute, type OtlpAttributeValue, type OtlpSpan, type OtlpTracesPayload, type ParsedTranscriptSession, type PreparedTaskResult, RECEIVED_VALUE_LIMIT, type RegisterApoTracingOptions, type RejudgeCheckStability, type RejudgeEndpoints, RejudgeError, type RejudgeOptions, type RejudgeOutcome, type ReplayAdapterTranscriptOptions, type RunTaskOptions, type StartSessionContext, TEST_METHOD_NAMES, type TaskConfig, type TaskDefinition, type TaskEvaluationResult, TaskFiles, type TaskRevisionManifestV1, type TaskRunResult, type TaskScope, type TaskTranscript, type TaskTranscriptTurn, type TestContext, type TestRegistration, type TokenBudgetOptions, type ToolCallOptions, type ToolCallStatus, type TranscriptCaptureResult, TranscriptReplayError, type TranscriptReplayOptions, type TranscriptSource, type TranscriptToolCall, type TranscriptTurn, type TranscriptUsage, type TurnContext, type TurnFn, type TurnRecord, type TypedAdapterDefinition, type ValueMatcher, buildApoAuthHeaders, buildManifest, canonicalManifestJson, compactChecksForSubmission, connectMcpServers, contentSha256, defineAdapter, defineTask, describe, detectTranscriptSource, discoverAgentTaskDirs, equals, expandSecretPlaceholders, exportOtlpTraces, fileArtifact, filePaths, fromAISDK, fromAnthropicMessages, fromOpenAIMessages, getActiveApoRun, getTaskTurn, includes, isFileArtifact, loadTask, loadTaskRuntime, matchValue, matches, normalizeManifestPath, parseAgentTaskCliArgs, parseClaudeCodeTranscript, parseCodexTranscript, persistFileArtifacts, registerApoTracing, rejudgeTaskRun, replayAdapterTranscript, resetApoTracing, resetFlowChecks, resetTaskRegistry, resetTaskTurn, resolveMcpServerPaths, resolveOtlpTracesUrl, resolveTurn, runAgentTaskCli, runTask, runTaskDir, satisfies, sha256Hex, similarity, task, defineCheck as test, transcriptSessionToOtlp, turn, withApoRun, withApoRunSync };
2602
+ export { type AISDKResult, type AdapterDefinition, type AdapterRuntimeState, type AdapterSession, type AdapterTranscriptCapture, type AgentBudget, type AgentEvidence, type AgentJudgeOptions, type AgentJudgeOutcome, type AgentJudgeSession, type AgentJudgeStep, type AgentProgressEvent, type AgentTaskCliOptions, type AgentTaskRunConfiguration, AgentTaskRunError, type AgentTaskRunSummary, type AgentTaskRuntime, AgentTaskTraceContext, AgentTaskTraceOptions, type AgentTurnResult, type AnthropicMessage, type ApoRunContext, ApoSpanProcessor, type ArtifactUploadConfig, type CheckCompaction, type CheckContext, type CheckGroup, type CleanupContext, type CollectDeliverablesContext, type CollectedDeliverables, type DeliverableDefinition, type DeliverableSummary, type DeliverableValidationResult, type DescribeRegistration, type DurationBudgetOptions, type EvidenceFingerprint, type ExportOtlpTracesOptions, type FileArtifact, type FileArtifactOptions, type FileEntry, type Flow, type FlowEvent, FlowView, type InitializeContext, JUDGE_SEGMENT_LIMIT, type JudgeConfig, type JudgeContext, type JudgePromptBuilder, type JudgeScope, type JudgeToolsConfig, type LoadedTask, type ManifestFile, type ManifestFileInput, type Matcher, type McpHttpTransport, type McpServerConfig, type McpStdioTransport, type ModeClass, NO_CHECKS_REGISTERED_MESSAGE, type NameMatcher, type OpenAIMessage, type OtlpAttribute, type OtlpAttributeValue, type OtlpSpan, type OtlpTracesPayload, type ParsedTranscriptSession, type PreparedTaskResult, RECEIVED_VALUE_LIMIT, type RegisterApoTracingOptions, type RejudgeCheckStability, type RejudgeEndpoints, RejudgeError, type RejudgeOptions, type RejudgeOutcome, type ReplayAdapterTranscriptOptions, type RunTaskOptions, type SecondJudgeFacts, type SecondJudgeFactsInput, type StartSessionContext, type SteerContext, type SteerDeliveryBoundary, type SteerRecord, type SteerResult, type SteerSpec, type SteerTrigger, TEST_METHOD_NAMES, type TaskConfig, type TaskDefinition, type TaskEvaluationResult, TaskFiles, type TaskRevisionManifestV1, type TaskRunResult, type TaskScope, type TaskTranscript, type TaskTranscriptTurn, type TestContext, type TestRegistration, type TokenBudgetOptions, type ToolCallOptions, type ToolCallStatus, type TranscriptCaptureResult, TranscriptReplayError, type TranscriptReplayOptions, type TranscriptSource, type TranscriptToolCall, type TranscriptTurn, type TranscriptUsage, type TurnContext, type TurnFn, type TurnRecord, type TypedAdapterDefinition, type ValueMatcher, buildApoAuthHeaders, buildManifest, canonicalManifestJson, compactChecksForSubmission, connectMcpServers, contentSha256, defineAdapter, defineTask, describe, detectTranscriptSource, discoverAgentTaskDirs, equals, expandSecretPlaceholders, exportOtlpTraces, fileArtifact, filePaths, fromAISDK, fromAnthropicMessages, fromOpenAIMessages, getActiveApoRun, getTaskSteers, getTaskTurn, includes, isFileArtifact, loadTask, loadTaskRuntime, matchValue, matches, normalizeManifestPath, parseAgentTaskCliArgs, parseClaudeCodeTranscript, parseCodexTranscript, persistFileArtifacts, registerApoTracing, rejudgeTaskRun, replayAdapterTranscript, resetApoTracing, resetFlowChecks, resetTaskRegistry, resetTaskSteers, resetTaskTurn, resolveMcpServerPaths, resolveOtlpTracesUrl, resolveTurn, runAgentTaskCli, runTask, runTaskDir, satisfies, secondJudgeFacts, secondJudgeTakeaway, sha256Hex, similarity, steer, task, defineCheck as test, transcriptSessionToOtlp, turn, withApoRun, withApoRunSync };