@apo-ai/sdk 0.9.4 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-task/integrations/ai-sdk.d.ts +1 -1
- package/dist/agent-task/integrations/ai-sdk.js +1 -1
- package/dist/agent-task/integrations/anthropic.d.ts +1 -1
- package/dist/agent-task/integrations/anthropic.js +2 -2
- package/dist/agent-task/integrations/openai.d.ts +1 -1
- package/dist/agent-task/integrations/openai.js +2 -2
- package/dist/agent-task/public.d.ts +302 -12
- package/dist/agent-task/public.js +810 -122
- package/dist/agent-task/public.js.map +1 -1
- package/dist/{chunk-43OI6GP7.js → chunk-6H2GACBG.js} +9 -3
- package/dist/chunk-6H2GACBG.js.map +1 -0
- package/dist/{chunk-IK4GJX7H.js → chunk-OZWOLCJ5.js} +5 -1
- package/dist/chunk-OZWOLCJ5.js.map +1 -0
- package/dist/{chunk-3B3L2H4Y.js → chunk-P43KL5AR.js} +9 -3
- package/dist/chunk-P43KL5AR.js.map +1 -0
- package/dist/{chunk-3OQ3JBFJ.js → chunk-Z64A566W.js} +36 -1
- package/dist/chunk-Z64A566W.js.map +1 -0
- package/dist/{tracing-Dqbuc7AK.d.ts → tracing-DoIGvJW1.d.ts} +8 -0
- package/package.json +1 -1
- package/dist/chunk-3B3L2H4Y.js.map +0 -1
- package/dist/chunk-3OQ3JBFJ.js.map +0 -1
- package/dist/chunk-43OI6GP7.js.map +0 -1
- package/dist/chunk-IK4GJX7H.js.map +0 -1
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
// Do not edit by hand — edit the src/ originals and rebuild.
|
|
3
3
|
import {
|
|
4
4
|
createApoAnthropic
|
|
5
|
-
} from "../../chunk-
|
|
6
|
-
import "../../chunk-
|
|
5
|
+
} from "../../chunk-6H2GACBG.js";
|
|
6
|
+
import "../../chunk-Z64A566W.js";
|
|
7
7
|
export {
|
|
8
8
|
createApoAnthropic
|
|
9
9
|
};
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
// Do not edit by hand — edit the src/ originals and rebuild.
|
|
3
3
|
import {
|
|
4
4
|
createApoOpenAI
|
|
5
|
-
} from "../../chunk-
|
|
6
|
-
import "../../chunk-
|
|
5
|
+
} from "../../chunk-P43KL5AR.js";
|
|
6
|
+
import "../../chunk-Z64A566W.js";
|
|
7
7
|
export {
|
|
8
8
|
createApoOpenAI
|
|
9
9
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { A as AgentTaskTraceContext, a as AgentTaskTraceOptions } from '../tracing-
|
|
1
|
+
import { A as AgentTaskTraceContext, a as AgentTaskTraceOptions } from '../tracing-DoIGvJW1.js';
|
|
2
2
|
export { CreateApoTracerOptions, createApoTracer } from './integrations/ai-sdk.js';
|
|
3
3
|
export { CreateApoOpenAIOptions, createApoOpenAI } from './integrations/openai.js';
|
|
4
4
|
export { CreateApoAnthropicOptions, createApoAnthropic } from './integrations/anthropic.js';
|
|
@@ -131,6 +131,37 @@ interface TraceProjectionSnapshot {
|
|
|
131
131
|
* the drift where concurrent tools appeared in completion order.
|
|
132
132
|
*/
|
|
133
133
|
|
|
134
|
+
/** One mid-run steer, derived from a `task.steer` observation. */
|
|
135
|
+
interface TraceSteer {
|
|
136
|
+
/** 1-based, registration order across the run. */
|
|
137
|
+
number: number;
|
|
138
|
+
/** Scripted turn the steer targeted. */
|
|
139
|
+
turn: number;
|
|
140
|
+
label?: string;
|
|
141
|
+
/**
|
|
142
|
+
* The serialized trigger (`{"toolResults":2}`, `"runStart"`). Records
|
|
143
|
+
* carry the trigger as an object — the local tee embeds the SteerRecord
|
|
144
|
+
* directly and the canonical path re-parses its JSON — while the event
|
|
145
|
+
* metadata carries the pre-stringified form; both render identically.
|
|
146
|
+
*/
|
|
147
|
+
trigger: string;
|
|
148
|
+
message: unknown;
|
|
149
|
+
status: "delivered" | "undelivered" | "error";
|
|
150
|
+
/** ISO timestamp of successful delivery (wall clock, display only). */
|
|
151
|
+
deliveredAt?: string;
|
|
152
|
+
/**
|
|
153
|
+
* The steer observation's own `startedAt` — the snapshot-clock position of
|
|
154
|
+
* the delivery. Window/delivery comparisons use THIS, not `deliveredAt`:
|
|
155
|
+
* local tee snapshots timestamp with a monotonic clock (performance.now
|
|
156
|
+
* epoch), so a wall-clock deliveredAt is not comparable to sibling
|
|
157
|
+
* observations, while the steer observation's own timestamp always is.
|
|
158
|
+
*/
|
|
159
|
+
spanStartedAt?: string;
|
|
160
|
+
/** The steer observation's span id — the deterministic tie-breaker. */
|
|
161
|
+
spanId: string;
|
|
162
|
+
boundary?: string;
|
|
163
|
+
reason?: string;
|
|
164
|
+
}
|
|
134
165
|
/** A tool call derived from a `TOOL` observation. */
|
|
135
166
|
interface TraceToolCall {
|
|
136
167
|
spanId: string;
|
|
@@ -199,6 +230,29 @@ declare class TraceView {
|
|
|
199
230
|
get skillLoads(): readonly TraceSkillLoad[];
|
|
200
231
|
/** Subagent delegations derived from `AGENT` observations, in invocation order. */
|
|
201
232
|
get subagentCalls(): readonly TraceSubagentCall[];
|
|
233
|
+
/**
|
|
234
|
+
* Mid-run steers, derived from `task.steer` observations, in number order.
|
|
235
|
+
* Empty when the trace carries none — steering assertions fail closed on
|
|
236
|
+
* the empty case instead of passing vacuously.
|
|
237
|
+
*/
|
|
238
|
+
get steers(): readonly TraceSteer[];
|
|
239
|
+
/**
|
|
240
|
+
* Whether an observation started after the steer observation, using the
|
|
241
|
+
* same ordering key as everywhere else in this class (startedAt, then
|
|
242
|
+
* span id). The id tie-break matters on the local tee path, whose ISO
|
|
243
|
+
* timestamps truncate to milliseconds — a steer and the generation that
|
|
244
|
+
* consumes it can share one.
|
|
245
|
+
*/
|
|
246
|
+
private static startedAfter;
|
|
247
|
+
/**
|
|
248
|
+
* A TraceView over the run after steer n was delivered: observations that
|
|
249
|
+
* started after the steer's delivery timestamp (the first generation that
|
|
250
|
+
* consumed it through run end). Missing timestamps sort after timestamped
|
|
251
|
+
* observations per the projection contract, so they belong to the window
|
|
252
|
+
* too. An undelivered steer yields an empty view — assertions inside it
|
|
253
|
+
* fail on missing evidence rather than passing vacuously.
|
|
254
|
+
*/
|
|
255
|
+
windowAfterSteer(n: number): TraceView;
|
|
202
256
|
/** Last assistant message content — the agent's "reply". Empty if none. */
|
|
203
257
|
get reply(): string;
|
|
204
258
|
/**
|
|
@@ -348,6 +402,38 @@ type CollectedDeliverables = Record<string, unknown>;
|
|
|
348
402
|
type AgentTurnResult = {
|
|
349
403
|
response: unknown;
|
|
350
404
|
};
|
|
405
|
+
/**
|
|
406
|
+
* Live progress signal from the agent under test, fired while a turn runs.
|
|
407
|
+
* The adapter forwards these from its harness's event stream via the
|
|
408
|
+
* `notifyAgentEvent` callback on `sendUserTurn`'s context — the runner cannot
|
|
409
|
+
* observe subprocess spans live (the projection tee only sees spans created
|
|
410
|
+
* through the runner's trace context), so the adapter is the source of
|
|
411
|
+
* progress. `run_start` MUST be the first event a turn emits.
|
|
412
|
+
*/
|
|
413
|
+
type AgentProgressEvent = {
|
|
414
|
+
kind: "run_start";
|
|
415
|
+
} | {
|
|
416
|
+
kind: "tool_result";
|
|
417
|
+
toolName?: string;
|
|
418
|
+
} | {
|
|
419
|
+
kind: "assistant_reply";
|
|
420
|
+
};
|
|
421
|
+
/** Where the harness actually queued/landed an injected message. */
|
|
422
|
+
type SteerDeliveryBoundary = "run_start" | "tool_results" | "assistant_reply";
|
|
423
|
+
type SteerResult = {
|
|
424
|
+
/**
|
|
425
|
+
* The boundary the harness reports for the injection. Harness truth wins
|
|
426
|
+
* over the scheduler's trigger intent — a `{ toolResults: 2 }` steer may
|
|
427
|
+
* land at `assistant_reply` on a harness that drains its queue there.
|
|
428
|
+
*/
|
|
429
|
+
boundary: SteerDeliveryBoundary;
|
|
430
|
+
};
|
|
431
|
+
/** Context handed to `session.steer()`. */
|
|
432
|
+
type SteerContext = {
|
|
433
|
+
trace: AgentTaskTraceContext;
|
|
434
|
+
turnNumber: number;
|
|
435
|
+
steerNumber: number;
|
|
436
|
+
};
|
|
351
437
|
/**
|
|
352
438
|
* The adapter-reported identity of the agent under test for one Task Run
|
|
353
439
|
* The adapter resolves its configuration (env vars, aliases,
|
|
@@ -431,7 +517,22 @@ type AdapterSession = {
|
|
|
431
517
|
trace: AgentTaskTraceContext;
|
|
432
518
|
turnNumber: number;
|
|
433
519
|
parentSpanId?: string;
|
|
520
|
+
/**
|
|
521
|
+
* The adapter calls this for every agent progress event while the turn
|
|
522
|
+
* runs. Fire-and-forget (sync void); the runner serializes internally
|
|
523
|
+
* and never lets scheduler errors break the turn. MUST be called with
|
|
524
|
+
* `{ kind: "run_start" }` before the first model call.
|
|
525
|
+
*/
|
|
526
|
+
notifyAgentEvent: (event: AgentProgressEvent) => void;
|
|
434
527
|
}) => Promise<AgentTurnResult>;
|
|
528
|
+
/**
|
|
529
|
+
* Mid-run injection primitive. Optional = negotiated capability: a task
|
|
530
|
+
* with registered steers fails closed before turn 1 when this is absent.
|
|
531
|
+
* Resolve when the harness has accepted/queued the message, not when the
|
|
532
|
+
* agent has reacted to it — reaction is asserted afterward by
|
|
533
|
+
* `t.steerDelivered` / `t.afterSteer`.
|
|
534
|
+
*/
|
|
535
|
+
steer?: (input: unknown, context: SteerContext) => Promise<SteerResult>;
|
|
435
536
|
close?: () => Promise<void>;
|
|
436
537
|
};
|
|
437
538
|
type AdapterDefinition = {
|
|
@@ -449,6 +550,47 @@ type TypedAdapterDefinition<TName extends string, TDeliverables extends Record<s
|
|
|
449
550
|
collectDeliverables: (ctx: CollectDeliverablesContext) => Promise<TCollected>;
|
|
450
551
|
};
|
|
451
552
|
|
|
553
|
+
/**
|
|
554
|
+
* Mid-run steering registration — the task-plane half of steering.
|
|
555
|
+
*
|
|
556
|
+
* `steer({...})` is called at eval-file top level next to `turn()`: it
|
|
557
|
+
* declares that a user message will be injected into a turn that is already
|
|
558
|
+
* running, at a boundary described relative to that turn's progress (after
|
|
559
|
+
* its n-th tool result, its n-th assistant reply, or right at run start).
|
|
560
|
+
*
|
|
561
|
+
* Registration mechanics mirror `turn.ts`: a module-global array under a
|
|
562
|
+
* `Symbol.for` key so the registry survives the eval module being imported
|
|
563
|
+
* from a temp path, reset by `loadTask` before each import and read by the
|
|
564
|
+
* runner's turn loop afterwards.
|
|
565
|
+
*/
|
|
566
|
+
type SteerTrigger =
|
|
567
|
+
/** After the n-th completed tool result of the target turn (1-based). */
|
|
568
|
+
{
|
|
569
|
+
toolResults: number;
|
|
570
|
+
}
|
|
571
|
+
/** After the n-th completed assistant message of the target turn (1-based). */
|
|
572
|
+
| {
|
|
573
|
+
assistantReply: number;
|
|
574
|
+
}
|
|
575
|
+
/** Immediately after the turn's user input is handed to the agent. */
|
|
576
|
+
| "runStart";
|
|
577
|
+
/** One scripted steer, registered from the eval file (or legacy checks.ts). */
|
|
578
|
+
type SteerSpec = {
|
|
579
|
+
when: SteerTrigger;
|
|
580
|
+
/** The injected user message — same shape a `turn()` return value has. */
|
|
581
|
+
message: unknown;
|
|
582
|
+
/** Human label shown in the trace, transcript, and check failures. */
|
|
583
|
+
label?: string;
|
|
584
|
+
/** 1-based scripted turn this steer belongs to. Default 1. */
|
|
585
|
+
turn?: number;
|
|
586
|
+
};
|
|
587
|
+
/** Register a mid-run steer. Call at eval-file top level, next to turn(). */
|
|
588
|
+
declare function steer(spec: SteerSpec): void;
|
|
589
|
+
/** All steers registered so far, in registration order (runner use). */
|
|
590
|
+
declare function getTaskSteers(): SteerSpec[];
|
|
591
|
+
/** Clear registrations (loadTask use — mirrors resetTaskTurn). */
|
|
592
|
+
declare function resetTaskSteers(): void;
|
|
593
|
+
|
|
452
594
|
/**
|
|
453
595
|
* The outcome of a single assertion.
|
|
454
596
|
*
|
|
@@ -524,6 +666,13 @@ type JudgeMetadata = {
|
|
|
524
666
|
* as extra signals per check, not a second vote.
|
|
525
667
|
*/
|
|
526
668
|
secondJudge?: SecondJudgeEvidence;
|
|
669
|
+
/**
|
|
670
|
+
* Present only when the second judge's verdict decided this check (cascade
|
|
671
|
+
* mode, `judge.mode: "cascade"`): the primary LLM judge was not called.
|
|
672
|
+
* Absent = the primary judge decided. The `model` field above then names
|
|
673
|
+
* the decision model, and `response` carries its typed verdict.
|
|
674
|
+
*/
|
|
675
|
+
verdict_by?: "second-judge";
|
|
527
676
|
};
|
|
528
677
|
/**
|
|
529
678
|
* Evidence from the second grader. `error` is set when the decisions
|
|
@@ -766,10 +915,36 @@ type TaskEvaluationResult = {
|
|
|
766
915
|
type TaskTranscript = {
|
|
767
916
|
turns: TaskTranscriptTurn[];
|
|
768
917
|
};
|
|
918
|
+
/**
|
|
919
|
+
* Outcome of one scripted steer. Steers never fail the run by themselves —
|
|
920
|
+
* a dropped or errored steer is recorded here and on the trace, and
|
|
921
|
+
* `t.steerDelivered` is what turns it red.
|
|
922
|
+
*/
|
|
923
|
+
type SteerRecord = {
|
|
924
|
+
/** 1-based, registration order across the whole run. */
|
|
925
|
+
number: number;
|
|
926
|
+
/** Scripted turn the steer targeted. */
|
|
927
|
+
turn: number;
|
|
928
|
+
label?: string;
|
|
929
|
+
trigger: SteerTrigger;
|
|
930
|
+
message: unknown;
|
|
931
|
+
status: "delivered" | "undelivered" | "error";
|
|
932
|
+
/** ISO timestamp of successful delivery. */
|
|
933
|
+
deliveredAt?: string;
|
|
934
|
+
boundary?: SteerDeliveryBoundary;
|
|
935
|
+
/**
|
|
936
|
+
* Why undelivered ("turn ended before trigger fired", "no progress events
|
|
937
|
+
* observed for this turn", "target turn never ran"), or the injection
|
|
938
|
+
* error message when status === "error".
|
|
939
|
+
*/
|
|
940
|
+
reason?: string;
|
|
941
|
+
};
|
|
769
942
|
type TaskTranscriptTurn = {
|
|
770
943
|
turnNumber: number;
|
|
771
944
|
userAction: unknown;
|
|
772
945
|
agentResponse: unknown;
|
|
946
|
+
/** Steers that targeted this turn, in steerNumber order. */
|
|
947
|
+
steers?: SteerRecord[];
|
|
773
948
|
};
|
|
774
949
|
type TaskRunResult = {
|
|
775
950
|
task: TaskDefinition;
|
|
@@ -1044,6 +1219,17 @@ type AgentJudgeOptions = {
|
|
|
1044
1219
|
};
|
|
1045
1220
|
budget?: AgentBudget;
|
|
1046
1221
|
label?: string;
|
|
1222
|
+
/**
|
|
1223
|
+
* The ground-truth verdict this check asserts: `"pass"` (default) or
|
|
1224
|
+
* `"fail"`. The session never sees it — the instruction is sent unchanged,
|
|
1225
|
+
* so the judge still decides independently. What changes is the recorded
|
|
1226
|
+
* outcome: the check passes only when the judge's verdict matches the
|
|
1227
|
+
* author's ground truth. `"fail"` is how a known-bad deliverable (a
|
|
1228
|
+
* fabricated figure, a claim the trace contradicts) becomes a passing
|
|
1229
|
+
* check when the judge catches it — judging the judge against cases whose
|
|
1230
|
+
* correct verdict is fixed by construction.
|
|
1231
|
+
*/
|
|
1232
|
+
expect?: "pass" | "fail";
|
|
1047
1233
|
};
|
|
1048
1234
|
/** The local evidence plane a session can investigate. */
|
|
1049
1235
|
type AgentEvidence = {
|
|
@@ -1107,6 +1293,13 @@ type JudgeConfig = {
|
|
|
1107
1293
|
* {@link JudgePromptBuilder}.
|
|
1108
1294
|
*/
|
|
1109
1295
|
prompt?: JudgePromptBuilder;
|
|
1296
|
+
/**
|
|
1297
|
+
* `"cascade"`: the second judge (decision model, `APO_SECOND_JUDGE_MODEL`)
|
|
1298
|
+
* answers first and a confident verdict (native confidence ≥ 0.95) stands
|
|
1299
|
+
* without calling the primary judge — the cost tier. Unset = the default
|
|
1300
|
+
* dual behavior (second judge as parallel, evidence-only shadow).
|
|
1301
|
+
*/
|
|
1302
|
+
mode?: "cascade";
|
|
1110
1303
|
};
|
|
1111
1304
|
/**
|
|
1112
1305
|
* What a check knows about its scope when it invokes the judge — the
|
|
@@ -1152,6 +1345,20 @@ interface TestContext {
|
|
|
1152
1345
|
minTokens(n: number, opts?: TokenBudgetOptions): void;
|
|
1153
1346
|
/** Escape hatch for a named predicate over the complete normalized run. */
|
|
1154
1347
|
assert(label: string, predicate: (flow: TraceView) => boolean): void;
|
|
1348
|
+
/**
|
|
1349
|
+
* Asserts steer n was delivered AND consumed: at least one generation
|
|
1350
|
+
* observation started after the steer's delivery timestamp. Fails closed —
|
|
1351
|
+
* a trace with no steering evidence at all is a failure, never a pass.
|
|
1352
|
+
*/
|
|
1353
|
+
steerDelivered(n: number): void;
|
|
1354
|
+
/**
|
|
1355
|
+
* Scopes trace-based assertions to what the agent did after steer n: the
|
|
1356
|
+
* fn receives a TestContext over `view.windowAfterSteer(n)`, and every
|
|
1357
|
+
* existing matcher works unchanged inside it. When the steer is missing
|
|
1358
|
+
* from the trace, records one failure and skips fn (inner assertions must
|
|
1359
|
+
* not run — they would pass vacuously against an empty window).
|
|
1360
|
+
*/
|
|
1361
|
+
afterSteer(n: number, fn: (t: TestContext) => void): void;
|
|
1155
1362
|
/** Grades any value (a deliverable, parsed JSON, anything) with a matcher. */
|
|
1156
1363
|
check<T>(value: T, matcher: Matcher<T>, label?: string): void;
|
|
1157
1364
|
/**
|
|
@@ -1170,6 +1377,11 @@ interface TestContext {
|
|
|
1170
1377
|
* only, when the full value exceeds its context limit: the primary judge
|
|
1171
1378
|
* still sees all of ``value``, so other criteria that need the complete
|
|
1172
1379
|
* deliverable are unaffected.
|
|
1380
|
+
*
|
|
1381
|
+
* ``opts.expect`` pins the ground-truth verdict (``"pass"`` default, or
|
|
1382
|
+
* ``"fail"``): the judge still decides independently, but the check passes
|
|
1383
|
+
* only when its verdict matches — how a known-bad value yields a passing
|
|
1384
|
+
* check exactly when the judge catches it.
|
|
1173
1385
|
*/
|
|
1174
1386
|
judge(values: unknown | unknown[], instruction: string, opts?: {
|
|
1175
1387
|
label?: string;
|
|
@@ -1181,6 +1393,14 @@ interface TestContext {
|
|
|
1181
1393
|
* the second judge sees exactly what the primary judge sees.
|
|
1182
1394
|
*/
|
|
1183
1395
|
secondJudgeValue?: unknown | unknown[];
|
|
1396
|
+
/**
|
|
1397
|
+
* The ground-truth verdict this check asserts: `"pass"` (default) or
|
|
1398
|
+
* `"fail"`. The judge never sees it — the instruction is sent
|
|
1399
|
+
* unchanged. The check passes only when the judge's verdict matches
|
|
1400
|
+
* the author's ground truth, so a known-bad value becomes a passing
|
|
1401
|
+
* check exactly when the judge catches it.
|
|
1402
|
+
*/
|
|
1403
|
+
expect?: "pass" | "fail";
|
|
1184
1404
|
}): Promise<void>;
|
|
1185
1405
|
/**
|
|
1186
1406
|
* Agentic judge: a tool-using LLM session that investigates the run's own
|
|
@@ -1195,6 +1415,10 @@ interface TestContext {
|
|
|
1195
1415
|
* 2 MiB read) and fail-closed: a session that ends without a verdict is a
|
|
1196
1416
|
* recorded failure, never a silent pass. Costs more and runs minutes, not
|
|
1197
1417
|
* seconds — reserve it for rubrics that need investigation.
|
|
1418
|
+
*
|
|
1419
|
+
* ``opts.expect`` pins the ground-truth verdict (``"pass"`` default, or
|
|
1420
|
+
* ``"fail"``): the judge still investigates and decides independently, but
|
|
1421
|
+
* the check passes only when its verdict matches.
|
|
1198
1422
|
*/
|
|
1199
1423
|
agent(instruction: string, opts?: AgentJudgeOptions): Promise<void>;
|
|
1200
1424
|
}
|
|
@@ -1210,7 +1434,7 @@ interface TestContext {
|
|
|
1210
1434
|
* adding a method to the interface without registering it here is a type
|
|
1211
1435
|
* error, and vice versa. See ``tests/test-context-methods.test.ts``.
|
|
1212
1436
|
*/
|
|
1213
|
-
declare const TEST_METHOD_NAMES: readonly ["calledTool", "notCalledTool", "toolOrder", "usedNoTools", "maxToolCalls", "noFailedActions", "loadedSkill", "calledSubagent", "messageIncludes", "maxTurns", "maxDurationMs", "maxTokens", "minTokens", "assert", "check", "judge", "agent"];
|
|
1437
|
+
declare const TEST_METHOD_NAMES: readonly ["calledTool", "notCalledTool", "toolOrder", "usedNoTools", "maxToolCalls", "noFailedActions", "loadedSkill", "calledSubagent", "messageIncludes", "maxTurns", "maxDurationMs", "maxTokens", "minTokens", "steerDelivered", "afterSteer", "assert", "check", "judge", "agent"];
|
|
1214
1438
|
|
|
1215
1439
|
type TaskDefinition<TAdapterName extends string = string, TDeliverable extends string = string> = {
|
|
1216
1440
|
id: string;
|
|
@@ -1365,15 +1589,6 @@ type TaskScope<TDeliverables> = {
|
|
|
1365
1589
|
* test("reviewed-methodically", (t, { deliverables }) => {
|
|
1366
1590
|
* t.check(deliverables.result, includes("finding"));
|
|
1367
1591
|
* });
|
|
1368
|
-
*
|
|
1369
|
-
* // A task that needs dev-machine resources (cloud creds, VPC, stage) can
|
|
1370
|
-
* // declare execution: "local" so `apo task run` runs it on the caller's
|
|
1371
|
-
* // machine while still recording a backend run row.
|
|
1372
|
-
* task("bind-e2e", {
|
|
1373
|
-
* adapter: bindAdapter,
|
|
1374
|
-
* deliverables: ["summary"],
|
|
1375
|
-
* execution: "local",
|
|
1376
|
-
* });
|
|
1377
1592
|
* ```
|
|
1378
1593
|
*/
|
|
1379
1594
|
declare function task<const TTaskId extends string, const TAdapterName extends string, const TDeliverableDefs extends Record<string, DeliverableDefinition>, TCollected extends CollectedDeliverables, const TSelected extends readonly (DeliverableKey<TCollected> & keyof TDeliverableDefs)[]>(name: TTaskId, config: Omit<TaskConfig<TSelected[number] & string>, "id" | "checks" | "deliverables"> & {
|
|
@@ -1497,6 +1712,65 @@ interface PreparedTaskResult {
|
|
|
1497
1712
|
*/
|
|
1498
1713
|
declare function persistFileArtifacts(deliverables: Record<string, unknown>, config: ArtifactUploadConfig): Promise<PreparedTaskResult>;
|
|
1499
1714
|
|
|
1715
|
+
/**
|
|
1716
|
+
* Second grader: a typed-decision model (System One / Jev) that grades the
|
|
1717
|
+
* same deliverable + instruction as the primary LLM judge, in parallel.
|
|
1718
|
+
*
|
|
1719
|
+
* Why a second grader at all: the primary judge's verdict is the product,
|
|
1720
|
+
* but its trustworthiness is invisible. A non-generative decision model
|
|
1721
|
+
* cannot fabricate quoted evidence the way text judges measurably do, and
|
|
1722
|
+
* its calibrated confidence acts as a difficulty meter — so agreement +
|
|
1723
|
+
* confidence turn every check into self-describing evidence (auto-verified /
|
|
1724
|
+
* borderline / disputed) at ~1/100th of a cent per call.
|
|
1725
|
+
*
|
|
1726
|
+
* The second opinion NEVER changes the verdict. It is opt-in
|
|
1727
|
+
* (`APO_SECOND_JUDGE_MODEL`), runs on a decisions endpoint (not
|
|
1728
|
+
* chat/completions — decision models reject it), and any failure is
|
|
1729
|
+
* recorded as `error` on the evidence instead of affecting the check.
|
|
1730
|
+
*
|
|
1731
|
+
* Providers: every decisions endpoint speaks its own request/response
|
|
1732
|
+
* dialect, selected via `APO_SECOND_JUDGE_PROVIDER` (or auto-detected from
|
|
1733
|
+
* the base URL). Adding a provider means adding one object to `PROVIDERS`
|
|
1734
|
+
* — nothing else in this file or the caller changes.
|
|
1735
|
+
*/
|
|
1736
|
+
|
|
1737
|
+
/** How a recorded second-judge opinion relates to the check's verdict. */
|
|
1738
|
+
type SecondJudgeFacts = {
|
|
1739
|
+
kind: "none";
|
|
1740
|
+
} | {
|
|
1741
|
+
kind: "error";
|
|
1742
|
+
} | {
|
|
1743
|
+
kind: "skipped";
|
|
1744
|
+
} | {
|
|
1745
|
+
kind: "split";
|
|
1746
|
+
confidence: number;
|
|
1747
|
+
} | {
|
|
1748
|
+
kind: "agree";
|
|
1749
|
+
confidence: number;
|
|
1750
|
+
} | {
|
|
1751
|
+
kind: "unsure";
|
|
1752
|
+
confidence: number;
|
|
1753
|
+
};
|
|
1754
|
+
/**
|
|
1755
|
+
* Structural input for {@link secondJudgeFacts}: any check shape carrying a
|
|
1756
|
+
* verdict and optional judge evidence — the SDK's own results and the wire
|
|
1757
|
+
* types the CLI and dashboard read from the backend are all assignable.
|
|
1758
|
+
*/
|
|
1759
|
+
type SecondJudgeFactsInput = {
|
|
1760
|
+
pass: boolean;
|
|
1761
|
+
judge?: {
|
|
1762
|
+
secondJudge?: {
|
|
1763
|
+
choice?: "pass" | "fail";
|
|
1764
|
+
confidence?: number;
|
|
1765
|
+
skipped?: string;
|
|
1766
|
+
error?: string;
|
|
1767
|
+
};
|
|
1768
|
+
};
|
|
1769
|
+
};
|
|
1770
|
+
declare function secondJudgeFacts(check: SecondJudgeFactsInput): SecondJudgeFacts;
|
|
1771
|
+
/** One honest sentence about the relation; null when corroborated. */
|
|
1772
|
+
declare function secondJudgeTakeaway(check: SecondJudgeFactsInput): string | null;
|
|
1773
|
+
|
|
1500
1774
|
/**
|
|
1501
1775
|
* Check submission compaction (issue #175).
|
|
1502
1776
|
*
|
|
@@ -1702,6 +1976,8 @@ type RunTaskOptions = {
|
|
|
1702
1976
|
*/
|
|
1703
1977
|
judgeTools?: JudgeToolsConfig;
|
|
1704
1978
|
onTurn?: (turnNumber: number, userAction: TaskTranscriptTurn["userAction"], agentResponse: unknown) => void;
|
|
1979
|
+
/** Called for every steer outcome (delivered / undelivered / error). */
|
|
1980
|
+
onSteer?: (record: SteerRecord) => void;
|
|
1705
1981
|
/**
|
|
1706
1982
|
* Skip the internal `loadTask(taskDir)` call and reuse an already-loaded
|
|
1707
1983
|
* task definition.
|
|
@@ -1722,6 +1998,20 @@ type RunTaskOptions = {
|
|
|
1722
1998
|
};
|
|
1723
1999
|
declare function runTask(taskDir: string, options?: RunTaskOptions): Promise<TaskRunResult>;
|
|
1724
2000
|
|
|
2001
|
+
/**
|
|
2002
|
+
* Issue #8: shown when a run ends with zero registered checks. A bare
|
|
2003
|
+
* `FAIL <task>` with no Checks section looked like a real failure but was
|
|
2004
|
+
* almost always a silent registration bug (e.g. a double-import that wiped
|
|
2005
|
+
* the check registry). Naming `test()` matches the documented registration
|
|
2006
|
+
* function — `apps/docs` reference/task.md.
|
|
2007
|
+
*
|
|
2008
|
+
* Lives next to {@link aggregateResult} because the empty-checks verdict is
|
|
2009
|
+
* the single source of truth for "why did this run fail with no checks?".
|
|
2010
|
+
* The CLI and the standalone e2e runner both import this so the wording
|
|
2011
|
+
* stays in one place.
|
|
2012
|
+
*/
|
|
2013
|
+
declare const NO_CHECKS_REGISTERED_MESSAGE = "No tests were registered by the eval module \u2014 a task must define at least one test().";
|
|
2014
|
+
|
|
1725
2015
|
/**
|
|
1726
2016
|
* Issue #159: replay Phase 2 of a completed Run against its stored
|
|
1727
2017
|
* Deliverables — without re-running the agent.
|
|
@@ -2309,4 +2599,4 @@ type TranscriptCaptureResult = {
|
|
|
2309
2599
|
};
|
|
2310
2600
|
declare function replayAdapterTranscript(capture: AdapterTranscriptCapture, options: ReplayAdapterTranscriptOptions): Promise<TranscriptCaptureResult>;
|
|
2311
2601
|
|
|
2312
|
-
export { type AISDKResult, type AdapterDefinition, type AdapterRuntimeState, type AdapterSession, type AdapterTranscriptCapture, type AgentBudget, type AgentEvidence, type AgentJudgeOptions, type AgentJudgeOutcome, type AgentJudgeSession, type AgentJudgeStep, type AgentTaskCliOptions, type AgentTaskRunConfiguration, AgentTaskRunError, type AgentTaskRunSummary, type AgentTaskRuntime, AgentTaskTraceContext, AgentTaskTraceOptions, type AgentTurnResult, type AnthropicMessage, type ApoRunContext, ApoSpanProcessor, type ArtifactUploadConfig, type CheckCompaction, type CheckContext, type CheckGroup, type CleanupContext, type CollectDeliverablesContext, type CollectedDeliverables, type DeliverableDefinition, type DeliverableSummary, type DeliverableValidationResult, type DescribeRegistration, type DurationBudgetOptions, type EvidenceFingerprint, type ExportOtlpTracesOptions, type FileArtifact, type FileArtifactOptions, type FileEntry, type Flow, type FlowEvent, FlowView, type InitializeContext, JUDGE_SEGMENT_LIMIT, type JudgeConfig, type JudgeContext, type JudgePromptBuilder, type JudgeScope, type JudgeToolsConfig, type LoadedTask, type ManifestFile, type ManifestFileInput, type Matcher, type McpHttpTransport, type McpServerConfig, type McpStdioTransport, type ModeClass, type NameMatcher, type OpenAIMessage, type OtlpAttribute, type OtlpAttributeValue, type OtlpSpan, type OtlpTracesPayload, type ParsedTranscriptSession, type PreparedTaskResult, RECEIVED_VALUE_LIMIT, type RegisterApoTracingOptions, type RejudgeCheckStability, type RejudgeEndpoints, RejudgeError, type RejudgeOptions, type RejudgeOutcome, type ReplayAdapterTranscriptOptions, type RunTaskOptions, type StartSessionContext, TEST_METHOD_NAMES, type TaskConfig, type TaskDefinition, type TaskEvaluationResult, TaskFiles, type TaskRevisionManifestV1, type TaskRunResult, type TaskScope, type TaskTranscript, type TaskTranscriptTurn, type TestContext, type TestRegistration, type TokenBudgetOptions, type ToolCallOptions, type ToolCallStatus, type TranscriptCaptureResult, TranscriptReplayError, type TranscriptReplayOptions, type TranscriptSource, type TranscriptToolCall, type TranscriptTurn, type TranscriptUsage, type TurnContext, type TurnFn, type TurnRecord, type TypedAdapterDefinition, type ValueMatcher, buildApoAuthHeaders, buildManifest, canonicalManifestJson, compactChecksForSubmission, connectMcpServers, contentSha256, defineAdapter, defineTask, describe, detectTranscriptSource, discoverAgentTaskDirs, equals, expandSecretPlaceholders, exportOtlpTraces, fileArtifact, filePaths, fromAISDK, fromAnthropicMessages, fromOpenAIMessages, getActiveApoRun, getTaskTurn, includes, isFileArtifact, loadTask, loadTaskRuntime, matchValue, matches, normalizeManifestPath, parseAgentTaskCliArgs, parseClaudeCodeTranscript, parseCodexTranscript, persistFileArtifacts, registerApoTracing, rejudgeTaskRun, replayAdapterTranscript, resetApoTracing, resetFlowChecks, resetTaskRegistry, resetTaskTurn, resolveMcpServerPaths, resolveOtlpTracesUrl, resolveTurn, runAgentTaskCli, runTask, runTaskDir, satisfies, sha256Hex, similarity, task, defineCheck as test, transcriptSessionToOtlp, turn, withApoRun, withApoRunSync };
|
|
2602
|
+
export { type AISDKResult, type AdapterDefinition, type AdapterRuntimeState, type AdapterSession, type AdapterTranscriptCapture, type AgentBudget, type AgentEvidence, type AgentJudgeOptions, type AgentJudgeOutcome, type AgentJudgeSession, type AgentJudgeStep, type AgentProgressEvent, type AgentTaskCliOptions, type AgentTaskRunConfiguration, AgentTaskRunError, type AgentTaskRunSummary, type AgentTaskRuntime, AgentTaskTraceContext, AgentTaskTraceOptions, type AgentTurnResult, type AnthropicMessage, type ApoRunContext, ApoSpanProcessor, type ArtifactUploadConfig, type CheckCompaction, type CheckContext, type CheckGroup, type CleanupContext, type CollectDeliverablesContext, type CollectedDeliverables, type DeliverableDefinition, type DeliverableSummary, type DeliverableValidationResult, type DescribeRegistration, type DurationBudgetOptions, type EvidenceFingerprint, type ExportOtlpTracesOptions, type FileArtifact, type FileArtifactOptions, type FileEntry, type Flow, type FlowEvent, FlowView, type InitializeContext, JUDGE_SEGMENT_LIMIT, type JudgeConfig, type JudgeContext, type JudgePromptBuilder, type JudgeScope, type JudgeToolsConfig, type LoadedTask, type ManifestFile, type ManifestFileInput, type Matcher, type McpHttpTransport, type McpServerConfig, type McpStdioTransport, type ModeClass, NO_CHECKS_REGISTERED_MESSAGE, type NameMatcher, type OpenAIMessage, type OtlpAttribute, type OtlpAttributeValue, type OtlpSpan, type OtlpTracesPayload, type ParsedTranscriptSession, type PreparedTaskResult, RECEIVED_VALUE_LIMIT, type RegisterApoTracingOptions, type RejudgeCheckStability, type RejudgeEndpoints, RejudgeError, type RejudgeOptions, type RejudgeOutcome, type ReplayAdapterTranscriptOptions, type RunTaskOptions, type SecondJudgeFacts, type SecondJudgeFactsInput, type StartSessionContext, type SteerContext, type SteerDeliveryBoundary, type SteerRecord, type SteerResult, type SteerSpec, type SteerTrigger, TEST_METHOD_NAMES, type TaskConfig, type TaskDefinition, type TaskEvaluationResult, TaskFiles, type TaskRevisionManifestV1, type TaskRunResult, type TaskScope, type TaskTranscript, type TaskTranscriptTurn, type TestContext, type TestRegistration, type TokenBudgetOptions, type ToolCallOptions, type ToolCallStatus, type TranscriptCaptureResult, TranscriptReplayError, type TranscriptReplayOptions, type TranscriptSource, type TranscriptToolCall, type TranscriptTurn, type TranscriptUsage, type TurnContext, type TurnFn, type TurnRecord, type TypedAdapterDefinition, type ValueMatcher, buildApoAuthHeaders, buildManifest, canonicalManifestJson, compactChecksForSubmission, connectMcpServers, contentSha256, defineAdapter, defineTask, describe, detectTranscriptSource, discoverAgentTaskDirs, equals, expandSecretPlaceholders, exportOtlpTraces, fileArtifact, filePaths, fromAISDK, fromAnthropicMessages, fromOpenAIMessages, getActiveApoRun, getTaskSteers, getTaskTurn, includes, isFileArtifact, loadTask, loadTaskRuntime, matchValue, matches, normalizeManifestPath, parseAgentTaskCliArgs, parseClaudeCodeTranscript, parseCodexTranscript, persistFileArtifacts, registerApoTracing, rejudgeTaskRun, replayAdapterTranscript, resetApoTracing, resetFlowChecks, resetTaskRegistry, resetTaskSteers, resetTaskTurn, resolveMcpServerPaths, resolveOtlpTracesUrl, resolveTurn, runAgentTaskCli, runTask, runTaskDir, satisfies, secondJudgeFacts, secondJudgeTakeaway, sha256Hex, similarity, steer, task, defineCheck as test, transcriptSessionToOtlp, turn, withApoRun, withApoRunSync };
|