@apo-ai/sdk 0.10.1 → 0.11.0

package/dist/agent-task/public.d.ts

@@ -131,6 +131,37 @@ interface TraceProjectionSnapshot {
  * the drift where concurrent tools appeared in completion order.
  */
 
+/** One mid-run steer, derived from a `task.steer` observation. */
+interface TraceSteer {
+    /** 1-based, registration order across the run. */
+    number: number;
+    /** Scripted turn the steer targeted. */
+    turn: number;
+    label?: string;
+    /**
+     * The serialized trigger (`{"toolResults":2}`, `"runStart"`). Records
+     * carry the trigger as an object — the local tee embeds the SteerRecord
+     * directly and the canonical path re-parses its JSON — while the event
+     * metadata carries the pre-stringified form; both render identically.
+     */
+    trigger: string;
+    message: unknown;
+    status: "delivered" | "undelivered" | "error";
+    /** ISO timestamp of successful delivery (wall clock, display only). */
+    deliveredAt?: string;
+    /**
+     * The steer observation's own `startedAt` — the snapshot-clock position of
+     * the delivery. Window/delivery comparisons use THIS, not `deliveredAt`:
+     * local tee snapshots timestamp with a monotonic clock (performance.now
+     * epoch), so a wall-clock deliveredAt is not comparable to sibling
+     * observations, while the steer observation's own timestamp always is.
+     */
+    spanStartedAt?: string;
+    /** The steer observation's span id — the deterministic tie-breaker. */
+    spanId: string;
+    boundary?: string;
+    reason?: string;
+}
 /** A tool call derived from a `TOOL` observation. */
 interface TraceToolCall {
     spanId: string;
@@ -199,6 +230,29 @@ declare class TraceView {
     get skillLoads(): readonly TraceSkillLoad[];
     /** Subagent delegations derived from `AGENT` observations, in invocation order. */
     get subagentCalls(): readonly TraceSubagentCall[];
+    /**
+     * Mid-run steers, derived from `task.steer` observations, in number order.
+     * Empty when the trace carries none — steering assertions fail closed on
+     * the empty case instead of passing vacuously.
+     */
+    get steers(): readonly TraceSteer[];
+    /**
+     * Whether an observation started after the steer observation, using the
+     * same ordering key as everywhere else in this class (startedAt, then
+     * span id). The id tie-break matters on the local tee path, whose ISO
+     * timestamps truncate to milliseconds — a steer and the generation that
+     * consumes it can share one.
+     */
+    private static startedAfter;
+    /**
+     * A TraceView over the run after steer n was delivered: observations that
+     * started after the steer's delivery timestamp (the first generation that
+     * consumed it through run end). Missing timestamps sort after timestamped
+     * observations per the projection contract, so they belong to the window
+     * too. An undelivered steer yields an empty view — assertions inside it
+     * fail on missing evidence rather than passing vacuously.
+     */
+    windowAfterSteer(n: number): TraceView;
     /** Last assistant message content — the agent's "reply". Empty if none. */
     get reply(): string;
     /**
@@ -348,6 +402,38 @@ type CollectedDeliverables = Record<string, unknown>;
 type AgentTurnResult = {
     response: unknown;
 };
+/**
+ * Live progress signal from the agent under test, fired while a turn runs.
+ * The adapter forwards these from its harness's event stream via the
+ * `notifyAgentEvent` callback on `sendUserTurn`'s context — the runner cannot
+ * observe subprocess spans live (the projection tee only sees spans created
+ * through the runner's trace context), so the adapter is the source of
+ * progress. `run_start` MUST be the first event a turn emits.
+ */
+type AgentProgressEvent = {
+    kind: "run_start";
+} | {
+    kind: "tool_result";
+    toolName?: string;
+} | {
+    kind: "assistant_reply";
+};
+/** Where the harness actually queued/landed an injected message. */
+type SteerDeliveryBoundary = "run_start" | "tool_results" | "assistant_reply";
+type SteerResult = {
+    /**
+     * The boundary the harness reports for the injection. Harness truth wins
+     * over the scheduler's trigger intent — a `{ toolResults: 2 }` steer may
+     * land at `assistant_reply` on a harness that drains its queue there.
+     */
+    boundary: SteerDeliveryBoundary;
+};
+/** Context handed to `session.steer()`. */
+type SteerContext = {
+    trace: AgentTaskTraceContext;
+    turnNumber: number;
+    steerNumber: number;
+};
 /**
  * The adapter-reported identity of the agent under test for one Task Run
  * The adapter resolves its configuration (env vars, aliases,
@@ -431,7 +517,22 @@ type AdapterSession = {
         trace: AgentTaskTraceContext;
         turnNumber: number;
         parentSpanId?: string;
+        /**
+         * The adapter calls this for every agent progress event while the turn
+         * runs. Fire-and-forget (sync void); the runner serializes internally
+         * and never lets scheduler errors break the turn. MUST be called with
+         * `{ kind: "run_start" }` before the first model call.
+         */
+        notifyAgentEvent: (event: AgentProgressEvent) => void;
     }) => Promise<AgentTurnResult>;
+    /**
+     * Mid-run injection primitive. Optional = negotiated capability: a task
+     * with registered steers fails closed before turn 1 when this is absent.
+     * Resolve when the harness has accepted/queued the message, not when the
+     * agent has reacted to it — reaction is asserted afterward by
+     * `t.steerDelivered` / `t.afterSteer`.
+     */
+    steer?: (input: unknown, context: SteerContext) => Promise<SteerResult>;
     close?: () => Promise<void>;
 };
 type AdapterDefinition = {
@@ -449,6 +550,47 @@ type TypedAdapterDefinition<TName extends string, TDeliverables extends Record<s
     collectDeliverables: (ctx: CollectDeliverablesContext) => Promise<TCollected>;
 };
 
+/**
+ * Mid-run steering registration — the task-plane half of steering.
+ *
+ * `steer({...})` is called at eval-file top level next to `turn()`: it
+ * declares that a user message will be injected into a turn that is already
+ * running, at a boundary described relative to that turn's progress (after
+ * its n-th tool result, its n-th assistant reply, or right at run start).
+ *
+ * Registration mechanics mirror `turn.ts`: a module-global array under a
+ * `Symbol.for` key so the registry survives the eval module being imported
+ * from a temp path, reset by `loadTask` before each import and read by the
+ * runner's turn loop afterwards.
+ */
+type SteerTrigger = 
+/** After the n-th completed tool result of the target turn (1-based). */
+{
+    toolResults: number;
+}
+/** After the n-th completed assistant message of the target turn (1-based). */
+ | {
+    assistantReply: number;
+}
+/** Immediately after the turn's user input is handed to the agent. */
+ | "runStart";
+/** One scripted steer, registered from the eval file (or legacy checks.ts). */
+type SteerSpec = {
+    when: SteerTrigger;
+    /** The injected user message — same shape a `turn()` return value has. */
+    message: unknown;
+    /** Human label shown in the trace, transcript, and check failures. */
+    label?: string;
+    /** 1-based scripted turn this steer belongs to. Default 1. */
+    turn?: number;
+};
+/** Register a mid-run steer. Call at eval-file top level, next to turn(). */
+declare function steer(spec: SteerSpec): void;
+/** All steers registered so far, in registration order (runner use). */
+declare function getTaskSteers(): SteerSpec[];
+/** Clear registrations (loadTask use — mirrors resetTaskTurn). */
+declare function resetTaskSteers(): void;
+
 /**
  * The outcome of a single assertion.
  *
@@ -524,6 +666,13 @@ type JudgeMetadata = {
      * as extra signals per check, not a second vote.
      */
     secondJudge?: SecondJudgeEvidence;
+    /**
+     * Present only when the second judge's verdict decided this check (cascade
+     * mode, `judge.mode: "cascade"`): the primary LLM judge was not called.
+     * Absent = the primary judge decided. The `model` field above then names
+     * the decision model, and `response` carries its typed verdict.
+     */
+    verdict_by?: "second-judge";
 };
 /**
  * Evidence from the second grader. `error` is set when the decisions
@@ -766,10 +915,36 @@ type TaskEvaluationResult = {
 type TaskTranscript = {
     turns: TaskTranscriptTurn[];
 };
+/**
+ * Outcome of one scripted steer. Steers never fail the run by themselves —
+ * a dropped or errored steer is recorded here and on the trace, and
+ * `t.steerDelivered` is what turns it red.
+ */
+type SteerRecord = {
+    /** 1-based, registration order across the whole run. */
+    number: number;
+    /** Scripted turn the steer targeted. */
+    turn: number;
+    label?: string;
+    trigger: SteerTrigger;
+    message: unknown;
+    status: "delivered" | "undelivered" | "error";
+    /** ISO timestamp of successful delivery. */
+    deliveredAt?: string;
+    boundary?: SteerDeliveryBoundary;
+    /**
+     * Why undelivered ("turn ended before trigger fired", "no progress events
+     * observed for this turn", "target turn never ran"), or the injection
+     * error message when status === "error".
+     */
+    reason?: string;
+};
 type TaskTranscriptTurn = {
     turnNumber: number;
     userAction: unknown;
     agentResponse: unknown;
+    /** Steers that targeted this turn, in steerNumber order. */
+    steers?: SteerRecord[];
 };
 type TaskRunResult = {
     task: TaskDefinition;
@@ -1118,6 +1293,13 @@ type JudgeConfig = {
      * {@link JudgePromptBuilder}.
      */
     prompt?: JudgePromptBuilder;
+    /**
+     * `"cascade"`: the second judge (decision model, `APO_SECOND_JUDGE_MODEL`)
+     * answers first and a confident verdict (native confidence ≥ 0.95) stands
+     * without calling the primary judge — the cost tier. Unset = the default
+     * dual behavior (second judge as parallel, evidence-only shadow).
+     */
+    mode?: "cascade";
 };
 /**
  * What a check knows about its scope when it invokes the judge — the
@@ -1163,6 +1345,20 @@ interface TestContext {
     minTokens(n: number, opts?: TokenBudgetOptions): void;
     /** Escape hatch for a named predicate over the complete normalized run. */
     assert(label: string, predicate: (flow: TraceView) => boolean): void;
+    /**
+     * Asserts steer n was delivered AND consumed: at least one generation
+     * observation started after the steer's delivery timestamp. Fails closed —
+     * a trace with no steering evidence at all is a failure, never a pass.
+     */
+    steerDelivered(n: number): void;
+    /**
+     * Scopes trace-based assertions to what the agent did after steer n: the
+     * fn receives a TestContext over `view.windowAfterSteer(n)`, and every
+     * existing matcher works unchanged inside it. When the steer is missing
+     * from the trace, records one failure and skips fn (inner assertions must
+     * not run — they would pass vacuously against an empty window).
+     */
+    afterSteer(n: number, fn: (t: TestContext) => void): void;
     /** Grades any value (a deliverable, parsed JSON, anything) with a matcher. */
     check<T>(value: T, matcher: Matcher<T>, label?: string): void;
     /**
@@ -1238,7 +1434,7 @@ interface TestContext {
  * adding a method to the interface without registering it here is a type
  * error, and vice versa. See ``tests/test-context-methods.test.ts``.
  */
-declare const TEST_METHOD_NAMES: readonly ["calledTool", "notCalledTool", "toolOrder", "usedNoTools", "maxToolCalls", "noFailedActions", "loadedSkill", "calledSubagent", "messageIncludes", "maxTurns", "maxDurationMs", "maxTokens", "minTokens", "assert", "check", "judge", "agent"];
+declare const TEST_METHOD_NAMES: readonly ["calledTool", "notCalledTool", "toolOrder", "usedNoTools", "maxToolCalls", "noFailedActions", "loadedSkill", "calledSubagent", "messageIncludes", "maxTurns", "maxDurationMs", "maxTokens", "minTokens", "steerDelivered", "afterSteer", "assert", "check", "judge", "agent"];
 
 type TaskDefinition<TAdapterName extends string = string, TDeliverable extends string = string> = {
     id: string;
@@ -1516,6 +1712,65 @@ interface PreparedTaskResult {
  */
 declare function persistFileArtifacts(deliverables: Record<string, unknown>, config: ArtifactUploadConfig): Promise<PreparedTaskResult>;
 
+/**
+ * Second grader: a typed-decision model (System One / Jev) that grades the
+ * same deliverable + instruction as the primary LLM judge, in parallel.
+ *
+ * Why a second grader at all: the primary judge's verdict is the product,
+ * but its trustworthiness is invisible. A non-generative decision model
+ * cannot fabricate quoted evidence the way text judges measurably do, and
+ * its calibrated confidence acts as a difficulty meter — so agreement +
+ * confidence turn every check into self-describing evidence (auto-verified /
+ * borderline / disputed) at ~1/100th of a cent per call.
+ *
+ * The second opinion NEVER changes the verdict. It is opt-in
+ * (`APO_SECOND_JUDGE_MODEL`), runs on a decisions endpoint (not
+ * chat/completions — decision models reject it), and any failure is
+ * recorded as `error` on the evidence instead of affecting the check.
+ *
+ * Providers: every decisions endpoint speaks its own request/response
+ * dialect, selected via `APO_SECOND_JUDGE_PROVIDER` (or auto-detected from
+ * the base URL). Adding a provider means adding one object to `PROVIDERS`
+ * — nothing else in this file or the caller changes.
+ */
+
+/** How a recorded second-judge opinion relates to the check's verdict. */
+type SecondJudgeFacts = {
+    kind: "none";
+} | {
+    kind: "error";
+} | {
+    kind: "skipped";
+} | {
+    kind: "split";
+    confidence: number;
+} | {
+    kind: "agree";
+    confidence: number;
+} | {
+    kind: "unsure";
+    confidence: number;
+};
+/**
+ * Structural input for {@link secondJudgeFacts}: any check shape carrying a
+ * verdict and optional judge evidence — the SDK's own results and the wire
+ * types the CLI and dashboard read from the backend are all assignable.
+ */
+type SecondJudgeFactsInput = {
+    pass: boolean;
+    judge?: {
+        secondJudge?: {
+            choice?: "pass" | "fail";
+            confidence?: number;
+            skipped?: string;
+            error?: string;
+        };
+    };
+};
+declare function secondJudgeFacts(check: SecondJudgeFactsInput): SecondJudgeFacts;
+/** One honest sentence about the relation; null when corroborated. */
+declare function secondJudgeTakeaway(check: SecondJudgeFactsInput): string | null;
+
 /**
  * Check submission compaction (issue #175).
  *
@@ -1721,6 +1976,8 @@ type RunTaskOptions = {
      */
     judgeTools?: JudgeToolsConfig;
     onTurn?: (turnNumber: number, userAction: TaskTranscriptTurn["userAction"], agentResponse: unknown) => void;
+    /** Called for every steer outcome (delivered / undelivered / error). */
+    onSteer?: (record: SteerRecord) => void;
     /**
      * Skip the internal `loadTask(taskDir)` call and reuse an already-loaded
      * task definition.
@@ -1741,6 +1998,20 @@ type RunTaskOptions = {
 };
 declare function runTask(taskDir: string, options?: RunTaskOptions): Promise<TaskRunResult>;
 
+/**
+ * Issue #8: shown when a run ends with zero registered checks. A bare
+ * `FAIL <task>` with no Checks section looked like a real failure but was
+ * almost always a silent registration bug (e.g. a double-import that wiped
+ * the check registry). Naming `test()` matches the documented registration
+ * function — `apps/docs` reference/task.md.
+ *
+ * Lives next to {@link aggregateResult} because the empty-checks verdict is
+ * the single source of truth for "why did this run fail with no checks?".
+ * The CLI and the standalone e2e runner both import this so the wording
+ * stays in one place.
+ */
+declare const NO_CHECKS_REGISTERED_MESSAGE = "No tests were registered by the eval module \u2014 a task must define at least one test().";
+
 /**
  * Issue #159: replay Phase 2 of a completed Run against its stored
  * Deliverables — without re-running the agent.
@@ -2328,4 +2599,4 @@ type TranscriptCaptureResult = {
 };
 declare function replayAdapterTranscript(capture: AdapterTranscriptCapture, options: ReplayAdapterTranscriptOptions): Promise<TranscriptCaptureResult>;
 
-export { type AISDKResult, type AdapterDefinition, type AdapterRuntimeState, type AdapterSession, type AdapterTranscriptCapture, type AgentBudget, type AgentEvidence, type AgentJudgeOptions, type AgentJudgeOutcome, type AgentJudgeSession, type AgentJudgeStep, type AgentTaskCliOptions, type AgentTaskRunConfiguration, AgentTaskRunError, type AgentTaskRunSummary, type AgentTaskRuntime, AgentTaskTraceContext, AgentTaskTraceOptions, type AgentTurnResult, type AnthropicMessage, type ApoRunContext, ApoSpanProcessor, type ArtifactUploadConfig, type CheckCompaction, type CheckContext, type CheckGroup, type CleanupContext, type CollectDeliverablesContext, type CollectedDeliverables, type DeliverableDefinition, type DeliverableSummary, type DeliverableValidationResult, type DescribeRegistration, type DurationBudgetOptions, type EvidenceFingerprint, type ExportOtlpTracesOptions, type FileArtifact, type FileArtifactOptions, type FileEntry, type Flow, type FlowEvent, FlowView, type InitializeContext, JUDGE_SEGMENT_LIMIT, type JudgeConfig, type JudgeContext, type JudgePromptBuilder, type JudgeScope, type JudgeToolsConfig, type LoadedTask, type ManifestFile, type ManifestFileInput, type Matcher, type McpHttpTransport, type McpServerConfig, type McpStdioTransport, type ModeClass, type NameMatcher, type OpenAIMessage, type OtlpAttribute, type OtlpAttributeValue, type OtlpSpan, type OtlpTracesPayload, type ParsedTranscriptSession, type PreparedTaskResult, RECEIVED_VALUE_LIMIT, type RegisterApoTracingOptions, type RejudgeCheckStability, type RejudgeEndpoints, RejudgeError, type RejudgeOptions, type RejudgeOutcome, type ReplayAdapterTranscriptOptions, type RunTaskOptions, type StartSessionContext, TEST_METHOD_NAMES, type TaskConfig, type TaskDefinition, type TaskEvaluationResult, TaskFiles, type TaskRevisionManifestV1, type TaskRunResult, type TaskScope, type TaskTranscript, type TaskTranscriptTurn, type TestContext, type TestRegistration, type TokenBudgetOptions, type ToolCallOptions, type ToolCallStatus, type TranscriptCaptureResult, TranscriptReplayError, type TranscriptReplayOptions, type TranscriptSource, type TranscriptToolCall, type TranscriptTurn, type TranscriptUsage, type TurnContext, type TurnFn, type TurnRecord, type TypedAdapterDefinition, type ValueMatcher, buildApoAuthHeaders, buildManifest, canonicalManifestJson, compactChecksForSubmission, connectMcpServers, contentSha256, defineAdapter, defineTask, describe, detectTranscriptSource, discoverAgentTaskDirs, equals, expandSecretPlaceholders, exportOtlpTraces, fileArtifact, filePaths, fromAISDK, fromAnthropicMessages, fromOpenAIMessages, getActiveApoRun, getTaskTurn, includes, isFileArtifact, loadTask, loadTaskRuntime, matchValue, matches, normalizeManifestPath, parseAgentTaskCliArgs, parseClaudeCodeTranscript, parseCodexTranscript, persistFileArtifacts, registerApoTracing, rejudgeTaskRun, replayAdapterTranscript, resetApoTracing, resetFlowChecks, resetTaskRegistry, resetTaskTurn, resolveMcpServerPaths, resolveOtlpTracesUrl, resolveTurn, runAgentTaskCli, runTask, runTaskDir, satisfies, sha256Hex, similarity, task, defineCheck as test, transcriptSessionToOtlp, turn, withApoRun, withApoRunSync };
+export { type AISDKResult, type AdapterDefinition, type AdapterRuntimeState, type AdapterSession, type AdapterTranscriptCapture, type AgentBudget, type AgentEvidence, type AgentJudgeOptions, type AgentJudgeOutcome, type AgentJudgeSession, type AgentJudgeStep, type AgentProgressEvent, type AgentTaskCliOptions, type AgentTaskRunConfiguration, AgentTaskRunError, type AgentTaskRunSummary, type AgentTaskRuntime, AgentTaskTraceContext, AgentTaskTraceOptions, type AgentTurnResult, type AnthropicMessage, type ApoRunContext, ApoSpanProcessor, type ArtifactUploadConfig, type CheckCompaction, type CheckContext, type CheckGroup, type CleanupContext, type CollectDeliverablesContext, type CollectedDeliverables, type DeliverableDefinition, type DeliverableSummary, type DeliverableValidationResult, type DescribeRegistration, type DurationBudgetOptions, type EvidenceFingerprint, type ExportOtlpTracesOptions, type FileArtifact, type FileArtifactOptions, type FileEntry, type Flow, type FlowEvent, FlowView, type InitializeContext, JUDGE_SEGMENT_LIMIT, type JudgeConfig, type JudgeContext, type JudgePromptBuilder, type JudgeScope, type JudgeToolsConfig, type LoadedTask, type ManifestFile, type ManifestFileInput, type Matcher, type McpHttpTransport, type McpServerConfig, type McpStdioTransport, type ModeClass, NO_CHECKS_REGISTERED_MESSAGE, type NameMatcher, type OpenAIMessage, type OtlpAttribute, type OtlpAttributeValue, type OtlpSpan, type OtlpTracesPayload, type ParsedTranscriptSession, type PreparedTaskResult, RECEIVED_VALUE_LIMIT, type RegisterApoTracingOptions, type RejudgeCheckStability, type RejudgeEndpoints, RejudgeError, type RejudgeOptions, type RejudgeOutcome, type ReplayAdapterTranscriptOptions, type RunTaskOptions, type SecondJudgeFacts, type SecondJudgeFactsInput, type StartSessionContext, type SteerContext, type SteerDeliveryBoundary, type SteerRecord, type SteerResult, type SteerSpec, type SteerTrigger, TEST_METHOD_NAMES, type TaskConfig, type TaskDefinition, type TaskEvaluationResult, TaskFiles, type TaskRevisionManifestV1, type TaskRunResult, type TaskScope, type TaskTranscript, type TaskTranscriptTurn, type TestContext, type TestRegistration, type TokenBudgetOptions, type ToolCallOptions, type ToolCallStatus, type TranscriptCaptureResult, TranscriptReplayError, type TranscriptReplayOptions, type TranscriptSource, type TranscriptToolCall, type TranscriptTurn, type TranscriptUsage, type TurnContext, type TurnFn, type TurnRecord, type TypedAdapterDefinition, type ValueMatcher, buildApoAuthHeaders, buildManifest, canonicalManifestJson, compactChecksForSubmission, connectMcpServers, contentSha256, defineAdapter, defineTask, describe, detectTranscriptSource, discoverAgentTaskDirs, equals, expandSecretPlaceholders, exportOtlpTraces, fileArtifact, filePaths, fromAISDK, fromAnthropicMessages, fromOpenAIMessages, getActiveApoRun, getTaskSteers, getTaskTurn, includes, isFileArtifact, loadTask, loadTaskRuntime, matchValue, matches, normalizeManifestPath, parseAgentTaskCliArgs, parseClaudeCodeTranscript, parseCodexTranscript, persistFileArtifacts, registerApoTracing, rejudgeTaskRun, replayAdapterTranscript, resetApoTracing, resetFlowChecks, resetTaskRegistry, resetTaskSteers, resetTaskTurn, resolveMcpServerPaths, resolveOtlpTracesUrl, resolveTurn, runAgentTaskCli, runTask, runTaskDir, satisfies, secondJudgeFacts, secondJudgeTakeaway, sha256Hex, similarity, steer, task, defineCheck as test, transcriptSessionToOtlp, turn, withApoRun, withApoRunSync };