@apo-ai/sdk 0.8.0 → 0.9.0

package/README.md

@@ -216,7 +216,8 @@ exposes only the deliverables selected by that task.
 `calledTool(name, opts?)`, `notCalledTool(name, opts?)`, `toolOrder([...])`,
 `usedNoTools()`, `maxToolCalls(n)`, `noFailedActions()`, `loadedSkill(skill)`,
 `calledSubagent(agent)`, `messageIncludes(token)`, `maxTurns(n)`,
-`maxDurationMs(n)`, `assert(label, predicate)`.
+`maxDurationMs(n, { turn? })`, `maxTokens(n, { turn?, kind? })`,
+`minTokens(n, { turn?, kind? })`, `assert(label, predicate)`.
 
 `calledTool` / `notCalledTool` take optional constraints to match a call's
 **input**, **output**, and **status** (and `count` for an exact number). Each

package/dist/agent-task/integrations/ai-sdk.js

@@ -2,8 +2,8 @@
 // Do not edit by hand — edit the src/ originals and rebuild.
 import {
   createApoTracer
-} from "../../chunk-CKB3SUDO.js";
-import "../../chunk-HLRXQAZ5.js";
+} from "../../chunk-IK4GJX7H.js";
+import "../../chunk-HRYXLRXJ.js";
 export {
   createApoTracer
 };

package/dist/agent-task/public.d.ts

@@ -47,6 +47,19 @@ interface TraceProjectionCapabilities {
     timing: EvidenceAvailability;
     skills: EvidenceAvailability;
     subagents: EvidenceAvailability;
+    /**
+     * Token usage on observations. Optional for snapshots written before usage
+     * was projected: absent reads as `unavailable`.
+     */
+    usage?: EvidenceAvailability;
+}
+/**
+ * Token counts one observation reported. Each dimension is present only when
+ * the source reported it — unknown stays unknown, never zero.
+ */
+interface TraceProjectionUsage {
+    inputTokens?: number;
+    outputTokens?: number;
 }
 /** One chat message reconstructed from a generation observation. */
 interface TraceProjectionMessage {
@@ -75,6 +88,8 @@ interface TraceProjectionObservation {
     toolParameters?: unknown;
     toolResult?: unknown;
     messages?: readonly TraceProjectionMessage[];
+    /** Token usage this observation reported. Absent when none was recorded. */
+    usage?: TraceProjectionUsage;
     metadata?: Readonly<Record<string, unknown>>;
 }
 /** Trace-level facts. All timing is optional; absence is honest. */
@@ -139,6 +154,34 @@ interface TraceSubagentCall {
     status: ObservationStatus;
     startedAt?: string;
 }
+/** One Task Turn, derived from a `task.turn` observation. */
+interface TraceTurn {
+    /** 1-based, in invocation order. */
+    turnNumber: number;
+    spanId: string;
+    durationMs?: number;
+    status: ObservationStatus;
+}
+type TokenKind = "input" | "output" | "total";
+/**
+ * Token usage summed over a scope (the whole agent execution, or one turn).
+ * When `unreported` is non-zero, `tokens` is only a lower bound.
+ */
+interface TraceTokenTally {
+    tokens: number;
+    /**
+     * Observations that reported a count, including a step whose count its
+     * parent call's own larger count superseded.
+     */
+    reported: number;
+    /**
+     * LLM calls in scope whose count is unknown or untrustworthy: a GENERATION
+     * that reported no usage for the dimension, or an errored call (a provider
+     * error often drops the final usage event, so its count may be short) —
+     * except a step beneath a call whose own count is complete, which covers it.
+     */
+    unreported: number;
+}
 declare class TraceView {
     readonly snapshot: TraceProjectionSnapshot;
     constructor(snapshot: TraceProjectionSnapshot);
@@ -174,6 +217,34 @@ declare class TraceView {
      * startedAt/endedAt when present.
      */
     get durationMs(): number | undefined;
+    /**
+     * Task Turns in invocation order, or `undefined` when timing evidence is
+     * not available. Each is one `task.turn` span: the adapter's whole
+     * `sendUserTurn` call for that turn.
+     */
+    get turns(): readonly TraceTurn[] | undefined;
+    /**
+     * Tokens the agent under test spent: usage on observations inside
+     * `task.turn` spans (every turn, or only turn `turn`). Work outside the
+     * turns — the evaluation phase's judges, adapter setup — is not the
+     * agent's and is excluded. `undefined` when usage evidence is not
+     * available or the requested turn does not exist.
+     */
+    /** How many Task Turns ran — independent of timing evidence. */
+    get turnCountFromSpans(): number;
+    tokens(kind: TokenKind, opts?: {
+        turn?: number;
+    }): TraceTokenTally | undefined;
+    /**
+     * The run's own Task Turns: `task.turn` observations with no `task.turn`
+     * ancestor (a nested run or a user span of the same name inside a turn is
+     * part of that turn, not a turn of its own). Turns run sequentially, so
+     * invocation order is turn order.
+     */
+    private get turnSpans();
+    private _turnSpans;
+    private get childrenByParent();
+    private _children;
     /**
      * Observations sorted deterministically by invocation time then span ID.
      * Memoized per TraceView instance since the snapshot is immutable.
@@ -381,7 +452,11 @@ type JudgeMetadata = {
 /**
  * Evidence from the second grader. `error` is set when the decisions
  * endpoint could not be reached — a failed second opinion must never
- * affect the check's verdict.
+ * affect the check's verdict. `skipped` is set when no verdict was
+ * attempted or possible because the second judge could not read the
+ * input — a different fact from a transport failure, and rendered
+ * distinctly so a run can say "N checks had no second opinion because
+ * the value was too large" without grepping error strings (issue #311).
  */
 type SecondJudgeEvidence = {
     /** Decision-model id that graded, e.g. `typesafe/jev-1.13`. */
@@ -398,8 +473,20 @@ type SecondJudgeEvidence = {
     costUsd?: number;
     /** Wall-clock latency of the decision call in milliseconds. */
     latencyMs?: number;
+    /**
+     * Why no verdict was possible (as opposed to attempted): the input
+     * exceeded the second judge model's context limit, so the provider
+     * rejected it before judging.
+     */
+    skipped?: string;
     /** Why no verdict was recorded (transport/HTTP/parse failure). */
     error?: string;
+    /**
+     * True when the second judge graded a `secondJudgeValue` projection
+     * instead of the full value the primary judge saw — the verdict
+     * corroborates the projection, not the whole deliverable.
+     */
+    projected?: boolean;
 };
 /**
  * One investigation step of an agentic judge session. Tool results are
@@ -590,6 +677,12 @@ type EvaluationItemResult = {
 type TaskEvaluationResult = {
     checks: EvaluationItemResult[];
     pass: boolean;
+    /**
+     * Set when the run has no verdict: every failing check got no answer from
+     * the judge (`outcome: "error"`) and nothing genuinely failed. `pass` stays
+     * `false`; readers should report "no verdict" rather than FAIL (issue #323).
+     */
+    noVerdict?: true;
 };
 type TaskTranscript = {
     turns: TaskTranscriptTurn[];
@@ -797,6 +890,19 @@ type ToolCallOptions = {
     output?: ValueMatcher<unknown>;
     status?: "ok" | "error";
 };
+/** Scope for duration budgets: the whole run, or one Task Turn (1-based). */
+type DurationBudgetOptions = {
+    turn?: number;
+};
+/**
+ * Scope for token budgets. Tokens are the agent's: usage inside `task.turn`
+ * spans, summed over every turn or only `turn`. `kind` defaults to `"total"`
+ * (input + output).
+ */
+type TokenBudgetOptions = {
+    turn?: number;
+    kind?: TokenKind;
+};
 /**
  * LLM judge model config for `t.judge(...)` calls in checks.ts. Threaded
  * through `runTask({ judge })` from the caller — the runner has no opinion
@@ -850,8 +956,15 @@ interface TestContext {
     messageIncludes(token: string | RegExp): void;
     /** Asserts the run took at most `n` turns — anti-flail. */
     maxTurns(n: number): void;
-    /** Asserts the run took at most `n` milliseconds — anti-flail. */
-    maxDurationMs(n: number): void;
+    /**
+     * Asserts the run took at most `n` milliseconds — anti-flail. With
+     * `{ turn }`, bounds that one Task Turn instead.
+     */
+    maxDurationMs(n: number, opts?: DurationBudgetOptions): void;
+    /** Asserts the agent spent at most `n` tokens (whole run, or `{ turn }`). */
+    maxTokens(n: number, opts?: TokenBudgetOptions): void;
+    /** Asserts the agent spent at least `n` tokens (whole run, or `{ turn }`). */
+    minTokens(n: number, opts?: TokenBudgetOptions): void;
     /** Escape hatch for a named predicate over the complete normalized run. */
     assert(label: string, predicate: (flow: TraceView) => boolean): void;
     /** Grades any value (a deliverable, parsed JSON, anything) with a matcher. */
@@ -867,10 +980,22 @@ interface TestContext {
      * merging field-by-field onto the config threaded in via ``runTask({ judge })``
      * — set ``{ model }`` to escalate a finicky criterion to a stronger model
      * without switching the whole run. Absent fields inherit from the base config.
+     *
+     * ``opts.secondJudgeValue`` grades a smaller view with the second judge
+     * only, when the full value exceeds its context limit: the primary judge
+     * still sees all of ``value``, so other criteria that need the complete
+     * deliverable are unaffected.
      */
     judge(values: unknown | unknown[], instruction: string, opts?: {
         label?: string;
         judge?: Partial<JudgeConfig>;
+        /**
+         * What the second judge reads instead of ``values`` — e.g. the
+         * tracked-changes section of a redline when the whole marked-up
+         * document is too large for the second judge's context. Absent,
+         * the second judge sees exactly what the primary judge sees.
+         */
+        secondJudgeValue?: unknown | unknown[];
     }): Promise<void>;
     /**
      * Agentic judge: a tool-using LLM session that investigates the run's own
@@ -900,7 +1025,7 @@ interface TestContext {
  * adding a method to the interface without registering it here is a type
  * error, and vice versa. See ``tests/test-context-methods.test.ts``.
  */
-declare const TEST_METHOD_NAMES: readonly ["calledTool", "notCalledTool", "toolOrder", "usedNoTools", "maxToolCalls", "noFailedActions", "loadedSkill", "calledSubagent", "messageIncludes", "maxTurns", "maxDurationMs", "assert", "check", "judge", "agent"];
+declare const TEST_METHOD_NAMES: readonly ["calledTool", "notCalledTool", "toolOrder", "usedNoTools", "maxToolCalls", "noFailedActions", "loadedSkill", "calledSubagent", "messageIncludes", "maxTurns", "maxDurationMs", "maxTokens", "minTokens", "assert", "check", "judge", "agent"];
 
 type TaskDefinition<TAdapterName extends string = string, TDeliverable extends string = string> = {
     id: string;
@@ -1456,6 +1581,8 @@ interface RejudgeOutcome {
     /** Full check results from the primary (first) sample. */
     checks: EvaluationItemResult[];
     pass: boolean;
+    /** Every failing check got no answer from the judge — see {@link TaskEvaluationResult.noVerdict}. */
+    noVerdict?: true;
     /** Only meaningful with samples > 1; ordered like the primary sample. */
     stability: RejudgeCheckStability[];
     /** False when the trace projection was unreadable — trajectory assertions were unsupported. */
@@ -1487,6 +1614,8 @@ type AgentTaskRunSummary = {
     taskDir: string;
     taskId: string;
     pass: boolean;
+    /** Every failing check got no answer from the judge — see `TaskEvaluationResult.noVerdict`. */
+    noVerdict?: true;
     checks: EvaluationItemResult[];
     /** Adapter that ran the task. Forwarded to the backend when recording locally. */
     adapterName?: string;
@@ -1741,4 +1870,4 @@ declare function canonicalManifestJson(manifest: TaskRevisionManifestV1): string
 /** SHA-256 (lowercase hex) of the canonical manifest JSON's UTF-8 bytes. */
 declare function contentSha256(manifest: TaskRevisionManifestV1): string;
 
-export { type AISDKResult, type AdapterDefinition, type AdapterRuntimeState, type AdapterSession, type AgentBudget, type AgentEvidence, type AgentJudgeOptions, type AgentJudgeOutcome, type AgentJudgeSession, type AgentJudgeStep, type AgentTaskCliOptions, type AgentTaskRunConfiguration, AgentTaskRunError, type AgentTaskRunSummary, type AgentTaskRuntime, AgentTaskTraceContext, AgentTaskTraceOptions, type AgentTurnResult, type AnthropicMessage, type ApoRunContext, ApoSpanProcessor, type ArtifactUploadConfig, type CheckCompaction, type CheckContext, type CheckGroup, type CleanupContext, type CollectDeliverablesContext, type CollectedDeliverables, type DeliverableDefinition, type DeliverableSummary, type DeliverableValidationResult, type DescribeRegistration, type EvidenceFingerprint, type FileArtifact, type FileArtifactOptions, type FileEntry, type Flow, type FlowEvent, FlowView, type InitializeContext, JUDGE_SEGMENT_LIMIT, type JudgeConfig, type JudgeContext, type JudgePromptBuilder, type JudgeScope, type LoadedTask, type ManifestFile, type ManifestFileInput, type Matcher, type ModeClass, type NameMatcher, type OpenAIMessage, type PreparedTaskResult, RECEIVED_VALUE_LIMIT, type RegisterApoTracingOptions, type RejudgeCheckStability, type RejudgeEndpoints, RejudgeError, type RejudgeOptions, type RejudgeOutcome, type RunTaskOptions, type StartSessionContext, TEST_METHOD_NAMES, type TaskConfig, type TaskDefinition, type TaskEvaluationResult, TaskFiles, type TaskRevisionManifestV1, type TaskRunResult, type TaskScope, type TaskTranscript, type TaskTranscriptTurn, type TestContext, type TestRegistration, type ToolCallOptions, type ToolCallStatus, type TurnContext, type TurnFn, type TurnRecord, type TypedAdapterDefinition, type ValueMatcher, buildManifest, canonicalManifestJson, compactChecksForSubmission, contentSha256, defineAdapter, defineTask, describe, discoverAgentTaskDirs, equals, fileArtifact, filePaths, fromAISDK, fromAnthropicMessages, fromOpenAIMessages, getActiveApoRun, getTaskTurn, includes, isFileArtifact, loadTask, loadTaskRuntime, matchValue, matches, normalizeManifestPath, parseAgentTaskCliArgs, persistFileArtifacts, registerApoTracing, rejudgeTaskRun, resetApoTracing, resetFlowChecks, resetTaskRegistry, resetTaskTurn, resolveTurn, runAgentTaskCli, runTask, runTaskDir, satisfies, sha256Hex, similarity, task, defineCheck as test, turn, withApoRun, withApoRunSync };
+export { type AISDKResult, type AdapterDefinition, type AdapterRuntimeState, type AdapterSession, type AgentBudget, type AgentEvidence, type AgentJudgeOptions, type AgentJudgeOutcome, type AgentJudgeSession, type AgentJudgeStep, type AgentTaskCliOptions, type AgentTaskRunConfiguration, AgentTaskRunError, type AgentTaskRunSummary, type AgentTaskRuntime, AgentTaskTraceContext, AgentTaskTraceOptions, type AgentTurnResult, type AnthropicMessage, type ApoRunContext, ApoSpanProcessor, type ArtifactUploadConfig, type CheckCompaction, type CheckContext, type CheckGroup, type CleanupContext, type CollectDeliverablesContext, type CollectedDeliverables, type DeliverableDefinition, type DeliverableSummary, type DeliverableValidationResult, type DescribeRegistration, type DurationBudgetOptions, type EvidenceFingerprint, type FileArtifact, type FileArtifactOptions, type FileEntry, type Flow, type FlowEvent, FlowView, type InitializeContext, JUDGE_SEGMENT_LIMIT, type JudgeConfig, type JudgeContext, type JudgePromptBuilder, type JudgeScope, type LoadedTask, type ManifestFile, type ManifestFileInput, type Matcher, type ModeClass, type NameMatcher, type OpenAIMessage, type PreparedTaskResult, RECEIVED_VALUE_LIMIT, type RegisterApoTracingOptions, type RejudgeCheckStability, type RejudgeEndpoints, RejudgeError, type RejudgeOptions, type RejudgeOutcome, type RunTaskOptions, type StartSessionContext, TEST_METHOD_NAMES, type TaskConfig, type TaskDefinition, type TaskEvaluationResult, TaskFiles, type TaskRevisionManifestV1, type TaskRunResult, type TaskScope, type TaskTranscript, type TaskTranscriptTurn, type TestContext, type TestRegistration, type TokenBudgetOptions, type ToolCallOptions, type ToolCallStatus, type TurnContext, type TurnFn, type TurnRecord, type TypedAdapterDefinition, type ValueMatcher, buildManifest, canonicalManifestJson, compactChecksForSubmission, contentSha256, defineAdapter, defineTask, describe, discoverAgentTaskDirs, equals, fileArtifact, filePaths, fromAISDK, fromAnthropicMessages, fromOpenAIMessages, getActiveApoRun, getTaskTurn, includes, isFileArtifact, loadTask, loadTaskRuntime, matchValue, matches, normalizeManifestPath, parseAgentTaskCliArgs, persistFileArtifacts, registerApoTracing, rejudgeTaskRun, resetApoTracing, resetFlowChecks, resetTaskRegistry, resetTaskTurn, resolveTurn, runAgentTaskCli, runTask, runTaskDir, satisfies, sha256Hex, similarity, task, defineCheck as test, turn, withApoRun, withApoRunSync };