@apo-ai/sdk 0.6.0 → 0.7.1

package/README.md

@@ -1,13 +1,16 @@
 # apo SDK
 
-TypeScript/JavaScript SDK for the [apo](.) backend — an agent testing and
-observability platform. The SDK's entry points:
+TypeScript/JavaScript SDK for [apo](https://github.com/samikuikka/apo), the
+testing and engineering system that turns expert knowledge into executable
+capability specifications for agent harnesses. The task API runs a real harness
+against that definition of done; the tracing API preserves the evidence a human
+or coding agent needs to improve it. The SDK's entry points:
 
 - **`@apo-ai/sdk/otel`** — OpenTelemetry-native tracing. Wrap your LLM calls and
   agent steps so they land in apo as structured spans, then attach scores.
-- **`@apo-ai/sdk/agent-task`** — the agent-task evaluation framework. Define tasks,
-  adapters, and checks; run them against an agent and collect structured
-  results.
+- **`@apo-ai/sdk/agent-task`** — the primary capability-testing surface. Define
+  tasks, adapters, and tests; run them against the real harness and collect a
+  verdict with structured evidence.
 - **`@apo-ai/sdk/agent-task/integrations/{ai-sdk,openai,anthropic}`** —
   per-framework tracing wrappers re-exported for direct import.
 

package/dist/agent-task/integrations/ai-sdk.d.ts

@@ -1,4 +1,4 @@
-import { A as AgentTaskTraceContext } from '../../tracing-CTmq4PSw.js';
+import { A as AgentTaskTraceContext } from '../../tracing-DVLUqD-i.js';
 
 /**
  * Vercel AI SDK tracing integration for apo.

package/dist/agent-task/integrations/ai-sdk.js

@@ -2,7 +2,7 @@
 // Do not edit by hand — edit the src/ originals and rebuild.
 import {
   createApoTracer
-} from "../../chunk-H7ITLBWU.js";
+} from "../../chunk-QHRWPVPZ.js";
 import "../../chunk-HWZVL6TS.js";
 export {
   createApoTracer

package/dist/agent-task/integrations/anthropic.d.ts

@@ -1,4 +1,4 @@
-import { A as AgentTaskTraceContext } from '../../tracing-CTmq4PSw.js';
+import { A as AgentTaskTraceContext } from '../../tracing-DVLUqD-i.js';
 
 /**
  * Anthropic JS SDK tracing integration for apo.

package/dist/agent-task/integrations/openai.d.ts

@@ -1,4 +1,4 @@
-import { A as AgentTaskTraceContext } from '../../tracing-CTmq4PSw.js';
+import { A as AgentTaskTraceContext } from '../../tracing-DVLUqD-i.js';
 
 /**
  * OpenAI JS SDK tracing integration for apo.

package/dist/agent-task/public.d.ts

@@ -1,4 +1,4 @@
-import { A as AgentTaskTraceContext, a as AgentTaskTraceOptions } from '../tracing-CTmq4PSw.js';
+import { A as AgentTaskTraceContext, a as AgentTaskTraceOptions } from '../tracing-DVLUqD-i.js';
 export { CreateApoTracerOptions, createApoTracer } from './integrations/ai-sdk.js';
 export { CreateApoOpenAIOptions, createApoOpenAI } from './integrations/openai.js';
 export { CreateApoAnthropicOptions, createApoAnthropic } from './integrations/anthropic.js';
@@ -350,6 +350,102 @@ type JudgeMetadata = {
     latency_ms?: number;
     /** Temperature or other sampling parameters, if relevant. */
     temperature?: number;
+    /**
+     * Agentic-judge session transcript (`t.agent`). Absent for single-shot
+     * `t.judge` calls. Records every investigation step (tool calls with
+     * truncated results), the session outcome, per-step usage, and the
+     * content-hashed evidence manifest (replay audit backbone).
+     */
+    session?: AgentJudgeSession;
+    /**
+     * Second-grader evidence: a typed-decision model (Jev via OpenRouter's
+     * `/alpha/decisions`) graded the same deliverable + instruction alongside
+     * the primary judge. Opt-in via `APO_SECOND_JUDGE_MODEL`. The primary
+     * verdict is never changed by it — the point is agreement and confidence
+     * as extra signals per check, not a second vote.
+     */
+    secondJudge?: SecondJudgeEvidence;
+};
+/**
+ * Evidence from the second grader. `error` is set when the decisions
+ * endpoint could not be reached — a failed second opinion must never
+ * affect the check's verdict.
+ */
+type SecondJudgeEvidence = {
+    /** Decision-model id that graded, e.g. `typesafe/jev-1.13`. */
+    model: string;
+    /** The typed verdict the model returned. */
+    choice?: "pass" | "fail";
+    /** P(pass) from the model's probability distribution over the choice. */
+    passProbability?: number;
+    /** The model's own confidence in its choice (0-1). */
+    confidence?: number;
+    /** Input tokens billed for the decision, when the provider reports usage. */
+    inputTokens?: number;
+    /** Cost in USD, when the provider reports it. */
+    costUsd?: number;
+    /** Wall-clock latency of the decision call in milliseconds. */
+    latencyMs?: number;
+    /** Why no verdict was recorded (transport/HTTP/parse failure). */
+    error?: string;
+};
+/**
+ * One investigation step of an agentic judge session. Tool results are
+ * truncated text; the sha256 + byte size are always kept so the full value
+ * stays recoverable from the evidence store (the transcript is an index,
+ * not a copy).
+ */
+type AgentJudgeStep = {
+    index: number;
+    tool_calls?: {
+        name: string;
+        input?: string;
+        result?: string;
+        result_sha256?: string;
+        result_bytes?: number;
+    }[];
+    /** Assistant prose, if the model produced text alongside tool calls. */
+    text?: string;
+    tokens?: {
+        input?: number;
+        output?: number;
+        cost?: number;
+    };
+    latency_ms?: number;
+};
+/** How an agentic judge session ended. */
+type AgentJudgeOutcome = "verdict" | "budget_exhausted" | "error";
+/**
+ * A fingerprint of one piece of evidence the judge consumed — the
+ * content-addressed manifest. Never truncated.
+ */
+type EvidenceFingerprint = {
+    step: number;
+    tool: string;
+    args_sha256?: string;
+    result_sha256: string;
+    result_bytes: number;
+};
+/** Transcript-shaped record of one `t.agent` session. */
+type AgentJudgeSession = {
+    /** Tool names offered to the judge. */
+    tools?: string[];
+    /** Turn-0 context: system briefing and the rubric (user message). */
+    briefing?: {
+        system?: string;
+        rubric?: string;
+    };
+    /** Ordered investigation steps. */
+    steps?: AgentJudgeStep[];
+    outcome: AgentJudgeOutcome;
+    /** Evidence manifest — what this session consumed, hashed. Never truncated. */
+    evidence?: EvidenceFingerprint[];
+    usage?: {
+        steps?: number;
+        input_tokens?: number;
+        output_tokens?: number;
+        cache_read_tokens?: number;
+    };
 };
 /**
  * A source location for a failed code check — lets the dashboard render the
@@ -393,7 +489,7 @@ type AssertionResult = {
      */
     received?: unknown;
     location?: CheckLocation;
-    evaluator_type?: "llm" | "code";
+    evaluator_type?: "llm" | "code" | "agent";
     judge?: JudgeMetadata;
 };
 /**
@@ -423,7 +519,7 @@ type EvaluationItemResult = {
      * - ``"regex"`` — pattern matching
      * Older persisted results may still use ``"llm"`` at this level.
      */
-    evaluator_type?: "llm" | "code" | "regex";
+    evaluator_type?: "llm" | "code" | "agent" | "regex";
     /**
      * If this check was judged by an LLM, details about the judge call
      * (model, prompt, response, tokens, cost, latency). Populated by the
@@ -564,6 +660,101 @@ type JudgePromptBuilder = (ctx: JudgeContext) => {
     user?: string;
 };
 
+/** One prior attempt of this task, as frozen at evaluation start. */
+type FrozenRunSummary = {
+    id: string;
+    status: string;
+    passed: boolean | null;
+    started_at?: string;
+    primary_model?: string | null;
+    passed_checks?: number | null;
+    failed_checks?: number | null;
+    corrected_tests?: unknown[];
+    /** True when this entry IS the run under judgment. */
+    is_run_under_judgment?: boolean;
+};
+/** A prior run's full check report (merged view, like the run detail page). */
+type HistoryRunDetail = {
+    id: string;
+    status: string;
+    checks: {
+        id: string;
+        pass: boolean;
+        evaluator: string;
+        reasoning?: string;
+        assertions?: {
+            pass: boolean;
+            expected?: string;
+            received?: string;
+        }[];
+    }[];
+    /** Human corrections recorded against this run; "none" when empty. */
+    human_corrections: unknown[] | "none";
+};
+/** The frozen evidence plane handed to every `t.agent` session. */
+type AgentHistoryPlane = {
+    /** Prior attempts of this task, frozen at evaluation start. */
+    runs: FrozenRunSummary[];
+    /**
+     * Fetch one prior run's merged check report. The run-under-judgment id
+     * returns an honest "you are deciding this run" answer instead of data
+     * whose checks are still being evaluated.
+     */
+    getRun(runId: string): Promise<HistoryRunDetail | {
+        error: string;
+    }>;
+    /** The backend's id for the run under judgment, when known. */
+    selfRunId?: string;
+};
+
+/**
+ * The agentic judge — `t.agent`. A tool-using LLM session that investigates
+ * the run's own evidence (deliverables, trace) before verdicting, powered by
+ * a lazy-loaded Vercel AI SDK engine. The judge semantics live here at the
+ * call site; the loop itself is engine-level: forced tool choice plus a
+ * done-tool verdict means the only exits are a verdict or the budget — every
+ * other ending is recorded as a failure with an explanation.
+ *
+ * Sessions are stateless per check: no memory crosses sessions (upstream
+ * ablations showed remembered judgments cascade errors). Everything the
+ * session consumed is recorded as a content-hashed manifest
+ * ({@link EvidenceFingerprint}) — the transcript is an index, not a copy.
+ */
+
+/** Hard ceilings for one agentic session. Every field overrides a default. */
+type AgentBudget = {
+    /** Max LLM turns (steps). Default 12. */
+    maxTurns?: number;
+    /** Max total tool executions. Default 24. */
+    maxToolCalls?: number;
+    /** Wall-clock ceiling. Default 300_000 ms. */
+    timeoutMs?: number;
+    /** Max cumulative bytes served by read/search tools. Default 2 MiB. */
+    maxReadBytes?: number;
+};
+/** Options for `t.agent(instruction, opts)`. */
+type AgentJudgeOptions = {
+    /** Pre-staged values rendered into turn 0, like `t.judge` values. */
+    exhibits?: unknown | unknown[];
+    /** Judge config override for this call only (merges field-by-field). */
+    judge?: Partial<JudgeConfig>;
+    /** Tool families; deliverables are always on. `trace: false` drops get_trace. */
+    tools?: {
+        trace?: boolean;
+    };
+    budget?: AgentBudget;
+    label?: string;
+};
+/** The local evidence plane a session can investigate. */
+type AgentEvidence = {
+    /** The run's deliverables, by name. */
+    deliverables: Record<string, unknown>;
+    /** The frozen trace projection; absent = no trace evidence plane. */
+    view?: TraceView;
+    /** This task's prior runs; absent = history plane unavailable. */
+    history?: AgentHistoryPlane;
+};
+
 /**
  * The assertion surface — `t`. Flat and eve-style: reads the trace (via
  * {@link TraceView}) for behavioural checks and grades explicit values via
@@ -659,6 +850,21 @@ interface TestContext {
         label?: string;
         judge?: Partial<JudgeConfig>;
     }): Promise<void>;
+    /**
+     * Agentic judge: a tool-using LLM session that investigates the run's own
+     * evidence (deliverables via `read_deliverable`/`search_deliverable`, the
+     * trace via `get_trace`) before verdicting with `finish_verdict`. Async —
+     * the check function must `await` it. Records a single assertion tagged
+     * ``evaluator_type: "agent"`` with the full session transcript attached.
+     *
+     * Unlike `t.judge`, the author states a rubric and the judge gathers its
+     * own evidence; `opts.exhibits` optionally pre-stages values into turn 0.
+     * The session is budgeted (defaults: 12 turns / 24 tool calls / 300 s /
+     * 2 MiB read) and fail-closed: a session that ends without a verdict is a
+     * recorded failure, never a silent pass. Costs more and runs minutes, not
+     * seconds — reserve it for rubrics that need investigation.
+     */
+    agent(instruction: string, opts?: AgentJudgeOptions): Promise<void>;
 }
 /**
  * Canonical names of every assertion method on {@link TestContext}.
@@ -672,7 +878,7 @@ interface TestContext {
  * adding a method to the interface without registering it here is a type
  * error, and vice versa. See ``tests/test-context-methods.test.ts``.
  */
-declare const TEST_METHOD_NAMES: readonly ["calledTool", "notCalledTool", "toolOrder", "usedNoTools", "maxToolCalls", "noFailedActions", "loadedSkill", "calledSubagent", "messageIncludes", "maxTurns", "maxDurationMs", "assert", "check", "judge"];
+declare const TEST_METHOD_NAMES: readonly ["calledTool", "notCalledTool", "toolOrder", "usedNoTools", "maxToolCalls", "noFailedActions", "loadedSkill", "calledSubagent", "messageIncludes", "maxTurns", "maxDurationMs", "assert", "check", "judge", "agent"];
 
 type TaskDefinition<TAdapterName extends string = string, TDeliverable extends string = string> = {
     id: string;
@@ -1505,4 +1711,4 @@ declare function canonicalManifestJson(manifest: TaskRevisionManifestV1): string
 /** SHA-256 (lowercase hex) of the canonical manifest JSON's UTF-8 bytes. */
 declare function contentSha256(manifest: TaskRevisionManifestV1): string;
 
-export { type AISDKResult, type AdapterDefinition, type AdapterRuntimeState, type AdapterSession, type AgentTaskCliOptions, type AgentTaskRunConfiguration, AgentTaskRunError, type AgentTaskRunSummary, type AgentTaskRuntime, AgentTaskTraceContext, AgentTaskTraceOptions, type AgentTurnResult, type AnthropicMessage, type ApoRunContext, ApoSpanProcessor, type ArtifactUploadConfig, type CheckCompaction, type CheckContext, type CheckGroup, type CleanupContext, type CollectDeliverablesContext, type CollectedDeliverables, type DeliverableDefinition, type DeliverableSummary, type DeliverableValidationResult, type DescribeRegistration, type FileArtifact, type FileArtifactOptions, type FileEntry, type Flow, type FlowEvent, FlowView, type InitializeContext, JUDGE_SEGMENT_LIMIT, type JudgeConfig, type JudgeContext, type JudgePromptBuilder, type JudgeScope, type LoadedTask, type ManifestFile, type ManifestFileInput, type Matcher, type ModeClass, type NameMatcher, type OpenAIMessage, type PreparedTaskResult, RECEIVED_VALUE_LIMIT, type RegisterApoTracingOptions, type RejudgeCheckStability, type RejudgeEndpoints, RejudgeError, type RejudgeOptions, type RejudgeOutcome, type RunTaskOptions, type StartSessionContext, TEST_METHOD_NAMES, type TaskConfig, type TaskDefinition, type TaskEvaluationResult, TaskFiles, type TaskRevisionManifestV1, type TaskRunResult, type TaskScope, type TaskTranscript, type TaskTranscriptTurn, type TestContext, type TestRegistration, type ToolCallOptions, type ToolCallStatus, type TurnContext, type TurnFn, type TurnRecord, type TypedAdapterDefinition, type ValueMatcher, buildManifest, canonicalManifestJson, compactChecksForSubmission, contentSha256, defineAdapter, defineTask, describe, discoverAgentTaskDirs, equals, fileArtifact, filePaths, fromAISDK, fromAnthropicMessages, fromOpenAIMessages, getActiveApoRun, getTaskTurn, includes, isFileArtifact, loadTask, loadTaskRuntime, matchValue, matches, normalizeManifestPath, parseAgentTaskCliArgs, persistFileArtifacts, registerApoTracing, rejudgeTaskRun, resetApoTracing, resetFlowChecks, resetTaskRegistry, resetTaskTurn, resolveTurn, runAgentTaskCli, runTask, runTaskDir, satisfies, sha256Hex, similarity, task, defineCheck as test, turn, withApoRun, withApoRunSync };
+export { type AISDKResult, type AdapterDefinition, type AdapterRuntimeState, type AdapterSession, type AgentBudget, type AgentEvidence, type AgentJudgeOptions, type AgentJudgeOutcome, type AgentJudgeSession, type AgentJudgeStep, type AgentTaskCliOptions, type AgentTaskRunConfiguration, AgentTaskRunError, type AgentTaskRunSummary, type AgentTaskRuntime, AgentTaskTraceContext, AgentTaskTraceOptions, type AgentTurnResult, type AnthropicMessage, type ApoRunContext, ApoSpanProcessor, type ArtifactUploadConfig, type CheckCompaction, type CheckContext, type CheckGroup, type CleanupContext, type CollectDeliverablesContext, type CollectedDeliverables, type DeliverableDefinition, type DeliverableSummary, type DeliverableValidationResult, type DescribeRegistration, type EvidenceFingerprint, type FileArtifact, type FileArtifactOptions, type FileEntry, type Flow, type FlowEvent, FlowView, type InitializeContext, JUDGE_SEGMENT_LIMIT, type JudgeConfig, type JudgeContext, type JudgePromptBuilder, type JudgeScope, type LoadedTask, type ManifestFile, type ManifestFileInput, type Matcher, type ModeClass, type NameMatcher, type OpenAIMessage, type PreparedTaskResult, RECEIVED_VALUE_LIMIT, type RegisterApoTracingOptions, type RejudgeCheckStability, type RejudgeEndpoints, RejudgeError, type RejudgeOptions, type RejudgeOutcome, type RunTaskOptions, type StartSessionContext, TEST_METHOD_NAMES, type TaskConfig, type TaskDefinition, type TaskEvaluationResult, TaskFiles, type TaskRevisionManifestV1, type TaskRunResult, type TaskScope, type TaskTranscript, type TaskTranscriptTurn, type TestContext, type TestRegistration, type ToolCallOptions, type ToolCallStatus, type TurnContext, type TurnFn, type TurnRecord, type TypedAdapterDefinition, type ValueMatcher, buildManifest, canonicalManifestJson, compactChecksForSubmission, contentSha256, defineAdapter, defineTask, describe, discoverAgentTaskDirs, equals, fileArtifact, filePaths, fromAISDK, fromAnthropicMessages, fromOpenAIMessages, getActiveApoRun, getTaskTurn, includes, isFileArtifact, loadTask, loadTaskRuntime, matchValue, matches, normalizeManifestPath, parseAgentTaskCliArgs, persistFileArtifacts, registerApoTracing, rejudgeTaskRun, resetApoTracing, resetFlowChecks, resetTaskRegistry, resetTaskTurn, resolveTurn, runAgentTaskCli, runTask, runTaskDir, satisfies, sha256Hex, similarity, task, defineCheck as test, turn, withApoRun, withApoRunSync };