@apo-ai/sdk 0.9.0 → 0.9.1

package/dist/agent-task/public.d.ts

@@ -822,6 +822,111 @@ type AgentHistoryPlane = {
     selfRunId?: string;
 };
 
+/**
+ * MCP plumbing for apo's two tool planes. The judge's budgeted toolset
+ * (`createMcpToolset`) wraps user-declared MCP servers into `t.agent`
+ * sessions under the same budget ledger and evidence-manifest discipline as
+ * the built-in evidence tools. `connectMcpServers` is the raw connect core
+ * shared with adapters — the agent under test is not apo's to budget, so
+ * adapters merge the namespaced `mcp__<server>__<tool>` tools themselves.
+ *
+ * Judge config layers exactly like the judge model config
+ * (`resolveJudgeConfig`): env/file (APO_JUDGE_MCP) ← runTask({ judgeTools })
+ * ← task.judgeTools ← per-call `t.agent(..., { tools: { mcp } })` — most
+ * specific wins, arrays replace, never concat. Task-level `mcpServers`
+ * (adapter plane) is a separate declaration honored by adapters that choose
+ * to (see TaskDefinition.mcpServers).
+ *
+ * The MCP client (`@ai-sdk/mcp`) is lazy-loaded: suites that never configure
+ * MCP servers must never pay for the dependency (pinned by the
+ * lazy-isolation test in tests/judge-mcp.test.ts).
+ */
+
+/** Stdio transport: the judge's machine spawns the server process. */
+type McpStdioTransport = {
+    type: "stdio";
+    command: string;
+    args?: string[];
+    /**
+     * Environment for the spawned process. Values support `${VAR}` expansion
+     * from the judge process's environment at connect time; an unset variable
+     * is a load error, never an empty string.
+     */
+    env?: Record<string, string>;
+};
+/** HTTP transport: an already-running remote server (Streamable HTTP). */
+type McpHttpTransport = {
+    type: "http";
+    url: string;
+    /** Header values support `${VAR}` expansion like stdio env values. */
+    headers?: Record<string, string>;
+};
+/** One user MCP server the judge may call. Mirrors the industry shape. */
+type McpServerConfig = {
+    /** Unique server name; its tools are exposed as `mcp__<name>__<tool>`. */
+    name: string;
+    transport: McpStdioTransport | McpHttpTransport;
+    /** Allowlist of raw server tool names (applied before namespacing). */
+    tools?: string[];
+    /** Denylist applied after the allowlist. */
+    excludeTools?: string[];
+    /** Per-tool-call timeout in ms. Default 30_000. */
+    timeoutMs?: number;
+};
+/** Layered judge-tools config, sibling of `JudgeConfig`. */
+type JudgeToolsConfig = {
+    mcp?: McpServerConfig[];
+};
+/**
+ * Expand `${VAR}` placeholders in secret-bearing string maps (stdio env,
+ * HTTP headers). Unset variables throw naming the variable — a missing
+ * credential must fail visibly at load time, never collapse to "".
+ */
+declare function expandSecretPlaceholders(values: Record<string, string>): Record<string, string>;
+/**
+ * Narrow, locally-owned view of an MCP client, mirroring the engine-facade
+ * discipline in agent-session.ts: never let tsc instantiate the library's
+ * generics. The real `@ai-sdk/mcp` client is structurally compatible.
+ */
+type McpRawTool = {
+    description?: unknown;
+    inputSchema?: unknown;
+    execute?: unknown;
+};
+type McpClientLike = {
+    tools: () => Promise<Record<string, McpRawTool>>;
+    close: () => Promise<void>;
+};
+type McpClientFactory = (args: {
+    server: McpServerConfig;
+    timeoutMs: number;
+}) => Promise<McpClientLike>;
+/**
+ * Resolve path-like stdio values against a base directory (the task dir):
+ * absolute values, "./" and "../" prefixes resolve; bare names stay bare so
+ * the spawn resolves them from PATH. Applied to TaskDefinition.mcpServers
+ * by adapters AND to layered judgeTools by runTask — "./mcp/server.mjs"
+ * means the same thing on both planes, independent of the runner's cwd.
+ */
+declare function resolveMcpServerPaths(servers: McpServerConfig[], baseDir: string): McpServerConfig[];
+/**
+ * Connects one client per declared server and returns the RAW namespaced
+ * tools — no budget wrapping, because the agent under test is not apo's to
+ * budget (that discipline is judge-only, in `createMcpToolset` below).
+ * Adapters that drive an AI-SDK agent loop merge `tools` into their own tool
+ * record; `byServer` maps each server name to its exposed raw tool names.
+ *
+ * Connect failures throw naming the server after closing what already
+ * connected. Duplicate server names throw before anything spawns.
+ */
+declare function connectMcpServers(servers: McpServerConfig[], opts?: {
+    clientFactory?: McpClientFactory;
+}): Promise<{
+    tools: Record<string, unknown>;
+    byServer: Record<string, string[]>;
+    cleanup: () => Promise<void>;
+}>;
+
 /**
  * The agentic judge — `t.agent`. A tool-using LLM session that investigates
  * the run's own evidence (deliverables, trace) before verdicting, powered by
@@ -856,6 +961,7 @@ type AgentJudgeOptions = {
     /** Tool families; deliverables are always on. `trace: false` drops get_trace. */
     tools?: {
         trace?: boolean;
+        mcp?: McpServerConfig[];
     };
     budget?: AgentBudget;
     label?: string;
@@ -1042,6 +1148,21 @@ type TaskDefinition<TAdapterName extends string = string, TDeliverable extends s
      * `prompt` that tells the judge what it is grading.
      */
     judge?: Partial<JudgeConfig>;
+    /**
+     * Task-level judge-tools layer: MCP evidence servers for `t.agent` sessions
+     * in this task. Overrides `runTask({ judgeTools })`, is overridden per
+     * `t.agent(..., { tools: { mcp } })` call. Arrays replace, never concat.
+     */
+    judgeTools?: JudgeToolsConfig;
+    /**
+     * MCP servers declared for the AGENT UNDER TEST (the adapter plane — a
+     * separate declaration from `judgeTools`). Adapters that honor it resolve
+     * relative `command`/`args` paths against the task directory and expose
+     * the tools as `mcp__<server>__<tool>`; trace assertions can match those
+     * stable names. Declaring the harness's tool surface here keeps the task
+     * portable across harnesses.
+     */
+    mcpServers?: McpServerConfig[];
 };
 type TaskConfig<TDeliverable extends string = string> = Omit<TaskDefinition<string, TDeliverable>, "adapter">;
 type FileEntry = {
@@ -1486,6 +1607,11 @@ type RunTaskOptions = {
     tracing?: AgentTaskTraceOptions;
     /** LLM judge model config for `t.judge(...)` calls in the task checks. */
     judge?: JudgeConfig;
+    /**
+     * Judge tool config for `t.agent(...)` sessions (MCP evidence servers).
+     * Layered under `TaskDefinition.judgeTools` and per-call `tools.mcp`.
+     */
+    judgeTools?: JudgeToolsConfig;
     onTurn?: (turnNumber: number, userAction: TaskTranscriptTurn["userAction"], agentResponse: unknown) => void;
     /**
      * Skip the internal `loadTask(taskDir)` call and reuse an already-loaded
@@ -1609,6 +1735,7 @@ declare function discoverAgentTaskDirs(rootDir: string): string[];
 
 type AgentTaskRuntime = {
     judge?: JudgeConfig;
+    judgeTools?: JudgeToolsConfig;
 };
 type AgentTaskRunSummary = {
     taskDir: string;
@@ -1870,4 +1997,204 @@ declare function canonicalManifestJson(manifest: TaskRevisionManifestV1): string
 /** SHA-256 (lowercase hex) of the canonical manifest JSON's UTF-8 bytes. */
 declare function contentSha256(manifest: TaskRevisionManifestV1): string;
 
-export { type AISDKResult, type AdapterDefinition, type AdapterRuntimeState, type AdapterSession, type AgentBudget, type AgentEvidence, type AgentJudgeOptions, type AgentJudgeOutcome, type AgentJudgeSession, type AgentJudgeStep, type AgentTaskCliOptions, type AgentTaskRunConfiguration, AgentTaskRunError, type AgentTaskRunSummary, type AgentTaskRuntime, AgentTaskTraceContext, AgentTaskTraceOptions, type AgentTurnResult, type AnthropicMessage, type ApoRunContext, ApoSpanProcessor, type ArtifactUploadConfig, type CheckCompaction, type CheckContext, type CheckGroup, type CleanupContext, type CollectDeliverablesContext, type CollectedDeliverables, type DeliverableDefinition, type DeliverableSummary, type DeliverableValidationResult, type DescribeRegistration, type DurationBudgetOptions, type EvidenceFingerprint, type FileArtifact, type FileArtifactOptions, type FileEntry, type Flow, type FlowEvent, FlowView, type InitializeContext, JUDGE_SEGMENT_LIMIT, type JudgeConfig, type JudgeContext, type JudgePromptBuilder, type JudgeScope, type LoadedTask, type ManifestFile, type ManifestFileInput, type Matcher, type ModeClass, type NameMatcher, type OpenAIMessage, type PreparedTaskResult, RECEIVED_VALUE_LIMIT, type RegisterApoTracingOptions, type RejudgeCheckStability, type RejudgeEndpoints, RejudgeError, type RejudgeOptions, type RejudgeOutcome, type RunTaskOptions, type StartSessionContext, TEST_METHOD_NAMES, type TaskConfig, type TaskDefinition, type TaskEvaluationResult, TaskFiles, type TaskRevisionManifestV1, type TaskRunResult, type TaskScope, type TaskTranscript, type TaskTranscriptTurn, type TestContext, type TestRegistration, type TokenBudgetOptions, type ToolCallOptions, type ToolCallStatus, type TurnContext, type TurnFn, type TurnRecord, type TypedAdapterDefinition, type ValueMatcher, buildManifest, canonicalManifestJson, compactChecksForSubmission, contentSha256, defineAdapter, defineTask, describe, discoverAgentTaskDirs, equals, fileArtifact, filePaths, fromAISDK, fromAnthropicMessages, fromOpenAIMessages, getActiveApoRun, getTaskTurn, includes, isFileArtifact, loadTask, loadTaskRuntime, matchValue, matches, normalizeManifestPath, parseAgentTaskCliArgs, persistFileArtifacts, registerApoTracing, rejudgeTaskRun, resetApoTracing, resetFlowChecks, resetTaskRegistry, resetTaskTurn, resolveTurn, runAgentTaskCli, runTask, runTaskDir, satisfies, sha256Hex, similarity, task, defineCheck as test, turn, withApoRun, withApoRunSync };
+/**
+ * Normalized transcript model — the common shape every harness transcript
+ * parser produces, and the input to the OTLP translator.
+ *
+ * The model is deliberately lossy-but-sufficient for trace reconstruction:
+ * per turn it keeps the user/assistant text, thinking, tool calls with their
+ * results, model identity, and token usage. Anything the projection cannot
+ * use (streaming chunk boundaries, harness housekeeping) is folded away.
+ */
+type TranscriptSource = "claude-code" | "codex";
+type TranscriptUsage = {
+    inputTokens?: number;
+    outputTokens?: number;
+    cacheReadTokens?: number;
+    cacheWriteTokens?: number;
+    reasoningTokens?: number;
+};
+type TranscriptToolCall = {
+    callId: string;
+    name: string;
+    /** Arguments exactly as recorded — parsed object when the harness allows, raw string otherwise. */
+    input: unknown;
+    /** Rendered result text or the raw output payload; absent when the transcript never recorded one. */
+    result?: unknown;
+    /** ISO timestamp of the call, when the transcript carries one. */
+    startedAt?: string;
+    /** ISO timestamp of the matching result, when the transcript carries one. */
+    endedAt?: string;
+};
+type TranscriptTurn = {
+    /** Zero-based position of the turn within the session. */
+    index: number;
+    /** ISO timestamp of the user message that opened the turn. */
+    startedAt: string;
+    /** ISO timestamp of the last observed event in the turn, when known. */
+    endedAt?: string;
+    model?: string;
+    userMessage?: string;
+    assistantMessage?: string;
+    /** Extended-thinking text (Claude) or reasoning summaries (Codex). */
+    thinkingText?: string;
+    /** Codex commentary-phase messages (progress narration, not reasoning). */
+    commentary?: string[];
+    toolCalls: TranscriptToolCall[];
+    usage?: TranscriptUsage;
+};
+type ParsedTranscriptSession = {
+    source: TranscriptSource;
+    sessionId: string;
+    cwd?: string;
+    turns: TranscriptTurn[];
+    /** Non-fatal parse issues: torn lines, unterminated turns, unmatched outputs. */
+    warnings: string[];
+};
+
+/**
+ * Claude Code session-transcript parser.
+ *
+ * Claude Code appends one JSON object per line to
+ * ``~/.claude/projects/<project>/<session-id>.jsonl`` for every session:
+ *
+ *   { type: "user" | "assistant", timestamp, requestId?, sessionId?, cwd?,
+ *     toolUseResult?, message: { id?, role, model?, usage?, stop_reason?,
+ *     content: string | Array<text | thinking | tool_use | tool_result> } }
+ *
+ * Two grouping rules do the real work:
+ * - Streamed assistant chunks of one API response share ``message.id``; they
+ *   are one logical message, and usage is read from the most complete chunk
+ *   (the first chunk's snapshot undercounts the request).
+ * - A user line whose content is a ``tool_result`` block array is the harness
+ *   feeding a tool result back to the model, not a new turn.
+ */
+
+declare function parseClaudeCodeTranscript(content: string): ParsedTranscriptSession;
+
+/**
+ * Codex CLI session-transcript parser.
+ *
+ * Codex writes rollout files under ``~/.codex/sessions/YYYY/MM/DD/``, one
+ * JSON object per line:
+ *
+ *   { type: "session_meta" | "turn_context" | "event_msg" | "response_item",
+ *     timestamp, payload }
+ *
+ * - ``event_msg`` payloads: ``task_started``, ``task_complete``,
+ *   ``user_message``, ``agent_message`` (phase ``final_answer`` |
+ *   ``commentary``), ``token_count`` (``info.last_token_usage``).
+ * - ``response_item`` payloads: ``function_call`` / ``custom_tool_call`` and
+ *   their ``*_output`` counterparts (paired by ``call_id``), ``reasoning``
+ *   (summary text), ``web_search_call``.
+ *
+ * A turn is bracketed by ``task_started`` … ``task_complete``. A rollout that
+ * ends mid-turn (harness killed, crash) still yields its turn — the activity
+ * happened, the trace should show it — with a warning.
+ */
+
+declare function parseCodexTranscript(content: string): ParsedTranscriptSession;
+
+/**
+ * Transcript → OTLP/JSON translator.
+ *
+ * Converts a parsed session into exactly the span vocabulary apo's receiver
+ * already normalizes best. Two rules make replayed traces indistinguishable
+ * from live-OTel ones in the projection:
+ *
+ * 1. Every span carries an explicit ``apo.observation.type`` override — the
+ *    priority-1 mapper — so typing (AGENT / GENERATION / TOOL / SPAN) never
+ *    depends on the backend's registry guessing from attributes.
+ * 2. Attributes use apo's established GenAI keys (``gen_ai.request.model``,
+ *    ``gen_ai.usage.*`` incl. the cache/reasoning buckets the pricing path
+ *    reads, ``gen_ai.tool.*``, ``gen_ai.input/output.messages``) — not a
+ *    replay-specific dialect.
+ *
+ * IDs are SHA-256-derived from the session identity, so re-exporting the same
+ * transcript is idempotent: the receiver upserts on
+ * ``(project_id, trace_id, span_id)``.
+ */
+
+type OtlpAttributeValue = {
+    stringValue: string;
+} | {
+    intValue: string;
+} | {
+    boolValue: boolean;
+};
+type OtlpAttribute = {
+    key: string;
+    value: OtlpAttributeValue;
+};
+type OtlpSpan = {
+    traceId: string;
+    spanId: string;
+    parentSpanId?: string;
+    name: string;
+    kind: number;
+    startTimeUnixNano: string;
+    endTimeUnixNano: string;
+    attributes: OtlpAttribute[];
+    status: {
+        code: number;
+    };
+};
+type OtlpTracesPayload = {
+    resourceSpans: Array<{
+        resource: {
+            attributes: OtlpAttribute[];
+        };
+        scopeSpans: Array<{
+            scope: {
+                name: string;
+                version: string;
+            };
+            spans: OtlpSpan[];
+        }>;
+    }>;
+};
+type TranscriptReplayOptions = {
+    /** Trace id (32 lowercase hex). Default: deterministic from source + session id. */
+    traceId?: string;
+    /** Parent for the first turn's interaction span — adapter capture mode nests it under the live task.turn span. */
+    parentSpanId?: string;
+    /** Run list label (`apo.run.flow_name`). Default: "<source> session <id prefix>". */
+    flowName?: string;
+    /** Run tags (`apo.run.tags`). Default: ["transcript-replay", source]. */
+    tags?: string[];
+};
+declare function transcriptSessionToOtlp(session: ParsedTranscriptSession, options?: TranscriptReplayOptions): OtlpTracesPayload;
+
+/**
+ * OTLP export for replayed transcripts.
+ *
+ * Posts an OTLP/JSON traces payload to apo's public OTLP endpoint using the
+ * same auth the live exporters use. Auth mode decides what the trace becomes:
+ *
+ * - API key (CLI import): the trace lands as an unclaimed project run.
+ * - Attempt/service token (adapter capture): the replayed spans join the live
+ *   task run's trace — the runner's root span already carries the claim.
+ *
+ * 5xx and network failures are retried; a 4xx is a caller bug (bad token,
+ * malformed payload) and fails fast with the response body excerpt.
+ */
+
+declare class TranscriptReplayError extends Error {
+    readonly status?: number;
+    constructor(message: string, status?: number);
+}
+type ExportOtlpTracesOptions = {
+    /** apo base URL (e.g. http://localhost:8000); a full …/v1/traces URL is accepted verbatim. */
+    endpoint: string;
+    /** API key or attempt token — sent as the OTLP Authorization bearer. */
+    token: string;
+    /** Per-request timeout. Default 30s. */
+    timeoutMs?: number;
+    /** Extra attempts after the first on 5xx/network failure. Default 2. */
+    retries?: number;
+};
+/** Normalize an apo base URL into the public OTLP traces endpoint. */
+declare function resolveOtlpTracesUrl(base: string): string;
+declare function exportOtlpTraces(payload: OtlpTracesPayload, options: ExportOtlpTracesOptions): Promise<void>;
+
+export { type AISDKResult, type AdapterDefinition, type AdapterRuntimeState, type AdapterSession, type AgentBudget, type AgentEvidence, type AgentJudgeOptions, type AgentJudgeOutcome, type AgentJudgeSession, type AgentJudgeStep, type AgentTaskCliOptions, type AgentTaskRunConfiguration, AgentTaskRunError, type AgentTaskRunSummary, type AgentTaskRuntime, AgentTaskTraceContext, AgentTaskTraceOptions, type AgentTurnResult, type AnthropicMessage, type ApoRunContext, ApoSpanProcessor, type ArtifactUploadConfig, type CheckCompaction, type CheckContext, type CheckGroup, type CleanupContext, type CollectDeliverablesContext, type CollectedDeliverables, type DeliverableDefinition, type DeliverableSummary, type DeliverableValidationResult, type DescribeRegistration, type DurationBudgetOptions, type EvidenceFingerprint, type ExportOtlpTracesOptions, type FileArtifact, type FileArtifactOptions, type FileEntry, type Flow, type FlowEvent, FlowView, type InitializeContext, JUDGE_SEGMENT_LIMIT, type JudgeConfig, type JudgeContext, type JudgePromptBuilder, type JudgeScope, type JudgeToolsConfig, type LoadedTask, type ManifestFile, type ManifestFileInput, type Matcher, type McpHttpTransport, type McpServerConfig, type McpStdioTransport, type ModeClass, type NameMatcher, type OpenAIMessage, type OtlpAttribute, type OtlpAttributeValue, type OtlpSpan, type OtlpTracesPayload, type ParsedTranscriptSession, type PreparedTaskResult, RECEIVED_VALUE_LIMIT, type RegisterApoTracingOptions, type RejudgeCheckStability, type RejudgeEndpoints, RejudgeError, type RejudgeOptions, type RejudgeOutcome, type RunTaskOptions, type StartSessionContext, TEST_METHOD_NAMES, type TaskConfig, type TaskDefinition, type TaskEvaluationResult, TaskFiles, type TaskRevisionManifestV1, type TaskRunResult, type TaskScope, type TaskTranscript, type TaskTranscriptTurn, type TestContext, type TestRegistration, type TokenBudgetOptions, type ToolCallOptions, type ToolCallStatus, TranscriptReplayError, type TranscriptReplayOptions, type TranscriptSource, type TranscriptToolCall, type TranscriptTurn, type TranscriptUsage, type TurnContext, type TurnFn, type TurnRecord, type TypedAdapterDefinition, type ValueMatcher, buildManifest, canonicalManifestJson, compactChecksForSubmission, connectMcpServers, contentSha256, defineAdapter, defineTask, describe, discoverAgentTaskDirs, equals, expandSecretPlaceholders, exportOtlpTraces, fileArtifact, filePaths, fromAISDK, fromAnthropicMessages, fromOpenAIMessages, getActiveApoRun, getTaskTurn, includes, isFileArtifact, loadTask, loadTaskRuntime, matchValue, matches, normalizeManifestPath, parseAgentTaskCliArgs, parseClaudeCodeTranscript, parseCodexTranscript, persistFileArtifacts, registerApoTracing, rejudgeTaskRun, resetApoTracing, resetFlowChecks, resetTaskRegistry, resetTaskTurn, resolveMcpServerPaths, resolveOtlpTracesUrl, resolveTurn, runAgentTaskCli, runTask, runTaskDir, satisfies, sha256Hex, similarity, task, defineCheck as test, transcriptSessionToOtlp, turn, withApoRun, withApoRunSync };