@tangle-network/agent-eval 0.172.1 → 0.173.0

package/dist/types-gvRsyJLh.d.ts

@@ -0,0 +1,831 @@
+import { t as AgentEvalError } from "./errors-DEE6u6ot.js";
+import { b as CustomTokenPricing, c as CostLedgerHandle, g as CostReceiptInput, x as MaximumCharge } from "./cost-ledger-DbQdN3nO.js";
+//#region src/judge-families.d.ts
+/**
+ * Judge model-family classification + cross-family enforcement.
+ *
+ * A judge ensemble built entirely from one provider family shares that
+ * family's blind spots and self-preference — its "agreement" is correlated
+ * bias, not independent signal. `assertCrossFamily` makes the consumer prove
+ * the ensemble spans ≥2 families; `judgeFamily` is the single regex map that
+ * replaces the per-consumer copies (tax/legal/creative/gtm each ship one).
+ */
+/** Provider family a model belongs to. `unknown` when no rule matches. */
+type JudgeFamily = 'anthropic' | 'openai' | 'google' | 'meta' | 'mistral' | 'deepseek' | 'xai' | 'qwen' | 'cohere' | 'amazon' | 'moonshot' | 'zhipu' | 'unknown';
+/**
+ * Classify a model id into its provider family. Strips a `@snapshot` suffix
+ * and prefers an explicit `provider/...` prefix; otherwise matches the model
+ * name. Returns `unknown` when nothing matches (callers decide whether that's
+ * acceptable — `assertCrossFamily` counts it as its own family).
+ */
+declare function judgeFamily(modelId: string): JudgeFamily;
+interface AssertCrossFamilyOptions {
+  /** Minimum number of distinct families the ensemble must span. Default 2. */
+  minFamilies?: number;
+  /** When false (default), `unknown`-family models do NOT count toward the
+   *  family total — an ensemble of all-unclassifiable models is not provably
+   *  cross-family. Set true to count `unknown` as one shared family. */
+  allowUnknown?: boolean;
+}
+declare class CrossFamilyError extends Error {
+  readonly families: JudgeFamily[];
+  readonly models: string[];
+  constructor(message: string, families: JudgeFamily[], models: string[]);
+}
+/**
+ * Throw unless the judge models span at least `minFamilies` distinct provider
+ * families. Pass the model ids backing your judge ensemble. Fail-loud by
+ * design — a correlated single-family ensemble silently inflates agreement.
+ *
+ * Scope: this reads the ids you REQUEST. It proves the panel was configured
+ * across families; it cannot prove the panel RAN across families, because a
+ * routing gateway may answer several different ids from one provider. Where
+ * the diversity claim is load-bearing (a published leaderboard, a
+ * certification, a non-self-judging exclusion), assert on the ids the
+ * provider echoed instead: `assertCrossFamilyServed` in
+ * ./integrity/served-model.
+ */
+declare function assertCrossFamily(models: string[], opts?: AssertCrossFamilyOptions): JudgeFamily[];
+//#endregion
+//#region src/integrity/served-model.d.ts
+/** How a served id relates to the id that was requested. */
+type ServedModelVerdict =
+/** Byte-identical after normalisation — the requested model answered. */
+'exact' |
+/** Same model, different spelling (provider prefix, snapshot, tier suffix). */
+'alias' |
+/** A different model of the SAME provider family answered. */
+'substituted-within-family' |
+/** A different provider's model answered. */
+'substituted-cross-family' |
+/** The response carried no model id — identity is unproven either way. */
+'unreported';
+interface ServedModelCheck {
+  /** The id the caller asked for. */
+  requested: string;
+  /** The id echoed on the response; `null` when the response omitted it. */
+  served: string | null;
+  requestedFamily: JudgeFamily;
+  /** `null` when `served` is null. */
+  servedFamily: JudgeFamily | null;
+  verdict: ServedModelVerdict;
+  /** True for every verdict except `exact` and `alias`. */
+  substituted: boolean;
+}
+/**
+ * Classify one requested/served pair. Pure — no I/O — so it is safe inside
+ * response handlers, reducers, and CI gates.
+ *
+ * `served` is the id echoed by the provider (OpenAI-compatible bodies put it
+ * at `model`). `null`/`undefined` means the body omitted it; that is
+ * `unreported`, NOT a pass — a provider that does not name what answered has
+ * not proven identity, and a transport that drops the field must not read as
+ * agreement.
+ */
+declare function checkServedModel(requested: string, served: string | null | undefined): ServedModelCheck;
+declare class ModelSubstitutionError extends AgentEvalError {
+  readonly checks: ReadonlyArray<ServedModelCheck>;
+  constructor(message: string, checks: ReadonlyArray<ServedModelCheck>);
+}
+/**
+ * Consumer-facing name for the substitution policy a metered surface applies.
+ * `'exact'` rejects every substitution. `'allow-within-family'` accepts a
+ * different model of the same provider family; it keeps family-level claims
+ * valid and forfeits per-model claims. Maps to
+ * `AssertServedModelOptions.allowWithinFamily`.
+ */
+type ServedModelPolicy = 'exact' | 'allow-within-family';
+interface AssertServedModelOptions {
+  /**
+   * Accept a different model of the same provider family (e.g. requested
+   * `deepseek-v3.2`, served `deepseek-v4-flash`). Default false. Setting this
+   * keeps family-level claims valid and forfeits per-model claims.
+   */
+  allowWithinFamily?: boolean;
+  /**
+   * Accept a response that carried no model id. Default false — an
+   * unidentified response cannot support a per-model or per-family claim.
+   */
+  allowUnreported?: boolean;
+  /** Prefixed to the thrown message, e.g. the judge or campaign cell name. */
+  context?: string;
+}
+/**
+ * The one place the accept/reject policy lives, so a caller that reports
+ * substitution (a preflight table, a run record) and a caller that throws on it
+ * can never drift apart. A cross-family substitution is never acceptable.
+ */
+declare function servedModelAcceptable(check: ServedModelCheck, opts?: AssertServedModelOptions): boolean;
+/**
+ * Throw `ModelSubstitutionError` unless the served id is the requested model.
+ * Returns the check on success so callers can record the served id alongside
+ * the result.
+ */
+declare function assertServedModel(requested: string, served: string | null | undefined, opts?: AssertServedModelOptions): ServedModelCheck;
+/**
+ * Batch form: check every pair and throw naming EVERY substitution, so one
+ * failure does not hide the rest. Returns all checks on success.
+ */
+declare function assertServedModels(pairs: ReadonlyArray<{
+  requested: string;
+  served: string | null | undefined;
+}>, opts?: AssertServedModelOptions): ServedModelCheck[];
+interface AssertCrossFamilyServedOptions extends AssertServedModelOptions {
+  /** Minimum distinct SERVED families required. Default 2. */
+  minFamilies?: number;
+  /** Count `unknown`-family served ids toward the total. Default false. */
+  allowUnknown?: boolean;
+}
+declare class ServedCrossFamilyError extends AgentEvalError {
+  readonly families: JudgeFamily[];
+  readonly checks: ReadonlyArray<ServedModelCheck>;
+  constructor(message: string, families: JudgeFamily[], checks: ReadonlyArray<ServedModelCheck>);
+}
+/**
+ * Family-diversity rule over the models that actually ANSWERED.
+ *
+ * `assertCrossFamily` (../judge-families) reads the requested ids and so
+ * cannot see a gateway that answers three "different" requests from one
+ * provider. This one asserts no substitution first, then counts families from
+ * the served ids — a panel that collapsed to one family under the hood fails
+ * here even though its request list looked diverse.
+ */
+declare function assertCrossFamilyServed(pairs: ReadonlyArray<{
+  requested: string;
+  served: string | null | undefined;
+}>, opts?: AssertCrossFamilyServedOptions): JudgeFamily[];
+//#endregion
+//#region src/trace/raw-provider-sink.d.ts
+/**
+ * RawProviderSink — first-class persistence for the actual HTTP-level
+ * request/response bodies of every LLM provider call.
+ *
+ * Why this is a separate sink from the structured `LlmSpan`:
+ *
+ *   - `LlmSpan` records the *intent* — model name, messages, output text,
+ *     usage. It's what dashboards read; it's NOT enough for forensics.
+ *   - When a downstream consumer reports "the verifier used the wrong route"
+ *     or "tokens look right but reasoning was missing," the only way to
+ *     answer is the raw HTTP body. Span fields can lie (a proxy can echo
+ *     a different `model` value than what actually answered); the raw
+ *     response is ground truth.
+ *
+ * Default behaviour: opt-in. Pass `rawSink` to `LlmClientOptions` (or the
+ * matrix runner / BuilderSession sets it up automatically) and every
+ * request, response, and error is recorded — including retries, with the
+ * attempt index attached so a flaky call's full event chain is recoverable.
+ *
+ * Redaction is enforced at sink time. The default redactor strips
+ * `Authorization`, `X-Api-Key`, `X-Auth-Token`, `Cookie` headers and any
+ * payload field whose key matches `apiKey | api_key | bearer | password |
+ * secret | token` (case-insensitive). Override via the sink constructor or
+ * the per-call `redactor`. The `redactedFields` array on the persisted
+ * event lets a reviewer see what was stripped without exposing the values.
+ */
+type RawProviderDirection = 'request' | 'response' | 'error';
+interface RawProviderEvent {
+  /** Stable id. Generated by the sink if omitted. */
+  eventId: string;
+  /** Trace context populated by `LlmClient` when the call is wrapped in a span. */
+  runId?: string;
+  spanId?: string;
+  /**
+   * Logical provider name. Free-form so callers can use whatever id matches
+   * their topology (`'openai'`, `'anthropic'`, `'tangle-router'`, …). When
+   * omitted, derived from `baseUrl` in `LlmClientOptions`.
+   */
+  provider: string;
+  model: string;
+  /** Endpoint path, e.g. `'/v1/chat/completions'`. */
+  endpoint: string;
+  /** Base URL used for the call (already-normalised — no trailing slash). */
+  baseUrl: string;
+  /** 0-indexed retry attempt. The first attempt is 0; a retried call gets 1, 2, … */
+  attemptIndex: number;
+  direction: RawProviderDirection;
+  /** Unix ms. */
+  timestamp: number;
+  /** Wall-clock duration of the call leg. Set on `response` and `error` events; null on `request`. */
+  durationMs?: number;
+  statusCode?: number;
+  requestHeaders?: Record<string, string>;
+  requestBody?: unknown;
+  responseHeaders?: Record<string, string>;
+  responseBody?: unknown;
+  /** Set on `direction: 'error'` events. */
+  errorMessage?: string;
+  /** Field paths the redactor stripped from this event ('header:Authorization', 'body.apiKey', …). */
+  redactedFields: string[];
+}
+interface RawProviderSinkFilter {
+  runId?: string;
+  spanId?: string;
+  direction?: RawProviderDirection;
+  attemptIndex?: number;
+}
+interface RawProviderSink {
+  record(event: RawProviderEvent): Promise<void>;
+  /** Optional listing — implementations that durably persist (file, db) should support this. */
+  list?(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
+  /** Optional teardown for backed implementations. */
+  close?(): Promise<void>;
+}
+type ProviderRedactor = (event: RawProviderEvent) => RawProviderEvent;
+/**
+ * Default redactor — strips well-known auth headers and any body field whose
+ * key matches the credential pattern. Records every redacted path on
+ * `event.redactedFields` so a downstream reviewer can see what was removed.
+ */
+declare function defaultProviderRedactor(event: RawProviderEvent): RawProviderEvent;
+interface InMemoryRawProviderSinkOptions {
+  redactor?: ProviderRedactor;
+}
+declare class InMemoryRawProviderSink implements RawProviderSink {
+  private events;
+  private redactor;
+  constructor(opts?: InMemoryRawProviderSinkOptions);
+  record(event: RawProviderEvent): Promise<void>;
+  list(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
+  size(): number;
+}
+declare class NoopRawProviderSink implements RawProviderSink {
+  record(): Promise<void>;
+  /**
+   * Returns an empty array. Implemented so `assertRunCaptured` does not
+   * trip the `no_raw_sink` issue when a caller explicitly opts out of
+   * capture by passing this sink — opt-out is a deliberate choice, not a
+   * misconfiguration.
+   */
+  list(): Promise<RawProviderEvent[]>;
+}
+interface FileSystemRawProviderSinkOptions {
+  /** Directory the NDJSON file is written into. Created if missing. */
+  dir: string;
+  /** File name; default `'raw-provider-events.ndjson'`. */
+  fileName?: string;
+  /** Bytes after which the writer rolls over to a new file (default 32 MiB). */
+  rollAtBytes?: number;
+  redactor?: ProviderRedactor;
+}
+declare class FileSystemRawProviderSink implements RawProviderSink {
+  private dir;
+  private fileName;
+  private rollAtBytes;
+  private redactor;
+  private bytesWritten;
+  private rollIndex;
+  private initPromise;
+  constructor(opts: FileSystemRawProviderSinkOptions);
+  private ensureInit;
+  private currentPath;
+  record(event: RawProviderEvent): Promise<void>;
+  list(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
+}
+/**
+ * Best-effort provider id from a base URL. Falls back to the URL host when
+ * none of the well-known patterns match.
+ */
+declare function providerFromBaseUrl(baseUrl: string): string;
+//#endregion
+//#region src/llm-client.d.ts
+interface LlmMessage {
+  role: 'system' | 'user' | 'assistant' | 'tool';
+  /**
+   * Either a plain text content string OR a multimodal content array
+   * (text + image_url parts) for vision-capable models.
+   */
+  content: string | Array<{
+    type: 'text';
+    text: string;
+  } | {
+    type: 'image_url';
+    image_url: {
+      url: string;
+      detail?: 'auto' | 'low' | 'high';
+    };
+  }>;
+  /** Tool invocations made by an `assistant` message earlier in the turn. */
+  toolCalls?: LlmToolCall[];
+  /** The invocation a `tool` message answers. Required when role is `tool`. */
+  toolCallId?: string;
+}
+type LlmThinkingMode = 'enabled' | 'disabled';
+/** Canonical function-tool definition offered to the model. */
+interface LlmToolDefinition {
+  type: 'function';
+  function: {
+    name: string;
+    description?: string;
+    parameters: Record<string, unknown>;
+  };
+}
+/** Canonical tool-choice policy; meaningful only when `tools` is present. */
+type LlmToolChoice = 'auto' | 'none' | 'required' | {
+  type: 'function';
+  function: {
+    name: string;
+  };
+};
+/** One tool invocation on a response or an assistant history message. */
+interface LlmToolCall {
+  id: string;
+  name: string;
+  /** JSON-encoded arguments, exactly as the provider produced them. */
+  argumentsJson: string;
+}
+interface LlmCallRequest {
+  model: string;
+  messages: LlmMessage[];
+  /** Optional JSON-mode response format (response_format: json_object). */
+  jsonMode?: boolean;
+  /** Optional structured output via JSON Schema. Falls back to json_object on 400. */
+  jsonSchema?: {
+    name: string;
+    schema: Record<string, unknown>;
+  };
+  /** Function tools offered to the model for this call. */
+  tools?: LlmToolDefinition[];
+  /** Tool-choice policy for `tools`. */
+  toolChoice?: LlmToolChoice;
+  temperature?: number;
+  maxTokens?: number;
+  /** Ask the provider for the log-probability of each sampled token and its
+   *  `topLogprobs` most likely alternatives. Sends `logprobs: true` with
+   *  `top_logprobs`. A provider that ignores the field returns
+   *  `LlmCallResult.logprobs === null`; nothing is inferred from its absence. */
+  logprobs?: {
+    topLogprobs: number;
+  };
+  /** OpenAI-compatible reasoning mode. Omitted when the provider default should apply. */
+  thinking?: LlmThinkingMode;
+  /** Per-call timeout, default 300s. */
+  timeoutMs?: number;
+}
+/** Conservative priced bound for the exact text request sent to a provider.
+ * Returns undefined when output or multimodal input is not bounded, causing a
+ * capped CostLedger to reject the call before execution. Pass
+ * `customTokenPricing` when package pricing does not cover the model or endpoint. */
+interface LlmChargeBounds {
+  /** Total provider attempts the transport may make for this call. Default 3. */
+  maximumAttempts?: number;
+  /** The transport sends JSON mode instead of a response schema. */
+  jsonSchemaTransport?: 'native' | 'json-object';
+  /** Default provider reasoning mode the transport applies. */
+  thinking?: LlmThinkingMode;
+  /** Token rates used when the provider omits cost or package pricing does not cover the model. */
+  customTokenPricing?: CustomTokenPricing;
+}
+declare function maximumChargeForLlmRequest(request: Pick<LlmCallRequest, 'model' | 'messages' | 'jsonSchema' | 'tools' | 'toolChoice' | 'maxTokens' | 'thinking'>, options?: LlmChargeBounds): MaximumCharge | undefined;
+interface LlmUsage {
+  promptTokens: number;
+  completionTokens: number;
+  totalTokens: number;
+  /** False when the provider omitted or malformed prompt/completion usage. */
+  captured?: boolean;
+  /** Reasoning-token subset of completionTokens, when reported. */
+  reasoningTokens?: number;
+  /** Proxies populate this when prompt caching is on. */
+  cachedPromptTokens?: number;
+}
+/** One sampled token with its own log probability and the alternatives the
+ *  provider ranked at that position. */
+interface LlmTokenLogprob {
+  token: string;
+  logprob: number;
+  top: ReadonlyArray<{
+    token: string;
+    logprob: number;
+  }>;
+}
+interface LlmCallResult {
+  /** The text content of the first choice. Empty string if none. */
+  content: string;
+  /** Tool invocations from the first choice, when the model called tools. */
+  toolCalls?: LlmToolCall[];
+  usage: LlmUsage;
+  /**
+   * Cost in USD. Uses the provider's reported cost when present, otherwise
+   * caller-supplied token pricing. `null` when neither is available.
+   */
+  costUsd: number | null;
+  /**
+   * Model id used for attribution (cost, pricing, log lines). The response's
+   * echoed id when the provider sent one, else the requested id.
+   *
+   * NOT evidence of which model answered — read `servedModel` for that. A
+   * provider that omits `model` makes this equal to the request, which is
+   * exactly the case an identity check must be able to distinguish.
+   */
+  model: string;
+  /**
+   * The model id the provider echoed on the response, verbatim; `null` when
+   * the body carried none. This is the only field that can witness a gateway
+   * substituting a different model for the one requested — compare it with
+   * `assertServedModel` / `checkServedModel` (src/integrity/served-model.ts).
+   *
+   * Optional so hand-built results (mock/custom transports) still typecheck,
+   * but omitting it is not a pass: the identity checks read `undefined` as
+   * `unreported` and reject it by default. A transport that knows which model
+   * answered should say so.
+   */
+  servedModel?: string | null;
+  /** Wall-clock duration of the HTTP call (last attempt, if retried). */
+  durationMs: number;
+  /**
+   * `finish_reason` echoed from the first choice (`stop`, `length`,
+   * `content_filter`, `tool_calls`, ...). `null` when the provider omits it.
+   * Exposed so a free-form `callLlm` caller CAN detect a truncated answer
+   * (`length`) instead of treating a cut-off completion as complete. Note:
+   * `callLlm` does not itself reject on it — acting on this signal is the
+   * caller's responsibility (in-repo free-form drivers do not yet enforce it).
+   */
+  finishReason?: string | null;
+  /**
+   * True when `content.trim()` is empty. An empty completion is a silent zero
+   * for free-form `callLlm` callers; this flag is the signal a caller can
+   * inspect to fail loud rather than proceed on an empty string. `callLlm`
+   * surfaces it but does not throw on it.
+   */
+  contentEmpty?: boolean;
+  /**
+   * Per-token log probabilities for the first choice, in emission order, when
+   * the request asked for them and the provider returned
+   * `choices[0].logprobs.content`. `null` when the provider returned none, so a
+   * caller can tell "not requested or not supported" from "requested and
+   * empty". Absent when the request did not ask for logprobs.
+   */
+  logprobs?: ReadonlyArray<LlmTokenLogprob> | null;
+  /** Raw response body. */
+  raw: Record<string, unknown>;
+}
+type LlmCallMetadata = Pick<LlmCallResult, 'usage' | 'costUsd' | 'model' | 'durationMs'>;
+/** Convert a provider result into the canonical paid-call receipt input.
+ * The receipt is JSON-clean: absent optional fields are omitted, never carried
+ * as explicit-undefined keys. Receipts cross JSON boundaries (the external
+ * optimizer's loopback proxy validates them with `assertJsonValue`), where an
+ * explicit-undefined value is rejected as non-serializable. */
+declare function costReceiptFromLlm(result: LlmCallResult, customTokenPricing?: CustomTokenPricing): CostReceiptInput;
+/** Structured-response failures retain their completed provider receipt. */
+declare function costReceiptFromLlmError(error: Error, customTokenPricing?: CustomTokenPricing): CostReceiptInput | undefined;
+declare class LlmCallError extends AgentEvalError {
+  readonly status: number;
+  readonly body: string;
+  readonly model: string;
+  constructor(message: string, status: number, body: string, model: string);
+}
+/** A provider response completed and incurred measurable usage, but its content
+ *  could not satisfy the caller's response contract. The response envelope is
+ *  retained so accounting can commit the receipt before the error propagates. */
+declare class LlmResponseError extends AgentEvalError {
+  readonly result: LlmCallResult;
+  constructor(message: string, result: LlmCallResult, options?: {
+    cause?: unknown;
+  });
+}
+interface LlmClientOptions extends LlmChargeBounds {
+  /** Base URL (without trailing slash), ending at the `/v1` prefix. Required: there is no default endpoint. */
+  baseUrl?: string;
+  /** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */
+  apiKey?: string;
+  bearer?: string;
+  /** Override for the `Authorization` header (e.g. `X-Auth: ...`). Takes precedence over apiKey/bearer. */
+  authHeader?: {
+    name: string;
+    value: string;
+  };
+  /** Stable provider idempotency key, reused across retries of this logical call. */
+  idempotencyKey?: string;
+  /** Default timeout in ms. Per-call can override. */
+  defaultTimeoutMs?: number;
+  /**
+   * Caller-supplied abort signal — e.g. a campaign-wide cancel. Linked to
+   * each attempt's per-attempt timeout controller, so aborting it cancels
+   * the in-flight fetch. A caller abort is FATAL: it is not retried even
+   * though an AbortError otherwise matches the transient patterns.
+   */
+  signal?: AbortSignal;
+  /**
+   * Cross-attempt wall-clock budget in ms, measured from the first attempt.
+   * Before launching each attempt the loop checks the remaining budget and
+   * stops retrying once it is exhausted, rather than waiting the full
+   * per-attempt timeout on every retry. Bounds total time independent of
+   * total attempts × `timeoutMs`.
+   */
+  deadlineMs?: number;
+  /**
+   * JSON payload parsing policy. `extract` accepts fenced or prose-prefixed JSON.
+   * `exact` requires the complete response content to be one JSON value.
+   * Default: `extract`.
+   */
+  jsonPayloadMode?: 'extract' | 'exact';
+  /** Fetch implementation — defaults to global `fetch`. Override for custom transport (e.g. tests). */
+  fetch?: typeof fetch;
+  /**
+   * Optional raw HTTP capture sink. When provided, every request, response,
+   * and error (across all retry attempts) is recorded to the sink, with auth
+   * headers and credential-shaped body fields redacted by default. This is
+   * the layer-1 forensics primitive: structured `LlmSpan`s record intent,
+   * raw events record what actually crossed the wire.
+   */
+  rawSink?: RawProviderSink;
+  /**
+   * Logical provider id attached to raw events. When omitted, derived from
+   * `baseUrl` via `providerFromBaseUrl`.
+   */
+  provider?: string;
+  /** Trace context attached to raw events; populated by emitter-aware callers. */
+  traceContext?: {
+    runId?: string;
+    spanId?: string;
+  };
+  /** Override the redaction strategy for this call. Defaults to `defaultProviderRedactor`. */
+  redactor?: ProviderRedactor;
+  /**
+   * Reject a response whose echoed model is not the model that was requested.
+   * A routing gateway can accept one id and answer from another, which
+   * silently invalidates every per-model and per-family claim downstream.
+   * `true` uses the strict default (aliases pass, substitutions and
+   * unidentified responses throw `ModelSubstitutionError`); pass an options
+   * object to relax a specific case. Off by default — turning it on for a
+   * measurement run is the point.
+   */
+  assertServedModel?: boolean | AssertServedModelOptions;
+}
+/**
+ * True when an error is a transient transport/network fault worth retrying,
+ * as opposed to a deterministic failure (4xx schema reject, JSON parse) that
+ * a retry cannot fix. Inspects `LlmCallError.status`, then the error's
+ * name/message/code, then recurses into `error.cause` — undici nests the
+ * real socket fault one or more levels under `.cause`.
+ *
+ * This is the retry classifier for the package: `callLlm` and
+ * `withJudgeRetry` both route through it, so connection failures are treated
+ * consistently across transports.
+ */
+declare function isTransientLlmError(err: unknown): boolean;
+/**
+ * Strip a ```json / ``` code fence if the model emitted one.
+ * Idempotent for naked JSON. Some models (claude-code via router, certain
+ * deepseek models) wrap output even under json_object.
+ */
+declare function stripFencedJson(raw: string): string;
+//#endregion
+//#region src/analyst/chat-client.d.ts
+/**
+ * Unified chat interface using the package's canonical LLM request and result.
+ */
+interface ChatClient {
+  /** Display name of the bound transport, included in telemetry. */
+  readonly transport: ChatTransport;
+  /** Default model when the caller omits one. */
+  readonly defaultModel?: string;
+  /** Total provider attempts this transport can make for one chat call. */
+  readonly maximumAttempts?: number;
+  /** Implementations must enforce `req.maxTokens` when it is present. */
+  chat(req: ChatRequest, opts?: ChatCallOpts): Promise<ChatResponse>;
+}
+type ChatTransport = 'sandbox-sdk' | 'custom' | 'openai-compatible' | 'mock';
+interface ChatRequest extends Omit<LlmCallRequest, 'model'> {
+  /** Optional — falls back to ChatClient.defaultModel. */
+  model?: string;
+}
+type ChatResponse = LlmCallResult;
+interface ChatCallOpts {
+  /** Cancel the in-flight request. */
+  signal?: AbortSignal;
+  /** Hard USD ceiling for this single call (informational; the underlying transport may not enforce). */
+  maxCostUsd?: number;
+  /** Correlation tag carried into request headers when the transport allows. */
+  correlationId?: string;
+  /** Stable provider idempotency key for retries/redrives of one paid call. */
+  idempotencyKey?: string;
+}
+type CreateChatClientOpts = SandboxSdkTransportOpts | CustomTransportOpts | OpenAiCompatibleTransportOpts | MockTransportOpts;
+interface BaseTransportOpts {
+  defaultModel?: string;
+  /** Total provider attempts. Required for opaque transports used in capped runs. */
+  maximumAttempts?: number;
+}
+/**
+ * Sandbox-SDK transport. The caller supplies a canonical chat function for an
+ * already-configured Sandbox handle, so agent-eval does not import the SDK.
+ */
+interface SandboxSdkTransportOpts extends BaseTransportOpts {
+  transport: 'sandbox-sdk';
+  chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>;
+}
+/** Caller-adapted SDK or transport returning the canonical ChatResponse shape. */
+interface CustomTransportOpts extends BaseTransportOpts {
+  transport: 'custom';
+  chat: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>;
+}
+/**
+ * OpenAI-compatible HTTP transport: the caller names a `/v1` endpoint and
+ * hands over a bearer, and this package drives `POST {baseUrl}/chat/completions`.
+ *
+ * Use this instead of hand-rolling a `custom` transport around `fetch`. The
+ * result carries `servedModel` — the model id the provider echoed, verbatim —
+ * which is the only field `assertServedModel` and `assertCrossFamilyServed`
+ * can read to witness a gateway answering from a different model than the one
+ * requested.
+ *
+ * `baseUrl` ends at the `/v1` prefix; the `/chat/completions` path is this
+ * package's to append. A `baseUrl` that already carries the path is a
+ * construction error, not a request to a doubled URL.
+ *
+ * Exactly one credential form is required — `apiKey`, `bearer`, or
+ * `authHeader`. There is no environment fallback: agent-eval never goes
+ * looking for a key.
+ */
+interface OpenAiCompatibleTransportOpts extends BaseTransportOpts, Pick<LlmClientOptions, 'assertServedModel' | 'customTokenPricing' | 'deadlineMs' | 'defaultTimeoutMs' | 'fetch' | 'jsonPayloadMode' | 'jsonSchemaTransport' | 'provider' | 'rawSink' | 'signal' | 'thinking'> {
+  transport: 'openai-compatible';
+  /** Endpoint ending at the `/v1` prefix. Required — there is no default endpoint. */
+  baseUrl: string;
+  /** Bearer credential. One of `apiKey`, `bearer`, or `authHeader` is required. */
+  apiKey?: string;
+  /** Bearer credential, alternate spelling. */
+  bearer?: string;
+  /** Non-bearer authorization header, for endpoints that want their own scheme. */
+  authHeader?: {
+    name: string;
+    value: string;
+  };
+}
+/**
+ * Mock transport for tests. The handler receives the request and returns
+ * whatever the test wants. No retries, no JSON-schema degrade.
+ */
+interface MockTransportOpts extends BaseTransportOpts {
+  transport: 'mock';
+  handler: (req: ChatRequest, opts?: ChatCallOpts) => Promise<ChatResponse>;
+}
+/**
+ * Build a ChatClient bound to a specific transport. The returned client
+ * is safe to share across analysts in a single registry run.
+ */
+declare function createChatClient(opts: CreateChatClientOpts): ChatClient;
+//#endregion
+//#region src/types.d.ts
+interface Scenario {
+  id: string;
+  persona: string;
+  label: string;
+  thesis: string;
+  dimensions: string[];
+  turns: Turn[];
+  artifactChecks: ArtifactCheck[];
+  systemPromptAppend?: string;
+}
+interface Turn {
+  user: string;
+  expectedBehaviors: string[];
+  adversarial?: boolean;
+  feedbackType?: 'correction' | 'rejection' | 'vague' | 'contradictory' | 'escalation';
+}
+interface ArtifactCheck {
+  type: 'vault_file_exists' | 'vault_file_contains' | 'block_extracted' | 'code_valid' | 'generation_produced' | 'tool_created' | string;
+  target: string;
+  contains?: string;
+  minCount?: number;
+  description: string;
+}
+interface JudgeRubric {
+  name: string;
+  description: string;
+  dimensions: RubricDimension[];
+}
+interface RubricDimension {
+  name: string;
+  description: string;
+  anchor_low: string;
+  anchor_high: string;
+  weight: number;
+}
+interface TurnResult {
+  turnIndex: number;
+  userMessage: string;
+  agentResponse: string;
+  durationMs: number;
+  blocksExtracted: {
+    type: string;
+    title: string;
+  }[];
+  containsCode: boolean;
+  containsToolCall: boolean;
+}
+interface JudgeScore {
+  judgeName: string;
+  dimension: string;
+  score: number;
+  reasoning: string;
+  evidence?: string;
+}
+interface CollectedArtifacts {
+  vaultFiles: {
+    path: string;
+    content: string;
+  }[];
+  blocksExtracted: {
+    type: string;
+    fields: Record<string, string>;
+  }[];
+  codeBlocks: {
+    language: string;
+    code: string;
+  }[];
+  toolCalls: string[];
+}
+interface RouteMap {
+  signup?: string;
+  login?: string;
+  workspaces?: string;
+  threads?: string;
+  chat?: string;
+  tasks?: string;
+  events?: string;
+  approvals?: string;
+  vault?: string;
+  generations?: string;
+  [key: string]: string | undefined;
+}
+interface ProductClientConfig {
+  baseUrl: string;
+  routes: RouteMap;
+  /** Per-request timeout in ms before the request is aborted. Default 30s. */
+  timeoutMs?: number;
+}
+interface CompletionCriterion {
+  name: string;
+  check: (state: DriverState) => boolean;
+  progress?: (state: DriverState) => number;
+}
+/**
+ * How hard the simulated user pushes back. The driver LLM scales its tone
+ * and follow-up aggression to this:
+ *   cooperative — forgiving early adopter; accepts reasonable answers.
+ *   demanding   — experienced professional; rejects vague or hedged answers.
+ *   relentless  — senior partner reviewing for a client who will litigate;
+ *                 interrogates every claim, accepts nothing undefended.
+ */
+type PersonaRigor = 'cooperative' | 'demanding' | 'relentless';
+interface PersonaConfig {
+  id: string;
+  role: string;
+  goal: string;
+  completionCriteria: CompletionCriterion[];
+  maxTurns: number;
+  /** How adversarial the simulated user is. Defaults to 'demanding'. */
+  rigor?: PersonaRigor;
+  /**
+   * Domain expertise the simulated user holds — quoted into the driver
+   * prompt so it challenges the agent with authority instead of vague
+   * dissatisfaction. e.g. "a 15-year M&A partner who knows GAAP
+   * working-capital mechanics cold".
+   */
+  expertise?: string;
+  /**
+   * Substantive issues a senior professional in this role would
+   * interrogate — traps the scenario hides, claims that must be defended.
+   * The driver probes these without revealing them verbatim; the agent
+   * must surface them on its own.
+   */
+  pressurePoints?: string[];
+  /**
+   * Curveballs the driver may inject once the agent is coasting — changed
+   * facts, a hostile counterparty position, a new constraint. Forces the
+   * agent to re-derive rather than recite.
+   */
+  curveballs?: string[];
+}
+interface DriverState {
+  tasks: number;
+  events: number;
+  proposals: {
+    pending: number;
+    approved: number;
+    rejected: number;
+  };
+  vaultFiles: string[];
+  codeBlocks: number;
+  generations: number;
+}
+interface JudgeInput {
+  scenario: Scenario;
+  turns: TurnResult[];
+  artifacts: CollectedArtifacts;
+  /** Shared ledger for paid built-in judges. Direct calls default to an uncapped ledger. */
+  costLedger?: CostLedgerHandle;
+  costPhase?: string;
+  costTags?: Record<string, string>;
+  signal?: AbortSignal;
+}
+type JudgeFn = (chat: ChatClient, input: JudgeInput) => Promise<JudgeScore[]>;
+interface CheckResult {
+  name: string;
+  passed: boolean;
+  expected: string;
+  actual: string;
+}
+//#endregion
+export { AssertCrossFamilyServedOptions as $, LlmThinkingMode as A, stripFencedJson as B, LlmCallError as C, LlmChargeBounds as D, LlmCallResult as E, LlmUsage as F, NoopRawProviderSink as G, FileSystemRawProviderSinkOptions as H, costReceiptFromLlm as I, RawProviderEvent as J, ProviderRedactor as K, costReceiptFromLlmError as L, LlmToolCall as M, LlmToolChoice as N, LlmMessage as O, LlmToolDefinition as P, providerFromBaseUrl as Q, isTransientLlmError as R, createChatClient as S, LlmCallRequest as T, InMemoryRawProviderSink as U, FileSystemRawProviderSink as V, InMemoryRawProviderSinkOptions as W, RawProviderSinkFilter as X, RawProviderSink as Y, defaultProviderRedactor as Z, CreateChatClientOpts as _, JudgeInput as a, ServedModelVerdict as at, OpenAiCompatibleTransportOpts as b, PersonaConfig as c, assertServedModels as ct, Scenario as d, AssertCrossFamilyOptions as dt, AssertServedModelOptions as et, ChatCallOpts as f, CrossFamilyError as ft, ChatTransport as g, ChatResponse as h, judgeFamily as ht, JudgeFn as i, ServedModelPolicy as it, LlmTokenLogprob as j, LlmResponseError as k, ProductClientConfig as l, checkServedModel as lt, ChatRequest as m, assertCrossFamily as mt, CompletionCriterion as n, ServedCrossFamilyError as nt, JudgeRubric as o, assertCrossFamilyServed as ot, ChatClient as p, JudgeFamily as pt, RawProviderDirection as q, DriverState as r, ServedModelCheck as rt, JudgeScore as s, assertServedModel as st, CheckResult as t, ModelSubstitutionError as tt, RouteMap as u, servedModelAcceptable as ut, CustomTransportOpts as v, LlmCallMetadata as w, SandboxSdkTransportOpts as x, MockTransportOpts as y, maximumChargeForLlmRequest as z };
+//# sourceMappingURL=types-gvRsyJLh.d.ts.map