@tangle-network/agent-eval 0.20.12 → 0.22.0

package/dist/integrity-K2oVlF57.d.ts

@@ -0,0 +1,210 @@
+import { T as TraceStore } from './store-u47QaJ9G.js';
+
+/**
+ * RawProviderSink — first-class persistence for the actual HTTP-level
+ * request/response bodies of every LLM provider call.
+ *
+ * Why this is a separate sink from the structured `LlmSpan`:
+ *
+ *   - `LlmSpan` records the *intent* — model name, messages, output text,
+ *     usage. It's what dashboards read; it's NOT enough for forensics.
+ *   - When a downstream consumer reports "the verifier used the wrong route"
+ *     or "tokens look right but reasoning was missing," the only way to
+ *     answer is the raw HTTP body. Span fields can lie (a proxy can echo
+ *     a different `model` value than what actually answered); the raw
+ *     response is ground truth.
+ *
+ * Default behaviour: opt-in. Pass `rawSink` to `LlmClientOptions` (or the
+ * matrix runner / BuilderSession sets it up automatically) and every
+ * request, response, and error is recorded — including retries, with the
+ * attempt index attached so a flaky call's full event chain is recoverable.
+ *
+ * Redaction is enforced at sink time. The default redactor strips
+ * `Authorization`, `X-Api-Key`, `X-Auth-Token`, `Cookie` headers and any
+ * payload field whose key matches `apiKey | api_key | bearer | password |
+ * secret | token` (case-insensitive). Override via the sink constructor or
+ * the per-call `redactor`. The `redactedFields` array on the persisted
+ * event lets a reviewer see what was stripped without exposing the values.
+ */
+type RawProviderDirection = 'request' | 'response' | 'error';
+interface RawProviderEvent {
+    /** Stable id. Generated by the sink if omitted. */
+    eventId: string;
+    /** Trace context populated by `LlmClient` when the call is wrapped in a span. */
+    runId?: string;
+    spanId?: string;
+    /**
+     * Logical provider name. Free-form so callers can use whatever id matches
+     * their topology (`'openai'`, `'anthropic'`, `'tangle-router'`, …). When
+     * omitted, derived from `baseUrl` in `LlmClientOptions`.
+     */
+    provider: string;
+    model: string;
+    /** Endpoint path, e.g. `'/v1/chat/completions'`. */
+    endpoint: string;
+    /** Base URL used for the call (already-normalised — no trailing slash). */
+    baseUrl: string;
+    /** 0-indexed retry attempt. The first attempt is 0; a retried call gets 1, 2, … */
+    attemptIndex: number;
+    direction: RawProviderDirection;
+    /** Unix ms. */
+    timestamp: number;
+    /** Wall-clock duration of the call leg. Set on `response` and `error` events; null on `request`. */
+    durationMs?: number;
+    statusCode?: number;
+    requestHeaders?: Record<string, string>;
+    requestBody?: unknown;
+    responseHeaders?: Record<string, string>;
+    responseBody?: unknown;
+    /** Set on `direction: 'error'` events. */
+    errorMessage?: string;
+    /** Field paths the redactor stripped from this event ('header:Authorization', 'body.apiKey', …). */
+    redactedFields: string[];
+}
+interface RawProviderSinkFilter {
+    runId?: string;
+    spanId?: string;
+    direction?: RawProviderDirection;
+    attemptIndex?: number;
+}
+interface RawProviderSink {
+    record(event: RawProviderEvent): Promise<void>;
+    /** Optional listing — implementations that durably persist (file, db) should support this. */
+    list?(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
+    /** Optional teardown for backed implementations. */
+    close?(): Promise<void>;
+}
+type ProviderRedactor = (event: RawProviderEvent) => RawProviderEvent;
+/**
+ * Default redactor — strips well-known auth headers and any body field whose
+ * key matches the credential pattern. Records every redacted path on
+ * `event.redactedFields` so a downstream reviewer can see what was removed.
+ */
+declare function defaultProviderRedactor(event: RawProviderEvent): RawProviderEvent;
+interface InMemoryRawProviderSinkOptions {
+    redactor?: ProviderRedactor;
+}
+declare class InMemoryRawProviderSink implements RawProviderSink {
+    private events;
+    private redactor;
+    constructor(opts?: InMemoryRawProviderSinkOptions);
+    record(event: RawProviderEvent): Promise<void>;
+    list(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
+    size(): number;
+}
+declare class NoopRawProviderSink implements RawProviderSink {
+    record(): Promise<void>;
+    /**
+     * Returns an empty array. Implemented so `assertRunCaptured` does not
+     * trip the `no_raw_sink` issue when a caller explicitly opts out of
+     * capture by passing this sink — opt-out is a deliberate choice, not a
+     * misconfiguration.
+     */
+    list(): Promise<RawProviderEvent[]>;
+}
+interface FileSystemRawProviderSinkOptions {
+    /** Directory the NDJSON file is written into. Created if missing. */
+    dir: string;
+    /** File name; default `'raw-provider-events.ndjson'`. */
+    fileName?: string;
+    /** Bytes after which the writer rolls over to a new file (default 32 MiB). */
+    rollAtBytes?: number;
+    redactor?: ProviderRedactor;
+}
+declare class FileSystemRawProviderSink implements RawProviderSink {
+    private dir;
+    private fileName;
+    private rollAtBytes;
+    private redactor;
+    private bytesWritten;
+    private rollIndex;
+    private initPromise;
+    constructor(opts: FileSystemRawProviderSinkOptions);
+    private ensureInit;
+    private currentPath;
+    record(event: RawProviderEvent): Promise<void>;
+    list(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
+}
+/**
+ * Best-effort provider id from a base URL. Falls back to the URL host when
+ * none of the well-known patterns match.
+ */
+declare function providerFromBaseUrl(baseUrl: string): string;
+
+/**
+ * Run-completion integrity check — at end of run, verify the expected event
+ * types were actually captured. The point is the launch-review failure mode:
+ * a run *appears* successful but the raw provider events were never written,
+ * so a downstream reviewer can't reconstruct what happened.
+ *
+ * Pattern:
+ *
+ *   const report = await assertRunCaptured(store, runId, {
+ *     llmSpansMin: 1,
+ *     judgeSpansMin: 1,
+ *     rawSink: providerSink,                  // must have ≥ 1 event for this run
+ *     requireRawCoverageOfLlmSpans: true,     // every llm span has matching raw events
+ *   })
+ *   if (!report.ok) throwIfRunIncomplete(report)  // or mark run failed and continue
+ *
+ * The function is read-only on the store and returns a structured report;
+ * the caller chooses the failure mode (throw, mark run failed, log warning).
+ * `throwIfRunIncomplete` is the convenient strict mode.
+ */
+
+interface RunIntegrityExpectations {
+    /** Minimum LLM span count. Default 0 (no requirement). */
+    llmSpansMin?: number;
+    /** Minimum judge span count. Default 0. */
+    judgeSpansMin?: number;
+    /** Minimum tool span count. Default 0. */
+    toolSpansMin?: number;
+    /**
+     * Raw provider sink to consult for capture verification. When present,
+     * the check requires at least one raw event for the run.
+     */
+    rawSink?: RawProviderSink;
+    /** Minimum raw provider event count. Default 0; ignored when `rawSink` absent. */
+    rawProviderEventsMin?: number;
+    /**
+     * Every LLM span must have at least one matching raw `request` event
+     * (matched by spanId). Catches the common bug where the structured span
+     * was emitted but the raw HTTP capture was wired to a different sink.
+     */
+    requireRawCoverageOfLlmSpans?: boolean;
+    /** Run outcome must be set (not null/undefined). Default false. */
+    requireOutcome?: boolean;
+}
+type RunIntegrityIssueCode = 'no_run' | 'missing_llm_spans' | 'missing_judge_spans' | 'missing_tool_spans' | 'missing_raw_events' | 'no_raw_sink' | 'orphan_llm_span' | 'missing_outcome';
+interface RunIntegrityIssue {
+    code: RunIntegrityIssueCode;
+    message: string;
+    detail?: Record<string, unknown>;
+}
+interface RunIntegrityReport {
+    ok: boolean;
+    runId: string;
+    llmSpanCount: number;
+    judgeSpanCount: number;
+    toolSpanCount: number;
+    rawProviderEventCount: number;
+    /**
+     * Coverage of LLM spans by raw provider events keyed on spanId.
+     * `total` is the number of LLM spans; `covered` is the count with at
+     * least one matching `request` raw event.
+     */
+    rawSpanCoverage: {
+        covered: number;
+        total: number;
+    };
+    issues: RunIntegrityIssue[];
+}
+declare class RunIntegrityError extends Error {
+    readonly report: RunIntegrityReport;
+    constructor(report: RunIntegrityReport);
+}
+declare function assertRunCaptured(store: TraceStore, runId: string, expectations?: RunIntegrityExpectations): Promise<RunIntegrityReport>;
+/** Strict mode: throws `RunIntegrityError` when the report isn't ok. */
+declare function throwIfRunIncomplete(report: RunIntegrityReport): void;
+
+export { FileSystemRawProviderSink as F, InMemoryRawProviderSink as I, NoopRawProviderSink as N, type ProviderRedactor as P, type RawProviderDirection as R, type FileSystemRawProviderSinkOptions as a, type InMemoryRawProviderSinkOptions as b, type RawProviderEvent as c, type RawProviderSink as d, type RawProviderSinkFilter as e, RunIntegrityError as f, type RunIntegrityExpectations as g, type RunIntegrityIssue as h, type RunIntegrityIssueCode as i, type RunIntegrityReport as j, assertRunCaptured as k, defaultProviderRedactor as l, providerFromBaseUrl as p, throwIfRunIncomplete as t };

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.20.12",
+    "version": "0.22.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",