@tangle-network/agent-eval 0.27.2 → 0.29.0

package/dist/replay-D7z0J43-.d.ts

@@ -1,225 +0,0 @@
-import { T as TraceStore } from './store-Db2Bv8Cf.js';
-import { R as ReplayError } from './errors-BZ9sTdz7.js';
-import { R as RawProviderSink, c as RawProviderEvent } from './integrity-DK2EBVZC.js';
-
-/**
- * OpenTelemetry JSON export — maps TraceSchema v1 to OTLP/JSON so
- * traces render natively in Jaeger / Honeycomb / Langfuse / Grafana.
- *
- * Wire format only. We do NOT depend on the @opentelemetry SDK — that
- * would drag in polyfills incompatible with Workers/Edge. Consumers
- * push the JSON to their collector of choice via HTTP.
- *
- * Reference: OTLP 1.3.2 (ResourceSpans / ScopeSpans / Span).
- */
-
-declare const OTEL_AGENT_EVAL_SCOPE: {
-    name: string;
-    version: string;
-};
-interface OtlpSpan {
-    traceId: string;
-    spanId: string;
-    parentSpanId?: string;
-    name: string;
-    kind: number;
-    startTimeUnixNano: string;
-    endTimeUnixNano: string;
-    attributes: Array<{
-        key: string;
-        value: {
-            stringValue?: string;
-            intValue?: string;
-            doubleValue?: number;
-            boolValue?: boolean;
-        };
-    }>;
-    events?: Array<{
-        timeUnixNano: string;
-        name: string;
-        attributes?: OtlpSpan['attributes'];
-    }>;
-    status?: {
-        code: number;
-        message?: string;
-    };
-}
-interface OtlpResourceSpans {
-    resource: {
-        attributes: OtlpSpan['attributes'];
-    };
-    scopeSpans: Array<{
-        scope: typeof OTEL_AGENT_EVAL_SCOPE;
-        spans: OtlpSpan[];
-    }>;
-}
-interface OtlpExport {
-    resourceSpans: OtlpResourceSpans[];
-}
-/** Export a single run's spans + events in OTLP/JSON. */
-declare function exportRunAsOtlp(store: TraceStore, runId: string, resourceAttrs?: Record<string, string | number | boolean>): Promise<OtlpExport>;
-
-/**
- * Redaction — remove PII / secrets from trace payloads before persist.
- *
- * Pre-persistence rules mean raw traces in storage are already scrubbed.
- * Unredacted variants (for debugging / post-mortems) live in a separate
- * storage layer with stricter access controls; this module only covers
- * the default scrub-then-persist path.
- *
- * Rules compose: pass an array of `RedactionRule`, each is applied in
- * order. Strings that match get replaced with a tagged sentinel so the
- * eval framework can count how many redactions happened per run
- * (surfaced via `redaction_applied` events).
- */
-interface RedactionRule {
-    id: string;
-    pattern: RegExp;
-    /** Replacement — e.g. '[PII:email]'. Defaults to `[redacted:{id}]`. */
-    replacement?: string;
-}
-interface RedactionReport {
-    redactionCount: number;
-    byRule: Record<string, number>;
-}
-/** OWASP / common-sense defaults — extend per-domain. */
-declare const DEFAULT_REDACTION_RULES: RedactionRule[];
-declare const REDACTION_VERSION = "1.0.0";
-/**
- * Redact a single string. Returns the new string and a per-rule count of
- * how many substitutions fired.
- */
-declare function redactString(input: string, rules?: RedactionRule[]): {
-    output: string;
-    report: RedactionReport;
-};
-/**
- * Walk a JSON-ish value applying `redactString` to every string leaf.
- * Arrays and plain objects are recursed; other types pass through
- * untouched. Circular references throw — traces should be tree-shaped.
- */
-declare function redactValue(value: unknown, rules?: RedactionRule[], report?: RedactionReport): {
-    value: unknown;
-    report: RedactionReport;
-};
-
-/**
- * Replay-from-raw-events — turn every captured campaign run into a
- * re-runnable artifact.
- *
- * `RawProviderSink` captures every provider HTTP envelope; `runEvalCampaign`
- * makes that capture the default. Together they make every past run a
- * complete fingerprint of what happened on the wire — enough to replay
- * the run without burning new LLM cost.
- *
- * Three use cases this primitive enables:
- *
- *   1. **Post-hoc judging** — apply a new judge / rubric / scoring callback
- *      to last week's runs without re-calling any LLM. The cost of trying
- *      a new rubric drops from "another full sweep" to a CPU-bound replay.
- *   2. **Determinism audits** — replay the same campaign and verify the
- *      raw responses match byte-for-byte. Any drift is a non-determinism
- *      bug (in the harness, the prompt builder, the sandbox, …).
- *   3. **Free judge calibration** — run two judges on identical responses
- *      and measure inter-judge agreement without doubling LLM spend.
- *
- * The interface is deliberately fetch-shaped. Inject `createReplayFetch`
- * into `LlmClientOptions.fetch` and every `callLlm` transparently reads
- * from the cache instead of calling the network. No new code path through
- * the LLM client is needed; the cache hit is invisible to the runner.
- */
-
-declare class ReplayCacheMissError extends ReplayError {
-    readonly url: string;
-    readonly requestKey: string;
-    constructor(url: string, requestKey: string, message?: string);
-}
-interface ReplayCacheEntry {
-    request: RawProviderEvent;
-    response: RawProviderEvent;
-}
-interface ReplayCacheStats {
-    total: number;
-    byProvider: Record<string, number>;
-    byModel: Record<string, number>;
-    /** Spans for which we have a request but no response (run aborted mid-call). */
-    orphanRequests: number;
-}
-/**
- * In-memory deterministic cache of (request → response) keyed on a stable
- * hash of the request body. Built from a `RawProviderSink` containing
- * paired `request` and `response` events from a previous run.
- *
- * The cache is the source of truth for replay; `createReplayFetch` is a
- * thin wrapper that reads from it.
- */
-declare class ReplayCache {
-    private byKey;
-    private orphans;
-    private byProvider;
-    private byModel;
-    /**
-     * Build a cache from a sink's events. The sink must implement `list()`.
-     * Filter by `runId` / `spanId` to scope to a specific replay.
-     */
-    static fromSink(sink: RawProviderSink, filter?: {
-        runId?: string;
-        spanId?: string;
-    }): Promise<ReplayCache>;
-    /** Build a cache from an in-memory event list. */
-    static fromEvents(events: RawProviderEvent[]): Promise<ReplayCache>;
-    /** Number of cacheable (request, response) pairs in the cache. */
-    size(): number;
-    stats(): ReplayCacheStats;
-    /** Iterate every cached `(request, response)` pair in insertion order. */
-    entries(): IterableIterator<ReplayCacheEntry>;
-    /**
-     * Look up a cached response by hashing the (model, messages, temperature,
-     * maxTokens, response_format) shape. Returns `undefined` on miss; the
-     * caller decides whether to throw, fall back to the network, or skip.
-     */
-    lookup(requestBody: unknown): Promise<ReplayCacheEntry | undefined>;
-}
-interface ReplayFetchOptions {
-    /**
-     * Behaviour on cache miss. Default `'throw'`. `'fallback'` calls the
-     * `fallbackFetch` (typically `globalThis.fetch`) so a partial replay can
-     * still complete; `'fail-closed'` returns a synthetic 599 response so the
-     * call site sees a non-retriable failure.
-     */
-    onMiss?: 'throw' | 'fallback' | 'fail-closed';
-    fallbackFetch?: typeof fetch;
-    /** Optional callback fired once per replayed call (for telemetry / counters). */
-    onHit?: (info: {
-        url: string;
-        provider: string;
-        model: string;
-    }) => void;
-    /** Optional callback fired on cache miss before the `onMiss` policy applies. */
-    onMissNotify?: (info: {
-        url: string;
-        requestBody: unknown;
-    }) => void;
-}
-/**
- * Build a `fetch`-shaped function that serves cached responses out of a
- * `ReplayCache` for any URL ending in `/chat/completions`. Pass through
- * `LlmClientOptions.fetch` and `callLlm` becomes free.
- *
- * Non-`/chat/completions` URLs are passed straight to the fallback fetch
- * (default: `globalThis.fetch`). This matters because non-LLM HTTP work
- * (judge HTTP servers, sandbox callbacks) sometimes flows through the same
- * `fetch` and shouldn't be intercepted.
- */
-declare function createReplayFetch(cache: ReplayCache, opts?: ReplayFetchOptions): typeof fetch;
-/**
- * Convenience iterator over `(request, response)` pairs in a sink — for
- * post-hoc scoring that doesn't need a `fetch` shim. The judge or scorer
- * runs purely in-process over cached LLM outputs.
- */
-declare function iterateRawCalls(sink: RawProviderSink, filter?: {
-    runId?: string;
-    spanId?: string;
-}): AsyncGenerator<ReplayCacheEntry>;
-
-export { DEFAULT_REDACTION_RULES as D, OTEL_AGENT_EVAL_SCOPE as O, REDACTION_VERSION as R, type OtlpExport as a, type OtlpResourceSpans as b, type OtlpSpan as c, type RedactionReport as d, type RedactionRule as e, ReplayCache as f, type ReplayCacheEntry as g, ReplayCacheMissError as h, type ReplayCacheStats as i, type ReplayFetchOptions as j, createReplayFetch as k, exportRunAsOtlp as l, iterateRawCalls as m, redactValue as n, redactString as r };