@tangle-network/agent-eval 0.20.11 → 0.21.0

package/dist/traces.d.ts

@@ -0,0 +1,914 @@
+import { T as TraceStore, L as LlmSpan, J as JudgeSpan, a as Run, F as FailureClass, c as ToolSpan } from './store-u47QaJ9G.js';
+export { A as Artifact, B as BudgetLedgerEntry, g as BudgetSpec, i as EventFilter, E as EventKind, j as FAILURE_CLASSES, k as FileSystemTraceStore, l as FileSystemTraceStoreOptions, G as GenericSpan, I as InMemoryTraceStore, M as Message, d as RetrievalSpan, h as RunFilter, m as RunLayer, R as RunOutcome, n as RunStatus, e as SandboxSpan, S as Span, o as SpanBase, p as SpanFilter, b as SpanKind, q as SpanStatus, r as TRACE_SCHEMA_VERSION, f as TraceEvent, s as isJudgeSpan, t as isLlmSpan, u as isRetrievalSpan, v as isSandboxSpan, w as isToolSpan } from './store-u47QaJ9G.js';
+import { a as RunCompleteHookContext, R as RunCompleteHook } from './emitter-B2XqDKFU.js';
+export { S as SpanHandle, T as TraceEmitter, b as TraceEmitterOptions, l as llmSpanFromProvider } from './emitter-B2XqDKFU.js';
+import { AxAIService, AxFunction } from '@ax-llm/ax';
+
+/**
+ * Typed query helpers over TraceStore.
+ *
+ * Not a full SQL engine — a minimal, composable set of operators that
+ * cover the canned-pipeline use cases. For ad-hoc analytics, persist to
+ * NDJSON and point DuckDB at it; the schema is stable so external SQL
+ * tooling works out of the box.
+ */
+
+declare function runsForScenario(store: TraceStore, scenarioId: string): Promise<Run[]>;
+declare function llmSpans(store: TraceStore, runId?: string): Promise<LlmSpan[]>;
+declare function toolSpans(store: TraceStore, runId?: string, toolName?: string): Promise<ToolSpan[]>;
+declare function judgeSpans(store: TraceStore, runId?: string): Promise<JudgeSpan[]>;
+/** Group spans by any key selector. */
+declare function groupBy<T, K extends string | number>(items: T[], key: (t: T) => K): Map<K, T[]>;
+/** Hash tool arguments to an orderless-key-stable string for de-duplication. */
+declare function argHash(args: unknown): string;
+/** Sum an LLM-span array into aggregate token + cost. */
+declare function aggregateLlm(spans: LlmSpan[]): {
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens: number;
+    costUsd: number;
+};
+/** Pick the outcome's failure class when present, else derive 'success' from run status. */
+declare function runFailureClass(run: Run): FailureClass;
+
+/**
+ * Redaction — remove PII / secrets from trace payloads before persist.
+ *
+ * Pre-persistence rules mean raw traces in storage are already scrubbed.
+ * Unredacted variants (for debugging / post-mortems) live in a separate
+ * storage layer with stricter access controls; this module only covers
+ * the default scrub-then-persist path.
+ *
+ * Rules compose: pass an array of `RedactionRule`, each is applied in
+ * order. Strings that match get replaced with a tagged sentinel so the
+ * eval framework can count how many redactions happened per run
+ * (surfaced via `redaction_applied` events).
+ */
+interface RedactionRule {
+    id: string;
+    pattern: RegExp;
+    /** Replacement — e.g. '[PII:email]'. Defaults to `[redacted:{id}]`. */
+    replacement?: string;
+}
+interface RedactionReport {
+    redactionCount: number;
+    byRule: Record<string, number>;
+}
+/** OWASP / common-sense defaults — extend per-domain. */
+declare const DEFAULT_REDACTION_RULES: RedactionRule[];
+declare const REDACTION_VERSION = "1.0.0";
+/**
+ * Redact a single string. Returns the new string and a per-rule count of
+ * how many substitutions fired.
+ */
+declare function redactString(input: string, rules?: RedactionRule[]): {
+    output: string;
+    report: RedactionReport;
+};
+/**
+ * Walk a JSON-ish value applying `redactString` to every string leaf.
+ * Arrays and plain objects are recursed; other types pass through
+ * untouched. Circular references throw — traces should be tree-shaped.
+ */
+declare function redactValue(value: unknown, rules?: RedactionRule[], report?: RedactionReport): {
+    value: unknown;
+    report: RedactionReport;
+};
+
+/**
+ * OpenTelemetry JSON export — maps TraceSchema v1 to OTLP/JSON so
+ * traces render natively in Jaeger / Honeycomb / Langfuse / Grafana.
+ *
+ * Wire format only. We do NOT depend on the @opentelemetry SDK — that
+ * would drag in polyfills incompatible with Workers/Edge. Consumers
+ * push the JSON to their collector of choice via HTTP.
+ *
+ * Reference: OTLP 1.3.2 (ResourceSpans / ScopeSpans / Span).
+ */
+
+declare const OTEL_AGENT_EVAL_SCOPE: {
+    name: string;
+    version: string;
+};
+interface OtlpSpan {
+    traceId: string;
+    spanId: string;
+    parentSpanId?: string;
+    name: string;
+    kind: number;
+    startTimeUnixNano: string;
+    endTimeUnixNano: string;
+    attributes: Array<{
+        key: string;
+        value: {
+            stringValue?: string;
+            intValue?: string;
+            doubleValue?: number;
+            boolValue?: boolean;
+        };
+    }>;
+    events?: Array<{
+        timeUnixNano: string;
+        name: string;
+        attributes?: OtlpSpan['attributes'];
+    }>;
+    status?: {
+        code: number;
+        message?: string;
+    };
+}
+interface OtlpResourceSpans {
+    resource: {
+        attributes: OtlpSpan['attributes'];
+    };
+    scopeSpans: Array<{
+        scope: typeof OTEL_AGENT_EVAL_SCOPE;
+        spans: OtlpSpan[];
+    }>;
+}
+interface OtlpExport {
+    resourceSpans: OtlpResourceSpans[];
+}
+/** Export a single run's spans + events in OTLP/JSON. */
+declare function exportRunAsOtlp(store: TraceStore, runId: string, resourceAttrs?: Record<string, string | number | boolean>): Promise<OtlpExport>;
+
+/**
+ * RawProviderSink — first-class persistence for the actual HTTP-level
+ * request/response bodies of every LLM provider call.
+ *
+ * Why this is a separate sink from the structured `LlmSpan`:
+ *
+ *   - `LlmSpan` records the *intent* — model name, messages, output text,
+ *     usage. It's what dashboards read; it's NOT enough for forensics.
+ *   - When a downstream consumer reports "the verifier used the wrong route"
+ *     or "tokens look right but reasoning was missing," the only way to
+ *     answer is the raw HTTP body. Span fields can lie (a proxy can echo
+ *     a different `model` value than what actually answered); the raw
+ *     response is ground truth.
+ *
+ * Default behaviour: opt-in. Pass `rawSink` to `LlmClientOptions` (or the
+ * matrix runner / BuilderSession sets it up automatically) and every
+ * request, response, and error is recorded — including retries, with the
+ * attempt index attached so a flaky call's full event chain is recoverable.
+ *
+ * Redaction is enforced at sink time. The default redactor strips
+ * `Authorization`, `X-Api-Key`, `X-Auth-Token`, `Cookie` headers and any
+ * payload field whose key matches `apiKey | api_key | bearer | password |
+ * secret | token` (case-insensitive). Override via the sink constructor or
+ * the per-call `redactor`. The `redactedFields` array on the persisted
+ * event lets a reviewer see what was stripped without exposing the values.
+ */
+type RawProviderDirection = 'request' | 'response' | 'error';
+interface RawProviderEvent {
+    /** Stable id. Generated by the sink if omitted. */
+    eventId: string;
+    /** Trace context populated by `LlmClient` when the call is wrapped in a span. */
+    runId?: string;
+    spanId?: string;
+    /**
+     * Logical provider name. Free-form so callers can use whatever id matches
+     * their topology (`'openai'`, `'anthropic'`, `'tangle-router'`, …). When
+     * omitted, derived from `baseUrl` in `LlmClientOptions`.
+     */
+    provider: string;
+    model: string;
+    /** Endpoint path, e.g. `'/v1/chat/completions'`. */
+    endpoint: string;
+    /** Base URL used for the call (already-normalised — no trailing slash). */
+    baseUrl: string;
+    /** 0-indexed retry attempt. The first attempt is 0; a retried call gets 1, 2, … */
+    attemptIndex: number;
+    direction: RawProviderDirection;
+    /** Unix ms. */
+    timestamp: number;
+    /** Wall-clock duration of the call leg. Set on `response` and `error` events; null on `request`. */
+    durationMs?: number;
+    statusCode?: number;
+    requestHeaders?: Record<string, string>;
+    requestBody?: unknown;
+    responseHeaders?: Record<string, string>;
+    responseBody?: unknown;
+    /** Set on `direction: 'error'` events. */
+    errorMessage?: string;
+    /** Field paths the redactor stripped from this event ('header:Authorization', 'body.apiKey', …). */
+    redactedFields: string[];
+}
+interface RawProviderSinkFilter {
+    runId?: string;
+    spanId?: string;
+    direction?: RawProviderDirection;
+    attemptIndex?: number;
+}
+interface RawProviderSink {
+    record(event: RawProviderEvent): Promise<void>;
+    /** Optional listing — implementations that durably persist (file, db) should support this. */
+    list?(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
+    /** Optional teardown for backed implementations. */
+    close?(): Promise<void>;
+}
+type ProviderRedactor = (event: RawProviderEvent) => RawProviderEvent;
+/**
+ * Default redactor — strips well-known auth headers and any body field whose
+ * key matches the credential pattern. Records every redacted path on
+ * `event.redactedFields` so a downstream reviewer can see what was removed.
+ */
+declare function defaultProviderRedactor(event: RawProviderEvent): RawProviderEvent;
+interface InMemoryRawProviderSinkOptions {
+    redactor?: ProviderRedactor;
+}
+declare class InMemoryRawProviderSink implements RawProviderSink {
+    private events;
+    private redactor;
+    constructor(opts?: InMemoryRawProviderSinkOptions);
+    record(event: RawProviderEvent): Promise<void>;
+    list(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
+    size(): number;
+}
+declare class NoopRawProviderSink implements RawProviderSink {
+    record(): Promise<void>;
+}
+interface FileSystemRawProviderSinkOptions {
+    /** Directory the NDJSON file is written into. Created if missing. */
+    dir: string;
+    /** File name; default `'raw-provider-events.ndjson'`. */
+    fileName?: string;
+    /** Bytes after which the writer rolls over to a new file (default 32 MiB). */
+    rollAtBytes?: number;
+    redactor?: ProviderRedactor;
+}
+declare class FileSystemRawProviderSink implements RawProviderSink {
+    private dir;
+    private fileName;
+    private rollAtBytes;
+    private redactor;
+    private bytesWritten;
+    private rollIndex;
+    private initPromise;
+    constructor(opts: FileSystemRawProviderSinkOptions);
+    private ensureInit;
+    private currentPath;
+    record(event: RawProviderEvent): Promise<void>;
+    list(filter?: RawProviderSinkFilter): Promise<RawProviderEvent[]>;
+}
+/**
+ * Best-effort provider id from a base URL. Falls back to the URL host when
+ * none of the well-known patterns match.
+ */
+declare function providerFromBaseUrl(baseUrl: string): string;
+
+/**
+ * Run-completion integrity check — at end of run, verify the expected event
+ * types were actually captured. The point is the launch-review failure mode:
+ * a run *appears* successful but the raw provider events were never written,
+ * so a downstream reviewer can't reconstruct what happened.
+ *
+ * Pattern:
+ *
+ *   const report = await assertRunCaptured(store, runId, {
+ *     llmSpansMin: 1,
+ *     judgeSpansMin: 1,
+ *     rawSink: providerSink,                  // must have ≥ 1 event for this run
+ *     requireRawCoverageOfLlmSpans: true,     // every llm span has matching raw events
+ *   })
+ *   if (!report.ok) throwIfRunIncomplete(report)  // or mark run failed and continue
+ *
+ * The function is read-only on the store and returns a structured report;
+ * the caller chooses the failure mode (throw, mark run failed, log warning).
+ * `throwIfRunIncomplete` is the convenient strict mode.
+ */
+
+interface RunIntegrityExpectations {
+    /** Minimum LLM span count. Default 0 (no requirement). */
+    llmSpansMin?: number;
+    /** Minimum judge span count. Default 0. */
+    judgeSpansMin?: number;
+    /** Minimum tool span count. Default 0. */
+    toolSpansMin?: number;
+    /**
+     * Raw provider sink to consult for capture verification. When present,
+     * the check requires at least one raw event for the run.
+     */
+    rawSink?: RawProviderSink;
+    /** Minimum raw provider event count. Default 0; ignored when `rawSink` absent. */
+    rawProviderEventsMin?: number;
+    /**
+     * Every LLM span must have at least one matching raw `request` event
+     * (matched by spanId). Catches the common bug where the structured span
+     * was emitted but the raw HTTP capture was wired to a different sink.
+     */
+    requireRawCoverageOfLlmSpans?: boolean;
+    /** Run outcome must be set (not null/undefined). Default false. */
+    requireOutcome?: boolean;
+}
+type RunIntegrityIssueCode = 'no_run' | 'missing_llm_spans' | 'missing_judge_spans' | 'missing_tool_spans' | 'missing_raw_events' | 'no_raw_sink' | 'orphan_llm_span' | 'missing_outcome';
+interface RunIntegrityIssue {
+    code: RunIntegrityIssueCode;
+    message: string;
+    detail?: Record<string, unknown>;
+}
+interface RunIntegrityReport {
+    ok: boolean;
+    runId: string;
+    llmSpanCount: number;
+    judgeSpanCount: number;
+    toolSpanCount: number;
+    rawProviderEventCount: number;
+    /**
+     * Coverage of LLM spans by raw provider events keyed on spanId.
+     * `total` is the number of LLM spans; `covered` is the count with at
+     * least one matching `request` raw event.
+     */
+    rawSpanCoverage: {
+        covered: number;
+        total: number;
+    };
+    issues: RunIntegrityIssue[];
+}
+declare class RunIntegrityError extends Error {
+    readonly report: RunIntegrityReport;
+    constructor(report: RunIntegrityReport);
+}
+declare function assertRunCaptured(store: TraceStore, runId: string, expectations?: RunIntegrityExpectations): Promise<RunIntegrityReport>;
+/** Strict mode: throws `RunIntegrityError` when the report isn't ok. */
+declare function throwIfRunIncomplete(report: RunIntegrityReport): void;
+
+/**
+ * Shared types for the trace-analyst module.
+ *
+ * Wire format. The store interface speaks `OtlpSpanLike` rows — one JSONL
+ * line per span, OTLP-shaped. We do NOT depend on a specific tracing
+ * vendor at the type level. Adapter
+ * layers map upstream shapes onto this interface.
+ *
+ * Design constraint. Every read operation that can return arbitrary
+ * payload must carry a byte budget so the agent's tool result stays
+ * bounded regardless of input trace size. Oversized responses
+ * substitute a deterministic summary instead of bytes — see
+ * `ViewTraceOversized`.
+ */
+/** OTLP span kind (subset we actually use). */
+type TraceAnalystSpanKind = 'AGENT' | 'LLM' | 'TOOL' | 'CHAIN' | 'GUARDRAIL' | 'SPAN' | 'UNKNOWN';
+type TraceAnalystSpanStatus = 'OK' | 'ERROR' | 'UNSET';
+/** Subset of OTLP span fields the analyst exposes to the agent. The
+ *  store's job is to project upstream's full span shape down to this
+ *  view — the analyst never sees vendor extensions directly. */
+interface TraceAnalystSpan {
+    trace_id: string;
+    span_id: string;
+    parent_span_id: string | null;
+    name: string;
+    kind: TraceAnalystSpanKind;
+    start_time: string;
+    end_time: string;
+    duration_ms: number;
+    status: TraceAnalystSpanStatus;
+    status_message?: string;
+    service_name: string | null;
+    agent_name: string | null;
+    model_name: string | null;
+    tool_name: string | null;
+    /** Raw JSON-serialisable attribute map. May contain large strings;
+     *  callers must respect the per-attribute byte cap. */
+    attributes: Record<string, unknown>;
+}
+interface TraceAnalystTraceSummary {
+    trace_id: string;
+    service_name: string | null;
+    agent_name: string | null;
+    span_count: number;
+    has_errors: boolean;
+    start_time: string;
+    end_time: string;
+    duration_ms: number;
+    raw_jsonl_bytes: number;
+    models: string[];
+    tools: string[];
+}
+interface TraceAnalystFilters {
+    /** Restrict to traces that contain at least one error span. */
+    has_errors?: boolean;
+    /** Match if any span's `service.name` is in this list. */
+    service_names?: string[];
+    /** Match if any span's `agent.name` is in this list. */
+    agent_names?: string[];
+    /** Match if any LLM span's `llm.model_name` is in this list. */
+    model_names?: string[];
+    /** Match if any tool span's `tool.name` is in this list. */
+    tool_names?: string[];
+    /** ISO-8601 lower bound on the trace's earliest start time. */
+    start_time_after?: string;
+    /** ISO-8601 upper bound on the trace's earliest start time. */
+    start_time_before?: string;
+    /** Single regex applied to raw JSONL bytes for the trace. Opt-in;
+     *  expensive on large datasets. Use the indexed filters above first. */
+    regex_pattern?: string;
+}
+interface DatasetOverview {
+    total_traces: number;
+    raw_jsonl_bytes: number;
+    services: string[];
+    agents: string[];
+    models: string[];
+    tool_names: string[];
+    /** Up to 20 real trace ids the agent may pass to view/search tools. */
+    sample_trace_ids: string[];
+    errors: {
+        trace_count: number;
+        span_count: number;
+    };
+    time_range: {
+        earliest: string;
+        latest: string;
+    } | null;
+}
+interface QueryTracesPage {
+    traces: TraceAnalystTraceSummary[];
+    total: number;
+    has_more: boolean;
+}
+/** Full-trace view. When the response would exceed the per-call byte
+ *  budget, `oversized` is populated INSTEAD of `spans` so the agent
+ *  knows to switch to `searchTrace` / `viewSpans`. */
+interface ViewTraceResult {
+    trace_id: string;
+    spans?: TraceAnalystSpan[];
+    oversized?: ViewTraceOversized;
+}
+interface ViewTraceOversized {
+    span_count: number;
+    /** Names with their counts, sorted desc. Capped at 20 entries. */
+    top_span_names: Array<[string, number]>;
+    /** Largest single span body (bytes after attribute-cap projection). */
+    span_response_bytes_max: number;
+    error_span_count: number;
+}
+interface ViewSpansResult {
+    trace_id: string;
+    spans: TraceAnalystSpan[];
+    /** Number of requested span ids that were not found in the trace. */
+    missing_span_ids: string[];
+    /** Number of attribute fields truncated to fit the per-attribute cap. */
+    truncated_attribute_count: number;
+}
+interface SpanMatchRecord {
+    trace_id: string;
+    span_id: string;
+    span_name: string;
+    span_kind: TraceAnalystSpanKind;
+    /** JSON pointer-style path to the matched value, e.g.
+     *  `attributes."llm.input_messages"[2].content`. */
+    attribute_path: string;
+    matched_text: string;
+    context_before: string;
+    context_after: string;
+    match_offset: number;
+}
+interface SearchTraceResult {
+    trace_id: string;
+    hits: SpanMatchRecord[];
+    total_matches: number;
+    has_more: boolean;
+}
+interface SearchSpanResult {
+    trace_id: string;
+    span_id: string;
+    hits: SpanMatchRecord[];
+    total_matches: number;
+    has_more: boolean;
+}
+/** Tunable byte budgets for bounded RLM tool output. */
+interface TraceAnalystByteBudgets {
+    /** Max bytes any single tool response may emit. Hard ceiling enforced
+     *  by the store; oversized → summary. Default 150_000. */
+    perCallByteCeiling: number;
+    /** Per-attribute string truncation cap on `viewTrace` (discovery scan).
+     *  Default 4096. */
+    perAttributeViewBudget: number;
+    /** Per-attribute string truncation cap on `viewSpans` (surgical reads).
+     *  Default 16384. */
+    perAttributeSpanBudget: number;
+    /** Per-attribute cap on a single match record's `matched_text` and
+     *  context window. Default 1024. */
+    perMatchTextBudget: number;
+}
+declare const DEFAULT_TRACE_ANALYST_BUDGETS: TraceAnalystByteBudgets;
+/** Marker substituted in place of truncated string payloads. Callers
+ *  parsing tool output can detect it deterministically. */
+declare const TRACE_ANALYST_TRUNCATION_MARKER_PREFIX = "[trace-analyst truncated:";
+
+/**
+ * `TraceAnalysisStore` — read-side interface the trace-analyst calls
+ * through. Six operations, all bounded:
+ *
+ *   - `getOverview(filters?)` — dataset rollup + sample trace ids.
+ *   - `queryTraces(filters?, limit, offset)` — paginated summaries.
+ *   - `countTraces(filters?)` — cheap count without materialisation.
+ *   - `viewTrace(trace_id, perAttrCap)` — full span list, oversized → summary.
+ *   - `viewSpans(trace_id, span_ids, perAttrCap)` — surgical span fetch.
+ *   - `searchTrace(trace_id, regex, max_matches)` — bounded regex hits.
+ *   - `searchSpan(trace_id, span_id, regex, max_matches)` — single-span search.
+ *
+ * Multiple implementations ship in the core (`OtlpFileTraceStore`).
+ * Downstream callers can supply their own — e.g. a DuckDB-backed
+ * adapter or an in-memory adapter for tests — by implementing this
+ * interface.
+ *
+ * Filters compose with AND semantics. Empty/undefined fields impose
+ * no constraint. `regex_pattern` is the only opt-in raw-bytes scan —
+ * implementations may skip it via `count`/`overview` when not set.
+ */
+
+interface TraceAnalysisStore {
+    getOverview(filters?: TraceAnalystFilters): Promise<DatasetOverview>;
+    queryTraces(opts: {
+        filters?: TraceAnalystFilters;
+        limit: number;
+        offset?: number;
+    }): Promise<QueryTracesPage>;
+    countTraces(filters?: TraceAnalystFilters): Promise<number>;
+    viewTrace(opts: {
+        trace_id: string;
+        /** Override per-attribute byte cap. Defaults to discovery budget. */
+        per_attribute_byte_cap?: number;
+    }): Promise<ViewTraceResult>;
+    viewSpans(opts: {
+        trace_id: string;
+        span_ids: readonly string[];
+        /** Override per-attribute byte cap. Defaults to surgical budget. */
+        per_attribute_byte_cap?: number;
+    }): Promise<ViewSpansResult>;
+    searchTrace(opts: {
+        trace_id: string;
+        regex_pattern: string;
+        /** Hard cap on matches returned. Default 50. */
+        max_matches?: number;
+    }): Promise<SearchTraceResult>;
+    searchSpan(opts: {
+        trace_id: string;
+        span_id: string;
+        regex_pattern: string;
+        max_matches?: number;
+    }): Promise<SearchSpanResult>;
+}
+
+interface AnalyzeTracesInput {
+    /** The user-facing question. Domain framing belongs here, not in the
+     *  actor description. */
+    question: string;
+}
+interface AnalyzeTracesResult {
+    /** The responder's prose answer. */
+    answer: string;
+    /** Bulleted findings extracted from the responder's structured output. */
+    findings: string[];
+    /** Per-actor-turn snapshots captured via `actorTurnCallback`. */
+    turns: AnalyzeTracesTurnSnapshot[];
+    /** Total turns the actor took. */
+    turnCount: number;
+    /** Token usage by role. */
+    usage: TraceAnalystUsage;
+    /** Full system + assistant + tool message log by role. */
+    chatLog: TraceAnalystChatLog;
+    /** Prompt version that produced this run. */
+    actorPromptVersion: string;
+}
+interface TraceAnalystUsage {
+    actor: TraceAnalystUsageEntry[];
+    responder: TraceAnalystUsageEntry[];
+}
+interface TraceAnalystUsageEntry {
+    [key: string]: unknown;
+}
+interface TraceAnalystChatLog {
+    actor: TraceAnalystChatMessage[];
+    responder: TraceAnalystChatMessage[];
+}
+interface TraceAnalystChatMessage {
+    [key: string]: unknown;
+}
+interface AnalyzeTracesTurnSnapshot {
+    turn: number;
+    isError: boolean;
+    /** The JS code the actor produced for this turn. */
+    code: string;
+    /** The formatted action-log entry the actor sees on the next turn. */
+    output: string;
+    /** Provider thought (when `actorOptions.showThoughts` is true and the
+     *  provider returns it). */
+    thought?: string;
+}
+interface AnalyzeTracesOptions {
+    /** Trace data source. Pass either an OTLP-JSONL path or a custom store. */
+    source: string | TraceAnalysisStore;
+    /** Caller-provided AxAIService. */
+    ai: AxAIService;
+    /** Model id forwarded to actor + responder. */
+    model?: string;
+    /** Recursion depth. 0 = no sub-agent dispatch. Default 1. */
+    maxDepth?: number;
+    /** Maximum actor turns. Default 12. */
+    maxTurns?: number;
+    /** Maximum parallel sub-agent calls in batched llmQuery. Default 2. */
+    maxParallelSubagents?: number;
+    /** Override the actor description. */
+    actorDescription?: string;
+    /** Override the subagent description. */
+    subagentDescription?: string;
+    /** Per-turn observability hook. */
+    onTurn?: (turn: AnalyzeTracesTurnSnapshot) => void | Promise<void>;
+    /** Override max runtime characters per turn. Default 6000. */
+    maxRuntimeChars?: number;
+    /** When set, every turn's snapshot is appended to this JSONL file
+     *  immediately. If the analyst crashes mid-loop (provider 503,
+     *  network error, validator reject) the partial reasoning is still
+     *  on disk. Replay the file with the responder afterward to recover
+     *  evidence. */
+    progressLogPath?: string;
+}
+/**
+ * Run the trace analyst.
+ *
+ * Throws:
+ *   - `TraceFileMissingError` if `source` is a path and doesn't exist.
+ *   - `AxAgentClarificationError` if the analyst asks for clarification.
+ *   - Provider errors (auth, rate limits) propagate from the AI service.
+ */
+declare function analyzeTraces(input: AnalyzeTracesInput, options: AnalyzeTracesOptions): Promise<AnalyzeTracesResult>;
+
+/**
+ * `OtlpFileTraceStore` — read-only OTLP-JSONL trace store for the
+ * trace-analyst.
+ *
+ * Wire shape. Each line of the input file is one OTLP-shaped span. The
+ * store understands flattened OTLP JSONL plus the OpenInference vocab.
+ * We project upstream's full
+ * span shape down to `TraceAnalystSpan` lazily — full materialisation
+ * only happens for the spans the agent actually requests.
+ *
+ * Indexing. On first read the store builds an in-memory index keyed
+ * by `trace_id` carrying:
+ *   - byte offsets + lengths for each span line (for surgical reads
+ *     without re-parsing the whole file)
+ *   - a `TraceAnalystTraceSummary` rollup
+ *   - sets of services / agents / models / tools / has_errors
+ *   - byte size of the trace's JSONL slab
+ *
+ * Memory bound. The index keeps span metadata only — names, kinds,
+ * offsets, status. Attribute payloads stay on disk until requested.
+ * For a 50MB JSONL with 50k spans, the index is ~5MB.
+ *
+ * Concurrency. The store builds the index once on first read and
+ * caches it. Subsequent reads reuse the index. The file is opened on
+ * each read; we never hold a long-lived FD.
+ */
+
+interface OtlpFileTraceStoreOptions {
+    /** Path to the OTLP-JSONL file. */
+    path: string;
+    /** Override the discovery (`viewTrace`) per-attribute byte cap. */
+    perAttributeViewBudget?: number;
+    /** Override the surgical (`viewSpans`) per-attribute byte cap. */
+    perAttributeSpanBudget?: number;
+    /** Override the per-call ceiling that triggers oversized summaries. */
+    perCallByteCeiling?: number;
+    /** Override the per-match text budget. */
+    perMatchTextBudget?: number;
+}
+declare class OtlpFileTraceStore implements TraceAnalysisStore {
+    private readonly path;
+    private readonly perAttributeViewBudget;
+    private readonly perAttributeSpanBudget;
+    private readonly perCallByteCeiling;
+    private readonly perMatchTextBudget;
+    private indexPromise?;
+    /** Cached UTF-8 buffer of the file. We pin it once because every
+     *  read needs slice access and re-reading on each call balloons the
+     *  syscall count. */
+    private bufferPromise?;
+    constructor(opts: OtlpFileTraceStoreOptions);
+    getOverview(filters?: TraceAnalystFilters): Promise<DatasetOverview>;
+    queryTraces(opts: {
+        filters?: TraceAnalystFilters;
+        limit: number;
+        offset?: number;
+    }): Promise<QueryTracesPage>;
+    countTraces(filters?: TraceAnalystFilters): Promise<number>;
+    viewTrace(opts: {
+        trace_id: string;
+        per_attribute_byte_cap?: number;
+    }): Promise<ViewTraceResult>;
+    viewSpans(opts: {
+        trace_id: string;
+        span_ids: readonly string[];
+        per_attribute_byte_cap?: number;
+    }): Promise<ViewSpansResult>;
+    searchTrace(opts: {
+        trace_id: string;
+        regex_pattern: string;
+        max_matches?: number;
+    }): Promise<SearchTraceResult>;
+    searchSpan(opts: {
+        trace_id: string;
+        span_id: string;
+        regex_pattern: string;
+        max_matches?: number;
+    }): Promise<SearchSpanResult>;
+    /** Force the index to materialise. Useful to amortise startup cost
+     *  before the first agent call. */
+    ensureIndexed(): Promise<void>;
+    private buffer;
+    private index;
+    private buildIndex;
+    private matchedTraces;
+    private toSummary;
+    private projectSpan;
+    private buildOversizedSummary;
+    private scanSpanForMatches;
+}
+declare class TraceFileMissingError extends Error {
+    constructor(path: string);
+}
+declare class TraceNotFoundError extends Error {
+    readonly trace_id: string;
+    constructor(trace_id: string);
+}
+declare class SpanNotFoundError extends Error {
+    readonly trace_id: string;
+    readonly span_id: string;
+    constructor(trace_id: string, span_id: string);
+}
+
+/**
+ * Trace-analyst tool surface — six namespaced AxFunctions the analyst
+ * agent calls from generated JS code via `traces.<name>(...)`.
+ *
+ * Discovery → narrow → deep-read protocol. Tool names + ordering
+ * support RLM discovery:
+ *
+ *   1. `getDatasetOverview` (cheap)  — first call, sizes the dataset
+ *   2. `queryTraces`                 — paginated summaries with `raw_jsonl_bytes`
+ *   3. `countTraces`                 — cheap pre-flight before regex
+ *   4. `viewTrace`                   — full span list, oversized → summary
+ *   5. `viewSpans`                   — surgical 16KB-cap reads
+ *   6. `searchTrace` / `searchSpan`  — bounded regex hits
+ *
+ * Failure mode. Tool handlers throw on bad input (invalid trace ids,
+ * out-of-range pagination, malformed regex). Ax converts thrown errors
+ * into actor-visible `[ERROR]` strings so the analyst can adjust on
+ * the next turn instead of looping.
+ */
+
+interface BuildTraceAnalystToolsOpts {
+    store: TraceAnalysisStore;
+    /** Override the default sample-trace-id slot count (20). Mostly for tests. */
+    sampleTraceLimit?: number;
+}
+/**
+ * Build the trace-analyst function set. Pass the result into
+ * `agent(...).functions.local`.
+ */
+declare function buildTraceAnalystTools(opts: BuildTraceAnalystToolsOpts): AxFunction[];
+/**
+ * Convenience: same shape as `buildTraceAnalystTools` but returns the
+ * grouped form expected when registering trace tools alongside other
+ * agent function modules. */
+declare function traceAnalystFunctionGroup(opts: BuildTraceAnalystToolsOpts): {
+    namespace: string;
+    title: string;
+    selectionCriteria: string;
+    description: string;
+    functions: AxFunction[];
+};
+
+/**
+ * Trace-analyst auto-execution hook.
+ *
+ * Wires `analyzeTraces` into a `TraceEmitter`'s `onRunComplete` so a
+ * direct matrix run produces an analysis artifact without an out-of-band
+ * step. Designed for the case where the consumer reports "the analyst
+ * never ran" — the cause is almost always orchestration, not the analyst.
+ *
+ * Usage:
+ *
+ *   const emitter = new TraceEmitter(store, {
+ *     onRunComplete: [traceAnalystOnRunComplete({ analyze: opts, save })],
+ *   })
+ *
+ * Hooks are best-effort by default — they never crash the underlying run.
+ * The caller decides whether to gate the run on the analysis result via
+ * the `gateOn` callback.
+ */
+
+interface TraceAnalystHookOptions {
+    /**
+     * Options forwarded to `analyzeTraces`. The hook supplies the question
+     * if you don't pass one — defaulting to a launch-grade prompt that asks
+     * for failure modes, surprising findings, and a recommendation.
+     */
+    analyze: Omit<AnalyzeTracesOptions, 'source'> & {
+        source?: AnalyzeTracesOptions['source'];
+    };
+    /**
+     * Override the question. The default is intentionally generic:
+     * "Summarise what happened in this run, surface any failure modes,
+     *  surprising findings, or evidence the verdict is wrong."
+     */
+    question?: string;
+    /**
+     * Persist the result. The hook calls this with the analysis output and
+     * the run context. Common implementations write to a TraceAnalysisStore
+     * or append to a per-run JSONL.
+     */
+    save?: (result: AnalyzeTracesResult, ctx: RunCompleteHookContext) => Promise<void>;
+    /**
+     * Predicate gating execution per run. Default: every completed run.
+     * Use to skip aborted runs, debug runs, or runs without LLM activity.
+     */
+    shouldRun?: (ctx: RunCompleteHookContext) => boolean;
+    /**
+     * Optional gate: if set and returns false, the hook records the failure
+     * as a log event on the run instead of staying quiet. The caller can
+     * then trigger downstream alerts off `analyst_gate_failed` log events.
+     */
+    gateOn?: (result: AnalyzeTracesResult, ctx: RunCompleteHookContext) => boolean;
+}
+declare function traceAnalystOnRunComplete(opts: TraceAnalystHookOptions): RunCompleteHook;
+
+/** Ax RLM prompt for bounded trace discovery and evidence-backed analysis. */
+declare const TRACE_ANALYST_ACTOR_DESCRIPTION = "You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the `traces` namespace.\n\nDISCOVERY \u2192 NARROW \u2192 DEEP-READ protocol \u2014 follow exactly:\n\n1. ALWAYS call `traces.getDatasetOverview({})` FIRST without a regex_pattern. The result tells you total_traces, raw_jsonl_bytes, services, agents, models, and sample_trace_ids (real ids \u2014 never fabricate one).\n\n2. Use raw_jsonl_bytes to gauge how expensive raw scans will be. `filters.regex_pattern` is the one scan-heavy filter on getDatasetOverview / queryTraces / countTraces \u2014 narrow with indexed fields (has_errors, model_names, service_names, agent_names, time bounds) BEFORE adding a regex on a large dataset.\n\n3. To list more traces than the sample, call `traces.queryTraces({ filters?, limit, offset? })`. Each summary carries raw_jsonl_bytes \u2014 use it to choose between viewTrace and searchTrace BEFORE calling either.\n\n4. Per-trace inspection:\n   - SMALL trace (raw_jsonl_bytes well under 150_000): call `traces.viewTrace({ trace_id })`. Returns all spans. Per-attribute payloads are head-capped at ~4KB; large `input.value` / `output.value` / `llm.input_messages` will show a `[trace-analyst truncated: N bytes]` marker.\n   - LARGE trace (raw_jsonl_bytes near or above 150_000, or you saw an `oversized` response): use `traces.searchTrace({ trace_id, regex_pattern })` to get bounded SpanMatchRecords (span metadata + matched text + surrounding context). Then call `traces.viewSpans({ trace_id, span_ids: [...] })` for surgical reads (~16KB cap, 4\u00D7 higher than discovery), or `traces.searchSpan({ trace_id, span_id, regex_pattern })` for one large span. Stays bounded regardless of trace size.\n   - Useful regex patterns: `STATUS_CODE_ERROR` (failures), tool names like `grep` or `view_trace`, error strings like `MaxTurnsExceeded`, model names, attribute keys.\n\n5. ONLY call viewTrace / viewSpans / searchTrace / searchSpan with trace/span ids you have already seen in sample_trace_ids, a queryTraces page, or a previous search result. Never invent ids.\n\n5a. **Result-shape contract** \u2014 searchTrace and searchSpan return `{ trace_id, hits, total_matches, has_more }`. Iterate `result.hits` (NOT result.matches). Each hit has `{ span_id, span_name, span_kind, attribute_path, matched_text, context_before, context_after, match_offset }`. viewTrace returns `{ trace_id, spans }` (or `oversized`). viewSpans returns `{ trace_id, spans, missing_span_ids, truncated_attribute_count }`. Never assume a field name \u2014 log the result shape first if unsure.\n\n6. If viewTrace returns an `oversized` summary instead of `spans`, DO NOT retry the same call. Read the summary's top_span_names, span_count, span_response_bytes_max, error_span_count to plan a follow-up: switch to searchTrace (or searchSpan for one large span), then viewSpans on a smaller, surgical span_ids set.\n\n7. If searchTrace or searchSpan returns has_more=true, REFINE the regex to be more specific rather than blindly raising max_matches.\n\n8. If a tool errors (invalid regex, range error), STOP and reconsider \u2014 don't retry with a guessed id or argument. Use the discovery tools above to recover.\n\n9. If a ~4KB-truncated payload from viewTrace / searchTrace matters for your answer, first try viewSpans on that span id (~16KB cap). If a 16KB-truncated payload from viewSpans still matters, narrow further with searchSpan against a more specific regex rather than asking for the full payload again.\n\n10. If maxDepth > 0 and the question splits into independent semantic branches, delegate well-defined subtasks to subagents using `await llmQuery(...)`. Pass narrow context and a focused query. Examples:\n\n    const reviews = await llmQuery([\n      { query: 'Drill into trace abc123 \u2014 what tool calls preceded the failure?', context: { trace_id: 'abc123' } },\n      { query: 'Drill into trace def456 \u2014 same failure mode?', context: { trace_id: 'def456' } },\n    ]);\n\nOBSERVABILITY rules:\n- Each non-final actor turn must emit at least one `console.log(...)` for evidence. Up to 3 logs per turn is fine when correlating multiple data sources (e.g. one log for findings list, one for source-file content, one for derived analysis).\n- Do NOT combine `console.log` with `final(...)` or `askClarification(...)` in the same turn \u2014 finish gathering data first, then call final on its own turn.\n- Reuse runtime variables across turns; don't recompute.\n- When done, call `await final(answer)` with the fully-formed report. The responder rewrites the answer into output fields; if you only pass a vague summary string the responder has nothing concrete to format.\n\nCRITICAL \u2014 `final()` payload contract for evidence-grounded analysis tasks:\n- Pass a STRUCTURED object as the second arg with the actual data the responder needs to format the answer. Do NOT pass abstract instructions; pass evidence.\n- Example for per-item verdict tasks:\n  ```js\n  await final(\"Format the per-item verdict report from the evidence below.\", {\n    findings: [\n      { id: 'sub-1-finding-1', claim: '...', verdict: 'TRUE-POSITIVE', evidence: 'lines 42-45 of contracts/X.sol show ...' },\n      ...all items\n    ],\n    systemic_summary: '3 sentences I wrote based on the evidence above'\n  });\n  ```\n- Calling `final(\"answer\", {})` with no evidence is a failure mode \u2014 the responder will hallucinate or echo back the field names. Always include the gathered data.\n- Premature final after a single viewSpans call is INSUFFICIENT for per-finding analysis tasks. Read the requested attributes (e.g. `spans[i].attributes['redteam.finding.title']`), and for each one perform the requested cross-reference (e.g. read the source SPAN's `attributes['source.content']`).\n\nOUTPUT contract \u2014 your final answer must include:\n- A clear prose conclusion answering the user's question.\n- Trace ids and span ids cited as evidence for each claim.\n- Failure modes named in the user's domain language, with frequency and concrete examples.\n\nDo NOT invent trace ids, span ids, error messages, or model names. Every fact must be traceable to a tool result.";
+declare const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-actor-v5-2026-05-06";
+/** Subagent prompt for focused trace-inspection subtasks. */
+declare const TRACE_ANALYST_SUBAGENT_DESCRIPTION = "You are a trace-analyst subagent. Your parent has delegated a focused trace-inspection question. Use the same DISCOVERY \u2192 NARROW \u2192 DEEP-READ protocol but stay tightly scoped: do exactly what was asked, return a concise compact answer, do NOT spawn further subagents unless the parent's question is genuinely multi-branch.\n\nCite trace ids and span ids for every claim. Do NOT invent ids.";
+
+interface TraceInsightTask {
+    id: string;
+    name: string;
+    prompt?: string;
+    difficulty?: string;
+    tags?: string[];
+    outcome?: string;
+    score?: number;
+    gaps?: string[];
+}
+interface TraceInsightSuite {
+    name: string;
+    collectionId?: string;
+    tasks: TraceInsightTask[];
+}
+interface TraceInsightFinding {
+    kind: string;
+    severity?: string;
+    taskIds: string[];
+    evidence?: string;
+    proposedFixClass?: string;
+}
+interface TraceInsightQuestion {
+    id: string;
+    question: string;
+    why: string;
+}
+interface TraceInsightPanelRole {
+    id: string;
+    name: string;
+    responsibility: string;
+}
+interface TraceInsightPromptInput {
+    suite: TraceInsightSuite;
+    findings?: TraceInsightFinding[];
+    agent?: Record<string, unknown>;
+    totals?: Record<string, unknown>;
+    maxRepresentativeTraces?: number;
+}
+interface TraceInsightContext {
+    suite: TraceInsightSuite;
+    scope: string;
+    keywords: string[];
+    questions: TraceInsightQuestion[];
+    panel: TraceInsightPanelRole[];
+    findings: TraceInsightFinding[];
+    agent: Record<string, unknown> | null;
+    totals: Record<string, unknown> | null;
+}
+interface TraceInsightQualityGate {
+    id: string;
+    label: string;
+    passed: boolean;
+    severity: 'critical' | 'high' | 'medium' | 'low';
+    detail: string;
+}
+interface TraceInsightReadiness {
+    score: number;
+    grade: 'external-ready' | 'internal-review' | 'raw-analysis';
+    gates: TraceInsightQualityGate[];
+}
+declare function tokenizeDomainWords(value: string): string[];
+declare function inferDomainKeywords(suite: TraceInsightSuite): string[];
+declare function domainEvidencePattern(keywords: string[]): RegExp;
+declare function describeTraceInsightScope(suite: TraceInsightSuite): string;
+declare function planTraceInsightQuestions(input: TraceInsightPromptInput): TraceInsightQuestion[];
+declare function buildTraceInsightContext(input: TraceInsightPromptInput): TraceInsightContext;
+declare function scoreTraceInsightReadiness(context: TraceInsightContext): TraceInsightReadiness;
+declare function defaultTraceInsightPanel(): TraceInsightPanelRole[];
+declare function buildTraceInsightPrompt(input: TraceInsightPromptInput): string;
+
+export { type AnalyzeTracesInput, type AnalyzeTracesOptions, type AnalyzeTracesResult, type AnalyzeTracesTurnSnapshot, DEFAULT_REDACTION_RULES, DEFAULT_TRACE_ANALYST_BUDGETS, type DatasetOverview, FailureClass, FileSystemRawProviderSink, type FileSystemRawProviderSinkOptions, InMemoryRawProviderSink, type InMemoryRawProviderSinkOptions, JudgeSpan, LlmSpan, NoopRawProviderSink, OTEL_AGENT_EVAL_SCOPE, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpResourceSpans, type OtlpSpan, type ProviderRedactor, type QueryTracesPage, REDACTION_VERSION, type RawProviderDirection, type RawProviderEvent, type RawProviderSink, type RawProviderSinkFilter, type RedactionReport, type RedactionRule, Run, RunCompleteHook, RunCompleteHookContext, RunIntegrityError, type RunIntegrityExpectations, type RunIntegrityIssue, type RunIntegrityIssueCode, type RunIntegrityReport, type SearchSpanResult, type SearchTraceResult, type SpanMatchRecord, SpanNotFoundError, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, ToolSpan, type TraceAnalysisStore, type TraceAnalystByteBudgets, type TraceAnalystFilters, type TraceAnalystHookOptions, type TraceAnalystSpan, type TraceAnalystSpanKind, type TraceAnalystSpanStatus, type TraceAnalystTraceSummary, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, TraceStore, type ViewSpansResult, type ViewTraceOversized, type ViewTraceResult, aggregateLlm, analyzeTraces, argHash, assertRunCaptured, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, defaultProviderRedactor, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, groupBy, inferDomainKeywords, judgeSpans, llmSpans, planTraceInsightQuestions, providerFromBaseUrl, redactString, redactValue, runFailureClass, runsForScenario, scoreTraceInsightReadiness, throwIfRunIncomplete, tokenizeDomainWords, toolSpans, traceAnalystFunctionGroup, traceAnalystOnRunComplete };