@tangle-network/agent-eval 0.23.0 → 0.24.0

package/dist/traces.d.ts

@@ -1,256 +1,11 @@
-import { T as TraceStore, L as LlmSpan, J as JudgeSpan, a as Run, F as FailureClass, c as ToolSpan } from './store-u47QaJ9G.js';
-export { A as Artifact, B as BudgetLedgerEntry, g as BudgetSpec, i as EventFilter, E as EventKind, j as FAILURE_CLASSES, k as FileSystemTraceStore, l as FileSystemTraceStoreOptions, G as GenericSpan, I as InMemoryTraceStore, M as Message, d as RetrievalSpan, h as RunFilter, m as RunLayer, R as RunOutcome, n as RunStatus, e as SandboxSpan, S as Span, o as SpanBase, p as SpanFilter, b as SpanKind, q as SpanStatus, r as TRACE_SCHEMA_VERSION, f as TraceEvent, s as isJudgeSpan, t as isLlmSpan, u as isRetrievalSpan, v as isSandboxSpan, w as isToolSpan } from './store-u47QaJ9G.js';
-import { a as RunCompleteHookContext, R as RunCompleteHook } from './emitter-B2XqDKFU.js';
-export { S as SpanHandle, T as TraceEmitter, b as TraceEmitterOptions, l as llmSpanFromProvider } from './emitter-B2XqDKFU.js';
-import { R as RawProviderSink, f as RawProviderEvent } from './integrity-Cr5YodSY.js';
-export { F as FileSystemRawProviderSink, c as FileSystemRawProviderSinkOptions, I as InMemoryRawProviderSink, d as InMemoryRawProviderSinkOptions, N as NoopRawProviderSink, P as ProviderRedactor, e as RawProviderDirection, g as RawProviderSinkFilter, h as RunIntegrityError, a as RunIntegrityExpectations, i as RunIntegrityIssue, j as RunIntegrityIssueCode, b as RunIntegrityReport, k as assertRunCaptured, l as defaultProviderRedactor, p as providerFromBaseUrl, t as throwIfRunIncomplete } from './integrity-Cr5YodSY.js';
+export { D as DEFAULT_REDACTION_RULES, O as OTEL_AGENT_EVAL_SCOPE, a as OtlpExport, b as OtlpResourceSpans, c as OtlpSpan, R as REDACTION_VERSION, d as RedactionReport, e as RedactionRule, f as ReplayCache, g as ReplayCacheEntry, h as ReplayCacheMissError, i as ReplayCacheStats, j as ReplayFetchOptions, k as createReplayFetch, l as exportRunAsOtlp, m as iterateRawCalls, r as redactString, n as redactValue } from './replay-BL96gCEP.js';
+import { a as RunCompleteHookContext, R as RunCompleteHook } from './emitter-DP_cSSiw.js';
+export { S as SpanHandle, T as TraceEmitter, b as TraceEmitterOptions, l as llmSpanFromProvider } from './emitter-DP_cSSiw.js';
+export { F as FileSystemRawProviderSink, d as FileSystemRawProviderSinkOptions, I as InMemoryRawProviderSink, e as InMemoryRawProviderSinkOptions, N as NoopRawProviderSink, P as ProviderRedactor, f as RawProviderDirection, c as RawProviderEvent, R as RawProviderSink, g as RawProviderSinkFilter, h as RunIntegrityError, a as RunIntegrityExpectations, i as RunIntegrityIssue, j as RunIntegrityIssueCode, b as RunIntegrityReport, k as assertRunCaptured, l as defaultProviderRedactor, p as providerFromBaseUrl, t as throwIfRunIncomplete } from './integrity-DK2EBVZC.js';
+export { a as aggregateLlm, b as argHash, g as groupBy, j as judgeSpans, l as llmSpans, r as runFailureClass, c as runsForScenario, t as toolSpans } from './query-DODUYdPg.js';
+export { A as Artifact, B as BudgetLedgerEntry, g as BudgetSpec, i as EventFilter, E as EventKind, j as FAILURE_CLASSES, F as FailureClass, k as FileSystemTraceStore, l as FileSystemTraceStoreOptions, G as GenericSpan, I as InMemoryTraceStore, J as JudgeSpan, L as LlmSpan, M as Message, e as RetrievalSpan, R as Run, h as RunFilter, m as RunLayer, c as RunOutcome, n as RunStatus, f as SandboxSpan, S as Span, o as SpanBase, p as SpanFilter, d as SpanKind, q as SpanStatus, r as TRACE_SCHEMA_VERSION, a as ToolSpan, b as TraceEvent, T as TraceStore, s as isJudgeSpan, t as isLlmSpan, u as isRetrievalSpan, v as isSandboxSpan, w as isToolSpan } from './store-Db2Bv8Cf.js';
 import { AxAIService, AxFunction } from '@ax-llm/ax';
-
-/**
- * Typed query helpers over TraceStore.
- *
- * Not a full SQL engine — a minimal, composable set of operators that
- * cover the canned-pipeline use cases. For ad-hoc analytics, persist to
- * NDJSON and point DuckDB at it; the schema is stable so external SQL
- * tooling works out of the box.
- */
-
-declare function runsForScenario(store: TraceStore, scenarioId: string): Promise<Run[]>;
-declare function llmSpans(store: TraceStore, runId?: string): Promise<LlmSpan[]>;
-declare function toolSpans(store: TraceStore, runId?: string, toolName?: string): Promise<ToolSpan[]>;
-declare function judgeSpans(store: TraceStore, runId?: string): Promise<JudgeSpan[]>;
-/** Group spans by any key selector. */
-declare function groupBy<T, K extends string | number>(items: T[], key: (t: T) => K): Map<K, T[]>;
-/** Hash tool arguments to an orderless-key-stable string for de-duplication. */
-declare function argHash(args: unknown): string;
-/** Sum an LLM-span array into aggregate token + cost. */
-declare function aggregateLlm(spans: LlmSpan[]): {
-    inputTokens: number;
-    outputTokens: number;
-    cachedTokens: number;
-    costUsd: number;
-};
-/** Pick the outcome's failure class when present, else derive 'success' from run status. */
-declare function runFailureClass(run: Run): FailureClass;
-
-/**
- * Redaction — remove PII / secrets from trace payloads before persist.
- *
- * Pre-persistence rules mean raw traces in storage are already scrubbed.
- * Unredacted variants (for debugging / post-mortems) live in a separate
- * storage layer with stricter access controls; this module only covers
- * the default scrub-then-persist path.
- *
- * Rules compose: pass an array of `RedactionRule`, each is applied in
- * order. Strings that match get replaced with a tagged sentinel so the
- * eval framework can count how many redactions happened per run
- * (surfaced via `redaction_applied` events).
- */
-interface RedactionRule {
-    id: string;
-    pattern: RegExp;
-    /** Replacement — e.g. '[PII:email]'. Defaults to `[redacted:{id}]`. */
-    replacement?: string;
-}
-interface RedactionReport {
-    redactionCount: number;
-    byRule: Record<string, number>;
-}
-/** OWASP / common-sense defaults — extend per-domain. */
-declare const DEFAULT_REDACTION_RULES: RedactionRule[];
-declare const REDACTION_VERSION = "1.0.0";
-/**
- * Redact a single string. Returns the new string and a per-rule count of
- * how many substitutions fired.
- */
-declare function redactString(input: string, rules?: RedactionRule[]): {
-    output: string;
-    report: RedactionReport;
-};
-/**
- * Walk a JSON-ish value applying `redactString` to every string leaf.
- * Arrays and plain objects are recursed; other types pass through
- * untouched. Circular references throw — traces should be tree-shaped.
- */
-declare function redactValue(value: unknown, rules?: RedactionRule[], report?: RedactionReport): {
-    value: unknown;
-    report: RedactionReport;
-};
-
-/**
- * OpenTelemetry JSON export — maps TraceSchema v1 to OTLP/JSON so
- * traces render natively in Jaeger / Honeycomb / Langfuse / Grafana.
- *
- * Wire format only. We do NOT depend on the @opentelemetry SDK — that
- * would drag in polyfills incompatible with Workers/Edge. Consumers
- * push the JSON to their collector of choice via HTTP.
- *
- * Reference: OTLP 1.3.2 (ResourceSpans / ScopeSpans / Span).
- */
-
-declare const OTEL_AGENT_EVAL_SCOPE: {
-    name: string;
-    version: string;
-};
-interface OtlpSpan {
-    traceId: string;
-    spanId: string;
-    parentSpanId?: string;
-    name: string;
-    kind: number;
-    startTimeUnixNano: string;
-    endTimeUnixNano: string;
-    attributes: Array<{
-        key: string;
-        value: {
-            stringValue?: string;
-            intValue?: string;
-            doubleValue?: number;
-            boolValue?: boolean;
-        };
-    }>;
-    events?: Array<{
-        timeUnixNano: string;
-        name: string;
-        attributes?: OtlpSpan['attributes'];
-    }>;
-    status?: {
-        code: number;
-        message?: string;
-    };
-}
-interface OtlpResourceSpans {
-    resource: {
-        attributes: OtlpSpan['attributes'];
-    };
-    scopeSpans: Array<{
-        scope: typeof OTEL_AGENT_EVAL_SCOPE;
-        spans: OtlpSpan[];
-    }>;
-}
-interface OtlpExport {
-    resourceSpans: OtlpResourceSpans[];
-}
-/** Export a single run's spans + events in OTLP/JSON. */
-declare function exportRunAsOtlp(store: TraceStore, runId: string, resourceAttrs?: Record<string, string | number | boolean>): Promise<OtlpExport>;
-
-/**
- * Replay-from-raw-events — turn every captured campaign run into a
- * re-runnable artifact.
- *
- * The premise: 0.21 made `RawProviderSink` capture every provider HTTP
- * envelope. 0.22's `runEvalCampaign` makes capture the default. Together
- * they mean every past run is a complete fingerprint of what happened on
- * the wire — and that fingerprint is enough to replay the run without
- * burning new LLM cost.
- *
- * Three use cases this primitive enables:
- *
- *   1. **Post-hoc judging** — apply a new judge / rubric / scoring callback
- *      to last week's runs without re-calling any LLM. The cost of trying
- *      a new rubric drops from "another full sweep" to a CPU-bound replay.
- *   2. **Determinism audits** — replay the same campaign and verify the
- *      raw responses match byte-for-byte. Any drift is a non-determinism
- *      bug (in the harness, the prompt builder, the sandbox, …).
- *   3. **Free judge calibration** — run two judges on identical responses
- *      and measure inter-judge agreement without doubling LLM spend.
- *
- * The interface is deliberately fetch-shaped. Inject `createReplayFetch`
- * into `LlmClientOptions.fetch` and every `callLlm` transparently reads
- * from the cache instead of calling the network. No new code path through
- * the LLM client is needed; the cache hit is invisible to the runner.
- */
-
-declare class ReplayCacheMissError extends Error {
-    readonly url: string;
-    readonly requestKey: string;
-    constructor(url: string, requestKey: string, message?: string);
-}
-interface ReplayCacheEntry {
-    request: RawProviderEvent;
-    response: RawProviderEvent;
-}
-interface ReplayCacheStats {
-    total: number;
-    byProvider: Record<string, number>;
-    byModel: Record<string, number>;
-    /** Spans for which we have a request but no response (run aborted mid-call). */
-    orphanRequests: number;
-}
-/**
- * In-memory deterministic cache of (request → response) keyed on a stable
- * hash of the request body. Built from a `RawProviderSink` containing
- * paired `request` and `response` events from a previous run.
- *
- * The cache is the source of truth for replay; `createReplayFetch` is a
- * thin wrapper that reads from it.
- */
-declare class ReplayCache {
-    private byKey;
-    private orphans;
-    private byProvider;
-    private byModel;
-    /**
-     * Build a cache from a sink's events. The sink must implement `list()`.
-     * Filter by `runId` / `spanId` to scope to a specific replay.
-     */
-    static fromSink(sink: RawProviderSink, filter?: {
-        runId?: string;
-        spanId?: string;
-    }): Promise<ReplayCache>;
-    /** Build a cache from an in-memory event list. */
-    static fromEvents(events: RawProviderEvent[]): Promise<ReplayCache>;
-    /** Number of cacheable (request, response) pairs in the cache. */
-    size(): number;
-    stats(): ReplayCacheStats;
-    /**
-     * Look up a cached response by hashing the (model, messages, temperature,
-     * maxTokens, response_format) shape. Returns `undefined` on miss; the
-     * caller decides whether to throw, fall back to the network, or skip.
-     */
-    lookup(requestBody: unknown): Promise<ReplayCacheEntry | undefined>;
-}
-interface ReplayFetchOptions {
-    /**
-     * Behaviour on cache miss. Default `'throw'`. `'fallback'` calls the
-     * `fallbackFetch` (typically `globalThis.fetch`) so a partial replay can
-     * still complete; `'fail-closed'` returns a synthetic 599 response so the
-     * call site sees a non-retriable failure.
-     */
-    onMiss?: 'throw' | 'fallback' | 'fail-closed';
-    fallbackFetch?: typeof fetch;
-    /** Optional callback fired once per replayed call (for telemetry / counters). */
-    onHit?: (info: {
-        url: string;
-        provider: string;
-        model: string;
-    }) => void;
-    /** Optional callback fired on cache miss before the `onMiss` policy applies. */
-    onMissNotify?: (info: {
-        url: string;
-        requestBody: unknown;
-    }) => void;
-}
-/**
- * Build a `fetch`-shaped function that serves cached responses out of a
- * `ReplayCache` for any URL ending in `/chat/completions`. Pass through
- * `LlmClientOptions.fetch` and `callLlm` becomes free.
- *
- * Non-`/chat/completions` URLs are passed straight to the fallback fetch
- * (default: `globalThis.fetch`). This matters because non-LLM HTTP work
- * (judge HTTP servers, sandbox callbacks) sometimes flows through the same
- * `fetch` and shouldn't be intercepted.
- */
-declare function createReplayFetch(cache: ReplayCache, opts?: ReplayFetchOptions): typeof fetch;
-/**
- * Convenience iterator over `(request, response)` pairs in a sink — for
- * post-hoc scoring that doesn't need a `fetch` shim. The judge or scorer
- * runs purely in-process over cached LLM outputs.
- */
-declare function iterateRawCalls(sink: RawProviderSink, filter?: {
-    runId?: string;
-    spanId?: string;
-}): AsyncGenerator<ReplayCacheEntry>;
+import { N as NotFoundError } from './errors-BZ9sTdz7.js';
 
 /**
  * Shared types for the trace-analyst module.
@@ -555,6 +310,137 @@ interface AnalyzeTracesOptions {
  */
 declare function analyzeTraces(input: AnalyzeTracesInput, options: AnalyzeTracesOptions): Promise<AnalyzeTracesResult>;
 
+/**
+ * Trace-analyst auto-execution hook.
+ *
+ * Wires `analyzeTraces` into a `TraceEmitter`'s `onRunComplete` so a
+ * direct matrix run produces an analysis artifact without an out-of-band
+ * step. Designed for the case where the consumer reports "the analyst
+ * never ran" — the cause is almost always orchestration, not the analyst.
+ *
+ * Usage:
+ *
+ *   const emitter = new TraceEmitter(store, {
+ *     onRunComplete: [traceAnalystOnRunComplete({ analyze: opts, save })],
+ *   })
+ *
+ * Hooks are best-effort by default — they never crash the underlying run.
+ * The caller decides whether to gate the run on the analysis result via
+ * the `gateOn` callback.
+ */
+
+interface TraceAnalystHookOptions {
+    /**
+     * Options forwarded to `analyzeTraces`. The hook supplies the question
+     * if you don't pass one — defaulting to a launch-grade prompt that asks
+     * for failure modes, surprising findings, and a recommendation.
+     */
+    analyze: Omit<AnalyzeTracesOptions, 'source'> & {
+        source?: AnalyzeTracesOptions['source'];
+    };
+    /**
+     * Override the question. The default is intentionally generic:
+     * "Summarise what happened in this run, surface any failure modes,
+     *  surprising findings, or evidence the verdict is wrong."
+     */
+    question?: string;
+    /**
+     * Persist the result. The hook calls this with the analysis output and
+     * the run context. Common implementations write to a TraceAnalysisStore
+     * or append to a per-run JSONL.
+     */
+    save?: (result: AnalyzeTracesResult, ctx: RunCompleteHookContext) => Promise<void>;
+    /**
+     * Predicate gating execution per run. Default: every completed run.
+     * Use to skip aborted runs, debug runs, or runs without LLM activity.
+     */
+    shouldRun?: (ctx: RunCompleteHookContext) => boolean;
+    /**
+     * Optional gate: if set and returns false, the hook records the failure
+     * as a log event on the run instead of staying quiet. The caller can
+     * then trigger downstream alerts off `analyst_gate_failed` log events.
+     */
+    gateOn?: (result: AnalyzeTracesResult, ctx: RunCompleteHookContext) => boolean;
+}
+declare function traceAnalystOnRunComplete(opts: TraceAnalystHookOptions): RunCompleteHook;
+
+interface TraceInsightTask {
+    id: string;
+    name: string;
+    prompt?: string;
+    difficulty?: string;
+    tags?: string[];
+    outcome?: string;
+    score?: number;
+    gaps?: string[];
+}
+interface TraceInsightSuite {
+    name: string;
+    collectionId?: string;
+    tasks: TraceInsightTask[];
+}
+interface TraceInsightFinding {
+    kind: string;
+    severity?: string;
+    taskIds: string[];
+    evidence?: string;
+    proposedFixClass?: string;
+}
+interface TraceInsightQuestion {
+    id: string;
+    question: string;
+    why: string;
+}
+interface TraceInsightPanelRole {
+    id: string;
+    name: string;
+    responsibility: string;
+}
+interface TraceInsightPromptInput {
+    suite: TraceInsightSuite;
+    findings?: TraceInsightFinding[];
+    agent?: Record<string, unknown>;
+    totals?: Record<string, unknown>;
+    maxRepresentativeTraces?: number;
+}
+interface TraceInsightContext {
+    suite: TraceInsightSuite;
+    scope: string;
+    keywords: string[];
+    questions: TraceInsightQuestion[];
+    panel: TraceInsightPanelRole[];
+    findings: TraceInsightFinding[];
+    agent: Record<string, unknown> | null;
+    totals: Record<string, unknown> | null;
+}
+interface TraceInsightQualityGate {
+    id: string;
+    label: string;
+    passed: boolean;
+    severity: 'critical' | 'high' | 'medium' | 'low';
+    detail: string;
+}
+interface TraceInsightReadiness {
+    score: number;
+    grade: 'external-ready' | 'internal-review' | 'raw-analysis';
+    gates: TraceInsightQualityGate[];
+}
+declare function tokenizeDomainWords(value: string): string[];
+declare function inferDomainKeywords(suite: TraceInsightSuite): string[];
+declare function domainEvidencePattern(keywords: string[]): RegExp;
+declare function describeTraceInsightScope(suite: TraceInsightSuite): string;
+declare function planTraceInsightQuestions(input: TraceInsightPromptInput): TraceInsightQuestion[];
+declare function buildTraceInsightContext(input: TraceInsightPromptInput): TraceInsightContext;
+declare function scoreTraceInsightReadiness(context: TraceInsightContext): TraceInsightReadiness;
+declare function defaultTraceInsightPanel(): TraceInsightPanelRole[];
+declare function buildTraceInsightPrompt(input: TraceInsightPromptInput): string;
+
+/** Ax RLM prompt for bounded trace discovery and evidence-backed analysis. */
+declare const TRACE_ANALYST_ACTOR_DESCRIPTION = "You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the `traces` namespace.\n\nDISCOVERY \u2192 NARROW \u2192 DEEP-READ protocol \u2014 follow exactly:\n\n1. ALWAYS call `traces.getDatasetOverview({})` FIRST without a regex_pattern. The result tells you total_traces, raw_jsonl_bytes, services, agents, models, and sample_trace_ids (real ids \u2014 never fabricate one).\n\n2. Use raw_jsonl_bytes to gauge how expensive raw scans will be. `filters.regex_pattern` is the one scan-heavy filter on getDatasetOverview / queryTraces / countTraces \u2014 narrow with indexed fields (has_errors, model_names, service_names, agent_names, time bounds) BEFORE adding a regex on a large dataset.\n\n3. To list more traces than the sample, call `traces.queryTraces({ filters?, limit, offset? })`. Each summary carries raw_jsonl_bytes \u2014 use it to choose between viewTrace and searchTrace BEFORE calling either.\n\n4. Per-trace inspection:\n   - SMALL trace (raw_jsonl_bytes well under 150_000): call `traces.viewTrace({ trace_id })`. Returns all spans. Per-attribute payloads are head-capped at ~4KB; large `input.value` / `output.value` / `llm.input_messages` will show a `[trace-analyst truncated: N bytes]` marker.\n   - LARGE trace (raw_jsonl_bytes near or above 150_000, or you saw an `oversized` response): use `traces.searchTrace({ trace_id, regex_pattern })` to get bounded SpanMatchRecords (span metadata + matched text + surrounding context). Then call `traces.viewSpans({ trace_id, span_ids: [...] })` for surgical reads (~16KB cap, 4\u00D7 higher than discovery), or `traces.searchSpan({ trace_id, span_id, regex_pattern })` for one large span. Stays bounded regardless of trace size.\n   - Useful regex patterns: `STATUS_CODE_ERROR` (failures), tool names like `grep` or `view_trace`, error strings like `MaxTurnsExceeded`, model names, attribute keys.\n\n5. ONLY call viewTrace / viewSpans / searchTrace / searchSpan with trace/span ids you have already seen in sample_trace_ids, a queryTraces page, or a previous search result. Never invent ids.\n\n5a. **Result-shape contract** \u2014 searchTrace and searchSpan return `{ trace_id, hits, total_matches, has_more }`. Iterate `result.hits` (NOT result.matches). Each hit has `{ span_id, span_name, span_kind, attribute_path, matched_text, context_before, context_after, match_offset }`. viewTrace returns `{ trace_id, spans }` (or `oversized`). viewSpans returns `{ trace_id, spans, missing_span_ids, truncated_attribute_count }`. Never assume a field name \u2014 log the result shape first if unsure.\n\n6. If viewTrace returns an `oversized` summary instead of `spans`, DO NOT retry the same call. Read the summary's top_span_names, span_count, span_response_bytes_max, error_span_count to plan a follow-up: switch to searchTrace (or searchSpan for one large span), then viewSpans on a smaller, surgical span_ids set.\n\n7. If searchTrace or searchSpan returns has_more=true, REFINE the regex to be more specific rather than blindly raising max_matches.\n\n8. If a tool errors (invalid regex, range error), STOP and reconsider \u2014 don't retry with a guessed id or argument. Use the discovery tools above to recover.\n\n9. If a ~4KB-truncated payload from viewTrace / searchTrace matters for your answer, first try viewSpans on that span id (~16KB cap). If a 16KB-truncated payload from viewSpans still matters, narrow further with searchSpan against a more specific regex rather than asking for the full payload again.\n\n10. If maxDepth > 0 and the question splits into independent semantic branches, delegate well-defined subtasks to subagents using `await llmQuery(...)`. Pass narrow context and a focused query. Examples:\n\n    const reviews = await llmQuery([\n      { query: 'Drill into trace abc123 \u2014 what tool calls preceded the failure?', context: { trace_id: 'abc123' } },\n      { query: 'Drill into trace def456 \u2014 same failure mode?', context: { trace_id: 'def456' } },\n    ]);\n\nOBSERVABILITY rules:\n- Each non-final actor turn must emit at least one `console.log(...)` for evidence. Up to 3 logs per turn is fine when correlating multiple data sources (e.g. one log for findings list, one for source-file content, one for derived analysis).\n- Do NOT combine `console.log` with `final(...)` or `askClarification(...)` in the same turn \u2014 finish gathering data first, then call final on its own turn.\n- Reuse runtime variables across turns; don't recompute.\n- When done, call `await final(answer)` with the fully-formed report. The responder rewrites the answer into output fields; if you only pass a vague summary string the responder has nothing concrete to format.\n\nCRITICAL \u2014 `final()` payload contract for evidence-grounded analysis tasks:\n- Pass a STRUCTURED object as the second arg with the actual data the responder needs to format the answer. Do NOT pass abstract instructions; pass evidence.\n- Example for per-item verdict tasks:\n  ```js\n  await final(\"Format the per-item verdict report from the evidence below.\", {\n    findings: [\n      { id: 'sub-1-finding-1', claim: '...', verdict: 'TRUE-POSITIVE', evidence: 'lines 42-45 of contracts/X.sol show ...' },\n      ...all items\n    ],\n    systemic_summary: '3 sentences I wrote based on the evidence above'\n  });\n  ```\n- Calling `final(\"answer\", {})` with no evidence is a failure mode \u2014 the responder will hallucinate or echo back the field names. Always include the gathered data.\n- Premature final after a single viewSpans call is INSUFFICIENT for per-finding analysis tasks. Read the requested attributes (e.g. `spans[i].attributes['redteam.finding.title']`), and for each one perform the requested cross-reference (e.g. read the source SPAN's `attributes['source.content']`).\n\nOUTPUT contract \u2014 your final answer must include:\n- A clear prose conclusion answering the user's question.\n- Trace ids and span ids cited as evidence for each claim.\n- Failure modes named in the user's domain language, with frequency and concrete examples.\n\nDo NOT invent trace ids, span ids, error messages, or model names. Every fact must be traceable to a tool result.";
+declare const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-actor-v5-2026-05-06";
+/** Subagent prompt for focused trace-inspection subtasks. */
+declare const TRACE_ANALYST_SUBAGENT_DESCRIPTION = "You are a trace-analyst subagent. Your parent has delegated a focused trace-inspection question. Use the same DISCOVERY \u2192 NARROW \u2192 DEEP-READ protocol but stay tightly scoped: do exactly what was asked, return a concise compact answer, do NOT spawn further subagents unless the parent's question is genuinely multi-branch.\n\nCite trace ids and span ids for every claim. Do NOT invent ids.";
+
 /**
  * `OtlpFileTraceStore` — read-only OTLP-JSONL trace store for the
  * trace-analyst.
@@ -645,14 +531,14 @@ declare class OtlpFileTraceStore implements TraceAnalysisStore {
     private buildOversizedSummary;
     private scanSpanForMatches;
 }
-declare class TraceFileMissingError extends Error {
+declare class TraceFileMissingError extends NotFoundError {
     constructor(path: string);
 }
-declare class TraceNotFoundError extends Error {
+declare class TraceNotFoundError extends NotFoundError {
     readonly trace_id: string;
     constructor(trace_id: string);
 }
-declare class SpanNotFoundError extends Error {
+declare class SpanNotFoundError extends NotFoundError {
     readonly trace_id: string;
     readonly span_id: string;
     constructor(trace_id: string, span_id: string);
@@ -700,135 +586,4 @@ declare function traceAnalystFunctionGroup(opts: BuildTraceAnalystToolsOpts): {
     functions: AxFunction[];
 };
 
-/**
- * Trace-analyst auto-execution hook.
- *
- * Wires `analyzeTraces` into a `TraceEmitter`'s `onRunComplete` so a
- * direct matrix run produces an analysis artifact without an out-of-band
- * step. Designed for the case where the consumer reports "the analyst
- * never ran" — the cause is almost always orchestration, not the analyst.
- *
- * Usage:
- *
- *   const emitter = new TraceEmitter(store, {
- *     onRunComplete: [traceAnalystOnRunComplete({ analyze: opts, save })],
- *   })
- *
- * Hooks are best-effort by default — they never crash the underlying run.
- * The caller decides whether to gate the run on the analysis result via
- * the `gateOn` callback.
- */
-
-interface TraceAnalystHookOptions {
-    /**
-     * Options forwarded to `analyzeTraces`. The hook supplies the question
-     * if you don't pass one — defaulting to a launch-grade prompt that asks
-     * for failure modes, surprising findings, and a recommendation.
-     */
-    analyze: Omit<AnalyzeTracesOptions, 'source'> & {
-        source?: AnalyzeTracesOptions['source'];
-    };
-    /**
-     * Override the question. The default is intentionally generic:
-     * "Summarise what happened in this run, surface any failure modes,
-     *  surprising findings, or evidence the verdict is wrong."
-     */
-    question?: string;
-    /**
-     * Persist the result. The hook calls this with the analysis output and
-     * the run context. Common implementations write to a TraceAnalysisStore
-     * or append to a per-run JSONL.
-     */
-    save?: (result: AnalyzeTracesResult, ctx: RunCompleteHookContext) => Promise<void>;
-    /**
-     * Predicate gating execution per run. Default: every completed run.
-     * Use to skip aborted runs, debug runs, or runs without LLM activity.
-     */
-    shouldRun?: (ctx: RunCompleteHookContext) => boolean;
-    /**
-     * Optional gate: if set and returns false, the hook records the failure
-     * as a log event on the run instead of staying quiet. The caller can
-     * then trigger downstream alerts off `analyst_gate_failed` log events.
-     */
-    gateOn?: (result: AnalyzeTracesResult, ctx: RunCompleteHookContext) => boolean;
-}
-declare function traceAnalystOnRunComplete(opts: TraceAnalystHookOptions): RunCompleteHook;
-
-/** Ax RLM prompt for bounded trace discovery and evidence-backed analysis. */
-declare const TRACE_ANALYST_ACTOR_DESCRIPTION = "You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the `traces` namespace.\n\nDISCOVERY \u2192 NARROW \u2192 DEEP-READ protocol \u2014 follow exactly:\n\n1. ALWAYS call `traces.getDatasetOverview({})` FIRST without a regex_pattern. The result tells you total_traces, raw_jsonl_bytes, services, agents, models, and sample_trace_ids (real ids \u2014 never fabricate one).\n\n2. Use raw_jsonl_bytes to gauge how expensive raw scans will be. `filters.regex_pattern` is the one scan-heavy filter on getDatasetOverview / queryTraces / countTraces \u2014 narrow with indexed fields (has_errors, model_names, service_names, agent_names, time bounds) BEFORE adding a regex on a large dataset.\n\n3. To list more traces than the sample, call `traces.queryTraces({ filters?, limit, offset? })`. Each summary carries raw_jsonl_bytes \u2014 use it to choose between viewTrace and searchTrace BEFORE calling either.\n\n4. Per-trace inspection:\n   - SMALL trace (raw_jsonl_bytes well under 150_000): call `traces.viewTrace({ trace_id })`. Returns all spans. Per-attribute payloads are head-capped at ~4KB; large `input.value` / `output.value` / `llm.input_messages` will show a `[trace-analyst truncated: N bytes]` marker.\n   - LARGE trace (raw_jsonl_bytes near or above 150_000, or you saw an `oversized` response): use `traces.searchTrace({ trace_id, regex_pattern })` to get bounded SpanMatchRecords (span metadata + matched text + surrounding context). Then call `traces.viewSpans({ trace_id, span_ids: [...] })` for surgical reads (~16KB cap, 4\u00D7 higher than discovery), or `traces.searchSpan({ trace_id, span_id, regex_pattern })` for one large span. Stays bounded regardless of trace size.\n   - Useful regex patterns: `STATUS_CODE_ERROR` (failures), tool names like `grep` or `view_trace`, error strings like `MaxTurnsExceeded`, model names, attribute keys.\n\n5. ONLY call viewTrace / viewSpans / searchTrace / searchSpan with trace/span ids you have already seen in sample_trace_ids, a queryTraces page, or a previous search result. Never invent ids.\n\n5a. **Result-shape contract** \u2014 searchTrace and searchSpan return `{ trace_id, hits, total_matches, has_more }`. Iterate `result.hits` (NOT result.matches). Each hit has `{ span_id, span_name, span_kind, attribute_path, matched_text, context_before, context_after, match_offset }`. viewTrace returns `{ trace_id, spans }` (or `oversized`). viewSpans returns `{ trace_id, spans, missing_span_ids, truncated_attribute_count }`. Never assume a field name \u2014 log the result shape first if unsure.\n\n6. If viewTrace returns an `oversized` summary instead of `spans`, DO NOT retry the same call. Read the summary's top_span_names, span_count, span_response_bytes_max, error_span_count to plan a follow-up: switch to searchTrace (or searchSpan for one large span), then viewSpans on a smaller, surgical span_ids set.\n\n7. If searchTrace or searchSpan returns has_more=true, REFINE the regex to be more specific rather than blindly raising max_matches.\n\n8. If a tool errors (invalid regex, range error), STOP and reconsider \u2014 don't retry with a guessed id or argument. Use the discovery tools above to recover.\n\n9. If a ~4KB-truncated payload from viewTrace / searchTrace matters for your answer, first try viewSpans on that span id (~16KB cap). If a 16KB-truncated payload from viewSpans still matters, narrow further with searchSpan against a more specific regex rather than asking for the full payload again.\n\n10. If maxDepth > 0 and the question splits into independent semantic branches, delegate well-defined subtasks to subagents using `await llmQuery(...)`. Pass narrow context and a focused query. Examples:\n\n    const reviews = await llmQuery([\n      { query: 'Drill into trace abc123 \u2014 what tool calls preceded the failure?', context: { trace_id: 'abc123' } },\n      { query: 'Drill into trace def456 \u2014 same failure mode?', context: { trace_id: 'def456' } },\n    ]);\n\nOBSERVABILITY rules:\n- Each non-final actor turn must emit at least one `console.log(...)` for evidence. Up to 3 logs per turn is fine when correlating multiple data sources (e.g. one log for findings list, one for source-file content, one for derived analysis).\n- Do NOT combine `console.log` with `final(...)` or `askClarification(...)` in the same turn \u2014 finish gathering data first, then call final on its own turn.\n- Reuse runtime variables across turns; don't recompute.\n- When done, call `await final(answer)` with the fully-formed report. The responder rewrites the answer into output fields; if you only pass a vague summary string the responder has nothing concrete to format.\n\nCRITICAL \u2014 `final()` payload contract for evidence-grounded analysis tasks:\n- Pass a STRUCTURED object as the second arg with the actual data the responder needs to format the answer. Do NOT pass abstract instructions; pass evidence.\n- Example for per-item verdict tasks:\n  ```js\n  await final(\"Format the per-item verdict report from the evidence below.\", {\n    findings: [\n      { id: 'sub-1-finding-1', claim: '...', verdict: 'TRUE-POSITIVE', evidence: 'lines 42-45 of contracts/X.sol show ...' },\n      ...all items\n    ],\n    systemic_summary: '3 sentences I wrote based on the evidence above'\n  });\n  ```\n- Calling `final(\"answer\", {})` with no evidence is a failure mode \u2014 the responder will hallucinate or echo back the field names. Always include the gathered data.\n- Premature final after a single viewSpans call is INSUFFICIENT for per-finding analysis tasks. Read the requested attributes (e.g. `spans[i].attributes['redteam.finding.title']`), and for each one perform the requested cross-reference (e.g. read the source SPAN's `attributes['source.content']`).\n\nOUTPUT contract \u2014 your final answer must include:\n- A clear prose conclusion answering the user's question.\n- Trace ids and span ids cited as evidence for each claim.\n- Failure modes named in the user's domain language, with frequency and concrete examples.\n\nDo NOT invent trace ids, span ids, error messages, or model names. Every fact must be traceable to a tool result.";
-declare const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-actor-v5-2026-05-06";
-/** Subagent prompt for focused trace-inspection subtasks. */
-declare const TRACE_ANALYST_SUBAGENT_DESCRIPTION = "You are a trace-analyst subagent. Your parent has delegated a focused trace-inspection question. Use the same DISCOVERY \u2192 NARROW \u2192 DEEP-READ protocol but stay tightly scoped: do exactly what was asked, return a concise compact answer, do NOT spawn further subagents unless the parent's question is genuinely multi-branch.\n\nCite trace ids and span ids for every claim. Do NOT invent ids.";
-
-interface TraceInsightTask {
-    id: string;
-    name: string;
-    prompt?: string;
-    difficulty?: string;
-    tags?: string[];
-    outcome?: string;
-    score?: number;
-    gaps?: string[];
-}
-interface TraceInsightSuite {
-    name: string;
-    collectionId?: string;
-    tasks: TraceInsightTask[];
-}
-interface TraceInsightFinding {
-    kind: string;
-    severity?: string;
-    taskIds: string[];
-    evidence?: string;
-    proposedFixClass?: string;
-}
-interface TraceInsightQuestion {
-    id: string;
-    question: string;
-    why: string;
-}
-interface TraceInsightPanelRole {
-    id: string;
-    name: string;
-    responsibility: string;
-}
-interface TraceInsightPromptInput {
-    suite: TraceInsightSuite;
-    findings?: TraceInsightFinding[];
-    agent?: Record<string, unknown>;
-    totals?: Record<string, unknown>;
-    maxRepresentativeTraces?: number;
-}
-interface TraceInsightContext {
-    suite: TraceInsightSuite;
-    scope: string;
-    keywords: string[];
-    questions: TraceInsightQuestion[];
-    panel: TraceInsightPanelRole[];
-    findings: TraceInsightFinding[];
-    agent: Record<string, unknown> | null;
-    totals: Record<string, unknown> | null;
-}
-interface TraceInsightQualityGate {
-    id: string;
-    label: string;
-    passed: boolean;
-    severity: 'critical' | 'high' | 'medium' | 'low';
-    detail: string;
-}
-interface TraceInsightReadiness {
-    score: number;
-    grade: 'external-ready' | 'internal-review' | 'raw-analysis';
-    gates: TraceInsightQualityGate[];
-}
-declare function tokenizeDomainWords(value: string): string[];
-declare function inferDomainKeywords(suite: TraceInsightSuite): string[];
-declare function domainEvidencePattern(keywords: string[]): RegExp;
-declare function describeTraceInsightScope(suite: TraceInsightSuite): string;
-declare function planTraceInsightQuestions(input: TraceInsightPromptInput): TraceInsightQuestion[];
-declare function buildTraceInsightContext(input: TraceInsightPromptInput): TraceInsightContext;
-declare function scoreTraceInsightReadiness(context: TraceInsightContext): TraceInsightReadiness;
-declare function defaultTraceInsightPanel(): TraceInsightPanelRole[];
-declare function buildTraceInsightPrompt(input: TraceInsightPromptInput): string;
-
-export { type AnalyzeTracesInput, type AnalyzeTracesOptions, type AnalyzeTracesResult, type AnalyzeTracesTurnSnapshot, DEFAULT_REDACTION_RULES, DEFAULT_TRACE_ANALYST_BUDGETS, type DatasetOverview, FailureClass, JudgeSpan, LlmSpan, OTEL_AGENT_EVAL_SCOPE, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpResourceSpans, type OtlpSpan, type QueryTracesPage, REDACTION_VERSION, RawProviderEvent, RawProviderSink, type RedactionReport, type RedactionRule, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, type ReplayFetchOptions, Run, RunCompleteHook, RunCompleteHookContext, type SearchSpanResult, type SearchTraceResult, type SpanMatchRecord, SpanNotFoundError, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, ToolSpan, type TraceAnalysisStore, type TraceAnalystByteBudgets, type TraceAnalystFilters, type TraceAnalystHookOptions, type TraceAnalystSpan, type TraceAnalystSpanKind, type TraceAnalystSpanStatus, type TraceAnalystTraceSummary, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, TraceStore, type ViewSpansResult, type ViewTraceOversized, type ViewTraceResult, aggregateLlm, analyzeTraces, argHash, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, groupBy, inferDomainKeywords, iterateRawCalls, judgeSpans, llmSpans, planTraceInsightQuestions, redactString, redactValue, runFailureClass, runsForScenario, scoreTraceInsightReadiness, tokenizeDomainWords, toolSpans, traceAnalystFunctionGroup, traceAnalystOnRunComplete };
+export { type AnalyzeTracesInput, type AnalyzeTracesOptions, type AnalyzeTracesResult, type AnalyzeTracesTurnSnapshot, DEFAULT_TRACE_ANALYST_BUDGETS, type DatasetOverview, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type QueryTracesPage, RunCompleteHook, RunCompleteHookContext, type SearchSpanResult, type SearchTraceResult, type SpanMatchRecord, SpanNotFoundError, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, type TraceAnalysisStore, type TraceAnalystByteBudgets, type TraceAnalystFilters, type TraceAnalystHookOptions, type TraceAnalystSpan, type TraceAnalystSpanKind, type TraceAnalystSpanStatus, type TraceAnalystTraceSummary, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, type ViewSpansResult, type ViewTraceOversized, type ViewTraceResult, analyzeTraces, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, inferDomainKeywords, planTraceInsightQuestions, scoreTraceInsightReadiness, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete };