@tangle-network/agent-eval 0.56.0 → 0.58.0

package/dist/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.56.0",
+    "version": "0.58.0",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/traces.d.ts

@@ -1,6 +1,6 @@
 import { N as NotFoundError, R as ReplayError } from './errors-mje_cKOs.js';
-import { R as RawProviderSink, d as RawProviderEvent } from './raw-provider-sink-C46HDghv.js';
-export { F as FileSystemRawProviderSink, a as FileSystemRawProviderSinkOptions, I as InMemoryRawProviderSink, b as InMemoryRawProviderSinkOptions, N as NoopRawProviderSink, P as ProviderRedactor, c as RawProviderDirection, e as RawProviderSinkFilter, f as defaultProviderRedactor, p as providerFromBaseUrl } from './raw-provider-sink-C46HDghv.js';
+import { P as ProviderRedactor, R as RawProviderSink, d as RawProviderEvent } from './raw-provider-sink-C46HDghv.js';
+export { F as FileSystemRawProviderSink, a as FileSystemRawProviderSinkOptions, I as InMemoryRawProviderSink, b as InMemoryRawProviderSinkOptions, N as NoopRawProviderSink, c as RawProviderDirection, e as RawProviderSinkFilter, f as defaultProviderRedactor, p as providerFromBaseUrl } from './raw-provider-sink-C46HDghv.js';
 import { R as RunCompleteHook, a as RunCompleteHookContext } from './emitter-DEZwY14K.js';
 export { S as SpanHandle, T as TraceEmitter, b as TraceEmitterOptions, l as llmSpanFromProvider } from './emitter-DEZwY14K.js';
 export { b as RunIntegrityError, R as RunIntegrityExpectations, c as RunIntegrityIssue, d as RunIntegrityIssueCode, a as RunIntegrityReport, e as assertRunCaptured, t as throwIfRunIncomplete } from './integrity-CfXjSqEv.js';
@@ -12,6 +12,45 @@ import { AxAIService, AxFunction } from '@ax-llm/ax';
 import { T as TraceAnalysisStore, f as TraceAnalystFilters, a as DatasetOverview, Q as QueryTracesPage, l as ViewTraceResult, V as ViewSpansResult, b as SearchTraceResult, S as SearchSpanResult } from './store-CJbzDxZ2.js';
 export { D as DEFAULT_TRACE_ANALYST_BUDGETS, c as SpanMatchRecord, d as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, e as TraceAnalystByteBudgets, g as TraceAnalystSpan, h as TraceAnalystSpanKind, i as TraceAnalystSpanStatus, j as TraceAnalystTraceSummary, k as ViewTraceOversized } from './store-CJbzDxZ2.js';
 
+/**
+ * `captureFetchToRawSink` — wrap a `fetch` so every request / response / error
+ * against a provider is recorded into a `RawProviderSink` as the canonical
+ * `RawProviderEvent` triple. The one substrate copy of the fetch-capture
+ * pattern four consumers hand-roll (legal ships two copies).
+ *
+ * The returned value is a plain `typeof fetch` — pass it as the `fetchImpl` to
+ * any OpenAI-compatible backend factory. Capture is best-effort by default: a
+ * sink write that throws does NOT take down the underlying LLM call (set
+ * `failClosed` to change that). Uses the existing `defaultProviderRedactor` +
+ * `providerFromBaseUrl` — no new redaction policy.
+ */
+
+interface CaptureFetchContext {
+    /** Logical run id stamped on every captured event. Required — without it
+     *  the raw events can't be paired with their parent `Run`. */
+    runId: string;
+    /** Optional logical span id (enables span-level sink filtering). */
+    spanId?: string;
+    /** Resolved base URL (normalised, no trailing slash). Used for the event's
+     *  `baseUrl` and for endpoint-path extraction. */
+    baseUrl: string;
+    /** Model id the caller intends to invoke. Stamped on every event. */
+    model: string;
+    /** Provider override. When omitted, `providerFromBaseUrl(baseUrl)`. */
+    provider?: string;
+}
+interface CaptureFetchOptions {
+    /** Override the capture-time redactor. Default `defaultProviderRedactor`. */
+    redactor?: ProviderRedactor;
+    /** Cap on captured response-body bytes; beyond it the body is truncated and
+     *  `body_truncated` is added to `redactedFields`. Default 2 MiB. */
+    responseBodyByteCap?: number;
+    /** When true, a sink-write failure propagates to the caller. Default false
+     *  — capture is best-effort so a sink failure never kills the LLM call. */
+    failClosed?: boolean;
+}
+declare function captureFetchToRawSink(fetch: typeof globalThis.fetch, sink: RawProviderSink, ctx: CaptureFetchContext, opts?: CaptureFetchOptions): typeof globalThis.fetch;
+
 /**
  * OpenTelemetry JSON export — maps TraceSchema v1 to OTLP/JSON so
  * traces render natively in Jaeger / Honeycomb / Langfuse / Grafana.
@@ -401,6 +440,50 @@ declare function scoreTraceInsightReadiness(context: TraceInsightContext): Trace
 declare function defaultTraceInsightPanel(): TraceInsightPanelRole[];
 declare function buildTraceInsightPrompt(input: TraceInsightPromptInput): string;
 
+/**
+ * `flattenOtlpExportToNdjson` — flatten an `OtlpExport` (the shape
+ * `exportRunAsOtlp` produces) into the per-line JSON the analyst's
+ * `OtlpFileTraceStore` index reads. Replaces three per-consumer OTLP
+ * flatteners with one canonical projection.
+ *
+ * Pure function, no I/O — the caller does `.map(JSON.stringify).join('\n')`
+ * and writes the file (consumers want control over rotation + naming).
+ */
+
+interface OtlpFlatLine {
+    trace_id: string;
+    span_id: string;
+    parent_span_id: string | null;
+    name: string;
+    kind: string;
+    start_time: string;
+    end_time: string;
+    status: {
+        code: 'STATUS_CODE_OK' | 'STATUS_CODE_ERROR' | 'STATUS_CODE_UNSET';
+        message?: string;
+    };
+    resource: {
+        attributes: Record<string, string | number | boolean>;
+    };
+    attributes: Record<string, string | number | boolean>;
+    events?: Array<{
+        name: string;
+        timeUnixNano?: string;
+        attributes?: Record<string, unknown>;
+    }>;
+}
+interface FlattenOtlpOptions {
+    /** `'openinference'` (default) mirrors per-span attributes into the
+     *  OpenInference vocabulary the analyst's `inferKind` reads
+     *  (`llm.model`→`llm.model_name`, `tool.name`→`inference.tool.name`,
+     *  `span.kind`→`openinference.span.kind` uppercased). `'none'` passes
+     *  attributes through untouched. */
+    attributeVocabulary?: 'openinference' | 'none';
+    /** Override the numeric-kind → otlp-string mapping. */
+    kindMap?: Partial<Record<number, string>>;
+}
+declare function flattenOtlpExportToNdjson(otlpExport: OtlpExport, opts?: FlattenOtlpOptions): OtlpFlatLine[];
+
 /** Ax RLM prompt for bounded trace discovery and evidence-backed analysis. */
 declare const TRACE_ANALYST_ACTOR_DESCRIPTION = "You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the `traces` namespace.\n\nDISCOVERY \u2192 NARROW \u2192 DEEP-READ protocol \u2014 follow exactly:\n\n1. ALWAYS call `traces.getDatasetOverview({})` FIRST without a regex_pattern. The result tells you total_traces, raw_jsonl_bytes, services, agents, models, and sample_trace_ids (real ids \u2014 never fabricate one).\n\n2. Use raw_jsonl_bytes to gauge how expensive raw scans will be. `filters.regex_pattern` is the one scan-heavy filter on getDatasetOverview / queryTraces / countTraces \u2014 narrow with indexed fields (has_errors, model_names, service_names, agent_names, time bounds) BEFORE adding a regex on a large dataset.\n\n3. To list more traces than the sample, call `traces.queryTraces({ filters?, limit, offset? })`. Each summary carries raw_jsonl_bytes \u2014 use it to choose between viewTrace and searchTrace BEFORE calling either.\n\n4. Per-trace inspection:\n   - SMALL trace (raw_jsonl_bytes well under 150_000): call `traces.viewTrace({ trace_id })`. Returns all spans. Per-attribute payloads are head-capped at ~4KB; large `input.value` / `output.value` / `llm.input_messages` will show a `[trace-analyst truncated: N bytes]` marker.\n   - LARGE trace (raw_jsonl_bytes near or above 150_000, or you saw an `oversized` response): use `traces.searchTrace({ trace_id, regex_pattern })` to get bounded SpanMatchRecords (span metadata + matched text + surrounding context). Then call `traces.viewSpans({ trace_id, span_ids: [...] })` for surgical reads (~16KB cap, 4\u00D7 higher than discovery), or `traces.searchSpan({ trace_id, span_id, regex_pattern })` for one large span. Stays bounded regardless of trace size.\n   - Useful regex patterns: `STATUS_CODE_ERROR` (failures), tool names like `grep` or `view_trace`, error strings like `MaxTurnsExceeded`, model names, attribute keys.\n\n5. ONLY call viewTrace / viewSpans / searchTrace / searchSpan with trace/span ids you have already seen in sample_trace_ids, a queryTraces page, or a previous search result. Never invent ids.\n\n5a. **Result-shape contract** \u2014 searchTrace and searchSpan return `{ trace_id, hits, total_matches, has_more }`. Iterate `result.hits` (NOT result.matches). Each hit has `{ span_id, span_name, span_kind, attribute_path, matched_text, context_before, context_after, match_offset }`. viewTrace returns `{ trace_id, spans }` (or `oversized`). viewSpans returns `{ trace_id, spans, missing_span_ids, truncated_attribute_count }`. Never assume a field name \u2014 log the result shape first if unsure.\n\n6. If viewTrace returns an `oversized` summary instead of `spans`, DO NOT retry the same call. Read the summary's top_span_names, span_count, span_response_bytes_max, error_span_count to plan a follow-up: switch to searchTrace (or searchSpan for one large span), then viewSpans on a smaller, surgical span_ids set.\n\n7. If searchTrace or searchSpan returns has_more=true, REFINE the regex to be more specific rather than blindly raising max_matches.\n\n8. If a tool errors (invalid regex, range error), STOP and reconsider \u2014 don't retry with a guessed id or argument. Use the discovery tools above to recover.\n\n9. If a ~4KB-truncated payload from viewTrace / searchTrace matters for your answer, first try viewSpans on that span id (~16KB cap). If a 16KB-truncated payload from viewSpans still matters, narrow further with searchSpan against a more specific regex rather than asking for the full payload again.\n\n10. If maxDepth > 0 and the question splits into independent semantic branches, delegate well-defined subtasks to subagents using `await llmQuery(...)`. Pass narrow context and a focused query. Examples:\n\n    const reviews = await llmQuery([\n      { query: 'Drill into trace abc123 \u2014 what tool calls preceded the failure?', context: { trace_id: 'abc123' } },\n      { query: 'Drill into trace def456 \u2014 same failure mode?', context: { trace_id: 'def456' } },\n    ]);\n\nOBSERVABILITY rules:\n- Each non-final actor turn must emit at least one `console.log(...)` for evidence. Up to 3 logs per turn is fine when correlating multiple data sources (e.g. one log for findings list, one for source-file content, one for derived analysis).\n- Do NOT combine `console.log` with `final(...)` or `askClarification(...)` in the same turn \u2014 finish gathering data first, then call final on its own turn.\n- Reuse runtime variables across turns; don't recompute.\n- When done, call `await final(answer)` with the fully-formed report. The responder rewrites the answer into output fields; if you only pass a vague summary string the responder has nothing concrete to format.\n\nCRITICAL \u2014 `final()` payload contract for evidence-grounded analysis tasks:\n- Pass a STRUCTURED object as the second arg with the actual data the responder needs to format the answer. Do NOT pass abstract instructions; pass evidence.\n- Example for per-item verdict tasks:\n  ```js\n  await final(\"Format the per-item verdict report from the evidence below.\", {\n    findings: [\n      { id: 'sub-1-finding-1', claim: '...', verdict: 'TRUE-POSITIVE', evidence: 'lines 42-45 of contracts/X.sol show ...' },\n      ...all items\n    ],\n    systemic_summary: '3 sentences I wrote based on the evidence above'\n  });\n  ```\n- Calling `final(\"answer\", {})` with no evidence is a failure mode \u2014 the responder will hallucinate or echo back the field names. Always include the gathered data.\n- Premature final after a single viewSpans call is INSUFFICIENT for per-finding analysis tasks. Read the requested attributes (e.g. `spans[i].attributes['redteam.finding.title']`), and for each one perform the requested cross-reference (e.g. read the source SPAN's `attributes['source.content']`).\n\nOUTPUT contract \u2014 your final answer must include:\n- A clear prose conclusion answering the user's question.\n- Trace ids and span ids cited as evidence for each claim.\n- Failure modes named in the user's domain language, with frequency and concrete examples.\n\nDo NOT invent trace ids, span ids, error messages, or model names. Every fact must be traceable to a tool result.";
 declare const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-actor-v5-2026-05-06";
@@ -671,4 +754,4 @@ declare function iterateRawCalls(sink: RawProviderSink, filter?: {
     spanId?: string;
 }): AsyncGenerator<ReplayCacheEntry>;
 
-export { type AnalyzeTracesInput, type AnalyzeTracesOptions, type AnalyzeTracesResult, type AnalyzeTracesTurnSnapshot, DEFAULT_REDACTION_RULES, DatasetOverview, type ExportableSpan, OTEL_AGENT_EVAL_SCOPE, type OtelExportConfig, type OtelExporter, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpResourceSpans, type OtlpSpan, QueryTracesPage, REDACTION_VERSION, RawProviderEvent, RawProviderSink, type RedactionReport, type RedactionRule, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, type ReplayFetchOptions, RunCompleteHook, RunCompleteHookContext, SearchSpanResult, SearchTraceResult, SpanNotFoundError, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, TraceAnalysisStore, TraceAnalystFilters, type TraceAnalystHookOptions, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, TraceStore, ViewSpansResult, ViewTraceResult, analyzeTraces, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, inferDomainKeywords, iterateRawCalls, otelRunCompleteHook, planTraceInsightQuestions, redactString, redactValue, scoreTraceInsightReadiness, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete };
+export { type AnalyzeTracesInput, type AnalyzeTracesOptions, type AnalyzeTracesResult, type AnalyzeTracesTurnSnapshot, type CaptureFetchContext, type CaptureFetchOptions, DEFAULT_REDACTION_RULES, DatasetOverview, type ExportableSpan, type FlattenOtlpOptions, OTEL_AGENT_EVAL_SCOPE, type OtelExportConfig, type OtelExporter, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpFlatLine, type OtlpResourceSpans, type OtlpSpan, ProviderRedactor, QueryTracesPage, REDACTION_VERSION, RawProviderEvent, RawProviderSink, type RedactionReport, type RedactionRule, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, type ReplayFetchOptions, RunCompleteHook, RunCompleteHookContext, SearchSpanResult, SearchTraceResult, SpanNotFoundError, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, TraceAnalysisStore, TraceAnalystFilters, type TraceAnalystHookOptions, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, TraceStore, ViewSpansResult, ViewTraceResult, analyzeTraces, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, captureFetchToRawSink, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, flattenOtlpExportToNdjson, inferDomainKeywords, iterateRawCalls, otelRunCompleteHook, planTraceInsightQuestions, redactString, redactValue, scoreTraceInsightReadiness, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete };

package/dist/traces.js

@@ -17,6 +17,7 @@ import {
   buildTraceAnalystTools,
   buildTraceInsightContext,
   buildTraceInsightPrompt,
+  captureFetchToRawSink,
   createOtelExporter,
   createOtelTracingStore,
   createReplayFetch,
@@ -24,6 +25,7 @@ import {
   describeTraceInsightScope,
   domainEvidencePattern,
   exportRunAsOtlp,
+  flattenOtlpExportToNdjson,
   inferDomainKeywords,
   iterateRawCalls,
   otelRunCompleteHook,
@@ -32,7 +34,7 @@ import {
   tokenizeDomainWords,
   traceAnalystFunctionGroup,
   traceAnalystOnRunComplete
-} from "./chunk-MAOZCN36.js";
+} from "./chunk-5GLYP2IQ.js";
 import {
   DEFAULT_REDACTION_RULES,
   REDACTION_VERSION,
@@ -108,6 +110,7 @@ export {
   buildTraceAnalystTools,
   buildTraceInsightContext,
   buildTraceInsightPrompt,
+  captureFetchToRawSink,
   createOtelExporter,
   createOtelTracingStore,
   createReplayFetch,
@@ -116,6 +119,7 @@ export {
   describeTraceInsightScope,
   domainEvidencePattern,
   exportRunAsOtlp,
+  flattenOtlpExportToNdjson,
   groupBy,
   inferDomainKeywords,
   isJudgeSpan,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.56.0",
+  "version": "0.58.0",
   "description": "Substrate for self-improving agents: traces, verifiable rewards, preferences, GEPA / reflective mutation, auto-research, replay, sequential anytime-valid stats, and release gates.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {