@tangle-network/agent-eval 0.36.0 → 0.38.0

package/dist/traces.d.ts

@@ -1,7 +1,7 @@
 import { N as NotFoundError, R as ReplayError } from './errors-mje_cKOs.js';
 import { R as RawProviderSink, f as RawProviderEvent } from './integrity-DYR5gWlb.js';
 export { F as FileSystemRawProviderSink, c as FileSystemRawProviderSinkOptions, I as InMemoryRawProviderSink, d as InMemoryRawProviderSinkOptions, N as NoopRawProviderSink, P as ProviderRedactor, e as RawProviderDirection, g as RawProviderSinkFilter, h as RunIntegrityError, a as RunIntegrityExpectations, i as RunIntegrityIssue, j as RunIntegrityIssueCode, b as RunIntegrityReport, k as assertRunCaptured, l as defaultProviderRedactor, p as providerFromBaseUrl, t as throwIfRunIncomplete } from './integrity-DYR5gWlb.js';
-import { a as RunCompleteHookContext, R as RunCompleteHook } from './emitter-DP_cSSiw.js';
+import { R as RunCompleteHook, a as RunCompleteHookContext } from './emitter-DP_cSSiw.js';
 export { S as SpanHandle, T as TraceEmitter, b as TraceEmitterOptions, l as llmSpanFromProvider } from './emitter-DP_cSSiw.js';
 import { T as TraceStore } from './store-Db2Bv8Cf.js';
 export { A as Artifact, B as BudgetLedgerEntry, g as BudgetSpec, i as EventFilter, E as EventKind, j as FAILURE_CLASSES, F as FailureClass, k as FileSystemTraceStore, l as FileSystemTraceStoreOptions, G as GenericSpan, I as InMemoryTraceStore, J as JudgeSpan, L as LlmSpan, M as Message, e as RetrievalSpan, R as Run, h as RunFilter, m as RunLayer, c as RunOutcome, n as RunStatus, f as SandboxSpan, S as Span, o as SpanBase, p as SpanFilter, d as SpanKind, q as SpanStatus, r as TRACE_SCHEMA_VERSION, a as ToolSpan, b as TraceEvent, s as isJudgeSpan, t as isLlmSpan, u as isRetrievalSpan, v as isSandboxSpan, w as isToolSpan } from './store-Db2Bv8Cf.js';
@@ -65,6 +65,85 @@ interface OtlpExport {
 /** Export a single run's spans + events in OTLP/JSON. */
 declare function exportRunAsOtlp(store: TraceStore, runId: string, resourceAttrs?: Record<string, string | number | boolean>): Promise<OtlpExport>;
 
+/**
+ * OTEL span exporter — streams spans to an OTLP/HTTP collector.
+ *
+ * Reads OTEL_EXPORTER_OTLP_ENDPOINT + OTEL_EXPORTER_OTLP_HEADERS from env
+ * when no explicit config is given. Batches spans and flushes periodically
+ * or when the batch fills. No @opentelemetry SDK dependency — minimal
+ * OTLP/JSON serializer (~120 LOC) using the existing otel.ts helpers.
+ */
+interface OtelExportConfig {
+    /** OTLP endpoint. Reads OTEL_EXPORTER_OTLP_ENDPOINT env by default. */
+    endpoint?: string;
+    /** OTLP headers. Reads OTEL_EXPORTER_OTLP_HEADERS env by default. */
+    headers?: Record<string, string>;
+    /** Batch size before flush. Default 64. */
+    batchSize?: number;
+    /** Flush interval ms. Default 5000. */
+    flushIntervalMs?: number;
+    /** Resource attributes stamped on every export. */
+    resourceAttributes?: Record<string, string | number | boolean>;
+    /** Service name. Default 'agent-eval'. */
+    serviceName?: string;
+}
+interface OtelExporter {
+    /** Called by the TraceEmitter on every span close. */
+    exportSpan(span: ExportableSpan): void;
+    /** Force flush pending spans. */
+    flush(): Promise<void>;
+    /** Shutdown cleanly — flushes remaining spans and stops the timer. */
+    shutdown(): Promise<void>;
+}
+interface ExportableSpan {
+    traceId: string;
+    spanId: string;
+    parentSpanId?: string;
+    name: string;
+    kind: string;
+    startedAt: number;
+    endedAt?: number;
+    status?: string;
+    error?: string;
+    model?: string;
+    inputTokens?: number;
+    outputTokens?: number;
+    costUsd?: number;
+    attributes?: Record<string, unknown>;
+}
+/**
+ * Create an OTEL exporter. Returns undefined when no endpoint is configured
+ * (neither via config nor env) — callers should check before attaching.
+ */
+declare function createOtelExporter(config?: OtelExportConfig): OtelExporter | undefined;
+
+/**
+ * OTEL bridge — connects TraceEmitter span lifecycle to the OtelExporter.
+ *
+ * When an OtelExporter is active, every span that closes through the
+ * TraceEmitter is also pushed to the exporter for real-time streaming to
+ * the user's OTEL collector.
+ *
+ * The bridge is opt-in: attach via `otelRunCompleteHook(exporter)` as a
+ * RunCompleteHook, or wrap the store with `createOtelTracingStore` for
+ * real-time per-span export.
+ */
+
+/**
+ * Create a RunCompleteHook that exports all spans from the completed run
+ * to the OTEL exporter, then flushes.
+ */
+declare function otelRunCompleteHook(exporter: OtelExporter): RunCompleteHook;
+/**
+ * Create an auto-exporting TraceStore wrapper that intercepts updateSpan
+ * calls. When a span gets an endedAt, it's exported immediately. This
+ * gives real-time streaming instead of batch-at-end.
+ *
+ * This is the preferred integration path: wrap the store before
+ * constructing the TraceEmitter.
+ */
+declare function createOtelTracingStore(inner: TraceStore, exporter: OtelExporter, traceId: string): TraceStore;
+
 /**
  * Redaction — remove PII / secrets from trace payloads before persist.
  *
@@ -807,4 +886,4 @@ declare function iterateRawCalls(sink: RawProviderSink, filter?: {
     spanId?: string;
 }): AsyncGenerator<ReplayCacheEntry>;
 
-export { type AnalyzeTracesInput, type AnalyzeTracesOptions, type AnalyzeTracesResult, type AnalyzeTracesTurnSnapshot, DEFAULT_REDACTION_RULES, DEFAULT_TRACE_ANALYST_BUDGETS, type DatasetOverview, OTEL_AGENT_EVAL_SCOPE, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpResourceSpans, type OtlpSpan, type QueryTracesPage, REDACTION_VERSION, RawProviderEvent, RawProviderSink, type RedactionReport, type RedactionRule, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, type ReplayFetchOptions, RunCompleteHook, RunCompleteHookContext, type SearchSpanResult, type SearchTraceResult, type SpanMatchRecord, SpanNotFoundError, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, type TraceAnalysisStore, type TraceAnalystByteBudgets, type TraceAnalystFilters, type TraceAnalystHookOptions, type TraceAnalystSpan, type TraceAnalystSpanKind, type TraceAnalystSpanStatus, type TraceAnalystTraceSummary, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, TraceStore, type ViewSpansResult, type ViewTraceOversized, type ViewTraceResult, analyzeTraces, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, inferDomainKeywords, iterateRawCalls, planTraceInsightQuestions, redactString, redactValue, scoreTraceInsightReadiness, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete };
+export { type AnalyzeTracesInput, type AnalyzeTracesOptions, type AnalyzeTracesResult, type AnalyzeTracesTurnSnapshot, DEFAULT_REDACTION_RULES, DEFAULT_TRACE_ANALYST_BUDGETS, type DatasetOverview, type ExportableSpan, OTEL_AGENT_EVAL_SCOPE, type OtelExportConfig, type OtelExporter, type OtlpExport, OtlpFileTraceStore, type OtlpFileTraceStoreOptions, type OtlpResourceSpans, type OtlpSpan, type QueryTracesPage, REDACTION_VERSION, RawProviderEvent, RawProviderSink, type RedactionReport, type RedactionRule, ReplayCache, type ReplayCacheEntry, ReplayCacheMissError, type ReplayCacheStats, type ReplayFetchOptions, RunCompleteHook, RunCompleteHookContext, type SearchSpanResult, type SearchTraceResult, type SpanMatchRecord, SpanNotFoundError, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, type TraceAnalysisStore, type TraceAnalystByteBudgets, type TraceAnalystFilters, type TraceAnalystHookOptions, type TraceAnalystSpan, type TraceAnalystSpanKind, type TraceAnalystSpanStatus, type TraceAnalystTraceSummary, TraceFileMissingError, type TraceInsightContext, type TraceInsightFinding, type TraceInsightPanelRole, type TraceInsightPromptInput, type TraceInsightQualityGate, type TraceInsightQuestion, type TraceInsightReadiness, type TraceInsightSuite, type TraceInsightTask, TraceNotFoundError, TraceStore, type ViewSpansResult, type ViewTraceOversized, type ViewTraceResult, analyzeTraces, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, inferDomainKeywords, iterateRawCalls, otelRunCompleteHook, planTraceInsightQuestions, redactString, redactValue, scoreTraceInsightReadiness, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete };

package/dist/traces.js

@@ -19,6 +19,8 @@ import {
   buildTraceAnalystTools,
   buildTraceInsightContext,
   buildTraceInsightPrompt,
+  createOtelExporter,
+  createOtelTracingStore,
   createReplayFetch,
   defaultTraceInsightPanel,
   describeTraceInsightScope,
@@ -26,6 +28,7 @@ import {
   exportRunAsOtlp,
   inferDomainKeywords,
   iterateRawCalls,
+  otelRunCompleteHook,
   planTraceInsightQuestions,
   redactString,
   redactValue,
@@ -33,7 +36,7 @@ import {
   tokenizeDomainWords,
   traceAnalystFunctionGroup,
   traceAnalystOnRunComplete
-} from "./chunk-HIO4UIS5.js";
+} from "./chunk-L5UNCDAJ.js";
 import {
   aggregateLlm,
   argHash,
@@ -103,6 +106,8 @@ export {
   buildTraceAnalystTools,
   buildTraceInsightContext,
   buildTraceInsightPrompt,
+  createOtelExporter,
+  createOtelTracingStore,
   createReplayFetch,
   defaultProviderRedactor,
   defaultTraceInsightPanel,
@@ -120,6 +125,7 @@ export {
   judgeSpans,
   llmSpanFromProvider,
   llmSpans,
+  otelRunCompleteHook,
   planTraceInsightQuestions,
   providerFromBaseUrl,
   redactString,

package/dist/types-DHqkLwEU.d.ts

@@ -0,0 +1,110 @@
+import { DefaultVerdict } from '@tangle-network/agent-runtime/loops';
+
+/**
+ * @experimental
+ *
+ * N-axis cartesian matrix over substrate types — types module.
+ *
+ * The matrix is a runner + aggregator. It iterates the cartesian product of
+ * caller-provided axes (any value type — `AgentProfile` from sandbox, `Driver`
+ * / `Validator` from agent-runtime, rubric records, thinking levels, anything)
+ * and aggregates per-axis pass/score/cost summaries. Substrate types are
+ * imported at the boundary by JSDoc only; the matrix never wraps them.
+ */
+
+/** One axis = one dimension to iterate. `V` is the value type — pass any
+ *  substrate type (AgentProfile, Driver, Validator, rubric record). */
+interface MatrixAxis<V> {
+    /** Axis name. Becomes the key in `MatrixResult.byAxis`. */
+    name: string;
+    /** Stable id per value. Used as the bucket key in aggregation. */
+    values: Array<{
+        id: string;
+        value: V;
+    }>;
+    /** Optional bucket label override. Receives the same `(value, id)` the
+     *  runner stored on the cell; default label is `id`. */
+    label?: (value: V, id: string) => string;
+}
+/** A cell carries one picked value from each axis, keyed by axis name. */
+interface MatrixCell {
+    axes: Record<string, {
+        id: string;
+        value: unknown;
+    }>;
+    /** 0-based replicate index within the same axis combination. */
+    rep: number;
+    /** Stable sort key — preserves cartesian order across concurrent execution. */
+    ordinal: number;
+}
+interface CellResult<Output> {
+    output: Output;
+    verdict: DefaultVerdict;
+    costUsd: number;
+    durationMs: number;
+    runId?: string;
+    /** Populated when `runCell` threw. The cell contributes 0 to passRate AND
+     *  meanScore regardless of `verdict`. */
+    error?: {
+        message: string;
+        kind: string;
+    };
+}
+interface AxisSummary {
+    axisName: string;
+    axisValue: string;
+    cells: number;
+    passRate: number;
+    meanScore: number;
+    p50Score: number;
+    p90Score: number;
+    totalCostUsd: number;
+    meanDurationMs: number;
+}
+interface MatrixResult<Output> {
+    cells: Array<{
+        cell: MatrixCell;
+        runs: CellResult<Output>[];
+    }>;
+    /** `byAxis[axisName][axisValueId] = summary`. Populated only for axes
+     *  named in `aggregateBy` (default = every axis in `axes`). */
+    byAxis: Record<string, Record<string, AxisSummary>>;
+    summary: {
+        totalCells: number;
+        runsExecuted: number;
+        /** Cells removed by `filter` plus cells unscheduled after the cost
+         *  ceiling or abort signal tripped. */
+        cellsSkipped: number;
+        overallPassRate: number;
+        overallMeanScore: number;
+        totalCostUsd: number;
+        durationMs: number;
+    };
+    /** Stable id-like string generated at the end of the run. */
+    matrixId: string;
+}
+interface RunAgentMatrixOptions<Output> {
+    axes: MatrixAxis<unknown>[];
+    /** User-supplied cell executor. May throw; the matrix captures throws as
+     *  `CellResult.error` and continues. */
+    runCell: (cell: MatrixCell) => Promise<CellResult<Output>>;
+    /** Replicates per cell. Default 1. */
+    reps?: number;
+    /** Prune cells from the cartesian BEFORE rep expansion. */
+    filter?: (cell: Omit<MatrixCell, 'rep' | 'ordinal'>) => boolean;
+    /** Axes to aggregate into `byAxis`. Default: every axis in `axes`. */
+    aggregateBy?: string[];
+    /** Max concurrent in-flight `runCell` invocations. Default 4. */
+    maxConcurrency?: number;
+    /** Cumulative-cost abort threshold (USD). When the running sum of
+     *  `result.costUsd` crosses this value, no new cells are scheduled.
+     *  In-flight cells finish. Default `Infinity`. */
+    costCeiling?: number;
+    /** Fires once per executed cell, after its promise settles. */
+    onCellComplete?: (cell: MatrixCell, result: CellResult<Output>) => void;
+    /** External cancellation. Aborts in-flight cells via a forwarded signal
+     *  and suppresses scheduling of new ones. */
+    signal?: AbortSignal;
+}
+
+export type { AxisSummary as A, CellResult as C, MatrixResult as M, RunAgentMatrixOptions as R, MatrixAxis as a, MatrixCell as b };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.36.0",
+  "version": "0.38.0",
   "description": "Substrate for self-improving agents: traces, verifiable rewards, preferences, GEPA / reflective mutation, auto-research, replay, sequential anytime-valid stats, and release gates.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -99,6 +99,11 @@
       "import": "./dist/matrix/index.js",
       "default": "./dist/matrix/index.js"
     },
+    "./multishot": {
+      "types": "./dist/multishot/index.d.ts",
+      "import": "./dist/multishot/index.js",
+      "default": "./dist/multishot/index.js"
+    },
     "./openapi.json": {
       "default": "./dist/openapi.json"
     }