@tangle-network/agent-eval 0.20.11 → 0.21.0

package/dist/traces.js

@@ -0,0 +1,120 @@
+import {
+  DEFAULT_REDACTION_RULES,
+  DEFAULT_TRACE_ANALYST_BUDGETS,
+  FAILURE_CLASSES,
+  FileSystemTraceStore,
+  InMemoryTraceStore,
+  OTEL_AGENT_EVAL_SCOPE,
+  OtlpFileTraceStore,
+  REDACTION_VERSION,
+  RunIntegrityError,
+  SpanNotFoundError,
+  TRACE_ANALYST_ACTOR_DESCRIPTION,
+  TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
+  TRACE_ANALYST_SUBAGENT_DESCRIPTION,
+  TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
+  TRACE_SCHEMA_VERSION,
+  TraceFileMissingError,
+  TraceNotFoundError,
+  aggregateLlm,
+  analyzeTraces,
+  argHash,
+  assertRunCaptured,
+  buildTraceAnalystTools,
+  buildTraceInsightContext,
+  buildTraceInsightPrompt,
+  defaultTraceInsightPanel,
+  describeTraceInsightScope,
+  domainEvidencePattern,
+  exportRunAsOtlp,
+  groupBy,
+  inferDomainKeywords,
+  isJudgeSpan,
+  isLlmSpan,
+  isRetrievalSpan,
+  isSandboxSpan,
+  isToolSpan,
+  judgeSpans,
+  llmSpans,
+  planTraceInsightQuestions,
+  redactString,
+  redactValue,
+  runFailureClass,
+  runsForScenario,
+  scoreTraceInsightReadiness,
+  throwIfRunIncomplete,
+  tokenizeDomainWords,
+  toolSpans,
+  traceAnalystFunctionGroup,
+  traceAnalystOnRunComplete
+} from "./chunk-WOK2RTWG.js";
+import {
+  TraceEmitter,
+  llmSpanFromProvider
+} from "./chunk-5IIQKMD5.js";
+import {
+  FileSystemRawProviderSink,
+  InMemoryRawProviderSink,
+  NoopRawProviderSink,
+  defaultProviderRedactor,
+  providerFromBaseUrl
+} from "./chunk-SNUHRBDL.js";
+import "./chunk-PZ5AY32C.js";
+export {
+  DEFAULT_REDACTION_RULES,
+  DEFAULT_TRACE_ANALYST_BUDGETS,
+  FAILURE_CLASSES,
+  FileSystemRawProviderSink,
+  FileSystemTraceStore,
+  InMemoryRawProviderSink,
+  InMemoryTraceStore,
+  NoopRawProviderSink,
+  OTEL_AGENT_EVAL_SCOPE,
+  OtlpFileTraceStore,
+  REDACTION_VERSION,
+  RunIntegrityError,
+  SpanNotFoundError,
+  TRACE_ANALYST_ACTOR_DESCRIPTION,
+  TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
+  TRACE_ANALYST_SUBAGENT_DESCRIPTION,
+  TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
+  TRACE_SCHEMA_VERSION,
+  TraceEmitter,
+  TraceFileMissingError,
+  TraceNotFoundError,
+  aggregateLlm,
+  analyzeTraces,
+  argHash,
+  assertRunCaptured,
+  buildTraceAnalystTools,
+  buildTraceInsightContext,
+  buildTraceInsightPrompt,
+  defaultProviderRedactor,
+  defaultTraceInsightPanel,
+  describeTraceInsightScope,
+  domainEvidencePattern,
+  exportRunAsOtlp,
+  groupBy,
+  inferDomainKeywords,
+  isJudgeSpan,
+  isLlmSpan,
+  isRetrievalSpan,
+  isSandboxSpan,
+  isToolSpan,
+  judgeSpans,
+  llmSpanFromProvider,
+  llmSpans,
+  planTraceInsightQuestions,
+  providerFromBaseUrl,
+  redactString,
+  redactValue,
+  runFailureClass,
+  runsForScenario,
+  scoreTraceInsightReadiness,
+  throwIfRunIncomplete,
+  tokenizeDomainWords,
+  toolSpans,
+  traceAnalystFunctionGroup,
+  traceAnalystOnRunComplete
+};
+//# sourceMappingURL=traces.js.map

package/dist/traces.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/wire/index.js

@@ -24,8 +24,9 @@ import {
   runRpcBatch,
   runRpcOnce,
   startServer
-} from "../chunk-LSR4IAYN.js";
-import "../chunk-JAOLXRIA.js";
+} from "../chunk-WOPGKVN4.js";
+import "../chunk-3GN6U53I.js";
+import "../chunk-SNUHRBDL.js";
 import "../chunk-PZ5AY32C.js";
 export {
   BUILTIN_RUBRICS,

package/docs/concepts.md

@@ -1,14 +1,15 @@
 # Concepts
 
-Read this once and the rest of agent-eval makes sense.
+`agent-eval` is for deciding whether an agent run should pass, keep working, be
+replayed, be optimized, or be promoted.
 
-## What is agent-eval?
+It exists because agent output is not evidence. A model can say a task is done
+while the build fails, the browser flow is broken, the integration was never
+connected, or the answer lacks required sources. The package gives products a
+shared way to record runs, check outcomes, classify failures, compare variants,
+and make release decisions.
 
-A library for **deciding whether a code generator or content generator did its job.** You give it a thing the generator produced (a scaffold, a patch, a tweet, a JSON config), and you get back a structured verdict: pass/fail, dimension scores, a reason in plain English.
-
-It exists because LLMs lie about whether they succeeded. A model will say "Done!" and ship code that doesn't compile. agent-eval is the layer between the model's output and your decision to ship.
-
-## The three things you'll touch most
+## Main Objects
 
 | Thing | What it is | One-line example |
 |---|---|---|
@@ -17,7 +18,8 @@ It exists because LLMs lie about whether they succeeded. A model will say "Done!
 | **Verifier** | A pipeline of judges run in order, with dependencies. | "install → typecheck → build → semantic" |
 | **Feedback trajectory** | A multi-shot record of attempts, approvals, rejections, edits, metrics, and policy outcomes. | "draft → user rejects → revised draft → approved → measured" |
 
-That's the whole framework. Everything else (sessions, traces, layers) is plumbing around those three.
+Everything else exists to make those objects useful in real product loops:
+traces, datasets, control runtime, optimizers, statistics, and reports.
 
 When the thing being evaluated is an agent that should keep working, use
 [`runAgentControlLoop`](./control-runtime.md). It turns validators into a
@@ -62,7 +64,7 @@ shape stays the same.
 Those trajectories can be converted into preference memory, `DatasetScenario`
 rows, optimizer rows, and held-out examples for overfit checks.
 
-## The three-layer eval (for code generators)
+## Code Generator Eval
 
 When the artifact is generated code, agent-eval scores it at three independent layers. Each layer fails differently, and you want to know which one broke:
 
@@ -125,7 +127,7 @@ Two rules that will save you bugs:
 
 2. **Pair LLM judges with build outcomes.** An LLM judge will rate non-compiling code as "looks right" (0.8). Always short-circuit on `buildOutcome.passed === false` before any LLM judging.
 
-## The trace model (skip on first read)
+## Trace Model
 
 Every operation emits structured spans into a `TraceStore`. A run is a tree:
 
@@ -142,7 +144,10 @@ builder-session                 [span]
 
 Spans are append-only and have stable ids — replay is reading the same store back. OTLP export ships them out for distributed tracing.
 
-You don't need to build the trace tree by hand. `BuilderSession` does it for you. Look at the trace store when you're debugging a flaky run; ignore it otherwise.
+You usually should not build this tree by hand. Product runtimes,
+`runAgentControlLoop`, harnesses, and verifiers should emit it while they run.
+Use traces when debugging a flaky run, building replay data, or explaining a
+release decision.
 
 ## Where to go next
 

package/docs/feature-guide.md

@@ -3,7 +3,7 @@
 This page explains the main `agent-eval` primitives in plain English first,
 then shows when to use each one.
 
-## ELI5
+## Overview
 
 LLM agents can write code, drafts, research, plans, and actions. The hard part
 is knowing whether they actually did a good job, whether they should keep
@@ -41,7 +41,7 @@ trying, and whether a change made them better or worse.
 
 ## Integration Patterns
 
-### Recommended Agent Product Shape
+### Recommended Product Shape
 
 Use this shape when the product needs to keep pushing work forward instead of
 only answering once:
@@ -175,21 +175,6 @@ Store as `FeedbackTrajectory`, then derive:
   logs, screenshots, or browser state. Use separate sandboxes for parallel
   variants or destructive checks.
 
-## Same-Sandbox Example
-
-`examples/same-sandbox-harness/` shows the common coding/browser pattern:
-
-```text
-one sandbox/workdir -> install/build/test -> inspect evidence -> emit judge span
-```
-
-Use this when a judge needs evidence produced by earlier harness phases. Use
-isolated sandboxes when variants run in parallel or a phase can corrupt the
-workspace.
-- Treat telemetry as evidence, not control flow. A trace sink outage should be
-  visible in `runtimeErrors`, but it should not stop the worker from completing
-  the user task.
-
 ## Highest-ROI Adoption Order
 
 1. Wrap one real product workflow in `runAgentControlLoop`.
@@ -211,3 +196,11 @@ reusable:
 
 Core should provide shapes, stores, runners, scoring, traces, and converters.
 Downstream integrations provide domain state, policy, tools, and storage.
+
+## Examples
+
+- `examples/same-sandbox-harness`: one workdir for install/build/test plus
+  evidence inspection.
+- `examples/multi-shot-optimization`: full-trajectory optimization with a
+  holdout gate.
+- `examples/benchmarks`: benchmark adapter contracts and reference wrappers.

package/docs/integration-launch-gates.md

@@ -0,0 +1,77 @@
+# Integration Launch Gates
+
+Use these gates when a product lets generated apps or agents use user-owned
+connections through an integration hub.
+
+The eval should wrap the real product path:
+
+```txt
+user prompt
+  -> product emits IntegrationManifest
+  -> platform resolves connections and grants
+  -> sandbox receives capability bundle
+  -> generated app invokes integration action
+  -> platform enforces policy, approval, idempotency, audit
+```
+
+## Deterministic Gates
+
+- The generated app declares an integration manifest before sandbox launch.
+- Manifest validation passes.
+- Required connections and scopes are present before execution.
+- Sandbox environment contains a capability bundle, not raw provider tokens.
+- Reads invoke through the platform bridge.
+- Writes return `approval_required` unless product policy explicitly allows
+  them.
+- Approved writes are bound to the same action, input hash, connection, and
+  subject.
+- Revoked grants or expired capabilities stop invocation.
+- Resumed or long-running sandboxes receive a refreshed bundle before expiry.
+- Audit includes grant creation, capability issue, invoke success/failure,
+  approval resolution, and revoke events.
+
+## Failure Classes
+
+`agent-eval` classifies integration failures separately from prompt/tool
+failures:
+
+- `bad_integration_manifest`
+- `missing_integration_connection`
+- `missing_integration_scope`
+- `integration_approval_required`
+- `integration_auth_expired`
+- `integration_provider_failure`
+- `unsafe_integration_write_denied`
+
+Use the helper payload builders and eval builders so products emit the same
+trace evidence:
+
+```ts
+const gate = {
+  connectorId: 'google-calendar',
+  actionId: 'events.create',
+  valid: true,
+  missingConnections: [],
+  missingScopes: ['calendar.events.write'],
+}
+
+const evals = integrationGateEvals(gate)
+
+await emitter.emit({
+  kind: 'custom',
+  payload: integrationManifestResolvedPayload(gate),
+})
+
+await emitter.emit({
+  kind: 'custom',
+  payload: integrationInvokeFailedPayload({
+    connectorId: 'google-calendar',
+    actionId: 'events.create',
+    code: 'scope_denied',
+    message: 'calendar.events.write was not granted',
+  }),
+})
+```
+
+The classifier then reports the real missing surface instead of burying the
+failure under `tool_recovery_failure` or `unknown`.

package/docs/product-eval-adoption.md

@@ -98,6 +98,23 @@ replayable eval data.
 production run -> feedback trajectory -> dataset scenario -> optimizer row
 ```
 
+For promotion-grade runs, also project the completed control result into a
+strict `RunRecord`:
+
+```ts
+const record = controlRunToRunRecord(controlResult, {
+  experimentId,
+  candidateId,
+  seed,
+  model: 'gpt-4o-2024-11-20',
+  promptHash,
+  configHash,
+  commitSha,
+  splitTag: 'holdout',
+  tokenUsage,
+})
+```
+
 ## Datasets And Holdouts
 
 Use four splits:
@@ -107,6 +124,10 @@ Use four splits:
 - `test`: normal reporting.
 - `holdout`: promotion-only gate.
 
+The low-level `RunRecord` schema uses `search | dev | holdout`; map `train`
+and normal non-holdout test/report rows to `search` when producing promotion
+tables.
+
 Do not inspect or tune against holdout failures during optimization. If a
 holdout failure reveals a real product bug, fix the bug and rotate the holdout
 set with a signed note.
@@ -149,6 +170,7 @@ A launch or promotion should require:
 - cost and latency within budget
 - no unresolved canary or contamination failures
 - trace evidence for representative successes and failures
+- TraceAnalyst findings for failure-heavy or regression-heavy corpora
 - human-readable report with failure clusters and next actions
 
 `evaluateReleaseConfidence()` and the paired statistics helpers provide the
@@ -177,6 +199,11 @@ Use `@tangle-network/agent-integrations` manifests as readiness inputs. Gate
 missing connections, missing scopes, approval-required writes, and stale tokens
 before blaming the agent prompt.
 
+For generated apps and sandbox agents, also run the
+[Integration Launch Gates](./integration-launch-gates.md). The eval should prove
+that app code invokes through the integration bridge, not provider SDKs with raw
+OAuth tokens.
+
 ### Voice Agent
 
 Record transcript, timing, interruptions, tool calls, and task outcome. Judge

package/docs/research-report-methodology.md

@@ -0,0 +1,155 @@
+# researchReport — methodology
+
+This document is the methodological brief for `researchReport` (exported from
+`@tangle-network/agent-eval` and `@tangle-network/agent-eval/reporting`). It
+exists so a launch reviewer, peer reviewer, or auditor can quickly verify that
+the verdict embedded in any rendered report is defensible, reproducible, and
+appropriate to the data.
+
+The companion code is `src/summary-report.ts`. Each item below names the
+corresponding function or option so the doc and the code don't drift.
+
+## Inputs
+
+- `runs: RunRecord[]` — every record carries `runId`, `candidateId`, `seed`,
+  `experimentId`, `splitTag`, and an `outcome` with the configured score.
+- `comparator: string` — the candidate id treated as the null reference. Must
+  be selected before data inspection; `preregistrationHash` should pin this.
+- `split: 'search' | 'holdout'` — defaults to `holdout`. Decisions on `search`
+  are descriptive only; promotion calls require the holdout.
+- `rope: { low, high }` — Region of Practical Equivalence on the paired delta,
+  in score units. Must come from the domain owner — there is no
+  statistically-defensible default.
+- `minPairs` (soft floor, default 20) and `RESEARCH_REPORT_HARD_PAIR_FLOOR`
+  (hard floor, 6). Below the soft floor, the verdict is `needs_more_data` and
+  the report carries the MDE at the current N.
+- `fdr` (default 0.05), `confidence` (default 0.95), `mdePower` (default 0.8),
+  `mdeAlpha` (default = `fdr`).
+
+## Pairing
+
+Pairs are joined by `(experimentId, seed)` so the comparator and candidate
+share scenario *and* seed. This is the same join `gainHistogram` uses; see
+`pairScoresByKey` in `src/summary-report.ts`. Records on the wrong split or
+with non-finite scores are dropped before pairing.
+
+## Decision rule
+
+In order — first match wins:
+
+1. `comparator` itself → `hold` (baseline).
+2. No comparator → `hold` if on the cost/quality Pareto frontier, else
+   `needs_more_data`. The verdict is descriptive, not causal.
+3. Held-out gate verdict ≠ `promote` → `reject`. The gate is *necessary but
+   not sufficient*; even a `promote` gate must clear the paired test below.
+4. Paired N < `RESEARCH_REPORT_HARD_PAIR_FLOOR` → `needs_more_data` with a
+   "below hard floor" reason. Bootstrap CIs degenerate at this size.
+5. ROPE configured AND paired-delta CI ⊂ ROPE → `equivalent`.
+6. Paired-delta CI upper bound < 0 → `reject` (CI excludes a non-negative
+   effect). Note: this uses **paired delta only** — not the marginal mean.
+7. Paired N < `minPairs` (soft floor) → `needs_more_data` with the MDE at
+   current N attached so the verdict is actionable.
+8. BH-adjusted q ≤ `fdr` AND CI lower bound > 0 → `promote`. The BH q-value
+   controls FDR across all candidates in the same sweep; the bootstrap CI
+   provides an effect-size guarantee independent of the test.
+9. Otherwise → `hold`.
+
+## Statistical primitives used
+
+| Quantity | Function | Source file |
+|---|---|---|
+| Marginal CI on score mean | `confidenceInterval` | `statistics.ts` |
+| Cohen's d vs comparator | `cohensD` | `statistics.ts` |
+| Wilcoxon signed-rank (paired) | `wilcoxonSignedRank` | `statistics.ts` |
+| BH-FDR q-values | `benjaminiHochberg` | `power-analysis.ts` |
+| Paired bootstrap CI on median delta | `pairedBootstrap` | `paired-stats.ts` |
+| Bayesian-bootstrap-style Pr(Δ>0), Pr(Δ∈ROPE) | `bootstrapMeanSamples` | `summary-report.ts` (private) |
+| Minimum detectable paired effect | `pairedMde` | `power-analysis.ts` |
+| Run fingerprint | `hashJson(canonicalize(...))` | `pre-registration.ts` |
+
+The Pr(Δ>0) and Pr(Δ∈ROPE) summaries use the bootstrap-prior duality of
+[Rubin 1981]: under a non-informative Dirichlet prior, the bootstrap
+distribution of a sample statistic is its posterior. We expose these as
+posterior summaries on the **mean** delta and the bootstrap CI on the
+**median** delta — the median is more robust to the heavy-tailed score
+distributions seen in agent benchmarks; the mean lets us read off the
+Bayesian-style probability of superiority in a single number.
+
+## MDE
+
+The minimum detectable paired effect at N pairs, two-sided α, and power β:
+
+$$d_\text{min} = \frac{z_{1-\alpha/2} + z_\beta}{\sqrt{n}}$$
+
+reported on the standardised scale, then multiplied by the observed paired-
+delta SD to get the MDE in score units. Consumers reading a `needs_more_data`
+verdict can use the MDE to budget the next round of runs:
+
+- Observed paired SD = 0.10 score units, paired N = 20, α = 0.05, β = 0.8 →
+  d_min ≈ 0.63 standardised → MDE ≈ 0.063 score units. If the smallest
+  effect that would change a launch decision is below this, run more pairs.
+
+## Provenance
+
+Every report carries:
+
+- `runFingerprint`: SHA-256 over the canonicalised list of
+  `(runId, candidateId, splitTag)` triples (sorted by runId), plus the
+  comparator id and split. Same `(runs, comparator, split)` produces the same
+  fingerprint regardless of input order.
+- `preregistrationHash`: the caller passes the hash of a signed
+  `HypothesisManifest` (see `pre-registration.ts`). The fingerprint and the
+  preregistration hash together let a reader verify both *what data the
+  report saw* and *what protocol it was supposed to run.*
+
+Reports without a `preregistrationHash` carry a "post-hoc" warning in the
+risks list and the executive summary. Treat them as descriptive only.
+
+## Alternatives considered
+
+- **Paired t-test instead of Wilcoxon + bootstrap.** Rejected: agent score
+  distributions are heavy-tailed (judges saturate near 0 and 1) and the t
+  approximation breaks down with the small N typical of holdouts.
+- **Unpaired Mann–Whitney.** Rejected: matched scenarios make pairing free,
+  and unpaired tests throw away the variance reduction. Use the paired test
+  by default.
+- **Sequential / always-valid inference (e-values, mSPRT, alpha-spending).**
+  Out of scope for a single-look report. If users iterate, wrap this report
+  in an alpha-spending schedule, or commit to one preregistered look.
+- **Hierarchical Bayesian shrinkage across many candidates.** Future work.
+  The current ranking is on raw paired statistics and over-credits the top
+  candidate when many are tested.
+- **Calibration / coverage simulation on the bootstrap CI.** Future work; we
+  rely on the asymptotic guarantee plus the hard pair floor to keep coverage
+  reasonable.
+
+## When NOT to apply
+
+- Paired N below the hard floor (6) on any candidate.
+- Comparator chosen by inspecting the data (post-hoc selection inflates
+  false-discovery rates beyond the BH guarantee).
+- Mid-run distribution shift: judge model swap, rubric change, infrastructure
+  outage. Pair exchangeability is violated and the bootstrap is not valid.
+- Scenarios drawn non-randomly from a stream the candidate can influence
+  (data-leak across runs). The pairing is no longer ignorable.
+- Highly skewed cost distributions: the Pareto frontier still works but the
+  marginal CI on cost may be misleading.
+
+## Citations
+
+- Benjamini, Y. & Hochberg, Y. (1995). Controlling the false discovery rate:
+  a practical and powerful approach to multiple testing. *JRSS B*,
+  57(1), 289–300.
+- Wilcoxon, F. (1945). Individual comparisons by ranking methods.
+  *Biometrics Bulletin*, 1(6), 80–83.
+- Efron, B. (1979). Bootstrap methods: another look at the jackknife.
+  *Annals of Statistics*, 7(1), 1–26.
+- Rubin, D. B. (1981). The Bayesian bootstrap.
+  *Annals of Statistics*, 9(1), 130–134.
+- Kruschke, J. K. (2018). Rejecting or accepting parameter values in
+  Bayesian estimation. *Advances in Methods and Practices in
+  Psychological Science*, 1(2), 270–280. (ROPE.)
+- Howard, S. R., Ramdas, A., McAuliffe, J., Sekhon, J. (2021).
+  Time-uniform, nonparametric, nonasymptotic confidence sequences.
+  *Annals of Statistics*, 49(2), 1055–1080. (Background reading on
+  always-valid inference for sequential extensions.)

package/docs/trace-analysis.md

@@ -0,0 +1,75 @@
+# Trace Analysis
+
+Trace analysis is the bridge between raw product telemetry and useful eval work.
+
+```txt
+live product run
+  -> TraceEmitter / TraceStore
+  -> TraceAnalyst investigates trace corpora
+  -> findings become ASI, failures, replay cases, and release actions
+```
+
+## When To Use TraceAnalyst
+
+Use `TraceAnalyst` when you have more than a few traces and need to answer:
+
+- which failure modes are recurring?
+- which spans explain a regression?
+- did retrieval, integrations, sandbox, or policy block the run?
+- are failed runs missing evidence that the optimizer needs?
+- which product surfaces deserve the next fix?
+
+Use summary tables and release confidence for promotion decisions. Use
+TraceAnalyst to explain the evidence behind those decisions.
+
+## Minimal Flow
+
+```ts
+import {
+  OtlpFileTraceStore,
+  analyzeTraces,
+} from '@tangle-network/agent-eval'
+
+const result = await analyzeTraces({
+  question: 'Why did app-runtime holdout runs fail this week?',
+}, {
+  source: new OtlpFileTraceStore({ path: 'traces/otlp.jsonl' }),
+  ai,
+  model: 'gpt-4o-2024-11-20',
+})
+
+console.log(result.findings)
+```
+
+Products can pass any `TraceAnalysisStore`; they do not need to use the file
+store in production.
+
+## Required Trace Shape
+
+Every serious product run should include:
+
+- `runId`, `projectId`, `scenarioId`, `variantId`, and `layer`
+- commit, prompt hash, config hash, model fingerprint, and dataset version
+- LLM spans with model, inputs, outputs, token counts, and cost
+- tool/integration spans with arguments, result summaries, and error codes
+- retrieval spans with query, source ids, hit scores, and freshness metadata
+- sandbox/build/test/deploy spans with exit codes and log artifacts
+- custom events for knowledge readiness and integration gates
+- final run outcome with pass/score/failure class
+
+Do not put secrets, raw OAuth tokens, or unredacted PII in traces.
+
+## Product Loop
+
+The product loop should not treat traces as a separate debug dump. The intended
+path is:
+
+1. Wrap the real workflow in `runAgentControlLoop` or the product runtime.
+2. Emit canonical spans/events while the user task runs.
+3. Convert the completed run to `FeedbackTrajectory` for replay.
+4. Convert promotion-grade runs to `RunRecord` with `controlRunToRunRecord`.
+5. Run TraceAnalyst over failure-heavy trace sets.
+6. Feed findings into `ActionableSideInfo`, failure clusters, and release
+   reports.
+
+That makes normal product usage become eval data instead of isolated logs.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.20.11",
+  "version": "0.21.0",
   "description": "Trace-first evaluation infrastructure for agent systems: traces, harnesses, verifier pipelines, judges, datasets, gates, optimization, and reporting.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -19,6 +19,26 @@
       "import": "./dist/index.js",
       "default": "./dist/index.js"
     },
+    "./control": {
+      "types": "./dist/control.d.ts",
+      "import": "./dist/control.js",
+      "default": "./dist/control.js"
+    },
+    "./optimization": {
+      "types": "./dist/optimization.d.ts",
+      "import": "./dist/optimization.js",
+      "default": "./dist/optimization.js"
+    },
+    "./reporting": {
+      "types": "./dist/reporting.d.ts",
+      "import": "./dist/reporting.js",
+      "default": "./dist/reporting.js"
+    },
+    "./traces": {
+      "types": "./dist/traces.d.ts",
+      "import": "./dist/traces.js",
+      "default": "./dist/traces.js"
+    },
     "./telemetry": {
       "types": "./dist/telemetry/index.d.ts",
       "import": "./dist/telemetry/index.js",
@@ -54,15 +74,6 @@
   "publishConfig": {
     "access": "public"
   },
-  "scripts": {
-    "build": "tsup && pnpm openapi",
-    "dev": "tsup --watch",
-    "prepare": "pnpm build",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "typecheck": "tsc --noEmit",
-    "openapi": "node dist/cli.js openapi --out dist/openapi.json"
-  },
   "dependencies": {
     "@asteasolutions/zod-to-openapi": "^8.5.0",
     "@ax-llm/ax": "^19.0.25",
@@ -82,5 +93,12 @@
     "node": ">=20"
   },
   "license": "MIT",
-  "packageManager": "pnpm@10.22.0"
-}
+  "scripts": {
+    "build": "tsup && pnpm openapi",
+    "dev": "tsup --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit",
+    "openapi": "node dist/cli.js openapi --out dist/openapi.json"
+  }
+}