@tangle-network/agent-eval 0.20.10 → 0.20.12

package/dist/traces.js

@@ -0,0 +1,100 @@
+import {
+  DEFAULT_REDACTION_RULES,
+  DEFAULT_TRACE_ANALYST_BUDGETS,
+  FAILURE_CLASSES,
+  FileSystemTraceStore,
+  InMemoryTraceStore,
+  OTEL_AGENT_EVAL_SCOPE,
+  OtlpFileTraceStore,
+  REDACTION_VERSION,
+  SpanNotFoundError,
+  TRACE_ANALYST_ACTOR_DESCRIPTION,
+  TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
+  TRACE_ANALYST_SUBAGENT_DESCRIPTION,
+  TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
+  TRACE_SCHEMA_VERSION,
+  TraceFileMissingError,
+  TraceNotFoundError,
+  aggregateLlm,
+  analyzeTraces,
+  argHash,
+  buildTraceAnalystTools,
+  buildTraceInsightContext,
+  buildTraceInsightPrompt,
+  defaultTraceInsightPanel,
+  describeTraceInsightScope,
+  domainEvidencePattern,
+  exportRunAsOtlp,
+  groupBy,
+  inferDomainKeywords,
+  isJudgeSpan,
+  isLlmSpan,
+  isRetrievalSpan,
+  isSandboxSpan,
+  isToolSpan,
+  judgeSpans,
+  llmSpans,
+  planTraceInsightQuestions,
+  redactString,
+  redactValue,
+  runFailureClass,
+  runsForScenario,
+  scoreTraceInsightReadiness,
+  tokenizeDomainWords,
+  toolSpans,
+  traceAnalystFunctionGroup
+} from "./chunk-KWUAAIHR.js";
+import {
+  TraceEmitter,
+  llmSpanFromProvider
+} from "./chunk-PKCVBYTQ.js";
+import "./chunk-PZ5AY32C.js";
+export {
+  DEFAULT_REDACTION_RULES,
+  DEFAULT_TRACE_ANALYST_BUDGETS,
+  FAILURE_CLASSES,
+  FileSystemTraceStore,
+  InMemoryTraceStore,
+  OTEL_AGENT_EVAL_SCOPE,
+  OtlpFileTraceStore,
+  REDACTION_VERSION,
+  SpanNotFoundError,
+  TRACE_ANALYST_ACTOR_DESCRIPTION,
+  TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
+  TRACE_ANALYST_SUBAGENT_DESCRIPTION,
+  TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
+  TRACE_SCHEMA_VERSION,
+  TraceEmitter,
+  TraceFileMissingError,
+  TraceNotFoundError,
+  aggregateLlm,
+  analyzeTraces,
+  argHash,
+  buildTraceAnalystTools,
+  buildTraceInsightContext,
+  buildTraceInsightPrompt,
+  defaultTraceInsightPanel,
+  describeTraceInsightScope,
+  domainEvidencePattern,
+  exportRunAsOtlp,
+  groupBy,
+  inferDomainKeywords,
+  isJudgeSpan,
+  isLlmSpan,
+  isRetrievalSpan,
+  isSandboxSpan,
+  isToolSpan,
+  judgeSpans,
+  llmSpanFromProvider,
+  llmSpans,
+  planTraceInsightQuestions,
+  redactString,
+  redactValue,
+  runFailureClass,
+  runsForScenario,
+  scoreTraceInsightReadiness,
+  tokenizeDomainWords,
+  toolSpans,
+  traceAnalystFunctionGroup
+};
+//# sourceMappingURL=traces.js.map

package/dist/traces.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/wire/index.js

@@ -24,8 +24,8 @@ import {
   runRpcBatch,
   runRpcOnce,
   startServer
-} from "../chunk-LSR4IAYN.js";
-import "../chunk-JAOLXRIA.js";
+} from "../chunk-HNJLMAJ2.js";
+import "../chunk-75MCTH7P.js";
 import "../chunk-PZ5AY32C.js";
 export {
   BUILTIN_RUBRICS,

package/docs/concepts.md

@@ -1,14 +1,15 @@
 # Concepts
 
-Read this once and the rest of agent-eval makes sense.
+`agent-eval` is for deciding whether an agent run should pass, keep working, be
+replayed, be optimized, or be promoted.
 
-## What is agent-eval?
+It exists because agent output is not evidence. A model can say a task is done
+while the build fails, the browser flow is broken, the integration was never
+connected, or the answer lacks required sources. The package gives products a
+shared way to record runs, check outcomes, classify failures, compare variants,
+and make release decisions.
 
-A library for **deciding whether a code generator or content generator did its job.** You give it a thing the generator produced (a scaffold, a patch, a tweet, a JSON config), and you get back a structured verdict: pass/fail, dimension scores, a reason in plain English.
-
-It exists because LLMs lie about whether they succeeded. A model will say "Done!" and ship code that doesn't compile. agent-eval is the layer between the model's output and your decision to ship.
-
-## The three things you'll touch most
+## Main Objects
 
 | Thing | What it is | One-line example |
 |---|---|---|
@@ -17,7 +18,8 @@ It exists because LLMs lie about whether they succeeded. A model will say "Done!
 | **Verifier** | A pipeline of judges run in order, with dependencies. | "install → typecheck → build → semantic" |
 | **Feedback trajectory** | A multi-shot record of attempts, approvals, rejections, edits, metrics, and policy outcomes. | "draft → user rejects → revised draft → approved → measured" |
 
-That's the whole framework. Everything else (sessions, traces, layers) is plumbing around those three.
+Everything else exists to make those objects useful in real product loops:
+traces, datasets, control runtime, optimizers, statistics, and reports.
 
 When the thing being evaluated is an agent that should keep working, use
 [`runAgentControlLoop`](./control-runtime.md). It turns validators into a
@@ -62,7 +64,7 @@ shape stays the same.
 Those trajectories can be converted into preference memory, `DatasetScenario`
 rows, optimizer rows, and held-out examples for overfit checks.
 
-## The three-layer eval (for code generators)
+## Code Generator Eval
 
 When the artifact is generated code, agent-eval scores it at three independent layers. Each layer fails differently, and you want to know which one broke:
 
@@ -125,7 +127,7 @@ Two rules that will save you bugs:
 
 2. **Pair LLM judges with build outcomes.** An LLM judge will rate non-compiling code as "looks right" (0.8). Always short-circuit on `buildOutcome.passed === false` before any LLM judging.
 
-## The trace model (skip on first read)
+## Trace Model
 
 Every operation emits structured spans into a `TraceStore`. A run is a tree:
 
@@ -142,7 +144,10 @@ builder-session                 [span]
 
 Spans are append-only and have stable ids — replay is reading the same store back. OTLP export ships them out for distributed tracing.
 
-You don't need to build the trace tree by hand. `BuilderSession` does it for you. Look at the trace store when you're debugging a flaky run; ignore it otherwise.
+You usually should not build this tree by hand. Product runtimes,
+`runAgentControlLoop`, harnesses, and verifiers should emit it while they run.
+Use traces when debugging a flaky run, building replay data, or explaining a
+release decision.
 
 ## Where to go next
 

package/docs/feature-guide.md

@@ -3,7 +3,7 @@
 This page explains the main `agent-eval` primitives in plain English first,
 then shows when to use each one.
 
-## ELI5
+## Overview
 
 LLM agents can write code, drafts, research, plans, and actions. The hard part
 is knowing whether they actually did a good job, whether they should keep
@@ -41,7 +41,7 @@ trying, and whether a change made them better or worse.
 
 ## Integration Patterns
 
-### Recommended Agent Product Shape
+### Recommended Product Shape
 
 Use this shape when the product needs to keep pushing work forward instead of
 only answering once:
@@ -175,21 +175,6 @@ Store as `FeedbackTrajectory`, then derive:
   logs, screenshots, or browser state. Use separate sandboxes for parallel
   variants or destructive checks.
 
-## Same-Sandbox Example
-
-`examples/same-sandbox-harness/` shows the common coding/browser pattern:
-
-```text
-one sandbox/workdir -> install/build/test -> inspect evidence -> emit judge span
-```
-
-Use this when a judge needs evidence produced by earlier harness phases. Use
-isolated sandboxes when variants run in parallel or a phase can corrupt the
-workspace.
-- Treat telemetry as evidence, not control flow. A trace sink outage should be
-  visible in `runtimeErrors`, but it should not stop the worker from completing
-  the user task.
-
 ## Highest-ROI Adoption Order
 
 1. Wrap one real product workflow in `runAgentControlLoop`.
@@ -211,3 +196,11 @@ reusable:
 
 Core should provide shapes, stores, runners, scoring, traces, and converters.
 Downstream integrations provide domain state, policy, tools, and storage.
+
+## Examples
+
+- `examples/same-sandbox-harness`: one workdir for install/build/test plus
+  evidence inspection.
+- `examples/multi-shot-optimization`: full-trajectory optimization with a
+  holdout gate.
+- `examples/benchmarks`: benchmark adapter contracts and reference wrappers.

package/docs/integration-launch-gates.md

@@ -0,0 +1,77 @@
+# Integration Launch Gates
+
+Use these gates when a product lets generated apps or agents use user-owned
+connections through an integration hub.
+
+The eval should wrap the real product path:
+
+```txt
+user prompt
+  -> product emits IntegrationManifest
+  -> platform resolves connections and grants
+  -> sandbox receives capability bundle
+  -> generated app invokes integration action
+  -> platform enforces policy, approval, idempotency, audit
+```
+
+## Deterministic Gates
+
+- The generated app declares an integration manifest before sandbox launch.
+- Manifest validation passes.
+- Required connections and scopes are present before execution.
+- Sandbox environment contains a capability bundle, not raw provider tokens.
+- Reads invoke through the platform bridge.
+- Writes return `approval_required` unless product policy explicitly allows
+  them.
+- Approved writes are bound to the same action, input hash, connection, and
+  subject.
+- Revoked grants or expired capabilities stop invocation.
+- Resumed or long-running sandboxes receive a refreshed bundle before expiry.
+- Audit includes grant creation, capability issue, invoke success/failure,
+  approval resolution, and revoke events.
+
+## Failure Classes
+
+`agent-eval` classifies integration failures separately from prompt/tool
+failures:
+
+- `bad_integration_manifest`
+- `missing_integration_connection`
+- `missing_integration_scope`
+- `integration_approval_required`
+- `integration_auth_expired`
+- `integration_provider_failure`
+- `unsafe_integration_write_denied`
+
+Use the helper payload builders and eval builders so products emit the same
+trace evidence:
+
+```ts
+const gate = {
+  connectorId: 'google-calendar',
+  actionId: 'events.create',
+  valid: true,
+  missingConnections: [],
+  missingScopes: ['calendar.events.write'],
+}
+
+const evals = integrationGateEvals(gate)
+
+await emitter.emit({
+  kind: 'custom',
+  payload: integrationManifestResolvedPayload(gate),
+})
+
+await emitter.emit({
+  kind: 'custom',
+  payload: integrationInvokeFailedPayload({
+    connectorId: 'google-calendar',
+    actionId: 'events.create',
+    code: 'scope_denied',
+    message: 'calendar.events.write was not granted',
+  }),
+})
+```
+
+The classifier then reports the real missing surface instead of burying the
+failure under `tool_recovery_failure` or `unknown`.

package/docs/product-eval-adoption.md

@@ -0,0 +1,221 @@
+# Product Eval Adoption
+
+This guide is for teams adding `@tangle-network/agent-eval` to a real agent
+product. The package supplies evaluation contracts and runtime primitives. Your
+product supplies the actual workflow adapter, state, credentials, tools, UI, and
+storage.
+
+## Goal
+
+Use the same loop for production, replay, and optimization:
+
+```txt
+real user task
+  -> product adapter observes state
+  -> validators and judges grade state
+  -> control loop decides next action
+  -> product agent acts in the real environment
+  -> trace + feedback trajectory are stored
+  -> datasets and optimizers replay the same adapter
+```
+
+If production and eval use different loops, benchmark gains will not transfer.
+
+## What The Product Owns
+
+The product owns:
+
+- task state and domain models
+- credentials, tenant policy, approval, and side-effect rules
+- browser, sandbox, CLI, connector, or voice drivers
+- database and trace persistence
+- user/reviewer feedback collection
+- deployment and live canary routing
+- model gateway configuration
+
+`agent-eval` owns:
+
+- trace, run, dataset, feedback, and score contracts
+- control-loop mechanics
+- verifier and judge orchestration
+- failure taxonomy
+- paired statistics and holdout gates
+- optimizer inputs and promotion reports
+
+## Minimal Production Adapter
+
+Start with a small adapter that mirrors one real workflow.
+
+```ts
+interface ProductEvalAdapter<TState, TAction> {
+  observe(taskId: string): Promise<TState>
+  validate(state: TState): Promise<ControlEvalResult[]>
+  decide(input: {
+    state: TState
+    evals: ControlEvalResult[]
+    history: unknown[]
+  }): Promise<TAction | 'stop'>
+  act(taskId: string, action: TAction): Promise<void>
+}
+```
+
+Keep the adapter product-owned until at least two products need the same shape.
+
+## Validator Order
+
+Use deterministic checks before judges.
+
+1. **State validity**: schema, required files, required DB rows, required
+   connections.
+2. **Runtime gates**: install, build, typecheck, tests, serve, deploy smoke.
+3. **Policy gates**: approvals, side effects, budget, credentials, data
+   freshness.
+4. **Behavior gates**: browser flows, API calls, generated app preview, voice
+   transcript checks.
+5. **Semantic judges**: intent fit, quality, completeness, safety,
+   professional correctness.
+
+Semantic judges should never turn a failed build into a pass.
+
+## Traces And Feedback
+
+Every serious run should record:
+
+- task id and scenario id
+- git commit
+- model and provider
+- prompt/config hashes
+- tool calls and retrieval spans
+- build/test/deploy output
+- cost, latency, and token use
+- user/reviewer feedback
+- final outcome and failure class
+
+Convert runs into `FeedbackTrajectory` records so normal product usage becomes
+replayable eval data.
+
+```txt
+production run -> feedback trajectory -> dataset scenario -> optimizer row
+```
+
+For promotion-grade runs, also project the completed control result into a
+strict `RunRecord`:
+
+```ts
+const record = controlRunToRunRecord(controlResult, {
+  experimentId,
+  candidateId,
+  seed,
+  model: 'gpt-4o-2024-11-20',
+  promptHash,
+  configHash,
+  commitSha,
+  splitTag: 'holdout',
+  tokenUsage,
+})
+```
+
+## Datasets And Holdouts
+
+Use four splits:
+
+- `train`: optimizer search.
+- `dev`: tuning and threshold selection.
+- `test`: normal reporting.
+- `holdout`: promotion-only gate.
+
+The low-level `RunRecord` schema uses `search | dev | holdout`; map `train`
+and normal non-holdout test/report rows to `search` when producing promotion
+tables.
+
+Do not inspect or tune against holdout failures during optimization. If a
+holdout failure reveals a real product bug, fix the bug and rotate the holdout
+set with a signed note.
+
+## Optimization
+
+Use `runMultiShotOptimization()` when the system is a multi-step agent, not a
+single prompt.
+
+Good optimization targets:
+
+- system prompt
+- tool descriptions
+- retrieval policy
+- data acquisition policy
+- user-question policy
+- evaluator threshold
+- agent topology
+- scaffold/template choice
+
+Bad optimization targets:
+
+- hidden holdout examples
+- production credentials
+- brittle string checks that do not match user value
+- fake workflows that do not call the product adapter
+
+Use actionable side information so the optimizer knows whether a failure belongs
+to prompt, tools, retrieval, data acquisition, sandbox, evaluator, or product
+runtime.
+
+## Release Gate
+
+A launch or promotion should require:
+
+- enough runs for the target risk level
+- paired improvement over the current baseline
+- no critical regression on test
+- holdout pass or explicit rejection
+- cost and latency within budget
+- no unresolved canary or contamination failures
+- trace evidence for representative successes and failures
+- TraceAnalyst findings for failure-heavy or regression-heavy corpora
+- human-readable report with failure clusters and next actions
+
+`evaluateReleaseConfidence()` and the paired statistics helpers provide the
+decision data. The product decides the business threshold.
+
+## Product Patterns
+
+### Coding Or Builder Agent
+
+Use sandbox/build/test/serve/browser validators. Add intent and semantic
+concept judges only after the generated app runs.
+
+### Browser Agent
+
+Record browser steps, screenshots, network errors, console errors, and final
+state. Use deterministic DOM/API assertions before visual or semantic judges.
+
+### Domain Agent
+
+Use domain fixtures, jurisdiction/date metadata, retrieval spans, and
+professional judges. Fail missing/stale evidence separately from bad reasoning.
+
+### Workflow Or Integration Agent
+
+Use `@tangle-network/agent-integrations` manifests as readiness inputs. Gate
+missing connections, missing scopes, approval-required writes, and stale tokens
+before blaming the agent prompt.
+
+For generated apps and sandbox agents, also run the
+[Integration Launch Gates](./integration-launch-gates.md). The eval should prove
+that app code invokes through the integration bridge, not provider SDKs with raw
+OAuth tokens.
+
+### Voice Agent
+
+Record transcript, timing, interruptions, tool calls, and task outcome. Judge
+conversation quality separately from tool success and policy compliance.
+
+## Anti-Patterns
+
+- Evaluating only final prose for an agent that actually builds, browses, or
+  calls tools.
+- Letting an LLM judge override failed tests.
+- Optimizing on examples that users will never hit.
+- Recording traces as logs but never converting them to datasets.
+- Calling every failure a prompt failure when context, data, auth, or runtime
+  readiness was missing.
+- Shipping reports without run ids, commits, model ids, or evidence links.

package/docs/trace-analysis.md

@@ -0,0 +1,75 @@
+# Trace Analysis
+
+Trace analysis is the bridge between raw product telemetry and useful eval work.
+
+```txt
+live product run
+  -> TraceEmitter / TraceStore
+  -> TraceAnalyst investigates trace corpora
+  -> findings become ASI, failures, replay cases, and release actions
+```
+
+## When To Use TraceAnalyst
+
+Use `TraceAnalyst` when you have more than a few traces and need to answer:
+
+- which failure modes are recurring?
+- which spans explain a regression?
+- did retrieval, integrations, sandbox, or policy block the run?
+- are failed runs missing evidence that the optimizer needs?
+- which product surfaces deserve the next fix?
+
+Use summary tables and release confidence for promotion decisions. Use
+TraceAnalyst to explain the evidence behind those decisions.
+
+## Minimal Flow
+
+```ts
+import {
+  OtlpFileTraceStore,
+  analyzeTraces,
+} from '@tangle-network/agent-eval'
+
+const result = await analyzeTraces({
+  question: 'Why did app-runtime holdout runs fail this week?',
+}, {
+  source: new OtlpFileTraceStore({ path: 'traces/otlp.jsonl' }),
+  ai,
+  model: 'gpt-4o-2024-11-20',
+})
+
+console.log(result.findings)
+```
+
+Products can pass any `TraceAnalysisStore`; they do not need to use the file
+store in production.
+
+## Required Trace Shape
+
+Every serious product run should include:
+
+- `runId`, `projectId`, `scenarioId`, `variantId`, and `layer`
+- commit, prompt hash, config hash, model fingerprint, and dataset version
+- LLM spans with model, inputs, outputs, token counts, and cost
+- tool/integration spans with arguments, result summaries, and error codes
+- retrieval spans with query, source ids, hit scores, and freshness metadata
+- sandbox/build/test/deploy spans with exit codes and log artifacts
+- custom events for knowledge readiness and integration gates
+- final run outcome with pass/score/failure class
+
+Do not put secrets, raw OAuth tokens, or unredacted PII in traces.
+
+## Product Loop
+
+The product loop should not treat traces as a separate debug dump. The intended
+path is:
+
+1. Wrap the real workflow in `runAgentControlLoop` or the product runtime.
+2. Emit canonical spans/events while the user task runs.
+3. Convert the completed run to `FeedbackTrajectory` for replay.
+4. Convert promotion-grade runs to `RunRecord` with `controlRunToRunRecord`.
+5. Run TraceAnalyst over failure-heavy trace sets.
+6. Feed findings into `ActionableSideInfo`, failure clusters, and release
+   reports.
+
+That makes normal product usage become eval data instead of isolated logs.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.20.10",
+  "version": "0.20.12",
   "description": "Trace-first evaluation infrastructure for agent systems: traces, harnesses, verifier pipelines, judges, datasets, gates, optimization, and reporting.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -19,6 +19,26 @@
       "import": "./dist/index.js",
       "default": "./dist/index.js"
     },
+    "./control": {
+      "types": "./dist/control.d.ts",
+      "import": "./dist/control.js",
+      "default": "./dist/control.js"
+    },
+    "./optimization": {
+      "types": "./dist/optimization.d.ts",
+      "import": "./dist/optimization.js",
+      "default": "./dist/optimization.js"
+    },
+    "./reporting": {
+      "types": "./dist/reporting.d.ts",
+      "import": "./dist/reporting.js",
+      "default": "./dist/reporting.js"
+    },
+    "./traces": {
+      "types": "./dist/traces.d.ts",
+      "import": "./dist/traces.js",
+      "default": "./dist/traces.js"
+    },
     "./telemetry": {
       "types": "./dist/telemetry/index.d.ts",
       "import": "./dist/telemetry/index.js",