@tangle-network/agent-eval 0.46.0 → 0.48.0

package/dist/{run-improvement-loop-Bfam3MT1.d.ts → run-improvement-loop-B-L8GgpW.d.ts}

@@ -1,7 +1,7 @@
 import { S as Scenario, d as CampaignResult, j as GateResult, o as Mutator, I as ImprovementDriver, G as Gate, g as DispatchFn, J as JudgeConfig, L as LabeledScenarioStore, e as CampaignTraceWriter, M as MutableSurface, l as GenerationRecord } from './types-8u72Gc76.js';
 import { L as LlmClientOptions } from './llm-client-BXVRUZyX.js';
-import { RunRecord } from '@tangle-network/agent-runtime';
 import { R as RedTeamCase } from './red-team-30II1T4o.js';
+import { R as RunRecord } from './run-record-BGY6bHRh.js';
 
 /**
  * @experimental

package/dist/{sequential-DdV5ShjT.d.ts → sequential-CbFH___X.d.ts}

@@ -35,7 +35,7 @@ import { F as FailureClusterReport } from './failure-cluster-Cw65_5FY.js';
  *     specific promotion path (still useful for replay-style evals).
  */
 
-type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap';
+type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
 interface HeldOutGateConfig {
     /** Minimum number of paired (candidate, baseline) holdout observations
      *  required before the gate will even consider promoting. Default 3. */
@@ -57,6 +57,19 @@ interface HeldOutGateConfig {
     /** Optional deterministic seed for the bootstrap. Default undefined
      *  (Math.random). */
     seed?: number;
+    /**
+     * Hard ceiling on the candidate's median per-task USD cost. When the
+     * candidate clears the quality gates (paired-delta + overfit-gap) but
+     * its median cost exceeds this number, the gate rejects with
+     * `cost_ceiling`. Default `undefined` = no cost ceiling, behaving
+     * exactly like the pre-cost gate.
+     *
+     * This exists because "we ship the better prompt" is only an honest
+     * pitch when the better prompt also fits a customer-stated budget.
+     * Cost is read from `RunRecord.costUsd` (already mandatory on every
+     * run) so no new schema is required.
+     */
+    costPerTaskCeiling?: number;
 }
 interface GateEvidence {
     /** Number of paired (candidate, baseline) holdout observations used. */
@@ -78,6 +91,14 @@ interface GateEvidence {
     overfitGap: number;
     /** Baseline (search − holdout) gap. */
     baselineOverfitGap: number;
+    /** Median per-task USD cost across the candidate's runs. Recorded
+     *  even when no `costPerTaskCeiling` is configured so downstream
+     *  dashboards (intelligence.tangle.tools) can render \$/task per
+     *  generation regardless of gating policy. */
+    medianCandidateCost: number;
+    /** Median per-task USD cost across the baseline runs, for
+     *  symmetric reporting. */
+    medianBaselineCost: number;
 }
 interface GateDecision {
     /** Final promote/no-promote verdict. */
@@ -106,6 +127,7 @@ declare class HeldOutGate {
     private readonly confidence;
     private readonly resamples;
     private readonly seed?;
+    private readonly costPerTaskCeiling?;
     constructor(config: HeldOutGateConfig);
     /** Decide whether `candidate` should replace `baseline`. Pairing
      *  is by (experimentId, seed) — identical experiment + seed pairs

package/dist/{types-DHqkLwEU.d.ts → types-CqPax19X.d.ts}

@@ -1,4 +1,4 @@
-import { DefaultVerdict } from '@tangle-network/agent-runtime/loops';
+import { D as DefaultVerdict } from './verdict-CeEgtjyI.js';
 
 /**
  * @experimental

package/dist/verdict-CeEgtjyI.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Validator-output verdict — substrate primitive for "did this output pass,
+ * and how well?"
+ *
+ * Used by:
+ *   - `@tangle-network/agent-eval/matrix` — verdict per cell in the cartesian.
+ *   - `@tangle-network/agent-runtime` — Validator<Output, Verdict = DefaultVerdict>.
+ *     Runtime keeps `Validator` because it's coupled to runtime-shaped
+ *     `ValidationCtx` (iteration, signal, traceEmitter); the verdict TYPE
+ *     itself is a substrate concept and lives here.
+ *
+ * Repo layering: agent-eval is the substrate (no upward deps). Both
+ * agent-runtime and agent-knowledge consume this type FROM agent-eval —
+ * never the other way around. See CLAUDE.md "Repo layering" for the rule.
+ */
+/**
+ * Minimal verdict shape — `valid` + `score` are required; `scores` +
+ * `notes` are optional surface. Validators that need richer shapes
+ * parameterise `Validator<Output, MyVerdict>` with their own type.
+ */
+interface DefaultVerdict {
+    /** Whether the output meets the validator's pass criteria. */
+    valid: boolean;
+    /** Aggregate score in [0, 1]. Drivers use this for winner selection. */
+    score: number;
+    /** Per-dimension scores. Free-form; weighted into `score` by the validator. */
+    scores?: Record<string, number>;
+    /** Human-readable rationale; surfaces in trace + final-result `winner.verdict`. */
+    notes?: string;
+}
+
+export type { DefaultVerdict as D };

package/docs/adapters-observability.md

@@ -35,6 +35,21 @@ it*. Unified at the trace level, you see both as one timeline per cell.
 - Compose: register TraceAI's instrumentations on the global tracer
   provider, then either point both at your OTLP collector or at
   TraceAI's hosted backend if you want their UI.
+- **Or use the bridge: `@tangle-network/agent-eval/adapters/traceai`.**
+  Forwards finished OTel spans (`ReadableSpan` shape) directly into the
+  hosted-tier ingest, lifting `tangle.runId` / `tangle.scenarioId` /
+  `tangle.cellId` / `tangle.generation` to first-class wire fields so
+  the dashboard pivots correctly. Zero dependency on `@opentelemetry/*`
+  at the substrate; consumers pass spans from their own OTel SDK.
+  ```ts
+  import { createHostedClient } from '@tangle-network/agent-eval/hosted'
+  import { createTraceAiBridge } from '@tangle-network/agent-eval/adapters/traceai'
+
+  const client = createHostedClient({ endpoint, apiKey, tenantId })
+  const bridge = createTraceAiBridge({ client, defaultRunId: substrateRunId })
+  processor.onEnd = (span) => { void bridge.ingest([span]) }
+  // ...or call `bridge.ingest(batch)` from a SpanProcessor.onShutdown.
+  ```
 
 ### Langfuse SDK
 

package/docs/design/phase-d-rfc.md

@@ -0,0 +1,125 @@
+# Phase D RFC — hosted-tier substrate
+
+Pinned scope decisions for the EXPAND tier. What we built, what we
+deliberately did NOT, and what's gated on Phase B evidence.
+
+---
+
+## What's in this version
+
+**Wire-format substrate (shipped):**
+
+1. `@tangle-network/agent-eval/hosted` — public client + types for shipping
+   eval-run events + trace spans to any orchestrator that speaks the wire
+   format.
+2. `docs/hosted-ingest-spec.md` — semver-committed wire spec
+   (`HostedWireVersion = "2026-05-26.v1"`).
+3. `examples/hosted-ingest-server/` — minimal hono-based reference
+   receiver (~200 LOC). Executable spec. Stays as the reference even
+   after the production orchestrator ships.
+4. `selfImprove({ hostedTenant })` opt-in — when set, the substrate
+   POSTs the final eval-run event to the configured endpoint. Failures
+   are logged but never fail the loop (LAND tier never blocks on
+   EXPAND-tier infra).
+
+**Production orchestrator (started):**
+
+5. HTTP ingest service in `@tangle-network/monorepo` accepting the wire
+   format. Lives under the orchestrator app. Tenant auth + isolation
+   + persistent storage + read endpoints. *Started this session — see
+   the @tangle-network/agent-dev-container PR. Not feature-complete:
+   tenant CRUD + adversarial isolation tests pending.*
+
+## What's deliberately deferred
+
+The wedge doc gates these on Phase B evidence — partner-validated
+signal about what the hosted product actually needs to do. Shipping
+them without that signal risks building the wrong thing.
+
+| Deferred until Phase B passes | Why |
+|---|---|
+| **Metered billing wire-up (Stripe + cost-ledger)** | The billable units (per-eval-run, per-ingested-MB, per-seat) depend on actual partner consumption patterns. Picking dimensions in a vacuum locks us into wrong pricing. |
+| **Multi-tenant dashboard UX** | Partners' first dashboard request defines the right default views. We have a stub list-runs page; the rest is post-signal. |
+| **Webhook callbacks per tenant** | The events partners want pushed (gate-decided, cost-threshold, regression-alert) are partner-shaped. Add them when a partner asks. |
+| **Cross-tenant aggregation / benchmarking** | This is the "Datadog for agents" tier — explicit roadmap, requires user volume we don't have. |
+| **Sandbox-cost roll-up into hosted billing** | Cross-product billing integration requires PLATFORM-tier partners. Out of scope until at least one. |
+| **Trace UI** | OTel-shape spans store fine. Visualization comes after partners ask. Phoenix / Jaeger / any OTLP-compatible viewer covers it in the interim. |
+| **Soc2 / compliance audit work** | Required for enterprise; not required for design partners. |
+
+## Architecture decisions locked
+
+These are committed and won't change without a major-version wire bump
+or a documented migration:
+
+1. **Wire format is JSON over HTTP**, not gRPC. Reasons: works in
+   browsers + edge + node + curl; OTel-compatible at the trace stream
+   level; lowest possible barrier to a self-hosted orchestrator.
+2. **Tenant auth is bearer-token + tenant-id header**, not OIDC /
+   service-account / mutual-TLS. Reasons: simplest thing that's
+   actually secure with proper key handling; defers complex IAM until
+   enterprise demand.
+3. **Idempotency via header, not transactional API**. Servers MUST
+   dedupe by `(tenantId, Idempotency-Key)` for 24h. Simpler than
+   making clients commit transactions.
+4. **Eval-runs and traces are SEPARATE streams** with pivot keys
+   (`tangle.runId` etc.) on spans. Reasons: traces can be best-effort
+   (lossy) without corrupting eval-run semantics; orchestrators can
+   prioritize eval-run durability without forcing trace durability.
+5. **Wire version is a date.v-N string**, not semver. Reasons: dates
+   communicate "when was this contract frozen"; v-N captures
+   incremental breaking changes between dates.
+
+## Open questions for Phase B to answer
+
+When the design-partner pairing happens, capture answers to these
+explicitly:
+
+1. **Surface confidentiality**: do partners want the verbatim surface
+   (system prompt) shipped, or just the hash? Today the wire format
+   has `surface?` as optional; partner default is what we ship.
+2. **Trace sampling**: at what cells-per-second do trace spans become
+   noise? What's the right default sampling rate?
+3. **Cost attribution granularity**: per cell? per generation? per
+   run? Per judge dimension? Partner needs determine what we surface
+   in billing reports.
+4. **Replay**: do partners want to re-run an old eval-run from the
+   stored data? That would require us to store more than the summary —
+   actual artifacts + prompts. Storage cost implication.
+5. **PII / sensitive scenarios**: how do partners want to handle
+   scenarios containing user data? Encryption-at-rest is table stakes;
+   redaction-at-ingest may be required for some.
+
+The partner pairing kit (`docs/phase-b-pairing-kit.md`) has discovery
+questions that probe these.
+
+## Non-goals (explicit)
+
+This RFC does NOT plan for:
+
+- Replacing Langfuse / Phoenix / Arize. We INGEST OTel; we don't
+  build a generic trace viewer. The dashboard is eval-run-shaped, not
+  trace-shaped.
+- Becoming a model gateway. Tangle Router exists; the hosted
+  orchestrator routes to Tangle Router by default but doesn't
+  duplicate its function.
+- Becoming an LLM-call CDN. Caching is the consumer's job (their
+  agent code, their HTTP client). We don't intercept LLM calls.
+- Building an "agents IDE." Substrate, not surface.
+
+## Migration path (post Phase B)
+
+When Phase B passes the gate, the production orchestrator finishes:
+
+1. Replace in-memory store with Postgres (tenant data) + S3 (large
+   artifacts) OR Cloudflare D1 + R2 (Workers-native).
+2. Wire metered events to Stripe + the cost-ledger.
+3. Tenant CRUD UI + onboarding flow.
+4. Multi-tenant dashboard MVP (list runs, drill into one, diff
+   generations, view shipped prompt).
+5. Adversarial tenant-isolation test battery in CI.
+6. Webhooks + observability for the orchestrator itself.
+
+Estimated effort post-Phase-B: ~1 week focused work for one engineer.
+This is fast precisely BECAUSE the wire format is locked and the
+reference receiver exists — the production server is a different
+implementation of the same contract.

package/docs/design/substrate-gaps-2026-05-27.md

@@ -0,0 +1,118 @@
+# Substrate gaps — design-partner readiness
+
+What's missing from `@tangle-network/agent-eval` substrate (this repo) and `~/code/agent-dev-container/products/intelligence/` (the orchestrator) to credibly hand to a first design partner.
+
+This doc is the engineering-side mirror of `~/company/gtm/experiments/2026-05-27/design-partner-readiness.md` — gtm tracks the partner-facing readiness, this tracks the code that backs each bar.
+
+## Current substrate state
+
+Shipped in v0.47:
+- `selfImprove({ scenarios, judges, dispatch, hostedTenant })` one-shot helper
+- `defaultProductionGate(deltaThreshold)` autonomous-ship gate
+- Wire format frozen at `HOSTED_WIRE_VERSION = '2026-05-26.v1'`
+- `/hosted/client.ts` — bearer auth, idempotency, bounded retries on 5xx/408/429
+- `examples/hosted-ingest-server/` — reference receiver implementing the spec
+- `docs/hosted-ingest-spec.md` — semver-locked wire spec
+- `docs/design/phase-d-rfc.md` — scope decisions + deferred items
+- `docs/quickstart-external.md` — foreign-agent quickstart
+- `docs/phase-b-pairing-kit.md` — partner discovery script
+- `adapters/langchain` + `adapters/http`
+
+## Engineering gaps keyed to first-partner readiness
+
+### Substrate gaps (this repo)
+
+**S1. TraceAI/OTel adapter (`adapters/traceai`).**
+Future AGI's `traceai` library is the strongest OTel-native instrumentation in the TS ecosystem. Partners using it should be able to wire its emitted spans into our hosted ingest via one config line. The adapter receives OTel spans, normalizes them to `TraceSpanEvent`, ensures the `tangle.runId` attribute is present, and forwards via the existing hosted client.
+
+Path: `src/adapters/traceai.ts`. Export from `tsup.config.ts`. Add to `docs/adapters-observability.md`.
+
+Estimate: 6-8h. Owner: claude. Priority: medium (Tier C in partner-readiness — defer until first partner asks, but pre-build the contract).
+
+**S2. Run-diff data primitive.**
+The orchestrator needs to render "v3 vs v4" comparisons. The substrate should expose a `diffRuns(runA: EvalRunEvent, runB: EvalRunEvent): RunDiff` helper that computes cell-by-cell judge-score deltas, artifact-text diff (using a stable diff algorithm), and lift summary. Without this, every consumer rebuilds the diff logic.
+
+Path: `src/contract/diff.ts`. Add to `/contract` entry.
+
+Estimate: 4-6h. Owner: claude. Priority: high (orchestrator's run-diff view depends on this).
+
+**S3. Sampling controls in hosted client.**
+The Phase D RFC flags trace sampling as an open question. Add `sampling: { traces: number /* 0-1 */ }` to `createHostedClient` options. Default 1.0. Document the cost implication. Reservoir-sample if over budget.
+
+Path: `src/hosted/client.ts`. Update `docs/hosted-ingest-spec.md` accordingly.
+
+Estimate: 2h. Owner: claude. Priority: medium (Tier B partner-readiness).
+
+**S4. Auto-instrumentation library (`@tangle-network/agent-eval/auto`).**
+LangSmith's `@traceable` decorator + auto-wrap of OpenAI/Anthropic SDKs is their highest-leverage adoption tool. Build the equivalent: a `traceable()` HOF that emits OTel spans with `tangle.runId` attribute and forwards via the hosted client. Optional auto-wrap of `OpenAI` / `Anthropic` SDK clients.
+
+This is the biggest unlock for non-LangChain TS partners. Defer until at least one partner asks — pre-shipping costs 16-20h and may be the wrong shape without partner signal.
+
+Path: `src/auto/index.ts` (new entry). Estimate: 16-20h. Owner: claude. Priority: low (Tier C — defer).
+
+**S5. Surface-confidentiality option in wire format.**
+RFC open question 1. Add `surfaceMode: 'verbatim' | 'hashed' | 'omitted'` to `selfImprove` config. When `'hashed'`, ship `surfaceHash` instead of `surface` on the eval-run event. When `'omitted'`, ship neither.
+
+This is partner-shaped — wait for the first conversation. But the wire format should accommodate it without a breaking version bump. Add the optional field to types now.
+
+Path: `src/hosted/types.ts` (add `surfaceHash?: string`). Estimate: 1h to land the type change; later partner work to wire selfImprove. Priority: low until asked.
+
+### Orchestrator gaps (`agent-dev-container/products/intelligence/`)
+
+These are the gtm-doc's Tier A items rephrased for engineering tracking.
+
+**O1. Adversarial tenant-isolation test suite.**
+`tests/auth.test.ts` exists. Need `tests/isolation.test.ts` covering: cross-tenant header mismatch, cross-tenant `/v1/runs/:id` reads, webhook tenant scoping, idempotency-key tenant scoping, raw-SQL cross-tenant query, JWT replay after revocation. Use `VITEST_INTEGRATION=1` with a real Postgres in CI.
+
+Estimate: 4-6h. Owner: claude. Priority: **critical** (blocker for any partner conversation).
+
+**O2. Web dashboard MVP.**
+List runs + run detail + login. See gtm doc A2 for shape. Pages: `/login`, `/runs`, `/runs/:id`, `/keys`. Use `intelligence-web` Vite scaffold; wire to `/v1/runs*` reads.
+
+Estimate: 12-16h. Owner: claude. Priority: **critical** (without UI, partner can't show anyone in their org).
+
+**O3. Free-tier plan limits enforcement.**
+`lib/plans.ts` defines limits. `routes/ingest.ts` does not enforce them. Add per-tenant counters (eval-runs/mo, trace-spans/day), check against plan, return 429 with clear message + reset time on exceed.
+
+Estimate: 3-4h. Owner: claude. Priority: medium (Tier B partner-readiness).
+
+**O4. Stale README sweep.**
+`api/README.md` lists T0-3..T0-8 as "next/pending" when 5 of them are shipped. Broken link to `../../../docs/intelligence-product-rfc.md` (gitignored path; should be `../RFC.md`).
+
+Estimate: 30min. Owner: claude. Priority: **must-fix-now** (5-min job, awful first impression for anyone reading the repo).
+
+**O5. Onboarding partner-facing doc.**
+Engineer-shaped `quickstart-external.md` exists in this repo. Partner-facing 10-minute walkthrough does not. Lives at `intelligence.tangle.tools/docs` once provisioned; for now, write at `products/intelligence/docs/partner-onboarding.md`.
+
+Estimate: 2-3h. Owner: claude. Priority: high (Tier A — needed for any partner call).
+
+## Recommended sequencing (engineering view)
+
+**Sprint 1 — partner-ready (≤ 1 week):**
+- O4 (README sweep) — ship today
+- O1 (isolation tests) — 4-6h
+- O2 (dashboard MVP) — 12-16h
+- O5 (partner onboarding doc) — 2-3h
+- S2 (run-diff primitive) — 4-6h (substrate side of O2 follow-up)
+
+Total: ~25-32h focused work. One engineer-week.
+
+**Sprint 2 — concurrent with first partner conversations:**
+- O3 (plan limits enforcement) — partner will hit it
+- S3 (trace sampling) — partner will ask about cost
+
+**Sprint 3 — after first partner ships to prod:**
+- S1 (TraceAI adapter)
+- Stripe billing wire-up in orchestrator
+
+**Holding for partner signal:**
+- S4 (auto-instrumentation library) — 16-20h speculative without ask
+- S5 wiring (surface confidentiality) — partner-shaped
+
+## Cross-references
+
+- `docs/design/phase-d-rfc.md` — substrate scope decisions (this doc operationalizes its "what's deferred until Phase B")
+- `docs/hosted-ingest-spec.md` — wire format spec (any change here is a wire-version bump)
+- `~/company/gtm/experiments/2026-05-27/design-partner-readiness.md` — partner-facing readiness, mirrors this
+- `~/company/gtm/products/tangle-intelligence.md` — product hub
+- `~/company/gtm/competitor-analysis/agent-improvement.md` — competitive frame

package/docs/hosted-ingest-spec.md

@@ -0,0 +1,204 @@
+# Hosted-ingest wire spec — `2026-05-26.v1`
+
+The schema **every** orchestrator (ours, partners' self-hosted ones,
+any future open implementation) must accept. Frozen under semver:
+**new minors only add optional fields. Breaking changes mean a major
+bump and a new `HostedWireVersion` literal.**
+
+This is the contract that decouples the LAND-tier substrate
+(`@tangle-network/agent-eval`) from the EXPAND-tier hosted product. A
+foreign builder can:
+
+- Use our orchestrator at `https://orchestrator.tangle.tools/v1`.
+- Self-host the reference receiver from
+  `examples/hosted-ingest-server/`.
+- Implement their own orchestrator against this spec.
+
+All three are wire-compatible by definition.
+
+---
+
+## Transport
+
+Two endpoints, both `POST`, both JSON. Headers on every request:
+
+| Header | Value |
+|---|---|
+| `Authorization` | `Bearer <tenant-key>` (the orchestrator issues this) |
+| `Content-Type` | `application/json` |
+| `X-Tangle-Tenant-Id` | The tenant's stable id (the orchestrator's primary key for the tenant) |
+| `X-Tangle-Wire-Version` | `2026-05-26.v1` (this spec) |
+| `Idempotency-Key` (optional) | UUID; servers MUST treat repeated keys as dedup |
+
+Responses are JSON of shape `{ accepted: number, rejected: Array<{ index, reason }> }`. The
+server SHOULD return 202 (accepted, async) or 200 (accepted, synchronous);
+both are equivalent for the wire's purposes.
+
+### `POST /v1/ingest/eval-runs`
+
+Body: `IngestEvalRunsRequest = { wireVersion, events: EvalRunEvent[] }`.
+
+One ingest call per logical eval-run; generations stream in
+incrementally via repeated calls with the same `runId`. The
+orchestrator deduplicates by `(tenantId, runId, generation.index)`.
+
+### `POST /v1/ingest/traces`
+
+Body: `IngestTracesRequest = { wireVersion, spans: TraceSpanEvent[] }`.
+
+Standard OTLP-shaped spans with a few additional attributes
+(`tangle.runId`, `tangle.generation`, `tangle.cellId`,
+`tangle.scenarioId`) so the orchestrator can pivot between the
+eval-run stream and the underlying execution trace.
+
+---
+
+## `EvalRunEvent`
+
+```ts
+interface EvalRunEvent {
+  runId: string                      // stable; same id across all generations of one run
+  runDir: string                     // logical run directory (mem://... or filesystem path)
+  timestamp: string                  // ISO-8601
+  status:                            // lifecycle stage this event represents
+    | 'started'
+    | 'baseline-complete'
+    | 'generation-complete'
+    | 'gate-decided'
+    | 'finished'
+    | 'errored'
+  labels: Record<string, string>     // free-form (env, branch, model id, etc.)
+  baseline?: EvalRunGenerationSnapshot   // present when status >= baseline-complete
+  generations: EvalRunGenerationSnapshot[]
+  gateDecision?:                     // present when status >= gate-decided
+    | 'ship' | 'hold' | 'need_more_work' | 'model_ceiling' | 'arch_ceiling'
+  holdoutLift?: number               // winner-on-holdout - baseline-on-holdout
+  totalCostUsd: number
+  totalDurationMs: number
+  errorMessage?: string              // present when status === 'errored'
+}
+```
+
+## `EvalRunGenerationSnapshot`
+
+```ts
+interface EvalRunGenerationSnapshot {
+  index: number                      // 0 is baseline; 1..N are improvement generations
+  surfaceHash: string                // stable hash of the candidate surface (pivot key)
+  surface?: MutableSurface           // OMITTED to avoid PII when consumer prefers
+  cells: EvalRunCellScore[]
+  compositeMean: number
+  costUsd: number
+  durationMs: number
+}
+```
+
+## `EvalRunCellScore`
+
+```ts
+interface EvalRunCellScore {
+  scenarioId: string
+  rep: number                        // 0 for the default; > 0 when reps > 1
+  compositeMean: number              // composite across all judges + dimensions
+  dimensions: Record<                // outer key = judge name; inner = dimension name → score
+    string,
+    Record<string, number>
+  >
+  errorMessage?: string              // present when the dispatch threw
+}
+```
+
+## `TraceSpanEvent`
+
+```ts
+interface TraceSpanEvent {
+  // Standard OTel
+  traceId: string
+  spanId: string
+  parentSpanId?: string
+  name: string
+  startTimeUnixNano: number
+  endTimeUnixNano: number
+  attributes: Record<string, string | number | boolean>
+  events?: Array<{ timeUnixNano, name, attributes? }>
+  status?: { code: 'OK' | 'ERROR' | 'UNSET', message? }
+
+  // Tangle additions (all optional) for pivoting
+  'tangle.runId'?: string
+  'tangle.generation'?: number
+  'tangle.cellId'?: string
+  'tangle.scenarioId'?: string
+}
+```
+
+---
+
+## Server requirements
+
+Any orchestrator implementing this spec MUST:
+
+1. **Validate auth**: reject without `Authorization` header (401), with a
+   mismatched bearer token (401), or without a recognized `X-Tangle-Tenant-Id`
+   (404).
+2. **Validate wire version**: reject incompatible wire versions (400 with
+   a clear error message). The major component is the breaking-change axis.
+3. **Validate tenant isolation**: queries with `tenantId` X never return
+   data tagged with `tenantId` Y. Test this adversarially.
+4. **Honor idempotency**: when an `Idempotency-Key` matches a prior
+   request from the same tenant in the last 24h, return the same response
+   without double-processing.
+5. **Persist eval-runs durably**: at least the event + cell scores must
+   survive an orchestrator restart. Trace spans MAY be best-effort.
+6. **Provide read access**: GET endpoints for the tenant to list + fetch
+   their own runs. Wire format for reads is NOT part of this spec — each
+   orchestrator can pick its own (REST + JSON, gRPC, GraphQL).
+
+Servers SHOULD also:
+
+- Provide a webhook callback per tenant for `gate-decided` events.
+- Provide a billable-events emitter (Stripe meter / equivalent) per ingest
+  call so consumption can be metered.
+- Provide a dashboard or API to view + diff per-scenario lifts over time.
+
+---
+
+## Reference implementation
+
+`examples/hosted-ingest-server/` — a minimal hono-based receiver. ~200
+LOC. Validates auth, accepts ingest, stores in memory, exposes a
+read endpoint. Runs anywhere Node runs.
+
+```sh
+TENANT_KEY=dev-token TENANT_ID=acme pnpm tsx examples/hosted-ingest-server/server.ts
+```
+
+In another terminal:
+
+```sh
+HOSTED_ENDPOINT=http://localhost:8080 \
+HOSTED_TENANT_KEY=dev-token \
+HOSTED_TENANT_ID=acme \
+pnpm tsx examples/foreign-agent-quickstart/index.ts
+```
+
+The quickstart's eval-run gets POSTed to the reference receiver; the
+receiver's `GET /v1/runs` lists it back.
+
+---
+
+## Versioning
+
+`HostedWireVersion` is `"2026-05-26.v1"`.
+
+- Adding an optional field → no version change.
+- Adding a new endpoint or new event type → minor wire bump
+  (`2026-05-26.v2`).
+- Changing the shape of an existing field, removing a field, or
+  changing semantics of an existing field → major wire bump
+  (`2026-11-XX.v1`); a server may accept both versions during a
+  transition window.
+
+Servers MUST reject requests with `X-Tangle-Wire-Version` they don't
+support, with a 400 listing the versions they DO accept.
+
+The version string IS the spec id — pin against it.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.46.0",
+  "version": "0.48.0",
   "description": "Substrate for self-improving agents: traces, verifiable rewards, preferences, GEPA / reflective mutation, auto-research, replay, sequential anytime-valid stats, and release gates.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -119,6 +119,16 @@
       "import": "./dist/adapters/http.js",
       "default": "./dist/adapters/http.js"
     },
+    "./adapters/traceai": {
+      "types": "./dist/adapters/traceai.d.ts",
+      "import": "./dist/adapters/traceai.js",
+      "default": "./dist/adapters/traceai.js"
+    },
+    "./hosted": {
+      "types": "./dist/hosted/index.d.ts",
+      "import": "./dist/hosted/index.js",
+      "default": "./dist/hosted/index.js"
+    },
     "./openapi.json": {
       "default": "./dist/openapi.json"
     }
@@ -134,18 +144,6 @@
   "publishConfig": {
     "access": "public"
   },
-  "scripts": {
-    "build": "tsup && pnpm openapi",
-    "dev": "tsup --watch",
-    "prepare": "husky",
-    "prepublishOnly": "pnpm build",
-    "test": "vitest run",
-    "test:watch": "vitest",
-    "typecheck": "tsc --noEmit",
-    "lint": "biome check src",
-    "format": "biome format --write src",
-    "openapi": "node dist/cli.js openapi --out dist/openapi.json"
-  },
   "dependencies": {
     "@asteasolutions/zod-to-openapi": "^8.5.0",
     "@ax-llm/ax": "^19.0.25",
@@ -155,20 +153,15 @@
     "zod": "^4.3.6"
   },
   "peerDependencies": {
-    "@tangle-network/agent-runtime": ">=0.21.0 <0.26.0",
     "@tangle-network/sandbox": ">=0.2.1 <0.4.0"
   },
   "peerDependenciesMeta": {
-    "@tangle-network/agent-runtime": {
-      "optional": true
-    },
     "@tangle-network/sandbox": {
       "optional": true
     }
   },
   "devDependencies": {
     "@biomejs/biome": "^2.4.15",
-    "@tangle-network/agent-runtime": ">=0.21.0 <0.26.0",
     "@tangle-network/sandbox": "0.3.0",
     "@types/node": "^25.6.0",
     "husky": "^9.1.7",
@@ -178,17 +171,6 @@
     "typescript": "^5.7.0",
     "vitest": "^3.0.0"
   },
-  "pnpm": {
-    "minimumReleaseAge": 4320,
-    "minimumReleaseAgeExclude": [
-      "@tangle-network/sandbox",
-      "@tangle-network/agent-runtime"
-    ],
-    "overrides": {
-      "postcss@<8.5.10": "^8.5.10",
-      "ws@>=8.0.0 <8.20.1": "^8.20.1"
-    }
-  },
   "engines": {
     "node": ">=20"
   },
@@ -201,5 +183,14 @@
     ]
   },
   "license": "MIT",
-  "packageManager": "pnpm@10.22.0"
-}
+  "scripts": {
+    "build": "tsup && pnpm openapi",
+    "dev": "tsup --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit",
+    "lint": "biome check src",
+    "format": "biome format --write src",
+    "openapi": "node dist/cli.js openapi --out dist/openapi.json"
+  }
+}