@tangle-network/agent-eval 0.93.0 → 0.95.0

package/dist/sink-fetch-B1Yg4Til.d.ts

@@ -1,101 +0,0 @@
-/**
- * Fleet telemetry envelope — agent-eval's portable observability shape.
- *
- * Designed so any consumer (Node CLI, Cloudflare Worker, Lambda, browser
- * extension) can emit structured rows describing one unit of work — a page
- * audit, a tool call, an evolve round, a full agent run — to a central sink.
- *
- * The schema is intentionally a strict superset of agent-eval's `Run` shape
- * so a future TraceStore adapter can promote envelopes into traces without
- * translation.
- */
-declare const TELEMETRY_SCHEMA_VERSION = 1;
-/** Discriminator for the unit of work this envelope describes. */
-type TelemetryKind = 'agent-run' | 'design-audit-page' | 'design-audit-run' | 'design-evolve-round' | 'design-evolve-run' | 'gepa-trial' | 'gepa-generation' | 'tool-call' | 'judge-verdict' | 'custom';
-interface TelemetryEnvelope {
-    schemaVersion: typeof TELEMETRY_SCHEMA_VERSION;
-    envelopeId: string;
-    runId: string;
-    timestamp: string;
-    parentRunId?: string;
-    source: TelemetrySource;
-    model?: TelemetryModel;
-    kind: TelemetryKind;
-    ok: boolean;
-    durationMs: number;
-    data: Record<string, unknown>;
-    metrics: Record<string, number>;
-    tags?: Record<string, string>;
-    error?: string;
-}
-interface TelemetrySource {
-    /** Repo identity — basename of cwd plus git remote if discoverable. */
-    repo: string;
-    cwd: string;
-    gitSha?: string;
-    gitBranch?: string;
-    cliVersion: string;
-    /** What was invoked, e.g. `design-audit`, `bad run`, `gepa --target`. */
-    invocation: string;
-    /** Sanitised argv minus secrets. */
-    argv?: string[];
-    /**
-     * Multi-tenant identity. Set when the consumer runs inside a hosted
-     * product so a fleet rollup can group by tenant without leaking customer
-     * URLs or PII.
-     */
-    tenantId?: string;
-    /** Optional sub-tenant identity (project, suite, walkthrough, customer). */
-    customerId?: string;
-    /** SHA-256 (12 hex) of the API key used to authenticate this run, when applicable. */
-    apiKeyHash?: string;
-}
-interface TelemetryModel {
-    provider: string;
-    name: string;
-    /** SHA-256 (12 hex chars) of the prompt(s) used. */
-    promptHash?: string;
-    /** SHA-256 (12 hex chars) of the composed rubric body, if applicable. */
-    rubricHash?: string;
-}
-
-/**
- * Workers-safe telemetry sinks — only `fetch` and pure JS. No `fs`, no
- * `child_process`. Safe to import from a Cloudflare Worker, Lambda, edge
- * function, or browser extension.
- *
- * For Node-only file persistence, import from '@tangle-network/agent-eval/telemetry/file'.
- */
-
-interface TelemetrySink {
-    emit(envelope: TelemetryEnvelope): Promise<void> | void;
-    close?(): Promise<void> | void;
-}
-/** Best-effort POST to a remote collector. Fire-and-forget; never throws. */
-declare class HttpTelemetrySink implements TelemetrySink {
-    private readonly endpoint;
-    private readonly bearer?;
-    private inflight;
-    constructor(endpoint: string, bearer?: string | undefined);
-    emit(envelope: TelemetryEnvelope): void;
-    close(): Promise<void>;
-}
-/** Fanout to multiple sinks — failures in one do not affect others. */
-declare class FanoutTelemetrySink implements TelemetrySink {
-    private readonly sinks;
-    constructor(sinks: TelemetrySink[]);
-    emit(envelope: TelemetryEnvelope): void;
-    close(): Promise<void>;
-}
-/** No-op sink — used when telemetry is explicitly disabled. */
-declare class NullTelemetrySink implements TelemetrySink {
-    emit(): void;
-}
-/** In-memory sink — useful for tests + downstream adapters. */
-declare class InMemoryTelemetrySink implements TelemetrySink {
-    readonly envelopes: TelemetryEnvelope[];
-    emit(envelope: TelemetryEnvelope): void;
-    clear(): void;
-}
-
-export { FanoutTelemetrySink as F, HttpTelemetrySink as H, InMemoryTelemetrySink as I, NullTelemetrySink as N, type TelemetryKind as T, type TelemetryModel as a, type TelemetrySource as b, type TelemetrySink as c, TELEMETRY_SCHEMA_VERSION as d, type TelemetryEnvelope as e };

package/docs/design/external-agent-wedge.md

@@ -1,89 +0,0 @@
-# External-Agent Wedge — plug-in self-improvement for any agent
-
-**Status:** proposed · **Owner:** Drew · **Tracking:** ops-board #507 (epic) · **Updated:** 2026-05-26
-
-> One sentence: **expose the self-improvement engine our own fleet runs on as a drop-in library any agent builder can install — land free, bill the hosted intelligence.**
-
-This doc locks direction so the team executes without thrash. It is deliberately short. If a question isn't answered here, the default is "do the cheapest thing that keeps the LAND tier free and the EXPAND tier billable."
-
----
-
-## Why now
-
-Repeated signal from agent founders (latest: a high-quality GTM/social-marketing agent, sharp CTO): **no evals, no observability, hand-built, no way for the agent to improve itself.** This is the median serious agent team. They don't need another trace viewer — they need their agent to *get better on its own* and proof that it did.
-
-We already built that engine and hardened it the hard way: **6 of our own agents** (tax, legal, creative, gtm, agent-builder, physim) now run the *same* `runCampaign` / `runImprovementLoop` / `gepaDriver` / gate surface. It is `@tangle-network/agent-eval` — already on npm, runtime-agnostic, dogfooded in production. Exposing it externally is distribution of an asset we already own, not a new product to invent.
-
-## Positioning — self-improvement first, not observability
-
-If we sell "observability," we are a worse Langfuse/Braintrust/Arize and it's a margin race. Our wedge is the **closed self-improvement loop** (eval campaign → reflective `gepaDriver` prompt optimization → gated promotion → Bradley-Terry tournaments → predictive-validity calibration). Observability is the *byproduct*, not the pitch.
-
-**The pitch:** "Plug us in. Your agent runs a closed self-improvement loop against your own use case, gated so it never ships a regression — and you get the eval + trace observability for free as it does."
-
-### Backend-agnostic by design — the sandbox is swappable, not required
-
-The whole stack already runs **without our sandbox**, at two granularities:
-- **`agent-eval`** (the self-improvement engine) touches the sandbox only as a *type* — the LAND tier needs no sandbox.
-- **`agent-runtime`** is runtime-decoupled too: every sandbox import is `import type`, and the sandbox backend takes an *injected* instance. Its `Backend` seam already ships `createOpenAICompatibleBackend` (pure LLM) and `createIterableBackend` (bring-your-own) next to `createSandboxPromptBackend`. A builder runs the full runtime + self-improvement loop on **their own execution backend**, no Tangle sandbox installed.
-
-So adoption is *graduated*, and the builder picks the depth: (1) **trace-analysis only** over their existing runtime; (2) the **full self-improvement loop** on their own backend; (3) **our sandbox as a swap-in backend** for the batteries. Our sandbox becomes the best *option*, never the cost of entry. The only work to make this explicit: mark the `@tangle-network/sandbox` peer `optional` in agent-runtime + document the `Backend` interface as the swap seam (Phase A).
-
-## The surface — three tiers (land → expand → platform)
-
-| Tier | What they do | What they get | Billing |
-|---|---|---|---|
-| **LAND** (exists today) | `npm i @tangle-network/agent-eval`, wrap their agent behind one `dispatch` seam, bring a judge | Full self-improvement loop + **local** trace/eval artifacts. Any infra, no sandbox. | Free (lib) — **with optional Tangle Router as a $0-friction inference upsell.** When a builder points `OPENAI_BASE_URL` at `router.tangle.tools/v1`, every campaign call (agent + judge + reflective mutation) routes through us; we earn the routing margin. Same code, opt-in monetization vector that ships today. |
-| **EXPAND** (the build) | Route trace/eval/labeled-scenario data to our orchestrator | Hosted dashboards, cross-run intelligence, the capture flywheel as a service | **Metered** — composes with existing sandbox Stripe + cost-ledger |
-| **PLATFORM** (the carrot) | Move execution into our sandbox (agent-dev-container) | Substrate + orchestrator data/intelligence pre-wired, batteries included | Sandbox usage |
-
-The free lib casts the widest possible net at near-zero cost (it's already published). LAND is **not actually zero-revenue** — pointing the loop at Tangle Router is a one-line config change with no other code differences, so we monetize inference for any LAND-tier adopter who opts in. The wedge ladder is therefore four steps: no-revenue install → router routing margin (LAND with router) → metered data hosting (EXPAND) → sandbox usage (PLATFORM). Each step a one-line config change, never a rewrite. Value capture concentrates at EXPAND (hosting their data/intelligence is the biggest billable surface), but LAND-with-router is the immediate upsell available from day one.
-
-## Plan & gates — land-first, validate, then build
-
-The non-negotiable discipline: **do not build the hosted/billing tier before the free LAND is validated on a foreign agent.** Reality on someone else's real agent is cheaper than our imagination.
-
-- **Phase A — unblock (ungated, now):** upgrade agent-dev-container to the latest substrate; export `OutcomeStore`/`DeploymentOutcome` from `/rl`; mark agent-runtime's `@tangle-network/sandbox` peer `optional` + document the `Backend` swap seam (make sandbox-free adoption explicitly supported). Pure wins, correct regardless of the wedge.
-- **Phase B — design-partner LAND validation (forcing function):** wrap the founder's agent behind `dispatch`, author a marketing-quality judge, run one real campaign + `gepaDriver` loop. Instrument integration friction, judge cold-start, and actual quality lift.
-- **GATE — go/no-go + pricing:** decided from Phase B evidence, not theory.
-- **Phase C — LAND ergonomics:** external 15-minute quickstart + a stable `dispatch`/judge/scenario contract + ≥1 reference framework adapter.
-- **Phase D — EXPAND (gated):** hosted OTLP/eval-run HTTP sink (client in agent-eval) + multi-tenant orchestrator ingest + metered billing + minimal dashboard (server in `@tangle-network/monorepo`).
-
-## Success metrics
-
-- **Phase B:** ≥1 measurable quality lift on the partner's own use case; integration ≤ a 1–2 day pairing.
-- **LAND:** time-to-first-self-improvement-loop for a new external agent < 1 day from the quickstart.
-- **EXPAND:** first external tenant routed + first metered dollar.
-
-## Risk register
-
-**Knowns (high confidence)**
-- The substrate is a portable lib; it wraps our 6 agents behind `dispatch`/judge/gate in production.
-- It's runtime-agnostic (FS + in-memory storage, Node + edge) and emits OTLP traces.
-- **agent-runtime is already sandbox-decoupled at runtime** — sandbox is type-only + dependency-injected; the `Backend` seam (`createOpenAICompatibleBackend` / `createIterableBackend` / `createSandboxPromptBackend`) makes the sandbox one swappable option. Only the peer-dep label needs to change.
-- The orchestrator/observability/billing platform lives in `@tangle-network/monorepo`.
-
-**Known-unknowns (Phase B answers these)**
-- Does `dispatch` wrap a *foreign* agent as cleanly as ours, or are there hidden Tangle assumptions?
-- Judge cold-start: can a team with no prior evals author a usable judge for a subjective domain (marketing quality)?
-- Data cold-start: no evals = no scenarios = no labels; how do they bootstrap the flywheel day 1?
-- Orchestrator multi-tenancy: it's only run internally — external auth/isolation/privacy is unbuilt.
-- Pricing line + OSS boundary (the lib is already public).
-
-**Unknown-unknowns (instrument, don't predict)**
-- Integration-surface explosion → keep a tiny stable contract + reference adapters; refuse to special-case.
-- Foreign-domain eval semantics we've never seen → the design partner is the discovery mechanism.
-- Multi-tenant orchestrator failure modes at external/adversarial scale.
-- Supply-chain/trust — we enter their dependency tree; our security posture becomes their procurement question.
-- *Mitigation for all:* land-first-with-a-partner surfaces surprises cheaply, on a real agent, before any hosted spend. Timebox Phase D behind the gate.
-
-## Non-goals (anti-thrash)
-
-- **Not** a standalone observability product. Observability ships only as a byproduct of self-improvement.
-- **Not** building the hosted/billing tier before Phase B validates the lib.
-- **Not** per-framework bespoke integrations — one stable contract + a couple of reference adapters.
-- **Not** re-architecting the substrate for hypothetical external needs — extend at the `dispatch`/judge/store seams only.
-- **Not** changing the sandbox/products roadmap — this *exposes the same engine*, it doesn't fork it.
-
-## Tracking
-
-ops-board epic **#507** + children: agent-dev-container stack upgrade (eng), foreign-agent adoption surface (eng), design-partner LAND validation (gtm), hosted orchestrator routing + billing (eng, blocked on the gate), GTM-fit + pricing decision (gtm).

package/docs/design/phase-d-rfc.md

@@ -1,125 +0,0 @@
-# Phase D RFC — hosted-tier substrate
-
-Pinned scope decisions for the EXPAND tier. What we built, what we
-deliberately did NOT, and what's gated on Phase B evidence.
-
----
-
-## What's in this version
-
-**Wire-format substrate (shipped):**
-
-1. `@tangle-network/agent-eval/hosted` — public client + types for shipping
-   eval-run events + trace spans to any orchestrator that speaks the wire
-   format.
-2. `docs/hosted-ingest-spec.md` — semver-committed wire spec
-   (`HostedWireVersion = "2026-05-26.v1"`).
-3. `examples/hosted-ingest-server/` — minimal hono-based reference
-   receiver (~200 LOC). Executable spec. Stays as the reference even
-   after the production orchestrator ships.
-4. `selfImprove({ hostedTenant })` opt-in — when set, the substrate
-   POSTs the final eval-run event to the configured endpoint. Failures
-   are logged but never fail the loop (LAND tier never blocks on
-   EXPAND-tier infra).
-
-**Production orchestrator (started):**
-
-5. HTTP ingest service in `@tangle-network/monorepo` accepting the wire
-   format. Lives under the orchestrator app. Tenant auth + isolation
-   + persistent storage + read endpoints. *Started this session — see
-   the @tangle-network/agent-dev-container PR. Not feature-complete:
-   tenant CRUD + adversarial isolation tests pending.*
-
-## What's deliberately deferred
-
-The wedge doc gates these on Phase B evidence — partner-validated
-signal about what the hosted product actually needs to do. Shipping
-them without that signal risks building the wrong thing.
-
-| Deferred until Phase B passes | Why |
-|---|---|
-| **Metered billing wire-up (Stripe + cost-ledger)** | The billable units (per-eval-run, per-ingested-MB, per-seat) depend on actual partner consumption patterns. Picking dimensions in a vacuum locks us into wrong pricing. |
-| **Multi-tenant dashboard UX** | Partners' first dashboard request defines the right default views. We have a stub list-runs page; the rest is post-signal. |
-| **Webhook callbacks per tenant** | The events partners want pushed (gate-decided, cost-threshold, regression-alert) are partner-shaped. Add them when a partner asks. |
-| **Cross-tenant aggregation / benchmarking** | This is the "Datadog for agents" tier — explicit roadmap, requires user volume we don't have. |
-| **Sandbox-cost roll-up into hosted billing** | Cross-product billing integration requires PLATFORM-tier partners. Out of scope until at least one. |
-| **Trace UI** | OTel-shape spans store fine. Visualization comes after partners ask. Phoenix / Jaeger / any OTLP-compatible viewer covers it in the interim. |
-| **Soc2 / compliance audit work** | Required for enterprise; not required for design partners. |
-
-## Architecture decisions locked
-
-These are committed and won't change without a major-version wire bump
-or a documented migration:
-
-1. **Wire format is JSON over HTTP**, not gRPC. Reasons: works in
-   browsers + edge + node + curl; OTel-compatible at the trace stream
-   level; lowest possible barrier to a self-hosted orchestrator.
-2. **Tenant auth is bearer-token + tenant-id header**, not OIDC /
-   service-account / mutual-TLS. Reasons: simplest thing that's
-   actually secure with proper key handling; defers complex IAM until
-   enterprise demand.
-3. **Idempotency via header, not transactional API**. Servers MUST
-   dedupe by `(tenantId, Idempotency-Key)` for 24h. Simpler than
-   making clients commit transactions.
-4. **Eval-runs and traces are SEPARATE streams** with pivot keys
-   (`tangle.runId` etc.) on spans. Reasons: traces can be best-effort
-   (lossy) without corrupting eval-run semantics; orchestrators can
-   prioritize eval-run durability without forcing trace durability.
-5. **Wire version is a date.v-N string**, not semver. Reasons: dates
-   communicate "when was this contract frozen"; v-N captures
-   incremental breaking changes between dates.
-
-## Open questions for Phase B to answer
-
-When the design-partner pairing happens, capture answers to these
-explicitly:
-
-1. **Surface confidentiality**: do partners want the verbatim surface
-   (system prompt) shipped, or just the hash? Today the wire format
-   has `surface?` as optional; partner default is what we ship.
-2. **Trace sampling**: at what cells-per-second do trace spans become
-   noise? What's the right default sampling rate?
-3. **Cost attribution granularity**: per cell? per generation? per
-   run? Per judge dimension? Partner needs determine what we surface
-   in billing reports.
-4. **Replay**: do partners want to re-run an old eval-run from the
-   stored data? That would require us to store more than the summary —
-   actual artifacts + prompts. Storage cost implication.
-5. **PII / sensitive scenarios**: how do partners want to handle
-   scenarios containing user data? Encryption-at-rest is table stakes;
-   redaction-at-ingest may be required for some.
-
-The partner pairing kit (`docs/phase-b-pairing-kit.md`) has discovery
-questions that probe these.
-
-## Non-goals (explicit)
-
-This RFC does NOT plan for:
-
-- Replacing Langfuse / Phoenix / Arize. We INGEST OTel; we don't
-  build a generic trace viewer. The dashboard is eval-run-shaped, not
-  trace-shaped.
-- Becoming a model gateway. Tangle Router exists; the hosted
-  orchestrator routes to Tangle Router by default but doesn't
-  duplicate its function.
-- Becoming an LLM-call CDN. Caching is the consumer's job (their
-  agent code, their HTTP client). We don't intercept LLM calls.
-- Building an "agents IDE." Substrate, not surface.
-
-## Migration path (post Phase B)
-
-When Phase B passes the gate, the production orchestrator finishes:
-
-1. Replace in-memory store with Postgres (tenant data) + S3 (large
-   artifacts) OR Cloudflare D1 + R2 (Workers-native).
-2. Wire metered events to Stripe + the cost-ledger.
-3. Tenant CRUD UI + onboarding flow.
-4. Multi-tenant dashboard MVP (list runs, drill into one, diff
-   generations, view shipped prompt).
-5. Adversarial tenant-isolation test battery in CI.
-6. Webhooks + observability for the orchestrator itself.
-
-Estimated effort post-Phase-B: ~1 week focused work for one engineer.
-This is fast precisely BECAUSE the wire format is locked and the
-reference receiver exists — the production server is a different
-implementation of the same contract.

package/docs/design/phase4-consumer-migration.md

@@ -1,70 +0,0 @@
-# Phase 4 — consumer migration tracking
-
-Migrate the product repos off their duplicated eval / prompt-evolution
-orchestration onto the published substrate (`@tangle-network/agent-eval@^0.40.3`
-+ `@tangle-network/agent-runtime@^0.25.0`). Integration contract:
-[`primitives-integration-spec.md`](./primitives-integration-spec.md).
-
-**Strategy:** prove **gtm end-to-end first** (the canonical consumer), then fan
-the proven migration pattern to the rest via parallel subagents, each briefed
-with the gtm reference diff + the spec's forbidden-anti-patterns list. Each
-migration is its own reviewable, rollback-able PR.
-
-## Status board
-
-| Repo | Deletable orchestration (LOC est.) | Dispatch seam | Status | PR |
-|---|---|---|---|---|
-| gtm-agent | ~2,420 | `runChatThroughRuntime` | **IN PROGRESS** | — |
-| legal-agent | tbd | tbd | queued | — |
-| tax-agent | tbd | tbd | queued | — |
-| creative-agent | tbd | tbd | queued | — |
-| agent-builder | tbd | tbd | queued | — |
-| blueprint-agent | tbd | tbd | queued (Drew dispatching via spec) | — |
-| physim | tbd (MultiLayerVerifier adapter) | tbd | queued | — |
-
-## Per-repo migration checklist
-
-For each repo, in order:
-
-- [ ] **Survey** — inventory eval + prompt-evolution wrappers (file:line + LOC).
-      Identify the dispatch seam, scenarios, judges, mutation strategy.
-- [ ] **Bump deps** — `@tangle-network/agent-eval` → `^0.40.0`,
-      `@tangle-network/agent-runtime` → `^0.25.0`; `pnpm update`; baseline
-      typecheck green.
-- [ ] **Rewire seams** — `dispatch`/`dispatchWithSurface`, `judges`,
-      `scenarios` extracted from the existing wrappers (KEEP domain logic).
-- [ ] **Replace orchestration** — swap the local generation/population/scorecard
-      loop for `runImprovementLoop` (or `runCampaign` for eval-only). DELETE the
-      wrapper body.
-- [ ] **Gate** — compose domain gates with `defaultProductionGate`.
-- [ ] **Dataset** — wire `FsLabeledScenarioStore` with correct `captureSource`.
-- [ ] **Tests** — port wrapper contract tests to assert the substrate wiring;
-      keep judge/scenario tests. Suite green.
-- [ ] **Prove** — one real eval/improve run end-to-end; confirm scorecard +
-      (if applicable) a PR opens on a shipping gate.
-- [ ] **Anti-pattern sweep** — no silent fallbacks, no reimplemented loop, no
-      train/holdout conflation, tracing on, dispatch named.
-- [ ] **PR** — open, independent-review, merge.
-
-## gtm-agent — migration map (from survey)
-
-- **Branch base:** off the repo's working branch (`feat/gtm-rich-chat-actions`)
-  or main — confirm before starting.
-- **Dispatch seam:** `runChatThroughRuntime(ctx)`
-  (`src/lib/.server/agent-runtime/chat.ts`) — prompt variant + scenario → real
-  agent run → artifact + events + token usage.
-- **Scenarios:** `src/lib/.server/production-loop/scenarios.ts` (3 holdout) +
-  `eval/business-owner/personas.json` (canonical personas).
-- **Judges:** `src/lib/.server/production-loop/judges.ts` (`runEnsembleJudge`,
-  3-model ensemble) + canonical 12-dimension judges in `eval/canonical.ts`.
-- **Delete (~2,420 LOC orchestration):** the generation/population/reps loop in
-  `src/lib/.server/production-loop/index.ts` (~450), the checkpoint loop in
-  `eval/canonical.ts` (~600), `eval/run-prompt-evolution.ts` wrapper (~800),
-  `eval/analyst-loop.ts` wrapper (~300), `eval/optimization-campaign.ts` (~170),
-  `scripts/evals/run-optimization-campaign.ts` (~100 scaffold).
-- **Rewire:** `buildHoldoutRunner` → `dispatchWithSurface`; `buildScorer` →
-  `judges`; `buildMutator` → `evolutionaryDriver({ mutator })`;
-  `runProductionLoop` → `runImprovementLoop`.
-- **Keep:** judges, scenarios, persona data + reactive driver, deterministic
-  anti-slop/brief checks, GitHub PR wiring, feedback/trace ingestion.
-- **Net:** ~1,400–1,600 LOC reduction.