@tangle-network/agent-eval 0.95.0 → 0.95.1

package/dist/hosted/index.d.ts

@@ -1,13 +1,242 @@
+import { M as MutableSurface, d as GateDecision } from '../types-DQRY8ZT-.js';
+import { I as InsightReport } from '../insight-report-BnRjTibG.js';
+import '../run-record-CP2ObebC.js';
+import '@tangle-network/agent-interface';
+import '../errors-CzMUYo7b.js';
+import '../schema-m0gsnbt3.js';
+import '../summary-report-CInXwsza.js';
+import '../failure-cluster-DH9Flgcf.js';
+import '../store-BcFXE6LG.js';
+import '../judge-calibration-0p2QcWNE.js';
+
 /**
- * # `@tangle-network/agent-eval/hosted`
+ * # Hosted-tier wire format — the schema that EVERY orchestrator (ours,
+ * a partner's self-hosted one, a future open implementation) must accept.
  *
- * Client substrate for shipping eval-run events + trace spans to a
- * hosted orchestrator. The wire format is the public contract — every
- * orchestrator (ours, partners' self-hosted ones, future open
- * implementations) speaks the same shape. Schema lives in
- * `docs/hosted-ingest-spec.md`, types live in `./types.ts`, code
- * lives in `./client.ts`.
+ * **Stability:** every type in this file is committed under semver. New
+ * minors only ADD optional fields. Breaking changes mean a major bump
+ * (`HostedWireVersion` literal increment).
+ *
+ * The wire format is two event streams in one transport:
+ *
+ *   1. **Eval-run events** (`POST /v1/ingest/eval-runs`). Posted when a
+ *      campaign / improvement-loop completes (or per-generation if
+ *      streaming). Carries the structured result + per-cell scores +
+ *      surface diffs the orchestrator stores for the dashboard.
+ *
+ *   2. **Trace spans** (`POST /v1/ingest/traces`). Standard OTLP-shaped
+ *      spans with a few additional attributes so the orchestrator can
+ *      pivot from eval-run → underlying execution. Compatible with any
+ *      OTel collector.
+ *
+ * Both endpoints are authenticated with a bearer token + a tenant id
+ * header. Tenants isolate everything downstream of ingest; no tenant
+ * ever sees another tenant's data.
+ */
+
+declare const HOSTED_WIRE_VERSION: "2026-05-26.v1";
+type HostedWireVersion = typeof HOSTED_WIRE_VERSION;
+/** Every ingest request carries these. */
+interface HostedIngestHeaders {
+    /** Bearer token. The orchestrator validates against the tenant key. */
+    authorization: `Bearer ${string}`;
+    /** Stable tenant id (the orchestrator-side primary key for the tenant). */
+    'x-tangle-tenant-id': string;
+    /** Wire-version pin so the server can reject incompatible payloads. */
+    'x-tangle-wire-version': HostedWireVersion;
+    /** Optional idempotency key for retry-safe ingest. */
+    'idempotency-key'?: string;
+}
+/** Lifecycle stages of an eval-run as the substrate reports them. */
+type EvalRunStatus = 'started' | 'baseline-complete' | 'generation-complete' | 'gate-decided' | 'finished' | 'errored';
+interface EvalRunCellScore {
+    /** Stable scenario id from the consumer's scenario set. */
+    scenarioId: string;
+    /** Repetition index when reps > 1; 0 for the default. */
+    rep: number;
+    /** Composite score across all judges + dimensions for this cell. */
+    compositeMean: number;
+    /** Per-judge → per-dimension scores; null where the judge did not run. */
+    dimensions: Record<string, Record<string, number>>;
+    /** Per-cell error message if the dispatch threw. Null on success. */
+    errorMessage?: string;
+}
+interface EvalRunGenerationSnapshot {
+    /** Generation index. 0 is baseline. */
+    index: number;
+    /** Candidate surface fingerprint (stable hash) — pivot key into the
+     *  trace stream to fetch the underlying execution. */
+    surfaceHash: string;
+    /** The candidate surface itself. May be omitted to avoid PII when the
+     *  consumer prefers not to ship verbatim prompts. */
+    surface?: MutableSurface;
+    /** Per-cell scores for this generation. */
+    cells: EvalRunCellScore[];
+    /** Aggregate composite mean across all cells in this generation. */
+    compositeMean: number;
+    /** Total $ spent across this generation. */
+    costUsd: number;
+    /** Wall-clock duration of this generation. */
+    durationMs: number;
+}
+/**
+ * The top-level eval-run event. One ingest call per logical eval-run;
+ * generations stream in incrementally via repeated calls with the same
+ * `runId`. The orchestrator deduplicates by `(runId, generation.index)`.
+ */
+interface EvalRunEvent {
+    /** Stable run id (the substrate's `runId`). UUID or substrate-generated. */
+    runId: string;
+    /** Where this run was happening — derived from `RunCampaignOptions.runDir`. */
+    runDir: string;
+    /** ISO-8601 timestamp the substrate recorded the event. */
+    timestamp: string;
+    /** Lifecycle stage this event represents. */
+    status: EvalRunStatus;
+    /** Free-form consumer tags (env, branch, model id, etc.). Searchable. */
+    labels: Record<string, string>;
+    /** Baseline campaign snapshot. Present when status >= baseline-complete. */
+    baseline?: EvalRunGenerationSnapshot;
+    /** Per-generation snapshots. Streams in; orchestrator appends. */
+    generations: EvalRunGenerationSnapshot[];
+    /** Final gate decision. Present when status >= gate-decided. */
+    gateDecision?: GateDecision;
+    /** Held-out lift = winner-on-holdout - baseline-on-holdout. */
+    holdoutLift?: number;
+    /** Total $ spent across baseline + every generation. */
+    totalCostUsd: number;
+    /** Total wall-clock duration. */
+    totalDurationMs: number;
+    /** Error message if status === 'errored'. */
+    errorMessage?: string;
+    /** Rigor packet emitted alongside the run — distributional summary,
+     *  paired-bootstrap lift CI, judge stats, inter-rater agreement,
+     *  contamination check, failure clusters (when an analyst is wired),
+     *  outcome correlation (when downstream signal is supplied), and the
+     *  recommendations the dashboard surfaces verbatim. Additive; older
+     *  clients that don't know about this field continue to work. */
+    insightReport?: InsightReport;
+}
+/**
+ * OTel-shape span with a few additional attributes for eval-run pivoting.
+ * Compatible with any OTLP collector — `name`, `traceId`, `spanId`,
+ * `startTimeUnixNano`, `endTimeUnixNano`, `attributes` are stock OTel.
+ */
+interface TraceSpanEvent {
+    traceId: string;
+    spanId: string;
+    parentSpanId?: string;
+    name: string;
+    startTimeUnixNano: number;
+    endTimeUnixNano: number;
+    attributes: Record<string, string | number | boolean>;
+    events?: Array<{
+        timeUnixNano: number;
+        name: string;
+        attributes?: Record<string, string | number | boolean>;
+    }>;
+    status?: {
+        code: 'OK' | 'ERROR' | 'UNSET';
+        message?: string;
+    };
+    /** Pivot back into the eval-run stream. */
+    'tangle.runId'?: string;
+    /** Pivot to the specific generation. */
+    'tangle.generation'?: number;
+    /** Pivot to the specific cell. */
+    'tangle.cellId'?: string;
+    /** Pivot to the specific scenario. */
+    'tangle.scenarioId'?: string;
+}
+interface IngestEvalRunsRequest {
+    wireVersion: HostedWireVersion;
+    events: EvalRunEvent[];
+}
+interface IngestTracesRequest {
+    wireVersion: HostedWireVersion;
+    spans: TraceSpanEvent[];
+}
+interface IngestResponse {
+    /** Accepted events / spans count. */
+    accepted: number;
+    /** Rejected events with reasons (validation failures, dup idempotency key, etc.). */
+    rejected: Array<{
+        index: number;
+        reason: string;
+    }>;
+}
+
+/**
+ * # Hosted-tier ingest client.
+ *
+ * Ships eval-run events + trace spans to any orchestrator (ours, a
+ * partner's self-hosted one, or a future open implementation) that
+ * speaks the wire format in `./types.ts`.
+ *
+ * Three modes:
+ *   - **Ours:** point at `https://orchestrator.tangle.tools` (the host root —
+ *     the client appends the versioned `/v1/ingest/...` path itself; a trailing
+ *     `/v1` on the endpoint is tolerated and normalized away). We handle ingest
+ *     + storage + dashboard.
+ *   - **Self-hosted:** point at whatever URL runs the reference receiver
+ *     from `examples/hosted-ingest-server/`.
+ *   - **Off (default):** when `hostedTenant` is unset, nothing is sent.
+ *     Everything stays local.
+ */
+
+interface HostedTenant {
+    /** Orchestrator endpoint base URL (no trailing slash). Required. */
+    endpoint: string;
+    /** Bearer token issued by the orchestrator. Required. */
+    apiKey: string;
+    /** Tenant id — the orchestrator's primary key for this consumer. Required. */
+    tenantId: string;
+    /** Optional `fetch` override (auth wrappers, custom agent, test mocks). */
+    fetchImpl?: typeof fetch;
+    /** Per-call timeout in ms. Default 30s. */
+    timeoutMs?: number;
+    /** Retries on 5xx / network errors. Default 2. */
+    retries?: number;
+}
+interface HostedClient {
+    ingestEvalRun(event: EvalRunEvent, idempotencyKey?: string): Promise<IngestResponse>;
+    ingestEvalRuns(events: EvalRunEvent[], idempotencyKey?: string): Promise<IngestResponse>;
+    ingestTraces(spans: TraceSpanEvent[], idempotencyKey?: string): Promise<IngestResponse>;
+    readonly tenant: HostedTenant;
+    readonly wireVersion: HostedWireVersion;
+}
+declare function createHostedClient(tenant: HostedTenant): HostedClient;
+/**
+ * Build a `HostedClient` from environment, or `undefined` when ingest is not
+ * configured — the canonical, fail-soft wiring every product uses so eval-run +
+ * trace provenance lands in the Intelligence dashboard with ONE call:
+ *
+ *   const hosted = hostedClientFromEnv()
+ *   // ...run the loop...
+ *   await emitLoopProvenance({ ..., hostedClient: hosted })  // no-op if undefined
+ *
+ * Returns `undefined` (NOT an error) when any of endpoint / apiKey / tenantId is
+ * missing — so a product wires the ship call unconditionally and it stays a
+ * no-op until the env is set. Env precedence:
+ *   - endpoint:  `TANGLE_INGEST_URL` → `TANGLE_ORCHESTRATOR_URL`
+ *   - apiKey:    `TANGLE_INGEST_API_KEY` → `TANGLE_API_KEY`
+ *   - tenantId:  `TANGLE_TENANT_ID`
+ * A trailing slash on the endpoint is stripped. Pass `overrides` to supply any
+ * field directly (e.g. a fixed `tenantId` per product) — overrides win over env.
+ */
+/**
+ * Build a {@link HostedTenant} config from env — the input `selfImprove`'s
+ * `hostedTenant` and `emitLoopProvenance` take. Same env precedence + overrides
+ * as {@link hostedClientFromEnv}; returns `undefined` (not an error) when any of
+ * endpoint / apiKey / tenantId is missing, so a product wires
+ * `hostedTenant: hostedTenantFromEnv({ tenantId: 'my-agent' })` unconditionally
+ * and it stays off until the env is set.
  */
-export { createHostedClient, type HostedClient, type HostedTenant, hostedClientFromEnv, hostedTenantFromEnv, } from './client';
-export { type EvalRunCellScore, type EvalRunEvent, type EvalRunGenerationSnapshot, type EvalRunStatus, HOSTED_WIRE_VERSION, type HostedIngestHeaders, type HostedWireVersion, type IngestEvalRunsRequest, type IngestResponse, type IngestTracesRequest, type TraceSpanEvent, } from './types';
-//# sourceMappingURL=index.d.ts.map
+declare function hostedTenantFromEnv(overrides?: Partial<HostedTenant> & {
+    env?: Record<string, string | undefined>;
+}): HostedTenant | undefined;
+declare function hostedClientFromEnv(overrides?: Partial<HostedTenant> & {
+    env?: Record<string, string | undefined>;
+}): HostedClient | undefined;
+
+export { type EvalRunCellScore, type EvalRunEvent, type EvalRunGenerationSnapshot, type EvalRunStatus, HOSTED_WIRE_VERSION, type HostedClient, type HostedIngestHeaders, type HostedTenant, type HostedWireVersion, type IngestEvalRunsRequest, type IngestResponse, type IngestTracesRequest, type TraceSpanEvent, createHostedClient, hostedClientFromEnv, hostedTenantFromEnv };

package/dist/index-_Y4oNOOb.d.ts

@@ -0,0 +1,159 @@
+import { a as RunSplitTag } from './run-record-CP2ObebC.js';
+
+/**
+ * Shared types for the reference benchmark wrappers under
+ * `src/benchmarks/`. Each wrapper exports the three functions in
+ * `BenchmarkAdapter` plus its own typed `DatasetItem` shape.
+ */
+
+interface BenchmarkDatasetItem<TPayload = unknown> {
+    /** Stable dataset-local item id (used for split assignment + paper
+     *  references). Unique within a benchmark. */
+    id: string;
+    /** Free-form payload. Each benchmark defines its own shape. */
+    payload: TPayload;
+}
+interface BenchmarkEvaluation {
+    /** [0, 1] score for the response on this item. Exact-match
+     *  benchmarks use 0/1; partial-credit benchmarks may return
+     *  fractional values. */
+    score: number;
+    /** Optional bag of raw scoring signals — e.g. parsed numeric
+     *  answer, regex match, judge sub-scores. */
+    raw: Record<string, unknown>;
+}
+/** Common signature implemented by every adapter under `src/benchmarks/*`. */
+interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown> {
+    /** Load the dataset for the given split. May hit the network on
+     *  first call but should be cache-friendly. Adapters that don't
+     *  ship the dataset itself MUST throw a clearly-marked error
+     *  pointing the caller at the loader script. */
+    loadDataset(split: RunSplitTag): Promise<BenchmarkDatasetItem<TPayload>[]>;
+    /** Score a single response. Pure with respect to the inputs. */
+    evaluate(item: BenchmarkDatasetItem<TPayload>, response: string): Promise<BenchmarkEvaluation>;
+    /** Deterministic split assignment via item id hashing. The
+     *  fraction of items in each split is implementation-defined but
+     *  MUST be stable across processes and platforms. */
+    assignSplit(itemId: string): RunSplitTag;
+}
+/** Split-assignment seed shared across all benchmarks. Bumping this
+ *  value reshuffles every split — do NOT do that lightly. */
+declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
+/**
+ * Assign an item id to one of `'search' | 'dev' | 'holdout'` using a
+ * stable 32-bit hash of `${seed}::${id}`. Default proportions:
+ *
+ *   search: 60%   (optimization-readable)
+ *   dev:    20%   (held-out for tuning, leak-on-purpose during dev)
+ *   holdout:20%   (paper-grade held-out, gated reads)
+ */
+declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
+
+/**
+ * Synthetic routing dataset. 16 tasks across 4 categories. Used as a
+ * deterministic, dependency-free benchmark for any router that maps a
+ * natural-language request to one of a fixed set of route labels.
+ *
+ * Format (see `routing/README.md` for prose):
+ *
+ *   {
+ *     id:           stable per-task ID (matches across processes).
+ *     category:     one of the four route labels.
+ *     prompt:       the user-facing request the router must classify.
+ *     route:        the ground-truth route the router should pick.
+ *     synonyms:     other strings that count as a correct answer.
+ *     hardNegatives:close-but-wrong route labels — used to detect the
+ *                   "always picks the popular route" failure mode.
+ *   }
+ *
+ * The four categories are intentionally cross-domain (file ops,
+ * math, search, conversation) so a router that collapses to one
+ * category is easy to spot.
+ */
+interface RoutingItem {
+    id: string;
+    category: 'file' | 'math' | 'search' | 'chat';
+    prompt: string;
+    /** Canonical correct route label. */
+    route: string;
+    /** Alternate route labels that also count as correct. */
+    synonyms: string[];
+    /** Wrong-but-tempting route labels (for analysis, not grading). */
+    hardNegatives: string[];
+}
+declare const ROUTING_DATASET: RoutingItem[];
+
+/**
+ * Routing benchmark — synthetic, dependency-free, ships in the
+ * package. 16 cross-category items in `dataset.ts`. See
+ * `routing/README.md` for the format.
+ *
+ * `evaluate` does case-insensitive exact match against the canonical
+ * route plus declared synonyms. The first valid route token in the
+ * response wins; everything else is ignored. Wrong answers also
+ * report whether they hit a hard negative — useful when triaging
+ * "always picks the popular route" failure modes.
+ */
+
+type RoutingPayload = RoutingItem;
+type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>;
+declare class RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {
+    loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]>;
+    evaluate(item: RoutingDatasetItem, response: string): Promise<BenchmarkEvaluation>;
+    assignSplit(itemId: string): RunSplitTag;
+}
+/**
+ * Pull route-shaped tokens out of a model response. Routes look like
+ * `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics
+ * are not routes, but `category.action` patterns are robust to most
+ * model wrappers (JSON output, prose explanations, code fences).
+ */
+declare function extractRouteTokens(response: string): string[];
+declare const loadDataset: (split: RunSplitTag) => Promise<RoutingDatasetItem[]>;
+declare const evaluate: (item: RoutingDatasetItem, response: string) => Promise<BenchmarkEvaluation>;
+declare const assignSplit: (itemId: string) => RunSplitTag;
+
+declare const index$1_ROUTING_DATASET: typeof ROUTING_DATASET;
+type index$1_RoutingAdapter = RoutingAdapter;
+declare const index$1_RoutingAdapter: typeof RoutingAdapter;
+type index$1_RoutingDatasetItem = RoutingDatasetItem;
+type index$1_RoutingItem = RoutingItem;
+type index$1_RoutingPayload = RoutingPayload;
+declare const index$1_assignSplit: typeof assignSplit;
+declare const index$1_evaluate: typeof evaluate;
+declare const index$1_extractRouteTokens: typeof extractRouteTokens;
+declare const index$1_loadDataset: typeof loadDataset;
+declare namespace index$1 {
+  export { index$1_ROUTING_DATASET as ROUTING_DATASET, index$1_RoutingAdapter as RoutingAdapter, type index$1_RoutingDatasetItem as RoutingDatasetItem, type index$1_RoutingItem as RoutingItem, type index$1_RoutingPayload as RoutingPayload, index$1_assignSplit as assignSplit, index$1_evaluate as evaluate, index$1_extractRouteTokens as extractRouteTokens, index$1_loadDataset as loadDataset };
+}
+
+/**
+ * Reference benchmark wrappers — entry point.
+ *
+ * Core surface (exported here):
+ *   - The `BenchmarkAdapter` contract.
+ *   - `deterministicSplit` + `BENCHMARK_SPLIT_SEED` for split assignment.
+ *   - `routing` — synthetic 16-task router benchmark. The only novel
+ *     benchmark we built; ships in the package.
+ *
+ * Example wrappers (under `examples/benchmarks/`, NOT in the bundle):
+ *   - `gsm8k`         — exact-match math reasoning (HF mirror, dataset
+ *                       not bundled).
+ *   - `swebench-lite` — 30-instance SWE-Bench subset via an external
+ *                       grader command.
+ *
+ * The example wrappers are reference implementations of `BenchmarkAdapter`.
+ * Read them, copy them, adapt them. They're intentionally not in the main
+ * entry — every team will configure them differently.
+ */
+
+declare const index_BENCHMARK_SPLIT_SEED: typeof BENCHMARK_SPLIT_SEED;
+type index_BenchmarkAdapter<_TItem = unknown, TPayload = unknown> = BenchmarkAdapter<_TItem, TPayload>;
+type index_BenchmarkDatasetItem<TPayload = unknown> = BenchmarkDatasetItem<TPayload>;
+type index_BenchmarkEvaluation = BenchmarkEvaluation;
+declare const index_deterministicSplit: typeof deterministicSplit;
+declare namespace index {
+  export { index_BENCHMARK_SPLIT_SEED as BENCHMARK_SPLIT_SEED, type index_BenchmarkAdapter as BenchmarkAdapter, type index_BenchmarkDatasetItem as BenchmarkDatasetItem, type index_BenchmarkEvaluation as BenchmarkEvaluation, index_deterministicSplit as deterministicSplit, index$1 as routing };
+}
+
+export { BENCHMARK_SPLIT_SEED as B, type BenchmarkAdapter as a, type BenchmarkDatasetItem as b, type BenchmarkEvaluation as c, deterministicSplit as d, index$1 as e, index as i };