@tangle-network/agent-eval 0.129.0 → 0.130.0

package/dist/hosted/index.d.ts

@@ -1,837 +1,16 @@
-import { z } from 'zod';
-
-type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
-
-/**
- * Paper-grade RunRecord schema + runtime validator.
- *
- * Every run that participates in a promotion gate, paper table, or
- * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
- * fields are exactly those the paper "Two Loops, Three Roles" requires
- * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
- * holdout split tag. A task score is optional because execution-only records
- * must preserve missing labels instead of converting errors into zero quality.
- *
- * This is intentionally NOT a replacement for the rich `Run` /
- * `ProposeReviewReport` / `ScenarioResult` types already in the
- * package. Those are runtime structures with full provenance. A
- * `RunRecord` is the analysis-time projection — the JSON-friendly
- * row you'd put in a parquet file or paste into a notebook.
- *
- * Validate at the boundary:
- *
- *   const rec = validateRunRecord(rawJson)         // throws on missing
- *   const ok  = isRunRecord(rawJson)               // boolean check
- *   const rec = parseRunRecordSafe(rawJson)        // { ok, value | error }
- *
- * The validator runs in pure TS — zod is intentionally NOT a
- * dependency. Round-trip tested in `tests/run-record.test.ts`.
- */
-
-/**
- * Explicit execution-lifecycle result for a run.
- *
- * This is separate from task quality (`outcome`) and failure classification.
- * Producers set it only from root-run or process evidence.
- */
-type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
-
-/**
- * Pass A substrate types — `runCampaign` is the one primitive every
- * eval flow composes from. Three contracts in this file:
- *
- *   - `Scenario`            input set
- *   - `DispatchFn`          how to run one scenario → artifact
- *   - `CampaignResult`      defined output schema (the contract downstream tools depend on)
- *
- * Three more lifted from earlier substrate work (re-exported):
- *
- *   - `JudgeConfig`         pluggable dimensional scorer (0.38)
- *   - `Mutator`             optimization-loop surface mutator
- *   - `Gate`                promotion gate (`HeldOutGate` and friends adapt to this)
- *
- * No new architecture vs 0.38 — Pass A formalizes the shapes so consumers
- * can build dashboards / CI gates / regression diffs against a stable schema.
- */
-
-/** A tier-4 code surface — a finalized candidate change to the agent's
- *  IMPLEMENTATION, not its prompt. Produced by autoresearch (reads codebase +
- *  trace findings → opens a worktree). `worktreeRef` locates the candidate;
- *  the exact commits, tree, and binary-patch digest identify it. See the
- *  improvement-tier table in `docs/design/loop-taxonomy.md`. */
-interface CodeSurface {
-    readonly kind: 'code';
-    /** Worktree path or git ref holding the candidate code change. This is a
-     *  mutable locator and is deliberately excluded from content hashes. */
-    readonly worktreeRef: string;
-    /** Human-readable ref the worktree was forked from. Not identity-bearing. */
-    readonly baseRef: string;
-    /** Exact commit the candidate was forked from. */
-    readonly baseCommit: string;
-    /** Exact tree object for `baseCommit`. */
-    readonly baseTree: string;
-    /** Exact finalized candidate commit. */
-    readonly candidateCommit: string;
-    /** Exact tree object for `candidateCommit`. */
-    readonly candidateTree: string;
-    /** Identity of the exact patch artifact. The deployable candidate bundle
-     *  carries the same descriptor plus its base64-encoded content. */
-    readonly patch: {
-        readonly format: 'git-diff-binary';
-        readonly sha256: `sha256:${string}`;
-        readonly byteLength: number;
-    };
-    /** Human summary of what changed — rendered into the auto-PR body. */
-    readonly summary?: string;
-}
-/** Named text components optimized together as one candidate. */
-interface ComponentSurface {
-    readonly kind: 'components';
-    readonly components: Readonly<Record<string, string>>;
-}
-/** The mutable surface a proposer changes. Tiers (see
- *  `docs/design/loop-taxonomy.md`):
- *   - `string`      — tiers 1-2: system-prompt addendum / serialized tool
- *                     config. Cheap, reversible, text-diffable.
- *   - `ComponentSurface` — named prompts, policies, and tool descriptions
- *                          optimized together.
- *   - `CodeSurface` — tier 4: an implementation change behind a worktree ref.
- *  Tier 3 (knowledge) is owned by agent-knowledge and rides its own adapter,
- *  not this type. */
-type MutableSurface = string | ComponentSurface | CodeSurface;
-/** Five-valued verdict taxonomy (MOSS-paper alignment). */
-type GateDecision = 'ship' | 'hold' | 'need_more_work' | 'model_ceiling' | 'arch_ceiling';
-
-/**
- * Reporting helpers — production summaries and paper-quality figures — sit alongside `reporter.ts` rather
- * than replacing it.
- *
- * Three artefacts:
- *
- *   - `summaryTable`           Markdown table of per-candidate means,
- *                            95% bootstrap CIs, BH-adjusted Wilcoxon
- *                            p-values, and Cohen's d versus a
- *                            comparator candidate.
- *   - `paretoChart`         Abstract spec for a cost vs quality
- *                            scatter, with gate decisions overlaid.
- *                            Returns numbers + labels — caller
- *                            chooses the plotting library.
- *   - `gainHistogram`
- *                            Per-item paired holdout deltas as a
- *                            histogram spec (bins + counts + median +
- *                            CI). Same "data, not images" contract.
- *
- * The figure types are PlotSpecs — JSON-friendly, library-agnostic.
- * They aren't React components and they aren't PNGs; they are
- * what you'd hand to vega-lite, plotly, matplotlib, or your own
- * Canvas renderer to draw the actual figure.
- */
-
-interface ParetoPoint {
-    candidateId: string;
-    /** Mean USD cost per run on the chosen split. */
-    cost: number;
-    /** Mean score on the chosen split. */
-    quality: number;
-    /** Number of runs that informed this point. */
-    n: number;
-    /** Whether this candidate is on the Pareto frontier — high
-     *  quality, low cost, no dominator. */
-    onFrontier: boolean;
-    /** Optional gate verdict for this candidate, if a `GateDecision`
-     *  for it was passed in. */
-    gate?: 'promote' | 'reject';
-}
-interface ParetoFigureSpec {
-    kind: 'pareto-cost-quality';
-    split: 'search' | 'holdout';
-    points: ParetoPoint[];
-    axes: {
-        x: 'costUsd';
-        y: 'score';
-    };
-}
-interface GainDistributionBin {
-    /** Inclusive lower edge. */
-    lo: number;
-    /** Exclusive upper edge (or inclusive if it's the last bin). */
-    hi: number;
-    /** Number of pairs whose delta lands in this bin. */
-    count: number;
-}
-
-interface ContinuousAgreement {
-    /** Cohen's κ_w with quadratic weights, computed on raw [0,1] scores. */
-    weightedKappa: number;
-    /** ICC(2,1): two-way random effects, absolute agreement, single rater. */
-    icc: number;
-    /** Pearson product-moment correlation (averaged over rater pairs if N>2). */
-    pearson: number;
-    /** Spearman rank correlation (averaged over rater pairs if N>2). */
-    spearman: number;
-    /** 95% bootstrap percentile CIs over items. */
-    ci: {
-        icc: [number, number];
-        weightedKappa: [number, number];
-    };
-    /** Number of complete items (no NaN across raters). */
-    n: number;
-    /** Number of raters. */
-    raters: number;
-}
-
-/**
- * # InsightReport — the rigorous decision packet for any set of agent runs.
- *
- * Returned by `analyzeRuns()` and embedded in `SelfImproveResult.insight` +
- * the hosted-tier `EvalRunEvent.insightReport`. One shape across two surfaces:
- *
- *   - **Customer who has a closed loop** (`selfImprove`): the report ships
- *     with the loop output. Their dashboard renders ship/hold + lift CI +
- *     calibration + cluster + Pareto in one packet.
- *   - **Customer who has observed runs but no loop** (`analyzeRuns` directly):
- *     same packet from a `RunRecord[]` they already have — production traces,
- *     approve/reject corpus, CSV gold set.
- *
- * Every field is optional except the distributional summary — fields are
- * populated when the input data supports them:
- *
- *   - `lift` requires both baseline and candidate splits to be present.
- *   - `interRater` requires multi-rater feedback (≥2 raters per run).
- *   - `judges` populates per-judge stats only when the run records carry
- *     `outcome.judgeScores`.
- *   - `failureClusters` requires the optional `analystRegistry` to be wired.
- *   - `contamination` requires canary scenarios to be passed in.
- *   - `outcomeCorrelation` requires a downstream outcome signal.
- *   - `sequential` requires the run set to be ordered (treats them as a
- *     stream and emits an anytime-valid interim decision).
- *
- * Consumers read the `recommendations` array first — that's the
- * actionable layer, ranked by priority. The numeric sections back it up.
- */
-
-interface InsightReport {
-    /** Number of runs analyzed. */
-    n: number;
-    /** Runtime facts carried by the run records. These describe execution,
-     *  not task quality: duration, queueing, token categories, models,
-     *  execution errors, and terminal outcomes. */
-    execution: ExecutionInsight;
-    /** Composite-score distribution across all runs. Always present. */
-    composite: ScalarDistribution;
-    /** Per-dimension distributions for every dimension that appeared in any
-     *  run's judge scores. Empty when no judge scores were recorded. */
-    perDimension: Record<string, ScalarDistribution>;
-    /** Cost/quality distribution and Pareto frontier. */
-    costQuality: {
-        cost: ScalarDistribution;
-        pareto: ParetoFigureSpec;
-        /** Cost source coverage. `uncaptured` rows are excluded from the USD
-         *  distribution and Pareto chart; observed and estimated totals remain
-         *  separate so reports never present estimates as billed spend. */
-        provenance?: CostProvenanceSummary;
-        /** Set when the cost/quality view is degraded because the input data
-         *  doesn't fully support it — e.g. all `costUsd` were zero, or only a
-         *  single candidate appears (so the Pareto is a single point). The
-         *  named fields name the degraded sub-view, free-text the reason. */
-        degraded?: {
-            cost?: string;
-            pareto?: string;
-        };
-    };
-    /** Per-judge calibration + bias detection. Populated for every judge name
-     *  that appears in `outcome.judgeScores`. Bias fields require either a
-     *  gold reference or multi-rater data. */
-    judges: Record<string, JudgeInsight>;
-    /** Inter-rater agreement when multiple judges scored the same runs.
-     *  Includes pairwise kappa and the specific run ids where raters
-     *  disagree — the cases worth a human meeting. */
-    interRater?: InterRaterInsight;
-    /** Pairwise lift (baseline → candidate) with bootstrap CI. Present when
-     *  `RunRecord.splitTag` includes both `holdout` and search/dev splits,
-     *  or when caller passes an explicit baseline/candidate split. */
-    lift?: LiftInsight;
-    /** Failure clusters with exemplars. Populated when an AnalystRegistry
-     *  is wired in `analyzeRuns({ analyst })`. */
-    failureClusters?: FailureClusterInsight;
-    /** Canary leak count + holdout audit status. Populated when canary
-     *  scenarios are passed in. */
-    contamination?: ContaminationInsight;
-    /** Correlation between judge composite and a downstream outcome the
-     *  caller supplies (engagement, revenue, downstream pass rate, etc.).
-     *  When present, the optional reward model is the model that maps
-     *  judge scores → predicted outcome. */
-    outcomeCorrelation?: OutcomeCorrelationInsight;
-    /** Aggregate release-readiness summary. A consumer needing the full
-     *  substrate `ReleaseConfidenceScorecard` (SLO-axis evaluation,
-     *  ActionableSideInfo bag) calls `evaluateReleaseConfidence()` directly;
-     *  this summary captures the analyzeRuns-derived axes. */
-    release: ReleaseSummary;
-    /** Delta vs a prior period when `baselineRuns` is passed. Per-metric
-     *  current vs baseline with Welch CI + Cohen's d + significance flag.
-     *  Answers "did my last change help?" — the customer-conversion question.
-     *  Surfaced metrics: composite, cost, duration, tokenUsage, plus any
-     *  per-dimension judge metric present in both windows. */
-    priorPeriodComparison?: PriorPeriodComparison;
-    /** Model-free task-failure breakdown from `RunRecord.failureClass`, ranked
-     *  by count descending. Domain-specific `failureMode` detail is retained on
-     *  each record but never creates a second aggregation vocabulary. */
-    failureClasses?: FailureClassTally[];
-    /** Top-N actionable recommendations, ranked by priority. The packet's
-     *  human-readable layer; the numeric sections are the evidence. */
-    recommendations: Recommendation[];
-}
-interface CostProvenanceSummary {
-    observed: {
-        n: number;
-        totalUsd: number;
-    };
-    estimated: {
-        n: number;
-        totalUsd: number;
-    };
-    uncaptured: {
-        n: number;
-    };
-    knownFraction: number;
-}
-interface ExecutionInsight {
-    /** End-to-end wall time for every run. */
-    durationMs: ScalarDistribution;
-    /** Queue time for the subset of runs that recorded it. */
-    queueMs: ScalarDistribution;
-    /** Token distributions plus corpus totals. Optional token categories use
-     *  distribution `n` to disclose how many runs recorded that category. */
-    tokenUsage: TokenUsageInsight;
-    /** Usage reported only by orchestration or agent aggregate spans.
-     *  Kept separate because it may duplicate model-call telemetry in other traces. */
-    aggregateUsage: {
-        runs: number;
-        tokenUsage: TokenUsageInsight;
-        costUsd: ScalarDistribution;
-        totalCostUsd: number;
-    };
-    /** Stable model counts, largest cohort first. */
-    models: Array<{
-        model: string;
-        runs: number;
-    }>;
-    /** Model-call coverage. `events` is available only from producers that
-     *  record `outcome.raw.llm_span_count`; `runs` also recognizes non-zero
-     *  token usage from other producers. */
-    modelCalls: {
-        runs: number;
-        events: number;
-        reportingRuns: number;
-    };
-    /** Runs with explicit execution-error telemetry. This is independent of
-     *  whether the root run ultimately succeeded, failed, or has no terminal
-     *  evidence. */
-    executionErrors: {
-        runs: number;
-        /** Share among runs that supplied an execution-error count.
-         *  `null` when no run supplied error telemetry. */
-        fraction: number | null;
-        /** Execution-error events reported through the canonical count. */
-        events: number;
-        /** Runs that supplied an execution-error count, including explicit zeroes. */
-        reportingRuns: number;
-        /** Exact sum of `outcome.raw.error_span_count`, kept separate from other errors. */
-        errorSpanEvents: number;
-        /** Runs that supplied `outcome.raw.error_span_count`, including explicit zeroes. */
-        errorSpanReportingRuns: number;
-        /**
-         * Error-telemetry coverage crossed with independently reported terminal
-         * outcomes. `unreported` is distinct from a reported zero.
-         */
-        byTerminalOutcome: Record<RunTerminalOutcome, ExecutionErrorOutcomeCell>;
-    };
-    /** Root-run or process outcomes. Missing `RunRecord.terminalOutcome` values
-     *  count as `unknown`; child-span status never changes these counts. */
-    terminalOutcomes: {
-        succeeded: number;
-        failed: number;
-        cancelled: number;
-        incomplete: number;
-        unknown: number;
-    };
-}
-interface ExecutionErrorOutcomeCell {
-    /** Runs that explicitly reported one or more execution errors. */
-    withErrors: number;
-    /** Runs that explicitly reported zero execution errors. */
-    withoutErrors: number;
-    /** Runs with no execution-error count from the producer. */
-    unreported: number;
-}
-interface TokenUsageInsight {
-    input: ScalarDistribution;
-    output: ScalarDistribution;
-    reasoning: ScalarDistribution;
-    cached: ScalarDistribution;
-    cacheWrite: ScalarDistribution;
-    totals: {
-        input: number;
-        output: number;
-        reasoning: number;
-        cached: number;
-        cacheWrite: number;
-    };
-}
-/** Distributional summary of a scalar-valued metric. */
-interface ScalarDistribution {
-    /** Sample count after dropping non-finite values. */
-    n: number;
-    /** Null when `n` is zero. */
-    mean: number | null;
-    /** Null when `n` is zero. */
-    p50: number | null;
-    /** Null when `n` is zero. */
-    p95: number | null;
-    /** Null when `n` is zero. */
-    stddev: number | null;
-    /** Null when `n` is zero. */
-    min: number | null;
-    /** Null when `n` is zero. */
-    max: number | null;
-    /** Histogram bins using `agent-eval`'s `gainHistogram` primitive. */
-    histogram: GainDistributionBin[];
-    /** Worst-N runs by score, ascending. Populated for the composite
-     *  distribution so the report names the runs a customer should
-     *  inspect first. Undefined when the distribution was computed from a
-     *  raw value list with no run identity (e.g. cost). */
-    tailRuns?: Array<{
-        runId: string;
-        score: number;
-    }>;
-}
-interface JudgeInsight {
-    /** Number of times this judge scored a run. */
-    n: number;
-    /** Mean composite over this judge's runs. */
-    meanScore: number;
-    /** Calibration against a gold reference, when provided. Cohen's κ for
-     *  binary thresholding + continuous agreement metrics. */
-    calibration?: ContinuousAgreement;
-    /** Positional bias — when the judge sees options in different orders,
-     *  do its preferences track the content or the position? */
-    positionalBias?: number;
-    /** Self-preference — when the judge sees its own model's output vs a
-     *  competitor, does it over-pick its own? */
-    selfPreference?: number;
-    /** Verbosity bias — does the judge reward longer outputs regardless of
-     *  quality? */
-    verbosityBias?: number;
-}
-interface InterRaterInsight {
-    /** Number of raters whose scores were aggregated. */
-    raters: number;
-    /** Number of runs every rater scored. */
-    jointlyRated: number;
-    /** Multi-rater weighted kappa over the jointly rated runs. */
-    kappa: number;
-    /** Absolute agreement across raters, using ICC(2,1). */
-    icc: number;
-    /** Mean pairwise Pearson correlation. Correlation is not agreement. */
-    pearson: number;
-    /** Mean pairwise Spearman rank correlation. */
-    spearman: number;
-    /** Pairwise weighted kappa per rater pair (key = `"raterA::raterB"`). */
-    perPair: Record<string, number>;
-    /** Run ids where raters disagree the most — the high-value triage list. */
-    disagreementCases: Array<{
-        runId: string;
-        ratings: Array<{
-            rater: string;
-            score: number;
-        }>;
-        range: number;
-    }>;
-}
-interface LiftInsight {
-    baselineMean: number;
-    candidateMean: number;
-    /** Candidate − baseline. */
-    delta: number;
-    /** Lower / upper bound of bootstrap CI on the delta. */
-    ci95: [number, number];
-    /** Paired-t-test p-value. */
-    pValue: number;
-    /** Number of paired observations. */
-    n: number;
-    /** Scored baseline observations without a candidate match. */
-    unpairedBaseline: number;
-    /** Scored candidate observations without a baseline match. */
-    unpairedCandidate: number;
-    /** Cohen's dz for paired deltas; null when the observed delta variance is zero. */
-    cohensD: number | null;
-    /** Minimum detectable effect at current n, 80% power. */
-    mde: number;
-    /** Paired sample size needed to detect the standardized effect at 80% power. */
-    requiredN: number | null;
-}
-interface FailureClusterInsight {
-    /** All clusters identified by the registry, ranked by share descending. */
-    clusters: Array<{
-        id: string;
-        name: string;
-        /** Fraction of failed runs in this cluster, 0..1. */
-        share: number;
-        /** Exemplar `runId`s (≤ 5) the consumer can drill into. */
-        exemplars: string[];
-        /** Short LLM-generated suggested fix when the registry supports it. */
-        suggestedFix?: string;
-    }>;
-    totalFailures: number;
-}
-/** Model-free task-failure breakdown over canonical `RunRecord.failureClass`
- *  values. Unlike semantic failure clusters, this is computed directly from
- *  run records and does not require a model analyst. */
-interface FailureClassTally {
-    /** Canonical task-failure class. */
-    failureClass: FailureClass;
-    /** Number of failed runs carrying this class. */
-    count: number;
-    /** Share of the whole corpus, 0..1. */
-    share: number;
-}
-interface ContaminationInsight {
-    /** Canary phrases that leaked into outputs. */
-    leaks: number;
-    /** Holdout audit verdict — did any holdout-tagged run end up in the
-     *  search/dev pool, or vice versa? */
-    holdoutAuditPassed: boolean;
-    details?: Array<{
-        runId: string;
-        canary: string;
-        matched: string;
-    }>;
-}
-interface OutcomeCorrelationInsight {
-    /** What outcome the consumer is correlating against (e.g.
-     *  `'engagement_rate'`, `'approval_rate'`, `'downstream_pass'`). */
-    metric: string;
-    /** Number of (run, outcome) pairs used. */
-    n: number;
-    /** Pearson correlation between composite score and outcome. */
-    pearson: number;
-    /** Spearman rank correlation — robust to monotonic non-linearity. */
-    spearman: number;
-    /** When present, the simple linear reward model fit to the data. */
-    rewardModel?: {
-        intercept: number;
-        slope: number;
-        r2: number;
-    };
-}
-interface ReleaseSummary {
-    /** Overall verdict across axes — fail if any axis fails, else warn if any
-     *  warns, else pass. */
-    status: 'pass' | 'warn' | 'fail';
-    axes: Array<{
-        name: 'quality-lift' | 'contamination' | 'composite-distribution';
-        status: 'pass' | 'warn' | 'fail' | 'not_evaluated';
-        detail: string;
-    }>;
-    /** Free-form issues surfaced beyond the standard axes. Empty by default;
-     *  consumers can post-process to populate. */
-    issues: string[];
-}
-interface MetricDelta {
-    /** Current-period mean. */
-    current: number;
-    /** Baseline-period mean. */
-    baseline: number;
-    /** current - baseline. Positive means improved (or, for cost/duration,
-     *  the consumer-side interpretation: "higher current" — semantic
-     *  direction depends on the metric). */
-    delta: number;
-    /** Welch 95% confidence interval on the delta. Two-sample, unpaired —
-     *  the baseline and current run sets may have different scenarios. */
-    ci95: [number, number];
-    /** Welch t-test p-value (two-sided). */
-    pValue: number;
-    /** Cohen's d (pooled stddev). Effect size, signed. */
-    cohensD: number;
-    /** Sample sizes. */
-    baselineN: number;
-    currentN: number;
-    /** True when p < 0.05 AND |d| >= 0.2 (small-effect threshold). The
-     *  conjunction prevents large-effect-but-noisy and significant-but-
-     *  tiny from triggering recommendations. */
-    significant: boolean;
-}
-interface PriorPeriodComparison {
-    /** Sample counts. */
-    baselineN: number;
-    currentN: number;
-    /** Optional human-readable label — "vs prior 7 days", "vs v3 release". */
-    windowLabel?: string;
-    /** Every metric we could compare. Keys: 'composite', 'cost', 'duration',
-     *  'tokenUsage' for always-present ones; per-dimension keys when both
-     *  windows have judge scores on the same dimension. */
-    metrics: Record<string, MetricDelta>;
-    /** Metric names where current is significantly WORSE than baseline.
-     *  Direction-aware: for cost/duration, higher current = worse. */
-    regressedMetrics: string[];
-    /** Metric names where current is significantly BETTER than baseline. */
-    improvedMetrics: string[];
-}
-interface Recommendation {
-    priority: 'critical' | 'high' | 'medium' | 'low';
-    kind: 'ship' | 'hold' | 'investigate' | 'fix' | 'recalibrate' | 'expand-corpus';
-    title: string;
-    detail: string;
-    /** Optional pointer back into the report for the evidence. */
-    evidencePath?: string;
-}
-
-/**
- * # Hosted-tier wire format — the schema that EVERY orchestrator (ours,
- * a partner's self-hosted one, a future open implementation) must accept.
- *
- * This package implements exactly one wire version. Servers reject every
- * other version instead of translating old payloads.
- *
- * The wire format is two event streams in one transport:
- *
- *   1. **Eval-run events** (`POST /v1/ingest/eval-runs`). Posted when a
- *      campaign / improvement-loop completes (or per-generation if
- *      streaming). Carries the structured result + per-cell scores +
- *      surface diffs the orchestrator stores for the dashboard.
- *
- *   2. **Trace spans** (`POST /v1/ingest/traces`). Standard OTLP-shaped
- *      spans with a few additional attributes so the orchestrator can
- *      pivot from eval-run → underlying execution. Compatible with any
- *      OTel collector.
- *
- * Both endpoints are authenticated with a bearer token + a tenant id
- * header. Tenants isolate everything downstream of ingest; no tenant
- * ever sees another tenant's data.
- */
-
-declare const HOSTED_WIRE_VERSION: "2026-07-24.v1";
-type HostedWireVersion = typeof HOSTED_WIRE_VERSION;
-/** Every ingest request carries these. */
-interface HostedIngestHeaders {
-    /** Bearer token. The orchestrator validates against the tenant key. */
-    authorization: `Bearer ${string}`;
-    /** Stable tenant id (the orchestrator-side primary key for the tenant). */
-    'x-tangle-tenant-id': string;
-    /** Wire-version pin so the server can reject incompatible payloads. */
-    'x-tangle-wire-version': HostedWireVersion;
-    /** Stable request key generated once and reused across retries. */
-    'idempotency-key': string;
-}
-/** Lifecycle stages of an eval-run as the substrate reports them. */
-type EvalRunStatus = 'started' | 'baseline-complete' | 'generation-complete' | 'gate-decided' | 'finished' | 'errored';
-interface EvalRunCellScore {
-    /** Stable scenario id from the consumer's scenario set. */
-    scenarioId: string;
-    /** Repetition index when reps > 1; 0 for the default. */
-    rep: number;
-    /** Composite score across successful judges, or null when unscored. */
-    compositeMean: number | null;
-    /** Per-judge and per-dimension scores; failed or missing judges are absent. */
-    dimensions: Record<string, Record<string, number>>;
-    /** Root execution result, kept separate from task quality. */
-    terminalOutcome: RunTerminalOutcome;
-    /** Canonical execution-error count, or null when the producer did not measure it. */
-    executionErrorCount: number | null;
-    /** Per-cell dispatch or judge error. Missing on success. */
-    errorMessage?: string;
-}
-interface EvalRunGenerationSnapshot {
-    /** Generation index. 0 is baseline. */
-    index: number;
-    /** Candidate surface fingerprint (stable hash) — pivot key into the
-     *  trace stream to fetch the underlying execution. */
-    surfaceHash: string;
-    /** The candidate surface itself. May be omitted to avoid PII when the
-     *  consumer prefers not to ship verbatim prompts. */
-    surface?: MutableSurface;
-    /** Per-cell scores for this generation. */
-    cells: EvalRunCellScore[];
-    /** Mean across scored cells, or null when no cell has a task-quality label. */
-    compositeMean: number | null;
-    /** Total $ spent across this generation. */
-    costUsd: number;
-    /** Wall-clock duration of this generation. */
-    durationMs: number;
-}
-/**
- * The top-level eval-run event. One ingest call per logical eval-run;
- * generations stream in incrementally via repeated calls with the same
- * `runId`. The orchestrator deduplicates by `(runId, generation.index)`.
- */
-interface EvalRunEvent {
-    /** Stable run id (the substrate's `runId`). UUID or substrate-generated. */
-    runId: string;
-    /** Where this run was happening — derived from `RunCampaignOptions.runDir`. */
-    runDir: string;
-    /** ISO-8601 timestamp the substrate recorded the event. */
-    timestamp: string;
-    /** Lifecycle stage this event represents. */
-    status: EvalRunStatus;
-    /** Free-form consumer tags (env, branch, model id, etc.). Searchable. */
-    labels: Record<string, string>;
-    /** Baseline campaign snapshot. Present when status >= baseline-complete. */
-    baseline?: EvalRunGenerationSnapshot;
-    /** Per-generation snapshots. Streams in; orchestrator appends. */
-    generations: EvalRunGenerationSnapshot[];
-    /** Final gate decision. Present when status >= gate-decided. */
-    gateDecision?: GateDecision;
-    /** Held-out lift = winner-on-holdout - baseline-on-holdout. */
-    holdoutLift?: number;
-    /** Total $ spent across baseline + every generation. */
-    totalCostUsd: number;
-    /** Total wall-clock duration. */
-    totalDurationMs: number;
-    /** Error message if status === 'errored'. */
-    errorMessage?: string;
-    /** Rigor packet emitted alongside the run — distributional summary,
-     *  paired-bootstrap lift CI, judge stats, inter-rater agreement,
-     *  contamination check, failure clusters (when an analyst is wired),
-     *  outcome correlation (when downstream signal is supplied), and the
-     *  recommendations the dashboard surfaces verbatim. */
-    insightReport?: InsightReport;
-}
-/**
- * Canonical unsigned 64-bit integer encoded as a base-10 string.
- * JSON numbers cannot represent OTLP nanosecond timestamps exactly.
- */
-type UnixNanoTimestamp = string;
-/**
- * OTel-shape span with a few additional attributes for eval-run pivoting.
- * Compatible with any OTLP collector — `name`, `traceId`, `spanId`,
- * `startTimeUnixNano`, `endTimeUnixNano`, `attributes` are stock OTel.
- */
-interface TraceSpanEvent {
-    traceId: string;
-    spanId: string;
-    parentSpanId?: string;
-    name: string;
-    startTimeUnixNano: UnixNanoTimestamp;
-    endTimeUnixNano: UnixNanoTimestamp;
-    attributes: Record<string, string | number | boolean>;
-    events?: Array<{
-        timeUnixNano: UnixNanoTimestamp;
-        name: string;
-        attributes?: Record<string, string | number | boolean>;
-    }>;
-    status?: {
-        code: 'OK' | 'ERROR' | 'UNSET';
-        message?: string;
-    };
-    /** Pivot back into the eval-run stream. */
-    'tangle.runId'?: string;
-    /** Pivot to the specific generation. */
-    'tangle.generation'?: number;
-    /** Pivot to the specific cell. */
-    'tangle.cellId'?: string;
-    /** Pivot to the specific scenario. */
-    'tangle.scenarioId'?: string;
-}
-interface IngestEvalRunsRequest {
-    wireVersion: HostedWireVersion;
-    events: EvalRunEvent[];
-}
-interface IngestTracesRequest {
-    wireVersion: HostedWireVersion;
-    spans: TraceSpanEvent[];
-}
-interface IngestResponse {
-    /** Accepted events / spans count. */
-    accepted: number;
-    /** Rejected events with reasons (validation failures, dup idempotency key, etc.). */
-    rejected: Array<{
-        index: number;
-        reason: string;
-    }>;
-}
-
-/**
- * # Hosted-tier ingest client.
- *
- * Ships eval-run events + trace spans to any orchestrator (ours, a
- * partner's self-hosted one, or a future open implementation) that
- * speaks the wire format in `./types.ts`.
- *
- * Three modes:
- *   - **Ours:** point at `https://orchestrator.tangle.tools` (the host root —
- *     the client appends the versioned `/v1/ingest/...` path itself; a trailing
- *     `/v1` on the endpoint is tolerated and normalized away). We handle ingest
- *     + storage + dashboard.
- *   - **Self-hosted:** point at whatever URL runs the reference receiver
- *     from `examples/hosted-ingest-server/`.
- *   - **Off (default):** when `hostedTenant` is unset, nothing is sent.
- *     Everything stays local.
- */
-
-interface HostedTenant {
-    /** Orchestrator endpoint base URL (no trailing slash). Required. */
-    endpoint: string;
-    /** Bearer token issued by the orchestrator. Required. */
-    apiKey: string;
-    /** Tenant id — the orchestrator's primary key for this consumer. Required. */
-    tenantId: string;
-    /** Optional `fetch` override (auth wrappers, custom agent, test mocks). */
-    fetchImpl?: typeof fetch;
-    /** Per-call timeout in ms. Default 30s. */
-    timeoutMs?: number;
-    /** Retries on 5xx / network errors. Default 2. */
-    retries?: number;
-}
-interface HostedClient {
-    ingestEvalRun(event: EvalRunEvent, idempotencyKey?: string): Promise<IngestResponse>;
-    ingestEvalRuns(events: EvalRunEvent[], idempotencyKey?: string): Promise<IngestResponse>;
-    ingestTraces(spans: TraceSpanEvent[], idempotencyKey?: string): Promise<IngestResponse>;
-    readonly tenant: HostedTenant;
-    readonly wireVersion: HostedWireVersion;
-}
-declare function createHostedClient(tenant: HostedTenant): HostedClient;
-/**
- * Build a `HostedClient` from environment, or `undefined` when ingest is not
- * configured — the canonical, fail-soft wiring every product uses so eval-run +
- * trace provenance lands in the Intelligence dashboard with ONE call:
- *
- *   const hosted = hostedClientFromEnv()
- *   // ...run the loop...
- *   await emitLoopProvenance({ ..., hostedClient: hosted })  // no-op if undefined
- *
- * Returns `undefined` (NOT an error) when any of endpoint / apiKey / tenantId is
- * missing — so a product wires the ship call unconditionally and it stays a
- * no-op until the env is set. Env precedence:
- *   - endpoint:  `TANGLE_INGEST_URL` → `TANGLE_ORCHESTRATOR_URL`
- *   - apiKey:    `TANGLE_INGEST_API_KEY` → `TANGLE_API_KEY`
- *   - tenantId:  `TANGLE_TENANT_ID`
- * A trailing slash on the endpoint is stripped. Pass `overrides` to supply any
- * field directly (e.g. a fixed `tenantId` per product) — overrides win over env.
- */
-/**
- * Build a {@link HostedTenant} config from env — the input `selfImprove`'s
- * `hostedTenant` and `emitLoopProvenance` take. Same env precedence + overrides
- * as {@link hostedClientFromEnv}; returns `undefined` (not an error) when any of
- * endpoint / apiKey / tenantId is missing, so a product wires
- * `hostedTenant: hostedTenantFromEnv({ tenantId: 'my-agent' })` unconditionally
- * and it stays off until the env is set.
- */
-declare function hostedTenantFromEnv(overrides?: Partial<HostedTenant> & {
-    env?: Record<string, string | undefined>;
-}): HostedTenant | undefined;
-declare function hostedClientFromEnv(overrides?: Partial<HostedTenant> & {
-    env?: Record<string, string | undefined>;
-}): HostedClient | undefined;
-
+import { j as MutableSurface } from "../types-k9tZGKUg.js";
+import { C as InsightReport, _ as UnixNanoTimestamp, a as hostedTenantFromEnv, c as EvalRunGenerationSnapshot, d as HostedIngestHeaders, f as HostedWireVersion, g as TraceSpanEvent, h as IngestTracesRequest, i as hostedClientFromEnv, l as EvalRunStatus, m as IngestResponse, n as HostedTenant, o as EvalRunCellScore, p as IngestEvalRunsRequest, r as createHostedClient, s as EvalRunEvent, t as HostedClient, u as HOSTED_WIRE_VERSION } from "../client-C97NMzqi.js";
+import { z } from "zod";
+//#region src/hosted/schemas.d.ts
 declare const UnixNanoTimestampSchema: z.ZodType<UnixNanoTimestamp>;
 declare const InsightReportSchema: z.ZodType<InsightReport>;
 declare const MutableSurfaceSchema: z.ZodType<MutableSurface>;
 declare const RunTerminalOutcomeSchema: z.ZodEnum<{
-    succeeded: "succeeded";
-    unknown: "unknown";
-    failed: "failed";
-    cancelled: "cancelled";
-    incomplete: "incomplete";
+  cancelled: "cancelled";
+  failed: "failed";
+  incomplete: "incomplete";
+  succeeded: "succeeded";
+  unknown: "unknown";
 }>;
 declare const EvalRunCellScoreSchema: z.ZodType<EvalRunCellScore>;
 declare const EvalRunGenerationSnapshotSchema: z.ZodType<EvalRunGenerationSnapshot>;
@@ -840,5 +19,6 @@ declare const TraceSpanEventSchema: z.ZodType<TraceSpanEvent>;
 declare const IngestEvalRunsRequestSchema: z.ZodType<IngestEvalRunsRequest>;
 declare const IngestTracesRequestSchema: z.ZodType<IngestTracesRequest>;
 declare const IngestResponseSchema: z.ZodType<IngestResponse>;
-
+//#endregion
 export { type EvalRunCellScore, EvalRunCellScoreSchema, type EvalRunEvent, EvalRunEventSchema, type EvalRunGenerationSnapshot, EvalRunGenerationSnapshotSchema, type EvalRunStatus, HOSTED_WIRE_VERSION, type HostedClient, type HostedIngestHeaders, type HostedTenant, type HostedWireVersion, type IngestEvalRunsRequest, IngestEvalRunsRequestSchema, type IngestResponse, IngestResponseSchema, type IngestTracesRequest, IngestTracesRequestSchema, type InsightReport, InsightReportSchema, MutableSurfaceSchema, RunTerminalOutcomeSchema, type TraceSpanEvent, TraceSpanEventSchema, type UnixNanoTimestamp, UnixNanoTimestampSchema, createHostedClient, hostedClientFromEnv, hostedTenantFromEnv };
+//# sourceMappingURL=index.d.ts.map