@tangle-network/agent-eval 0.126.6 → 0.127.0

package/dist/control.d.ts

@@ -770,7 +770,8 @@ interface AgentProfileCell {
  * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
  * fields are exactly those the paper "Two Loops, Three Roles" requires
  * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
- * holdout split tag and either a `searchScore` or a `holdoutScore`.
+ * holdout split tag. A task score is optional because execution-only records
+ * must preserve missing labels instead of converting errors into zero quality.
  *
  * This is intentionally NOT a replacement for the rich `Run` /
  * `ProposeReviewReport` / `ScenarioResult` types already in the
@@ -791,6 +792,13 @@ interface AgentProfileCell {
 /** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
  *  combined train+test pool that the optimizer is allowed to read. */
 type RunSplitTag = 'search' | 'dev' | 'holdout';
+/**
+ * Explicit execution-lifecycle result for a run.
+ *
+ * This is separate from task quality (`outcome`) and failure classification.
+ * Producers set it only from root-run or process evidence.
+ */
+type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
 interface RunTokenUsage {
     input: number;
     /** All generated tokens charged as output, including reasoning tokens. */
@@ -804,11 +812,6 @@ interface RunTokenUsage {
 }
 /**
  * How a run's USD amount was obtained.
- *
- * `costUsd` remains mandatory for wire compatibility. New producers should
- * always populate this discriminated union so a missing bill is never
- * mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
- * the legacy `0` sentinel while this field carries the truthful null.
  */
 type RunCostProvenance = {
     kind: 'observed';
@@ -854,8 +857,8 @@ interface JudgeScoresRecord {
     perJudge: Record<string, Record<string, number>>;
     /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
     perDimMean: Record<string, number>;
-    /** Composite mean across all dims and judges. Mirrors the score
-     *  the gate sees on `outcome.searchScore` / `holdoutScore`. */
+    /** Composite mean across successful judges. Mirrors the task score only
+     *  when `failedJudges` is empty. */
     composite: number;
     /** Judges that errored or returned an unparseable verdict. Recorded
      *  by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
@@ -866,11 +869,11 @@ interface JudgeScoresRecord {
     notes?: string;
 }
 interface RunOutcome {
-    /** Score on the search/optimization split. Optional because a
-     *  holdout-only evaluation only fills `holdoutScore`. */
+    /** Score on the search/optimization split. Optional for holdout-only and
+     *  execution-only records. */
     searchScore?: number;
-    /** Score on the held-out split. Optional because a search-only run
-     *  only fills `searchScore`. At least one must be present. */
+    /** Score on the held-out split. Optional for search-only and execution-only
+     *  records. When both scores are absent, the run is explicitly unlabeled. */
     holdoutScore?: number;
     /** Bag of any other metric the run produced — judge dimensions,
      *  pass/fail counters, latency stats, etc. Numeric only — keeps
@@ -934,42 +937,39 @@ interface RunRecord {
     wallMs: number;
     /** Time spent queued before execution started, if known. */
     queueMs?: number;
-    /** Total USD cost. Mandatory — runs without a cost number are
-     *  unbounded by definition and must not be admitted into the gate.
-     *  `0` is retained as the compatibility sentinel for an uncaptured amount;
-     *  inspect `costProvenance` before treating it as observed. */
-    costUsd: number;
-    /** Observed, model-priced estimate, or genuinely uncaptured USD amount.
-     *  Optional only so existing serialized RunRecords remain valid. */
-    costProvenance?: RunCostProvenance;
+    /** Total USD cost, or null when the producer could not capture one. */
+    costUsd: number | null;
+    /** Whether `costUsd` came from billing data, a price calculation, or is unavailable. */
+    costProvenance: RunCostProvenance;
     /** Token usage breakdown. */
     tokenUsage: RunTokenUsage;
+    /** Root-run or process terminal result. Never inferred from a child span. */
+    terminalOutcome: RunTerminalOutcome;
+    /** Root-run or process failure reason. Valid only for a failed, cancelled,
+     *  or incomplete terminal result; never populated from a child span. */
+    terminalFailureReason?: string;
     /** Judge-side metadata, if a judge was used. */
     judgeMetadata?: RunJudgeMetadata;
     /** Per-split scores + raw bag. */
     outcome: RunOutcome;
-    /** Canonical, cross-agent failure class drawn from the shared
+    /** Canonical task-failure class drawn from the shared
      *  `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
      *  "which failure dominates across the whole fleet" answerable in ONE
      *  vocabulary — every agent classifies against the same enum. Producers
-     *  set it via the substrate classifier; leave unset only when the failure
-     *  genuinely can't be classified. */
+     *  set it only from task-result evidence. Execution errors belong in
+     *  `outcome.raw.execution_error_count`, even when the run later fails. */
     failureClass?: FailureClass;
-    /** Free-form domain-specific failure detail, scoped UNDER `failureClass`
+    /** Free-form task-failure detail, scoped UNDER `failureClass`
      *  (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
-     *  The within-agent drill-down; `failureClass` is the cross-agent key. */
+     *  Do not populate this from a child execution error alone. */
     failureMode?: string;
     /** Which split this run was drawn from. */
     splitTag: RunSplitTag;
     /**
-     * Stable scenario identifier the run was scored against. Optional for
-     * backwards compatibility, but **strongly recommended**: every primitive
-     * that pairs runs by scenario (preferences, paired stats, BT tournament)
-     * keys on this. The campaign artifact populates it canonically; legacy
-     * runs without it fall back to inference from `outcome.raw.scenario_id`
-     * or `experimentId`.
+     * Stable scenario identifier the run observed or was scored against.
+     * Comparison primitives match this identity rather than input order.
      */
-    scenarioId?: string;
+    scenarioId: string;
     /**
      * Canonical identity for the agent profile cell that produced this row:
      * profile artifact hash plus optional harness/model/prompt/reporting
@@ -982,6 +982,7 @@ interface RunRecord {
 
 interface RunEvidenceMetadata {
     experimentId: string;
+    scenarioId: string;
     candidateId: string;
     seed: number;
     model: string;
@@ -990,6 +991,7 @@ interface RunEvidenceMetadata {
     commitSha: string;
     splitTag: RunSplitTag;
     tokenUsage: RunTokenUsage;
+    costProvenance: RunRecord['costProvenance'];
     queueMs?: number;
     judgeMetadata?: RunRecord['judgeMetadata'];
     raw?: Record<string, number>;

package/dist/control.js

@@ -10,9 +10,9 @@ import {
   stopOnNoProgress,
   stopOnRepeatedAction,
   subjectiveEval
-} from "./chunk-474LBSOX.js";
+} from "./chunk-OD5NQ2CS.js";
 import "./chunk-VQMK5FMP.js";
-import "./chunk-IILEIWGW.js";
+import "./chunk-NL33OXEP.js";
 import "./chunk-MA6HLL3S.js";
 import "./chunk-VSMTAMNK.js";
 import "./chunk-ONWEPEDO.js";

package/dist/hosted/index.d.ts

@@ -1,3 +1,39 @@
+import { z } from 'zod';
+
+/**
+ * Paper-grade RunRecord schema + runtime validator.
+ *
+ * Every run that participates in a promotion gate, paper table, or
+ * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
+ * fields are exactly those the paper "Two Loops, Three Roles" requires
+ * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
+ * holdout split tag. A task score is optional because execution-only records
+ * must preserve missing labels instead of converting errors into zero quality.
+ *
+ * This is intentionally NOT a replacement for the rich `Run` /
+ * `ProposeReviewReport` / `ScenarioResult` types already in the
+ * package. Those are runtime structures with full provenance. A
+ * `RunRecord` is the analysis-time projection — the JSON-friendly
+ * row you'd put in a parquet file or paste into a notebook.
+ *
+ * Validate at the boundary:
+ *
+ *   const rec = validateRunRecord(rawJson)         // throws on missing
+ *   const ok  = isRunRecord(rawJson)               // boolean check
+ *   const rec = parseRunRecordSafe(rawJson)        // { ok, value | error }
+ *
+ * The validator runs in pure TS — zod is intentionally NOT a
+ * dependency. Round-trip tested in `tests/run-record.test.ts`.
+ */
+
+/**
+ * Explicit execution-lifecycle result for a run.
+ *
+ * This is separate from task quality (`outcome`) and failure classification.
+ * Producers set it only from root-run or process evidence.
+ */
+type RunTerminalOutcome = 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown';
+
 /**
  * Pass A substrate types — `runCampaign` is the one primitive every
  * eval flow composes from. Three contracts in this file:
@@ -102,7 +138,7 @@ interface ParetoPoint {
     onFrontier: boolean;
     /** Optional gate verdict for this candidate, if a `GateDecision`
      *  for it was passed in. */
-    gate?: 'promote' | 'reject_few_runs' | 'reject_negative_delta' | 'reject_overfit_gap' | null;
+    gate?: 'promote' | 'reject';
 }
 interface ParetoFigureSpec {
     kind: 'pareto-cost-quality';
@@ -176,8 +212,8 @@ interface InsightReport {
     /** Number of runs analyzed. */
     n: number;
     /** Runtime facts carried by the run records. These describe execution,
-     *  not task quality: duration, queueing, token categories, models, and
-     *  explicitly recorded failures. */
+     *  not task quality: duration, queueing, token categories, models,
+     *  execution errors, and terminal outcomes. */
     execution: ExecutionInsight;
     /** Composite-score distribution across all runs. Always present. */
     composite: ScalarDistribution;
@@ -235,10 +271,9 @@ interface InsightReport {
      *  Surfaced metrics: composite, cost, duration, tokenUsage, plus any
      *  per-dimension judge metric present in both windows. */
     priorPeriodComparison?: PriorPeriodComparison;
-    /** Model-free failure-mode breakdown from `RunRecord.failureMode`, ranked
-     *  by count descending. Present when any run carries a `failureMode`.
-     *  Complements `failureClusters` (LLM-semantic) with the structured tags
-     *  the harness already recorded — actionable with no analyst wired. */
+    /** Model-free task-failure breakdown from `RunRecord.failureClass` or
+     *  `failureMode`, ranked by count descending. Tags from successful or
+     *  otherwise non-failed tasks are excluded. */
     failureModes?: FailureModeTally[];
     /** Top-N actionable recommendations, ranked by priority. The packet's
      *  human-readable layer; the numeric sections are the evidence. */
@@ -287,15 +322,46 @@ interface ExecutionInsight {
         events: number;
         reportingRuns: number;
     };
-    /** Failure counts remain separate from outcome scores. `reportedErrorEvents`
-     *  sums `outcome.raw.error_span_count` only where a producer supplied it. */
-    failures: {
+    /** Runs with explicit execution-error telemetry. This is independent of
+     *  whether the root run ultimately succeeded, failed, or has no terminal
+     *  evidence. */
+    executionErrors: {
         runs: number;
-        fraction: number;
-        reportedErrorEvents: number;
+        /** Share among runs that supplied an execution-error count.
+         *  `null` when no run supplied error telemetry. */
+        fraction: number | null;
+        /** Execution-error events reported through the canonical count. */
+        events: number;
+        /** Runs that supplied an execution-error count, including explicit zeroes. */
         reportingRuns: number;
+        /** Exact sum of `outcome.raw.error_span_count`, kept separate from other errors. */
+        errorSpanEvents: number;
+        /** Runs that supplied `outcome.raw.error_span_count`, including explicit zeroes. */
+        errorSpanReportingRuns: number;
+        /**
+         * Error-telemetry coverage crossed with independently reported terminal
+         * outcomes. `unreported` is distinct from a reported zero.
+         */
+        byTerminalOutcome: Record<RunTerminalOutcome, ExecutionErrorOutcomeCell>;
+    };
+    /** Root-run or process outcomes. Missing `RunRecord.terminalOutcome` values
+     *  count as `unknown`; child-span status never changes these counts. */
+    terminalOutcomes: {
+        succeeded: number;
+        failed: number;
+        cancelled: number;
+        incomplete: number;
+        unknown: number;
     };
 }
+interface ExecutionErrorOutcomeCell {
+    /** Runs that explicitly reported one or more execution errors. */
+    withErrors: number;
+    /** Runs that explicitly reported zero execution errors. */
+    withoutErrors: number;
+    /** Runs with no execution-error count from the producer. */
+    unreported: number;
+}
 interface TokenUsageInsight {
     input: ScalarDistribution;
     output: ScalarDistribution;
@@ -314,12 +380,18 @@ interface TokenUsageInsight {
 interface ScalarDistribution {
     /** Sample count after dropping non-finite values. */
     n: number;
-    mean: number;
-    p50: number;
-    p95: number;
-    stddev: number;
-    min: number;
-    max: number;
+    /** Null when `n` is zero. */
+    mean: number | null;
+    /** Null when `n` is zero. */
+    p50: number | null;
+    /** Null when `n` is zero. */
+    p95: number | null;
+    /** Null when `n` is zero. */
+    stddev: number | null;
+    /** Null when `n` is zero. */
+    min: number | null;
+    /** Null when `n` is zero. */
+    max: number | null;
     /** Histogram bins using `agent-eval`'s `gainHistogram` primitive. */
     histogram: GainDistributionBin[];
     /** Worst-N runs by score, ascending. Populated for the composite
@@ -385,12 +457,16 @@ interface LiftInsight {
     pValue: number;
     /** Number of paired observations. */
     n: number;
-    /** Cohen's d for the delta. */
-    cohensD: number;
+    /** Scored baseline observations without a candidate match. */
+    unpairedBaseline: number;
+    /** Scored candidate observations without a baseline match. */
+    unpairedCandidate: number;
+    /** Cohen's dz for paired deltas; null when the observed delta variance is zero. */
+    cohensD: number | null;
     /** Minimum detectable effect at current n, 80% power. */
     mde: number;
-    /** Sample size needed to detect the observed delta at 80% power. */
-    requiredN: number;
+    /** Paired sample size needed to detect the standardized effect at 80% power. */
+    requiredN: number | null;
 }
 interface FailureClusterInsight {
     /** All clusters identified by the registry, ranked by share descending. */
@@ -454,7 +530,7 @@ interface ReleaseSummary {
     status: 'pass' | 'warn' | 'fail';
     axes: Array<{
         name: 'quality-lift' | 'contamination' | 'composite-distribution';
-        status: 'pass' | 'warn' | 'fail';
+        status: 'pass' | 'warn' | 'fail' | 'not_evaluated';
         detail: string;
     }>;
     /** Free-form issues surfaced beyond the standard axes. Empty by default;
@@ -514,9 +590,8 @@ interface Recommendation {
  * # Hosted-tier wire format — the schema that EVERY orchestrator (ours,
  * a partner's self-hosted one, a future open implementation) must accept.
  *
- * **Stability:** every type in this file is committed under semver. New
- * minors only ADD optional fields. Breaking changes mean a major bump
- * (`HostedWireVersion` literal increment).
+ * This package implements exactly one wire version. Servers reject every
+ * other version instead of translating old payloads.
  *
  * The wire format is two event streams in one transport:
  *
@@ -535,7 +610,7 @@ interface Recommendation {
  * ever sees another tenant's data.
  */
 
-declare const HOSTED_WIRE_VERSION: "2026-05-26.v1";
+declare const HOSTED_WIRE_VERSION: "2026-07-24.v1";
 type HostedWireVersion = typeof HOSTED_WIRE_VERSION;
 /** Every ingest request carries these. */
 interface HostedIngestHeaders {
@@ -545,8 +620,8 @@ interface HostedIngestHeaders {
     'x-tangle-tenant-id': string;
     /** Wire-version pin so the server can reject incompatible payloads. */
     'x-tangle-wire-version': HostedWireVersion;
-    /** Optional idempotency key for retry-safe ingest. */
-    'idempotency-key'?: string;
+    /** Stable request key generated once and reused across retries. */
+    'idempotency-key': string;
 }
 /** Lifecycle stages of an eval-run as the substrate reports them. */
 type EvalRunStatus = 'started' | 'baseline-complete' | 'generation-complete' | 'gate-decided' | 'finished' | 'errored';
@@ -555,11 +630,15 @@ interface EvalRunCellScore {
     scenarioId: string;
     /** Repetition index when reps > 1; 0 for the default. */
     rep: number;
-    /** Composite score across all judges + dimensions for this cell. */
-    compositeMean: number;
-    /** Per-judge → per-dimension scores; null where the judge did not run. */
+    /** Composite score across successful judges, or null when unscored. */
+    compositeMean: number | null;
+    /** Per-judge and per-dimension scores; failed or missing judges are absent. */
     dimensions: Record<string, Record<string, number>>;
-    /** Per-cell error message if the dispatch threw. Null on success. */
+    /** Root execution result, kept separate from task quality. */
+    terminalOutcome: RunTerminalOutcome;
+    /** Canonical execution-error count, or null when the producer did not measure it. */
+    executionErrorCount: number | null;
+    /** Per-cell dispatch or judge error. Missing on success. */
     errorMessage?: string;
 }
 interface EvalRunGenerationSnapshot {
@@ -573,8 +652,8 @@ interface EvalRunGenerationSnapshot {
     surface?: MutableSurface;
     /** Per-cell scores for this generation. */
     cells: EvalRunCellScore[];
-    /** Aggregate composite mean across all cells in this generation. */
-    compositeMean: number;
+    /** Mean across scored cells, or null when no cell has a task-quality label. */
+    compositeMean: number | null;
     /** Total $ spent across this generation. */
     costUsd: number;
     /** Wall-clock duration of this generation. */
@@ -614,10 +693,14 @@ interface EvalRunEvent {
      *  paired-bootstrap lift CI, judge stats, inter-rater agreement,
      *  contamination check, failure clusters (when an analyst is wired),
      *  outcome correlation (when downstream signal is supplied), and the
-     *  recommendations the dashboard surfaces verbatim. Additive; older
-     *  clients that don't know about this field continue to work. */
+     *  recommendations the dashboard surfaces verbatim. */
     insightReport?: InsightReport;
 }
+/**
+ * Canonical unsigned 64-bit integer encoded as a base-10 string.
+ * JSON numbers cannot represent OTLP nanosecond timestamps exactly.
+ */
+type UnixNanoTimestamp = string;
 /**
  * OTel-shape span with a few additional attributes for eval-run pivoting.
  * Compatible with any OTLP collector — `name`, `traceId`, `spanId`,
@@ -628,11 +711,11 @@ interface TraceSpanEvent {
     spanId: string;
     parentSpanId?: string;
     name: string;
-    startTimeUnixNano: number;
-    endTimeUnixNano: number;
+    startTimeUnixNano: UnixNanoTimestamp;
+    endTimeUnixNano: UnixNanoTimestamp;
     attributes: Record<string, string | number | boolean>;
     events?: Array<{
-        timeUnixNano: number;
+        timeUnixNano: UnixNanoTimestamp;
         name: string;
         attributes?: Record<string, string | number | boolean>;
     }>;
@@ -740,4 +823,22 @@ declare function hostedClientFromEnv(overrides?: Partial<HostedTenant> & {
     env?: Record<string, string | undefined>;
 }): HostedClient | undefined;
 
-export { type EvalRunCellScore, type EvalRunEvent, type EvalRunGenerationSnapshot, type EvalRunStatus, HOSTED_WIRE_VERSION, type HostedClient, type HostedIngestHeaders, type HostedTenant, type HostedWireVersion, type IngestEvalRunsRequest, type IngestResponse, type IngestTracesRequest, type TraceSpanEvent, createHostedClient, hostedClientFromEnv, hostedTenantFromEnv };
+declare const UnixNanoTimestampSchema: z.ZodType<UnixNanoTimestamp>;
+declare const InsightReportSchema: z.ZodType<InsightReport>;
+declare const MutableSurfaceSchema: z.ZodType<MutableSurface>;
+declare const RunTerminalOutcomeSchema: z.ZodEnum<{
+    succeeded: "succeeded";
+    unknown: "unknown";
+    failed: "failed";
+    cancelled: "cancelled";
+    incomplete: "incomplete";
+}>;
+declare const EvalRunCellScoreSchema: z.ZodType<EvalRunCellScore>;
+declare const EvalRunGenerationSnapshotSchema: z.ZodType<EvalRunGenerationSnapshot>;
+declare const EvalRunEventSchema: z.ZodType<EvalRunEvent>;
+declare const TraceSpanEventSchema: z.ZodType<TraceSpanEvent>;
+declare const IngestEvalRunsRequestSchema: z.ZodType<IngestEvalRunsRequest>;
+declare const IngestTracesRequestSchema: z.ZodType<IngestTracesRequest>;
+declare const IngestResponseSchema: z.ZodType<IngestResponse>;
+
+export { type EvalRunCellScore, EvalRunCellScoreSchema, type EvalRunEvent, EvalRunEventSchema, type EvalRunGenerationSnapshot, EvalRunGenerationSnapshotSchema, type EvalRunStatus, HOSTED_WIRE_VERSION, type HostedClient, type HostedIngestHeaders, type HostedTenant, type HostedWireVersion, type IngestEvalRunsRequest, IngestEvalRunsRequestSchema, type IngestResponse, IngestResponseSchema, type IngestTracesRequest, IngestTracesRequestSchema, type InsightReport, InsightReportSchema, MutableSurfaceSchema, RunTerminalOutcomeSchema, type TraceSpanEvent, TraceSpanEventSchema, type UnixNanoTimestamp, UnixNanoTimestampSchema, createHostedClient, hostedClientFromEnv, hostedTenantFromEnv };

package/dist/hosted/index.js

@@ -1,12 +1,34 @@
 import {
+  EvalRunCellScoreSchema,
+  EvalRunEventSchema,
+  EvalRunGenerationSnapshotSchema,
   HOSTED_WIRE_VERSION,
+  IngestEvalRunsRequestSchema,
+  IngestResponseSchema,
+  IngestTracesRequestSchema,
+  InsightReportSchema,
+  MutableSurfaceSchema,
+  RunTerminalOutcomeSchema,
+  TraceSpanEventSchema,
+  UnixNanoTimestampSchema,
   createHostedClient,
   hostedClientFromEnv,
   hostedTenantFromEnv
-} from "../chunk-ZZUXHH3R.js";
+} from "../chunk-IG35YDN3.js";
 import "../chunk-PZ5AY32C.js";
 export {
+  EvalRunCellScoreSchema,
+  EvalRunEventSchema,
+  EvalRunGenerationSnapshotSchema,
   HOSTED_WIRE_VERSION,
+  IngestEvalRunsRequestSchema,
+  IngestResponseSchema,
+  IngestTracesRequestSchema,
+  InsightReportSchema,
+  MutableSurfaceSchema,
+  RunTerminalOutcomeSchema,
+  TraceSpanEventSchema,
+  UnixNanoTimestampSchema,
   createHostedClient,
   hostedClientFromEnv,
   hostedTenantFromEnv