@tangle-network/agent-eval 0.126.6 → 0.127.0

package/dist/traces.js

@@ -27,8 +27,8 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-OCFJACJU.js";
-import "./chunk-H5UD2323.js";
+} from "./chunk-X6VOPMLD.js";
+import "./chunk-IN455XKS.js";
 import {
   extractUsage,
   extractUsageFromResponse,
@@ -49,7 +49,7 @@ import {
   TRACE_ANALYST_ACTOR_DESCRIPTION,
   TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
   analyzeTraces
-} from "./chunk-NUKSVU3W.js";
+} from "./chunk-5DTSBUL2.js";
 import {
   DEFAULT_REDACTION_RULES,
   REDACTION_VERSION,
@@ -66,15 +66,17 @@ import {
   applyToolSpanOtlpAttributes,
   asString,
   buildTraceAnalystTools,
+  classifyOtlpSpanRole,
   extractOtlpAttributes,
   firstStringAttr,
   inferOtlpKind,
+  isOtlpModelCall,
   projectOtlpFlatLine,
   readOtlpStatus,
   stringField,
   traceAnalystFunctionGroup,
   traceSpanKindToOpenInferenceKind
-} from "./chunk-IR3KBHOY.js";
+} from "./chunk-P6FYH6K4.js";
 import {
   RunIntegrityError,
   assertRunCaptured,
@@ -91,7 +93,7 @@ import {
   TraceEmitter,
   llmSpanFromProvider
 } from "./chunk-VQMK5FMP.js";
-import "./chunk-IILEIWGW.js";
+import "./chunk-NL33OXEP.js";
 import {
   FAILURE_CLASSES,
   TRACE_SCHEMA_VERSION,
@@ -193,6 +195,7 @@ export {
   buildTraceInsightContext,
   buildTraceInsightPrompt,
   captureFetchToRawSink,
+  classifyOtlpSpanRole,
   contextInputTokens,
   convertTraceStoresToOtlp,
   createOtelExporter,
@@ -216,6 +219,7 @@ export {
   inferOtlpKind,
   isJudgeSpan,
   isLlmSpan,
+  isOtlpModelCall,
   isRetrievalSpan,
   isSandboxSpan,
   isToolSpan,

package/dist/wire/index.d.ts

@@ -640,6 +640,7 @@ type ProviderRedactor = (event: RawProviderEvent) => RawProviderEvent;
  * OpenAI-compatible `/v1/chat/completions` client with:
  *   - Exponential-backoff retry on 429 + 5xx gateway errors (502/503/504).
  *   - Retry on transient network errors (fetch failed, AbortError, ECONNRESET).
+ *   - One retry at temperature 1 when a model explicitly requires it.
  *   - Graceful json_schema → json_object degrade on 400 with schema-reject body.
  *   - Fenced-JSON stripping (```json ... ```) for models that wrap structured output.
  *   - Configurable base URL + api key / bearer, works with LiteLLM proxies, OpenAI

package/dist/wire/index.js

@@ -34,8 +34,8 @@ import {
   runRpcOnce,
   startServer,
   startServerAsync
-} from "../chunk-FO7HEH76.js";
-import "../chunk-J5SQWP6Y.js";
+} from "../chunk-YJBNWCAA.js";
+import "../chunk-PBE2LOSS.js";
 import "../chunk-WS3NZZQQ.js";
 import "../chunk-VI2UW6B6.js";
 import "../chunk-PC4UYEBM.js";

package/docs/concepts.md

@@ -43,6 +43,18 @@ Agent sessions, worker coordination, sandbox execution, and runtime-specific pro
 
 Traces, datasets, optimization, statistics, and reports build on these objects.
 
+## Release check results
+
+Every entry in `GateResult.contributingGates` has a `status` of `pass`, `fail`, or `not_evaluated`.
+`pass` and `fail` mean the check ran with sufficient input.
+`not_evaluated` means the check lacked enough evidence to run.
+`defaultProductionGate` always requires held-out significance.
+Its other checks are optional until their input is configured or their name is included in `requiredChecks`.
+A required check with missing or insufficient evidence remains `not_evaluated` and holds the release decision.
+An absent optional check records `not_evaluated` and never appears as a successful check.
+Run history is shared input only.
+Enable reward-hacking and canary monitoring independently with `rewardHacking` and `canary`.
+
 When the thing being evaluated is an agent that should keep working, use
 [`runAgentControlLoop`](./control-runtime.md). It turns validators into a
 runtime loop: observe typed state, validate it, decide the next action, act,
@@ -138,10 +150,15 @@ const verifier = new MultiLayerVerifier([
 
 const report = await verifier.run({ env: { runner, workdir, ... } })
 report.allPass        // boolean: every layer passed
-report.blendedScore   // 0..1: weighted aggregate
+report.taskScore      // complete task score, or undefined
+report.blendedScore   // diagnostic weighted aggregate, possibly partial
 report.layers         // per-layer status, findings, duration
 ```
 
+Use `taskScore` when creating task labels or training data.
+An errored, timed-out, skipped, or incomplete scoring panel leaves `taskScore` undefined.
+Use `blendedScore` only to inspect the measurements that did complete.
+
 Two rules that will save you bugs:
 
 1. **Run both gates.** Build gates catch code that doesn't compile; structural assertions catch missing files. Run both unconditionally: they catch orthogonal failures.

package/docs/hosted-ingest-spec.md

@@ -1,9 +1,7 @@
-# Hosted-ingest wire spec: `2026-05-26.v1`
+# Hosted-ingest wire spec: `2026-07-24.v1`
 
-The schema **every** orchestrator (ours, partners' self-hosted ones,
-any future open implementation) must accept. Frozen under semver:
-**new minors only add optional fields. Breaking changes mean a major
-bump and a new `HostedWireVersion` literal.**
+This is the only hosted-ingest wire format implemented by the current package.
+Clients and servers reject every other wire version.
 
 This is the contract between `@tangle-network/agent-eval` and any hosted
 or self-hosted orchestrator. A builder can:
@@ -26,12 +24,12 @@ Two endpoints, both `POST`, both JSON. Headers on every request:
 | `Authorization` | `Bearer <tenant-key>` (the orchestrator issues this) |
 | `Content-Type` | `application/json` |
 | `X-Tangle-Tenant-Id` | The tenant's stable id (the orchestrator's primary key for the tenant) |
-| `X-Tangle-Wire-Version` | `2026-05-26.v1` (this spec) |
-| `Idempotency-Key` (optional) | UUID; servers MUST treat repeated keys as dedup |
+| `X-Tangle-Wire-Version` | `2026-07-24.v1` (this spec) |
+| `Idempotency-Key` | Non-empty request key, at most 256 characters; clients generate one per call and reuse it across retries |
 
-Responses are JSON of shape `{ accepted: number, rejected: Array<{ index, reason }> }`. The
-server SHOULD return 202 (accepted, async) or 200 (accepted, synchronous);
-both are equivalent for the wire's purposes.
+Responses are JSON of shape `{ accepted: number, rejected: Array<{ index, reason }> }`.
+Clients validate the complete response before returning it.
+The server can return 202 for asynchronous acceptance or 200 for synchronous acceptance.
 
 ### `POST /v1/ingest/eval-runs`
 
@@ -75,6 +73,7 @@ interface EvalRunEvent {
   totalCostUsd: number
   totalDurationMs: number
   errorMessage?: string              // present when status === 'errored'
+  insightReport?: InsightReport      // current report contract
 }
 ```
 
@@ -86,7 +85,7 @@ interface EvalRunGenerationSnapshot {
   surfaceHash: string                // stable hash of the candidate surface (pivot key)
   surface?: MutableSurface           // OMITTED to avoid PII when consumer prefers
   cells: EvalRunCellScore[]
-  compositeMean: number
+  compositeMean: number | null       // null when no cell has a task-quality label
   costUsd: number
   durationMs: number
 }
@@ -98,12 +97,14 @@ interface EvalRunGenerationSnapshot {
 interface EvalRunCellScore {
   scenarioId: string
   rep: number                        // 0 for the default; > 0 when reps > 1
-  compositeMean: number              // composite across all judges + dimensions
-  dimensions: Record<                // outer key = judge name; inner = dimension name → score
+  compositeMean: number | null       // null when the cell is unscored
+  dimensions: Record<                // successful scores; failed or missing judges are absent
     string,
     Record<string, number>
   >
-  errorMessage?: string              // present when the dispatch threw
+  terminalOutcome: 'succeeded' | 'failed' | 'cancelled' | 'incomplete' | 'unknown'
+  executionErrorCount: number | null // null when the producer cannot classify errors
+  errorMessage?: string              // present for a dispatch or judge error
 }
 ```
 
@@ -116,10 +117,14 @@ interface TraceSpanEvent {
   spanId: string
   parentSpanId?: string
   name: string
-  startTimeUnixNano: number
-  endTimeUnixNano: number
+  startTimeUnixNano: string // canonical unsigned 64-bit integer encoded in base 10
+  endTimeUnixNano: string   // canonical unsigned 64-bit integer encoded in base 10
   attributes: Record<string, string | number | boolean>
-  events?: Array<{ timeUnixNano, name, attributes? }>
+  events?: Array<{
+    timeUnixNano: string    // canonical unsigned 64-bit integer encoded in base 10
+    name: string
+    attributes?: Record<string, string | number | boolean>
+  }>
   status?: { code: 'OK' | 'ERROR' | 'UNSET', message? }
 
   // Tangle additions (all optional) for pivoting
@@ -143,12 +148,14 @@ Any orchestrator implementing this spec MUST:
    a clear error message). The major component is the breaking-change axis.
 3. **Validate tenant isolation**: queries with `tenantId` X never return
    data tagged with `tenantId` Y. Test this adversarially.
-4. **Honor idempotency**: when an `Idempotency-Key` matches a prior
-   request from the same tenant in the last 24h, return the same response
-   without double-processing.
-5. **Persist eval-runs durably**: at least the event + cell scores must
+4. **Honor idempotency**: require an `Idempotency-Key`; when it matches a prior request to the same endpoint from the same tenant in the last 24 hours, return the same response without processing the body again.
+5. **Deduplicate spans**: store at most one span for each `(tenantId, traceId, spanId)` identity.
+   Accept an exact duplicate as already stored and reject a conflicting payload with the same identity.
+6. **Keep run state monotonic**: accept forward lifecycle transitions, but never replace `finished` or `errored` with a late event.
+   Delayed events can add a missing generation without replacing terminal totals, labels, or status.
+7. **Persist eval-runs durably**: at least the event + cell scores must
    survive an orchestrator restart. Trace spans MAY be best-effort.
-6. **Provide read access**: GET endpoints for the tenant to list + fetch
+8. **Provide read access**: GET endpoints for the tenant to list + fetch
    their own runs. Wire format for reads is NOT part of this spec: each
    orchestrator can pick its own (REST + JSON, gRPC, GraphQL).
 
@@ -163,9 +170,9 @@ Servers SHOULD also:
 
 ## Reference implementation
 
-`examples/hosted-ingest-server/`: a minimal hono-based receiver. ~200
-LOC. Validates auth, accepts ingest, stores in memory, exposes a
-read endpoint. Runs anywhere Node runs.
+`examples/hosted-ingest-server/` is a Hono receiver for local development and contract tests.
+It validates auth, request and response shapes, versions, exact nanosecond strings, request keys, span identity, and monotonic run state.
+It is not production storage because process restart clears its in-memory data.
 
 ```sh
 TENANT_KEY=dev-token TENANT_ID=acme pnpm tsx examples/hosted-ingest-server/server.ts
@@ -185,19 +192,9 @@ receiver's `GET /v1/runs` lists it back.
 
 ---
 
-## Versioning
+## Version
 
-`HostedWireVersion` is `"2026-05-26.v1"`.
-
-- Adding an optional field → no version change.
-- Adding a new endpoint or new event type → minor wire bump
-  (`2026-05-26.v2`).
-- Changing the shape of an existing field, removing a field, or
-  changing semantics of an existing field → major wire bump
-  (`2026-11-XX.v1`); a server may accept both versions during a
-  transition window.
-
-Servers MUST reject requests with `X-Tangle-Wire-Version` they don't
-support, with a 400 listing the versions they DO accept.
-
-The version string IS the spec id: pin against it.
+`HostedWireVersion` is `"2026-07-24.v1"`.
+The current clients emit only this value.
+Servers return 400 for every other value and list the accepted version.
+There are no compatibility readers or migration branches.

package/docs/insight-report.md

@@ -16,7 +16,7 @@ This page walks every section with a real (synthetic) example and explains how t
 ```ts
 interface InsightReport {
   n: number                              // runs analyzed
-  execution: ExecutionInsight            // duration, tokens, models, failures
+  execution: ExecutionInsight            // duration, tokens, errors, terminal outcomes
   composite: ScalarDistribution          // always
   perDimension: Record<string, ScalarDistribution>   // when judgeScores carry dimensions
   costQuality: { cost: ScalarDistribution; pareto: ParetoFigureSpec }   // always
@@ -36,14 +36,28 @@ interface InsightReport {
 ## `execution`: runtime facts, separate from quality
 
 Always present.
-It reports duration, optional queue time, direct input, output, reasoning, cache-read, and cache-write tokens, model-call coverage, model cohorts, explicit failures, and separately reported orchestration aggregates.
+It reports duration, optional queue time, direct input, output, reasoning, cache-read, and cache-write tokens, model-call coverage, model cohorts, execution errors, terminal outcomes, and separately reported orchestration aggregates.
 These fields describe what ran; they do not claim whether the task succeeded.
+`executionErrors` counts child or internal errors reported by the producer.
+`terminalOutcomes` reads only `RunRecord.terminalOutcome`, which must come from root-run or process evidence.
+A child tool error can therefore appear in a run whose terminal outcome is `succeeded`.
+Current OTel and code-agent adapters also preserve process, guardrail, judge, propagated-parent, and unknown error counts in `RunRecord.outcome.raw`.
+These counters are diagnostic and never become task-quality scores.
 
 ```jsonc
 {
   "execution": {
     "durationMs": { "n": 30, "p50": 5400, "p95": 82000, "min": 900, "max": 190000 },
-    "queueMs": { "n": 0, "histogram": [] },
+    "queueMs": {
+      "n": 0,
+      "mean": null,
+      "p50": null,
+      "p95": null,
+      "stddev": null,
+      "min": null,
+      "max": null,
+      "histogram": []
+    },
     "tokenUsage": {
       "totals": { "input": 50132, "output": 471783, "reasoning": 12000, "cached": 60489565, "cacheWrite": 3032227 },
       "input": { "n": 30, "p50": 25, "p95": 56 },
@@ -62,12 +76,40 @@ These fields describe what ran; they do not claim whether the task succeeded.
     },
     "modelCalls": { "runs": 20, "events": 42, "reportingRuns": 30 },
     "models": [{ "model": "claude-opus@2026-07-01", "runs": 20 }],
-    "failures": { "runs": 2, "fraction": 0.067, "reportedErrorEvents": 3, "reportingRuns": 30 }
+    "executionErrors": {
+      "runs": 2,
+      "fraction": 0.067,
+      "events": 3,
+      "reportingRuns": 30,
+      "errorSpanEvents": 3,
+      "errorSpanReportingRuns": 30,
+      "byTerminalOutcome": {
+        "succeeded": { "withErrors": 1, "withoutErrors": 26, "unreported": 0 },
+        "failed": { "withErrors": 0, "withoutErrors": 1, "unreported": 0 },
+        "cancelled": { "withErrors": 0, "withoutErrors": 0, "unreported": 0 },
+        "incomplete": { "withErrors": 0, "withoutErrors": 0, "unreported": 0 },
+        "unknown": { "withErrors": 1, "withoutErrors": 1, "unreported": 0 }
+      }
+    },
+    "terminalOutcomes": {
+      "succeeded": 27,
+      "failed": 1,
+      "cancelled": 0,
+      "incomplete": 0,
+      "unknown": 2
+    }
   }
 }
 ```
 
 Use `distribution.n` for optional fields to distinguish an uncaptured category from a recorded zero.
+When `distribution.n` is zero, `mean`, percentiles, standard deviation, minimum, and maximum are `null`.
+Use `executionErrors.reportingRuns` to assess error-telemetry coverage.
+`errorSpanEvents` preserves the exact child-span error count separately from other reported execution errors.
+The error fraction uses `reportingRuns` as its denominator and is `null` when no run reported error telemetry, so missing telemetry is not treated as a clean run.
+`byTerminalOutcome` is a cross-tab, not a causal recovery claim.
+It keeps reported errors, reported zeroes, and missing error telemetry separate for every terminal result.
+Missing terminal evidence counts as `unknown`, not `failed`.
 Never add `aggregateUsage` to direct `tokenUsage`: orchestration spans may repeat model-call usage from other traces.
 Cost remains in `costQuality`, where observed, estimated, and uncaptured USD stay separate.
 
@@ -97,7 +139,9 @@ Always present. The basic "where are my numbers" view.
 }
 ```
 
-**Read first:** the `composite.mean`. If it's < 0.5, your agent has a ceiling problem, not a tuning problem.
+Read `composite.mean` only when `composite.n > 0`.
+A `null` mean means task quality was not measured, not that quality was zero.
+When a measured mean is below 0.5, inspect the lowest-scoring runs before tuning.
 
 **Read next:** `perDimension`. If `clarity` is high but `concision` is low, your prompts get the right ideas in too many words: different fix than "wrong ideas."
 
@@ -204,13 +248,19 @@ Populated when baseline + candidate candidates are present (auto-detected from t
     "ci95": [0.04, 0.10],          // bootstrap CI on the delta
     "pValue": 0.0008,              // paired t-test
     "n": 40,                       // paired observations
-    "cohensD": 0.41,
+    "unpairedBaselineRuns": 2,
+    "unpairedCandidateRuns": 1,
+    "cohensD": 0.41,              // paired Cohen's dz; null when delta variance is zero
     "mde": 0.06,                   // min detectable effect at current n, 80% power
-    "requiredN": 38                // n needed for observed delta at 80% power
+    "requiredN": 38                // paired n needed at 80% power; null when dz is undefined
   }
 }
 ```
 
+Rows pair only when `(experimentId, scenarioId, seed)` matches.
+Missing `scenarioId` and duplicate identities fail loudly.
+Unmatched rows are reported and excluded from paired statistics.
+
 **Decision rule:**
 - `ci95[0] > threshold` → **SHIP.** Lower bound above your delta threshold means the lift is real at 95% confidence.
 - `ci95[0] ≤ threshold < ci95[1]` → **INCONCLUSIVE.** Expand the corpus or wait for more data.

package/docs/research-report-methodology.md

@@ -28,10 +28,10 @@ corresponding function or option so the doc and the code don't drift.
 
 ## Pairing
 
-Pairs are joined by `(experimentId, seed)` so the comparator and candidate
-share scenario *and* seed. This is the same join `gainHistogram` uses; see
-`pairScoresByKey` in `src/summary-report.ts`. Records on the wrong split or
-with non-finite scores are dropped before pairing.
+Pairs are joined by `(experimentId, scenarioId, seed)` so the comparator and candidate share the same experiment, scenario, and repetition.
+This is the same join used by `gainHistogram`; see `pairRunRecords` in `src/paired-arms.ts`.
+Missing or duplicate identities are rejected.
+Records on the wrong split or with non-finite scores are excluded before pairing, and unmatched valid rows are reported.
 
 ## Decision rule
 
@@ -44,36 +44,35 @@ In order: first match wins:
    not sufficient*; even a `promote` gate must clear the paired test below.
 4. Paired N < `RESEARCH_REPORT_HARD_PAIR_FLOOR` → `needs_more_data` with a
    "below hard floor" reason. Bootstrap CIs degenerate at this size.
-5. ROPE configured AND paired-delta CI ⊂ ROPE → `equivalent`.
-6. Paired-delta CI upper bound < 0 → `reject` (CI excludes a non-negative
+5. `split: 'search'` → `hold`.
+   Search results guide optimization but cannot authorize a release.
+6. ROPE configured AND paired-delta CI ⊂ ROPE → `equivalent`.
+7. Paired-delta CI upper bound < 0 → `reject` (CI excludes a non-negative
    effect). Note: this uses **paired delta only**: not the marginal mean.
-7. Paired N < `minPairs` (soft floor) → `needs_more_data` with the MDE at
+8. Paired N < `minPairs` (soft floor) → `needs_more_data` with the MDE at
    current N attached so the verdict is actionable.
-8. BH-adjusted q ≤ `fdr` AND CI lower bound > 0 → `promote`. The BH q-value
+9. BH-adjusted q ≤ `fdr` AND CI lower bound > 0 → `promote`. The BH q-value
    controls FDR across all candidates in the same sweep; the bootstrap CI
    provides an effect-size guarantee independent of the test.
-9. Otherwise → `hold`.
+10. Otherwise → `hold`.
 
 ## Statistical primitives used
 
 | Quantity | Function | Source file |
 |---|---|---|
 | Marginal CI on score mean | `confidenceInterval` | `statistics.ts` |
-| Cohen's d vs comparator | `cohensD` | `statistics.ts` |
+| Paired Cohen's dz vs comparator | `pairedCohensDz` | `statistics.ts` |
 | Wilcoxon signed-rank (paired) | `wilcoxonSignedRank` | `statistics.ts` |
-| BH-FDR q-values | `benjaminiHochberg` | `power-analysis.ts` |
-| Paired bootstrap CI on median delta | `pairedBootstrap` | `paired-stats.ts` |
-| Bayesian-bootstrap-style Pr(Δ>0), Pr(Δ∈ROPE) | `bootstrapMeanSamples` | `summary-report.ts` (private) |
-| Minimum detectable paired effect | `pairedMde` | `power-analysis.ts` |
+| BH-FDR q-values | `benjaminiHochberg` | `statistics.ts` |
+| Paired bootstrap CI on median delta | `pairedBootstrap` | `statistics.ts` |
+| Bayesian-bootstrap Pr(Δ>0), Pr(Δ∈ROPE) | `bayesianBootstrapMeanSamples` | `summary-report.ts` (private) |
+| Minimum detectable paired effect | `pairedMde` | `statistics.ts` |
 | Run fingerprint | `hashJson(canonicalize(...))` | `pre-registration.ts` |
 
-The Pr(Δ>0) and Pr(Δ∈ROPE) summaries use the bootstrap-prior duality of
-[Rubin 1981]: under a non-informative Dirichlet prior, the bootstrap
-distribution of a sample statistic is its posterior. We expose these as
-posterior summaries on the **mean** delta and the bootstrap CI on the
-**median** delta: the median is more robust to the heavy-tailed score
-distributions seen in agent benchmarks; the mean lets us read off the
-Bayesian-style probability of superiority in a single number.
+The Pr(Δ>0) and Pr(Δ∈ROPE) summaries use Rubin's Bayesian bootstrap.
+Each posterior draw assigns the observed paired deltas Dirichlet(1, ..., 1) weights, implemented as normalized independent Exponential(1) draws.
+The posterior summaries apply to the **mean** delta.
+The separate frequentist bootstrap CI applies to the **median** delta because the median is more robust to heavy-tailed agent scores.
 
 ## MDE
 

package/docs/rollout.md

@@ -25,24 +25,29 @@ Rule applied: where fields conflicted, RunRecord-derived semantics won; the ledg
 | `schema: "tangle.rollout.v1"` | ledger | wire key is `schema` (PR #410's `format` key retired) |
 | `rollout_id` / `parent_rollout_id` | ledger | minted lines use `runId` as `rollout_id` (deterministic); multi-agent producers mint UUIDs and point workers at their supervisor |
 | `run_id` | both | `RunRecord.runId` |
-| `experiment_id`, `candidate_id` | PR #410 | added to the wire (nullable, optional-on-read for pre-unification ledgers) |
+| `experiment_id`, `candidate_id` | PR #410 | required keys; `null` means the producer did not record the value |
 | `generation`, `candidate_index` | ledger | kept as improvement-loop coordinates; now `integer \| null` (`null` = not an improvement loop, `-1` = baseline) |
 | `role` | ledger | enum extended with `agent` for solo eval runs (mint default) |
-| `task.split` | conflict | **RunRecord semantics win**: canonical trainable value is `search` (the optimizer-readable pool); `dev`/`holdout` as in `RunSplitTag`; ledger's `canary` kept; ledger's `train` validates as a legacy alias of `search` and counts as trainable, but new producers emit `search` |
+| `task.split` | conflict | **RunRecord semantics win**: `search` is the trainable pool; `dev` and `holdout` follow `RunSplitTag`; `canary` is retained for release checks |
 | `task.seed`, `task.rep` | ledger | seed from `RunRecord.seed`; rep 0 for minted solo runs |
 | `policy.*` | ledger | + `prompt_hash`, `config_hash`, `agent_profile_cell_id` from PR #410's RunRecord provenance |
 | `messages` | ledger | canonical OpenAI chat-with-tools incl. `reasoning_content`; minted lines inline the final llm span's conversation |
 | `steps` | PR #410 | optional trace-span projections (llm/tool), absent on harness-store-derived lines |
-| `outcome.reward` | conflict | **merged**: `number \| null`; `null` means no verdict exists (a labeled gap, never 0); minted lines always carry a number (`holdoutScore ?? searchScore ?? 0`), forced to 0 when realness-gated |
-| `outcome.realness_gated` | PR #410 | added (optional-on-read, absent = false); the anti-Goodhart gate travels into the data and SFT export refuses gated lines |
-| `outcome.reward_source` / `verdict` / `metrics` / `is_completed` / `is_truncated` / `error` | ledger | unchanged; mint fills `metrics` from `RunRecord.outcome.raw` |
+| `outcome.reward` | conflict | **merged**: `number \| null`; `null` means no verdict exists (a labeled gap, never 0); minting requires an explicit RunRecord task score and rejects execution-only records instead of labeling them 0; realness-gated scores are forced to 0 |
+| `outcome.realness_gated` | PR #410 | required boolean; the anti-Goodhart decision travels with the row and every training export refuses gated lines |
+| `outcome.reward_source` / `verdict` / `metrics` | ledger | unchanged; mint fills `metrics` from `RunRecord.outcome.raw` |
+| `outcome.is_completed` / `is_truncated` / `error` | conflict | mint derives terminal fields from the required `RunRecord.terminalOutcome`; producers use the explicit `unknown` value when terminal evidence is unavailable |
 | `cost.*` | ledger | superset of PR #410's costUsd/totalTokens; `cost.usd` is `null` when `costProvenance.kind === 'uncaptured'` (never a fake 0) |
 | `artifacts.*`, `provenance.*` | ledger | `provenance.capture` gains `mint` alongside `settle-time` / `backfill` |
 | gap discipline | ledger | records without trace spans become labeled gap lines (`messages: []`, `provenance.gap`) AND are listed in `missingTraces`; PR #410's silent skip retired |
 
 ## Export filters (fail-closed)
 
-- SFT: `reward ≥ 1` (configurable) ∧ not realness-gated ∧ trainable split ∧ non-empty transcript. Rows are `{messages, metadata}`.
-- Reward rows: every line with a non-null reward, failures included; unlabeled lines excluded (a gap is not a zero).
-- Verifiers / RFT: transcript-bearing lines only; RFT additionally requires prompt turns before the first assistant turn.
+- SFT, reward rows, Verifiers, and RFT default to completed, non-truncated, error-free, ungated `search` runs with reward greater than `0`.
+- `minimumQualityExclusive` can raise the quality floor.
+- Held-out rows require `allowHeldOutTrainingData: true`.
+- SFT also requires a non-empty transcript.
+- Reward rows require a non-empty user prompt.
+- Verifiers requires both prompt and completion turns.
+- RFT requires prompt turns before the first assistant turn.
 - Release (`rollout-release`): trainable split only; proposer lines dropped unless `--include-proposers`; every string value scrubbed by the 9 deterministic rules (idempotent, so a second pass counts zero); `--push` requires `huggingface-cli` + `HF_TOKEN` and never prints the token.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.126.6",
+  "version": "0.127.0",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {
@@ -160,30 +160,37 @@
     "verify:package": "pnpm run check:skill && node scripts/verify-package-exports.mjs"
   },
   "dependencies": {
-    "@asteasolutions/zod-to-openapi": "^8.5.0",
-    "@ax-llm/ax": "^23.0.1",
-    "@hono/node-server": "^2.0.0",
+    "@asteasolutions/zod-to-openapi": "^9.1.0",
+    "@ax-llm/ax": "^23.0.5",
+    "@hono/node-server": "^2.0.11",
     "@tangle-network/agent-core": "^0.4.20",
     "@tangle-network/agent-interface": "^0.32.0",
     "@tangle-network/tcloud": "^0.4.14",
-    "hono": "^4.12.30",
-    "zod": "^4.3.6"
+    "hono": "^4.12.32",
+    "zod": "^4.4.3"
   },
   "devDependencies": {
-    "@biomejs/biome": "^2.4.15",
-    "@types/node": "^25.6.0",
+    "@biomejs/biome": "^2.5.5",
+    "@types/node": "^26.1.1",
+    "esbuild": "^0.28.1",
     "husky": "^9.1.7",
-    "lint-staged": "^17.0.5",
-    "openapi3-ts": "^4.5.0",
+    "lint-staged": "^17.2.0",
+    "openapi3-ts": "^4.6.0",
     "tsup": "^8.0.0",
-    "typescript": "^5.7.0",
-    "vitest": "^3.0.0",
+    "typescript": "^6.0.3",
+    "vitest": "^4.1.10",
     "yaml": "2.9.0"
   },
   "pnpm": {
     "minimumReleaseAge": 4320,
+    "minimumReleaseAgeExclude": [
+      "esbuild",
+      "vite"
+    ],
     "overrides": {
-      "postcss@<8.5.10": "^8.5.10",
+      "esbuild@>=0.27.3 <0.28.1": "^0.28.1",
+      "postcss@<8.5.18": "^8.5.18",
+      "vite@>=7.0.0 <=7.3.4": "^7.3.5",
       "ws@>=8.0.0 <8.21.0": "^8.21.0"
     }
   },
@@ -199,5 +206,5 @@
     ]
   },
   "license": "MIT",
-  "packageManager": "pnpm@10.22.0"
+  "packageManager": "pnpm@10.34.5"
 }