@tangle-network/agent-eval 0.216.0 → 0.217.1

package/docs/design/mlbenchmarks-review/probes.mts

@@ -8,7 +8,8 @@ import { sequentialPairedGate } from '../../../src/campaign/gates/sequential.ts'
 import type { JudgeScore } from '../../../src/campaign/types.ts'
 import { HoldoutAuditor } from '../../../src/contamination-guard.ts'
 import { selfImprove } from '../../../src/contract/self-improve.ts'
-import { evaluatePowerFloorGate } from '../../../src/experiment/ast.ts'
+import { ValidationError } from '../../../src/errors.ts'
+import { evaluatePowerFloorGate } from '../../../src/experiment/ast/validity.ts'
 import { compareCodeUnits, hashCanonical } from '../../../src/ledger-core/canonical.ts'
 import { correlationStudy } from '../../../src/meta-eval/correlation-study.ts'
 import { InMemoryOutcomeStore } from '../../../src/meta-eval/outcome-store.ts'
@@ -21,7 +22,8 @@ import { TraceEmitter } from '../../../src/trace/emitter.ts'
 import { InMemoryTraceStore } from '../../../src/trace/store.ts'
 
 // These diagnostics record behavior without asserting that a defect must remain.
-const reviewedBaseRevision = 'fe1cc5111aab5d588bf7db3a3785325635937a91'
+const reviewedObservations = JSON.parse(await readFile(new URL('./observations.json', import.meta.url), 'utf8'))
+const reviewedBaseRevision = reviewedObservations.reviewedBaseRevision
 const command = 'pnpm exec tsx docs/design/mlbenchmarks-review/probes.mts'
 const repositoryRoot = fileURLToPath(new URL('../../../', import.meta.url))
 const sourcePaths = ['src', 'package.json', 'pnpm-lock.yaml', 'tsconfig.json']
@@ -223,6 +225,13 @@ async function adaptationPairing() {
   const b = await runAdaptationCurve<{ scenarioId: string; score: number }>({
     scenarios: scenariosB, ks, reps, runner,
   })
+  let comparison: ReturnType<typeof compareAdaptationCurves> | { refused: true; code: string; message: string }
+  try {
+    comparison = compareAdaptationCurves(a, b, { seed: 1 })
+  } catch (error) {
+    if (!(error instanceof ValidationError) || !error.message.includes('scenario pairs differ')) throw error
+    comparison = { refused: true, code: error.code, message: error.message }
+  }
   return {
     inputs: {
       scenariosA,
@@ -234,7 +243,7 @@ async function adaptationPairing() {
       sameRunnerForBothArms: true,
       bootstrapSeed: 1,
     },
-    results: compareAdaptationCurves(a, b, { seed: 1 }),
+    results: comparison,
     limitations: [
       'The two arms differ in task difficulty; zero task identities overlap.',
       'This probes the adaptation helper, not the separately implemented campaign paired comparison.',
@@ -278,6 +287,7 @@ async function negativeOutcomeDirection() {
   }))
   const runs: RunRecord[] = []
   const outcomes = new InMemoryOutcomeStore()
+  const targetOutcome = { id: 'success_rate', direction: 'higher-is-better' as const }
   for (const [i, row] of rows.entries()) {
     const runId = `direction-fixture-${i}`
     runs.push({
@@ -307,14 +317,14 @@ async function negativeOutcomeDirection() {
   const report = await rubricPredictiveValidity({
     runs,
     outcomes,
-    outcomeMetrics: ['success_rate'],
+    outcomeMetrics: [targetOutcome],
     rubrics: ['quality'],
     seed: 1,
     bootstrapResamples: 100,
   })
   const researcher = new PredictiveValidityResearcher({
     outcomes,
-    outcomeMetrics: ['success_rate'],
+    targetOutcome,
     rubrics: ['quality'],
   })
   const researcherReport = await researcher.runValidityCheck(runs)
@@ -325,8 +335,7 @@ async function negativeOutcomeDirection() {
       n: rows.length,
       rows,
       rubric: 'quality',
-      outcome: 'success_rate',
-      desiredOutcomeDirection: 'increase',
+      outcome: targetOutcome,
       seed: 1,
       bootstrapResamples: 100,
       researcherBootstrapResamples: 500,
@@ -347,9 +356,8 @@ async function negativeOutcomeDirection() {
       },
     },
     limitations: [
-      'Magnitude-based bucketing is intentional in existing tests, despite contradictory interface prose.',
-      'A negative association can be desirable for an outcome such as failure rate; direction needs explicit interpretation.',
-      'The researcher recommends increasing rubric weight despite its negative association with desired success rate; it does not execute or deploy that recommendation.',
+      'The desired outcome direction is explicit; association remains descriptive and does not authorize a rubric change.',
+      'The researcher reports recommendations; this diagnostic does not execute or deploy them.',
       'Constructed perfect correlation establishes neither causal validity nor held-out predictive performance.',
     ],
   }
@@ -411,6 +419,7 @@ function powerFloor() {
   const gate = {
     kind: 'power-floor' as const,
     target: 0.8,
+    minimumEffect: 0.01,
     effectGrid: [0.01, 1],
     sim: { trials: 1, resamples: 1, seed: 1 },
   }
@@ -419,14 +428,13 @@ function powerFloor() {
     inputs: {
       gate,
       curve,
-      practicalEffectForInterpretation: 0.01,
+      practicalEffectForInterpretation: gate.minimumEffect,
       curveSource: 'Supplied deterministic fixture; no power simulation is run.',
     },
     results: evaluatePowerFloorGate('floor', gate, curve),
     limitations: [
-      'The inspected gate documents a maximum-over-grid structural feasibility check; this output matches that contract.',
-      'Passing this gate does not establish target power at the practical effect of 0.01.',
-      'The fixture powers are inputs, not measured or simulated power estimates.',
+      'The gate compares power at minimumEffect with target; power 0.1 at effect 0.01 fails despite maximum power 1.',
+      'The fixture powers are supplied diagnostic inputs, not measured or simulated estimates.',
     ],
   }
 }
@@ -454,23 +462,7 @@ const observations = {
     sequentialDependence: await sequentialDependence(),
     powerFloor: powerFloor(),
   },
-  separateVerificationAtReviewedBase: {
-    provenance: 'Historical checks at reviewedBaseRevision; this diagnostic does not rerun them.',
-    sourceRevision: reviewedBaseRevision,
-    checks: [
-      { command: 'pnpm typecheck', result: 'passed' },
-      { command: 'pnpm build', result: 'passed' },
-      { command: 'pnpm verify:package', result: 'passed' },
-    ],
-    tests: {
-      command: 'pnpm test -- tests/experiment/preregistration-acceptance.test.ts tests/experiment/power.test.ts tests/contamination-guard.test.ts tests/rl-predictive-validity-researcher.test.ts tests/rubric-predictive-validity.test.ts tests/meta-eval.test.ts',
-      observedScope: 'The package command expanded to the full Vitest suite.',
-      files: { passed: 399, skipped: 2 },
-      tests: { passed: 5876, skipped: 3 },
-      result: 'passed',
-    },
-    limitation: 'Passing repository checks do not establish correctness of the counterexample behaviors recorded above.',
-  },
+  separateVerificationAtReviewedBase: reviewedObservations.separateVerificationAtReviewedBase,
 }
 
 console.log(JSON.stringify(observations, null, 2))

package/docs/design/statistics-decisions.md

@@ -1,6 +1,6 @@
 # Statistics: adoption decisions
 
-This is the standing decision record for `src/statistics.ts` and the estimators layered on it.
+This is the standing decision record for `src/statistics/` and the estimators layered on it.
 It answers three questions: which statistic is trustworthy, where each number comes from, and what a caller is allowed to ask for at 3–10 repetitions per arm.
 The API itself is documented in [`concepts.md`](../concepts.md); this file records *why* each statistic is kept, fixed, or refused.
 
@@ -63,7 +63,7 @@ All are repaired, each with a regression test that fails on the pristine blob an
 | `pairedTTest` (zero-variance branch) | defective, **fixed** | keep-and-fix | Line 199 returns `p = 0` when the standard error is zero. `pairedTTest([0, 0, 0.5], [0.5, 0.5, 1])` — a constant `+0.5` shift on three pairs — returns `{t: null, df: 2, p: 0}`. That is absolute certainty from three observations where the exact signed-rank floor at `n = 3` is `0.25`, and the object is internally inconsistent (`t` serialized as `null` alongside `p = 0`). | `pairedCohensDz` handles the identical condition correctly by returning `null` and documents why. The module should answer the degenerate case once, the same way, everywhere. |
 | `cohensD` (degenerate branches) | defective, **fixed** | keep-and-fix | Returns a silent `0` twice: when either sample has fewer than two observations, and when the pooled standard deviation is zero. `cohensD([1,1,1],[2,2,2])` returns `0` — "no effect" for a maximal, zero-variance separation. | Third distinct answer to the same degenerate condition, after `pairedTTest`'s `p = 0` and `pairedCohensDz`'s `null`. Only `pairedCohensDz` is right. |
 | `mulberry32` | defective, **fixed** | keep-and-fix | `let s = seed \| 0 \|\| 0x9e3779b9` collapses seed `0` to the golden-ratio constant. `mulberry32(0)` and `mulberry32(0x9e3779b9 \| 0)` both emit `[0.3588899802, 0.1059032613, 0.6752904793]`. | Seed `0` is a common default, so two runs a caller believes are independent replicates are the same run. |
-| `makeRng` / bootstrap seeding | defective, **fixed** | keep-and-fix | `makeRng` returns raw `Math.random` when `opts.seed` is undefined. Two back-to-back `confidenceInterval` calls on identical input returned `lower = 0.30000000000000004` and `lower = 0.3`, `upper = 0.6749999999999999` and `upper = 0.6499999999999999`. Seeding works when supplied (`seed: 7` reproduced exactly). | This contradicts `mulberry32`'s own docstring, which states that a seed is required because unseeded randomness in gate verdicts is non-reproducible by construction. `src/contract/analyze-runs.ts:908` is the live call site that passes no seed; the gates in `held-out-gate.ts`, `promotion-policy.ts`, `statistical-heldout.ts`, `measured-comparison.ts` and `summary-report.ts` all thread one through. |
+| `makeRng` / bootstrap seeding | defective, **fixed** | keep-and-fix | `makeRng` returns raw `Math.random` when `opts.seed` is undefined. Two back-to-back `confidenceInterval` calls on identical input returned `lower = 0.30000000000000004` and `lower = 0.3`, `upper = 0.6749999999999999` and `upper = 0.6499999999999999`. Seeding works when supplied (`seed: 7` reproduced exactly). | This contradicts `mulberry32`'s own docstring, which states that a seed is required because unseeded randomness in gate verdicts is non-reproducible by construction. `src/contract/analyze-runs.ts` is the live call site that passes no seed; the gates in `held-out-gate.ts`, `promotion-policy.ts`, `statistical-heldout.ts`, `measured-comparison.ts` and `summary-report.ts` all thread one through. |
 
 ## Correct mathematics, misleading regime
 
@@ -100,12 +100,12 @@ These 17 matched their references with zero mismatches and stay as they are, pin
 | `cliffsDelta` | keep-with-CI-oracle | Matched a hand-computed `(#gt − #lt)/(n₁n₂)` reference exactly (`0.312500`). Note the orientation is *after over before*, the opposite sign to the textbook `δ` written over `(a, b)`; the docstring states this and the parameter names `(before, after)` carry it. |
 | `pairedCohensDz` | keep-with-CI-oracle | Matched `mean(d)/sd(d)` exactly (`2.184070`). It is the only function in the module that handles the zero-variance degenerate case correctly, returning `null` with a docstring explaining that the standardized effect is undefined rather than an arbitrarily large finite number. Treat it as the reference behaviour the other degenerate branches should adopt. |
 | `weightedMean`, `partialCredit`, `weightedComposite`, `interpretCliffs`, `normalizeScores` | keep-with-CI-oracle | Arithmetic and thresholding, verified by direct evaluation (`weightedMean` of `[{1,w2},{4,w1}]` is `2.000000`; `partialCredit(3,4)` is `0.750000`). |
-| `welchsTTest`, `compareToBaseline` (`src/baseline.ts`) | keep-with-CI-oracle, **covered** | Correct, and previously unguarded: no test file imported either, which is the structural reason a duplicated broken CDF survived here. It is exported publicly and it gates improved / regressed / stable verdicts. Now pinned against `scipy.stats.ttest_ind(equal_var=False)` in the oracle fixture and exercised through `compareToBaseline` in `tests/statistics.test.ts`. |
+| `welchsTTest`, `compareToBaseline` (`src/baseline.ts`) | keep-with-CI-oracle, **covered** | Correct, and previously unguarded: no test file imported either, which is the structural reason a duplicated broken CDF survived here. It is exported publicly and it gates improved / regressed / stable verdicts. Now pinned against `scipy.stats.ttest_ind(equal_var=False)` in the oracle fixture and exercised through `compareToBaseline` in a statistics test file that is no longer in the tree. |
 
 ## Public surface change
 
 `normalCdf` and `studentTCdf` were private on `origin/main` and are now exported with explicit accuracy contracts.
-`baseline.ts` owns the single Welch implementation, and `contract/analyze-runs.ts` consumes that result instead of carrying another normal-approximation copy.
+`baseline.ts` owns the single Welch implementation, and `src/contract/analyze-runs.ts` consumes that result instead of carrying another normal-approximation copy.
 That is the right trade: one implementation with one accuracy contract beats three copies of which two were wrong.
 It also means both functions are now public API and owe callers a stated accuracy bound.
 `normalCdf` is documented at `7.5e-8` absolute.
@@ -225,7 +225,7 @@ Promotion paths now use `pairedDeltaTest`: an exact one-sided sign test carries
 ## Consumer notice
 
 Every published version from **0.1.0 (2026-04-20)** through **0.133.0 (2026-07-27)** reported p-values that are too small from the functions listed below.
-The defect entered at the initial commit (`7d5032b`, `src/statistics.ts:747`) and was present in every release since.
+The defect entered at the initial commit (`7d5032b`, in the then single-file statistics module) and was present in every release since.
 
 The normal CDF itself was corrected in 0.133.1; every other row below was still open at that release.
 
@@ -290,18 +290,18 @@ That is why the option stays optional — a required seed would push the caller
 
 | Site | Formula | Consumer | Seed source |
 | --- | --- | --- | --- |
-| `statistics/internal.ts` `makeRng` | mulberry32 | every resampling path in `src/statistics` | caller seed, else FNV-1a over the IEEE-754 bytes of the observations |
-| `statistics/descriptive.ts` `confidenceInterval` | percentile bootstrap | reported interval | `makeRng(opts.seed, scores)` |
-| `statistics/paired-tests.ts` `pairedBootstrap` | paired percentile bootstrap | promotion intervals | `makeRng(opts.seed, deltas)` |
-| `statistics/rank-tests.ts` `mannWhitneyU`, `wilcoxonSignedRank` | permutation null | exact rank-test p | `makeRng(opts.seed, …)`, order-independent for the symmetric two-sample case |
+| `src/statistics/internal.ts` `makeRng` | mulberry32 | every resampling path in `src/statistics` | caller seed, else FNV-1a over the IEEE-754 bytes of the observations |
+| `src/statistics/descriptive.ts` `confidenceInterval` | percentile bootstrap | reported interval | `makeRng(opts.seed, scores)` |
+| `src/statistics/paired-tests.ts` `pairedBootstrap` | paired percentile bootstrap | promotion intervals | `makeRng(opts.seed, deltas)` |
+| `src/statistics/rank-tests.ts` `mannWhitneyU`, `wilcoxonSignedRank` | permutation null | exact rank-test p | `makeRng(opts.seed, …)`, order-independent for the symmetric two-sample case |
 | `promotion-gate.ts` | bootstrap over the two arms | ship / hold verdict | `options.seed`, else `hashSeed(baseline, candidate)` |
 | `held-out-gate.ts` | delegates to `pairedBootstrap` | held-out gate verdict | caller seed, else the paired observations |
 | `summary-report.ts` `bayesianBootstrapMeanSamples` | Dirichlet-weight bootstrap | reported posterior and gain histogram | `makeRng(seed, deltas)` |
-| `meta-eval/rubric-predictive-validity.ts` `bootstrapCi` | percentile bootstrap of Pearson r | rubric verdict (`load_bearing` / `informative`) | `makeRng(input.seed, xs, ys)` |
-| `meta-eval/correlation-study.ts` `bootstrapPearsonCi` | percentile bootstrap of Pearson r | reported correlation CI | `makeRng(options.seed, xs, ys)` |
-| `campaign/search-policy.ts` `aide` | Thompson draw over node posteriors, the debug coin, the debug and uniform draws | the parent an expansion extends (spends budget, claims nothing) | `mulberry32(seed ^ imul(completed expansions + 1, 0x9e3779b1) ^ salt)`, a pure function of the ledger |
-| `rl/adaptation-eval.ts` `compareAdaptationCurves` | bootstrap of per-k mean deltas | per-k CI and AUC delta | `makeRng(opts.seed, a-means, b-means)` |
-| `campaign/gates/sequential.ts` | seeded Fisher-Yates over paired deltas | exchangeability guard before the e-process | `shuffleSeed`, default 1337, data-independent by construction |
+| `src/meta-eval/rubric-predictive-validity.ts` `bootstrapCi` | percentile bootstrap of Pearson r | rubric verdict (`load_bearing` / `informative`) | `makeRng(input.seed, xs, ys)` |
+| `src/meta-eval/correlation-study.ts` `bootstrapPearsonCi` | percentile bootstrap of Pearson r | reported correlation CI | `makeRng(options.seed, xs, ys)` |
+| `src/campaign/search-policy.ts` `aide` | Thompson draw over node posteriors, the debug coin, the debug and uniform draws | the parent an expansion extends (spends budget, claims nothing) | `mulberry32(seed ^ imul(completed expansions + 1, 0x9e3779b1) ^ salt)`, a pure function of the ledger |
+| `src/rl/adaptation-eval.ts` `compareAdaptationCurves` | bootstrap of per-k mean deltas | per-k CI and AUC delta | `makeRng(opts.seed, a-means, b-means)` |
+| `src/campaign/gates/sequential.ts` | seeded Fisher-Yates over paired deltas | exchangeability guard before the e-process | `shuffleSeed`, default 1337, data-independent by construction |
 
 ### Draws that reach no result
 
@@ -319,9 +319,9 @@ These stay on `Math.random`. Each produces an identifier or a retry delay; no re
 
 ### What changed in this pass
 
-Five byte-identical private copies of mulberry32 existed alongside the canonical one, in `meta-eval/rubric-predictive-validity.ts`, `rl/active-curriculum.ts`, `rl/adaptation-eval.ts`, `summary-report.ts`, and `promotion-gate.ts`.
+Five byte-identical private copies of mulberry32 existed alongside the canonical one, in `src/meta-eval/rubric-predictive-validity.ts`, `src/rl/active-curriculum.ts`, `src/rl/adaptation-eval.ts`, `summary-report.ts`, and `promotion-gate.ts`.
 Four of them fell back to `Math.random` when the caller passed no seed, so four reported statistics were silently non-reproducible.
-`meta-eval/correlation-study.ts` called `Math.random` directly inside its bootstrap and had no seed option at all.
+`src/meta-eval/correlation-study.ts` called `Math.random` directly inside its bootstrap and had no seed option at all.
 All six now route through `makeRng`, and `correlationStudy` gained the `seed` option its sibling already had.
 `Math.random` references in `src/` fell from 19 to 10, and none of the remaining ten is result-bearing.
 
@@ -334,7 +334,7 @@ The audit in #411 counted assertions that prove nothing, module mocks, and condi
 - *A guard's accept case.* `assertLlmRoute`, `assertNoHiddenLeak`, `assertCapabilityHeadroom`, `assertMultishotShotResult`, `assertMatchedMethodLimits`, `assertGateReport`, `assertAnalystBenchmarkObservation`, `assertDeterministicOracle`, `assertDenominatorIntact`, and `assertJsonValue` return nothing. For these the no-throw IS the assertion: it pins the false-positive boundary of a fail-closed gate, and a gate that refuses a legal input is as broken as one that admits an illegal one. Each sits next to a `toThrow` sibling that proves the refusal. Kept.
 - *A value-returning call.* `validateRunRecord` and friends return the validated value, so the no-throw discarded the only thing worth checking. Converted to assert the returned value.
 
-**Module mocks.** 5 sites, each now carrying a one-line justification in its file header: four wrap a `node:` builtin to reproduce a race or an absent native binding that a real filesystem cannot be timed into (`trace/store.test.ts`, `analyst/benchmark-verification-artifacts.test.ts`, `analyst/benchmark-command-public-data.test.ts`, `rollout/readers/opencode-sqlite-lazy.test.ts`); one replaces the provider call in the wire handler test, which is about request validation and error mapping (`tests/wire/handlers.test.ts`).
+**Module mocks.** 5 sites, each now carrying a one-line justification in its file header: four wrap a `node:` builtin to reproduce a race or an absent native binding that a real filesystem cannot be timed into (`src/trace/store.test.ts`, `src/analyst/benchmark-verification-artifacts.test.ts`, `src/analyst/benchmark-command-public-data.test.ts`, the opencode-sqlite lazy-reader test (no longer in the tree)); one replaces the provider call in the wire handler test, which is about request validation and error mapping (`tests/wire/handlers.test.ts`).
 
 **Conditional skips.** 6 sites, all environment-gated, none silent:
 

package/docs/design.md

@@ -31,7 +31,7 @@ The stack context matters only if you adopt more of it later.
 ## The dependency rule
 
 This section is the public rationale.
-The enforceable maintainer rule lives in [`CLAUDE.md`](../CLAUDE.md#dependency-and-evidence-boundaries).
+The enforceable maintainer rule lives in [`AGENTS.md`](../AGENTS.md#dependency-and-evidence-boundaries).
 
 **`agent-eval` has zero upward dependencies on a consumer.**
 This is what keeps the package reusable outside our own stack: nothing in here imports from `agent-runtime`, `agent-knowledge`, or `sandbox`, whether at runtime, in development dependencies, or as type-only imports.

package/docs/evaluation-integrity.md

@@ -189,7 +189,7 @@ For outcome associations and direct score calibration, use [the outcome-validity
 
 These integrity checks establish execution and measurement behavior.
 They do not establish that a particular optimizer improves agents across domains.
-The [historical evidence audit](./design/self-improvement-evidence-audit.md) records prior gains, nulls, regressions, and their limits.
+The [historical evidence audit](../archive/2026-09-13-self-improvement-evidence-audit.md) records prior gains, nulls, regressions, and their limits.
 
 For a benefit experiment, define the user behavior and useful effect before search.
 Compare the starting agent, a direct edit or simple search baseline, and the proposed improvement method at equal actual resources.

package/docs/experiment.md

@@ -21,7 +21,7 @@ Calling `registered.decide()` does not automatically run admission, power, budge
 
 | Object | Entry point | Purpose |
 | --- | --- | --- |
-| Registered rules | [Rule types](../src/experiment/ast.ts) and [`ExperimentSpec`](../src/experiment/define.ts) | Describe admission, estimation, intervals, and decisions as data. |
+| Registered rules | [Rule types](../src/experiment/ast/) and [`ExperimentSpec`](../src/experiment/define.ts) | Describe admission, estimation, intervals, and decisions as data. |
 | Define / seal / execute | `defineExperiment`, `sealExperiment`, `amendExperiment`, `openSealedExperiment` | Validate, identify, and execute the registered rules. |
 | Cluster-aware power | `clusteredPower`, `assertDesignAdequate` | Assess a declared effect under a simulated outcome model and cluster-count policy. |
 | Denominator chain | `buildFunnel`, `executeAdmissionRule`, `composeFunnels`, `renderFunnelTable` | Reconcile retained and excluded evidence. |

package/docs/proofs/campaign-attempt-retention-20261001/README.md

@@ -44,8 +44,8 @@ Run [qualify.mjs](artifacts/current/qualify.mjs) with these environment variable
 
 - `PROOF_ROOT`: a new output directory.
 - `BASELINE_ROOT`: the preserved predecessor evidence directory containing `before-resume`.
-- `EVAL_MODULE`: the candidate's built `dist/campaign/index.js`.
-- `EVAL_COST_MODULE`: the candidate's built `dist/index.js`.
+- `EVAL_MODULE`: the candidate's built campaign entry point.
+- `EVAL_COST_MODULE`: the candidate's built package entry point (`main` in `package.json`).
 
 Build, typecheck, and package verification completed successfully.
 No unit tests were added or run.