@tangle-network/agent-eval 0.179.0 → 0.181.0

package/docs/design/mlbenchmarks-review/probes.mts

@@ -0,0 +1,476 @@
+import { createHash } from 'node:crypto'
+import { lstat, mkdtemp, readFile, readdir, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { campaignSplitDigest } from '../../../src/campaign/coverage.ts'
+import { sequentialPairedGate } from '../../../src/campaign/gates/sequential.ts'
+import type { JudgeScore } from '../../../src/campaign/types.ts'
+import { HoldoutAuditor } from '../../../src/contamination-guard.ts'
+import { selfImprove } from '../../../src/contract/self-improve.ts'
+import { evaluatePowerFloorGate } from '../../../src/experiment/ast.ts'
+import { compareCodeUnits, hashCanonical } from '../../../src/ledger-core/canonical.ts'
+import { correlationStudy } from '../../../src/meta-eval/correlation-study.ts'
+import { InMemoryOutcomeStore } from '../../../src/meta-eval/outcome-store.ts'
+import { rubricPredictiveValidity } from '../../../src/meta-eval/rubric-predictive-validity.ts'
+import { compareAdaptationCurves, runAdaptationCurve } from '../../../src/rl/adaptation-eval.ts'
+import { runContaminationProbe } from '../../../src/rl/contamination.ts'
+import { PredictiveValidityResearcher } from '../../../src/rl/predictive-validity-researcher.ts'
+import type { RunRecord } from '../../../src/run-record.ts'
+import { TraceEmitter } from '../../../src/trace/emitter.ts'
+import { InMemoryTraceStore } from '../../../src/trace/store.ts'
+
+// These diagnostics record behavior without asserting that a defect must remain.
+const reviewedBaseRevision = 'fe1cc5111aab5d588bf7db3a3785325635937a91'
+const command = 'pnpm exec tsx docs/design/mlbenchmarks-review/probes.mts'
+const repositoryRoot = fileURLToPath(new URL('../../../', import.meta.url))
+const sourcePaths = ['src', 'package.json', 'pnpm-lock.yaml', 'tsconfig.json']
+
+async function fileEntries(path: string): Promise<Array<{ path: string; sha256: string }>> {
+  const absolutePath = join(repositoryRoot, path)
+  const metadata = await lstat(absolutePath)
+  if (metadata.isDirectory()) {
+    const children = await readdir(absolutePath)
+    return (await Promise.all(children.map(child => fileEntries(`${path}/${child}`)))).flat()
+  }
+  if (!metadata.isFile()) throw new Error(`Source identity requires a regular file: ${path}`)
+  return [{ path, sha256: createHash('sha256').update(await readFile(absolutePath)).digest('hex') }]
+}
+
+// Hash working files, including untracked files, so Git index state cannot hide changes.
+const sourceFiles = (await Promise.all(sourcePaths.map(fileEntries)))
+  .flat()
+  .sort((left, right) => compareCodeUnits(left.path, right.path))
+const sourceIdentity = {
+  algorithm: 'sha256-canonical-file-manifest',
+  paths: sourcePaths,
+  fileCount: sourceFiles.length,
+  digest: hashCanonical({ domain: 'agent-eval-mlbenchmarks-review-source-v1', files: sourceFiles }),
+  dependencyScope: 'Records the manifest and lockfile; assumes dependencies were installed from that lockfile.',
+}
+const diagnosticPath = 'docs/design/mlbenchmarks-review/probes.mts'
+const diagnosticIdentity = {
+  path: diagnosticPath,
+  sha256: createHash('sha256').update(await readFile(join(repositoryRoot, diagnosticPath))).digest('hex'),
+}
+
+async function holdoutReuse() {
+  const runRoot = await mkdtemp(join(tmpdir(), 'agent-eval-mlbenchmarks-review-'))
+  const train = Array.from({ length: 6 }, (_, i) => ({
+    id: `train-${i}`,
+    kind: 'offline-probe',
+  }))
+  const final = Array.from({ length: 6 }, (_, i) => ({
+    id: `final-${i}`,
+    kind: 'offline-probe',
+  }))
+  const rounds = []
+  try {
+    for (const round of [1, 2]) {
+      const finalDispatches: string[] = []
+      let agentCallbacks = 0
+      let judgeCallbacks = 0
+      let proposerCallbacks = 0
+      const result = await selfImprove({
+        scenarios: train,
+        baselineSurface: 'baseline',
+        model: 'deterministic-probe@2026-09-12',
+        agent: async (surface, scenario) => {
+          agentCallbacks++
+          if (scenario.id.startsWith('final-')) finalDispatches.push(scenario.id)
+          return String(surface)
+        },
+        judge: {
+          name: 'marker',
+          dimensions: [{ key: 'pass', description: 'Output has the marker' }],
+          score: ({ artifact }) => {
+            judgeCallbacks++
+            const pass = artifact.includes('marker') ? 1 : 0
+            return { dimensions: { pass }, composite: pass, notes: '' }
+          },
+        },
+        proposer: {
+          kind: 'offline-probe',
+          propose: async () => {
+            proposerCallbacks++
+            return [{
+              surface: 'marker',
+              label: 'marker',
+              rationale: 'Synthetic repeat-access probe',
+            }]
+          },
+        },
+        budget: { generations: 1, populationSize: 1, holdoutScenarios: final },
+        runDir: join(runRoot, `round-${round}`),
+        expectUsage: 'off',
+      })
+      rounds.push({
+        round,
+        gateDecision: result.gateDecision,
+        finalDispatches: finalDispatches.length,
+        distinctFinalIds: new Set(finalDispatches).size,
+        finalSplitDigest: campaignSplitDigest(final, 1),
+        agentCallbacks,
+        judgeCallbacks,
+        proposerCallbacks,
+      })
+    }
+  } finally {
+    // Remove only the directory allocated for this invocation.
+    await rm(runRoot, { recursive: true, force: true })
+  }
+  const auditor = new HoldoutAuditor([
+    { id: 'heldout', payload: 'hidden', split: 'holdout' },
+  ])
+  auditor.get('heldout', 'debugging')
+  auditor.get('heldout', 'evaluation')
+  return {
+    inputs: {
+      calls: 2,
+      trainCases: train.length,
+      finalCases: final.length,
+      generationsPerCall: 1,
+      populationPerGeneration: 1,
+      replicatesPerCase: 1,
+      sameFinalPayloadsOnBothCalls: true,
+      firstResultFedToSecondProposer: false,
+      baselineSurface: 'baseline',
+      proposedSurface: 'marker',
+      scoring: '1 if artifact contains marker, otherwise 0',
+      expectUsage: 'off',
+    },
+    results: {
+      rounds,
+      accessPurposes: auditor.getAccessLog().map(entry => entry.purpose),
+      temporaryRunDirectoriesRemoved: true,
+    },
+    limitations: [
+      'Measures repeated final-set access and debugging access, not empirical overfitting.',
+      'The calls use deterministic local callbacks and independent temporary run directories.',
+      'Does not estimate false-promotion frequency or test a downstream access-control service.',
+    ],
+  }
+}
+
+async function outcomeKeyOrder() {
+  const rows = Array.from({ length: 10 }, (_, i) => {
+    const score = (i + 1) / 10
+    return { score, retention: 1 - score, csat: score }
+  })
+  const results = []
+  for (const keyOrder of ['retention-first', 'csat-first'] as const) {
+    const traces = new InMemoryTraceStore()
+    const outcomes = new InMemoryOutcomeStore()
+    let tick = 0
+    for (const [i, row] of rows.entries()) {
+      const emitter = new TraceEmitter(traces, {
+        runId: `outcome-fixture-${i}`,
+        now: () => ++tick,
+      })
+      await emitter.startRun({ scenarioId: `scenario-${i}` })
+      await emitter.endRun({ pass: true, score: row.score })
+      await outcomes.append({
+        runId: emitter.runId,
+        capturedAt: ++tick,
+        metrics: keyOrder === 'retention-first'
+          ? { retention: row.retention, csat: row.csat }
+          : { csat: row.csat, retention: row.retention },
+      })
+    }
+    results.push({
+      keyOrder,
+      latest: await correlationStudy(traces, outcomes, [{ id: 'score' }], ['csat'], {
+        seed: 1,
+        bootstrapIterations: 500,
+      }),
+      mean: await correlationStudy(traces, outcomes, [{ id: 'score' }], ['csat'], {
+        seed: 1,
+        bootstrapIterations: 500,
+        reduction: 'mean',
+      }),
+    })
+  }
+  return {
+    inputs: {
+      n: rows.length,
+      rows,
+      outcomeRowsPerRun: 1,
+      requestedMetric: 'csat',
+      expectedPearsonForRequestedMetric: 1,
+      expectedSpearmanForRequestedMetric: 1,
+      seed: 1,
+      bootstrapIterations: 500,
+    },
+    results,
+    limitations: [
+      'Tests metric selection and JSON key order with constructed data, not deployment validity.',
+      'Each run has one outcome row, so latest and mean refer to the same requested observation.',
+    ],
+  }
+}
+
+async function adaptationPairing() {
+  const scenariosA = [{ scenarioId: 'easy-only', score: 0.9 }]
+  const scenariosB = [{ scenarioId: 'hard-only', score: 0.1 }]
+  const runner = {
+    run: async ({ scenario }: { scenario: { score: number } }) => scenario.score,
+  }
+  const ks = [0, 1]
+  const reps = 1
+  const a = await runAdaptationCurve<{ scenarioId: string; score: number }>({
+    scenarios: scenariosA, ks, reps, runner,
+  })
+  const b = await runAdaptationCurve<{ scenarioId: string; score: number }>({
+    scenarios: scenariosB, ks, reps, runner,
+  })
+  return {
+    inputs: {
+      scenariosA,
+      scenariosB,
+      ks,
+      reps,
+      observationsPerArm: ks.length * reps,
+      commonScenarios: scenariosA.filter(a => scenariosB.some(b => a.scenarioId === b.scenarioId)).length,
+      sameRunnerForBothArms: true,
+      bootstrapSeed: 1,
+    },
+    results: compareAdaptationCurves(a, b, { seed: 1 }),
+    limitations: [
+      'The two arms differ in task difficulty; zero task identities overlap.',
+      'This probes the adaptation helper, not the separately implemented campaign paired comparison.',
+    ],
+  }
+}
+
+async function contaminationDisplay() {
+  const originals = Array.from({ length: 12 }, (_, i) => ({
+    id: `case-${i}`,
+    score: 1,
+  }))
+  const perturbed = originals.map(scenario => ({ ...scenario, score: 0.4 }))
+  const result = await runContaminationProbe({
+    scenarioId: scenario => scenario.id,
+    originals,
+    perturbed,
+    scoreFn: async scenario => scenario.score,
+  })
+  return {
+    inputs: {
+      n: originals.length,
+      originalScorePerCase: 1,
+      perturbedScorePerCase: 0.4,
+      observationsPerCasePerCondition: 1,
+      modelTrainingExposure: 'No model is used; scores are constructed fixture values.',
+    },
+    results: result,
+    limitations: [
+      'The global Wilcoxon test measures the constructed paired difference; it does not identify contamination as its cause.',
+      'Per-item qValue uses BH on 1 - abs(delta), without a per-item sampling null; it is a display aid in the inspected source.',
+      'The per-item qValues do not drive the global contaminationSuspected result.',
+    ],
+  }
+}
+
+async function negativeOutcomeDirection() {
+  const rows = Array.from({ length: 8 }, (_, i) => ({
+    quality: i / 7,
+    successRate: 1 - i / 7,
+  }))
+  const runs: RunRecord[] = []
+  const outcomes = new InMemoryOutcomeStore()
+  for (const [i, row] of rows.entries()) {
+    const runId = `direction-fixture-${i}`
+    runs.push({
+      runId,
+      experimentId: 'book-review-fixture',
+      candidateId: 'same-candidate',
+      scenarioId: `direction-scenario-${i}`,
+      seed: 0,
+      model: 'fixture@1',
+      promptHash: '0'.repeat(64),
+      configHash: '1'.repeat(64),
+      commitSha: reviewedBaseRevision,
+      wallMs: 0,
+      costUsd: 0,
+      costProvenance: { kind: 'observed', usd: 0 },
+      tokenUsage: { input: 0, output: 0 },
+      terminalOutcome: 'succeeded',
+      splitTag: 'holdout',
+      outcome: { holdoutScore: row.quality, raw: { quality: row.quality } },
+    })
+    await outcomes.append({
+      runId,
+      capturedAt: i,
+      metrics: { success_rate: row.successRate },
+    })
+  }
+  const report = await rubricPredictiveValidity({
+    runs,
+    outcomes,
+    outcomeMetrics: ['success_rate'],
+    rubrics: ['quality'],
+    seed: 1,
+    bootstrapResamples: 100,
+  })
+  const researcher = new PredictiveValidityResearcher({
+    outcomes,
+    outcomeMetrics: ['success_rate'],
+    rubrics: ['quality'],
+  })
+  const researcherReport = await researcher.runValidityCheck(runs)
+  const failures = await researcher.inspectFailures(runs)
+  const changes = await researcher.proposeChange(failures)
+  return {
+    inputs: {
+      n: rows.length,
+      rows,
+      rubric: 'quality',
+      outcome: 'success_rate',
+      desiredOutcomeDirection: 'increase',
+      seed: 1,
+      bootstrapResamples: 100,
+      researcherBootstrapResamples: 500,
+      researcherSeed: 'Derived deterministically by the validity helper',
+      researcherFailureThreshold: 0.5,
+    },
+    results: {
+      report,
+      researcher: {
+        report: researcherReport,
+        failureGroups: failures.length,
+        failures: failures.map(failure => ({
+          code: failure.code,
+          description: failure.description,
+          samples: failure.evidence.samples,
+        })),
+        proposedChanges: changes,
+      },
+    },
+    limitations: [
+      'Magnitude-based bucketing is intentional in existing tests, despite contradictory interface prose.',
+      'A negative association can be desirable for an outcome such as failure rate; direction needs explicit interpretation.',
+      'The researcher recommends increasing rubric weight despite its negative association with desired success rate; it does not execute or deploy that recommendation.',
+      'Constructed perfect correlation establishes neither causal validity nor held-out predictive performance.',
+    ],
+  }
+}
+
+async function sequentialDependence() {
+  const options = { alpha: 0.05, minN: 5, maxN: 100, shuffleSeed: 1337 }
+  const branches = []
+  for (const delta of [-1, 1]) {
+    const scores = (composite: number): Record<string, JudgeScore> => ({
+      judge: { composite, dimensions: {}, notes: '' },
+    })
+    const baseline = new Map(Array.from({ length: 100 }, (_, i) => [
+      `task:${i}`,
+      scores(delta > 0 ? 0 : 1),
+    ]))
+    const candidate = new Map(Array.from({ length: 100 }, (_, i) => [
+      `task:${i}`,
+      scores(delta > 0 ? 1 : 0),
+    ]))
+    const result = await sequentialPairedGate(options).decide({
+      scenarios: [{ id: 'task', kind: 'synthetic-common-sign' }],
+      judgeScores: candidate,
+      baselineJudgeScores: baseline,
+      candidateArtifacts: new Map(),
+      baselineArtifacts: new Map(),
+      cost: { candidate: 0, baseline: 0 },
+      signal: new AbortController().signal,
+    })
+    branches.push({ commonDelta: delta, probability: 0.5, result })
+  }
+  return {
+    inputs: {
+      ...options,
+      branchesEnumerated: branches.length,
+      cellsPerBranch: 100,
+      independentRandomSignsPerExperiment: 1,
+      dataGeneratingProcess: 'Draw one fair sign Z; set all 100 paired deltas equal to Z.',
+      exchangeable: true,
+      marginalMeanDelta: 0,
+      conditionalMeanAfterFirstObservation: 'Z, not necessarily <= 0',
+    },
+    results: {
+      branches,
+      promotionProbabilityUnderMarginalZeroProcess: branches.reduce(
+        (sum, branch) => sum + (branch.result.decision === 'ship' ? branch.probability : 0),
+        0,
+      ),
+    },
+    limitations: [
+      'Enumerates both equiprobable branches exactly; this is not a Monte Carlo estimate.',
+      'The process violates the conditional-mean null required by the e-process core.',
+      'This refutes sufficiency of exchangeability and shuffling, not the valid e-process theorem or any measured production dataset.',
+    ],
+  }
+}
+
+function powerFloor() {
+  const gate = {
+    kind: 'power-floor' as const,
+    target: 0.8,
+    effectGrid: [0.01, 1],
+    sim: { trials: 1, resamples: 1, seed: 1 },
+  }
+  const curve = [{ effect: 0.01, power: 0.1 }, { effect: 1, power: 1 }]
+  return {
+    inputs: {
+      gate,
+      curve,
+      practicalEffectForInterpretation: 0.01,
+      curveSource: 'Supplied deterministic fixture; no power simulation is run.',
+    },
+    results: evaluatePowerFloorGate('floor', gate, curve),
+    limitations: [
+      'The inspected gate documents a maximum-over-grid structural feasibility check; this output matches that contract.',
+      'Passing this gate does not establish target power at the practical effect of 0.01.',
+      'The fixture powers are inputs, not measured or simulated power estimates.',
+    ],
+  }
+}
+
+const observations = {
+  reviewedBaseRevision,
+  sourceIdentity,
+  diagnosticIdentity,
+  command,
+  paidModelCalls: 0,
+  execution: {
+    kind: 'offline deterministic diagnostic',
+    modelCalls: 0,
+    callbackImplementation: 'Local arithmetic and string checks only; no provider clients are supplied.',
+    temporaryRunStorage: 'Allocated under the OS temporary directory and removed in finally.',
+    outputPolicy: 'Preserves current returned measurements; excludes temporary paths, run IDs, and wallclock fields.',
+    assertionPolicy: 'No assertions require the observed defects or policy boundaries to persist.',
+  },
+  probes: {
+    holdoutReuse: await holdoutReuse(),
+    outcomeKeyOrder: await outcomeKeyOrder(),
+    adaptationPairing: await adaptationPairing(),
+    contaminationDisplay: await contaminationDisplay(),
+    negativeOutcomeDirection: await negativeOutcomeDirection(),
+    sequentialDependence: await sequentialDependence(),
+    powerFloor: powerFloor(),
+  },
+  separateVerificationAtReviewedBase: {
+    provenance: 'Historical checks at reviewedBaseRevision; this diagnostic does not rerun them.',
+    sourceRevision: reviewedBaseRevision,
+    checks: [
+      { command: 'pnpm typecheck', result: 'passed' },
+      { command: 'pnpm build', result: 'passed' },
+      { command: 'pnpm verify:package', result: 'passed' },
+    ],
+    tests: {
+      command: 'pnpm test -- tests/experiment/preregistration-acceptance.test.ts tests/experiment/power.test.ts tests/contamination-guard.test.ts tests/rl-predictive-validity-researcher.test.ts tests/rubric-predictive-validity.test.ts tests/meta-eval.test.ts',
+      observedScope: 'The package command expanded to the full Vitest suite.',
+      files: { passed: 399, skipped: 2 },
+      tests: { passed: 5876, skipped: 3 },
+      result: 'passed',
+    },
+    limitation: 'Passing repository checks do not establish correctness of the counterexample behaviors recorded above.',
+  },
+}
+
+console.log(JSON.stringify(observations, null, 2))

package/docs/design/mlbenchmarks-review/sources.json

@@ -0,0 +1,200 @@
+{
+  "accessed_at_utc": "2026-09-13T01:01:32.062338+00:00",
+  "index_url": "https://mlbenchmarks.org/",
+  "index_sha256": "edd63754d75578741714d1fcbf8766e97782c865a4e65345f00967a26b8b4589",
+  "pages": [
+    {
+      "file": "00-preface.html",
+      "url": "https://mlbenchmarks.org/00-preface.html",
+      "title": "Preface - The Emerging Science of Machine Learning Benchmarks",
+      "html_sha256": "df83e9d6dd7ee845e05a4086db47f53af0d85f807be0f512f22f2323f7c00391",
+      "body_words": 2689,
+      "review_status": "complete-text"
+    },
+    {
+      "file": "00-prologue.html",
+      "url": "https://mlbenchmarks.org/00-prologue.html",
+      "title": "Prologue - The Emerging Science of Machine Learning Benchmarks",
+      "html_sha256": "1885c0e15e19f55c2f267d96a0ebd06ff694582f0ad8905ccddd6e85b8033553",
+      "body_words": 625,
+      "review_status": "complete-text"
+    },
+    {
+      "file": "01-introduction.html",
+      "url": "https://mlbenchmarks.org/01-introduction.html",
+      "title": "1 – Introduction - The Emerging Science of Machine Learning Benchmarks",
+      "html_sha256": "507797a0b6c72a0036b8367602ee41a0d0c339a7269a3ea7934128bb3a9185a0",
+      "body_words": 3377,
+      "review_status": "complete-text"
+    },
+    {
+      "file": "02-populations-predictions.html",
+      "url": "https://mlbenchmarks.org/02-populations-predictions.html",
+      "title": "2 – Populations and predictions - The Emerging Science of Machine Learning Benchmarks",
+      "html_sha256": "e6ff7c7ddd73154d199f04d3a43f92b3a16195456f4f23cc0296e257bbb225bc",
+      "body_words": 5563,
+      "review_status": "complete-text"
+    },
+    {
+      "file": "03-detecting-differences.html",
+      "url": "https://mlbenchmarks.org/03-detecting-differences.html",
+      "title": "3 – Detecting differences - The Emerging Science of Machine Learning Benchmarks",
+      "html_sha256": "b9b1df7d9339a71a70d825f0d51c27d17293a0d3dd55a6272f84591c78285cde",
+      "body_words": 5542,
+      "review_status": "complete-text"
+    },
+    {
+      "file": "04-holdout-method.html",
+      "url": "https://mlbenchmarks.org/04-holdout-method.html",
+      "title": "4 – Holdout method - The Emerging Science of Machine Learning Benchmarks",
+      "html_sha256": "c3dd581b0fbc8227d989f0c5863b2060d902cf866c1167dc1fb840a224a04907",
+      "body_words": 8441,
+      "review_status": "complete-text"
+    },
+    {
+      "file": "05-test-set-reuse.html",
+      "url": "https://mlbenchmarks.org/05-test-set-reuse.html",
+      "title": "5 – Test set reuse - The Emerging Science of Machine Learning Benchmarks",
+      "html_sha256": "09261cb5ddeea95fcb84b042eaa2b721393e1a52c40377d13247085d394d6138",
+      "body_words": 5585,
+      "review_status": "complete-text"
+    },
+    {
+      "file": "06-scientific-crisis.html",
+      "url": "https://mlbenchmarks.org/06-scientific-crisis.html",
+      "title": "6 – Scientific crisis - The Emerging Science of Machine Learning Benchmarks",
+      "html_sha256": "500d42c5bce9fbd67b4908e5a5a3fd956d954969ab73c0828e22211be952ef15",
+      "body_words": 7256,
+      "review_status": "complete-text"
+    },
+    {
+      "file": "07-replication-machine-learning.html",
+      "url": "https://mlbenchmarks.org/07-replication-machine-learning.html",
+      "title": "7 – Replication in machine learning - The Emerging Science of Machine Learning Benchmarks",
+      "html_sha256": "c5103477aa1a72b2bbd75fe58e8f1ab2254b247e6529f06c0bebb12950335bf4",
+      "body_words": 8899,
+      "review_status": "complete-text"
+    },
+    {
+      "file": "08-forces-against-crisis.html",
+      "url": "https://mlbenchmarks.org/08-forces-against-crisis.html",
+      "title": "8 – Forces against crisis - The Emerging Science of Machine Learning Benchmarks",
+      "html_sha256": "874764d22ba2c712877fc86fa2e465297670071ca8c58fd011769c1dfb1be4f8",
+      "body_words": 8128,
+      "review_status": "complete-text"
+    },
+    {
+      "file": "10-generative-models.html",
+      "url": "https://mlbenchmarks.org/10-generative-models.html",
+      "title": "10 – Generative models - The Emerging Science of Machine Learning Benchmarks",
+      "html_sha256": "2e89354b87cc473e1190f687a4286f80754efc86257ec06b3569ea13d6f23729",
+      "body_words": 11490,
+      "review_status": "complete-text"
+    },
+    {
+      "file": "11-evaluating-language-models.html",
+      "url": "https://mlbenchmarks.org/11-evaluating-language-models.html",
+      "title": "11 – Evaluating language models - The Emerging Science of Machine Learning Benchmarks",
+      "html_sha256": "01a08570e33f9446bc33f0b0ac6fa60c18ec9d356672944da51b4ff802059bad",
+      "body_words": 13399,
+      "review_status": "complete-text"
+    },
+    {
+      "file": "12-problem-aggregation.html",
+      "url": "https://mlbenchmarks.org/12-problem-aggregation.html",
+      "title": "12 – The problem of aggregation - The Emerging Science of Machine Learning Benchmarks",
+      "html_sha256": "f09917ed330f01aa573d9609469e82745ebc50a218484c88685ff21065217104",
+      "body_words": 8857,
+      "review_status": "complete-text"
+    },
+    {
+      "file": "13-model-moves-data.html",
+      "url": "https://mlbenchmarks.org/13-model-moves-data.html",
+      "title": "13 – When the model moves the data - The Emerging Science of Machine Learning Benchmarks",
+      "html_sha256": "673f0e4f300cf4d27b4bdab92593a0e2b72fcd63092a51afbebf614ed4173ae8",
+      "body_words": 8832,
+      "review_status": "complete-text"
+    },
+    {
+      "file": "14-evaluation-frontier.html",
+      "url": "https://mlbenchmarks.org/14-evaluation-frontier.html",
+      "title": "14 – Evaluation at the frontier - The Emerging Science of Machine Learning Benchmarks",
+      "html_sha256": "71a5d336dc99e9135651218344564ffaea2e9140e67295860735c3b76df15154",
+      "body_words": 9464,
+      "review_status": "complete-text"
+    },
+    {
+      "file": "15-epilogue.html",
+      "url": "https://mlbenchmarks.org/15-epilogue.html",
+      "title": "15 – Epilogue - The Emerging Science of Machine Learning Benchmarks",
+      "html_sha256": "a0b57ccdb46107e3a82d2a9f3bd16601e950fd3fb13006030062edc61379c651",
+      "body_words": 1624,
+      "review_status": "complete-text"
+    }
+  ],
+  "review_date_local": "2026-09-12",
+  "review_timezone": "America/Los_Angeles",
+  "repository": {
+    "url": "https://github.com/tangle-network/agent-eval",
+    "revision": "fe1cc5111aab5d588bf7db3a3785325635937a91",
+    "package_version": "0.180.0",
+    "base": "origin/main"
+  },
+  "coverage": {
+    "linked_reading_pages": 16,
+    "reviewed_pages": 16,
+    "body_words": 109771,
+    "word_count_method": "Whitespace split of BeautifulSoup chapter-body get_text with spaces; includes headings, tables, equations, notes, and references.",
+    "extraction_audit": "Paragraph extraction plus every omitted non-whitespace text node were read; figures/tables supporting conclusions were checked in HTML or PDF context.",
+    "missing_chapter": {
+      "number": 9,
+      "status": "unavailable from inspected live index",
+      "checked_404_paths": [
+        "https://mlbenchmarks.org/09-annotation.html",
+        "https://mlbenchmarks.org/09-data-annotation.html",
+        "https://mlbenchmarks.org/09-annotations.html"
+      ]
+    },
+    "independent_reproduction_of_cited_studies": false,
+    "print_edition_reviewed": false
+  },
+  "review_partitions": [
+    {
+      "pages": [
+        "00-preface.html",
+        "00-prologue.html",
+        "01-introduction.html",
+        "02-populations-predictions.html",
+        "03-detecting-differences.html",
+        "04-holdout-method.html",
+        "05-test-set-reuse.html",
+        "06-scientific-crisis.html"
+      ],
+      "body_words": 39078
+    },
+    {
+      "pages": [
+        "07-replication-machine-learning.html",
+        "08-forces-against-crisis.html",
+        "10-generative-models.html",
+        "11-evaluating-language-models.html"
+      ],
+      "body_words": 41916
+    },
+    {
+      "pages": [
+        "12-problem-aggregation.html",
+        "13-model-moves-data.html",
+        "14-evaluation-frontier.html",
+        "15-epilogue.html"
+      ],
+      "body_words": 28777
+    }
+  ],
+  "visual_inspection": {
+    "chapter_7": "Figures 7.2–7.4 checked using linked chapter PDF; HTML used for references.",
+    "chapter_11": "Figures 11.3–11.6 checked using linked chapter PDF; HTML used for references.",
+    "chapters_12_to_14": "All eight SVG figures checked; all three chapter-12 tables checked in HTML.",
+    "chapters_1_to_6": "Captions and surrounding arguments read; chapter-2 confusion matrix and chapter-5 rate table checked in HTML."
+  }
+}