@hecer/yoke 1.21.1 → 1.22.0

package/.claude-plugin/plugin.json

@@ -2,7 +2,7 @@
   "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
   "name": "yoke",
   "displayName": "Yoke",
-  "version": "1.21.1",
+  "version": "1.22.0",
   "description": "Cross-agent coding harness for Claude, Codex, Gemini, Qwen, OpenCode, Kilo, Pi and Hermes: one curated skill canon plus mechanical safety gates and an autonomous loop via the yoke CLI.",
   "author": { "name": "HECer", "url": "https://github.com/HECer" },
   "homepage": "https://github.com/HECer/yoke#readme",

package/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "yoke",
-  "version": "1.21.1",
+  "version": "1.22.0",
   "description": "Cross-agent coding discipline, mechanical gates, and release workflows for eight supported harnesses",
   "skills": "./canon/skills/",
   "hooks": "./hooks/hooks.json"

package/CHANGELOG.md

@@ -1,5 +1,32 @@
 # Changelog
 
+## 1.22.0 — 2026-10-03
+
+### Added
+- Add durable routing attempt reservations independent of optional analytics retention, per-call goal admission and persistent consumption accounting, including planning and interrupted calls.
+- Add optional evidence-based economic ranking inside capability routing. Cost, speed and balanced objectives require comparable, complete execution evidence; configured capability floors and fallback limits remain authoritative.
+- Add `yoke goal assess` to prepare a bound goal contract explicitly, and record versioned provenance for setup model profiles as configuration priors rather than measured price or capability claims.
+- Add compact persistent failure signatures: repeated unchanged failures request diagnosis and then stop instead of exhausting repeated identical turns. Completion failures carry typed reasons into repair planning.
+- Add optional multi-step browser journeys and structured proof reports. Project checks can bind declared artifacts and journeys to an acceptance digest, source fingerprint, runtime environment and individual requirement results.
+- Add versioned direct-Codex benchmark manifests containing actual source/build, fixture, acceptance, model and startup-policy provenance. Only compatible verified pairs contribute to comparisons; legacy data remains diagnostic.
+
+### Fixed
+- Retain incomplete parallel work across failure, pause and decision boundaries, including unsuccessful candidate races, and resume implementation separately from already prepared integration candidates.
+- Honor ambiguity aborts consistently in serial and parallel execution. Reuse successful gates only on unchanged source, task and gate inputs; rerun them after relevant changes and integration.
+- Prevent exhausted routing budgets from resetting after statistics eviction or failed registry writes. Preserve the cost of planning even when routing blocks before implementation.
+- Preserve known partial provider usage without converting unknown calls to zero-cost work or discarding measured worker usage. Record PRD drafting, decomposition and change-planning/coverage calls on success and failure.
+- Apply goal assessment policy and routing rules, reset provider-specific defaults when changing runners, and separate historic native Codex bindings from the currently executing provider.
+- Correct Code Intelligence reference edges, report unverified index freshness and traversal coverage honestly, and enforce shared request deadlines and bounded UTF-8 responses including evidence metadata.
+- Synchronize package, lockfile, Canon, provider manifests and README release metadata.
+
+### Migration and validation limits
+- Existing configurations retain their routing ranking unless `routing.optimization` is added. New setup configurations select `balanced` with a minimum of 20 comparable samples per candidate; insufficient or incomplete evidence keeps the conservative order. Profile tiers and `costTier` are configuration priors, not live provider prices.
+- Goal token ceilings are checked before each additional call. A provider reporting only at call completion can still exceed a ceiling within that call; the overrun is retained and prevents further budgeted dispatch. Unknown interrupted consumption requires an explicit budget decision.
+- Retained work consumes disk space until successful integration or explicit reviewed cleanup. Candidate recovery resumes the first retained alternative; additional alternatives remain available for inspection and cleanup. Custom lifecycles without `retain` keep their existing cleanup policy. Interrupted attempts remain spent budget and are excluded from model-quality comparisons; recovery does not invent missing accounting history.
+- Delivery declarations map project-authored commands to artifacts and journeys. Hashes compare pre-check and post-check artifact content; they do not detect changes reverted between snapshots or independently prove that a command exercised a particular device or deployment. Artifact snapshots are limited to 512 MiB per file, 1 GiB total and 10 seconds per snapshot. Build artifacts before `yoke check`; inspect and explicitly refresh protected acceptance after intentional manifest changes.
+- Code Intelligence token budgets use a documented conservative UTF-8 estimate, not provider-measured token counts. A tiny budget can only return a bounded `BUDGET_EXCEEDED` error envelope; backend freshness stays unknown without independent index evidence.
+- This release uses deterministic regression and packaging checks. No new paid model benchmark or general speed/cost improvement is claimed; see the [validation record](docs/RELEASE-VALIDATION-1.22.0.md).
+
 ## 1.21.1 — 2026-10-03
 
 ### Fixed

package/README.md

@@ -11,7 +11,7 @@
 [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
 ![Node.js 20+](https://img.shields.io/badge/node-%E2%89%A520-339933?logo=node.js&logoColor=white)
 
-<!-- yoke:version:start -->1.21.1<!-- yoke:version:end --> · <!-- yoke:tests:start -->1426<!-- yoke:tests:end --> test cases · <!-- yoke:skills:start -->34<!-- yoke:skills:end --> skills
+<!-- yoke:version:start -->1.22.0<!-- yoke:version:end --> · <!-- yoke:tests:start -->1610<!-- yoke:tests:end --> test cases · <!-- yoke:skills:start -->34<!-- yoke:skills:end --> skills
 
 <!-- yoke:agents:start -->Claude | Codex | Gemini | Qwen | OpenCode | Kilo | Pi | Hermes<!-- yoke:agents:end -->
 
@@ -41,6 +41,8 @@ Each story carries observable acceptance criteria and targeted test commands. Pa
 | **Verified completion** | Checks acceptance criteria, the project verify command, and any configured completion, review, quality, or browser gates before accepting work. |
 | **Parallel execution** | Schedules independent stories in isolated worktrees, respects dependencies and write scopes, and coordinates a shared worker limit across projects. |
 | **Durable goals** | Binds objectives to executable acceptance, shares worker capacity, records budgets, and optionally resumes native Codex goal threads. |
+| **Measured routing** | Keeps hard attempt budgets independent of statistics, accounts for planning and reviews, and uses comparable complete evidence for optional economic model selection. |
+| **Delivery evidence** | Records executable user journeys, acceptance results and stable declared build artifacts with their source and acceptance fingerprints. |
 | **Long-running autonomy** | Recovers from provider failures and blocked work. Optional exploration discovers new, repository-evidenced tasks after the planned backlog drains. |
 | **A choice of agents** | Uses the native CLI for Claude, Codex, Gemini, Qwen, OpenCode, Kilo, Pi, or Hermes. Install and authenticate the CLI you choose; Yoke does not bundle model runtimes or credentials. |
 | **Project visibility** | A local dashboard shows project status, **Workspace analytics**, **History**, and controls such as **Queue a change**, safe-boundary pause/resume, and operator notes. It runs with the local Yoke process. Dark and light themes are available. |
@@ -119,9 +121,12 @@ The dashboard is a local control room. It does not discover every process or run
 | [Continuous exploration](docs/CONTINUOUS-EXPLORATION.md) | Autonomous discovery, runtime limits, pause/resume, and stop detection |
 | [Project workflows](docs/VERIFIED-PROJECTS.md) | Setup, verification, goals, and execution defaults |
 | [Verified goals](docs/GOALS.md) | Acceptance binding, native Codex goals, budgets, and resource admission |
+| [Economic routing](docs/ECONOMIC-ROUTING.md) | Durable attempt budgets, per-call costs, capability floors, and conservative model selection |
+| [Delivery journeys](docs/DELIVERY-JOURNEYS.md) | Browser steps, project acceptance, artifact hashes, and proof limits |
 | [Code Intelligence](docs/CODE-INTELLIGENCE.md) | Optional Graphify, Serena, and Graft evidence providers |
 | [Dashboard](docs/DASHBOARD-EVOLUTION.md) | Project views, history, controls, and reporting boundaries |
 | [Benchmarks](bench/RESULTS.md) | Direct Codex comparison, routing studies, sample limits, and integration findings |
+| [Benchmark manifests](docs/BENCHMARK-MANIFEST.md) | Comparable inputs, model identity, build provenance, and legacy-data limitations |
 | [Changelog](CHANGELOG.md) | Release features, behavior changes, and migration notes |
 
 ## Safety and limits

package/bench/analyze-codex-comparison.mjs

@@ -1,22 +1,95 @@
 // node bench/analyze-codex-comparison.mjs <results.json> ... > summary.json
 import { readFileSync } from 'node:fs'
+import { assessComparison, stableComparisonJson } from './result-schema.mjs'
+
 const sources = process.argv.slice(2).map(path => JSON.parse(readFileSync(path, 'utf8')))
 if (!sources.length) throw new Error('Supply comparison results.json files')
-const rows = sources.flatMap(s => s.results).map(({ dir, ...row }) => ({
-  ...row,
-  freshInputTokens: Number.isFinite(row.usage?.inputTokens) && Number.isFinite(row.usage?.cachedInputTokens)
-    ? row.usage.inputTokens - row.usage.cachedInputTokens : null,
+const runs = []
+const cohorts = new Map()
+const legacyReports = []
+const median = values => {
+  const sorted = [...values].sort((a, b) => a - b)
+  const n = sorted.length
+  return n ? (sorted[Math.floor((n - 1) / 2)] + sorted[Math.floor(n / 2)]) / 2 : null
+}
+const measured = value => Number.isFinite(value) && value >= 0
+const metric = (rows, read) => rows.every(row => measured(read(row))) ? median(rows.map(read)) : null
+function summarize(rows) {
+  return {
+    fixture: rows[0].fixture, arm: rows[0].arm, runs: rows.length,
+    accepted: rows.filter(row => row.accepted === true).length,
+    medianWallMs: median(rows.map(row => row.wallMs)),
+    medianElapsedToAcceptanceMs: metric(rows, row => measured(row.acceptanceMs) ? row.wallMs + row.acceptanceMs : null),
+    medianFreshInputTokens: metric(rows, row => row.usage?.measurementComplete === false ? null : row.freshInputTokens),
+    medianOutputTokens: metric(rows, row => row.usage?.measurementComplete === false ? null : row.usage?.outputTokens),
+    medianAllInputTokens: metric(rows, row => row.usage?.measurementComplete === false ? null : row.usage?.inputTokens),
+  }
+}
+function armSummaries(rows) {
+  return [...new Set(rows.map(row => row.arm))].map(arm => summarize(rows.filter(row => row.arm === arm)))
+}
+for (const [index, source] of sources.entries()) {
+  const input = source.results ?? source.runs
+  if (!Array.isArray(input) || !input.length) throw new Error('No completed runs in input ' + (index + 1))
+  const rows = input.map(({ dir, ...row }) => {
+    if (!measured(row.wallMs) || row.wallMs <= 0) throw new Error('Invalid measurement: wall time')
+    for (const field of ['inputTokens', 'cachedInputTokens', 'outputTokens']) {
+      if (row.usage?.[field] != null && !measured(row.usage[field])) throw new Error('Invalid measurement: ' + field)
+    }
+    if (row.acceptanceMs != null && !measured(row.acceptanceMs)) throw new Error('Invalid measurement: acceptance time')
+    const freshInputTokens = measured(row.usage?.inputTokens) && measured(row.usage?.cachedInputTokens)
+      ? row.usage.inputTokens - row.usage.cachedInputTokens : null
+    if (freshInputTokens !== null && freshInputTokens < 0) throw new Error('Invalid measurement: cache exceeds input')
+    return { ...row, freshInputTokens }
+  })
+  if (!source.manifest) {
+    legacyReports.push({
+      status: 'legacy/unverified', reportedVersion: source.version ?? source.auditedVersion ?? null,
+      reportedCommit: source.commit ?? source.auditedCommit ?? null, runs: rows.length,
+      reason: 'No versioned manifest; provenance and cross-arm conditions cannot be verified',
+      diagnosticSummaries: armSummaries(rows),
+    })
+    runs.push(...rows.map(row => ({ ...row, evidenceStatus: 'legacy/unverified' })))
+    continue
+  }
+  // A malformed identity remains its own unverified cohort, never joins another file.
+  const id = typeof source.manifest.id === 'string' && source.manifest.id ? source.manifest.id : 'invalid-input-' + index
+  const cohort = cohorts.get(id) ?? { id, manifests: [], runs: [] }
+  cohort.manifests.push(source.manifest)
+  cohort.runs.push(...rows)
+  cohorts.set(id, cohort)
+}
+const comparisons = [...cohorts.values()].map(cohort => ({
+  id: cohort.id,
+  ...assessComparison(cohort.manifests, cohort.runs),
+  manifest: cohort.manifests[0],
+  provenance: [...new Map(cohort.runs.map(row => [stableComparisonJson(row.context?.source), row.context?.source ?? null])).values()],
+  diagnosticSummaries: armSummaries(cohort.runs),
 }))
-if (!rows.length) throw new Error('No completed runs')
-for (const row of rows) if (!Number.isFinite(row.wallMs) || row.wallMs <= 0 || (row.freshInputTokens !== null && row.freshInputTokens < 0)) throw new Error('Invalid measurement')
-const median = values => { const sorted = [...values].sort((a,b) => a-b); const n = sorted.length; return n ? (sorted[Math.floor((n-1)/2)] + sorted[Math.floor(n/2)]) / 2 : null }
-const groups = [...new Set(rows.map(r => `${r.fixture}:${r.arm}`))].map(key => {
-  const runs = rows.filter(r => `${r.fixture}:${r.arm}` === key)
-  if (new Set(runs.map(r => `${r.model}:${r.effort}:${r.routing}`)).size !== 1) throw new Error('Incompatible execution policies in one comparison group')
-  return { fixture: runs[0].fixture, arm: runs[0].arm, runs: runs.length, accepted: runs.filter(r => r.accepted).length,
-    medianWallMs: median(runs.map(r => r.wallMs)),
-    medianFreshInputTokens: runs.every(r => r.freshInputTokens !== null) ? median(runs.map(r => r.freshInputTokens)) : null,
-    medianOutputTokens: runs.every(r => Number.isFinite(r.usage?.outputTokens)) ? median(runs.map(r => r.usage.outputTokens)) : null,
-    medianAllInputTokens: runs.every(r => Number.isFinite(r.usage?.inputTokens)) ? median(runs.map(r => r.usage.inputTokens)) : null }
-})
-console.log(JSON.stringify({ auditedVersion: '1.19.0', auditedCommit: '05fac3db1a51412563005e07f80e6e5a9aab8e06', models: [...new Set(rows.map(r => r.model))], effort: 'low', groups, runs: rows, limitations: [...new Set(sources.flatMap(s => s.limits))], pricesMeasured: false, cpuMeasured: false, memoryMeasured: false }, null, 2))
+const groups = []
+for (const comparison of comparisons) {
+  const rows = cohorts.get(comparison.id).runs
+  runs.push(...rows.map(row => ({ ...row, comparisonId: comparison.id, evidenceStatus: comparison.status })))
+  if (comparison.status === 'verified') {
+    groups.push(...armSummaries(rows).map(group => ({ comparisonId: comparison.id, ...group })))
+  }
+}
+const verifiedSources = comparisons.filter(comparison => comparison.status === 'verified').flatMap(comparison => comparison.provenance)
+const unique = values => [...new Set(values)]
+const versions = unique(verifiedSources.map(source => source.version))
+const commits = unique(verifiedSources.map(source => source.commit))
+console.log(JSON.stringify({
+  schemaVersion: 2,
+  auditedVersion: versions.length === 1 ? versions[0] : null,
+  auditedCommit: commits.length === 1 ? commits[0] : null,
+  requestedModels: unique(runs.map(row => row.context?.execution?.requestedModel ?? row.model).filter(Boolean)),
+  reportedModels: unique(runs.flatMap(row => row.context?.execution?.actualModels ?? [])),
+  comparisons, groups, legacyReports, runs,
+  limitations: unique([
+    ...sources.flatMap(source => source.limits ?? source.limitations ?? []),
+    'Verified means the recorded manifest conditions match; it does not certify model behavior, host isolation, or statistical significance',
+    'Diagnostic summaries include unsuccessful or unverified runs and must not be used as accepted performance comparisons',
+    'Missing or partial token telemetry remains unknown; no USD price is inferred',
+  ]),
+  pricesMeasured: false, cpuMeasured: false, memoryMeasured: false,
+}, null, 2))

package/bench/compare-codex.mjs

@@ -1,71 +1,194 @@
-// Controlled live comparison. Run after npm run build.
-// node bench/compare-codex.mjs --repeats=2 --fixture=routing-queue
-import { cpSync, mkdirSync, readFileSync, writeFileSync, readdirSync } from 'node:fs'
-import { createHash } from 'node:crypto'
+// Controlled live workflow comparison. Run only when authenticated runs are authorized.
+// Run after npm run build. See docs/BENCHMARK-MANIFEST.md for scope and limits.
+import { cpSync, mkdirSync, readFileSync, writeFileSync, readdirSync, lstatSync, existsSync } from 'node:fs'
+import { createHash, randomUUID } from 'node:crypto'
 import { spawn, spawnSync } from 'node:child_process'
+import { arch, cpus, hostname, release, tmpdir, totalmem } from 'node:os'
 import { dirname, join, resolve } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { parse, stringify } from 'yaml'
 import { buildProviderInvocation, startProviderProcess } from '../dist/agents/providers.js'
+import { readEvents } from '../dist/observability/events.js'
+import { stableComparisonJson } from './result-schema.mjs'
+
 const repo = dirname(dirname(fileURLToPath(import.meta.url)))
 const version = JSON.parse(readFileSync(join(repo, 'package.json'), 'utf8')).version
-const opts = Object.fromEntries(process.argv.slice(2).map(x => x.replace(/^--/, '').split('=')))
+const opts = Object.fromEntries(process.argv.slice(2).map(value => value.replace(/^--/, '').split('=')))
 const fixture = opts.fixture ?? 'routing-queue'
 if (!['routing-queue', 'string-kit', 'independent-utils'].includes(fixture)) throw new Error('Unsupported fixture')
 const repeats = Number(opts.repeats ?? 2)
 if (!Number.isInteger(repeats) || repeats < 1 || repeats > 10) throw new Error('repeats must be 1..10')
 const model = opts.model ?? 'gpt-6.1-sol'
-const root = resolve(opts.root ?? `G:/NN-Developed/Yoke-Testground/b${Date.now().toString(36)}`)
+const effort = opts.effort ?? 'low'
+const root = resolve(opts.root ?? join(tmpdir(), 'ykb-' + Date.now().toString(36)))
 mkdirSync(root, { recursive: true })
 const seed = join(repo, 'bench', 'fixtures', fixture)
-const stories = parse(readFileSync(join(seed, '.yoke/prd.yaml'), 'utf8'))
-const originalTests = readdirSync(join(seed, 'tests')).filter(f => f.endsWith('.test.mjs')).sort()
-const protectedFiles = new Map(['bench-verify.mjs', ...originalTests.map(f => join('tests', f))].map(file => [file, readFileSync(join(seed, file))]))
-const acceptanceDigest = createHash('sha256').update(Buffer.concat([...protectedFiles].flatMap(([file, bytes]) => [Buffer.from(file + '\0'), bytes]))).digest('hex')
-const prompt = 'Implement all requirements below. Run node bench-verify.mjs. Preserve tests and the verification script. Do not commit.\n' + stories.map(s => `${s.id}: ${s.title}\n${s.acceptance.join('\n')}`).join('\n\n')
+const prdInput = readFileSync(join(seed, '.yoke/prd.yaml'), 'utf8')
+const stories = parse(prdInput)
+const originalTests = readdirSync(join(seed, 'tests')).filter(file => file.endsWith('.test.mjs')).sort()
+const protectedFiles = new Map(['bench-verify.mjs', ...originalTests.map(file => join('tests', file))].map(file => [file, readFileSync(join(seed, file))]))
+const digest = value => createHash('sha256').update(value).digest('hex')
+const acceptanceDigest = digest(Buffer.concat([...protectedFiles].flatMap(([file, bytes]) => [Buffer.from(file.replaceAll('\\', '/') + '\0'), bytes])))
+const prompt = 'Implement all requirements below. Run node bench-verify.mjs. Preserve tests and the verification script. Do not commit.\n' + stories.map(story => story.id + ': ' + story.title + '\n' + story.acceptance.join('\n')).join('\n\n')
 const results = []
 const requestedArms = opts.arms?.split(',')
 if (requestedArms && (!requestedArms.length || requestedArms.some(arm => !['codex', 'yoke-serial', 'yoke-parallel'].includes(arm)))) throw new Error('arms must name codex,yoke-serial,yoke-parallel')
+const baseArms = fixture === 'independent-utils' ? ['codex', 'yoke-serial', 'yoke-parallel'] : ['codex', 'yoke-serial']
+const selectedArms = requestedArms ? baseArms.filter(arm => requestedArms.includes(arm)) : baseArms
+if (!selectedArms.length) throw new Error('No requested arms apply to this fixture')
+const manifest = {
+  schemaVersion: 1, id: randomUUID(), kind: 'workflow', arms: selectedArms, repeats,
+  allowedDifferences: [
+    { field: 'execution.workflow', reason: 'One direct provider session versus the normal Yoke story workflow' },
+    { field: 'execution.parallel', reason: 'The explicitly selected parallel arm uses three Yoke workers' },
+    { field: 'execution.promptDigest', reason: 'Workflow inputs differ while the fixture requirements and acceptance remain identical' },
+    { field: 'execution.promptScope', reason: 'Direct provider prompt versus PRD/configuration inputs expanded by Yoke' },
+    { field: 'startup.ignoreRules', reason: 'Preserve the documented direct-arm ignore-rules flag; Yoke retains normal project rule handling' },
+    { field: 'startup.commitPolicy', reason: 'Direct arm is instructed not to commit; Yoke performs its normal story commits' },
+    { field: 'startup.isolation', reason: 'Yoke uses its normal isolated worktrees inside a fresh fixture' },
+    { field: 'startup.timeoutPolicy', reason: 'Direct session deadline versus bounded Yoke story dispatches' },
+    { field: 'startup.userStatePolicy', reason: 'Yoke runtime registry is isolated using LOCALAPPDATA; direct CLI keeps inherited state' },
+  ],
+}
+const limits = [
+  'Small fixture sample; not representative of production projects',
+  'Workflow comparison includes declared prompt, commit, isolation, timeout and rule-policy differences',
+  'CPU and RAM not measured; host background load is uncontrolled',
+  'Routing disabled; fixed requested model and effort; missing reported model identity prevents a verified comparison',
+  'Immutable tests replayed after execution; visible to agents, not hidden tests',
+  'User config disabled; installed plugins and discovered skills are not fully audited',
+  'Prompt digests bind submitted workflow inputs, not provider system prompts or every dynamically expanded Yoke prompt',
+  'Wall time starts after fixture setup and ends at runner exit; independent acceptance time is recorded separately',
+]
+function hashPaths(base, paths) {
+  const hash = createHash('sha256')
+  function visit(relative) {
+    const path = join(base, relative)
+    const label = relative.replaceAll('\\', '/')
+    if (!existsSync(path)) { hash.update('missing:' + label + '\0'); return }
+    const stat = lstatSync(path)
+    if (stat.isSymbolicLink()) throw new Error('Cannot certify linked benchmark input: ' + label)
+    if (stat.isDirectory()) {
+      hash.update('directory:' + label + '\0')
+      for (const name of readdirSync(path).sort()) visit(join(relative, name))
+    } else if (stat.isFile()) {
+      const bytes = readFileSync(path)
+      hash.update(Buffer.byteLength(label) + ':' + label + ':' + bytes.length + ':')
+      hash.update(bytes)
+    } else throw new Error('Unsupported benchmark input: ' + label)
+  }
+  for (const path of [...paths].sort()) visit(path)
+  return hash.digest('hex')
+}
+function git(args) {
+  const result = spawnSync('git', args, { cwd: repo, encoding: 'utf8' })
+  return result.status === 0 ? result.stdout.trim() : null
+}
+function sourceSnapshot() {
+  const gitRoot = git(['rev-parse', '--show-toplevel'])
+  const sameRoot = gitRoot && (process.platform === 'win32' ? resolve(gitRoot).toLowerCase() === repo.toLowerCase() : resolve(gitRoot) === repo)
+  const status = sameRoot ? git(['status', '--porcelain', '--untracked-files=normal']) : null
+  return {
+    version: JSON.parse(readFileSync(join(repo, 'package.json'), 'utf8')).version,
+    commit: sameRoot ? git(['rev-parse', 'HEAD']) : null,
+    dirty: status === null ? null : status.length > 0,
+    // Hash actual runtime artifacts even on a clean checkout: dist may be stale or untracked.
+    buildDigest: hashPaths(repo, ['dist', 'canon', 'agents', 'hooks', 'package.json', 'package-lock.json', 'bench/compare-codex.mjs', 'bench/result-schema.mjs']),
+  }
+}
+function environmentSnapshot() {
+  const provider = spawnSync('codex', ['--version'], { encoding: 'utf8', timeout: 20_000, shell: process.platform === 'win32' })
+  return {
+    platform: process.platform, arch: arch(), nodeVersion: process.version,
+    providerVersion: provider.status === 0 ? provider.stdout.trim() || null : null,
+    hostDigest: digest(stableComparisonJson({ hostname: hostname(), platform: process.platform, release: release(), arch: arch(), cpus: cpus().map(cpu => cpu.model), memory: totalmem() })),
+    hostLoad: 'uncontrolled', skillsPlugins: 'not-audited',
+  }
+}
+function reportedModelsFromEvents(dir) {
+  const records = readEvents(dir, 1000).filter(event => event.type === 'tokens')
+  const calls = records.flatMap(event => Array.isArray(event.data?.calls) && event.data.calls.length ? event.data.calls : [event.data ?? {}])
+  const models = calls.map(call => call.actualModel ?? call.model)
+  return models.length && models.every(value => typeof value === 'string' && value) ? [...new Set(models)].sort() : null
+}
+const fixtureContext = {
+  id: fixture, seedDigest: hashPaths(seed, ['.']),
+  acceptanceDigest, requirementsDigest: digest(stableComparisonJson(stories)),
+}
 for (let repeat = 0; repeat < repeats; repeat++) {
-  const baseArms = fixture === 'independent-utils' ? ['codex', 'yoke-serial', 'yoke-parallel'] : ['codex', 'yoke-serial']
-  const selectedArms = requestedArms ? baseArms.filter(arm => requestedArms.includes(arm)) : baseArms
-  if (!selectedArms.length) throw new Error('No requested arms apply to this fixture')
   const arms = repeat % 2 ? [...selectedArms].reverse() : selectedArms
   for (const arm of arms) {
-    const dir = join(root, `${repeat + 1}-${arm}`)
-    mkdirSync(dir); cpSync(seed, dir, { recursive: true })
+    const before = sourceSnapshot()
+    const environment = environmentSnapshot()
+    const dir = join(root, String(repeat + 1) + '-' + arm)
+    mkdirSync(dir)
+    cpSync(seed, dir, { recursive: true })
     const parallel = arm === 'yoke-parallel' ? 3 : 1
-    writeFileSync(join(dir, '.yoke/config.yaml'), stringify({ canonVersion: '1.2.0', agents: ['codex'], loop: { enabled: true, parallel, isolate: true }, runner: { agent: 'codex', model, reasoningEffort: 'low', bare: true }, routing: { enabled: false }, verify: { command: 'node bench-verify.mjs' } }))
+    const configInput = stringify({ canonVersion: '1.2.0', agents: ['codex'], loop: { enabled: true, parallel, isolate: true }, runner: { agent: 'codex', model, reasoningEffort: effort, bare: true }, routing: { enabled: false }, verify: { command: 'node bench-verify.mjs' } })
+    writeFileSync(join(dir, '.yoke/config.yaml'), configInput)
     for (const args of [['init', '-q'], ['add', '-A'], ['-c', 'user.name=benchmark', '-c', 'user.email=benchmark@yoke.local', 'commit', '-qm', 'Immutable fixture seed']]) {
-      const r = spawnSync('git', args, { cwd: dir, encoding: 'utf8' }); if (r.status !== 0) throw new Error(r.stderr)
+      const result = spawnSync('git', args, { cwd: dir, encoding: 'utf8' })
+      if (result.status !== 0) throw new Error(result.stderr)
     }
-    console.log(JSON.stringify({ type: 'start', fixture, repeat: repeat + 1, arm, model, dir }))
-    const start = Date.now(); let outcome; let events = []; let usage
+    console.log(JSON.stringify({ type: 'start', comparisonId: manifest.id, fixture, repeat: repeat + 1, arm, model, effort, dir }))
+    const start = Date.now()
+    let outcome, usage, actualModels = null
+    let events = []
     if (arm === 'codex') {
-      const inv = buildProviderInvocation('codex', prompt, dir, 'unsafe', { model, reasoningEffort: 'low', bare: true, nativeMultiAgent: false })
-      inv.args.push('--ignore-rules')
-      const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), 600_000)
+      const invocation = buildProviderInvocation('codex', prompt, dir, 'unsafe', { model, reasoningEffort: effort, bare: true, nativeMultiAgent: false })
+      // Historical workflow policy is retained and explicitly declared, not silently equalized.
+      invocation.args.push('--ignore-rules')
+      const controller = new AbortController()
+      const timer = setTimeout(() => controller.abort(), 600_000)
       try {
-        const r = await startProviderProcess('codex', inv, { signal: controller.signal, idleTimeoutMs: 180_000 }).completion
-        outcome = r.kind; usage = r.telemetry.tokens
-        writeFileSync(join(root, `${fixture}-${repeat + 1}-${arm}.log`), JSON.stringify(r, null, 2))
+        const result = await startProviderProcess('codex', invocation, { signal: controller.signal, idleTimeoutMs: 180_000 }).completion
+        outcome = result.kind
+        usage = result.telemetry.tokens
+        const reported = result.telemetry.reportedModels ?? (usage?.model ? [usage.model] : [])
+        actualModels = reported.length ? [...new Set(reported)].sort() : null
+        writeFileSync(join(root, fixture + '-' + (repeat + 1) + '-' + arm + '.log'), JSON.stringify(result, null, 2))
       } finally { clearTimeout(timer) }
     } else {
-      const child = spawn(process.execPath, [join(repo, 'dist/cli.js'), 'loop', 'run', dir, '--json', '--runner=codex', '--no-routing', `--parallel=${parallel}`, '--max=3', '--timeout=10', '--unsafe'], { cwd: dir, env: { ...process.env, LOCALAPPDATA: join(root, 'state') }, stdio: ['ignore', 'pipe', 'pipe'] })
-      let output = '', errors = ''; child.stdout.on('data', d => { output += d }); child.stderr.on('data', d => { errors += d })
-      outcome = await new Promise((res, rej) => { child.on('error', rej); child.on('close', res) })
+      const child = spawn(process.execPath, [join(repo, 'dist/cli.js'), 'loop', 'run', dir, '--json', '--runner=codex', '--no-routing', '--parallel=' + parallel, '--max=3', '--timeout=10', '--unsafe'], { cwd: dir, env: { ...process.env, LOCALAPPDATA: join(root, 'state') }, stdio: ['ignore', 'pipe', 'pipe'] })
+      let output = '', errors = ''
+      child.stdout.on('data', data => { output += data })
+      child.stderr.on('data', data => { errors += data })
+      outcome = await new Promise((resolveExit, reject) => { child.on('error', reject); child.on('close', resolveExit) })
       events = output.split('\n').flatMap(line => { try { return [JSON.parse(line)] } catch { return [] } })
-      writeFileSync(join(root, `${fixture}-${repeat + 1}-${arm}.log`), output + '\nSTDERR\n' + errors)
-      usage = events.filter(e => e.type === 'status' && e.tokens).at(-1)?.tokens
+      writeFileSync(join(root, fixture + '-' + (repeat + 1) + '-' + arm + '.log'), output + '\nSTDERR\n' + errors)
+      usage = events.filter(event => event.type === 'status' && event.tokens).at(-1)?.tokens
+      actualModels = reportedModelsFromEvents(dir)
     }
     const wallMs = Date.now() - start
     // Replay immutable originals after the agent exits. Agent-written tests never determine quality.
     mkdirSync(join(dir, 'tests'), { recursive: true })
     for (const [file, bytes] of protectedFiles) writeFileSync(join(dir, file), bytes)
-    const testFiles = originalTests.map(f => join('tests', f))
-    const checkStart = Date.now(); const check = spawnSync(process.execPath, ['--test', ...testFiles], { cwd: dir, encoding: 'utf8', timeout: 60_000 })
-    writeFileSync(join(root, `${fixture}-${repeat + 1}-${arm}.acceptance.log`), check.stdout + check.stderr)
-    const result = { fixture, repeat: repeat + 1, arm, model, effort: 'low', routing: false, nativeMultiAgent: false, parallel, acceptanceDigest, wallMs, acceptanceMs: Date.now() - checkStart, accepted: check.status === 0, outcome, usage: usage ?? null, events: events.length, dir }
-    results.push(result); writeFileSync(join(root, 'results.json'), JSON.stringify({ version, root, results, limits: ['Small fixture sample; not representative of production projects', 'CPU and RAM not measured', 'Routing disabled; fixed model and effort', 'Immutable tests replayed after execution; not hidden from agents', 'User config disabled; not an audit of every installed plugin'] }, null, 2)); console.log(JSON.stringify({ type: 'result', ...result }))
+    const checkStart = Date.now()
+    const check = spawnSync(process.execPath, ['--test', ...originalTests.map(file => join('tests', file))], { cwd: dir, encoding: 'utf8', timeout: 60_000 })
+    const acceptanceMs = Date.now() - checkStart
+    writeFileSync(join(root, fixture + '-' + (repeat + 1) + '-' + arm + '.acceptance.log'), check.stdout + check.stderr)
+    const after = sourceSnapshot()
+    const context = {
+      source: { ...before, stable: stableComparisonJson(before) === stableComparisonJson(after) && hashPaths(seed, ['.']) === fixtureContext.seedDigest },
+      fixture: fixtureContext,
+      execution: {
+        provider: 'codex', requestedModel: model, actualModels, effort, routing: false, nativeMultiAgent: false, nativeGoal: false, parallel,
+        workflow: arm === 'codex' ? 'direct-provider-session' : 'yoke-story-loop',
+        promptDigest: digest(arm === 'codex' ? prompt : stableComparisonJson({ prdInput, configInput })),
+        promptScope: arm === 'codex' ? 'provider-prompt' : 'workflow-input-bundle',
+      },
+      startup: {
+        permissionProfile: 'unsafe', bare: true, ignoreRules: arm === 'codex',
+        commitPolicy: arm === 'codex' ? 'instructed-no-commit' : 'normal-story-commits',
+        isolation: arm === 'codex' ? 'fresh-fixture' : 'fresh-fixture-and-story-worktrees',
+        timeoutPolicy: arm === 'codex' ? '600000ms session; 180000ms idle' : 'three dispatches; 600000ms per runner',
+        userStatePolicy: arm === 'codex' ? 'inherited' : 'isolated-LOCALAPPDATA',
+      },
+      environment,
+    }
+    const result = { fixture, repeat: repeat + 1, arm, model, effort, routing: false, nativeMultiAgent: false, parallel, acceptanceDigest, context, wallMs, acceptanceMs, accepted: check.status === 0, outcome, usage: usage ?? null, events: events.length, dir }
+    results.push(result)
+    writeFileSync(join(root, 'results.json'), JSON.stringify({ schemaVersion: 2, version, manifest, root, results, limits }, null, 2))
+    console.log(JSON.stringify({ type: 'result', comparisonId: manifest.id, ...result }))
   }
 }

package/bench/result-schema.mjs

@@ -10,3 +10,135 @@ export function validateResult(result) {
   if (!['completed', 'blocked', 'unavailable', 'auth-failed'].includes(result.verdict)) throw new Error(`invalid verdict: ${result.verdict}`)
   return result
 }
+
+// Comparison manifests are separate from the historical individual-run schema above.
+// A "verified" manifest describes comparable recorded conditions, not a causal or
+// statistically conclusive performance result.
+const record = value => value !== null && typeof value === 'object' && !Array.isArray(value)
+const nonempty = value => typeof value === 'string' && value.trim().length > 0
+const boolean = value => typeof value === 'boolean'
+const sha256 = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value)
+const positiveInteger = value => Number.isSafeInteger(value) && value > 0
+
+export const comparisonContextFields = {
+  'source.version': nonempty,
+  'source.commit': value => typeof value === 'string' && /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(value),
+  'source.dirty': boolean,
+  'source.buildDigest': sha256,
+  'source.stable': value => value === true,
+  'fixture.id': nonempty,
+  'fixture.seedDigest': sha256,
+  'fixture.acceptanceDigest': sha256,
+  'fixture.requirementsDigest': sha256,
+  'execution.provider': nonempty,
+  'execution.requestedModel': nonempty,
+  'execution.actualModels': value => Array.isArray(value) && value.length > 0 && value.every(nonempty) && new Set(value).size === value.length,
+  'execution.effort': nonempty,
+  'execution.routing': boolean,
+  'execution.nativeMultiAgent': boolean,
+  'execution.nativeGoal': boolean,
+  'execution.parallel': positiveInteger,
+  'execution.workflow': nonempty,
+  'execution.promptDigest': sha256,
+  'execution.promptScope': value => ['provider-prompt', 'workflow-input-bundle'].includes(value),
+  'startup.permissionProfile': value => ['safe', 'read-only', 'unsafe'].includes(value),
+  'startup.bare': boolean,
+  'startup.ignoreRules': boolean,
+  'startup.commitPolicy': nonempty,
+  'startup.isolation': nonempty,
+  'startup.timeoutPolicy': nonempty,
+  'startup.userStatePolicy': nonempty,
+  'environment.platform': nonempty,
+  'environment.arch': nonempty,
+  'environment.nodeVersion': nonempty,
+  'environment.providerVersion': nonempty,
+  'environment.hostDigest': sha256,
+  'environment.hostLoad': value => ['uncontrolled', 'isolated'].includes(value),
+  'environment.skillsPlugins': value => ['not-audited', 'audited'].includes(value),
+}
+
+export const allowedComparisonVariables = Object.keys(comparisonContextFields)
+  .filter(field => !field.startsWith('fixture.') && field !== 'source.stable')
+
+export function comparisonValue(context, field) {
+  return field.split('.').reduce((value, key) => record(value) ? value[key] : undefined, context)
+}
+
+export function stableComparisonJson(value) {
+  if (Array.isArray(value)) return '[' + value.map(stableComparisonJson).join(',') + ']'
+  if (record(value)) return '{' + Object.keys(value).sort().map(key => JSON.stringify(key) + ':' + stableComparisonJson(value[key])).join(',') + '}'
+  return JSON.stringify(value)
+}
+
+export function comparisonManifestIssues(manifest) {
+  if (!record(manifest)) return ['Missing versioned comparison manifest']
+  const issues = []
+  if (manifest.schemaVersion !== 1) issues.push('Unsupported comparison manifest version')
+  if (!nonempty(manifest.id)) issues.push('Missing comparison identity')
+  if (!['workflow', 'controlled'].includes(manifest.kind)) issues.push('Unknown comparison kind')
+  if (!Array.isArray(manifest.arms) || manifest.arms.length < 2 || !manifest.arms.every(nonempty) || new Set(manifest.arms).size !== manifest.arms.length) issues.push('Declare at least two distinct comparison arms')
+  if (!positiveInteger(manifest.repeats)) issues.push('Declare the planned number of paired repeats')
+  if (!Array.isArray(manifest.allowedDifferences)) issues.push('Declare allowed differences explicitly, including an empty list when appropriate')
+  else {
+    const fields = new Set()
+    for (const difference of manifest.allowedDifferences) {
+      if (!record(difference) || !allowedComparisonVariables.includes(difference.field) || !nonempty(difference.reason)) {
+        issues.push('Invalid comparison variable or missing rationale')
+        continue
+      }
+      if (fields.has(difference.field)) issues.push('Duplicate comparison variable: ' + difference.field)
+      fields.add(difference.field)
+    }
+  }
+  return issues
+}
+
+export function comparisonContextIssues(context) {
+  if (!record(context)) return ['Missing per-run comparison context']
+  const issues = []
+  for (const [field, validate] of Object.entries(comparisonContextFields)) {
+    if (!validate(comparisonValue(context, field))) issues.push('Missing, unknown or invalid condition: ' + field)
+  }
+  // Refuse silently ignored startup/source conditions introduced by a future writer.
+  for (const [section, fields] of Object.entries(context)) {
+    if (!record(fields)) { issues.push('Invalid context section: ' + section); continue }
+    for (const field of Object.keys(fields)) {
+      if (!(section + '.' + field in comparisonContextFields)) issues.push('Unknown comparison condition: ' + section + '.' + field)
+    }
+  }
+  return issues
+}
+
+export function assessComparison(manifests, runs) {
+  const manifest = manifests[0]
+  let reasons = manifests.flatMap(comparisonManifestIssues)
+  if (reasons.length) return { status: 'unverified', reasons: [...new Set(reasons)] }
+  if (manifests.some(value => stableComparisonJson(value) !== stableComparisonJson(manifest))) return { status: 'incompatible', reasons: ['Comparison manifests differ for the same identity'] }
+  reasons = runs.flatMap(run => comparisonContextIssues(run.context).map(issue => run.arm + ': ' + issue))
+  for (const run of runs) {
+    if (!manifest.arms.includes(run.arm) || !positiveInteger(run.repeat) || run.repeat > manifest.repeats) reasons.push('Unexpected arm or repeat')
+    if (run.fixture !== run.context?.fixture?.id) reasons.push('Fixture label disagrees with measured context')
+    if (typeof run.accepted !== 'boolean') reasons.push('Missing acceptance outcome')
+  }
+  if (reasons.length) return { status: 'unverified', reasons: [...new Set(reasons)] }
+  const keys = runs.map(run => run.arm + ':' + run.repeat)
+  if (new Set(keys).size !== keys.length) return { status: 'incompatible', reasons: ['Duplicate arm/repeat measurements'] }
+  const permitted = new Set(manifest.allowedDifferences.map(value => value.field))
+  for (const field of Object.keys(comparisonContextFields)) {
+    const valueOf = run => {
+      const value = comparisonValue(run.context, field)
+      return stableComparisonJson(field === 'execution.actualModels' ? [...value].sort() : value)
+    }
+    for (const arm of manifest.arms) {
+      if (new Set(runs.filter(run => run.arm === arm).map(valueOf)).size > 1) reasons.push('Conditions changed within arm ' + arm + ': ' + field)
+    }
+    if (!permitted.has(field) && new Set(runs.map(valueOf)).size > 1) reasons.push('Undeclared difference between arms: ' + field)
+  }
+  if (reasons.length) return { status: 'incompatible', reasons: [...new Set(reasons)] }
+  if (runs.length !== manifest.arms.length * manifest.repeats) return { status: 'incomplete', reasons: ['Not all declared paired runs are present'] }
+  if (runs.some(run => !run.accepted)) return { status: 'acceptance-failed', reasons: ['At least one arm failed immutable acceptance; failed elapsed time is not a speedup'] }
+  if (runs.some(run => run.usage?.measurementComplete === false || ['inputTokens', 'cachedInputTokens', 'outputTokens'].some(field => !Number.isFinite(run.usage?.[field]) || run.usage[field] < 0))) {
+    return { status: 'unverified', reasons: ['Incomplete token telemetry; retain diagnostic timings without an efficiency comparison'] }
+  }
+  return { status: 'verified', reasons: [] }
+}

package/canon/manifest.yaml

@@ -1,5 +1,5 @@
 name: yoke-canon
-version: 1.19.0
+version: 1.22.0
 agents: [claude, codex, gemini, qwen, opencode, kilo, pi, hermes]
 skills:
   - { id: tdd, path: skills/tdd, kind: methodology, invocation: auto }

package/dist/agents/pi-telemetry.js

@@ -39,10 +39,11 @@ export function createPiTelemetry() {
             if (complete) {
                 // Optional totals must also cover every turn; omitted is not measured zero.
                 const measured = Object.fromEntries(Object.entries(totals).filter(([key]) => counts[key] === turns));
+                const partial = Object.fromEntries(Object.entries(totals).filter(([key]) => counts[key] !== turns));
                 return { usageAvailable: true, tokens: {
                         ...measured, inputTokens: totals.inputTokens, outputTokens: totals.outputTokens,
                         ...(reportedModels.length === 1 ? { model: reportedModels[0] } : {}),
-                    }, ...(reportedModels.length > 1 ? { reportedModels } : {}) };
+                    }, ...(Object.keys(partial).length ? { partialUsage: partial } : {}), ...(reportedModels.length > 1 ? { reportedModels } : {}) };
             }
             return { usageAvailable: false,
                 ...(Object.keys(totals).length ? { partialUsage: { ...totals } } : {}),