@hecer/yoke 1.21.1 → 1.23.0

package/dist/smoke/command.js

@@ -1,7 +1,10 @@
-import { mkdirSync, rmSync, renameSync } from 'node:fs';
+import { lstatSync, mkdirSync, rmSync, renameSync, writeFileSync } from 'node:fs';
 import { join, resolve } from 'node:path';
 import { createRequire } from 'node:module';
+import { createHash } from 'node:crypto';
 import { loadConfig } from '../retrofit/config.js';
+import { workspaceFingerprint } from '../workspace/fingerprint.js';
+import { statePath } from '../workspace/state.js';
 const CONFIG_GUIDANCE = [
     'No smoke flows configured. Add a smoke section to .yoke/config.yaml, e.g.:',
     '',
@@ -13,26 +16,41 @@ const CONFIG_GUIDANCE = [
     '      landmark: "main h1"',
 ].join('\n');
 export async function launchPlaywright(targetDir) {
+    const req = createRequire(join(resolve(targetDir), 'package.json'));
     try {
-        // createRequire needs an absolute anchor — a relative targetDir (the CLI
-        // default '.') would throw and masquerade as "playwright not found".
-        // Playwright is CJS, so load it with native require() rather than a
-        // file:// dynamic import — the URL round-trip breaks under Windows 8.3
-        // short paths (e.g. RUNNER~1 on CI) and test-runner import interception.
-        const req = createRequire(join(resolve(targetDir), 'package.json'));
-        const pw = req('playwright');
-        const chromium = pw.chromium ?? pw.default?.chromium;
-        if (!chromium)
+        req.resolve('playwright');
+    }
+    catch (error) {
+        if (error.code === 'MODULE_NOT_FOUND')
             return null;
-        return await chromium.launch({ headless: true });
+        throw Object.assign(new Error(`Playwright resolution failed: ${String(error)}`, { cause: error }), { code: 'browser-package-failed' });
     }
-    catch {
-        return null;
+    // Native CJS loading also preserves Windows short-path compatibility.
+    let chromium;
+    try {
+        const pw = req('playwright');
+        chromium = pw.chromium ?? pw.default?.chromium;
+    }
+    catch (error) {
+        throw Object.assign(new Error(`Playwright package could not load: ${String(error)}`, { cause: error }), { code: 'browser-package-failed' });
+    }
+    if (!chromium)
+        throw Object.assign(new Error('Installed Playwright package has no chromium export'), { code: 'browser-export-missing' });
+    try {
+        return await chromium.launch({ headless: true, timeout: 30_000 });
+    }
+    catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        const code = /executable doesn't exist|executable does not exist/i.test(detail) ? 'browser-binary-missing'
+            : /no usable sandbox|running as root without --no-sandbox/i.test(detail) ? 'browser-sandbox-failed'
+                : /\bEPERM\b|\bEACCES\b|permission denied|operation not permitted/i.test(detail) ? 'browser-permission-denied'
+                    : 'browser-launch-failed';
+        throw Object.assign(new Error(detail, { cause: error }), { code });
     }
 }
 // Flow names come from user config and become filenames — keep them safe.
 function safeName(name) {
-    return name.replace(/[^\w.-]+/g, '-');
+    return name.replace(/[^\w.-]+/g, '-').slice(0, 120) || 'flow';
 }
 // The label names a directory that gets rmSync'd recursively — it must never
 // carry path semantics ('..', separators). Dots are stripped entirely so a
@@ -41,6 +59,230 @@ export function safeLabel(label) {
     const cleaned = label.replace(/[^\w-]+/g, '-').replace(/^-+|-+$/g, '');
     return cleaned || 'latest';
 }
+async function verifyServedSource(baseUrl, identity) {
+    const origin = new URL(baseUrl);
+    const url = new URL(identity.path, origin);
+    if (!['http:', 'https:'].includes(url.protocol) || url.origin !== origin.origin)
+        return false;
+    const response = await fetch(url, { signal: AbortSignal.timeout(5000), redirect: 'error' });
+    if (!response.ok || !response.body)
+        return false;
+    const reader = response.body.getReader(), hash = createHash('sha256');
+    let bytes = 0;
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            bytes += value.byteLength;
+            if (bytes > 1024 * 1024)
+                return false;
+            hash.update(value);
+        }
+        return hash.digest('hex') === identity.sha256;
+    }
+    finally {
+        await reader.cancel().catch(() => { });
+    }
+}
+class SmokeFailure extends Error {
+    code;
+    constructor(code) {
+        super(code);
+        this.code = code;
+    }
+}
+function failureCode(error, fallback) {
+    if (error instanceof SmokeFailure)
+        return error.code;
+    return error?.name === 'TimeoutError' ? 'timeout' : fallback;
+}
+async function bounded(operation, timeout) {
+    if (timeout <= 0)
+        throw new SmokeFailure('timeout');
+    const deadline = Date.now() + timeout;
+    let timer;
+    try {
+        const result = await Promise.race([
+            Promise.resolve().then(operation),
+            new Promise((_, reject) => { timer = setTimeout(() => reject(new SmokeFailure('timeout')), timeout); }),
+        ]);
+        if (Date.now() >= deadline)
+            throw new SmokeFailure('timeout');
+        return result;
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+async function measured(stage, operation, fallback) {
+    const started = Date.now();
+    try {
+        await operation();
+        stage.status = 'passed';
+    }
+    catch (error) {
+        stage.status = 'failed';
+        stage.failure = failureCode(error, fallback);
+        throw new SmokeFailure(stage.failure);
+    }
+    finally {
+        stage.durationMs = Date.now() - started;
+    }
+}
+async function executeStep(page, step, baseUrl, timeout, fillValue) {
+    switch (step.action) {
+        case 'click':
+            if (!page.click)
+                throw new SmokeFailure('unsupported-action');
+            await page.click(step.selector, { timeout });
+            return;
+        case 'fill':
+            if (fillValue === undefined || fillValue.length > 8192)
+                throw new SmokeFailure('fill-value-unavailable');
+            if (!page.fill)
+                throw new SmokeFailure('unsupported-action');
+            await page.fill(step.selector, fillValue, { timeout });
+            return;
+        case 'press':
+            if (!page.press)
+                throw new SmokeFailure('unsupported-action');
+            await page.press(step.selector, step.key, { timeout });
+            return;
+        case 'expect-visible':
+            await page.waitForSelector(step.selector, { state: 'visible', timeout });
+            return;
+        case 'expect-text': {
+            if (!page.textContent)
+                throw new SmokeFailure('unsupported-action');
+            const deadline = Date.now() + timeout;
+            await page.waitForSelector(step.selector, { state: 'visible', timeout });
+            const normalize = (value) => value.replace(/\s+/gu, ' ').trim();
+            const expected = normalize(step.text);
+            // Both a fixed attempt bound and the enclosing deadline apply. No unbounded
+            // polling, even when a test adapter ignores Playwright's own timeout option.
+            for (let attempt = 0; attempt <= Math.ceil(timeout / 100); attempt++) {
+                const remaining = deadline - Date.now();
+                if (remaining <= 0)
+                    break;
+                const actual = normalize(await page.textContent(step.selector, { timeout: remaining }) ?? '');
+                if (Date.now() < deadline && (step.exact ? actual === expected : actual.includes(expected)))
+                    return;
+                await new Promise(resolveValue => setTimeout(resolveValue, Math.max(0, Math.min(100, deadline - Date.now()))));
+            }
+            throw new SmokeFailure('timeout');
+        }
+        case 'expect-url': {
+            if (!page.waitForURL)
+                throw new SmokeFailure('unsupported-action');
+            const expected = new URL(step.url, baseUrl).href;
+            await page.waitForURL(url => url.href === expected, { waitUntil: 'load', timeout });
+            return;
+        }
+        case 'reload': {
+            if (!page.reload)
+                throw new SmokeFailure('unsupported-action');
+            const response = await page.reload({ waitUntil: 'load', timeout });
+            if (response && !response.ok())
+                throw new SmokeFailure('http-error');
+            return;
+        }
+        default: {
+            const unexpected = step;
+            void unexpected;
+            throw new SmokeFailure('unsupported-action');
+        }
+    }
+}
+async function runFlow(browser, flow, baseUrl, proofDir, fileStem, fillValues, name) {
+    const started = Date.now(), deadline = started + (flow.timeoutMs ?? 60_000);
+    const report = {
+        name, status: 'failed', durationMs: 0, navigation: { status: 'skipped', durationMs: 0 },
+        ...(flow.landmark ? { landmark: { status: 'skipped', durationMs: 0 } } : {}),
+        steps: (flow.steps ?? []).map((step, index) => ({ index: index + 1, action: step.action, status: 'skipped', durationMs: 0 })),
+    };
+    let context, page, consoleErrors = 0;
+    let video = null;
+    const remaining = (limit) => Math.min(limit, deadline - Date.now());
+    const assertNoConsoleErrors = () => { if (consoleErrors > 0)
+        throw new SmokeFailure('console-errors'); };
+    try {
+        context = await bounded(() => browser.newContext({ recordVideo: { dir: join(proofDir, '.video-tmp') }, viewport: { width: 1280, height: 720 } }), remaining(30_000));
+        page = await bounded(() => context.newPage(), remaining(30_000));
+        const activePage = page;
+        // Browser diagnostics can echo input values. Count them; never persist their
+        // text, call logs, selectors or parameters in the machine-readable report.
+        page.on('console', msg => { if (msg?.type?.() === 'error')
+            consoleErrors++; });
+        page.on('pageerror', () => { consoleErrors++; });
+        await measured(report.navigation, async () => {
+            const timeout = remaining(30_000);
+            const response = await bounded(() => activePage.goto(baseUrl + flow.path, { waitUntil: 'load', timeout }), timeout);
+            if (response && !response.ok())
+                throw new SmokeFailure('http-error');
+            assertNoConsoleErrors();
+        }, 'navigation-failed');
+        if (flow.landmark && report.landmark) {
+            await measured(report.landmark, async () => {
+                const timeout = remaining(10_000);
+                await bounded(() => activePage.waitForSelector(flow.landmark, { state: 'visible', timeout }), timeout);
+                assertNoConsoleErrors();
+            }, 'landmark-missing');
+        }
+        for (const [index, step] of (flow.steps ?? []).entries()) {
+            await measured(report.steps[index], async () => {
+                const timeout = remaining(step.timeoutMs ?? 10_000);
+                await bounded(() => executeStep(activePage, step, baseUrl, timeout, fillValues.get(step)), timeout);
+                assertNoConsoleErrors();
+            }, 'step-failed');
+        }
+    }
+    catch (error) {
+        report.failure = failureCode(error, 'context-failed');
+    }
+    finally {
+        if (page) {
+            try {
+                await bounded(() => page.screenshot({ path: join(proofDir, `${fileStem}.png`), fullPage: true, timeout: 5000 }), 5000);
+                const saved = lstatSync(join(proofDir, `${fileStem}.png`));
+                if (!saved.isFile() || saved.size === 0)
+                    throw new SmokeFailure('screenshot-failed');
+                report.screenshot = `${fileStem}.png`;
+            }
+            catch {
+                report.failure ??= 'screenshot-failed';
+            }
+            try {
+                video = page.video();
+            }
+            catch { /* Video is best-effort evidence. */ }
+        }
+        if (context) {
+            try {
+                await bounded(() => context.close(), 5000);
+            }
+            catch {
+                report.failure ??= 'context-cleanup-failed';
+            }
+        }
+        if (video) {
+            try {
+                const path = await bounded(() => video.path(), 5000);
+                if (report.failure) {
+                    renameSync(path, join(proofDir, `${fileStem}.webm`));
+                    report.failureVideo = `${fileStem}.webm`;
+                }
+                else
+                    rmSync(path, { force: true });
+            }
+            catch { /* Preserve the original failure if video is unavailable. */ }
+        }
+    }
+    report.status = report.failure ? 'failed' : 'passed';
+    report.durationMs = Date.now() - started;
+    return report;
+}
 export async function runFlowSmoke(targetDir, opts = {}) {
     const config = loadConfig(targetDir);
     const smoke = config?.smoke;
@@ -51,91 +293,151 @@ export async function runFlowSmoke(targetDir, opts = {}) {
     const baseUrl = opts.url ?? smoke.baseUrl;
     const label = safeLabel(opts.label ?? process.env.YOKE_STORY ?? 'latest');
     const proofRel = join('.yoke', 'proof', label);
-    const proofDir = join(targetDir, proofRel);
+    let proofDir, sourceFingerprint;
+    try {
+        proofDir = statePath(targetDir, 'proof', label);
+        sourceFingerprint = workspaceFingerprint(targetDir);
+    }
+    catch {
+        console.error('Smoke evidence path or source fingerprint could not be verified; previous evidence was preserved.');
+        return 2;
+    }
+    const started = Date.now();
+    const fillValues = new Map();
+    for (const flow of smoke.flows)
+        for (const step of flow.steps ?? []) {
+            if (step.action === 'fill')
+                fillValues.set(step, step.value ?? (step.valueEnv ? process.env[step.valueEnv] : undefined));
+        }
+    const configDigest = createHash('sha256').update(JSON.stringify({ smoke, baseUrl, resolvedFillValues: [...fillValues.values()] })).digest('hex');
+    const redact = (text) => [...fillValues.values()].filter((value) => !!value).sort((a, b) => b.length - a.length).reduce((current, value) => current.split(value).join('[redacted]'), text);
     const launch = opts.launch ?? launchPlaywright;
-    const browser = await launch(targetDir);
+    if (!opts.launch && !smoke.sourceIdentity) {
+        console.error('Smoke sourceIdentity is required for a production browser run: configure a served path and expected SHA-256 for this source build; previous evidence was preserved.');
+        return 2;
+    }
+    if (smoke.sourceIdentity) {
+        let matched = false;
+        try {
+            matched = await verifyServedSource(baseUrl, smoke.sourceIdentity);
+        }
+        catch { /* Unavailable identity cannot pass. */ }
+        if (!matched) {
+            console.error('Smoke served-source identity could not be verified; previous evidence was preserved.');
+            return 2;
+        }
+    }
+    let browser;
+    try {
+        browser = await bounded(() => launch(targetDir), 30_000);
+    }
+    catch (error) {
+        const rawCode = error?.code;
+        const code = typeof rawCode === 'string' && ['browser-package-failed', 'browser-export-missing', 'browser-binary-missing', 'browser-sandbox-failed', 'browser-permission-denied'].includes(rawCode) ? rawCode : 'browser-launch-failed';
+        const detail = redact(error instanceof Error ? error.message : String(error))
+            .replace(/(https?:\/\/)[^\s/@]+:[^\s/@]+@/gi, '$1[redacted]@')
+            .replace(/\b((?:access_|refresh_|id_)?token|password|secret|api[_-]?key)=([^\s&]+)/gi, '$1=[redacted]')
+            .replace(/\b(authorization\s*:\s*(?:bearer|basic)\s+)[^\s,;]+/gi, '$1[redacted]');
+        console.error(`Smoke browser could not start (${code}): ${detail.slice(0, 2000)}; previous evidence was preserved.`);
+        return 2;
+    }
     if (!browser) {
         console.error(`Playwright not found in ${targetDir}. Install it: npm i -D playwright && npx playwright install chromium`);
         return 2;
     }
     // Wipe only once the run is actually going to happen — an exit-2 run must
     // not destroy the previous run's evidence.
-    rmSync(proofDir, { recursive: true, force: true }); // fresh evidence per run
-    mkdirSync(proofDir, { recursive: true });
+    try {
+        statePath(targetDir, 'proof', label);
+        rmSync(proofDir, { recursive: true, force: true }); // fresh evidence per run
+        mkdirSync(proofDir, { recursive: true });
+    }
+    catch {
+        try {
+            await bounded(() => browser.close(), 5000);
+        }
+        catch { /* Report setup failed. */ }
+        console.error('Smoke evidence directory could not be prepared.');
+        return 2;
+    }
     const videoTmp = join(proofDir, '.video-tmp');
-    let green = 0;
+    let baseOrigin = 'unavailable', browserVersion;
     try {
-        for (const flow of smoke.flows) {
-            const context = await browser.newContext({ recordVideo: { dir: videoTmp } });
-            const page = await context.newPage();
-            const errors = [];
-            page.on('console', (msg) => {
-                const m = msg;
-                if (m.type?.() === 'error')
-                    errors.push(String(m.text?.() ?? msg));
-            });
-            page.on('pageerror', (err) => {
-                errors.push(String(err?.message ?? err));
-            });
-            let reason;
-            try {
-                const resp = await page.goto(baseUrl + flow.path, { waitUntil: 'load', timeout: 30_000 });
-                if (resp && !resp.ok())
-                    reason = `HTTP ${resp.status()}`;
-                if (!reason && flow.landmark) {
-                    try {
-                        await page.waitForSelector(flow.landmark, { timeout: 10_000 });
-                    }
-                    catch {
-                        reason = `landmark "${flow.landmark}" not found`;
-                    }
-                }
-                if (!reason && errors.length > 0) {
-                    reason = `${errors.length} console error(s): ${errors[0].slice(0, 200)}`;
-                }
-            }
-            catch (e) {
-                reason = e.message.split('\n')[0];
-            }
-            // The screenshot IS the evidence — taken on success AND failure; a crashed
-            // page must not mask the original failure.
-            const shotName = `${safeName(flow.name)}.png`;
-            let shotOk = false;
-            try {
-                await page.screenshot({ path: join(proofDir, shotName), fullPage: true });
-                shotOk = true;
-            }
-            catch { /* keep the original reason */ }
-            const video = page.video();
-            await context.close();
-            let videoKept = false;
-            if (video) {
-                try {
-                    const vpath = await video.path();
-                    if (reason) {
-                        renameSync(vpath, join(proofDir, `${safeName(flow.name)}.webm`));
-                        videoKept = true;
-                    }
-                    else {
-                        rmSync(vpath, { force: true });
-                    }
-                }
-                catch { /* video is best-effort evidence */ }
-            }
-            if (reason) {
-                const saved = [shotOk ? 'screenshot' : null, videoKept ? 'video' : null].filter(Boolean).join(' + ');
-                console.log(`✘ ${flow.name} — ${reason}${saved ? ` (${saved} saved under ${proofRel})` : ''}`);
-            }
-            else {
-                green++;
-                console.log(`✔ ${flow.name} (screenshot: ${join(proofRel, shotName)})`);
+        baseOrigin = new URL(baseUrl).origin;
+    }
+    catch { /* Navigation reports the invalid address. */ }
+    try {
+        const version = browser.version?.();
+        if (version)
+            browserVersion = redact(version.slice(0, 120));
+    }
+    catch { /* Optional runtime metadata. */ }
+    const report = {
+        serverIdentity: smoke.sourceIdentity ? 'verified' : 'unverified',
+        version: 1, startedAt: new Date(started).toISOString(), generatedAt: '', durationMs: 0, status: 'failed',
+        sourceFingerprint, afterSourceFingerprint: null, sourceStable: false, configDigest,
+        environment: { platform: process.platform, architecture: process.arch, nodeVersion: process.version, driver: opts.launch ? 'injected' : 'playwright-chromium', ...(browserVersion ? { browserVersion } : {}), baseOrigin: redact(baseOrigin), viewport: { width: 1280, height: 720 } },
+        flows: [],
+    };
+    const filenames = new Set();
+    try {
+        for (const [index, flow] of smoke.flows.entries()) {
+            const baseName = safeName(redact(flow.name));
+            let fileStem = baseName, suffix = index + 1;
+            while (filenames.has(fileStem.toLowerCase()))
+                fileStem = `${baseName}-${suffix++}`;
+            filenames.add(fileStem.toLowerCase());
+            const result = await runFlow(browser, flow, baseUrl, proofDir, fileStem, fillValues, redact(flow.name));
+            report.flows.push(result);
+            if (result.status === 'failed') {
+                const saved = [result.screenshot ? 'screenshot' : null, result.failureVideo ? 'video' : null].filter(Boolean).join(' + ');
+                const failedStep = result.steps.find(step => step.status === 'failed');
+                console.log(`✘ ${result.name} — ${failedStep ? `step ${failedStep.index} (${failedStep.action}): ` : ''}${result.failure}${saved ? ` (${saved} saved under ${proofRel})` : ''}`);
             }
+            else
+                console.log(`✔ ${result.name} (screenshot: ${join(proofRel, result.screenshot)})`);
         }
     }
     finally {
-        await browser.close();
+        try {
+            await bounded(() => browser.close(), 5000);
+        }
+        catch {
+            report.failure = 'browser-cleanup-failed';
+        }
         rmSync(videoTmp, { recursive: true, force: true });
     }
+    try {
+        report.afterSourceFingerprint = workspaceFingerprint(targetDir);
+    }
+    catch { /* Missing identity prevents a pass. */ }
+    if (smoke.sourceIdentity) {
+        let matched = false;
+        try {
+            matched = await verifyServedSource(baseUrl, smoke.sourceIdentity);
+        }
+        catch { /* Fail closed. */ }
+        if (!matched) {
+            report.serverIdentity = 'failed';
+            report.failure ??= 'server-identity-failed';
+        }
+    }
+    report.sourceStable = report.sourceFingerprint === report.afterSourceFingerprint;
+    if (!report.sourceStable)
+        report.failure ??= 'source-changed-or-unavailable';
+    const green = report.flows.filter(flow => flow.status === 'passed').length;
+    report.status = !report.failure && green === smoke.flows.length ? 'passed' : 'failed';
+    report.generatedAt = new Date().toISOString();
+    report.durationMs = Date.now() - started;
+    try {
+        writeFileSync(statePath(targetDir, 'proof', label, 'report.json'), JSON.stringify(report, null, 2), { mode: 0o600 });
+    }
+    catch {
+        console.error('Smoke report could not be saved.');
+        return 2;
+    }
+    if (report.failure)
+        console.log(`✘ Flow-smoke evidence: ${report.failure}`);
     console.log(`Flow-smoke: ${green}/${smoke.flows.length} flows green — proof: ${proofRel}`);
-    return green === smoke.flows.length ? 0 : 1;
+    return report.status === 'passed' ? 0 : 1;
 }

package/dist/update/check.js

@@ -69,7 +69,7 @@ export function maybeNotifyUpdate(currentVersion, opts = {}) {
     if (env.YOKE_NO_UPDATE_CHECK || env.CI)
         return;
     const argv = opts.argv ?? process.argv;
-    if (argv.includes('--json'))
+    if (argv.some(arg => ['--json', '--help', '-h'].includes(arg)))
         return;
     const tty = opts.tty ?? process.stderr.isTTY === true;
     if (!tty)

package/docs/BENCHMARK-MANIFEST.md

@@ -0,0 +1,131 @@
+# Benchmark comparison manifests
+
+Yoke 1.22.0 adds a versioned provenance and comparability contract to the direct
+Codex comparison tools. This changes the tooling, not the historical results.
+No authenticated development run or new savings measurement is implied.
+
+## What a new result records
+
+The result document has schemaVersion 2 and a manifest with schemaVersion 1.
+The manifest declares a unique comparison identity, its purpose (workflow or
+controlled), the expected arms, the planned paired repeat count, and every
+condition that is intentionally allowed to differ between arms. Each declared
+difference needs a field name and a reason.
+
+Each run contains its own context:
+
+| Section | Recorded conditions |
+| --- | --- |
+| Source | Package version, Git commit, dirty state, SHA-256 build digest and whether source/build inputs stayed stable during the run |
+| Fixture | Fixture identity and SHA-256 digests of the seed, original acceptance files and shared requirements |
+| Execution | Provider, requested model, provider-reported model identities, effort, routing, native delegation, native goals, worker count and workflow |
+| Prompt | Digest and scope of the submitted input: direct provider prompt or Yoke PRD/configuration bundle |
+| Startup | Permissions, bare mode, ignore-rules, commit policy, isolation, timeout policy and user-state policy |
+| Environment | Platform, architecture, Node version, provider CLI version, hashed host identity, background-load policy and whether skills/plugins were audited |
+
+The build digest hashes the actual runtime files, canon, adapters/hooks, package
+metadata, dependency lockfile and comparison tooling. It is recorded even for a
+clean checkout, because an ignored dist directory can be stale. Dirty work is
+therefore identified by both its base commit and runtime digest. Changes during a
+run invalidate the stable-source condition. Missing Git or CLI provenance remains
+unknown; it is never filled from a version written into an older report.
+
+These digests identify recorded inputs. They are not signatures and do not prove
+that a provider served a particular model. Reported model identity comes from
+provider telemetry, never from the requested alias as a fallback.
+
+## Submitted inputs and effective prompts
+
+The direct arm records a digest of the prompt sent to the provider. The Yoke arm
+records a digest of the PRD and generated configuration submitted to the workflow.
+Those inputs are expanded into story prompts by the separately hashed runtime.
+The manifest labels these different scopes explicitly.
+
+The tool does not claim to capture every dynamically expanded prompt, retry
+message, provider system instruction, discovered skill or native context.
+The seed and requirement digests still have to agree across all arms. A prompt
+digest difference is permitted only when the manifest declares it.
+
+## Current workflow comparison
+
+The comparison still measures one direct Codex session against Yoke's normal
+story execution. Yoke can add worktrees, per-story provider starts, verification
+and commits. These are deliberate workflow differences, not isolated measurements
+of a scheduler or model-call overhead.
+
+The existing direct-arm ignore-rules flag is retained and explicitly declared as
+a startup-policy difference. Yoke retains its normal project rule handling.
+Both arms request bare mode and disable native multi-agent delegation. The tool
+does not silently modify global provider configuration or user rules.
+
+Model and effort must match across arms unless a different experiment explicitly
+declares those fields as variables. The same applies to actual model identities:
+matching aliases alone do not establish a same-model comparison. An allowed
+difference never makes a missing value valid. Fixture, requirement and acceptance
+differences cannot be waived.
+
+Conditions must stay stable within repeated runs of the same arm. A changing
+adaptive model mix therefore needs a separately designed comparison contract;
+this fixed-arm workflow tool does not silently pool such runs.
+
+## Analysis and output states
+
+Run the analyzer on already recorded result files:
+
+    node bench/analyze-codex-comparison.mjs path/to/results.json
+
+Files may contribute different arms of the same declared comparison. They must
+agree on the manifest and supply every planned arm/repeat exactly once.
+Different comparison identities are never pooled merely because their fixture
+and arm labels match.
+
+| State | Meaning |
+| --- | --- |
+| verified | All declared paired runs exist, recorded conditions match the contract, immutable acceptance passes, and input/cache/output telemetry is present |
+| incompatible | Undeclared differences, changing within-arm conditions, conflicting manifests or duplicate measurements |
+| incomplete | Some planned arms or repeats are missing |
+| acceptance-failed | An arm did not pass immutable acceptance; its short failure time cannot count as a speedup |
+| unverified | Missing/unknown provenance or telemetry, an unsupported schema, invalid context or an invalid variable declaration |
+| legacy/unverified | The input predates manifests; its reported provenance and measurements remain readable but are not promoted to a verified comparison |
+
+Only verified comparisons produce entries in the top-level groups array.
+Other measurements remain in runs and explicitly labeled diagnostic summaries.
+Missing or partial token measurements remain null. The analyzer does not invent
+USD costs, CPU/memory measurements or savings percentages.
+
+A successfully parsed diagnostic report can exit 0 while containing no verified
+comparison. Consumers must inspect comparison status and groups. Malformed JSON,
+impossible cache accounting, invalid timing and empty measurement inputs fail
+the command.
+
+## Historical records
+
+The analyzer reads both older results arrays and the runs arrays in dated
+aggregate reports. It preserves their reported version and commit under
+legacyReports, marks them legacy/unverified and leaves performance groups empty.
+It no longer hardcodes Yoke 1.19.0 or the September audit commit.
+
+The September 29 comparison and September 30 corrected smoke remain historical
+evidence with their original sample sizes and limitations. The latter has no
+contemporaneous baseline and must not be combined with the former to claim a
+new relative speedup.
+
+## Measurement limits and authorized execution
+
+Manifest verification establishes recorded comparability, not statistical
+confidence, complete environment isolation or general product quality.
+Current fixtures have visible tests and are small. Host load is uncontrolled,
+skills/plugins are not exhaustively audited, and the dependency lockfile does
+not certify every installed dependency or provider-internal setting.
+
+Wall time starts after fixture preparation and ends when the runner exits.
+Independent acceptance duration is separate; summaries also report elapsed time
+through that acceptance. Source/environment hashing and fixture setup are
+outside that timed interval.
+
+The comparison launcher makes real authenticated provider calls. Run it only
+under the user's authorized experiment scope. It accepts explicit arms, repeats,
+model and effort; its default result root is a short OS-temporary directory
+whose logs are retained. Tooling tests use synthetic records and do not launch
+providers. An existing authorization for exactly two Curiuma development runs
+is not permission for an additional fixture matrix or model-routing study.