codehere 1.0.1 → 1.0.2

package/dist/application/use-cases/verify-claim.js

@@ -1,456 +0,0 @@
-/**
- * v0.7 · verify-claim use case
- *
- * Pure function that compares what the Final Report TEXT claims against
- * the structured reality (artifacts + steps) captured during the session.
- *
- * Codehere's #1 differentiator (per claim_market_position v4 + landing copy):
- * agents round to optimism, the verifier catches the rounding.
- *
- * Examples this catches today:
- *   - Report says "18 tests passed" but artifacts.tests.passed = 17 → MISMATCH
- *   - Report says "all tests green" but artifacts.tests.failed > 0 → MISMATCH
- *   - Report says "no files changed" but artifacts.files has entries → MISMATCH
- *   - Report says "5 files modified" but artifacts.files.length === 3 → MISMATCH
- *   - Report says "Added subtract() to calc.js" and calc.js IS in the recorded
- *     changes → VERIFIED (named-file claim, the dominant real-run shape)
- *   - Report names a changed file the recorded changes don't include → WARN
- *
- * NOT a substitute for human review · this is a tripwire for the easy lies.
- * The verifier is intentionally conservative: only flag claims it can
- * compare against structured data. Unverifiable claims (prose-only,
- * intent-only) pass through silently.
- *
- * I/O is deferred to the CLI/UI surface. This file is pure: same input,
- * same output, no side-effects, no clock, no filesystem.
- */
-/* ──────────────────────────────────────────────────────────────────────── */
-/* Main verifier                                                             */
-/* ──────────────────────────────────────────────────────────────────────── */
-export function verifyClaim(input) {
-    if (!input.summary || !input.summary.trim()) {
-        return {
-            status: 'no-report',
-            findings: [
-                {
-                    severity: 'warn',
-                    kind: 'no-report',
-                    claim: '(no Final Report generated)',
-                    reality: 'nothing to verify',
-                    message: 'No Final Report available for this session. Nothing to verify against.',
-                },
-            ],
-            summary: 'No Final Report to verify.',
-        };
-    }
-    const findings = [
-        ...verifyTestsClaim(input.summary, input.artifacts),
-        ...verifyFilesClaim(input.summary, input.artifacts),
-        ...verifyNamedFileClaims(input.summary, input.artifacts),
-    ];
-    // Tier 1.1 · codehere RE-RAN the suite (observed) and it did NOT pass. This is
-    // a WARN, NOT a hard `broken` — a re-run can fail for reasons the agent's run
-    // didn't have (a monorepo where the root `npm test` is the wrong suite, a test
-    // needing a server/DB, a flaky test). A verifier that fails honest work is worse
-    // than no verifier. We ADD the env-caveat warn but do NOT touch any `fail` the
-    // scrape-based verifyTestsClaim already produced — a prose-vs-own-stdout
-    // self-contradiction stays `broken` (a real lie), the re-run failure is a
-    // separate, softer signal (self-audit 2026-06-09, fixing the over-correction
-    // that had laundered every re-run failure — including caught lies — to warn).
-    const obs = input.artifacts?.tests;
-    if (obs?.observed &&
-        ((obs.exitCode != null && obs.exitCode !== 0) || (obs.failed ?? 0) > 0) &&
-        !findings.some((f) => f.kind === 'tests' && f.severity === 'fail')) {
-        findings.push({
-            severity: 'warn',
-            kind: 'tests',
-            claim: 'the agent reported its run as done',
-            reality: `codehere re-ran ${obs.command || 'the test suite'} · exit ${obs.exitCode ?? '?'}, ${obs.failed ?? 0} failing`,
-            message: "codehere re-ran the suite and it did NOT pass — but a re-run can differ from the agent's environment (a monorepo's root suite, a test needing a server/DB). Worth a look, not a confirmed failure.",
-        });
-    }
-    const hasFail = findings.some((f) => f.severity === 'fail');
-    const hasNon = findings.length > 0;
-    // user-#1 dogfood 2026-06-07 (session 4b6bc481): an aider run that did
-    // NOTHING produced a Final Report that was just the startup banner. The
-    // banner makes no test/file claim, so the verifier found zero findings and
-    // returned 'verified' — laundering a no-op into "claims match reality". When
-    // there are no findings AND the report made no checkable claim at all, there
-    // is nothing to verify; say so honestly instead of vacuously passing.
-    const status = hasFail
-        ? 'broken'
-        : hasNon
-            ? 'partial'
-            : hasCheckableClaim(input.summary, input.artifacts)
-                ? 'verified'
-                : 'nothing-to-verify';
-    // Was the green earned purely by codehere's own re-run (no agent prose claim)?
-    // If so, say that honestly instead of "Claims match reality" (there was no claim).
-    const observedOnly = status === 'verified' && findings.length === 0 && !proseMakesCheckableClaim(input.summary, input.artifacts);
-    const summary = buildVerdictSummary(status, findings, observedOnly);
-    return { status, findings, summary };
-}
-/**
- * True when the report makes at least one assertion the verifier can check
- * against structured reality (a test claim or a file claim). If nothing is
- * checkable, a green "verified" verdict would be vacuous — see status
- * 'nothing-to-verify'.
- */
-function hasCheckableClaim(summary, artifacts) {
-    // Tier 1.1 · a clean OBSERVED re-run is itself a fact codehere confirmed, so it
-    // earns 'verified' even if the prose phrased no claim (the green ✓ here means
-    // "codehere ran the suite and it passed" — observed, not the agent's word).
-    const ot = artifacts?.tests;
-    if (ot?.observed && (ot.exitCode == null || ot.exitCode === 0) && (ot.failed ?? 0) === 0)
-        return true;
-    return proseMakesCheckableClaim(summary, artifacts);
-}
-/**
- * True when the REPORT PROSE itself asserts something checkable — a count, a
- * named file that matches reality, or an "all green"/"no files changed"
- * qualifier — independent of any observed re-run. Lets the verdict tell apart a
- * green the AGENT earned with a real claim from one codehere produced purely by
- * re-running the suite itself (so the summary doesn't say "Claims match reality"
- * when there was no claim — R5 refuter, 2026-06-09).
- */
-function proseMakesCheckableClaim(summary, artifacts) {
-    const lower = summary.toLowerCase();
-    if (matchTestCountClaim(summary) !== null)
-        return true;
-    if (matchFilesCountClaim(summary) !== null)
-        return true;
-    // Named-file claims ("Added subtract() to calc.js") count as checkable when
-    // the named file appears in the recorded file changes — the dominant shape
-    // of a REAL run's report (2026-06-07 v1 walk, session ab9cdcc9: a perfect
-    // run earned only "nothing to verify" because it phrased no numbers).
-    // A named file that does NOT match artifacts produced a warn finding in
-    // verifyNamedFileClaims, so status is 'partial' before this is consulted.
-    if (extractNamedFileClaims(summary).some((p) => fileMatchesArtifacts(p, artifacts)))
-        return true;
-    // Universal "all tests pass / green / no failures" qualifiers.
-    if (/\b(?:all|every|each)\s+tests?\s+(?:pass(?:ed|ing)?|green|succeed(?:ed)?)\b/i.test(lower) ||
-        /\btests?\s+all\s+(?:pass(?:ed|ing)?|green)\b/i.test(lower) ||
-        /\ball\s+(?:tests?\s+)?(?:green|passing)\b/i.test(lower) ||
-        /\bno\s+(?:test\s+)?(?:failures|regressions?)\b/i.test(lower))
-        return true;
-    // "no files changed / read-only / nothing written" claims. Deliberate: a
-    // no-op run that SAYS "no files were changed" gets 'verified' (the claim is
-    // true and checked against reality), not 'nothing-to-verify'. The vacuous
-    // green this function kills is the report that claims NOTHING (e.g. a bare
-    // startup banner) — an honest no-op claim is exactly what verification is for.
-    if (/\bno\s+files?\s+(were\s+)?(changed|modified|touched|written|created)\b/i.test(summary) ||
-        /\bsession\s+was\s+read-only\b/i.test(summary) ||
-        /\bnothing\s+(was\s+)?(written|changed|modified)\b/i.test(summary))
-        return true;
-    return false;
-}
-/* ──────────────────────────────────────────────────────────────────────── */
-/* Tests claim verification                                                  */
-/* ──────────────────────────────────────────────────────────────────────── */
-/**
- * Detect tests-related claims in the report and cross-check against
- * artifacts.tests. Catches "X tests passed" exaggeration AND "all tests
- * green" when failed > 0.
- */
-export function verifyTestsClaim(summary, artifacts) {
-    const out = [];
-    const lower = summary.toLowerCase();
-    // Check the prose against the AGENT'S OWN scraped numbers. When codehere
-    // re-ran (observed), `artifacts.tests` holds the re-run result and the original
-    // scrape is preserved in `.claimed` — use the scrape here so a prose-vs-own-run
-    // self-contradiction ("all pass" while its own stdout showed failures) is still
-    // caught as a real lie, separate from a re-run that fails on an env difference
-    // (handled as a warn in verifyClaim). Self-audit 2026-06-09.
-    const scrape = artifacts?.tests?.claimed;
-    const tests = scrape ? { passed: scrape.passed, failed: scrape.failed } : artifacts?.tests;
-    // Extract numeric "X tests passed" / "X passed" / "X of Y" patterns.
-    // We deliberately keep the regex tight to avoid false positives on prose
-    // like "this is the 5th attempt".
-    const countClaim = matchTestCountClaim(summary);
-    if (countClaim !== null) {
-        if (!tests) {
-            // The agent claimed a test count but the adapter recorded no test
-            // artifact. Suspicious but not necessarily wrong (the adapter may not
-            // capture test results yet). Flag as warn.
-            out.push({
-                severity: 'warn',
-                kind: 'tests',
-                claim: countClaim.quote,
-                reality: 'no test artifact captured',
-                message: `Final Report claims "${countClaim.quote}" but no test results were recorded by the agent.`,
-            });
-        }
-        else {
-            const actualPassed = tests.passed;
-            const actualFailed = tests.failed;
-            const actualTotal = actualPassed + actualFailed;
-            // Per codex Round 1: "X of Y tests passed" must check BOTH numbers.
-            // A claim like "18 of 18 tests passed" with reality {passed: 17, failed: 1}
-            // has matching total (18 ran) but mismatched passed (17 ≠ 18) · we flag.
-            if (countClaim.passedCount !== undefined && countClaim.passedCount !== actualPassed) {
-                out.push({
-                    severity: 'fail',
-                    kind: 'tests',
-                    claim: countClaim.quote,
-                    reality: `${actualPassed} passed, ${actualFailed} failed (${actualTotal} ran)`,
-                    message: `Final Report claims ${countClaim.passedCount} tests passed but reality is ${actualPassed} passed and ${actualFailed} failed.`,
-                });
-            }
-            if (countClaim.totalCount !== undefined && countClaim.totalCount !== actualTotal) {
-                out.push({
-                    severity: 'fail',
-                    kind: 'tests',
-                    claim: countClaim.quote,
-                    reality: `${actualTotal} ran (${actualPassed} passed, ${actualFailed} failed)`,
-                    message: `Final Report claims ${countClaim.totalCount} tests ran but reality is ${actualTotal} (${actualPassed} passed, ${actualFailed} failed).`,
-                });
-            }
-        }
-    }
-    // Universal "all tests pass" / "all green" claims. Requires explicit
-    // universal qualifier ("all", "every", "each", "no failures") so that
-    // count claims like "18 tests passed" don't also match — count
-    // mismatches are caught above by matchTestCountClaim.
-    const claimsAllPass = /\b(?:all|every|each)\s+tests?\s+(?:pass(?:ed|ing)?|green|succeed(?:ed)?)\b/i.test(lower) ||
-        /\btests?\s+all\s+(?:pass(?:ed|ing)?|green)\b/i.test(lower) ||
-        /\ball\s+(?:tests?\s+)?(?:green|passing)\b/i.test(lower) ||
-        /\bno\s+(?:test\s+)?(?:failures|regressions?)\b/i.test(lower);
-    if (claimsAllPass && tests && tests.failed > 0) {
-        out.push({
-            severity: 'fail',
-            kind: 'tests',
-            claim: 'all tests pass / green / no failures',
-            reality: `${tests.failed} test${tests.failed === 1 ? '' : 's'} failed`,
-            message: `Final Report claims all tests pass but ${tests.failed} actually failed.`,
-        });
-    }
-    return out;
-}
-/**
- * Parse the first explicit test-count claim from the summary.
- * Returns null if no specific count is mentioned.
- *
- * Patterns matched (a single claim can assert BOTH a passed-count AND a total):
- *   "18 tests passed"          → { passedCount: 18 }
- *   "18 of 18 tests passed"    → { passedCount: 18, totalCount: 18 }
- *   "passed 18 tests"          → { passedCount: 18 }
- *   "ran 25 tests"             → { totalCount: 25 }
- *
- * Per codex Round 1 review on commit 8481412: "X of Y tests passed" must
- * check BOTH numbers, not just total, otherwise "18 of 18 passed" with
- * reality {passed: 17, failed: 1} slips through (total still equals 18).
- */
-function matchTestCountClaim(summary) {
-    // "X of Y tests passed/green/succeeded" → X is passed AND Y is total.
-    // Per codex Round 2: require explicit pass-context to populate passedCount
-    // so negative phrasing ("1 of 3 tests failed") doesn't become a false-positive
-    // "1 passed" claim.
-    const ofPassMatch = summary.match(/\b(\d+)\s+(?:of|\/)\s+(\d+)\s+tests?\s+(?:pass(?:ed|ing)?|green|succeed(?:ed)?)\b/i);
-    if (ofPassMatch) {
-        return {
-            quote: ofPassMatch[0],
-            passedCount: parseInt(ofPassMatch[1], 10),
-            totalCount: parseInt(ofPassMatch[2], 10),
-        };
-    }
-    // "X of Y tests" without pass-context → only Y is a total-count claim. X
-    // is ambient (failed / ran / partial / etc) and we don't pretend to know.
-    // Per codex Round 3: this branch preserves total-count detection for
-    // bare X-of-Y phrasing so the verifier still catches mismatched totals.
-    const ofMatch = summary.match(/\b(\d+)\s+(?:of|\/)\s+(\d+)\s+tests?\b/i);
-    if (ofMatch) {
-        return {
-            quote: ofMatch[0],
-            totalCount: parseInt(ofMatch[2], 10),
-        };
-    }
-    // "X tests passed" / "X tests pass"
-    const passedMatch = summary.match(/\b(\d+)\s+tests?\s+(?:pass(?:ed|ing)?|green)\b/i);
-    if (passedMatch) {
-        return { quote: passedMatch[0], passedCount: parseInt(passedMatch[1], 10) };
-    }
-    // "passed X tests" / "ran X tests"
-    const reverseMatch = summary.match(/\b(?:pass(?:ed)?|ran)\s+(\d+)\s+tests?\b/i);
-    if (reverseMatch) {
-        const isPass = /pass/i.test(reverseMatch[0]);
-        return {
-            quote: reverseMatch[0],
-            passedCount: isPass ? parseInt(reverseMatch[1], 10) : undefined,
-            totalCount: isPass ? undefined : parseInt(reverseMatch[1], 10),
-        };
-    }
-    return null;
-}
-/* ──────────────────────────────────────────────────────────────────────── */
-/* Files claim verification                                                  */
-/* ──────────────────────────────────────────────────────────────────────── */
-/**
- * Detect files-related claims in the report and cross-check against
- * artifacts.files. Catches "no files changed" when files DID change,
- * and "X files changed" count mismatches.
- */
-export function verifyFilesClaim(summary, artifacts) {
-    const out = [];
-    // Claim: "no files were changed" / "no files modified" / similar
-    const claimsNoFiles = /\bno\s+files?\s+(were\s+)?(changed|modified|touched|written|created)\b/i.test(summary) ||
-        /\bsession\s+was\s+read-only\b/i.test(summary) ||
-        /\bnothing\s+(was\s+)?(written|changed|modified)\b/i.test(summary);
-    // Claim: "X files changed" / "modified X files" / "wrote X files"
-    const countClaim = matchFilesCountClaim(summary);
-    // Per codex Round 1: if the adapter didn't capture file artifacts at all,
-    // we can't verify file claims · warn rather than fail. Mirrors the tests
-    // path (missing tests artifact → warn). Distinguishes from `files: []`
-    // which is a valid "explicitly captured zero files" reality.
-    if (artifacts?.files === undefined) {
-        if (claimsNoFiles) {
-            out.push({
-                severity: 'warn',
-                kind: 'files',
-                claim: 'no files changed / read-only',
-                reality: 'no file artifact captured',
-                message: 'Final Report claims no files changed but no file results were recorded by the agent.',
-            });
-        }
-        if (countClaim !== null) {
-            out.push({
-                severity: 'warn',
-                kind: 'files',
-                claim: countClaim.quote,
-                reality: 'no file artifact captured',
-                message: `Final Report claims "${countClaim.quote}" but no file results were recorded by the agent.`,
-            });
-        }
-        return out;
-    }
-    const actualCount = artifacts.files.length;
-    if (claimsNoFiles && actualCount > 0) {
-        out.push({
-            severity: 'fail',
-            kind: 'files',
-            claim: 'no files changed / read-only',
-            reality: `${actualCount} file${actualCount === 1 ? '' : 's'} changed`,
-            message: `Final Report claims no files were changed but ${actualCount} actually were.`,
-        });
-    }
-    if (countClaim !== null && countClaim.value !== actualCount) {
-        out.push({
-            severity: 'fail',
-            kind: 'files',
-            claim: countClaim.quote,
-            reality: `${actualCount} file${actualCount === 1 ? '' : 's'} changed`,
-            message: `Final Report claims ${countClaim.value} file${countClaim.value === 1 ? '' : 's'} were changed but ${actualCount} actually were.`,
-        });
-    }
-    return out;
-}
-/* ──────────────────────────────────────────────────────────────────────── */
-/* Named-file claims · "Added subtract() to calc.js"                         */
-/* ──────────────────────────────────────────────────────────────────────── */
-// Library/runtime names that look like files but never are ("migrated to
-// Next.js", "Node.js 20"). Skipped to keep the tripwire conservative.
-const NON_FILE_NAMES = /^(node|next|vue|react|express|d3|three|angular|nuxt|deno|bun|ember|backbone)\.js$/i;
-/**
- * Extract file paths the report claims to have CHANGED — an edit verb followed
- * (within the same sentence) by a path-shaped token. Deliberately verb-gated:
- * "read calc.js" or a bare mention is NOT a change claim.
- */
-export function extractNamedFileClaims(summary) {
-    const out = new Set();
-    const re = /\b(?:added|created|wrote|modified|updated|changed|edited|fixed|patched|refactored|renamed|implemented|extended)\b[^.!?\n]{0,80}?`?([\w][\w./-]*\.[A-Za-z0-9]{1,8})(?::\d+)?`?/gi;
-    let m;
-    while ((m = re.exec(summary)) !== null) {
-        const path = m[1];
-        if (NON_FILE_NAMES.test(path))
-            continue;
-        out.add(path);
-    }
-    return [...out];
-}
-/** True when a claimed path matches a recorded file change (exact, suffix, or basename). */
-function fileMatchesArtifacts(claimPath, artifacts) {
-    if (!artifacts?.files?.length)
-        return false;
-    // LOW #26 (atomic audit) · the bare-basename fallback only applies when the
-    // CLAIM is a bare basename. A directory-qualified claim ('src/a/calc.js')
-    // must not green-match a same-named file elsewhere ('lib/b/calc.js').
-    const claimIsBare = !claimPath.includes('/');
-    return artifacts.files.some((f) => {
-        const p = f.path ?? '';
-        if (p === claimPath || p.endsWith('/' + claimPath) || claimPath.endsWith('/' + p))
-            return true;
-        return claimIsBare && (p.split('/').pop() ?? p) === claimPath;
-    });
-}
-/**
- * Tripwire side of named-file claims: the report says it changed a specific
- * file but the recorded changes don't include it. Warn, not fail — the token
- * might be a tool/library mention the verb regex over-read; the count/no-files
- * checks above own the hard contradictions.
- */
-export function verifyNamedFileClaims(summary, artifacts) {
-    if (artifacts?.files === undefined)
-        return []; // can't check (e.g. non-git cwd) — stays unverifiable
-    const out = [];
-    for (const path of extractNamedFileClaims(summary)) {
-        if (!fileMatchesArtifacts(path, artifacts)) {
-            out.push({
-                severity: 'warn',
-                kind: 'files',
-                claim: `changed ${path}`,
-                reality: artifacts.files.length === 0
-                    ? 'no file changes recorded'
-                    : `recorded changes: ${artifacts.files.map((f) => f.path).join(', ')}`,
-                message: `Final Report says it changed ${path} but the recorded file changes don't include it.`,
-            });
-        }
-    }
-    return out;
-}
-function matchFilesCountClaim(summary) {
-    // "X files changed" / "X files modified" / "X files were touched"
-    const m1 = summary.match(/\b(\d+)\s+files?\s+(were\s+)?(?:changed|modified|touched|written|created|added|updated)\b/i);
-    if (m1)
-        return { quote: m1[0], value: parseInt(m1[1], 10) };
-    // "changed X files" / "modified X files"
-    const m2 = summary.match(/\b(?:changed|modified|touched|wrote|created|added|updated)\s+(\d+)\s+files?\b/i);
-    if (m2)
-        return { quote: m2[0], value: parseInt(m2[1], 10) };
-    return null;
-}
-/**
- * The "verified requires git" SURFACE invariant, single-sourced (R11 / M2).
- * A rendered verifier panel must never show 'verified' while its git-tracked
- * flag is off: without git-tracked artifacts the file-claims are unverifiable
- * (see verifyNamedFileClaims's undefined-files early-return above), so a green
- * "verified" there would be an overclaim. The /verify dogfood page (H1 in
- * verify-page.ts) interpolates THIS function's compiled source into its client
- * script and executes it — it no longer re-encodes the rule, so the rule can't
- * drift between the engine and the harness. Keep it self-contained (no captured
- * scope, no imports) — its .toString() must run standalone in a browser.
- */
-export function violatesGitInvariant(verdict, gitTracked) {
-    return verdict === 'verified' && !gitTracked;
-}
-/* ──────────────────────────────────────────────────────────────────────── */
-/* Verdict summary                                                           */
-/* ──────────────────────────────────────────────────────────────────────── */
-function buildVerdictSummary(status, findings, observedOnly = false) {
-    if (status === 'verified')
-        return observedOnly
-            ? "codehere re-ran the test suite and it passed — observed by codehere, not just the agent's word."
-            : 'Claims match the recorded reality. No gaps detected.';
-    if (status === 'no-report')
-        return 'No Final Report to verify.';
-    if (status === 'nothing-to-verify')
-        return "The report makes no checkable claim, so there's nothing to verify against the recorded reality.";
-    const failCount = findings.filter((f) => f.severity === 'fail').length;
-    const warnCount = findings.filter((f) => f.severity === 'warn').length;
-    if (failCount > 0 && warnCount > 0) {
-        return `${failCount} claim${failCount === 1 ? '' : 's'} contradict the recorded reality, plus ${warnCount} unverifiable claim${warnCount === 1 ? '' : 's'}.`;
-    }
-    if (failCount > 0) {
-        return `${failCount} claim${failCount === 1 ? '' : 's'} contradict the recorded reality.`;
-    }
-    return `${warnCount} unverifiable claim${warnCount === 1 ? '' : 's'}.`;
-}
-//# sourceMappingURL=verify-claim.js.map