@gobing-ai/spur 0.4.2 → 0.4.3

package/plugins/sp/lib/dogfood-report.ts

@@ -0,0 +1,278 @@
+/**
+ * dogfood report contract — the pure `validateReport(markdown)` core.
+ *
+ * Extracted from `plugins/sp/scripts/dogfood-testing/validate-report.ts` (task 1173, plan P7/ADR-130):
+ * the reusable validation logic lives in lib, and the script keeps only the CLI entry that opens and
+ * prints. Callers and tests may keep importing the script — it re-exports this module.
+ *
+ * Pure function over a report markdown string → { ok, errors[] } with stable error
+ * codes, callable from tests and by agents finalizing a run. It validates the
+ * *complete-report* shape: the six unique section headings, Issues subheads, the
+ * mandatory summary footer with both delivery paths, frontmatter protocol string,
+ * ledger↔declared-steps cardinality, and the Cost evidence contract (task 0913):
+ * a complete report must carry a Cost block (Ledger estimate / Method / Meter),
+ * percentages within 0..100, observable token totals consistent with the ledger
+ * row sums (fresh + cached), and no fabricated share when no observable rows
+ * exist. Unknown cells are never folded into zero; all-unknown evidence must
+ * render `n/a`. Aborted/partial reports remain out of scope — they legitimately
+ * lack the footer and Steps line.
+ */
+
+export interface ReportValidation {
+    ok: boolean;
+    errors: string[];
+}
+
+const REQUIRED_SECTIONS = [1, 2, 3, 4, 5, 6] as const;
+const CANONICAL_PROTOCOL = 'sp:dogfood-testing@1.2';
+
+function countSectionHeadings(markdown: string, section: number): number {
+    const re = new RegExp(`^### ${section}\\.`, 'gm');
+    return (markdown.match(re) ?? []).length;
+}
+
+function countLedgerDataRows(markdown: string): number | null {
+    const heading = markdown.match(/^### 3\. Monitor Ledger\s*$/m);
+    if (!heading || heading.index === undefined) return null;
+    const after = markdown.slice(heading.index + heading[0].length);
+    const nextHeading = after.search(/^### /m);
+    const body = nextHeading === -1 ? after : after.slice(0, nextHeading);
+    const rows = body
+        .split('\n')
+        .filter((line) => line.trim().startsWith('|'))
+        .filter((line) => !/^\|[\s:|-]+\|?\s*$/.test(line.trim()))
+        // drift:external rows are documentary (task 0296) — included in the table but
+        // subtracted from the executed-step count, per the @1.2 cardinality contract;
+        // the prescribed code-span form (`drift:external`) matches too (task 0701 R5b).
+        .filter((line) => !/^\|\s*`?drift:/.test(line.trim()));
+    // Minus the header row; what remains are data rows.
+    return Math.max(rows.length - 1, 0);
+}
+
+function declaredExecutedSteps(markdown: string): number | null {
+    const match = markdown.match(/\*\*Steps:\*\*\s*\d+\s+derived,\s*(\d+)\s+executed/)?.[1];
+    return match ? Number.parseInt(match, 10) : null;
+}
+
+// ── Cost evidence contract (task 0913, R2) ──────────────────────────────────
+
+const UNKNOWN_CELLS = new Set(['~unknown', 'unknown', '—', '-', '?', 'n/a', '~n/a']);
+
+/** Display-rounding tolerance: a whole-percent share may differ from the exact
+ * value by at most 1 point (e.g. 700/2100 = 33.3% may display as 33%). */
+const CACHE_SHARE_TOLERANCE = 1;
+
+function sectionBody(markdown: string, headingRe: RegExp): string | null {
+    const heading = markdown.match(headingRe);
+    if (!heading || heading.index === undefined) return null;
+    const after = markdown.slice(heading.index + heading[0].length);
+    const nextHeading = after.search(/^###? /m);
+    return nextHeading === -1 ? after : after.slice(0, nextHeading);
+}
+
+interface CostTotals {
+    total: number | 'n/a' | null;
+    cached: number | 'n/a' | null;
+    pct: number | 'n/a' | null;
+}
+
+function parseTokensLine(line: string): CostTotals | null {
+    const match = line.match(
+        /~(?<total>n\/a|\d+)\s+total\s*\|\s*~(?<cached>n\/a|\d+)\s+cached(?:\s*\(\s*~?(?<pct>n\/a|\d+(?:\.\d+)?)%\s*hit rate\))?/i,
+    );
+    if (!match) return null;
+    const num = (v: string | undefined): number | 'n/a' | null =>
+        v === undefined ? null : v === 'n/a' ? 'n/a' : Number.parseInt(v, 10);
+    return { total: num(match.groups?.total), cached: num(match.groups?.cached), pct: num(match.groups?.pct) };
+}
+
+interface LedgerCostRow {
+    step: string;
+    fresh: number | null;
+    cached: number | null;
+    cachePct: number | null;
+    unknownCell: boolean;
+}
+
+function parseTokenCell(cell: string): number | null {
+    const t = cell.trim();
+    if (UNKNOWN_CELLS.has(t)) return null;
+    const m = t.match(/~?(\d+)/)?.[1];
+    return m ? Number.parseInt(m, 10) : null;
+}
+
+function parsePctCell(cell: string): number | null {
+    const t = cell.trim();
+    if (UNKNOWN_CELLS.has(t)) return null;
+    const m = t.match(/(\d+(?:\.\d+)?)\s*%/)?.[1];
+    return m ? Number.parseFloat(m) : null;
+}
+
+function ledgerCostRows(markdown: string): LedgerCostRow[] {
+    const body = sectionBody(markdown, /^### 3\. Monitor Ledger\s*$/m);
+    if (body === null) return [];
+    return body
+        .split('\n')
+        .filter((line) => line.trim().startsWith('|'))
+        .filter((line) => !/^\|[\s:|-]+\|?\s*$/.test(line.trim()))
+        .filter((line) => !/^\|\s*`?drift:/.test(line.trim()))
+        .slice(1) // drop header row
+        .map((line) => {
+            const cells = line
+                .split('|')
+                .map((c) => c.trim())
+                .filter((c, i, a) => !(i === 0 && c === '') && !(i === a.length - 1 && c === ''));
+            // Columns: 0 Step … 5 Fresh, 6 Cached, 7 Cache %
+            const freshCell = cells[5] ?? '';
+            const cachedCell = cells[6] ?? '';
+            return {
+                step: cells[0] ?? '',
+                fresh: parseTokenCell(freshCell),
+                cached: parseTokenCell(cachedCell),
+                cachePct: parsePctCell(cells[7] ?? ''),
+                unknownCell: UNKNOWN_CELLS.has(freshCell) || UNKNOWN_CELLS.has(cachedCell),
+            };
+        });
+}
+
+function validateCostEvidence(markdown: string, errors: string[]): void {
+    const costBlock = sectionBody(markdown, /^#### Cost\s*$/m);
+    if (costBlock === null) {
+        errors.push('missing_cost_block');
+        return;
+    }
+    for (const field of ['Ledger estimate', 'Method', 'Meter']) {
+        if (!costBlock.includes(`**${field}:**`)) errors.push(`missing_cost_field:${field}`);
+    }
+
+    const ledgerLine = costBlock.split('\n').find((l) => l.includes('**Ledger estimate:**')) ?? '';
+    const costTotals = parseTokensLine(ledgerLine);
+    // A label present but unparseable is its own defect (task 0913 review P3-1): report it and
+    // keep validating — an early return here disabled every downstream cost check, including
+    // the per-row impossible-percentage scan. An absent label is already reported above as
+    // missing_cost_field; with no line to parse there is nothing further to check for it.
+    if (ledgerLine !== '' && costTotals === null) errors.push('malformed_cost_line');
+    const footerLine = markdown.split('\n').find((l) => l.trim().startsWith('Tokens:'));
+    const footerTotals = footerLine ? parseTokensLine(footerLine) : null;
+    // Same contract for the footer: a present-but-unparseable Tokens: line must not
+    // silently skip the footer totals comparison. An absent Tokens: line with the
+    // summary block present is its own defect (task 0913 verify advisory).
+    if (footerLine !== undefined && footerTotals === null) errors.push('malformed_footer');
+    if (footerLine === undefined && markdown.includes('── Dogfood Summary ──')) {
+        errors.push('missing_footer_tokens');
+    }
+
+    // Percentages must be within 0..100 wherever cost evidence renders one.
+    const pctScopes = [costBlock, footerLine ?? ''];
+    for (const row of ledgerCostRows(markdown)) {
+        if (row.cachePct !== null) pctScopes.push(`${row.cachePct}%`);
+    }
+    for (const scope of pctScopes) {
+        for (const m of scope.matchAll(/(\d+(?:\.\d+)?)\s*%/g)) {
+            const pct = Number.parseFloat(m[1] ?? '');
+            if (pct < 0 || pct > 100) errors.push(`impossible_percentage:${m[0].trim()}`);
+        }
+    }
+
+    const rows = ledgerCostRows(markdown);
+
+    // Unknown cells are never folded into a numeric share: a row whose basis is
+    // unknown must carry `—`, never a computed percent.
+    for (const row of rows) {
+        if (row.unknownCell && row.cachePct !== null) errors.push('unknown_row_with_numeric_cache');
+    }
+
+    // Per-row arithmetic: a numeric Cache % must match the row's own fresh/cached cells
+    // (±1pt display rounding), not merely the aggregate share (task 0913 verify advisory).
+    for (const row of rows) {
+        if (row.fresh === null || row.cached === null || row.cachePct === null) continue;
+        const basis = row.fresh + row.cached;
+        if (basis === 0) continue;
+        const expected = Math.round((row.cached / basis) * 100);
+        if (Math.abs(row.cachePct - expected) > CACHE_SHARE_TOLERANCE) {
+            errors.push(`cache_pct_mismatch:${row.step}_expected_${expected}`);
+        }
+    }
+
+    const observable = rows.filter((r) => r.fresh !== null && r.cached !== null);
+    const sumFresh = observable.reduce((acc, r) => acc + (r.fresh ?? 0), 0);
+    const sumCached = observable.reduce((acc, r) => acc + (r.cached ?? 0), 0);
+
+    const checkTotals = (label: string, totals: CostTotals | null): void => {
+        if (totals === null) return;
+        if (observable.length > 0) {
+            const expectedTotal = sumFresh + sumCached;
+            if (typeof totals.total === 'number' && totals.total !== expectedTotal) {
+                errors.push(`cost_total_mismatch:${label}_expected_${expectedTotal}`);
+            }
+            if (typeof totals.cached === 'number' && totals.cached !== sumCached) {
+                errors.push(`cost_total_mismatch:${label}_cached_expected_${sumCached}`);
+            }
+            const expectedPct = expectedTotal > 0 ? Math.round((sumCached / expectedTotal) * 100) : null;
+            if (expectedPct !== null) {
+                if (totals.pct === 'n/a' || totals.pct === null) {
+                    errors.push(`missing_cache_share:${label}`);
+                } else if (
+                    typeof totals.pct === 'number' &&
+                    Math.abs(totals.pct - expectedPct) > CACHE_SHARE_TOLERANCE
+                ) {
+                    errors.push(`cache_share_mismatch:${label}_expected_${expectedPct}`);
+                }
+            }
+        } else if (
+            typeof totals.total === 'number' ||
+            typeof totals.cached === 'number' ||
+            typeof totals.pct === 'number'
+        ) {
+            // No observable denominator: numeric totals/shares fabricate evidence
+            // from unknown rows (unknown folded into zero). `n/a` is the honest form.
+            errors.push(`fabricated_cache_share:${label}`);
+        }
+    };
+    checkTotals('cost', costTotals);
+    checkTotals('footer', footerTotals);
+}
+
+export function validateReport(markdown: string): ReportValidation {
+    const errors: string[] = [];
+
+    // Footer (W2/D1) — the mirrored footer block at the report end.
+    if (!markdown.includes('── Dogfood Summary ──')) errors.push('missing_footer');
+    if (!markdown.includes('[Live:')) errors.push('missing_live_path');
+    if (!markdown.includes('[Report:')) errors.push('missing_report_path');
+
+    // Six unique section headings (W2/D3).
+    for (const section of REQUIRED_SECTIONS) {
+        const count = countSectionHeadings(markdown, section);
+        if (count === 0) errors.push(`missing_section:${section}`);
+        if (count > 1) errors.push(`duplicate_section:${section}`);
+    }
+
+    // Issues subheads (W2/D3).
+    if (!markdown.includes('#### Fixed') || !markdown.includes('#### Unresolved')) {
+        errors.push('missing_issues_subheads');
+    }
+
+    // Protocol string (W1/D5) — colon form, exact version.
+    const protocolMatch = markdown.match(/^protocol:\s*(\S+)\s*$/m);
+    if (!protocolMatch || protocolMatch[1] !== CANONICAL_PROTOCOL) {
+        errors.push('protocol_string');
+    }
+
+    // Ledger cardinality (W3/D4) — data rows == declared executed steps.
+    const executed = declaredExecutedSteps(markdown);
+    if (executed === null) {
+        errors.push('missing_steps_declared');
+    } else {
+        const rows = countLedgerDataRows(markdown);
+        if (rows === null || rows !== executed) {
+            errors.push('ledger_cardinality');
+        }
+    }
+
+    // Cost evidence (task 0913, R2) — required block, arithmetic consistency,
+    // honest unknowns.
+    validateCostEvidence(markdown, errors);
+
+    return { ok: errors.length === 0, errors };
+}