@event4u/agent-config 14.12.0 → 14.13.0

package/src/scripts/_lib/judge_hygiene.ts

@@ -0,0 +1,228 @@
+/**
+ * Judge hygiene — what an assertion is worth, and what a pass verdict must show.
+ *
+ * `road-to-skill-ecosystem-eval-integrity` Phase 3, Steps 3–6. Steps 1 and 2
+ * (blinding and the order-swap) were already met by
+ * `check_quality_regression.evaluatePair`, which judges every pair in both
+ * orders and resolves a flip to `inconsistent` rather than to a winner — that
+ * is stronger than the step's own "default to a tie", because a flagged tie is
+ * still a tie in a denominator while `inconsistent` is its own bucket with its
+ * own reported rate.
+ *
+ * ── Why an assertion that always passes is worse than no assertion ──────────
+ * It inflates the treatment pass rate without reflecting any value: both arms
+ * satisfy it, so it contributes a guaranteed point to each and narrows nothing.
+ * The reverse — an assertion that always fails — is not noise but a broken
+ * check or an impossible expectation, and it is worth finding for a different
+ * reason. The interesting set is neither: the assertions that pass in treatment
+ * and fail in control ARE the measurement, and everything else is overhead
+ * masquerading as coverage.
+ */
+
+/** How a rubric item or assertion is phrased, which is what overfitting looks like. */
+export type OverfitShape = 'outcome' | 'technique' | 'vocabulary';
+
+export interface OverfitClassification {
+    shape: OverfitShape;
+    /** Why this shape, in one clause, printable in an advisory report. */
+    reason: string;
+}
+
+/**
+ * Phrases that name HOW rather than WHAT.
+ *
+ * `technique` is an assertion that prescribes a mechanism — it passes only for
+ * an implementation that took the route the author had in mind, so a better
+ * solution scores lower. `vocabulary` is narrower and worse: it checks that a
+ * particular WORD appears, which any arm can satisfy by saying the word.
+ */
+const TECHNIQUE_MARKERS = [
+    'using ',
+    'by calling',
+    'via ',
+    'with the helper',
+    'implemented as',
+    'should call',
+    'must call',
+    'invokes',
+];
+
+const VOCABULARY_MARKERS = [
+    'mentions',
+    'contains the word',
+    'uses the term',
+    'the phrase',
+    'says ',
+    'literally',
+];
+
+/**
+ * Classify one criterion by shape. ADVISORY — this gates nothing.
+ *
+ * Deliberately a keyword heuristic and deliberately never a verdict input. A
+ * classifier that decided anything would need to be right, and no phrase list
+ * is right about natural language; what it can do honestly is put a shape
+ * beside each item so a human reviewing a rubric sees that nine of twelve items
+ * check vocabulary. That is a real finding and it costs nothing to be wrong
+ * about one of them.
+ */
+export function classifyOverfit(criterion: string): OverfitClassification {
+    const c = criterion.toLowerCase();
+    for (const m of VOCABULARY_MARKERS) {
+        if (c.includes(m)) {
+            return {
+                shape: 'vocabulary',
+                reason: `phrased around wording ("${m.trim()}") — any arm passes by saying the word`,
+            };
+        }
+    }
+    for (const m of TECHNIQUE_MARKERS) {
+        if (c.includes(m)) {
+            return {
+                shape: 'technique',
+                reason: `prescribes a mechanism ("${m.trim()}") — a better route scores lower`,
+            };
+        }
+    }
+    return { shape: 'outcome', reason: 'phrased as a result rather than a route or a wording' };
+}
+
+/** One assertion's behaviour across both arms of a paired run. */
+export interface AssertionObservation {
+    id: string;
+    /** Trials where this assertion passed in the treatment arm. */
+    treatment_passes: number;
+    /** Trials where it passed in the control arm. */
+    control_passes: number;
+    trials: number;
+}
+
+export type AssertionVerdict =
+    | 'discriminating'
+    | 'non-discriminating-always-pass'
+    | 'non-discriminating-always-fail'
+    | 'inverted'
+    | 'partial';
+
+export interface AssertionFinding {
+    id: string;
+    verdict: AssertionVerdict;
+    /** The prescribed action, so the report does not stop at a label. */
+    action: string;
+}
+
+/**
+ * Classify an assertion by whether it separates the arms.
+ *
+ * The pruning rule of Step 5 is expressed HERE rather than as prose in a
+ * document, because a rule that lives only in a document is applied by whoever
+ * remembers it:
+ *
+ *   · always passes in both  → REMOVE or replace. It is pure inflation.
+ *   · always fails in both   → INVESTIGATE. A broken check or an impossible bar.
+ *   · treatment > control    → KEEP. This is where the value is.
+ *   · control > treatment    → KEEP and read it. An inverted assertion is a
+ *                              finding about the treatment, not a defect here.
+ */
+export function classifyAssertion(o: AssertionObservation): AssertionFinding {
+    if (o.trials <= 0) {
+        return {
+            id: o.id,
+            verdict: 'partial',
+            action: 'no trials — the assertion was never exercised, which is not a result',
+        };
+    }
+    const allT = o.treatment_passes === o.trials;
+    const allC = o.control_passes === o.trials;
+    const noneT = o.treatment_passes === 0;
+    const noneC = o.control_passes === 0;
+    if (allT && allC) {
+        return {
+            id: o.id,
+            verdict: 'non-discriminating-always-pass',
+            action:
+                'remove or replace — it passes in both arms, so it adds a guaranteed point to each ' +
+                'and inflates the treatment pass rate without reflecting value',
+        };
+    }
+    if (noneT && noneC) {
+        return {
+            id: o.id,
+            verdict: 'non-discriminating-always-fail',
+            action:
+                'investigate — an assertion nothing satisfies is a broken check or an impossible ' +
+                'expectation, not a measurement of the treatment',
+        };
+    }
+    if (o.treatment_passes > o.control_passes) {
+        return { id: o.id, verdict: 'discriminating', action: 'keep — this is where the value is' };
+    }
+    if (o.control_passes > o.treatment_passes) {
+        return {
+            id: o.id,
+            verdict: 'inverted',
+            action:
+                'keep and read it — the control satisfies this more often, which is a finding about ' +
+                'the treatment rather than a defect in the assertion',
+        };
+    }
+    return {
+        id: o.id,
+        verdict: 'partial',
+        action: 'keep — equal pass counts but not saturated, so it still carries information',
+    };
+}
+
+/** Summary a report can print, with the inflation number named. */
+export interface AssertionAudit {
+    findings: readonly AssertionFinding[];
+    discriminating: number;
+    non_discriminating: number;
+    /** Points every arm is guaranteed by always-pass assertions. */
+    guaranteed_points: number;
+}
+
+export function auditAssertions(obs: readonly AssertionObservation[]): AssertionAudit {
+    const findings = obs.map(classifyAssertion);
+    const nd = findings.filter((f) => f.verdict.startsWith('non-discriminating')).length;
+    return {
+        findings,
+        discriminating: findings.filter((f) => f.verdict === 'discriminating').length,
+        non_discriminating: nd,
+        guaranteed_points: findings.filter((f) => f.verdict === 'non-discriminating-always-pass').length,
+    };
+}
+
+/**
+ * Does a section carry EVIDENCE, or only the label a pass was looking for?
+ *
+ * Step 6. The failure it names is specific and common: a section headed with the
+ * expected label followed by one vague sentence scores as a pass, because the
+ * label being present was mistaken for the substance being present. The check is
+ * deliberately crude — a length floor plus a concreteness signal — because the
+ * alternative is a judge, and a judge is the thing whose output this is meant to
+ * check.
+ *
+ * Returns `null` when the section passes, or the reason it does not.
+ */
+export function evidenceDeficit(section: string, opts: { minWords?: number } = {}): string | null {
+    const minWords = opts.minWords ?? 25;
+    const body = section.replace(/^#{1,6} .*$/gm, '').trim();
+    const words = body.split(/\s+/).filter((w) => w !== '').length;
+    if (words < minWords) {
+        return `carries ${String(words)} word(s) of body, below the ${String(minWords)}-word floor — ` +
+            'the label is present and the substance is not';
+    }
+    // A concrete claim cites something a reader can check: a path, a number, a
+    // command, an identifier. Prose with none of those is an assertion about
+    // itself.
+    const concrete =
+        /[\w./-]+\.(?:ts|js|md|json|ya?ml|py|sh)\b/.test(body) ||
+        /\b\d+(?:\.\d+)?\s*(?:%|ms|s|MB|KB|lines?|files?|cases?|trials?)\b/.test(body) ||
+        /`[^`]+`/.test(body);
+    if (!concrete) {
+        return 'no checkable reference — no path, no measured figure, no identifier. A section that ' +
+            'cites nothing is a claim about itself.';
+    }
+    return null;
+}

package/src/scripts/_lib/legacy_boundary_map.ts

@@ -0,0 +1,228 @@
+/**
+ * A per-path legacy/modern verdict for a half-migrated repository.
+ *
+ * `road-to-consumer-repo-reality` Phase 4, step 4.1. Incremental replacement of
+ * a legacy system alongside the system it replaces is a named, published
+ * migration pattern; the artifact that pattern needs and nobody produces is a
+ * decidable answer to *"which conventions hold HERE"*.
+ *
+ * Current guidance says to respect existing patterns and to keep diffs minimal.
+ * Neither is decidable without this: in a tree that is two-fifths namespaced,
+ * strict-typed, PSR-shaped code and three-fifths an older include-and-dispatch
+ * module system, "the existing pattern" is not a property of the repository.
+ *
+ * A MIXED VERDICT IS NOT A REFUSAL TO DECIDE. That is the step's own wording and
+ * it is the hardest requirement here: a verdict that says only "mixed" leaves
+ * the caller exactly where it started. So a mixed file reports WHICH convention
+ * governs WHICH REGION, by line range, and `conventionAt(verdict, line)` answers
+ * the only question an editor actually has — *what must my edit at this point
+ * follow?*
+ *
+ * WHAT THIS MAY AND MAY NOT ENCODE. The phase states the trap in its own body:
+ * 4.2 and 4.3 were each observed in exactly one tree, so what may be encoded is
+ * the QUESTION, never the particular mechanism that tree happened to use. 4.1 is
+ * not in that category — namespaces, strict-type declarations and PSR-4
+ * autoloading are language-level and specification-level facts, not one
+ * repository's arrangement — but the same restraint is applied anyway: the
+ * signals below are all language constructs, and none is a project-specific
+ * naming convention, directory layout or framework marker.
+ *
+ * PER PATH, NEVER PER REPOSITORY. A repository-wide verdict is the thing that
+ * makes this artifact useless: the whole point is that the answer differs by
+ * file, and inside a file by region.
+ */
+import * as fs from 'node:fs';
+
+/** Which convention governs. `unknown` is returned rather than guessed. */
+export type Convention = 'modern' | 'legacy' | 'mixed' | 'unknown';
+
+/** A contiguous run of lines governed by one convention. */
+export interface Region {
+    /** 1-indexed, inclusive. */
+    startLine: number;
+    /** 1-indexed, inclusive. */
+    endLine: number;
+    convention: 'modern' | 'legacy';
+    /** The construct that decided it. Never empty. */
+    signal: string;
+}
+
+export interface PathVerdict {
+    /** The path as given. */
+    path: string;
+    convention: Convention;
+    /**
+     * Populated for EVERY verdict that is not `unknown`, including a
+     * single-convention file — a caller should not need two code paths.
+     */
+    regions: Region[];
+    /**
+     * A single foreign signal inside an otherwise uniform file — one `$GLOBALS`
+     * read in a namespaced class, one class-like declaration in an
+     * include-and-dispatch file. Reported as a POINT and deliberately not as a
+     * region: a lone signal says something true about ITS line and nothing about
+     * the lines after it, and letting it open a region made every line to the end
+     * of the file inherit a convention it does not have.
+     */
+    foreignPoints: { line: number; convention: 'modern' | 'legacy'; signal: string }[];
+    /** One sentence naming what decided it. Never empty. */
+    reason: string;
+}
+
+/**
+ * Signals, as language constructs only.
+ *
+ * `modern` markers are things the older include-and-dispatch style cannot
+ * express; `legacy` markers are things the namespaced style does not use. A line
+ * matching neither is inherited from the enclosing region rather than guessed —
+ * most lines in any file carry no signal at all.
+ */
+const MODERN: readonly { re: RegExp; signal: string }[] = [
+    { re: /^\s*declare\s*\(\s*strict_types\s*=\s*1\s*\)/, signal: 'declare(strict_types=1)' },
+    { re: /^\s*namespace\s+[A-Za-z_]/, signal: 'namespace declaration' },
+    { re: /^\s*use\s+[A-Za-z_][A-Za-z0-9_]*\\/, signal: 'namespaced use statement' },
+    { re: /^\s*(?:final\s+|abstract\s+)?(?:readonly\s+)?(?:class|interface|trait|enum)\s+/, signal: 'class-like declaration' },
+];
+
+const LEGACY: readonly { re: RegExp; signal: string }[] = [
+    { re: /^\s*(?:require|include)(?:_once)?\s*[\s(]/, signal: 'require/include' },
+    { re: /^\s*global\s+\$/, signal: 'global statement' },
+    { re: /\$GLOBALS\s*\[/, signal: '$GLOBALS access' },
+    { re: /^\s*function\s+[a-z][A-Za-z0-9_]*\s*\(/, signal: 'unnamespaced snake/camel function' },
+];
+
+/**
+ * Requiring the Composer autoloader is the MODERN ecosystem's own bootstrap, not
+ * the include-and-dispatch pattern. Without this exception the canonical PSR-4
+ * entry point — `require __DIR__ . '/../vendor/autoload.php';` — makes every
+ * modern application's front controller come back `mixed`, which is the single
+ * most common file shape in the ecosystem this module is meant to classify.
+ */
+const AUTOLOAD_REQUIRE = /vendor\/autoload\.php/;
+
+/**
+ * A line that is only a comment carries no convention. `// never touch
+ * $GLOBALS['x']` is advice ABOUT legacy code, and reading it as legacy code
+ * flips the region a reader is standing in. Block-comment interiors are not
+ * tracked — that needs a lexer — so only whole-line `//`, `#` and `*`
+ * continuations are skipped.
+ */
+const COMMENT_ONLY = /^\s*(?:\/\/|#|\*|\/\*)/;
+
+function _signalFor(line: string): { convention: 'modern' | 'legacy'; signal: string } | null {
+    if (COMMENT_ONLY.test(line)) return null;
+    for (const { re, signal } of MODERN) if (re.test(line)) return { convention: 'modern', signal };
+    if (AUTOLOAD_REQUIRE.test(line)) return null;
+    for (const { re, signal } of LEGACY) if (re.test(line)) return { convention: 'legacy', signal };
+    return null;
+}
+
+/**
+ * Classify one file's text into regions.
+ *
+ * A signal starts a region that runs until the next signal of the OTHER
+ * convention. Lines before the first signal belong to the first region, so every
+ * line of a classified file is covered — a caller asking about line 1 of a file
+ * whose first signal is on line 3 gets an answer rather than a hole.
+ */
+export function classifyText(pathName: string, text: string): PathVerdict {
+    const lines = text.split('\n');
+    const hits: { line: number; convention: 'modern' | 'legacy'; signal: string }[] = [];
+    for (let i = 0; i < lines.length; i += 1) {
+        const hit = _signalFor(lines[i] ?? '');
+        if (hit !== null) hits.push({ line: i + 1, ...hit });
+    }
+
+    // A lone signal of one convention, against two or more of the other, is a
+    // POINT rather than a region boundary. Two is the threshold because one
+    // occurrence is as likely to be an artifact — a legacy global read inside a
+    // modern service, a single class declaration in an old dispatch file — as a
+    // statement about the surrounding code.
+    const counts = { modern: 0, legacy: 0 };
+    for (const h of hits) counts[h.convention] += 1;
+    const isolated: 'modern' | 'legacy' | null =
+        counts.modern === 1 && counts.legacy >= 2 ? 'modern'
+        : counts.legacy === 1 && counts.modern >= 2 ? 'legacy'
+        : null;
+    const foreignPoints = isolated === null ? [] : hits.filter((h) => h.convention === isolated);
+    const structural = isolated === null ? hits : hits.filter((h) => h.convention !== isolated);
+
+    const regions: Region[] = [];
+    for (const hit of structural) {
+        const i = hit.line - 1;
+        const last = regions[regions.length - 1];
+        if (last !== undefined && last.convention === hit.convention) {
+            last.endLine = i + 1;
+            continue;
+        }
+        regions.push({ startLine: i + 1, endLine: i + 1, convention: hit.convention, signal: hit.signal });
+    }
+
+    if (regions.length === 0) {
+        return {
+            path: pathName,
+            convention: 'unknown',
+            regions: [],
+            foreignPoints,
+            reason: 'no convention signal found — neither a namespaced construct nor an include-and-dispatch one',
+        };
+    }
+
+    // Extend each region to meet the next, and the first back to line 1, so the
+    // whole file is covered and `conventionAt` never returns a hole.
+    const first = regions[0] as Region;
+    first.startLine = 1;
+    for (let i = 0; i < regions.length - 1; i += 1) {
+        (regions[i] as Region).endLine = (regions[i + 1] as Region).startLine - 1;
+    }
+    (regions[regions.length - 1] as Region).endLine = Math.max(lines.length, 1);
+
+    const kinds = new Set(regions.map((r) => r.convention));
+    if (kinds.size === 1) {
+        const only = [...kinds][0] as 'modern' | 'legacy';
+        return {
+            path: pathName,
+            convention: only,
+            regions,
+            foreignPoints,
+            reason:
+                `every structural signal in this file is ${only} (${regions.map((r) => r.signal).join(', ')})` +
+                (foreignPoints.length === 0
+                    ? ''
+                    : `. One isolated ${isolated} signal at L${foreignPoints.map((f) => f.line).join(', L')} ` +
+                      `(${foreignPoints.map((f) => f.signal).join(', ')}) — true of that line, not of the lines after it`),
+        };
+    }
+    return {
+        path: pathName,
+        convention: 'mixed',
+        regions,
+        foreignPoints,
+        reason:
+            `both conventions appear in this file, in ${regions.length} regions. ` +
+            regions.map((r) => `L${r.startLine}-${r.endLine} ${r.convention} (${r.signal})`).join('; ') +
+            '. An edit follows the convention of the region it lands in, not the file as a whole.',
+    };
+}
+
+/** Read and classify one file. An unreadable file is `unknown`, never guessed. */
+export function classifyPath(absPath: string, displayPath = absPath): PathVerdict {
+    try {
+        return classifyText(displayPath, fs.readFileSync(absPath, 'utf8'));
+    } catch {
+        return { path: displayPath, convention: 'unknown', regions: [], foreignPoints: [], reason: 'unreadable' };
+    }
+}
+
+/**
+ * The only question an editor actually has: what must an edit at THIS line
+ * follow? Returns `null` when the file carries no signal at all — which is a
+ * real answer, and different from "modern by default".
+ */
+export function conventionAt(verdict: PathVerdict, line: number): 'modern' | 'legacy' | null {
+    for (const f of verdict.foreignPoints) if (f.line === line) return f.convention;
+    for (const r of verdict.regions) {
+        if (line >= r.startLine && line <= r.endLine) return r.convention;
+    }
+    return null;
+}