@skyramp/mcp 0.3.6 → 0.3.8

package/build/utils/utils-verify/verify.js

@@ -2,10 +2,12 @@ import * as fs from "fs";
 import * as path from "path";
 import { logger } from "../logger.js";
 import { locateUtilsFiles } from "./locate.js";
-import { parseUtilsFile } from "./parse.js";
-import { findSiblingInlineCallSites, } from "./call-sites.js";
-import { parseUtilsAllows } from "./allow.js";
-import { helperFamilyFor, scenarioNameViolation, utilsSpecFor, utilsSpecForFile, } from "./language-spec.js";
+import { blankCommentsAndStrings, countAssertions, hasPageErrorGuard, parseUtilsFile, } from "./parse.js";
+import { escapeRegExp } from "../regex.js";
+import { findSiblingInlineCallSites, generatedSiblings, } from "./call-sites.js";
+import { parseUtilsAllows, } from "./allow.js";
+import { singleImporterHelpers, } from "./importers.js";
+import { helperFamilyFor, helperFamilyOf, scenarioNameViolation, utilsSpecFor, utilsSpecForFile, } from "./language-spec.js";
 /**
  * Deterministic post-reuse check of the shared utils file(s) a spec depends on.
  *
@@ -42,10 +44,11 @@ export async function verifyUtils(params) {
         advisories: [],
         malformedAllows: [],
         spec,
-        family,
+        fallbackFamily: family,
         inlineCallSites: [],
+        singleImporters: [],
     };
-    if (!spec || utilsFiles.length === 0)
+    if (!spec)
         return result;
     let specContent = "";
     try {
@@ -54,8 +57,14 @@ export async function verifyUtils(params) {
     catch {
         /* the spec may be absent outside a run; import counting then reports 0 */
     }
+    if (spec.language === "python")
+        result.violations.push(...importFormViolations(params.testFile, specContent, utilsFiles));
     const allHelpers = [];
+    const helpersByFile = new Map();
+    const allAllows = [];
     const importedNames = new Set();
+    const importedByFile = new Map();
+    const helpersOfFile = new Map();
     // Route map spans every utils file: the reuse prompt allows the helpers to split
     // across files, and the same method+path in two of them is still one duplicate.
     const byRoute = new Map();
@@ -73,16 +82,36 @@ export async function verifyUtils(params) {
             });
             throw err; // the caller records a verify error, which fails OPEN — never a silent PASS
         }
-        const helpers = parseUtilsFile(content, spec, (expr) => family.isAllowed(spec, expr));
+        const helpers = parseUtilsFile(content, spec);
+        for (const h of helpers) {
+            // An untyped parameter is an error under `noImplicitAny`; whether the host repo's
+            // tsconfig reaches the tests directory is its call, so this advises and never blocks.
+            if (h.untypedParams.length > 0)
+                result.advisories.push({
+                    kind: "untyped-parameter",
+                    file,
+                    helper: h.name,
+                    line: h.line,
+                    // Names only — the remediation is stated once, in the advisory footer.
+                    detail: h.untypedParams.map((p) => `\`${p}\``).join(", "),
+                });
+        }
         allHelpers.push(...helpers);
+        helpersByFile.set(file, helpers);
+        helpersOfFile.set(file, helpers);
         const { allows, malformed } = parseUtilsAllows(content, spec);
+        allAllows.push(...allows);
         result.helpers += helpers.length;
         result.malformedAllows.push(...malformed.map((line) => ({ file, line })));
         const stem = path.basename(file).replace(/\.[^.]+$/, "");
         const defined = new Set(helpers.map((h) => h.name));
+        const importedHere = new Set();
         for (const n of spec.importedNames(specContent, stem))
-            if (defined.has(n))
+            if (defined.has(n)) {
                 importedNames.add(n);
+                importedHere.add(n);
+            }
+        importedByFile.set(file, importedHere);
         // One marker documents ONE violation of that kind in that helper: a second,
         // undocumented scenario claim in the same helper is not covered by the first
         // decline. Markers are consumed in order.
@@ -91,7 +120,7 @@ export async function verifyUtils(params) {
             const k = `${a.kind}\0${a.helper}`;
             budget.set(k, (budget.get(k) ?? 0) + 1);
         }
-        for (const v of violationsIn(helpers, file, family, byRoute)) {
+        for (const v of violationsIn(helpers, file, family, spec, byRoute)) {
             const k = `${v.kind}\0${v.helper}`;
             const left = budget.get(k) ?? 0;
             if (left > 0) {
@@ -107,6 +136,51 @@ export async function verifyUtils(params) {
         }
     }
     result.imported = importedNames.size;
+    // One directory walk feeds both advisories: every sibling is read exactly once.
+    let siblings = [];
+    try {
+        siblings = await generatedSiblings(params.testFile, spec, {
+            includeSelf: true,
+        });
+        result.singleImporters = await singleImporterHelpers(params.testFile, utilsFiles, helpersByFile, spec, siblings);
+    }
+    catch {
+        /* advisory only — never fails a verify */
+    }
+    if (params.baseline && specContent) {
+        // Delivered assertions live in the spec and in the helpers it imports — counted
+        // per utils FILE (a same-named helper in another module is not this import) and
+        // per CALL SITE on a comment/string-blanked copy of the spec: the baseline
+        // counted executions in source order, so a 2-assertion block extracted from 3
+        // repetitions delivers 6, and a commented-out call delivers nothing.
+        const blankedSpec = blankCommentsAndStrings(specContent, spec);
+        let inHelpers = 0;
+        // The guard may legitimately move INTO a shared helper — registered there,
+        // asserted in the spec, it still guards. Credited exactly like assertions.
+        let guardInHelpers = false;
+        for (const [file, imported] of importedByFile) {
+            for (const h of helpersOfFile.get(file) ?? []) {
+                if (!imported.has(h.name))
+                    continue;
+                const calls = (blankedSpec.match(new RegExp(`\\b${escapeRegExp(h.name)}\\s*\\(`, "g")) ?? []).length;
+                if (calls === 0)
+                    continue;
+                inHelpers += calls * h.assertions.length;
+                if (h.hasGuard)
+                    guardInHelpers = true;
+            }
+        }
+        for (const v of baselineViolations(params.baseline, params.testFile, spec, specContent, inHelpers, guardInHelpers)) {
+            // A marker in any utils file documents an assertion-loss (never a removed
+            // guard). At most one loss finding exists per verify, so a matching marker's
+            // presence is the whole budget — nothing to consume.
+            if (v.kind === "assertion-loss" &&
+                allAllows.some((a) => a.kind === v.kind && a.helper === v.helper))
+                result.allowed.push(v);
+            else
+                result.violations.push(v);
+        }
+    }
     result.ok =
         result.violations.length === 0 && result.malformedAllows.length === 0;
     // Integration only — the same gate as the prompt rule. `contract`, `load` and an
@@ -115,7 +189,7 @@ export async function verifyUtils(params) {
     // not have taken.
     if ((params.testType ?? "").toLowerCase() === "integration") {
         try {
-            result.inlineCallSites = await findSiblingInlineCallSites(params.testFile, utilsFiles, allHelpers, spec);
+            result.inlineCallSites = await findSiblingInlineCallSites(params.testFile, utilsFiles, allHelpers, spec, siblings);
         }
         catch {
             /* advisory only — never fails a verify */
@@ -123,9 +197,147 @@ export async function verifyUtils(params) {
     }
     return result;
 }
-function violationsIn(helpers, file, family, byRoute) {
+/** The baseline (A7/A8) findings for a delivered spec — pure; marker handling is the
+ *  caller's. Runs whether or not a utils file was located: deleting the utils file
+ *  (with the moved assertions and guard in it) must not read as "nothing to verify". */
+function baselineViolations(baseline, testFile, spec, specContent, inHelpers, guardInHelpers) {
+    const found = [];
+    const inSpec = countAssertions(specContent, spec);
+    const delivered = inSpec + inHelpers;
+    // Marker grammar takes an identifier: `orders.spec.ts` → `orders_spec_ts`.
+    const stem = path.basename(testFile).replace(/\W/g, "_");
+    if (delivered < baseline.assertions)
+        found.push({
+            kind: "assertion-loss",
+            file: testFile,
+            helper: stem,
+            line: 1,
+            detail: `${baseline.assertions} assertions at the modularization hand-out, ${delivered} delivered (${inSpec} in the test, ${inHelpers} in the helpers it imports and calls) — restore each removed check in the test body after the helper call it verifies, or document a deliberate consolidation with the allow marker`,
+        });
+    if (baseline.pageErrorGuard &&
+        !guardInHelpers &&
+        !hasPageErrorGuard(specContent, spec))
+        found.push({
+            kind: "guard-removed",
+            file: testFile,
+            helper: stem,
+            line: 1,
+            detail: `the page.on('pageerror', …) guard present at the modularization hand-out is gone — restore the registration (in this spec, or in a shared helper this spec imports and calls) and its final expect over the collected errors; it is never removed and cannot be documented away`,
+        });
+    return found;
+}
+/**
+ * Python only. The import form that works here is not the one that works in the
+ * customer's repo, and the difference is invisible in this run.
+ *
+ * pytest imports a test inside a package directory (one carrying `__init__.py`) from
+ * that package's first NON-package ancestor, so a bare `from SkyrampUtils import …`
+ * resolves only when the runner puts the test's own directory on `sys.path`. The
+ * Skyramp executor flattens its bundle and drops the `__init__.py` files, so it does;
+ * the customer running `pytest` from their repo root does not, and the test dies at
+ * collection with `ModuleNotFoundError`, taking the whole file with it. Measured on
+ * prefect (`tests/`, `tests/server/`, `tests/server/api/` all packages) and present in
+ * five of the six real Python repos in the eval corpus — the majority shape, not an
+ * edge case.
+ *
+ * The relative form is not simply the answer: under the flattened bundle it fails with
+ * `attempted relative import with no known parent package`. The two layouts want
+ * opposite forms, so a package directory must carry BOTH — the relative import first,
+ * falling back to the bare one:
+ *
+ *     try:
+ *         from .SkyrampUtils import create_order
+ *     except ImportError:  # flattened test bundle (no package context)
+ *         from SkyrampUtils import create_order
+ *
+ * Verified in both layouts. Outside a package the bare form alone is correct, and the
+ * fallback would be noise, so it is required only where `__init__.py` sits beside the
+ * spec — which is exactly where pytest changes its rule.
+ */
+export function importFormViolations(testFile, specContent, utilsFiles) {
+    const dir = path.dirname(testFile);
+    if (!fs.existsSync(path.join(dir, "__init__.py")))
+        return [];
+    const out = [];
+    // Docstrings and other triple-quoted text can carry an example import; blanked
+    // (length- and line-preserving) so an example is never read as a live import.
+    const lines = blankTripleQuoted(specContent).split("\n");
+    for (const file of utilsFiles) {
+        // Only a module beside the spec is addressable relatively.
+        if (path.dirname(file) !== dir)
+            continue;
+        const stem = path.basename(file).replace(/\.[^.]+$/, "");
+        const esc = escapeRegExp(stem);
+        // Both supported spellings: `from M import a, b` and `import M as alias`.
+        const bareFrom = new RegExp(`^\\s*from\\s+${esc}\\s+import\\b(.*)$`);
+        const bareModule = new RegExp(`^\\s*import\\s+${esc}\\b(.*)$`);
+        const relFrom = new RegExp(`^\\s*from\\s+\\.${esc}\\s+import\\b`);
+        const relModule = new RegExp(`^\\s*from\\s+\\.\\s+import\\s+${esc}\\b`);
+        lines.forEach((text, i) => {
+            const from = bareFrom.exec(text);
+            const mod = from ? null : bareModule.exec(text);
+            if (!from && !mod)
+                return;
+            // The bare form is legitimate ONLY as the `except ImportError` arm of a pair
+            // whose relative import comes first. Checked per import against what precedes
+            // IT — a relative import elsewhere in the file (or of other names) does not
+            // make this line a fallback, and a bare import above the pair still runs first
+            // and still fails collection.
+            const before = lines.slice(0, i);
+            const lastIndexWhere = (pred) => {
+                for (let k = before.length - 1; k >= 0; k--)
+                    if (pred(before[k]))
+                        return k;
+                return -1;
+            };
+            // ANY except clause counts, not just `except ImportError`. `except
+            // ModuleNotFoundError` (the error actually raised, and an ImportError
+            // subclass) and `except (ImportError, ModuleNotFoundError)` are both things a
+            // careful author writes, and this gate REFUSES THE REPORT — so the structural
+            // requirement below (a relative import first, this import indented inside the
+            // handler) is what carries the meaning. Narrowing on the exception name buys
+            // no detection and costs blocked runs on correct code.
+            const exceptAt = lastIndexWhere((l) => /^\s*except\b/.test(l));
+            // Either relative spelling satisfies either bare spelling: a pair mixing
+            // `from . import M` with `from M import x` is unusual but not wrong, and
+            // refusing it would again block correct code.
+            const relAt = lastIndexWhere((l) => relFrom.test(l) || relModule.test(l));
+            // Textual order is not enough: the bare import must sit INSIDE that except
+            // suite. `except ImportError: pass` followed by a top-level bare import passes
+            // an ordering test, still runs, and still dies at collection.
+            const indentOf = (l) => l.length - l.trimStart().length;
+            const insideExcept = exceptAt !== -1 &&
+                indentOf(text) > indentOf(lines[exceptAt]) &&
+                lines
+                    .slice(exceptAt + 1, i)
+                    .every((l) => l.trim() === "" || indentOf(l) > indentOf(lines[exceptAt]));
+            if (insideExcept && relAt !== -1 && relAt < exceptAt)
+                return;
+            const names = (from ? from[1] : (mod?.[1] ?? "")).trim();
+            const fixed = from
+                ? `from .${stem} import ${names}`
+                : `from . import ${stem}${names ? " " + names : ""}`;
+            out.push({
+                kind: "import-form",
+                file: testFile,
+                helper: stem,
+                line: i + 1,
+                detail: `\`${text.trim()}\` — this directory is a package (\`__init__.py\`), so pytest resolves imports from the package root and will NOT find \`${stem}\` on sys.path when the customer runs it; the test dies at collection. ` +
+                    `Replace it with the pair that works in both layouts: \`try:\` / \`    ${fixed}\` / \`except ImportError:\` / \`    ${text.trim()}\``,
+            });
+        });
+    }
+    return out;
+}
+/** Triple-quoted regions blanked, preserving length and line count, so an example
+ *  import inside a docstring is never read as a live one. */
+function blankTripleQuoted(src) {
+    return src.replace(/("""|''')[\s\S]*?\1/g, (m) => m.replace(/[^\n]/g, " "));
+}
+function violationsIn(helpers, file, fallback, spec, byRoute) {
     const out = [];
     for (const h of helpers) {
+        const family = helperFamilyOf(h, fallback);
         if (family.routeDuplicates && h.method && h.normalizedPath) {
             const key = `${h.method} ${h.normalizedPath}`;
             const first = byRoute.get(key);
@@ -143,7 +355,7 @@ function violationsIn(helpers, file, family, byRoute) {
             }
         }
         for (const a of h.assertions) {
-            if (!a.allowed) {
+            if (!family.isAllowed(spec, a.text)) {
                 out.push({
                     kind: "body-assertion",
                     file,