@skyramp/mcp 0.3.6 → 0.3.8

package/build/utils/utils-verify/retrofit-equivalence.js

@@ -0,0 +1,218 @@
+import { execFile } from "child_process";
+import * as fs from "fs";
+import * as path from "path";
+import { promisify } from "util";
+import { logger } from "../logger.js";
+import { detectGitRoot } from "../gitStaging.js";
+import { blankCommentsAndStrings, parseUtilsFile } from "./parse.js";
+import { normalizePath } from "./language-spec.js";
+import { callText, hasSchemaCheck, requestFields, REQUEST_CALL_RE, SCHEMA_SCAN_LINES, shapeKeys, } from "./call-sites.js";
+import { escapeRegExp } from "../regex.js";
+const exec = promisify(execFile);
+/**
+ * Whether a retrofit changed the request the rewired test sends.
+ *
+ * STEP 5c edits a PRE-EXISTING test — code the customer already owns and whose
+ * behaviour nothing in this run chose — to call a shared helper instead of its own
+ * inline request block. That is only safe if the helper sends the same request. It
+ * usually does, because the helper was extracted from an identical block; but nothing
+ * checked it, and a helper that hardcodes a field the block never sent (or omits one
+ * it did) silently changes what the customer's test exercises. The failure is quiet:
+ * the test still runs, and a red result reads like any other red.
+ *
+ * The advisory path already refuses to OFFER such a site (see `substitutable` in
+ * call-sites.ts). This is the same question asked after the fact, about the edit that
+ * was actually made, and the rule is stricter: offering tolerates a site with an extra
+ * field, because extending the helper with a defaulted parameter is the sanctioned
+ * move; a completed retrofit must leave the request byte-equivalent in shape, so the
+ * field sets must match exactly in both directions.
+ *
+ * LIMIT: this compares the request's SHAPE — which fields and options it carries —
+ * not the values passed. A baseline sending `status: "draft"` rewired to
+ * `createPost(..., "published")` has the same field set and is not reported. Value
+ * equivalence needs argument-to-parameter binding, default resolution and expression
+ * normalisation; a wrong answer here REFUSES the report, so it is deliberately left
+ * out rather than approximated. The value case is also visible in the rewired file's
+ * own diff, where a hardcoded helper field is not.
+ *
+ * Compares against the file's committed baseline (`git show HEAD:<path>`), which is
+ * the only record of what the block sent before the edit. Fails OPEN — an untracked
+ * file, a detached worktree, or an unreadable baseline yields no mismatches rather
+ * than a refusal the agent cannot act on.
+ */
+export async function retrofitMismatches(retrofitFile, utilsFiles, spec) {
+    const baseline = await committedBaseline(retrofitFile);
+    if (baseline === undefined)
+        return [];
+    let delivered;
+    try {
+        delivered = await fs.promises.readFile(retrofitFile, "utf8");
+    }
+    catch {
+        return [];
+    }
+    const byRoute = new Map();
+    for (const utilsFile of utilsFiles) {
+        let content;
+        try {
+            content = await fs.promises.readFile(utilsFile, "utf8");
+        }
+        catch {
+            continue;
+        }
+        const helpers = parseUtilsFile(content, spec);
+        // Positions come from a comment/string-blanked copy (offsets preserved), so a
+        // commented-out or quoted request is never read as a real one; the text itself
+        // is taken from the original.
+        const scan = blankCommentsAndStrings(content, spec);
+        for (const m of scan.matchAll(new RegExp(REQUEST_CALL_RE))) {
+            const start = m.index ?? 0;
+            const line = content.slice(0, start).split("\n").length;
+            const owner = helpers.find((h) => h.line <= line && line <= h.endLine);
+            if (!owner?.method || !owner.normalizedPath)
+                continue;
+            const call = callText(content, start + m[0].length - 1);
+            const route = `${owner.method} ${owner.normalizedPath}`;
+            const list = byRoute.get(route) ?? [];
+            // One helper may issue several requests; judge it by its first for this route.
+            if (!list.some((h) => h.helper === owner.name))
+                list.push({
+                    helper: owner.name,
+                    keys: shapeKeys(call),
+                    fields: requestFields(content, call, start),
+                    schemaChecked: hasSchemaCheck(content, owner.line, owner.endLine),
+                });
+            byRoute.set(route, list);
+        }
+    }
+    if (byRoute.size === 0)
+        return [];
+    // The requests the file used to make itself, before the edit.
+    const baselineByRoute = new Map();
+    const ambiguous = new Set();
+    const baselineScan = blankCommentsAndStrings(baseline, spec);
+    for (const m of baselineScan.matchAll(new RegExp(REQUEST_CALL_RE))) {
+        const start = m.index ?? 0;
+        const line = baseline.slice(0, start).split("\n").length;
+        const call = callText(baseline, start + m[0].length - 1);
+        const method = spec.methodRe.exec(call)?.[1]?.toUpperCase();
+        const p = spec.pathRe.exec(call)?.[1];
+        if (!method || !p)
+            continue;
+        const key = `${method} ${normalizePath(p)}`;
+        const seen = baselineByRoute.get(key);
+        const here = {
+            keys: shapeKeys(call),
+            fields: requestFields(baseline, call, start),
+            schemaChecked: hasSchemaCheck(baseline, line, line + SCHEMA_SCAN_LINES),
+        };
+        if (!seen) {
+            baselineByRoute.set(key, here);
+            continue;
+        }
+        // The same route more than once: only one of them was replaced, and nothing here
+        // says which. Identical blocks compare the same either way; DIFFERING ones are
+        // ambiguous, so the route is dropped rather than judged against a block that may
+        // not be the one the helper took over. Occurrence-level attribution would need to
+        // know which block disappeared, and getting that wrong refuses a safe edit.
+        if (!sameSet(seen.fields, here.fields) || !sameSet(seen.keys, here.keys))
+            ambiguous.add(key);
+    }
+    for (const key of ambiguous)
+        baselineByRoute.delete(key);
+    if (baselineByRoute.size === 0)
+        return [];
+    const calls = blankCommentsAndStrings(delivered, spec);
+    const out = [];
+    for (const [route, helpers] of byRoute) {
+        const before = baselineByRoute.get(route);
+        if (!before)
+            continue; // the file never made this request; nothing was replaced
+        for (const helper of helpers) {
+            // Only a helper this file actually calls now can have replaced anything —
+            // under whatever local name it was imported as (`import create_order as mk`).
+            const local = localBindings(calls, helper.helper);
+            if (!local.some((n) => new RegExp(`\\b${escapeRegExp(n)}\\s*\\(`).test(calls)))
+                continue;
+            const added = [...helper.fields].filter((f) => !before.fields.has(f));
+            const dropped = [...before.fields].filter((f) => !helper.fields.has(f));
+            const reasons = [];
+            // Non-liftable call options (query_params, timeout, …): the advisory compares
+            // these before offering a site, so the completed edit must be held to them too.
+            const optAdded = [...helper.keys].filter((k) => !before.keys.has(k));
+            const optDropped = [...before.keys].filter((k) => !helper.keys.has(k));
+            if (optAdded.length > 0 || optDropped.length > 0)
+                reasons.push(`the request options changed (${[
+                    ...optAdded.map((k) => `+\`${k}\``),
+                    ...optDropped.map((k) => `-\`${k}\``),
+                ].join(", ")})`);
+            if (added.length > 0)
+                reasons.push(`it now also sends ${added.map((f) => `\`${f}\``).join(", ")}`);
+            if (dropped.length > 0)
+                reasons.push(`it no longer sends ${dropped.map((f) => `\`${f}\``).join(", ")}`);
+            if (before.schemaChecked && !helper.schemaChecked)
+                reasons.push("the response-schema check the block carried is gone");
+            if (reasons.length === 0)
+                continue;
+            const [method, ...rest] = route.split(" ");
+            out.push({
+                file: retrofitFile,
+                helper: helper.helper,
+                method,
+                path: rest.join(" "),
+                detail: reasons.join("; "),
+            });
+        }
+    }
+    return out;
+}
+/**
+ * The local names a module's export can be called by in this file: the export's own
+ * name, plus any `import X as Y` / `{ X as Y }` alias. A retrofit that imported the
+ * helper under an alias would otherwise look uncalled and fail open.
+ */
+function localBindings(content, exported) {
+    const esc = escapeRegExp(exported);
+    const names = new Set([exported]);
+    // Python `from M import x as y`; TS `import { x as y }` / `{ x: y } = require(...)`.
+    for (const re of [
+        new RegExp(`\\b${esc}\\s+as\\s+([A-Za-z_$][\\w$]*)`, "g"),
+        new RegExp(`\\b${esc}\\s*:\\s*([A-Za-z_$][\\w$]*)`, "g"),
+    ])
+        for (const m of content.matchAll(re))
+            names.add(m[1]);
+    return [...names];
+}
+function sameSet(a, b) {
+    if (a.size !== b.size)
+        return false;
+    for (const v of a)
+        if (!b.has(v))
+            return false;
+    return true;
+}
+/** The file's content at HEAD, or undefined when there is nothing to compare against
+ *  (untracked, no git root, or git unavailable). Never throws. */
+async function committedBaseline(file) {
+    const root = detectGitRoot(file);
+    if (!root)
+        return undefined;
+    const rel = path.relative(root, file);
+    if (!rel || rel.startsWith(".."))
+        return undefined;
+    try {
+        const { stdout } = await exec("git", ["show", `HEAD:${rel}`], {
+            cwd: root,
+            maxBuffer: 10 * 1024 * 1024,
+        });
+        return stdout;
+    }
+    catch (err) {
+        // A file added by this run has no HEAD version — the common case, not an error.
+        logger.debug("No committed baseline for a retrofit; skipping equivalence", {
+            file,
+            error: String(err),
+        });
+        return undefined;
+    }
+}

package/build/utils/utils-verify/stage.d.ts

@@ -16,6 +16,12 @@
 export interface StagingResult {
     /** Files that reached the index. */
     staged: string[];
+    /** Utils files located for the test (absolute), staged or not. */
+    utilsFiles: string[];
+    /** Pre-existing generated siblings this run edited to import from those files
+     *  (absolute) — the caller records them as retrofits so the report must disclose
+     *  them. */
+    retrofits: string[];
     /** Utils files that `git add` refused (gitignored, outside the repo, git failure) —
      *  the tests importing them will fail in the output commit, so the verify pass
      *  treats a non-empty list as a failed verification. */

package/build/utils/utils-verify/stage.js

@@ -12,7 +12,9 @@ const execFileAsync = promisify(execFile);
 export async function stageUtilsArtifacts(testFile, language) {
     const staged = [];
     const failed = [];
-    const result = { staged, failed };
+    const utilsFilesOut = [];
+    const retrofits = [];
+    const result = { staged, failed, utilsFiles: utilsFilesOut, retrofits };
     // Outside a testbot run stageGeneratedPaths is a no-op; report nothing as staged
     // rather than a list of files nothing touched.
     if (!isTestbotEnabled())
@@ -46,7 +48,9 @@ export async function stageUtilsArtifacts(testFile, language) {
     // file, so without this the sibling's edit drops out of the output commit while the
     // utils change lands — the same class of hole as the unstaged utils file.
     if (files.length > 0) {
+        utilsFilesOut.push(...files);
         for (const sibling of await modifiedSiblingsImporting(testFile, files, language)) {
+            retrofits.push(sibling);
             try {
                 await stageGeneratedPaths(sibling);
                 staged.push(sibling);
@@ -69,7 +73,13 @@ async function modifiedSiblingsImporting(testFile, utilsFiles, language) {
     let modified;
     try {
         // `-z`: unquoted, NUL-separated — git C-quotes non-ASCII paths otherwise.
-        const { stdout } = await execFileAsync("git", ["diff", "-z", "--name-only"], {
+        const { stdout } = await execFileAsync("git", 
+        // Against HEAD, modified only: a test THIS run created is staged at discovery, so
+        // its later STEP 5b edits show in a plain `git diff` and it would masquerade as a
+        // pre-existing sibling (observed: the report gate then demanded an execution no
+        // record could satisfy, and the agent un-shared the helpers to escape). Files
+        // added since HEAD are excluded; they are staged by their own tool calls.
+        ["diff", "-z", "--name-only", "--diff-filter=M", "HEAD"], {
             cwd: root,
             encoding: "utf8",
         });

package/build/utils/utils-verify/verify.d.ts

@@ -1,14 +1,23 @@
 import { type InlineCallSite } from "./call-sites.js";
-import { type UtilsViolationKind } from "./allow.js";
+import { type UtilsAdvisoryKind, type UtilsViolationKind } from "./allow.js";
+import { type SingleImporterHelper } from "./importers.js";
 import { type HelperFamilySpec, type UtilsLanguageSpec } from "./language-spec.js";
 export interface UtilsViolation {
     kind: UtilsViolationKind;
     /** Absolute path of the utils file. */
     file: string;
+    /** The helper the finding is about — or, for a spec-level finding (assertion-loss,
+     *  guard-removed), the spec's marker token: its basename with non-word characters
+     *  as `_` (`orders.spec.ts` → `orders_spec_ts`), what the allow grammar accepts. */
     helper: string;
     line: number;
     detail: string;
 }
+/** A violation-shaped finding whose kind may also be advisory-only. Only the
+ *  `advisories` list carries these; `violations` never does. */
+export type UtilsFinding = Omit<UtilsViolation, "kind"> & {
+    kind: UtilsViolationKind | UtilsAdvisoryKind;
+};
 export interface UtilsVerifyResult {
     /** No unallowed violations and no malformed allow markers. */
     ok: boolean;
@@ -24,18 +33,23 @@ export interface UtilsVerifyResult {
     /** Findings that inform but never fail the pass. `scenario-name` lives here: a naming
      *  heuristic is a quality signal, not a broken shared helper, and blocking a
      *  customer's run on a false positive of it is the worse outcome. */
-    advisories: UtilsViolation[];
+    advisories: UtilsFinding[];
     /** Allow-marker lines the grammar cannot read, per file. */
     malformedAllows: {
         file: string;
         line: string;
     }[];
     spec?: UtilsLanguageSpec;
-    /** Which invariant set applied — decided by the test type, see helperFamilyFor. */
-    family: HelperFamilySpec;
+    /** The test type's invariant set — the FALLBACK for helpers that wrap no request;
+     *  a helper that issues an SDK request is judged by the api rules regardless. See
+     *  helperFamilyOf. */
+    fallbackFamily: HelperFamilySpec;
     /** Advisory (does not affect `ok`): inline request calls in sibling generated tests
      *  that a utils helper already wraps — see findSiblingInlineCallSites. API family only. */
     inlineCallSites: InlineCallSite[];
+    /** Helpers imported by 0 or 1 delivered test (SKYR-4276 A6). Advisory: a first run
+     *  seeds every helper single-use by design; the count is the trend to watch. */
+    singleImporters: SingleImporterHelper[];
 }
 export interface VerifyUtilsParams {
     testFile: string;
@@ -43,6 +57,13 @@ export interface VerifyUtilsParams {
     /** Selects the helper family (api vs browser). Absent → api. */
     testType?: string;
     cwd?: string;
+    /** The spec at the modularization hand-out (SKYR-4276 A7/A8). When present, the
+     *  delivered spec plus its imported helpers must hold at least as many assertions,
+     *  and a page-error guard that was there must still be. */
+    baseline?: {
+        assertions: number;
+        pageErrorGuard: boolean;
+    };
 }
 /**
  * Deterministic post-reuse check of the shared utils file(s) a spec depends on.
@@ -61,3 +82,32 @@ export interface VerifyUtilsParams {
  * except to count what the spec imports.
  */
 export declare function verifyUtils(params: VerifyUtilsParams): Promise<UtilsVerifyResult>;
+/**
+ * Python only. The import form that works here is not the one that works in the
+ * customer's repo, and the difference is invisible in this run.
+ *
+ * pytest imports a test inside a package directory (one carrying `__init__.py`) from
+ * that package's first NON-package ancestor, so a bare `from SkyrampUtils import …`
+ * resolves only when the runner puts the test's own directory on `sys.path`. The
+ * Skyramp executor flattens its bundle and drops the `__init__.py` files, so it does;
+ * the customer running `pytest` from their repo root does not, and the test dies at
+ * collection with `ModuleNotFoundError`, taking the whole file with it. Measured on
+ * prefect (`tests/`, `tests/server/`, `tests/server/api/` all packages) and present in
+ * five of the six real Python repos in the eval corpus — the majority shape, not an
+ * edge case.
+ *
+ * The relative form is not simply the answer: under the flattened bundle it fails with
+ * `attempted relative import with no known parent package`. The two layouts want
+ * opposite forms, so a package directory must carry BOTH — the relative import first,
+ * falling back to the bare one:
+ *
+ *     try:
+ *         from .SkyrampUtils import create_order
+ *     except ImportError:  # flattened test bundle (no package context)
+ *         from SkyrampUtils import create_order
+ *
+ * Verified in both layouts. Outside a package the bare form alone is correct, and the
+ * fallback would be noise, so it is required only where `__init__.py` sits beside the
+ * spec — which is exactly where pytest changes its rule.
+ */
+export declare function importFormViolations(testFile: string, specContent: string, utilsFiles: string[]): UtilsViolation[];