@quikit/quikpilot-runner 8.2.5 → 8.2.6

package/README.md

@@ -1,27 +1,27 @@
-# @quikit/quikpilot-runner
-
-QuikPilot's CI runner. Invoked automatically by the `.github/workflows/quikpilot.yml` file QuikPilot
-installs in an onboarded repository — it fetches the run's work packet, runs your setup/lint/test
-commands, reports results to QuikTest, enforces the denied-path guardrail, commits, pushes, and
-reports back to QuikTrack.
-
-This package is not meant to be installed or run by hand. It exists so a fix to QuikPilot's runner
-logic ships as a new package version rather than a new pull request into every onboarded repository
-— the workflow file itself lives in your repository (QuikIT cannot update it directly), so it stays a
-thin, ~25-line launcher that invokes an exact, pinned version of this package via `npx`.
-
-It carries no secrets and needs none of your credentials: it authenticates to QuikTrack and to GitHub
-using short-lived tokens exchanged at runtime, never a value baked into the workflow file or this
-package.
-
-## Versioning and support
-
-Every workflow file pins an **exact** version of this package (`npx --yes
-@quikit/quikpilot-runner@<version>`, never a range or `latest`) — a repository never picks up a new
-release on its own. Older published versions are not removed from npm and keep working as long as a
-repository's workflow file keeps pinning them; upgrading to a newer version only happens through a new
-pull request into that repository (QuikTrack's "re-bootstrap" flow), never silently.
-
-Releases follow [SemVer](https://semver.org/): a major version bump means the workflow file's own
-shape changed (inputs, permissions) and requires a new pull request regardless of this package's
-version; minor/patch releases are runner-logic fixes safe to pick up on the next re-bootstrap.
+# @quikit/quikpilot-runner
+
+QuikPilot's CI runner. Invoked automatically by the `.github/workflows/quikpilot.yml` file QuikPilot
+installs in an onboarded repository — it fetches the run's work packet, runs your setup/lint/test
+commands, reports results to QuikTest, enforces the denied-path guardrail, commits, pushes, and
+reports back to QuikTrack.
+
+This package is not meant to be installed or run by hand. It exists so a fix to QuikPilot's runner
+logic ships as a new package version rather than a new pull request into every onboarded repository
+— the workflow file itself lives in your repository (QuikIT cannot update it directly), so it stays a
+thin, ~25-line launcher that invokes an exact, pinned version of this package via `npx`.
+
+It carries no secrets and needs none of your credentials: it authenticates to QuikTrack and to GitHub
+using short-lived tokens exchanged at runtime, never a value baked into the workflow file or this
+package.
+
+## Versioning and support
+
+Every workflow file pins an **exact** version of this package (`npx --yes
+@quikit/quikpilot-runner@<version>`, never a range or `latest`) — a repository never picks up a new
+release on its own. Older published versions are not removed from npm and keep working as long as a
+repository's workflow file keeps pinning them; upgrading to a newer version only happens through a new
+pull request into that repository (QuikTrack's "re-bootstrap" flow), never silently.
+
+Releases follow [SemVer](https://semver.org/): a major version bump means the workflow file's own
+shape changed (inputs, permissions) and requires a new pull request regardless of this package's
+version; minor/patch releases are runner-logic fixes safe to pick up on the next re-bootstrap.

package/dist/index.js

@@ -73,11 +73,12 @@ async function describeQuikTrackFailure(res) {
 
 // src/callback.ts
 async function reportBack(params) {
-  const { callbackUrl, runToken, status, branch, warnings, targeting } = params;
+  const { callbackUrl, runToken, status, branch, warnings, targeting, baselineFindings } = params;
   const body = { status };
   if (branch) body.branch = branch;
   if (warnings && warnings.length > 0) body.warnings = warnings;
   if (targeting) body.targeting = targeting;
+  if (baselineFindings) body.baselineFindings = baselineFindings;
   const res = await fetch(callbackUrl, {
     method: "POST",
     headers: { Authorization: `Bearer ${runToken}`, "Content-Type": "application/json" },
@@ -109,10 +110,36 @@ function runTestCommand(command) {
 }
 function runTestCommandAsync(command) {
   if (!command) return Promise.resolve("success");
-  return new Promise((resolve3) => {
+  return new Promise((resolve4) => {
     const child = (0, import_node_child_process.spawn)(command, { shell: "/bin/bash", stdio: "inherit" });
-    child.on("exit", (code) => resolve3(code === 0 ? "success" : "failure"));
-    child.on("error", () => resolve3("failure"));
+    child.on("exit", (code) => resolve4(code === 0 ? "success" : "failure"));
+    child.on("error", () => resolve4("failure"));
+  });
+}
+function runCommandWithTimeout(command, timeoutMs) {
+  return new Promise((resolve4) => {
+    let settled = false;
+    const finish = (outcome) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      resolve4(outcome);
+    };
+    const child = (0, import_node_child_process.spawn)(command, { shell: "/bin/bash", stdio: "inherit", detached: true });
+    const timer = setTimeout(() => {
+      try {
+        if (child.pid !== void 0) process.kill(-child.pid, "SIGKILL");
+        else child.kill("SIGKILL");
+      } catch {
+        try {
+          child.kill("SIGKILL");
+        } catch {
+        }
+      }
+      finish("timeout");
+    }, timeoutMs);
+    child.on("exit", () => finish("exited"));
+    child.on("error", () => finish("error"));
   });
 }
 
@@ -959,8 +986,8 @@ async function reportToQuikTest(params) {
   try {
     console.log(`reporting ${report}`);
     const ciUrl = buildCiUrl(githubServerUrl, githubRepository, githubRunId);
-    const { readFileSync: readFileSync5 } = await import("node:fs");
-    const body = readFileSync5(report);
+    const { readFileSync: readFileSync6 } = await import("node:fs");
+    const body = readFileSync6(report);
     let url = `${resultsUrl}?ciUrl=${encodeURIComponent(ciUrl)}&automationIdFormat=bracket`;
     if (generatedFiles) {
       url += `&generatedFiles=${encodeURIComponent(generatedFiles.join(","))}`;
@@ -987,12 +1014,18 @@ async function reportToQuikTest(params) {
   }
 }
 
+// src/baselineCheck.ts
+var import_node_child_process3 = require("node:child_process");
+var import_node_fs6 = require("node:fs");
+var import_node_path4 = require("node:path");
+
 // src/regressionCheck.ts
 var import_node_fs5 = require("node:fs");
 var TESTCASE_RE = /<testcase\b([^>]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/g;
 var NAME_ATTR_RE = /\bname="([^"]*)"/;
 var CLASSNAME_ATTR_RE = /\bclassname="([^"]*)"/;
 var FAILURE_CHILD_RE = /<failure\b|<error\b/;
+var SKIPPED_CHILD_RE = /<skipped\b/;
 var TESTSUITE_OPEN_RE = /<testsuite\b([^>]*?)>/g;
 var FILE_ATTR_RE = /\bfile="([^"]*)"/;
 var QP_TOKEN_BRACKET_RE = /\[(QP-[A-Z][A-Z0-9]*-\d+-\d+)\]/;
@@ -1014,8 +1047,10 @@ function extractFailureText(testcaseBody) {
   const text = unescapeXmlEntities(raw);
   return text.length > MAX_FAILURE_TEXT ? `${text.slice(0, MAX_FAILURE_TEXT)}\u2026` : text;
 }
+var RESULT_RANK = { pass: 0, skipped: 1, fail: 2 };
 function parseJUnitOutcomes(xml) {
   const outcomes = /* @__PURE__ */ new Map();
+  const automationIdResults = /* @__PURE__ */ new Map();
   const failureText = /* @__PURE__ */ new Map();
   const discoveredFiles = /* @__PURE__ */ new Set();
   const discoveredAutomationIds = /* @__PURE__ */ new Set();
@@ -1043,9 +1078,14 @@ function parseJUnitOutcomes(xml) {
     if (file) discoveredFiles.add(normalizeSlashes(file));
     if (classname) discoveredFiles.add(normalizeSlashes(classname));
     const bracket = QP_TOKEN_BRACKET_RE.exec(name)?.[1];
-    if (bracket) discoveredAutomationIds.add(bracket);
+    if (bracket) {
+      discoveredAutomationIds.add(bracket);
+      const result = failed ? "fail" : SKIPPED_CHILD_RE.test(body) ? "skipped" : "pass";
+      const seen = automationIdResults.get(bracket);
+      if (!seen || RESULT_RANK[result] > RESULT_RANK[seen]) automationIdResults.set(bracket, result);
+    }
   }
-  return { outcomes, failureText, discoveredFiles, discoveredAutomationIds };
+  return { outcomes, automationIdResults, failureText, discoveredFiles, discoveredAutomationIds };
 }
 function readJUnitOutcomes(cwd) {
   const report = findJUnitReport(cwd);
@@ -1056,6 +1096,9 @@ function readJUnitOutcomes(cwd) {
     return null;
   }
 }
+function readAutomationIdResults(cwd = process.cwd()) {
+  return readJUnitOutcomes(cwd)?.automationIdResults ?? null;
+}
 function captureTestSnapshot(command, cwd = process.cwd()) {
   const outcome = runTestCommand(command);
   const parsed = readJUnitOutcomes(cwd);
@@ -1126,8 +1169,186 @@ function describeRegressions(check) {
   return parts.join("; ");
 }
 
+// src/baselineCheck.ts
+var BASELINE_CHECK_TIMEOUT_MS = 12e4;
+function shellQuote(value) {
+  return `'${value.split("'").join("'\\''")}'`;
+}
+function buildSingleFileCommand(command, path) {
+  const quoted = shellQuote(path);
+  return command.includes("{file}") ? command.split("{file}").join(quoted) : `${command} ${quoted}`;
+}
+var isTestTarget2 = (t) => (t.automationIds?.length ?? 0) > 0;
+function checkableIds(ids, types) {
+  if (!types) return [];
+  return ids.filter((id) => {
+    if (!Object.prototype.hasOwnProperty.call(types, id)) return false;
+    const type = types[id];
+    return typeof type === "string" && type.trim() !== "" && type.trim().toUpperCase() !== "REGRESSION";
+  });
+}
+function insideCheckout(cwd, path) {
+  const abs = (0, import_node_path4.resolve)(cwd, path);
+  const rel = (0, import_node_path4.relative)(cwd, abs);
+  return rel === "" || rel.startsWith("..") || (0, import_node_path4.isAbsolute)(rel) ? null : abs;
+}
+function writeForCheck(abs, content) {
+  const original = (0, import_node_fs6.existsSync)(abs) ? (0, import_node_fs6.readFileSync)(abs) : null;
+  const createdDirs = [];
+  if (original === null) {
+    const missing = [];
+    for (let d = (0, import_node_path4.dirname)(abs); !(0, import_node_fs6.existsSync)(d); d = (0, import_node_path4.dirname)(d)) missing.push(d);
+    for (const d of missing.reverse()) {
+      (0, import_node_fs6.mkdirSync)(d);
+      createdDirs.push(d);
+    }
+  }
+  (0, import_node_fs6.writeFileSync)(abs, content);
+  return { abs, original, createdDirs };
+}
+function restore(written) {
+  for (const file of [...written].reverse()) {
+    try {
+      if (file.original !== null) (0, import_node_fs6.writeFileSync)(file.abs, file.original);
+      else (0, import_node_fs6.rmSync)(file.abs, { force: true });
+      for (const d of [...file.createdDirs].reverse()) {
+        try {
+          (0, import_node_fs6.rmdirSync)(d);
+        } catch {
+        }
+      }
+    } catch (err) {
+      console.error(`Baseline check could not restore ${file.abs}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+}
+function untrackedFiles(cwd) {
+  try {
+    const out = (0, import_node_child_process3.execFileSync)("git", ["ls-files", "--others", "--exclude-standard", "-z"], {
+      cwd,
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"],
+      maxBuffer: 64 * 1024 * 1024
+    });
+    return new Set(out.split("\0").filter(Boolean));
+  } catch {
+    return null;
+  }
+}
+var mtimeOf = (path) => {
+  try {
+    return (0, import_node_fs6.statSync)(path).mtimeMs;
+  } catch {
+    return null;
+  }
+};
+function decideCause(targets, results, cwd) {
+  const codeTargets = targets.filter((t) => !isTestTarget2(t));
+  if (codeTargets.length === 0) return "ALREADY_TRUE";
+  for (const t of codeTargets) {
+    if (t.kind === "NEW_FILE") return "ALREADY_TRUE";
+    const next = results.get(t.path);
+    const abs = insideCheckout(cwd, t.path);
+    if (next === void 0 || abs === null) return "ALREADY_TRUE";
+    try {
+      if ((0, import_node_fs6.readFileSync)(abs, "utf8") !== next) return "ALREADY_TRUE";
+    } catch {
+      return "ALREADY_TRUE";
+    }
+  }
+  return "NO_OP";
+}
+async function runBaselineCheck(params) {
+  const { targets, results, testFileCmd, requiredTestTypes } = params;
+  const cwd = params.cwd ?? process.cwd();
+  const runFile = params.runFile ?? runCommandWithTimeout;
+  const nothing = { findings: null, warnings: [] };
+  if (!testFileCmd || testFileCmd.trim() === "") return nothing;
+  const testTargets = targets.filter(isTestTarget2);
+  const toCheck = testTargets.map((target) => ({ target, ids: checkableIds(target.automationIds ?? [], requiredTestTypes) })).filter((entry) => entry.ids.length > 0);
+  if (toCheck.length === 0) {
+    console.log("Baseline check: no case with a known, non-Regression label \u2014 nothing to check.");
+    return nothing;
+  }
+  const warnings = [];
+  const flagged = [];
+  const written = [];
+  const unsafe = /* @__PURE__ */ new Set();
+  const skip = (path, why) => {
+    const message = `Baseline check skipped for ${path}: ${why}`;
+    console.warn(`::warning::${message}`);
+    warnings.push(message);
+  };
+  try {
+    const contentByPath = new Map(results.map((r) => [r.targetPath, r.newContent]));
+    const cause = decideCause(targets, contentByPath, cwd);
+    for (const target of testTargets) {
+      const content = contentByPath.get(target.path);
+      if (content === void 0) continue;
+      const abs = insideCheckout(cwd, target.path);
+      if (abs === null) {
+        unsafe.add(target.path);
+        continue;
+      }
+      written.push(writeForCheck(abs, content));
+    }
+    for (const { target, ids } of toCheck) {
+      if (unsafe.has(target.path)) {
+        skip(target.path, "the path is outside the checkout, so the file was not written or run.");
+        continue;
+      }
+      if (!contentByPath.has(target.path)) continue;
+      const reportBefore = findJUnitReport(cwd);
+      const mtimeBefore = reportBefore ? mtimeOf(reportBefore) : null;
+      const untrackedBefore = untrackedFiles(cwd);
+      const outcome = await runFile(buildSingleFileCommand(testFileCmd, target.path), BASELINE_CHECK_TIMEOUT_MS);
+      if (outcome === "timeout") {
+        skip(
+          target.path,
+          `the single-file command did not finish within ${BASELINE_CHECK_TIMEOUT_MS / 1e3} seconds and was stopped. No test case from this file was flagged.`
+        );
+      } else if (outcome === "error") {
+        skip(target.path, "the single-file command could not be started. No test case from this file was flagged.");
+      } else {
+        const reportAfter = findJUnitReport(cwd);
+        const fresh = reportAfter !== null && (reportAfter !== reportBefore || mtimeOf(reportAfter) !== mtimeBefore);
+        if (!fresh) {
+          skip(
+            target.path,
+            "the single-file command wrote no new JUnit report (it must write the same JUnit report as the test command), so nothing could be checked."
+          );
+        } else {
+          const byId = readAutomationIdResults(cwd);
+          for (const id of ids) if (byId?.get(id) === "pass") flagged.push(id);
+        }
+      }
+      const untrackedAfter = untrackedFiles(cwd);
+      if (untrackedBefore && untrackedAfter) {
+        for (const file of untrackedAfter) {
+          if (untrackedBefore.has(file)) continue;
+          const abs = insideCheckout(cwd, file);
+          if (abs) (0, import_node_fs6.rmSync)(abs, { force: true });
+        }
+      }
+    }
+    if (flagged.length === 0) return { findings: null, warnings };
+    const automationIds = [...new Set(flagged)].slice(0, 200);
+    const list = automationIds.join(", ");
+    warnings.push(
+      cause === "NO_OP" ? `Baseline check: ${automationIds.length} test case(s) passed against the unmodified code, and every file the generated change edited came out unchanged, so the change itself may have been a no-op: ${list}.` : `Baseline check: ${automationIds.length} test case(s) passed against the unmodified code, so they may assert something already true: ${list}. This check cannot tell that from a change that did nothing.`
+    );
+    return { findings: { cause, automationIds }, warnings };
+  } catch (err) {
+    const message = `Baseline check could not run: ${err instanceof Error ? err.message : String(err)}`;
+    console.warn(`::warning::${message}`);
+    return { findings: null, warnings: [...warnings, message] };
+  } finally {
+    restore(written);
+  }
+}
+
 // src/scopeCheck.ts
-var import_node_fs6 = require("node:fs");
+var import_node_fs7 = require("node:fs");
 var ROUTE_METHOD_RE = /\b(GET|POST|PUT|PATCH|DELETE)\b/i;
 var ROUTE_PATH_RE = /\/[A-Za-z0-9_\-{}[\]]+(?:\/[A-Za-z0-9_\-:{}[\]]+)+/;
 var ROUTE_EXPORT_RE = /\bexport\s+(?:async\s+)?function\s+(GET|POST|PUT|PATCH|DELETE)\s*\(/;
@@ -1184,7 +1405,7 @@ function checkApiScope(results, targets, apiChanges) {
     } else {
       let currentContent;
       try {
-        currentContent = (0, import_node_fs6.readFileSync)(targetPath, "utf8");
+        currentContent = (0, import_node_fs7.readFileSync)(targetPath, "utf8");
       } catch {
         currentContent = "";
       }
@@ -1203,7 +1424,7 @@ function describeScopeViolations(check) {
 }
 
 // src/targeting.ts
-var import_node_path4 = require("node:path");
+var import_node_path5 = require("node:path");
 var CONTRACT_VERSION2 = "test-targets-1";
 function listWorkspaceFiles(workspaceRoot) {
   return listFiles(workspaceRoot ?? ".");
@@ -1232,9 +1453,9 @@ function validateTarget(target, workspaceRoot, workspaceListing, deniedPaths) {
     }
     return;
   }
-  const root = (0, import_node_path4.resolve)(workspaceRoot ?? ".");
-  const rel = (0, import_node_path4.relative)(root, (0, import_node_path4.resolve)(target.path));
-  if (rel.startsWith("..") || (0, import_node_path4.isAbsolute)(rel)) {
+  const root = (0, import_node_path5.resolve)(workspaceRoot ?? ".");
+  const rel = (0, import_node_path5.relative)(root, (0, import_node_path5.resolve)(target.path));
+  if (rel.startsWith("..") || (0, import_node_path5.isAbsolute)(rel)) {
     throw new Error(
       `pilot.targeting named a NEW_FILE path that escapes the workspace root: ${target.path}`
     );
@@ -1455,10 +1676,11 @@ async function run(env2) {
   let packet;
   let branch;
   let runToken;
-  let warnings = [];
+  const warnings = [];
   let targeted = false;
   let generatedFilePaths = [];
   let targetingResult;
+  let baselineFindings;
   const contextLog = [];
   try {
     runToken = await exchangeForRunToken(env2.tokenUrl, await fetchGithubOidcToken());
@@ -1493,8 +1715,9 @@ async function run(env2) {
       const blockingOverlaps = overlaps.filter((o) => o.severity === "identical");
       const softOverlaps = overlaps.filter((o) => o.severity === "similar");
       if (softOverlaps.length > 0) {
-        warnings = describeOverlapWarnings(softOverlaps);
-        for (const w of warnings) {
+        const overlapWarnings = describeOverlapWarnings(softOverlaps);
+        warnings.push(...overlapWarnings);
+        for (const w of overlapWarnings) {
           console.warn(`QuikPilot targeting warning (not blocking): ${w}`);
         }
       }
@@ -1544,6 +1767,20 @@ async function run(env2) {
           `QuikPilot's generated tests are not where the test runner will find them \u2014 ${describeTestPlacementViolations(placementCheck)}`
         );
       }
+      try {
+        const baselineCheck = await runBaselineCheck({
+          targets: targeting.targets,
+          results,
+          testFileCmd: packet.commands.testFile,
+          requiredTestTypes: packet.requiredTestTypes
+        });
+        warnings.push(...baselineCheck.warnings);
+        baselineFindings = baselineCheck.findings ?? void 0;
+      } catch (err) {
+        const message = `Baseline check could not run: ${err instanceof Error ? err.message : String(err)}`;
+        console.error(message);
+        warnings.push(message);
+      }
       generatedFilePaths = results.map((r) => r.targetPath);
       flushCodegenResults(results);
       try {
@@ -1615,7 +1852,8 @@ async function run(env2) {
     branch,
     warnings,
     // Omitted when targeting itself never produced a result (nothing exists to record).
-    targeting: targetingResult ? buildTargetingRecord(targetingResult, contextLog) : void 0
+    targeting: targetingResult ? buildTargetingRecord(targetingResult, contextLog) : void 0,
+    baselineFindings
   });
   return { hardFailure };
 }