agent-roast 0.22.0 → 0.24.0

package/README.md

@@ -112,7 +112,7 @@ Measured, with the denominators and the caveats in [docs/benchmarks.md](docs/ben
 deliberate, and not specific to AI. agent-roast is a tripwire that asks about them. On repositories it had not seen, between a quarter and
 two thirds of its blocks were questions with a harmless answer, depending on the sample.**
 
-- **162 of 162** hand-labeled cheating edits are caught and **150 of 150** harmless look-alikes (reformats, moves, raised
+- **165 of 165** hand-labeled cheating edits are caught and **154 of 154** harmless look-alikes (reformats, moves, raised
   thresholds, environment-gated skips) stay silent. The cases were written with the detector, so they guard against regressions;
   they do not prove accuracy. One gap is documented and still open.
 - **A catalog of cheats that someone else wrote.** Of the nine hacks in [reward-hack-zoo](https://github.com/MatthewWaller/reward-hack-zoo) (hand-written specimens; none from a model) four are reported (a skip, a patched target, a `conftest.py` that rewrites reports,
@@ -131,8 +131,8 @@ two thirds of its blocks were questions with a harmless answer, depending on the
   test that asserts nothing counts as a block for a reviewer; counted as false alarms, 13 of 33 on the fifth sample and 72 of 100 on the sixth. Most of the rest, in the
   earlier measurements too, were deliberate and explained. One reader (an AI assistant) sorted them, and the fixes were made from the same
   sorting, so treat the figures as a first estimate. The published 0.19.0 blocked 31 of the 12,000 commits of the seventh sample: 16 for a reviewer and 15 false alarms (48%, 6 of them one
-  repository's lint configuration); the build with the fixes from reading it, tried on an eighth sample that it had not seen, blocked 26 of 11,081 commits: 16 for a reviewer and 10 false alarms (38%), and the fixes from reading that one are in this release too; this release as it was frozen blocked 46 of 11,286 commits of a ninth sample that it had not seen: 28 for a reviewer and 18 false alarms (39%), and five of those are fixed in it; this release as it ships blocked 40 of the 11,191 commits of a tenth sample that it had not seen either: 27 for a reviewer and 13 false alarms (33%) ([section 9](docs/benchmarks.md#9-repositories-nobody-had-tuned-on),
-  [section 10](docs/benchmarks.md#10-two-more-samples-and-what-the-fixes-from-sample-4-did), [section 11](docs/benchmarks.md#11-a-seventh-sample-and-two-things-that-were-missing), [section 12](docs/benchmarks.md#12-an-eighth-sample-the-first-measurement-of-the-fixes-from-sample-7), [section 14](docs/benchmarks.md#14-a-ninth-sample-the-shipped-build-on-repositories-it-had-not-seen), [section 15](docs/benchmarks.md#15-a-tenth-sample-the-build-that-ships-on-repositories-it-had-not-seen)).
+  repository's lint configuration); the build with the fixes from reading it, tried on an eighth sample that it had not seen, blocked 26 of 11,081 commits: 16 for a reviewer and 10 false alarms (38%), and the fixes from reading that one are in this release too; 0.21.0 as it was frozen blocked 46 of 11,286 commits of a ninth sample that it had not seen: 28 for a reviewer and 18 false alarms (39%), and five of those are fixed in it; 0.21.0 as it shipped blocked 40 of the 11,191 commits of a tenth sample that it had not seen either: 27 for a reviewer and 13 false alarms (33%), and three of those 13 are fixed in this release ([section 9](docs/benchmarks.md#9-repositories-nobody-had-tuned-on),
+  [section 10](docs/benchmarks.md#10-two-more-samples-and-what-the-fixes-from-sample-4-did), [section 11](docs/benchmarks.md#11-a-seventh-sample-and-two-things-that-were-missing), [section 12](docs/benchmarks.md#12-an-eighth-sample-the-first-measurement-of-the-fixes-from-sample-7), [section 14](docs/benchmarks.md#14-a-ninth-sample-the-shipped-build-on-repositories-it-had-not-seen), [section 15](docs/benchmarks.md#15-a-tenth-sample-the-build-that-ships-on-repositories-it-had-not-seen), [section 16](docs/benchmarks.md#16-what-sample-10-led-to-three-fixes-fitted-and-a-guard-that-did-not-pay)).
 - **9,732 commits compared in full** from 164 public repositories that use coding agents (5,454 of them AI-signed) were reviewed again for the
   last two releases, and every new finding was read. 0.15.0's rules reported 30 new findings there and 7 were false alarms (fixed in 0.16.0);
   0.17.0's rule for an exact check turned into a bound reported 18 in 14 commits, all of them loosened checks, after two false alarms were

package/dist/{chunk-HDC5C2D3.js → chunk-U5MFYERX.js}

@@ -1508,6 +1508,7 @@ function isCodeFile(filePath, customIgnorePaths) {
 function isTestFile(filePath) {
   return TEST_PATH_REGEX.test(filePath);
 }
+var PYTHON_TEST_PATH = /(?:^|[/._-])(?:tests?|testing|specs?|conftest)(?:[/._-]|$)/i;
 function auditFileDiffs(files, commitHash = "HEAD", commitDate = (/* @__PURE__ */ new Date()).toISOString(), options = {}) {
   const infractions = [];
   let hasSupportedChanges = false;
@@ -1538,7 +1539,7 @@ function auditFileDiffs(files, commitHash = "HEAD", commitDate = (/* @__PURE__ *
       for (const start of masked.templateStarts) testCode[start] = "
";
       maskedLines.push({ code: cleanedLine, testCode: testCode.join(""), raw: rawLine, number: added.lineNumber });
       const isComment = !cleanedLine.trim();
-      if (python2 && !isComment) {
+      if (python2 && !isComment && PYTHON_TEST_PATH.test(file.path)) {
         for (const rule of PYTHON_TEST_SKIP_PATTERNS) {
           if (rule.regex.test(cleanedLine)) {
             if (rule.imperative && isGuardedSkip(maskedLines)) break;
@@ -1644,7 +1645,7 @@ function auditFileDiffs(files, commitHash = "HEAD", commitDate = (/* @__PURE__ *
     }
     if (python2) {
       const reportedSkips = new Set(infractions.slice(firstFinding).filter((f) => f.type === "test-skip").map((f) => f.lineNumber));
-      for (const match of source.matchAll(PYTHON_MARKER)) {
+      for (const match of PYTHON_TEST_PATH.test(file.path) ? source.matchAll(PYTHON_MARKER) : []) {
         const xfail = match[2] === "xfail";
         let open2 = match.index + match[0].length;
         while (source[open2] === " " || source[open2] === "	") open2++;
@@ -4407,11 +4408,13 @@ var THRESHOLD_PATTERNS = [
 var SWITCHES = ["strict", "noImplicitAny", "strictNullChecks", "noImplicitReturns", "useUnknownInCatchVariables"];
 var GATE_SCRIPT = /^(?:pre|post)?(?:test|lint|typecheck|type-check|check|ci|verify|validate)(?::[\w:.-]+)?$/;
 var CI_RUNNER = new RegExp(
-  String.raw`(?<![\w./-])(?:pytest|vitest|jest|mocha|tox|nox)(?![\w/-])` + String.raw`|(?<![\w./-])(?:npm|yarn|pnpm|bun|go|cargo|mvn|gradle|make|dotnet|flutter)\s+(?:run\s+)?(?:-\S+\s+)*test(?![\w/])` + // A package script that runs the tests with coverage: `yarn coverage`, `npm run coverage`.
-  String.raw`|(?<![\w./-])(?:npm|yarn|pnpm|bun)\s+(?:run\s+)?coverage(?![\w/-])` + // Monorepo runners: `nx affected --target=test`, `turbo run test`, `lerna run test`, `bazel test //...`.
+  String.raw`(?<![\w./-])(?:pytest|vitest|jest|mocha|tox|nox)(?![\w/-])` + String.raw`|(?<![\w./-])(?:npm|yarn|pnpm|bun|go|cargo|mvn|gradle|make|dotnet|flutter)\s+(?:run\s+)?(?:-\S+\s+)*test(?![\w/])` + // A package script that runs the tests with coverage: `yarn coverage`, `npm run coverage`, `npm run cover`.
+  String.raw`|(?<![\w./-])(?:npm|yarn|pnpm|bun)\s+(?:run\s+)?cover(?:age)?(?![\w/-])` + // Monorepo runners: `nx affected --target=test`, `turbo run test`, `lerna run test`, `bazel test //...`.
   String.raw`|(?<![\w./-])(?:nx|turbo|lerna|rush)\s+(?:run-many|affected|run|test)\b[^\n]*(?<![\w-])(?:test|tests)(?![\w-])` + String.raw`|(?<![\w./-])(?:bazel\s+test|ctest|phpunit|rspec|deno\s+test)(?![\w/-])` + // Python's own runners and the browser runners: `python3 -m unittest`, `nosetests`, `playwright test`, `cypress run`, `karma start`.
   String.raw`|(?<![\w./-])python[\d.]*\s+-m\s+(?:unittest|pytest|nose2?)(?![\w/-])` + String.raw`|(?<![\w./-])(?:nosetests|nose2|py\.test|stestr|behave|jasmine|playwright\s+test|cypress\s+run|karma\s+start|hatch\s+(?:run\s+)?test|swift\s+test)(?![\w/-])` + // Task runners: `deno task test`, `just test`, `rake test`, `composer test`, `mix test`, `sbt test`, `./gradlew test`, and Node's own `node --test`.
-  String.raw`|(?<![\w./-])(?:deno\s+(?:task|run)|just|task|poe|rake|composer(?:\s+run)?|mix|sbt|lein|stack|cabal|(?:\./)?(?:gradlew|mvnw)(?:\.bat|\.cmd)?)\s+(?:-\S+\s+)*test(?![\w/-])` + String.raw`|(?<![\w./-])(?:node|tsx)(?:\s+-\S+)*\s+--test(?![\w-])` + // A project's own test script, run through its interpreter or by path: `python scripts/run_tests.py`, `./scripts/test.sh`, `python tests/test_cli.py`.
+  String.raw`|(?<![\w./-])(?:deno\s+(?:task|run)|just|task|poe|rake|composer(?:\s+run)?|mix|sbt|lein|stack|cabal|(?:\./)?(?:gradlew|mvnw)(?:\.bat|\.cmd)?)\s+(?:-\S+\s+)*test(?![\w/-])` + String.raw`|(?<![\w./-])(?:node|tsx)(?:\s+-\S+)*\s+--test(?![\w-])` + // A target of a task runner that has the word as one of its own words and not as the first: `make unit-test`, `just test-asyncio py314`, `pnpm run docs:test`. A target that sets up, installs, builds or type-checks
+  // (`make setup-tests`, `pnpm run typecheck:tests`, `pnpm run build:native:test`) runs no tests.
+  String.raw`|(?<![\w./-])(?:make|just|task|(?:npm|pnpm|bun)\s+run|yarn(?:\s+run)?|pnpm)\s+(?:-\S+\s+)*(?![\w.:-]*(?:setup|teardown|install|deps|typecheck|lint|build|docker|clean|prepare|publish|deploy|release|format|image|download|data))[\w.:-]*(?<![A-Za-z0-9])tests?(?![A-Za-z0-9])` + // A project's own test script, run through its interpreter or by path: `python scripts/run_tests.py`, `./scripts/test.sh`, `python tests/test_cli.py`.
   String.raw`|(?<![\w./-])(?:(?:python[\d.]*|node|bash|sh|zsh|ruby|pwsh|tsx|ts-node)(?:\s+-\S+)*\s+|\./)(?:[\w.-]+/)*(?:(?:run[_-]?)?tests?(?:[_-](?:all|suite|runner|unit|ci))?|test_[\w-]+|[\w-]+_test)\.(?:py|sh|bash|js|mjs|cjs|ts|rb|ps1)(?![\w-])` + String.raw`|\brun:.*(?<=\s)(?:tests?|unit|e2e)(?![\w/.-])`
 );
 var SHELL_TEST = /(?:^|run:\s*|[;&|({]\s*|\b(?:if|then|do|else|elif|while|until)\s+|!\s+)test\s+!?\s*(?:-[bcdefghkLnOGprsStuwxzN](?=\s)|["$'])/g;
@@ -4531,12 +4534,42 @@ function ciTestSteps(file, oldText, newText) {
   const after = commandsAfter.length;
   return { file, before: commandsBefore.length, after, reusableAfter: matchingLines(now, CI_REUSABLE), ungatedAfter: after === 0 ? ciTestLines(now).length : 0, commandsBefore, commandsAfter };
 }
-function resolveCiTestSteps(counts, commit, elsewhere = /* @__PURE__ */ new Set()) {
+function gatingCalls(file, text) {
+  if (!(/\.gitlab-ci/.test(file) || CI_GATE.test(gatingTriggers(text) ?? text))) return [];
+  return [...text.matchAll(/^\s*-?\s*uses:\s*["']?\.\/(\.github\/workflows\/[^\s"'#@]+\.ya?ml)/gm)].map((match) => match[1]);
+}
+function lostCalls(file, oldText, newText) {
+  if (!CI_FILE.test(file)) return [];
+  const before = gatingCalls(file, oldText ?? "");
+  if (!before.length) return [];
+  const after = new Set(gatingCalls(file, newText ?? ""));
+  return before.filter((workflow) => !after.has(workflow));
+}
+function calledTestCommands(text) {
+  const directories = [...new Set([...text.matchAll(/^\s*(?:-\s+)?working-directory\s*:\s*["']?([^\s"'#]+)/gm)].map((match) => match[1]))].sort().join(",");
+  return ciTestEntries(text).map((entry) => `${entry.command.replace(/\s#.*$/, "").replace(/^\s*-?\s*(?:run\s*:\s*)?/, "").replace(/\s+/g, " ").trim()} @${directories}`);
+}
+function resolveLostCallers(lost, commit) {
+  const out = [];
+  for (const [file, { content, stillCalled }] of lost) {
+    if (stillCalled || gatingTestCommands(file, content).length > 0 || calledTestCommands(content).length === 0) continue;
+    out.push(finding(
+      "guard-weakened",
+      file,
+      1,
+      "nothing calls this workflow on a push or pull request any more",
+      "Workflow keeps its tests, but the workflow that called it is gone",
+      commit
+    ));
+  }
+  return out;
+}
+function resolveCiTestSteps(counts, commit, elsewhere = /* @__PURE__ */ new Set(), called = /* @__PURE__ */ new Set()) {
   const before = counts.reduce((sum, item) => sum + item.before, 0);
   const after = counts.reduce((sum, item) => sum + item.after + item.reusableAfter, 0);
   if (after >= before) return [];
   const running = /* @__PURE__ */ new Set([...counts.flatMap((item) => item.commandsAfter ?? []), ...elsewhere]);
-  return counts.filter((item) => item.before > 0 && item.after === 0 && item.reusableAfter === 0).filter((item) => item.commandsBefore === void 0 || item.commandsBefore.some((command) => !running.has(command))).map((item) => item.ungatedAfter > 0 ? finding("guard-weakened", item.file, 1, "tests no longer run on push or pull_request", "Workflow keeps its tests but no longer runs on pushes or pull requests", commit) : finding("guard-weakened", item.file, 1, "test command no longer runs in CI", "Test step removed from CI", commit));
+  return counts.filter((item) => item.before > 0 && item.after === 0 && item.reusableAfter === 0).filter((item) => !(item.ungatedAfter > 0 && called.has(item.file))).filter((item) => item.commandsBefore === void 0 || item.commandsBefore.some((command) => !running.has(command))).map((item) => item.ungatedAfter > 0 ? finding("guard-weakened", item.file, 1, "tests no longer run on push or pull_request", "Workflow keeps its tests but no longer runs on pushes or pull requests", commit) : finding("guard-weakened", item.file, 1, "test command no longer runs in CI", "Test step removed from CI", commit));
 }
 var LENIENT = /continue-on-error:\s*true|allow_failure:\s*true|^\s*(?:-\s+)?if:\s*(?:\$\{\{\s*)?false\s*(?:\}\})?\s*(?:#.*)?$/;
 var SHELL_SWALLOW = /\|\|\s*(?:true|exit\s+0|:|echo\b)|;\s*(?:true|exit\s+0)\b/;
@@ -4792,6 +4825,9 @@ async function compareSnapshot(options, cwd, snapshot) {
   const tamperingFiles = [];
   const guardFindings = [];
   const ciCounts = [];
+  const calledInChange = /* @__PURE__ */ new Set();
+  const lostCalledWorkflows = /* @__PURE__ */ new Set();
+  const touchedCiContent = /* @__PURE__ */ new Map();
   let productionChanged = false;
   const decode = (content) => {
     totalBytes += content.length;
@@ -4829,7 +4865,12 @@ async function compareSnapshot(options, cwd, snapshot) {
         const newText = await readNew(change);
         guardFindings.push(...analyzeGuardFile(change.newPath, oldText, newText, refs.headSha));
         const ci = ciTestSteps(change.newPath, oldText, newText);
-        if (ci) ciCounts.push(ci);
+        if (ci) {
+          ciCounts.push(ci);
+          touchedCiContent.set(change.newPath, newText);
+          for (const workflow of lostCalls(change.newPath, oldText, newText)) lostCalledWorkflows.add(workflow);
+          if (newText !== null) for (const workflow of gatingCalls(change.newPath, newText)) calledInChange.add(workflow);
+        }
         result.filesInspected++;
       } catch (error) {
         assertNotAborted(options.signal);
@@ -4887,14 +4928,18 @@ async function compareSnapshot(options, cwd, snapshot) {
   await snapshot?.verify();
   result.complete = result.skippedFiles.length === 0;
   const rules = policy.config.rules ?? {};
-  const elsewhere = /* @__PURE__ */ new Set();
-  if (ciCounts.some((item) => item.before > 0 && item.after === 0 && item.reusableAfter === 0)) {
+  const elsewhere = /* @__PURE__ */ new Set(), called = new Set(calledInChange);
+  const untouchedLostCallerContent = /* @__PURE__ */ new Map();
+  if (ciCounts.some((item) => item.before > 0 && item.after === 0 && item.reusableAfter === 0) || lostCalledWorkflows.size > 0) {
     const touched = new Set(changes.flatMap((change) => [change.oldPath, change.newPath]));
     const tree = /^[a-f0-9]{40,64}$/.test(refs.headSha) ? refs.headSha : refs.baseSha;
     try {
       for (const file of (await ciFilesAt(cwd, tree, options.signal)).filter((candidate) => !touched.has(candidate.path)).slice(0, 100)) {
         try {
-          for (const command of enforcedTestCommands(file.path, new TextDecoder().decode(await readBlob(cwd, file.oid, options.signal, 512 * 1024)))) elsewhere.add(command);
+          const text = new TextDecoder().decode(await readBlob(cwd, file.oid, options.signal, 512 * 1024));
+          for (const command of enforcedTestCommands(file.path, text)) elsewhere.add(command);
+          for (const workflow of gatingCalls(file.path, text)) called.add(workflow);
+          if (lostCalledWorkflows.has(file.path)) untouchedLostCallerContent.set(file.path, text);
         } catch (error) {
           assertNotAborted(options.signal);
         }
@@ -4903,7 +4948,16 @@ async function compareSnapshot(options, cwd, snapshot) {
       assertNotAborted(options.signal);
     }
   }
-  result.infractions.push(...[...resolveTampering(tamperingFiles, productionChanged), ...guardFindings, ...resolveCiTestSteps(ciCounts, refs.headSha, elsewhere)].filter((finding2) => rules[finding2.type] !== "off"));
+  const ciFindings = resolveCiTestSteps(ciCounts, refs.headSha, elsewhere, called);
+  const alreadyFlagged = new Set(ciFindings.map((found) => found.file));
+  const lostCallers = /* @__PURE__ */ new Map();
+  for (const workflow of lostCalledWorkflows) {
+    if (alreadyFlagged.has(workflow)) continue;
+    const content = touchedCiContent.has(workflow) ? touchedCiContent.get(workflow) ?? null : untouchedLostCallerContent.get(workflow) ?? null;
+    if (content === null) continue;
+    lostCallers.set(workflow, { content, stillCalled: called.has(workflow) });
+  }
+  result.infractions.push(...[...resolveTampering(tamperingFiles, productionChanged), ...guardFindings, ...ciFindings, ...resolveLostCallers(lostCallers, refs.headSha)].filter((finding2) => rules[finding2.type] !== "off"));
   const exceptions = applyExceptions(result.infractions, options.ignoreExceptions ? [] : policy.config.exceptions, /* @__PURE__ */ new Date(), existingCounts);
   result.infractions = exceptions.findings;
   result.suppressedFindings = exceptions.suppressedFindings;

package/dist/cli.js

@@ -38,7 +38,7 @@ import {
   terminalText,
   uninstallHooks,
   writeBaseline
-} from "./chunk-HDC5C2D3.js";
+} from "./chunk-U5MFYERX.js";
 
 // src/cli.ts
 import { Command } from "commander";

package/dist/index.js

@@ -92,7 +92,7 @@ import {
   unquoteGitPath,
   validateExceptions,
   writeBaseline
-} from "./chunk-HDC5C2D3.js";
+} from "./chunk-U5MFYERX.js";
 export {
   AGENT_LABELS,
   CONFIG_FILENAMES,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-roast",
-  "version": "0.22.0",
+  "version": "0.24.0",
   "description": "Stops AI agents (and rushed humans) weakening tests, coverage gates or CI to get a green build. Hook for Claude Code, Codex, Cursor, Gemini CLI, Copilot, Droid; pre-commit; GitHub Action. Offline.",
   "mcpName": "io.github.Nari-Ab/agent-roast",
   "type": "module",