agent-roast 0.16.0 → 0.17.0

package/README.md

@@ -78,7 +78,7 @@ Policy is read from the **base** branch, so a pull request cannot switch off the
 
 | Rule | Fires when | By default |
 | --- | --- | :-: |
-| `assertion-weakened` | an exact check is replaced by a weaker one: `toBe(42)` → `toBeDefined()`, `assert x == 1` → `assert x` | blocks |
+| `assertion-weakened` | an exact check is replaced by a weaker one: `toBe(42)` → `toBeDefined()`, `toBe(2)` → `toBeLessThanOrEqual(2)`, `toEqual(x)` → `toMatchObject(x)`, `assert x == 1` → `assert x` | blocks |
 | `guard-weakened` | a coverage threshold is lowered, `strict` is switched off, a lint rule is downgraded, a CI step may fail, a workflow stops running tests, a `conftest.py` hook sets a report to `passed` | blocks |
 | `test-skip` | a test is switched off or expected to fail: `it.skip`, `@pytest.mark.skip`, `@pytest.mark.xfail`, `it.fails`, `test.fixme`, `t.Skip`, `#[ignore]`, `@Disabled`, `[Ignore]`, `xit`, ... | blocks |
 | `test-focus`, `trivial-assert` | `.only` / `fit`; `expect(true).toBe(true)` and its relatives in ten languages; a test that asserts on a function it mocked itself | blocks |
@@ -111,7 +111,7 @@ old debt with a baseline: [docs/reference.md](docs/reference.md#configuration).
 Measured, with the denominators and the caveats in [docs/benchmarks.md](docs/benchmarks.md). **Weakened checks are rare, mostly
 deliberate, and not specific to AI. agent-roast is a tripwire that stays silent until a check gets weaker.**
 
-- **119 of 119** hand-labeled cheating edits are caught and **84 of 84** harmless look-alikes (reformats, moves, raised
+- **126 of 126** hand-labeled cheating edits are caught and **90 of 90** harmless look-alikes (reformats, moves, raised
   thresholds, environment-gated skips) stay silent. The cases were written with the detector, so they guard against regressions;
   they do not prove accuracy. One gap is documented and still open.
 - **12,645 commits** that touch tests or CI/config in 166 public repositories: blocking rules fired on 0.3% of AI-signed commits
@@ -120,6 +120,11 @@ deliberate, and not specific to AI. agent-roast is a tripwire that stays silent
   checks more often.
 - **About one finding in five was a false alarm**, and most of the rest were deliberate, explained changes a reviewer should
   see rather than mistakes. The labels come from one reader (an AI assistant), so treat them as a first estimate.
+- **9,732 commits compared in full** from 164 public repositories that use coding agents (5,454 of them AI-signed) were reviewed again for the
+  last two releases, and every new finding was read. 0.15.0's rules reported 30 new findings there and 7 were false alarms (fixed in 0.16.0);
+  0.17.0's rule for an exact check turned into a bound reported 18 in 14 commits, all of them loosened checks, after two false alarms were
+  fixed. One reader, and a convenience sample ([section 7](docs/benchmarks.md#7-ai-signed-commits-and-a-test-that-checks-its-own-mock),
+  [section 8](docs/benchmarks.md#8-an-exact-check-turned-into-a-bound)).
 - `replay` ran on 179 public repositories that use coding agents: no commit would have been blocked in 149 of them.
 - The rules for **eight more languages** (Go, Rust, Java, Kotlin, C#, Ruby, PHP, Swift) ran on the newest 300 commits of 39
   repositories and every finding was read: 26 of 99 were skips that depend on the environment, now fixed; of the 73 left, 56 are

package/dist/{chunk-52FUQ4EP.js → chunk-IKXGHCWB.js}

@@ -788,9 +788,9 @@ var RULE_GUIDES = {
   },
   "assertion-weakened": {
     title: "Exact check replaced by a weaker one",
-    why: "Swapping an exact assertion for toBeDefined, a truthiness check or `is not None` keeps the test green while it verifies far less.",
+    why: "Swapping an exact assertion for toBeDefined, a truthiness check, `is not None`, a bound (`>= 2` in place of `== 2`) or toMatchObject in place of toEqual keeps the test green while it verifies far less.",
     suggestion: "Restore the exact check, or assert the specific value the code should produce.",
-    example: "expect(total).toBe(42) \u2192 expect(total).toBeDefined()  (restore the exact check)"
+    example: "expect(total).toBe(42) \u2192 expect(total).toBeDefined()  (restore the exact check); expect(cost).toBe(2) \u2192 expect(cost).toBeLessThanOrEqual(2)  (pin the value again)"
   },
   "assertion-removed": {
     title: "Test checks less than before",
@@ -3673,10 +3673,66 @@ var BOUND_JS = /\.toBeGreaterThan\s*\(\s*(?:0|-1)\s*\)|\.toBeGreaterThanOrEqual\
 var ZERO_JS = /\.(?:toBe|toEqual|toStrictEqual|toHaveLength|toHaveBeenCalledTimes)\s*\(\s*(?:0|\[\s*\]|\{\s*\}|null|undefined|false|"")\s*\)/;
 var WEAK_PY = /^\s*assert\s+[\w.\[\]]+\s*(?:,.*)?$|^\s*assert\s+.+\bis\s+not\s+None\s*(?:,.*)?$|^\s*assert\s+True\b/;
 var BOUND_PY = /^\s*assert\s+.+(?:>\s*0|>=\s*[01])\s*(?:,.*)?$/;
-var EMPTY_PY = /==\s*(?:0|\[\s*\]|\{\s*\}|""|''|None|False)\b|\bis\s+None\b|\bnot\b|!=\s*(?:0|None)\b/;
+var EMPTY_PY = /==\s*(?:0(?:\.0+)?(?![\w.])|\[\s*\]|\{\s*\}|""|''|\(\s*\)|(?:set|dict|list|tuple|frozenset)\(\s*\)|None\b|False\b)|\bis\s+None\b|\bnot\b|!=\s*(?:0|None)\b/;
 var PY_SPECIFIC = /==|!=|<=|>=|\bin\b|\bis\b|\bisinstance\s*\(|\.startswith\s*\(|\.endswith\s*\(|\bcallable\s*\(/;
 var JS_SPECIFIC = /\.(?:toBe|toEqual|toStrictEqual|toHaveLength|toContain|toContainEqual|toMatch|toMatchObject|toHaveProperty|toHaveBeenCalledWith|toHaveBeenLastCalledWith|toHaveBeenNthCalledWith|toBeCalledWith|toHaveBeenCalledTimes|toHaveBeenCalledOnce|toHaveTextContent|toHaveAttribute|toHaveValue|toThrow\w*|toBeGreaterThan\w*|toBeLessThan\w*)\s*\(/;
 var THROWING_QUERY = /\b(?:get|find)(?:All)?By\w*\s*\(/;
+function checkedExpression(assertion, language) {
+  const squash = (text) => text.replace(/\s+/g, "").replace(/^len\((.*)\)$/, "$1").replace(/\.length$/, "");
+  if (language === "py") {
+    const operand = /^\s*assert\s+(?:not\s+)?(.+?)\s*(?:==|!=|>=|<=|>|<|\bin\b|\bis\b)/.exec(assertion)?.[1] ?? /\bself\s*\.\s*assert\w*\s*\(\s*([^,]+?)\s*,/.exec(assertion)?.[1];
+    return operand ? squash(operand) : null;
+  }
+  const start = assertion.search(/(?<![\w$.])expect\s*\(/);
+  if (start >= 0) {
+    const open2 = assertion.indexOf("(", start);
+    let depth = 0;
+    for (let i = open2; i < assertion.length; i++) {
+      if (assertion[i] === "(") depth++;
+      else if (assertion[i] === ")" && --depth === 0) return squash(assertion.slice(open2 + 1, i));
+    }
+    return null;
+  }
+  const node = /assert\s*\.\s*(?:equal|strictEqual|deepEqual|deepStrictEqual|ok)\s*\(\s*([^,)]+?)\s*(?:,|\)|>=|<=|>|<)/.exec(assertion)?.[1];
+  return node ? squash(node) : null;
+}
+function matcherArgument(assertion, name) {
+  const open2 = new RegExp(`\\.\\s*${name}\\s*\\(`).exec(assertion);
+  if (!open2) return null;
+  let depth = 1;
+  for (let i = open2.index + open2[0].length; i < assertion.length; i++) {
+    if ("([{".includes(assertion[i])) depth++;
+    else if (")]}".includes(assertion[i]) && --depth === 0) return assertion.slice(open2.index + open2[0].length, i).replace(/\s+/g, "");
+  }
+  return null;
+}
+function isExact(assertion, language) {
+  if (language === "js") {
+    return !/\.\s*not\s*\./.test(assertion) && (/\.(?:toBe|toEqual|toStrictEqual|toHaveLength)\s*\(/.test(assertion) || /assert\s*\.\s*(?:equal|strictEqual|deepEqual|deepStrictEqual)\s*\(/.test(assertion));
+  }
+  if (language === "py") return /^\s*assert\s+.+?\s*==\s*\S/.test(assertion) || /\bself\s*\.\s*assertEqual\s*\(/.test(assertion);
+  return false;
+}
+function relaxes(oldAssertion, newAssertion, language, siblings = []) {
+  const subject = checkedExpression(oldAssertion, language);
+  if (!subject || subject !== checkedExpression(newAssertion, language)) return false;
+  if (siblings.some((other) => isExact(other, language) && checkedExpression(other, language) === subject)) return false;
+  if (language === "js") {
+    if (/\.\s*not\s*\./.test(oldAssertion) || /\.\s*not\s*\./.test(newAssertion)) return false;
+    const exact = /\.(?:toBe|toEqual|toStrictEqual|toHaveLength)\s*\(\s*(-?\d+(?:\.\d+)?)\s*\)/.exec(oldAssertion)?.[1] ?? /assert\s*\.\s*(?:equal|strictEqual|deepEqual|deepStrictEqual)\s*\([^,]+,\s*(-?\d+(?:\.\d+)?)\s*[,)]/.exec(oldAssertion)?.[1];
+    const bound = /\.(?:toBeGreaterThan|toBeGreaterThanOrEqual|toBeLessThan|toBeLessThanOrEqual)\s*\(/.test(newAssertion) || /assert\s*\.\s*ok\s*\([^;]*(?:>=|<=|>|<)/.test(newAssertion);
+    if (exact !== void 0 && Number(exact) !== 0 && bound) return true;
+    const same = matcherArgument(oldAssertion, "toEqual") ?? matcherArgument(oldAssertion, "toStrictEqual");
+    return same !== null && same !== "" && same === matcherArgument(newAssertion, "toMatchObject");
+  }
+  if (language === "py") {
+    if (EMPTY_PY.test(oldAssertion)) return false;
+    const exact = /^\s*assert\s+.+?\s*==\s*\S/.test(oldAssertion) || /\bself\s*\.\s*assertEqual\s*\(/.test(oldAssertion);
+    const bound = /^\s*assert\s+.+?\s*(?:>=|<=|>|<)\s*\S/.test(newAssertion) && !/==/.test(newAssertion.replace(/[<>]=/g, "")) || /\bself\s*\.\s*assert(?:Greater|Less)(?:Equal)?\s*\(/.test(newAssertion);
+    return exact && bound;
+  }
+  return false;
+}
 function pinsLess(oldAssertion, newAssertion, language) {
   if (language === "js") {
     const was = /\.toThrow(?:Error)?\s*\(\s*([^)\s][^)]*)\)/.exec(oldAssertion);
@@ -3698,7 +3754,7 @@ function pinsLess(oldAssertion, newAssertion, language) {
   return false;
 }
 function isWeaker(oldAssertion, newAssertion, language) {
-  if (pinsLess(oldAssertion, newAssertion, language)) return true;
+  if (pinsLess(oldAssertion, newAssertion, language) || relaxes(oldAssertion, newAssertion, language)) return true;
   if (language === "js") {
     if (THROWING_QUERY.test(newAssertion) || !JS_SPECIFIC.test(oldAssertion) || WEAK_JS.test(oldAssertion) || BOUND_JS.test(oldAssertion)) return false;
     return WEAK_JS.test(newAssertion) || BOUND_JS.test(newAssertion) && !ZERO_JS.test(oldAssertion);
@@ -3724,6 +3780,34 @@ var HELPER_CALL = /(?<![\w$])((?:expect|assert|verify|check|require)[A-Z]\w*|_{0
 function finding(type, file, line, snippet, reason, commit) {
   return { type, file, lineNumber: Math.max(1, line), lineSnippet: snippet, commitHash: commit, commitDate: "", reason, suggestion: RULE_GUIDES[type].suggestion };
 }
+function alignAssertions(before, after) {
+  const table = Array.from({ length: before.length + 1 }, () => new Array(after.length + 1).fill(0));
+  for (let i = before.length - 1; i >= 0; i--) {
+    for (let j = after.length - 1; j >= 0; j--) table[i][j] = before[i] === after[j] ? table[i + 1][j + 1] + 1 : Math.max(table[i + 1][j], table[i][j + 1]);
+  }
+  const pairs = /* @__PURE__ */ new Map();
+  for (let i = 0, j = 0; i < before.length && j < after.length; ) {
+    if (before[i] === after[j]) {
+      pairs.set(i, j);
+      i++;
+      j++;
+    } else if (table[i + 1][j] >= table[i][j + 1]) i++;
+    else j++;
+  }
+  return pairs;
+}
+function unmatchedIndexes(from, against) {
+  const budget = /* @__PURE__ */ new Map();
+  for (const assertion of against) budget.set(assertion, (budget.get(assertion) ?? 0) + 1);
+  return from.map((assertion, i) => ({ assertion, i })).filter(({ assertion }) => {
+    const available = budget.get(assertion) ?? 0;
+    if (available > 0) {
+      budget.set(assertion, available - 1);
+      return false;
+    }
+    return true;
+  }).map(({ i }) => i);
+}
 function analyzeTestFile(oldPath, newPath, oldText, newText, commit) {
   const language = languageOf(newText === null ? oldPath : newPath);
   if (!language || !(isTestFile(newPath) || isTestFile(oldPath))) return EMPTY;
@@ -3765,6 +3849,34 @@ function analyzeTestFile(oldPath, newPath, oldText, newText, commit) {
     if (current.parametrized && !old.parametrized) continue;
     const snapshot = (test) => test.raw.some((statement) => /\.toMatch(?:Inline)?Snapshot\b|\.matchSnapshot\b|\bsnapshot\s*\(/.test(statement));
     if (snapshot(current) && !snapshot(old)) continue;
+    if (old.assertions.length && current.assertions.length && current.assertions.length !== old.assertions.length && old.assertions.length * current.assertions.length <= 25e4) {
+      const aligned = alignAssertions(old.assertions, current.assertions);
+      const fresh = unmatchedIndexes(current.assertions, old.assertions);
+      const gone = unmatchedIndexes(old.assertions, current.assertions).filter((index) => (newTotals.get(old.assertions[index]) ?? 0) < (oldTotals.get(old.assertions[index]) ?? 0));
+      for (const index of gone) {
+        const was = checkedExpression(old.raw[index], language);
+        if (was === null) continue;
+        let low = -1, high = current.assertions.length;
+        for (const [from, to] of aligned) {
+          if (from < index) low = Math.max(low, to);
+          else if (from > index) high = Math.min(high, to);
+        }
+        const successors = fresh.filter((candidate) => candidate > low && candidate < high);
+        const siblings = successors.map((candidate) => current.raw[candidate]);
+        const partner = successors.find((candidate) => checkedExpression(current.raw[candidate], language) === was && relaxes(old.raw[index], current.raw[candidate], language, siblings));
+        if (partner !== void 0) {
+          result.immediate.push(finding(
+            "assertion-weakened",
+            newPath,
+            current.line,
+            `${old.name}: ${old.raw[index].slice(0, 80)} \u2192 ${current.raw[partner].slice(0, 80)}`,
+            "Assertion replaced by a weaker check",
+            commit
+          ));
+          break;
+        }
+      }
+    }
     if (current.assertions.length < old.assertions.length) {
       if ([...current.text.matchAll(HELPER_CALL)].some((call) => !(oldText ?? "").includes(call[1]))) continue;
       const budget = tally([current]);
@@ -3793,19 +3905,7 @@ ${addedText}`;
       continue;
     }
     if (current.assertions.length === old.assertions.length && old.assertions.length > 0) {
-      const left = (from, against) => {
-        const budget = /* @__PURE__ */ new Map();
-        for (const assertion of against) budget.set(assertion, (budget.get(assertion) ?? 0) + 1);
-        return from.map((assertion, i) => ({ assertion, i })).filter(({ assertion }) => {
-          const available = budget.get(assertion) ?? 0;
-          if (available > 0) {
-            budget.set(assertion, available - 1);
-            return false;
-          }
-          return true;
-        }).map(({ i }) => i);
-      };
-      const goneIndexes = left(old.assertions, current.assertions), freshIndexes = left(current.assertions, old.assertions);
+      const goneIndexes = unmatchedIndexes(old.assertions, current.assertions), freshIndexes = unmatchedIndexes(current.assertions, old.assertions);
       const changed = goneIndexes.map((index, k) => ({ before: old.assertions[index], after: current.assertions[freshIndexes[k]], rawBefore: old.raw[index], rawAfter: current.raw[freshIndexes[k]] })).filter((pair) => pair.after !== void 0);
       const weaker = changed.find((pair) => isWeaker(pair.rawBefore, pair.rawAfter, language));
       if (weaker) {

package/dist/cli.js

@@ -38,7 +38,7 @@ import {
   terminalText,
   uninstallHooks,
   writeBaseline
-} from "./chunk-52FUQ4EP.js";
+} from "./chunk-IKXGHCWB.js";
 
 // src/cli.ts
 import { Command } from "commander";

package/dist/index.js

@@ -91,7 +91,7 @@ import {
   unquoteGitPath,
   validateExceptions,
   writeBaseline
-} from "./chunk-52FUQ4EP.js";
+} from "./chunk-IKXGHCWB.js";
 export {
   AGENT_LABELS,
   CONFIG_FILENAMES,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-roast",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "description": "Stops AI coding agents (and rushed humans) from turning a build green by weakening tests, coverage gates or CI. Hook for Claude Code, Codex and Cursor, pre-commit, GitHub Action. Offline.",
   "mcpName": "io.github.Nari-Ab/agent-roast",
   "type": "module",