agent-roast 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/{chunk-MZBP537U.js → chunk-BWGYT2YZ.js} +344 -49
- package/dist/cli.js +1 -1
- package/dist/index.js +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -79,11 +79,11 @@ Policy is read from the **base** branch, so a pull request cannot switch off the
|
|
|
79
79
|
| Rule | Fires when | By default |
|
|
80
80
|
| --- | --- | :-: |
|
|
81
81
|
| `assertion-weakened` | an exact check is replaced by a weaker one: `toBe(42)` → `toBeDefined()`, `assert x == 1` → `assert x` | blocks |
|
|
82
|
-
| `guard-weakened` | a coverage threshold is lowered, `strict` is switched off, a lint rule is downgraded, a CI step may fail, a workflow stops running tests | blocks |
|
|
83
|
-
| `test-skip` | a test is switched off: `it.skip`, `@pytest.mark.skip`, `t.Skip`, `#[ignore]`, `@Disabled`, `[Ignore]`, `xit`, ... | blocks |
|
|
82
|
+
| `guard-weakened` | a coverage threshold is lowered, `strict` is switched off, a lint rule is downgraded, a CI step may fail, a workflow stops running tests, a `conftest.py` hook sets a report to `passed` | blocks |
|
|
83
|
+
| `test-skip` | a test is switched off or expected to fail: `it.skip`, `@pytest.mark.skip`, `@pytest.mark.xfail`, `it.fails`, `test.fixme`, `t.Skip`, `#[ignore]`, `@Disabled`, `[Ignore]`, `xit`, ... | blocks |
|
|
84
84
|
| `test-focus`, `trivial-assert` | `.only` / `fit`; `expect(true).toBe(true)` and its relatives in ten languages | blocks |
|
|
85
85
|
| `assertion-removed`, `test-removed`, `expectation-rewritten` | a test loses assertions, disappears, or its expected value is edited next to production code | warns |
|
|
86
|
-
| `swallowed-error`, `type-escape`, `stub-abandonment` | `catch {}` / `except: pass`, `as any` / `# type: ignore`, `throw new Error("Not implemented")` | warns |
|
|
86
|
+
| `swallowed-error`, `type-escape`, `stub-abandonment` | `catch {}` / `except: pass`, `as any` / `# type: ignore` / `# pragma: no cover` / `istanbul ignore`, `throw new Error("Not implemented")` | warns |
|
|
87
87
|
|
|
88
88
|
Renames, moved tests, parametrization, moved assertions and reformatting are recognized and not reported, and neither is a skip
|
|
89
89
|
that depends on the environment (`if testing.Short() { t.Skip() }`, `test.skip(isMobile, ...)`). JavaScript/TypeScript, Python
|
|
@@ -111,9 +111,9 @@ old debt with a baseline: [docs/reference.md](docs/reference.md#configuration).
|
|
|
111
111
|
Measured, with the denominators and the caveats in [docs/benchmarks.md](docs/benchmarks.md). **Weakened checks are rare, mostly
|
|
112
112
|
deliberate, and not specific to AI. agent-roast is a tripwire that stays silent until a check gets weaker.**
|
|
113
113
|
|
|
114
|
-
- **
|
|
114
|
+
- **114 of 114** hand-labeled cheating edits are caught and **80 of 80** harmless look-alikes (reformats, moves, raised
|
|
115
115
|
thresholds, environment-gated skips) stay silent. The cases were written with the detector, so they guard against regressions;
|
|
116
|
-
they do not prove accuracy.
|
|
116
|
+
they do not prove accuracy. Two gaps are documented and still open.
|
|
117
117
|
- **12,645 commits** that touch tests or CI/config in 166 public repositories: blocking rules fired on 0.3% of AI-signed commits
|
|
118
118
|
(Claude, Copilot, Cursor, Codex, Devin, aider) and on 0.3% of the others. Inside the same repositories the odds for an
|
|
119
119
|
AI-signed commit were 0.81 times those of another commit (95% CI 0.40–1.67). There is no sign that AI-signed commits weaken
|
|
@@ -757,10 +757,10 @@ var RULE_GUIDES = {
|
|
|
757
757
|
example: 'test.only("auth", ...) \u2192 test("auth", ...), then run the full suite'
|
|
758
758
|
},
|
|
759
759
|
"test-skip": {
|
|
760
|
-
title: "Skipped or
|
|
761
|
-
why: "The test runner may report success without executing this behavior check.",
|
|
762
|
-
suggestion: "Re-enable the test and fix its failure. If a skip is intentional, document its reason and track when to restore it.",
|
|
763
|
-
example: 'it.skip("auth", ...) \u2192
|
|
760
|
+
title: "Skipped, disabled or expected-to-fail test",
|
|
761
|
+
why: "The test runner may report success without executing this behavior check, or without letting it fail.",
|
|
762
|
+
suggestion: "Re-enable the test and fix its failure. If a skip or an expected failure is intentional, document its reason and track when to restore it.",
|
|
763
|
+
example: 'it.skip("auth", ...) or @pytest.mark.xfail \u2192 remove the marker after fixing the failing behavior'
|
|
764
764
|
},
|
|
765
765
|
"trivial-assert": {
|
|
766
766
|
title: "Assertion that always passes",
|
|
@@ -769,10 +769,10 @@ var RULE_GUIDES = {
|
|
|
769
769
|
example: "expect(true).toBe(true) \u2192 expect(canSignIn(validUser)).toBe(true)"
|
|
770
770
|
},
|
|
771
771
|
"type-escape": {
|
|
772
|
-
title: "Type or
|
|
773
|
-
why: "A suppression can hide incompatible values
|
|
774
|
-
suggestion: "Narrow or validate the value at the boundary. Keep a suppression only with a specific reason and the smallest possible scope.",
|
|
775
|
-
example: "response as any \u2192 parse and validate response before using it"
|
|
772
|
+
title: "Type, lint or coverage check bypass",
|
|
773
|
+
why: "A suppression can hide incompatible values, a disabled correctness check, or code that no test reaches. Some suppressions are justified.",
|
|
774
|
+
suggestion: "Narrow or validate the value at the boundary, or test the excluded code. Keep a suppression only with a specific reason and the smallest possible scope.",
|
|
775
|
+
example: "response as any \u2192 parse and validate response before using it; # pragma: no cover \u2192 a test for that branch"
|
|
776
776
|
},
|
|
777
777
|
"swallowed-error": {
|
|
778
778
|
title: "Error discarded without handling",
|
|
@@ -812,9 +812,9 @@ var RULE_GUIDES = {
|
|
|
812
812
|
},
|
|
813
813
|
"guard-weakened": {
|
|
814
814
|
title: "Quality gate made easier to pass",
|
|
815
|
-
why: "Lower coverage thresholds, disabled compiler strictness, downgraded lint rules, tolerated CI failures
|
|
815
|
+
why: "Lower coverage thresholds, disabled compiler strictness, downgraded lint rules, tolerated CI failures, an always-passing test script or a test-harness hook that rewrites the result let defects through without touching the code.",
|
|
816
816
|
suggestion: "Keep the gate and fix the underlying failure. If the change is deliberate, record the reason and a plan to restore it.",
|
|
817
|
-
example: 'coverageThreshold lines: 80 \u2192 60, "strict": true \u2192 false, continue-on-error: true'
|
|
817
|
+
example: 'coverageThreshold lines: 80 \u2192 60, "strict": true \u2192 false, continue-on-error: true, report.outcome = "passed"'
|
|
818
818
|
},
|
|
819
819
|
"panic-loop": {
|
|
820
820
|
title: "Rapid overlapping fix commits",
|
|
@@ -1163,8 +1163,19 @@ var PYTHON_TEST_SKIP_PATTERNS = [
|
|
|
1163
1163
|
{ regex: /@pytest\s*\.\s*mark\s*\.\s*skip\b/, name: "skipped pytest (@pytest.mark.skip)" },
|
|
1164
1164
|
{ regex: /(?<![\w.])pytest\s*\.\s*skip\s*\(/, name: "imperative pytest skip (pytest.skip)", imperative: true },
|
|
1165
1165
|
{ regex: /@unittest\s*\.\s*skip\s*\(/, name: "skipped unittest (@unittest.skip)", imperative: false },
|
|
1166
|
-
{ regex: /\bself\s*\.\s*skipTest\s*\(/, name: "imperative unittest skip (self.skipTest)", imperative: true }
|
|
1166
|
+
{ regex: /\bself\s*\.\s*skipTest\s*\(/, name: "imperative unittest skip (self.skipTest)", imperative: true },
|
|
1167
|
+
// A test that is expected to fail cannot fail the run, which is what a skip does too.
|
|
1168
|
+
{ regex: /(?<![\w.])pytest\s*\.\s*xfail\s*\(/, name: "imperative pytest xfail (pytest.xfail)", imperative: true },
|
|
1169
|
+
{ regex: /@(?:unittest\s*\.\s*)?expectedFailure\b/, name: "test expected to fail (@unittest.expectedFailure)", imperative: false }
|
|
1167
1170
|
];
|
|
1171
|
+
var PYTHON_MARKER = /(@\s*|(?<![\w.])marks\s*=\s*\[?\s*|^[ \t]*pytestmark\s*=\s*\[?\s*)pytest\s*\.\s*mark\s*\.\s*(skip|xfail)\b/gm;
|
|
1172
|
+
function isConditionalXfail(args) {
|
|
1173
|
+
if (!args?.length) return false;
|
|
1174
|
+
const keyword = (arg) => /^(\w+)\s*=(?!=)/.exec(arg)?.[1];
|
|
1175
|
+
const condition = keyword(args[0]) ? args.find((arg) => keyword(arg) === "condition")?.replace(/^condition\s*=\s*/, "") : args[0];
|
|
1176
|
+
if (condition === void 0 || /^(?:True|1)$/.test(condition.trim())) return false;
|
|
1177
|
+
return !/\bCI\b|\bisCI\b|GITHUB_ACTIONS/.test(condition);
|
|
1178
|
+
}
|
|
1168
1179
|
function isGuardedSkip(lines) {
|
|
1169
1180
|
const indentOf = (text) => text.length - text.trimStart().length;
|
|
1170
1181
|
const mentionsCi = (raw) => /\bCI\b|\bGITHUB_ACTIONS\b/.test(raw);
|
|
@@ -1305,44 +1316,67 @@ function isGuardedBlockSkip(lines, profile) {
|
|
|
1305
1316
|
}
|
|
1306
1317
|
return compiledConditionally(i + 1);
|
|
1307
1318
|
}
|
|
1308
|
-
var JS_TEST_API = /(?<![\w$.])(?:(?:it|test|describe|suite)(?:\s*\.\s*(?:concurrent|serial|parallel|describe|failing|sequential))*\s*\.\s*(skip|todo|only)|((?:xit|xtest|xdescribe|fit|fdescribe)))\b((?:\s*\.\s*(?:each|failing|concurrent|serial|parallel|sequential))*)\s*(?=\(|\u0001)/g;
|
|
1319
|
+
var JS_TEST_API = /(?<![\w$.])(?:(?:it|test|describe|suite)(?:\s*\.\s*(?:concurrent|serial|parallel|describe|failing|sequential))*\s*\.\s*(skip|todo|only|fixme|failing|fails|fail)|((?:xit|xtest|xdescribe|fit|fdescribe)))\b((?:\s*\.\s*(?:each|failing|concurrent|serial|parallel|sequential))*)\s*(?=\(|\u0001)/g;
|
|
1309
1320
|
function isConditionalSkip(source, raw, open2) {
|
|
1310
|
-
|
|
1321
|
+
const args = callArguments(source, raw, open2);
|
|
1322
|
+
if (!args) return false;
|
|
1323
|
+
const isString = (arg) => /^(['"`])[\s\S]*\1$/.test(arg);
|
|
1324
|
+
if (!args.length || isString(args[0]) || args[0] === "true") return false;
|
|
1325
|
+
if (args.length > 1 && !isString(args.at(-1))) return false;
|
|
1326
|
+
return !/\bCI\b|\bisCI\b|GITHUB_ACTIONS/.test(args[0]);
|
|
1327
|
+
}
|
|
1328
|
+
function callArguments(source, raw, open2) {
|
|
1329
|
+
if (source[open2] !== "(") return null;
|
|
1311
1330
|
const args = [];
|
|
1312
1331
|
let depth = 0;
|
|
1313
1332
|
let start = open2 + 1;
|
|
1314
1333
|
for (let i = start; i < source.length && i - open2 <= 2e3; i++) {
|
|
1315
1334
|
const char = source[i];
|
|
1316
|
-
if (char === "\0") return
|
|
1335
|
+
if (char === "\0") return null;
|
|
1317
1336
|
if ("([{".includes(char)) depth++;
|
|
1318
1337
|
else if (")]}".includes(char)) {
|
|
1319
1338
|
if (depth) {
|
|
1320
1339
|
depth--;
|
|
1321
1340
|
continue;
|
|
1322
1341
|
}
|
|
1323
|
-
if (char !== ")") return
|
|
1342
|
+
if (char !== ")") return null;
|
|
1324
1343
|
args.push(raw.slice(start, i).trim());
|
|
1325
1344
|
while (args.length && !args.at(-1)) args.pop();
|
|
1326
|
-
|
|
1327
|
-
if (!args.length || isString(args[0]) || args[0] === "true") return false;
|
|
1328
|
-
if (args.length > 1 && !isString(args.at(-1))) return false;
|
|
1329
|
-
return !/\bCI\b|\bisCI\b|GITHUB_ACTIONS/.test(args[0]);
|
|
1345
|
+
return args;
|
|
1330
1346
|
} else if (char === "," && !depth) {
|
|
1331
1347
|
args.push(raw.slice(start, i).trim());
|
|
1332
1348
|
start = i + 1;
|
|
1333
1349
|
}
|
|
1334
1350
|
}
|
|
1335
|
-
return
|
|
1351
|
+
return null;
|
|
1336
1352
|
}
|
|
1337
1353
|
var JS_COMMENT_DIRECTIVE_PATTERNS = [
|
|
1338
1354
|
{ regex: /^\s*\/\/\s*@ts-ignore\b/, name: "compiler ignore (@ts-ignore)" },
|
|
1339
1355
|
{ regex: /^\s*\/\/\s*@ts-expect-error\b/, name: "compiler suppression (@ts-expect-error)" },
|
|
1340
1356
|
{ regex: /^\s*\/\/\s*@ts-nocheck\b/, name: "compiler check disable (@ts-nocheck)" },
|
|
1341
|
-
{ regex: /^\s*\/\/\s*eslint-disable(?:-next-line|-line)?\b/, name: "linter suppression (eslint-disable)" }
|
|
1357
|
+
{ regex: /^\s*\/\/\s*eslint-disable(?:-next-line|-line)?\b/, name: "linter suppression (eslint-disable)" },
|
|
1358
|
+
{ regex: /^\s*\/\/\s*biome-ignore\b/, name: "linter suppression (biome-ignore)" },
|
|
1359
|
+
{ regex: /^\s*\/\/\s*(?:istanbul|c8|v8|node:coverage)\s+(?:ignore\b(?!\s+(?:stop|end)\b)|disable\b)/, name: "coverage exclusion (istanbul / c8 / v8 ignore)" }
|
|
1360
|
+
];
|
|
1361
|
+
var JS_BLOCK_DIRECTIVE_PATTERNS = [
|
|
1362
|
+
{ regex: /^\/\*\s*eslint-disable(?:-next-line|-line)?\b/, name: "linter suppression (eslint-disable)" },
|
|
1363
|
+
{ regex: /^\/\*\s*(?:istanbul|c8|v8|node:coverage)\s+(?:ignore\b(?!\s+(?:stop|end)\b)|disable\b)/, name: "coverage exclusion (istanbul / c8 / v8 ignore)" }
|
|
1342
1364
|
];
|
|
1365
|
+
var COVERAGE_IDIOM = /^\s*(?:if\s+(?:(?:typing|t)\s*\.\s*)?TYPE_CHECKING\b|if\s+__name__\s*==|raise\s+(?:NotImplementedError|AssertionError)\b|(?:\w+\s*\.\s*)?assert_never\s*\(|@(?:abc\s*\.\s*)?abstractmethod\b|except\s*\(?\s*(?:ImportError|ModuleNotFoundError)\b|import\s|from\s+\S+\s+import\b|\.\.\.\s*$|pass\s*$|else\s*:|case\s+_\s*:|(?:el)?if\s+sys\s*\.\s*(?:version_info|platform)\b)/;
|
|
1343
1366
|
var PYTHON_COMMENT_DIRECTIVE_PATTERNS = [
|
|
1344
1367
|
{ regex: /#\s*type:\s*ignore\b/, name: "python type ignore" },
|
|
1345
|
-
{ regex: /#\s*noqa\b/, name: "python linter suppression (# noqa)" }
|
|
1368
|
+
{ regex: /#\s*noqa\b/, name: "python linter suppression (# noqa)" },
|
|
1369
|
+
{ regex: /#\s*pragma:\s*no\s*(?:cover|branch)\b/i, name: "coverage exclusion (# pragma: no cover)", unless: COVERAGE_IDIOM },
|
|
1370
|
+
{ regex: /#\s*pylint:\s*(?:disable|skip-file)\b/, name: "python linter suppression (# pylint: disable)" },
|
|
1371
|
+
{ regex: /#\s*pyright:\s*ignore\b/, name: "python type ignore (# pyright: ignore)" },
|
|
1372
|
+
{ regex: /#\s*mypy:\s*ignore-errors\b/, name: "python type check disabled for the file (# mypy: ignore-errors)" },
|
|
1373
|
+
{ regex: /#\s*(?:flake8|ruff):\s*noqa\b/i, name: "python linter suppression for the file (# flake8: noqa)" }
|
|
1374
|
+
];
|
|
1375
|
+
var PYTHON_HARNESS_FILE = /(?:^|\/)(?:conftest|sitecustomize|usercustomize)\.py$/i;
|
|
1376
|
+
var PYTHON_HARNESS_PATTERNS = [
|
|
1377
|
+
{ regex: /\.\s*outcome\s*=\s*(["'])passed\1/, name: "test report outcome forced to passed" },
|
|
1378
|
+
{ regex: /(?<![\w.])call\s*\.\s*excinfo\s*=\s*None\b/, name: "failure cleared from a test report hook (call.excinfo = None)" },
|
|
1379
|
+
{ regex: /(?<![\w.])session\s*\.\s*exitstatus\s*=\s*(?:0\b|(?:pytest\s*\.\s*)?ExitCode\s*\.\s*OK\b)/, name: "pytest exit status forced to 0" }
|
|
1346
1380
|
];
|
|
1347
1381
|
var CODE_TYPE_ESCAPE_PATTERNS = [
|
|
1348
1382
|
{ regex: /\bas\s+any\b/, name: "type bypass (as any)" },
|
|
@@ -1465,7 +1499,7 @@ function auditFileDiffs(files, commitHash = "HEAD", commitDate = (/* @__PURE__ *
|
|
|
1465
1499
|
}
|
|
1466
1500
|
let foundEscape = false;
|
|
1467
1501
|
for (const rule of directives) {
|
|
1468
|
-
if (rule.regex.test(masked.comments)) {
|
|
1502
|
+
if (rule.regex.test(masked.comments) && !rule.unless?.test(cleanedLine)) {
|
|
1469
1503
|
infractions.push({
|
|
1470
1504
|
type: "type-escape",
|
|
1471
1505
|
file: file.path,
|
|
@@ -1479,8 +1513,9 @@ function auditFileDiffs(files, commitHash = "HEAD", commitDate = (/* @__PURE__ *
|
|
|
1479
1513
|
break;
|
|
1480
1514
|
}
|
|
1481
1515
|
}
|
|
1482
|
-
|
|
1483
|
-
|
|
1516
|
+
const blockDirective = javascript && !foundEscape ? JS_BLOCK_DIRECTIVE_PATTERNS.find((rule) => masked.blockStarts.some((start) => rule.regex.test(rawLine.slice(start)))) : void 0;
|
|
1517
|
+
if (blockDirective) {
|
|
1518
|
+
infractions.push({ type: "type-escape", file: file.path, lineNumber: added.lineNumber, lineSnippet: rawLine.trim(), commitHash, commitDate, reason: blockDirective.name });
|
|
1484
1519
|
foundEscape = true;
|
|
1485
1520
|
}
|
|
1486
1521
|
if (python && !isComment && isTestFile(file.path)) {
|
|
@@ -1543,6 +1578,34 @@ function auditFileDiffs(files, commitHash = "HEAD", commitDate = (/* @__PURE__ *
|
|
|
1543
1578
|
infractions.push({ type: "stub-abandonment", file: file.path, ...location, commitHash, commitDate, reason: rule.name });
|
|
1544
1579
|
}
|
|
1545
1580
|
}
|
|
1581
|
+
if (python) {
|
|
1582
|
+
const reportedSkips = new Set(infractions.slice(firstFinding).filter((f) => f.type === "test-skip").map((f) => f.lineNumber));
|
|
1583
|
+
for (const match of source.matchAll(PYTHON_MARKER)) {
|
|
1584
|
+
const xfail = match[2] === "xfail";
|
|
1585
|
+
let open2 = match.index + match[0].length;
|
|
1586
|
+
while (source[open2] === " " || source[open2] === " ") open2++;
|
|
1587
|
+
if (xfail && isConditionalXfail(callArguments(source, raw, open2))) continue;
|
|
1588
|
+
const location = evidence(match.index, match[0].length);
|
|
1589
|
+
if (reportedSkips.has(location.lineNumber)) continue;
|
|
1590
|
+
reportedSkips.add(location.lineNumber);
|
|
1591
|
+
infractions.push({
|
|
1592
|
+
type: "test-skip",
|
|
1593
|
+
file: file.path,
|
|
1594
|
+
...location,
|
|
1595
|
+
commitHash,
|
|
1596
|
+
commitDate,
|
|
1597
|
+
reason: xfail ? "test marked as expected to fail (pytest.mark.xfail)" : "test skipped by a marker (pytest.mark.skip)"
|
|
1598
|
+
});
|
|
1599
|
+
}
|
|
1600
|
+
if (isTestFile(file.path) || PYTHON_HARNESS_FILE.test(file.path)) {
|
|
1601
|
+
for (const rule of PYTHON_HARNESS_PATTERNS) {
|
|
1602
|
+
for (const match of raw.matchAll(new RegExp(rule.regex.source, "g"))) {
|
|
1603
|
+
if (code[match.index] !== raw[match.index]) continue;
|
|
1604
|
+
infractions.push({ type: "guard-weakened", file: file.path, ...evidence(match.index, match[0].length), commitHash, commitDate, reason: rule.name });
|
|
1605
|
+
}
|
|
1606
|
+
}
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1546
1609
|
if (typescript) {
|
|
1547
1610
|
const reported = new Set(infractions.slice(firstFinding).filter((f) => f.type === "type-escape").map((f) => f.lineNumber));
|
|
1548
1611
|
for (const rule of CODE_TYPE_ESCAPE_PATTERNS) {
|
|
@@ -1562,15 +1625,16 @@ function auditFileDiffs(files, commitHash = "HEAD", commitDate = (/* @__PURE__ *
|
|
|
1562
1625
|
if (match[2] && !isTestFile(file.path)) continue;
|
|
1563
1626
|
if (/\bfunction$/.test(source.slice(Math.max(0, previous - 8), previous + 1))) continue;
|
|
1564
1627
|
if (source[match.index + match[0].length] === "" && !/\.\s*each\s*$/.test(match[3])) continue;
|
|
1565
|
-
if (match[1] === "skip" && !match[3] && isConditionalSkip(source, raw, match.index + match[0].length)) continue;
|
|
1628
|
+
if ((match[1] === "skip" || match[1] === "fixme" || match[1] === "fail") && !match[3] && isConditionalSkip(source, raw, match.index + match[0].length)) continue;
|
|
1566
1629
|
const focus = match[1] === "only" || match[2] === "fit" || match[2] === "fdescribe";
|
|
1630
|
+
const expectedToFail = match[1] === "failing" || match[1] === "fails" || match[1] === "fail";
|
|
1567
1631
|
infractions.push({
|
|
1568
1632
|
type: focus ? "test-focus" : "test-skip",
|
|
1569
1633
|
file: file.path,
|
|
1570
1634
|
...evidence(match.index, match[0].length),
|
|
1571
1635
|
commitHash,
|
|
1572
1636
|
commitDate,
|
|
1573
|
-
reason: focus ? "focused test or suite excludes other tests from the run" : `skipped, disabled or placeholder test registration (${match[0].replace(/\s/g, "")})`
|
|
1637
|
+
reason: focus ? "focused test or suite excludes other tests from the run" : expectedToFail ? `test expected to fail, so its failure cannot fail the run (${match[0].replace(/\s/g, "")})` : `skipped, disabled or placeholder test registration (${match[0].replace(/\s/g, "")})`
|
|
1574
1638
|
});
|
|
1575
1639
|
}
|
|
1576
1640
|
if (isTestFile(file.path)) {
|
|
@@ -3100,6 +3164,113 @@ import path6 from "path";
|
|
|
3100
3164
|
|
|
3101
3165
|
// src/tampering.ts
|
|
3102
3166
|
import { createHash as createHash3 } from "crypto";
|
|
3167
|
+
|
|
3168
|
+
// src/pyconfig.ts
|
|
3169
|
+
function openBrackets(text) {
|
|
3170
|
+
const bare = text.replace(/"(?:\\.|[^"\\\n])*"|'[^'\n]*'/g, '""').replace(/#.*$/gm, "");
|
|
3171
|
+
let depth = 0;
|
|
3172
|
+
for (const char of bare) {
|
|
3173
|
+
if (char === "[" || char === "{") depth++;
|
|
3174
|
+
else if (char === "]" || char === "}") depth--;
|
|
3175
|
+
}
|
|
3176
|
+
return depth;
|
|
3177
|
+
}
|
|
3178
|
+
function configEntries(text, toml) {
|
|
3179
|
+
const lines = text.split(/\r?\n/);
|
|
3180
|
+
const entries = [];
|
|
3181
|
+
let section = "";
|
|
3182
|
+
for (let i = 0; i < lines.length; i++) {
|
|
3183
|
+
const trimmed = lines[i].trim();
|
|
3184
|
+
if (!trimmed || trimmed.startsWith("#") || trimmed.startsWith(";")) continue;
|
|
3185
|
+
const header = /^\[\[?\s*([^\]]+?)\s*\]\]?\s*(?:[#;].*)?$/.exec(trimmed);
|
|
3186
|
+
if (header) {
|
|
3187
|
+
section = header[1].replace(/["']/g, "");
|
|
3188
|
+
continue;
|
|
3189
|
+
}
|
|
3190
|
+
const pair = /^("[^"]*"|'[^']*'|[\w.-]+)\s*=\s*(.*)$/.exec(trimmed);
|
|
3191
|
+
if (!pair) continue;
|
|
3192
|
+
const start = i;
|
|
3193
|
+
let value = pair[2];
|
|
3194
|
+
if (toml) {
|
|
3195
|
+
while (openBrackets(value) > 0 && i + 1 < lines.length) value += "\n" + lines[++i];
|
|
3196
|
+
} else {
|
|
3197
|
+
while (i + 1 < lines.length && /^\s+\S/.test(lines[i + 1]) && !/^\s*[#;]/.test(lines[i + 1])) value += "\n" + lines[++i].trim();
|
|
3198
|
+
}
|
|
3199
|
+
entries.push({ section, key: pair[1].replace(/["']/g, ""), value, line: start + 1 });
|
|
3200
|
+
}
|
|
3201
|
+
return entries;
|
|
3202
|
+
}
|
|
3203
|
+
function members(value) {
|
|
3204
|
+
const quoted = [...value.matchAll(/"((?:\\.|[^"\\\n])*)"|'([^'\n]*)'/g)].map((match) => (match[1] ?? match[2]).trim()).filter(Boolean);
|
|
3205
|
+
if (quoted.length) return quoted;
|
|
3206
|
+
return value.replace(/[#;].*$/gm, "").split(/[\s,]+/).map((word) => word.trim()).filter(Boolean);
|
|
3207
|
+
}
|
|
3208
|
+
var truthy = (value) => /^(?:true|1|yes|on)$/i.test(value.replace(/\s*[#;].*$/, "").trim());
|
|
3209
|
+
function toolOf(file, section) {
|
|
3210
|
+
const name = section.toLowerCase();
|
|
3211
|
+
if (/(?:^|\/)\.?ruff\.toml$/i.test(file) || name.startsWith("tool.ruff")) return "ruff";
|
|
3212
|
+
if (name === "flake8" || /(?:^|\/)\.flake8$/i.test(file)) return "flake8";
|
|
3213
|
+
if (/messages[ _]control/.test(name)) return "pylint";
|
|
3214
|
+
if (name === "mypy" || name.startsWith("mypy-") || name.startsWith("tool.mypy") || /(?:^|\/)\.?mypy\.ini$/i.test(file)) return "mypy";
|
|
3215
|
+
return null;
|
|
3216
|
+
}
|
|
3217
|
+
var IGNORE_LISTS = {
|
|
3218
|
+
ruff: /^(?:extend-)?(?:ignore|per-file-ignores)$/,
|
|
3219
|
+
flake8: /^(?:extend[-_])?ignore$|^per[-_]file[-_]ignores$/,
|
|
3220
|
+
pylint: /^disable$/,
|
|
3221
|
+
mypy: /^disable_error_code$/
|
|
3222
|
+
};
|
|
3223
|
+
var MYPY_SWITCHES = /^(?:strict|disallow_untyped_defs|disallow_incomplete_defs|disallow_untyped_calls|disallow_any_generics|disallow_untyped_decorators|disallow_subclassing_any|check_untyped_defs|warn_return_any|warn_unreachable|warn_redundant_casts|no_implicit_optional|strict_optional)$/;
|
|
3224
|
+
var looksLikeCode = (word) => !/[/*:]/.test(word);
|
|
3225
|
+
function analyzePythonConfig(file, oldText, newText, report) {
|
|
3226
|
+
const toml = /\.toml$/i.test(file);
|
|
3227
|
+
const before = configEntries(oldText, toml), after = configEntries(newText, toml);
|
|
3228
|
+
const configured = new Set(before.map((entry) => toolOf(file, entry.section)).filter(Boolean));
|
|
3229
|
+
const ignored = (entries) => {
|
|
3230
|
+
const found = /* @__PURE__ */ new Map();
|
|
3231
|
+
for (const entry of entries) {
|
|
3232
|
+
const tool = toolOf(file, entry.section);
|
|
3233
|
+
if (!tool) continue;
|
|
3234
|
+
const perFile = tool === "ruff" ? /per-file-ignores$/i.test(entry.section) : false;
|
|
3235
|
+
if (!perFile && !IGNORE_LISTS[tool].test(entry.key)) continue;
|
|
3236
|
+
const bucket = found.get(tool) ?? /* @__PURE__ */ new Map();
|
|
3237
|
+
found.set(tool, bucket);
|
|
3238
|
+
for (const code of members(entry.value)) if (looksLikeCode(code) && !bucket.has(code)) bucket.set(code, entry.line);
|
|
3239
|
+
}
|
|
3240
|
+
return found;
|
|
3241
|
+
};
|
|
3242
|
+
const was = ignored(before);
|
|
3243
|
+
for (const [tool, bucket] of ignored(after)) {
|
|
3244
|
+
if (!configured.has(tool)) continue;
|
|
3245
|
+
const old = [...was.get(tool)?.keys() ?? []];
|
|
3246
|
+
const fresh = [...bucket].filter(([code]) => !old.some((known) => code.startsWith(known)));
|
|
3247
|
+
if (fresh.length) report(Math.min(...fresh.map(([, line]) => line)), `${tool} ignore: ${fresh.slice(0, 4).map(([code]) => "+" + code).join(", ")}${fresh.length > 4 ? ", \u2026" : ""}`, "Lint or type checks switched off in the configuration");
|
|
3248
|
+
}
|
|
3249
|
+
const selected = (entries, tool) => entries.filter((entry) => toolOf(file, entry.section) === tool && entry.key === "select").flatMap((entry) => members(entry.value));
|
|
3250
|
+
for (const tool of ["ruff", "flake8"]) {
|
|
3251
|
+
const old = selected(before, tool), now = selected(after, tool);
|
|
3252
|
+
if (!old.length || !now.length || now.includes("ALL")) continue;
|
|
3253
|
+
const dropped = old.filter((code) => !now.some((kept) => code.startsWith(kept)));
|
|
3254
|
+
if (dropped.length) {
|
|
3255
|
+
const line = after.find((entry) => toolOf(file, entry.section) === tool && entry.key === "select")?.line ?? 1;
|
|
3256
|
+
report(line, `${tool} select: ${dropped.slice(0, 4).map((code) => "-" + code).join(", ")}${dropped.length > 4 ? ", \u2026" : ""}`, "Lint rules removed from the selected set");
|
|
3257
|
+
}
|
|
3258
|
+
}
|
|
3259
|
+
const switches = new Map(before.filter((entry) => toolOf(file, entry.section) === "mypy" && MYPY_SWITCHES.test(entry.key)).map((entry) => [`${entry.section}|${entry.key}`, truthy(entry.value)]));
|
|
3260
|
+
for (const entry of after) {
|
|
3261
|
+
if (toolOf(file, entry.section) !== "mypy" || !MYPY_SWITCHES.test(entry.key)) continue;
|
|
3262
|
+
if (switches.get(`${entry.section}|${entry.key}`) === true && !truthy(entry.value)) report(entry.line, `mypy ${entry.key}: true \u2192 false`, "Type checker strictness switched off");
|
|
3263
|
+
}
|
|
3264
|
+
const ignoring = (entries) => entries.filter((entry) => toolOf(file, entry.section) === "mypy" && entry.key === "ignore_errors" && truthy(entry.value));
|
|
3265
|
+
const before_ = ignoring(before), after_ = ignoring(after);
|
|
3266
|
+
if (configured.has("mypy") && after_.length > before_.length) {
|
|
3267
|
+
const known = new Set(before_.map((entry) => entry.section));
|
|
3268
|
+
const fresh = after_.find((entry) => !known.has(entry.section)) ?? after_[after_.length - 1];
|
|
3269
|
+
report(fresh.line, `mypy ignore_errors = true (${fresh.section})`, "Type errors ignored for a module");
|
|
3270
|
+
}
|
|
3271
|
+
}
|
|
3272
|
+
|
|
3273
|
+
// src/tampering.ts
|
|
3103
3274
|
var EMPTY = { immediate: [], removedTests: [], addedHashes: [], rewritten: [] };
|
|
3104
3275
|
function languageOf(file) {
|
|
3105
3276
|
if (/\.(?:[cm]?[jt]sx?)$/i.test(file)) return "js";
|
|
@@ -3259,14 +3430,44 @@ function parseTests(text, language) {
|
|
|
3259
3430
|
function skeleton(assertion) {
|
|
3260
3431
|
return assertion.replace(/(['"`])(?:\\.|(?!\1)[^\\])*\1/g, "S").replace(/\b\d+(?:\.\d+)?\b/g, "N").replace(/\b(?:true|false|null|undefined|None|True|False|nil)\b/g, "V").replace(/\s+/g, "");
|
|
3261
3432
|
}
|
|
3262
|
-
var WEAK_JS = /\.(?:toBeDefined|toBeTruthy|toBeFalsy|toBeUndefined)\s*\(|\.not\s*\.\s*(?:toBeNull|toBeUndefined|toThrow)\s*\(|\.toBeGreaterThanOrEqual\s*\(\s*0\s*\)/;
|
|
3433
|
+
var WEAK_JS = /\.(?:toBeDefined|toBeTruthy|toBeFalsy|toBeUndefined)\s*\(|\.not\s*\.\s*(?:toBeNull|toBeUndefined|toThrow)\s*\(|\.toBeGreaterThanOrEqual\s*\(\s*0\s*\)|\.(?:toHaveBeenCalled|toBeCalled)\s*\(\s*\)|\.(?:toBe|toEqual|toStrictEqual)\s*\(\s*expect\s*\.\s*(?:anything\s*\(\s*\)|any\s*\(\s*\w+\s*\))\s*\)/;
|
|
3434
|
+
var BOUND_JS = /\.toBeGreaterThan\s*\(\s*(?:0|-1)\s*\)|\.toBeGreaterThanOrEqual\s*\(\s*[01]\s*\)/;
|
|
3435
|
+
var ZERO_JS = /\.(?:toBe|toEqual|toStrictEqual|toHaveLength|toHaveBeenCalledTimes)\s*\(\s*(?:0|\[\s*\]|\{\s*\}|null|undefined|false|"")\s*\)/;
|
|
3263
3436
|
var WEAK_PY = /^\s*assert\s+[\w.\[\]]+\s*(?:,.*)?$|^\s*assert\s+.+\bis\s+not\s+None\s*(?:,.*)?$|^\s*assert\s+True\b/;
|
|
3437
|
+
var BOUND_PY = /^\s*assert\s+.+(?:>\s*0|>=\s*[01])\s*(?:,.*)?$/;
|
|
3438
|
+
var EMPTY_PY = /==\s*(?:0|\[\s*\]|\{\s*\}|""|''|None|False)\b|\bis\s+None\b|\bnot\b|!=\s*(?:0|None)\b/;
|
|
3264
3439
|
var PY_SPECIFIC = /==|!=|<=|>=|\bin\b|\bis\b|\bisinstance\s*\(|\.startswith\s*\(|\.endswith\s*\(|\bcallable\s*\(/;
|
|
3265
|
-
var JS_SPECIFIC = /\.(?:toBe|toEqual|toStrictEqual|toHaveLength|toContain|toContainEqual|toMatch|toMatchObject|toHaveProperty|toHaveBeenCalledWith|toHaveBeenCalledTimes|toHaveTextContent|toHaveAttribute|toHaveValue|toThrow\w*|toBeGreaterThan\w*|toBeLessThan\w*)\s*\(/;
|
|
3440
|
+
var JS_SPECIFIC = /\.(?:toBe|toEqual|toStrictEqual|toHaveLength|toContain|toContainEqual|toMatch|toMatchObject|toHaveProperty|toHaveBeenCalledWith|toHaveBeenLastCalledWith|toHaveBeenNthCalledWith|toBeCalledWith|toHaveBeenCalledTimes|toHaveBeenCalledOnce|toHaveTextContent|toHaveAttribute|toHaveValue|toThrow\w*|toBeGreaterThan\w*|toBeLessThan\w*)\s*\(/;
|
|
3266
3441
|
var THROWING_QUERY = /\b(?:get|find)(?:All)?By\w*\s*\(/;
|
|
3442
|
+
function pinsLess(oldAssertion, newAssertion, language) {
|
|
3443
|
+
if (language === "js") {
|
|
3444
|
+
const was = /\.toThrow(?:Error)?\s*\(\s*([^)\s][^)]*)\)/.exec(oldAssertion);
|
|
3445
|
+
const now = /\.toThrow(?:Error)?\s*\(\s*([^)]*)\)/.exec(newAssertion);
|
|
3446
|
+
return !!was && !!now && (now[1].trim() === "" || now[1].trim() === "Error" && was[1].trim() !== "Error");
|
|
3447
|
+
}
|
|
3448
|
+
if (language === "py") {
|
|
3449
|
+
const raised = (text) => {
|
|
3450
|
+
const marker = /\bpytest\s*\.\s*raises\s*\(\s*([\w.]+|\([^)]*\))([^)]*)\)/.exec(text);
|
|
3451
|
+
if (marker) return { type: marker[1], message: /\bmatch\s*=/.test(marker[2]) };
|
|
3452
|
+
const unit = /\bassertRaises(Regex)?\s*\(\s*([\w.]+|\([^)]*\))/.exec(text);
|
|
3453
|
+
return unit ? { type: unit[2], message: !!unit[1] } : null;
|
|
3454
|
+
};
|
|
3455
|
+
const was = raised(oldAssertion), now = raised(newAssertion);
|
|
3456
|
+
const broad = (type) => /^(?:builtins\.)?(?:Base)?Exception$/.test(type);
|
|
3457
|
+
return !!was && !!now && (was.message && !now.message || !broad(was.type) && broad(now.type));
|
|
3458
|
+
}
|
|
3459
|
+
return false;
|
|
3460
|
+
}
|
|
3267
3461
|
function isWeaker(oldAssertion, newAssertion, language) {
|
|
3268
|
-
if (
|
|
3269
|
-
if (language === "
|
|
3462
|
+
if (pinsLess(oldAssertion, newAssertion, language)) return true;
|
|
3463
|
+
if (language === "js") {
|
|
3464
|
+
if (THROWING_QUERY.test(newAssertion) || !JS_SPECIFIC.test(oldAssertion) || WEAK_JS.test(oldAssertion) || BOUND_JS.test(oldAssertion)) return false;
|
|
3465
|
+
return WEAK_JS.test(newAssertion) || BOUND_JS.test(newAssertion) && !ZERO_JS.test(oldAssertion);
|
|
3466
|
+
}
|
|
3467
|
+
if (language === "py") {
|
|
3468
|
+
if (WEAK_PY.test(oldAssertion) || BOUND_PY.test(oldAssertion) || !PY_SPECIFIC.test(oldAssertion)) return false;
|
|
3469
|
+
return WEAK_PY.test(newAssertion) || BOUND_PY.test(newAssertion) && !EMPTY_PY.test(oldAssertion);
|
|
3470
|
+
}
|
|
3270
3471
|
return false;
|
|
3271
3472
|
}
|
|
3272
3473
|
var GENERIC_ARGUMENT = /^(?:not\.)?(?:be|have|contain|include|exist|match|eq|equal|length|text|value|attr|class|css|visible|checked|disabled|enabled|selected|focus|empty|true|false|null|undefined|deep)(?:\.\w+)*$/;
|
|
@@ -3382,7 +3583,58 @@ function resolveTampering(files, productionChanged) {
|
|
|
3382
3583
|
...productionChanged ? files.flatMap((file) => file.rewritten) : []
|
|
3383
3584
|
];
|
|
3384
3585
|
}
|
|
3385
|
-
|
|
3586
|
+
function isKnownFailureMarker(finding2) {
|
|
3587
|
+
return finding2.type === "test-skip" && /\bxfail\b|\bexpectedFailure\b|\bfixme\b|expected to fail/i.test(finding2.reason);
|
|
3588
|
+
}
|
|
3589
|
+
function markedTest(finding2, lines) {
|
|
3590
|
+
const snippet = finding2.lineSnippet ?? "";
|
|
3591
|
+
const index = (finding2.lineNumber ?? 1) - 1;
|
|
3592
|
+
if (/\.py$/i.test(finding2.file ?? "")) {
|
|
3593
|
+
if (/^\s*pytestmark\b/.test(snippet)) return null;
|
|
3594
|
+
if (/imperative/.test(finding2.reason)) {
|
|
3595
|
+
const indent = (lines[index] ?? "").search(/\S/);
|
|
3596
|
+
for (let i = index - 1; i >= 0 && i > index - 200; i--) {
|
|
3597
|
+
const def = /^(\s*)(?:async\s+)?def\s+(\w+)\s*\(/.exec(lines[i]);
|
|
3598
|
+
if (def && def[1].length < indent) return { key: `py:${def[2]}`, title: def[2] };
|
|
3599
|
+
}
|
|
3600
|
+
return null;
|
|
3601
|
+
}
|
|
3602
|
+
for (let i = index; i < lines.length && i < index + 80; i++) {
|
|
3603
|
+
const def = /^\s*(?:async\s+)?def\s+(\w+)\s*\(/.exec(lines[i]);
|
|
3604
|
+
if (def) return { key: `py:${def[1]}`, title: def[1] };
|
|
3605
|
+
}
|
|
3606
|
+
return null;
|
|
3607
|
+
}
|
|
3608
|
+
if (/\.each\b/.test(snippet)) return null;
|
|
3609
|
+
const title = /(['"`])((?:\\.|(?!\1)[^\\\n])+)\1/.exec(snippet);
|
|
3610
|
+
return title ? { key: `js:${title[0]}`, title: title[0] } : null;
|
|
3611
|
+
}
|
|
3612
|
+
function newKnownFailureMarkers(before, after, oldText, newText) {
|
|
3613
|
+
const markers = after.filter(isKnownFailureMarker);
|
|
3614
|
+
if (!markers.length || oldText === null) return [];
|
|
3615
|
+
const oldLines = oldText.split(/\r?\n/), newLines = (newText ?? "").split(/\r?\n/);
|
|
3616
|
+
const existed = (test) => test.key.startsWith("py:") ? new RegExp(`^\\s*(?:async\\s+)?def\\s+${test.title}\\s*\\(`, "m").test(oldText) : oldText.includes(test.title);
|
|
3617
|
+
const markedBefore = /* @__PURE__ */ new Set();
|
|
3618
|
+
let unnamedBefore = 0;
|
|
3619
|
+
for (const finding2 of before.filter(isKnownFailureMarker)) {
|
|
3620
|
+
const test = markedTest(finding2, oldLines);
|
|
3621
|
+
if (test) markedBefore.add(test.key);
|
|
3622
|
+
else unnamedBefore++;
|
|
3623
|
+
}
|
|
3624
|
+
const found = [], unnamed = [], seen = /* @__PURE__ */ new Set();
|
|
3625
|
+
for (const finding2 of markers) {
|
|
3626
|
+
const test = markedTest(finding2, newLines);
|
|
3627
|
+
if (!test) {
|
|
3628
|
+
unnamed.push(finding2);
|
|
3629
|
+
continue;
|
|
3630
|
+
}
|
|
3631
|
+
if (markedBefore.has(test.key) || seen.has(test.key) || !existed(test)) continue;
|
|
3632
|
+
seen.add(test.key);
|
|
3633
|
+
found.push(finding2);
|
|
3634
|
+
}
|
|
3635
|
+
return [...found, ...unnamed.slice(unnamedBefore)];
|
|
3636
|
+
}
|
|
3637
|
+
var GUARD_FILE = /(?:^|\/)(?:package\.json|tsconfig[^/]*\.json|jsconfig\.json|(?:jest|vitest|vite)\.config\.[cm]?[jt]s|\.eslintrc(?:\.[a-z]+)?|eslint\.config\.[cm]?[jt]s|pyproject\.toml|setup\.cfg|pytest\.ini|tox\.ini|\.?ruff\.toml|\.flake8|\.?mypy\.ini|\.?pylintrc|\.coveragerc|codecov\.ya?ml|\.gitlab-ci\.ya?ml|\.github\/workflows\/[^/]+\.ya?ml)$/i;
|
|
3386
3638
|
function isGuardFile(file) {
|
|
3387
3639
|
return GUARD_FILE.test(file);
|
|
3388
3640
|
}
|
|
@@ -3403,9 +3655,13 @@ function numbers(text, pattern) {
|
|
|
3403
3655
|
var THRESHOLD_PATTERNS = [
|
|
3404
3656
|
/\b(lines|branches|functions|statements)\b["']?\s*[:=]\s*(\d+(?:\.\d+)?)/g,
|
|
3405
3657
|
/\b(fail_under)\s*[:=]\s*(\d+(?:\.\d+)?)/g,
|
|
3406
|
-
/(--cov-fail-under)[= ](\d+(?:\.\d+)?)/g
|
|
3658
|
+
/(--cov-fail-under)[= ](\d+(?:\.\d+)?)/g,
|
|
3659
|
+
// `coverage report --fail-under=90` on a command line, and pylint's `fail-under=9.5` in its own config (a lint score).
|
|
3660
|
+
/(?<![\w-])(--fail-under)[= ](\d+(?:\.\d+)?)/g,
|
|
3661
|
+
/(?<![\w-])(fail-under)\s*[:=]\s*(\d+(?:\.\d+)?)/g
|
|
3407
3662
|
];
|
|
3408
3663
|
var SWITCHES = ["strict", "noImplicitAny", "strictNullChecks", "noImplicitReturns", "useUnknownInCatchVariables"];
|
|
3664
|
+
var GATE_SCRIPT = /^(?:pre|post)?(?:test|lint|typecheck|type-check|check|ci|verify|validate)(?::[\w:.-]+)?$/;
|
|
3409
3665
|
var CI_RUNNER = new RegExp(
|
|
3410
3666
|
String.raw`(?<![\w./-])(?:pytest|vitest|jest|mocha|tox|nox)(?![\w/-])` + String.raw`|(?<![\w./-])(?:npm|yarn|pnpm|bun|go|cargo|mvn|gradle|make|dotnet|flutter)\s+(?:run\s+)?(?:-\S+\s+)*test(?![\w/])` + String.raw`|\brun:.*(?<=\s)(?:tests?|unit|e2e)(?![\w/.-])`
|
|
3411
3667
|
);
|
|
@@ -3430,7 +3686,8 @@ function resolveCiTestSteps(counts, commit) {
|
|
|
3430
3686
|
if (after >= before) return [];
|
|
3431
3687
|
return counts.filter((item) => item.before > 0 && item.after === 0 && item.reusableAfter === 0).map((item) => item.ungatedAfter > 0 ? finding("guard-weakened", item.file, 1, "tests no longer run on push or pull_request", "Workflow keeps its tests but no longer runs on pushes or pull requests", commit) : finding("guard-weakened", item.file, 1, "test command no longer runs in CI", "Test step removed from CI", commit));
|
|
3432
3688
|
}
|
|
3433
|
-
var LENIENT = /continue-on-error:\s*true|allow_failure:\s*true
|
|
3689
|
+
var LENIENT = /continue-on-error:\s*true|allow_failure:\s*true|^\s*(?:-\s+)?if:\s*(?:\$\{\{\s*)?false\s*(?:\}\})?\s*(?:#.*)?$/;
|
|
3690
|
+
var SHELL_SWALLOW = /\|\|\s*(?:true|exit\s+0|:|echo\b)|;\s*(?:true|exit\s+0)\b/;
|
|
3434
3691
|
function lenientSteps(text) {
|
|
3435
3692
|
const lines = text.split(/\r?\n/);
|
|
3436
3693
|
const indentOf = (line) => line.length - line.trimStart().length;
|
|
@@ -3438,8 +3695,15 @@ function lenientSteps(text) {
|
|
|
3438
3695
|
lines.forEach((line, i) => {
|
|
3439
3696
|
if (line.trim().startsWith("#") || !LENIENT.test(line)) return;
|
|
3440
3697
|
let j = i - 1;
|
|
3441
|
-
|
|
3442
|
-
|
|
3698
|
+
let identity;
|
|
3699
|
+
if (/^\s*-\s+if:/.test(line)) {
|
|
3700
|
+
j = i + 1;
|
|
3701
|
+
while (j < lines.length && (!lines[j].trim() || lines[j].trim().startsWith("#"))) j++;
|
|
3702
|
+
identity = j < lines.length ? lines[j].trim() : "";
|
|
3703
|
+
} else {
|
|
3704
|
+
while (j >= 0 && (!lines[j].trim() || lines[j].trim().startsWith("#") || indentOf(lines[j]) >= indentOf(line))) j--;
|
|
3705
|
+
identity = j >= 0 ? lines[j].trim() : "";
|
|
3706
|
+
}
|
|
3443
3707
|
const entry = found.get(identity);
|
|
3444
3708
|
if (entry) entry.count++;
|
|
3445
3709
|
else found.set(identity, { count: 1, line: i + 1 });
|
|
@@ -3453,22 +3717,25 @@ function analyzeGuardFile(file, oldText, newText, commit) {
|
|
|
3453
3717
|
if (oldText === null || newText === null) return [];
|
|
3454
3718
|
const out = [];
|
|
3455
3719
|
const add = (index, snippet, reason) => out.push(finding("guard-weakened", file, lineOf(newText, index), snippet, reason, commit));
|
|
3456
|
-
if (/threshold|
|
|
3720
|
+
if (/threshold|fail[_-]under|coverage/i.test(newText)) {
|
|
3457
3721
|
for (const pattern of THRESHOLD_PATTERNS) {
|
|
3458
3722
|
const old = new Map(numbers(oldText, pattern).map((item) => [item.key, item.value]));
|
|
3459
3723
|
for (const item of numbers(newText, pattern)) {
|
|
3460
3724
|
const previous = old.get(item.key);
|
|
3461
3725
|
if (previous !== void 0 && item.value < previous) {
|
|
3462
|
-
add(item.index, `${item.key.split("#")[0]}: ${previous} \u2192 ${item.value}`, "Coverage threshold lowered");
|
|
3726
|
+
add(item.index, `${item.key.split("#")[0]}: ${previous} \u2192 ${item.value}`, item.key.startsWith("fail-under#") ? "Lint score threshold lowered" : "Coverage threshold lowered");
|
|
3463
3727
|
}
|
|
3464
3728
|
}
|
|
3465
3729
|
}
|
|
3466
3730
|
}
|
|
3467
3731
|
if (/(?:^|\/)(?:tsconfig|jsconfig)[^/]*\.json$/i.test(file)) {
|
|
3732
|
+
const setting = (text, name) => new RegExp(`"${name}"\\s*:\\s*(true|false)`).exec(text);
|
|
3733
|
+
const strictBefore = setting(oldText, "strict")?.[1] === "true";
|
|
3468
3734
|
for (const name of SWITCHES) {
|
|
3469
|
-
const
|
|
3470
|
-
|
|
3471
|
-
|
|
3735
|
+
const before = setting(oldText, name), after = setting(newText, name);
|
|
3736
|
+
if (after?.[1] === "false" && (before?.[1] === "true" || before === null && strictBefore && name !== "strict")) {
|
|
3737
|
+
add(after.index, `${name}: ${before ? "true \u2192 false" : "false (strict stays on)"}`, "Compiler strictness switched off");
|
|
3738
|
+
}
|
|
3472
3739
|
}
|
|
3473
3740
|
}
|
|
3474
3741
|
if (/eslint/i.test(file)) {
|
|
@@ -3488,20 +3755,48 @@ function analyzeGuardFile(file, oldText, newText, commit) {
|
|
|
3488
3755
|
if (previous && previous.level === 2 && now.level < 2 && !replaced) add(now.index, `${name}: error \u2192 ${now.level ? "warn" : "off"}`, "Lint rule downgraded");
|
|
3489
3756
|
}
|
|
3490
3757
|
}
|
|
3758
|
+
if (/(?:^|\/)(?:pytest\.ini|tox\.ini|setup\.cfg|pyproject\.toml)$/i.test(file)) {
|
|
3759
|
+
const strictness = (text) => /^[ \t]*xfail_strict[ \t]*=[ \t]*(true|false|1|0|yes|no|on|off)\b/im.exec(text);
|
|
3760
|
+
const before = strictness(oldText), after = strictness(newText);
|
|
3761
|
+
if (before && after && /^(?:true|1|yes|on)$/i.test(before[1]) && /^(?:false|0|no|off)$/i.test(after[1])) {
|
|
3762
|
+
add(after.index, "xfail_strict: true \u2192 false", "Strict xfail switched off");
|
|
3763
|
+
}
|
|
3764
|
+
}
|
|
3765
|
+
if (/(?:^|\/)(?:pyproject\.toml|setup\.cfg|tox\.ini|\.?ruff\.toml|\.flake8|\.?mypy\.ini|\.?pylintrc)$/i.test(file)) {
|
|
3766
|
+
analyzePythonConfig(file, oldText, newText, (line, snippet, reason) => {
|
|
3767
|
+
out.push(finding("guard-weakened", file, line, snippet, reason, commit));
|
|
3768
|
+
});
|
|
3769
|
+
}
|
|
3491
3770
|
if (CI_FILE.test(file)) {
|
|
3492
|
-
const before = lenientSteps(oldText), existing = new Set(oldText.split(/\r?\n/).
|
|
3493
|
-
|
|
3494
|
-
|
|
3495
|
-
|
|
3771
|
+
const before = lenientSteps(oldText), existing = new Set(oldText.split(/\r?\n/).flatMap((line) => [line.trim(), line.trim().replace(/^-\s+/, "")]));
|
|
3772
|
+
const newLines = newText.split(/\r?\n/);
|
|
3773
|
+
for (const [identity, now2] of lenientSteps(newText)) {
|
|
3774
|
+
if (identity && existing.has(identity) && now2.count > (before.get(identity)?.count ?? 0)) {
|
|
3775
|
+
const marker = (newLines[now2.line - 1] ?? "").trim();
|
|
3776
|
+
out.push(finding("guard-weakened", file, now2.line, marker, /^(?:-\s+)?if:/.test(marker) ? "CI step is now switched off" : "CI step is now allowed to fail", commit));
|
|
3496
3777
|
break;
|
|
3497
3778
|
}
|
|
3498
3779
|
}
|
|
3780
|
+
const swallowing = (text) => text.split(/\r?\n/).map((line, index) => ({ line: line.trim(), index })).filter((item) => !item.line.startsWith("#") && CI_RUNNER.test(item.line) && SHELL_SWALLOW.test(item.line));
|
|
3781
|
+
const was = swallowing(oldText), now = swallowing(newText);
|
|
3782
|
+
if (now.length > was.length) {
|
|
3783
|
+
const known = new Set(was.map((item) => item.line));
|
|
3784
|
+
const fresh = now.find((item) => !known.has(item.line)) ?? now[0];
|
|
3785
|
+
out.push(finding("guard-weakened", file, fresh.index + 1, fresh.line.slice(0, 80), "Test command can no longer fail the CI step", commit));
|
|
3786
|
+
}
|
|
3499
3787
|
}
|
|
3500
3788
|
if (/(?:^|\/)package\.json$/i.test(file)) {
|
|
3501
3789
|
try {
|
|
3502
|
-
const before = JSON.parse(oldText)?.scripts
|
|
3503
|
-
|
|
3504
|
-
|
|
3790
|
+
const before = JSON.parse(oldText)?.scripts, after = JSON.parse(newText)?.scripts;
|
|
3791
|
+
const noop = (script) => /^\s*(?:echo\b|true\b|exit\s+0|:)/.test(script) || /\|\|\s*(?:true|exit\s+0)\b/.test(script);
|
|
3792
|
+
if (before && after) for (const name of Object.keys(after)) {
|
|
3793
|
+
const was = before[name], now = after[name];
|
|
3794
|
+
if (!GATE_SCRIPT.test(name) || typeof was !== "string" || typeof now !== "string" || was === now) continue;
|
|
3795
|
+
const tests = /^(?:pre|post)?test(?::|$)/.test(name);
|
|
3796
|
+
if (noop(now) && !noop(was) || tests && /--passWithNoTests/.test(now) && !/--passWithNoTests/.test(was)) {
|
|
3797
|
+
add(Math.max(0, newText.indexOf(`"${name}"`)), `${name}: ${now}`, tests ? "Test script can no longer fail" : `The ${name} script can no longer fail`);
|
|
3798
|
+
break;
|
|
3799
|
+
}
|
|
3505
3800
|
}
|
|
3506
3801
|
} catch {
|
|
3507
3802
|
}
|
|
@@ -3668,7 +3963,7 @@ async function compareSnapshot(options, cwd, snapshot) {
|
|
|
3668
3963
|
const fingerprint = findingFingerprint(finding2);
|
|
3669
3964
|
existingCounts.set(fingerprint, (existingCounts.get(fingerprint) ?? 0) + 1);
|
|
3670
3965
|
}
|
|
3671
|
-
result.infractions.push(...compared.introduced);
|
|
3966
|
+
result.infractions.push(...compared.introduced.filter((finding2) => !isKnownFailureMarker(finding2)), ...newKnownFailureMarkers(before, after, oldText, newText));
|
|
3672
3967
|
result.removedFindings.push(...compared.removed);
|
|
3673
3968
|
result.unchangedCount += compared.unchanged;
|
|
3674
3969
|
result.beforeFindings += before.length;
|
package/dist/cli.js
CHANGED
package/dist/index.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-roast",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.15.0",
|
|
4
4
|
"description": "Stops AI coding agents (and rushed humans) from turning a build green by weakening tests, coverage gates or CI. Hook for Claude Code, Codex and Cursor, pre-commit, GitHub Action. Offline.",
|
|
5
5
|
"mcpName": "io.github.Nari-Ab/agent-roast",
|
|
6
6
|
"type": "module",
|