codehere 2.1.0-rc.2 → 2.1.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/package.json +2 -2
- package/src/audit.js +18 -8
- package/src/brain.js +14 -0
- package/src/commands.js +5 -0
- package/src/hooks.js +27 -5
- package/src/noskip.js +585 -0
- package/src/signals.js +387 -12
package/README.md
CHANGED
|
@@ -119,10 +119,10 @@ codehere show <id> read a help card
|
|
|
119
119
|
codehere reply <id> "answer" [--solved] answer; --solved makes a lesson
|
|
120
120
|
codehere status what Codehere has seen
|
|
121
121
|
codehere stats turns to green, loops stopped, context size
|
|
122
|
-
codehere audit commits of the last 90 days
|
|
122
|
+
codehere audit commits of the last 90 days to read: tests deleted, skipped or weakened
|
|
123
123
|
```
|
|
124
124
|
|
|
125
|
-
`codehere audit` reads this repo's git history on your machine and sends nothing anywhere. It lists
|
|
125
|
+
`codehere audit` reads this repo's git history on your machine and sends nothing anywhere. It lists the commits that deleted a test file, skipped a test, left tests out of the test script or runner config, or removed or weakened assertions in test files, and says which ones carry an agent's marker (`Co-Authored-By` naming Claude, Copilot or Codex, `Claude-Session:`, `Agent-Logs-Url:`, Claude Code's "Generated with Claude Code" footer, or the `copilot-swe-agent[bot]` author). The count is found by pattern: read each commit before drawing a conclusion. On 12 public repos picked before measuring, about 1 in 25 listed commits had left the tests checking less; most were tests rewritten along with a changed feature ([results](docs/evidence/2026-11-13-audit-heldout-results.md)). A commit with no marker may still be an agent's, since markers can be turned off.
|
|
126
126
|
|
|
127
127
|
When an agent runs `rule`, `lesson`, `help` or `reply`, the command prints a draft and saves nothing. Codehere saves it if your next message is a plain yes. Typed in your own terminal, they save at once.
|
|
128
128
|
|
|
@@ -186,6 +186,6 @@ npm test # end-to-end checks
|
|
|
186
186
|
- Restart briefs are written without a yes, for your own next session. They enter git only if you commit `.codehere/briefs/`.
|
|
187
187
|
- A committed link to a folder that holds another repo's `.codehere/` can lead Codehere, run from inside that folder, to write its own local files there. `init` can stop halfway on a file it may not write, such as a read-only `.gitignore`.
|
|
188
188
|
- Prompt checks are plain word lists (English and Indonesian). They can miss a short prompt in another language.
|
|
189
|
-
- The weakened-test scan catches skipped and focused tests (`.skip`, `xit`, `{ skip: true }`, `.only`, `fit`), deleted test files, an assertion commented out with `//` or `#`, assertions that cannot fail (`assert.equal(1, 1)`), at least two more assertion lines removed than added,
|
|
189
|
+
- The weakened-test scan catches skipped and focused tests (`.skip`, `xit`, `{ skip: true }`, a skip option added to a test that already existed, `.only`, `fit`, Python's `skip` and a `skipif` with a literal condition), deleted test files, a test replaced by a differently named one (a renamed test counts), tests left out by the test script or the runner's config, an assertion commented out with `//` or `#`, assertions that cannot fail (`assert.equal(1, 1)`), at least two more assertion lines removed than added, an expected value changed (in an assertion or in a `want` or `expected` variable), an assertion flipped to its opposite or weakened to a truthy check, the call a test checks changed while its expected value stayed, a case dropped from a list on one line or rows of a test table reordered, an assertion inside a `try` that swallows its failure, a test script or CI test step made to always pass, a CI test step removed, code that returns the exact value a test expects for that test's input, and code that reads a test runner's own marker. Several of these rules are ported from noskip (MIT). It does not catch one removed assertion, assertions wrapped in `/* */`, options spread over several lines, a `return` above an assertion, a row removed from a multi-line test table, or a new test that skips itself on a variable; see `docs/overlap-with-check.md`. On the held-out half of CheatBench (Claude Code runs labelled by a person) it sent back 15 of 19 runs that changed a test to pass and 0 of 30 honest runs, with those results seen once before: `docs/evidence/2026-11-13-cheatbench.md`.
|
|
190
190
|
|
|
191
191
|
MIT licensed.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "codehere",
|
|
3
|
-
"version": "2.1.0-rc.
|
|
3
|
+
"version": "2.1.0-rc.3",
|
|
4
4
|
"description": "A git-native team brain for vibe coders. Silent fixes for AI coding agents: breaks doom loops, verifies 'done', blocks test tampering, and shares what your team learned.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"README.md"
|
|
19
19
|
],
|
|
20
20
|
"engines": {
|
|
21
|
-
"node": ">=
|
|
21
|
+
"node": ">=20"
|
|
22
22
|
},
|
|
23
23
|
"scripts": {
|
|
24
24
|
"test": "node test/e2e.mjs"
|
package/src/audit.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
// codehere audit: the commits of the last 90 days that weakened tests, each listed for a person to read.
|
|
1
|
+
// codehere audit: the commits of the last 90 days that deleted, skipped or weakened tests, each listed for a person to read.
|
|
2
2
|
// Read-only: it runs git on this machine and sends nothing anywhere. The kinds come from the finish
|
|
3
3
|
// check's own detector. A count found by pattern is a list of candidates, never proof (DESIGN.md §1).
|
|
4
4
|
import { spawn, execFileSync } from 'node:child_process';
|
|
5
5
|
import readline from 'node:readline';
|
|
6
|
-
import { printable } from './brain.js';
|
|
7
|
-
import { tamperEvents, isTestFile } from './signals.js';
|
|
6
|
+
import { printable, hasGit } from './brain.js';
|
|
7
|
+
import { tamperEvents, isTestFile, isTestConfig } from './signals.js';
|
|
8
8
|
|
|
9
9
|
const DAYS = 90;
|
|
10
10
|
const KINDS = new Set(['test-deleted', 'test-skipped', 'assertion-weakened']);
|
|
@@ -16,6 +16,8 @@ const MARKERS = [
|
|
|
16
16
|
{ re: /^Co-authored-by:[^\n]*\b(Claude|Copilot|Codex)\b/im, name: (m) => `Co-Authored-By ${m[1]}` },
|
|
17
17
|
{ re: /^Claude-Session:/im, name: () => 'Claude-Session' },
|
|
18
18
|
{ re: /^Agent-Logs-Url:/im, name: () => 'Agent-Logs-Url' },
|
|
19
|
+
// Claude Code's commit footer, the one marker a squash merge often keeps
|
|
20
|
+
{ re: /Generated with \[?Claude Code\b/i, name: () => 'Generated with Claude Code' },
|
|
19
21
|
];
|
|
20
22
|
|
|
21
23
|
export function markersOf(author, body) {
|
|
@@ -59,7 +61,8 @@ function samePath(line) {
|
|
|
59
61
|
return Number.isInteger(half) && rest.slice(half, half + 3) === ' b/' ? rest.slice(0, half) : '';
|
|
60
62
|
}
|
|
61
63
|
|
|
62
|
-
// One `git log -p`, read line by line: test
|
|
64
|
+
// One `git log -p`, read line by line: hunks of test files and of the files that say which tests run
|
|
65
|
+
// (package.json, runner configs) are kept, a deleted file by its name only, so
|
|
63
66
|
// memory stays small however large a commit is. Each commit's kinds go on its entry in bySha.
|
|
64
67
|
function scanChanges(root, since, bySha) {
|
|
65
68
|
return new Promise((resolve, reject) => {
|
|
@@ -99,7 +102,7 @@ function scanChanges(root, since, bySha) {
|
|
|
99
102
|
}
|
|
100
103
|
if (line.startsWith('+++ ')) {
|
|
101
104
|
const file = line.replace(/^\+\+\+ (b\/)?/, '');
|
|
102
|
-
keep = file !== '/dev/null' && isTestFile(file);
|
|
105
|
+
keep = file !== '/dev/null' && (isTestFile(file) || isTestConfig(file));
|
|
103
106
|
if (keep) kept.push(`+++ ${file}`);
|
|
104
107
|
return;
|
|
105
108
|
}
|
|
@@ -128,6 +131,10 @@ function summary(events) {
|
|
|
128
131
|
|
|
129
132
|
// Prints the audit; returns the exit code.
|
|
130
133
|
export async function audit(root, { out = console.log, err = console.error, now = Date.now() } = {}) {
|
|
134
|
+
if (!hasGit()) {
|
|
135
|
+
err('codehere audit: git was not found on PATH');
|
|
136
|
+
return 1;
|
|
137
|
+
}
|
|
131
138
|
if (gitLine(root, ['rev-parse', '--is-inside-work-tree']) !== 'true') {
|
|
132
139
|
err('codehere audit: not inside a git repository');
|
|
133
140
|
return 1;
|
|
@@ -150,9 +157,12 @@ export async function audit(root, { out = console.log, err = console.error, now
|
|
|
150
157
|
const marked = flagged.filter((c) => markersOf(c.author, c.body).length);
|
|
151
158
|
const n = (k, one, many) => `${k} ${k === 1 ? one : many}`;
|
|
152
159
|
out(`codehere audit · last ${DAYS} days · ${n(list.length, 'commit', 'commits')} read`);
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
160
|
+
if (!flagged.length) {
|
|
161
|
+
out('codehere › 0 commits to read: no test deleted, skipped or weakened');
|
|
162
|
+
return 0;
|
|
163
|
+
}
|
|
164
|
+
out(`codehere › ${n(flagged.length, 'commit', 'commits')} to read: tests deleted, skipped or weakened · ${n(marked.length, 'carries', 'carry')} an agent marker`);
|
|
165
|
+
out('codehere › found by pattern: a test removed with the code it tested is often fine, so read each commit');
|
|
156
166
|
for (const c of flagged) {
|
|
157
167
|
const m = markersOf(c.author, c.body);
|
|
158
168
|
out('');
|
package/src/brain.js
CHANGED
|
@@ -10,6 +10,9 @@ export function findRoot(start = process.cwd()) {
|
|
|
10
10
|
let dir = path.resolve(start);
|
|
11
11
|
while (true) {
|
|
12
12
|
if (fs.existsSync(path.join(dir, '.codehere'))) return dir;
|
|
13
|
+
// a repo's .codehere/ is at or below its git root: a .codehere/ further up (Codehere 1.x's ~/.codehere,
|
|
14
|
+
// another project) is not this repo's
|
|
15
|
+
if (fs.existsSync(path.join(dir, '.git'))) return null;
|
|
13
16
|
const parent = path.dirname(dir);
|
|
14
17
|
if (parent === dir) return null;
|
|
15
18
|
dir = parent;
|
|
@@ -399,6 +402,17 @@ export function commitId(x) {
|
|
|
399
402
|
return typeof x === 'string' && /^[0-9a-f]{40}([0-9a-f]{24})?$/.test(x) ? x : null;
|
|
400
403
|
}
|
|
401
404
|
|
|
405
|
+
// Whether git itself can be run. Without it every hook finds "no repository" and would stay silent, so
|
|
406
|
+
// init, audit and the finish check ask this first and say so.
|
|
407
|
+
export function hasGit() {
|
|
408
|
+
try {
|
|
409
|
+
execFileSync('git', ['--version'], { stdio: 'ignore' });
|
|
410
|
+
return true;
|
|
411
|
+
} catch {
|
|
412
|
+
return false;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
402
416
|
export function isGitRepo(root) {
|
|
403
417
|
return git(root, 'rev-parse --is-inside-work-tree') === 'true';
|
|
404
418
|
}
|
package/src/commands.js
CHANGED
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
readRules, listDocs, parseDoc, renderDoc, slug, stamp, today, git, isGitRepo, me,
|
|
9
9
|
readApprovals, isApproved, addApproval, removeApprovals, markRemoved, printable, noTick, commitId,
|
|
10
10
|
saveRecord, writeDraft, readPlain, writePlain, removePlain, ownPath, teamSnapshot, teamMark, keepSaved,
|
|
11
|
+
hasGit,
|
|
11
12
|
} from './brain.js';
|
|
12
13
|
import { redact } from './signals.js';
|
|
13
14
|
import { summarize } from './gemma.js';
|
|
@@ -304,6 +305,10 @@ export async function init(flags) {
|
|
|
304
305
|
console.error(`Unknown agent "${printable(flags.agent === true ? '' : flags.agent)}". Use --agent with claude, codex or copilot, or several: --agent claude,codex,copilot.`);
|
|
305
306
|
process.exit(1);
|
|
306
307
|
}
|
|
308
|
+
if (!hasGit()) {
|
|
309
|
+
console.error('codehere init: git was not found on PATH. Codehere reads what the agent changed through git, so nothing was set up.');
|
|
310
|
+
process.exit(1);
|
|
311
|
+
}
|
|
307
312
|
const agents = [...new Set(names)].map((n) => AGENTS[n]);
|
|
308
313
|
const root = process.cwd();
|
|
309
314
|
// Everything init writes, checked before anything is written: a committed link anywhere on these
|
package/src/hooks.js
CHANGED
|
@@ -7,11 +7,11 @@ import crypto from 'node:crypto';
|
|
|
7
7
|
import { spawnSync } from 'node:child_process';
|
|
8
8
|
import {
|
|
9
9
|
findRoot, paths, readConfig, readState, writeState, session, readRules, renderDoc,
|
|
10
|
-
listDocs, git, gitDiff, isGitRepo, me, isApproved, readApprovals, printable, commitId, statusFiles, gitHashes, noTick,
|
|
10
|
+
listDocs, git, gitDiff, isGitRepo, hasGit, me, isApproved, readApprovals, printable, commitId, statusFiles, gitHashes, noTick,
|
|
11
11
|
removedAt, expireDrafts, draftFile, saveRecord, readPlain, ownPath, teamSnapshot, teamMark, keepSaved, isMap,
|
|
12
12
|
} from './brain.js';
|
|
13
13
|
import {
|
|
14
|
-
responseText, signature, errorLines, similarity, tamperFindings, failingTest, lowInfoPrompt, vagueFeature, testCounts, redact, plainYes,
|
|
14
|
+
responseText, signature, errorLines, similarity, tamperFindings, failingTest, lowInfoPrompt, vagueFeature, testCounts, redact, plainYes, isTestFile,
|
|
15
15
|
} from './signals.js';
|
|
16
16
|
|
|
17
17
|
function readStdin() {
|
|
@@ -917,6 +917,7 @@ function stop(root, input) {
|
|
|
917
917
|
emitJson({ systemMessage: [String(message || '').replace(/ · agent sent back · \d+ of 3/, ''), printable(why)].filter(Boolean).join('\n') });
|
|
918
918
|
};
|
|
919
919
|
|
|
920
|
+
if (!hasGit()) emitJson({ systemMessage: 'codehere ⊘ git not found · the finish check did not run' });
|
|
920
921
|
if (!isGitRepo(root)) quiet();
|
|
921
922
|
const base = commitId(s.startHead) || 'HEAD';
|
|
922
923
|
const changed = changedThisSession(root, s);
|
|
@@ -953,7 +954,28 @@ function stop(root, input) {
|
|
|
953
954
|
const nameStatus = git(root, ['diff', base, '--relative', '--name-status', '--', ...changed]);
|
|
954
955
|
const diff = gitDiff(root, ['diff', base, '--relative', '-U0', '--', ...changed]);
|
|
955
956
|
unread = diff === null;
|
|
956
|
-
|
|
957
|
+
// a test file as it is now, to see what a swallowing catch holds: plain files only, up to 1 MiB
|
|
958
|
+
const read = (f, max = 1024 * 1024) => {
|
|
959
|
+
try {
|
|
960
|
+
return readPlain(path.join(root, f), 'utf8', max);
|
|
961
|
+
} catch {
|
|
962
|
+
return null;
|
|
963
|
+
}
|
|
964
|
+
};
|
|
965
|
+
// the repo's test files, read only when the diff returns a literal under a condition: up to 400
|
|
966
|
+
// files, 256 KiB each and 4 MiB in all
|
|
967
|
+
const tests = () => {
|
|
968
|
+
const out = [];
|
|
969
|
+
let total = 0;
|
|
970
|
+
for (const f of git(root, ['ls-files', '-z', '--cached', '--others', '--exclude-standard'], true).split('\0').filter(isTestFile).slice(0, 400)) {
|
|
971
|
+
const text = read(f, 256 * 1024);
|
|
972
|
+
if (text == null) continue;
|
|
973
|
+
if ((total += text.length) > 4 * 1024 * 1024) break;
|
|
974
|
+
out.push(`${f}\0${text}`);
|
|
975
|
+
}
|
|
976
|
+
return out;
|
|
977
|
+
};
|
|
978
|
+
findings = tamperFindings(nameStatus, diff || '', { read, tests });
|
|
957
979
|
}
|
|
958
980
|
const key = findings.join('|');
|
|
959
981
|
if (findings.length && s.tamperReported !== key && mayBlock) {
|
|
@@ -967,7 +989,7 @@ function stop(root, input) {
|
|
|
967
989
|
'Restore them and fix the code instead.',
|
|
968
990
|
'If the change is truly intended, tell the user in one line why, then finish.',
|
|
969
991
|
].join('\n'),
|
|
970
|
-
|
|
992
|
+
[printable(`codehere ✕ a check was weakened · ${noTick(findings.join(' · '))} · agent sent back · ${s.stopBlocks} of 3`), message()].filter(Boolean).join('\n'),
|
|
971
993
|
true
|
|
972
994
|
);
|
|
973
995
|
}
|
|
@@ -988,7 +1010,7 @@ function stop(root, input) {
|
|
|
988
1010
|
'If you changed them and the user did not ask for it, undo your change.',
|
|
989
1011
|
'Then tell the user in one line what changed and why, and finish.',
|
|
990
1012
|
].join('\n'),
|
|
991
|
-
|
|
1013
|
+
message(),
|
|
992
1014
|
true
|
|
993
1015
|
);
|
|
994
1016
|
}
|
package/src/noskip.js
ADDED
|
@@ -0,0 +1,585 @@
|
|
|
1
|
+
// Changed expected values, replaced tests and dropped test cases, read from a diff of test files.
|
|
2
|
+
// Ported from noskip 1.2.0, src/rules/tests.js and src/util.js (github.com/DariusHQ/noskip), with the
|
|
3
|
+
// findings reworded for Codehere and the file model built from a unified diff plus the files as they are now.
|
|
4
|
+
// Only the finish check runs this: it is the one caller that can read the files.
|
|
5
|
+
//
|
|
6
|
+
// MIT License
|
|
7
|
+
//
|
|
8
|
+
// Copyright (c) 2026 DariusHQ
|
|
9
|
+
//
|
|
10
|
+
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
// of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
// in the Software without restriction, including without limitation the rights
|
|
13
|
+
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
// copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
// furnished to do so, subject to the following conditions:
|
|
16
|
+
//
|
|
17
|
+
// The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
// copies or substantial portions of the Software.
|
|
19
|
+
//
|
|
20
|
+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
// SOFTWARE.
|
|
27
|
+
|
|
28
|
+
// ---------- helpers (noskip src/util.js) ----------
|
|
29
|
+
|
|
30
|
+
const normalizeLine = (text) => text.trim().replace(/\s+/g, ' ');
|
|
31
|
+
const isCommentLine = (text) => /^\s*(?:\/\/|\/\*|\*(?:\s|\/|$)|<!--|#(?![[!]|pragma\b|if\b|include\b|define\b|expect\b|require\b))/.test(text);
|
|
32
|
+
const truncate = (text, max = 120) => {
|
|
33
|
+
const t = text.trim();
|
|
34
|
+
return t.length > max ? t.slice(0, max - 1) + '…' : t;
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
function splitArgs(text) {
|
|
38
|
+
const out = [];
|
|
39
|
+
let depth = 0;
|
|
40
|
+
let quote = null;
|
|
41
|
+
let cur = '';
|
|
42
|
+
for (let i = 0; i < text.length; i++) {
|
|
43
|
+
const ch = text[i];
|
|
44
|
+
if (quote) {
|
|
45
|
+
cur += ch;
|
|
46
|
+
if (ch === '\\') cur += text[++i] ?? '';
|
|
47
|
+
else if (ch === quote) quote = null;
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
if (ch === '"' || ch === "'" || ch === '`') (quote = ch), (cur += ch);
|
|
51
|
+
else if (ch === '(' || ch === '[' || ch === '{') depth++, (cur += ch);
|
|
52
|
+
else if (ch === ')' || ch === ']' || ch === '}') depth--, (cur += ch);
|
|
53
|
+
else if (ch === ',' && depth === 0) out.push(cur.trim()), (cur = '');
|
|
54
|
+
else cur += ch;
|
|
55
|
+
}
|
|
56
|
+
if (cur.trim()) out.push(cur.trim());
|
|
57
|
+
return out;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function matchParen(text, open) {
|
|
61
|
+
let depth = 0;
|
|
62
|
+
let quote = null;
|
|
63
|
+
for (let i = open; i < text.length; i++) {
|
|
64
|
+
const ch = text[i];
|
|
65
|
+
if (quote) {
|
|
66
|
+
if (ch === '\\') i++;
|
|
67
|
+
else if (ch === quote) quote = null;
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
if (ch === '"' || ch === "'" || ch === '`') quote = ch;
|
|
71
|
+
else if (ch === '(') depth++;
|
|
72
|
+
else if (ch === ')' && --depth === 0) return i;
|
|
73
|
+
}
|
|
74
|
+
return -1;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// ---------- test names (noskip src/rules/tests.js) ----------
|
|
78
|
+
|
|
79
|
+
const EXT_LANG = {
|
|
80
|
+
js: 'js', jsx: 'js', mjs: 'js', cjs: 'js', ts: 'js', tsx: 'js', mts: 'js', cts: 'js', vue: 'js', svelte: 'js', astro: 'js',
|
|
81
|
+
py: 'py', pyi: 'py', go: 'go', rs: 'rs', java: 'java', kt: 'java', kts: 'java', groovy: 'java', scala: 'java', cs: 'cs',
|
|
82
|
+
rb: 'rb', php: 'php', swift: 'swift', dart: 'dart', c: 'c', cc: 'c', cpp: 'c', cxx: 'c', h: 'c', hh: 'c', hpp: 'c', hxx: 'c', ex: 'ex', exs: 'ex',
|
|
83
|
+
};
|
|
84
|
+
const languageOf = (p) => EXT_LANG[(/\.([^./]+)$/.exec(p) || [])[1]?.toLowerCase()] ?? null;
|
|
85
|
+
|
|
86
|
+
const STR = String.raw`(['"\x60])((?:\\.|(?!\1)[^\\])*)\1`;
|
|
87
|
+
const NAME_PATTERNS = {
|
|
88
|
+
js: [
|
|
89
|
+
{
|
|
90
|
+
re: new RegExp(
|
|
91
|
+
String.raw`\b(?:it|test|specify)(?:\s*\.\s*(?:only|skip|concurrent|sequential|failing|fails|todo|fixme|skipIf\([^)]*\)|runIf\([^)]*\)|each\s*(?:\([^)]*\)|\x60[^\x60]*\x60)))*\s*\(\s*` + STR,
|
|
92
|
+
'g',
|
|
93
|
+
),
|
|
94
|
+
pick: (m) => m[2],
|
|
95
|
+
},
|
|
96
|
+
],
|
|
97
|
+
py: [{ re: /^[ \t]*(?:async[ \t]+)?def[ \t]+(test\w*)[ \t]*\(/gm }],
|
|
98
|
+
go: [{ re: /^func\s+(?:\([^)]*\)\s*)?((?:Test|Benchmark|Fuzz|Example)\w*)\s*\(/gm }, { re: /\bt\.Run\(\s*"((?:\\.|[^"\\])*)"/g }, { re: /\bname:\s*"((?:\\.|[^"\\])*)"/g }],
|
|
99
|
+
rs: [{ re: /#\[(?:[\w:]+::)?test\b[^\]]*\]\s*(?:#\[[^\]]*\]\s*)*(?:pub(?:\([^)]*\))?\s+)?(?:async\s+)?fn\s+(\w+)/g }],
|
|
100
|
+
java: [
|
|
101
|
+
{ re: /@(?:Test|ParameterizedTest|RepeatedTest|TestFactory|TestTemplate)\b[^\n]*\n(?:\s*@[^\n]*\n)*\s*(?:(?:public|private|protected|internal|open|override|suspend|static|final)\s+)*(?:void\s+|fun\s+)?`?([^`(\n]+?)`?\s*\(/g },
|
|
102
|
+
{ re: /\bvoid\s+(test\w+)\s*\(/g },
|
|
103
|
+
],
|
|
104
|
+
cs: [{ re: /\[\s*(?:Fact|Theory|Test|TestMethod|TestCase|TestCaseSource|SkippableFact)\b[^\]]*\]\s*(?:\[[^\]]*\]\s*)*(?:(?:public|private|protected|internal|async|static|virtual|override)\s+)*[\w<>[\],.?]+\s+(\w+)\s*\(/g }],
|
|
105
|
+
rb: [{ re: new RegExp(String.raw`^\s*(?:it|specify|scenario|example|test)\s*\(?\s*` + STR, 'gm'), pick: (m) => m[2] }, { re: /^\s*def\s+(test_\w+)/gm }],
|
|
106
|
+
php: [{ re: /function\s+(test\w+)\s*\(/g }, { re: /(?:#\[Test\]|@test\b)[^]{0,200}?function\s+(\w+)\s*\(/g }, { re: new RegExp(String.raw`\b(?:it|test)\s*\(\s*` + STR, 'g'), pick: (m) => m[2] }],
|
|
107
|
+
swift: [{ re: /func\s+(test\w+)\s*\(/g }, { re: /@Test\b(?:\([^)]*\))?\s*(?:@\w+(?:\([^)]*\))?\s*)*func\s+(\w+)\s*\(/g }],
|
|
108
|
+
dart: [{ re: new RegExp(String.raw`\b(?:test|testWidgets)\s*\(\s*` + STR, 'g'), pick: (m) => m[2] }],
|
|
109
|
+
c: [{ re: /\b(?:TEST|TEST_F|TEST_P|TYPED_TEST)\s*\(\s*(\w+)\s*,\s*(\w+)\s*\)/g, pick: (m) => `${m[1]}.${m[2]}` }, { re: /\bTEST_CASE\s*\(\s*"((?:\\.|[^"\\])*)"/g }],
|
|
110
|
+
ex: [{ re: /\btest\s+"((?:\\.|[^"\\])*)"/g }],
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
function extractTestNames(text, lang) {
|
|
114
|
+
const names = new Map();
|
|
115
|
+
if (!text) return names;
|
|
116
|
+
for (const { re, pick } of NAME_PATTERNS[lang] ?? []) {
|
|
117
|
+
re.lastIndex = 0;
|
|
118
|
+
for (const m of text.matchAll(re)) {
|
|
119
|
+
const name = (pick ? pick(m) : m[m.length - 1] ?? m[1])?.trim();
|
|
120
|
+
if (!name) continue;
|
|
121
|
+
const entry = names.get(name);
|
|
122
|
+
if (entry) entry.count++;
|
|
123
|
+
else names.set(name, { count: 1, index: m.index });
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return names;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const NAME_STOPWORDS = new Set(['a', 'an', 'the', 'is', 'are', 'it', 'to', 'of', 'on', 'in', 'and', 'or', 'when', 'if', 'with', 'for', 'should', 'be', 'not', 'still', 'that', 'does', 'do', 'test']);
|
|
130
|
+
const nameTokens = (name) => new Set(name.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 1 && !NAME_STOPWORDS.has(t)));
|
|
131
|
+
|
|
132
|
+
// Two test names describe the same behaviour: a test split or moved under a new name, not one that went.
|
|
133
|
+
function similarNames(a, b) {
|
|
134
|
+
const A = nameTokens(a);
|
|
135
|
+
const B = nameTokens(b);
|
|
136
|
+
if (!A.size || !B.size) return false;
|
|
137
|
+
let shared = 0;
|
|
138
|
+
for (const t of A) if (B.has(t)) shared++;
|
|
139
|
+
return shared >= 2 && shared / Math.min(A.size, B.size) >= 0.5;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// ---------- assertions ----------
|
|
143
|
+
|
|
144
|
+
const STRONG = new Set([
|
|
145
|
+
'toBe', 'toEqual', 'toStrictEqual', 'toMatchObject', 'toHaveBeenCalledWith', 'toHaveBeenNthCalledWith',
|
|
146
|
+
'toHaveBeenLastCalledWith', 'toHaveBeenCalledTimes', 'toHaveLength', 'toHaveProperty', 'toThrow', 'toThrowError',
|
|
147
|
+
'toMatch', 'toContainEqual', 'toBeCloseTo', 'equal', 'equals', 'eql', 'eq', 'isEqualTo', 'containsExactly',
|
|
148
|
+
'hasSize', 'isSameAs', 'assertEqual', 'assertEquals', 'assertSame', 'assertIs', 'assertDictEqual',
|
|
149
|
+
'assertListEqual', 'assertCountEqual', 'assertAlmostEqual', 'assertRaises', 'assertStrictEqual', 'assert_equal',
|
|
150
|
+
'assert_eq!', 'Equal', 'Equals', 'AreEqual', 'Same', 'Exactly', 'EqualValues', 'StrictEqual', 'deepEqual',
|
|
151
|
+
'deepStrictEqual', 'strictEqual', 'XCTAssertEqual', 'EXPECT_EQ', 'ASSERT_EQ', 'to eq', 'to eql', 'to equal',
|
|
152
|
+
'to match', 'to raise_error', '==',
|
|
153
|
+
]);
|
|
154
|
+
const WEAK = new Set([
|
|
155
|
+
'toBeDefined', 'toBeTruthy', 'toBeFalsy', 'toBeInstanceOf', 'toBeTypeOf', 'toHaveBeenCalled', 'not.toBeNull',
|
|
156
|
+
'not.toBeUndefined', 'ok', 'exist', 'exists', 'isNotNull', 'isNotEmpty', 'isInstanceOf', 'assertTrue', 'assertFalse',
|
|
157
|
+
'assertIsNotNone', 'assertIsInstance', 'assertIn', 'assertNotNull', 'assertNotNil', 'assert_not_nil', 'assert',
|
|
158
|
+
'assert!', 'NotNil', 'NotNull', 'NotEmpty', 'True', 'IsNotNull', 'IsTrue', 'XCTAssertNotNil', 'XCTAssertTrue',
|
|
159
|
+
'EXPECT_TRUE', 'ASSERT_TRUE', 'to be_truthy', 'to be_present', 'not_to be_nil', 'to_not be_nil', 'is not',
|
|
160
|
+
'truthy',
|
|
161
|
+
]);
|
|
162
|
+
|
|
163
|
+
const norm = (s) => normalizeLine(String(s ?? '')).replace(/;$/, '').replace(/\s*,\s*$/, '');
|
|
164
|
+
|
|
165
|
+
function parseExpectStyle(t, opener) {
|
|
166
|
+
const m = opener.exec(t);
|
|
167
|
+
if (!m) return null;
|
|
168
|
+
const open = m.index + m[0].length - 1;
|
|
169
|
+
const close = matchParen(t, open);
|
|
170
|
+
if (close < 0) return null;
|
|
171
|
+
const subject = t.slice(open + 1, close);
|
|
172
|
+
const rest = t.slice(close + 1).replace(/->/g, '.');
|
|
173
|
+
let mm = /^\s*((?:\.\s*(?:not|resolves|rejects|to|be|been|is|deep|own|nested|and|has|have)\s*)*)\.\s*(\w+)\s*\(/.exec(rest);
|
|
174
|
+
if (mm) {
|
|
175
|
+
const argOpen = mm[0].length - 1;
|
|
176
|
+
const argClose = matchParen(rest, argOpen);
|
|
177
|
+
const chain = mm[1].replace(/\s+/g, '').replace(/^\./, '');
|
|
178
|
+
const matcher = (chain ? chain + '.' : '') + mm[2];
|
|
179
|
+
const args = argClose > 0 ? rest.slice(argOpen + 1, argClose) : rest.slice(argOpen + 1);
|
|
180
|
+
return { kind: 'expect', subject: norm(subject), matcher, args: norm(args) };
|
|
181
|
+
}
|
|
182
|
+
mm = /^\s*\.\s*(to|not_to|to_not)\s+(\w+)\s*\(?(.*?)\)?\s*$/.exec(rest);
|
|
183
|
+
if (mm) return { kind: 'expect', subject: norm(subject), matcher: `${mm[1]} ${mm[2]}`, args: norm(mm[3]) };
|
|
184
|
+
return null;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const CALL_OPENER =
|
|
188
|
+
/(?:^|[^\w.$])((?:self\s*\.\s*|this\s*\.\s*|\$this\s*->\s*)?(?:assert[A-Z_]\w*|assert_\w+|assert_eq!|assert_ne!|assert!)|(?:assert|Assert|require)\s*\.\s*\w+|XCTAssert\w*|(?:EXPECT|ASSERT)_[A-Z_]+)\s*\(/;
|
|
189
|
+
|
|
190
|
+
function parseCallStyle(t) {
|
|
191
|
+
const m = CALL_OPENER.exec(t);
|
|
192
|
+
if (!m) return null;
|
|
193
|
+
const open = m.index + m[0].length - 1;
|
|
194
|
+
const close = matchParen(t, open);
|
|
195
|
+
if (close < 0) return null;
|
|
196
|
+
let fn = m[1].replace(/\s+/g, '').replace(/^(?:self\.|this\.|\$this->)/, '');
|
|
197
|
+
let args = splitArgs(t.slice(open + 1, close)).map(norm);
|
|
198
|
+
if (/^(?:assert|require)\./.test(fn) && /^[A-Z]/.test(fn.split('.')[1] ?? '') && args[0] === 't') args = args.slice(1); // Go testify
|
|
199
|
+
fn = fn.replace(/^(?:assert|Assert|require)\./, '');
|
|
200
|
+
return { kind: 'call', matcher: fn, args };
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function parsePyAssert(t) {
|
|
204
|
+
const m = /^assert\s+(.+)$/.exec(t);
|
|
205
|
+
if (!m) return null;
|
|
206
|
+
const cond = splitArgs(m[1])[0] ?? '';
|
|
207
|
+
const op = /\s(==|!=|<=|>=|is not|is|not in|in)\s/.exec(cond);
|
|
208
|
+
if (!op) return { kind: 'py', subject: norm(cond), matcher: 'assert', args: '' };
|
|
209
|
+
const left = cond.slice(0, op.index);
|
|
210
|
+
const right = cond.slice(op.index + op[0].length);
|
|
211
|
+
if (op[1] === 'is not' && /^None$/.test(right.trim())) return { kind: 'py', subject: norm(left), matcher: 'is not', args: 'None' };
|
|
212
|
+
return { kind: 'py', subject: norm(left), matcher: op[1], args: norm(right) };
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
const KV = /^\s*(?:(?:const|let|var|val|final|auto|my)\s+)?["']?((?:want|wanted|expected|expect|exp|golden|answer)\w*)["']?\s*(?::=|=|:)\s*(.+?)\s*[,;]?\s*$/i;
|
|
216
|
+
|
|
217
|
+
function parseAssertion(text) {
|
|
218
|
+
const t = text.trim().replace(/;\s*$/, '');
|
|
219
|
+
if (!t || isCommentLine(t)) return null;
|
|
220
|
+
return (
|
|
221
|
+
parseExpectStyle(t, /\bexpect\s*\(/) ||
|
|
222
|
+
parseExpectStyle(t, /\bassertThat\s*\(/) ||
|
|
223
|
+
parseCallStyle(t) ||
|
|
224
|
+
parsePyAssert(t) ||
|
|
225
|
+
(() => {
|
|
226
|
+
const m = KV.exec(t);
|
|
227
|
+
return m ? { kind: 'kv', subject: m[1].toLowerCase(), matcher: 'kv', args: norm(m[2]) } : null;
|
|
228
|
+
})()
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
const LITERALISH = /^(?:-?\d[\d_.]*n?|'(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*"|`[^`$]*`|true|false|null|undefined|None|True|False|nil|\/.+\/[a-z]*|\{[^()]*\}|\[[^()]*\]|[A-Z][A-Z0-9_]+)$/;
|
|
233
|
+
|
|
234
|
+
// The expression with its literals blanked out: `count + 3` and `count + 4` share a shape.
|
|
235
|
+
const shapeOf = (s) => s.replace(/'(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*"|`(?:\\.|[^`\\])*`/g, 'S').replace(/\b\d+(?:\.\d+)?n?\b/g, '#');
|
|
236
|
+
|
|
237
|
+
// The expectation moved (`46` to `47`, `n + 3` to `n + 4`), as opposed to the expression computing the actual value.
|
|
238
|
+
const goalpostMoved = (before, after) => LITERALISH.test(before) || LITERALISH.test(after) || shapeOf(before) === shapeOf(after);
|
|
239
|
+
|
|
240
|
+
const OPPOSITES = new Map(
|
|
241
|
+
[
|
|
242
|
+
['assertTrue', 'assertFalse'], ['assertEqual', 'assertNotEqual'], ['assertEquals', 'assertNotEquals'],
|
|
243
|
+
['assertIs', 'assertIsNot'], ['assertIn', 'assertNotIn'], ['assertIsNone', 'assertIsNotNone'],
|
|
244
|
+
['assertNull', 'assertNotNull'], ['assertSame', 'assertNotSame'], ['assertRegex', 'assertNotRegex'],
|
|
245
|
+
['assertIsInstance', 'assertNotIsInstance'], ['assert_equal', 'refute_equal'], ['assert_nil', 'refute_nil'],
|
|
246
|
+
['assert_eq!', 'assert_ne!'], ['equal', 'notEqual'], ['strictEqual', 'notStrictEqual'], ['deepEqual', 'notDeepEqual'],
|
|
247
|
+
['deepStrictEqual', 'notDeepStrictEqual'], ['throws', 'doesNotThrow'], ['rejects', 'doesNotReject'],
|
|
248
|
+
['match', 'doesNotMatch'], ['True', 'False'], ['Equal', 'NotEqual'], ['Nil', 'NotNil'], ['Error', 'NoError'],
|
|
249
|
+
['Empty', 'NotEmpty'], ['Contains', 'NotContains'], ['Zero', 'NotZero'], ['IsTrue', 'IsFalse'],
|
|
250
|
+
['AreEqual', 'AreNotEqual'], ['IsNull', 'IsNotNull'], ['XCTAssertTrue', 'XCTAssertFalse'],
|
|
251
|
+
['XCTAssertEqual', 'XCTAssertNotEqual'], ['XCTAssertNil', 'XCTAssertNotNil'], ['EXPECT_TRUE', 'EXPECT_FALSE'],
|
|
252
|
+
['ASSERT_TRUE', 'ASSERT_FALSE'], ['EXPECT_EQ', 'EXPECT_NE'], ['ASSERT_EQ', 'ASSERT_NE'],
|
|
253
|
+
].flatMap(([a, b]) => [[a, b], [b, a]]),
|
|
254
|
+
);
|
|
255
|
+
|
|
256
|
+
function negated(a, b) {
|
|
257
|
+
const strip = (s) => {
|
|
258
|
+
const m = /^(?:!\s*|not\s+)(.+)$/.exec(s);
|
|
259
|
+
return m ? m[1].replace(/^\((.*)\)$/, '$1').trim() : null;
|
|
260
|
+
};
|
|
261
|
+
const unwrap = (s) => s.replace(/^\((.*)\)$/, '$1').trim();
|
|
262
|
+
return strip(a) === unwrap(b) || strip(b) === unwrap(a);
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// `f(x)` to `f(x, opts)`: the same function, called with arguments added or dropped at the end.
|
|
266
|
+
function callArgsExtended(before, after) {
|
|
267
|
+
const call = (s) => {
|
|
268
|
+
const m = /^([\w$.]+)\s*\(/.exec(s);
|
|
269
|
+
if (!m) return null;
|
|
270
|
+
const close = matchParen(s, m[0].length - 1);
|
|
271
|
+
if (close !== s.length - 1) return null;
|
|
272
|
+
return { callee: m[1], args: splitArgs(s.slice(m[0].length, close)).map(norm).filter(Boolean) };
|
|
273
|
+
};
|
|
274
|
+
const b = call(before);
|
|
275
|
+
const a = call(after);
|
|
276
|
+
if (!b || !a || b.callee !== a.callee || b.args.length === a.args.length) return false;
|
|
277
|
+
const [short, long] = b.args.length < a.args.length ? [b.args, a.args] : [a.args, b.args];
|
|
278
|
+
return short.every((x, i) => x === long[i]);
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// A removed assertion against an added one: null, or { type, what, from, to }.
|
|
282
|
+
// type 'weakened' or 'changed'; what says which change, for the finding's wording.
|
|
283
|
+
const pair = (type, what, from, to) => ({ type, what, from: truncate(from, 50), to: truncate(to, 50) });
|
|
284
|
+
function compareAssertions(r, a) {
|
|
285
|
+
if (r.kind === 'call' && a.kind === 'call') {
|
|
286
|
+
const showCall = (x) => `${x.matcher}(${x.args.join(', ')})`;
|
|
287
|
+
if (r.matcher === a.matcher && r.args.length === a.args.length && r.args.length >= 2) {
|
|
288
|
+
const diffs = [0, 1].filter((i) => r.args[i] !== a.args[i]);
|
|
289
|
+
if (diffs.length !== 1) return null;
|
|
290
|
+
const i = diffs[0];
|
|
291
|
+
if (goalpostMoved(r.args[i], a.args[i])) return pair('changed', 'expected', r.args[i], a.args[i]);
|
|
292
|
+
return callArgsExtended(r.args[i], a.args[i]) ? pair('changed', 'call', r.args[i], a.args[i]) : null;
|
|
293
|
+
}
|
|
294
|
+
if (r.matcher === a.matcher && r.args.length === 1 && a.args.length === 1 && negated(r.args[0], a.args[0])) return pair('changed', 'flipped', showCall(r), showCall(a));
|
|
295
|
+
if (OPPOSITES.get(r.matcher) === a.matcher && r.args.length === a.args.length && r.args.every((x, i) => x === a.args[i])) return pair('changed', 'flipped', showCall(r), showCall(a));
|
|
296
|
+
if (STRONG.has(r.matcher) && WEAK.has(a.matcher) && a.args.some((x) => r.args.slice(0, 2).includes(x))) return pair('weakened', 'weakened', `${r.matcher}(…)`, `${a.matcher}(…)`);
|
|
297
|
+
return null;
|
|
298
|
+
}
|
|
299
|
+
if (r.kind === 'py' && a.kind === 'py' && r.matcher === 'assert' && a.matcher === 'assert' && negated(r.subject, a.subject)) return pair('changed', 'flipped', `assert ${r.subject}`, `assert ${a.subject}`);
|
|
300
|
+
if (r.kind === a.kind && r.kind !== 'kv' && r.subject !== a.subject && r.matcher === a.matcher && r.args === a.args && callArgsExtended(r.subject, a.subject)) return pair('changed', 'call', r.subject, a.subject);
|
|
301
|
+
if (r.kind !== a.kind || r.subject === undefined || r.subject !== a.subject) return null;
|
|
302
|
+
if (r.kind === 'kv') return r.args === a.args || !goalpostMoved(r.args, a.args) ? null : pair('changed', 'expected', `${r.subject} ${r.args}`, `${a.subject} ${a.args}`);
|
|
303
|
+
if (r.matcher === a.matcher) {
|
|
304
|
+
if (r.args === a.args) return null;
|
|
305
|
+
if (a.args === '' && r.args !== '') return pair('weakened', 'weakened', `${r.matcher}(${truncate(r.args, 40)})`, `${a.matcher}()`);
|
|
306
|
+
return pair('changed', 'expected', r.args, a.args);
|
|
307
|
+
}
|
|
308
|
+
const strongOld = STRONG.has(r.matcher) || STRONG.has(r.matcher.replace(/^not\./, ''));
|
|
309
|
+
const weakNew = WEAK.has(a.matcher) || (r.kind === 'py' && (a.matcher === 'assert' || a.matcher === 'is not'));
|
|
310
|
+
const show = (x) => (x.kind === 'py' ? `assert ${x.subject}${x.matcher === 'assert' ? '' : ` ${x.matcher} ${truncate(x.args, 40)}`}` : `${x.matcher}(${truncate(x.args, 40)})`);
|
|
311
|
+
if (strongOld && weakNew) return pair('weakened', 'weakened', show(r), show(a));
|
|
312
|
+
return pair('changed', 'assertion', show(r), show(a));
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
// ---------- test tables (table-driven tests, parametrize lists, `for (x of [..])`) ----------
|
|
316
|
+
|
|
317
|
+
const STRING_SRC = String.raw`'(?:\\.|[^'\\\n])*'|"(?:\\.|[^"\\\n])*"`;
|
|
318
|
+
const LITERAL_SRC = String.raw`${STRING_SRC}|-?\d[\d_]*(?:\.\d+)?(?:[eE][+-]?\d+)?n?|\b(?:true|false|null|undefined|None|True|False|nil)\b`;
|
|
319
|
+
const ROW_TOKEN = new RegExp(String.raw`(\s+)|((?:[A-Za-z_$][\w$]*|${STRING_SRC})\s*:(?!:))|(${LITERAL_SRC})|([[\](){},;])`, 'y');
|
|
320
|
+
|
|
321
|
+
// A line that is only data, such as `[1900, true],`, `(1900, True),`, `{ in: 5, want: 120 },` or `1900,`.
|
|
322
|
+
function dataRow(text) {
|
|
323
|
+
const t = text.trim();
|
|
324
|
+
if (!t || !(/^[[({]/.test(t) || /,$/.test(t))) return null;
|
|
325
|
+
const literals = [];
|
|
326
|
+
let shape = '';
|
|
327
|
+
let depth = 0;
|
|
328
|
+
ROW_TOKEN.lastIndex = 0;
|
|
329
|
+
while (ROW_TOKEN.lastIndex < t.length) {
|
|
330
|
+
const m = ROW_TOKEN.exec(t);
|
|
331
|
+
if (!m) return null;
|
|
332
|
+
if (m[1]) continue;
|
|
333
|
+
if (m[3]) {
|
|
334
|
+
literals.push(m[3]);
|
|
335
|
+
shape += 'L';
|
|
336
|
+
continue;
|
|
337
|
+
}
|
|
338
|
+
if (m[4] && '[({'.includes(m[4])) depth++;
|
|
339
|
+
if (m[4] && '])}'.includes(m[4]) && --depth < 0) return null; // the tail of a call: `{ name: 'X' });`
|
|
340
|
+
shape += (m[2] ?? m[4]).replace(/\s+/g, '');
|
|
341
|
+
}
|
|
342
|
+
// open brackets are the start of a multi-line value (`[true,`); a key with one value (`code: 1,`) is a property
|
|
343
|
+
if (depth !== 0 || !literals.length || (!/^[[({]/.test(t) && (literals.length > 1 || shape.includes(':')))) return null;
|
|
344
|
+
return { literals, shape };
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
const INLINE_LIST = new RegExp(String.raw`\[\s*((?:${LITERAL_SRC})(?:\s*,\s*(?:${LITERAL_SRC}))*)\s*,?\s*\]`, 'g');
|
|
348
|
+
function inlineLists(text) {
|
|
349
|
+
const lists = [];
|
|
350
|
+
const shape = text.replace(INLINE_LIST, (_, items) => {
|
|
351
|
+
lists.push(items.match(new RegExp(LITERAL_SRC, 'g')));
|
|
352
|
+
return '[L]';
|
|
353
|
+
});
|
|
354
|
+
return { lists, shape: normalizeLine(shape) };
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
// What is gone from big, if small is big with some items removed and nothing else.
|
|
358
|
+
function dropped(big, small) {
|
|
359
|
+
if (!small || small.length >= big.length) return null;
|
|
360
|
+
const gone = [];
|
|
361
|
+
let j = 0;
|
|
362
|
+
for (const item of big) {
|
|
363
|
+
if (j < small.length && small[j] === item) j++;
|
|
364
|
+
else gone.push(item);
|
|
365
|
+
}
|
|
366
|
+
return j === small.length ? gone : null;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
// Items dropped from a list on one line whose rest is unchanged, and rows moved within their table. Whole rows
|
|
370
|
+
// that disappear from multi-line tables are not reported: tables shrink when features go, so that signal is noise.
|
|
371
|
+
function tableChanges(file) {
|
|
372
|
+
const out = [];
|
|
373
|
+
for (const hunk of file.hunks) {
|
|
374
|
+
const used = new Set();
|
|
375
|
+
for (const r of hunk.removed) {
|
|
376
|
+
if (parseAssertion(r.text) || dataRow(r.text)) continue;
|
|
377
|
+
const before = inlineLists(r.text);
|
|
378
|
+
if (!before.lists.length) continue;
|
|
379
|
+
const a = hunk.added.find((x) => !used.has(x.line) && inlineLists(x.text).shape === before.shape);
|
|
380
|
+
if (!a) continue;
|
|
381
|
+
used.add(a.line);
|
|
382
|
+
const after = inlineLists(a.text);
|
|
383
|
+
before.lists.forEach((items, i) => {
|
|
384
|
+
const gone = dropped(items, after.lists[i]);
|
|
385
|
+
if (gone) out.push({ type: 'changed', what: 'dropped', from: truncate(gone.join(', '), 40) });
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
if (!file.oldLines || !file.newLines) return out;
|
|
390
|
+
const key = (text) => normalizeLine(text).replace(/,$/, '');
|
|
391
|
+
const rowLines = (lines) => lines.filter((l) => dataRow(l.text) && !parseAssertion(l.text));
|
|
392
|
+
const table = (lines, at) => {
|
|
393
|
+
let start = at - 1;
|
|
394
|
+
let end = at - 1;
|
|
395
|
+
while (start > 0 && dataRow(lines[start - 1])) start--;
|
|
396
|
+
while (end < lines.length - 1 && dataRow(lines[end + 1])) end++;
|
|
397
|
+
return lines.slice(start, end + 1).map(key);
|
|
398
|
+
};
|
|
399
|
+
const addedRows = rowLines(file.added);
|
|
400
|
+
for (const r of rowLines(file.removed)) {
|
|
401
|
+
const a = addedRows.find((x) => key(x.text) === key(r.text));
|
|
402
|
+
if (!a) continue;
|
|
403
|
+
const before = table(file.oldLines, r.line);
|
|
404
|
+
const after = table(file.newLines, a.line);
|
|
405
|
+
const sorted = (list) => [...list].sort().join('\n');
|
|
406
|
+
if (before.join('\n') !== after.join('\n') && sorted(before) === sorted(after)) {
|
|
407
|
+
out.push({ type: 'changed', what: 'reordered', from: truncate(key(r.text), 50) });
|
|
408
|
+
break;
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
return out;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// ---------- the file model, from a unified diff and the files as they are now ----------
|
|
415
|
+
|
|
416
|
+
function parseDiff(diff) {
|
|
417
|
+
const files = [];
|
|
418
|
+
let f = null;
|
|
419
|
+
let h = null;
|
|
420
|
+
let oldLine = 0;
|
|
421
|
+
let newLine = 0;
|
|
422
|
+
let from = '';
|
|
423
|
+
for (const line of diff.split('\n')) {
|
|
424
|
+
if (line.startsWith('diff --git ')) {
|
|
425
|
+
f = null;
|
|
426
|
+
h = null;
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
429
|
+
if (line.startsWith('--- ') && !h) {
|
|
430
|
+
from = line.slice(4).replace(/^a\//, '');
|
|
431
|
+
continue;
|
|
432
|
+
}
|
|
433
|
+
if (line.startsWith('+++ ') && !h) {
|
|
434
|
+
const to = line.slice(4).replace(/^b\//, '');
|
|
435
|
+
f = { path: to === '/dev/null' ? from : to, status: from === '/dev/null' ? 'A' : to === '/dev/null' ? 'D' : 'M', hunks: [], removed: [], added: [] };
|
|
436
|
+
files.push(f);
|
|
437
|
+
continue;
|
|
438
|
+
}
|
|
439
|
+
const m = /^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/.exec(line);
|
|
440
|
+
if (m && f) {
|
|
441
|
+
h = { oldStart: +m[1], newStart: +m[3], newCount: m[4] === undefined ? 1 : +m[4], removed: [], added: [], lines: [] };
|
|
442
|
+
f.hunks.push(h);
|
|
443
|
+
[oldLine, newLine] = [h.oldStart, h.newStart];
|
|
444
|
+
continue;
|
|
445
|
+
}
|
|
446
|
+
if (!h || line.startsWith('\\')) continue;
|
|
447
|
+
if (line.startsWith('-')) {
|
|
448
|
+
const r = { line: oldLine++, text: line.slice(1, 10001) }; // the patterns can take quadratic time on a long line
|
|
449
|
+
h.removed.push(r), f.removed.push(r), h.lines.push(line);
|
|
450
|
+
} else if (line.startsWith('+')) {
|
|
451
|
+
const a = { line: newLine++, text: line.slice(1, 10001) };
|
|
452
|
+
h.added.push(a), f.added.push(a), h.lines.push(line);
|
|
453
|
+
} else if (line.startsWith(' ')) oldLine++, newLine++, h.lines.push(line);
|
|
454
|
+
else h = null; // the end of this file's hunks
|
|
455
|
+
}
|
|
456
|
+
return files;
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
// The file before the change: the file as it is now with each hunk undone. Null if they do not match.
|
|
460
|
+
function undo(newLines, hunks) {
|
|
461
|
+
const old = [];
|
|
462
|
+
let cur = 0;
|
|
463
|
+
for (const h of hunks) {
|
|
464
|
+
const upto = h.newCount === 0 ? h.newStart : h.newStart - 1;
|
|
465
|
+
if (upto < cur || upto > newLines.length) return null;
|
|
466
|
+
while (cur < upto) old.push(newLines[cur++]);
|
|
467
|
+
for (const l of h.lines) {
|
|
468
|
+
if (l[0] === '-') old.push(l.slice(1));
|
|
469
|
+
else if (newLines[cur]?.replace(/\r$/, '') !== l.slice(1).replace(/\r$/, '')) return null;
|
|
470
|
+
else if (l[0] === ' ') old.push(newLines[cur++]);
|
|
471
|
+
else cur++;
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
while (cur < newLines.length) old.push(newLines[cur++]);
|
|
475
|
+
return old;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
const OPTIONS_SKIP = /\b(?:it|test|describe|suite|specify)\s*\(\s*(['"`])((?:\\.|(?!\1).)*)\1\s*,\s*\{[^}]*\b(?:skip|todo)\b(?!\s*:\s*(?:false|null|undefined|0)\b)/;
|
|
479
|
+
const lineOfIndex = (text, index) => text.slice(0, index).split('\n').length;
|
|
480
|
+
|
|
481
|
+
// The changes, as { type: 'changed' | 'weakened' | 'replaced' | 'skipped', file, what, from, to }.
|
|
482
|
+
// isTestFile picks the test files; read(file) returns a file as it is now, or null.
|
|
483
|
+
export function testChanges(diff, { read, isTestFile }) {
|
|
484
|
+
const files = parseDiff(diff).filter((f) => isTestFile(f.path) && languageOf(f.path));
|
|
485
|
+
for (const f of files) {
|
|
486
|
+
const now = f.status === 'D' ? null : read(f.path);
|
|
487
|
+
f.newLines = now == null ? null : now.split('\n');
|
|
488
|
+
f.oldLines = f.status === 'A' ? [] : f.status === 'D' ? f.removed.map((r) => r.text) : f.newLines && undo(f.newLines, f.hunks);
|
|
489
|
+
f.newText = f.status === 'D' ? '' : f.newLines && f.newLines.join('\n');
|
|
490
|
+
f.oldText = f.oldLines && f.oldLines.join('\n');
|
|
491
|
+
}
|
|
492
|
+
const changes = [];
|
|
493
|
+
|
|
494
|
+
// Tests that went: moved to another file, replaced by a new one, or removed. Only replaced ones are reported
|
|
495
|
+
// here; a removed test is counted with the removed assertions, and a deleted file on its own.
|
|
496
|
+
const perFile = [];
|
|
497
|
+
const addedPool = new Map();
|
|
498
|
+
for (const file of files.filter((x) => x.oldText != null && x.newText != null)) {
|
|
499
|
+
const lang = languageOf(file.path);
|
|
500
|
+
const oldNames = extractTestNames(file.oldText, lang);
|
|
501
|
+
const newNames = extractTestNames(file.newText, lang);
|
|
502
|
+
const removed = [];
|
|
503
|
+
const added = [];
|
|
504
|
+
for (const [name, { count, index }] of oldNames) for (let i = newNames.get(name)?.count ?? 0; i < count; i++) removed.push({ name, index });
|
|
505
|
+
for (const [name, { count, index }] of newNames) {
|
|
506
|
+
const before = oldNames.get(name)?.count ?? 0;
|
|
507
|
+
for (let i = before; i < count; i++) added.push({ name, index });
|
|
508
|
+
if (count > before) addedPool.set(name, (addedPool.get(name) ?? 0) + (count - before));
|
|
509
|
+
}
|
|
510
|
+
perFile.push({ file, removed, added });
|
|
511
|
+
}
|
|
512
|
+
for (const entry of perFile) {
|
|
513
|
+
entry.removed = entry.removed.filter(({ name }) => {
|
|
514
|
+
const n = addedPool.get(name) ?? 0;
|
|
515
|
+
if (n === 0) return true;
|
|
516
|
+
addedPool.set(name, n - 1);
|
|
517
|
+
return false;
|
|
518
|
+
});
|
|
519
|
+
}
|
|
520
|
+
const removedIn = new Set();
|
|
521
|
+
const addedIn = (pred) => perFile.filter((e) => e.file.status !== 'D' && pred(e.file)).flatMap((e) => e.added.map((a) => ({ ...a, file: e.file })));
|
|
522
|
+
const take = (list, pick) => {
|
|
523
|
+
const i = list.findIndex((a) => (addedPool.get(a.name) ?? 0) > 0 && pick(a));
|
|
524
|
+
if (i < 0) return null;
|
|
525
|
+
const [hit] = list.splice(i, 1);
|
|
526
|
+
addedPool.set(hit.name, addedPool.get(hit.name) - 1);
|
|
527
|
+
return hit;
|
|
528
|
+
};
|
|
529
|
+
for (const { file, removed } of perFile) {
|
|
530
|
+
if (file.status === 'D') {
|
|
531
|
+
if (removed.length) removedIn.add(file.path);
|
|
532
|
+
continue;
|
|
533
|
+
}
|
|
534
|
+
const own = addedIn((x) => x === file);
|
|
535
|
+
const elsewhere = addedIn((x) => x !== file);
|
|
536
|
+
for (const r of removed) {
|
|
537
|
+
const replacement = take(own, () => true) ?? take(elsewhere, (a) => similarNames(r.name, a.name));
|
|
538
|
+
if (replacement) changes.push({ type: 'replaced', file: file.path, what: 'replaced', from: truncate(r.name, 50), to: truncate(replacement.name, 50), where: replacement.file === file ? null : replacement.file.path, line: lineOfIndex(file.newText, replacement.index) });
|
|
539
|
+
else removedIn.add(file.path);
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
// Assertions changed in place: each removed assertion against the first added one in its hunk that it matches.
|
|
544
|
+
for (const file of files) {
|
|
545
|
+
for (const hunk of file.hunks) {
|
|
546
|
+
if (!hunk.removed.length || !hunk.added.length) continue;
|
|
547
|
+
const rem = hunk.removed.map((r) => ({ ...r, a: parseAssertion(r.text) })).filter((x) => x.a);
|
|
548
|
+
const add = hunk.added.map((x) => ({ ...x, a: parseAssertion(x.text) })).filter((x) => x.a);
|
|
549
|
+
if (rem.length * add.length > 250000) continue; // a rewritten test file, not an edit; the other checks still read it
|
|
550
|
+
const used = new Set();
|
|
551
|
+
for (const r of rem) {
|
|
552
|
+
for (const a of add) {
|
|
553
|
+
if (used.has(a.line)) continue;
|
|
554
|
+
if (normalizeLine(r.text) === normalizeLine(a.text)) {
|
|
555
|
+
used.add(a.line);
|
|
556
|
+
break;
|
|
557
|
+
}
|
|
558
|
+
const cmp = compareAssertions(r.a, a.a);
|
|
559
|
+
if (!cmp) continue;
|
|
560
|
+
used.add(a.line);
|
|
561
|
+
changes.push({ ...cmp, file: file.path });
|
|
562
|
+
break;
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
// A skip option added to a test that already existed: test('name', { skip }, fn). New tests use the same
|
|
569
|
+
// option to declare what they need (`{ skip: !hasCrypto }`), so a test whose name is new does not count.
|
|
570
|
+
for (const file of files) {
|
|
571
|
+
if (languageOf(file.path) !== 'js' || file.oldText == null) continue;
|
|
572
|
+
const before = new Set(file.removed.map((r) => normalizeLine(r.text)));
|
|
573
|
+
for (const a of file.added) {
|
|
574
|
+
const m = OPTIONS_SKIP.exec(a.text);
|
|
575
|
+
if (m && !isCommentLine(a.text) && !before.has(normalizeLine(a.text)) && file.oldText.includes(m[2])) {
|
|
576
|
+
changes.push({ type: 'skipped', file: file.path, what: 'skipped', from: truncate(m[2], 50) });
|
|
577
|
+
break;
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
// Test data dropped or reordered, in files that lost no whole test.
|
|
583
|
+
for (const file of files) if (file.status === 'M' && !removedIn.has(file.path)) for (const c of tableChanges(file)) changes.push({ ...c, file: file.path });
|
|
584
|
+
return changes;
|
|
585
|
+
}
|
package/src/signals.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
// Deterministic signals. No AI in the decision path.
|
|
2
|
+
import { testChanges } from './noskip.js';
|
|
2
3
|
|
|
3
4
|
const ERROR_LINE =
|
|
4
5
|
/(error|exception|traceback|failed|failure|fatal|cannot|can't|not found|undefined is not|is not defined|is not a function|segmentation fault|ERR!|panic:|denied|refused|unexpected token|syntaxerror|typeerror|referenceerror|modulenotfound|no such file)/i;
|
|
@@ -132,19 +133,29 @@ export function similarity(a, b) {
|
|
|
132
133
|
// ---------- test tampering ----------
|
|
133
134
|
|
|
134
135
|
const TEST_FILE = /(^|\/)(test|tests|__tests__|spec)\/|\.(test|spec)\.[a-z]+$|_test\.(go|py)$|(^|\/)test_[^/]+\.py$/i;
|
|
135
|
-
|
|
136
|
+
// Data the tests read, not tests: deleting or editing it is not counted as weakening a test.
|
|
137
|
+
const FIXTURE = /(^|\/)(__fixtures__|fixtures?|__snapshots__|__mocks__|testdata)\//i;
|
|
138
|
+
export const isTestFile = (file) => TEST_FILE.test(file) && !FIXTURE.test(file);
|
|
136
139
|
|
|
137
140
|
// Kinds: the first three weaken the tests themselves, which is what `codehere audit` counts;
|
|
138
|
-
//
|
|
141
|
+
// expected-changed is an assertion whose expected value changed, which the finish check reports and the
|
|
142
|
+
// audit does not (in history it is mostly ordinary upkeep); checks-off turns off a check around the tests
|
|
143
|
+
// (hooks, types, the test script's exit code).
|
|
139
144
|
const DELETED = 'test-deleted';
|
|
140
145
|
const SKIPPED = 'test-skipped';
|
|
141
146
|
const WEAKENED = 'assertion-weakened';
|
|
147
|
+
const EXPECTED = 'expected-changed';
|
|
148
|
+
const HARDCODED = 'hardcoded-answer';
|
|
149
|
+
const UNDER_TEST = 'test-env';
|
|
142
150
|
const CHECKS_OFF = 'checks-off';
|
|
151
|
+
const REPLACED = 'test-replaced';
|
|
143
152
|
|
|
144
153
|
const TAMPER_ADDED = [
|
|
145
154
|
{ re: /\b(it|test|describe)\.skip\s*\(/, kind: SKIPPED, why: 'a test was skipped (.skip)' },
|
|
146
155
|
{ re: /\b(xit|xdescribe|xtest)\s*\(/, kind: SKIPPED, why: 'a test was disabled (xit/xdescribe)' },
|
|
147
|
-
|
|
156
|
+
// skip, and skipif or skipIf only when the condition is a literal: skipUnless(shutil.which("uv")) is a
|
|
157
|
+
// test that cannot run on this machine, as { skip: process.platform === 'win32' } is in JS
|
|
158
|
+
{ re: /@pytest\.mark\.skip\b|\bunittest\.skip\s*\(|\b(skipif|skipIf)\s*\(\s*True\b|\bskipUnless\s*\(\s*False\b/, kind: SKIPPED, why: 'a Python test was skipped' },
|
|
148
159
|
{ re: /@ts-nocheck/, kind: CHECKS_OFF, why: 'type checking was turned off (@ts-nocheck)' },
|
|
149
160
|
{ re: /--no-verify/, kind: CHECKS_OFF, why: 'git hooks were bypassed (--no-verify)' },
|
|
150
161
|
{ re: /expect\([^)]*\)\.(toBe|toEqual)\(\s*(true|1)\s*\)\s*;?\s*\/\/\s*(todo|temp|hack)/i, kind: WEAKENED, why: 'an assertion was replaced with a placeholder' },
|
|
@@ -169,14 +180,246 @@ const ASSERT = /\b(expect\s*\(|assert[\w.]*\s*\()|^[+-]\s*assert\s/;
|
|
|
169
180
|
const ALWAYS_PASSES = /"test"\s*:\s*"[^"]*(\|\|\s*(true|:|exit\s+0|echo\b[^"&|;]*)|;\s*(true|exit\s+0))\s*"/;
|
|
170
181
|
const COMMENT = /^[+-]\s*(\/\/|#|\/\*|\*)\s*/;
|
|
171
182
|
|
|
183
|
+
// Options that make the runner leave tests out (node --test, jest, vitest, mocha, playwright, pytest,
|
|
184
|
+
// go, cargo), on a test script or a pytest addopts line. Reporter, coverage and other options are not.
|
|
185
|
+
const FILTER = /(?:^|\s)(--test-skip-pattern|--test-name-pattern|--testPathIgnorePatterns|--testNamePattern|--testPathPatterns?|--grep-invert|--grep|--fgrep|--invert|--ignore(?:-glob)?|--deselect|--skip|-skip|-run|-k)(?=[\s=]|$)/g;
|
|
186
|
+
const TEST_SCRIPT = /"test(?::[^"]*)?"\s*:\s*"((?:[^"\\]|\\.)*)"/;
|
|
187
|
+
const ADDOPTS = /^[+-]\s*"?addopts"?\s*=(.*)$/;
|
|
188
|
+
const PYTEST_CONFIG = /(^|\/)(pytest\.ini|pyproject\.toml|setup\.cfg|tox\.ini)$/;
|
|
189
|
+
// The same, as keys in a runner's own config file
|
|
190
|
+
const RUNNER_CONFIG = /(^|\/)(jest|vitest|playwright)\.config\.[\w.]+$|(^|\/)\.mocharc(\.[\w]+)?$/;
|
|
191
|
+
const CONFIG_KEY = /(?:^|[\s{,'"])(testPathIgnorePatterns|testNamePattern|testIgnore|grepInvert|grep|invert|ignore)['"]?\s*:/g;
|
|
192
|
+
export const isTestConfig = (file) => /(^|\/)package\.json$/.test(file) || PYTEST_CONFIG.test(file) || RUNNER_CONFIG.test(file);
|
|
193
|
+
|
|
194
|
+
// The options a line of this file sets that leave tests out, with where they were set.
|
|
195
|
+
function filtersOn(file, line) {
|
|
196
|
+
let text = null;
|
|
197
|
+
let where = '';
|
|
198
|
+
if (/(^|\/)package\.json$/.test(file)) (text = (TEST_SCRIPT.exec(line) || [])[1] ?? null), (where = 'by the test script');
|
|
199
|
+
else if (PYTEST_CONFIG.test(file)) (text = (ADDOPTS.exec(line) || [])[1] ?? null), (where = "by the test runner's options");
|
|
200
|
+
else if (RUNNER_CONFIG.test(file)) (text = line.slice(1)), (where = "by the test runner's config");
|
|
201
|
+
if (text == null) return [];
|
|
202
|
+
const re = RUNNER_CONFIG.test(file) ? CONFIG_KEY : FILTER;
|
|
203
|
+
return [...text.matchAll(re)].map((m) => ({ option: m[1], where }));
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// Which characters of a line are inside a string literal: test code often writes another test as
|
|
207
|
+
// text, and a skip in that text is not a skip. One line at a time, so a string that spans lines (a
|
|
208
|
+
// template literal, a docstring) is not seen; after // or a # comment, no quote opens a string.
|
|
209
|
+
function stringMask(line) {
|
|
210
|
+
const mask = new Uint8Array(line.length);
|
|
211
|
+
let quote = '';
|
|
212
|
+
for (let i = 0; i < line.length; i++) {
|
|
213
|
+
const c = line[i];
|
|
214
|
+
if (quote) {
|
|
215
|
+
mask[i] = 1;
|
|
216
|
+
if (c === '\\') mask[++i] = 1;
|
|
217
|
+
else if (c === quote) quote = '';
|
|
218
|
+
} else if (c === '"' || c === "'" || c === '`') (quote = c), (mask[i] = 1);
|
|
219
|
+
else if ((c === '/' && line[i + 1] === '/') || (c === '#' && (i <= 1 || /\s/.test(line[i - 1])))) break;
|
|
220
|
+
}
|
|
221
|
+
return mask;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// An assertion line with its values taken out. A removed and an added line with the same shape and other
|
|
225
|
+
// values are one assertion whose expected value changed; assertTrue and assertFalse share a shape.
|
|
226
|
+
// numbers may use _ as a separator (10_000), as JS and Python allow
|
|
227
|
+
const VALUES = /(?<![\w.$])(?:\d[\d_]*(?:\.\d+)?|true|false|True|False|null|None|undefined)(?![\w$])|'(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*"|`(?:[^`\\]|\\.)*`/g;
|
|
228
|
+
function shape(line) {
|
|
229
|
+
const values = [];
|
|
230
|
+
const text = line
|
|
231
|
+
.slice(1)
|
|
232
|
+
.trim()
|
|
233
|
+
.replace(/\bassert(True|False)\b/g, (m, b) => (values.push(b), 'assert<b>'))
|
|
234
|
+
.replace(VALUES, (v) => (values.push(v.slice(0, 40)), '<v>'));
|
|
235
|
+
return { text, values };
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// The arguments of the call whose '(' is at open, split at top-level commas; null if it does not close.
|
|
239
|
+
function argsAt(t, open) {
|
|
240
|
+
const args = [];
|
|
241
|
+
let depth = 0;
|
|
242
|
+
let quote = '';
|
|
243
|
+
let start = open + 1;
|
|
244
|
+
for (let i = open; i < t.length; i++) {
|
|
245
|
+
const c = t[i];
|
|
246
|
+
if (quote) {
|
|
247
|
+
if (c === '\\') i++;
|
|
248
|
+
else if (c === quote) quote = '';
|
|
249
|
+
} else if (c === '"' || c === "'" || c === '`') quote = c;
|
|
250
|
+
else if ('([{'.includes(c)) depth++;
|
|
251
|
+
else if (')]}'.includes(c)) {
|
|
252
|
+
if (--depth === 0) return (args.push(t.slice(start, i).trim()), { args, end: i });
|
|
253
|
+
} else if (c === ',' && depth === 1) args.push(t.slice(start, i).trim()), (start = i + 1);
|
|
254
|
+
}
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// Where a top-level operator (==) starts in a Python assert, outside brackets and strings; -1 if none.
|
|
259
|
+
function topLevel(t, op) {
|
|
260
|
+
let depth = 0;
|
|
261
|
+
let quote = '';
|
|
262
|
+
for (let i = 0; i < t.length; i++) {
|
|
263
|
+
const c = t[i];
|
|
264
|
+
if (quote) {
|
|
265
|
+
if (c === '\\') i++;
|
|
266
|
+
else if (c === quote) quote = '';
|
|
267
|
+
} else if (c === '"' || c === "'") quote = c;
|
|
268
|
+
else if ('([{'.includes(c)) depth++;
|
|
269
|
+
else if (')]}'.includes(c)) depth--;
|
|
270
|
+
else if (depth === 0 && t.startsWith(op, i)) return i;
|
|
271
|
+
}
|
|
272
|
+
return -1;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
// An assertion's subject and whether it checks a value (equal, toBe, assertEqual, ==) or only that the
|
|
276
|
+
// subject is set or truthy (ok, assert(), toBeDefined, toBeTruthy, assertTrue, a bare Python assert).
|
|
277
|
+
// The same subject going from the first kind to the second is a weakened assertion.
|
|
278
|
+
const VALUE_CALL = /\b(equal|strictEqual|deepEqual|deepStrictEqual|assertEqual|assertEquals)\s*\(/;
|
|
279
|
+
const TRUTHY_CALL = /(?:^|[^\w.])(assert\.ok|assert)\s*\(|\b(assertTrue|assertIsNotNone)\s*\(/;
|
|
280
|
+
const EXPECT_VALUE = /^\.(toBe|toEqual|toStrictEqual)\s*\(/;
|
|
281
|
+
const EXPECT_TRUTHY = /^\.(toBeDefined|toBeTruthy|not\.toBeNull|not\.toBeUndefined)\s*\(\s*\)/;
|
|
282
|
+
function strength(line) {
|
|
283
|
+
const t = line.slice(1).trim().replace(/;$/, '');
|
|
284
|
+
let m = VALUE_CALL.exec(t);
|
|
285
|
+
if (m) {
|
|
286
|
+
const c = argsAt(t, m.index + m[0].length - 1);
|
|
287
|
+
return c && c.args.length >= 2 ? { subject: c.args[0], value: true, form: `${m[1]}(…)`, expected: c.args[1] } : null;
|
|
288
|
+
}
|
|
289
|
+
m = /\bexpect\s*\(/.exec(t);
|
|
290
|
+
if (m) {
|
|
291
|
+
const c = argsAt(t, m.index + m[0].length - 1);
|
|
292
|
+
if (!c) return null;
|
|
293
|
+
const rest = t.slice(c.end + 1);
|
|
294
|
+
const v = EXPECT_VALUE.exec(rest);
|
|
295
|
+
if (v) {
|
|
296
|
+
const e = argsAt(rest, rest.indexOf('('));
|
|
297
|
+
return { subject: c.args[0], value: true, form: `${v[1]}(…)`, expected: e ? e.args[0] : '' };
|
|
298
|
+
}
|
|
299
|
+
const w = EXPECT_TRUTHY.exec(rest);
|
|
300
|
+
return w ? { subject: c.args[0], value: false, form: `${w[1]}()` } : null;
|
|
301
|
+
}
|
|
302
|
+
m = TRUTHY_CALL.exec(t);
|
|
303
|
+
if (m) {
|
|
304
|
+
const c = argsAt(t, m.index + m[0].length - 1);
|
|
305
|
+
return c ? { subject: c.args[0], value: false, form: `${(m[1] || m[2]).replace('assert.', '')}(…)` } : null;
|
|
306
|
+
}
|
|
307
|
+
m = /^assert\s+(.+)$/.exec(t);
|
|
308
|
+
if (m) {
|
|
309
|
+
const comma = topLevel(m[1], ',');
|
|
310
|
+
const body = (comma >= 0 ? m[1].slice(0, comma) : m[1]).trim(); // drop a message
|
|
311
|
+
const eq = topLevel(body, '==');
|
|
312
|
+
if (eq >= 0) return { subject: body.slice(0, eq).trim(), value: true, form: '==', expected: body.slice(eq + 2).trim() };
|
|
313
|
+
const none = /^(.+?)\s+is\s+not\s+None$/.exec(body);
|
|
314
|
+
if (none) return { subject: none[1].trim(), value: false, form: 'is not None' };
|
|
315
|
+
if (/[<>!]=?|\bin\b|\bis\b/.test(body)) return null;
|
|
316
|
+
return { subject: body, value: false, form: 'a bare assert' };
|
|
317
|
+
}
|
|
318
|
+
return null;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// A catch that swallows what was thrown: catch {}, catch (e) {} or catch (e) { /* ignore */ } on one line;
|
|
322
|
+
// a catch line that opens a body which the next line closes; Python's except ...: pass, on one or two lines.
|
|
323
|
+
const CATCH_EMPTY = /\bcatch\s*(?:\(\s*[\w$]*\s*\))?\s*\{\s*(?:\/\*[^*]*\*\/|\/\/[^\n]*)?\s*\}/;
|
|
324
|
+
const CATCH_OPEN = /\bcatch\s*(?:\(\s*[\w$]*\s*\))?\s*\{\s*$/;
|
|
325
|
+
const EXCEPT_PASS = /^\s*except\b[^:]*:\s*pass\s*(#.*)?$/;
|
|
326
|
+
const EXCEPT_OPEN = /^\s*except\b[^:]*:\s*(#.*)?$/;
|
|
327
|
+
const TRY_OPEN = /\btry\s*\{|^\s*try\s*:/;
|
|
328
|
+
|
|
329
|
+
// Whether the try that a swallowing catch at line `at` (1-based) closes holds an assertion. The lines are the
|
|
330
|
+
// file as it is now when it can be read, else the lines of the diff around the catch (git's context).
|
|
331
|
+
function swallowsAssertion(lines, at, catchText) {
|
|
332
|
+
let i = at - 1;
|
|
333
|
+
if ((lines[i] || '').trim() !== catchText.trim()) {
|
|
334
|
+
const near = [-3, -2, -1, 1, 2, 3].map((d) => at - 1 + d).find((k) => (lines[k] || '').trim() === catchText.trim());
|
|
335
|
+
if (near === undefined) return false;
|
|
336
|
+
i = near;
|
|
337
|
+
}
|
|
338
|
+
for (let k = i, n = 0; k >= 0 && n < 60; k--, n++) {
|
|
339
|
+
let text = lines[k] || '';
|
|
340
|
+
const cut = k === i ? text.search(/\bcatch\b|\bexcept\b/) : -1;
|
|
341
|
+
if (cut >= 0) text = text.slice(0, cut); // try { assert(...) } catch {} on one line: the part before the catch
|
|
342
|
+
const open = text.search(TRY_OPEN);
|
|
343
|
+
if (ASSERT.test('+' + (open >= 0 ? text.slice(open).replace(TRY_OPEN, '') : text))) return true;
|
|
344
|
+
if (open >= 0) return false;
|
|
345
|
+
}
|
|
346
|
+
return false;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
// Literals in a piece of code: numbers (10_000 is 10000) and the contents of quoted strings, tagged so the
|
|
350
|
+
// number 500 and the string "500" stay apart.
|
|
351
|
+
const LITERAL_RE = /(?<![\w.$])-?\d[\d_]*(?:\.\d+)?(?![\w$])|'((?:[^'\\]|\\.)*)'|"((?:[^"\\]|\\.)*)"/g;
|
|
352
|
+
function literals(text) {
|
|
353
|
+
return [...String(text).matchAll(LITERAL_RE)].map((m) => (m[1] !== undefined || m[2] !== undefined ? `s:${m[1] ?? m[2]}` : `n:${Number(m[0].replace(/_/g, ''))}`));
|
|
354
|
+
}
|
|
355
|
+
// Values too common to say anything when code returns them
|
|
356
|
+
const DULL = new Set(['n:0', 'n:1', 'n:-1', 'n:2', 's:']);
|
|
357
|
+
|
|
358
|
+
// What a test line expects, when it compares a call on literal inputs with one literal value.
|
|
359
|
+
function expectationOf(line) {
|
|
360
|
+
const st = strength('+' + line);
|
|
361
|
+
if (!st || !st.value || !st.expected) return null;
|
|
362
|
+
const want = literals(st.expected);
|
|
363
|
+
if (want.length !== 1 || st.expected.replace(/^[-'"]|['"]$/g, '').length === 0) return null;
|
|
364
|
+
return { inputs: new Set(literals(st.subject)), expected: want[0] };
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
// A return of one literal under a condition, on the same line or the line before: if (x === 250) return 500;
|
|
368
|
+
const RETURN_LITERAL = /\breturn\s+(-?\d[\d_]*(?:\.\d+)?|'(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*")\s*;?\s*\}?\s*$/;
|
|
369
|
+
const SAME_LINE_IF = /\bif\s*\(?(.*?)\)?\s*(?:\{|:)?\s*return\b/;
|
|
370
|
+
|
|
371
|
+
// A test runner's own markers, read by code outside the tests: the code can then behave one way under the
|
|
372
|
+
// runner and another way for users. NODE_ENV === 'test' is not one: apps use it to pick a test database.
|
|
373
|
+
const RUNNER_MARKER = /\b(NODE_TEST_CONTEXT|JEST_WORKER_ID|VITEST_WORKER_ID|VITEST_POOL_ID|PYTEST_CURRENT_TEST)\b|(['"]pytest['"]\s+in\s+sys\.modules)|\b(testing\.Testing\(\))/;
|
|
374
|
+
// Test setup that may read them: pytest's conftest.py and the setup files jest and vitest load
|
|
375
|
+
const TEST_SETUP = /(^|\/)(conftest\.py|setupTests\.[cm]?[jt]sx?|(jest|vitest)\.setup\.[cm]?[jt]sx?)$/;
|
|
376
|
+
|
|
377
|
+
// CI files, and the commands in them that run the tests
|
|
378
|
+
const CI_FILE = /(^|\/)\.github\/workflows\/[^/]+\.ya?ml$|(^|\/)\.gitlab-ci\.ya?ml$|(^|\/)\.circleci\/config\.ya?ml$|(^|\/)(azure|bitbucket)-pipelines\.ya?ml$/;
|
|
379
|
+
const TEST_CMD = /\b(?:(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?test\b|node\s+--test\b|(?:npx\s+)?(?:jest|vitest|mocha)\b|(?:python3?\s+-m\s+)?pytest\b|python3?\s+-m\s+unittest\b|go\s+test\b|cargo\s+test\b|make\s+test\b)/;
|
|
380
|
+
const CI_ALWAYS_PASSES = new RegExp(TEST_CMD.source + String.raw`[^\n]*?(\|\|\s*(true|:|exit\s+0|echo\b)|;\s*(true|exit\s+0)\s*$)`);
|
|
381
|
+
const CI_IGNORE = /^\s*(continue-on-error|allow_failure)\s*:\s*true\b/;
|
|
382
|
+
|
|
383
|
+
// Whether the YAML block a key at line `at` belongs to (its step or job) runs the tests.
|
|
384
|
+
function blockRunsTests(lines, at, text) {
|
|
385
|
+
let i = at - 1;
|
|
386
|
+
if ((lines[i] || '').trim() !== text.trim()) {
|
|
387
|
+
const near = [-3, -2, -1, 1, 2, 3].map((d) => at - 1 + d).find((k) => (lines[k] || '').trim() === text.trim());
|
|
388
|
+
if (near === undefined) return false;
|
|
389
|
+
i = near;
|
|
390
|
+
}
|
|
391
|
+
const indent = (l) => l.length - l.trimStart().length;
|
|
392
|
+
const own = indent(lines[i]);
|
|
393
|
+
let start = i;
|
|
394
|
+
while (start > 0 && !((lines[start - 1] || '').trim() && indent(lines[start - 1]) < own)) start--;
|
|
395
|
+
start = Math.max(0, start - 1);
|
|
396
|
+
const top = indent(lines[start] || '');
|
|
397
|
+
for (let k = start; k < lines.length && k < start + 200; k++) {
|
|
398
|
+
if (k > start && (lines[k] || '').trim() && indent(lines[k]) <= top) break;
|
|
399
|
+
if (TEST_CMD.test(lines[k] || '')) return true;
|
|
400
|
+
}
|
|
401
|
+
return false;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
// re found in the code of the line, not only inside its strings
|
|
405
|
+
function inCode(re, line, mask) {
|
|
406
|
+
const g = new RegExp(re.source, re.flags.includes('g') ? re.flags : re.flags + 'g');
|
|
407
|
+
for (const m of line.matchAll(g)) if (!mask()[m.index + (m[0].length - m[0].trimStart().length)]) return true;
|
|
408
|
+
return false;
|
|
409
|
+
}
|
|
410
|
+
|
|
172
411
|
// What the finish check prints: the events' text, in order, each once.
|
|
173
|
-
export function tamperFindings(nameStatus, diff) {
|
|
174
|
-
return tamperEvents(nameStatus, diff).map((e) => e.why);
|
|
412
|
+
export function tamperFindings(nameStatus, diff, opts) {
|
|
413
|
+
return tamperEvents(nameStatus, diff, opts).map((e) => e.why);
|
|
175
414
|
}
|
|
176
415
|
|
|
177
416
|
// Each finding as { kind, file, why }; file is undefined for the count of removed assertions,
|
|
178
417
|
// which spans the whole diff. A why already seen is not repeated.
|
|
179
|
-
|
|
418
|
+
// read(file), when given, returns the file as it is now (or null): the finish check passes one, the audit
|
|
419
|
+
// does not and relies on the diff's context lines. tests, when given, is the repo's test files as
|
|
420
|
+
// "path\0content" (or a function returning them, called only when it is needed), for the expected values a
|
|
421
|
+
// hardcoded answer would copy.
|
|
422
|
+
export function tamperEvents(nameStatus, diff, { read, tests } = {}) {
|
|
180
423
|
const events = [];
|
|
181
424
|
const seen = new Set();
|
|
182
425
|
const add = (kind, file, why) => {
|
|
@@ -186,37 +429,169 @@ export function tamperEvents(nameStatus, diff) {
|
|
|
186
429
|
};
|
|
187
430
|
for (const line of nameStatus.split('\n')) {
|
|
188
431
|
const [status, file] = line.split(/\s+/);
|
|
189
|
-
if (status === 'D' && file &&
|
|
432
|
+
if (status === 'D' && file && isTestFile(file)) add(DELETED, file, `test file deleted: ${file}`);
|
|
190
433
|
}
|
|
191
434
|
let current = '';
|
|
192
435
|
let removedAsserts = 0;
|
|
193
436
|
let addedAsserts = 0;
|
|
194
437
|
let removed = new Set(); // this file's removed assertions, to spot one coming back as a comment
|
|
438
|
+
let removedShapes = new Map(); // this file's removed assertions by shape, to spot a changed expected value
|
|
439
|
+
let removedValueChecks = new Map(); // this file's removed value checks by subject, to spot a weakened one
|
|
440
|
+
const changedIn = new Set(); // one changed-value finding per file
|
|
441
|
+
let newLine = 0; // the line number in the file as it is now, from the hunk headers
|
|
442
|
+
let fileLines = []; // this file's lines from the diff (added and context), by line number, for the audit
|
|
443
|
+
let pendingCatch = 0; // a catch or except line whose body may be closed or passed on the next line
|
|
444
|
+
let prevText = ''; // the previous added or context line of this file
|
|
445
|
+
const expectations = []; // from test lines in the diff, then from the repo's test files
|
|
446
|
+
const returns = []; // literal returns under a condition in code files, checked after the diff is read
|
|
447
|
+
const ciRemoved = []; // CI files that lost a line running the tests
|
|
448
|
+
let ciAdded = 0; // lines running the tests added to any CI file
|
|
449
|
+
const weakenedIn = new Set(); // one weakened-matcher finding per file
|
|
195
450
|
let hadFallback = false; // the test script already ended in `|| true` before this session
|
|
451
|
+
let hadFilters = new Set(); // options that left tests out before this session
|
|
196
452
|
for (const line of diff.split('\n')) {
|
|
197
453
|
if (line.startsWith('+++ ')) {
|
|
198
454
|
current = line.replace(/^\+\+\+ (b\/)?/, '');
|
|
199
455
|
removed = new Set();
|
|
456
|
+
removedShapes = new Map();
|
|
457
|
+
removedValueChecks = new Map();
|
|
458
|
+
fileLines = [];
|
|
459
|
+
pendingCatch = 0;
|
|
460
|
+
prevText = '';
|
|
200
461
|
hadFallback = false;
|
|
462
|
+
hadFilters = new Set();
|
|
201
463
|
continue;
|
|
202
464
|
}
|
|
203
465
|
if (line.startsWith('--- ')) continue;
|
|
204
|
-
|
|
466
|
+
if (line.startsWith('@@')) {
|
|
467
|
+
newLine = Number((/\+(\d+)/.exec(line) || [])[1] || 0);
|
|
468
|
+
continue;
|
|
469
|
+
}
|
|
470
|
+
const at = line.startsWith('-') ? 0 : newLine;
|
|
471
|
+
if (!line.startsWith('-')) newLine++;
|
|
472
|
+
if (at && at <= 20000) fileLines[at - 1] = line.slice(1);
|
|
473
|
+
const inTest = isTestFile(current);
|
|
474
|
+
if (CI_FILE.test(current) && !COMMENT.test(line)) {
|
|
475
|
+
const text = line.slice(1, 10001);
|
|
476
|
+
if (line.startsWith('-') && TEST_CMD.test(text)) ciRemoved.push(current);
|
|
477
|
+
if (line.startsWith('+') && TEST_CMD.test(text)) ciAdded++;
|
|
478
|
+
if (line.startsWith('+') && CI_ALWAYS_PASSES.test(text)) add(CHECKS_OFF, current, `the CI test step was changed to always pass in ${current}`);
|
|
479
|
+
if (line.startsWith('+') && CI_IGNORE.test(text)) {
|
|
480
|
+
const now = read ? read(current) : null;
|
|
481
|
+
if (blockRunsTests(now == null ? fileLines : now.split('\n'), at, text)) add(CHECKS_OFF, current, `the CI test step was changed to always pass in ${current}`);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
const before = prevText;
|
|
485
|
+
if (!line.startsWith('-')) prevText = line.slice(1, 10001);
|
|
486
|
+
if (inTest && !line.startsWith('-')) {
|
|
487
|
+
const e = expectationOf(line.slice(1, 10001));
|
|
488
|
+
if (e) expectations.push(e);
|
|
489
|
+
} else if (!inTest && line.startsWith('+') && !isTestConfig(current)) {
|
|
490
|
+
const marker = !COMMENT.test(line) && !TEST_SETUP.test(current) && !FIXTURE.test(current) && RUNNER_MARKER.exec(line.slice(1, 10001));
|
|
491
|
+
if (marker) add(UNDER_TEST, current, `${current} checks whether it is running under a test (${(marker[1] || marker[2] || marker[3]).replace(/['"]/g, '')})`);
|
|
492
|
+
const r = RETURN_LITERAL.exec(line.slice(1, 10001));
|
|
493
|
+
if (r) {
|
|
494
|
+
const same = SAME_LINE_IF.exec(line.slice(1, 10001));
|
|
495
|
+
const cond = same ? same[1] : /^\s*if\b/.test(before) ? before : null;
|
|
496
|
+
if (cond != null) returns.push({ file: current, text: r[1], value: literals(r[1])[0], given: literals(cond) });
|
|
497
|
+
}
|
|
498
|
+
}
|
|
205
499
|
const probe = line.slice(0, 10000); // the patterns can take quadratic time on a long line
|
|
500
|
+
let lineMask = null;
|
|
501
|
+
const mask = () => (lineMask ||= stringMask(probe));
|
|
206
502
|
if (/(^|\/)package\.json$/.test(current) && ALWAYS_PASSES.test(probe)) {
|
|
207
503
|
if (line.startsWith('-')) hadFallback = true;
|
|
208
504
|
else if (!hadFallback) add(CHECKS_OFF, current, `the test script was changed to always exit 0 in ${current}`);
|
|
209
505
|
}
|
|
506
|
+
if (isTestConfig(current)) {
|
|
507
|
+
for (const f of filtersOn(current, probe)) {
|
|
508
|
+
if (line.startsWith('-')) hadFilters.add(f.option);
|
|
509
|
+
else if (line.startsWith('+') && !hadFilters.has(f.option)) add(SKIPPED, current, `tests were left out ${f.where} (${f.option}) in ${current}`);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
210
512
|
if (line.startsWith('+')) {
|
|
211
|
-
for (const t of TAMPER_ADDED) if (t.re
|
|
513
|
+
for (const t of TAMPER_ADDED) if (inCode(t.re, probe, mask)) add(t.kind, current, `${t.why} in ${current}`);
|
|
212
514
|
if (!inTest) continue;
|
|
213
|
-
|
|
515
|
+
const text = probe.slice(1);
|
|
516
|
+
let swallowAt = 0;
|
|
517
|
+
let swallowText = '';
|
|
518
|
+
if (pendingCatch && at === pendingCatch.at + 1 && (text.trim() === '}' || text.trim() === 'pass')) [swallowAt, swallowText] = [pendingCatch.at, pendingCatch.text];
|
|
519
|
+
pendingCatch = 0;
|
|
520
|
+
if (inCode(CATCH_EMPTY, probe, mask) || EXCEPT_PASS.test(text)) [swallowAt, swallowText] = [at, text];
|
|
521
|
+
else if (inCode(CATCH_OPEN, probe, mask) || EXCEPT_OPEN.test(text)) pendingCatch = { at, text };
|
|
522
|
+
if (swallowAt) {
|
|
523
|
+
const now = read ? read(current) : null;
|
|
524
|
+
const lines = now == null ? fileLines : now.split('\n');
|
|
525
|
+
if (swallowsAssertion(lines, swallowAt, swallowText)) add(WEAKENED, current, `an assertion was wrapped in a try that swallows its failure in ${current}`);
|
|
526
|
+
}
|
|
527
|
+
for (const t of TAMPER_ADDED_IN_TESTS) if (inCode(t.re, probe, mask)) add(t.kind, current, `${t.why} in ${current}`);
|
|
214
528
|
if (COMMENT.test(line)) {
|
|
215
529
|
if (removed.has(line.replace(COMMENT, '').trim())) add(WEAKENED, current, `an assertion was commented out in ${current}`);
|
|
216
|
-
} else if (ASSERT
|
|
217
|
-
|
|
530
|
+
} else if (inCode(ASSERT, probe, mask)) {
|
|
531
|
+
addedAsserts++;
|
|
532
|
+
const now = shape(probe);
|
|
533
|
+
const before = removedShapes.get(now.text);
|
|
534
|
+
const i = before ? before.findIndex((v, k) => v !== now.values[k]) : -1;
|
|
535
|
+
if (i >= 0 && !changedIn.has(current)) {
|
|
536
|
+
changedIn.add(current);
|
|
537
|
+
add(EXPECTED, current, `an expected value was changed in ${current} (${before[i]} → ${now.values[i]})`);
|
|
538
|
+
}
|
|
539
|
+
const st = strength(probe);
|
|
540
|
+
if (st && !st.value && removedValueChecks.has(st.subject) && !weakenedIn.has(current)) {
|
|
541
|
+
weakenedIn.add(current);
|
|
542
|
+
add(WEAKENED, current, `an assertion was weakened in ${current} (${removedValueChecks.get(st.subject)} became ${st.form})`);
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
} else if (line.startsWith('-') && inTest && !COMMENT.test(line) && inCode(ASSERT, probe, mask)) {
|
|
218
546
|
removedAsserts++;
|
|
219
547
|
removed.add(line.slice(1).trim());
|
|
548
|
+
const was = shape(probe);
|
|
549
|
+
removedShapes.set(was.text, was.values);
|
|
550
|
+
const st = strength(probe);
|
|
551
|
+
if (st && st.value) removedValueChecks.set(st.subject, st.form);
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
if (ciRemoved.length > ciAdded) add(CHECKS_OFF, ciRemoved[0], `the test step was removed from CI in ${ciRemoved[0]}`);
|
|
555
|
+
const candidates = returns.filter((r) => !DULL.has(r.value));
|
|
556
|
+
if (candidates.length) {
|
|
557
|
+
for (const t of (typeof tests === 'function' ? tests() : tests) || []) {
|
|
558
|
+
for (const l of String(t).slice(String(t).indexOf('\0') + 1).split('\n')) {
|
|
559
|
+
const e = expectationOf(l.slice(0, 10000));
|
|
560
|
+
if (e) expectations.push(e);
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
for (const r of candidates) {
|
|
564
|
+
// numbers first: a string in the condition is often a key ("price"), the number is the input
|
|
565
|
+
const order = [...r.given.filter((g) => g.startsWith('n:')), ...r.given.filter((g) => g.startsWith('s:'))];
|
|
566
|
+
const given = order.find((g) => !DULL.has(g) && expectations.some((e) => e.expected === r.value && e.inputs.has(g)));
|
|
567
|
+
const show = (g) => (g.startsWith('n:') ? g.slice(2) : JSON.stringify(g.slice(2)));
|
|
568
|
+
if (given) add(HARDCODED, r.file, `${r.file} returns ${r.text} when given ${show(given)}, the exact value a test expects`);
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
// Expected values, replaced tests and test data, by the rules ported from noskip (src/noskip.js). They need
|
|
572
|
+
// the files as they are now, so only the finish check runs them.
|
|
573
|
+
if (read) {
|
|
574
|
+
const replacedIn = new Set();
|
|
575
|
+
const WORDS = {
|
|
576
|
+
expected: (c) => `an expected value was changed in ${c.file} (${c.from} → ${c.to})`,
|
|
577
|
+
call: (c) => `the call a test checks was changed and its expected value kept in ${c.file} (${c.from} → ${c.to})`,
|
|
578
|
+
flipped: (c) => `an assertion was flipped to its opposite in ${c.file} (${c.from} → ${c.to})`,
|
|
579
|
+
assertion: (c) => `an assertion was changed in ${c.file} (${c.from} → ${c.to})`,
|
|
580
|
+
dropped: (c) => `a case was removed from a test list in ${c.file} (${c.from})`,
|
|
581
|
+
reordered: (c) => `test data was reordered in ${c.file} (${c.from} moved)`,
|
|
582
|
+
};
|
|
583
|
+
for (const c of testChanges(diff, { read, isTestFile })) {
|
|
584
|
+
if (c.type === 'skipped') add(SKIPPED, c.file, `a test that already existed was given a skip option in ${c.file} ("${c.from}")`);
|
|
585
|
+
else if (c.type === 'replaced') {
|
|
586
|
+
if (!replacedIn.has(c.file)) add(REPLACED, c.file, `a test was replaced by a different one in ${c.file} ("${c.from}" → "${c.to}"${c.where ? ` in ${c.where}` : ''})`);
|
|
587
|
+
replacedIn.add(c.file);
|
|
588
|
+
} else if (c.type === 'weakened') {
|
|
589
|
+
if (!weakenedIn.has(c.file)) add(WEAKENED, c.file, `an assertion was weakened in ${c.file} (${c.from} became ${c.to})`);
|
|
590
|
+
weakenedIn.add(c.file);
|
|
591
|
+
} else {
|
|
592
|
+
if (!changedIn.has(c.file)) add(EXPECTED, c.file, WORDS[c.what](c));
|
|
593
|
+
changedIn.add(c.file);
|
|
594
|
+
}
|
|
220
595
|
}
|
|
221
596
|
}
|
|
222
597
|
if (removedAsserts - addedAsserts >= 2) {
|