codehere 2.1.0-rc.0 → 2.1.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -3
- package/bin/codehere.js +7 -0
- package/package.json +2 -2
- package/src/audit.js +173 -0
- package/src/brain.js +78 -0
- package/src/commands.js +11 -1
- package/src/hooks.js +82 -8
- package/src/noskip.js +585 -0
- package/src/signals.js +419 -23
package/src/signals.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
// Deterministic signals. No AI in the decision path.
|
|
2
|
+
import { testChanges } from './noskip.js';
|
|
2
3
|
|
|
3
4
|
const ERROR_LINE =
|
|
4
5
|
/(error|exception|traceback|failed|failure|fatal|cannot|can't|not found|undefined is not|is not defined|is not a function|segmentation fault|ERR!|panic:|denied|refused|unexpected token|syntaxerror|typeerror|referenceerror|modulenotfound|no such file)/i;
|
|
@@ -132,27 +133,45 @@ export function similarity(a, b) {
|
|
|
132
133
|
// ---------- test tampering ----------
|
|
133
134
|
|
|
134
135
|
const TEST_FILE = /(^|\/)(test|tests|__tests__|spec)\/|\.(test|spec)\.[a-z]+$|_test\.(go|py)$|(^|\/)test_[^/]+\.py$/i;
|
|
136
|
+
// Data the tests read, not tests: deleting or editing it is not counted as weakening a test.
|
|
137
|
+
const FIXTURE = /(^|\/)(__fixtures__|fixtures?|__snapshots__|__mocks__|testdata)\//i;
|
|
138
|
+
export const isTestFile = (file) => TEST_FILE.test(file) && !FIXTURE.test(file);
|
|
139
|
+
|
|
140
|
+
// Kinds: the first three weaken the tests themselves, which is what `codehere audit` counts;
|
|
141
|
+
// expected-changed is an assertion whose expected value changed, which the finish check reports and the
|
|
142
|
+
// audit does not (in history it is mostly ordinary upkeep); checks-off turns off a check around the tests
|
|
143
|
+
// (hooks, types, the test script's exit code).
|
|
144
|
+
const DELETED = 'test-deleted';
|
|
145
|
+
const SKIPPED = 'test-skipped';
|
|
146
|
+
const WEAKENED = 'assertion-weakened';
|
|
147
|
+
const EXPECTED = 'expected-changed';
|
|
148
|
+
const HARDCODED = 'hardcoded-answer';
|
|
149
|
+
const UNDER_TEST = 'test-env';
|
|
150
|
+
const CHECKS_OFF = 'checks-off';
|
|
151
|
+
const REPLACED = 'test-replaced';
|
|
135
152
|
|
|
136
153
|
const TAMPER_ADDED = [
|
|
137
|
-
{ re: /\b(it|test|describe)\.skip\s*\(/, why: 'a test was skipped (.skip)' },
|
|
138
|
-
{ re: /\b(xit|xdescribe|xtest)\s*\(/, why: 'a test was disabled (xit/xdescribe)' },
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
{ re:
|
|
142
|
-
{ re:
|
|
154
|
+
{ re: /\b(it|test|describe)\.skip\s*\(/, kind: SKIPPED, why: 'a test was skipped (.skip)' },
|
|
155
|
+
{ re: /\b(xit|xdescribe|xtest)\s*\(/, kind: SKIPPED, why: 'a test was disabled (xit/xdescribe)' },
|
|
156
|
+
// skip, and skipif or skipIf only when the condition is a literal: skipUnless(shutil.which("uv")) is a
|
|
157
|
+
// test that cannot run on this machine, as { skip: process.platform === 'win32' } is in JS
|
|
158
|
+
{ re: /@pytest\.mark\.skip\b|\bunittest\.skip\s*\(|\b(skipif|skipIf)\s*\(\s*True\b|\bskipUnless\s*\(\s*False\b/, kind: SKIPPED, why: 'a Python test was skipped' },
|
|
159
|
+
{ re: /@ts-nocheck/, kind: CHECKS_OFF, why: 'type checking was turned off (@ts-nocheck)' },
|
|
160
|
+
{ re: /--no-verify/, kind: CHECKS_OFF, why: 'git hooks were bypassed (--no-verify)' },
|
|
161
|
+
{ re: /expect\([^)]*\)\.(toBe|toEqual)\(\s*(true|1)\s*\)\s*;?\s*\/\/\s*(todo|temp|hack)/i, kind: WEAKENED, why: 'an assertion was replaced with a placeholder' },
|
|
143
162
|
];
|
|
144
163
|
|
|
145
164
|
// Only in test files: elsewhere `{ skip: true }` or `assert.equal(1, 1)` can be ordinary code.
|
|
146
165
|
const LITERAL = String.raw`(\d+(?:\.\d+)?|true|false|null|'[^']*'|"[^"]*")`;
|
|
147
166
|
const TAMPER_ADDED_IN_TESTS = [
|
|
148
167
|
// test('name', { skip: true }, ...): the name cannot run past its closing quote, so the options are this test's
|
|
149
|
-
{ re: /(?<![.\w])(it|test|describe|suite)\s*\(\s*(['"`])(?:(?!\2)[^\\]|\\.)*\2\s*,\s*\{[^}]*\b(skip|todo)\s*:\s*(true\b|['"`])/, why: 'a test was skipped ({ skip: true })' },
|
|
168
|
+
{ re: /(?<![.\w])(it|test|describe|suite)\s*\(\s*(['"`])(?:(?!\2)[^\\]|\\.)*\2\s*,\s*\{[^}]*\b(skip|todo)\s*:\s*(true\b|['"`])/, kind: SKIPPED, why: 'a test was skipped ({ skip: true })' },
|
|
150
169
|
// fit/fdescribe only in jest's form, starting a statement with a quoted name then a callback:
|
|
151
170
|
// `fit(` alone is a common function name
|
|
152
|
-
{ re: /(?<![.\w])(it|test|describe|context|suite)\.only\s*\(|^[+-]\s*(fit|fdescribe)\s*\(\s*(['"`])(?:(?!\3)[^\\]|\\.)*\3\s*,\s*(async\s+)?(\(|function\b|\w+\s*=>)/, why: 'other tests were turned off (.only)' },
|
|
153
|
-
{ re: /\bassert(\.ok)?\(\s*true\s*\)|^\+\s*assert\s+True\s*$/, why: 'an assertion that cannot fail was added' },
|
|
154
|
-
{ re: new RegExp(String.raw`\bassert\.\w+\(\s*${LITERAL}\s*,\s*\1\s*\)`), why: 'an assertion that cannot fail was added' },
|
|
155
|
-
{ re: new RegExp(String.raw`\bexpect\(\s*${LITERAL}\s*\)\.(toBe|toEqual|toStrictEqual)\(\s*\1\s*\)`), why: 'an assertion that cannot fail was added' },
|
|
171
|
+
{ re: /(?<![.\w])(it|test|describe|context|suite)\.only\s*\(|^[+-]\s*(fit|fdescribe)\s*\(\s*(['"`])(?:(?!\3)[^\\]|\\.)*\3\s*,\s*(async\s+)?(\(|function\b|\w+\s*=>)/, kind: SKIPPED, why: 'other tests were turned off (.only)' },
|
|
172
|
+
{ re: /\bassert(\.ok)?\(\s*true\s*\)|^\+\s*assert\s+True\s*$/, kind: WEAKENED, why: 'an assertion that cannot fail was added' },
|
|
173
|
+
{ re: new RegExp(String.raw`\bassert\.\w+\(\s*${LITERAL}\s*,\s*\1\s*\)`), kind: WEAKENED, why: 'an assertion that cannot fail was added' },
|
|
174
|
+
{ re: new RegExp(String.raw`\bexpect\(\s*${LITERAL}\s*\)\.(toBe|toEqual|toStrictEqual)\(\s*\1\s*\)`), kind: WEAKENED, why: 'an assertion that cannot fail was added' },
|
|
156
175
|
];
|
|
157
176
|
// assert(...), assert.equal(...), expect(...), self.assertEqual(...), and pytest's bare `assert x`;
|
|
158
177
|
// not expect.objectContaining(...) and the other matchers inside one assertion
|
|
@@ -161,47 +180,424 @@ const ASSERT = /\b(expect\s*\(|assert[\w.]*\s*\()|^[+-]\s*assert\s/;
|
|
|
161
180
|
const ALWAYS_PASSES = /"test"\s*:\s*"[^"]*(\|\|\s*(true|:|exit\s+0|echo\b[^"&|;]*)|;\s*(true|exit\s+0))\s*"/;
|
|
162
181
|
const COMMENT = /^[+-]\s*(\/\/|#|\/\*|\*)\s*/;
|
|
163
182
|
|
|
164
|
-
|
|
165
|
-
|
|
183
|
+
// Options that make the runner leave tests out (node --test, jest, vitest, mocha, playwright, pytest,
|
|
184
|
+
// go, cargo), on a test script or a pytest addopts line. Reporter, coverage and other options are not.
|
|
185
|
+
const FILTER = /(?:^|\s)(--test-skip-pattern|--test-name-pattern|--testPathIgnorePatterns|--testNamePattern|--testPathPatterns?|--grep-invert|--grep|--fgrep|--invert|--ignore(?:-glob)?|--deselect|--skip|-skip|-run|-k)(?=[\s=]|$)/g;
|
|
186
|
+
const TEST_SCRIPT = /"test(?::[^"]*)?"\s*:\s*"((?:[^"\\]|\\.)*)"/;
|
|
187
|
+
const ADDOPTS = /^[+-]\s*"?addopts"?\s*=(.*)$/;
|
|
188
|
+
const PYTEST_CONFIG = /(^|\/)(pytest\.ini|pyproject\.toml|setup\.cfg|tox\.ini)$/;
|
|
189
|
+
// The same, as keys in a runner's own config file
|
|
190
|
+
const RUNNER_CONFIG = /(^|\/)(jest|vitest|playwright)\.config\.[\w.]+$|(^|\/)\.mocharc(\.[\w]+)?$/;
|
|
191
|
+
const CONFIG_KEY = /(?:^|[\s{,'"])(testPathIgnorePatterns|testNamePattern|testIgnore|grepInvert|grep|invert|ignore)['"]?\s*:/g;
|
|
192
|
+
export const isTestConfig = (file) => /(^|\/)package\.json$/.test(file) || PYTEST_CONFIG.test(file) || RUNNER_CONFIG.test(file);
|
|
193
|
+
|
|
194
|
+
// The options a line of this file sets that leave tests out, with where they were set.
|
|
195
|
+
function filtersOn(file, line) {
|
|
196
|
+
let text = null;
|
|
197
|
+
let where = '';
|
|
198
|
+
if (/(^|\/)package\.json$/.test(file)) (text = (TEST_SCRIPT.exec(line) || [])[1] ?? null), (where = 'by the test script');
|
|
199
|
+
else if (PYTEST_CONFIG.test(file)) (text = (ADDOPTS.exec(line) || [])[1] ?? null), (where = "by the test runner's options");
|
|
200
|
+
else if (RUNNER_CONFIG.test(file)) (text = line.slice(1)), (where = "by the test runner's config");
|
|
201
|
+
if (text == null) return [];
|
|
202
|
+
const re = RUNNER_CONFIG.test(file) ? CONFIG_KEY : FILTER;
|
|
203
|
+
return [...text.matchAll(re)].map((m) => ({ option: m[1], where }));
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// Which characters of a line are inside a string literal: test code often writes another test as
|
|
207
|
+
// text, and a skip in that text is not a skip. One line at a time, so a string that spans lines (a
|
|
208
|
+
// template literal, a docstring) is not seen; after // or a # comment, no quote opens a string.
|
|
209
|
+
function stringMask(line) {
|
|
210
|
+
const mask = new Uint8Array(line.length);
|
|
211
|
+
let quote = '';
|
|
212
|
+
for (let i = 0; i < line.length; i++) {
|
|
213
|
+
const c = line[i];
|
|
214
|
+
if (quote) {
|
|
215
|
+
mask[i] = 1;
|
|
216
|
+
if (c === '\\') mask[++i] = 1;
|
|
217
|
+
else if (c === quote) quote = '';
|
|
218
|
+
} else if (c === '"' || c === "'" || c === '`') (quote = c), (mask[i] = 1);
|
|
219
|
+
else if ((c === '/' && line[i + 1] === '/') || (c === '#' && (i <= 1 || /\s/.test(line[i - 1])))) break;
|
|
220
|
+
}
|
|
221
|
+
return mask;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// An assertion line with its values taken out. A removed and an added line with the same shape and other
|
|
225
|
+
// values are one assertion whose expected value changed; assertTrue and assertFalse share a shape.
|
|
226
|
+
// numbers may use _ as a separator (10_000), as JS and Python allow
|
|
227
|
+
const VALUES = /(?<![\w.$])(?:\d[\d_]*(?:\.\d+)?|true|false|True|False|null|None|undefined)(?![\w$])|'(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*"|`(?:[^`\\]|\\.)*`/g;
|
|
228
|
+
function shape(line) {
|
|
229
|
+
const values = [];
|
|
230
|
+
const text = line
|
|
231
|
+
.slice(1)
|
|
232
|
+
.trim()
|
|
233
|
+
.replace(/\bassert(True|False)\b/g, (m, b) => (values.push(b), 'assert<b>'))
|
|
234
|
+
.replace(VALUES, (v) => (values.push(v.slice(0, 40)), '<v>'));
|
|
235
|
+
return { text, values };
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// The arguments of the call whose '(' is at open, split at top-level commas; null if it does not close.
|
|
239
|
+
function argsAt(t, open) {
|
|
240
|
+
const args = [];
|
|
241
|
+
let depth = 0;
|
|
242
|
+
let quote = '';
|
|
243
|
+
let start = open + 1;
|
|
244
|
+
for (let i = open; i < t.length; i++) {
|
|
245
|
+
const c = t[i];
|
|
246
|
+
if (quote) {
|
|
247
|
+
if (c === '\\') i++;
|
|
248
|
+
else if (c === quote) quote = '';
|
|
249
|
+
} else if (c === '"' || c === "'" || c === '`') quote = c;
|
|
250
|
+
else if ('([{'.includes(c)) depth++;
|
|
251
|
+
else if (')]}'.includes(c)) {
|
|
252
|
+
if (--depth === 0) return (args.push(t.slice(start, i).trim()), { args, end: i });
|
|
253
|
+
} else if (c === ',' && depth === 1) args.push(t.slice(start, i).trim()), (start = i + 1);
|
|
254
|
+
}
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// Where a top-level operator (==) starts in a Python assert, outside brackets and strings; -1 if none.
|
|
259
|
+
function topLevel(t, op) {
|
|
260
|
+
let depth = 0;
|
|
261
|
+
let quote = '';
|
|
262
|
+
for (let i = 0; i < t.length; i++) {
|
|
263
|
+
const c = t[i];
|
|
264
|
+
if (quote) {
|
|
265
|
+
if (c === '\\') i++;
|
|
266
|
+
else if (c === quote) quote = '';
|
|
267
|
+
} else if (c === '"' || c === "'") quote = c;
|
|
268
|
+
else if ('([{'.includes(c)) depth++;
|
|
269
|
+
else if (')]}'.includes(c)) depth--;
|
|
270
|
+
else if (depth === 0 && t.startsWith(op, i)) return i;
|
|
271
|
+
}
|
|
272
|
+
return -1;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
// An assertion's subject and whether it checks a value (equal, toBe, assertEqual, ==) or only that the
|
|
276
|
+
// subject is set or truthy (ok, assert(), toBeDefined, toBeTruthy, assertTrue, a bare Python assert).
|
|
277
|
+
// The same subject going from the first kind to the second is a weakened assertion.
|
|
278
|
+
const VALUE_CALL = /\b(equal|strictEqual|deepEqual|deepStrictEqual|assertEqual|assertEquals)\s*\(/;
|
|
279
|
+
const TRUTHY_CALL = /(?:^|[^\w.])(assert\.ok|assert)\s*\(|\b(assertTrue|assertIsNotNone)\s*\(/;
|
|
280
|
+
const EXPECT_VALUE = /^\.(toBe|toEqual|toStrictEqual)\s*\(/;
|
|
281
|
+
const EXPECT_TRUTHY = /^\.(toBeDefined|toBeTruthy|not\.toBeNull|not\.toBeUndefined)\s*\(\s*\)/;
|
|
282
|
+
function strength(line) {
|
|
283
|
+
const t = line.slice(1).trim().replace(/;$/, '');
|
|
284
|
+
let m = VALUE_CALL.exec(t);
|
|
285
|
+
if (m) {
|
|
286
|
+
const c = argsAt(t, m.index + m[0].length - 1);
|
|
287
|
+
return c && c.args.length >= 2 ? { subject: c.args[0], value: true, form: `${m[1]}(…)`, expected: c.args[1] } : null;
|
|
288
|
+
}
|
|
289
|
+
m = /\bexpect\s*\(/.exec(t);
|
|
290
|
+
if (m) {
|
|
291
|
+
const c = argsAt(t, m.index + m[0].length - 1);
|
|
292
|
+
if (!c) return null;
|
|
293
|
+
const rest = t.slice(c.end + 1);
|
|
294
|
+
const v = EXPECT_VALUE.exec(rest);
|
|
295
|
+
if (v) {
|
|
296
|
+
const e = argsAt(rest, rest.indexOf('('));
|
|
297
|
+
return { subject: c.args[0], value: true, form: `${v[1]}(…)`, expected: e ? e.args[0] : '' };
|
|
298
|
+
}
|
|
299
|
+
const w = EXPECT_TRUTHY.exec(rest);
|
|
300
|
+
return w ? { subject: c.args[0], value: false, form: `${w[1]}()` } : null;
|
|
301
|
+
}
|
|
302
|
+
m = TRUTHY_CALL.exec(t);
|
|
303
|
+
if (m) {
|
|
304
|
+
const c = argsAt(t, m.index + m[0].length - 1);
|
|
305
|
+
return c ? { subject: c.args[0], value: false, form: `${(m[1] || m[2]).replace('assert.', '')}(…)` } : null;
|
|
306
|
+
}
|
|
307
|
+
m = /^assert\s+(.+)$/.exec(t);
|
|
308
|
+
if (m) {
|
|
309
|
+
const comma = topLevel(m[1], ',');
|
|
310
|
+
const body = (comma >= 0 ? m[1].slice(0, comma) : m[1]).trim(); // drop a message
|
|
311
|
+
const eq = topLevel(body, '==');
|
|
312
|
+
if (eq >= 0) return { subject: body.slice(0, eq).trim(), value: true, form: '==', expected: body.slice(eq + 2).trim() };
|
|
313
|
+
const none = /^(.+?)\s+is\s+not\s+None$/.exec(body);
|
|
314
|
+
if (none) return { subject: none[1].trim(), value: false, form: 'is not None' };
|
|
315
|
+
if (/[<>!]=?|\bin\b|\bis\b/.test(body)) return null;
|
|
316
|
+
return { subject: body, value: false, form: 'a bare assert' };
|
|
317
|
+
}
|
|
318
|
+
return null;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// A catch that swallows what was thrown: catch {}, catch (e) {} or catch (e) { /* ignore */ } on one line;
|
|
322
|
+
// a catch line that opens a body which the next line closes; Python's except ...: pass, on one or two lines.
|
|
323
|
+
const CATCH_EMPTY = /\bcatch\s*(?:\(\s*[\w$]*\s*\))?\s*\{\s*(?:\/\*[^*]*\*\/|\/\/[^\n]*)?\s*\}/;
|
|
324
|
+
const CATCH_OPEN = /\bcatch\s*(?:\(\s*[\w$]*\s*\))?\s*\{\s*$/;
|
|
325
|
+
const EXCEPT_PASS = /^\s*except\b[^:]*:\s*pass\s*(#.*)?$/;
|
|
326
|
+
const EXCEPT_OPEN = /^\s*except\b[^:]*:\s*(#.*)?$/;
|
|
327
|
+
const TRY_OPEN = /\btry\s*\{|^\s*try\s*:/;
|
|
328
|
+
|
|
329
|
+
// Whether the try that a swallowing catch at line `at` (1-based) closes holds an assertion. The lines are the
|
|
330
|
+
// file as it is now when it can be read, else the lines of the diff around the catch (git's context).
|
|
331
|
+
function swallowsAssertion(lines, at, catchText) {
|
|
332
|
+
let i = at - 1;
|
|
333
|
+
if ((lines[i] || '').trim() !== catchText.trim()) {
|
|
334
|
+
const near = [-3, -2, -1, 1, 2, 3].map((d) => at - 1 + d).find((k) => (lines[k] || '').trim() === catchText.trim());
|
|
335
|
+
if (near === undefined) return false;
|
|
336
|
+
i = near;
|
|
337
|
+
}
|
|
338
|
+
for (let k = i, n = 0; k >= 0 && n < 60; k--, n++) {
|
|
339
|
+
let text = lines[k] || '';
|
|
340
|
+
const cut = k === i ? text.search(/\bcatch\b|\bexcept\b/) : -1;
|
|
341
|
+
if (cut >= 0) text = text.slice(0, cut); // try { assert(...) } catch {} on one line: the part before the catch
|
|
342
|
+
const open = text.search(TRY_OPEN);
|
|
343
|
+
if (ASSERT.test('+' + (open >= 0 ? text.slice(open).replace(TRY_OPEN, '') : text))) return true;
|
|
344
|
+
if (open >= 0) return false;
|
|
345
|
+
}
|
|
346
|
+
return false;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
// Literals in a piece of code: numbers (10_000 is 10000) and the contents of quoted strings, tagged so the
|
|
350
|
+
// number 500 and the string "500" stay apart.
|
|
351
|
+
const LITERAL_RE = /(?<![\w.$])-?\d[\d_]*(?:\.\d+)?(?![\w$])|'((?:[^'\\]|\\.)*)'|"((?:[^"\\]|\\.)*)"/g;
|
|
352
|
+
function literals(text) {
|
|
353
|
+
return [...String(text).matchAll(LITERAL_RE)].map((m) => (m[1] !== undefined || m[2] !== undefined ? `s:${m[1] ?? m[2]}` : `n:${Number(m[0].replace(/_/g, ''))}`));
|
|
354
|
+
}
|
|
355
|
+
// Values too common to say anything when code returns them
|
|
356
|
+
const DULL = new Set(['n:0', 'n:1', 'n:-1', 'n:2', 's:']);
|
|
357
|
+
|
|
358
|
+
// What a test line expects, when it compares a call on literal inputs with one literal value.
|
|
359
|
+
function expectationOf(line) {
|
|
360
|
+
const st = strength('+' + line);
|
|
361
|
+
if (!st || !st.value || !st.expected) return null;
|
|
362
|
+
const want = literals(st.expected);
|
|
363
|
+
if (want.length !== 1 || st.expected.replace(/^[-'"]|['"]$/g, '').length === 0) return null;
|
|
364
|
+
return { inputs: new Set(literals(st.subject)), expected: want[0] };
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
// A return of one literal under a condition, on the same line or the line before: if (x === 250) return 500;
|
|
368
|
+
const RETURN_LITERAL = /\breturn\s+(-?\d[\d_]*(?:\.\d+)?|'(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*")\s*;?\s*\}?\s*$/;
|
|
369
|
+
const SAME_LINE_IF = /\bif\s*\(?(.*?)\)?\s*(?:\{|:)?\s*return\b/;
|
|
370
|
+
|
|
371
|
+
// A test runner's own markers, read by code outside the tests: the code can then behave one way under the
|
|
372
|
+
// runner and another way for users. NODE_ENV === 'test' is not one: apps use it to pick a test database.
|
|
373
|
+
const RUNNER_MARKER = /\b(NODE_TEST_CONTEXT|JEST_WORKER_ID|VITEST_WORKER_ID|VITEST_POOL_ID|PYTEST_CURRENT_TEST)\b|(['"]pytest['"]\s+in\s+sys\.modules)|\b(testing\.Testing\(\))/;
|
|
374
|
+
// Test setup that may read them: pytest's conftest.py and the setup files jest and vitest load
|
|
375
|
+
const TEST_SETUP = /(^|\/)(conftest\.py|setupTests\.[cm]?[jt]sx?|(jest|vitest)\.setup\.[cm]?[jt]sx?)$/;
|
|
376
|
+
|
|
377
|
+
// CI files, and the commands in them that run the tests
|
|
378
|
+
const CI_FILE = /(^|\/)\.github\/workflows\/[^/]+\.ya?ml$|(^|\/)\.gitlab-ci\.ya?ml$|(^|\/)\.circleci\/config\.ya?ml$|(^|\/)(azure|bitbucket)-pipelines\.ya?ml$/;
|
|
379
|
+
const TEST_CMD = /\b(?:(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?test\b|node\s+--test\b|(?:npx\s+)?(?:jest|vitest|mocha)\b|(?:python3?\s+-m\s+)?pytest\b|python3?\s+-m\s+unittest\b|go\s+test\b|cargo\s+test\b|make\s+test\b)/;
|
|
380
|
+
const CI_ALWAYS_PASSES = new RegExp(TEST_CMD.source + String.raw`[^\n]*?(\|\|\s*(true|:|exit\s+0|echo\b)|;\s*(true|exit\s+0)\s*$)`);
|
|
381
|
+
const CI_IGNORE = /^\s*(continue-on-error|allow_failure)\s*:\s*true\b/;
|
|
382
|
+
|
|
383
|
+
// Whether the YAML block a key at line `at` belongs to (its step or job) runs the tests.
|
|
384
|
+
function blockRunsTests(lines, at, text) {
|
|
385
|
+
let i = at - 1;
|
|
386
|
+
if ((lines[i] || '').trim() !== text.trim()) {
|
|
387
|
+
const near = [-3, -2, -1, 1, 2, 3].map((d) => at - 1 + d).find((k) => (lines[k] || '').trim() === text.trim());
|
|
388
|
+
if (near === undefined) return false;
|
|
389
|
+
i = near;
|
|
390
|
+
}
|
|
391
|
+
const indent = (l) => l.length - l.trimStart().length;
|
|
392
|
+
const own = indent(lines[i]);
|
|
393
|
+
let start = i;
|
|
394
|
+
while (start > 0 && !((lines[start - 1] || '').trim() && indent(lines[start - 1]) < own)) start--;
|
|
395
|
+
start = Math.max(0, start - 1);
|
|
396
|
+
const top = indent(lines[start] || '');
|
|
397
|
+
for (let k = start; k < lines.length && k < start + 200; k++) {
|
|
398
|
+
if (k > start && (lines[k] || '').trim() && indent(lines[k]) <= top) break;
|
|
399
|
+
if (TEST_CMD.test(lines[k] || '')) return true;
|
|
400
|
+
}
|
|
401
|
+
return false;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
// re found in the code of the line, not only inside its strings
|
|
405
|
+
function inCode(re, line, mask) {
|
|
406
|
+
const g = new RegExp(re.source, re.flags.includes('g') ? re.flags : re.flags + 'g');
|
|
407
|
+
for (const m of line.matchAll(g)) if (!mask()[m.index + (m[0].length - m[0].trimStart().length)]) return true;
|
|
408
|
+
return false;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
// What the finish check prints: the events' text, in order, each once.
|
|
412
|
+
export function tamperFindings(nameStatus, diff, opts) {
|
|
413
|
+
return tamperEvents(nameStatus, diff, opts).map((e) => e.why);
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
// Each finding as { kind, file, why }; file is undefined for the count of removed assertions,
|
|
417
|
+
// which spans the whole diff. A why already seen is not repeated.
|
|
418
|
+
// read(file), when given, returns the file as it is now (or null): the finish check passes one, the audit
|
|
419
|
+
// does not and relies on the diff's context lines. tests, when given, is the repo's test files as
|
|
420
|
+
// "path\0content" (or a function returning them, called only when it is needed), for the expected values a
|
|
421
|
+
// hardcoded answer would copy.
|
|
422
|
+
export function tamperEvents(nameStatus, diff, { read, tests } = {}) {
|
|
423
|
+
const events = [];
|
|
424
|
+
const seen = new Set();
|
|
425
|
+
const add = (kind, file, why) => {
|
|
426
|
+
if (seen.has(why)) return;
|
|
427
|
+
seen.add(why);
|
|
428
|
+
events.push({ kind, file, why });
|
|
429
|
+
};
|
|
166
430
|
for (const line of nameStatus.split('\n')) {
|
|
167
431
|
const [status, file] = line.split(/\s+/);
|
|
168
|
-
if (status === 'D' && file &&
|
|
432
|
+
if (status === 'D' && file && isTestFile(file)) add(DELETED, file, `test file deleted: ${file}`);
|
|
169
433
|
}
|
|
170
434
|
let current = '';
|
|
171
435
|
let removedAsserts = 0;
|
|
172
436
|
let addedAsserts = 0;
|
|
173
437
|
let removed = new Set(); // this file's removed assertions, to spot one coming back as a comment
|
|
438
|
+
let removedShapes = new Map(); // this file's removed assertions by shape, to spot a changed expected value
|
|
439
|
+
let removedValueChecks = new Map(); // this file's removed value checks by subject, to spot a weakened one
|
|
440
|
+
const changedIn = new Set(); // one changed-value finding per file
|
|
441
|
+
let newLine = 0; // the line number in the file as it is now, from the hunk headers
|
|
442
|
+
let fileLines = []; // this file's lines from the diff (added and context), by line number, for the audit
|
|
443
|
+
let pendingCatch = 0; // a catch or except line whose body may be closed or passed on the next line
|
|
444
|
+
let prevText = ''; // the previous added or context line of this file
|
|
445
|
+
const expectations = []; // from test lines in the diff, then from the repo's test files
|
|
446
|
+
const returns = []; // literal returns under a condition in code files, checked after the diff is read
|
|
447
|
+
const ciRemoved = []; // CI files that lost a line running the tests
|
|
448
|
+
let ciAdded = 0; // lines running the tests added to any CI file
|
|
449
|
+
const weakenedIn = new Set(); // one weakened-matcher finding per file
|
|
174
450
|
let hadFallback = false; // the test script already ended in `|| true` before this session
|
|
451
|
+
let hadFilters = new Set(); // options that left tests out before this session
|
|
175
452
|
for (const line of diff.split('\n')) {
|
|
176
453
|
if (line.startsWith('+++ ')) {
|
|
177
454
|
current = line.replace(/^\+\+\+ (b\/)?/, '');
|
|
178
455
|
removed = new Set();
|
|
456
|
+
removedShapes = new Map();
|
|
457
|
+
removedValueChecks = new Map();
|
|
458
|
+
fileLines = [];
|
|
459
|
+
pendingCatch = 0;
|
|
460
|
+
prevText = '';
|
|
179
461
|
hadFallback = false;
|
|
462
|
+
hadFilters = new Set();
|
|
180
463
|
continue;
|
|
181
464
|
}
|
|
182
465
|
if (line.startsWith('--- ')) continue;
|
|
183
|
-
|
|
466
|
+
if (line.startsWith('@@')) {
|
|
467
|
+
newLine = Number((/\+(\d+)/.exec(line) || [])[1] || 0);
|
|
468
|
+
continue;
|
|
469
|
+
}
|
|
470
|
+
const at = line.startsWith('-') ? 0 : newLine;
|
|
471
|
+
if (!line.startsWith('-')) newLine++;
|
|
472
|
+
if (at && at <= 20000) fileLines[at - 1] = line.slice(1);
|
|
473
|
+
const inTest = isTestFile(current);
|
|
474
|
+
if (CI_FILE.test(current) && !COMMENT.test(line)) {
|
|
475
|
+
const text = line.slice(1, 10001);
|
|
476
|
+
if (line.startsWith('-') && TEST_CMD.test(text)) ciRemoved.push(current);
|
|
477
|
+
if (line.startsWith('+') && TEST_CMD.test(text)) ciAdded++;
|
|
478
|
+
if (line.startsWith('+') && CI_ALWAYS_PASSES.test(text)) add(CHECKS_OFF, current, `the CI test step was changed to always pass in ${current}`);
|
|
479
|
+
if (line.startsWith('+') && CI_IGNORE.test(text)) {
|
|
480
|
+
const now = read ? read(current) : null;
|
|
481
|
+
if (blockRunsTests(now == null ? fileLines : now.split('\n'), at, text)) add(CHECKS_OFF, current, `the CI test step was changed to always pass in ${current}`);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
const before = prevText;
|
|
485
|
+
if (!line.startsWith('-')) prevText = line.slice(1, 10001);
|
|
486
|
+
if (inTest && !line.startsWith('-')) {
|
|
487
|
+
const e = expectationOf(line.slice(1, 10001));
|
|
488
|
+
if (e) expectations.push(e);
|
|
489
|
+
} else if (!inTest && line.startsWith('+') && !isTestConfig(current)) {
|
|
490
|
+
const marker = !COMMENT.test(line) && !TEST_SETUP.test(current) && !FIXTURE.test(current) && RUNNER_MARKER.exec(line.slice(1, 10001));
|
|
491
|
+
if (marker) add(UNDER_TEST, current, `${current} checks whether it is running under a test (${(marker[1] || marker[2] || marker[3]).replace(/['"]/g, '')})`);
|
|
492
|
+
const r = RETURN_LITERAL.exec(line.slice(1, 10001));
|
|
493
|
+
if (r) {
|
|
494
|
+
const same = SAME_LINE_IF.exec(line.slice(1, 10001));
|
|
495
|
+
const cond = same ? same[1] : /^\s*if\b/.test(before) ? before : null;
|
|
496
|
+
if (cond != null) returns.push({ file: current, text: r[1], value: literals(r[1])[0], given: literals(cond) });
|
|
497
|
+
}
|
|
498
|
+
}
|
|
184
499
|
const probe = line.slice(0, 10000); // the patterns can take quadratic time on a long line
|
|
500
|
+
let lineMask = null;
|
|
501
|
+
const mask = () => (lineMask ||= stringMask(probe));
|
|
185
502
|
if (/(^|\/)package\.json$/.test(current) && ALWAYS_PASSES.test(probe)) {
|
|
186
503
|
if (line.startsWith('-')) hadFallback = true;
|
|
187
|
-
else if (!hadFallback)
|
|
504
|
+
else if (!hadFallback) add(CHECKS_OFF, current, `the test script was changed to always exit 0 in ${current}`);
|
|
505
|
+
}
|
|
506
|
+
if (isTestConfig(current)) {
|
|
507
|
+
for (const f of filtersOn(current, probe)) {
|
|
508
|
+
if (line.startsWith('-')) hadFilters.add(f.option);
|
|
509
|
+
else if (line.startsWith('+') && !hadFilters.has(f.option)) add(SKIPPED, current, `tests were left out ${f.where} (${f.option}) in ${current}`);
|
|
510
|
+
}
|
|
188
511
|
}
|
|
189
512
|
if (line.startsWith('+')) {
|
|
190
|
-
for (const t of TAMPER_ADDED) if (t.re
|
|
513
|
+
for (const t of TAMPER_ADDED) if (inCode(t.re, probe, mask)) add(t.kind, current, `${t.why} in ${current}`);
|
|
191
514
|
if (!inTest) continue;
|
|
192
|
-
|
|
515
|
+
const text = probe.slice(1);
|
|
516
|
+
let swallowAt = 0;
|
|
517
|
+
let swallowText = '';
|
|
518
|
+
if (pendingCatch && at === pendingCatch.at + 1 && (text.trim() === '}' || text.trim() === 'pass')) [swallowAt, swallowText] = [pendingCatch.at, pendingCatch.text];
|
|
519
|
+
pendingCatch = 0;
|
|
520
|
+
if (inCode(CATCH_EMPTY, probe, mask) || EXCEPT_PASS.test(text)) [swallowAt, swallowText] = [at, text];
|
|
521
|
+
else if (inCode(CATCH_OPEN, probe, mask) || EXCEPT_OPEN.test(text)) pendingCatch = { at, text };
|
|
522
|
+
if (swallowAt) {
|
|
523
|
+
const now = read ? read(current) : null;
|
|
524
|
+
const lines = now == null ? fileLines : now.split('\n');
|
|
525
|
+
if (swallowsAssertion(lines, swallowAt, swallowText)) add(WEAKENED, current, `an assertion was wrapped in a try that swallows its failure in ${current}`);
|
|
526
|
+
}
|
|
527
|
+
for (const t of TAMPER_ADDED_IN_TESTS) if (inCode(t.re, probe, mask)) add(t.kind, current, `${t.why} in ${current}`);
|
|
193
528
|
if (COMMENT.test(line)) {
|
|
194
|
-
if (removed.has(line.replace(COMMENT, '').trim()))
|
|
195
|
-
} else if (ASSERT
|
|
196
|
-
|
|
529
|
+
if (removed.has(line.replace(COMMENT, '').trim())) add(WEAKENED, current, `an assertion was commented out in ${current}`);
|
|
530
|
+
} else if (inCode(ASSERT, probe, mask)) {
|
|
531
|
+
addedAsserts++;
|
|
532
|
+
const now = shape(probe);
|
|
533
|
+
const before = removedShapes.get(now.text);
|
|
534
|
+
const i = before ? before.findIndex((v, k) => v !== now.values[k]) : -1;
|
|
535
|
+
if (i >= 0 && !changedIn.has(current)) {
|
|
536
|
+
changedIn.add(current);
|
|
537
|
+
add(EXPECTED, current, `an expected value was changed in ${current} (${before[i]} → ${now.values[i]})`);
|
|
538
|
+
}
|
|
539
|
+
const st = strength(probe);
|
|
540
|
+
if (st && !st.value && removedValueChecks.has(st.subject) && !weakenedIn.has(current)) {
|
|
541
|
+
weakenedIn.add(current);
|
|
542
|
+
add(WEAKENED, current, `an assertion was weakened in ${current} (${removedValueChecks.get(st.subject)} became ${st.form})`);
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
} else if (line.startsWith('-') && inTest && !COMMENT.test(line) && inCode(ASSERT, probe, mask)) {
|
|
197
546
|
removedAsserts++;
|
|
198
547
|
removed.add(line.slice(1).trim());
|
|
548
|
+
const was = shape(probe);
|
|
549
|
+
removedShapes.set(was.text, was.values);
|
|
550
|
+
const st = strength(probe);
|
|
551
|
+
if (st && st.value) removedValueChecks.set(st.subject, st.form);
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
if (ciRemoved.length > ciAdded) add(CHECKS_OFF, ciRemoved[0], `the test step was removed from CI in ${ciRemoved[0]}`);
|
|
555
|
+
const candidates = returns.filter((r) => !DULL.has(r.value));
|
|
556
|
+
if (candidates.length) {
|
|
557
|
+
for (const t of (typeof tests === 'function' ? tests() : tests) || []) {
|
|
558
|
+
for (const l of String(t).slice(String(t).indexOf('\0') + 1).split('\n')) {
|
|
559
|
+
const e = expectationOf(l.slice(0, 10000));
|
|
560
|
+
if (e) expectations.push(e);
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
for (const r of candidates) {
|
|
564
|
+
// numbers first: a string in the condition is often a key ("price"), the number is the input
|
|
565
|
+
const order = [...r.given.filter((g) => g.startsWith('n:')), ...r.given.filter((g) => g.startsWith('s:'))];
|
|
566
|
+
const given = order.find((g) => !DULL.has(g) && expectations.some((e) => e.expected === r.value && e.inputs.has(g)));
|
|
567
|
+
const show = (g) => (g.startsWith('n:') ? g.slice(2) : JSON.stringify(g.slice(2)));
|
|
568
|
+
if (given) add(HARDCODED, r.file, `${r.file} returns ${r.text} when given ${show(given)}, the exact value a test expects`);
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
// Expected values, replaced tests and test data, by the rules ported from noskip (src/noskip.js). They need
|
|
572
|
+
// the files as they are now, so only the finish check runs them.
|
|
573
|
+
if (read) {
|
|
574
|
+
const replacedIn = new Set();
|
|
575
|
+
const WORDS = {
|
|
576
|
+
expected: (c) => `an expected value was changed in ${c.file} (${c.from} → ${c.to})`,
|
|
577
|
+
call: (c) => `the call a test checks was changed and its expected value kept in ${c.file} (${c.from} → ${c.to})`,
|
|
578
|
+
flipped: (c) => `an assertion was flipped to its opposite in ${c.file} (${c.from} → ${c.to})`,
|
|
579
|
+
assertion: (c) => `an assertion was changed in ${c.file} (${c.from} → ${c.to})`,
|
|
580
|
+
dropped: (c) => `a case was removed from a test list in ${c.file} (${c.from})`,
|
|
581
|
+
reordered: (c) => `test data was reordered in ${c.file} (${c.from} moved)`,
|
|
582
|
+
};
|
|
583
|
+
for (const c of testChanges(diff, { read, isTestFile })) {
|
|
584
|
+
if (c.type === 'skipped') add(SKIPPED, c.file, `a test that already existed was given a skip option in ${c.file} ("${c.from}")`);
|
|
585
|
+
else if (c.type === 'replaced') {
|
|
586
|
+
if (!replacedIn.has(c.file)) add(REPLACED, c.file, `a test was replaced by a different one in ${c.file} ("${c.from}" → "${c.to}"${c.where ? ` in ${c.where}` : ''})`);
|
|
587
|
+
replacedIn.add(c.file);
|
|
588
|
+
} else if (c.type === 'weakened') {
|
|
589
|
+
if (!weakenedIn.has(c.file)) add(WEAKENED, c.file, `an assertion was weakened in ${c.file} (${c.from} became ${c.to})`);
|
|
590
|
+
weakenedIn.add(c.file);
|
|
591
|
+
} else {
|
|
592
|
+
if (!changedIn.has(c.file)) add(EXPECTED, c.file, WORDS[c.what](c));
|
|
593
|
+
changedIn.add(c.file);
|
|
594
|
+
}
|
|
199
595
|
}
|
|
200
596
|
}
|
|
201
597
|
if (removedAsserts - addedAsserts >= 2) {
|
|
202
|
-
|
|
598
|
+
add(WEAKENED, undefined, `${removedAsserts - addedAsserts} assertions removed from test files`);
|
|
203
599
|
}
|
|
204
|
-
return
|
|
600
|
+
return events;
|
|
205
601
|
}
|
|
206
602
|
|
|
207
603
|
// ---------- redaction for anything that leaves the machine ----------
|