jules-orchestrator-kit 0.72.2 → 0.73.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agent/prompts/{Overseer.md → Auditor.md} +3 -3
- package/.agent/prompts/{Alchemist.md → Database.md} +1 -1
- package/.agent/prompts/Debugger.md +25 -0
- package/.agent/prompts/{Scribe.md → Docs.md} +8 -5
- package/.agent/prompts/{Spectator.md → E2E.md} +9 -6
- package/.agent/prompts/{Janitor.md → Hygiene.md} +2 -2
- package/.agent/prompts/{Bolt.md → Performance.md} +1 -1
- package/.agent/prompts/Resilience.md +20 -0
- package/.agent/prompts/Security.md +21 -0
- package/.agent/prompts/Testing.md +30 -0
- package/.agent/prompts/Types.md +19 -0
- package/.agent/rules/jules-protocol.md +4 -3
- package/AGENTS.md +78 -96
- package/CHANGELOG.md +193 -0
- package/JULES_RULES_TEMPLATE.md +83 -96
- package/LICENSE +1 -1
- package/README.md +77 -439
- package/ROADMAP_V1.md +22 -132
- package/bin/agentctl.mjs +443 -144
- package/bin/init.js +6 -3
- package/index.mjs +9 -6
- package/package.json +1 -1
- package/scripts/asset-integrity-check.mjs +1 -1
- package/scripts/doc-sync-check.mjs +47 -0
- package/scripts/generate-command-reference.mjs +39 -0
- package/scripts/jules-dispatch.mjs +12 -113
- package/scripts/jules-merge-swarm.mjs +8 -196
- package/scripts/jules-patch.mjs +7 -8
- package/scripts/jules-queue-runner.mjs +6 -8
- package/scripts/jules-scan-todos.mjs +10 -38
- package/scripts/jules-self-audit.mjs +8 -139
- package/scripts/jules-status.mjs +32 -38
- package/scripts/jules-webhook-receiver.mjs +1 -1
- package/src/assertions.mjs +5 -50
- package/src/bidi-guard.mjs +36 -0
- package/src/budget.mjs +3 -14
- package/src/config.mjs +2 -6
- package/src/dashboard.mjs +7 -9
- package/src/dispatch.mjs +212 -0
- package/src/engine.mjs +40 -16
- package/src/evidence.mjs +10 -41
- package/src/execution-envelope.mjs +13 -1
- package/src/flaky-ledger.mjs +1 -1
- package/src/fs-atomic.mjs +72 -0
- package/src/git.mjs +298 -27
- package/src/mcp.mjs +296 -7
- package/src/memory.mjs +0 -0
- package/src/merge-swarm.mjs +202 -0
- package/src/ops/cli-intent.mjs +1 -0
- package/src/ops/command-registry.mjs +796 -70
- package/src/ops/doctor-registry.mjs +134 -47
- package/src/ops/handover.mjs +3 -27
- package/src/ops/pr-harvest.mjs +1 -1
- package/src/prompt-guard.mjs +33 -3
- package/src/provider.mjs +51 -7
- package/src/remediation.mjs +2 -2
- package/src/role-resolver.mjs +113 -3
- package/src/router.mjs +19 -11
- package/src/runtime-env.mjs +67 -0
- package/src/scaffold.mjs +3 -1
- package/src/scope-guard.mjs +249 -0
- package/src/secret-scanner.mjs +530 -0
- package/src/security.mjs +81 -2972
- package/src/self-audit.mjs +140 -0
- package/src/session-ops.mjs +29 -0
- package/src/stability.mjs +8 -1
- package/src/stack-detector.mjs +5 -2
- package/src/state.mjs +45 -0
- package/src/swarm.mjs +76 -0
- package/src/task-optimizer.mjs +34 -7
- package/src/telemetry.mjs +23 -0
- package/src/test-tamper-guard.mjs +2173 -0
- package/src/todo-scanner.mjs +129 -0
- package/src/web-templates.mjs +3 -3
- package/src/webhook.mjs +10 -3
- package/src/wizard-init.mjs +12 -20
- package/src/wizard-oracle.mjs +4 -3
- package/src/wizard-task.mjs +37 -9
- package/.agent/prompts/Sentinel.md +0 -18
- package/scripts/utils.mjs +0 -241
|
@@ -0,0 +1,2173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test-tamper detection: did the change edit the oracle instead of the code?
|
|
3
|
+
*
|
|
4
|
+
* Split out of src/security.mjs (P05). Six checks (skip injection, vacuous
|
|
5
|
+
* assertions, commented-out assertions, removal, weakening, expectation
|
|
6
|
+
* rewrites) run over the test files a diff touches, with the multi-language
|
|
7
|
+
* statement parser they depend on: literal blanking, comment stripping,
|
|
8
|
+
* statement reassembly across physical lines, and the expectation-rewrite
|
|
9
|
+
* pairing that runs on statements rather than lines.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { isTestPath } from "./test-paths.mjs";
|
|
13
|
+
|
|
14
|
+
// ---------------------------------------------------------------------------
|
|
15
|
+
// Statement-level expectation-rewrite detection (multi-line aware)
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
//
|
|
18
|
+
// The original pairing ran on physical lines. That caught
|
|
19
|
+
// `assert.equal(add(1, 2), 3);` becoming `assert.equal(add(1, 2), -1);`, but
|
|
20
|
+
// the same edit walked straight through the moment it was wrapped across
|
|
21
|
+
// lines — which every formatter does the day a line runs long, and which an
|
|
22
|
+
// agent doing an ordinary reformat does on its own:
|
|
23
|
+
//
|
|
24
|
+
// -assert.equal(
|
|
25
|
+
// - add(1, 2),
|
|
26
|
+
// - 3
|
|
27
|
+
// -);
|
|
28
|
+
// +assert.equal(
|
|
29
|
+
// + add(1, 2),
|
|
30
|
+
// + -1
|
|
31
|
+
// +);
|
|
32
|
+
//
|
|
33
|
+
// The value lives on a line that carries no assertion keyword, so neither
|
|
34
|
+
// side ever paired, and the suite went from checking that addition works to
|
|
35
|
+
// certifying that it is broken. The statement, not the line, is the unit an
|
|
36
|
+
// agent rewrites, so the pairing now runs on reassembled statements: a run
|
|
37
|
+
// of physical lines joined while its delimiters are unbalanced, one of its
|
|
38
|
+
// strings or comments is still open, a Python line-continuation is pending,
|
|
39
|
+
// or the next line cannot start a statement of its own. The hunk's context
|
|
40
|
+
// lines belong to both images and are what make the reassembly possible;
|
|
41
|
+
// when they are absent (a zero-context diff) the only pair that can survive
|
|
42
|
+
// is the one where each image is a single fragment, and that pair is taken
|
|
43
|
+
// too, requiring a literal placeholder so a code change cannot masquerade as
|
|
44
|
+
// a value change.
|
|
45
|
+
//
|
|
46
|
+
// The pairing rule is unchanged in spirit: the two sides must be the *same*
|
|
47
|
+
// assertion — identical once every literal is blanked out — with different
|
|
48
|
+
// values. That does not distinguish an attack from a deliberate change of
|
|
49
|
+
// spec; nothing can, from a diff alone. This reports rather than decides,
|
|
50
|
+
// and `--allow-test-change expectation` is the answer when the new
|
|
51
|
+
// expectation is the correct one. Narrow on purpose: the blunt
|
|
52
|
+
// `--allow-test-modifications` turns off the other five checks too, and a
|
|
53
|
+
// check that can only be answered by disabling its neighbours ends up
|
|
54
|
+
// disabling its neighbours.
|
|
55
|
+
|
|
56
|
+
// An assertion that states a *specific* expected value. Counting assertions
|
|
57
|
+
// alone let a test be gutted while looking untouched: swapping
|
|
58
|
+
// `assert.strictEqual(add(2,3), 5)` for `assert.ok(add(2,3) !== undefined)`
|
|
59
|
+
// removes one and adds one, so `removed > added` stayed false and the guard
|
|
60
|
+
// said nothing — while the suite stopped checking the answer.
|
|
61
|
+
//
|
|
62
|
+
// The `expect` argument span is a bounded lazy match rather than `[^)]*` so
|
|
63
|
+
// that a call split across lines with a nested call in its arguments
|
|
64
|
+
// (`expect(\n formatInvoice(bill)\n).toBe(…`) still recognises the chain.
|
|
65
|
+
// The bound is a guess: an argument list longer than 240 characters is
|
|
66
|
+
// rarer than a missed chain.
|
|
67
|
+
// The dialect list is not decoration. `assertEqual` was recognised only
|
|
68
|
+
// because `\.?` made the dot optional and the `i` flag let `Equal` match
|
|
69
|
+
// `equal`; `assertEquals`, one letter longer, fell out of the pattern and
|
|
70
|
+
// took JUnit, PHPUnit, Minitest, RSpec and XCTest with it. The weak forms
|
|
71
|
+
// — assertTrue, assertNotNull, XCTAssertTrue — are deliberately absent:
|
|
72
|
+
// they state no expected value, so their arrival in place of one of these
|
|
73
|
+
// is a weakening, which is a finding of its own.
|
|
74
|
+
const SPECIFIC_ASSERTION = new RegExp(
|
|
75
|
+
[
|
|
76
|
+
"\\bassert(?:\\.strict)?\\.?(?:strictEqual|deepStrictEqual|deepEqual|notStrictEqual|notDeepStrictEqual|equal|notEqual|match|doesNotMatch|throws|rejects|doesNotThrow)\\s*\\(",
|
|
77
|
+
"\\bexpect\\s*\\([\\s\\S]{0,240}?\\)\\s*\\.(?:toBe|toEqual|toStrictEqual|toMatch|toMatchObject|toContain|toHaveBeenCalledWith|toThrow|toHaveLength|toBeCloseTo)\\s*\\(",
|
|
78
|
+
"\\bassert\\.(?:equals|deepEquals|include|lengthOf)\\s*\\(",
|
|
79
|
+
"assert_eq!|assert_ne!",
|
|
80
|
+
// The bare comparison form. `assert add(1, 2) == 3` is how pytest is
|
|
81
|
+
// actually written, and Rust's `assert!(a == b)` and Elixir's
|
|
82
|
+
// `assert f(x) == 3` follow it; none of them name a comparison
|
|
83
|
+
// function, so a list of function names could never reach them.
|
|
84
|
+
// Equality only. `assert!(x != 0)` names no expected value — it is the
|
|
85
|
+
// weaker claim you arrive at by giving one up, and counting it as
|
|
86
|
+
// specific would make the downgrade from `assert_eq!(x, 5)` invisible to
|
|
87
|
+
// the weakening check.
|
|
88
|
+
"\\bassert\\s+[^\\n]*(?:===|==)(?!=)",
|
|
89
|
+
"\\bassert!\\s*\\([^\\n]*==(?!=)",
|
|
90
|
+
"\\bt\\.(?:Errorf|Fatalf)\\s*\\(",
|
|
91
|
+
"\\brequire\\.(?:Equal|NotEqual|Len|Contains|Error|NoError)\\s*\\(",
|
|
92
|
+
// Python unittest, stated rather than inherited from the optional dot.
|
|
93
|
+
"\\bassert(?:Equal|NotEqual|AlmostEqual|NotAlmostEqual|Regex|NotRegex|Raises|In|NotIn|Is|IsNot|ListEqual|DictEqual|SetEqual|TupleEqual|CountEqual|Greater|Less|GreaterEqual|LessEqual)\\s*\\(",
|
|
94
|
+
// JUnit / TestNG / PHPUnit
|
|
95
|
+
"\\bassert(?:Equals|NotEquals|Same|NotSame|ArrayEquals|IterableEquals|LinesMatch|Count|StringContainsString|StringEqualsFile|InstanceOf|Contains|Throws)\\s*\\(",
|
|
96
|
+
"\\bassertThat\\s*\\([\\s\\S]{0,240}?\\)\\s*\\.(?:isEqualTo|isSameAs|contains|containsExactly|hasSize|isCloseTo|matches)\\s*\\(",
|
|
97
|
+
// Minitest
|
|
98
|
+
"\\b(?:assert|refute)_(?:equal|includes|match|nil|same|in_delta|in_epsilon|raises|empty|operator|predicate)\\b",
|
|
99
|
+
// RSpec
|
|
100
|
+
"\\bexpect\\s*\\([\\s\\S]{0,240}?\\)\\s*\\.(?:to|not_to|to_not)\\s+(?:eq|eql|equal|be|be_within|match|include|contain_exactly|match_array|have_attributes|raise_error|start_with|end_with)\\b",
|
|
101
|
+
// XCTest
|
|
102
|
+
"\\bXCTAssert(?:Equal|NotEqual|EqualWithAccuracy|Identical|NotIdentical|GreaterThan|LessThan|GreaterThanOrEqual|LessThanOrEqual|ThrowsError|NoThrow)\\s*\\(",
|
|
103
|
+
// chai — a dot chain, where RSpec's is a space. `expect(x).to.equal(3)`
|
|
104
|
+
// never reached the RSpec branch, so swapping it for `.toBeDefined()`
|
|
105
|
+
// lost no *specific* assertion and the weakening check stayed quiet.
|
|
106
|
+
"\\bexpect\\s*\\([\\s\\S]{0,240}?\\)\\s*\\.to(?:\\.[a-z]+)*\\.(?:equal|equals|eql|eqls|closeTo|match|include|contain|members|throw|string|lengthOf|above|below|least|most|within)\\s*\\(",
|
|
107
|
+
// node-tap and its relatives, where the assertion hangs off whatever the
|
|
108
|
+
// sub-test callback named its argument — `ct` as often as `t`. Bounded to
|
|
109
|
+
// a short receiver so `results.match(...)` on an ordinary object is not
|
|
110
|
+
// mistaken for an assertion; a heuristic, and stated as one.
|
|
111
|
+
// `is`/`not` are AVA's value assertions (`t.is(actual, expected)`),
|
|
112
|
+
// measurable on P-Limit's root `test.js`: without them the guard watched a
|
|
113
|
+
// suite whose every check was `t.is(...)` and counted no assertions at
|
|
114
|
+
// all.
|
|
115
|
+
"\\b[a-z_$][a-z0-9_$]{0,2}\\.(?:equal|equals|same|strictSame|deepEqual|notEqual|notSame|match|hasStrict|type|throws|rejects|is|not|like)\\s*\\(",
|
|
116
|
+
].join("|"),
|
|
117
|
+
"i"
|
|
118
|
+
);
|
|
119
|
+
const isSpecificAssertion = (str) => SPECIFIC_ASSERTION.test(str);
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* An assertion with every literal value replaced by a placeholder.
|
|
123
|
+
*
|
|
124
|
+
* Two lines that normalize to the same string are the same assertion about
|
|
125
|
+
* the same expression; whatever differs between them is a value.
|
|
126
|
+
*
|
|
127
|
+
* The number form covers hex, octal, binary, underscores and exponents: the
|
|
128
|
+
* original decimal-only regex never blanked `0xFF`, so an expectation
|
|
129
|
+
* rewritten from `0xFF` to `0xFE` normalized to two *different* shapes and
|
|
130
|
+
* the pair was never formed.
|
|
131
|
+
*/
|
|
132
|
+
const blankLiterals = (str) =>
|
|
133
|
+
str
|
|
134
|
+
.replace(/(['"`])(?:\\.|(?!\1)[^\\])*\1/g, "\u0000S")
|
|
135
|
+
// A regex literal is an expected value like any other. Without this,
|
|
136
|
+
// `toMatch(/Hello World/)` and `toMatch(/Hello Tampered/)` normalized to
|
|
137
|
+
// two different shapes, never met in a bucket, and the rewrite was
|
|
138
|
+
// reported as neither a change nor a loss — one specific assertion out,
|
|
139
|
+
// one in, and silence. Runs after the string pass so a `/` inside a
|
|
140
|
+
// string is already gone, and before the number pass so a pattern
|
|
141
|
+
// containing digits collapses whole.
|
|
142
|
+
//
|
|
143
|
+
// The lookbehind is what separates a regex from a division: an operand
|
|
144
|
+
// never precedes `/` here, only an opening paren, a comma, or an
|
|
145
|
+
// operator, which is where a test's expected pattern actually sits.
|
|
146
|
+
.replace(
|
|
147
|
+
/(?<=[(,=:[!&|?{;]\s{0,8})\/(?![*/])(?:\\.|\[(?:\\.|[^\]\\\n])*\]|[^/\\\n])+\/[dgimsuvy]*/g,
|
|
148
|
+
"\u0000R"
|
|
149
|
+
)
|
|
150
|
+
// The sign belongs to the literal: without it `3` and `-1` normalized to
|
|
151
|
+
// different shapes and the rewritten expectation was never paired.
|
|
152
|
+
.replace(
|
|
153
|
+
/(?<![\w$])(?:0[xX][0-9a-fA-F_]+|0[bB][01_]+|0[oO][0-7_]+|-?\d[\d_]*(?:\.[\d_]+)?(?:[eE][+-]?\d+)?)/g,
|
|
154
|
+
"\u0000N"
|
|
155
|
+
)
|
|
156
|
+
// A JS conditional expectation `cond ? a : b` whose branches carry
|
|
157
|
+
// literals is an expected value, whatever it evaluates to. Without this,
|
|
158
|
+
// `expect(x).toBe(3)` becoming `expect(x).toBe(x === 2 ? 3 : -1)` — the
|
|
159
|
+
// JavaScript spelling of F03's Python ternary — landed in a different
|
|
160
|
+
// shape bucket and never paired. Only a conditional holding a collapsed
|
|
161
|
+
// literal collapses (so the `?` of an optional chain or a ternary over
|
|
162
|
+
// bare variables is left untouched), and Python's `x if c else y` cannot
|
|
163
|
+
// match this JS punctuation.
|
|
164
|
+
.replace(/\?[^?\n;:]*[\u0000][SN][^?\n;:]*:[^?\n;:]*[\u0000][SN][^?\n;:]*/g, "\u0000C")
|
|
165
|
+
.replace(/\b(?:true|false|null|undefined|None|True|False|nil)\b/g, "\u0000B")
|
|
166
|
+
// Whitespace is dropped, not collapsed: the shape is compared for
|
|
167
|
+
// equality only, and a reformatted statement must normalize to the same
|
|
168
|
+
// shape as the original — ` <N> );` and ` <N>);` are the same assertion.
|
|
169
|
+
.replace(/\s+/g, "");
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Split a *bare-comparison* assertion into the compared subject and the
|
|
173
|
+
* expected expression: `assert dec == value`, `assert!(x == y)` (Rust) and
|
|
174
|
+
* `assert add(1, 2) == 3` — the forms SPECIFIC_ASSERTION names that carry no
|
|
175
|
+
* call argument list, so splitAssertionArgs cannot see their operands.
|
|
176
|
+
*
|
|
177
|
+
* The split happens at the first top-level equality comparison, which is
|
|
178
|
+
* the assertion's own operator in every supported form; comparisons nested
|
|
179
|
+
* deeper (the one inside a conditional expectation) belong to the expected
|
|
180
|
+
* expression and are returned as part of `rhs`.
|
|
181
|
+
*
|
|
182
|
+
* @returns {{lhs: string, rhs: string} | null}
|
|
183
|
+
*/
|
|
184
|
+
function splitBareComparison(clean, lang) {
|
|
185
|
+
const trySplit = (body) => {
|
|
186
|
+
// First equality comparison after the body starts; comparisons nested in
|
|
187
|
+
// the expected expression (e.g. inside a conditional) are further right
|
|
188
|
+
// and therefore part of the rhs.
|
|
189
|
+
const m = /(?:^|[^=!<>])==(?!=)/.exec(body);
|
|
190
|
+
if (!m) return null;
|
|
191
|
+
const at = m.index + m[0].length - 2; // index of the first `=`
|
|
192
|
+
return { lhs: body.slice(0, at).trim(), rhs: body.slice(at + m[0].length - 1).trim() };
|
|
193
|
+
};
|
|
194
|
+
|
|
195
|
+
// Python/Elixir: `assert <subj> == <expect>`.
|
|
196
|
+
let m = /\bassert\s+([\s\S]+)$/.exec(clean);
|
|
197
|
+
if (m) {
|
|
198
|
+
const parts = trySplit(m[1]);
|
|
199
|
+
if (parts) return parts;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// Rust: `assert!(<subj> == <expect>)`.
|
|
203
|
+
m = /\bassert!\s*\(\s*([\s\S]*?)\s*\)\s*;?\s*$/.exec(clean);
|
|
204
|
+
if (m) {
|
|
205
|
+
const parts = trySplit(m[1]);
|
|
206
|
+
if (parts) return parts;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// JS/Java-style call form handed in here as well (shape pairing covers
|
|
210
|
+
// most; this only needs to expose the subject for a conditional rhs).
|
|
211
|
+
const cm = lang === "js" || lang === "java" ? /\bexpect\s*\(([^)]*)\)\s*\.[\s\S]*?\(\s*([\s\S]*?)\s*\)\s*;?\s*$/.exec(clean) : null;
|
|
212
|
+
if (cm) {
|
|
213
|
+
return { lhs: cm[1].trim(), rhs: cm[2].trim() };
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
return null;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* True when an expected expression is conditional rather than a single value:
|
|
221
|
+
* Python's `x if cond else y` (including a comparison inside, which is the
|
|
222
|
+
* F03 spelling — `(193 if value == 192 else value)`) or a JS/Java
|
|
223
|
+
* `cond ? x : y` ternary. Conditional expectations keep the suite green for
|
|
224
|
+
* both the old and the broken output, which is exactly the point of replacing
|
|
225
|
+
* a value with one.
|
|
226
|
+
*/
|
|
227
|
+
function containsConditional(expr) {
|
|
228
|
+
if (/\bif\b[^?:\n]*\belse\b/.test(expr)) return true;
|
|
229
|
+
// A question mark that is a ternary, not optional chaining (`?.`) or
|
|
230
|
+
// nullish (`??`).
|
|
231
|
+
if (/[)\]\w"']\s*\?(?![.?])[^?:\n]*:/.test(expr)) return true;
|
|
232
|
+
return false;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// The test languages the gate runs over. The scanner below is written for
|
|
236
|
+
// these four and nothing else; an unrecognised extension falls back to `js`,
|
|
237
|
+
// which is the strictest of the four for line joining.
|
|
238
|
+
const TEST_LANG_BY_EXT = new Map([
|
|
239
|
+
[".js", "js"], [".mjs", "js"], [".cjs", "js"], [".jsx", "js"],
|
|
240
|
+
[".ts", "js"], [".mts", "js"], [".cts", "js"], [".tsx", "js"],
|
|
241
|
+
[".py", "python"], [".pyi", "python"],
|
|
242
|
+
[".go", "go"],
|
|
243
|
+
[".rs", "rust"],
|
|
244
|
+
// Approximations, chosen for comment and continuation syntax rather than
|
|
245
|
+
// for kinship: the C-like family reads correctly under the `js` scanner,
|
|
246
|
+
// and Ruby under the `python` one because both end a comment at `#` and a
|
|
247
|
+
// statement at the newline. Naming them beats falling through to `js` by
|
|
248
|
+
// default, which is how a `#` comment came to be read as code.
|
|
249
|
+
[".java", "js"], [".kt", "js"], [".kts", "js"], [".scala", "js"], [".groovy", "js"],
|
|
250
|
+
[".swift", "js"], [".cs", "js"], [".php", "js"], [".c", "js"], [".cc", "js"],
|
|
251
|
+
[".cpp", "js"], [".h", "js"], [".hpp", "js"], [".m", "js"], [".sol", "js"],
|
|
252
|
+
[".rb", "python"],
|
|
253
|
+
]);
|
|
254
|
+
|
|
255
|
+
function langForTestFile(file) {
|
|
256
|
+
const n = String(file || "").toLowerCase();
|
|
257
|
+
const dot = n.lastIndexOf(".");
|
|
258
|
+
if (dot === -1) return "js";
|
|
259
|
+
return TEST_LANG_BY_EXT.get(n.slice(dot)) || "js";
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function freshScanState() {
|
|
263
|
+
// str: the open string, or null.
|
|
264
|
+
// q the quote character
|
|
265
|
+
// tri Python triple-quoted
|
|
266
|
+
// raw raw string, no escapes (Go backtick)
|
|
267
|
+
// rawHashes Rust raw string r#"…"#: terminator is " plus that many #
|
|
268
|
+
// block: depth of an open /* … */ (nested only in Rust)
|
|
269
|
+
// accDelta: counted delimiters still open in the current statement
|
|
270
|
+
// specialStack: counted depth recorded at each non-joining call (see below)
|
|
271
|
+
return { str: null, block: 0, accDelta: 0, specialStack: [] };
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// A call whose opening paren must not join lines: the test name is not the
|
|
275
|
+
// expectation. Without this, `it("old name", () => { expect(f()).toBe(3); })`
|
|
276
|
+
// would pair against its renamed copy, because a name is a string and so
|
|
277
|
+
// blanks to the same placeholder as any value change would. The closer of a
|
|
278
|
+
// non-joining paren is recognised by the depth it was opened at, so the
|
|
279
|
+
// running balance stays exact.
|
|
280
|
+
const NON_JOINING_CALL = /\b(?:it|test|describe|context)\s*\($|\bt\.Run\s*\($/;
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Scan one physical line of source.
|
|
284
|
+
*
|
|
285
|
+
* Returns { delta, trailingBackslash }. `delta` is the net number of
|
|
286
|
+
* still-open ( [ { delimiters outside strings and comments; a statement
|
|
287
|
+
* continues to the next physical line while it is positive, while a string
|
|
288
|
+
* or block comment is open (tracked on `state`), or on a Python line
|
|
289
|
+
* continuation.
|
|
290
|
+
*
|
|
291
|
+
* This is not a parser, and the approximations are deliberate: JS regex
|
|
292
|
+
* literals are detected with a one-token look-behind (a `/` that cannot
|
|
293
|
+
* follow an identifier, number, `)` or `]` starts one), template
|
|
294
|
+
* interpolation is treated as opaque string content, and Rust lifetimes are
|
|
295
|
+
* told apart from char literals by shape alone. `stripComments` below
|
|
296
|
+
* walks the same constructs, so the two must stay in lock-step.
|
|
297
|
+
*/
|
|
298
|
+
function scanSourceLine(text, lang, state) {
|
|
299
|
+
let delta = 0;
|
|
300
|
+
let lastSig = "\n";
|
|
301
|
+
const n = text.length;
|
|
302
|
+
let i = 0;
|
|
303
|
+
|
|
304
|
+
while (i < n) {
|
|
305
|
+
const c = text[i];
|
|
306
|
+
const c2 = i + 1 < n ? text[i + 1] : "";
|
|
307
|
+
|
|
308
|
+
if (state.str) {
|
|
309
|
+
const s = state.str;
|
|
310
|
+
let closed = false;
|
|
311
|
+
if (s.rawHashes !== undefined) {
|
|
312
|
+
if (c === '"') {
|
|
313
|
+
let j = i + 1;
|
|
314
|
+
let h = 0;
|
|
315
|
+
while (j < n && text[j] === "#") { h++; j++; }
|
|
316
|
+
if (h >= s.rawHashes) { i = j; closed = true; }
|
|
317
|
+
}
|
|
318
|
+
} else if (s.raw) {
|
|
319
|
+
closed = c === s.q;
|
|
320
|
+
} else if (c === "\\") {
|
|
321
|
+
i += s.tri ? 1 : 2;
|
|
322
|
+
continue;
|
|
323
|
+
} else if (c === s.q) {
|
|
324
|
+
if (s.tri) {
|
|
325
|
+
if (text[i + 1] === s.q && text[i + 2] === s.q) { i += 3; closed = true; }
|
|
326
|
+
else { i += 1; }
|
|
327
|
+
} else {
|
|
328
|
+
i += 1;
|
|
329
|
+
closed = true;
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
if (closed) { state.str = null; lastSig = s.q; continue; }
|
|
333
|
+
i += 1;
|
|
334
|
+
continue;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
if (state.block > 0) {
|
|
338
|
+
if (c === "/" && c2 === "*") {
|
|
339
|
+
if (lang === "rust") state.block += 1;
|
|
340
|
+
i += 2;
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
if (c === "*" && c2 === "/") {
|
|
344
|
+
state.block -= 1;
|
|
345
|
+
i += 2;
|
|
346
|
+
lastSig = "/";
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
i += 1;
|
|
350
|
+
continue;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// A line comment ends the line.
|
|
354
|
+
if (c === "/" && c2 === "/") break;
|
|
355
|
+
if (lang === "python" && c === "#") break;
|
|
356
|
+
|
|
357
|
+
if (c === "/" && c2 === "*") {
|
|
358
|
+
state.block = 1;
|
|
359
|
+
i += 2;
|
|
360
|
+
continue;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
// JS regex literal, best effort. Delimiters inside are not counted.
|
|
364
|
+
if ((lang === "js" || lang === "ts") && c === "/" && !/[\w$)\]}]/.test(lastSig)) {
|
|
365
|
+
i += 1;
|
|
366
|
+
let inClass = false;
|
|
367
|
+
while (i < n) {
|
|
368
|
+
const rc = text[i];
|
|
369
|
+
if (rc === "\\") { i += 2; continue; }
|
|
370
|
+
if (rc === "[") inClass = true;
|
|
371
|
+
else if (rc === "]") inClass = false;
|
|
372
|
+
else if (rc === "/" && !inClass) { i += 1; break; }
|
|
373
|
+
i += 1;
|
|
374
|
+
}
|
|
375
|
+
while (i < n && /[a-z]/i.test(text[i])) i += 1; // flags
|
|
376
|
+
lastSig = "/";
|
|
377
|
+
continue;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
if (c === '"' || c === "'" || (lang === "go" && c === "`")) {
|
|
381
|
+
if (lang === "python" && c2 === c && text[i + 2] === c) {
|
|
382
|
+
state.str = { q: c, tri: true };
|
|
383
|
+
i += 3;
|
|
384
|
+
} else if (lang === "rust" && c === '"') {
|
|
385
|
+
if (i > 0 && text[i - 1] === "r") {
|
|
386
|
+
let j = i - 1;
|
|
387
|
+
let h = 0;
|
|
388
|
+
while (j >= 1 && text[j - 1] === "#") { h++; j--; }
|
|
389
|
+
state.str = { q: '"', rawHashes: h };
|
|
390
|
+
i += 1;
|
|
391
|
+
} else {
|
|
392
|
+
state.str = { q: c };
|
|
393
|
+
i += 1;
|
|
394
|
+
}
|
|
395
|
+
} else if (lang === "rust" && c === "'") {
|
|
396
|
+
// A char literal is 'X' or '\X' within four characters; anything
|
|
397
|
+
// else starting with a quote is a lifetime and only the quote is
|
|
398
|
+
// skipped, or the next line would see a string that never closed.
|
|
399
|
+
if (c2 === "\\") {
|
|
400
|
+
const end = text.indexOf("'", i + 2);
|
|
401
|
+
if (end !== -1 && end - i <= 4) { i = end + 1; lastSig = "'"; continue; }
|
|
402
|
+
} else if (text[i + 2] === "'" && c2 !== "'") {
|
|
403
|
+
i += 3;
|
|
404
|
+
lastSig = "'";
|
|
405
|
+
continue;
|
|
406
|
+
}
|
|
407
|
+
i += 1;
|
|
408
|
+
continue;
|
|
409
|
+
} else {
|
|
410
|
+
state.str = { q: c };
|
|
411
|
+
i += 1;
|
|
412
|
+
}
|
|
413
|
+
lastSig = c;
|
|
414
|
+
continue;
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
// A backslash on the very last character is a Python line continuation.
|
|
418
|
+
if (c === "\\" && i + 1 === n) {
|
|
419
|
+
return { delta, trailingBackslash: true };
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
if (c === "(") {
|
|
423
|
+
// `before` ends with the paren itself; NON_JOINING_CALL matches on it.
|
|
424
|
+
const before = text.slice(0, i + 1).replace(/\s+$/, "");
|
|
425
|
+
if (NON_JOINING_CALL.test(before)) state.specialStack.push(state.accDelta);
|
|
426
|
+
else { delta += 1; state.accDelta += 1; }
|
|
427
|
+
lastSig = c;
|
|
428
|
+
i += 1;
|
|
429
|
+
continue;
|
|
430
|
+
}
|
|
431
|
+
if (c === "[") { delta += 1; state.accDelta += 1; lastSig = c; i += 1; continue; }
|
|
432
|
+
if (c === "{") {
|
|
433
|
+
// Go joins braces: the `if got != want { t.Errorf(…) }` block is the
|
|
434
|
+
// idiomatic Go assertion, and the value lives on its first line. The
|
|
435
|
+
// other three languages get no brace joining, so a rename of a test
|
|
436
|
+
// inside a block cannot pair as a value change on its own.
|
|
437
|
+
if (lang === "go") { delta += 1; state.accDelta += 1; }
|
|
438
|
+
lastSig = c;
|
|
439
|
+
i += 1;
|
|
440
|
+
continue;
|
|
441
|
+
}
|
|
442
|
+
if (c === ")") {
|
|
443
|
+
const top = state.specialStack[state.specialStack.length - 1];
|
|
444
|
+
if (top === state.accDelta) state.specialStack.pop();
|
|
445
|
+
else { delta -= 1; state.accDelta -= 1; }
|
|
446
|
+
lastSig = c;
|
|
447
|
+
i += 1;
|
|
448
|
+
continue;
|
|
449
|
+
}
|
|
450
|
+
if (c === "]") { delta -= 1; state.accDelta -= 1; lastSig = c; i += 1; continue; }
|
|
451
|
+
if (c === "}") {
|
|
452
|
+
if (lang === "go") { delta -= 1; state.accDelta -= 1; }
|
|
453
|
+
lastSig = c;
|
|
454
|
+
i += 1;
|
|
455
|
+
continue;
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
if (!/\s/.test(c)) lastSig = c;
|
|
459
|
+
i += 1;
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
return { delta, trailingBackslash: false };
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
/**
|
|
466
|
+
* The same source with every comment blanked out, strings and line
|
|
467
|
+
* structure untouched. Shapes and keywords are computed on this text so a
|
|
468
|
+
* commented-out `expect(…)` cannot make a block an assertion, and a number
|
|
469
|
+
* changed inside a comment cannot pair as a value change.
|
|
470
|
+
*/
|
|
471
|
+
function stripComments(text, lang) {
|
|
472
|
+
const state = freshScanState();
|
|
473
|
+
return text.split("\n").map((line) => {
|
|
474
|
+
let out = "";
|
|
475
|
+
let pending = 0;
|
|
476
|
+
const copyCode = (to) => {
|
|
477
|
+
out += line.slice(pending, to);
|
|
478
|
+
pending = to;
|
|
479
|
+
};
|
|
480
|
+
let i = 0;
|
|
481
|
+
const n = line.length;
|
|
482
|
+
|
|
483
|
+
while (i < n) {
|
|
484
|
+
const c = line[i];
|
|
485
|
+
const c2 = i + 1 < n ? line[i + 1] : "";
|
|
486
|
+
|
|
487
|
+
if (state.block > 0) {
|
|
488
|
+
if (c === "/" && c2 === "*") {
|
|
489
|
+
if (lang === "rust") state.block += 1;
|
|
490
|
+
i += 2;
|
|
491
|
+
continue;
|
|
492
|
+
}
|
|
493
|
+
if (c === "*" && c2 === "/") {
|
|
494
|
+
state.block -= 1;
|
|
495
|
+
i += 2;
|
|
496
|
+
if (state.block === 0) pending = i;
|
|
497
|
+
continue;
|
|
498
|
+
}
|
|
499
|
+
i += 1;
|
|
500
|
+
continue;
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
if (state.str) {
|
|
504
|
+
const s = state.str;
|
|
505
|
+
let closed = false;
|
|
506
|
+
if (s.rawHashes !== undefined) {
|
|
507
|
+
if (c === '"') {
|
|
508
|
+
let j = i + 1;
|
|
509
|
+
let h = 0;
|
|
510
|
+
while (j < n && line[j] === "#") { h++; j++; }
|
|
511
|
+
if (h >= s.rawHashes) { i = j; closed = true; }
|
|
512
|
+
}
|
|
513
|
+
} else if (s.raw) {
|
|
514
|
+
closed = c === s.q;
|
|
515
|
+
} else if (c === "\\") {
|
|
516
|
+
i += s.tri ? 1 : 2;
|
|
517
|
+
continue;
|
|
518
|
+
} else if (c === s.q) {
|
|
519
|
+
if (s.tri) {
|
|
520
|
+
if (line[i + 1] === s.q && line[i + 2] === s.q) { i += 3; closed = true; }
|
|
521
|
+
else { i += 1; }
|
|
522
|
+
} else {
|
|
523
|
+
i += 1;
|
|
524
|
+
closed = true;
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
if (closed) {
|
|
528
|
+
state.str = null;
|
|
529
|
+
copyCode(i);
|
|
530
|
+
continue;
|
|
531
|
+
}
|
|
532
|
+
i += 1;
|
|
533
|
+
continue;
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
// `pending = n` is the whole fix. `copyCode(i)` copies the code up to
|
|
537
|
+
// the comment and leaves `pending` sitting at its start; the
|
|
538
|
+
// `copyCode(n)` after this loop then copied the comment straight back
|
|
539
|
+
// in, so no line comment has ever been stripped. Block comments were,
|
|
540
|
+
// which is why `/* … */` behaved and `// …` did not.
|
|
541
|
+
if (c === "/" && c2 === "/") { copyCode(i); pending = n; break; }
|
|
542
|
+
if (lang === "python" && c === "#") { copyCode(i); pending = n; break; }
|
|
543
|
+
if (c === "/" && c2 === "*") {
|
|
544
|
+
copyCode(i);
|
|
545
|
+
state.block = 1;
|
|
546
|
+
i += 2;
|
|
547
|
+
continue;
|
|
548
|
+
}
|
|
549
|
+
if (c === '"' || c === "'" || (lang === "go" && c === "`")) {
|
|
550
|
+
copyCode(i);
|
|
551
|
+
if (lang === "python" && c2 === c && line[i + 2] === c) {
|
|
552
|
+
state.str = { q: c, tri: true };
|
|
553
|
+
i += 3;
|
|
554
|
+
} else if (lang === "rust" && c === '"') {
|
|
555
|
+
if (i > 0 && line[i - 1] === "r") {
|
|
556
|
+
let j = i - 1;
|
|
557
|
+
let h = 0;
|
|
558
|
+
while (j >= 1 && line[j - 1] === "#") { h++; j--; }
|
|
559
|
+
state.str = { q: '"', rawHashes: h };
|
|
560
|
+
i += 1;
|
|
561
|
+
} else {
|
|
562
|
+
state.str = { q: c };
|
|
563
|
+
i += 1;
|
|
564
|
+
}
|
|
565
|
+
} else if (lang === "rust" && c === "'") {
|
|
566
|
+
if (c2 === "\\") {
|
|
567
|
+
const end = line.indexOf("'", i + 2);
|
|
568
|
+
if (end !== -1 && end - i <= 4) { i = end + 1; copyCode(i); continue; }
|
|
569
|
+
} else if (line[i + 2] === "'" && c2 !== "'") {
|
|
570
|
+
i += 3;
|
|
571
|
+
copyCode(i);
|
|
572
|
+
continue;
|
|
573
|
+
}
|
|
574
|
+
i += 1;
|
|
575
|
+
copyCode(i);
|
|
576
|
+
continue;
|
|
577
|
+
} else {
|
|
578
|
+
state.str = { q: c };
|
|
579
|
+
i += 1;
|
|
580
|
+
}
|
|
581
|
+
continue;
|
|
582
|
+
}
|
|
583
|
+
i += 1;
|
|
584
|
+
}
|
|
585
|
+
copyCode(n);
|
|
586
|
+
return out;
|
|
587
|
+
}).join("\n");
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
// A line that cannot start a statement of its own continues the previous
|
|
591
|
+
// statement: a closing delimiter, a member-access, or an operator.
|
|
592
|
+
const CONTINUATION_START = /^[)\],.]/;
|
|
593
|
+
const CONTINUATION_OP_START = /^[+\-*/%<>=&|^:]/;
|
|
594
|
+
|
|
595
|
+
// A comment is not a continuation, however much it looks like one.
|
|
596
|
+
//
|
|
597
|
+
// `//` begins with a division sign and `--` with a minus, so both matched
|
|
598
|
+
// CONTINUATION_OP_START and folded the following comment line into the
|
|
599
|
+
// statement above it. The cost was a false accusation on a virtuous act:
|
|
600
|
+
// adding an assertion next to a `// ...` line made the new assertion absorb
|
|
601
|
+
// the comment, stop matching its unchanged twin, and get reported as a
|
|
602
|
+
// rewritten expectation. Python was unaffected only because `#` is not an
|
|
603
|
+
// operator — which is why the same fixture passed in pytest and failed in
|
|
604
|
+
// Jest, and why it survived every suite written against the pytest layout.
|
|
605
|
+
//
|
|
606
|
+
// A comment inside an open delimiter still joins: `cur.delta > 0` decides
|
|
607
|
+
// that before this test is ever reached.
|
|
608
|
+
const COMMENT_LINE_START = /^(?:\/\/|\/\*|#|--)/;
|
|
609
|
+
|
|
610
|
+
// A scanner miscount (an unbalanced delimiter inside a regex literal is the
|
|
611
|
+
// usual cause) must not be able to merge a whole file into one statement,
|
|
612
|
+
// which would pair *any* literal change anywhere in the file.
|
|
613
|
+
const MAX_STATEMENT_LINES = 100;
|
|
614
|
+
const MAX_STATEMENT_CHARS = 12000;
|
|
615
|
+
|
|
616
|
+
/**
|
|
617
|
+
* Reassemble physical lines into statements.
|
|
618
|
+
*
|
|
619
|
+
* `sliceLines` is one image of a hunk in file order: context lines plus the
|
|
620
|
+
* removed (or added) lines. Context lines are ordinary file text; a
|
|
621
|
+
* statement spans them freely, which is exactly what makes a value edit
|
|
622
|
+
* inside a formatter-wrapped assertion visible to the pairing.
|
|
623
|
+
*
|
|
624
|
+
* @param {Array<{ kind: string, text: string, oldNo: number|null, newNo: number|null }>} sliceLines
|
|
625
|
+
* @param {string} lang
|
|
626
|
+
* @returns {Array<{ text: string, firstOld: number|null, lastOld: number|null, firstNew: number|null, lastNew: number|null, removedLines: Array, addedLines: Array }>}
|
|
627
|
+
*/
|
|
628
|
+
function assembleStatements(sliceLines, lang) {
|
|
629
|
+
const stmts = [];
|
|
630
|
+
let cur = null;
|
|
631
|
+
|
|
632
|
+
const flush = () => {
|
|
633
|
+
if (!cur) return;
|
|
634
|
+
stmts.push({
|
|
635
|
+
text: cur.lines.join("\n"),
|
|
636
|
+
firstOld: cur.firstOld,
|
|
637
|
+
lastOld: cur.lastOld,
|
|
638
|
+
firstNew: cur.firstNew,
|
|
639
|
+
lastNew: cur.lastNew,
|
|
640
|
+
removedLines: cur.removedLines,
|
|
641
|
+
addedLines: cur.addedLines,
|
|
642
|
+
});
|
|
643
|
+
cur = null;
|
|
644
|
+
};
|
|
645
|
+
|
|
646
|
+
for (const L of sliceLines) {
|
|
647
|
+
const trimmed = L.text.replace(/^\s+/, "");
|
|
648
|
+
const startsComment = COMMENT_LINE_START.test(trimmed);
|
|
649
|
+
const joins =
|
|
650
|
+
cur !== null &&
|
|
651
|
+
(cur.delta > 0 ||
|
|
652
|
+
cur.state.str !== null ||
|
|
653
|
+
cur.state.block > 0 ||
|
|
654
|
+
cur.trailingBackslash ||
|
|
655
|
+
(!startsComment &&
|
|
656
|
+
(CONTINUATION_START.test(trimmed) || CONTINUATION_OP_START.test(trimmed))));
|
|
657
|
+
|
|
658
|
+
if (
|
|
659
|
+
joins &&
|
|
660
|
+
cur.lines.length < MAX_STATEMENT_LINES &&
|
|
661
|
+
cur.chars + L.text.length + 1 <= MAX_STATEMENT_CHARS
|
|
662
|
+
) {
|
|
663
|
+
cur.lines.push(L.text);
|
|
664
|
+
cur.chars += L.text.length + 1;
|
|
665
|
+
const sc = scanSourceLine(L.text, lang, cur.state);
|
|
666
|
+
cur.delta += sc.delta;
|
|
667
|
+
cur.trailingBackslash = sc.trailingBackslash;
|
|
668
|
+
cur.lastOld = L.oldNo;
|
|
669
|
+
cur.lastNew = L.newNo;
|
|
670
|
+
if (L.kind === "-") cur.removedLines.push(L);
|
|
671
|
+
else if (L.kind === "+") cur.addedLines.push(L);
|
|
672
|
+
} else {
|
|
673
|
+
flush();
|
|
674
|
+
const st = freshScanState();
|
|
675
|
+
const sc = scanSourceLine(L.text, lang, st);
|
|
676
|
+
cur = {
|
|
677
|
+
lines: [L.text],
|
|
678
|
+
chars: L.text.length,
|
|
679
|
+
firstOld: L.oldNo,
|
|
680
|
+
lastOld: L.oldNo,
|
|
681
|
+
firstNew: L.newNo,
|
|
682
|
+
lastNew: L.newNo,
|
|
683
|
+
delta: sc.delta,
|
|
684
|
+
state: st,
|
|
685
|
+
trailingBackslash: sc.trailingBackslash,
|
|
686
|
+
removedLines: L.kind === "-" ? [L] : [],
|
|
687
|
+
addedLines: L.kind === "+" ? [L] : [],
|
|
688
|
+
};
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
flush();
|
|
692
|
+
return stmts;
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
const hasLiteralPlaceholder = (shape) =>
|
|
696
|
+
shape.includes("\u0000S") || shape.includes("\u0000N") || shape.includes("\u0000B");
|
|
697
|
+
|
|
698
|
+
const collapseWhitespace = (s) => s.replace(/\s+/g, " ").trim();
|
|
699
|
+
const shorten = (s) => (s.length > 160 ? `${s.slice(0, 157)}…` : s);
|
|
700
|
+
|
|
701
|
+
/**
|
|
702
|
+
* Split the argument list of the outermost assertion call in `clean`.
|
|
703
|
+
*
|
|
704
|
+
* Comments are already stripped by the caller, so only string state has to be
|
|
705
|
+
* tracked. Returns null whenever the shape is not confidently understood — a
|
|
706
|
+
* truncated fragment, an unbalanced hunk, a quoting form not handled here —
|
|
707
|
+
* because every caller uses this to *suppress* a finding, and failing to
|
|
708
|
+
* understand a statement must never become a reason to stay quiet about it.
|
|
709
|
+
*
|
|
710
|
+
* @param {string} clean - comment-stripped statement text
|
|
711
|
+
* @param {string} lang
|
|
712
|
+
* @returns {string[] | null} top-level arguments, trimmed
|
|
713
|
+
*/
|
|
714
|
+
function splitAssertionArgs(clean, lang) {
|
|
715
|
+
SPECIFIC_ASSERTION.lastIndex = 0;
|
|
716
|
+
const m = SPECIFIC_ASSERTION.exec(clean);
|
|
717
|
+
if (!m) return null;
|
|
718
|
+
|
|
719
|
+
// Not every branch of SPECIFIC_ASSERTION ends at an opening paren:
|
|
720
|
+
// `assert_eq!`, `assert_equal` and RSpec's `.to eq` all match a bare name.
|
|
721
|
+
// Starting the walk one character early made every argument boundary wrong,
|
|
722
|
+
// so a reworded message read as a rewritten value.
|
|
723
|
+
let i = m.index + m[0].length;
|
|
724
|
+
if (clean[i - 1] !== "(") {
|
|
725
|
+
let j = i;
|
|
726
|
+
while (j < clean.length && /\s/.test(clean[j])) j++;
|
|
727
|
+
if (clean[j] !== "(") return null;
|
|
728
|
+
i = j + 1;
|
|
729
|
+
}
|
|
730
|
+
let depth = 1;
|
|
731
|
+
let quote = null;
|
|
732
|
+
let triple = false;
|
|
733
|
+
const args = [];
|
|
734
|
+
let start = i;
|
|
735
|
+
|
|
736
|
+
while (i < clean.length) {
|
|
737
|
+
const c = clean[i];
|
|
738
|
+
|
|
739
|
+
if (quote !== null) {
|
|
740
|
+
if (c === "\\") { i += 2; continue; }
|
|
741
|
+
if (triple && c === quote && clean[i + 1] === quote && clean[i + 2] === quote) {
|
|
742
|
+
quote = null; triple = false; i += 3; continue;
|
|
743
|
+
}
|
|
744
|
+
if (!triple && c === quote) { quote = null; i += 1; continue; }
|
|
745
|
+
i += 1;
|
|
746
|
+
continue;
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
if (c === '"' || c === "'" || c === "`") {
|
|
750
|
+
if (lang === "python" && clean[i + 1] === c && clean[i + 2] === c) {
|
|
751
|
+
quote = c; triple = true; i += 3; continue;
|
|
752
|
+
}
|
|
753
|
+
quote = c; i += 1; continue;
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
if (c === "(" || c === "[" || c === "{") { depth += 1; i += 1; continue; }
|
|
757
|
+
if (c === ")" || c === "]" || c === "}") {
|
|
758
|
+
depth -= 1;
|
|
759
|
+
if (depth === 0) {
|
|
760
|
+
args.push(clean.slice(start, i).trim());
|
|
761
|
+
return args;
|
|
762
|
+
}
|
|
763
|
+
i += 1;
|
|
764
|
+
continue;
|
|
765
|
+
}
|
|
766
|
+
if (c === "," && depth === 1) {
|
|
767
|
+
args.push(clean.slice(start, i).trim());
|
|
768
|
+
start = i + 1;
|
|
769
|
+
i += 1;
|
|
770
|
+
continue;
|
|
771
|
+
}
|
|
772
|
+
i += 1;
|
|
773
|
+
}
|
|
774
|
+
return null; // never closed: an unbalanced fragment, so no suppression
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
/** One plain string literal and nothing else. */
|
|
778
|
+
const PURE_STRING_LITERAL = new RegExp(
|
|
779
|
+
[
|
|
780
|
+
"^'(?:\\\\.|[^'\\\\])*'$",
|
|
781
|
+
'^"(?:\\\\.|[^"\\\\])*"$',
|
|
782
|
+
"^`(?:\\\\.|[^`\\\\])*`$",
|
|
783
|
+
'^"""[\\s\\S]*"""$',
|
|
784
|
+
"^'''[\\s\\S]*'''$",
|
|
785
|
+
].join("|")
|
|
786
|
+
);
|
|
787
|
+
|
|
788
|
+
function isPureStringLiteral(arg) {
|
|
789
|
+
if (!arg) return false;
|
|
790
|
+
return PURE_STRING_LITERAL.test(arg.trim());
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
/**
|
|
794
|
+
* Argument positions that carry a message for a human rather than an expected
|
|
795
|
+
* value.
|
|
796
|
+
*
|
|
797
|
+
* Trailing, for `assert.equal(got, want, "message")` and
|
|
798
|
+
* `assert_eq!(a, b, "message")`; leading, for Go's
|
|
799
|
+
* `t.Errorf("got %d want %d", got, want)`. Two arguments is the classic
|
|
800
|
+
* `(actual, expected)` shape, so a string in last position *there* is the
|
|
801
|
+
* expected value: `assert.equal(name, "Alice")` must still be judged when
|
|
802
|
+
* "Alice" becomes "Bob".
|
|
803
|
+
*/
|
|
804
|
+
function messageArgIndices(args) {
|
|
805
|
+
const idx = new Set();
|
|
806
|
+
const lastIsMessage = args.length >= 3 && isPureStringLiteral(args[args.length - 1]);
|
|
807
|
+
if (lastIsMessage) idx.add(args.length - 1);
|
|
808
|
+
|
|
809
|
+
// JUnit 4 is the one common dialect that puts the message *first*:
|
|
810
|
+
// `assertEquals("why this matters", expected, actual)`. It is also
|
|
811
|
+
// distinguishable, because its trailing argument is the actual value rather
|
|
812
|
+
// than prose — so a call that already carries a trailing message is not
|
|
813
|
+
// that shape, whatever its first argument looks like.
|
|
814
|
+
//
|
|
815
|
+
// Reading argument 0 as prose whenever it happened to be a string is what
|
|
816
|
+
// made a whole family of assertions invisible: `assertEquals(expected,
|
|
817
|
+
// actual)` — JUnit's and PHPUnit's own two-argument order — along with
|
|
818
|
+
// Python's `assertIn(member, container)` and `assertNotIn`. A rewritten
|
|
819
|
+
// expectation in any of them was dismissed as a reworded message, and the
|
|
820
|
+
// guard reported PASS on a check it had not performed.
|
|
821
|
+
if (!lastIsMessage && args.length >= 3 && isPureStringLiteral(args[0])) idx.add(0);
|
|
822
|
+
return idx;
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
/**
|
|
826
|
+
* True when two assertions differ only in text written to be read by a person.
|
|
827
|
+
*
|
|
828
|
+
* Rewording the message on a failing assertion is among the most common edits
|
|
829
|
+
* any test file receives, and it says nothing whatsoever about what the suite
|
|
830
|
+
* checks. But a message is a literal, so blanking literals made the two
|
|
831
|
+
* statements the same shape and the pairing reported a rewritten expectation
|
|
832
|
+
* every time somebody improved the wording of a failure. Firing on that is
|
|
833
|
+
* how an operator learns to pass the override without reading it.
|
|
834
|
+
*/
|
|
835
|
+
/**
|
|
836
|
+
* Split a statement at a trailing `, "message"` written outside the call.
|
|
837
|
+
*
|
|
838
|
+
* RSpec puts the message there — `expect(x).to eq(3), "explain"` — and so do
|
|
839
|
+
* Ruby and Elixir assertions generally. An argument-position check can never
|
|
840
|
+
* see it, so rewording one read as a rewritten expectation.
|
|
841
|
+
*/
|
|
842
|
+
function splitTrailingMessage(clean) {
|
|
843
|
+
let depth = 0;
|
|
844
|
+
let quote = null;
|
|
845
|
+
let lastComma = -1;
|
|
846
|
+
for (let i = 0; i < clean.length; i++) {
|
|
847
|
+
const c = clean[i];
|
|
848
|
+
if (quote !== null) {
|
|
849
|
+
if (c === "\\") { i += 1; continue; }
|
|
850
|
+
if (c === quote) quote = null;
|
|
851
|
+
continue;
|
|
852
|
+
}
|
|
853
|
+
if (c === '"' || c === "'" || c === "`") { quote = c; continue; }
|
|
854
|
+
if (c === "(" || c === "[" || c === "{") depth++;
|
|
855
|
+
else if (c === ")" || c === "]" || c === "}") depth--;
|
|
856
|
+
else if (c === "," && depth === 0) lastComma = i;
|
|
857
|
+
}
|
|
858
|
+
if (lastComma === -1) return { head: clean, msg: null };
|
|
859
|
+
const tail = clean.slice(lastComma + 1).trim();
|
|
860
|
+
if (!isPureStringLiteral(tail)) return { head: clean, msg: null };
|
|
861
|
+
return { head: clean.slice(0, lastComma), msg: tail };
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
/**
|
|
865
|
+
* Test declarations that a runner finds by the *name* of the function.
|
|
866
|
+
*
|
|
867
|
+
* pytest collects `def test_*`, Go collects `func Test*`, and unittest and
|
|
868
|
+
* Minitest collect `def test_*` off the case class. For those runners the
|
|
869
|
+
* name is not prose — it is the registration. Renaming `test_totals` to
|
|
870
|
+
* `totals` deletes the test from the run as completely as removing the file,
|
|
871
|
+
* and the diff shows a rename.
|
|
872
|
+
*
|
|
873
|
+
* Only these name-driven runners are listed. `it("...")`, `#[test]` and
|
|
874
|
+
* `@Test` register by call, attribute or annotation, so renaming what they
|
|
875
|
+
* declare removes nothing, and the ordinary rename rules already cover them.
|
|
876
|
+
*/
|
|
877
|
+
const NAME_REGISTERED_DECLS = [
|
|
878
|
+
{ lang: "python", re: /^\s*(?:async\s+)?def\s+([A-Za-z_]\w*)\s*\(/, discovered: /^test/i },
|
|
879
|
+
{ lang: "go", re: /^\s*func\s+([A-Za-z_]\w*)\s*\(/, discovered: /^(?:Test|Benchmark|Fuzz|Example)/ },
|
|
880
|
+
];
|
|
881
|
+
|
|
882
|
+
/**
|
|
883
|
+
* The declared name on this line, and whether the runner would collect it.
|
|
884
|
+
*
|
|
885
|
+
* @returns {{ name: string, collected: boolean }|null}
|
|
886
|
+
*/
|
|
887
|
+
function declaredTestName(text) {
|
|
888
|
+
for (const rule of NAME_REGISTERED_DECLS) {
|
|
889
|
+
const m = rule.re.exec(text);
|
|
890
|
+
if (m) return { name: m[1], collected: rule.discovered.test(m[1]) };
|
|
891
|
+
}
|
|
892
|
+
return null;
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
/**
|
|
896
|
+
* Did a collected test become a declaration the runner no longer collects?
|
|
897
|
+
*
|
|
898
|
+
* The name *is* the registration for pytest (`test*`) and Go (`Test*` with
|
|
899
|
+
* an uppercase letter or underscore after), so the signal is purely whether
|
|
900
|
+
* the runner would still find it: `test_want_bytes` → `check_want_bytes`,
|
|
901
|
+
* `TestLoadComment` → `checkLoadComment`, `test_x` → `disabled_x` all remove
|
|
902
|
+
* the test from the run while leaving every assertion in place.
|
|
903
|
+
*
|
|
904
|
+
* The earlier rule required the new name to be the old one with its prefix
|
|
905
|
+
* literally stripped, so `test_x` → `x` was caught and every other prefix
|
|
906
|
+
* swap sailed through. It was written narrow to avoid flagging
|
|
907
|
+
* `test_x` → `test_x_renamed` — an honest rename — but that case never needs
|
|
908
|
+
* the strip rule: the new name is *still collected*, so the collected check
|
|
909
|
+
* already keeps it silent, along with pytest's `test*` glob collecting
|
|
910
|
+
* `testx`, Go's `TestX` → `TestXRenamed`, and case-class `test_x` →
|
|
911
|
+
* `test_y`. Pairing is still required (a pure deletion is an assertion
|
|
912
|
+
* removal, not a rename), and names identical on both sides never pair.
|
|
913
|
+
*/
|
|
914
|
+
function isDeregistration(before, after) {
|
|
915
|
+
return Boolean(before.collected && !after.collected && before.name !== after.name);
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
// A test declaration whose first argument is the test's name. The name is
|
|
919
|
+
// prose about the test, not a value the test asserts — `test("adds", ...)`
|
|
920
|
+
// renamed to `test("adds positives", ...)` is the rename the diff says it is.
|
|
921
|
+
const TEST_DECL_CALL =
|
|
922
|
+
/\b(?:it|test|describe|context|suite|specify|bench|scenario)\s*\(|\b[a-z_$][\w$]{0,2}\.(?:test|Run|describe)\s*\(/;
|
|
923
|
+
|
|
924
|
+
/**
|
|
925
|
+
* Replace the name argument of a test declaration with a placeholder.
|
|
926
|
+
*
|
|
927
|
+
* @param {string} clean - comment-stripped statement text
|
|
928
|
+
* @returns {string|null} the text with the name blanked, or null when the
|
|
929
|
+
* statement is not a test declaration with a literal name.
|
|
930
|
+
*/
|
|
931
|
+
function blankTestName(clean) {
|
|
932
|
+
const m = TEST_DECL_CALL.exec(clean);
|
|
933
|
+
if (!m) return null;
|
|
934
|
+
|
|
935
|
+
let i = m.index + m[0].length;
|
|
936
|
+
while (i < clean.length && /\s/.test(clean[i])) i++;
|
|
937
|
+
const quote = clean[i];
|
|
938
|
+
if (quote !== '"' && quote !== "'" && quote !== "`") return null;
|
|
939
|
+
|
|
940
|
+
let j = i + 1;
|
|
941
|
+
while (j < clean.length) {
|
|
942
|
+
if (clean[j] === "\\") { j += 2; continue; }
|
|
943
|
+
if (clean[j] === quote) break;
|
|
944
|
+
j += 1;
|
|
945
|
+
}
|
|
946
|
+
if (j >= clean.length) return null;
|
|
947
|
+
|
|
948
|
+
return clean.slice(0, i) + "\u0000T" + clean.slice(j + 1);
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
/**
|
|
952
|
+
* True when the only thing that changed was the test's name.
|
|
953
|
+
*
|
|
954
|
+
* A one-line `test("adds", () => { assert.strictEqual(add(2, 3), 5); });`
|
|
955
|
+
* blanks to the same shape as its renamed copy, so the two pair — and the
|
|
956
|
+
* pair was then reported as a rewritten expectation, quoting the whole line
|
|
957
|
+
* back at an author who had renamed a test and nothing else. Renaming a test
|
|
958
|
+
* is one of the most ordinary edits there is, and a gate that calls it
|
|
959
|
+
* tampering is a gate that gets switched off.
|
|
960
|
+
*
|
|
961
|
+
* The multi-line form was never affected: NON_JOINING_CALL already keeps a
|
|
962
|
+
* test name from joining to the assertion below it. This is the same rule for
|
|
963
|
+
* the statements that fit on one line.
|
|
964
|
+
*
|
|
965
|
+
* A rename that also moves the expectation still differs after the name is
|
|
966
|
+
* blanked, so it is still reported.
|
|
967
|
+
*/
|
|
968
|
+
function differsOnlyInTestName(cleanRemoved, cleanAdded) {
|
|
969
|
+
const a = blankTestName(cleanRemoved);
|
|
970
|
+
const b = blankTestName(cleanAdded);
|
|
971
|
+
if (a === null || b === null) return false;
|
|
972
|
+
return a.replace(/\s+/g, "") === b.replace(/\s+/g, "");
|
|
973
|
+
}
|
|
974
|
+
|
|
975
|
+
function differsOnlyInMessage(cleanRemoved, cleanAdded, lang) {
|
|
976
|
+
const ta = splitTrailingMessage(cleanRemoved);
|
|
977
|
+
const tb = splitTrailingMessage(cleanAdded);
|
|
978
|
+
if (
|
|
979
|
+
(ta.msg !== null || tb.msg !== null) &&
|
|
980
|
+
ta.head.replace(/\s+/g, "") === tb.head.replace(/\s+/g, "")
|
|
981
|
+
) {
|
|
982
|
+
return true;
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
const a = splitAssertionArgs(cleanRemoved, lang);
|
|
986
|
+
const b = splitAssertionArgs(cleanAdded, lang);
|
|
987
|
+
if (!a || !b || a.length !== b.length || a.length === 0) return false;
|
|
988
|
+
|
|
989
|
+
const msgIdx = messageArgIndices(a);
|
|
990
|
+
if (msgIdx.size === 0) return false;
|
|
991
|
+
|
|
992
|
+
let sawDifference = false;
|
|
993
|
+
for (let i = 0; i < a.length; i++) {
|
|
994
|
+
if (a[i].replace(/\s+/g, "") === b[i].replace(/\s+/g, "")) continue;
|
|
995
|
+
// A difference outside a message position, or in a position that stopped
|
|
996
|
+
// being a plain string, is a real change.
|
|
997
|
+
if (!msgIdx.has(i) || !isPureStringLiteral(b[i])) return false;
|
|
998
|
+
sawDifference = true;
|
|
999
|
+
}
|
|
1000
|
+
return sawDifference;
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
/**
|
|
1004
|
+
* True when a paired difference is something other than a rewritten
|
|
1005
|
+
* expectation — a reworded message, or a renamed test.
|
|
1006
|
+
*/
|
|
1007
|
+
function isNonExpectationDifference(cleanRemoved, cleanAdded, lang) {
|
|
1008
|
+
return (
|
|
1009
|
+
differsOnlyInMessage(cleanRemoved, cleanAdded, lang) ||
|
|
1010
|
+
differsOnlyInTestName(cleanRemoved, cleanAdded)
|
|
1011
|
+
);
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
/**
|
|
1015
|
+
* Pair rewritten expectations across the removed and added images of every
|
|
1016
|
+
* hunk of one file, and report each pair.
|
|
1017
|
+
*
|
|
1018
|
+
* A statement is only a candidate when it actually contains a removed
|
|
1019
|
+
* (resp. added) line: a context-only statement is unchanged text on both
|
|
1020
|
+
* sides, and letting it pair would flag a genuinely new assertion that
|
|
1021
|
+
* merely has the same shape as one that stayed put.
|
|
1022
|
+
*
|
|
1023
|
+
* @param {string} file
|
|
1024
|
+
* @param {Array<{ lines: Array }>} hunks
|
|
1025
|
+
* @param {object} stats - per-file stats; the paired physical lines are
|
|
1026
|
+
* spliced out of the count pools so the count-based checks below do not
|
|
1027
|
+
* report the same edit a second time.
|
|
1028
|
+
* @param {Array} violations
|
|
1029
|
+
* @returns {Array<{ r: object, a: object }>} the pairs, for the caller
|
|
1030
|
+
*/
|
|
1031
|
+
function detectExpectationRewrites(file, hunks, stats, violations) {
|
|
1032
|
+
const lang = langForTestFile(file);
|
|
1033
|
+
const allPairs = [];
|
|
1034
|
+
|
|
1035
|
+
for (const hunk of hunks) {
|
|
1036
|
+
const oldSlice = [];
|
|
1037
|
+
const newSlice = [];
|
|
1038
|
+
for (const L of hunk.lines) {
|
|
1039
|
+
if (L.kind !== "+") oldSlice.push(L);
|
|
1040
|
+
if (L.kind !== "-") newSlice.push(L);
|
|
1041
|
+
}
|
|
1042
|
+
const oldStmts = assembleStatements(oldSlice, lang);
|
|
1043
|
+
const newStmts = assembleStatements(newSlice, lang);
|
|
1044
|
+
|
|
1045
|
+
// Candidate statements in file order, per image.
|
|
1046
|
+
const oldCands = [];
|
|
1047
|
+
for (const s of oldStmts) {
|
|
1048
|
+
if (s.removedLines.length === 0) continue;
|
|
1049
|
+
const clean = stripComments(s.text, lang);
|
|
1050
|
+
if (!isSpecificAssertion(clean)) continue;
|
|
1051
|
+
oldCands.push({ s, clean, shape: blankLiterals(clean), canon: clean.replace(/\s+/g, "") });
|
|
1052
|
+
}
|
|
1053
|
+
const newCands = [];
|
|
1054
|
+
for (const s of newStmts) {
|
|
1055
|
+
if (s.addedLines.length === 0) continue;
|
|
1056
|
+
const clean = stripComments(s.text, lang);
|
|
1057
|
+
if (!isSpecificAssertion(clean)) continue;
|
|
1058
|
+
newCands.push({ s, clean, shape: blankLiterals(clean), canon: clean.replace(/\s+/g, "") });
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
// The t-th removed candidate of a shape pairs with the t-th added
|
|
1062
|
+
// candidate of the same shape. Position alignment is what keeps a
|
|
1063
|
+
// formatter run over a block of same-shape assertions silent: a greedy
|
|
1064
|
+
// "first different text" pairing would match a re-indented
|
|
1065
|
+
// `assert.equal(f(0), 0)` against its neighbour's value and report a
|
|
1066
|
+
// rewrite that did not happen. It also keeps the pairing linear in the
|
|
1067
|
+
// number of statements, which a 4000-line same-shape table would not
|
|
1068
|
+
// survive as a square of comparisons.
|
|
1069
|
+
const oldByShape = new Map();
|
|
1070
|
+
const newByShape = new Map();
|
|
1071
|
+
for (const c of oldCands) {
|
|
1072
|
+
let arr = oldByShape.get(c.shape);
|
|
1073
|
+
if (!arr) arr = oldByShape.set(c.shape, []).get(c.shape);
|
|
1074
|
+
arr.push(c);
|
|
1075
|
+
}
|
|
1076
|
+
for (const c of newCands) {
|
|
1077
|
+
let arr = newByShape.get(c.shape);
|
|
1078
|
+
if (!arr) arr = newByShape.set(c.shape, []).get(c.shape);
|
|
1079
|
+
arr.push(c);
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
const pairs = [];
|
|
1083
|
+
const pairedOld = new Set();
|
|
1084
|
+
const pairedNew = new Set();
|
|
1085
|
+
const cancelled = new Set();
|
|
1086
|
+
for (const [shape, olds] of oldByShape) {
|
|
1087
|
+
const news = newByShape.get(shape) || [];
|
|
1088
|
+
|
|
1089
|
+
// Cancel the assertions that are byte-identical on both sides before
|
|
1090
|
+
// aligning anything.
|
|
1091
|
+
//
|
|
1092
|
+
// Reordering two assertions removes both and adds both back unchanged.
|
|
1093
|
+
// Positional alignment then matched the first removed against the first
|
|
1094
|
+
// added — a different assertion — and reported two rewritten
|
|
1095
|
+
// expectations for an edit that changed no expected value at all. The
|
|
1096
|
+
// same happened to an assertion that simply moved within its block.
|
|
1097
|
+
// What is present unchanged on both sides did not change; only the
|
|
1098
|
+
// residue can have been rewritten.
|
|
1099
|
+
const survivingNew = news.slice();
|
|
1100
|
+
const survivingOld = [];
|
|
1101
|
+
for (const o of olds) {
|
|
1102
|
+
const twin = survivingNew.findIndex((n) => n.canon === o.canon);
|
|
1103
|
+
if (twin === -1) {
|
|
1104
|
+
survivingOld.push(o);
|
|
1105
|
+
} else {
|
|
1106
|
+
// Present unchanged on both sides: this assertion did not change,
|
|
1107
|
+
// and the argument-level pass below must not be allowed to pair it
|
|
1108
|
+
// with something else and call that a rewrite.
|
|
1109
|
+
cancelled.add(o);
|
|
1110
|
+
cancelled.add(survivingNew[twin]);
|
|
1111
|
+
survivingNew.splice(twin, 1);
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
const k = Math.min(survivingOld.length, survivingNew.length);
|
|
1116
|
+
for (let t = 0; t < k; t++) {
|
|
1117
|
+
const r = survivingOld[t];
|
|
1118
|
+
const a = survivingNew[t];
|
|
1119
|
+
// Whatever this pass decides about a pair — reported, or deliberately
|
|
1120
|
+
// let go as a reorder or a reworded message — is the decision. The
|
|
1121
|
+
// argument-level pass below exists only for statements whose shapes
|
|
1122
|
+
// differ so much that they never met in a bucket here; letting it
|
|
1123
|
+
// re-open a case that was already judged turned a comment edit into a
|
|
1124
|
+
// rewritten expectation.
|
|
1125
|
+
pairedOld.add(r);
|
|
1126
|
+
pairedNew.add(a);
|
|
1127
|
+
if (r.canon === a.canon) continue;
|
|
1128
|
+
if (isNonExpectationDifference(r.clean, a.clean, lang)) continue;
|
|
1129
|
+
pairs.push({ r: r.s, a: a.s });
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
// Same assertion, same subject, different expected value.
|
|
1134
|
+
//
|
|
1135
|
+
// Shape pairing compares the statement with its literals blanked, so it
|
|
1136
|
+
// only ever matched assertions whose structure survived the edit. Shrink
|
|
1137
|
+
// a five-element expected list to one element to match broken output and
|
|
1138
|
+
// the two images land in different shape buckets, never pair, and the
|
|
1139
|
+
// rewrite is not reported at all — measured on a real repository, where
|
|
1140
|
+
// it collected five green phases.
|
|
1141
|
+
//
|
|
1142
|
+
// The arguments are the better witness here: when both sides call the
|
|
1143
|
+
// same assertion with the same number of arguments and the *subject*
|
|
1144
|
+
// argument is untouched, what changed is what the test expects of it.
|
|
1145
|
+
for (const r of oldCands) {
|
|
1146
|
+
if (pairedOld.has(r) || cancelled.has(r)) continue;
|
|
1147
|
+
const ra = splitAssertionArgs(r.clean, lang);
|
|
1148
|
+
if (!ra || ra.length < 2) continue;
|
|
1149
|
+
for (const a of newCands) {
|
|
1150
|
+
if (pairedNew.has(a) || cancelled.has(a)) continue;
|
|
1151
|
+
const aa = splitAssertionArgs(a.clean, lang);
|
|
1152
|
+
if (!aa || aa.length !== ra.length) continue;
|
|
1153
|
+
const same = (i) => ra[i].replace(/\s+/g, "") === aa[i].replace(/\s+/g, "");
|
|
1154
|
+
// The subject has to be the same expression, or these are two
|
|
1155
|
+
// different assertions that merely resemble each other.
|
|
1156
|
+
if (!same(0)) continue;
|
|
1157
|
+
if (ra.every((_, i) => same(i))) continue;
|
|
1158
|
+
if (isNonExpectationDifference(r.clean, a.clean, lang)) continue;
|
|
1159
|
+
pairs.push({ r: r.s, a: a.s });
|
|
1160
|
+
pairedOld.add(r);
|
|
1161
|
+
pairedNew.add(a);
|
|
1162
|
+
break;
|
|
1163
|
+
}
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
// A bare-comparison assertion whose expectation became a conditional
|
|
1167
|
+
// value. `assert dec == value` rewritten as
|
|
1168
|
+
// `assert dec == (193 if value == 192 else value)` keeps a comparison on
|
|
1169
|
+
// both sides of the new `==`, so both statements still parse as
|
|
1170
|
+
// assertions, but the expected value is now a conditional that bends to
|
|
1171
|
+
// broken output — F03, measured approving a deliberately broken function.
|
|
1172
|
+
// The call-argument passes above cannot see this spelling: a Python/Rust
|
|
1173
|
+
// bare comparison has no argument list, and the conditional introduces a
|
|
1174
|
+
// second comparison so the two images never share a shape bucket.
|
|
1175
|
+
//
|
|
1176
|
+
// The subject (the left operand of the assertion) must survive, and the
|
|
1177
|
+
// new right-hand side has to be a *conditional expression* —
|
|
1178
|
+
// Python's `a if c else b` or a JS/Java `c ? a : b` — so an honest
|
|
1179
|
+
// assertion whose expected value is a ternary from the start is only
|
|
1180
|
+
// reported when it replaces a non-conditional expectation of the same
|
|
1181
|
+
// subject, and an identifier renamed in the expectation (no conditional)
|
|
1182
|
+
// is not reported here either.
|
|
1183
|
+
for (const r of oldCands) {
|
|
1184
|
+
if (pairedOld.has(r) || cancelled.has(r)) continue;
|
|
1185
|
+
const oldParts = splitBareComparison(r.clean, lang);
|
|
1186
|
+
if (!oldParts) continue;
|
|
1187
|
+
for (const a of newCands) {
|
|
1188
|
+
if (pairedNew.has(a) || cancelled.has(a)) continue;
|
|
1189
|
+
const newParts = splitBareComparison(a.clean, lang);
|
|
1190
|
+
if (!newParts) continue;
|
|
1191
|
+
if (oldParts.lhs.replace(/\s+/g, "") !== newParts.lhs.replace(/\s+/g, "")) continue;
|
|
1192
|
+
if (oldParts.rhs.replace(/\s+/g, "") === newParts.rhs.replace(/\s+/g, "")) continue;
|
|
1193
|
+
if (!containsConditional(newParts.rhs)) continue;
|
|
1194
|
+
if (isNonExpectationDifference(r.clean, a.clean, lang)) continue;
|
|
1195
|
+
pairs.push({ r: r.s, a: a.s });
|
|
1196
|
+
pairedOld.add(r);
|
|
1197
|
+
pairedNew.add(a);
|
|
1198
|
+
break;
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
// Zero-context hunk: each image is a single fragment and the assertion
|
|
1203
|
+
// keyword may sit outside the hunk entirely. The fragment pair is taken
|
|
1204
|
+
// only when both sides normalize to the same shape *and* that shape
|
|
1205
|
+
// holds a literal — a code change cannot fake it.
|
|
1206
|
+
if (pairs.length === 0 && oldStmts.length === 1 && newStmts.length === 1) {
|
|
1207
|
+
const r = oldStmts[0];
|
|
1208
|
+
const a = newStmts[0];
|
|
1209
|
+
if (r.removedLines.length > 0 && a.addedLines.length > 0) {
|
|
1210
|
+
const sr = blankLiterals(stripComments(r.text, lang));
|
|
1211
|
+
const sa = blankLiterals(stripComments(a.text, lang));
|
|
1212
|
+
if (sr === sa && hasLiteralPlaceholder(sr)) {
|
|
1213
|
+
const clr = stripComments(r.text, lang);
|
|
1214
|
+
const cla = stripComments(a.text, lang);
|
|
1215
|
+
if (clr.replace(/\s+/g, "") !== cla.replace(/\s+/g, "") && !isNonExpectationDifference(clr, cla, lang)) {
|
|
1216
|
+
pairs.push({ r, a });
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
for (const p of pairs) {
|
|
1223
|
+
allPairs.push(p);
|
|
1224
|
+
const addedNo = p.a.addedLines.length > 0 ? p.a.addedLines[0].newNo : (p.r.removedLines[0] ? p.r.removedLines[0].oldNo : null);
|
|
1225
|
+
violations.push({
|
|
1226
|
+
file,
|
|
1227
|
+
line: addedNo,
|
|
1228
|
+
type: "ASSERTION_EXPECTATION_CHANGED",
|
|
1229
|
+
reason:
|
|
1230
|
+
`Test Tamper Guard: Expected value rewritten in ${file}${addedNo ? `:${addedNo}` : ""} — ` +
|
|
1231
|
+
`"${shorten(collapseWhitespace(p.r.text))}" became "${shorten(collapseWhitespace(p.a.text))}". ` +
|
|
1232
|
+
`A deliberately changed spec looks identical to a test bent to match broken ` +
|
|
1233
|
+
`output, and a diff alone cannot tell the two apart, so this is flagged for ` +
|
|
1234
|
+
`review rather than assumed. If the new expectation is the correct one, ` +
|
|
1235
|
+
`re-run with --allow-test-change expectation — which allows exactly this ` +
|
|
1236
|
+
`check and leaves the skip, vacuous, commented, removal and weakening ` +
|
|
1237
|
+
`checks doing their job.`,
|
|
1238
|
+
});
|
|
1239
|
+
|
|
1240
|
+
// Both sides are accounted for here, so they must not also feed the
|
|
1241
|
+
// count-based checks below — the same line reported twice under two
|
|
1242
|
+
// names tells the operator nothing extra. Consumption is symmetric so
|
|
1243
|
+
// a statement that absorbed two old assertion lines but only one new
|
|
1244
|
+
// one still leaves the surplus to the removal check.
|
|
1245
|
+
const removedMatches = p.r.removedLines.filter(
|
|
1246
|
+
(L) => stats.removed.some((e) => e.line === L.oldNo && e.text === L.text)
|
|
1247
|
+
);
|
|
1248
|
+
const addedMatches = p.a.addedLines.filter(
|
|
1249
|
+
(L) => stats.addedTexts.some((e) => e.line === L.newNo && e.text === L.text)
|
|
1250
|
+
);
|
|
1251
|
+
const take = Math.min(removedMatches.length, addedMatches.length);
|
|
1252
|
+
for (let k = 0; k < take; k++) {
|
|
1253
|
+
const L = removedMatches[k];
|
|
1254
|
+
const idx = stats.removed.findIndex((e) => e.line === L.oldNo && e.text === L.text);
|
|
1255
|
+
if (idx === -1) continue;
|
|
1256
|
+
const entry = stats.removed.splice(idx, 1)[0];
|
|
1257
|
+
const rsIdx = stats.removedSpecific.indexOf(entry);
|
|
1258
|
+
if (rsIdx !== -1) {
|
|
1259
|
+
stats.removedSpecific.splice(rsIdx, 1);
|
|
1260
|
+
stats.addedSpecific--;
|
|
1261
|
+
}
|
|
1262
|
+
stats.added--;
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
return allPairs;
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
/**
|
|
1271
|
+
* The tamper checks, by the name an operator uses to allow one of them.
|
|
1272
|
+
*
|
|
1273
|
+
* There was one override for all six, and it was a switch marked "off". A
|
|
1274
|
+
* deliberate change of spec rewrites what a test expects, which is
|
|
1275
|
+
* indistinguishable from bending a test to match broken output — so the honest
|
|
1276
|
+
* answer to that finding is sometimes an override. But reaching for it also
|
|
1277
|
+
* silenced injected `.skip()`, `expect(true).toBe(true)`, commented-out
|
|
1278
|
+
* assertions and outright deletions, none of which the operator had looked at.
|
|
1279
|
+
* The check with the highest firing rate therefore set the ceiling for every
|
|
1280
|
+
* other check in the bundle: the more useful this one became, the more often
|
|
1281
|
+
* it would be used to turn the others off.
|
|
1282
|
+
*/
|
|
1283
|
+
export const TAMPER_KINDS = new Map([
|
|
1284
|
+
["TEST_SKIP_INJECTION", "skip"],
|
|
1285
|
+
["VACUOUS_ASSERTION", "vacuous"],
|
|
1286
|
+
["COMMENTED_ASSERTION", "commented"],
|
|
1287
|
+
["ASSERTION_REMOVAL", "removal"],
|
|
1288
|
+
["ASSERTION_WEAKENED", "weakening"],
|
|
1289
|
+
["ASSERTION_EXPECTATION_CHANGED", "expectation"],
|
|
1290
|
+
["TEST_DEREGISTERED", "deregistration"],
|
|
1291
|
+
]);
|
|
1292
|
+
|
|
1293
|
+
/** Every kind name, for CLI validation and help text. */
|
|
1294
|
+
export const TAMPER_KIND_NAMES = Object.freeze([...new Set(TAMPER_KINDS.values())].sort());
|
|
1295
|
+
|
|
1296
|
+
/**
|
|
1297
|
+
* Conditions a compiler or the language's own rules make impossible.
|
|
1298
|
+
*
|
|
1299
|
+
* A Go `len(...)` is never negative, so `if len(comment) < 0` is false on
|
|
1300
|
+
* every input; a C unsigned/size comparison against 0 is the same shape in
|
|
1301
|
+
* the dialects scanned under the JS lexer. These are the cases where the
|
|
1302
|
+
* condition governing a failure call can be proven dead from the diff line
|
|
1303
|
+
* alone — anything fuzzier (a flag constant flipped elsewhere, an unreachable
|
|
1304
|
+
* branch behind real state) is not guessable and is deliberately left alone.
|
|
1305
|
+
*/
|
|
1306
|
+
const DEAD_GUARD_CONDITION =
|
|
1307
|
+
/\bif\b[^;{}]*\b(?:len|len\s+of|count|size|length|num\w*|total)\s*\([^)]*\)\s*(?:<\s*0|<\s*-0\b|<=\s*-1\b)|\bif\s+(?:False|false|0)\s*(?::|\{|$|\b)|\bif\s*\(\s*(?:False|false|0)\s*\)/;
|
|
1308
|
+
|
|
1309
|
+
/**
|
|
1310
|
+
* The calls a test uses to say "this failed": the assertion's actual teeth.
|
|
1311
|
+
* When one of these sits inside a dead condition, the assertion survives in
|
|
1312
|
+
* name only.
|
|
1313
|
+
*/
|
|
1314
|
+
const FAILURE_CALL =
|
|
1315
|
+
/\b(?:t\.(?:Errorf|Fatalf|Fatal|Error)\s*\(|require\.(?:Fail|FailNow|Error|Errorf|Equal|NotEqual|Len|Contains|NoError)\s*\(|assert\.(?:fail|fail!|equal|deepEqual|strictEqual)\b|assert_eq!\s*\(|assert!\s*\(|pytest\.fail\s*\(|self\.fail(?:ure)?\s*\(|fail(?:ure)?\s*\(|throw\s+new\s+(?:AssertionError|Error)\b|raise\s+AssertionError\b)/i;
|
|
1316
|
+
|
|
1317
|
+
/**
|
|
1318
|
+
* Go build-constraint terms. `//go:build ignore` never matches a release
|
|
1319
|
+
* build; conjoining a private tag (`go1.7 && cold_start_never`) gates a file
|
|
1320
|
+
* unless CI sets the tag. Version (`go1.x`), OS and arch terms are legitimate
|
|
1321
|
+
* CI gating and stay silent.
|
|
1322
|
+
*/
|
|
1323
|
+
const GO_BUILD_TAG_LINE = /^\s*\/\/go:build\s+(.+?)\s*$/;
|
|
1324
|
+
const GO_LEGACY_TAG_LINE = /^\s*\/\/\s*\+build\s+(.+?)\s*$/;
|
|
1325
|
+
const goBuildTerms = (line) => {
|
|
1326
|
+
const m = GO_BUILD_TAG_LINE.exec(line) || GO_LEGACY_TAG_LINE.exec(line);
|
|
1327
|
+
if (!m) return null;
|
|
1328
|
+
// `&&`/`||` separate constraint expressions; spaces/commas separate terms.
|
|
1329
|
+
// Negated terms (`!tag`) are normal platform guards; strip the leading `!`.
|
|
1330
|
+
return m[1]
|
|
1331
|
+
.split(/\s*&&\s*|\s*\|\|\s*|[\s,]+/)
|
|
1332
|
+
.filter(Boolean)
|
|
1333
|
+
.map((t) => t.replace(/^!/, ""));
|
|
1334
|
+
};
|
|
1335
|
+
const goKnownBuildTerm = (term) =>
|
|
1336
|
+
/^go1\.\d+/.test(term) ||
|
|
1337
|
+
/^(?:linux|darwin|windows|freebsd|openbsd|netbsd|dragonfly|solaris|aix|js|wasip1|plan9|ios|android)$/.test(term) ||
|
|
1338
|
+
/^(?:amd64|386|arm|arm64|ppc64|ppc64le|mips|mipsle|mips64|mips64le|riscv64|s390x|wasm|loong64)$/.test(term) ||
|
|
1339
|
+
term === "ignore";
|
|
1340
|
+
|
|
1341
|
+
/**
|
|
1342
|
+
* A bare, unconditional `return` (optionally returning a constant) — never a
|
|
1343
|
+
* `return value`, never attached to an `if` on the same line.
|
|
1344
|
+
*/
|
|
1345
|
+
/**
|
|
1346
|
+
* A bare, unconditional `return` (optionally returning a constant) — never a
|
|
1347
|
+
* `return value`, never attached to an `if` on the same line.
|
|
1348
|
+
*
|
|
1349
|
+
* The boundary of what this check can see: the body-first bare `return` is
|
|
1350
|
+
* caught; a `return` behind a condition the test author believes cannot hold
|
|
1351
|
+
* is not. Judging the latter requires knowing whether the branch is
|
|
1352
|
+
* reachable at runtime — a whole-program control-flow question no line pair
|
|
1353
|
+
* answers — and flagging any `return` above an assertion would hard-red the
|
|
1354
|
+
* ordinary `if (process.platform === "win32") return;` guard clause. The rule
|
|
1355
|
+
* stops at the shape whose intent is unambiguous from the text. Runtime
|
|
1356
|
+
* attestation (counting tests collected before and after) is the complete
|
|
1357
|
+
* answer, and a separate check from the text guard.
|
|
1358
|
+
*/
|
|
1359
|
+
const BARE_EARLY_RETURN = /^\s*return\s*(?:(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`|true|false|None|nil|null|undefined|0|-?\d+)\s*)?;?\s*$/;
|
|
1360
|
+
|
|
1361
|
+
/**
|
|
1362
|
+
* A line that opens a test body: a pytest/unittest `def test*` signature, a
|
|
1363
|
+
* Go `func Test*` / Rust `fn` immediately preceded by a test attribute (best
|
|
1364
|
+
* effort at line scope), or a JS test registration whose callback opens.
|
|
1365
|
+
*/
|
|
1366
|
+
const TEST_BODY_OPEN =
|
|
1367
|
+
/(?:^|\s)(?:def\s+test\w*\s*\([^)]*\)\s*(?:->[^:]+)?\s*:|func\s+(?:Test|Benchmark|Fuzz|Example)\w*\s*\([^)]*\)\s*\{|fn\s+\w+\s*\([^)]*\)\s*\{|\b(?:it|test|describe|context)\s*(?:\.[a-zA-Z]+)?\s*\(\s*["'`][^"'`]*["'`]\s*,?\s*(?:async\s*)?(?:function)?\s*\w*\s*=>?\s*\{?)$/i;
|
|
1368
|
+
|
|
1369
|
+
/**
|
|
1370
|
+
* Which tamper checks this run is allowed to stay quiet about.
|
|
1371
|
+
*
|
|
1372
|
+
* @param {object} options
|
|
1373
|
+
* @param {boolean} [options.allowTestModifications] - the blunt form: all of them.
|
|
1374
|
+
* @param {string|string[]} [options.allowTestChanges] - kind names, comma-separated or an array.
|
|
1375
|
+
* @returns {{ all: boolean, kinds: Set<string>, unknown: string[] }}
|
|
1376
|
+
*/
|
|
1377
|
+
export function resolveAllowedTamperKinds(options = {}) {
|
|
1378
|
+
if (options.allowTestModifications === true) {
|
|
1379
|
+
return { all: true, kinds: new Set(TAMPER_KIND_NAMES), unknown: [] };
|
|
1380
|
+
}
|
|
1381
|
+
const raw = options.allowTestChanges;
|
|
1382
|
+
const list = (Array.isArray(raw) ? raw : [raw])
|
|
1383
|
+
.flatMap((v) => String(v == null ? "" : v).split(","))
|
|
1384
|
+
.map((v) => v.trim().toLowerCase())
|
|
1385
|
+
.filter(Boolean);
|
|
1386
|
+
|
|
1387
|
+
if (list.includes("all")) {
|
|
1388
|
+
return { all: true, kinds: new Set(TAMPER_KIND_NAMES), unknown: [] };
|
|
1389
|
+
}
|
|
1390
|
+
const kinds = new Set();
|
|
1391
|
+
const unknown = [];
|
|
1392
|
+
for (const name of list) {
|
|
1393
|
+
if (TAMPER_KIND_NAMES.includes(name)) kinds.add(name);
|
|
1394
|
+
else unknown.push(name);
|
|
1395
|
+
}
|
|
1396
|
+
return { all: false, kinds, unknown };
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
/**
|
|
1400
|
+
* Detects test file assertion tampering, weakening, or test skips.
|
|
1401
|
+
*
|
|
1402
|
+
* @param {string} diffOrText - Unified git diff
|
|
1403
|
+
* @param {Object} [options]
|
|
1404
|
+
* @param {boolean} [options.allowTestModifications=false]
|
|
1405
|
+
* @returns {{ ok: boolean, violations: Array<object>, inputsSeen: number, filesSeen: number,
|
|
1406
|
+
* assertionsSeen: number, unreadable: Array<{file: string, count: number, samples: string[]}>,
|
|
1407
|
+
* status: "PASS"|"FAIL"|"UNREADABLE"|"NOT_APPLICABLE" }}
|
|
1408
|
+
* `status` distinguishes "checked and clean" from "nothing was checked";
|
|
1409
|
+
* `ok: true` alone cannot, and that ambiguity is the defect class this
|
|
1410
|
+
* field exists to make visible.
|
|
1411
|
+
*/
|
|
1412
|
+
export function checkTestTampering(diffOrText = "", options = {}) {
|
|
1413
|
+
if (!diffOrText || typeof diffOrText !== "string") {
|
|
1414
|
+
return { ok: true, violations: [], inputsSeen: 0, status: "NOT_APPLICABLE", reason: "empty diff" };
|
|
1415
|
+
}
|
|
1416
|
+
const allowed = resolveAllowedTamperKinds(options);
|
|
1417
|
+
if (allowed.all) {
|
|
1418
|
+
return { ok: true, violations: [], inputsSeen: 0, status: "NOT_APPLICABLE", reason: "all kinds allowed" };
|
|
1419
|
+
}
|
|
1420
|
+
|
|
1421
|
+
// Which predicate decides what this guard even looks at. Injectable so the
|
|
1422
|
+
// meta-check can mutate it: a canary that still passes when the predicate is
|
|
1423
|
+
// replaced by `() => false` was never requiring this guard to activate.
|
|
1424
|
+
const isTestPath_ = typeof options.isTestPath === "function" ? options.isTestPath : isTestPath;
|
|
1425
|
+
|
|
1426
|
+
const violations = [];
|
|
1427
|
+
const lines = diffOrText.split("\n");
|
|
1428
|
+
let lastOldFile = null;
|
|
1429
|
+
let currentFile = null;
|
|
1430
|
+
let currentOldLineNo = null;
|
|
1431
|
+
let currentNewLineNo = null;
|
|
1432
|
+
|
|
1433
|
+
const isTestFile = isTestPath_;
|
|
1434
|
+
|
|
1435
|
+
const SKIP_INJECTIONS = [
|
|
1436
|
+
{ pattern: /\b(?:it|test|describe|context)\.skip\s*\(/i, desc: "Injected test skip (.skip())" },
|
|
1437
|
+
{ pattern: /\b(?:xit|xtest|xdescribe)\s*\(/i, desc: "Injected disabled test (xit/xtest/xdescribe)" },
|
|
1438
|
+
{ pattern: /@pytest\.mark\.skip/i, desc: "Injected Pytest skip decorator (@pytest.mark.skip)" },
|
|
1439
|
+
{ pattern: /\bt\.Skip\s*\(/i, desc: "Injected Go test skip (t.Skip())" },
|
|
1440
|
+
{ pattern: /#\[ignore\]/i, desc: "Injected Rust test ignore attribute (#[ignore])" },
|
|
1441
|
+
{ pattern: /\/\/\s*@ts-ignore/i, desc: "Injected TypeScript ignore comment (// @ts-ignore)" },
|
|
1442
|
+
{ pattern: /\/\/\s*@ts-nocheck/i, desc: "Injected TypeScript nocheck directive (// @ts-nocheck)" },
|
|
1443
|
+
// A suite that never runs cannot fail, and `@Disabled` is exactly as
|
|
1444
|
+
// effective at arranging that as `it.skip` is.
|
|
1445
|
+
{ pattern: /@Disabled\b/, desc: "Injected JUnit 5 disable annotation (@Disabled)" },
|
|
1446
|
+
{ pattern: /@Ignore\b/, desc: "Injected JUnit 4 / TestNG ignore annotation (@Ignore)" },
|
|
1447
|
+
{ pattern: /@Test\s*\([^)]*enabled\s*=\s*false/i, desc: "Injected TestNG disabled test (enabled = false)" },
|
|
1448
|
+
{ pattern: /@unittest\.skip/i, desc: "Injected unittest skip decorator (@unittest.skip)" },
|
|
1449
|
+
{ pattern: /\bmarkTest(?:Skipped|Incomplete)\s*\(/i, desc: "Injected PHPUnit skip (markTestSkipped())" },
|
|
1450
|
+
{ pattern: /\bXCTSkip(?:If|Unless|IfNot)?\s*\(/, desc: "Injected XCTest skip (XCTSkip())" },
|
|
1451
|
+
{ pattern: /\b(?:xit|xdescribe|xcontext|xspecify)\b\s*["\x27]/i, desc: "Injected RSpec disabled example (xit)" },
|
|
1452
|
+
{ pattern: /,\s*skip:\s*(?:true|["\x27])/i, desc: "Injected RSpec skip metadata (skip:)" },
|
|
1453
|
+
// node-tap, node:test and ava pass it as a property of an options object,
|
|
1454
|
+
// so the comma sits before the brace and not before the key.
|
|
1455
|
+
{ pattern: /\{[^}]*\bskip\s*:\s*true/i, desc: "Injected skip option ({ skip: true })" },
|
|
1456
|
+
{ pattern: /\{[^}]*\btodo\s*:\s*true/i, desc: "Injected todo option ({ todo: true })" },
|
|
1457
|
+
{ pattern: /^\s*(?:skip|pending)\s*(?:["\x27(]|$)/i, desc: "Injected Minitest/RSpec skip statement" },
|
|
1458
|
+
// Skipping from inside the body, which is how these ecosystems actually
|
|
1459
|
+
// do it. Only the decorator and annotation forms were covered, so a test
|
|
1460
|
+
// could be silenced with the standard library's own method and the guard
|
|
1461
|
+
// said nothing: measured silent on six of seven in-body forms.
|
|
1462
|
+
{ pattern: /\bself\.skipTest\s*\(/i, desc: "Injected unittest skip (self.skipTest())" },
|
|
1463
|
+
{ pattern: /\braise\s+(?:unittest\.)?SkipTest\b/i, desc: "Injected unittest skip (raise SkipTest)" },
|
|
1464
|
+
{ pattern: /\bpytest\.skip\s*\(/i, desc: "Injected Pytest skip call (pytest.skip())" },
|
|
1465
|
+
{ pattern: /\bpytest\.xfail\s*\(/i, desc: "Injected Pytest expected-failure (pytest.xfail())" },
|
|
1466
|
+
// The decorator form the call above does not cover. `strict=False` (the
|
|
1467
|
+
// default) lets a *broken* test pass as xpass-with-no-failure; strict only
|
|
1468
|
+
// fails on an unexpected pass, so either spelling blesses a failing suite.
|
|
1469
|
+
{ pattern: /@pytest\.mark\.xfail\b/i, desc: "Injected Pytest expected-failure mark (@pytest.mark.xfail)" },
|
|
1470
|
+
{ pattern: /@(?:unittest\.)?expectedFailure\b/i, desc: "Injected unittest expected-failure decorator (@expectedFailure)" },
|
|
1471
|
+
{ pattern: /\bthis\.skip\s*\(/i, desc: "Injected Mocha skip (this.skip())" },
|
|
1472
|
+
{ pattern: /\b(?:it|test|describe|context)\.todo\s*\(/i, desc: "Injected todo placeholder (test.todo())" },
|
|
1473
|
+
{ pattern: /\bt\.Skip(?:Now|f)?\s*\(/, desc: "Injected Go test skip (t.Skip/t.Skipf/t.SkipNow)" },
|
|
1474
|
+
// `#[cfg(any())]` is false under every target — the idiomatic Rust way to
|
|
1475
|
+
// compile a test out without a skip or a rename. A body the cfg can never
|
|
1476
|
+
// be true for is a body the runner never executes.
|
|
1477
|
+
{ pattern: /#\s*\[\s*cfg\s*\(\s*any\s*\(\s*\)\s*\)\s*\]/, desc: "Injected impossible cfg attribute (#[cfg(any())]) — the test never compiles" },
|
|
1478
|
+
];
|
|
1479
|
+
|
|
1480
|
+
// `#` and `--` belong here for the same reason the dialects belong in
|
|
1481
|
+
// ASSERTION_PATTERN: a Ruby or Python assertion commented out is exactly
|
|
1482
|
+
// as gone as a JavaScript one, and was previously not looked for.
|
|
1483
|
+
const COMMENTED_ASSERTION =
|
|
1484
|
+
/^\+\s*(?:\/\/|\/\*|#|--)\s*(?:expect\s*\(|assert(?!ion|ing|ed\b|s\b)[a-zA-Z0-9_$]*\s*[.(]|assert\s|refute_|XCTAssert|t\.expect|t\.assert)/i;
|
|
1485
|
+
|
|
1486
|
+
const VACUOUS_ASSERTIONS = [
|
|
1487
|
+
{ pattern: /\bassert(?:\.ok)?\s*\(\s*true\s*(?:,[^)]*)?\)/i, desc: "Vacuous truth assertion (assert.ok(true))" },
|
|
1488
|
+
{ pattern: /\bassert\.(?:strictEqual|deepStrictEqual|equal|deepEqual)\s*\(\s*([^,]+?)\s*,\s*\1\s*(?:,[^)]*)?\)/i, desc: "Vacuous identity assertion (assert.equal(X, X))" },
|
|
1489
|
+
{ pattern: /\bexpect\s*\(\s*true\s*\)\s*\.(?:toBe|toEqual)\s*\(\s*true\s*\)/i, desc: "Vacuous truth expectation (expect(true).toBe(true))" },
|
|
1490
|
+
{ pattern: /\bexpect\s*\(\s*([^)]+?)\s*\)\s*\.(?:toBe|toEqual)\s*\(\s*\1\s*\)/i, desc: "Vacuous identity expectation (expect(X).toBe(X))" },
|
|
1491
|
+
{ pattern: /\bexpect\s*\(\s*true\s*\)\s*\.toBeTruthy\s*\(/i, desc: "Vacuous truth expectation (expect(true).toBeTruthy())" },
|
|
1492
|
+
{ pattern: /\bexpect\s*\(\s*false\s*\)\s*\.toBeFalsy\s*\(/i, desc: "Vacuous falsity expectation (expect(false).toBeFalsy())" },
|
|
1493
|
+
{ pattern: /\bassert\.(?:isTrue|isOk)\s*\(\s*true\s*(?:,[^)]*)?\)/i, desc: "Vacuous truth assertion (assert.isTrue(true))" },
|
|
1494
|
+
{ pattern: /\bassert\.(?:isFalse|isNotOk)\s*\(\s*false\s*(?:,[^)]*)?\)/i, desc: "Vacuous falsity assertion (assert.isFalse(false))" },
|
|
1495
|
+
{ pattern: /\b(?:XCT)?assertTrue\s*\(\s*true\s*[,)]/i, desc: "Vacuous truth assertion (assertTrue(true))" },
|
|
1496
|
+
{ pattern: /\b(?:XCT)?assertFalse\s*\(\s*false\s*[,)]/i, desc: "Vacuous falsity assertion (assertFalse(false))" },
|
|
1497
|
+
{ pattern: /\b(?:assertEquals|assertSame|XCTAssertEqual)\s*\(\s*([^,]+?)\s*,\s*\1\s*[,)]/i, desc: "Vacuous identity assertion (assertEquals(X, X))" },
|
|
1498
|
+
{ pattern: /\bassert_equal\s*\(?\s*([^,]+?)\s*,\s*\1\s*\)?\s*$/i, desc: "Vacuous identity assertion (assert_equal X, X)" },
|
|
1499
|
+
{ pattern: /\bexpect\s*\(\s*true\s*\)\s*\.to\s+be(?:\s+true)?\b/i, desc: "Vacuous truth expectation (expect(true).to be true)" },
|
|
1500
|
+
// AVA and node:test: `t.true(true)` / `t.assert(true)` assert a constant
|
|
1501
|
+
// the test itself supplied. The trial replaced `t.is(limit.activeCount,
|
|
1502
|
+
// 0)` with `t.true(true)` in a file the guard did not even classify.
|
|
1503
|
+
{ pattern: /\b[a-z_$][a-z0-9_$]{0,2}\.(?:true|truthy|assert|ok)\s*\(\s*true\s*\)/i, desc: "Vacuous truth assertion (t.true(true))" },
|
|
1504
|
+
{ pattern: /\b[a-z_$][a-z0-9_$]{0,2}\.(?:false|falsy|notOk)\s*\(\s*false\s*\)/i, desc: "Vacuous falsity assertion (t.false(false))" },
|
|
1505
|
+
];
|
|
1506
|
+
|
|
1507
|
+
// Broad on purpose: this is the denominator, not the verdict. A word
|
|
1508
|
+
// boundary immediately after `assert` never falls in `assertEquals`,
|
|
1509
|
+
// `assert_equal` or `XCTAssertEqual`, so five ecosystems contributed no
|
|
1510
|
+
// assertions to count at all and a gutted JUnit suite was arithmetically
|
|
1511
|
+
// indistinguishable from an untouched one. The lookahead keeps prose and
|
|
1512
|
+
// identifiers — `assertion`, `asserts`, `asserted` — out of the count.
|
|
1513
|
+
const ASSERTION_PATTERN =
|
|
1514
|
+
/(?:\b(?:assert(?!ion|ing|ed\b|s\b)[a-zA-Z0-9_$]*(?:\.[a-zA-Z0-9_$]+)?|refute[a-zA-Z0-9_$]*|XCTAssert[a-zA-Z0-9_$]*|XCTFail|expect|[a-z_$][a-z0-9_$]{0,2}\.(?:equal|equals|same|strictSame|deepEqual|notEqual|notSame|match|hasStrict|type|throws|rejects|ok|notOk)|t\.(?:assert|expect|is|equal|true|false|Errorf|Fatalf)|require\.[a-zA-Z0-9_$]+)\b|assert!|assert_eq!|assert_ne!)/i;
|
|
1515
|
+
// The loose net. Not a verdict and never a block — its only job is to
|
|
1516
|
+
// notice that a line was plainly an assertion in *some* dialect that
|
|
1517
|
+
// ASSERTION_PATTERN did not recognise. Without it, adding the seventh
|
|
1518
|
+
// ecosystem is indistinguishable from having covered it all along: the
|
|
1519
|
+
// guard returns the same clean PASS either way. This is the denominator
|
|
1520
|
+
// for the denominator.
|
|
1521
|
+
// Deliberately not call-shaped. Haskell's `x `shouldBe` 3` is an
|
|
1522
|
+
// assertion with no parentheses anywhere near it, and a net that only
|
|
1523
|
+
// catches `name(` reports the same confident PASS on it as on a clean
|
|
1524
|
+
// Node suite. `require` and `check` are absent on purpose: in a
|
|
1525
|
+
// CommonJS test file `require("./calc")` is an import, not a claim.
|
|
1526
|
+
const ASSERTION_SHAPED =
|
|
1527
|
+
/\b(?:assert(?!ion|ing|ed\b|s\b)|expect(?!ed\b|ation)|refute)[a-zA-Z0-9_$]*\b|`\s*should[a-zA-Z0-9_$]*\s*`|\b(?:should|must|verify|ensure|confirm)[a-zA-Z0-9_$]*\s*[(!]|\.\s*(?:should|to|to_not|not_to|must)\b|\bBOOST_[A-Z_]+\s*\(|\b[A-Z]+_(?:EQ|NE|TRUE|FALSE|THAT)\s*\(/;
|
|
1528
|
+
// `#` starts a line comment in Python/Ruby — but in Rust it opens an
|
|
1529
|
+
// attribute (`#[test]`, `#![...]`), which is code the runner keys off:
|
|
1530
|
+
// reading it as a comment is how `#[test]` removal used to slip past both
|
|
1531
|
+
// the declaration scan and the attribute check.
|
|
1532
|
+
const isCommentLine = (str) => /^\s*(?:\/\/|\/\*|\*|#(?![![])|--|;)/.test(str);
|
|
1533
|
+
|
|
1534
|
+
/** Book-keeping only: what this run looked at, before deciding anything. */
|
|
1535
|
+
const countExamined = (stats, text) => {
|
|
1536
|
+
if (!text.trim() || isCommentLine(text)) return;
|
|
1537
|
+
stats.examined++;
|
|
1538
|
+
if (ASSERTION_PATTERN.test(text)) stats.recognised++;
|
|
1539
|
+
else if (ASSERTION_SHAPED.test(text) && stats.unreadable.length < 5) stats.unreadable.push(text.trim().slice(0, 120));
|
|
1540
|
+
};
|
|
1541
|
+
|
|
1542
|
+
const fileAssertions = new Map();
|
|
1543
|
+
let pendingHunk = false;
|
|
1544
|
+
|
|
1545
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1546
|
+
const line = lines[i];
|
|
1547
|
+
|
|
1548
|
+
if ((line.startsWith("--- ") || line.startsWith("--- a/")) && !line.startsWith("----")) {
|
|
1549
|
+
const orig = line.slice(3).split("\t")[0].trim().replace(/^a\//, "");
|
|
1550
|
+
lastOldFile = orig && orig !== "/dev/null" ? orig : null;
|
|
1551
|
+
continue;
|
|
1552
|
+
}
|
|
1553
|
+
|
|
1554
|
+
if ((line.startsWith("+++ ") || line.startsWith("+++ b/") || line.startsWith("+++ /dev/null")) && !line.startsWith("++++")) {
|
|
1555
|
+
const target = line.slice(3).split("\t")[0].trim().replace(/^b\//, "");
|
|
1556
|
+
currentFile = target && target !== "/dev/null" ? target : lastOldFile;
|
|
1557
|
+
currentOldLineNo = null;
|
|
1558
|
+
currentNewLineNo = null;
|
|
1559
|
+
pendingHunk = false;
|
|
1560
|
+
continue;
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1563
|
+
const hunkMatch = /^@@ -(\d+)(?:,\d+)? \+(\d+)/.exec(line);
|
|
1564
|
+
if (hunkMatch) {
|
|
1565
|
+
currentOldLineNo = Number(hunkMatch[1]);
|
|
1566
|
+
currentNewLineNo = Number(hunkMatch[2]);
|
|
1567
|
+
pendingHunk = true;
|
|
1568
|
+
continue;
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1571
|
+
if (!currentFile || !isTestFile(currentFile)) {
|
|
1572
|
+
pendingHunk = false;
|
|
1573
|
+
continue;
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
if (!fileAssertions.has(currentFile)) {
|
|
1577
|
+
fileAssertions.set(currentFile, { removed: [], added: 0, addedTexts: [], removedSpecific: [], addedSpecific: 0, hunks: [], examined: 0, recognised: 0, unreadable: [], declRemoved: [], declAdded: [] });
|
|
1578
|
+
}
|
|
1579
|
+
const fileStats = fileAssertions.get(currentFile);
|
|
1580
|
+
if (pendingHunk) {
|
|
1581
|
+
fileStats.hunks.push({ lines: [] });
|
|
1582
|
+
pendingHunk = false;
|
|
1583
|
+
}
|
|
1584
|
+
const hunk = fileStats.hunks.length > 0 ? fileStats.hunks[fileStats.hunks.length - 1] : null;
|
|
1585
|
+
|
|
1586
|
+
if (line.startsWith("-") && !line.startsWith("---")) {
|
|
1587
|
+
const deletedText = line.slice(1);
|
|
1588
|
+
if (hunk) hunk.lines.push({ kind: "-", text: deletedText, oldNo: currentOldLineNo, newNo: null });
|
|
1589
|
+
countExamined(fileStats, deletedText);
|
|
1590
|
+
if (!isCommentLine(deletedText)) {
|
|
1591
|
+
const decl = declaredTestName(deletedText);
|
|
1592
|
+
if (decl) fileStats.declRemoved.push({ ...decl, line: currentOldLineNo, text: deletedText });
|
|
1593
|
+
}
|
|
1594
|
+
if (!isCommentLine(deletedText) && ASSERTION_PATTERN.test(deletedText)) {
|
|
1595
|
+
fileStats.removed.push({ line: currentOldLineNo, text: deletedText });
|
|
1596
|
+
if (isSpecificAssertion(deletedText)) {
|
|
1597
|
+
fileStats.removedSpecific.push({ line: currentOldLineNo, text: deletedText });
|
|
1598
|
+
}
|
|
1599
|
+
}
|
|
1600
|
+
if (currentOldLineNo !== null) currentOldLineNo++;
|
|
1601
|
+
} else if (line.startsWith("+") && !line.startsWith("+++")) {
|
|
1602
|
+
const addedText = line.slice(1);
|
|
1603
|
+
if (hunk) hunk.lines.push({ kind: "+", text: addedText, oldNo: null, newNo: currentNewLineNo });
|
|
1604
|
+
countExamined(fileStats, addedText);
|
|
1605
|
+
if (!isCommentLine(addedText)) {
|
|
1606
|
+
const decl = declaredTestName(addedText);
|
|
1607
|
+
if (decl) fileStats.declAdded.push({ ...decl, line: currentNewLineNo, text: addedText });
|
|
1608
|
+
}
|
|
1609
|
+
let isVacuous = false;
|
|
1610
|
+
|
|
1611
|
+
// Check skip injections
|
|
1612
|
+
for (const skipRule of SKIP_INJECTIONS) {
|
|
1613
|
+
if (skipRule.pattern.test(addedText)) {
|
|
1614
|
+
violations.push({
|
|
1615
|
+
file: currentFile,
|
|
1616
|
+
line: currentNewLineNo,
|
|
1617
|
+
type: "TEST_SKIP_INJECTION",
|
|
1618
|
+
reason: `Test Tamper Guard: ${skipRule.desc} found in ${currentFile}${currentNewLineNo ? `:${currentNewLineNo}` : ""}`,
|
|
1619
|
+
});
|
|
1620
|
+
}
|
|
1621
|
+
}
|
|
1622
|
+
|
|
1623
|
+
// Check vacuous assertions
|
|
1624
|
+
for (const vacuousRule of VACUOUS_ASSERTIONS) {
|
|
1625
|
+
if (vacuousRule.pattern.test(addedText)) {
|
|
1626
|
+
isVacuous = true;
|
|
1627
|
+
violations.push({
|
|
1628
|
+
file: currentFile,
|
|
1629
|
+
line: currentNewLineNo,
|
|
1630
|
+
type: "VACUOUS_ASSERTION",
|
|
1631
|
+
reason: `Test Tamper Guard: ${vacuousRule.desc} detected in ${currentFile}${currentNewLineNo ? `:${currentNewLineNo}` : ""}`,
|
|
1632
|
+
});
|
|
1633
|
+
}
|
|
1634
|
+
}
|
|
1635
|
+
|
|
1636
|
+
// Go build constraints (ignore / private-tag tightening) are assessed
|
|
1637
|
+
// as a hunk post-pass below, because their witness — the removed
|
|
1638
|
+
// constraint line — is not present while this added line is scanned.
|
|
1639
|
+
|
|
1640
|
+
// Check commented-out assertions
|
|
1641
|
+
let isCommented = false;
|
|
1642
|
+
if (COMMENTED_ASSERTION.test(line)) {
|
|
1643
|
+
isCommented = true;
|
|
1644
|
+
violations.push({
|
|
1645
|
+
file: currentFile,
|
|
1646
|
+
line: currentNewLineNo,
|
|
1647
|
+
type: "COMMENTED_ASSERTION",
|
|
1648
|
+
reason: `Test Tamper Guard: Commented-out test assertion detected in ${currentFile}${currentNewLineNo ? `:${currentNewLineNo}` : ""}`,
|
|
1649
|
+
});
|
|
1650
|
+
}
|
|
1651
|
+
|
|
1652
|
+
// If valid non-vacuous, non-commented assertion is added, increment added count
|
|
1653
|
+
if (!isVacuous && !isCommented && !isCommentLine(addedText) && ASSERTION_PATTERN.test(addedText)) {
|
|
1654
|
+
fileStats.added++;
|
|
1655
|
+
fileStats.addedTexts.push({ line: currentNewLineNo, text: addedText });
|
|
1656
|
+
if (isSpecificAssertion(addedText)) fileStats.addedSpecific++;
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
if (currentNewLineNo !== null) currentNewLineNo++;
|
|
1660
|
+
} else if (!line.startsWith("\\")) {
|
|
1661
|
+
// Context line. It is file text in both images, so the statement
|
|
1662
|
+
// assembler needs it to reassemble assertions that span a changed
|
|
1663
|
+
// line; `diff --git`/`index` lines that sneak in here are not
|
|
1664
|
+
// diff body and are not collected.
|
|
1665
|
+
if (hunk && (line.startsWith(" ") || line === "")) {
|
|
1666
|
+
hunk.lines.push({ kind: " ", text: line.slice(1), oldNo: currentOldLineNo, newNo: currentNewLineNo });
|
|
1667
|
+
}
|
|
1668
|
+
if (currentOldLineNo !== null) currentOldLineNo++;
|
|
1669
|
+
if (currentNewLineNo !== null) currentNewLineNo++;
|
|
1670
|
+
}
|
|
1671
|
+
}
|
|
1672
|
+
|
|
1673
|
+
// An assertion is a statement, not a line.
|
|
1674
|
+
//
|
|
1675
|
+
// Counting `+`/`-` lines missed the commonest shape in every language with
|
|
1676
|
+
// multi-line calls: `self.assertEqual(` sits on an unchanged context line
|
|
1677
|
+
// and only its argument lines are edited. Nothing among the changed lines
|
|
1678
|
+
// matched an assertion pattern, nothing looked assertion-shaped either, so
|
|
1679
|
+
// the guard reported `assertionsSeen: 0` and — because no line looked
|
|
1680
|
+
// suspicious — a clean PASS. A five-element expected list rewritten to one
|
|
1681
|
+
// element to match broken output sailed through five green phases.
|
|
1682
|
+
//
|
|
1683
|
+
// The statement machinery that already exists for pairing knows better:
|
|
1684
|
+
// it assembles context lines together with changed ones. Ask it.
|
|
1685
|
+
for (const [file, stats] of fileAssertions.entries()) {
|
|
1686
|
+
const lang = langForTestFile(file);
|
|
1687
|
+
let touched = 0;
|
|
1688
|
+
let stmtSpecificRemoved = 0;
|
|
1689
|
+
let stmtSpecificAdded = 0;
|
|
1690
|
+
for (const hunk of stats.hunks) {
|
|
1691
|
+
const oldSlice = [];
|
|
1692
|
+
const newSlice = [];
|
|
1693
|
+
for (const L of hunk.lines) {
|
|
1694
|
+
if (L.kind !== "+") oldSlice.push(L);
|
|
1695
|
+
if (L.kind !== "-") newSlice.push(L);
|
|
1696
|
+
}
|
|
1697
|
+
// Specific assertions are counted per side, on the reassembled
|
|
1698
|
+
// statement. The weakening check was still line-based: `assert (` alone
|
|
1699
|
+
// on a line names no value, so Black or Ruff splitting one assertion
|
|
1700
|
+
// across three lines removed one specific assertion and added none, and
|
|
1701
|
+
// a reformat was reported as CRITICAL tampering. Collapsing the joined
|
|
1702
|
+
// statement is what lets the bare-comparison form be recognised at all
|
|
1703
|
+
// — its pattern cannot cross a newline.
|
|
1704
|
+
const sides = [
|
|
1705
|
+
{ stmts: assembleStatements(oldSlice, lang), kind: "-" },
|
|
1706
|
+
{ stmts: assembleStatements(newSlice, lang), kind: "+" },
|
|
1707
|
+
];
|
|
1708
|
+
for (const { stmts, kind } of sides) {
|
|
1709
|
+
for (const st of stmts) {
|
|
1710
|
+
const changed = kind === "-" ? (st.removedLines?.length || 0) > 0 : (st.addedLines?.length || 0) > 0;
|
|
1711
|
+
if (!changed) continue;
|
|
1712
|
+
const clean = stripComments(st.text, lang);
|
|
1713
|
+
if (ASSERTION_PATTERN.test(clean)) touched++;
|
|
1714
|
+
if (isSpecificAssertion(collapseWhitespace(clean))) {
|
|
1715
|
+
if (kind === "-") stmtSpecificRemoved++;
|
|
1716
|
+
else stmtSpecificAdded++;
|
|
1717
|
+
}
|
|
1718
|
+
}
|
|
1719
|
+
}
|
|
1720
|
+
}
|
|
1721
|
+
stats.statementAssertions = touched;
|
|
1722
|
+
stats.stmtSpecificRemoved = stmtSpecificRemoved;
|
|
1723
|
+
stats.stmtSpecificAdded = stmtSpecificAdded;
|
|
1724
|
+
}
|
|
1725
|
+
|
|
1726
|
+
// Assertions added in some *other* file of this same diff.
|
|
1727
|
+
//
|
|
1728
|
+
// Tracking is strictly per file, so moving a test from one file to another
|
|
1729
|
+
// — ordinary refactoring — read as deleting three assertions and was
|
|
1730
|
+
// reported as CRITICAL tampering. The assertion still exists and still
|
|
1731
|
+
// runs; it is in a different file. What the removal check is for is
|
|
1732
|
+
// verification that *disappeared*, and this has not.
|
|
1733
|
+
//
|
|
1734
|
+
// Deliberately exact on the assertion text: something that changed on the
|
|
1735
|
+
// way across is not a move, and is judged normally.
|
|
1736
|
+
const movedIn = new Map();
|
|
1737
|
+
for (const [file, stats] of fileAssertions.entries()) {
|
|
1738
|
+
for (const t of stats.addedTexts || []) {
|
|
1739
|
+
const key = collapseWhitespace(String(t.text ?? t)).trim();
|
|
1740
|
+
if (!key) continue;
|
|
1741
|
+
if (!movedIn.has(key)) movedIn.set(key, []);
|
|
1742
|
+
movedIn.get(key).push(file);
|
|
1743
|
+
}
|
|
1744
|
+
}
|
|
1745
|
+
|
|
1746
|
+
// A test renamed out of its runner's discovery convention.
|
|
1747
|
+
//
|
|
1748
|
+
// pytest collects `test*` and nothing else, so `def test_totals` becoming
|
|
1749
|
+
// `def check_totals` deletes the test from every future run while leaving it
|
|
1750
|
+
// in the file, fully written, with all its assertions intact. Every count in
|
|
1751
|
+
// this guard stays level: nothing removed, weakened or rewritten.
|
|
1752
|
+
//
|
|
1753
|
+
// The earlier rule required the new name to be the old one with its prefix
|
|
1754
|
+
// literally stripped (`test_x` -> `x` caught; `test_x` -> `check_x` not).
|
|
1755
|
+
// The collected check is what actually matters, and it already keeps the
|
|
1756
|
+
// honest renames silent — `test_x` -> `test_x_renamed`, pytest's `test*`
|
|
1757
|
+
// glob still collecting `testx`, Go's `TestX` -> `TestXRenamed` — so the
|
|
1758
|
+
// narrow strip is gone.
|
|
1759
|
+
for (const [file, stats] of fileAssertions.entries()) {
|
|
1760
|
+
const takenAdds = new Set();
|
|
1761
|
+
for (const before of stats.declRemoved || []) {
|
|
1762
|
+
if (!before.collected) continue;
|
|
1763
|
+
const idx = (stats.declAdded || []).findIndex((after, i) => !takenAdds.has(i) && isDeregistration(before, after));
|
|
1764
|
+
if (idx === -1) continue;
|
|
1765
|
+
takenAdds.add(idx);
|
|
1766
|
+
const after = stats.declAdded[idx];
|
|
1767
|
+
violations.push({
|
|
1768
|
+
file,
|
|
1769
|
+
line: after.line ?? before.line,
|
|
1770
|
+
type: "TEST_DEREGISTERED",
|
|
1771
|
+
reason:
|
|
1772
|
+
`Test Tamper Guard: ${JSON.stringify(before.name)} was renamed to ${JSON.stringify(after.name)} in ${file}` +
|
|
1773
|
+
`${after.line ? `:${after.line}` : ""}. The runner collects tests by name, so the test still exists in ` +
|
|
1774
|
+
`the file and no longer runs — the same effect as deleting it, with none of the signs. ` +
|
|
1775
|
+
`If the test is genuinely obsolete, delete it; if it is being turned into a helper, say so with ` +
|
|
1776
|
+
`--allow-test-change deregistration.`,
|
|
1777
|
+
});
|
|
1778
|
+
}
|
|
1779
|
+
}
|
|
1780
|
+
|
|
1781
|
+
// Runners that register by attribute/annotation rather than by name: Rust's
|
|
1782
|
+
// `#[test]` (the name above the function is free-form, so the name pair
|
|
1783
|
+
// above cannot see this family) and JUnit's `@Test`. Removing the attribute
|
|
1784
|
+
// from an existing function keeps the body and loses the test — the same
|
|
1785
|
+
// uncollect with no line deleted.
|
|
1786
|
+
const TEST_ATTR_PATTERNS = [
|
|
1787
|
+
{ lang: "rust", re: /^\s*#\s*\[\s*(?:test|tokio::test|async_std::test)\s*\]/ },
|
|
1788
|
+
// JUnit/TestNG annotations live in files the scanner lexes as `js`
|
|
1789
|
+
// (the C-like family), so the lang here is the scanner's lang, not the
|
|
1790
|
+
// source language's name.
|
|
1791
|
+
{ lang: "js", re: /^\s*@(?:org\.junit\.)?(?:jupiter\.api\.)?Test\b/ },
|
|
1792
|
+
];
|
|
1793
|
+
const attrRegistration = (text, file) => {
|
|
1794
|
+
const lang = langForTestFile(file);
|
|
1795
|
+
return TEST_ATTR_PATTERNS.some((rule) => rule.lang === lang && rule.re.test(text));
|
|
1796
|
+
};
|
|
1797
|
+
|
|
1798
|
+
// Function signature text of the declaration the attribute at `start`
|
|
1799
|
+
// governs. Attributes sit immediately above `fn x()` / `void x()`, so the
|
|
1800
|
+
// next declaration line in the hunk carries the signature; scanning
|
|
1801
|
+
// backwards covers an attribute written on a context line position.
|
|
1802
|
+
const FN_SIG_RE = /\b(?:fn|func|def)\s+[A-Za-z_]\w*\s*\(|\b(?:void|[A-Za-z_][\w.<>\[\]]*)\s+[A-Za-z_]\w*\s*\([^;]*\)\s*(?:\{|$|throws\b)/;
|
|
1803
|
+
const adjacentSignature = (hunkLines, start) => {
|
|
1804
|
+
for (let k = start + 1; k < hunkLines.length; k++) {
|
|
1805
|
+
const t = hunkLines[k].text || "";
|
|
1806
|
+
if (FN_SIG_RE.test(t)) return collapseWhitespace(t).trim();
|
|
1807
|
+
}
|
|
1808
|
+
for (let k = start - 1; k >= 0; k--) {
|
|
1809
|
+
const t = hunkLines[k].text || "";
|
|
1810
|
+
if (FN_SIG_RE.test(t)) return collapseWhitespace(t).trim();
|
|
1811
|
+
}
|
|
1812
|
+
return null;
|
|
1813
|
+
};
|
|
1814
|
+
|
|
1815
|
+
// Attribute arrivals across the whole diff: a registration lost in one
|
|
1816
|
+
// file is forgiven when the same signature gained one in another — a move
|
|
1817
|
+
// between test files is ordinary refactoring, the same allowance the
|
|
1818
|
+
// assertion-removal check makes for moved assertions.
|
|
1819
|
+
const attrArrivals = new Map(); // signature -> [files]
|
|
1820
|
+
for (const [file, stats] of fileAssertions.entries()) {
|
|
1821
|
+
for (const hunk of stats.hunks) {
|
|
1822
|
+
for (let i = 0; i < hunk.lines.length; i++) {
|
|
1823
|
+
const L = hunk.lines[i];
|
|
1824
|
+
if (L.kind !== "+" || !attrRegistration(L.text, file)) continue;
|
|
1825
|
+
const sig = adjacentSignature(hunk.lines, i);
|
|
1826
|
+
if (!sig) continue;
|
|
1827
|
+
if (!attrArrivals.has(sig)) attrArrivals.set(sig, []);
|
|
1828
|
+
attrArrivals.get(sig).push(file);
|
|
1829
|
+
}
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
|
|
1833
|
+
for (const [file, stats] of fileAssertions.entries()) {
|
|
1834
|
+
for (const hunk of stats.hunks) {
|
|
1835
|
+
const removedAttrs = [];
|
|
1836
|
+
let addedInHunk = 0;
|
|
1837
|
+
for (let i = 0; i < hunk.lines.length; i++) {
|
|
1838
|
+
const L = hunk.lines[i];
|
|
1839
|
+
if (L.kind === "-" && !isCommentLine(L.text) && attrRegistration(L.text, file)) {
|
|
1840
|
+
removedAttrs.push({ line: L.oldNo, text: L.text, sig: adjacentSignature(hunk.lines, i) });
|
|
1841
|
+
}
|
|
1842
|
+
if (L.kind === "+" && attrRegistration(L.text, file)) addedInHunk++;
|
|
1843
|
+
}
|
|
1844
|
+
// Net loss within the hunk; same-file additions cancel one for one
|
|
1845
|
+
// (a test renamed or re-attributed counts even).
|
|
1846
|
+
let deficit = removedAttrs.length - addedInHunk;
|
|
1847
|
+
for (const attr of removedAttrs) {
|
|
1848
|
+
if (deficit <= 0) break;
|
|
1849
|
+
const landed = attr.sig ? attrArrivals.get(attr.sig) : null;
|
|
1850
|
+
if (landed) {
|
|
1851
|
+
const elsewhere = landed.findIndex((f) => f !== file);
|
|
1852
|
+
if (elsewhere !== -1) {
|
|
1853
|
+
landed.splice(elsewhere, 1);
|
|
1854
|
+
continue;
|
|
1855
|
+
}
|
|
1856
|
+
}
|
|
1857
|
+
deficit--;
|
|
1858
|
+
violations.push({
|
|
1859
|
+
file,
|
|
1860
|
+
line: attr.line,
|
|
1861
|
+
type: "TEST_DEREGISTERED",
|
|
1862
|
+
reason:
|
|
1863
|
+
`Test Tamper Guard: a test-registration attribute was removed from an existing function in ${file}` +
|
|
1864
|
+
`${attr.line ? `:${attr.line}` : ""} (${collapseWhitespace(attr.text).trim()}). ` +
|
|
1865
|
+
`The runner only executes functions carrying that attribute, so the test still exists in the file ` +
|
|
1866
|
+
`and no longer runs — the same effect as deleting it, with none of the signs. If the test is ` +
|
|
1867
|
+
`genuinely obsolete, delete it; if it is becoming a helper, say so with ` +
|
|
1868
|
+
`--allow-test-change deregistration.`,
|
|
1869
|
+
});
|
|
1870
|
+
}
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
|
|
1874
|
+
// A failure call parked behind a condition that cannot hold. Keeping the
|
|
1875
|
+
// `t.Errorf` while swapping its guard for `if len(s) < 0` (a Go string or
|
|
1876
|
+
// slice length is never negative) preserves every line of the old
|
|
1877
|
+
// assertion in code that can never run — F05, measured approving the
|
|
1878
|
+
// neutralised test. This is a hunk post-pass rather than an added-line
|
|
1879
|
+
// rule because the failure call it protects is untouched code: it sits on
|
|
1880
|
+
// a context line, which the line-by-line scan has not walked over yet.
|
|
1881
|
+
//
|
|
1882
|
+
// Only conditions that are impossible on their face are looked at
|
|
1883
|
+
// (`len(...) < 0`, an unsigned/size count `<= -1`); anything fuzzier — a
|
|
1884
|
+
// flag flipped elsewhere, a branch behind real state — is not guessable
|
|
1885
|
+
// from a diff and is deliberately left alone. The failure call has to be
|
|
1886
|
+
// reachable inside the condition's block, or an impossible condition in
|
|
1887
|
+
// ordinary test setup would be misread as one.
|
|
1888
|
+
for (const [file, stats] of fileAssertions.entries()) {
|
|
1889
|
+
for (const hunk of stats.hunks) {
|
|
1890
|
+
for (let i = 0; i < hunk.lines.length; i++) {
|
|
1891
|
+
const L = hunk.lines[i];
|
|
1892
|
+
if (L.kind !== "+" || !DEAD_GUARD_CONDITION.test(L.text)) continue;
|
|
1893
|
+
const guardLineNo = L.newNo;
|
|
1894
|
+
const guardText = L.text;
|
|
1895
|
+
const isFailureOrAssertion = (text) => FAILURE_CALL.test(text) || ASSERTION_PATTERN.test(text);
|
|
1896
|
+
let found = false;
|
|
1897
|
+
|
|
1898
|
+
if (guardText.includes("{")) {
|
|
1899
|
+
let depth = (guardText.match(/\{/g) || []).length - (guardText.match(/\}/g) || []).length;
|
|
1900
|
+
if (depth > 0 && isFailureOrAssertion(guardText.slice(guardText.indexOf("{")))) {
|
|
1901
|
+
found = true;
|
|
1902
|
+
}
|
|
1903
|
+
for (let k = i + 1; k < hunk.lines.length && depth > 0; k++) {
|
|
1904
|
+
const t = hunk.lines[k].text || "";
|
|
1905
|
+
const opens = (t.match(/\{/g) || []).length;
|
|
1906
|
+
const closes = (t.match(/\}/g) || []).length;
|
|
1907
|
+
if (depth > 0 && isFailureOrAssertion(t)) { found = true; break; }
|
|
1908
|
+
depth += opens - closes;
|
|
1909
|
+
}
|
|
1910
|
+
} else {
|
|
1911
|
+
// Indentation-based block (Python, etc.) or single-line guard without braces
|
|
1912
|
+
const colonIdx = guardText.indexOf(":");
|
|
1913
|
+
const parenIdx = guardText.lastIndexOf(")");
|
|
1914
|
+
const afterGuard = colonIdx !== -1 ? guardText.slice(colonIdx + 1) : (parenIdx !== -1 ? guardText.slice(parenIdx + 1) : "");
|
|
1915
|
+
if (afterGuard && isFailureOrAssertion(afterGuard)) {
|
|
1916
|
+
found = true;
|
|
1917
|
+
} else {
|
|
1918
|
+
const guardIndent = (guardText.match(/^\s*/)?.[0] || "").length;
|
|
1919
|
+
for (let k = i + 1; k < hunk.lines.length; k++) {
|
|
1920
|
+
const t = hunk.lines[k].text || "";
|
|
1921
|
+
if (!t.trim() || isCommentLine(t)) continue;
|
|
1922
|
+
const lineIndent = (t.match(/^\s*/)?.[0] || "").length;
|
|
1923
|
+
if (lineIndent <= guardIndent) break;
|
|
1924
|
+
if (isFailureOrAssertion(t)) { found = true; break; }
|
|
1925
|
+
}
|
|
1926
|
+
}
|
|
1927
|
+
}
|
|
1928
|
+
|
|
1929
|
+
if (found) {
|
|
1930
|
+
violations.push({
|
|
1931
|
+
file,
|
|
1932
|
+
line: guardLineNo,
|
|
1933
|
+
type: "VACUOUS_ASSERTION",
|
|
1934
|
+
reason:
|
|
1935
|
+
`Test Tamper Guard: the assertion or failure call in ${file}${guardLineNo ? `:${guardLineNo}` : ""} ` +
|
|
1936
|
+
`is guarded by a condition that can never be true ("${collapseWhitespace(guardText).trim()}"), so the ` +
|
|
1937
|
+
`assertion cannot fail. Keeping the assertion in dead logic is not preservation of verification.`,
|
|
1938
|
+
});
|
|
1939
|
+
}
|
|
1940
|
+
}
|
|
1941
|
+
}
|
|
1942
|
+
}
|
|
1943
|
+
|
|
1944
|
+
// Go build constraints that exclude a test file from the ordinary run.
|
|
1945
|
+
// `//go:build ignore` never matches a release build; conjoining a private
|
|
1946
|
+
// tag onto an existing constraint (`go1.7` -> `go1.7 && my_tag`) excludes
|
|
1947
|
+
// the file unless a CI job sets that tag. Version, OS and arch terms are
|
|
1948
|
+
// legitimate gating, so only an impossible ignore or a new private term
|
|
1949
|
+
// counts.
|
|
1950
|
+
for (const [file, stats] of fileAssertions.entries()) {
|
|
1951
|
+
if (langForTestFile(file) !== "go") continue;
|
|
1952
|
+
for (const hunk of stats.hunks) {
|
|
1953
|
+
for (let i = 0; i < hunk.lines.length; i++) {
|
|
1954
|
+
const L = hunk.lines[i];
|
|
1955
|
+
if (L.kind !== "+") continue;
|
|
1956
|
+
const terms = goBuildTerms(L.text);
|
|
1957
|
+
if (!terms) continue;
|
|
1958
|
+
if (terms.every((t) => t === "ignore") || terms.includes("ignore")) {
|
|
1959
|
+
violations.push({
|
|
1960
|
+
file,
|
|
1961
|
+
line: L.newNo,
|
|
1962
|
+
type: "TEST_SKIP_INJECTION",
|
|
1963
|
+
reason:
|
|
1964
|
+
`Test Tamper Guard: Go build constraint ${JSON.stringify(collapseWhitespace(L.text).trim())} in ${file}` +
|
|
1965
|
+
`${L.newNo ? `:${L.newNo}` : ""} excludes this test file from the ordinary 'go test' run — the ignore tag never ` +
|
|
1966
|
+
`matches a release build. Use --allow-test-change skip if the file is built by a deliberately separate command.`,
|
|
1967
|
+
});
|
|
1968
|
+
continue;
|
|
1969
|
+
}
|
|
1970
|
+
const oldTerms = new Set();
|
|
1971
|
+
for (const O of hunk.lines) {
|
|
1972
|
+
if (O.kind !== "-") continue;
|
|
1973
|
+
const t = goBuildTerms(O.text);
|
|
1974
|
+
if (t) for (const term of t) oldTerms.add(term);
|
|
1975
|
+
}
|
|
1976
|
+
const privateTag = terms.find((t) => !goKnownBuildTerm(t) && !oldTerms.has(t));
|
|
1977
|
+
if (privateTag) {
|
|
1978
|
+
violations.push({
|
|
1979
|
+
file,
|
|
1980
|
+
line: L.newNo,
|
|
1981
|
+
type: "TEST_SKIP_INJECTION",
|
|
1982
|
+
reason:
|
|
1983
|
+
`Test Tamper Guard: Go build constraint tightened with the private tag ${JSON.stringify(privateTag)} in ` +
|
|
1984
|
+
`${file}${L.newNo ? `:${L.newNo}` : ""} — a tag no ordinary 'go test' run sets excludes this test file from the ` +
|
|
1985
|
+
`suite. Version/OS/arch gating stays silent; a tag nobody passes is a skip. Use --allow-test-change skip ` +
|
|
1986
|
+
`if the tag is gated in CI on purpose.`,
|
|
1987
|
+
});
|
|
1988
|
+
}
|
|
1989
|
+
}
|
|
1990
|
+
}
|
|
1991
|
+
}
|
|
1992
|
+
|
|
1993
|
+
// An unconditional early return as the test body's first statement. A test
|
|
1994
|
+
// that returns before its assertions keeps the suite green while
|
|
1995
|
+
// verifying nothing — the trial made a broken function pass this way.
|
|
1996
|
+
// Unlike the skip above this spelling has no marker, so the declaration
|
|
1997
|
+
// has to be visible as the nearest non-blank line above the return:
|
|
1998
|
+
// `return` as a genuine guard clause (after a real condition or check)
|
|
1999
|
+
// never sits with a test signature as its predecessor, and a `return`
|
|
2000
|
+
// inside the body an honest refactor edited is past body-open already.
|
|
2001
|
+
for (const [file, stats] of fileAssertions.entries()) {
|
|
2002
|
+
for (const hunk of stats.hunks) {
|
|
2003
|
+
for (let i = 0; i < hunk.lines.length; i++) {
|
|
2004
|
+
const L = hunk.lines[i];
|
|
2005
|
+
if (L.kind !== "+" || !BARE_EARLY_RETURN.test(L.text)) continue;
|
|
2006
|
+
for (let k = i - 1; k >= 0; k--) {
|
|
2007
|
+
const t = (hunk.lines[k].text || "").trim();
|
|
2008
|
+
if (!t) continue;
|
|
2009
|
+
if (TEST_BODY_OPEN.test(hunk.lines[k].text || "")) {
|
|
2010
|
+
violations.push({
|
|
2011
|
+
file,
|
|
2012
|
+
line: L.newNo,
|
|
2013
|
+
type: "TEST_SKIP_INJECTION",
|
|
2014
|
+
reason:
|
|
2015
|
+
`Test Tamper Guard: an unconditional early return opens the test body in ` +
|
|
2016
|
+
`${file}${L.newNo ? `:${L.newNo}` : ""}, so none of the assertions after it run while the test still ` +
|
|
2017
|
+
`passes. A suite cannot verify what control flow never reaches. Use --allow-test-change skip if this ` +
|
|
2018
|
+
`test is deliberately dormant.`,
|
|
2019
|
+
});
|
|
2020
|
+
}
|
|
2021
|
+
break;
|
|
2022
|
+
}
|
|
2023
|
+
}
|
|
2024
|
+
}
|
|
2025
|
+
}
|
|
2026
|
+
|
|
2027
|
+
for (const [file, stats] of fileAssertions.entries()) {
|
|
2028
|
+
// An expectation that was rewritten rather than removed.
|
|
2029
|
+
//
|
|
2030
|
+
// Counting assertions cannot see this one: `assert.equal(add(1,2), 3)`
|
|
2031
|
+
// becoming `assert.equal(add(1,2), -1)` takes one specific assertion out
|
|
2032
|
+
// and puts one specific assertion back, so every total stayed level and
|
|
2033
|
+
// the guard said nothing — while the suite went from checking that
|
|
2034
|
+
// addition works to certifying that it is broken. It is the single
|
|
2035
|
+
// cheapest way to make a red suite green, and the one this tool exists
|
|
2036
|
+
// to refuse. The line-based version of this pairing could only see the
|
|
2037
|
+
// single-line spelling of the edit; the reassembled-statement version
|
|
2038
|
+
// above sees every spelling, and explains why on the way.
|
|
2039
|
+
detectExpectationRewrites(file, stats.hunks, stats, violations);
|
|
2040
|
+
|
|
2041
|
+
if (stats.removed.length > stats.added) {
|
|
2042
|
+
const unreplaced = stats.removed.slice(stats.added);
|
|
2043
|
+
for (const item of unreplaced) {
|
|
2044
|
+
// Did this exact assertion arrive somewhere else in the same diff?
|
|
2045
|
+
const key = collapseWhitespace(item.text).trim();
|
|
2046
|
+
const landing = movedIn.get(key);
|
|
2047
|
+
const elsewhere = landing ? landing.findIndex((f) => f !== file) : -1;
|
|
2048
|
+
if (elsewhere !== -1) {
|
|
2049
|
+
// Consume the landing so two removals cannot both claim one arrival.
|
|
2050
|
+
landing.splice(elsewhere, 1);
|
|
2051
|
+
continue;
|
|
2052
|
+
}
|
|
2053
|
+
violations.push({
|
|
2054
|
+
file,
|
|
2055
|
+
line: item.line,
|
|
2056
|
+
type: "ASSERTION_REMOVAL",
|
|
2057
|
+
reason: `Test Tamper Guard: Assertion removed without replacement in ${file}${item.line ? `:${item.line}` : ""}: "${item.text.trim()}"`,
|
|
2058
|
+
});
|
|
2059
|
+
}
|
|
2060
|
+
}
|
|
2061
|
+
|
|
2062
|
+
// Replacing an assertion is not the same as keeping one. Counting totals
|
|
2063
|
+
// let a specific expectation be swapped for a vague one at no cost — one
|
|
2064
|
+
// out, one in, guard silent, suite no longer checking the answer. What must
|
|
2065
|
+
// not fall is the number of assertions that name an expected value.
|
|
2066
|
+
//
|
|
2067
|
+
// Only the *replaced* ones are reported here. An assertion deleted outright
|
|
2068
|
+
// is already an ASSERTION_REMOVAL above, and emitting both would report the
|
|
2069
|
+
// same line twice under two names.
|
|
2070
|
+
const alreadyReportedSpecific = stats.removed
|
|
2071
|
+
.slice(stats.added)
|
|
2072
|
+
.filter((item) => isSpecificAssertion(item.text)).length;
|
|
2073
|
+
// Believe whichever unit saw more arrive. A statement is the honest unit,
|
|
2074
|
+
// but the line count still carries cases the statement scanner cannot
|
|
2075
|
+
// assemble, so the loss is only what *both* agree was lost.
|
|
2076
|
+
const lineLost = Math.max(0, stats.removedSpecific.length - stats.addedSpecific);
|
|
2077
|
+
const stmtLost = Math.max(0, (stats.stmtSpecificRemoved || 0) - (stats.stmtSpecificAdded || 0));
|
|
2078
|
+
const specificLost = Math.min(lineLost, stmtLost);
|
|
2079
|
+
const weakenedCount = Math.max(0, specificLost - alreadyReportedSpecific);
|
|
2080
|
+
|
|
2081
|
+
if (weakenedCount > 0) {
|
|
2082
|
+
for (const item of stats.removedSpecific.slice(stats.addedSpecific, stats.addedSpecific + weakenedCount)) {
|
|
2083
|
+
violations.push({
|
|
2084
|
+
file,
|
|
2085
|
+
line: item.line,
|
|
2086
|
+
type: "ASSERTION_WEAKENED",
|
|
2087
|
+
reason: `Test Tamper Guard: Assertion weakened in ${file}${item.line ? `:${item.line}` : ""} — an assertion naming an expected value was replaced by one that does not: "${item.text.trim()}"`,
|
|
2088
|
+
});
|
|
2089
|
+
}
|
|
2090
|
+
}
|
|
2091
|
+
}
|
|
2092
|
+
|
|
2093
|
+
// A kind the operator has already looked at and accepted is dropped here
|
|
2094
|
+
// rather than never being computed, so the reasoning above stays one code
|
|
2095
|
+
// path regardless of what any given run allows.
|
|
2096
|
+
const reported =
|
|
2097
|
+
allowed.kinds.size === 0
|
|
2098
|
+
? violations
|
|
2099
|
+
: violations.filter((v) => !allowed.kinds.has(TAMPER_KINDS.get(v.type)));
|
|
2100
|
+
|
|
2101
|
+
// What was examined, not only what was found.
|
|
2102
|
+
//
|
|
2103
|
+
// `ok: true` from a guard that looked at nothing is byte-identical to
|
|
2104
|
+
// `ok: true` from a guard that looked at everything and approved it. That
|
|
2105
|
+
// ambiguity is how a substring bug in the file classifier switched this
|
|
2106
|
+
// entire guard off for the standard pytest, Rust and RSpec layouts while
|
|
2107
|
+
// every signal stayed green.
|
|
2108
|
+
//
|
|
2109
|
+
// Counting *files* was not enough. A JUnit diff that rewrote an expected
|
|
2110
|
+
// value produced `inputsSeen: 1` and a clean PASS while not one assertion
|
|
2111
|
+
// in it had been recognised — the same ambiguity, one level down, inside
|
|
2112
|
+
// the mechanism built to remove it. So the denominator is now the thing
|
|
2113
|
+
// the rules actually consume: lines examined, and of those, assertions
|
|
2114
|
+
// understood. `UNREADABLE` is the state that has no business being silent
|
|
2115
|
+
// — assertion-shaped lines were present and none of them parsed, which
|
|
2116
|
+
// means this repository speaks a dialect the guard does not.
|
|
2117
|
+
let examined = 0;
|
|
2118
|
+
let assertionsSeen = 0;
|
|
2119
|
+
const unreadable = [];
|
|
2120
|
+
for (const [file, stats] of fileAssertions.entries()) {
|
|
2121
|
+
examined += stats.examined;
|
|
2122
|
+
assertionsSeen += Math.max(stats.recognised, stats.statementAssertions || 0);
|
|
2123
|
+
if (stats.unreadable.length > 0) {
|
|
2124
|
+
unreadable.push({ file, count: stats.unreadable.length, samples: stats.unreadable.slice(0, 3) });
|
|
2125
|
+
}
|
|
2126
|
+
}
|
|
2127
|
+
|
|
2128
|
+
// Changed lines inside a test file, none of them recognisable as part of an
|
|
2129
|
+
// assertion, is not the same as "checked and clean" — it is the state where
|
|
2130
|
+
// this guard has nothing to say. Saying nothing and saying "approved" have
|
|
2131
|
+
// to look different, which is the whole reason `status` exists.
|
|
2132
|
+
// `unreadable` is the evidence, and it is required.
|
|
2133
|
+
//
|
|
2134
|
+
// `|| examined > 0` used to stand here, and it threw away the distinction
|
|
2135
|
+
// this whole apparatus exists to draw. `ASSERTION_SHAPED` and `unreadable[]`
|
|
2136
|
+
// were built to separate "assertion-shaped lines were present and none of
|
|
2137
|
+
// them parsed" — a dialect the guard cannot read — from "there were no
|
|
2138
|
+
// assertions in these lines at all", which is most ordinary work on a test
|
|
2139
|
+
// file. That clause collapsed the two, so *any* changed substantive line in
|
|
2140
|
+
// a test file with no recognised assertion became a CRITICAL block:
|
|
2141
|
+
// measured on `pytest-dev/iniconfig`, renaming a test function did it, and
|
|
2142
|
+
// so did adding `import os`.
|
|
2143
|
+
//
|
|
2144
|
+
// The tell was in the finding itself: it carried `file: null`, `line: null`
|
|
2145
|
+
// and no sample, because `unreadable` was empty — the guard blocked while
|
|
2146
|
+
// holding no evidence of anything, and advised a pytest repository that its
|
|
2147
|
+
// assertion library might be unsupported, from a list that names pytest.
|
|
2148
|
+
//
|
|
2149
|
+
// Nothing is weakened by requiring the evidence. A removed or rewritten
|
|
2150
|
+
// assertion is a recognised assertion line, so it raises `assertionsSeen`
|
|
2151
|
+
// and goes to the ordinary removal and weakening checks; it never reached
|
|
2152
|
+
// this branch. What is lost is only the blanket, and a blanket that fires
|
|
2153
|
+
// on `import os` teaches its way around itself: the remedy it printed was
|
|
2154
|
+
// `tamperGuard: "warn"`, which switches the real guard off too.
|
|
2155
|
+
const status =
|
|
2156
|
+
reported.length > 0
|
|
2157
|
+
? "FAIL"
|
|
2158
|
+
: assertionsSeen === 0 && unreadable.length > 0
|
|
2159
|
+
? "UNREADABLE"
|
|
2160
|
+
: examined > 0
|
|
2161
|
+
? "PASS"
|
|
2162
|
+
: "NOT_APPLICABLE";
|
|
2163
|
+
|
|
2164
|
+
return {
|
|
2165
|
+
ok: reported.length === 0,
|
|
2166
|
+
violations: reported,
|
|
2167
|
+
inputsSeen: examined,
|
|
2168
|
+
filesSeen: fileAssertions.size,
|
|
2169
|
+
assertionsSeen,
|
|
2170
|
+
unreadable,
|
|
2171
|
+
status,
|
|
2172
|
+
};
|
|
2173
|
+
}
|