jules-orchestrator-kit 0.58.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +1 -1
- package/src/coverage.mjs +2 -6
- package/src/engine.mjs +2 -4
- package/src/evidence.mjs +2 -14
- package/src/mutation.mjs +2 -8
- package/src/security.mjs +698 -94
- package/src/test-paths.mjs +68 -0
package/README.md
CHANGED
|
@@ -204,7 +204,7 @@ To maximize PR merge rates, dispatch tasks according to deterministic boundaries
|
|
|
204
204
|
* **Fail-Closed Security & Secret Redaction:** Evaluates explicit Deny rules before Allow rules against canonicalized, case-folded paths. Redacts high-entropy keys and base64-encoded credentials (such as Kubernetes `Secret` manifests).
|
|
205
205
|
* **Complexity & Cost Router:** Zero-dependency heuristic classifier (`src/router.mjs`) routing mechanical tasks to lightweight models while reserving primary models for complex refactors, with a `node --check` syntax-verification gate that transparently escalates a FAST-tier result to the primary provider if it left broken JS on disk.
|
|
206
206
|
* **Terminal UI & Diagnostic Matrix (`agentctl doctor`):** Interactive terminal dashboard, task sidecar manager, and automated transactional self-repair.
|
|
207
|
-
* **Verified Test Suite:** Tested with **
|
|
207
|
+
* **Verified Test Suite:** Tested with **940 unit tests across 130 suites passing in < 15.0s**.
|
|
208
208
|
|
|
209
209
|
<br/>
|
|
210
210
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "jules-orchestrator-kit",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.59.0",
|
|
4
4
|
"description": "Zero-dependency safety gatekeeper, test oracle generator, and multi-agent coordination protocol for autonomous coding agents — Google Jules, Claude Code, Codex and Gemini CLI.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
package/src/coverage.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readFileSync, readdirSync, mkdtempSync, rmSync, existsSync, realpathSync } from "node:fs";
|
|
2
|
+
import { isTestPath } from "./test-paths.mjs";
|
|
2
3
|
import { join, resolve, relative, isAbsolute } from "node:path";
|
|
3
4
|
import { tmpdir } from "node:os";
|
|
4
5
|
import { fileURLToPath } from "node:url";
|
|
@@ -18,12 +19,7 @@ export function isExcludedFromCoverage(filePath = "") {
|
|
|
18
19
|
norm.startsWith(".agent/") ||
|
|
19
20
|
norm.startsWith(".github/") ||
|
|
20
21
|
norm.startsWith(".git/") ||
|
|
21
|
-
norm
|
|
22
|
-
norm.startsWith("tests/") ||
|
|
23
|
-
norm.includes("/__tests__/") ||
|
|
24
|
-
norm.includes(".test.") ||
|
|
25
|
-
norm.includes(".spec.") ||
|
|
26
|
-
norm.includes("_test.") ||
|
|
22
|
+
isTestPath(norm) ||
|
|
27
23
|
norm.endsWith(".md") ||
|
|
28
24
|
norm.endsWith(".json") ||
|
|
29
25
|
norm.endsWith(".yml") ||
|
package/src/engine.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { loadConfig, parseYaml, normalizeScope } from "./config.mjs";
|
|
2
|
+
import { isTestPath } from "./test-paths.mjs";
|
|
2
3
|
import { checkScope, scanDiff, scanBinaryPayloads, redactSecrets } from "./security.mjs";
|
|
3
4
|
import { changedFiles, diffBytes, diffText, binaryDiffEntries, symlinkChanges, showFromOrigin, runCmd } from "./git.mjs";
|
|
4
5
|
import { createProvider, ProviderRateLimitError, ProviderUnavailableError } from "./provider.mjs";
|
|
@@ -494,10 +495,7 @@ export async function gate(opts = {}) {
|
|
|
494
495
|
const postTestHash = postTestHashResult.treeHash;
|
|
495
496
|
let testTampered = false;
|
|
496
497
|
if (config.evidence?.strictTestLock && !opts.allowTestModifications && preTestHashResult.fileCount > 0) {
|
|
497
|
-
const changedTestFile = files.find((f) =>
|
|
498
|
-
const lower = f.toLowerCase();
|
|
499
|
-
return lower.startsWith("test/") || lower.startsWith("tests/") || lower.includes(".test.") || lower.includes(".spec.");
|
|
500
|
-
});
|
|
498
|
+
const changedTestFile = files.find((f) => isTestPath(f));
|
|
501
499
|
if (changedTestFile && preTestHash !== postTestHash) {
|
|
502
500
|
testTampered = true;
|
|
503
501
|
}
|
package/src/evidence.mjs
CHANGED
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
import { join, resolve, relative, isAbsolute, sep, extname } from "node:path";
|
|
15
15
|
import { createHash, randomUUID } from "node:crypto";
|
|
16
16
|
import { execSync } from "node:child_process";
|
|
17
|
+
import { isTestPath } from "./test-paths.mjs";
|
|
17
18
|
|
|
18
19
|
/**
|
|
19
20
|
* Normalizes file path to POSIX slashes.
|
|
@@ -164,20 +165,7 @@ export function computeDirectoryHash(root, options = {}) {
|
|
|
164
165
|
|
|
165
166
|
// Filter test files if specifically looking for test suites
|
|
166
167
|
if (options.testOnly) {
|
|
167
|
-
fileList = fileList.filter((f) =>
|
|
168
|
-
const lower = f.toLowerCase();
|
|
169
|
-
return (
|
|
170
|
-
lower.startsWith("test/") ||
|
|
171
|
-
lower.startsWith("tests/") ||
|
|
172
|
-
lower.startsWith("__tests__/") ||
|
|
173
|
-
lower.startsWith("spec/") ||
|
|
174
|
-
lower.includes(".test.") ||
|
|
175
|
-
lower.includes(".spec.") ||
|
|
176
|
-
lower.includes("_test.") ||
|
|
177
|
-
lower.endsWith("test.sol") ||
|
|
178
|
-
lower.endsWith(".t.sol")
|
|
179
|
-
);
|
|
180
|
-
});
|
|
168
|
+
fileList = fileList.filter((f) => isTestPath(f));
|
|
181
169
|
}
|
|
182
170
|
|
|
183
171
|
const fileHashes = {};
|
package/src/mutation.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readFileSync, writeFileSync, existsSync } from "node:fs";
|
|
2
|
+
import { isTestPath } from "./test-paths.mjs";
|
|
2
3
|
import { resolve, isAbsolute } from "node:path";
|
|
3
4
|
import { createHash } from "node:crypto";
|
|
4
5
|
import { diffText, runCmd } from "./git.mjs";
|
|
@@ -195,14 +196,7 @@ export function isExcludedFromMutation(filePath = "") {
|
|
|
195
196
|
const normalized = filePath.replace(/\\/g, "/").toLowerCase();
|
|
196
197
|
|
|
197
198
|
// Exclude test files
|
|
198
|
-
if (
|
|
199
|
-
normalized.includes(".test.") ||
|
|
200
|
-
normalized.includes(".spec.") ||
|
|
201
|
-
normalized.includes("_test.") ||
|
|
202
|
-
normalized.includes("/test/") ||
|
|
203
|
-
normalized.includes("/tests/") ||
|
|
204
|
-
normalized.includes("/__tests__/")
|
|
205
|
-
) {
|
|
199
|
+
if (isTestPath(normalized)) {
|
|
206
200
|
return true;
|
|
207
201
|
}
|
|
208
202
|
|
package/src/security.mjs
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { openSync, readFileSync, writeSync, fsyncSync, closeSync, renameSync, realpathSync, existsSync, lstatSync, unlinkSync } from "node:fs";
|
|
2
|
+
import { isTestPath } from "./test-paths.mjs";
|
|
2
3
|
import { dirname, join, basename } from "node:path";
|
|
3
4
|
import { randomBytes } from "node:crypto";
|
|
4
5
|
import { canonicalizePath, isWindowsAbsolutePath } from "./config.mjs";
|
|
@@ -1101,6 +1102,675 @@ function locateFindingLine(lines, type, file = null) {
|
|
|
1101
1102
|
return null;
|
|
1102
1103
|
}
|
|
1103
1104
|
|
|
1105
|
+
// ---------------------------------------------------------------------------
|
|
1106
|
+
// Statement-level expectation-rewrite detection (multi-line aware)
|
|
1107
|
+
// ---------------------------------------------------------------------------
|
|
1108
|
+
//
|
|
1109
|
+
// The original pairing ran on physical lines. That caught
|
|
1110
|
+
// `assert.equal(add(1, 2), 3);` becoming `assert.equal(add(1, 2), -1);`, but
|
|
1111
|
+
// the same edit walked straight through the moment it was wrapped across
|
|
1112
|
+
// lines — which every formatter does the day a line runs long, and which an
|
|
1113
|
+
// agent doing an ordinary reformat does on its own:
|
|
1114
|
+
//
|
|
1115
|
+
// -assert.equal(
|
|
1116
|
+
// - add(1, 2),
|
|
1117
|
+
// - 3
|
|
1118
|
+
// -);
|
|
1119
|
+
// +assert.equal(
|
|
1120
|
+
// + add(1, 2),
|
|
1121
|
+
// + -1
|
|
1122
|
+
// +);
|
|
1123
|
+
//
|
|
1124
|
+
// The value lives on a line that carries no assertion keyword, so neither
|
|
1125
|
+
// side ever paired, and the suite went from checking that addition works to
|
|
1126
|
+
// certifying that it is broken. The statement, not the line, is the unit an
|
|
1127
|
+
// agent rewrites, so the pairing now runs on reassembled statements: a run
|
|
1128
|
+
// of physical lines joined while its delimiters are unbalanced, one of its
|
|
1129
|
+
// strings or comments is still open, a Python line-continuation is pending,
|
|
1130
|
+
// or the next line cannot start a statement of its own. The hunk's context
|
|
1131
|
+
// lines belong to both images and are what make the reassembly possible;
|
|
1132
|
+
// when they are absent (a zero-context diff) the only pair that can survive
|
|
1133
|
+
// is the one where each image is a single fragment, and that pair is taken
|
|
1134
|
+
// too, requiring a literal placeholder so a code change cannot masquerade as
|
|
1135
|
+
// a value change.
|
|
1136
|
+
//
|
|
1137
|
+
// The pairing rule is unchanged in spirit: the two sides must be the *same*
|
|
1138
|
+
// assertion — identical once every literal is blanked out — with different
|
|
1139
|
+
// values. That does not distinguish an attack from a deliberate change of
|
|
1140
|
+
// spec; nothing can, from a diff alone. This reports rather than decides,
|
|
1141
|
+
// and `--allow-test-modifications` is the answer when the new expectation is
|
|
1142
|
+
// the correct one.
|
|
1143
|
+
|
|
1144
|
+
// An assertion that states a *specific* expected value. Counting assertions
|
|
1145
|
+
// alone let a test be gutted while looking untouched: swapping
|
|
1146
|
+
// `assert.strictEqual(add(2,3), 5)` for `assert.ok(add(2,3) !== undefined)`
|
|
1147
|
+
// removes one and adds one, so `removed > added` stayed false and the guard
|
|
1148
|
+
// said nothing — while the suite stopped checking the answer.
|
|
1149
|
+
//
|
|
1150
|
+
// The `expect` argument span is a bounded lazy match rather than `[^)]*` so
|
|
1151
|
+
// that a call split across lines with a nested call in its arguments
|
|
1152
|
+
// (`expect(\n formatInvoice(bill)\n).toBe(…`) still recognises the chain.
|
|
1153
|
+
// The bound is a guess: an argument list longer than 240 characters is
|
|
1154
|
+
// rarer than a missed chain.
|
|
1155
|
+
const SPECIFIC_ASSERTION = new RegExp(
|
|
1156
|
+
[
|
|
1157
|
+
"\\bassert(?:\\.strict)?\\.?(?:strictEqual|deepStrictEqual|deepEqual|notStrictEqual|notDeepStrictEqual|equal|notEqual|match|doesNotMatch|throws|rejects|doesNotThrow)\\s*\\(",
|
|
1158
|
+
"\\bexpect\\s*\\([\\s\\S]{0,240}?\\)\\s*\\.(?:toBe|toEqual|toStrictEqual|toMatch|toMatchObject|toContain|toHaveBeenCalledWith|toThrow|toHaveLength|toBeCloseTo)\\s*\\(",
|
|
1159
|
+
"\\bassert\\.(?:equals|deepEquals|include|lengthOf)\\s*\\(",
|
|
1160
|
+
"assert_eq!|assert_ne!",
|
|
1161
|
+
"\\bt\\.(?:Errorf|Fatalf)\\s*\\(",
|
|
1162
|
+
"\\brequire\\.(?:Equal|NotEqual|Len|Contains|Error|NoError)\\s*\\(",
|
|
1163
|
+
].join("|"),
|
|
1164
|
+
"i"
|
|
1165
|
+
);
|
|
1166
|
+
const isSpecificAssertion = (str) => SPECIFIC_ASSERTION.test(str);
|
|
1167
|
+
|
|
1168
|
+
/**
|
|
1169
|
+
* An assertion with every literal value replaced by a placeholder.
|
|
1170
|
+
*
|
|
1171
|
+
* Two lines that normalize to the same string are the same assertion about
|
|
1172
|
+
* the same expression; whatever differs between them is a value.
|
|
1173
|
+
*
|
|
1174
|
+
* The number form covers hex, octal, binary, underscores and exponents: the
|
|
1175
|
+
* original decimal-only regex never blanked `0xFF`, so an expectation
|
|
1176
|
+
* rewritten from `0xFF` to `0xFE` normalized to two *different* shapes and
|
|
1177
|
+
* the pair was never formed.
|
|
1178
|
+
*/
|
|
1179
|
+
const blankLiterals = (str) =>
|
|
1180
|
+
str
|
|
1181
|
+
.replace(/(['"`])(?:\\.|(?!\1)[^\\])*\1/g, "\u0000S")
|
|
1182
|
+
// The sign belongs to the literal: without it `3` and `-1` normalized to
|
|
1183
|
+
// different shapes and the rewritten expectation was never paired.
|
|
1184
|
+
.replace(
|
|
1185
|
+
/(?<![\w$])(?:0[xX][0-9a-fA-F_]+|0[bB][01_]+|0[oO][0-7_]+|-?\d[\d_]*(?:\.[\d_]+)?(?:[eE][+-]?\d+)?)/g,
|
|
1186
|
+
"\u0000N"
|
|
1187
|
+
)
|
|
1188
|
+
.replace(/\b(?:true|false|null|undefined|None|True|False|nil)\b/g, "\u0000B")
|
|
1189
|
+
// Whitespace is dropped, not collapsed: the shape is compared for
|
|
1190
|
+
// equality only, and a reformatted statement must normalize to the same
|
|
1191
|
+
// shape as the original — ` <N> );` and ` <N>);` are the same assertion.
|
|
1192
|
+
.replace(/\s+/g, "");
|
|
1193
|
+
|
|
1194
|
+
// The test languages the gate runs over. The scanner below is written for
|
|
1195
|
+
// these four and nothing else; an unrecognised extension falls back to `js`,
|
|
1196
|
+
// which is the strictest of the four for line joining.
|
|
1197
|
+
const TEST_LANG_BY_EXT = new Map([
|
|
1198
|
+
[".js", "js"], [".mjs", "js"], [".cjs", "js"], [".jsx", "js"],
|
|
1199
|
+
[".ts", "js"], [".mts", "js"], [".cts", "js"], [".tsx", "js"],
|
|
1200
|
+
[".py", "python"], [".pyi", "python"],
|
|
1201
|
+
[".go", "go"],
|
|
1202
|
+
[".rs", "rust"],
|
|
1203
|
+
]);
|
|
1204
|
+
|
|
1205
|
+
function langForTestFile(file) {
|
|
1206
|
+
const n = String(file || "").toLowerCase();
|
|
1207
|
+
const dot = n.lastIndexOf(".");
|
|
1208
|
+
if (dot === -1) return "js";
|
|
1209
|
+
return TEST_LANG_BY_EXT.get(n.slice(dot)) || "js";
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
function freshScanState() {
|
|
1213
|
+
// str: the open string, or null.
|
|
1214
|
+
// q the quote character
|
|
1215
|
+
// tri Python triple-quoted
|
|
1216
|
+
// raw raw string, no escapes (Go backtick)
|
|
1217
|
+
// rawHashes Rust raw string r#"…"#: terminator is " plus that many #
|
|
1218
|
+
// block: depth of an open /* … */ (nested only in Rust)
|
|
1219
|
+
// accDelta: counted delimiters still open in the current statement
|
|
1220
|
+
// specialStack: counted depth recorded at each non-joining call (see below)
|
|
1221
|
+
return { str: null, block: 0, accDelta: 0, specialStack: [] };
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
// A call whose opening paren must not join lines: the test name is not the
|
|
1225
|
+
// expectation. Without this, `it("old name", () => { expect(f()).toBe(3); })`
|
|
1226
|
+
// would pair against its renamed copy, because a name is a string and so
|
|
1227
|
+
// blanks to the same placeholder as any value change would. The closer of a
|
|
1228
|
+
// non-joining paren is recognised by the depth it was opened at, so the
|
|
1229
|
+
// running balance stays exact.
|
|
1230
|
+
const NON_JOINING_CALL = /\b(?:it|test|describe|context)\s*\($|\bt\.Run\s*\($/;
|
|
1231
|
+
|
|
1232
|
+
/**
|
|
1233
|
+
* Scan one physical line of source.
|
|
1234
|
+
*
|
|
1235
|
+
* Returns { delta, trailingBackslash }. `delta` is the net number of
|
|
1236
|
+
* still-open ( [ { delimiters outside strings and comments; a statement
|
|
1237
|
+
* continues to the next physical line while it is positive, while a string
|
|
1238
|
+
* or block comment is open (tracked on `state`), or on a Python line
|
|
1239
|
+
* continuation.
|
|
1240
|
+
*
|
|
1241
|
+
* This is not a parser, and the approximations are deliberate: JS regex
|
|
1242
|
+
* literals are detected with a one-token look-behind (a `/` that cannot
|
|
1243
|
+
* follow an identifier, number, `)` or `]` starts one), template
|
|
1244
|
+
* interpolation is treated as opaque string content, and Rust lifetimes are
|
|
1245
|
+
* told apart from char literals by shape alone. `stripComments` below
|
|
1246
|
+
* walks the same constructs, so the two must stay in lock-step.
|
|
1247
|
+
*/
|
|
1248
|
+
function scanSourceLine(text, lang, state) {
|
|
1249
|
+
let delta = 0;
|
|
1250
|
+
let lastSig = "\n";
|
|
1251
|
+
const n = text.length;
|
|
1252
|
+
let i = 0;
|
|
1253
|
+
|
|
1254
|
+
while (i < n) {
|
|
1255
|
+
const c = text[i];
|
|
1256
|
+
const c2 = i + 1 < n ? text[i + 1] : "";
|
|
1257
|
+
|
|
1258
|
+
if (state.str) {
|
|
1259
|
+
const s = state.str;
|
|
1260
|
+
let closed = false;
|
|
1261
|
+
if (s.rawHashes !== undefined) {
|
|
1262
|
+
if (c === '"') {
|
|
1263
|
+
let j = i + 1;
|
|
1264
|
+
let h = 0;
|
|
1265
|
+
while (j < n && text[j] === "#") { h++; j++; }
|
|
1266
|
+
if (h >= s.rawHashes) { i = j; closed = true; }
|
|
1267
|
+
}
|
|
1268
|
+
} else if (s.raw) {
|
|
1269
|
+
closed = c === s.q;
|
|
1270
|
+
} else if (c === "\\") {
|
|
1271
|
+
i += s.tri ? 1 : 2;
|
|
1272
|
+
continue;
|
|
1273
|
+
} else if (c === s.q) {
|
|
1274
|
+
if (s.tri) {
|
|
1275
|
+
if (text[i + 1] === s.q && text[i + 2] === s.q) { i += 3; closed = true; }
|
|
1276
|
+
else { i += 1; }
|
|
1277
|
+
} else {
|
|
1278
|
+
i += 1;
|
|
1279
|
+
closed = true;
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
if (closed) { state.str = null; lastSig = s.q; continue; }
|
|
1283
|
+
i += 1;
|
|
1284
|
+
continue;
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1287
|
+
if (state.block > 0) {
|
|
1288
|
+
if (c === "/" && c2 === "*") {
|
|
1289
|
+
if (lang === "rust") state.block += 1;
|
|
1290
|
+
i += 2;
|
|
1291
|
+
continue;
|
|
1292
|
+
}
|
|
1293
|
+
if (c === "*" && c2 === "/") {
|
|
1294
|
+
state.block -= 1;
|
|
1295
|
+
i += 2;
|
|
1296
|
+
lastSig = "/";
|
|
1297
|
+
continue;
|
|
1298
|
+
}
|
|
1299
|
+
i += 1;
|
|
1300
|
+
continue;
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
// A line comment ends the line.
|
|
1304
|
+
if (c === "/" && c2 === "/") break;
|
|
1305
|
+
if (lang === "python" && c === "#") break;
|
|
1306
|
+
|
|
1307
|
+
if (c === "/" && c2 === "*") {
|
|
1308
|
+
state.block = 1;
|
|
1309
|
+
i += 2;
|
|
1310
|
+
continue;
|
|
1311
|
+
}
|
|
1312
|
+
|
|
1313
|
+
// JS regex literal, best effort. Delimiters inside are not counted.
|
|
1314
|
+
if ((lang === "js" || lang === "ts") && c === "/" && !/[\w$)\]}]/.test(lastSig)) {
|
|
1315
|
+
i += 1;
|
|
1316
|
+
let inClass = false;
|
|
1317
|
+
while (i < n) {
|
|
1318
|
+
const rc = text[i];
|
|
1319
|
+
if (rc === "\\") { i += 2; continue; }
|
|
1320
|
+
if (rc === "[") inClass = true;
|
|
1321
|
+
else if (rc === "]") inClass = false;
|
|
1322
|
+
else if (rc === "/" && !inClass) { i += 1; break; }
|
|
1323
|
+
i += 1;
|
|
1324
|
+
}
|
|
1325
|
+
while (i < n && /[a-z]/i.test(text[i])) i += 1; // flags
|
|
1326
|
+
lastSig = "/";
|
|
1327
|
+
continue;
|
|
1328
|
+
}
|
|
1329
|
+
|
|
1330
|
+
if (c === '"' || c === "'" || (lang === "go" && c === "`")) {
|
|
1331
|
+
if (lang === "python" && c2 === c && text[i + 2] === c) {
|
|
1332
|
+
state.str = { q: c, tri: true };
|
|
1333
|
+
i += 3;
|
|
1334
|
+
} else if (lang === "rust" && c === '"') {
|
|
1335
|
+
if (i > 0 && text[i - 1] === "r") {
|
|
1336
|
+
let j = i - 1;
|
|
1337
|
+
let h = 0;
|
|
1338
|
+
while (j >= 1 && text[j - 1] === "#") { h++; j--; }
|
|
1339
|
+
state.str = { q: '"', rawHashes: h };
|
|
1340
|
+
i += 1;
|
|
1341
|
+
} else {
|
|
1342
|
+
state.str = { q: c };
|
|
1343
|
+
i += 1;
|
|
1344
|
+
}
|
|
1345
|
+
} else if (lang === "rust" && c === "'") {
|
|
1346
|
+
// A char literal is 'X' or '\X' within four characters; anything
|
|
1347
|
+
// else starting with a quote is a lifetime and only the quote is
|
|
1348
|
+
// skipped, or the next line would see a string that never closed.
|
|
1349
|
+
if (c2 === "\\") {
|
|
1350
|
+
const end = text.indexOf("'", i + 2);
|
|
1351
|
+
if (end !== -1 && end - i <= 4) { i = end + 1; lastSig = "'"; continue; }
|
|
1352
|
+
} else if (text[i + 2] === "'" && c2 !== "'") {
|
|
1353
|
+
i += 3;
|
|
1354
|
+
lastSig = "'";
|
|
1355
|
+
continue;
|
|
1356
|
+
}
|
|
1357
|
+
i += 1;
|
|
1358
|
+
continue;
|
|
1359
|
+
} else {
|
|
1360
|
+
state.str = { q: c };
|
|
1361
|
+
i += 1;
|
|
1362
|
+
}
|
|
1363
|
+
lastSig = c;
|
|
1364
|
+
continue;
|
|
1365
|
+
}
|
|
1366
|
+
|
|
1367
|
+
// A backslash on the very last character is a Python line continuation.
|
|
1368
|
+
if (c === "\\" && i + 1 === n) {
|
|
1369
|
+
return { delta, trailingBackslash: true };
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1372
|
+
if (c === "(") {
|
|
1373
|
+
// `before` ends with the paren itself; NON_JOINING_CALL matches on it.
|
|
1374
|
+
const before = text.slice(0, i + 1).replace(/\s+$/, "");
|
|
1375
|
+
if (NON_JOINING_CALL.test(before)) state.specialStack.push(state.accDelta);
|
|
1376
|
+
else { delta += 1; state.accDelta += 1; }
|
|
1377
|
+
lastSig = c;
|
|
1378
|
+
i += 1;
|
|
1379
|
+
continue;
|
|
1380
|
+
}
|
|
1381
|
+
if (c === "[") { delta += 1; state.accDelta += 1; lastSig = c; i += 1; continue; }
|
|
1382
|
+
if (c === "{") {
|
|
1383
|
+
// Go joins braces: the `if got != want { t.Errorf(…) }` block is the
|
|
1384
|
+
// idiomatic Go assertion, and the value lives on its first line. The
|
|
1385
|
+
// other three languages get no brace joining, so a rename of a test
|
|
1386
|
+
// inside a block cannot pair as a value change on its own.
|
|
1387
|
+
if (lang === "go") { delta += 1; state.accDelta += 1; }
|
|
1388
|
+
lastSig = c;
|
|
1389
|
+
i += 1;
|
|
1390
|
+
continue;
|
|
1391
|
+
}
|
|
1392
|
+
if (c === ")") {
|
|
1393
|
+
const top = state.specialStack[state.specialStack.length - 1];
|
|
1394
|
+
if (top === state.accDelta) state.specialStack.pop();
|
|
1395
|
+
else { delta -= 1; state.accDelta -= 1; }
|
|
1396
|
+
lastSig = c;
|
|
1397
|
+
i += 1;
|
|
1398
|
+
continue;
|
|
1399
|
+
}
|
|
1400
|
+
if (c === "]") { delta -= 1; state.accDelta -= 1; lastSig = c; i += 1; continue; }
|
|
1401
|
+
if (c === "}") {
|
|
1402
|
+
if (lang === "go") { delta -= 1; state.accDelta -= 1; }
|
|
1403
|
+
lastSig = c;
|
|
1404
|
+
i += 1;
|
|
1405
|
+
continue;
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1408
|
+
if (!/\s/.test(c)) lastSig = c;
|
|
1409
|
+
i += 1;
|
|
1410
|
+
}
|
|
1411
|
+
|
|
1412
|
+
return { delta, trailingBackslash: false };
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1415
|
+
/**
|
|
1416
|
+
* The same source with every comment blanked out, strings and line
|
|
1417
|
+
* structure untouched. Shapes and keywords are computed on this text so a
|
|
1418
|
+
* commented-out `expect(…)` cannot make a block an assertion, and a number
|
|
1419
|
+
* changed inside a comment cannot pair as a value change.
|
|
1420
|
+
*/
|
|
1421
|
+
function stripComments(text, lang) {
|
|
1422
|
+
const state = freshScanState();
|
|
1423
|
+
return text.split("\n").map((line) => {
|
|
1424
|
+
let out = "";
|
|
1425
|
+
let pending = 0;
|
|
1426
|
+
const copyCode = (to) => {
|
|
1427
|
+
out += line.slice(pending, to);
|
|
1428
|
+
pending = to;
|
|
1429
|
+
};
|
|
1430
|
+
let i = 0;
|
|
1431
|
+
const n = line.length;
|
|
1432
|
+
|
|
1433
|
+
while (i < n) {
|
|
1434
|
+
const c = line[i];
|
|
1435
|
+
const c2 = i + 1 < n ? line[i + 1] : "";
|
|
1436
|
+
|
|
1437
|
+
if (state.block > 0) {
|
|
1438
|
+
if (c === "/" && c2 === "*") {
|
|
1439
|
+
if (lang === "rust") state.block += 1;
|
|
1440
|
+
i += 2;
|
|
1441
|
+
continue;
|
|
1442
|
+
}
|
|
1443
|
+
if (c === "*" && c2 === "/") {
|
|
1444
|
+
state.block -= 1;
|
|
1445
|
+
i += 2;
|
|
1446
|
+
if (state.block === 0) pending = i;
|
|
1447
|
+
continue;
|
|
1448
|
+
}
|
|
1449
|
+
i += 1;
|
|
1450
|
+
continue;
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
if (state.str) {
|
|
1454
|
+
const s = state.str;
|
|
1455
|
+
let closed = false;
|
|
1456
|
+
if (s.rawHashes !== undefined) {
|
|
1457
|
+
if (c === '"') {
|
|
1458
|
+
let j = i + 1;
|
|
1459
|
+
let h = 0;
|
|
1460
|
+
while (j < n && line[j] === "#") { h++; j++; }
|
|
1461
|
+
if (h >= s.rawHashes) { i = j; closed = true; }
|
|
1462
|
+
}
|
|
1463
|
+
} else if (s.raw) {
|
|
1464
|
+
closed = c === s.q;
|
|
1465
|
+
} else if (c === "\\") {
|
|
1466
|
+
i += s.tri ? 1 : 2;
|
|
1467
|
+
continue;
|
|
1468
|
+
} else if (c === s.q) {
|
|
1469
|
+
if (s.tri) {
|
|
1470
|
+
if (line[i + 1] === s.q && line[i + 2] === s.q) { i += 3; closed = true; }
|
|
1471
|
+
else { i += 1; }
|
|
1472
|
+
} else {
|
|
1473
|
+
i += 1;
|
|
1474
|
+
closed = true;
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
if (closed) {
|
|
1478
|
+
state.str = null;
|
|
1479
|
+
copyCode(i);
|
|
1480
|
+
continue;
|
|
1481
|
+
}
|
|
1482
|
+
i += 1;
|
|
1483
|
+
continue;
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
if (c === "/" && c2 === "/") { copyCode(i); break; }
|
|
1487
|
+
if (lang === "python" && c === "#") { copyCode(i); break; }
|
|
1488
|
+
if (c === "/" && c2 === "*") {
|
|
1489
|
+
copyCode(i);
|
|
1490
|
+
state.block = 1;
|
|
1491
|
+
i += 2;
|
|
1492
|
+
continue;
|
|
1493
|
+
}
|
|
1494
|
+
if (c === '"' || c === "'" || (lang === "go" && c === "`")) {
|
|
1495
|
+
copyCode(i);
|
|
1496
|
+
if (lang === "python" && c2 === c && line[i + 2] === c) {
|
|
1497
|
+
state.str = { q: c, tri: true };
|
|
1498
|
+
i += 3;
|
|
1499
|
+
} else if (lang === "rust" && c === '"') {
|
|
1500
|
+
if (i > 0 && line[i - 1] === "r") {
|
|
1501
|
+
let j = i - 1;
|
|
1502
|
+
let h = 0;
|
|
1503
|
+
while (j >= 1 && line[j - 1] === "#") { h++; j--; }
|
|
1504
|
+
state.str = { q: '"', rawHashes: h };
|
|
1505
|
+
i += 1;
|
|
1506
|
+
} else {
|
|
1507
|
+
state.str = { q: c };
|
|
1508
|
+
i += 1;
|
|
1509
|
+
}
|
|
1510
|
+
} else if (lang === "rust" && c === "'") {
|
|
1511
|
+
if (c2 === "\\") {
|
|
1512
|
+
const end = line.indexOf("'", i + 2);
|
|
1513
|
+
if (end !== -1 && end - i <= 4) { i = end + 1; copyCode(i); continue; }
|
|
1514
|
+
} else if (line[i + 2] === "'" && c2 !== "'") {
|
|
1515
|
+
i += 3;
|
|
1516
|
+
copyCode(i);
|
|
1517
|
+
continue;
|
|
1518
|
+
}
|
|
1519
|
+
i += 1;
|
|
1520
|
+
copyCode(i);
|
|
1521
|
+
continue;
|
|
1522
|
+
} else {
|
|
1523
|
+
state.str = { q: c };
|
|
1524
|
+
i += 1;
|
|
1525
|
+
}
|
|
1526
|
+
continue;
|
|
1527
|
+
}
|
|
1528
|
+
i += 1;
|
|
1529
|
+
}
|
|
1530
|
+
copyCode(n);
|
|
1531
|
+
return out;
|
|
1532
|
+
}).join("\n");
|
|
1533
|
+
}
|
|
1534
|
+
|
|
1535
|
+
// A line that cannot start a statement of its own continues the previous
|
|
1536
|
+
// statement: a closing delimiter, a member-access, or an operator.
|
|
1537
|
+
const CONTINUATION_START = /^[)\],.]/;
|
|
1538
|
+
const CONTINUATION_OP_START = /^[+\-*/%<>=&|^:]/;
|
|
1539
|
+
|
|
1540
|
+
// A scanner miscount (an unbalanced delimiter inside a regex literal is the
|
|
1541
|
+
// usual cause) must not be able to merge a whole file into one statement,
|
|
1542
|
+
// which would pair *any* literal change anywhere in the file.
|
|
1543
|
+
const MAX_STATEMENT_LINES = 100;
|
|
1544
|
+
const MAX_STATEMENT_CHARS = 12000;
|
|
1545
|
+
|
|
1546
|
+
/**
|
|
1547
|
+
* Reassemble physical lines into statements.
|
|
1548
|
+
*
|
|
1549
|
+
* `sliceLines` is one image of a hunk in file order: context lines plus the
|
|
1550
|
+
* removed (or added) lines. Context lines are ordinary file text; a
|
|
1551
|
+
* statement spans them freely, which is exactly what makes a value edit
|
|
1552
|
+
* inside a formatter-wrapped assertion visible to the pairing.
|
|
1553
|
+
*
|
|
1554
|
+
* @param {Array<{ kind: string, text: string, oldNo: number|null, newNo: number|null }>} sliceLines
|
|
1555
|
+
* @param {string} lang
|
|
1556
|
+
* @returns {Array<{ text: string, firstOld: number|null, lastOld: number|null, firstNew: number|null, lastNew: number|null, removedLines: Array, addedLines: Array }>}
|
|
1557
|
+
*/
|
|
1558
|
+
function assembleStatements(sliceLines, lang) {
|
|
1559
|
+
const stmts = [];
|
|
1560
|
+
let cur = null;
|
|
1561
|
+
|
|
1562
|
+
const flush = () => {
|
|
1563
|
+
if (!cur) return;
|
|
1564
|
+
stmts.push({
|
|
1565
|
+
text: cur.lines.join("\n"),
|
|
1566
|
+
firstOld: cur.firstOld,
|
|
1567
|
+
lastOld: cur.lastOld,
|
|
1568
|
+
firstNew: cur.firstNew,
|
|
1569
|
+
lastNew: cur.lastNew,
|
|
1570
|
+
removedLines: cur.removedLines,
|
|
1571
|
+
addedLines: cur.addedLines,
|
|
1572
|
+
});
|
|
1573
|
+
cur = null;
|
|
1574
|
+
};
|
|
1575
|
+
|
|
1576
|
+
for (const L of sliceLines) {
|
|
1577
|
+
const trimmed = L.text.replace(/^\s+/, "");
|
|
1578
|
+
const joins =
|
|
1579
|
+
cur !== null &&
|
|
1580
|
+
(cur.delta > 0 ||
|
|
1581
|
+
cur.state.str !== null ||
|
|
1582
|
+
cur.state.block > 0 ||
|
|
1583
|
+
cur.trailingBackslash ||
|
|
1584
|
+
CONTINUATION_START.test(trimmed) ||
|
|
1585
|
+
CONTINUATION_OP_START.test(trimmed));
|
|
1586
|
+
|
|
1587
|
+
if (
|
|
1588
|
+
joins &&
|
|
1589
|
+
cur.lines.length < MAX_STATEMENT_LINES &&
|
|
1590
|
+
cur.chars + L.text.length + 1 <= MAX_STATEMENT_CHARS
|
|
1591
|
+
) {
|
|
1592
|
+
cur.lines.push(L.text);
|
|
1593
|
+
cur.chars += L.text.length + 1;
|
|
1594
|
+
const sc = scanSourceLine(L.text, lang, cur.state);
|
|
1595
|
+
cur.delta += sc.delta;
|
|
1596
|
+
cur.trailingBackslash = sc.trailingBackslash;
|
|
1597
|
+
cur.lastOld = L.oldNo;
|
|
1598
|
+
cur.lastNew = L.newNo;
|
|
1599
|
+
if (L.kind === "-") cur.removedLines.push(L);
|
|
1600
|
+
else if (L.kind === "+") cur.addedLines.push(L);
|
|
1601
|
+
} else {
|
|
1602
|
+
flush();
|
|
1603
|
+
const st = freshScanState();
|
|
1604
|
+
const sc = scanSourceLine(L.text, lang, st);
|
|
1605
|
+
cur = {
|
|
1606
|
+
lines: [L.text],
|
|
1607
|
+
chars: L.text.length,
|
|
1608
|
+
firstOld: L.oldNo,
|
|
1609
|
+
lastOld: L.oldNo,
|
|
1610
|
+
firstNew: L.newNo,
|
|
1611
|
+
lastNew: L.newNo,
|
|
1612
|
+
delta: sc.delta,
|
|
1613
|
+
state: st,
|
|
1614
|
+
trailingBackslash: sc.trailingBackslash,
|
|
1615
|
+
removedLines: L.kind === "-" ? [L] : [],
|
|
1616
|
+
addedLines: L.kind === "+" ? [L] : [],
|
|
1617
|
+
};
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
flush();
|
|
1621
|
+
return stmts;
|
|
1622
|
+
}
|
|
1623
|
+
|
|
1624
|
+
const hasLiteralPlaceholder = (shape) =>
|
|
1625
|
+
shape.includes("\u0000S") || shape.includes("\u0000N") || shape.includes("\u0000B");
|
|
1626
|
+
|
|
1627
|
+
const collapseWhitespace = (s) => s.replace(/\s+/g, " ").trim();
|
|
1628
|
+
const shorten = (s) => (s.length > 160 ? `${s.slice(0, 157)}…` : s);
|
|
1629
|
+
|
|
1630
|
+
/**
|
|
1631
|
+
* Pair rewritten expectations across the removed and added images of every
|
|
1632
|
+
* hunk of one file, and report each pair.
|
|
1633
|
+
*
|
|
1634
|
+
* A statement is only a candidate when it actually contains a removed
|
|
1635
|
+
* (resp. added) line: a context-only statement is unchanged text on both
|
|
1636
|
+
* sides, and letting it pair would flag a genuinely new assertion that
|
|
1637
|
+
* merely has the same shape as one that stayed put.
|
|
1638
|
+
*
|
|
1639
|
+
* @param {string} file
|
|
1640
|
+
* @param {Array<{ lines: Array }>} hunks
|
|
1641
|
+
* @param {object} stats - per-file stats; the paired physical lines are
|
|
1642
|
+
* spliced out of the count pools so the count-based checks below do not
|
|
1643
|
+
* report the same edit a second time.
|
|
1644
|
+
* @param {Array} violations
|
|
1645
|
+
* @returns {Array<{ r: object, a: object }>} the pairs, for the caller
|
|
1646
|
+
*/
|
|
1647
|
+
function detectExpectationRewrites(file, hunks, stats, violations) {
|
|
1648
|
+
const lang = langForTestFile(file);
|
|
1649
|
+
const allPairs = [];
|
|
1650
|
+
|
|
1651
|
+
for (const hunk of hunks) {
|
|
1652
|
+
const oldSlice = [];
|
|
1653
|
+
const newSlice = [];
|
|
1654
|
+
for (const L of hunk.lines) {
|
|
1655
|
+
if (L.kind !== "+") oldSlice.push(L);
|
|
1656
|
+
if (L.kind !== "-") newSlice.push(L);
|
|
1657
|
+
}
|
|
1658
|
+
const oldStmts = assembleStatements(oldSlice, lang);
|
|
1659
|
+
const newStmts = assembleStatements(newSlice, lang);
|
|
1660
|
+
|
|
1661
|
+
// Candidate statements in file order, per image.
|
|
1662
|
+
const oldCands = [];
|
|
1663
|
+
for (const s of oldStmts) {
|
|
1664
|
+
if (s.removedLines.length === 0) continue;
|
|
1665
|
+
const clean = stripComments(s.text, lang);
|
|
1666
|
+
if (!isSpecificAssertion(clean)) continue;
|
|
1667
|
+
oldCands.push({ s, shape: blankLiterals(clean), canon: clean.replace(/\s+/g, "") });
|
|
1668
|
+
}
|
|
1669
|
+
const newCands = [];
|
|
1670
|
+
for (const s of newStmts) {
|
|
1671
|
+
if (s.addedLines.length === 0) continue;
|
|
1672
|
+
const clean = stripComments(s.text, lang);
|
|
1673
|
+
if (!isSpecificAssertion(clean)) continue;
|
|
1674
|
+
newCands.push({ s, shape: blankLiterals(clean), canon: clean.replace(/\s+/g, "") });
|
|
1675
|
+
}
|
|
1676
|
+
|
|
1677
|
+
// The t-th removed candidate of a shape pairs with the t-th added
|
|
1678
|
+
// candidate of the same shape. Position alignment is what keeps a
|
|
1679
|
+
// formatter run over a block of same-shape assertions silent: a greedy
|
|
1680
|
+
// "first different text" pairing would match a re-indented
|
|
1681
|
+
// `assert.equal(f(0), 0)` against its neighbour's value and report a
|
|
1682
|
+
// rewrite that did not happen. It also keeps the pairing linear in the
|
|
1683
|
+
// number of statements, which a 4000-line same-shape table would not
|
|
1684
|
+
// survive as a square of comparisons.
|
|
1685
|
+
const oldByShape = new Map();
|
|
1686
|
+
const newByShape = new Map();
|
|
1687
|
+
for (const c of oldCands) {
|
|
1688
|
+
let arr = oldByShape.get(c.shape);
|
|
1689
|
+
if (!arr) arr = oldByShape.set(c.shape, []).get(c.shape);
|
|
1690
|
+
arr.push(c);
|
|
1691
|
+
}
|
|
1692
|
+
for (const c of newCands) {
|
|
1693
|
+
let arr = newByShape.get(c.shape);
|
|
1694
|
+
if (!arr) arr = newByShape.set(c.shape, []).get(c.shape);
|
|
1695
|
+
arr.push(c);
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1698
|
+
const pairs = [];
|
|
1699
|
+
for (const [shape, olds] of oldByShape) {
|
|
1700
|
+
const news = newByShape.get(shape) || [];
|
|
1701
|
+
const k = Math.min(olds.length, news.length);
|
|
1702
|
+
for (let t = 0; t < k; t++) {
|
|
1703
|
+
if (olds[t].canon !== news[t].canon) {
|
|
1704
|
+
pairs.push({ r: olds[t].s, a: news[t].s });
|
|
1705
|
+
}
|
|
1706
|
+
}
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
// Zero-context hunk: each image is a single fragment and the assertion
|
|
1710
|
+
// keyword may sit outside the hunk entirely. The fragment pair is taken
|
|
1711
|
+
// only when both sides normalize to the same shape *and* that shape
|
|
1712
|
+
// holds a literal — a code change cannot fake it.
|
|
1713
|
+
if (pairs.length === 0 && oldStmts.length === 1 && newStmts.length === 1) {
|
|
1714
|
+
const r = oldStmts[0];
|
|
1715
|
+
const a = newStmts[0];
|
|
1716
|
+
if (r.removedLines.length > 0 && a.addedLines.length > 0) {
|
|
1717
|
+
const sr = blankLiterals(stripComments(r.text, lang));
|
|
1718
|
+
const sa = blankLiterals(stripComments(a.text, lang));
|
|
1719
|
+
if (sr === sa && hasLiteralPlaceholder(sr)) {
|
|
1720
|
+
const cr = stripComments(r.text, lang).replace(/\s+/g, "");
|
|
1721
|
+
const ca = stripComments(a.text, lang).replace(/\s+/g, "");
|
|
1722
|
+
if (cr !== ca) pairs.push({ r, a });
|
|
1723
|
+
}
|
|
1724
|
+
}
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1727
|
+
for (const p of pairs) {
|
|
1728
|
+
allPairs.push(p);
|
|
1729
|
+
const addedNo = p.a.addedLines.length > 0 ? p.a.addedLines[0].newNo : (p.r.removedLines[0] ? p.r.removedLines[0].oldNo : null);
|
|
1730
|
+
violations.push({
|
|
1731
|
+
file,
|
|
1732
|
+
line: addedNo,
|
|
1733
|
+
type: "ASSERTION_EXPECTATION_CHANGED",
|
|
1734
|
+
reason:
|
|
1735
|
+
`Test Tamper Guard: Expected value rewritten in ${file}${addedNo ? `:${addedNo}` : ""} — ` +
|
|
1736
|
+
`"${shorten(collapseWhitespace(p.r.text))}" became "${shorten(collapseWhitespace(p.a.text))}". ` +
|
|
1737
|
+
`A deliberately changed spec looks identical to a test bent to match broken ` +
|
|
1738
|
+
`output, and a diff alone cannot tell the two apart, so this is flagged for ` +
|
|
1739
|
+
`review rather than assumed. If the new expectation is correct, re-run with ` +
|
|
1740
|
+
`--allow-test-modifications (which also silences the skip, vacuous, removal ` +
|
|
1741
|
+
`and weakening checks for this diff).`,
|
|
1742
|
+
});
|
|
1743
|
+
|
|
1744
|
+
// Both sides are accounted for here, so they must not also feed the
|
|
1745
|
+
// count-based checks below — the same line reported twice under two
|
|
1746
|
+
// names tells the operator nothing extra. Consumption is symmetric so
|
|
1747
|
+
// a statement that absorbed two old assertion lines but only one new
|
|
1748
|
+
// one still leaves the surplus to the removal check.
|
|
1749
|
+
const removedMatches = p.r.removedLines.filter(
|
|
1750
|
+
(L) => stats.removed.some((e) => e.line === L.oldNo && e.text === L.text)
|
|
1751
|
+
);
|
|
1752
|
+
const addedMatches = p.a.addedLines.filter(
|
|
1753
|
+
(L) => stats.addedTexts.some((e) => e.line === L.newNo && e.text === L.text)
|
|
1754
|
+
);
|
|
1755
|
+
const take = Math.min(removedMatches.length, addedMatches.length);
|
|
1756
|
+
for (let k = 0; k < take; k++) {
|
|
1757
|
+
const L = removedMatches[k];
|
|
1758
|
+
const idx = stats.removed.findIndex((e) => e.line === L.oldNo && e.text === L.text);
|
|
1759
|
+
if (idx === -1) continue;
|
|
1760
|
+
const entry = stats.removed.splice(idx, 1)[0];
|
|
1761
|
+
const rsIdx = stats.removedSpecific.indexOf(entry);
|
|
1762
|
+
if (rsIdx !== -1) {
|
|
1763
|
+
stats.removedSpecific.splice(rsIdx, 1);
|
|
1764
|
+
stats.addedSpecific--;
|
|
1765
|
+
}
|
|
1766
|
+
stats.added--;
|
|
1767
|
+
}
|
|
1768
|
+
}
|
|
1769
|
+
}
|
|
1770
|
+
|
|
1771
|
+
return allPairs;
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1104
1774
|
/**
|
|
1105
1775
|
* Detects test file assertion tampering, weakening, or test skips.
|
|
1106
1776
|
*
|
|
@@ -1120,18 +1790,7 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
1120
1790
|
let currentOldLineNo = null;
|
|
1121
1791
|
let currentNewLineNo = null;
|
|
1122
1792
|
|
|
1123
|
-
const isTestFile =
|
|
1124
|
-
if (!f) return false;
|
|
1125
|
-
const n = f.replace(/\\/g, "/").toLowerCase();
|
|
1126
|
-
return (
|
|
1127
|
-
n.includes(".test.") ||
|
|
1128
|
-
n.includes(".spec.") ||
|
|
1129
|
-
n.includes("_test.") ||
|
|
1130
|
-
n.includes("/test/") ||
|
|
1131
|
-
n.includes("/tests/") ||
|
|
1132
|
-
n.includes("/__tests__/")
|
|
1133
|
-
);
|
|
1134
|
-
};
|
|
1793
|
+
const isTestFile = isTestPath;
|
|
1135
1794
|
|
|
1136
1795
|
const SKIP_INJECTIONS = [
|
|
1137
1796
|
{ pattern: /\b(?:it|test|describe|context)\.skip\s*\(/i, desc: "Injected test skip (.skip())" },
|
|
@@ -1159,41 +1818,8 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
1159
1818
|
const ASSERTION_PATTERN = /(?:\b(?:assert(?:\.[a-zA-Z0-9_$]+)?|expect|t\.(?:assert|expect|is|equal|true|false|Errorf|Fatalf)|require\.[a-zA-Z0-9_$]+)\b|assert!|assert_eq!|assert_ne!)/i;
|
|
1160
1819
|
const isCommentLine = (str) => /^\s*(?:\/\/|\/\*|\*|#|--|;)/.test(str);
|
|
1161
1820
|
|
|
1162
|
-
// An assertion that states a *specific* expected value. Counting assertions
|
|
1163
|
-
// alone let a test be gutted while looking untouched: swapping
|
|
1164
|
-
// `assert.strictEqual(add(2,3), 5)` for `assert.ok(add(2,3) !== undefined)`
|
|
1165
|
-
// removes one and adds one, so `removed > added` stayed false and the guard
|
|
1166
|
-
// said nothing — while the suite stopped checking the answer.
|
|
1167
|
-
const SPECIFIC_ASSERTION = new RegExp(
|
|
1168
|
-
[
|
|
1169
|
-
"\\bassert(?:\\.strict)?\\.?(?:strictEqual|deepStrictEqual|deepEqual|notStrictEqual|notDeepStrictEqual|equal|notEqual|match|doesNotMatch|throws|rejects|doesNotThrow)\\s*\\(",
|
|
1170
|
-
"\\bexpect\\s*\\([^)]*\\)\\s*\\.(?:toBe|toEqual|toStrictEqual|toMatch|toMatchObject|toContain|toHaveBeenCalledWith|toThrow|toHaveLength|toBeCloseTo)\\s*\\(",
|
|
1171
|
-
"\\bassert\\.(?:equals|deepEquals|include|lengthOf)\\s*\\(",
|
|
1172
|
-
"assert_eq!|assert_ne!",
|
|
1173
|
-
"\\bt\\.(?:Errorf|Fatalf)\\s*\\(",
|
|
1174
|
-
"\\brequire\\.(?:Equal|NotEqual|Len|Contains|Error|NoError)\\s*\\(",
|
|
1175
|
-
].join("|"),
|
|
1176
|
-
"i"
|
|
1177
|
-
);
|
|
1178
|
-
const isSpecificAssertion = (str) => SPECIFIC_ASSERTION.test(str);
|
|
1179
|
-
|
|
1180
|
-
/**
|
|
1181
|
-
* An assertion with every literal value replaced by a placeholder.
|
|
1182
|
-
*
|
|
1183
|
-
* Two lines that normalize to the same string are the same assertion about
|
|
1184
|
-
* the same expression; whatever differs between them is a value.
|
|
1185
|
-
*/
|
|
1186
|
-
const blankLiterals = (str) =>
|
|
1187
|
-
str
|
|
1188
|
-
.replace(/(['"`])(?:\\.|(?!\1)[^\\])*\1/g, "\u0000S")
|
|
1189
|
-
// The sign belongs to the literal: without it `3` and `-1` normalized to
|
|
1190
|
-
// different shapes and the rewritten expectation was never paired.
|
|
1191
|
-
.replace(/(?<![\w$])-?\d+(?:\.\d+)?\b/g, "\u0000N")
|
|
1192
|
-
.replace(/\b(?:true|false|null|undefined|None|True|False|nil)\b/g, "\u0000B")
|
|
1193
|
-
.replace(/\s+/g, " ")
|
|
1194
|
-
.trim();
|
|
1195
|
-
|
|
1196
1821
|
const fileAssertions = new Map();
|
|
1822
|
+
let pendingHunk = false;
|
|
1197
1823
|
|
|
1198
1824
|
for (let i = 0; i < lines.length; i++) {
|
|
1199
1825
|
const line = lines[i];
|
|
@@ -1209,6 +1835,7 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
1209
1835
|
currentFile = target && target !== "/dev/null" ? target : lastOldFile;
|
|
1210
1836
|
currentOldLineNo = null;
|
|
1211
1837
|
currentNewLineNo = null;
|
|
1838
|
+
pendingHunk = false;
|
|
1212
1839
|
continue;
|
|
1213
1840
|
}
|
|
1214
1841
|
|
|
@@ -1216,20 +1843,28 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
1216
1843
|
if (hunkMatch) {
|
|
1217
1844
|
currentOldLineNo = Number(hunkMatch[1]);
|
|
1218
1845
|
currentNewLineNo = Number(hunkMatch[2]);
|
|
1846
|
+
pendingHunk = true;
|
|
1219
1847
|
continue;
|
|
1220
1848
|
}
|
|
1221
1849
|
|
|
1222
1850
|
if (!currentFile || !isTestFile(currentFile)) {
|
|
1851
|
+
pendingHunk = false;
|
|
1223
1852
|
continue;
|
|
1224
1853
|
}
|
|
1225
1854
|
|
|
1226
1855
|
if (!fileAssertions.has(currentFile)) {
|
|
1227
|
-
fileAssertions.set(currentFile, { removed: [], added: 0, addedTexts: [], removedSpecific: [], addedSpecific: 0 });
|
|
1856
|
+
fileAssertions.set(currentFile, { removed: [], added: 0, addedTexts: [], removedSpecific: [], addedSpecific: 0, hunks: [] });
|
|
1228
1857
|
}
|
|
1229
1858
|
const fileStats = fileAssertions.get(currentFile);
|
|
1859
|
+
if (pendingHunk) {
|
|
1860
|
+
fileStats.hunks.push({ lines: [] });
|
|
1861
|
+
pendingHunk = false;
|
|
1862
|
+
}
|
|
1863
|
+
const hunk = fileStats.hunks.length > 0 ? fileStats.hunks[fileStats.hunks.length - 1] : null;
|
|
1230
1864
|
|
|
1231
1865
|
if (line.startsWith("-") && !line.startsWith("---")) {
|
|
1232
1866
|
const deletedText = line.slice(1);
|
|
1867
|
+
if (hunk) hunk.lines.push({ kind: "-", text: deletedText, oldNo: currentOldLineNo, newNo: null });
|
|
1233
1868
|
if (!isCommentLine(deletedText) && ASSERTION_PATTERN.test(deletedText)) {
|
|
1234
1869
|
fileStats.removed.push({ line: currentOldLineNo, text: deletedText });
|
|
1235
1870
|
if (isSpecificAssertion(deletedText)) {
|
|
@@ -1239,6 +1874,7 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
1239
1874
|
if (currentOldLineNo !== null) currentOldLineNo++;
|
|
1240
1875
|
} else if (line.startsWith("+") && !line.startsWith("+++")) {
|
|
1241
1876
|
const addedText = line.slice(1);
|
|
1877
|
+
if (hunk) hunk.lines.push({ kind: "+", text: addedText, oldNo: null, newNo: currentNewLineNo });
|
|
1242
1878
|
let isVacuous = false;
|
|
1243
1879
|
|
|
1244
1880
|
// Check skip injections
|
|
@@ -1287,6 +1923,13 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
1287
1923
|
|
|
1288
1924
|
if (currentNewLineNo !== null) currentNewLineNo++;
|
|
1289
1925
|
} else if (!line.startsWith("\\")) {
|
|
1926
|
+
// Context line. It is file text in both images, so the statement
|
|
1927
|
+
// assembler needs it to reassemble assertions that span a changed
|
|
1928
|
+
// line; `diff --git`/`index` lines that sneak in here are not
|
|
1929
|
+
// diff body and are not collected.
|
|
1930
|
+
if (hunk && (line.startsWith(" ") || line === "")) {
|
|
1931
|
+
hunk.lines.push({ kind: " ", text: line.slice(1), oldNo: currentOldLineNo, newNo: currentNewLineNo });
|
|
1932
|
+
}
|
|
1290
1933
|
if (currentOldLineNo !== null) currentOldLineNo++;
|
|
1291
1934
|
if (currentNewLineNo !== null) currentNewLineNo++;
|
|
1292
1935
|
}
|
|
@@ -1297,53 +1940,14 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
1297
1940
|
//
|
|
1298
1941
|
// Counting assertions cannot see this one: `assert.equal(add(1,2), 3)`
|
|
1299
1942
|
// becoming `assert.equal(add(1,2), -1)` takes one specific assertion out
|
|
1300
|
-
// and puts one specific assertion back, so every total stayed level and
|
|
1301
|
-
// guard said nothing — while the suite went from checking that
|
|
1302
|
-
// works to certifying that it is broken. It is the single
|
|
1303
|
-
// make a red suite green, and the one this tool exists
|
|
1304
|
-
//
|
|
1305
|
-
//
|
|
1306
|
-
//
|
|
1307
|
-
|
|
1308
|
-
// spec — nothing can, from the diff alone — so this reports rather than
|
|
1309
|
-
// decides, and `--allow-test-modifications` is the answer when the new
|
|
1310
|
-
// expectation is the correct one.
|
|
1311
|
-
// Shapes are computed once per added line, not once per (removed, added)
|
|
1312
|
-
// pair: a large test refactor is O(n²) comparisons and the normalisation is
|
|
1313
|
-
// the expensive half of each one.
|
|
1314
|
-
const unpairedAdded = stats.addedTexts
|
|
1315
|
-
.filter((a) => isSpecificAssertion(a.text))
|
|
1316
|
-
.map((a) => ({ ...a, shape: blankLiterals(a.text) }));
|
|
1317
|
-
for (const removedItem of stats.removed.slice()) {
|
|
1318
|
-
if (!isSpecificAssertion(removedItem.text)) continue;
|
|
1319
|
-
const shape = blankLiterals(removedItem.text);
|
|
1320
|
-
const idx = unpairedAdded.findIndex(
|
|
1321
|
-
(a) => a.shape === shape && a.text.trim() !== removedItem.text.trim()
|
|
1322
|
-
);
|
|
1323
|
-
if (idx === -1) continue;
|
|
1324
|
-
const addedItem = unpairedAdded.splice(idx, 1)[0];
|
|
1325
|
-
|
|
1326
|
-
violations.push({
|
|
1327
|
-
file,
|
|
1328
|
-
line: addedItem.line ?? removedItem.line,
|
|
1329
|
-
type: "ASSERTION_EXPECTATION_CHANGED",
|
|
1330
|
-
reason:
|
|
1331
|
-
`Test Tamper Guard: Expected value rewritten in ${file}${addedItem.line ? `:${addedItem.line}` : ""} — ` +
|
|
1332
|
-
`"${removedItem.text.trim()}" became "${addedItem.text.trim()}". ` +
|
|
1333
|
-
`If the new value is the correct one, re-run with --allow-test-modifications.`,
|
|
1334
|
-
});
|
|
1335
|
-
|
|
1336
|
-
// Both sides are accounted for here, so they must not also feed the
|
|
1337
|
-
// count-based checks below — the same line reported twice under two
|
|
1338
|
-
// names tells the operator nothing extra.
|
|
1339
|
-
stats.removed.splice(stats.removed.indexOf(removedItem), 1);
|
|
1340
|
-
const rsIdx = stats.removedSpecific.indexOf(removedItem);
|
|
1341
|
-
if (rsIdx !== -1) {
|
|
1342
|
-
stats.removedSpecific.splice(rsIdx, 1);
|
|
1343
|
-
stats.addedSpecific--;
|
|
1344
|
-
}
|
|
1345
|
-
stats.added--;
|
|
1346
|
-
}
|
|
1943
|
+
// and puts one specific assertion back, so every total stayed level and
|
|
1944
|
+
// the guard said nothing — while the suite went from checking that
|
|
1945
|
+
// addition works to certifying that it is broken. It is the single
|
|
1946
|
+
// cheapest way to make a red suite green, and the one this tool exists
|
|
1947
|
+
// to refuse. The line-based version of this pairing could only see the
|
|
1948
|
+
// single-line spelling of the edit; the reassembled-statement version
|
|
1949
|
+
// above sees every spelling, and explains why on the way.
|
|
1950
|
+
detectExpectationRewrites(file, stats.hunks, stats, violations);
|
|
1347
1951
|
|
|
1348
1952
|
if (stats.removed.length > stats.added) {
|
|
1349
1953
|
const unreplaced = stats.removed.slice(stats.added);
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One answer to "is this path a test file?".
|
|
3
|
+
*
|
|
4
|
+
* There were five, in five modules, and they disagreed with each other:
|
|
5
|
+
*
|
|
6
|
+
* - `security.mjs` matched the substring `/test/`, so a file in the
|
|
7
|
+
* repository's own root-level `tests/` directory had no leading slash and
|
|
8
|
+
* was not a test file. The entire tamper guard — skip injection, vacuous
|
|
9
|
+
* assertions, commented-out assertions, removal, weakening, expectation
|
|
10
|
+
* rewrites — was therefore switched off for the standard pytest layout
|
|
11
|
+
* (`tests/test_calc.py`), the standard Rust integration layout
|
|
12
|
+
* (`tests/integration.rs`) and every RSpec suite (`spec/`).
|
|
13
|
+
* - `mutation.mjs` had the same substring bug, in the opposite direction:
|
|
14
|
+
* those same files were not excluded, so the harness mutated operators
|
|
15
|
+
* inside the tests themselves and scored the result.
|
|
16
|
+
* - `engine.mjs` never looked for `_test.`, so `strictTestLock` did not
|
|
17
|
+
* consider a Go test file to be a test file.
|
|
18
|
+
* - `coverage.mjs` and `evidence.mjs` each had a third and fourth spelling.
|
|
19
|
+
*
|
|
20
|
+
* A predicate this load-bearing cannot have five definitions. This is the one
|
|
21
|
+
* they all use.
|
|
22
|
+
*
|
|
23
|
+
* The classification errs toward "yes". Four of the five callers get stricter
|
|
24
|
+
* when it does — the tamper guard applies, the integrity hash covers more, the
|
|
25
|
+
* lock triggers — and the fifth (mutation) has nothing to lose, because
|
|
26
|
+
* mutating an operator inside a test proves nothing about the code under test.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Directory names that mean "everything below here is a test".
|
|
31
|
+
*
|
|
32
|
+
* Matched as whole path segments, not substrings: `latest/` is not `test/`,
|
|
33
|
+
* and `myspec/` is not `spec/`.
|
|
34
|
+
*/
|
|
35
|
+
const TEST_DIR_SEGMENTS = new Set(["test", "tests", "spec", "specs", "__tests__", "__test__"]);
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* True when `file` names a test file under any convention the kit supports.
|
|
39
|
+
*
|
|
40
|
+
* Covers, by directory: `test/`, `tests/`, `spec/`, `specs/`, `__tests__/` at
|
|
41
|
+
* any depth including the repository root. By filename: `foo.test.js`,
|
|
42
|
+
* `foo.spec.ts`, `foo_test.go`, `foo_spec.rb`, `test_calc.py` (pytest),
|
|
43
|
+
* `spec_helper.rb`, and Foundry's `Foo.t.sol` / `FooTest.sol`.
|
|
44
|
+
*
|
|
45
|
+
* @param {string} file - Repo-relative path, either separator.
|
|
46
|
+
* @returns {boolean}
|
|
47
|
+
*/
|
|
48
|
+
export function isTestPath(file) {
|
|
49
|
+
if (!file || typeof file !== "string") return false;
|
|
50
|
+
const segments = file.replace(/\\/g, "/").toLowerCase().split("/").filter(Boolean);
|
|
51
|
+
if (segments.length === 0) return false;
|
|
52
|
+
|
|
53
|
+
for (let i = 0; i < segments.length - 1; i++) {
|
|
54
|
+
if (TEST_DIR_SEGMENTS.has(segments[i])) return true;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const base = segments[segments.length - 1];
|
|
58
|
+
return (
|
|
59
|
+
base.includes(".test.") ||
|
|
60
|
+
base.includes(".spec.") ||
|
|
61
|
+
base.includes("_test.") ||
|
|
62
|
+
base.includes("_spec.") ||
|
|
63
|
+
base.startsWith("test_") ||
|
|
64
|
+
base.startsWith("spec_") ||
|
|
65
|
+
base.endsWith("test.sol") ||
|
|
66
|
+
base.endsWith(".t.sol")
|
|
67
|
+
);
|
|
68
|
+
}
|