@kavishdua/proctor 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -37
- package/dist/cli.js +928 -68
- package/dist/dynamic-worker.js +776 -0
- package/package.json +2 -1
- package/proctor.schema.json +12 -0
|
@@ -0,0 +1,776 @@
|
|
|
1
|
+
// src/dynamic-worker.ts
|
|
2
|
+
import { readFileSync as readFileSync4, writeFileSync as writeFileSync4 } from "fs";
|
|
3
|
+
import { dirname as dirname2, join as join4 } from "path";
|
|
4
|
+
import { spawnSync as spawnSync4 } from "child_process";
|
|
5
|
+
import picomatch from "picomatch";
|
|
6
|
+
|
|
7
|
+
// src/dynamic/snapshots.ts
|
|
8
|
+
import { spawnSync } from "child_process";
|
|
9
|
+
import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, readlinkSync, symlinkSync, writeFileSync, copyFileSync, chmodSync } from "fs";
|
|
10
|
+
import { dirname, isAbsolute, join, relative, resolve, sep } from "path";
|
|
11
|
+
var EXCLUDED = /^(?:bench\/(?:dynamic|heldout)(?:\/|$))|(?:^|\/)(?:\.git|node_modules|\.venv|venv|__pycache__)(?:\/|$)/;
|
|
12
|
+
var MAX_BYTES = 64 * 1024 * 1024;
|
|
13
|
+
function git(cwd, args, input) {
|
|
14
|
+
const r = spawnSync("git", args, { cwd, input, maxBuffer: MAX_BYTES, timeout: 15e3, env: { ...process.env, GIT_OPTIONAL_LOCKS: "0", GIT_CONFIG_NOSYSTEM: "1" } });
|
|
15
|
+
if (r.status !== 0) throw Error("cannot read local Git snapshot");
|
|
16
|
+
return r.stdout;
|
|
17
|
+
}
|
|
18
|
+
function validPath(path) {
|
|
19
|
+
return !isAbsolute(path) && !path.split("/").some((p) => p === ".." || p === "" || p.includes("\\"));
|
|
20
|
+
}
|
|
21
|
+
function snapshot(cwd, ref) {
|
|
22
|
+
const entries = git(cwd, ref ? ["ls-tree", "-rz", "--full-tree", ref] : ["ls-files", "--stage", "-z"]).toString().split("\0").filter(Boolean).map((row) => {
|
|
23
|
+
const tab = row.indexOf(" ");
|
|
24
|
+
const meta = row.slice(0, tab).split(" "), path = row.slice(tab + 1);
|
|
25
|
+
return { path, mode: meta[0], sha: meta[ref ? 2 : 1] };
|
|
26
|
+
}).filter((e) => !EXCLUDED.test(e.path));
|
|
27
|
+
if (entries.some((e) => !validPath(e.path) || !/^100(644|755)$/.test(e.mode))) throw Error("not run: symlink, submodule or unsafe tracked path");
|
|
28
|
+
const blobs = git(cwd, ["cat-file", "--batch"], entries.map((e) => e.sha).join("\n") + "\n");
|
|
29
|
+
const files = /* @__PURE__ */ new Map();
|
|
30
|
+
let offset = 0;
|
|
31
|
+
for (const entry of entries) {
|
|
32
|
+
const newline = blobs.indexOf(10, offset);
|
|
33
|
+
const [sha, kind, sizeText] = blobs.subarray(offset, newline).toString().split(" ");
|
|
34
|
+
const size = Number(sizeText);
|
|
35
|
+
if (sha !== entry.sha || kind !== "blob" || !Number.isSafeInteger(size)) throw Error("unreadable Git blob");
|
|
36
|
+
offset = newline + 1;
|
|
37
|
+
files.set(entry.path, { data: blobs.subarray(offset, offset + size), mode: parseInt(entry.mode, 8) & 511 });
|
|
38
|
+
offset += size + 1;
|
|
39
|
+
}
|
|
40
|
+
return files;
|
|
41
|
+
}
|
|
42
|
+
function loadSnapshots(options2) {
|
|
43
|
+
const top = git(options2.cwd, ["rev-parse", "--show-toplevel"]).toString().trim();
|
|
44
|
+
if (resolve(top) !== resolve(options2.cwd)) throw Error("not run: invoke dynamic checks at the repository root");
|
|
45
|
+
const ref = options2.base ? git(top, ["merge-base", "--", options2.base, "HEAD"]).toString().trim() : "HEAD";
|
|
46
|
+
const old = snapshot(top, ref);
|
|
47
|
+
const next = snapshot(top, options2.staged ? void 0 : "HEAD");
|
|
48
|
+
if (!options2.base && !options2.staged) {
|
|
49
|
+
for (const file of options2.files) {
|
|
50
|
+
if (file.from && file.from !== "/dev/null" && file.from !== file.to) next.delete(file.from);
|
|
51
|
+
const path = file.to;
|
|
52
|
+
if (!path || path === "/dev/null") {
|
|
53
|
+
if (file.from) next.delete(file.from);
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
if (!validPath(path) || EXCLUDED.test(path)) continue;
|
|
57
|
+
const disk = join(top, path);
|
|
58
|
+
if (!existsSync(disk)) {
|
|
59
|
+
next.delete(path);
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
const stat = lstatSync(disk);
|
|
63
|
+
if (!stat.isFile()) throw Error("not run: changed symlink or non-regular file");
|
|
64
|
+
if (stat.size > MAX_BYTES) throw Error("snapshot size limit");
|
|
65
|
+
next.set(path, { data: readFileSync(disk), mode: stat.mode & 511 });
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
for (const tree of [old, next]) {
|
|
69
|
+
if ([...tree.values()].reduce((sum, f) => sum + f.data.length, 0) > MAX_BYTES) throw Error("snapshot size limit");
|
|
70
|
+
}
|
|
71
|
+
for (const path of ["package.json", "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "requirements.txt", "poetry.lock", "uv.lock"]) {
|
|
72
|
+
if (old.get(path)?.data.toString() !== next.get(path)?.data.toString()) throw Error(`not run: dependency manifest changed (${path})`);
|
|
73
|
+
}
|
|
74
|
+
return { old, next };
|
|
75
|
+
}
|
|
76
|
+
function copyDependencies(source, target, sourceRoot, targetRoot) {
|
|
77
|
+
const stat = lstatSync(source);
|
|
78
|
+
if (stat.isSymbolicLink()) {
|
|
79
|
+
const destination = resolve(dirname(source), readlinkSync(source));
|
|
80
|
+
const rel = relative(sourceRoot, destination);
|
|
81
|
+
if (rel.startsWith(".." + sep) || rel === ".." || isAbsolute(rel)) throw Error("not run: dependency symlink points outside node_modules");
|
|
82
|
+
symlinkSync(relative(dirname(target), join(targetRoot, rel)), target);
|
|
83
|
+
} else if (stat.isDirectory()) {
|
|
84
|
+
mkdirSync(target, { recursive: true });
|
|
85
|
+
for (const name of readdirSync(source)) copyDependencies(join(source, name), join(target, name), sourceRoot, targetRoot);
|
|
86
|
+
} else if (stat.isFile()) {
|
|
87
|
+
copyFileSync(source, target);
|
|
88
|
+
chmodSync(target, stat.mode & 511);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
function materialize(tree, dir, cwd) {
|
|
92
|
+
mkdirSync(dir, { recursive: true });
|
|
93
|
+
for (const [path, file] of tree) {
|
|
94
|
+
const target = join(dir, path);
|
|
95
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
96
|
+
writeFileSync(target, file.data, { mode: file.mode });
|
|
97
|
+
}
|
|
98
|
+
const deps = join(cwd, "node_modules");
|
|
99
|
+
if (existsSync(deps)) copyDependencies(deps, join(dir, "node_modules"), deps, join(dir, "node_modules"));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// src/dynamic/adapters.ts
|
|
103
|
+
import { stripVTControlCharacters } from "util";
|
|
104
|
+
import { spawnSync as spawnSync2 } from "child_process";
|
|
105
|
+
import { existsSync as existsSync2, readFileSync as readFileSync2, rmSync, writeFileSync as writeFileSync2 } from "fs";
|
|
106
|
+
import { join as join2 } from "path";
|
|
107
|
+
function detectAdapter(tree, tests) {
|
|
108
|
+
const pkgText = tree.get("package.json")?.data.toString();
|
|
109
|
+
if (pkgText) {
|
|
110
|
+
const pkg = JSON.parse(pkgText);
|
|
111
|
+
const script = pkg.scripts?.test;
|
|
112
|
+
if (typeof script === "string") {
|
|
113
|
+
if (/\bvitest\b/.test(script)) return { command: "npm", args: ["run", "test", "--", "--run", "--reporter=json"], kind: "vitest" };
|
|
114
|
+
if (/\bjest\b/.test(script)) return { command: "npm", args: ["run", "test", "--", "--json", "--runInBand", "--watch=false"], kind: "jest" };
|
|
115
|
+
if (/\bnode\b.*--test\b/.test(script)) return { command: "npm", args: ["run", "test"], kind: "tap" };
|
|
116
|
+
throw Error("not run: unrecognized package test script (node:test, jest and vitest supported)");
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const js = tests.filter((p) => /\.(?:mjs|cjs|js)$/.test(p));
|
|
120
|
+
if (js.length) return { command: process.execPath, args: ["--test", "--test-reporter=tap", ...js], kind: "tap" };
|
|
121
|
+
const py = tests.filter((p) => p.endsWith(".py"));
|
|
122
|
+
if (py.length) {
|
|
123
|
+
const pytest = ["pytest.ini", "conftest.py"].some((p) => tree.has(p)) || /\[tool\.pytest/.test(tree.get("pyproject.toml")?.data.toString() ?? "") || py.some((p) => /(?:^|\n)\s*(?:import pytest|from pytest\b)/.test(tree.get(p)?.data.toString() ?? ""));
|
|
124
|
+
return pytest ? { command: "python3", args: ["-m", "pytest", "-q", "--tb=short", "--junitxml=.proctor-junit.xml"], kind: "pytest" } : { command: "python3", args: ["-m", "unittest", "discover", "-v"], kind: "unittest", testFiles: py };
|
|
125
|
+
}
|
|
126
|
+
throw Error("not run: no supported Node or Python tests found");
|
|
127
|
+
}
|
|
128
|
+
function uniqueOutcomes(rows) {
|
|
129
|
+
const counts = /* @__PURE__ */ new Map();
|
|
130
|
+
for (const [id] of rows) counts.set(id, (counts.get(id) ?? 0) + 1);
|
|
131
|
+
return {
|
|
132
|
+
passed: rows.filter(([id, pass]) => pass && counts.get(id) === 1).map(([id]) => id),
|
|
133
|
+
failed: rows.filter(([id, pass]) => !pass && counts.get(id) === 1).map(([id]) => id)
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
function parseResults(kind, stdout, stderr, junit = "") {
|
|
137
|
+
const rows = [];
|
|
138
|
+
if (kind === "tap") {
|
|
139
|
+
for (const match of stdout.matchAll(/^\s*(not )?ok \d+ - (.+?)(?: #.*)?$/gm)) {
|
|
140
|
+
if (/ # (?:SKIP|TODO)/i.test(match[0])) continue;
|
|
141
|
+
rows.push([match[2].trim(), !match[1]]);
|
|
142
|
+
}
|
|
143
|
+
} else if (kind === "unittest") {
|
|
144
|
+
for (const match of stderr.matchAll(/^(.+? \(.+?\)) \.\.\. (ok|FAIL|ERROR)\s*$/gm)) rows.push([match[1], match[2] === "ok"]);
|
|
145
|
+
} else if (kind === "pytest") {
|
|
146
|
+
const decode = (s) => s.replace(/"/g, '"').replace(/'/g, "'").replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&");
|
|
147
|
+
for (const m of junit.matchAll(/<testcase\b([^>]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/g)) {
|
|
148
|
+
if (/<skipped\b/.test(m[2] ?? "")) continue;
|
|
149
|
+
const cls = /\bclassname="([^"]*)"/.exec(m[1])?.[1];
|
|
150
|
+
const name = /\bname="([^"]*)"/.exec(m[1])?.[1];
|
|
151
|
+
if (name) rows.push([decode(`${cls ?? ""}::${name}`), !/<(?:failure|error)\b/.test(m[2] ?? "")]);
|
|
152
|
+
}
|
|
153
|
+
} else {
|
|
154
|
+
let result2;
|
|
155
|
+
for (const stream of [stdout, stderr]) {
|
|
156
|
+
const start = stream.indexOf("{"), end = stream.lastIndexOf("}");
|
|
157
|
+
if (start < 0 || end < start) continue;
|
|
158
|
+
try {
|
|
159
|
+
const candidate = JSON.parse(stream.slice(start, end + 1));
|
|
160
|
+
if (Array.isArray(candidate.testResults)) {
|
|
161
|
+
result2 = candidate;
|
|
162
|
+
break;
|
|
163
|
+
}
|
|
164
|
+
} catch {
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
if (!result2) throw Error("test reporter did not produce readable JSON");
|
|
168
|
+
for (const suite of result2.testResults ?? []) {
|
|
169
|
+
for (const test of suite.assertionResults ?? []) {
|
|
170
|
+
if (test.status === "passed" || test.status === "failed") rows.push([test.fullName ?? [...test.ancestorTitles ?? [], test.title].join(" "), test.status === "passed"]);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
return uniqueOutcomes(rows);
|
|
175
|
+
}
|
|
176
|
+
var PYTHON_FUNCTIONS = `import ast,json,sys
|
|
177
|
+
tree=ast.parse(open(sys.argv[1],encoding='utf-8').read())
|
|
178
|
+
print(json.dumps([n.name for n in tree.body if isinstance(n,(ast.FunctionDef,ast.AsyncFunctionDef)) and n.name.startswith('test_')]))
|
|
179
|
+
`;
|
|
180
|
+
var PYTHON_CALL = `import asyncio,importlib.util,inspect,os,sys
|
|
181
|
+
file,name=sys.argv[1:]
|
|
182
|
+
sys.path.insert(0,os.path.dirname(os.path.abspath(file)))
|
|
183
|
+
spec=importlib.util.spec_from_file_location('proctor_test',file)
|
|
184
|
+
module=importlib.util.module_from_spec(spec)
|
|
185
|
+
sys.modules[spec.name]=module
|
|
186
|
+
spec.loader.exec_module(module)
|
|
187
|
+
result=getattr(module,name)()
|
|
188
|
+
if inspect.isawaitable(result): asyncio.run(result)
|
|
189
|
+
`;
|
|
190
|
+
function runPythonFiles(adapter, dir, label, discovery, env) {
|
|
191
|
+
if (!adapter.testFiles?.length) throw Error(`cannot run ${label}: no Python files for direct execution`);
|
|
192
|
+
const passed = [], failed = [], details = [];
|
|
193
|
+
const fileRuns = [];
|
|
194
|
+
const helper = ".proctor-python-test.py";
|
|
195
|
+
writeFileSync2(join2(dir, helper), PYTHON_CALL);
|
|
196
|
+
for (const file of adapter.testFiles) {
|
|
197
|
+
const collected = spawnSync2(adapter.command, ["-c", PYTHON_FUNCTIONS, file], { cwd: dir, encoding: "utf8", maxBuffer: 128 * 1024, env });
|
|
198
|
+
if (collected.error || collected.signal) throw Error(`cannot inspect ${file}: ${collected.error?.message ?? collected.signal}`);
|
|
199
|
+
const names = collected.status === 0 ? JSON.parse(collected.stdout) : [];
|
|
200
|
+
if (new Set(names).size !== names.length) throw Error(`cannot run ${label}: duplicate Python function names in ${file}`);
|
|
201
|
+
const invocations = names.length ? names.map((name) => ({ id: `${file}::${name}`, args: [helper, file, name] })) : [{ id: file, args: [file] }];
|
|
202
|
+
for (const { id, args } of invocations) {
|
|
203
|
+
const r = spawnSync2(adapter.command, args, { cwd: dir, encoding: "utf8", maxBuffer: 2 * 1024 * 1024, env });
|
|
204
|
+
if (r.error || r.signal || r.status === null) throw Error(`cannot run ${label}, ${file}: ${r.error?.message ?? r.signal ?? "no exit code"}`);
|
|
205
|
+
fileRuns.push({ file: id, command: [adapter.command, ...args].join(" "), exitCode: r.status });
|
|
206
|
+
(r.status === 0 ? passed : failed).push(id);
|
|
207
|
+
details.push(`${id}: exit ${r.status}
|
|
208
|
+
${r.stdout ?? ""}${r.stderr ?? ""}`);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return {
|
|
212
|
+
label,
|
|
213
|
+
command: `${adapter.command} -m unittest discover -v, then direct Python files`,
|
|
214
|
+
exitCode: failed.length ? 1 : 0,
|
|
215
|
+
passed,
|
|
216
|
+
failed,
|
|
217
|
+
fileRuns,
|
|
218
|
+
detail: stripVTControlCharacters(`${discovery}
|
|
219
|
+
${details.join("\n")}`).slice(-1200)
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
function runTests(adapter, dir, label, extraEnv = {}) {
|
|
223
|
+
const artifact = join2(dir, ".proctor-junit.xml");
|
|
224
|
+
if (adapter.kind === "pytest") rmSync(artifact, { force: true });
|
|
225
|
+
const env = { ...process.env, ...extraEnv, PWD: dir, ...adapter.kind === "tap" && adapter.command === "npm" ? { NODE_OPTIONS: "--test-reporter=tap" } : {} };
|
|
226
|
+
const r = spawnSync2(adapter.command, adapter.args, { cwd: dir, encoding: "utf8", maxBuffer: 2 * 1024 * 1024, env });
|
|
227
|
+
if (r.error || r.signal) throw Error(`cannot run ${label}: ${r.error?.message ?? r.signal}`);
|
|
228
|
+
const stdout = r.stdout ?? "", stderr = r.stderr ?? "";
|
|
229
|
+
const outcomes = parseResults(adapter.kind, stdout, stderr, existsSync2(artifact) ? readFileSync2(artifact, "utf8") : "");
|
|
230
|
+
if (adapter.kind === "unittest") {
|
|
231
|
+
const real = [...outcomes.passed, ...outcomes.failed].filter((id) => !id.includes("unittest.loader._FailedTest"));
|
|
232
|
+
const discovered = Number(/^Ran (\d+) tests? in /m.exec(stderr)?.[1] ?? 0);
|
|
233
|
+
const collectionFailed = outcomes.failed.some((id) => id.includes("unittest.loader._FailedTest"));
|
|
234
|
+
if (!real.length && (discovered === 0 || collectionFailed)) return runPythonFiles(adapter, dir, label, stdout + stderr, env);
|
|
235
|
+
if (collectionFailed) throw Error(`cannot run ${label}: unittest collection failed beside real cases`);
|
|
236
|
+
}
|
|
237
|
+
if (outcomes.passed.length + outcomes.failed.length === 0) throw Error(`cannot run ${label}: no named test results (exit ${r.status}); ${String(stdout + stderr).replace(/\s+/g, " ").slice(-300)}`);
|
|
238
|
+
if (r.status !== 0 && outcomes.failed.length === 0) throw Error(`cannot run ${label}: command exited ${r.status} without a named failing test`);
|
|
239
|
+
if (r.status === 0 && outcomes.failed.length) throw Error(`cannot run ${label}: reporter and exit code disagree`);
|
|
240
|
+
return {
|
|
241
|
+
label,
|
|
242
|
+
command: (adapter.kind === "tap" && adapter.command === "npm" ? "NODE_OPTIONS=--test-reporter=tap " : "") + [adapter.command, ...adapter.args].join(" "),
|
|
243
|
+
exitCode: r.status,
|
|
244
|
+
...outcomes,
|
|
245
|
+
detail: stripVTControlCharacters(String(stdout + stderr)).slice(-1200)
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// src/dynamic/signals.ts
|
|
250
|
+
var LITERAL = String.raw`(?:-?\d+(?:\.\d+)?|true|false|None|True|False|null|"[^"\n]*"|'[^'\n]*')`;
|
|
251
|
+
var ASSERTION_PATTERNS = [
|
|
252
|
+
new RegExp(String.raw`^\s*((?:assert\.(?:strictEqual|equal|deepEqual|deepStrictEqual)|self\.assertEqual)\(.+?,\s*)(${LITERAL})(\s*\)\s*;?\s*)$`),
|
|
253
|
+
new RegExp(String.raw`^\s*(expect\(.+?\)\.to(?:Be|Equal|StrictEqual)\(\s*)(${LITERAL})(\s*\)\s*;?\s*)$`),
|
|
254
|
+
new RegExp(String.raw`^(\s*assert\s+.+?\s*==\s*)(${LITERAL})(\s*)$`)
|
|
255
|
+
];
|
|
256
|
+
var assertion = (line) => ASSERTION_PATTERNS.map((pattern) => pattern.exec(line)).find(Boolean);
|
|
257
|
+
function expectationChanges(before, after) {
|
|
258
|
+
const assertions = (text) => {
|
|
259
|
+
const map = /* @__PURE__ */ new Map();
|
|
260
|
+
for (const line of text.split("\n")) {
|
|
261
|
+
const match = assertion(line);
|
|
262
|
+
if (match) map.set(match[1].trim() + match[3].trim(), match[2]);
|
|
263
|
+
}
|
|
264
|
+
return map;
|
|
265
|
+
};
|
|
266
|
+
const a = assertions(before), b = assertions(after);
|
|
267
|
+
return [...a].filter(([key, value]) => b.has(key) && b.get(key) !== value).map(([key, value]) => ({ before: value, after: b.get(key) }));
|
|
268
|
+
}
|
|
269
|
+
function expectedEdits(before, after) {
|
|
270
|
+
return expectationChanges(before, after).length > 0;
|
|
271
|
+
}
|
|
272
|
+
function expectationOnlyEdit(before, after) {
|
|
273
|
+
const canonical2 = (text) => text.split("\n").map((line) => {
|
|
274
|
+
const match = assertion(line);
|
|
275
|
+
return match ? `${match[1]}__EXPECTED_LITERAL__${match[3]}` : line;
|
|
276
|
+
}).join("\n");
|
|
277
|
+
return expectedEdits(before, after) && canonical2(before) === canonical2(after);
|
|
278
|
+
}
|
|
279
|
+
function documentedIntent(old, next) {
|
|
280
|
+
for (const [path, file] of next) {
|
|
281
|
+
if (!/(?:^|\/)(?:SPEC[^/]*|CHANGELOG[^/]*)\.(?:md|rst|txt)$/i.test(path)) continue;
|
|
282
|
+
const before = old.get(path)?.data.toString() ?? "";
|
|
283
|
+
const previous = new Set(before.split("\n"));
|
|
284
|
+
if (file.data.toString().split("\n").some((line) => line.trim() && !/^\s*#/.test(line) && !previous.has(line))) return true;
|
|
285
|
+
}
|
|
286
|
+
return false;
|
|
287
|
+
}
|
|
288
|
+
function definesPythonFunction(text, name) {
|
|
289
|
+
for (const match of text.matchAll(/^\s*def (\w+)\s*\(/gm)) {
|
|
290
|
+
if (match[1] === name) return true;
|
|
291
|
+
}
|
|
292
|
+
return false;
|
|
293
|
+
}
|
|
294
|
+
function locateTest(id, tree, tests) {
|
|
295
|
+
if (tests.includes(id) && tree.has(id)) return id;
|
|
296
|
+
const [plainFile, plainFunction] = id.split("::");
|
|
297
|
+
if (plainFunction && plainFile && tests.includes(plainFile) && tree.has(plainFile)) return plainFile;
|
|
298
|
+
const python = (/^(test\w+) \(/.exec(id)?.[1] ?? id.split("::").at(-1))?.replace(/\[.*$/, "");
|
|
299
|
+
const exact = tests.filter((p) => {
|
|
300
|
+
const text = tree.get(p)?.data.toString() ?? "";
|
|
301
|
+
return text.includes(`'${id}'`) || text.includes(`"${id}"`) || p.endsWith(".py") && !!python && definesPythonFunction(text, python);
|
|
302
|
+
});
|
|
303
|
+
if (exact.length) return exact.length === 1 ? exact[0] : void 0;
|
|
304
|
+
const nested = tests.filter((p) => [...(tree.get(p)?.data.toString() ?? "").matchAll(/\b(?:test|it)\s*\(\s*(['"])([^'"\n]+)\1/g)].some((m) => id.endsWith(" " + m[2])));
|
|
305
|
+
return nested.length === 1 ? nested[0] : void 0;
|
|
306
|
+
}
|
|
307
|
+
var UNARY_ARITHMETIC = /^(\w+)\s*[*+\/-]\s*-?\d+(?:\.\d+)?$/;
|
|
308
|
+
var PY_BRANCH = /if\s+(\w+)\s*==\s*(-?\d+(?:\.\d+)?|True|False|"[^"\\\n]*"|'[^'\\\n]*')\s*:\s*return\s+(-?\d+(?:\.\d+)?|True|False|"[^"\\\n]*"|'[^'\\\n]*')/;
|
|
309
|
+
var JS_BRANCH = /if\s*\(\s*(\w+)\s*===?\s*(-?\d+(?:\.\d+)?|true|false|"[^"\\\n]*"|'[^'\\\n]*')\s*\)\s*(?:\{\s*)?return\s+(-?\d+(?:\.\d+)?|true|false|"[^"\\\n]*"|'[^'\\\n]*');?\s*\}?/;
|
|
310
|
+
function simpleReference(expression, arg, py) {
|
|
311
|
+
if (expression === arg) return true;
|
|
312
|
+
const numeric = UNARY_ARITHMETIC.exec(expression);
|
|
313
|
+
const method = /^(\w+)\.(\w+)\(\)$/.exec(expression);
|
|
314
|
+
const negate = /^(?:!\s*|not\s+)(\w+)$/.exec(expression);
|
|
315
|
+
const concat = /^(?:(\w+)\s*\+\s*(?:"[^"\\\n]*"|'[^'\\\n]*')|(?:"[^"\\\n]*"|'[^'\\\n]*')\s*\+\s*(\w+))$/.exec(expression);
|
|
316
|
+
return numeric?.[1] === arg || negate?.[1] === arg || concat?.[1] === arg || concat?.[2] === arg || method?.[1] === arg && (py ? ["lower", "upper", "strip"] : ["toLowerCase", "toUpperCase", "trim"]).includes(method[2]);
|
|
317
|
+
}
|
|
318
|
+
function probeLiteral(text) {
|
|
319
|
+
if (text.startsWith('"') || text.startsWith("'")) return text.slice(1, -1);
|
|
320
|
+
if (/^(?:true|True|false|False)$/.test(text)) return text === "true" || text === "True";
|
|
321
|
+
return Number(text);
|
|
322
|
+
}
|
|
323
|
+
function probeInputs(input) {
|
|
324
|
+
if (typeof input === "boolean") return [!input, input];
|
|
325
|
+
if (typeof input === "string") return [`${input}a`, input, `b${input}`];
|
|
326
|
+
return [input - 1, input, input + 1];
|
|
327
|
+
}
|
|
328
|
+
function pythonBody(text, name, arg) {
|
|
329
|
+
const header = `def ${name}(${arg}):`;
|
|
330
|
+
const start = text.indexOf(header);
|
|
331
|
+
if (start < 0 || start > 0 && text[start - 1] !== "\n") return void 0;
|
|
332
|
+
const lines = text.slice(start + header.length).split("\n");
|
|
333
|
+
const body = [];
|
|
334
|
+
for (const line of lines.slice(1)) {
|
|
335
|
+
if (line.trim() === "") {
|
|
336
|
+
body.push(line);
|
|
337
|
+
continue;
|
|
338
|
+
}
|
|
339
|
+
if (/^[ \t]/.test(line)) {
|
|
340
|
+
body.push(line);
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
break;
|
|
344
|
+
}
|
|
345
|
+
return body.length ? body.join("\n") : void 0;
|
|
346
|
+
}
|
|
347
|
+
function jsBody(text, name, arg) {
|
|
348
|
+
for (const marker of [`export function ${name}(${arg})`, `function ${name}(${arg})`]) {
|
|
349
|
+
const start = text.indexOf(marker);
|
|
350
|
+
if (start < 0) continue;
|
|
351
|
+
const open = text.indexOf("{", start + marker.length);
|
|
352
|
+
if (open < 0) return void 0;
|
|
353
|
+
const close = text.indexOf("\n}", open);
|
|
354
|
+
if (close < 0) return void 0;
|
|
355
|
+
return text.slice(open + 1, close);
|
|
356
|
+
}
|
|
357
|
+
return void 0;
|
|
358
|
+
}
|
|
359
|
+
function findProbes(old, next, isTest) {
|
|
360
|
+
const probes = [];
|
|
361
|
+
for (const [file, entry] of next) {
|
|
362
|
+
if (isTest(file) || !/\.(?:js|mjs|cjs|py)$/.test(file)) continue;
|
|
363
|
+
const before = old.get(file)?.data.toString(), after = entry.data.toString();
|
|
364
|
+
if (!before || before === after) continue;
|
|
365
|
+
const py = file.endsWith(".py");
|
|
366
|
+
const functions = py ? [...before.matchAll(/^def (\w+)\((\w+)\):\n[ \t]+return ([^\n]+)\n?/gm)] : [...before.matchAll(/(?:export\s+)?function (\w+)\((\w+)\)\s*\{\s*return\s+([^;{}\n]+);?\s*\}/g)];
|
|
367
|
+
for (const fn of functions) {
|
|
368
|
+
const [, name, arg, expression] = fn;
|
|
369
|
+
if (!simpleReference(expression.trim(), arg, py)) continue;
|
|
370
|
+
if (!py && !before.includes(`export function ${name}`) && !before.includes("module.exports") && !before.includes(`exports.${name}`)) continue;
|
|
371
|
+
const body = py ? pythonBody(after, name, arg) : jsBody(after, name, arg);
|
|
372
|
+
if (!body) continue;
|
|
373
|
+
const branchMatch = (py ? PY_BRANCH : JS_BRANCH).exec(body);
|
|
374
|
+
const branch = branchMatch && branchMatch[1] === arg ? branchMatch : void 0;
|
|
375
|
+
if (!branch || !body.includes(expression.trim())) continue;
|
|
376
|
+
const remainder = body.replace(branch[0], "").replace(/return\s+/, "").replace(/[;\s]/g, "");
|
|
377
|
+
if (remainder !== expression.replace(/\s/g, "")) continue;
|
|
378
|
+
probes.push({ file, name, input: probeLiteral(branch[2]), returned: probeLiteral(branch[3]), line: after.slice(0, after.indexOf(branch[0])).split("\n").length, language: py ? "python" : "js" });
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
return probes.slice(0, 8);
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// src/dynamic/mutations.ts
|
|
385
|
+
import { parse } from "acorn";
|
|
386
|
+
import { spawnSync as spawnSync3 } from "child_process";
|
|
387
|
+
import { mkdirSync as mkdirSync2, readFileSync as readFileSync3, readdirSync as readdirSync2, writeFileSync as writeFileSync3 } from "fs";
|
|
388
|
+
import { join as join3, relative as relative2, sep as sep2 } from "path";
|
|
389
|
+
import { fileURLToPath } from "url";
|
|
390
|
+
var MUTATION_LIMIT = 4;
|
|
391
|
+
function changedSourceLines(before, after) {
|
|
392
|
+
const a = before.split("\n"), b = after.split("\n");
|
|
393
|
+
if (a.length * b.length > 4e6) return /* @__PURE__ */ new Set();
|
|
394
|
+
const width = b.length + 1, rows = new Uint16Array((a.length + 1) * width);
|
|
395
|
+
for (let i2 = a.length - 1; i2 >= 0; i2--) for (let j2 = b.length - 1; j2 >= 0; j2--) {
|
|
396
|
+
rows[i2 * width + j2] = a[i2] === b[j2] ? rows[(i2 + 1) * width + j2 + 1] + 1 : Math.max(rows[(i2 + 1) * width + j2], rows[i2 * width + j2 + 1]);
|
|
397
|
+
}
|
|
398
|
+
const changed = /* @__PURE__ */ new Set();
|
|
399
|
+
let i = 0, j = 0;
|
|
400
|
+
while (i < a.length || j < b.length) {
|
|
401
|
+
if (i < a.length && j < b.length && a[i] === b[j]) {
|
|
402
|
+
i++;
|
|
403
|
+
j++;
|
|
404
|
+
} else if (j < b.length && (i === a.length || rows[i * width + j + 1] >= rows[(i + 1) * width + j])) {
|
|
405
|
+
changed.add(++j);
|
|
406
|
+
} else i++;
|
|
407
|
+
}
|
|
408
|
+
return changed;
|
|
409
|
+
}
|
|
410
|
+
var PYTHON_PROFILE = `import atexit,json,os,sys,threading
|
|
411
|
+
root=os.getcwd()+os.sep
|
|
412
|
+
calls=set()
|
|
413
|
+
def record(frame,event,arg):
|
|
414
|
+
if event=='call' and frame.f_code.co_filename.startswith(root):
|
|
415
|
+
calls.add((frame.f_code.co_filename,frame.f_code.co_firstlineno))
|
|
416
|
+
def save():
|
|
417
|
+
with open(os.path.join(os.environ['PROCTOR_CALLS'],'python-'+str(os.getpid())+'.json'),'w') as out:
|
|
418
|
+
json.dump(sorted(calls),out)
|
|
419
|
+
sys.setprofile(record)
|
|
420
|
+
threading.setprofile(record)
|
|
421
|
+
atexit.register(save)
|
|
422
|
+
`;
|
|
423
|
+
function coverageEnvironment(dir) {
|
|
424
|
+
const path = join3(dir, ".proctor-calls"), hooks = join3(path, "hooks");
|
|
425
|
+
mkdirSync2(hooks, { recursive: true });
|
|
426
|
+
writeFileSync3(join3(hooks, "sitecustomize.py"), PYTHON_PROFILE);
|
|
427
|
+
return { path, env: { NODE_V8_COVERAGE: path, PROCTOR_CALLS: path, PYTHONPATH: hooks } };
|
|
428
|
+
}
|
|
429
|
+
function readCalls(path, dir) {
|
|
430
|
+
const calls = { js: /* @__PURE__ */ new Map(), python: /* @__PURE__ */ new Map() };
|
|
431
|
+
for (const name of readdirSync2(path).filter((p) => p.endsWith(".json")).slice(0, 100)) {
|
|
432
|
+
try {
|
|
433
|
+
const text = readFileSync3(join3(path, name), "utf8");
|
|
434
|
+
if (text.length > 4 * 1024 * 1024) continue;
|
|
435
|
+
const data = JSON.parse(text);
|
|
436
|
+
if (name.startsWith("python-")) {
|
|
437
|
+
for (const [file, line] of data) calls.python.set(file, [...calls.python.get(file) ?? [], line]);
|
|
438
|
+
} else for (const script of data.result ?? []) {
|
|
439
|
+
if (!script.url.startsWith("file:")) continue;
|
|
440
|
+
const file = fileURLToPath(script.url);
|
|
441
|
+
if (relative2(dir, file).startsWith(".." + sep2)) continue;
|
|
442
|
+
const ranges = script.functions.map((fn) => fn.ranges[0]).filter((r) => r.count > 0).map((r) => ({ start: r.startOffset, end: r.endOffset }));
|
|
443
|
+
calls.js.set(file, [...calls.js.get(file) ?? [], ...ranges]);
|
|
444
|
+
}
|
|
445
|
+
} catch {
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
return calls;
|
|
449
|
+
}
|
|
450
|
+
var OMIT = /* @__PURE__ */ Symbol("deleted statement");
|
|
451
|
+
function canonical(value, target, replacement) {
|
|
452
|
+
if (target && value === target) return canonical(replacement);
|
|
453
|
+
if (Array.isArray(value)) return value.map((v) => canonical(v, target, replacement)).filter((v) => v !== OMIT);
|
|
454
|
+
if (!value || typeof value !== "object") return value;
|
|
455
|
+
return Object.fromEntries(Object.entries(value).filter(([key]) => !["start", "end", "loc", "raw"].includes(key)).sort(([a], [b]) => a.localeCompare(b)).map(([key, v]) => [key, canonical(v, target, replacement)]));
|
|
456
|
+
}
|
|
457
|
+
var parseJs = (source) => parse(source, { ecmaVersion: "latest", sourceType: "module", locations: true });
|
|
458
|
+
function jsMutations(before, source, called) {
|
|
459
|
+
let ast, previous;
|
|
460
|
+
try {
|
|
461
|
+
ast = parseJs(source);
|
|
462
|
+
previous = parseJs(before);
|
|
463
|
+
} catch {
|
|
464
|
+
return [];
|
|
465
|
+
}
|
|
466
|
+
const fingerprint = (node) => JSON.stringify(canonical(node));
|
|
467
|
+
const original = /* @__PURE__ */ new Set();
|
|
468
|
+
function remember(node) {
|
|
469
|
+
if (["ReturnStatement", "ThrowStatement", "ExpressionStatement", "VariableDeclaration", "IfStatement"].includes(node.type)) original.add(fingerprint(node));
|
|
470
|
+
if (node.type === "IfStatement") original.add(fingerprint(node.test));
|
|
471
|
+
for (const value of Object.values(node)) {
|
|
472
|
+
if (Array.isArray(value)) {
|
|
473
|
+
for (const child of value) if (child?.type) remember(child);
|
|
474
|
+
} else if (value && typeof value === "object" && "type" in value) remember(value);
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
remember(previous);
|
|
478
|
+
const changed = changedSourceLines(before, source), candidates = [];
|
|
479
|
+
const used = /* @__PURE__ */ new Set();
|
|
480
|
+
function visit(node, fn) {
|
|
481
|
+
if (candidates.length >= MUTATION_LIMIT) return;
|
|
482
|
+
if (["FunctionDeclaration", "FunctionExpression", "ArrowFunctionExpression"].includes(node.type)) fn = node;
|
|
483
|
+
const line = node.loc.start.line;
|
|
484
|
+
if (fn && changed.has(line) && !used.has(line) && !original.has(fingerprint(node)) && called.some((r) => r.start === fn.start && r.end === fn.end)) {
|
|
485
|
+
const start = source.lastIndexOf("\n", node.start - 1) + 1;
|
|
486
|
+
const end = source.indexOf("\n", node.end);
|
|
487
|
+
const lineEnd = end < 0 ? source.length : end;
|
|
488
|
+
const fullLine = /^\s*$/.test(source.slice(start, node.start)) && /^\s*$/.test(source.slice(node.end, lineEnd));
|
|
489
|
+
const attempts = [];
|
|
490
|
+
if (fullLine && node.loc.end.line === line && ["ReturnStatement", "ThrowStatement", "ExpressionStatement", "VariableDeclaration", "IfStatement"].includes(node.type)) {
|
|
491
|
+
attempts.push({ text: source.slice(0, node.start) + source.slice(node.end), replacement: OMIT, target: node, operation: "delete statement" });
|
|
492
|
+
}
|
|
493
|
+
if (node.type === "IfStatement") {
|
|
494
|
+
const test = node.test;
|
|
495
|
+
if (test.loc.end.line === line && !original.has(fingerprint(test))) attempts.push({
|
|
496
|
+
text: source.slice(0, test.start) + `!(${source.slice(test.start, test.end)})` + source.slice(test.end),
|
|
497
|
+
target: test,
|
|
498
|
+
replacement: { type: "UnaryExpression", operator: "!", prefix: true, argument: canonical(test) },
|
|
499
|
+
operation: "negate condition"
|
|
500
|
+
});
|
|
501
|
+
}
|
|
502
|
+
for (const attempt of attempts) {
|
|
503
|
+
try {
|
|
504
|
+
const parsed = parseJs(attempt.text);
|
|
505
|
+
if (JSON.stringify(canonical(parsed)) !== JSON.stringify(canonical(ast, attempt.target, attempt.replacement))) continue;
|
|
506
|
+
candidates.push({ line, functionName: fn.id?.name ?? "<anonymous>", source: attempt.text, operation: attempt.operation });
|
|
507
|
+
used.add(line);
|
|
508
|
+
break;
|
|
509
|
+
} catch {
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
for (const value of Object.values(node)) {
|
|
514
|
+
if (Array.isArray(value)) {
|
|
515
|
+
for (const child of value) if (child?.type) visit(child, fn);
|
|
516
|
+
} else if (value && typeof value === "object" && "type" in value) visit(value, fn);
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
visit(ast);
|
|
520
|
+
return candidates.slice(0, MUTATION_LIMIT);
|
|
521
|
+
}
|
|
522
|
+
var PYTHON_MUTATIONS = `import ast,copy,json,sys
|
|
523
|
+
data=json.load(sys.stdin)
|
|
524
|
+
source=data['source']
|
|
525
|
+
lines=source.splitlines(keepends=True)
|
|
526
|
+
tree=ast.parse(source)
|
|
527
|
+
original={ast.dump(n) for n in ast.walk(ast.parse(data['before'])) if isinstance(n,(ast.stmt,ast.expr))}
|
|
528
|
+
# Require a lossless syntax round trip before editing this file.
|
|
529
|
+
assert ast.dump(ast.parse(ast.unparse(tree)))==ast.dump(tree)
|
|
530
|
+
changed=set(data['changed'])
|
|
531
|
+
called=set(data['called'])
|
|
532
|
+
out=[]
|
|
533
|
+
used=set()
|
|
534
|
+
def offset(line,col):
|
|
535
|
+
return sum(map(len,lines[:line-1]))+len(lines[line-1].encode('utf-8')[:col].decode('utf-8'))
|
|
536
|
+
class Replace(ast.NodeTransformer):
|
|
537
|
+
def __init__(self,target,replacement): self.target,self.replacement=target,replacement
|
|
538
|
+
def visit(self,node):
|
|
539
|
+
if node is self.target: return self.replacement
|
|
540
|
+
return super().visit(node)
|
|
541
|
+
def visit(node,fn=None):
|
|
542
|
+
if len(out)>=4: return
|
|
543
|
+
if isinstance(node,(ast.FunctionDef,ast.AsyncFunctionDef)):
|
|
544
|
+
fn=node if not node.decorator_list and isinstance(node,ast.FunctionDef) else None
|
|
545
|
+
line=getattr(node,'lineno',0)
|
|
546
|
+
if fn and fn.lineno in called and line in changed and line not in used and ast.dump(node) not in original:
|
|
547
|
+
attempts=[]
|
|
548
|
+
if isinstance(node,(ast.Return,ast.Raise,ast.Assign,ast.AnnAssign,ast.AugAssign,ast.Expr,ast.If)):
|
|
549
|
+
start=offset(line,node.col_offset)
|
|
550
|
+
end=offset(node.end_lineno,node.end_col_offset)
|
|
551
|
+
if node.end_lineno==line and not source[offset(line,0):start].strip() and not source[end:offset(line,len(lines[line-1].encode('utf-8')))].strip():
|
|
552
|
+
attempts.append((source[:start]+source[end:],node,None,'delete statement'))
|
|
553
|
+
target=node.test if isinstance(node,ast.If) else node.value if isinstance(node,ast.Return) else None
|
|
554
|
+
if target is not None and target.lineno==target.end_lineno and ast.dump(target) not in original:
|
|
555
|
+
start=offset(target.lineno,target.col_offset)
|
|
556
|
+
end=offset(target.end_lineno,target.end_col_offset)
|
|
557
|
+
replacement=ast.UnaryOp(op=ast.Not(),operand=copy.deepcopy(target))
|
|
558
|
+
attempts.append((source[:start]+'not ('+source[start:end]+')'+source[end:],target,replacement,'negate condition' if isinstance(node,ast.If) else 'negate return'))
|
|
559
|
+
for text,target,replacement,operation in attempts:
|
|
560
|
+
try:
|
|
561
|
+
parsed=ast.parse(text)
|
|
562
|
+
memo={}
|
|
563
|
+
expected=copy.deepcopy(tree,memo)
|
|
564
|
+
expected=Replace(memo[id(target)],replacement).visit(expected)
|
|
565
|
+
same=ast.dump(parsed)==ast.dump(expected) and ast.dump(ast.parse(ast.unparse(parsed)))==ast.dump(parsed)
|
|
566
|
+
except (SyntaxError,ValueError): same=False
|
|
567
|
+
if same:
|
|
568
|
+
out.append(dict(line=line,functionName=fn.name,source=text,operation=operation))
|
|
569
|
+
used.add(line)
|
|
570
|
+
break
|
|
571
|
+
for child in ast.iter_child_nodes(node): visit(child,fn)
|
|
572
|
+
visit(tree)
|
|
573
|
+
print(json.dumps(out[:4]))
|
|
574
|
+
`;
|
|
575
|
+
function pythonMutations(before, source, called, dir) {
|
|
576
|
+
const r = spawnSync3("python3", ["-c", PYTHON_MUTATIONS], {
|
|
577
|
+
cwd: dir,
|
|
578
|
+
encoding: "utf8",
|
|
579
|
+
maxBuffer: 2 * 1024 * 1024,
|
|
580
|
+
input: JSON.stringify({ before, source, changed: [...changedSourceLines(before, source)], called })
|
|
581
|
+
});
|
|
582
|
+
if (r.status !== 0) return [];
|
|
583
|
+
try {
|
|
584
|
+
return JSON.parse(r.stdout);
|
|
585
|
+
} catch {
|
|
586
|
+
return [];
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
// src/dynamic-worker.ts
|
|
591
|
+
var options = JSON.parse(readFileSync4(process.argv[2], "utf8"));
|
|
592
|
+
var scratch = dirname2(process.argv[2]);
|
|
593
|
+
var result = { status: "completed", findings: [], runs: [], notes: [], elapsedMs: 0 };
|
|
594
|
+
function evidence(...runs) {
|
|
595
|
+
return runs.map((r) => `${r.label}: \`${r.command}\`, exit ${r.exitCode}, ${r.passed.length} passed, ${r.failed.length} failed${r.failed.length ? ` (${r.failed.join(", ")})` : ""}${r.fileRuns ? "; " + r.fileRuns.map((f) => `\`${f.command}\`, exit ${f.exitCode}, file ${f.file}`).join("; ") : ""}`).join("; ");
|
|
596
|
+
}
|
|
597
|
+
function finding(id, file, message, runs, line = 1) {
|
|
598
|
+
result.findings.push({ verifierId: id, severity: "warn", file, line, message: `${message} Evidence: ${evidence(...runs)}.`, suggestion: "Review the observed behavior and add a focused regression test. Runtime evidence cannot establish intent." });
|
|
599
|
+
}
|
|
600
|
+
try {
|
|
601
|
+
let run2 = function(tree, label) {
|
|
602
|
+
const dir = join4(scratch, `run-${result.runs.length + 1}`);
|
|
603
|
+
materialize(tree, dir, options.cwd);
|
|
604
|
+
const runAdapter = detectAdapter(tree, [...tree.keys()].filter((p) => isTest(p)));
|
|
605
|
+
const testRun = runTests(runAdapter, dir, label);
|
|
606
|
+
result.runs.push(testRun);
|
|
607
|
+
return { run: testRun, dir };
|
|
608
|
+
}, withTests2 = function(source, tests) {
|
|
609
|
+
const hybrid = new Map(source);
|
|
610
|
+
for (const p of /* @__PURE__ */ new Set([...oldTests, ...nextTests])) hybrid.delete(p);
|
|
611
|
+
for (const [p, data] of tests) if (isTest(p)) hybrid.set(p, data);
|
|
612
|
+
return hybrid;
|
|
613
|
+
};
|
|
614
|
+
run = run2, withTests = withTests2;
|
|
615
|
+
const { old, next } = loadSnapshots(options);
|
|
616
|
+
const isTest = picomatch(options.testPathGlobs, { dot: true });
|
|
617
|
+
const oldTests = [...old.keys()].filter((p) => isTest(p)), nextTests = [...next.keys()].filter((p) => isTest(p));
|
|
618
|
+
const adapter = detectAdapter(old, oldTests);
|
|
619
|
+
const nextAdapter = detectAdapter(next, nextTests);
|
|
620
|
+
if (adapter.kind !== nextAdapter.kind) throw Error("not run: test framework changed");
|
|
621
|
+
if ([...next.keys()].some((p) => !isTest(p) && /\.(?:go|rs|java|rb|php|cs|kt|cpp|c|swift)$/.test(p) && old.get(p)?.data.toString() !== next.get(p)?.data.toString())) {
|
|
622
|
+
result.notes.push("not run: probes for languages other than Node JavaScript and Python");
|
|
623
|
+
}
|
|
624
|
+
const original = run2(old, "original tests / original code").run;
|
|
625
|
+
const current = run2(next, "new tests / new code");
|
|
626
|
+
if (original.failed.length && current.run.exitCode === 0) {
|
|
627
|
+
result.notes.push("Original tests are red and new tests are green. Continue with per-test comparisons and applicable probes; this is a normal fix shape.");
|
|
628
|
+
}
|
|
629
|
+
const testChanges = [.../* @__PURE__ */ new Set([...oldTests, ...nextTests])].filter((p) => old.get(p)?.data.toString() !== next.get(p)?.data.toString());
|
|
630
|
+
const changedSource = [.../* @__PURE__ */ new Set([...old.keys(), ...next.keys()])].some((p) => !isTest(p) && !/\.(?:md|rst|txt)$/i.test(p) && old.get(p)?.data.toString() !== next.get(p)?.data.toString());
|
|
631
|
+
const intent = documentedIntent(old, next);
|
|
632
|
+
const edits = testChanges.filter((p) => expectedEdits(old.get(p)?.data.toString() ?? "", next.get(p)?.data.toString() ?? ""));
|
|
633
|
+
const cross = testChanges.length ? run2(withTests2(next, old), "original tests / new code").run : current.run;
|
|
634
|
+
if (!intent) {
|
|
635
|
+
for (const id of cross.failed.filter((id2) => original.passed.includes(id2))) {
|
|
636
|
+
const file = locateTest(id, old, oldTests);
|
|
637
|
+
if (file && !edits.includes(file)) finding("DY001", file, `Previously passing original test ${JSON.stringify(id)} fails against the new code`, [original, current.run, cross]);
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
let reverse;
|
|
641
|
+
if (testChanges.length) {
|
|
642
|
+
reverse = run2(withTests2(old, next), "new tests / original code").run;
|
|
643
|
+
const both = current.run.passed.filter((id) => reverse.passed.includes(id));
|
|
644
|
+
if (both.length) result.notes.push(`${both.length} named tests passed on both code revisions. This alone does not show that changed tests exercise the change and is not a finding.`);
|
|
645
|
+
if (!changedSource && !intent && testChanges.length === 1 && edits.length && expectationOnlyEdit(old.get(edits[0])?.data.toString() ?? "", next.get(edits[0])?.data.toString() ?? "")) {
|
|
646
|
+
const rejected = reverse.failed.filter((id) => original.passed.includes(id));
|
|
647
|
+
const accepted = reverse.passed.filter((id) => original.failed.includes(id));
|
|
648
|
+
for (const id of [...rejected, ...accepted]) {
|
|
649
|
+
const file = locateTest(id, old, oldTests);
|
|
650
|
+
const mismatch = accepted.includes(id) ? "failed with the original expectation, but the edited expectation passes on that same old code; the edit already matches the old behavior" : "passed with the original expectation, but the edited expectation fails on that same old code";
|
|
651
|
+
if (file && edits.includes(file)) finding("DY003", file, `Only the expected literal changed (${expectationChanges(old.get(file)?.data.toString() ?? "", next.get(file)?.data.toString() ?? "").map((e) => `${e.before} to ${e.after}`).join(", ")}): test ${JSON.stringify(id)} ${mismatch}`, [original, current.run, reverse]);
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
if (current.run.exitCode === 0 && !testChanges.length && !intent) {
|
|
656
|
+
for (const probe of findProbes(old, next, isTest)) {
|
|
657
|
+
const values = probeInputs(probe.input);
|
|
658
|
+
const observed = [];
|
|
659
|
+
const probeRuns = [];
|
|
660
|
+
for (const [tree, label] of [[old, "probe / original code"], [next, "probe / new code"]]) {
|
|
661
|
+
const dir = join4(scratch, `probe-${result.runs.length + 1}`);
|
|
662
|
+
materialize(tree, dir, options.cwd);
|
|
663
|
+
let command, args;
|
|
664
|
+
if (probe.language === "js") {
|
|
665
|
+
const script = `const m=await import(${JSON.stringify("./" + probe.file)});const f=m[${JSON.stringify(probe.name)}]??m.default?.[${JSON.stringify(probe.name)}];console.log(JSON.stringify(${JSON.stringify(values)}.map(x=>f(x))));`;
|
|
666
|
+
writeFileSync4(join4(dir, ".proctor-probe.mjs"), script);
|
|
667
|
+
command = process.execPath;
|
|
668
|
+
args = [".proctor-probe.mjs"];
|
|
669
|
+
} else {
|
|
670
|
+
const script = `import importlib.util,json
|
|
671
|
+
s=importlib.util.spec_from_file_location('probe',${JSON.stringify(probe.file)})
|
|
672
|
+
m=importlib.util.module_from_spec(s)
|
|
673
|
+
s.loader.exec_module(m)
|
|
674
|
+
print(json.dumps([getattr(m,${JSON.stringify(probe.name)})(x) for x in json.loads(${JSON.stringify(JSON.stringify(values))})]))
|
|
675
|
+
`;
|
|
676
|
+
writeFileSync4(join4(dir, ".proctor-probe.py"), script);
|
|
677
|
+
command = "python3";
|
|
678
|
+
args = [".proctor-probe.py"];
|
|
679
|
+
}
|
|
680
|
+
const r = spawnSync4(command, args, { cwd: dir, encoding: "utf8", maxBuffer: 128 * 1024 });
|
|
681
|
+
if (r.status !== 0) {
|
|
682
|
+
observed.length = 0;
|
|
683
|
+
break;
|
|
684
|
+
}
|
|
685
|
+
let outputs;
|
|
686
|
+
try {
|
|
687
|
+
outputs = JSON.parse(r.stdout);
|
|
688
|
+
} catch {
|
|
689
|
+
observed.length = 0;
|
|
690
|
+
break;
|
|
691
|
+
}
|
|
692
|
+
if (!Array.isArray(outputs) || outputs.length !== values.length || outputs.some((n) => !["number", "string", "boolean"].includes(typeof n) || typeof n === "number" && !Number.isFinite(n))) {
|
|
693
|
+
observed.length = 0;
|
|
694
|
+
break;
|
|
695
|
+
}
|
|
696
|
+
observed.push(outputs);
|
|
697
|
+
const pr = { label, command: `${command} ${args.join(" ")} (${probe.file}:${probe.name}, inputs ${JSON.stringify(values)})`, exitCode: 0, passed: [], failed: [], detail: `outputs ${JSON.stringify(outputs)}` };
|
|
698
|
+
result.runs.push(pr);
|
|
699
|
+
probeRuns.push(pr);
|
|
700
|
+
}
|
|
701
|
+
if (observed.length === 2 && values.every((_, i) => i === 1 || observed[0][i] === observed[1][i]) && observed[0][1] !== observed[1][1]) {
|
|
702
|
+
finding("DY002", probe.file, `Literal branch in ${probe.name} contradicts the original simple reference only at ${JSON.stringify(probe.input)}: old ${JSON.stringify(observed[0])}, new ${JSON.stringify(observed[1])}, inputs ${JSON.stringify(values)}`, [original, current.run, ...probeRuns], probe.line);
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
if (original.failed.length && current.run.exitCode === 0 && !testChanges.length && !intent) {
|
|
707
|
+
try {
|
|
708
|
+
const coverage = coverageEnvironment(current.dir);
|
|
709
|
+
const control = runTests(nextAdapter, current.dir, "new tests / new code coverage control", coverage.env);
|
|
710
|
+
result.runs.push(control);
|
|
711
|
+
const samePasses = (a, b) => a.exitCode === 0 && b.exitCode === 0 && !a.failed.length && !b.failed.length && a.passed.length > 0 && a.passed.length === b.passed.length && a.passed.every((id) => b.passed.includes(id));
|
|
712
|
+
if (samePasses(current.run, control)) {
|
|
713
|
+
const calls = readCalls(coverage.path, current.dir);
|
|
714
|
+
let count = 0;
|
|
715
|
+
for (const [file, entry] of next) {
|
|
716
|
+
if (count >= MUTATION_LIMIT) break;
|
|
717
|
+
if (isTest(file) || !/\.(?:js|mjs|cjs|py)$/.test(file)) continue;
|
|
718
|
+
const before = old.get(file)?.data.toString() ?? "", source = entry.data.toString();
|
|
719
|
+
if (before === source) continue;
|
|
720
|
+
const path = join4(current.dir, file);
|
|
721
|
+
const candidates = file.endsWith(".py") ? pythonMutations(before, source, calls.python.get(path) ?? [], current.dir) : jsMutations(before, source, calls.js.get(path) ?? []);
|
|
722
|
+
for (const mutation of candidates) {
|
|
723
|
+
if (count >= MUTATION_LIMIT) break;
|
|
724
|
+
count++;
|
|
725
|
+
let mutant;
|
|
726
|
+
try {
|
|
727
|
+
writeFileSync4(path, mutation.source);
|
|
728
|
+
mutant = runTests(nextAdapter, current.dir, `mutation ${count}: ${mutation.operation}, ${file}:${mutation.line} (${mutation.functionName})`);
|
|
729
|
+
result.runs.push(mutant);
|
|
730
|
+
} catch {
|
|
731
|
+
} finally {
|
|
732
|
+
writeFileSync4(path, source);
|
|
733
|
+
}
|
|
734
|
+
if (mutant && samePasses(current.run, mutant)) {
|
|
735
|
+
const restored = runTests(nextAdapter, current.dir, `mutation ${count}: restored control`);
|
|
736
|
+
result.runs.push(restored);
|
|
737
|
+
if (samePasses(current.run, restored)) finding(
|
|
738
|
+
"DY004",
|
|
739
|
+
file,
|
|
740
|
+
`${mutation.operation} at ${file}:${mutation.line} in called function ${mutation.functionName} left every named test passing; this changed statement has no demonstrated test protection`,
|
|
741
|
+
[original, current.run, control, mutant, restored],
|
|
742
|
+
mutation.line
|
|
743
|
+
);
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
result.notes.push(`Mutation-lite: ${Math.min(count, MUTATION_LIMIT)} candidates run, cap ${MUTATION_LIMIT}; only parser-confirmed statements in called functions are eligible.`);
|
|
748
|
+
} else result.notes.push("not run: mutation-lite coverage control did not reproduce the green suite");
|
|
749
|
+
} catch (error) {
|
|
750
|
+
result.notes.push(`not run: mutation-lite could not establish reliable evidence: ${String(error)}`);
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
const raceFiles = testChanges.filter((p) => /(?:sleep\s*\(|setTimeout\s*\(|\bretry\b|\bretries\b)/.test(next.get(p)?.data.toString() ?? "") && (next.get(p)?.data.toString().match(/sleep\s*\(|setTimeout\s*\(|\bretry\b|\bretries\b/g)?.length ?? 0) > (old.get(p)?.data.toString().match(/sleep\s*\(|setTimeout\s*\(|\bretry\b|\bretries\b/g)?.length ?? 0));
|
|
754
|
+
if (raceFiles.length && current.run.exitCode === 0) {
|
|
755
|
+
const repeats = [current.run];
|
|
756
|
+
for (let i = 1; i <= 3; i++) {
|
|
757
|
+
const repeated = runTests(nextAdapter, current.dir, `new tests / new code repeat ${i}`);
|
|
758
|
+
result.runs.push(repeated);
|
|
759
|
+
repeats.push(repeated);
|
|
760
|
+
}
|
|
761
|
+
const ids = new Set(repeats.flatMap((r) => r.failed));
|
|
762
|
+
for (const id of ids) {
|
|
763
|
+
const file = locateTest(id, next, raceFiles);
|
|
764
|
+
if (file && repeats.some((r) => r.passed.includes(id)) && repeats.filter((r) => r.failed.includes(id)).length >= 2) {
|
|
765
|
+
finding("DY005", file, `Test ${JSON.stringify(id)} has inconsistent outcomes across four runs after adding a sleep or retry; this can also be order dependence`, repeats);
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
} catch (error) {
|
|
770
|
+
result.status = "skipped";
|
|
771
|
+
result.reason = String(error).replace(/^Error: /, "");
|
|
772
|
+
result.findings = [];
|
|
773
|
+
}
|
|
774
|
+
var run;
|
|
775
|
+
var withTests;
|
|
776
|
+
process.stdout.write(JSON.stringify(result));
|