jules-orchestrator-kit 0.63.0 → 0.64.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +3 -2
- package/scripts/guard-reach-check.mjs +56 -6
- package/scripts/package-integrity-check.mjs +251 -0
- package/scripts/release.mjs +16 -0
- package/src/assertions.mjs +16 -0
- package/src/guard-policy.mjs +481 -0
- package/src/security.mjs +194 -14
package/README.md
CHANGED
|
@@ -204,7 +204,7 @@ To maximize PR merge rates, dispatch tasks according to deterministic boundaries
|
|
|
204
204
|
* **Fail-Closed Security & Secret Redaction:** Evaluates explicit Deny rules before Allow rules against canonicalized, case-folded paths. Redacts high-entropy keys and base64-encoded credentials (such as Kubernetes `Secret` manifests).
|
|
205
205
|
* **Complexity & Cost Router:** Zero-dependency heuristic classifier (`src/router.mjs`) routing mechanical tasks to lightweight models while reserving primary models for complex refactors, with a `node --check` syntax-verification gate that transparently escalates a FAST-tier result to the primary provider if it left broken JS on disk.
|
|
206
206
|
* **Terminal UI & Diagnostic Matrix (`agentctl doctor`):** Interactive terminal dashboard, task sidecar manager, and automated transactional self-repair.
|
|
207
|
-
* **Verified Test Suite:** Tested with **
|
|
207
|
+
* **Verified Test Suite:** Tested with **1085 unit tests across 150 suites passing in < 15.0s**.
|
|
208
208
|
|
|
209
209
|
<br/>
|
|
210
210
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "jules-orchestrator-kit",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.64.0",
|
|
4
4
|
"description": "Zero-dependency safety gatekeeper, test oracle generator, and multi-agent coordination protocol for autonomous coding agents — Google Jules, Claude Code, Codex and Gemini CLI.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -56,7 +56,8 @@
|
|
|
56
56
|
"jules:doc-sync": "node scripts/doc-sync-check.mjs",
|
|
57
57
|
"release": "node scripts/release.mjs",
|
|
58
58
|
"guard-reach": "node scripts/guard-reach-check.mjs",
|
|
59
|
-
"lint": "eslint ."
|
|
59
|
+
"lint": "eslint .",
|
|
60
|
+
"package-integrity": "node scripts/package-integrity-check.mjs"
|
|
60
61
|
},
|
|
61
62
|
"engines": {
|
|
62
63
|
"node": ">=20.0.0"
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
*
|
|
28
28
|
* Three steps, all in-process, no dependencies, well under a second:
|
|
29
29
|
*
|
|
30
|
-
* 1. POLICY — the hand-written witness table in
|
|
30
|
+
* 1. POLICY — the hand-written witness table in src/guard-policy.mjs
|
|
31
31
|
* must hold. It is derived from what the tool advertises, never
|
|
32
32
|
* from the regexes that implement it.
|
|
33
33
|
* 2. CANARIES — every known-bad input must produce the finding it names. A
|
|
@@ -52,14 +52,24 @@ import {
|
|
|
52
52
|
PREDICATE_MUTANTS,
|
|
53
53
|
EMPTY_RUN_CANARIES,
|
|
54
54
|
SCOPE_CANARIES,
|
|
55
|
-
|
|
55
|
+
INNOCENT_EDITS,
|
|
56
|
+
UNREADABLE_DIALECTS,
|
|
57
|
+
} from "../src/guard-policy.mjs";
|
|
56
58
|
|
|
57
|
-
/**
|
|
59
|
+
/**
|
|
60
|
+
* Build a unified diff for one canary.
|
|
61
|
+
*
|
|
62
|
+
* The context line carries the comment syntax of the file's own language.
|
|
63
|
+
* A `//` line in a `.py` fixture is not a comment, and a fixture that lies
|
|
64
|
+
* about the language under test measures the fixture rather than the guard —
|
|
65
|
+
* which is how two false results were once read as two defects.
|
|
66
|
+
*/
|
|
58
67
|
function canaryDiff(c) {
|
|
59
|
-
const
|
|
68
|
+
const ctx = c.context || "// context";
|
|
69
|
+
const lines = [`--- a/${c.file}`, `+++ b/${c.file}`, "@@ -1,20 +1,20 @@", ` ${ctx}`];
|
|
60
70
|
for (const l of c.removed) lines.push(`-${l}`);
|
|
61
71
|
for (const l of c.added) lines.push(`+${l}`);
|
|
62
|
-
lines.push(
|
|
72
|
+
lines.push(` ${ctx}`);
|
|
63
73
|
return lines.join("\n");
|
|
64
74
|
}
|
|
65
75
|
|
|
@@ -107,6 +117,7 @@ const canaryResults = new Map();
|
|
|
107
117
|
{
|
|
108
118
|
const silent = [];
|
|
109
119
|
const noDenominator = [];
|
|
120
|
+
const noAssertions = [];
|
|
110
121
|
for (const c of TAMPER_CANARIES) {
|
|
111
122
|
const res = checkTestTampering(canaryDiff(c));
|
|
112
123
|
const hit = (res.violations || []).some((v) => v.type === c.expect);
|
|
@@ -114,12 +125,51 @@ const canaryResults = new Map();
|
|
|
114
125
|
if (!hit) silent.push(`${c.id} expected ${c.expect}, got ${JSON.stringify((res.violations || []).map((v) => v.type))}`);
|
|
115
126
|
// A finding with no denominator is the shape this script exists to reject.
|
|
116
127
|
if (hit && !(res.inputsSeen > 0)) noDenominator.push(c.id);
|
|
128
|
+
// Counting lines was not enough: a JUnit diff reported one input examined
|
|
129
|
+
// and a clean PASS while every assertion in it went unrecognised. A rule
|
|
130
|
+
// about assertions has to say how many assertions it actually read.
|
|
131
|
+
if (hit && c.expect !== "TEST_SKIP_INJECTION" && !(res.assertionsSeen > 0)) {
|
|
132
|
+
noAssertions.push(`${c.id} (${res.assertionsSeen} assertions parsed)`);
|
|
133
|
+
}
|
|
117
134
|
}
|
|
118
135
|
add("canaries: every tamper rule still fires", silent.length === 0, silent.length ? silent.join("; ") : `${TAMPER_CANARIES.length} canaries red as required`);
|
|
119
136
|
add("canaries: every finding carries a denominator", noDenominator.length === 0, noDenominator.length ? noDenominator.join(", ") : "inputsSeen > 0 on every hit");
|
|
137
|
+
add("canaries: assertion rules parsed an assertion", noAssertions.length === 0, noAssertions.length ? noAssertions.join(", ") : "assertionsSeen > 0 on every assertion finding");
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// --- 3. The opposite failure: flagging what is innocent ---------------------
|
|
141
|
+
{
|
|
142
|
+
const noisy = [];
|
|
143
|
+
for (const e of INNOCENT_EDITS) {
|
|
144
|
+
const res = checkTestTampering(canaryDiff(e));
|
|
145
|
+
const types = (res.violations || []).map((v) => v.type);
|
|
146
|
+
if (types.length > 0) noisy.push(`${e.id} → ${JSON.stringify(types)} (${e.why})`);
|
|
147
|
+
}
|
|
148
|
+
add(
|
|
149
|
+
"innocent edits stay silent",
|
|
150
|
+
noisy.length === 0,
|
|
151
|
+
noisy.length
|
|
152
|
+
? noisy.join("; ")
|
|
153
|
+
: `${INNOCENT_EDITS.length} ordinary edits produce no finding`
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
{
|
|
158
|
+
const quiet = [];
|
|
159
|
+
for (const d of UNREADABLE_DIALECTS) {
|
|
160
|
+
const res = checkTestTampering(canaryDiff(d));
|
|
161
|
+
if (res.status !== "UNREADABLE") quiet.push(`${d.id} → ${res.status} (${d.why})`);
|
|
162
|
+
}
|
|
163
|
+
add(
|
|
164
|
+
"an unparsable dialect says so",
|
|
165
|
+
quiet.length === 0,
|
|
166
|
+
quiet.length
|
|
167
|
+
? `${quiet.join("; ")} — coverage ending is fine, ending silently is not`
|
|
168
|
+
: `${UNREADABLE_DIALECTS.length} unsupported dialects reported, not passed`
|
|
169
|
+
);
|
|
120
170
|
}
|
|
121
171
|
|
|
122
|
-
// ---
|
|
172
|
+
// --- 4. Predicate mutants ---------------------------------------------------
|
|
123
173
|
{
|
|
124
174
|
const survivors = [];
|
|
125
175
|
for (const mutant of PREDICATE_MUTANTS) {
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Does the package we publish actually work?
|
|
5
|
+
*
|
|
6
|
+
* Every test in this repository runs against the source tree, where every
|
|
7
|
+
* file is present by definition. What users install is a tarball built from
|
|
8
|
+
* the `files` list in package.json, and nothing checked that the two agreed.
|
|
9
|
+
*
|
|
10
|
+
* They did not. v0.63.0 shipped `scripts/guard-reach-check.mjs` — the check
|
|
11
|
+
* whose entire purpose is to prove no guard has silently gone missing —
|
|
12
|
+
* while leaving behind the policy contract it imports. Unpacked and run, it
|
|
13
|
+
* threw ERR_MODULE_NOT_FOUND. The CLI was fine, 1015 tests were green, nine
|
|
14
|
+
* CI cells passed, and the published artefact still had a hole in it,
|
|
15
|
+
* because every one of those signals was measured somewhere the file existed.
|
|
16
|
+
*
|
|
17
|
+
* This asks the packer what it would ship, and then resolves the import
|
|
18
|
+
* graph inside that answer.
|
|
19
|
+
*
|
|
20
|
+
* Usage: node scripts/package-integrity-check.mjs [--json]
|
|
21
|
+
* Exit codes: 0 = the tarball is self-contained, 1 = it is not.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { execFileSync } from "node:child_process";
|
|
25
|
+
import { readFileSync, existsSync } from "node:fs";
|
|
26
|
+
import { resolve, dirname, relative, sep } from "node:path";
|
|
27
|
+
import { fileURLToPath } from "node:url";
|
|
28
|
+
|
|
29
|
+
import { IMPORT_EXTRACTION_CASES } from "../src/guard-policy.mjs";
|
|
30
|
+
|
|
31
|
+
const root = fileURLToPath(new URL("..", import.meta.url));
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Mark every character that sits inside a string literal or a comment.
|
|
35
|
+
*
|
|
36
|
+
* Without this, an import quoted *inside* a fixture string reads as an
|
|
37
|
+
* import of the file itself — this check's first run reported six broken
|
|
38
|
+
* imports in the policy contract, all of them example text. A guard that
|
|
39
|
+
* cries wolf about its own fixtures gets switched off in a week.
|
|
40
|
+
*
|
|
41
|
+
* Approximate on purpose, and biased on purpose: a regex literal holding a
|
|
42
|
+
* quote can open a phantom string, so the counts reported below exist to
|
|
43
|
+
* make a mask that swallowed the file visible rather than silent.
|
|
44
|
+
*/
|
|
45
|
+
function stringMask(src) {
|
|
46
|
+
const mask = new Uint8Array(src.length);
|
|
47
|
+
let i = 0;
|
|
48
|
+
let quote = null;
|
|
49
|
+
let comment = null;
|
|
50
|
+
while (i < src.length) {
|
|
51
|
+
const c = src[i];
|
|
52
|
+
const d = src[i + 1];
|
|
53
|
+
if (comment === "line") {
|
|
54
|
+
if (c === "\n") comment = null;
|
|
55
|
+
else mask[i] = 1;
|
|
56
|
+
i++;
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
if (comment === "block") {
|
|
60
|
+
mask[i] = 1;
|
|
61
|
+
if (c === "*" && d === "/") {
|
|
62
|
+
mask[i + 1] = 1;
|
|
63
|
+
comment = null;
|
|
64
|
+
i += 2;
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
i++;
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
if (quote !== null) {
|
|
71
|
+
mask[i] = 1;
|
|
72
|
+
if (c === "\\") {
|
|
73
|
+
if (i + 1 < src.length) mask[i + 1] = 1;
|
|
74
|
+
i += 2;
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
if (c === quote) quote = null;
|
|
78
|
+
i++;
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
if (c === "/" && d === "/") { comment = "line"; mask[i] = 1; i++; continue; }
|
|
82
|
+
if (c === "/" && d === "*") { comment = "block"; mask[i] = 1; i++; continue; }
|
|
83
|
+
// A regex literal holding a quote — `/["']/` — opens a string that never
|
|
84
|
+
// closes, and everything after it is misread. This file's own subject
|
|
85
|
+
// matter is regexes full of quote characters, so the mask desynchronised
|
|
86
|
+
// and a `require("./calc")` written inside a comment was reported as a
|
|
87
|
+
// missing module. Deciding regex-versus-division on the previous token is
|
|
88
|
+
// the same approximation the diff scanner already makes.
|
|
89
|
+
if (c === "/") {
|
|
90
|
+
let k = i - 1;
|
|
91
|
+
while (k >= 0 && /\s/.test(src[k])) k--;
|
|
92
|
+
const prev = k >= 0 ? src[k] : "";
|
|
93
|
+
if (prev === "" || "(,=:[!&|?{};+-*%~^<>".includes(prev)) {
|
|
94
|
+
mask[i] = 1;
|
|
95
|
+
let j = i + 1;
|
|
96
|
+
let cls = false;
|
|
97
|
+
while (j < src.length) {
|
|
98
|
+
const e = src[j];
|
|
99
|
+
mask[j] = 1;
|
|
100
|
+
if (e === "\\") { if (j + 1 < src.length) mask[j + 1] = 1; j += 2; continue; }
|
|
101
|
+
if (e === "[") cls = true;
|
|
102
|
+
else if (e === "]") cls = false;
|
|
103
|
+
else if (e === "/" && !cls) { j++; break; }
|
|
104
|
+
else if (e === "\n") break;
|
|
105
|
+
j++;
|
|
106
|
+
}
|
|
107
|
+
i = j;
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
if (c === '"' || c === "'" || c === "`") { quote = c; mask[i] = 1; i++; continue; }
|
|
112
|
+
i++;
|
|
113
|
+
}
|
|
114
|
+
return mask;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Every module specifier in a source file.
|
|
119
|
+
*
|
|
120
|
+
* Deliberately newline-tolerant. A matcher bounded by `[^;\n]*?` cannot see
|
|
121
|
+
* a multi-line named import, which is exactly the import that was missing
|
|
122
|
+
* from the tarball, so the first version of this check certified the broken
|
|
123
|
+
* package as sound. The extraction cases in the policy contract exist to
|
|
124
|
+
* keep that from being a private mistake twice.
|
|
125
|
+
*/
|
|
126
|
+
export function extractSpecifiers(src, { skipQuoted = true } = {}) {
|
|
127
|
+
const mask = skipQuoted ? stringMask(src) : null;
|
|
128
|
+
const found = new Set();
|
|
129
|
+
const collect = (re) => {
|
|
130
|
+
for (const m of src.matchAll(re)) {
|
|
131
|
+
// What decides is the token immediately before the specifier — the
|
|
132
|
+
// `from`, or the `(` of a call. In a real import it is code; in a
|
|
133
|
+
// fixture it is the middle of a string. Testing the *keyword* instead
|
|
134
|
+
// was not enough: `export const CASES = [` at the top of a file
|
|
135
|
+
// matched lazily forward into the first `from "…"` inside an example,
|
|
136
|
+
// so a genuine keyword lent its authority to quoted text.
|
|
137
|
+
if (mask) {
|
|
138
|
+
let j = m.indices[1][0] - 2;
|
|
139
|
+
while (j >= 0 && /\s/.test(src[j])) j--;
|
|
140
|
+
if (j >= 0 && mask[j]) continue;
|
|
141
|
+
}
|
|
142
|
+
found.add(m[1]);
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
collect(/(?:^|[\s;}])(?:import|export)\s[\s\S]{0,500}?from\s*["']([^"']+)["']/gd);
|
|
146
|
+
collect(/(?:^|[\s;}])import\s*["']([^"']+)["']/gd);
|
|
147
|
+
collect(/\bimport\s*\(\s*["']([^"']+)["']/gd);
|
|
148
|
+
collect(/\brequire\s*\(\s*["']([^"']+)["']/gd);
|
|
149
|
+
return [...found];
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Run every integrity check and return the result.
|
|
154
|
+
*
|
|
155
|
+
* Exported as a function rather than run on import: a module that checks the
|
|
156
|
+
* package must not exit the process of anything that merely imports it.
|
|
157
|
+
*/
|
|
158
|
+
export function checkPackageIntegrity() {
|
|
159
|
+
const failures = [];
|
|
160
|
+
const checks = [];
|
|
161
|
+
const add = (name, ok, detail) => {
|
|
162
|
+
checks.push({ name, ok, detail });
|
|
163
|
+
if (!ok) failures.push(`${name}: ${detail}`);
|
|
164
|
+
};
|
|
165
|
+
|
|
166
|
+
// --- 1. The extractor must be able to see what it claims to look for -------
|
|
167
|
+
{
|
|
168
|
+
const wrong = [];
|
|
169
|
+
for (const c of IMPORT_EXTRACTION_CASES) {
|
|
170
|
+
const got = extractSpecifiers(c.src).sort();
|
|
171
|
+
const want = [...c.expect].sort();
|
|
172
|
+
if (JSON.stringify(got) !== JSON.stringify(want)) {
|
|
173
|
+
wrong.push(`${c.id}: found ${JSON.stringify(got)}, contract says ${JSON.stringify(want)}`);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
add("extractor: every import form is visible", wrong.length === 0, wrong.length ? wrong.join("; ") : `${IMPORT_EXTRACTION_CASES.length} forms found`);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// --- 2. Ask the packer what it would actually ship -------------------------
|
|
180
|
+
let shipped = new Set();
|
|
181
|
+
try {
|
|
182
|
+
const out = execFileSync("npm", ["pack", "--dry-run", "--json"], { cwd: root, encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"] });
|
|
183
|
+
shipped = new Set(JSON.parse(out)[0].files.map((f) => f.path.split(/[\\/]/).join("/")));
|
|
184
|
+
} catch (err) {
|
|
185
|
+
add("packer: `npm pack --dry-run` answers", false, err.message);
|
|
186
|
+
}
|
|
187
|
+
add("packer: the tarball is not empty", shipped.size > 0, `${shipped.size} files`);
|
|
188
|
+
|
|
189
|
+
// --- 3. Resolve the import graph inside the tarball ------------------------
|
|
190
|
+
{
|
|
191
|
+
const broken = [];
|
|
192
|
+
let resolved = 0;
|
|
193
|
+
let scanned = 0;
|
|
194
|
+
|
|
195
|
+
for (const file of shipped) {
|
|
196
|
+
if (!/\.(mjs|cjs|js)$/.test(file)) continue;
|
|
197
|
+
let src;
|
|
198
|
+
try {
|
|
199
|
+
src = readFileSync(resolve(root, file), "utf-8");
|
|
200
|
+
} catch {
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
scanned++;
|
|
204
|
+
for (const spec of extractSpecifiers(src)) {
|
|
205
|
+
if (!spec.startsWith(".")) continue; // bare and node: specifiers are not ours to resolve
|
|
206
|
+
resolved++;
|
|
207
|
+
const target = relative(root, resolve(dirname(resolve(root, file)), spec)).split(sep).join("/");
|
|
208
|
+
if (!shipped.has(target)) {
|
|
209
|
+
broken.push(`${file} imports ${spec} — ${target} is not in the tarball (on disk: ${existsSync(resolve(root, target)) ? "yes" : "no"})`);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
add(
|
|
215
|
+
"tarball: every relative import resolves",
|
|
216
|
+
broken.length === 0,
|
|
217
|
+
broken.length ? broken.join("; ") : `${resolved} relative imports across ${scanned} shipped modules`
|
|
218
|
+
);
|
|
219
|
+
// A resolver that resolved nothing would report the same clean line.
|
|
220
|
+
add("tarball: the graph was actually walked", resolved > 0 && scanned > 0, `${scanned} modules, ${resolved} relative imports`);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// --- 4. Every advertised entry point has to be in the box ------------------
|
|
224
|
+
{
|
|
225
|
+
const pkg = JSON.parse(readFileSync(resolve(root, "package.json"), "utf-8"));
|
|
226
|
+
const entries = [...Object.values(pkg.bin || {}), pkg.main, pkg.module].filter(Boolean);
|
|
227
|
+
const missing = entries.map((e) => e.replace(/^\.\//, "")).filter((e) => !shipped.has(e));
|
|
228
|
+
add("entry points: every bin and main ships", missing.length === 0, missing.length ? missing.join(", ") : `${entries.length} entry points present`);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
return { ok: failures.length === 0, checks, failures };
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
const isMain = process.argv[1] && process.argv[1].endsWith("package-integrity-check.mjs");
|
|
235
|
+
if (isMain) {
|
|
236
|
+
const { ok, checks: rows, failures: bad } = checkPackageIntegrity();
|
|
237
|
+
if (process.argv.includes("--json")) {
|
|
238
|
+
console.log(JSON.stringify({ ok, checks: rows }, null, 2));
|
|
239
|
+
} else {
|
|
240
|
+
console.log("\n📦 Package Integrity Check (what we actually publish)");
|
|
241
|
+
console.log("-------------------------------------------------------");
|
|
242
|
+
for (const c of rows) console.log(` ${c.ok ? "✅" : "❌"} ${c.name.padEnd(46)} ${c.detail}`);
|
|
243
|
+
console.log("-------------------------------------------------------");
|
|
244
|
+
console.log(
|
|
245
|
+
ok
|
|
246
|
+
? "✅ The published package is self-contained.\n"
|
|
247
|
+
: `\n❌ ${bad.length} problem(s). The tarball is not what the source tree looks like.\n`
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
process.exit(ok ? 0 : 1);
|
|
251
|
+
}
|
package/scripts/release.mjs
CHANGED
|
@@ -58,6 +58,22 @@ try {
|
|
|
58
58
|
process.exit(1);
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
+
// Step 1a proved the guards work here. Here is not where users run them.
|
|
62
|
+
//
|
|
63
|
+
// Every signal so far was measured in the source tree, where every file
|
|
64
|
+
// exists by construction. What ships is a tarball built from the `files`
|
|
65
|
+
// list, and v0.63.0 published the guard-reach check without the policy
|
|
66
|
+
// contract it imports — green suite, green matrix, green release, and a
|
|
67
|
+
// module that threw ERR_MODULE_NOT_FOUND the moment anyone installed it.
|
|
68
|
+
console.log("1a-2. Verifying the package we would publish is self-contained...");
|
|
69
|
+
try {
|
|
70
|
+
execSync("node scripts/package-integrity-check.mjs", { cwd: root, stdio: "inherit" });
|
|
71
|
+
console.log("");
|
|
72
|
+
} catch (_) {
|
|
73
|
+
console.error("\n❌ Release Aborted: the tarball is not what the source tree looks like.");
|
|
74
|
+
process.exit(1);
|
|
75
|
+
}
|
|
76
|
+
|
|
61
77
|
// 1b. Documentation / version consistency gate (blocking).
|
|
62
78
|
console.log("1b. Verifying documentation is in sync with package.json & test suite...");
|
|
63
79
|
{
|
package/src/assertions.mjs
CHANGED
|
@@ -561,12 +561,28 @@ export function assertTestIntegrity(config = {}, root = process.cwd()) {
|
|
|
561
561
|
const res = checkTestTampering(diffStr, config);
|
|
562
562
|
const diagnostics = (res.violations || []).map((v) => v.reason);
|
|
563
563
|
|
|
564
|
+
// A clean result from a guard that could not read the dialect is not a
|
|
565
|
+
// clean result, and it must not be reported as one. This does not fail the
|
|
566
|
+
// check — an unlisted assertion library is the user's normal, not their
|
|
567
|
+
// fault — but they get to know the guard is not covering them.
|
|
568
|
+
for (const u of res.unreadable || []) {
|
|
569
|
+
diagnostics.push(
|
|
570
|
+
`Test Tamper Guard: ${u.count} assertion-shaped line(s) in ${u.file} matched no known dialect, ` +
|
|
571
|
+
`so this file was not checked for tampering (e.g. ${JSON.stringify(u.samples[0])}). ` +
|
|
572
|
+
`The guard reports what it could not read rather than passing silently.`
|
|
573
|
+
);
|
|
574
|
+
}
|
|
575
|
+
|
|
564
576
|
return {
|
|
565
577
|
ok: res.ok,
|
|
566
578
|
violations: res.violations || [],
|
|
567
579
|
diagnostics,
|
|
568
580
|
metrics: {
|
|
569
581
|
violationCount: res.violations?.length || 0,
|
|
582
|
+
assertionsSeen: res.assertionsSeen ?? 0,
|
|
583
|
+
filesSeen: res.filesSeen ?? 0,
|
|
584
|
+
unreadableFiles: (res.unreadable || []).length,
|
|
585
|
+
status: res.status,
|
|
570
586
|
},
|
|
571
587
|
};
|
|
572
588
|
}
|
|
@@ -0,0 +1,481 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The policy this kit claims to enforce, written by hand.
|
|
3
|
+
*
|
|
4
|
+
* Every entry here is derived from what the tool *advertises* — the stacks
|
|
5
|
+
* `detectStack` declares, the layouts each ecosystem actually uses — and never
|
|
6
|
+
* from the regexes, path lists or registries that implement the checks. That
|
|
7
|
+
* separation is the whole point: a contract generated from the implementation
|
|
8
|
+
* makes the implementation its own oracle, and an implementation that is its
|
|
9
|
+
* own oracle cannot be wrong.
|
|
10
|
+
*
|
|
11
|
+
* This is what the substring bug in the file classifier cost. `isTestFile`
|
|
12
|
+
* matched `/test/`, which does not occur in `tests/test_calc.py`, so the entire
|
|
13
|
+
* tamper guard was off for the standard pytest, Rust and RSpec layouts — and
|
|
14
|
+
* every mechanism that should have caught it (a large suite, a doc-sync gate,
|
|
15
|
+
* a nine-way CI matrix, two cold reviews, a blocking release) was sampling the
|
|
16
|
+
* same distribution the implementation was written from. Nine runs of
|
|
17
|
+
* `test/foo.test.js` do not explore `tests/test_calc.py`.
|
|
18
|
+
*
|
|
19
|
+
* Adding a stack to `detectStack` is not finished until it has a row here.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Paths the policy says are test files, and near-misses it says are not.
|
|
24
|
+
*
|
|
25
|
+
* The near-misses matter as much as the hits: a predicate that answers "yes"
|
|
26
|
+
* to everything also has no denominator.
|
|
27
|
+
*/
|
|
28
|
+
export const TEST_PATH_CASES = [
|
|
29
|
+
// Node / JavaScript
|
|
30
|
+
{ path: "test/calc.test.js", expected: true, why: "node, conventional" },
|
|
31
|
+
{ path: "src/calc.spec.ts", expected: true, why: "co-located spec" },
|
|
32
|
+
{ path: "src/__tests__/calc.js", expected: true, why: "jest convention" },
|
|
33
|
+
// Python
|
|
34
|
+
{ path: "tests/test_calc.py", expected: true, why: "pytest, repository root — the reported gap" },
|
|
35
|
+
{ path: "test/test_calc.py", expected: true, why: "pytest, singular directory" },
|
|
36
|
+
{ path: "backend/tests/test_api.py", expected: true, why: "pytest, nested" },
|
|
37
|
+
// Go
|
|
38
|
+
{ path: "internal/calc/calc_test.go", expected: true, why: "go, co-located with the code" },
|
|
39
|
+
{ path: "cmd/api/main_test.go", expected: true, why: "go, command package" },
|
|
40
|
+
// Rust
|
|
41
|
+
{ path: "tests/integration.rs", expected: true, why: "rust integration tests" },
|
|
42
|
+
// Ruby
|
|
43
|
+
{ path: "spec/models/user_spec.rb", expected: true, why: "rspec" },
|
|
44
|
+
{ path: "test/user_test.rb", expected: true, why: "minitest" },
|
|
45
|
+
// JVM / .NET / PHP
|
|
46
|
+
{ path: "src/test/java/com/x/CalcTest.java", expected: true, why: "maven layout" },
|
|
47
|
+
{ path: "tests/Unit/CalcTest.php", expected: true, why: "phpunit" },
|
|
48
|
+
// Solidity
|
|
49
|
+
{ path: "test/Token.t.sol", expected: true, why: "foundry" },
|
|
50
|
+
// Monorepo position
|
|
51
|
+
{ path: "packages/api/test/handler.test.js", expected: true, why: "monorepo package" },
|
|
52
|
+
|
|
53
|
+
// Near-misses: segment match, not substring
|
|
54
|
+
{ path: "latest/build.js", expected: false, why: "`latest` is not `test`" },
|
|
55
|
+
{ path: "contest/result.js", expected: false, why: "`contest` is not `test`" },
|
|
56
|
+
{ path: "lib/myspec/render.js", expected: false, why: "`myspec` is not `spec`" },
|
|
57
|
+
{ path: "src/attestation.js", expected: false, why: "contains 'test', is not one" },
|
|
58
|
+
{ path: "src/index.js", expected: false, why: "ordinary source" },
|
|
59
|
+
{ path: "docs/testing-guide.md", expected: false, why: "documentation about testing" },
|
|
60
|
+
];
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Known-bad diffs, one per guard rule, each with the finding it MUST produce.
|
|
64
|
+
*
|
|
65
|
+
* A canary that comes back clean is not a passing test — it is proof that the
|
|
66
|
+
* rule it names has stopped being reachable. `layout` is deliberately varied
|
|
67
|
+
* across ecosystems so that a canary set cannot be green by only ever handing
|
|
68
|
+
* the guard the layout the implementation was written from.
|
|
69
|
+
*/
|
|
70
|
+
export const TAMPER_CANARIES = [
|
|
71
|
+
{
|
|
72
|
+
id: "skip-injection/node",
|
|
73
|
+
file: "test/calc.test.js",
|
|
74
|
+
removed: [],
|
|
75
|
+
added: ['it.skip("adds", () => { assert.equal(add(1,2), 3); });'],
|
|
76
|
+
expect: "TEST_SKIP_INJECTION",
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
id: "vacuous/node",
|
|
80
|
+
file: "test/calc.test.js",
|
|
81
|
+
removed: ["assert.equal(add(1, 2), 3);"],
|
|
82
|
+
added: ["assert.ok(true);"],
|
|
83
|
+
expect: "VACUOUS_ASSERTION",
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
id: "commented/node",
|
|
87
|
+
file: "test/calc.test.js",
|
|
88
|
+
removed: ["assert.equal(add(1, 2), 3);"],
|
|
89
|
+
added: ["// assert.equal(add(1, 2), 3);"],
|
|
90
|
+
expect: "COMMENTED_ASSERTION",
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
id: "removal/node",
|
|
94
|
+
file: "test/calc.test.js",
|
|
95
|
+
removed: ["assert.equal(add(1, 2), 3);", "assert.equal(add(2, 2), 4);"],
|
|
96
|
+
added: [],
|
|
97
|
+
expect: "ASSERTION_REMOVAL",
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
id: "weakening/node",
|
|
101
|
+
file: "test/calc.test.js",
|
|
102
|
+
removed: ["assert.strictEqual(add(1, 2), 3);"],
|
|
103
|
+
added: ["assert.ok(add(1, 2) !== undefined);"],
|
|
104
|
+
expect: "ASSERTION_WEAKENED",
|
|
105
|
+
},
|
|
106
|
+
{
|
|
107
|
+
id: "expectation/node",
|
|
108
|
+
file: "test/calc.test.js",
|
|
109
|
+
removed: ["assert.equal(add(1, 2), 3);"],
|
|
110
|
+
added: ["assert.equal(add(1, 2), -1);"],
|
|
111
|
+
expect: "ASSERTION_EXPECTATION_CHANGED",
|
|
112
|
+
},
|
|
113
|
+
// The same attacks, in the layouts the classifier used to be blind to.
|
|
114
|
+
// These are the canaries that were silent when the classifier matched a
|
|
115
|
+
// substring instead of a path segment.
|
|
116
|
+
{
|
|
117
|
+
id: "expectation/pytest-root",
|
|
118
|
+
file: "tests/test_calc.py",
|
|
119
|
+
removed: [" self.assertEqual(add(1, 2), 3)"],
|
|
120
|
+
added: [" self.assertEqual(add(1, 2), -1)"],
|
|
121
|
+
expect: "ASSERTION_EXPECTATION_CHANGED",
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
id: "skip-injection/pytest-root",
|
|
125
|
+
file: "tests/test_calc.py",
|
|
126
|
+
removed: [],
|
|
127
|
+
added: ["@pytest.mark.skip", "def test_add():"],
|
|
128
|
+
expect: "TEST_SKIP_INJECTION",
|
|
129
|
+
},
|
|
130
|
+
{
|
|
131
|
+
id: "expectation/rust",
|
|
132
|
+
file: "tests/integration.rs",
|
|
133
|
+
removed: [" assert_eq!(add(1, 2), 3);"],
|
|
134
|
+
added: [" assert_eq!(add(1, 2), -1);"],
|
|
135
|
+
expect: "ASSERTION_EXPECTATION_CHANGED",
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
id: "expectation/go-colocated",
|
|
139
|
+
file: "internal/calc/calc_test.go",
|
|
140
|
+
removed: ['\t\tt.Errorf("got %d want %d", got, 3)'],
|
|
141
|
+
added: ['\t\tt.Errorf("got %d want %d", got, 999)'],
|
|
142
|
+
expect: "ASSERTION_EXPECTATION_CHANGED",
|
|
143
|
+
},
|
|
144
|
+
{
|
|
145
|
+
id: "removal/monorepo",
|
|
146
|
+
file: "packages/api/test/handler.test.js",
|
|
147
|
+
removed: ["expect(handler(req)).toBe(200);", "expect(handler(bad)).toBe(400);"],
|
|
148
|
+
added: [],
|
|
149
|
+
expect: "ASSERTION_REMOVAL",
|
|
150
|
+
},
|
|
151
|
+
// Dialects the guard was measured silent on. `assertEqual` matched only
|
|
152
|
+
// because the pattern's optional dot and case-insensitive flag happened to
|
|
153
|
+
// line up; `assertEquals` — one letter longer — did not, and neither did
|
|
154
|
+
// RSpec's `.to eq(`, PHPUnit's `$this->assertSame`, Minitest's
|
|
155
|
+
// `assert_equal` or XCTest's `XCTAssertEqual`. Every one of them returned
|
|
156
|
+
// PASS with a non-zero denominator, which is the exact shape this file
|
|
157
|
+
// exists to reject, one level down inside the mechanism built to catch it.
|
|
158
|
+
{
|
|
159
|
+
id: "expectation/junit",
|
|
160
|
+
file: "src/test/java/com/x/CalcTest.java",
|
|
161
|
+
removed: [" assertEquals(3, calc.add(1, 2));"],
|
|
162
|
+
added: [" assertEquals(-1, calc.add(1, 2));"],
|
|
163
|
+
expect: "ASSERTION_EXPECTATION_CHANGED",
|
|
164
|
+
},
|
|
165
|
+
{
|
|
166
|
+
id: "expectation/rspec",
|
|
167
|
+
file: "spec/models/user_spec.rb",
|
|
168
|
+
removed: [" expect(user.age).to eq(30)"],
|
|
169
|
+
added: [" expect(user.age).to eq(-1)"],
|
|
170
|
+
expect: "ASSERTION_EXPECTATION_CHANGED",
|
|
171
|
+
},
|
|
172
|
+
{
|
|
173
|
+
id: "expectation/phpunit",
|
|
174
|
+
file: "tests/Unit/CalcTest.php",
|
|
175
|
+
removed: [" $this->assertSame(3, $c->add(1, 2));"],
|
|
176
|
+
added: [" $this->assertSame(-1, $c->add(1, 2));"],
|
|
177
|
+
expect: "ASSERTION_EXPECTATION_CHANGED",
|
|
178
|
+
},
|
|
179
|
+
{
|
|
180
|
+
id: "expectation/minitest",
|
|
181
|
+
file: "test/user_test.rb",
|
|
182
|
+
removed: [" assert_equal(3, add(1, 2))"],
|
|
183
|
+
added: [" assert_equal(-1, add(1, 2))"],
|
|
184
|
+
expect: "ASSERTION_EXPECTATION_CHANGED",
|
|
185
|
+
},
|
|
186
|
+
{
|
|
187
|
+
id: "expectation/xctest",
|
|
188
|
+
file: "Tests/CalcTests/CalcTests.swift",
|
|
189
|
+
removed: [" XCTAssertEqual(add(1, 2), 3)"],
|
|
190
|
+
added: [" XCTAssertEqual(add(1, 2), -1)"],
|
|
191
|
+
expect: "ASSERTION_EXPECTATION_CHANGED",
|
|
192
|
+
},
|
|
193
|
+
{
|
|
194
|
+
id: "weakening/junit",
|
|
195
|
+
file: "src/test/java/com/x/CalcTest.java",
|
|
196
|
+
removed: [" assertEquals(3, calc.add(1, 2));"],
|
|
197
|
+
added: [" assertTrue(calc.add(1, 2) != null);"],
|
|
198
|
+
expect: "ASSERTION_WEAKENED",
|
|
199
|
+
},
|
|
200
|
+
{
|
|
201
|
+
id: "removal/phpunit",
|
|
202
|
+
file: "tests/Unit/CalcTest.php",
|
|
203
|
+
removed: [" $this->assertSame(3, $c->add(1, 2));", " $this->assertSame(4, $c->add(2, 2));"],
|
|
204
|
+
added: [],
|
|
205
|
+
expect: "ASSERTION_REMOVAL",
|
|
206
|
+
},
|
|
207
|
+
{
|
|
208
|
+
id: "vacuous/xctest",
|
|
209
|
+
file: "Tests/CalcTests/CalcTests.swift",
|
|
210
|
+
removed: [" XCTAssertEqual(add(1, 2), 3)"],
|
|
211
|
+
added: [" XCTAssertTrue(true)"],
|
|
212
|
+
expect: "VACUOUS_ASSERTION",
|
|
213
|
+
},
|
|
214
|
+
// Skip injection is the same blindness in the other five ecosystems: a
|
|
215
|
+
// suite that never runs cannot fail, and `@Disabled` is as effective as
|
|
216
|
+
// `it.skip` at making that happen.
|
|
217
|
+
{
|
|
218
|
+
id: "skip-injection/junit",
|
|
219
|
+
file: "src/test/java/com/x/CalcTest.java",
|
|
220
|
+
removed: [],
|
|
221
|
+
added: [" @Disabled(\"flaky\")", " void addsTwoNumbers() {"],
|
|
222
|
+
expect: "TEST_SKIP_INJECTION",
|
|
223
|
+
},
|
|
224
|
+
{
|
|
225
|
+
id: "skip-injection/rspec",
|
|
226
|
+
file: "spec/models/user_spec.rb",
|
|
227
|
+
removed: [],
|
|
228
|
+
added: [" xit \"computes the age\" do"],
|
|
229
|
+
expect: "TEST_SKIP_INJECTION",
|
|
230
|
+
},
|
|
231
|
+
{
|
|
232
|
+
id: "skip-injection/phpunit",
|
|
233
|
+
file: "tests/Unit/CalcTest.php",
|
|
234
|
+
removed: [],
|
|
235
|
+
added: [" $this->markTestSkipped(\"later\");"],
|
|
236
|
+
expect: "TEST_SKIP_INJECTION",
|
|
237
|
+
},
|
|
238
|
+
{
|
|
239
|
+
id: "skip-injection/xctest",
|
|
240
|
+
file: "Tests/CalcTests/CalcTests.swift",
|
|
241
|
+
removed: [],
|
|
242
|
+
added: [" throw XCTSkip(\"not now\")"],
|
|
243
|
+
expect: "TEST_SKIP_INJECTION",
|
|
244
|
+
},
|
|
245
|
+
];
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Mutants of the applicability predicate.
|
|
250
|
+
*
|
|
251
|
+
* Each one must kill at least one canary. A mutant that survives means no
|
|
252
|
+
* canary ever required the guard to *activate* — the suite would stay green if
|
|
253
|
+
* the guard silently stopped looking, which is precisely the defect.
|
|
254
|
+
*
|
|
255
|
+
* These are hand-written rather than generated: the original bug was not an
|
|
256
|
+
* untested branch, it was a branch nobody wrote, and no mutation operator
|
|
257
|
+
* invents the case the code never handled.
|
|
258
|
+
*/
|
|
259
|
+
export const PREDICATE_MUTANTS = [
|
|
260
|
+
{ id: "alwaysFalse", fn: () => false, why: "the guard looks at nothing" },
|
|
261
|
+
{ id: "rootBlind", fn: (p) => String(p).includes("/test/"), why: "the original substring bug" },
|
|
262
|
+
{ id: "nodeOnly", fn: (p) => /\.(test|spec)\.[jt]sx?$/.test(String(p)), why: "only the layout the code was written from" },
|
|
263
|
+
{ id: "caseSensitive", fn: (p) => String(p) === String(p).toLowerCase() && /(^|\/)tests?\//.test(String(p)), why: "case and separator drift" },
|
|
264
|
+
];
|
|
265
|
+
|
|
266
|
+
/** Runner outputs that state zero collected tests, per ecosystem. */
|
|
267
|
+
export const EMPTY_RUN_CANARIES = [
|
|
268
|
+
{ id: "pytest", output: "collected 0 items\n\nno tests ran in 0.01s" },
|
|
269
|
+
{ id: "jest", output: "No tests found, exiting with code 0" },
|
|
270
|
+
{ id: "vitest", output: "No test files found, exiting with code 0" },
|
|
271
|
+
{ id: "cargo", output: "running 0 tests\ntest result: ok. 0 passed" },
|
|
272
|
+
{ id: "mocha", output: " 0 passing (1ms)" },
|
|
273
|
+
{ id: "go", output: "? example.com/app\t[no test files]" },
|
|
274
|
+
{ id: "surefire", output: "Tests run: 0, Failures: 0, Errors: 0, Skipped: 0" },
|
|
275
|
+
{ id: "gradle", output: "> Task :test NO-SOURCE" },
|
|
276
|
+
{ id: "phpunit", output: "No tests executed!" },
|
|
277
|
+
{ id: "rspec", output: "0 examples, 0 failures" },
|
|
278
|
+
{ id: "dotnet", output: "Total tests: 0" },
|
|
279
|
+
{ id: "xctest", output: "Executed 0 tests" },
|
|
280
|
+
{ id: "ctest", output: "No tests were found!!!" },
|
|
281
|
+
];
|
|
282
|
+
|
|
283
|
+
/** Paths the policy says an agent must never modify without an override. */
|
|
284
|
+
export const SCOPE_CANARIES = [
|
|
285
|
+
{ path: ".github/workflows/ci.yml", rule: "deny", why: "runs with repo credentials" },
|
|
286
|
+
{ path: ".gitlab-ci.yml", rule: "deny", why: "same, another forge" },
|
|
287
|
+
{ path: "Jenkinsfile", rule: "deny", why: "same, another forge" },
|
|
288
|
+
{ path: ".envrc", rule: "deny", why: "direnv executes it on cd" },
|
|
289
|
+
{ path: "package-lock.json", rule: "protect", why: "decides which code installs" },
|
|
290
|
+
{ path: "Cargo.lock", rule: "protect", why: "same, another ecosystem" },
|
|
291
|
+
{ path: "conftest.py", rule: "protect", why: "runs before every pytest collection" },
|
|
292
|
+
{ path: "jest.config.js", rule: "protect", why: "decides which tests run" },
|
|
293
|
+
{ path: "CODEOWNERS", rule: "protect", why: "decides who must approve" },
|
|
294
|
+
];
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Edits that must produce no finding at all.
|
|
298
|
+
*
|
|
299
|
+
* A guard that answers "yes" to everything has no more discrimination than
|
|
300
|
+
* one that answers "no" to everything, and it is worse in practice: the
|
|
301
|
+
* operator learns to pass the override without reading it, and the day it
|
|
302
|
+
* reports something real, nobody looks. Every entry here is an edit an
|
|
303
|
+
* honest agent makes constantly.
|
|
304
|
+
*
|
|
305
|
+
* The first two are not hypothetical. `//` begins with a division sign, so a
|
|
306
|
+
* comment line matched the operator-continuation test and folded itself into
|
|
307
|
+
* the assertion above it — which meant *adding* an assertion next to a
|
|
308
|
+
* comment was reported as rewriting an expectation. Python was immune,
|
|
309
|
+
* because `#` is not an operator, so the fixtures this project was written
|
|
310
|
+
* from never showed it.
|
|
311
|
+
*/
|
|
312
|
+
export const INNOCENT_EDITS = [
|
|
313
|
+
{
|
|
314
|
+
id: "add-assertion-beside-comment/node",
|
|
315
|
+
file: "test/calc.test.js",
|
|
316
|
+
context: "// arithmetic",
|
|
317
|
+
removed: [" assert.equal(add(1, 2), 3);"],
|
|
318
|
+
added: [" assert.equal(add(1, 2), 3);", " assert.equal(add(2, 2), 4);"],
|
|
319
|
+
why: "adding a test is the behaviour the gate exists to encourage",
|
|
320
|
+
},
|
|
321
|
+
{
|
|
322
|
+
id: "add-assertion-beside-comment/rust",
|
|
323
|
+
file: "tests/integration.rs",
|
|
324
|
+
context: "// arithmetic",
|
|
325
|
+
removed: [" assert!(add(1, 2) == 3);"],
|
|
326
|
+
added: [" assert!(add(1, 2) == 3);", " assert!(add(2, 2) == 4);"],
|
|
327
|
+
why: "same edit, same comment syntax, another ecosystem",
|
|
328
|
+
},
|
|
329
|
+
{
|
|
330
|
+
id: "reorder-assertions/pytest",
|
|
331
|
+
file: "tests/test_calc.py",
|
|
332
|
+
context: "# arithmetic",
|
|
333
|
+
removed: [" assert a() == 1", " assert b() == 2"],
|
|
334
|
+
added: [" assert b() == 2", " assert a() == 1"],
|
|
335
|
+
why: "moving an assertion changes nothing it checks",
|
|
336
|
+
},
|
|
337
|
+
{
|
|
338
|
+
id: "reindent/pytest",
|
|
339
|
+
file: "tests/test_calc.py",
|
|
340
|
+
context: "# arithmetic",
|
|
341
|
+
removed: [" assert add(1, 2) == 3"],
|
|
342
|
+
added: [" assert add(1, 2) == 3"],
|
|
343
|
+
why: "a formatter run must not read as tampering",
|
|
344
|
+
},
|
|
345
|
+
{
|
|
346
|
+
id: "reword-message/rspec",
|
|
347
|
+
file: "spec/models/user_spec.rb",
|
|
348
|
+
context: "# age",
|
|
349
|
+
removed: [' expect(u.age).to eq(30), "wrong"'],
|
|
350
|
+
added: [' expect(u.age).to eq(30), "unexpected age"'],
|
|
351
|
+
why: "RSpec writes the message outside the call, where an argument check cannot see it",
|
|
352
|
+
},
|
|
353
|
+
{
|
|
354
|
+
id: "reword-message/minitest",
|
|
355
|
+
file: "test/user_test.rb",
|
|
356
|
+
context: "# age",
|
|
357
|
+
removed: [' assert_equal 3, add(1, 2), "wrong"'],
|
|
358
|
+
added: [' assert_equal 3, add(1, 2), "unexpected sum"'],
|
|
359
|
+
why: "same, with the message in argument position and no parentheses",
|
|
360
|
+
},
|
|
361
|
+
{
|
|
362
|
+
id: "reword-message/node",
|
|
363
|
+
file: "test/calc.test.js",
|
|
364
|
+
context: "// arithmetic",
|
|
365
|
+
removed: [' assert.equal(add(1, 2), 3, "wrong");'],
|
|
366
|
+
added: [' assert.equal(add(1, 2), 3, "unexpected sum");'],
|
|
367
|
+
why: "rewording a failure message says nothing about what is checked",
|
|
368
|
+
},
|
|
369
|
+
{
|
|
370
|
+
id: "rename-test/junit",
|
|
371
|
+
file: "src/test/java/com/x/CalcTest.java",
|
|
372
|
+
context: "// arithmetic",
|
|
373
|
+
removed: [" void addsNumbers() {"],
|
|
374
|
+
added: [" void addsTwoNumbers() {"],
|
|
375
|
+
why: "a test name is not an expectation",
|
|
376
|
+
},
|
|
377
|
+
{
|
|
378
|
+
id: "change-import/node",
|
|
379
|
+
file: "test/calc.test.js",
|
|
380
|
+
context: "// setup",
|
|
381
|
+
removed: ['const calc = require("./calc");'],
|
|
382
|
+
added: ['const calc = require("../src/calc");'],
|
|
383
|
+
why: "`require(` is an import, not a claim — the loose net must not read it as one",
|
|
384
|
+
},
|
|
385
|
+
{
|
|
386
|
+
id: "rename-helper/node",
|
|
387
|
+
file: "test/calc.test.js",
|
|
388
|
+
context: "// setup",
|
|
389
|
+
removed: [" const shouldRetry = false;"],
|
|
390
|
+
added: [" const shouldRetry = true;"],
|
|
391
|
+
why: "`shouldRetry` and `expected` are identifiers; reading them as assertions makes the dialect warning worthless",
|
|
392
|
+
},
|
|
393
|
+
];
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* Dialects the guard genuinely cannot parse, which it must say out loud.
|
|
397
|
+
*
|
|
398
|
+
* This is the case the whole denominator exists for. A JUnit diff used to
|
|
399
|
+
* return `PASS` with `inputsSeen: 1` while not one assertion in it had been
|
|
400
|
+
* recognised — a verdict indistinguishable from a clean Node suite. Coverage
|
|
401
|
+
* will always end somewhere; what must never happen again is that the edge
|
|
402
|
+
* is silent.
|
|
403
|
+
*/
|
|
404
|
+
export const UNREADABLE_DIALECTS = [
|
|
405
|
+
{
|
|
406
|
+
id: "hspec",
|
|
407
|
+
file: "tests/CalcSpec.hs",
|
|
408
|
+
context: "-- arithmetic",
|
|
409
|
+
removed: [" calc `shouldBe` 3"],
|
|
410
|
+
added: [" calc `shouldBe` (-1)"],
|
|
411
|
+
why: "an infix assertion with no parentheses anywhere near it",
|
|
412
|
+
},
|
|
413
|
+
{
|
|
414
|
+
id: "googletest",
|
|
415
|
+
file: "tests/calc_test.cc",
|
|
416
|
+
context: "// arithmetic",
|
|
417
|
+
removed: [" EXPECT_EQ(add(1, 2), 3);"],
|
|
418
|
+
added: [" EXPECT_EQ(add(1, 2), -1);"],
|
|
419
|
+
why: "a macro dialect the pattern list does not cover",
|
|
420
|
+
},
|
|
421
|
+
];
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* Import forms the package-integrity extractor must find.
|
|
425
|
+
*
|
|
426
|
+
* The first pass of that check reported "every relative import resolves" on
|
|
427
|
+
* a package whose newest script could not start: its matcher was written as
|
|
428
|
+
* `[^;\n]*?from`, and the import it needed to see spanned several lines. The
|
|
429
|
+
* check was confidently green about a file that threw ERR_MODULE_NOT_FOUND
|
|
430
|
+
* on load — the same failure the tool exists to prevent, committed by the
|
|
431
|
+
* tool's own integrity check.
|
|
432
|
+
*
|
|
433
|
+
* Every entry is a source fragment and the specifiers it must yield.
|
|
434
|
+
*/
|
|
435
|
+
export const IMPORT_EXTRACTION_CASES = [
|
|
436
|
+
{ id: "single-line named", src: 'import { a, b } from "./one.mjs";', expect: ["./one.mjs"] },
|
|
437
|
+
{
|
|
438
|
+
id: "multi-line named",
|
|
439
|
+
src: 'import {\n a,\n b,\n} from "./two.mjs";',
|
|
440
|
+
expect: ["./two.mjs"],
|
|
441
|
+
why: "the form the first version of the check could not see",
|
|
442
|
+
},
|
|
443
|
+
{ id: "default", src: 'import three from "./three.mjs";', expect: ["./three.mjs"] },
|
|
444
|
+
{ id: "namespace", src: 'import * as four from "./four.mjs";', expect: ["./four.mjs"] },
|
|
445
|
+
{ id: "side-effect only", src: 'import "./five.mjs";', expect: ["./five.mjs"] },
|
|
446
|
+
{ id: "re-export", src: 'export { six } from "./six.mjs";', expect: ["./six.mjs"] },
|
|
447
|
+
{ id: "re-export all", src: 'export * from "./seven.mjs";', expect: ["./seven.mjs"] },
|
|
448
|
+
{ id: "dynamic", src: 'const m = await import("./eight.mjs");', expect: ["./eight.mjs"] },
|
|
449
|
+
{ id: "require", src: 'const nine = require("./nine.js");', expect: ["./nine.js"] },
|
|
450
|
+
{ id: "single quotes", src: "import ten from './ten.mjs';", expect: ["./ten.mjs"] },
|
|
451
|
+
{
|
|
452
|
+
id: "bare specifiers are not files",
|
|
453
|
+
src: 'import { readFileSync } from "node:fs";\nimport x from "some-package";',
|
|
454
|
+
expect: ["node:fs", "some-package"],
|
|
455
|
+
why: "found, then ignored by the resolver — never resolved against the tarball",
|
|
456
|
+
},
|
|
457
|
+
];
|
|
458
|
+
|
|
459
|
+
// Cases that exercise the mask rather than the matcher. Appended separately
|
|
460
|
+
// because each one is a fixture *about* fixtures: the check has to tell a
|
|
461
|
+
// module reference from a picture of one.
|
|
462
|
+
IMPORT_EXTRACTION_CASES.push(
|
|
463
|
+
{
|
|
464
|
+
id: "regex holding a quote, then a real import",
|
|
465
|
+
src: 'const q = /["\']/;\nimport x from "./after-regex.mjs";',
|
|
466
|
+
expect: ["./after-regex.mjs"],
|
|
467
|
+
why: "the phantom string this file's own subject matter opens",
|
|
468
|
+
},
|
|
469
|
+
{
|
|
470
|
+
id: "an import quoted inside a string",
|
|
471
|
+
src: 'const example = \'import a from "./not-real.mjs";\';',
|
|
472
|
+
expect: [],
|
|
473
|
+
why: "a picture of an import is not an import",
|
|
474
|
+
},
|
|
475
|
+
{
|
|
476
|
+
id: "an import written in a comment",
|
|
477
|
+
src: '// import a from "./commented.mjs";\nconst x = 1;',
|
|
478
|
+
expect: [],
|
|
479
|
+
why: "same, in the other place examples live",
|
|
480
|
+
}
|
|
481
|
+
);
|
package/src/security.mjs
CHANGED
|
@@ -1155,14 +1155,42 @@ function locateFindingLine(lines, type, file = null) {
|
|
|
1155
1155
|
// (`expect(\n formatInvoice(bill)\n).toBe(…`) still recognises the chain.
|
|
1156
1156
|
// The bound is a guess: an argument list longer than 240 characters is
|
|
1157
1157
|
// rarer than a missed chain.
|
|
1158
|
+
// The dialect list is not decoration. `assertEqual` was recognised only
|
|
1159
|
+
// because `\.?` made the dot optional and the `i` flag let `Equal` match
|
|
1160
|
+
// `equal`; `assertEquals`, one letter longer, fell out of the pattern and
|
|
1161
|
+
// took JUnit, PHPUnit, Minitest, RSpec and XCTest with it. The weak forms
|
|
1162
|
+
// — assertTrue, assertNotNull, XCTAssertTrue — are deliberately absent:
|
|
1163
|
+
// they state no expected value, so their arrival in place of one of these
|
|
1164
|
+
// is a weakening, which is a finding of its own.
|
|
1158
1165
|
const SPECIFIC_ASSERTION = new RegExp(
|
|
1159
1166
|
[
|
|
1160
1167
|
"\\bassert(?:\\.strict)?\\.?(?:strictEqual|deepStrictEqual|deepEqual|notStrictEqual|notDeepStrictEqual|equal|notEqual|match|doesNotMatch|throws|rejects|doesNotThrow)\\s*\\(",
|
|
1161
1168
|
"\\bexpect\\s*\\([\\s\\S]{0,240}?\\)\\s*\\.(?:toBe|toEqual|toStrictEqual|toMatch|toMatchObject|toContain|toHaveBeenCalledWith|toThrow|toHaveLength|toBeCloseTo)\\s*\\(",
|
|
1162
1169
|
"\\bassert\\.(?:equals|deepEquals|include|lengthOf)\\s*\\(",
|
|
1163
1170
|
"assert_eq!|assert_ne!",
|
|
1171
|
+
// The bare comparison form. `assert add(1, 2) == 3` is how pytest is
|
|
1172
|
+
// actually written, and Rust's `assert!(a == b)` and Elixir's
|
|
1173
|
+
// `assert f(x) == 3` follow it; none of them name a comparison
|
|
1174
|
+
// function, so a list of function names could never reach them.
|
|
1175
|
+
// Equality only. `assert!(x != 0)` names no expected value — it is the
|
|
1176
|
+
// weaker claim you arrive at by giving one up, and counting it as
|
|
1177
|
+
// specific would make the downgrade from `assert_eq!(x, 5)` invisible to
|
|
1178
|
+
// the weakening check.
|
|
1179
|
+
"\\bassert\\s+[^\\n]*(?:===|==)(?!=)",
|
|
1180
|
+
"\\bassert!\\s*\\([^\\n]*==(?!=)",
|
|
1164
1181
|
"\\bt\\.(?:Errorf|Fatalf)\\s*\\(",
|
|
1165
1182
|
"\\brequire\\.(?:Equal|NotEqual|Len|Contains|Error|NoError)\\s*\\(",
|
|
1183
|
+
// Python unittest, stated rather than inherited from the optional dot.
|
|
1184
|
+
"\\bassert(?:Equal|NotEqual|AlmostEqual|NotAlmostEqual|Regex|NotRegex|Raises|In|NotIn|Is|IsNot|ListEqual|DictEqual|SetEqual|TupleEqual|CountEqual|Greater|Less|GreaterEqual|LessEqual)\\s*\\(",
|
|
1185
|
+
// JUnit / TestNG / PHPUnit
|
|
1186
|
+
"\\bassert(?:Equals|NotEquals|Same|NotSame|ArrayEquals|IterableEquals|LinesMatch|Count|StringContainsString|StringEqualsFile|InstanceOf|Contains|Throws)\\s*\\(",
|
|
1187
|
+
"\\bassertThat\\s*\\([\\s\\S]{0,240}?\\)\\s*\\.(?:isEqualTo|isSameAs|contains|containsExactly|hasSize|isCloseTo|matches)\\s*\\(",
|
|
1188
|
+
// Minitest
|
|
1189
|
+
"\\b(?:assert|refute)_(?:equal|includes|match|nil|same|in_delta|in_epsilon|raises|empty|operator|predicate)\\b",
|
|
1190
|
+
// RSpec
|
|
1191
|
+
"\\bexpect\\s*\\([\\s\\S]{0,240}?\\)\\s*\\.(?:to|not_to|to_not)\\s+(?:eq|eql|equal|be|be_within|match|include|contain_exactly|match_array|have_attributes|raise_error|start_with|end_with)\\b",
|
|
1192
|
+
// XCTest
|
|
1193
|
+
"\\bXCTAssert(?:Equal|NotEqual|EqualWithAccuracy|Identical|NotIdentical|GreaterThan|LessThan|GreaterThanOrEqual|LessThanOrEqual|ThrowsError|NoThrow)\\s*\\(",
|
|
1166
1194
|
].join("|"),
|
|
1167
1195
|
"i"
|
|
1168
1196
|
);
|
|
@@ -1203,6 +1231,15 @@ const TEST_LANG_BY_EXT = new Map([
|
|
|
1203
1231
|
[".py", "python"], [".pyi", "python"],
|
|
1204
1232
|
[".go", "go"],
|
|
1205
1233
|
[".rs", "rust"],
|
|
1234
|
+
// Approximations, chosen for comment and continuation syntax rather than
|
|
1235
|
+
// for kinship: the C-like family reads correctly under the `js` scanner,
|
|
1236
|
+
// and Ruby under the `python` one because both end a comment at `#` and a
|
|
1237
|
+
// statement at the newline. Naming them beats falling through to `js` by
|
|
1238
|
+
// default, which is how a `#` comment came to be read as code.
|
|
1239
|
+
[".java", "js"], [".kt", "js"], [".kts", "js"], [".scala", "js"], [".groovy", "js"],
|
|
1240
|
+
[".swift", "js"], [".cs", "js"], [".php", "js"], [".c", "js"], [".cc", "js"],
|
|
1241
|
+
[".cpp", "js"], [".h", "js"], [".hpp", "js"], [".m", "js"], [".sol", "js"],
|
|
1242
|
+
[".rb", "python"],
|
|
1206
1243
|
]);
|
|
1207
1244
|
|
|
1208
1245
|
function langForTestFile(file) {
|
|
@@ -1540,6 +1577,21 @@ function stripComments(text, lang) {
|
|
|
1540
1577
|
const CONTINUATION_START = /^[)\],.]/;
|
|
1541
1578
|
const CONTINUATION_OP_START = /^[+\-*/%<>=&|^:]/;
|
|
1542
1579
|
|
|
1580
|
+
// A comment is not a continuation, however much it looks like one.
|
|
1581
|
+
//
|
|
1582
|
+
// `//` begins with a division sign and `--` with a minus, so both matched
|
|
1583
|
+
// CONTINUATION_OP_START and folded the following comment line into the
|
|
1584
|
+
// statement above it. The cost was a false accusation on a virtuous act:
|
|
1585
|
+
// adding an assertion next to a `// ...` line made the new assertion absorb
|
|
1586
|
+
// the comment, stop matching its unchanged twin, and get reported as a
|
|
1587
|
+
// rewritten expectation. Python was unaffected only because `#` is not an
|
|
1588
|
+
// operator — which is why the same fixture passed in pytest and failed in
|
|
1589
|
+
// Jest, and why it survived every suite written against the pytest layout.
|
|
1590
|
+
//
|
|
1591
|
+
// A comment inside an open delimiter still joins: `cur.delta > 0` decides
|
|
1592
|
+
// that before this test is ever reached.
|
|
1593
|
+
const COMMENT_LINE_START = /^(?:\/\/|\/\*|#|--)/;
|
|
1594
|
+
|
|
1543
1595
|
// A scanner miscount (an unbalanced delimiter inside a regex literal is the
|
|
1544
1596
|
// usual cause) must not be able to merge a whole file into one statement,
|
|
1545
1597
|
// which would pair *any* literal change anywhere in the file.
|
|
@@ -1578,14 +1630,15 @@ function assembleStatements(sliceLines, lang) {
|
|
|
1578
1630
|
|
|
1579
1631
|
for (const L of sliceLines) {
|
|
1580
1632
|
const trimmed = L.text.replace(/^\s+/, "");
|
|
1633
|
+
const startsComment = COMMENT_LINE_START.test(trimmed);
|
|
1581
1634
|
const joins =
|
|
1582
1635
|
cur !== null &&
|
|
1583
1636
|
(cur.delta > 0 ||
|
|
1584
1637
|
cur.state.str !== null ||
|
|
1585
1638
|
cur.state.block > 0 ||
|
|
1586
1639
|
cur.trailingBackslash ||
|
|
1587
|
-
|
|
1588
|
-
|
|
1640
|
+
(!startsComment &&
|
|
1641
|
+
(CONTINUATION_START.test(trimmed) || CONTINUATION_OP_START.test(trimmed))));
|
|
1589
1642
|
|
|
1590
1643
|
if (
|
|
1591
1644
|
joins &&
|
|
@@ -1648,7 +1701,17 @@ function splitAssertionArgs(clean, lang) {
|
|
|
1648
1701
|
const m = SPECIFIC_ASSERTION.exec(clean);
|
|
1649
1702
|
if (!m) return null;
|
|
1650
1703
|
|
|
1651
|
-
|
|
1704
|
+
// Not every branch of SPECIFIC_ASSERTION ends at an opening paren:
|
|
1705
|
+
// `assert_eq!`, `assert_equal` and RSpec's `.to eq` all match a bare name.
|
|
1706
|
+
// Starting the walk one character early made every argument boundary wrong,
|
|
1707
|
+
// so a reworded message read as a rewritten value.
|
|
1708
|
+
let i = m.index + m[0].length;
|
|
1709
|
+
if (clean[i - 1] !== "(") {
|
|
1710
|
+
let j = i;
|
|
1711
|
+
while (j < clean.length && /\s/.test(clean[j])) j++;
|
|
1712
|
+
if (clean[j] !== "(") return null;
|
|
1713
|
+
i = j + 1;
|
|
1714
|
+
}
|
|
1652
1715
|
let depth = 1;
|
|
1653
1716
|
let quote = null;
|
|
1654
1717
|
let triple = false;
|
|
@@ -1740,7 +1803,45 @@ function messageArgIndices(args) {
|
|
|
1740
1803
|
* every time somebody improved the wording of a failure. Firing on that is
|
|
1741
1804
|
* how an operator learns to pass the override without reading it.
|
|
1742
1805
|
*/
|
|
1806
|
+
/**
|
|
1807
|
+
* Split a statement at a trailing `, "message"` written outside the call.
|
|
1808
|
+
*
|
|
1809
|
+
* RSpec puts the message there — `expect(x).to eq(3), "explain"` — and so do
|
|
1810
|
+
* Ruby and Elixir assertions generally. An argument-position check can never
|
|
1811
|
+
* see it, so rewording one read as a rewritten expectation.
|
|
1812
|
+
*/
|
|
1813
|
+
function splitTrailingMessage(clean) {
|
|
1814
|
+
let depth = 0;
|
|
1815
|
+
let quote = null;
|
|
1816
|
+
let lastComma = -1;
|
|
1817
|
+
for (let i = 0; i < clean.length; i++) {
|
|
1818
|
+
const c = clean[i];
|
|
1819
|
+
if (quote !== null) {
|
|
1820
|
+
if (c === "\\") { i += 1; continue; }
|
|
1821
|
+
if (c === quote) quote = null;
|
|
1822
|
+
continue;
|
|
1823
|
+
}
|
|
1824
|
+
if (c === '"' || c === "'" || c === "`") { quote = c; continue; }
|
|
1825
|
+
if (c === "(" || c === "[" || c === "{") depth++;
|
|
1826
|
+
else if (c === ")" || c === "]" || c === "}") depth--;
|
|
1827
|
+
else if (c === "," && depth === 0) lastComma = i;
|
|
1828
|
+
}
|
|
1829
|
+
if (lastComma === -1) return { head: clean, msg: null };
|
|
1830
|
+
const tail = clean.slice(lastComma + 1).trim();
|
|
1831
|
+
if (!isPureStringLiteral(tail)) return { head: clean, msg: null };
|
|
1832
|
+
return { head: clean.slice(0, lastComma), msg: tail };
|
|
1833
|
+
}
|
|
1834
|
+
|
|
1743
1835
|
function differsOnlyInMessage(cleanRemoved, cleanAdded, lang) {
|
|
1836
|
+
const ta = splitTrailingMessage(cleanRemoved);
|
|
1837
|
+
const tb = splitTrailingMessage(cleanAdded);
|
|
1838
|
+
if (
|
|
1839
|
+
(ta.msg !== null || tb.msg !== null) &&
|
|
1840
|
+
ta.head.replace(/\s+/g, "") === tb.head.replace(/\s+/g, "")
|
|
1841
|
+
) {
|
|
1842
|
+
return true;
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1744
1845
|
const a = splitAssertionArgs(cleanRemoved, lang);
|
|
1745
1846
|
const b = splitAssertionArgs(cleanAdded, lang);
|
|
1746
1847
|
if (!a || !b || a.length !== b.length || a.length === 0) return false;
|
|
@@ -1988,7 +2089,9 @@ export function resolveAllowedTamperKinds(options = {}) {
|
|
|
1988
2089
|
* @param {string} diffOrText - Unified git diff
|
|
1989
2090
|
* @param {Object} [options]
|
|
1990
2091
|
* @param {boolean} [options.allowTestModifications=false]
|
|
1991
|
-
* @returns {{ ok: boolean, violations: Array<object>, inputsSeen: number,
|
|
2092
|
+
* @returns {{ ok: boolean, violations: Array<object>, inputsSeen: number, filesSeen: number,
|
|
2093
|
+
* assertionsSeen: number, unreadable: Array<{file: string, count: number, samples: string[]}>,
|
|
2094
|
+
* status: "PASS"|"FAIL"|"UNREADABLE"|"NOT_APPLICABLE" }}
|
|
1992
2095
|
* `status` distinguishes "checked and clean" from "nothing was checked";
|
|
1993
2096
|
* `ok: true` alone cannot, and that ambiguity is the defect class this
|
|
1994
2097
|
* field exists to make visible.
|
|
@@ -2024,9 +2127,24 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
2024
2127
|
{ pattern: /#\[ignore\]/i, desc: "Injected Rust test ignore attribute (#[ignore])" },
|
|
2025
2128
|
{ pattern: /\/\/\s*@ts-ignore/i, desc: "Injected TypeScript ignore comment (// @ts-ignore)" },
|
|
2026
2129
|
{ pattern: /\/\/\s*@ts-nocheck/i, desc: "Injected TypeScript nocheck directive (// @ts-nocheck)" },
|
|
2130
|
+
// A suite that never runs cannot fail, and `@Disabled` is exactly as
|
|
2131
|
+
// effective at arranging that as `it.skip` is.
|
|
2132
|
+
{ pattern: /@Disabled\b/, desc: "Injected JUnit 5 disable annotation (@Disabled)" },
|
|
2133
|
+
{ pattern: /@Ignore\b/, desc: "Injected JUnit 4 / TestNG ignore annotation (@Ignore)" },
|
|
2134
|
+
{ pattern: /@Test\s*\([^)]*enabled\s*=\s*false/i, desc: "Injected TestNG disabled test (enabled = false)" },
|
|
2135
|
+
{ pattern: /@unittest\.skip/i, desc: "Injected unittest skip decorator (@unittest.skip)" },
|
|
2136
|
+
{ pattern: /\bmarkTest(?:Skipped|Incomplete)\s*\(/i, desc: "Injected PHPUnit skip (markTestSkipped())" },
|
|
2137
|
+
{ pattern: /\bXCTSkip(?:If|Unless|IfNot)?\s*\(/, desc: "Injected XCTest skip (XCTSkip())" },
|
|
2138
|
+
{ pattern: /\b(?:xit|xdescribe|xcontext|xspecify)\b\s*["\x27]/i, desc: "Injected RSpec disabled example (xit)" },
|
|
2139
|
+
{ pattern: /,\s*skip:\s*(?:true|["\x27])/i, desc: "Injected RSpec skip metadata (skip:)" },
|
|
2140
|
+
{ pattern: /^\s*(?:skip|pending)\s*(?:["\x27(]|$)/i, desc: "Injected Minitest/RSpec skip statement" },
|
|
2027
2141
|
];
|
|
2028
2142
|
|
|
2029
|
-
|
|
2143
|
+
// `#` and `--` belong here for the same reason the dialects belong in
|
|
2144
|
+
// ASSERTION_PATTERN: a Ruby or Python assertion commented out is exactly
|
|
2145
|
+
// as gone as a JavaScript one, and was previously not looked for.
|
|
2146
|
+
const COMMENTED_ASSERTION =
|
|
2147
|
+
/^\+\s*(?:\/\/|\/\*|#|--)\s*(?:expect\s*\(|assert(?!ion|ing|ed\b|s\b)[a-zA-Z0-9_$]*\s*[.(]|assert\s|refute_|XCTAssert|t\.expect|t\.assert)/i;
|
|
2030
2148
|
|
|
2031
2149
|
const VACUOUS_ASSERTIONS = [
|
|
2032
2150
|
{ pattern: /\bassert(?:\.ok)?\s*\(\s*true\s*(?:,[^)]*)?\)/i, desc: "Vacuous truth assertion (assert.ok(true))" },
|
|
@@ -2037,11 +2155,44 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
2037
2155
|
{ pattern: /\bexpect\s*\(\s*false\s*\)\s*\.toBeFalsy\s*\(/i, desc: "Vacuous falsity expectation (expect(false).toBeFalsy())" },
|
|
2038
2156
|
{ pattern: /\bassert\.(?:isTrue|isOk)\s*\(\s*true\s*(?:,[^)]*)?\)/i, desc: "Vacuous truth assertion (assert.isTrue(true))" },
|
|
2039
2157
|
{ pattern: /\bassert\.(?:isFalse|isNotOk)\s*\(\s*false\s*(?:,[^)]*)?\)/i, desc: "Vacuous falsity assertion (assert.isFalse(false))" },
|
|
2158
|
+
{ pattern: /\b(?:XCT)?assertTrue\s*\(\s*true\s*[,)]/i, desc: "Vacuous truth assertion (assertTrue(true))" },
|
|
2159
|
+
{ pattern: /\b(?:XCT)?assertFalse\s*\(\s*false\s*[,)]/i, desc: "Vacuous falsity assertion (assertFalse(false))" },
|
|
2160
|
+
{ pattern: /\b(?:assertEquals|assertSame|XCTAssertEqual)\s*\(\s*([^,]+?)\s*,\s*\1\s*[,)]/i, desc: "Vacuous identity assertion (assertEquals(X, X))" },
|
|
2161
|
+
{ pattern: /\bassert_equal\s*\(?\s*([^,]+?)\s*,\s*\1\s*\)?\s*$/i, desc: "Vacuous identity assertion (assert_equal X, X)" },
|
|
2162
|
+
{ pattern: /\bexpect\s*\(\s*true\s*\)\s*\.to\s+be(?:\s+true)?\b/i, desc: "Vacuous truth expectation (expect(true).to be true)" },
|
|
2040
2163
|
];
|
|
2041
2164
|
|
|
2042
|
-
|
|
2165
|
+
// Broad on purpose: this is the denominator, not the verdict. A word
|
|
2166
|
+
// boundary immediately after `assert` never falls in `assertEquals`,
|
|
2167
|
+
// `assert_equal` or `XCTAssertEqual`, so five ecosystems contributed no
|
|
2168
|
+
// assertions to count at all and a gutted JUnit suite was arithmetically
|
|
2169
|
+
// indistinguishable from an untouched one. The lookahead keeps prose and
|
|
2170
|
+
// identifiers — `assertion`, `asserts`, `asserted` — out of the count.
|
|
2171
|
+
const ASSERTION_PATTERN =
|
|
2172
|
+
/(?:\b(?:assert(?!ion|ing|ed\b|s\b)[a-zA-Z0-9_$]*(?:\.[a-zA-Z0-9_$]+)?|refute[a-zA-Z0-9_$]*|XCTAssert[a-zA-Z0-9_$]*|XCTFail|expect|t\.(?:assert|expect|is|equal|true|false|Errorf|Fatalf)|require\.[a-zA-Z0-9_$]+)\b|assert!|assert_eq!|assert_ne!)/i;
|
|
2173
|
+
// The loose net. Not a verdict and never a block — its only job is to
|
|
2174
|
+
// notice that a line was plainly an assertion in *some* dialect that
|
|
2175
|
+
// ASSERTION_PATTERN did not recognise. Without it, adding the seventh
|
|
2176
|
+
// ecosystem is indistinguishable from having covered it all along: the
|
|
2177
|
+
// guard returns the same clean PASS either way. This is the denominator
|
|
2178
|
+
// for the denominator.
|
|
2179
|
+
// Deliberately not call-shaped. Haskell's `x `shouldBe` 3` is an
|
|
2180
|
+
// assertion with no parentheses anywhere near it, and a net that only
|
|
2181
|
+
// catches `name(` reports the same confident PASS on it as on a clean
|
|
2182
|
+
// Node suite. `require` and `check` are absent on purpose: in a
|
|
2183
|
+
// CommonJS test file `require("./calc")` is an import, not a claim.
|
|
2184
|
+
const ASSERTION_SHAPED =
|
|
2185
|
+
/\b(?:assert(?!ion|ing|ed\b|s\b)|expect(?!ed\b|ation)|refute)[a-zA-Z0-9_$]*\b|`\s*should[a-zA-Z0-9_$]*\s*`|\b(?:should|must|verify|ensure|confirm)[a-zA-Z0-9_$]*\s*[(!]|\.\s*(?:should|to|to_not|not_to|must)\b|\bBOOST_[A-Z_]+\s*\(|\b[A-Z]+_(?:EQ|NE|TRUE|FALSE|THAT)\s*\(/;
|
|
2043
2186
|
const isCommentLine = (str) => /^\s*(?:\/\/|\/\*|\*|#|--|;)/.test(str);
|
|
2044
2187
|
|
|
2188
|
+
/** Book-keeping only: what this run looked at, before deciding anything. */
|
|
2189
|
+
const countExamined = (stats, text) => {
|
|
2190
|
+
if (!text.trim() || isCommentLine(text)) return;
|
|
2191
|
+
stats.examined++;
|
|
2192
|
+
if (ASSERTION_PATTERN.test(text)) stats.recognised++;
|
|
2193
|
+
else if (ASSERTION_SHAPED.test(text) && stats.unreadable.length < 5) stats.unreadable.push(text.trim().slice(0, 120));
|
|
2194
|
+
};
|
|
2195
|
+
|
|
2045
2196
|
const fileAssertions = new Map();
|
|
2046
2197
|
let pendingHunk = false;
|
|
2047
2198
|
|
|
@@ -2077,7 +2228,7 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
2077
2228
|
}
|
|
2078
2229
|
|
|
2079
2230
|
if (!fileAssertions.has(currentFile)) {
|
|
2080
|
-
fileAssertions.set(currentFile, { removed: [], added: 0, addedTexts: [], removedSpecific: [], addedSpecific: 0, hunks: [] });
|
|
2231
|
+
fileAssertions.set(currentFile, { removed: [], added: 0, addedTexts: [], removedSpecific: [], addedSpecific: 0, hunks: [], examined: 0, recognised: 0, unreadable: [] });
|
|
2081
2232
|
}
|
|
2082
2233
|
const fileStats = fileAssertions.get(currentFile);
|
|
2083
2234
|
if (pendingHunk) {
|
|
@@ -2089,6 +2240,7 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
2089
2240
|
if (line.startsWith("-") && !line.startsWith("---")) {
|
|
2090
2241
|
const deletedText = line.slice(1);
|
|
2091
2242
|
if (hunk) hunk.lines.push({ kind: "-", text: deletedText, oldNo: currentOldLineNo, newNo: null });
|
|
2243
|
+
countExamined(fileStats, deletedText);
|
|
2092
2244
|
if (!isCommentLine(deletedText) && ASSERTION_PATTERN.test(deletedText)) {
|
|
2093
2245
|
fileStats.removed.push({ line: currentOldLineNo, text: deletedText });
|
|
2094
2246
|
if (isSpecificAssertion(deletedText)) {
|
|
@@ -2099,6 +2251,7 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
2099
2251
|
} else if (line.startsWith("+") && !line.startsWith("+++")) {
|
|
2100
2252
|
const addedText = line.slice(1);
|
|
2101
2253
|
if (hunk) hunk.lines.push({ kind: "+", text: addedText, oldNo: null, newNo: currentNewLineNo });
|
|
2254
|
+
countExamined(fileStats, addedText);
|
|
2102
2255
|
let isVacuous = false;
|
|
2103
2256
|
|
|
2104
2257
|
// Check skip injections
|
|
@@ -2225,17 +2378,44 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
2225
2378
|
// `ok: true` from a guard that looked at everything and approved it. That
|
|
2226
2379
|
// ambiguity is how a substring bug in the file classifier switched this
|
|
2227
2380
|
// entire guard off for the standard pytest, Rust and RSpec layouts while
|
|
2228
|
-
// every signal stayed green.
|
|
2229
|
-
//
|
|
2230
|
-
//
|
|
2231
|
-
//
|
|
2232
|
-
|
|
2381
|
+
// every signal stayed green.
|
|
2382
|
+
//
|
|
2383
|
+
// Counting *files* was not enough. A JUnit diff that rewrote an expected
|
|
2384
|
+
// value produced `inputsSeen: 1` and a clean PASS while not one assertion
|
|
2385
|
+
// in it had been recognised — the same ambiguity, one level down, inside
|
|
2386
|
+
// the mechanism built to remove it. So the denominator is now the thing
|
|
2387
|
+
// the rules actually consume: lines examined, and of those, assertions
|
|
2388
|
+
// understood. `UNREADABLE` is the state that has no business being silent
|
|
2389
|
+
// — assertion-shaped lines were present and none of them parsed, which
|
|
2390
|
+
// means this repository speaks a dialect the guard does not.
|
|
2391
|
+
let examined = 0;
|
|
2392
|
+
let assertionsSeen = 0;
|
|
2393
|
+
const unreadable = [];
|
|
2394
|
+
for (const [file, stats] of fileAssertions.entries()) {
|
|
2395
|
+
examined += stats.examined;
|
|
2396
|
+
assertionsSeen += stats.recognised;
|
|
2397
|
+
if (stats.unreadable.length > 0) {
|
|
2398
|
+
unreadable.push({ file, count: stats.unreadable.length, samples: stats.unreadable.slice(0, 3) });
|
|
2399
|
+
}
|
|
2400
|
+
}
|
|
2401
|
+
|
|
2402
|
+
const status =
|
|
2403
|
+
reported.length > 0
|
|
2404
|
+
? "FAIL"
|
|
2405
|
+
: assertionsSeen === 0 && unreadable.length > 0
|
|
2406
|
+
? "UNREADABLE"
|
|
2407
|
+
: examined > 0
|
|
2408
|
+
? "PASS"
|
|
2409
|
+
: "NOT_APPLICABLE";
|
|
2233
2410
|
|
|
2234
2411
|
return {
|
|
2235
2412
|
ok: reported.length === 0,
|
|
2236
2413
|
violations: reported,
|
|
2237
|
-
inputsSeen,
|
|
2238
|
-
|
|
2414
|
+
inputsSeen: examined,
|
|
2415
|
+
filesSeen: fileAssertions.size,
|
|
2416
|
+
assertionsSeen,
|
|
2417
|
+
unreadable,
|
|
2418
|
+
status,
|
|
2239
2419
|
};
|
|
2240
2420
|
}
|
|
2241
2421
|
|