jules-orchestrator-kit 0.63.0 → 0.64.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -204,7 +204,7 @@ To maximize PR merge rates, dispatch tasks according to deterministic boundaries
204
204
  * **Fail-Closed Security & Secret Redaction:** Evaluates explicit Deny rules before Allow rules against canonicalized, case-folded paths. Redacts high-entropy keys and base64-encoded credentials (such as Kubernetes `Secret` manifests).
205
205
  * **Complexity & Cost Router:** Zero-dependency heuristic classifier (`src/router.mjs`) routing mechanical tasks to lightweight models while reserving primary models for complex refactors, with a `node --check` syntax-verification gate that transparently escalates a FAST-tier result to the primary provider if it left broken JS on disk.
206
206
  * **Terminal UI & Diagnostic Matrix (`agentctl doctor`):** Interactive terminal dashboard, task sidecar manager, and automated transactional self-repair.
207
- * **Verified Test Suite:** Tested with **1015 unit tests across 145 suites passing in < 15.0s**.
207
+ * **Verified Test Suite:** Tested with **1085 unit tests across 150 suites passing in < 15.0s**.
208
208
 
209
209
  <br/>
210
210
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "jules-orchestrator-kit",
3
- "version": "0.63.0",
3
+ "version": "0.64.0",
4
4
  "description": "Zero-dependency safety gatekeeper, test oracle generator, and multi-agent coordination protocol for autonomous coding agents — Google Jules, Claude Code, Codex and Gemini CLI.",
5
5
  "repository": {
6
6
  "type": "git",
@@ -56,7 +56,8 @@
56
56
  "jules:doc-sync": "node scripts/doc-sync-check.mjs",
57
57
  "release": "node scripts/release.mjs",
58
58
  "guard-reach": "node scripts/guard-reach-check.mjs",
59
- "lint": "eslint ."
59
+ "lint": "eslint .",
60
+ "package-integrity": "node scripts/package-integrity-check.mjs"
60
61
  },
61
62
  "engines": {
62
63
  "node": ">=20.0.0"
@@ -27,7 +27,7 @@
27
27
  *
28
28
  * Three steps, all in-process, no dependencies, well under a second:
29
29
  *
30
- * 1. POLICY — the hand-written witness table in test/fixtures/guard-policy.mjs
30
+ * 1. POLICY — the hand-written witness table in src/guard-policy.mjs
31
31
  * must hold. It is derived from what the tool advertises, never
32
32
  * from the regexes that implement it.
33
33
  * 2. CANARIES — every known-bad input must produce the finding it names. A
@@ -52,14 +52,24 @@ import {
52
52
  PREDICATE_MUTANTS,
53
53
  EMPTY_RUN_CANARIES,
54
54
  SCOPE_CANARIES,
55
- } from "../test/fixtures/guard-policy.mjs";
55
+ INNOCENT_EDITS,
56
+ UNREADABLE_DIALECTS,
57
+ } from "../src/guard-policy.mjs";
56
58
 
57
- /** Build a unified diff for one canary. */
59
+ /**
60
+ * Build a unified diff for one canary.
61
+ *
62
+ * The context line carries the comment syntax of the file's own language.
63
+ * A `//` line in a `.py` fixture is not a comment, and a fixture that lies
64
+ * about the language under test measures the fixture rather than the guard —
65
+ * which is how two false results were once read as two defects.
66
+ */
58
67
  function canaryDiff(c) {
59
- const lines = [`--- a/${c.file}`, `+++ b/${c.file}`, "@@ -1,20 +1,20 @@", " // context"];
68
+ const ctx = c.context || "// context";
69
+ const lines = [`--- a/${c.file}`, `+++ b/${c.file}`, "@@ -1,20 +1,20 @@", ` ${ctx}`];
60
70
  for (const l of c.removed) lines.push(`-${l}`);
61
71
  for (const l of c.added) lines.push(`+${l}`);
62
- lines.push(" // context");
72
+ lines.push(` ${ctx}`);
63
73
  return lines.join("\n");
64
74
  }
65
75
 
@@ -107,6 +117,7 @@ const canaryResults = new Map();
107
117
  {
108
118
  const silent = [];
109
119
  const noDenominator = [];
120
+ const noAssertions = [];
110
121
  for (const c of TAMPER_CANARIES) {
111
122
  const res = checkTestTampering(canaryDiff(c));
112
123
  const hit = (res.violations || []).some((v) => v.type === c.expect);
@@ -114,12 +125,51 @@ const canaryResults = new Map();
114
125
  if (!hit) silent.push(`${c.id} expected ${c.expect}, got ${JSON.stringify((res.violations || []).map((v) => v.type))}`);
115
126
  // A finding with no denominator is the shape this script exists to reject.
116
127
  if (hit && !(res.inputsSeen > 0)) noDenominator.push(c.id);
128
+ // Counting lines was not enough: a JUnit diff reported one input examined
129
+ // and a clean PASS while every assertion in it went unrecognised. A rule
130
+ // about assertions has to say how many assertions it actually read.
131
+ if (hit && c.expect !== "TEST_SKIP_INJECTION" && !(res.assertionsSeen > 0)) {
132
+ noAssertions.push(`${c.id} (${res.assertionsSeen} assertions parsed)`);
133
+ }
117
134
  }
118
135
  add("canaries: every tamper rule still fires", silent.length === 0, silent.length ? silent.join("; ") : `${TAMPER_CANARIES.length} canaries red as required`);
119
136
  add("canaries: every finding carries a denominator", noDenominator.length === 0, noDenominator.length ? noDenominator.join(", ") : "inputsSeen > 0 on every hit");
137
+ add("canaries: assertion rules parsed an assertion", noAssertions.length === 0, noAssertions.length ? noAssertions.join(", ") : "assertionsSeen > 0 on every assertion finding");
138
+ }
139
+
140
+ // --- 3. The opposite failure: flagging what is innocent ---------------------
141
+ {
142
+ const noisy = [];
143
+ for (const e of INNOCENT_EDITS) {
144
+ const res = checkTestTampering(canaryDiff(e));
145
+ const types = (res.violations || []).map((v) => v.type);
146
+ if (types.length > 0) noisy.push(`${e.id} → ${JSON.stringify(types)} (${e.why})`);
147
+ }
148
+ add(
149
+ "innocent edits stay silent",
150
+ noisy.length === 0,
151
+ noisy.length
152
+ ? noisy.join("; ")
153
+ : `${INNOCENT_EDITS.length} ordinary edits produce no finding`
154
+ );
155
+ }
156
+
157
+ {
158
+ const quiet = [];
159
+ for (const d of UNREADABLE_DIALECTS) {
160
+ const res = checkTestTampering(canaryDiff(d));
161
+ if (res.status !== "UNREADABLE") quiet.push(`${d.id} → ${res.status} (${d.why})`);
162
+ }
163
+ add(
164
+ "an unparsable dialect says so",
165
+ quiet.length === 0,
166
+ quiet.length
167
+ ? `${quiet.join("; ")} — coverage ending is fine, ending silently is not`
168
+ : `${UNREADABLE_DIALECTS.length} unsupported dialects reported, not passed`
169
+ );
120
170
  }
121
171
 
122
- // --- 3. Predicate mutants ---------------------------------------------------
172
+ // --- 4. Predicate mutants ---------------------------------------------------
123
173
  {
124
174
  const survivors = [];
125
175
  for (const mutant of PREDICATE_MUTANTS) {
@@ -0,0 +1,251 @@
1
+ #!/usr/bin/env node
2
+
3
+ /**
4
+ * Does the package we publish actually work?
5
+ *
6
+ * Every test in this repository runs against the source tree, where every
7
+ * file is present by definition. What users install is a tarball built from
8
+ * the `files` list in package.json, and nothing checked that the two agreed.
9
+ *
10
+ * They did not. v0.63.0 shipped `scripts/guard-reach-check.mjs` — the check
11
+ * whose entire purpose is to prove no guard has silently gone missing —
12
+ * while leaving behind the policy contract it imports. Unpacked and run, it
13
+ * threw ERR_MODULE_NOT_FOUND. The CLI was fine, 1015 tests were green, nine
14
+ * CI cells passed, and the published artefact still had a hole in it,
15
+ * because every one of those signals was measured somewhere the file existed.
16
+ *
17
+ * This asks the packer what it would ship, and then resolves the import
18
+ * graph inside that answer.
19
+ *
20
+ * Usage: node scripts/package-integrity-check.mjs [--json]
21
+ * Exit codes: 0 = the tarball is self-contained, 1 = it is not.
22
+ */
23
+
24
+ import { execFileSync } from "node:child_process";
25
+ import { readFileSync, existsSync } from "node:fs";
26
+ import { resolve, dirname, relative, sep } from "node:path";
27
+ import { fileURLToPath } from "node:url";
28
+
29
+ import { IMPORT_EXTRACTION_CASES } from "../src/guard-policy.mjs";
30
+
31
+ const root = fileURLToPath(new URL("..", import.meta.url));
32
+
33
+ /**
34
+ * Mark every character that sits inside a string literal or a comment.
35
+ *
36
+ * Without this, an import quoted *inside* a fixture string reads as an
37
+ * import of the file itself — this check's first run reported six broken
38
+ * imports in the policy contract, all of them example text. A guard that
39
+ * cries wolf about its own fixtures gets switched off in a week.
40
+ *
41
+ * Approximate on purpose, and biased on purpose: a regex literal holding a
42
+ * quote can open a phantom string, so the counts reported below exist to
43
+ * make a mask that swallowed the file visible rather than silent.
44
+ */
45
+ function stringMask(src) {
46
+ const mask = new Uint8Array(src.length);
47
+ let i = 0;
48
+ let quote = null;
49
+ let comment = null;
50
+ while (i < src.length) {
51
+ const c = src[i];
52
+ const d = src[i + 1];
53
+ if (comment === "line") {
54
+ if (c === "\n") comment = null;
55
+ else mask[i] = 1;
56
+ i++;
57
+ continue;
58
+ }
59
+ if (comment === "block") {
60
+ mask[i] = 1;
61
+ if (c === "*" && d === "/") {
62
+ mask[i + 1] = 1;
63
+ comment = null;
64
+ i += 2;
65
+ continue;
66
+ }
67
+ i++;
68
+ continue;
69
+ }
70
+ if (quote !== null) {
71
+ mask[i] = 1;
72
+ if (c === "\\") {
73
+ if (i + 1 < src.length) mask[i + 1] = 1;
74
+ i += 2;
75
+ continue;
76
+ }
77
+ if (c === quote) quote = null;
78
+ i++;
79
+ continue;
80
+ }
81
+ if (c === "/" && d === "/") { comment = "line"; mask[i] = 1; i++; continue; }
82
+ if (c === "/" && d === "*") { comment = "block"; mask[i] = 1; i++; continue; }
83
+ // A regex literal holding a quote — `/["']/` — opens a string that never
84
+ // closes, and everything after it is misread. This file's own subject
85
+ // matter is regexes full of quote characters, so the mask desynchronised
86
+ // and a `require("./calc")` written inside a comment was reported as a
87
+ // missing module. Deciding regex-versus-division on the previous token is
88
+ // the same approximation the diff scanner already makes.
89
+ if (c === "/") {
90
+ let k = i - 1;
91
+ while (k >= 0 && /\s/.test(src[k])) k--;
92
+ const prev = k >= 0 ? src[k] : "";
93
+ if (prev === "" || "(,=:[!&|?{};+-*%~^<>".includes(prev)) {
94
+ mask[i] = 1;
95
+ let j = i + 1;
96
+ let cls = false;
97
+ while (j < src.length) {
98
+ const e = src[j];
99
+ mask[j] = 1;
100
+ if (e === "\\") { if (j + 1 < src.length) mask[j + 1] = 1; j += 2; continue; }
101
+ if (e === "[") cls = true;
102
+ else if (e === "]") cls = false;
103
+ else if (e === "/" && !cls) { j++; break; }
104
+ else if (e === "\n") break;
105
+ j++;
106
+ }
107
+ i = j;
108
+ continue;
109
+ }
110
+ }
111
+ if (c === '"' || c === "'" || c === "`") { quote = c; mask[i] = 1; i++; continue; }
112
+ i++;
113
+ }
114
+ return mask;
115
+ }
116
+
117
+ /**
118
+ * Every module specifier in a source file.
119
+ *
120
+ * Deliberately newline-tolerant. A matcher bounded by `[^;\n]*?` cannot see
121
+ * a multi-line named import, which is exactly the import that was missing
122
+ * from the tarball, so the first version of this check certified the broken
123
+ * package as sound. The extraction cases in the policy contract exist to
124
+ * keep that from being a private mistake twice.
125
+ */
126
+ export function extractSpecifiers(src, { skipQuoted = true } = {}) {
127
+ const mask = skipQuoted ? stringMask(src) : null;
128
+ const found = new Set();
129
+ const collect = (re) => {
130
+ for (const m of src.matchAll(re)) {
131
+ // What decides is the token immediately before the specifier — the
132
+ // `from`, or the `(` of a call. In a real import it is code; in a
133
+ // fixture it is the middle of a string. Testing the *keyword* instead
134
+ // was not enough: `export const CASES = [` at the top of a file
135
+ // matched lazily forward into the first `from "…"` inside an example,
136
+ // so a genuine keyword lent its authority to quoted text.
137
+ if (mask) {
138
+ let j = m.indices[1][0] - 2;
139
+ while (j >= 0 && /\s/.test(src[j])) j--;
140
+ if (j >= 0 && mask[j]) continue;
141
+ }
142
+ found.add(m[1]);
143
+ }
144
+ };
145
+ collect(/(?:^|[\s;}])(?:import|export)\s[\s\S]{0,500}?from\s*["']([^"']+)["']/gd);
146
+ collect(/(?:^|[\s;}])import\s*["']([^"']+)["']/gd);
147
+ collect(/\bimport\s*\(\s*["']([^"']+)["']/gd);
148
+ collect(/\brequire\s*\(\s*["']([^"']+)["']/gd);
149
+ return [...found];
150
+ }
151
+
152
+ /**
153
+ * Run every integrity check and return the result.
154
+ *
155
+ * Exported as a function rather than run on import: a module that checks the
156
+ * package must not exit the process of anything that merely imports it.
157
+ */
158
+ export function checkPackageIntegrity() {
159
+ const failures = [];
160
+ const checks = [];
161
+ const add = (name, ok, detail) => {
162
+ checks.push({ name, ok, detail });
163
+ if (!ok) failures.push(`${name}: ${detail}`);
164
+ };
165
+
166
+ // --- 1. The extractor must be able to see what it claims to look for -------
167
+ {
168
+ const wrong = [];
169
+ for (const c of IMPORT_EXTRACTION_CASES) {
170
+ const got = extractSpecifiers(c.src).sort();
171
+ const want = [...c.expect].sort();
172
+ if (JSON.stringify(got) !== JSON.stringify(want)) {
173
+ wrong.push(`${c.id}: found ${JSON.stringify(got)}, contract says ${JSON.stringify(want)}`);
174
+ }
175
+ }
176
+ add("extractor: every import form is visible", wrong.length === 0, wrong.length ? wrong.join("; ") : `${IMPORT_EXTRACTION_CASES.length} forms found`);
177
+ }
178
+
179
+ // --- 2. Ask the packer what it would actually ship -------------------------
180
+ let shipped = new Set();
181
+ try {
182
+ const out = execFileSync("npm", ["pack", "--dry-run", "--json"], { cwd: root, encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"] });
183
+ shipped = new Set(JSON.parse(out)[0].files.map((f) => f.path.split(/[\\/]/).join("/")));
184
+ } catch (err) {
185
+ add("packer: `npm pack --dry-run` answers", false, err.message);
186
+ }
187
+ add("packer: the tarball is not empty", shipped.size > 0, `${shipped.size} files`);
188
+
189
+ // --- 3. Resolve the import graph inside the tarball ------------------------
190
+ {
191
+ const broken = [];
192
+ let resolved = 0;
193
+ let scanned = 0;
194
+
195
+ for (const file of shipped) {
196
+ if (!/\.(mjs|cjs|js)$/.test(file)) continue;
197
+ let src;
198
+ try {
199
+ src = readFileSync(resolve(root, file), "utf-8");
200
+ } catch {
201
+ continue;
202
+ }
203
+ scanned++;
204
+ for (const spec of extractSpecifiers(src)) {
205
+ if (!spec.startsWith(".")) continue; // bare and node: specifiers are not ours to resolve
206
+ resolved++;
207
+ const target = relative(root, resolve(dirname(resolve(root, file)), spec)).split(sep).join("/");
208
+ if (!shipped.has(target)) {
209
+ broken.push(`${file} imports ${spec} — ${target} is not in the tarball (on disk: ${existsSync(resolve(root, target)) ? "yes" : "no"})`);
210
+ }
211
+ }
212
+ }
213
+
214
+ add(
215
+ "tarball: every relative import resolves",
216
+ broken.length === 0,
217
+ broken.length ? broken.join("; ") : `${resolved} relative imports across ${scanned} shipped modules`
218
+ );
219
+ // A resolver that resolved nothing would report the same clean line.
220
+ add("tarball: the graph was actually walked", resolved > 0 && scanned > 0, `${scanned} modules, ${resolved} relative imports`);
221
+ }
222
+
223
+ // --- 4. Every advertised entry point has to be in the box ------------------
224
+ {
225
+ const pkg = JSON.parse(readFileSync(resolve(root, "package.json"), "utf-8"));
226
+ const entries = [...Object.values(pkg.bin || {}), pkg.main, pkg.module].filter(Boolean);
227
+ const missing = entries.map((e) => e.replace(/^\.\//, "")).filter((e) => !shipped.has(e));
228
+ add("entry points: every bin and main ships", missing.length === 0, missing.length ? missing.join(", ") : `${entries.length} entry points present`);
229
+ }
230
+
231
+ return { ok: failures.length === 0, checks, failures };
232
+ }
233
+
234
+ const isMain = process.argv[1] && process.argv[1].endsWith("package-integrity-check.mjs");
235
+ if (isMain) {
236
+ const { ok, checks: rows, failures: bad } = checkPackageIntegrity();
237
+ if (process.argv.includes("--json")) {
238
+ console.log(JSON.stringify({ ok, checks: rows }, null, 2));
239
+ } else {
240
+ console.log("\n📦 Package Integrity Check (what we actually publish)");
241
+ console.log("-------------------------------------------------------");
242
+ for (const c of rows) console.log(` ${c.ok ? "✅" : "❌"} ${c.name.padEnd(46)} ${c.detail}`);
243
+ console.log("-------------------------------------------------------");
244
+ console.log(
245
+ ok
246
+ ? "✅ The published package is self-contained.\n"
247
+ : `\n❌ ${bad.length} problem(s). The tarball is not what the source tree looks like.\n`
248
+ );
249
+ }
250
+ process.exit(ok ? 0 : 1);
251
+ }
@@ -58,6 +58,22 @@ try {
58
58
  process.exit(1);
59
59
  }
60
60
 
61
+ // Step 1a proved the guards work here. Here is not where users run them.
62
+ //
63
+ // Every signal so far was measured in the source tree, where every file
64
+ // exists by construction. What ships is a tarball built from the `files`
65
+ // list, and v0.63.0 published the guard-reach check without the policy
66
+ // contract it imports — green suite, green matrix, green release, and a
67
+ // module that threw ERR_MODULE_NOT_FOUND the moment anyone installed it.
68
+ console.log("1a-2. Verifying the package we would publish is self-contained...");
69
+ try {
70
+ execSync("node scripts/package-integrity-check.mjs", { cwd: root, stdio: "inherit" });
71
+ console.log("");
72
+ } catch (_) {
73
+ console.error("\n❌ Release Aborted: the tarball is not what the source tree looks like.");
74
+ process.exit(1);
75
+ }
76
+
61
77
  // 1b. Documentation / version consistency gate (blocking).
62
78
  console.log("1b. Verifying documentation is in sync with package.json & test suite...");
63
79
  {
@@ -561,12 +561,28 @@ export function assertTestIntegrity(config = {}, root = process.cwd()) {
561
561
  const res = checkTestTampering(diffStr, config);
562
562
  const diagnostics = (res.violations || []).map((v) => v.reason);
563
563
 
564
+ // A clean result from a guard that could not read the dialect is not a
565
+ // clean result, and it must not be reported as one. This does not fail the
566
+ // check — an unlisted assertion library is the user's normal, not their
567
+ // fault — but they get to know the guard is not covering them.
568
+ for (const u of res.unreadable || []) {
569
+ diagnostics.push(
570
+ `Test Tamper Guard: ${u.count} assertion-shaped line(s) in ${u.file} matched no known dialect, ` +
571
+ `so this file was not checked for tampering (e.g. ${JSON.stringify(u.samples[0])}). ` +
572
+ `The guard reports what it could not read rather than passing silently.`
573
+ );
574
+ }
575
+
564
576
  return {
565
577
  ok: res.ok,
566
578
  violations: res.violations || [],
567
579
  diagnostics,
568
580
  metrics: {
569
581
  violationCount: res.violations?.length || 0,
582
+ assertionsSeen: res.assertionsSeen ?? 0,
583
+ filesSeen: res.filesSeen ?? 0,
584
+ unreadableFiles: (res.unreadable || []).length,
585
+ status: res.status,
570
586
  },
571
587
  };
572
588
  }
@@ -0,0 +1,481 @@
1
+ /**
2
+ * The policy this kit claims to enforce, written by hand.
3
+ *
4
+ * Every entry here is derived from what the tool *advertises* — the stacks
5
+ * `detectStack` declares, the layouts each ecosystem actually uses — and never
6
+ * from the regexes, path lists or registries that implement the checks. That
7
+ * separation is the whole point: a contract generated from the implementation
8
+ * makes the implementation its own oracle, and an implementation that is its
9
+ * own oracle cannot be wrong.
10
+ *
11
+ * This is what the substring bug in the file classifier cost. `isTestFile`
12
+ * matched `/test/`, which does not occur in `tests/test_calc.py`, so the entire
13
+ * tamper guard was off for the standard pytest, Rust and RSpec layouts — and
14
+ * every mechanism that should have caught it (a large suite, a doc-sync gate,
15
+ * a nine-way CI matrix, two cold reviews, a blocking release) was sampling the
16
+ * same distribution the implementation was written from. Nine runs of
17
+ * `test/foo.test.js` do not explore `tests/test_calc.py`.
18
+ *
19
+ * Adding a stack to `detectStack` is not finished until it has a row here.
20
+ */
21
+
22
+ /**
23
+ * Paths the policy says are test files, and near-misses it says are not.
24
+ *
25
+ * The near-misses matter as much as the hits: a predicate that answers "yes"
26
+ * to everything also has no denominator.
27
+ */
28
+ export const TEST_PATH_CASES = [
29
+ // Node / JavaScript
30
+ { path: "test/calc.test.js", expected: true, why: "node, conventional" },
31
+ { path: "src/calc.spec.ts", expected: true, why: "co-located spec" },
32
+ { path: "src/__tests__/calc.js", expected: true, why: "jest convention" },
33
+ // Python
34
+ { path: "tests/test_calc.py", expected: true, why: "pytest, repository root — the reported gap" },
35
+ { path: "test/test_calc.py", expected: true, why: "pytest, singular directory" },
36
+ { path: "backend/tests/test_api.py", expected: true, why: "pytest, nested" },
37
+ // Go
38
+ { path: "internal/calc/calc_test.go", expected: true, why: "go, co-located with the code" },
39
+ { path: "cmd/api/main_test.go", expected: true, why: "go, command package" },
40
+ // Rust
41
+ { path: "tests/integration.rs", expected: true, why: "rust integration tests" },
42
+ // Ruby
43
+ { path: "spec/models/user_spec.rb", expected: true, why: "rspec" },
44
+ { path: "test/user_test.rb", expected: true, why: "minitest" },
45
+ // JVM / .NET / PHP
46
+ { path: "src/test/java/com/x/CalcTest.java", expected: true, why: "maven layout" },
47
+ { path: "tests/Unit/CalcTest.php", expected: true, why: "phpunit" },
48
+ // Solidity
49
+ { path: "test/Token.t.sol", expected: true, why: "foundry" },
50
+ // Monorepo position
51
+ { path: "packages/api/test/handler.test.js", expected: true, why: "monorepo package" },
52
+
53
+ // Near-misses: segment match, not substring
54
+ { path: "latest/build.js", expected: false, why: "`latest` is not `test`" },
55
+ { path: "contest/result.js", expected: false, why: "`contest` is not `test`" },
56
+ { path: "lib/myspec/render.js", expected: false, why: "`myspec` is not `spec`" },
57
+ { path: "src/attestation.js", expected: false, why: "contains 'test', is not one" },
58
+ { path: "src/index.js", expected: false, why: "ordinary source" },
59
+ { path: "docs/testing-guide.md", expected: false, why: "documentation about testing" },
60
+ ];
61
+
62
+ /**
63
+ * Known-bad diffs, one per guard rule, each with the finding it MUST produce.
64
+ *
65
+ * A canary that comes back clean is not a passing test — it is proof that the
66
+ * rule it names has stopped being reachable. `layout` is deliberately varied
67
+ * across ecosystems so that a canary set cannot be green by only ever handing
68
+ * the guard the layout the implementation was written from.
69
+ */
70
+ export const TAMPER_CANARIES = [
71
+ {
72
+ id: "skip-injection/node",
73
+ file: "test/calc.test.js",
74
+ removed: [],
75
+ added: ['it.skip("adds", () => { assert.equal(add(1,2), 3); });'],
76
+ expect: "TEST_SKIP_INJECTION",
77
+ },
78
+ {
79
+ id: "vacuous/node",
80
+ file: "test/calc.test.js",
81
+ removed: ["assert.equal(add(1, 2), 3);"],
82
+ added: ["assert.ok(true);"],
83
+ expect: "VACUOUS_ASSERTION",
84
+ },
85
+ {
86
+ id: "commented/node",
87
+ file: "test/calc.test.js",
88
+ removed: ["assert.equal(add(1, 2), 3);"],
89
+ added: ["// assert.equal(add(1, 2), 3);"],
90
+ expect: "COMMENTED_ASSERTION",
91
+ },
92
+ {
93
+ id: "removal/node",
94
+ file: "test/calc.test.js",
95
+ removed: ["assert.equal(add(1, 2), 3);", "assert.equal(add(2, 2), 4);"],
96
+ added: [],
97
+ expect: "ASSERTION_REMOVAL",
98
+ },
99
+ {
100
+ id: "weakening/node",
101
+ file: "test/calc.test.js",
102
+ removed: ["assert.strictEqual(add(1, 2), 3);"],
103
+ added: ["assert.ok(add(1, 2) !== undefined);"],
104
+ expect: "ASSERTION_WEAKENED",
105
+ },
106
+ {
107
+ id: "expectation/node",
108
+ file: "test/calc.test.js",
109
+ removed: ["assert.equal(add(1, 2), 3);"],
110
+ added: ["assert.equal(add(1, 2), -1);"],
111
+ expect: "ASSERTION_EXPECTATION_CHANGED",
112
+ },
113
+ // The same attacks, in the layouts the classifier used to be blind to.
114
+ // These are the canaries that were silent when the classifier matched a
115
+ // substring instead of a path segment.
116
+ {
117
+ id: "expectation/pytest-root",
118
+ file: "tests/test_calc.py",
119
+ removed: [" self.assertEqual(add(1, 2), 3)"],
120
+ added: [" self.assertEqual(add(1, 2), -1)"],
121
+ expect: "ASSERTION_EXPECTATION_CHANGED",
122
+ },
123
+ {
124
+ id: "skip-injection/pytest-root",
125
+ file: "tests/test_calc.py",
126
+ removed: [],
127
+ added: ["@pytest.mark.skip", "def test_add():"],
128
+ expect: "TEST_SKIP_INJECTION",
129
+ },
130
+ {
131
+ id: "expectation/rust",
132
+ file: "tests/integration.rs",
133
+ removed: [" assert_eq!(add(1, 2), 3);"],
134
+ added: [" assert_eq!(add(1, 2), -1);"],
135
+ expect: "ASSERTION_EXPECTATION_CHANGED",
136
+ },
137
+ {
138
+ id: "expectation/go-colocated",
139
+ file: "internal/calc/calc_test.go",
140
+ removed: ['\t\tt.Errorf("got %d want %d", got, 3)'],
141
+ added: ['\t\tt.Errorf("got %d want %d", got, 999)'],
142
+ expect: "ASSERTION_EXPECTATION_CHANGED",
143
+ },
144
+ {
145
+ id: "removal/monorepo",
146
+ file: "packages/api/test/handler.test.js",
147
+ removed: ["expect(handler(req)).toBe(200);", "expect(handler(bad)).toBe(400);"],
148
+ added: [],
149
+ expect: "ASSERTION_REMOVAL",
150
+ },
151
+ // Dialects the guard was measured silent on. `assertEqual` matched only
152
+ // because the pattern's optional dot and case-insensitive flag happened to
153
+ // line up; `assertEquals` — one letter longer — did not, and neither did
154
+ // RSpec's `.to eq(`, PHPUnit's `$this->assertSame`, Minitest's
155
+ // `assert_equal` or XCTest's `XCTAssertEqual`. Every one of them returned
156
+ // PASS with a non-zero denominator, which is the exact shape this file
157
+ // exists to reject, one level down inside the mechanism built to catch it.
158
+ {
159
+ id: "expectation/junit",
160
+ file: "src/test/java/com/x/CalcTest.java",
161
+ removed: [" assertEquals(3, calc.add(1, 2));"],
162
+ added: [" assertEquals(-1, calc.add(1, 2));"],
163
+ expect: "ASSERTION_EXPECTATION_CHANGED",
164
+ },
165
+ {
166
+ id: "expectation/rspec",
167
+ file: "spec/models/user_spec.rb",
168
+ removed: [" expect(user.age).to eq(30)"],
169
+ added: [" expect(user.age).to eq(-1)"],
170
+ expect: "ASSERTION_EXPECTATION_CHANGED",
171
+ },
172
+ {
173
+ id: "expectation/phpunit",
174
+ file: "tests/Unit/CalcTest.php",
175
+ removed: [" $this->assertSame(3, $c->add(1, 2));"],
176
+ added: [" $this->assertSame(-1, $c->add(1, 2));"],
177
+ expect: "ASSERTION_EXPECTATION_CHANGED",
178
+ },
179
+ {
180
+ id: "expectation/minitest",
181
+ file: "test/user_test.rb",
182
+ removed: [" assert_equal(3, add(1, 2))"],
183
+ added: [" assert_equal(-1, add(1, 2))"],
184
+ expect: "ASSERTION_EXPECTATION_CHANGED",
185
+ },
186
+ {
187
+ id: "expectation/xctest",
188
+ file: "Tests/CalcTests/CalcTests.swift",
189
+ removed: [" XCTAssertEqual(add(1, 2), 3)"],
190
+ added: [" XCTAssertEqual(add(1, 2), -1)"],
191
+ expect: "ASSERTION_EXPECTATION_CHANGED",
192
+ },
193
+ {
194
+ id: "weakening/junit",
195
+ file: "src/test/java/com/x/CalcTest.java",
196
+ removed: [" assertEquals(3, calc.add(1, 2));"],
197
+ added: [" assertTrue(calc.add(1, 2) != null);"],
198
+ expect: "ASSERTION_WEAKENED",
199
+ },
200
+ {
201
+ id: "removal/phpunit",
202
+ file: "tests/Unit/CalcTest.php",
203
+ removed: [" $this->assertSame(3, $c->add(1, 2));", " $this->assertSame(4, $c->add(2, 2));"],
204
+ added: [],
205
+ expect: "ASSERTION_REMOVAL",
206
+ },
207
+ {
208
+ id: "vacuous/xctest",
209
+ file: "Tests/CalcTests/CalcTests.swift",
210
+ removed: [" XCTAssertEqual(add(1, 2), 3)"],
211
+ added: [" XCTAssertTrue(true)"],
212
+ expect: "VACUOUS_ASSERTION",
213
+ },
214
+ // Skip injection is the same blindness in the other five ecosystems: a
215
+ // suite that never runs cannot fail, and `@Disabled` is as effective as
216
+ // `it.skip` at making that happen.
217
+ {
218
+ id: "skip-injection/junit",
219
+ file: "src/test/java/com/x/CalcTest.java",
220
+ removed: [],
221
+ added: [" @Disabled(\"flaky\")", " void addsTwoNumbers() {"],
222
+ expect: "TEST_SKIP_INJECTION",
223
+ },
224
+ {
225
+ id: "skip-injection/rspec",
226
+ file: "spec/models/user_spec.rb",
227
+ removed: [],
228
+ added: [" xit \"computes the age\" do"],
229
+ expect: "TEST_SKIP_INJECTION",
230
+ },
231
+ {
232
+ id: "skip-injection/phpunit",
233
+ file: "tests/Unit/CalcTest.php",
234
+ removed: [],
235
+ added: [" $this->markTestSkipped(\"later\");"],
236
+ expect: "TEST_SKIP_INJECTION",
237
+ },
238
+ {
239
+ id: "skip-injection/xctest",
240
+ file: "Tests/CalcTests/CalcTests.swift",
241
+ removed: [],
242
+ added: [" throw XCTSkip(\"not now\")"],
243
+ expect: "TEST_SKIP_INJECTION",
244
+ },
245
+ ];
246
+
247
+
248
+ /**
249
+ * Mutants of the applicability predicate.
250
+ *
251
+ * Each one must kill at least one canary. A mutant that survives means no
252
+ * canary ever required the guard to *activate* — the suite would stay green if
253
+ * the guard silently stopped looking, which is precisely the defect.
254
+ *
255
+ * These are hand-written rather than generated: the original bug was not an
256
+ * untested branch, it was a branch nobody wrote, and no mutation operator
257
+ * invents the case the code never handled.
258
+ */
259
+ export const PREDICATE_MUTANTS = [
260
+ { id: "alwaysFalse", fn: () => false, why: "the guard looks at nothing" },
261
+ { id: "rootBlind", fn: (p) => String(p).includes("/test/"), why: "the original substring bug" },
262
+ { id: "nodeOnly", fn: (p) => /\.(test|spec)\.[jt]sx?$/.test(String(p)), why: "only the layout the code was written from" },
263
+ { id: "caseSensitive", fn: (p) => String(p) === String(p).toLowerCase() && /(^|\/)tests?\//.test(String(p)), why: "case and separator drift" },
264
+ ];
265
+
266
+ /** Runner outputs that state zero collected tests, per ecosystem. */
267
+ export const EMPTY_RUN_CANARIES = [
268
+ { id: "pytest", output: "collected 0 items\n\nno tests ran in 0.01s" },
269
+ { id: "jest", output: "No tests found, exiting with code 0" },
270
+ { id: "vitest", output: "No test files found, exiting with code 0" },
271
+ { id: "cargo", output: "running 0 tests\ntest result: ok. 0 passed" },
272
+ { id: "mocha", output: " 0 passing (1ms)" },
273
+ { id: "go", output: "? example.com/app\t[no test files]" },
274
+ { id: "surefire", output: "Tests run: 0, Failures: 0, Errors: 0, Skipped: 0" },
275
+ { id: "gradle", output: "> Task :test NO-SOURCE" },
276
+ { id: "phpunit", output: "No tests executed!" },
277
+ { id: "rspec", output: "0 examples, 0 failures" },
278
+ { id: "dotnet", output: "Total tests: 0" },
279
+ { id: "xctest", output: "Executed 0 tests" },
280
+ { id: "ctest", output: "No tests were found!!!" },
281
+ ];
282
+
283
+ /** Paths the policy says an agent must never modify without an override. */
284
+ export const SCOPE_CANARIES = [
285
+ { path: ".github/workflows/ci.yml", rule: "deny", why: "runs with repo credentials" },
286
+ { path: ".gitlab-ci.yml", rule: "deny", why: "same, another forge" },
287
+ { path: "Jenkinsfile", rule: "deny", why: "same, another forge" },
288
+ { path: ".envrc", rule: "deny", why: "direnv executes it on cd" },
289
+ { path: "package-lock.json", rule: "protect", why: "decides which code installs" },
290
+ { path: "Cargo.lock", rule: "protect", why: "same, another ecosystem" },
291
+ { path: "conftest.py", rule: "protect", why: "runs before every pytest collection" },
292
+ { path: "jest.config.js", rule: "protect", why: "decides which tests run" },
293
+ { path: "CODEOWNERS", rule: "protect", why: "decides who must approve" },
294
+ ];
295
+
296
+ /**
297
+ * Edits that must produce no finding at all.
298
+ *
299
+ * A guard that answers "yes" to everything has no more discrimination than
300
+ * one that answers "no" to everything, and it is worse in practice: the
301
+ * operator learns to pass the override without reading it, and the day it
302
+ * reports something real, nobody looks. Every entry here is an edit an
303
+ * honest agent makes constantly.
304
+ *
305
+ * The first two are not hypothetical. `//` begins with a division sign, so a
306
+ * comment line matched the operator-continuation test and folded itself into
307
+ * the assertion above it — which meant *adding* an assertion next to a
308
+ * comment was reported as rewriting an expectation. Python was immune,
309
+ * because `#` is not an operator, so the fixtures this project was written
310
+ * from never showed it.
311
+ */
312
+ export const INNOCENT_EDITS = [
313
+ {
314
+ id: "add-assertion-beside-comment/node",
315
+ file: "test/calc.test.js",
316
+ context: "// arithmetic",
317
+ removed: [" assert.equal(add(1, 2), 3);"],
318
+ added: [" assert.equal(add(1, 2), 3);", " assert.equal(add(2, 2), 4);"],
319
+ why: "adding a test is the behaviour the gate exists to encourage",
320
+ },
321
+ {
322
+ id: "add-assertion-beside-comment/rust",
323
+ file: "tests/integration.rs",
324
+ context: "// arithmetic",
325
+ removed: [" assert!(add(1, 2) == 3);"],
326
+ added: [" assert!(add(1, 2) == 3);", " assert!(add(2, 2) == 4);"],
327
+ why: "same edit, same comment syntax, another ecosystem",
328
+ },
329
+ {
330
+ id: "reorder-assertions/pytest",
331
+ file: "tests/test_calc.py",
332
+ context: "# arithmetic",
333
+ removed: [" assert a() == 1", " assert b() == 2"],
334
+ added: [" assert b() == 2", " assert a() == 1"],
335
+ why: "moving an assertion changes nothing it checks",
336
+ },
337
+ {
338
+ id: "reindent/pytest",
339
+ file: "tests/test_calc.py",
340
+ context: "# arithmetic",
341
+ removed: [" assert add(1, 2) == 3"],
342
+ added: [" assert add(1, 2) == 3"],
343
+ why: "a formatter run must not read as tampering",
344
+ },
345
+ {
346
+ id: "reword-message/rspec",
347
+ file: "spec/models/user_spec.rb",
348
+ context: "# age",
349
+ removed: [' expect(u.age).to eq(30), "wrong"'],
350
+ added: [' expect(u.age).to eq(30), "unexpected age"'],
351
+ why: "RSpec writes the message outside the call, where an argument check cannot see it",
352
+ },
353
+ {
354
+ id: "reword-message/minitest",
355
+ file: "test/user_test.rb",
356
+ context: "# age",
357
+ removed: [' assert_equal 3, add(1, 2), "wrong"'],
358
+ added: [' assert_equal 3, add(1, 2), "unexpected sum"'],
359
+ why: "same, with the message in argument position and no parentheses",
360
+ },
361
+ {
362
+ id: "reword-message/node",
363
+ file: "test/calc.test.js",
364
+ context: "// arithmetic",
365
+ removed: [' assert.equal(add(1, 2), 3, "wrong");'],
366
+ added: [' assert.equal(add(1, 2), 3, "unexpected sum");'],
367
+ why: "rewording a failure message says nothing about what is checked",
368
+ },
369
+ {
370
+ id: "rename-test/junit",
371
+ file: "src/test/java/com/x/CalcTest.java",
372
+ context: "// arithmetic",
373
+ removed: [" void addsNumbers() {"],
374
+ added: [" void addsTwoNumbers() {"],
375
+ why: "a test name is not an expectation",
376
+ },
377
+ {
378
+ id: "change-import/node",
379
+ file: "test/calc.test.js",
380
+ context: "// setup",
381
+ removed: ['const calc = require("./calc");'],
382
+ added: ['const calc = require("../src/calc");'],
383
+ why: "`require(` is an import, not a claim — the loose net must not read it as one",
384
+ },
385
+ {
386
+ id: "rename-helper/node",
387
+ file: "test/calc.test.js",
388
+ context: "// setup",
389
+ removed: [" const shouldRetry = false;"],
390
+ added: [" const shouldRetry = true;"],
391
+ why: "`shouldRetry` and `expected` are identifiers; reading them as assertions makes the dialect warning worthless",
392
+ },
393
+ ];
394
+
395
+ /**
396
+ * Dialects the guard genuinely cannot parse, which it must say out loud.
397
+ *
398
+ * This is the case the whole denominator exists for. A JUnit diff used to
399
+ * return `PASS` with `inputsSeen: 1` while not one assertion in it had been
400
+ * recognised — a verdict indistinguishable from a clean Node suite. Coverage
401
+ * will always end somewhere; what must never happen again is that the edge
402
+ * is silent.
403
+ */
404
+ export const UNREADABLE_DIALECTS = [
405
+ {
406
+ id: "hspec",
407
+ file: "tests/CalcSpec.hs",
408
+ context: "-- arithmetic",
409
+ removed: [" calc `shouldBe` 3"],
410
+ added: [" calc `shouldBe` (-1)"],
411
+ why: "an infix assertion with no parentheses anywhere near it",
412
+ },
413
+ {
414
+ id: "googletest",
415
+ file: "tests/calc_test.cc",
416
+ context: "// arithmetic",
417
+ removed: [" EXPECT_EQ(add(1, 2), 3);"],
418
+ added: [" EXPECT_EQ(add(1, 2), -1);"],
419
+ why: "a macro dialect the pattern list does not cover",
420
+ },
421
+ ];
422
+
423
+ /**
424
+ * Import forms the package-integrity extractor must find.
425
+ *
426
+ * The first pass of that check reported "every relative import resolves" on
427
+ * a package whose newest script could not start: its matcher was written as
428
+ * `[^;\n]*?from`, and the import it needed to see spanned several lines. The
429
+ * check was confidently green about a file that threw ERR_MODULE_NOT_FOUND
430
+ * on load — the same failure the tool exists to prevent, committed by the
431
+ * tool's own integrity check.
432
+ *
433
+ * Every entry is a source fragment and the specifiers it must yield.
434
+ */
435
+ export const IMPORT_EXTRACTION_CASES = [
436
+ { id: "single-line named", src: 'import { a, b } from "./one.mjs";', expect: ["./one.mjs"] },
437
+ {
438
+ id: "multi-line named",
439
+ src: 'import {\n a,\n b,\n} from "./two.mjs";',
440
+ expect: ["./two.mjs"],
441
+ why: "the form the first version of the check could not see",
442
+ },
443
+ { id: "default", src: 'import three from "./three.mjs";', expect: ["./three.mjs"] },
444
+ { id: "namespace", src: 'import * as four from "./four.mjs";', expect: ["./four.mjs"] },
445
+ { id: "side-effect only", src: 'import "./five.mjs";', expect: ["./five.mjs"] },
446
+ { id: "re-export", src: 'export { six } from "./six.mjs";', expect: ["./six.mjs"] },
447
+ { id: "re-export all", src: 'export * from "./seven.mjs";', expect: ["./seven.mjs"] },
448
+ { id: "dynamic", src: 'const m = await import("./eight.mjs");', expect: ["./eight.mjs"] },
449
+ { id: "require", src: 'const nine = require("./nine.js");', expect: ["./nine.js"] },
450
+ { id: "single quotes", src: "import ten from './ten.mjs';", expect: ["./ten.mjs"] },
451
+ {
452
+ id: "bare specifiers are not files",
453
+ src: 'import { readFileSync } from "node:fs";\nimport x from "some-package";',
454
+ expect: ["node:fs", "some-package"],
455
+ why: "found, then ignored by the resolver — never resolved against the tarball",
456
+ },
457
+ ];
458
+
459
+ // Cases that exercise the mask rather than the matcher. Appended separately
460
+ // because each one is a fixture *about* fixtures: the check has to tell a
461
+ // module reference from a picture of one.
462
+ IMPORT_EXTRACTION_CASES.push(
463
+ {
464
+ id: "regex holding a quote, then a real import",
465
+ src: 'const q = /["\']/;\nimport x from "./after-regex.mjs";',
466
+ expect: ["./after-regex.mjs"],
467
+ why: "the phantom string this file's own subject matter opens",
468
+ },
469
+ {
470
+ id: "an import quoted inside a string",
471
+ src: 'const example = \'import a from "./not-real.mjs";\';',
472
+ expect: [],
473
+ why: "a picture of an import is not an import",
474
+ },
475
+ {
476
+ id: "an import written in a comment",
477
+ src: '// import a from "./commented.mjs";\nconst x = 1;',
478
+ expect: [],
479
+ why: "same, in the other place examples live",
480
+ }
481
+ );
package/src/security.mjs CHANGED
@@ -1155,14 +1155,42 @@ function locateFindingLine(lines, type, file = null) {
1155
1155
  // (`expect(\n formatInvoice(bill)\n).toBe(…`) still recognises the chain.
1156
1156
  // The bound is a guess: an argument list longer than 240 characters is
1157
1157
  // rarer than a missed chain.
1158
+ // The dialect list is not decoration. `assertEqual` was recognised only
1159
+ // because `\.?` made the dot optional and the `i` flag let `Equal` match
1160
+ // `equal`; `assertEquals`, one letter longer, fell out of the pattern and
1161
+ // took JUnit, PHPUnit, Minitest, RSpec and XCTest with it. The weak forms
1162
+ // — assertTrue, assertNotNull, XCTAssertTrue — are deliberately absent:
1163
+ // they state no expected value, so their arrival in place of one of these
1164
+ // is a weakening, which is a finding of its own.
1158
1165
  const SPECIFIC_ASSERTION = new RegExp(
1159
1166
  [
1160
1167
  "\\bassert(?:\\.strict)?\\.?(?:strictEqual|deepStrictEqual|deepEqual|notStrictEqual|notDeepStrictEqual|equal|notEqual|match|doesNotMatch|throws|rejects|doesNotThrow)\\s*\\(",
1161
1168
  "\\bexpect\\s*\\([\\s\\S]{0,240}?\\)\\s*\\.(?:toBe|toEqual|toStrictEqual|toMatch|toMatchObject|toContain|toHaveBeenCalledWith|toThrow|toHaveLength|toBeCloseTo)\\s*\\(",
1162
1169
  "\\bassert\\.(?:equals|deepEquals|include|lengthOf)\\s*\\(",
1163
1170
  "assert_eq!|assert_ne!",
1171
+ // The bare comparison form. `assert add(1, 2) == 3` is how pytest is
1172
+ // actually written, and Rust's `assert!(a == b)` and Elixir's
1173
+ // `assert f(x) == 3` follow it; none of them name a comparison
1174
+ // function, so a list of function names could never reach them.
1175
+ // Equality only. `assert!(x != 0)` names no expected value — it is the
1176
+ // weaker claim you arrive at by giving one up, and counting it as
1177
+ // specific would make the downgrade from `assert_eq!(x, 5)` invisible to
1178
+ // the weakening check.
1179
+ "\\bassert\\s+[^\\n]*(?:===|==)(?!=)",
1180
+ "\\bassert!\\s*\\([^\\n]*==(?!=)",
1164
1181
  "\\bt\\.(?:Errorf|Fatalf)\\s*\\(",
1165
1182
  "\\brequire\\.(?:Equal|NotEqual|Len|Contains|Error|NoError)\\s*\\(",
1183
+ // Python unittest, stated rather than inherited from the optional dot.
1184
+ "\\bassert(?:Equal|NotEqual|AlmostEqual|NotAlmostEqual|Regex|NotRegex|Raises|In|NotIn|Is|IsNot|ListEqual|DictEqual|SetEqual|TupleEqual|CountEqual|Greater|Less|GreaterEqual|LessEqual)\\s*\\(",
1185
+ // JUnit / TestNG / PHPUnit
1186
+ "\\bassert(?:Equals|NotEquals|Same|NotSame|ArrayEquals|IterableEquals|LinesMatch|Count|StringContainsString|StringEqualsFile|InstanceOf|Contains|Throws)\\s*\\(",
1187
+ "\\bassertThat\\s*\\([\\s\\S]{0,240}?\\)\\s*\\.(?:isEqualTo|isSameAs|contains|containsExactly|hasSize|isCloseTo|matches)\\s*\\(",
1188
+ // Minitest
1189
+ "\\b(?:assert|refute)_(?:equal|includes|match|nil|same|in_delta|in_epsilon|raises|empty|operator|predicate)\\b",
1190
+ // RSpec
1191
+ "\\bexpect\\s*\\([\\s\\S]{0,240}?\\)\\s*\\.(?:to|not_to|to_not)\\s+(?:eq|eql|equal|be|be_within|match|include|contain_exactly|match_array|have_attributes|raise_error|start_with|end_with)\\b",
1192
+ // XCTest
1193
+ "\\bXCTAssert(?:Equal|NotEqual|EqualWithAccuracy|Identical|NotIdentical|GreaterThan|LessThan|GreaterThanOrEqual|LessThanOrEqual|ThrowsError|NoThrow)\\s*\\(",
1166
1194
  ].join("|"),
1167
1195
  "i"
1168
1196
  );
@@ -1203,6 +1231,15 @@ const TEST_LANG_BY_EXT = new Map([
1203
1231
  [".py", "python"], [".pyi", "python"],
1204
1232
  [".go", "go"],
1205
1233
  [".rs", "rust"],
1234
+ // Approximations, chosen for comment and continuation syntax rather than
1235
+ // for kinship: the C-like family reads correctly under the `js` scanner,
1236
+ // and Ruby under the `python` one because both end a comment at `#` and a
1237
+ // statement at the newline. Naming them beats falling through to `js` by
1238
+ // default, which is how a `#` comment came to be read as code.
1239
+ [".java", "js"], [".kt", "js"], [".kts", "js"], [".scala", "js"], [".groovy", "js"],
1240
+ [".swift", "js"], [".cs", "js"], [".php", "js"], [".c", "js"], [".cc", "js"],
1241
+ [".cpp", "js"], [".h", "js"], [".hpp", "js"], [".m", "js"], [".sol", "js"],
1242
+ [".rb", "python"],
1206
1243
  ]);
1207
1244
 
1208
1245
  function langForTestFile(file) {
@@ -1540,6 +1577,21 @@ function stripComments(text, lang) {
1540
1577
  const CONTINUATION_START = /^[)\],.]/;
1541
1578
  const CONTINUATION_OP_START = /^[+\-*/%<>=&|^:]/;
1542
1579
 
1580
+ // A comment is not a continuation, however much it looks like one.
1581
+ //
1582
+ // `//` begins with a division sign and `--` with a minus, so both matched
1583
+ // CONTINUATION_OP_START and folded the following comment line into the
1584
+ // statement above it. The cost was a false accusation on a virtuous act:
1585
+ // adding an assertion next to a `// ...` line made the new assertion absorb
1586
+ // the comment, stop matching its unchanged twin, and get reported as a
1587
+ // rewritten expectation. Python was unaffected only because `#` is not an
1588
+ // operator — which is why the same fixture passed in pytest and failed in
1589
+ // Jest, and why it survived every suite written against the pytest layout.
1590
+ //
1591
+ // A comment inside an open delimiter still joins: `cur.delta > 0` decides
1592
+ // that before this test is ever reached.
1593
+ const COMMENT_LINE_START = /^(?:\/\/|\/\*|#|--)/;
1594
+
1543
1595
  // A scanner miscount (an unbalanced delimiter inside a regex literal is the
1544
1596
  // usual cause) must not be able to merge a whole file into one statement,
1545
1597
  // which would pair *any* literal change anywhere in the file.
@@ -1578,14 +1630,15 @@ function assembleStatements(sliceLines, lang) {
1578
1630
 
1579
1631
  for (const L of sliceLines) {
1580
1632
  const trimmed = L.text.replace(/^\s+/, "");
1633
+ const startsComment = COMMENT_LINE_START.test(trimmed);
1581
1634
  const joins =
1582
1635
  cur !== null &&
1583
1636
  (cur.delta > 0 ||
1584
1637
  cur.state.str !== null ||
1585
1638
  cur.state.block > 0 ||
1586
1639
  cur.trailingBackslash ||
1587
- CONTINUATION_START.test(trimmed) ||
1588
- CONTINUATION_OP_START.test(trimmed));
1640
+ (!startsComment &&
1641
+ (CONTINUATION_START.test(trimmed) || CONTINUATION_OP_START.test(trimmed))));
1589
1642
 
1590
1643
  if (
1591
1644
  joins &&
@@ -1648,7 +1701,17 @@ function splitAssertionArgs(clean, lang) {
1648
1701
  const m = SPECIFIC_ASSERTION.exec(clean);
1649
1702
  if (!m) return null;
1650
1703
 
1651
- let i = m.index + m[0].length; // just past the opening paren
1704
+ // Not every branch of SPECIFIC_ASSERTION ends at an opening paren:
1705
+ // `assert_eq!`, `assert_equal` and RSpec's `.to eq` all match a bare name.
1706
+ // Starting the walk one character early made every argument boundary wrong,
1707
+ // so a reworded message read as a rewritten value.
1708
+ let i = m.index + m[0].length;
1709
+ if (clean[i - 1] !== "(") {
1710
+ let j = i;
1711
+ while (j < clean.length && /\s/.test(clean[j])) j++;
1712
+ if (clean[j] !== "(") return null;
1713
+ i = j + 1;
1714
+ }
1652
1715
  let depth = 1;
1653
1716
  let quote = null;
1654
1717
  let triple = false;
@@ -1740,7 +1803,45 @@ function messageArgIndices(args) {
1740
1803
  * every time somebody improved the wording of a failure. Firing on that is
1741
1804
  * how an operator learns to pass the override without reading it.
1742
1805
  */
1806
+ /**
1807
+ * Split a statement at a trailing `, "message"` written outside the call.
1808
+ *
1809
+ * RSpec puts the message there — `expect(x).to eq(3), "explain"` — and so do
1810
+ * Ruby and Elixir assertions generally. An argument-position check can never
1811
+ * see it, so rewording one read as a rewritten expectation.
1812
+ */
1813
+ function splitTrailingMessage(clean) {
1814
+ let depth = 0;
1815
+ let quote = null;
1816
+ let lastComma = -1;
1817
+ for (let i = 0; i < clean.length; i++) {
1818
+ const c = clean[i];
1819
+ if (quote !== null) {
1820
+ if (c === "\\") { i += 1; continue; }
1821
+ if (c === quote) quote = null;
1822
+ continue;
1823
+ }
1824
+ if (c === '"' || c === "'" || c === "`") { quote = c; continue; }
1825
+ if (c === "(" || c === "[" || c === "{") depth++;
1826
+ else if (c === ")" || c === "]" || c === "}") depth--;
1827
+ else if (c === "," && depth === 0) lastComma = i;
1828
+ }
1829
+ if (lastComma === -1) return { head: clean, msg: null };
1830
+ const tail = clean.slice(lastComma + 1).trim();
1831
+ if (!isPureStringLiteral(tail)) return { head: clean, msg: null };
1832
+ return { head: clean.slice(0, lastComma), msg: tail };
1833
+ }
1834
+
1743
1835
  function differsOnlyInMessage(cleanRemoved, cleanAdded, lang) {
1836
+ const ta = splitTrailingMessage(cleanRemoved);
1837
+ const tb = splitTrailingMessage(cleanAdded);
1838
+ if (
1839
+ (ta.msg !== null || tb.msg !== null) &&
1840
+ ta.head.replace(/\s+/g, "") === tb.head.replace(/\s+/g, "")
1841
+ ) {
1842
+ return true;
1843
+ }
1844
+
1744
1845
  const a = splitAssertionArgs(cleanRemoved, lang);
1745
1846
  const b = splitAssertionArgs(cleanAdded, lang);
1746
1847
  if (!a || !b || a.length !== b.length || a.length === 0) return false;
@@ -1988,7 +2089,9 @@ export function resolveAllowedTamperKinds(options = {}) {
1988
2089
  * @param {string} diffOrText - Unified git diff
1989
2090
  * @param {Object} [options]
1990
2091
  * @param {boolean} [options.allowTestModifications=false]
1991
- * @returns {{ ok: boolean, violations: Array<object>, inputsSeen: number, status: "PASS"|"FAIL"|"NOT_APPLICABLE" }}
2092
+ * @returns {{ ok: boolean, violations: Array<object>, inputsSeen: number, filesSeen: number,
2093
+ * assertionsSeen: number, unreadable: Array<{file: string, count: number, samples: string[]}>,
2094
+ * status: "PASS"|"FAIL"|"UNREADABLE"|"NOT_APPLICABLE" }}
1992
2095
  * `status` distinguishes "checked and clean" from "nothing was checked";
1993
2096
  * `ok: true` alone cannot, and that ambiguity is the defect class this
1994
2097
  * field exists to make visible.
@@ -2024,9 +2127,24 @@ export function checkTestTampering(diffOrText = "", options = {}) {
2024
2127
  { pattern: /#\[ignore\]/i, desc: "Injected Rust test ignore attribute (#[ignore])" },
2025
2128
  { pattern: /\/\/\s*@ts-ignore/i, desc: "Injected TypeScript ignore comment (// @ts-ignore)" },
2026
2129
  { pattern: /\/\/\s*@ts-nocheck/i, desc: "Injected TypeScript nocheck directive (// @ts-nocheck)" },
2130
+ // A suite that never runs cannot fail, and `@Disabled` is exactly as
2131
+ // effective at arranging that as `it.skip` is.
2132
+ { pattern: /@Disabled\b/, desc: "Injected JUnit 5 disable annotation (@Disabled)" },
2133
+ { pattern: /@Ignore\b/, desc: "Injected JUnit 4 / TestNG ignore annotation (@Ignore)" },
2134
+ { pattern: /@Test\s*\([^)]*enabled\s*=\s*false/i, desc: "Injected TestNG disabled test (enabled = false)" },
2135
+ { pattern: /@unittest\.skip/i, desc: "Injected unittest skip decorator (@unittest.skip)" },
2136
+ { pattern: /\bmarkTest(?:Skipped|Incomplete)\s*\(/i, desc: "Injected PHPUnit skip (markTestSkipped())" },
2137
+ { pattern: /\bXCTSkip(?:If|Unless|IfNot)?\s*\(/, desc: "Injected XCTest skip (XCTSkip())" },
2138
+ { pattern: /\b(?:xit|xdescribe|xcontext|xspecify)\b\s*["\x27]/i, desc: "Injected RSpec disabled example (xit)" },
2139
+ { pattern: /,\s*skip:\s*(?:true|["\x27])/i, desc: "Injected RSpec skip metadata (skip:)" },
2140
+ { pattern: /^\s*(?:skip|pending)\s*(?:["\x27(]|$)/i, desc: "Injected Minitest/RSpec skip statement" },
2027
2141
  ];
2028
2142
 
2029
- const COMMENTED_ASSERTION = /^\+\s*(?:\/\/|\/\*)\s*(?:expect\(|assert\.|assert\(|t\.expect|t\.assert)/i;
2143
+ // `#` and `--` belong here for the same reason the dialects belong in
2144
+ // ASSERTION_PATTERN: a Ruby or Python assertion commented out is exactly
2145
+ // as gone as a JavaScript one, and was previously not looked for.
2146
+ const COMMENTED_ASSERTION =
2147
+ /^\+\s*(?:\/\/|\/\*|#|--)\s*(?:expect\s*\(|assert(?!ion|ing|ed\b|s\b)[a-zA-Z0-9_$]*\s*[.(]|assert\s|refute_|XCTAssert|t\.expect|t\.assert)/i;
2030
2148
 
2031
2149
  const VACUOUS_ASSERTIONS = [
2032
2150
  { pattern: /\bassert(?:\.ok)?\s*\(\s*true\s*(?:,[^)]*)?\)/i, desc: "Vacuous truth assertion (assert.ok(true))" },
@@ -2037,11 +2155,44 @@ export function checkTestTampering(diffOrText = "", options = {}) {
2037
2155
  { pattern: /\bexpect\s*\(\s*false\s*\)\s*\.toBeFalsy\s*\(/i, desc: "Vacuous falsity expectation (expect(false).toBeFalsy())" },
2038
2156
  { pattern: /\bassert\.(?:isTrue|isOk)\s*\(\s*true\s*(?:,[^)]*)?\)/i, desc: "Vacuous truth assertion (assert.isTrue(true))" },
2039
2157
  { pattern: /\bassert\.(?:isFalse|isNotOk)\s*\(\s*false\s*(?:,[^)]*)?\)/i, desc: "Vacuous falsity assertion (assert.isFalse(false))" },
2158
+ { pattern: /\b(?:XCT)?assertTrue\s*\(\s*true\s*[,)]/i, desc: "Vacuous truth assertion (assertTrue(true))" },
2159
+ { pattern: /\b(?:XCT)?assertFalse\s*\(\s*false\s*[,)]/i, desc: "Vacuous falsity assertion (assertFalse(false))" },
2160
+ { pattern: /\b(?:assertEquals|assertSame|XCTAssertEqual)\s*\(\s*([^,]+?)\s*,\s*\1\s*[,)]/i, desc: "Vacuous identity assertion (assertEquals(X, X))" },
2161
+ { pattern: /\bassert_equal\s*\(?\s*([^,]+?)\s*,\s*\1\s*\)?\s*$/i, desc: "Vacuous identity assertion (assert_equal X, X)" },
2162
+ { pattern: /\bexpect\s*\(\s*true\s*\)\s*\.to\s+be(?:\s+true)?\b/i, desc: "Vacuous truth expectation (expect(true).to be true)" },
2040
2163
  ];
2041
2164
 
2042
- const ASSERTION_PATTERN = /(?:\b(?:assert(?:\.[a-zA-Z0-9_$]+)?|expect|t\.(?:assert|expect|is|equal|true|false|Errorf|Fatalf)|require\.[a-zA-Z0-9_$]+)\b|assert!|assert_eq!|assert_ne!)/i;
2165
+ // Broad on purpose: this is the denominator, not the verdict. A word
2166
+ // boundary immediately after `assert` never falls in `assertEquals`,
2167
+ // `assert_equal` or `XCTAssertEqual`, so five ecosystems contributed no
2168
+ // assertions to count at all and a gutted JUnit suite was arithmetically
2169
+ // indistinguishable from an untouched one. The lookahead keeps prose and
2170
+ // identifiers — `assertion`, `asserts`, `asserted` — out of the count.
2171
+ const ASSERTION_PATTERN =
2172
+ /(?:\b(?:assert(?!ion|ing|ed\b|s\b)[a-zA-Z0-9_$]*(?:\.[a-zA-Z0-9_$]+)?|refute[a-zA-Z0-9_$]*|XCTAssert[a-zA-Z0-9_$]*|XCTFail|expect|t\.(?:assert|expect|is|equal|true|false|Errorf|Fatalf)|require\.[a-zA-Z0-9_$]+)\b|assert!|assert_eq!|assert_ne!)/i;
2173
+ // The loose net. Not a verdict and never a block — its only job is to
2174
+ // notice that a line was plainly an assertion in *some* dialect that
2175
+ // ASSERTION_PATTERN did not recognise. Without it, adding the seventh
2176
+ // ecosystem is indistinguishable from having covered it all along: the
2177
+ // guard returns the same clean PASS either way. This is the denominator
2178
+ // for the denominator.
2179
+ // Deliberately not call-shaped. Haskell's `x `shouldBe` 3` is an
2180
+ // assertion with no parentheses anywhere near it, and a net that only
2181
+ // catches `name(` reports the same confident PASS on it as on a clean
2182
+ // Node suite. `require` and `check` are absent on purpose: in a
2183
+ // CommonJS test file `require("./calc")` is an import, not a claim.
2184
+ const ASSERTION_SHAPED =
2185
+ /\b(?:assert(?!ion|ing|ed\b|s\b)|expect(?!ed\b|ation)|refute)[a-zA-Z0-9_$]*\b|`\s*should[a-zA-Z0-9_$]*\s*`|\b(?:should|must|verify|ensure|confirm)[a-zA-Z0-9_$]*\s*[(!]|\.\s*(?:should|to|to_not|not_to|must)\b|\bBOOST_[A-Z_]+\s*\(|\b[A-Z]+_(?:EQ|NE|TRUE|FALSE|THAT)\s*\(/;
2043
2186
  const isCommentLine = (str) => /^\s*(?:\/\/|\/\*|\*|#|--|;)/.test(str);
2044
2187
 
2188
+ /** Book-keeping only: what this run looked at, before deciding anything. */
2189
+ const countExamined = (stats, text) => {
2190
+ if (!text.trim() || isCommentLine(text)) return;
2191
+ stats.examined++;
2192
+ if (ASSERTION_PATTERN.test(text)) stats.recognised++;
2193
+ else if (ASSERTION_SHAPED.test(text) && stats.unreadable.length < 5) stats.unreadable.push(text.trim().slice(0, 120));
2194
+ };
2195
+
2045
2196
  const fileAssertions = new Map();
2046
2197
  let pendingHunk = false;
2047
2198
 
@@ -2077,7 +2228,7 @@ export function checkTestTampering(diffOrText = "", options = {}) {
2077
2228
  }
2078
2229
 
2079
2230
  if (!fileAssertions.has(currentFile)) {
2080
- fileAssertions.set(currentFile, { removed: [], added: 0, addedTexts: [], removedSpecific: [], addedSpecific: 0, hunks: [] });
2231
+ fileAssertions.set(currentFile, { removed: [], added: 0, addedTexts: [], removedSpecific: [], addedSpecific: 0, hunks: [], examined: 0, recognised: 0, unreadable: [] });
2081
2232
  }
2082
2233
  const fileStats = fileAssertions.get(currentFile);
2083
2234
  if (pendingHunk) {
@@ -2089,6 +2240,7 @@ export function checkTestTampering(diffOrText = "", options = {}) {
2089
2240
  if (line.startsWith("-") && !line.startsWith("---")) {
2090
2241
  const deletedText = line.slice(1);
2091
2242
  if (hunk) hunk.lines.push({ kind: "-", text: deletedText, oldNo: currentOldLineNo, newNo: null });
2243
+ countExamined(fileStats, deletedText);
2092
2244
  if (!isCommentLine(deletedText) && ASSERTION_PATTERN.test(deletedText)) {
2093
2245
  fileStats.removed.push({ line: currentOldLineNo, text: deletedText });
2094
2246
  if (isSpecificAssertion(deletedText)) {
@@ -2099,6 +2251,7 @@ export function checkTestTampering(diffOrText = "", options = {}) {
2099
2251
  } else if (line.startsWith("+") && !line.startsWith("+++")) {
2100
2252
  const addedText = line.slice(1);
2101
2253
  if (hunk) hunk.lines.push({ kind: "+", text: addedText, oldNo: null, newNo: currentNewLineNo });
2254
+ countExamined(fileStats, addedText);
2102
2255
  let isVacuous = false;
2103
2256
 
2104
2257
  // Check skip injections
@@ -2225,17 +2378,44 @@ export function checkTestTampering(diffOrText = "", options = {}) {
2225
2378
  // `ok: true` from a guard that looked at everything and approved it. That
2226
2379
  // ambiguity is how a substring bug in the file classifier switched this
2227
2380
  // entire guard off for the standard pytest, Rust and RSpec layouts while
2228
- // every signal stayed green. A verdict without a denominator is not a
2229
- // verdict, so `inputsSeen` reports the number of test files this run
2230
- // actually reasoned about, and `status` distinguishes "nothing to check"
2231
- // from "checked and clean".
2232
- const inputsSeen = fileAssertions.size;
2381
+ // every signal stayed green.
2382
+ //
2383
+ // Counting *files* was not enough. A JUnit diff that rewrote an expected
2384
+ // value produced `inputsSeen: 1` and a clean PASS while not one assertion
2385
+ // in it had been recognised — the same ambiguity, one level down, inside
2386
+ // the mechanism built to remove it. So the denominator is now the thing
2387
+ // the rules actually consume: lines examined, and of those, assertions
2388
+ // understood. `UNREADABLE` is the state that has no business being silent
2389
+ // — assertion-shaped lines were present and none of them parsed, which
2390
+ // means this repository speaks a dialect the guard does not.
2391
+ let examined = 0;
2392
+ let assertionsSeen = 0;
2393
+ const unreadable = [];
2394
+ for (const [file, stats] of fileAssertions.entries()) {
2395
+ examined += stats.examined;
2396
+ assertionsSeen += stats.recognised;
2397
+ if (stats.unreadable.length > 0) {
2398
+ unreadable.push({ file, count: stats.unreadable.length, samples: stats.unreadable.slice(0, 3) });
2399
+ }
2400
+ }
2401
+
2402
+ const status =
2403
+ reported.length > 0
2404
+ ? "FAIL"
2405
+ : assertionsSeen === 0 && unreadable.length > 0
2406
+ ? "UNREADABLE"
2407
+ : examined > 0
2408
+ ? "PASS"
2409
+ : "NOT_APPLICABLE";
2233
2410
 
2234
2411
  return {
2235
2412
  ok: reported.length === 0,
2236
2413
  violations: reported,
2237
- inputsSeen,
2238
- status: reported.length > 0 ? "FAIL" : inputsSeen > 0 ? "PASS" : "NOT_APPLICABLE",
2414
+ inputsSeen: examined,
2415
+ filesSeen: fileAssertions.size,
2416
+ assertionsSeen,
2417
+ unreadable,
2418
+ status,
2239
2419
  };
2240
2420
  }
2241
2421