jules-orchestrator-kit 0.65.0 → 0.67.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/agentctl.mjs +24 -3
- package/package.json +1 -1
- package/scripts/guard-reach-check.mjs +31 -3
- package/scripts/run-tests.mjs +18 -1
- package/src/engine.mjs +7 -0
- package/src/guard-policy.mjs +147 -0
- package/src/memory.mjs +0 -0
- package/src/ops/test-collection.mjs +18 -1
- package/src/security.mjs +159 -6
package/README.md
CHANGED
|
@@ -207,7 +207,7 @@ To maximize PR merge rates, dispatch tasks according to deterministic boundaries
|
|
|
207
207
|
* **Fail-Closed Security & Secret Redaction:** Evaluates explicit Deny rules before Allow rules against canonicalized, case-folded paths. Redacts high-entropy keys and base64-encoded credentials (such as Kubernetes `Secret` manifests).
|
|
208
208
|
* **Complexity & Cost Router:** Zero-dependency heuristic classifier (`src/router.mjs`) routing mechanical tasks to lightweight models while reserving primary models for complex refactors, with a `node --check` syntax-verification gate that transparently escalates a FAST-tier result to the primary provider if it left broken JS on disk.
|
|
209
209
|
* **Terminal UI & Diagnostic Matrix (`agentctl doctor`):** Interactive terminal dashboard, task sidecar manager, and automated transactional self-repair.
|
|
210
|
-
* **Verified Test Suite:** Tested with **
|
|
210
|
+
* **Verified Test Suite:** Tested with **1154 unit tests across 166 suites passing in < 15.0s**.
|
|
211
211
|
|
|
212
212
|
<br/>
|
|
213
213
|
|
package/bin/agentctl.mjs
CHANGED
|
@@ -425,10 +425,23 @@ async function main() {
|
|
|
425
425
|
console.log(`-----------------------------------------------------`);
|
|
426
426
|
for (const p of res.phases) {
|
|
427
427
|
const status = p.ok ? "✅ PASS" : "❌ FAIL";
|
|
428
|
-
|
|
428
|
+
// Test tampering shares this phase, and its exit code, with the
|
|
429
|
+
// secret scanner. The codes are frozen, but reporting an assertion
|
|
430
|
+
// deletion under a bare "SECRETS" heading sent operators looking for
|
|
431
|
+
// a credential to rotate. Name what actually failed.
|
|
432
|
+
const tamperOnly =
|
|
433
|
+
p.phase === "secrets" &&
|
|
434
|
+
!p.ok &&
|
|
435
|
+
(p.findings || []).length > 0 &&
|
|
436
|
+
(p.findings || []).every((f) => f.type !== "SECRET_DETECTED" && !/SECRET/i.test(f.type || ""));
|
|
437
|
+
const label = tamperOnly ? "SECRETS (test integrity — no secret found)" : p.phase.toUpperCase();
|
|
438
|
+
console.log(` Phase [${label}] : ${status}`);
|
|
429
439
|
if (p.violations) {
|
|
430
440
|
p.violations.forEach((v) => console.log(` - Violation: ${v.file} (Rule: ${v.rule})`));
|
|
431
441
|
}
|
|
442
|
+
if (p.unverified) {
|
|
443
|
+
console.log(` - Unverified: ${p.unverified}`);
|
|
444
|
+
}
|
|
432
445
|
if (p.setup) {
|
|
433
446
|
console.log(` - Setup: accepted ${p.setup.length} gate scaffold file(s) this repository did not have yet`);
|
|
434
447
|
p.setup.forEach((f) => console.log(` ${f}`));
|
|
@@ -2667,13 +2680,21 @@ async function main() {
|
|
|
2667
2680
|
const taskId = values.task || "unknown";
|
|
2668
2681
|
const agent = values.agent || "jules";
|
|
2669
2682
|
|
|
2683
|
+
const { CONFIRM_AFTER } = await import("../src/memory.mjs");
|
|
2670
2684
|
const res = harvestFailure(root, { exitCode, logPath, diffText, taskId, agent });
|
|
2671
2685
|
if (values.json) {
|
|
2672
2686
|
console.log(JSON.stringify(res, null, 2));
|
|
2673
2687
|
} else {
|
|
2674
2688
|
if (res.status === "HARVESTED") {
|
|
2675
|
-
|
|
2676
|
-
|
|
2689
|
+
// No "Solution:" line any more, because there is no solution: the
|
|
2690
|
+
// repair loop exhausting its budget is the definition of not having
|
|
2691
|
+
// found one. It used to print a fixed sentence here and store it as
|
|
2692
|
+
// if it were a remedy.
|
|
2693
|
+
console.log(`🌾 Recorded failure observation: ${res.candidate.trigger}`);
|
|
2694
|
+
console.log(` Seen ${res.occurrences}× — ${res.confirmed ? "stated as a rule" : `not injected into prompts until it recurs ${CONFIRM_AFTER}×`}`);
|
|
2695
|
+
if (!res.confirmed) {
|
|
2696
|
+
console.log(` Record an actual fix for it with: agentctl learning add "<trigger>" "<solution>"`);
|
|
2697
|
+
}
|
|
2677
2698
|
} else {
|
|
2678
2699
|
console.log(`⚠️ Harvest rejected: ${res.reason}`);
|
|
2679
2700
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "jules-orchestrator-kit",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.67.0",
|
|
4
4
|
"description": "Zero-dependency safety gatekeeper, test oracle generator, and multi-agent coordination protocol for autonomous coding agents — Google Jules, Claude Code, Codex and Gemini CLI.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -49,12 +49,14 @@ import { parseCollectedTests } from "../src/ops/test-collection.mjs";
|
|
|
49
49
|
import {
|
|
50
50
|
TEST_PATH_CASES,
|
|
51
51
|
TAMPER_CANARIES,
|
|
52
|
+
MULTILINE_CANARIES,
|
|
52
53
|
PREDICATE_MUTANTS,
|
|
53
54
|
EMPTY_RUN_CANARIES,
|
|
54
55
|
COUNTED_RUN_CANARIES,
|
|
55
56
|
SCOPE_CANARIES,
|
|
56
57
|
INNOCENT_EDITS,
|
|
57
58
|
UNREADABLE_DIALECTS,
|
|
59
|
+
CROSS_FILE_CASES,
|
|
58
60
|
} from "../src/guard-policy.mjs";
|
|
59
61
|
|
|
60
62
|
/**
|
|
@@ -68,6 +70,9 @@ import {
|
|
|
68
70
|
function canaryDiff(c) {
|
|
69
71
|
const ctx = c.context || "// context";
|
|
70
72
|
const lines = [`--- a/${c.file}`, `+++ b/${c.file}`, "@@ -1,20 +1,20 @@", ` ${ctx}`];
|
|
73
|
+
// Unchanged lines the change sits inside. An assertion whose keyword is on
|
|
74
|
+
// one of these is the shape the line-level denominator could not see.
|
|
75
|
+
for (const l of c.lead || []) lines.push(` ${l}`);
|
|
71
76
|
for (const l of c.removed) lines.push(`-${l}`);
|
|
72
77
|
for (const l of c.added) lines.push(`+${l}`);
|
|
73
78
|
lines.push(` ${ctx}`);
|
|
@@ -130,7 +135,7 @@ const canaryResults = new Map();
|
|
|
130
135
|
const silent = [];
|
|
131
136
|
const noDenominator = [];
|
|
132
137
|
const noAssertions = [];
|
|
133
|
-
for (const c of TAMPER_CANARIES) {
|
|
138
|
+
for (const c of [...TAMPER_CANARIES, ...MULTILINE_CANARIES]) {
|
|
134
139
|
const res = checkTestTampering(canaryDiff(c));
|
|
135
140
|
const hit = (res.violations || []).some((v) => v.type === c.expect);
|
|
136
141
|
canaryResults.set(c.id, hit);
|
|
@@ -144,7 +149,7 @@ const canaryResults = new Map();
|
|
|
144
149
|
noAssertions.push(`${c.id} (${res.assertionsSeen} assertions parsed)`);
|
|
145
150
|
}
|
|
146
151
|
}
|
|
147
|
-
add("canaries: every tamper rule still fires", silent.length === 0, silent.length ? silent.join("; ") : `${TAMPER_CANARIES.length} canaries red as required`);
|
|
152
|
+
add("canaries: every tamper rule still fires", silent.length === 0, silent.length ? silent.join("; ") : `${TAMPER_CANARIES.length + MULTILINE_CANARIES.length} canaries red as required`);
|
|
148
153
|
add("canaries: every finding carries a denominator", noDenominator.length === 0, noDenominator.length ? noDenominator.join(", ") : "inputsSeen > 0 on every hit");
|
|
149
154
|
add("canaries: assertion rules parsed an assertion", noAssertions.length === 0, noAssertions.length ? noAssertions.join(", ") : "assertionsSeen > 0 on every assertion finding");
|
|
150
155
|
}
|
|
@@ -181,12 +186,35 @@ const canaryResults = new Map();
|
|
|
181
186
|
);
|
|
182
187
|
}
|
|
183
188
|
|
|
189
|
+
{
|
|
190
|
+
const wrong = [];
|
|
191
|
+
for (const c of CROSS_FILE_CASES) {
|
|
192
|
+
const lines = [];
|
|
193
|
+
for (const f of c.files) {
|
|
194
|
+
lines.push(`--- a/${f.file}`, `+++ b/${f.file}`, "@@ -1,20 +1,20 @@", ` ${c.context}`);
|
|
195
|
+
for (const l of f.removed) lines.push(`-${l}`);
|
|
196
|
+
for (const l of f.added) lines.push(`+${l}`);
|
|
197
|
+
lines.push(` ${c.context}`);
|
|
198
|
+
}
|
|
199
|
+
const got = (checkTestTampering(lines.join("\n")).violations || []).map((v) => v.type).sort();
|
|
200
|
+
const want = [...c.expect].sort();
|
|
201
|
+
if (JSON.stringify(got) !== JSON.stringify(want)) {
|
|
202
|
+
wrong.push(`${c.id}: got ${JSON.stringify(got)}, policy says ${JSON.stringify(want)} (${c.why})`);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
add(
|
|
206
|
+
"a move across files is judged as a move",
|
|
207
|
+
wrong.length === 0,
|
|
208
|
+
wrong.length ? wrong.join("; ") : `${CROSS_FILE_CASES.length} two-file diffs judged as declared`
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
|
|
184
212
|
// --- 4. Predicate mutants ---------------------------------------------------
|
|
185
213
|
{
|
|
186
214
|
const survivors = [];
|
|
187
215
|
for (const mutant of PREDICATE_MUTANTS) {
|
|
188
216
|
let killed = false;
|
|
189
|
-
for (const c of TAMPER_CANARIES) {
|
|
217
|
+
for (const c of [...TAMPER_CANARIES, ...MULTILINE_CANARIES]) {
|
|
190
218
|
// Only canaries the healthy predicate catches can kill a mutant.
|
|
191
219
|
if (!canaryResults.get(c.id)) continue;
|
|
192
220
|
const res = checkTestTampering(canaryDiff(c), { isTestPath: mutant.fn });
|
package/scripts/run-tests.mjs
CHANGED
|
@@ -1,11 +1,28 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { readdirSync } from "node:fs";
|
|
2
|
+
import { readdirSync, existsSync } from "node:fs";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { spawn, spawnSync } from "node:child_process";
|
|
5
5
|
import { cpus } from "node:os";
|
|
6
6
|
|
|
7
7
|
const root = process.cwd();
|
|
8
8
|
const testDir = join(root, "test");
|
|
9
|
+
|
|
10
|
+
// The published package ships `scripts/` and not `test/`, so running
|
|
11
|
+
// `npm test` inside an installed copy crashed with a raw
|
|
12
|
+
// `ENOENT: no such file or directory, scandir '.../test'`. The suite is not
|
|
13
|
+
// missing; it was never part of the tarball. Say that, rather than leaving
|
|
14
|
+
// someone to work it out from a stack trace.
|
|
15
|
+
if (!existsSync(testDir)) {
|
|
16
|
+
console.error("No test/ directory here.");
|
|
17
|
+
console.error("");
|
|
18
|
+
console.error("If this is an installed copy of jules-orchestrator-kit, that is expected:");
|
|
19
|
+
console.error("the test suite is not part of the published package. Clone the repository");
|
|
20
|
+
console.error("to run it: git clone https://github.com/FullThrottle83/jules-orchestrator-kit");
|
|
21
|
+
console.error("");
|
|
22
|
+
console.error("To check an installed copy instead, run: npm run guard-reach");
|
|
23
|
+
process.exit(1);
|
|
24
|
+
}
|
|
25
|
+
|
|
9
26
|
const testFiles = readdirSync(testDir)
|
|
10
27
|
.filter((f) => f.endsWith(".test.mjs"))
|
|
11
28
|
.sort()
|
package/src/engine.mjs
CHANGED
|
@@ -692,6 +692,13 @@ export async function gate(opts = {}) {
|
|
|
692
692
|
postTestHash,
|
|
693
693
|
tamperDetected: testTampered,
|
|
694
694
|
},
|
|
695
|
+
// Verified by exit code alone. Not a failure, and not an approval either:
|
|
696
|
+
// an unrecognised runner passes on purpose, but the operator has to be
|
|
697
|
+
// able to tell that apart from a suite the gate actually counted.
|
|
698
|
+
...(collectionFloor.unverified ? { unverified: collectionFloor.note } : {}),
|
|
699
|
+
...(collectionFloor.count !== null && collectionFloor.count !== undefined
|
|
700
|
+
? { testsCollected: collectionFloor.count, runner: collectionFloor.runner }
|
|
701
|
+
: {}),
|
|
695
702
|
});
|
|
696
703
|
phases.push({
|
|
697
704
|
phase: "evidence",
|
package/src/guard-policy.mjs
CHANGED
|
@@ -235,6 +235,58 @@ export const TAMPER_CANARIES = [
|
|
|
235
235
|
added: [" $this->markTestSkipped(\"later\");"],
|
|
236
236
|
expect: "TEST_SKIP_INJECTION",
|
|
237
237
|
},
|
|
238
|
+
// Skipping from inside the test body, which is how every one of these
|
|
239
|
+
// ecosystems actually does it. The decorator and annotation forms were
|
|
240
|
+
// covered; the standard library's own method was not, so a test could be
|
|
241
|
+
// silenced with the most ordinary call in the language.
|
|
242
|
+
{
|
|
243
|
+
id: "skip-injection/unittest-body",
|
|
244
|
+
file: "tests/test_app.py",
|
|
245
|
+
context: "# app",
|
|
246
|
+
removed: [],
|
|
247
|
+
added: [' self.skipTest("disabled")'],
|
|
248
|
+
expect: "TEST_SKIP_INJECTION",
|
|
249
|
+
},
|
|
250
|
+
{
|
|
251
|
+
id: "skip-injection/pytest-body",
|
|
252
|
+
file: "tests/test_app.py",
|
|
253
|
+
context: "# app",
|
|
254
|
+
removed: [],
|
|
255
|
+
added: [' pytest.skip("not now")'],
|
|
256
|
+
expect: "TEST_SKIP_INJECTION",
|
|
257
|
+
},
|
|
258
|
+
{
|
|
259
|
+
id: "skip-injection/raise-skiptest",
|
|
260
|
+
file: "tests/test_app.py",
|
|
261
|
+
context: "# app",
|
|
262
|
+
removed: [],
|
|
263
|
+
added: [' raise unittest.SkipTest("nope")'],
|
|
264
|
+
expect: "TEST_SKIP_INJECTION",
|
|
265
|
+
},
|
|
266
|
+
{
|
|
267
|
+
id: "skip-injection/mocha-body",
|
|
268
|
+
file: "test/app.test.js",
|
|
269
|
+
context: "// app",
|
|
270
|
+
removed: [],
|
|
271
|
+
added: [" this.skip();"],
|
|
272
|
+
expect: "TEST_SKIP_INJECTION",
|
|
273
|
+
},
|
|
274
|
+
{
|
|
275
|
+
id: "skip-injection/jest-todo",
|
|
276
|
+
file: "test/app.test.js",
|
|
277
|
+
context: "// app",
|
|
278
|
+
removed: [],
|
|
279
|
+
added: [' test.todo("adds two numbers");'],
|
|
280
|
+
expect: "TEST_SKIP_INJECTION",
|
|
281
|
+
},
|
|
282
|
+
{
|
|
283
|
+
id: "skip-injection/go-skipnow",
|
|
284
|
+
file: "internal/calc/calc_test.go",
|
|
285
|
+
context: "// calc",
|
|
286
|
+
removed: [],
|
|
287
|
+
added: ["\tt.SkipNow()"],
|
|
288
|
+
expect: "TEST_SKIP_INJECTION",
|
|
289
|
+
},
|
|
238
290
|
{
|
|
239
291
|
id: "skip-injection/xctest",
|
|
240
292
|
file: "Tests/CalcTests/CalcTests.swift",
|
|
@@ -382,6 +434,15 @@ export const INNOCENT_EDITS = [
|
|
|
382
434
|
added: ['const calc = require("../src/calc");'],
|
|
383
435
|
why: "`require(` is an import, not a claim — the loose net must not read it as one",
|
|
384
436
|
},
|
|
437
|
+
{
|
|
438
|
+
id: "reword-comment-inside-assertion/node",
|
|
439
|
+
file: "test/calc.test.js",
|
|
440
|
+
context: "// arithmetic",
|
|
441
|
+
lead: [" assert.equal(", " add(1, 2),"],
|
|
442
|
+
removed: [" 3, // the answer", " );"],
|
|
443
|
+
added: [" 3, // the correct answer", " );"],
|
|
444
|
+
why: "line comments were never stripped — `copyCode(i)` left `pending` at the comment and the copy after the loop put it back",
|
|
445
|
+
},
|
|
385
446
|
{
|
|
386
447
|
id: "rename-helper/node",
|
|
387
448
|
file: "test/calc.test.js",
|
|
@@ -510,3 +571,89 @@ export const COUNTED_RUN_CANARIES = [
|
|
|
510
571
|
why: "Go's own `ok <package>` line, which a bare `^ok\\s` confused with TAP's `ok 1 - name`",
|
|
511
572
|
},
|
|
512
573
|
];
|
|
574
|
+
|
|
575
|
+
/**
|
|
576
|
+
* Changes that sit inside an assertion whose keyword never appears in the diff.
|
|
577
|
+
*
|
|
578
|
+
* Counting `+`/`-` lines answered zero here, and nothing among the changed
|
|
579
|
+
* lines looked assertion-shaped either, so the guard reported a clean PASS on
|
|
580
|
+
* a five-element expected list rewritten to one element to match broken
|
|
581
|
+
* output. Measured on a real repository: five green phases, `APPROVED`.
|
|
582
|
+
*
|
|
583
|
+
* The statement machinery that pairs rewrites already assembles context lines
|
|
584
|
+
* together with changed ones. The denominator has to use it.
|
|
585
|
+
*/
|
|
586
|
+
export const MULTILINE_CANARIES = [
|
|
587
|
+
{
|
|
588
|
+
id: "expectation/python-multiline",
|
|
589
|
+
file: "tests/test_headers.py",
|
|
590
|
+
context: "# headers",
|
|
591
|
+
lead: [" def test_parse(self):", " self.assertEqual(", " _parse_http_header(x),"],
|
|
592
|
+
removed: [' [("text/xml", {}),', ' ("text/plain", {}),', ' ("more", {})])'],
|
|
593
|
+
added: [' [("text/xml", {"tampered": True})])'],
|
|
594
|
+
expect: "ASSERTION_EXPECTATION_CHANGED",
|
|
595
|
+
},
|
|
596
|
+
{
|
|
597
|
+
id: "expectation/js-multiline",
|
|
598
|
+
file: "test/headers.test.js",
|
|
599
|
+
context: "// headers",
|
|
600
|
+
lead: [" assert.deepStrictEqual(", " parse(input),"],
|
|
601
|
+
removed: [" [{ type: \"a\" }, { type: \"b\" }, { type: \"c\" }]", " );"],
|
|
602
|
+
added: [" [{ type: \"a\" }]", " );"],
|
|
603
|
+
expect: "ASSERTION_EXPECTATION_CHANGED",
|
|
604
|
+
},
|
|
605
|
+
];
|
|
606
|
+
|
|
607
|
+
/**
|
|
608
|
+
* Diffs that touch two files at once, where the verdict depends on both.
|
|
609
|
+
*
|
|
610
|
+
* Assertion tracking is per file, so moving a test between files — ordinary
|
|
611
|
+
* refactoring, done constantly — read as deleting every assertion in it and
|
|
612
|
+
* was reported as CRITICAL tampering. The assertion still exists and still
|
|
613
|
+
* runs. What the removal check is for is verification that *disappeared*.
|
|
614
|
+
*
|
|
615
|
+
* `expect: []` means the whole diff must produce no finding.
|
|
616
|
+
*/
|
|
617
|
+
export const CROSS_FILE_CASES = [
|
|
618
|
+
{
|
|
619
|
+
id: "move-a-test-between-files",
|
|
620
|
+
context: "# ctx",
|
|
621
|
+
files: [
|
|
622
|
+
{
|
|
623
|
+
file: "test/test_app.py",
|
|
624
|
+
removed: [" def test_setattr(self):", " self.assertEqual(5, app.test)", " self.assertRaises(AttributeError, setattr, app, 't', 6)"],
|
|
625
|
+
added: [],
|
|
626
|
+
},
|
|
627
|
+
{
|
|
628
|
+
file: "test/test_html.py",
|
|
629
|
+
removed: [],
|
|
630
|
+
added: [" def test_setattr_moved(self):", " self.assertEqual(5, app.test)", " self.assertRaises(AttributeError, setattr, app, 't', 6)"],
|
|
631
|
+
},
|
|
632
|
+
],
|
|
633
|
+
expect: [],
|
|
634
|
+
why: "a move is not a removal",
|
|
635
|
+
},
|
|
636
|
+
{
|
|
637
|
+
id: "delete-a-test-outright",
|
|
638
|
+
context: "# ctx",
|
|
639
|
+
files: [
|
|
640
|
+
{
|
|
641
|
+
file: "test/test_app.py",
|
|
642
|
+
removed: [" def test_setattr(self):", " self.assertEqual(5, app.test)", " self.assertRaises(AttributeError, setattr, app, 't', 6)"],
|
|
643
|
+
added: [],
|
|
644
|
+
},
|
|
645
|
+
],
|
|
646
|
+
expect: ["ASSERTION_REMOVAL", "ASSERTION_REMOVAL"],
|
|
647
|
+
why: "nothing caught it on the way, so verification really did disappear",
|
|
648
|
+
},
|
|
649
|
+
{
|
|
650
|
+
id: "change-it-on-the-way-across",
|
|
651
|
+
context: "# ctx",
|
|
652
|
+
files: [
|
|
653
|
+
{ file: "test/test_app.py", removed: [" self.assertEqual(5, app.test)"], added: [] },
|
|
654
|
+
{ file: "test/test_html.py", removed: [], added: [" self.assertEqual(99, app.test)"] },
|
|
655
|
+
],
|
|
656
|
+
expect: ["ASSERTION_REMOVAL"],
|
|
657
|
+
why: "what arrived is a different claim; only an exact arrival is a move",
|
|
658
|
+
},
|
|
659
|
+
];
|
package/src/memory.mjs
CHANGED
|
Binary file
|
|
@@ -148,7 +148,24 @@ export function checkCollectionFloor(testResult, opts = {}) {
|
|
|
148
148
|
}
|
|
149
149
|
|
|
150
150
|
const { count, runner } = parseCollectedTests(testResult.stdout, testResult.stderr);
|
|
151
|
-
|
|
151
|
+
|
|
152
|
+
// Deliberately one-sided: only a *stated* zero fails, because failing on
|
|
153
|
+
// "I could not tell" would break every runner not on the list. But passing
|
|
154
|
+
// and saying nothing makes `echo "all tests passed"` indistinguishable from
|
|
155
|
+
// a real suite, so the absence of evidence is reported as an absence.
|
|
156
|
+
if (count === null) {
|
|
157
|
+
return {
|
|
158
|
+
ok: true,
|
|
159
|
+
count: null,
|
|
160
|
+
runner: null,
|
|
161
|
+
reason: null,
|
|
162
|
+
unverified: true,
|
|
163
|
+
note:
|
|
164
|
+
`The verification command exited 0, but no recognised test runner stated how many tests it ran, ` +
|
|
165
|
+
`so the gate cannot tell a full suite from a command that ran nothing. Verified by exit code alone.`,
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
if (count >= minTests) {
|
|
152
169
|
return { ok: true, count, runner, reason: null };
|
|
153
170
|
}
|
|
154
171
|
|
package/src/security.mjs
CHANGED
|
@@ -1523,8 +1523,13 @@ function stripComments(text, lang) {
|
|
|
1523
1523
|
continue;
|
|
1524
1524
|
}
|
|
1525
1525
|
|
|
1526
|
-
|
|
1527
|
-
|
|
1526
|
+
// `pending = n` is the whole fix. `copyCode(i)` copies the code up to
|
|
1527
|
+
// the comment and leaves `pending` sitting at its start; the
|
|
1528
|
+
// `copyCode(n)` after this loop then copied the comment straight back
|
|
1529
|
+
// in, so no line comment has ever been stripped. Block comments were,
|
|
1530
|
+
// which is why `/* … */` behaved and `// …` did not.
|
|
1531
|
+
if (c === "/" && c2 === "/") { copyCode(i); pending = n; break; }
|
|
1532
|
+
if (lang === "python" && c === "#") { copyCode(i); pending = n; break; }
|
|
1528
1533
|
if (c === "/" && c2 === "*") {
|
|
1529
1534
|
copyCode(i);
|
|
1530
1535
|
state.block = 1;
|
|
@@ -1929,6 +1934,9 @@ function detectExpectationRewrites(file, hunks, stats, violations) {
|
|
|
1929
1934
|
}
|
|
1930
1935
|
|
|
1931
1936
|
const pairs = [];
|
|
1937
|
+
const pairedOld = new Set();
|
|
1938
|
+
const pairedNew = new Set();
|
|
1939
|
+
const cancelled = new Set();
|
|
1932
1940
|
for (const [shape, olds] of oldByShape) {
|
|
1933
1941
|
const news = newByShape.get(shape) || [];
|
|
1934
1942
|
|
|
@@ -1946,20 +1954,69 @@ function detectExpectationRewrites(file, hunks, stats, violations) {
|
|
|
1946
1954
|
const survivingOld = [];
|
|
1947
1955
|
for (const o of olds) {
|
|
1948
1956
|
const twin = survivingNew.findIndex((n) => n.canon === o.canon);
|
|
1949
|
-
if (twin === -1)
|
|
1950
|
-
|
|
1957
|
+
if (twin === -1) {
|
|
1958
|
+
survivingOld.push(o);
|
|
1959
|
+
} else {
|
|
1960
|
+
// Present unchanged on both sides: this assertion did not change,
|
|
1961
|
+
// and the argument-level pass below must not be allowed to pair it
|
|
1962
|
+
// with something else and call that a rewrite.
|
|
1963
|
+
cancelled.add(o);
|
|
1964
|
+
cancelled.add(survivingNew[twin]);
|
|
1965
|
+
survivingNew.splice(twin, 1);
|
|
1966
|
+
}
|
|
1951
1967
|
}
|
|
1952
1968
|
|
|
1953
1969
|
const k = Math.min(survivingOld.length, survivingNew.length);
|
|
1954
1970
|
for (let t = 0; t < k; t++) {
|
|
1955
1971
|
const r = survivingOld[t];
|
|
1956
1972
|
const a = survivingNew[t];
|
|
1973
|
+
// Whatever this pass decides about a pair — reported, or deliberately
|
|
1974
|
+
// let go as a reorder or a reworded message — is the decision. The
|
|
1975
|
+
// argument-level pass below exists only for statements whose shapes
|
|
1976
|
+
// differ so much that they never met in a bucket here; letting it
|
|
1977
|
+
// re-open a case that was already judged turned a comment edit into a
|
|
1978
|
+
// rewritten expectation.
|
|
1979
|
+
pairedOld.add(r);
|
|
1980
|
+
pairedNew.add(a);
|
|
1957
1981
|
if (r.canon === a.canon) continue;
|
|
1958
1982
|
if (differsOnlyInMessage(r.clean, a.clean, lang)) continue;
|
|
1959
1983
|
pairs.push({ r: r.s, a: a.s });
|
|
1960
1984
|
}
|
|
1961
1985
|
}
|
|
1962
1986
|
|
|
1987
|
+
// Same assertion, same subject, different expected value.
|
|
1988
|
+
//
|
|
1989
|
+
// Shape pairing compares the statement with its literals blanked, so it
|
|
1990
|
+
// only ever matched assertions whose structure survived the edit. Shrink
|
|
1991
|
+
// a five-element expected list to one element to match broken output and
|
|
1992
|
+
// the two images land in different shape buckets, never pair, and the
|
|
1993
|
+
// rewrite is not reported at all — measured on a real repository, where
|
|
1994
|
+
// it collected five green phases.
|
|
1995
|
+
//
|
|
1996
|
+
// The arguments are the better witness here: when both sides call the
|
|
1997
|
+
// same assertion with the same number of arguments and the *subject*
|
|
1998
|
+
// argument is untouched, what changed is what the test expects of it.
|
|
1999
|
+
for (const r of oldCands) {
|
|
2000
|
+
if (pairedOld.has(r) || cancelled.has(r)) continue;
|
|
2001
|
+
const ra = splitAssertionArgs(r.clean, lang);
|
|
2002
|
+
if (!ra || ra.length < 2) continue;
|
|
2003
|
+
for (const a of newCands) {
|
|
2004
|
+
if (pairedNew.has(a) || cancelled.has(a)) continue;
|
|
2005
|
+
const aa = splitAssertionArgs(a.clean, lang);
|
|
2006
|
+
if (!aa || aa.length !== ra.length) continue;
|
|
2007
|
+
const same = (i) => ra[i].replace(/\s+/g, "") === aa[i].replace(/\s+/g, "");
|
|
2008
|
+
// The subject has to be the same expression, or these are two
|
|
2009
|
+
// different assertions that merely resemble each other.
|
|
2010
|
+
if (!same(0)) continue;
|
|
2011
|
+
if (ra.every((_, i) => same(i))) continue;
|
|
2012
|
+
if (differsOnlyInMessage(r.clean, a.clean, lang)) continue;
|
|
2013
|
+
pairs.push({ r: r.s, a: a.s });
|
|
2014
|
+
pairedOld.add(r);
|
|
2015
|
+
pairedNew.add(a);
|
|
2016
|
+
break;
|
|
2017
|
+
}
|
|
2018
|
+
}
|
|
2019
|
+
|
|
1963
2020
|
// Zero-context hunk: each image is a single fragment and the assertion
|
|
1964
2021
|
// keyword may sit outside the hunk entirely. The fragment pair is taken
|
|
1965
2022
|
// only when both sides normalize to the same shape *and* that shape
|
|
@@ -2138,6 +2195,17 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
2138
2195
|
{ pattern: /\b(?:xit|xdescribe|xcontext|xspecify)\b\s*["\x27]/i, desc: "Injected RSpec disabled example (xit)" },
|
|
2139
2196
|
{ pattern: /,\s*skip:\s*(?:true|["\x27])/i, desc: "Injected RSpec skip metadata (skip:)" },
|
|
2140
2197
|
{ pattern: /^\s*(?:skip|pending)\s*(?:["\x27(]|$)/i, desc: "Injected Minitest/RSpec skip statement" },
|
|
2198
|
+
// Skipping from inside the body, which is how these ecosystems actually
|
|
2199
|
+
// do it. Only the decorator and annotation forms were covered, so a test
|
|
2200
|
+
// could be silenced with the standard library's own method and the guard
|
|
2201
|
+
// said nothing: measured silent on six of seven in-body forms.
|
|
2202
|
+
{ pattern: /\bself\.skipTest\s*\(/i, desc: "Injected unittest skip (self.skipTest())" },
|
|
2203
|
+
{ pattern: /\braise\s+(?:unittest\.)?SkipTest\b/i, desc: "Injected unittest skip (raise SkipTest)" },
|
|
2204
|
+
{ pattern: /\bpytest\.skip\s*\(/i, desc: "Injected Pytest skip call (pytest.skip())" },
|
|
2205
|
+
{ pattern: /\bpytest\.xfail\s*\(/i, desc: "Injected Pytest expected-failure (pytest.xfail())" },
|
|
2206
|
+
{ pattern: /\bthis\.skip\s*\(/i, desc: "Injected Mocha skip (this.skip())" },
|
|
2207
|
+
{ pattern: /\b(?:it|test|describe|context)\.todo\s*\(/i, desc: "Injected todo placeholder (test.todo())" },
|
|
2208
|
+
{ pattern: /\bt\.Skip(?:Now|f)?\s*\(/, desc: "Injected Go test skip (t.Skip/t.Skipf/t.SkipNow)" },
|
|
2141
2209
|
];
|
|
2142
2210
|
|
|
2143
2211
|
// `#` and `--` belong here for the same reason the dialects belong in
|
|
@@ -2312,6 +2380,26 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
2312
2380
|
}
|
|
2313
2381
|
}
|
|
2314
2382
|
|
|
2383
|
+
// Assertions added in some *other* file of this same diff.
|
|
2384
|
+
//
|
|
2385
|
+
// Tracking is strictly per file, so moving a test from one file to another
|
|
2386
|
+
// — ordinary refactoring — read as deleting three assertions and was
|
|
2387
|
+
// reported as CRITICAL tampering. The assertion still exists and still
|
|
2388
|
+
// runs; it is in a different file. What the removal check is for is
|
|
2389
|
+
// verification that *disappeared*, and this has not.
|
|
2390
|
+
//
|
|
2391
|
+
// Deliberately exact on the assertion text: something that changed on the
|
|
2392
|
+
// way across is not a move, and is judged normally.
|
|
2393
|
+
const movedIn = new Map();
|
|
2394
|
+
for (const [file, stats] of fileAssertions.entries()) {
|
|
2395
|
+
for (const t of stats.addedTexts || []) {
|
|
2396
|
+
const key = collapseWhitespace(String(t.text ?? t)).trim();
|
|
2397
|
+
if (!key) continue;
|
|
2398
|
+
if (!movedIn.has(key)) movedIn.set(key, []);
|
|
2399
|
+
movedIn.get(key).push(file);
|
|
2400
|
+
}
|
|
2401
|
+
}
|
|
2402
|
+
|
|
2315
2403
|
for (const [file, stats] of fileAssertions.entries()) {
|
|
2316
2404
|
// An expectation that was rewritten rather than removed.
|
|
2317
2405
|
//
|
|
@@ -2329,6 +2417,15 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
2329
2417
|
if (stats.removed.length > stats.added) {
|
|
2330
2418
|
const unreplaced = stats.removed.slice(stats.added);
|
|
2331
2419
|
for (const item of unreplaced) {
|
|
2420
|
+
// Did this exact assertion arrive somewhere else in the same diff?
|
|
2421
|
+
const key = collapseWhitespace(item.text).trim();
|
|
2422
|
+
const landing = movedIn.get(key);
|
|
2423
|
+
const elsewhere = landing ? landing.findIndex((f) => f !== file) : -1;
|
|
2424
|
+
if (elsewhere !== -1) {
|
|
2425
|
+
// Consume the landing so two removals cannot both claim one arrival.
|
|
2426
|
+
landing.splice(elsewhere, 1);
|
|
2427
|
+
continue;
|
|
2428
|
+
}
|
|
2332
2429
|
violations.push({
|
|
2333
2430
|
file,
|
|
2334
2431
|
line: item.line,
|
|
@@ -2388,21 +2485,57 @@ export function checkTestTampering(diffOrText = "", options = {}) {
|
|
|
2388
2485
|
// understood. `UNREADABLE` is the state that has no business being silent
|
|
2389
2486
|
// — assertion-shaped lines were present and none of them parsed, which
|
|
2390
2487
|
// means this repository speaks a dialect the guard does not.
|
|
2488
|
+
// An assertion is a statement, not a line.
|
|
2489
|
+
//
|
|
2490
|
+
// Counting `+`/`-` lines missed the commonest shape in every language with
|
|
2491
|
+
// multi-line calls: `self.assertEqual(` sits on an unchanged context line
|
|
2492
|
+
// and only its argument lines are edited. Nothing among the changed lines
|
|
2493
|
+
// matched an assertion pattern, nothing looked assertion-shaped either, so
|
|
2494
|
+
// the guard reported `assertionsSeen: 0` and — because no line looked
|
|
2495
|
+
// suspicious — a clean PASS. A five-element expected list rewritten to one
|
|
2496
|
+
// element to match broken output sailed through five green phases.
|
|
2497
|
+
//
|
|
2498
|
+
// The statement machinery that already exists for pairing knows better:
|
|
2499
|
+
// it assembles context lines together with changed ones. Ask it.
|
|
2500
|
+
for (const [file, stats] of fileAssertions.entries()) {
|
|
2501
|
+
const lang = langForTestFile(file);
|
|
2502
|
+
let touched = 0;
|
|
2503
|
+
for (const hunk of stats.hunks) {
|
|
2504
|
+
const oldSlice = [];
|
|
2505
|
+
const newSlice = [];
|
|
2506
|
+
for (const L of hunk.lines) {
|
|
2507
|
+
if (L.kind !== "+") oldSlice.push(L);
|
|
2508
|
+
if (L.kind !== "-") newSlice.push(L);
|
|
2509
|
+
}
|
|
2510
|
+
for (const stmts of [assembleStatements(oldSlice, lang), assembleStatements(newSlice, lang)]) {
|
|
2511
|
+
for (const st of stmts) {
|
|
2512
|
+
const changed = (st.removedLines?.length || 0) + (st.addedLines?.length || 0) > 0;
|
|
2513
|
+
if (changed && ASSERTION_PATTERN.test(stripComments(st.text, lang))) touched++;
|
|
2514
|
+
}
|
|
2515
|
+
}
|
|
2516
|
+
}
|
|
2517
|
+
stats.statementAssertions = touched;
|
|
2518
|
+
}
|
|
2519
|
+
|
|
2391
2520
|
let examined = 0;
|
|
2392
2521
|
let assertionsSeen = 0;
|
|
2393
2522
|
const unreadable = [];
|
|
2394
2523
|
for (const [file, stats] of fileAssertions.entries()) {
|
|
2395
2524
|
examined += stats.examined;
|
|
2396
|
-
assertionsSeen += stats.recognised;
|
|
2525
|
+
assertionsSeen += Math.max(stats.recognised, stats.statementAssertions || 0);
|
|
2397
2526
|
if (stats.unreadable.length > 0) {
|
|
2398
2527
|
unreadable.push({ file, count: stats.unreadable.length, samples: stats.unreadable.slice(0, 3) });
|
|
2399
2528
|
}
|
|
2400
2529
|
}
|
|
2401
2530
|
|
|
2531
|
+
// Changed lines inside a test file, none of them recognisable as part of an
|
|
2532
|
+
// assertion, is not the same as "checked and clean" — it is the state where
|
|
2533
|
+
// this guard has nothing to say. Saying nothing and saying "approved" have
|
|
2534
|
+
// to look different, which is the whole reason `status` exists.
|
|
2402
2535
|
const status =
|
|
2403
2536
|
reported.length > 0
|
|
2404
2537
|
? "FAIL"
|
|
2405
|
-
: assertionsSeen === 0 && unreadable.length > 0
|
|
2538
|
+
: assertionsSeen === 0 && (unreadable.length > 0 || examined > 0)
|
|
2406
2539
|
? "UNREADABLE"
|
|
2407
2540
|
: examined > 0
|
|
2408
2541
|
? "PASS"
|
|
@@ -2480,6 +2613,26 @@ export function scanDiff(diffTextStr = "", options = {}) {
|
|
|
2480
2613
|
}
|
|
2481
2614
|
}
|
|
2482
2615
|
|
|
2616
|
+
// The gate calls `scanDiff`, not `assertTestIntegrity` — so wiring the
|
|
2617
|
+
// dialect warning into the latter meant it reached nobody. The guard
|
|
2618
|
+
// computed `UNREADABLE`, and the operator was shown an unblemished pass.
|
|
2619
|
+
// A boundary that is not reported is not a boundary.
|
|
2620
|
+
if (tamperingRes.status === "UNREADABLE") {
|
|
2621
|
+
const where = (tamperingRes.unreadable || []).map((u) => u.file);
|
|
2622
|
+
const sample = tamperingRes.unreadable?.[0]?.samples?.[0];
|
|
2623
|
+
findings.push({
|
|
2624
|
+
severity: "MEDIUM",
|
|
2625
|
+
type: "TEST_DIALECT_UNREADABLE",
|
|
2626
|
+
file: where[0] ?? null,
|
|
2627
|
+
line: null,
|
|
2628
|
+
description:
|
|
2629
|
+
`Test Tamper Guard: changed ${tamperingRes.inputsSeen} line(s) in ${tamperingRes.filesSeen} test file(s) ` +
|
|
2630
|
+
`and recognised no assertion among them${sample ? ` (e.g. ${JSON.stringify(sample)})` : ""}. ` +
|
|
2631
|
+
`This change was NOT checked for tampering. That is not a failure — an unlisted assertion library is ` +
|
|
2632
|
+
`normal — but it is not an approval either, so it is reported rather than passed silently.`,
|
|
2633
|
+
});
|
|
2634
|
+
}
|
|
2635
|
+
|
|
2483
2636
|
return {
|
|
2484
2637
|
ok: secretsOk && edgeRes.ok && crossPkgRes.ok && tamperingRes.ok,
|
|
2485
2638
|
findings,
|