@orangepro/orangepro-mcp 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +195 -251
- package/dist/local/analyze/analyzer.js +46 -6
- package/dist/local/analyze/treeSitter/engine.js +172 -17
- package/dist/local/autoProve.js +402 -28
- package/dist/local/cli.js +93 -10
- package/dist/local/cliArgs.js +1 -0
- package/dist/local/generate/runHints.js +9 -4
- package/dist/local/graph/factories.js +5 -2
- package/dist/local/ledger.js +1 -1
- package/dist/local/mcp.js +26 -13
- package/dist/local/operations.js +396 -52
- package/dist/local/pack/coverageReport.js +3 -3
- package/dist/local/proofDoctor.js +312 -0
- package/dist/local/rtm.js +40 -10
- package/dist/local/viz/behaviorReportData.js +79 -7
- package/dist/local/viz/behaviorReportHtml.js +526 -615
- package/dist/local/viz/html.js +1 -1
- package/docs/agent-workflow.md +10 -38
- package/docs/agents/claude-code.md +3 -9
- package/docs/agents/codex.md +3 -20
- package/docs/agents/cursor.md +1 -1
- package/docs/agents/opencode.md +1 -1
- package/docs/agents/vscode.md +1 -1
- package/docs/local-proof-kit.md +52 -19
- package/package.json +39 -6
- package/scripts/spikes/go-dynamic-proof-spike.mjs +637 -0
- package/scripts/spikes/go-mutate.go +182 -0
- package/scripts/spikes/java-dynamic-proof-spike.mjs +571 -0
- package/scripts/spikes/java-mutate.mjs +264 -0
- package/scripts/spikes/python-dynamic-proof-spike.mjs +244 -0
- package/scripts/spikes/python-mutate.py +89 -0
|
@@ -0,0 +1,637 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// go-dynamic-proof-spike.mjs — Go dynamic-proof MECHANISM (G-1).
|
|
3
|
+
//
|
|
4
|
+
// Proves ONE free function 0->1 on a single Go module by mutation: it byte-copies
|
|
5
|
+
// the module into a hermetic sandbox, runs `go test -json` baseline, replaces the
|
|
6
|
+
// target function body with a signature-derived sentinel (via go-mutate.go), reruns
|
|
7
|
+
// the SAME test, and classifies. It emits a JSON verdict mirroring the TS/JS spike's
|
|
8
|
+
// shape: { status: "proven" | "associated_survived" | "unrunnable", ... }.
|
|
9
|
+
//
|
|
10
|
+
// TRUST: no false Proven. Both `go test` runs are scoped to the TARGET's PACKAGE
|
|
11
|
+
// ONLY (the directory of --target, `./<dir>` — never `./...`), so a same-named
|
|
12
|
+
// test in another package can never be credited to this target. `proven` requires
|
|
13
|
+
// (0) --test-run binds to EXACTLY ONE target test — a fully anchored plain literal
|
|
14
|
+
// `^TestName$` (broad/regex patterns that could match >1 test are rejected
|
|
15
|
+
// upfront), (a) baseline builds AND that
|
|
16
|
+
// target test PASSES, (b) the mutant builds, (c) the SAME target test FAILS, (d)
|
|
17
|
+
// the failure is TEST-LEVEL — a `fail` Action on the target test that is NOT a
|
|
18
|
+
// build error and NOT a panic, AND (e) it fails at a GENUINE value ASSERTION by
|
|
19
|
+
// SOURCE-LINE BINDING — the failing frame's `file:line` (Go's `\t<file>:<line>:`
|
|
20
|
+
// or testify's `Error Trace:`) is read back in the test SOURCE and must be a real
|
|
21
|
+
// assertion call (`t.Error`/`t.Errorf`, or testify `assert.`/`require.`), NOT a
|
|
22
|
+
// `t.Fatal`/`t.Fatalf`/`t.FailNow`/`t.SkipNow` hard-stop or a helper call. A build
|
|
23
|
+
// error, a panic, a t.Fatal precondition, a setup/helper failure, an unbindable
|
|
24
|
+
// failure, and an ambiguous/method/no-return name all classify as `unrunnable`,
|
|
25
|
+
// never `proven`. An equivalent-value mutation survives -> `associated_survived`.
|
|
26
|
+
//
|
|
27
|
+
// This is a spike harness only: it writes no graph edges or product artifacts and is
|
|
28
|
+
// NOT wired into autoProve / cert / RTM / the mint path. Use only on trusted checkouts
|
|
29
|
+
// for local measurement.
|
|
30
|
+
import { spawnSync } from "node:child_process";
|
|
31
|
+
import { cpSync, existsSync, lstatSync, mkdtempSync, mkdirSync, readFileSync, rmSync, symlinkSync } from "node:fs";
|
|
32
|
+
import { tmpdir } from "node:os";
|
|
33
|
+
import path from "node:path";
|
|
34
|
+
import { fileURLToPath } from "node:url";
|
|
35
|
+
|
|
36
|
+
const DEFAULT_TIMEOUT_MS = 60_000;
|
|
37
|
+
|
|
38
|
+
function usage() {
|
|
39
|
+
return [
|
|
40
|
+
"Usage: node scripts/spikes/go-dynamic-proof-spike.mjs --root <module> --test-run <^TestName$> --target <rel.go> --func <name> [--json]",
|
|
41
|
+
"",
|
|
42
|
+
"Runs a Go baseline test, mutates the target free function body in an isolated byte-copy, reruns the same test, and classifies the result.",
|
|
43
|
+
"--test-run is passed verbatim to `go test -run` and should anchor a single test, e.g. '^TestCompute$'.",
|
|
44
|
+
"--go-assertion-line <n> (optional): 1-based test-source line of the target's assertion. When set, the mutant's failure must bind to a frame at EXACTLY that line and subtest frames are considered — so a runtime-named subtest can prove while a sibling asserting elsewhere is refused.",
|
|
45
|
+
"G-1 scope: FREE FUNCTIONS ONLY. Methods are refused (unrunnable). Equivalent-value mutations survive (associated_survived).",
|
|
46
|
+
"This is a spike harness only; it does not write graph edges or product artifacts and is not wired into prove/RTM/mint."
|
|
47
|
+
].join("\n");
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function parseArgs(argv) {
|
|
51
|
+
const args = { json: false, mode: "sentinel" };
|
|
52
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
53
|
+
const arg = argv[i];
|
|
54
|
+
if (arg === "--json") {
|
|
55
|
+
args.json = true;
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
if (!arg.startsWith("--")) {
|
|
59
|
+
throw new Error(`Unexpected positional argument: ${arg}`);
|
|
60
|
+
}
|
|
61
|
+
const key = arg.slice(2).replace(/-([a-z])/g, (_, c) => c.toUpperCase());
|
|
62
|
+
const value = argv[i + 1];
|
|
63
|
+
if (value === undefined || value.startsWith("--")) {
|
|
64
|
+
throw new Error(`Missing value for ${arg}`);
|
|
65
|
+
}
|
|
66
|
+
args[key] = value;
|
|
67
|
+
i += 1;
|
|
68
|
+
}
|
|
69
|
+
for (const required of ["root", "testRun", "target", "func"]) {
|
|
70
|
+
if (!Object.prototype.hasOwnProperty.call(args, required)) {
|
|
71
|
+
throw new Error(`Missing required --${required.replace(/[A-Z]/g, c => `-${c.toLowerCase()}`)}`);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
if (args.mode !== "sentinel" && args.mode !== "equivalent") {
|
|
75
|
+
throw new Error("--mode must be sentinel or equivalent");
|
|
76
|
+
}
|
|
77
|
+
return args;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function parseTimeoutMs(value) {
|
|
81
|
+
if (value === undefined) {
|
|
82
|
+
return DEFAULT_TIMEOUT_MS;
|
|
83
|
+
}
|
|
84
|
+
const parsed = Number(value);
|
|
85
|
+
if (!Number.isInteger(parsed) || parsed <= 0) {
|
|
86
|
+
throw new Error("--timeout-ms must be a positive integer");
|
|
87
|
+
}
|
|
88
|
+
return parsed;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// Slice 2 (OPTIONAL): 1-based test-source line of the assertion that witnesses the target.
|
|
92
|
+
// When set, the mutant's failure must bind to a frame at EXACTLY this line, and output is
|
|
93
|
+
// collected from the target test AND its subtests (so a runtime-named subtest counts). When
|
|
94
|
+
// absent (undefined), behavior is UNCHANGED: exact `e.Test === name`, any-assertion line.
|
|
95
|
+
function parseAssertionLine(value) {
|
|
96
|
+
if (value === undefined) {
|
|
97
|
+
return undefined;
|
|
98
|
+
}
|
|
99
|
+
const parsed = Number(value);
|
|
100
|
+
if (!Number.isInteger(parsed) || parsed <= 0) {
|
|
101
|
+
throw new Error("--go-assertion-line must be a positive integer");
|
|
102
|
+
}
|
|
103
|
+
return parsed;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function isSecretEnvKey(key) {
|
|
107
|
+
return /TOKEN|SECRET|PASSWORD|API[_-]?KEY|ACCESS[_-]?KEY|PRIVATE[_-]?KEY|SECRET[_-]?KEY|PASSPHRASE|CREDENTIAL|PIN|AUTH|COOKIE|SESSION/i.test(key);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function resolveInside(root, relOrAbs) {
|
|
111
|
+
const resolved = path.resolve(root, relOrAbs);
|
|
112
|
+
const relative = path.relative(root, resolved);
|
|
113
|
+
if (relative.startsWith("..") || path.isAbsolute(relative)) {
|
|
114
|
+
throw new Error(`Path escapes root: ${relOrAbs}`);
|
|
115
|
+
}
|
|
116
|
+
return resolved;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Byte-copy the module into a temp sandbox. Never follow symlinks (a source-repo
|
|
120
|
+
// symlink must not leak an outside dir into the sandbox); exclude .git and
|
|
121
|
+
// .orangepro. node_modules is a JS concept and irrelevant to Go, but excluded for
|
|
122
|
+
// symmetry/safety. Local source is copied, never writable-symlinked.
|
|
123
|
+
function copyModuleRoot(root, label) {
|
|
124
|
+
const tmpRoot = mkdtempSync(path.join(tmpdir(), `opro-go-proof-${label}-`));
|
|
125
|
+
const repoRoot = path.join(tmpRoot, "module");
|
|
126
|
+
cpSync(root, repoRoot, {
|
|
127
|
+
recursive: true,
|
|
128
|
+
filter(source) {
|
|
129
|
+
const name = path.basename(source);
|
|
130
|
+
if (source !== root && lstatSync(source).isSymbolicLink()) {
|
|
131
|
+
return false;
|
|
132
|
+
}
|
|
133
|
+
return name !== "node_modules" && name !== ".git" && name !== ".orangepro";
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
return { tmpRoot, repoRoot };
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Resolve the developer's EXISTING Go module cache so the sandbox module can
|
|
140
|
+
// resolve deps it already downloaded (a real `opro` user has built the repo).
|
|
141
|
+
// `go env GOMODCACHE` is canonical; fall back to $GOMODCACHE then $HOME/go/pkg/mod.
|
|
142
|
+
// Result is cached (queried once). Returns null if nothing resolvable exists, in
|
|
143
|
+
// which case we degrade to a per-run empty cache (self-contained fixtures with no
|
|
144
|
+
// external deps still build). The cache is used READ-ONLY: with GOPROXY=off +
|
|
145
|
+
// -mod=readonly (below) `go` never downloads and so never writes to it — a missing
|
|
146
|
+
// dep errors out instead of mutating the cache.
|
|
147
|
+
let cachedModCache;
|
|
148
|
+
function resolveModCache() {
|
|
149
|
+
if (cachedModCache !== undefined) {
|
|
150
|
+
return cachedModCache;
|
|
151
|
+
}
|
|
152
|
+
let dir = "";
|
|
153
|
+
try {
|
|
154
|
+
const r = spawnSync(goBin(), ["env", "GOMODCACHE"], { encoding: "utf8" });
|
|
155
|
+
if ((r.status ?? 1) === 0) {
|
|
156
|
+
dir = String(r.stdout ?? "").trim();
|
|
157
|
+
}
|
|
158
|
+
} catch {
|
|
159
|
+
dir = "";
|
|
160
|
+
}
|
|
161
|
+
if (!dir) {
|
|
162
|
+
dir = process.env.GOMODCACHE || (process.env.HOME ? path.join(process.env.HOME, "go", "pkg", "mod") : "");
|
|
163
|
+
}
|
|
164
|
+
cachedModCache = dir && existsSync(dir) ? dir : null;
|
|
165
|
+
return cachedModCache;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// A per-run hermetic GOCACHE inside the sandbox tmp dir, plus a sanitized allowlist
|
|
169
|
+
// env. The module cache (GOMODCACHE) is the developer's EXISTING read-only cache so
|
|
170
|
+
// a real repo's already-downloaded deps resolve; with GOPROXY=off + -mod=readonly
|
|
171
|
+
// nothing writes to that cache or mutates go.mod/go.sum. No ambient secrets are
|
|
172
|
+
// forwarded to `go test`: only a fixed set of process-control vars is passed, and
|
|
173
|
+
// any secret-looking key is stripped defensively. This keeps proofs deterministic
|
|
174
|
+
// and prevents credentials from reaching repo test code.
|
|
175
|
+
function hermeticEnv(cacheRoot) {
|
|
176
|
+
const gocache = path.join(cacheRoot, "gocache");
|
|
177
|
+
mkdirSync(gocache, { recursive: true });
|
|
178
|
+
// Reuse the developer's read-only module cache; degrade to a per-run empty one
|
|
179
|
+
// (used only by self-contained fixtures) when none is resolvable.
|
|
180
|
+
let gomodcache = resolveModCache();
|
|
181
|
+
if (!gomodcache) {
|
|
182
|
+
gomodcache = path.join(cacheRoot, "gomodcache");
|
|
183
|
+
mkdirSync(gomodcache, { recursive: true });
|
|
184
|
+
}
|
|
185
|
+
const env = {
|
|
186
|
+
PATH: process.env.PATH ?? "",
|
|
187
|
+
HOME: process.env.HOME ?? "",
|
|
188
|
+
TMPDIR: process.env.TMPDIR ?? tmpdir(),
|
|
189
|
+
GOCACHE: gocache,
|
|
190
|
+
GOMODCACHE: gomodcache,
|
|
191
|
+
// -mod=readonly: never mutate the module or download into the reused cache.
|
|
192
|
+
GOFLAGS: "-mod=readonly",
|
|
193
|
+
GOTOOLCHAIN: "local",
|
|
194
|
+
GOPROXY: "off",
|
|
195
|
+
GONOSUMCHECK: "1",
|
|
196
|
+
GOFLAGS_TEST: "",
|
|
197
|
+
CGO_ENABLED: "0",
|
|
198
|
+
CI: "1",
|
|
199
|
+
NO_COLOR: "1"
|
|
200
|
+
};
|
|
201
|
+
// GOPATH is derived from HOME by default; keep it explicit and inside the sandbox
|
|
202
|
+
// so nothing writes to the developer's real GOPATH.
|
|
203
|
+
env.GOPATH = path.join(cacheRoot, "gopath");
|
|
204
|
+
mkdirSync(env.GOPATH, { recursive: true });
|
|
205
|
+
for (const key of Object.keys(env)) {
|
|
206
|
+
if (isSecretEnvKey(key)) {
|
|
207
|
+
delete env[key];
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
return env;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
function goBin() {
|
|
214
|
+
return process.env.OPRO_GO_BIN || "go";
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function runGoTest({ repoRoot, testRun, timeoutMs, cacheRoot, pkgPath }) {
|
|
218
|
+
const started = performance.now();
|
|
219
|
+
// TRUST (cross-package): scope BOTH runs to the TARGET's package ONLY, never
|
|
220
|
+
// `./...`. Go's `-run '^TestName$'` matches by name, so a same-named test in
|
|
221
|
+
// ANOTHER package could fail the mutant and be miscredited to the target. A
|
|
222
|
+
// single-package path (`./<dir>` — no `...`) makes `-run` reach only the target
|
|
223
|
+
// package's tests. `pkgPath` is derived from --target's directory.
|
|
224
|
+
const result = spawnSync(goBin(), ["test", "-json", "-count=1", "-run", testRun, pkgPath], {
|
|
225
|
+
cwd: repoRoot,
|
|
226
|
+
encoding: "utf8",
|
|
227
|
+
timeout: timeoutMs,
|
|
228
|
+
env: hermeticEnv(cacheRoot),
|
|
229
|
+
maxBuffer: 32 * 1024 * 1024
|
|
230
|
+
});
|
|
231
|
+
const elapsedMs = Math.round(performance.now() - started);
|
|
232
|
+
return {
|
|
233
|
+
exitCode: result.status ?? 1,
|
|
234
|
+
signal: result.signal ?? null,
|
|
235
|
+
timedOut: Boolean(result.error && result.error.code === "ETIMEDOUT"),
|
|
236
|
+
events: parseTestEvents(result.stdout ?? ""),
|
|
237
|
+
stdout: result.stdout ?? "",
|
|
238
|
+
stderr: result.stderr ?? "",
|
|
239
|
+
elapsedMs
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// go test -json emits one JSON object per line. Non-JSON lines (rare, e.g. a raw
|
|
244
|
+
// panic before the framework starts) are ignored here and surface via the raw
|
|
245
|
+
// `hadBuildFailure` / stderr signals instead.
|
|
246
|
+
function parseTestEvents(stdout) {
|
|
247
|
+
const events = [];
|
|
248
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
249
|
+
const trimmed = line.trim();
|
|
250
|
+
if (!trimmed.startsWith("{")) {
|
|
251
|
+
continue;
|
|
252
|
+
}
|
|
253
|
+
try {
|
|
254
|
+
events.push(JSON.parse(trimmed));
|
|
255
|
+
} catch {
|
|
256
|
+
// ignore malformed line
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
return events;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// Resolve --test-run to EXACTLY ONE target test name, or null if the pattern is
|
|
263
|
+
// broad/ambiguous. TRUST: a broad pattern (e.g. `TestCompute`, `^Test`, or any
|
|
264
|
+
// regex metacharacter) could match more than one test, and Go's `-run` treats an
|
|
265
|
+
// unanchored value as a substring match. So we accept ONLY a fully anchored plain
|
|
266
|
+
// literal `^Name$` (optionally a subtest path `^Outer$/^Inner$`): both `^` and `$`
|
|
267
|
+
// present, and the body containing only `[A-Za-z0-9_]` plus the `$/^` subtest
|
|
268
|
+
// joiner — nothing that Go's regexp could expand to a second test. Anything else
|
|
269
|
+
// (no anchors, partial anchors, `.`, `*`, `|`, `()`, `[]`, `?`, `+`, etc.) → null,
|
|
270
|
+
// which the classifier rejects as unrunnable. Never match-any.
|
|
271
|
+
function targetTestName(testRun) {
|
|
272
|
+
const trimmed = testRun.trim();
|
|
273
|
+
// Fully anchored: ^...$ — reject if either anchor is missing.
|
|
274
|
+
if (!trimmed.startsWith("^") || !trimmed.endsWith("$")) {
|
|
275
|
+
return null;
|
|
276
|
+
}
|
|
277
|
+
const inner = trimmed.slice(1, -1);
|
|
278
|
+
// The inner literal may only contain identifier chars, optionally split into a
|
|
279
|
+
// subtest path by the anchored joiner `$/^` (e.g. `^TestA$/^sub$` → `TestA/sub`).
|
|
280
|
+
// Any other regexp metacharacter makes the match potentially non-unique → reject.
|
|
281
|
+
const segments = inner.split("$/^");
|
|
282
|
+
if (segments.some(seg => seg.length === 0 || !/^[A-Za-z0-9_]+$/.test(seg))) {
|
|
283
|
+
return null;
|
|
284
|
+
}
|
|
285
|
+
return segments.join("/");
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function hasBuildFailure(run) {
|
|
289
|
+
return run.events.some(e => e.Action === "build-fail")
|
|
290
|
+
|| run.events.some(e => e.Action === "fail" && typeof e.FailedBuild === "string");
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// All target-test predicates require a CONCRETE exact name and match ONLY that
|
|
294
|
+
// test. A null name (broad/ambiguous --test-run) is rejected upfront by classify,
|
|
295
|
+
// so these never fall back to match-any — a null name yields false everywhere.
|
|
296
|
+
function targetTestPassed(run, name) {
|
|
297
|
+
return Boolean(name) && run.events.some(e => e.Action === "pass" && e.Test === name);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
function targetTestFailed(run, name) {
|
|
301
|
+
return Boolean(name) && run.events.some(e => e.Action === "fail" && e.Test === name);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// A panic aborts the test with a `fail` action on the target test just like an
|
|
305
|
+
// assertion does, so we must inspect the target test's output lines for a panic
|
|
306
|
+
// marker and reject it (a panic is NOT a trusted assertion signal).
|
|
307
|
+
function targetTestPanicked(run, name, includeChildren = false) {
|
|
308
|
+
if (!name) {
|
|
309
|
+
return false;
|
|
310
|
+
}
|
|
311
|
+
const prefix = name + "/";
|
|
312
|
+
return run.events.some(e =>
|
|
313
|
+
e.Action === "output"
|
|
314
|
+
&& typeof e.Output === "string"
|
|
315
|
+
&& (e.Test === name || (includeChildren && typeof e.Test === "string" && e.Test.startsWith(prefix)))
|
|
316
|
+
&& /^panic:|\bpanic:\s|\[signal SIGSEGV/.test(e.Output));
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
// Collect the target test's output lines (in order) so FIX 2 can inspect the
|
|
320
|
+
// failure shape for a TRUSTED assertion signal.
|
|
321
|
+
function targetTestOutput(run, name, includeChildren = false) {
|
|
322
|
+
if (!name) {
|
|
323
|
+
return [];
|
|
324
|
+
}
|
|
325
|
+
// Default: EXACT match only (unchanged). With `includeChildren` (Slice 2, line-gated),
|
|
326
|
+
// also collect the target test's SUBTESTS (`TestX/...`) so a runtime-named child's
|
|
327
|
+
// assertion frame is visible — the exact-line gate then rejects a sibling's frame.
|
|
328
|
+
const prefix = name + "/";
|
|
329
|
+
return run.events
|
|
330
|
+
.filter(e => e.Action === "output" && typeof e.Output === "string"
|
|
331
|
+
&& (e.Test === name || (includeChildren && typeof e.Test === "string" && e.Test.startsWith(prefix))))
|
|
332
|
+
.map(e => e.Output);
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
// FIX 2 — the mutant's failure for the EXACT target test must fail at a GENUINE
|
|
336
|
+
// value ASSERTION in the test SOURCE, not a t.Fatal precondition, a setup/helper
|
|
337
|
+
// failure, or an unrecognized error shape. This is SOURCE-LINE BINDING, mirroring
|
|
338
|
+
// the TS/JS gate's `isAssertionFailure` (parse the failing frame's file:line, read
|
|
339
|
+
// the test source at that line, require a real assertion call there). It replaces
|
|
340
|
+
// the earlier TEXT heuristic, which Codex reproduced a false-Proven against: a
|
|
341
|
+
// mutant-triggered `t.Fatalf("got %v, want %v", got, want)` carries "got/want"
|
|
342
|
+
// text but is a hard-stop precondition, not a value assertion of the target.
|
|
343
|
+
//
|
|
344
|
+
// Go prints the failing frame as `\t<file>:<line>: <message>` and, for testify,
|
|
345
|
+
// an `Error Trace:\t<file>:<line>` frame. The reported <line> is the SOURCE line
|
|
346
|
+
// of the assertion/Fatal CALL (Go's t.Helper() re-attributes a helper failure to
|
|
347
|
+
// the CALLER line — so a helper failure binds to the helper-call line, which is
|
|
348
|
+
// not an assertion, and is rejected). We read the copied test source at that line
|
|
349
|
+
// and require a genuine assertion: `t.Error(`/`t.Errorf(` (stdlib) or a testify
|
|
350
|
+
// `assert.`/`require.` call. We REJECT `t.Fatal(`/`t.Fatalf(`/`t.FailNow(`/
|
|
351
|
+
// `t.SkipNow(` and any line that is a helper call rather than an assertion.
|
|
352
|
+
//
|
|
353
|
+
// Fail CLOSED: if we cannot bind the failure to a genuine-assertion source line
|
|
354
|
+
// (no parseable frame, unreadable source, or the bound line is a Fatal/helper),
|
|
355
|
+
// this returns false and the classifier calls it unrunnable — never Proven.
|
|
356
|
+
|
|
357
|
+
// A genuine value-assertion call: stdlib t.Error/t.Errorf, or a testify
|
|
358
|
+
// assert.*/require.* call. A Fatal/FailNow/Skip hard-stop and a plain helper call
|
|
359
|
+
// do NOT match. Anchored to the assertion so `t.Fatalf` cannot pass as `t.Error`.
|
|
360
|
+
const GO_ASSERTION_LINE = /\bt\.Errorf?\s*\(|\b(?:assert|require)\.[A-Za-z]\w*\s*\(/;
|
|
361
|
+
// Explicit hard-stop reject list. These abort the test as a PRECONDITION, not a
|
|
362
|
+
// value assertion of the target, so a line bound to one is never trusted.
|
|
363
|
+
const GO_HARD_STOP_LINE = /\bt\.(?:Fatal|Fatalf|FailNow|SkipNow)\s*\(/;
|
|
364
|
+
|
|
365
|
+
// Parse Go per-line failure frames from the target test's output. Both the stdlib
|
|
366
|
+
// `\t<file>:<line>: <msg>` line and testify's `Error Trace:\t<file>:<line>` frame
|
|
367
|
+
// carry a basename + line. Return { file, line } for every frame, in order.
|
|
368
|
+
function parseGoFailFrames(lines) {
|
|
369
|
+
const frames = [];
|
|
370
|
+
for (const raw of lines) {
|
|
371
|
+
for (const segment of String(raw).split(/\r?\n/)) {
|
|
372
|
+
// Go stdlib prints `\t<file>:<line>: <msg>` (trailing colon); testify's
|
|
373
|
+
// `Error Trace:\t<file>:<line>` has none — so the trailing `:` is optional.
|
|
374
|
+
const match = /(?:^|\bError Trace:\s*)\s*([^\s:]+\.go):(\d+)(?::|\b)/.exec(segment);
|
|
375
|
+
if (match) {
|
|
376
|
+
frames.push({ file: match[1], line: Number(match[2]) });
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
return frames;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
// Read the copied test SOURCE at `line` and decide if it is a genuine assertion.
|
|
384
|
+
// Bind to the EXACT reported line (Go reports the call-start line, even for a
|
|
385
|
+
// multi-line call), and reject a hard-stop even if it also looks assertion-shaped.
|
|
386
|
+
function goLineIsAssertion(sourceLines, line) {
|
|
387
|
+
const index = line - 1;
|
|
388
|
+
if (index < 0 || index >= sourceLines.length) {
|
|
389
|
+
return false;
|
|
390
|
+
}
|
|
391
|
+
const text = sourceLines[index];
|
|
392
|
+
if (GO_HARD_STOP_LINE.test(text)) {
|
|
393
|
+
return false;
|
|
394
|
+
}
|
|
395
|
+
return GO_ASSERTION_LINE.test(text);
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
// SOURCE-LINE BINDING: at least one failing frame must resolve to a test source
|
|
399
|
+
// line that is a genuine assertion. `pkgDirAbs` is the copied module's package
|
|
400
|
+
// directory (Go prints frame files as basenames relative to it). Reads are
|
|
401
|
+
// wrapped so an unreadable/absent source fails CLOSED (returns false).
|
|
402
|
+
function mutantFailedAtTrustedAssertion(run, name, pkgDirAbs, assertionLine) {
|
|
403
|
+
// Slice 2 (line-gated): when an assertion line is provided, widen frame collection to
|
|
404
|
+
// the target test's subtests too, then require a failing frame at EXACTLY that line.
|
|
405
|
+
// A SIBLING subtest asserting at a DIFFERENT line is refused (its frame line != assertionLine).
|
|
406
|
+
// No line ⇒ unchanged: exact-name frames, any genuine-assertion line.
|
|
407
|
+
const useLine = typeof assertionLine === "number";
|
|
408
|
+
const frames = parseGoFailFrames(targetTestOutput(run, name, useLine));
|
|
409
|
+
if (frames.length === 0) {
|
|
410
|
+
return false;
|
|
411
|
+
}
|
|
412
|
+
const sourceCache = new Map();
|
|
413
|
+
for (const frame of frames) {
|
|
414
|
+
// Line-gated: the frame MUST be at the recorded assertion line (exact). Then
|
|
415
|
+
// `goLineIsAssertion` still runs as defense-in-depth (that line must be a real assertion).
|
|
416
|
+
if (useLine && frame.line !== assertionLine) {
|
|
417
|
+
continue;
|
|
418
|
+
}
|
|
419
|
+
const abs = path.join(pkgDirAbs, path.basename(frame.file));
|
|
420
|
+
if (!sourceCache.has(abs)) {
|
|
421
|
+
try {
|
|
422
|
+
sourceCache.set(abs, readFileSync(abs, "utf8").split(/\r?\n/));
|
|
423
|
+
} catch {
|
|
424
|
+
sourceCache.set(abs, null);
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
const sourceLines = sourceCache.get(abs);
|
|
428
|
+
if (sourceLines && goLineIsAssertion(sourceLines, frame.line)) {
|
|
429
|
+
return true;
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
return false;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
function failureSummary(run, name) {
|
|
436
|
+
for (const e of run.events) {
|
|
437
|
+
if (e.Action === "output" && typeof e.Output === "string" && name && e.Test === name) {
|
|
438
|
+
const line = e.Output.trim();
|
|
439
|
+
if (line && !/^(=== RUN|=== PAUSE|=== CONT|--- FAIL|--- PASS)/.test(line)) {
|
|
440
|
+
return redactSecrets(line);
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
for (const e of run.events) {
|
|
445
|
+
if (e.Action === "build-output" && typeof e.Output === "string" && e.Output.trim()) {
|
|
446
|
+
return redactSecrets(e.Output.trim().split("\n", 1)[0]);
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
const stderr = String(run.stderr ?? "").trim();
|
|
450
|
+
return stderr ? redactSecrets(stderr.split("\n", 1)[0]) : null;
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
function redactSecrets(text) {
|
|
454
|
+
return String(text)
|
|
455
|
+
.replace(/([A-Z0-9_]*(?:TOKEN|SECRET|PASSWORD|API[_-]?KEY|ACCESS[_-]?KEY|PRIVATE[_-]?KEY|SECRET[_-]?KEY|PASSPHRASE|CREDENTIAL|PIN|AUTH|COOKIE|SESSION)[A-Z0-9_]*=)[^\s'"]+/gi, "$1[REDACTED]")
|
|
456
|
+
.replace(/(:\/\/[^:/@\s]+:)[^@/\s]+(@)/g, "$1[REDACTED]$2")
|
|
457
|
+
.replace(/(Bearer\s+)[A-Za-z0-9._~+/=-]+/gi, "$1[REDACTED]");
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
// Run the AST mutator (go run go-mutate.go). Because `go run` collapses any
|
|
461
|
+
// non-zero child status to 1, we classify on the MUTATE_ERROR:<code> marker the
|
|
462
|
+
// helper prints to stderr, not the exit code.
|
|
463
|
+
function mutateFunc({ targetAbs, func, mode, cacheRoot, timeoutMs }) {
|
|
464
|
+
const helper = path.join(path.dirname(fileURLToPath(import.meta.url)), "go-mutate.go");
|
|
465
|
+
const result = spawnSync(goBin(), ["run", helper, "--file", targetAbs, "--func", func, "--mode", mode], {
|
|
466
|
+
encoding: "utf8",
|
|
467
|
+
timeout: timeoutMs,
|
|
468
|
+
env: hermeticEnv(cacheRoot),
|
|
469
|
+
maxBuffer: 8 * 1024 * 1024
|
|
470
|
+
});
|
|
471
|
+
const stderr = String(result.stderr ?? "");
|
|
472
|
+
const marker = /MUTATE_ERROR:(\d+)/.exec(stderr);
|
|
473
|
+
if (marker) {
|
|
474
|
+
return { ok: false, code: Number(marker[1]), message: redactSecrets(stderr.split("\n").filter(Boolean).slice(-1)[0] ?? "") };
|
|
475
|
+
}
|
|
476
|
+
if ((result.status ?? 1) !== 0) {
|
|
477
|
+
return { ok: false, code: 2, message: redactSecrets((stderr.split("\n", 1)[0] || "mutation failed").trim()) };
|
|
478
|
+
}
|
|
479
|
+
return { ok: true };
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
function mutateErrorReason(code) {
|
|
483
|
+
switch (code) {
|
|
484
|
+
case 3: return "target function name is ambiguous (more than one free function)";
|
|
485
|
+
case 4: return "target free function was not found";
|
|
486
|
+
case 5: return "target is a method (out of scope for G-1)";
|
|
487
|
+
case 6: return "target function has no return value (not mutable)";
|
|
488
|
+
default: return "mutation could not be applied";
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
function classify({ baseline, mutant, name, pkgDirAbs, assertionLine }) {
|
|
493
|
+
// (FIX 1) --test-run must bind to EXACTLY ONE target test. A broad/ambiguous
|
|
494
|
+
// pattern resolves to a null name; reject it before any pass/fail reasoning so a
|
|
495
|
+
// Proven can never rest on an unrelated test's event.
|
|
496
|
+
if (!name) {
|
|
497
|
+
return {
|
|
498
|
+
status: "unrunnable",
|
|
499
|
+
proven: false,
|
|
500
|
+
reason: "ambiguous or broad --test-run; require exactly one target test"
|
|
501
|
+
};
|
|
502
|
+
}
|
|
503
|
+
// (a) baseline must build AND the target test must pass.
|
|
504
|
+
if (baseline.exitCode !== 0 || baseline.timedOut || hasBuildFailure(baseline) || !targetTestPassed(baseline, name)) {
|
|
505
|
+
return { status: "unrunnable", proven: false, reason: "baseline target test did not pass" };
|
|
506
|
+
}
|
|
507
|
+
// (b) mutant must build.
|
|
508
|
+
if (hasBuildFailure(mutant)) {
|
|
509
|
+
return { status: "unrunnable", proven: false, reason: "mutant did not compile" };
|
|
510
|
+
}
|
|
511
|
+
if (mutant.timedOut) {
|
|
512
|
+
return { status: "unrunnable", proven: false, reason: "mutant timed out" };
|
|
513
|
+
}
|
|
514
|
+
// Equivalent-value mutation: the target test still passes -> survives.
|
|
515
|
+
if (mutant.exitCode === 0 && targetTestPassed(mutant, name) && !targetTestFailed(mutant, name)) {
|
|
516
|
+
return { status: "associated_survived", proven: false, reason: "mutated target did not change the test outcome" };
|
|
517
|
+
}
|
|
518
|
+
// (d) a panic is NOT a trusted assertion signal (line-gated: also reject a subtest panic).
|
|
519
|
+
if (targetTestPanicked(mutant, name, typeof assertionLine === "number")) {
|
|
520
|
+
return { status: "unrunnable", proven: false, reason: "mutant failed with a panic, not a test assertion" };
|
|
521
|
+
}
|
|
522
|
+
// (c) the SAME target test must fail (a subtest failure fails the parent too, so the
|
|
523
|
+
// parent-name `fail` action already reflects a runtime-named child's failure)...
|
|
524
|
+
if (!targetTestFailed(mutant, name)) {
|
|
525
|
+
return { status: "unrunnable", proven: false, reason: "mutant failed, but not at the target test" };
|
|
526
|
+
}
|
|
527
|
+
// (FIX 2) ...and that failure must bind to a GENUINE value-assertion source
|
|
528
|
+
// line. A t.Fatal precondition, a setup/helper failure, or any failure we cannot
|
|
529
|
+
// bind to an assertion line is NOT trusted -> unrunnable, never Proven (fail
|
|
530
|
+
// closed). Slice 2: when an assertion line is given, the frame must be at THAT line.
|
|
531
|
+
if (!mutantFailedAtTrustedAssertion(mutant, name, pkgDirAbs, assertionLine)) {
|
|
532
|
+
return {
|
|
533
|
+
status: "unrunnable",
|
|
534
|
+
proven: false,
|
|
535
|
+
reason: "mutant failed the target test, but not at a trusted value assertion"
|
|
536
|
+
};
|
|
537
|
+
}
|
|
538
|
+
return { status: "proven", proven: true, reason: "baseline passed and the mutant failed the same target test at a trusted assertion" };
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
function main() {
|
|
542
|
+
const args = parseArgs(process.argv.slice(2));
|
|
543
|
+
const root = path.resolve(args.root);
|
|
544
|
+
const targetAbs = resolveInside(root, args.target);
|
|
545
|
+
const targetRel = path.relative(root, targetAbs);
|
|
546
|
+
const timeoutMs = parseTimeoutMs(args.timeoutMs);
|
|
547
|
+
const name = targetTestName(args.testRun);
|
|
548
|
+
const assertionLine = parseAssertionLine(args.goAssertionLine);
|
|
549
|
+
// TARGET-PACKAGE SCOPE: Go packages ARE directories. The target's package is the
|
|
550
|
+
// directory of --target; scope `go test` to just it (`./<dir>`, no `...`) so a
|
|
551
|
+
// same-named test in another package can never be credited to this target. Use
|
|
552
|
+
// POSIX separators (Go accepts `./a/b` even on Windows) and `./` for the module
|
|
553
|
+
// root (dirname of a root-level file is ".").
|
|
554
|
+
const pkgDirRel = path.dirname(targetRel);
|
|
555
|
+
const pkgPath = pkgDirRel === "." ? "./" : `./${pkgDirRel.split(path.sep).join("/")}`;
|
|
556
|
+
|
|
557
|
+
const baselineCopy = copyModuleRoot(root, "baseline");
|
|
558
|
+
const mutantCopy = copyModuleRoot(root, "mutant");
|
|
559
|
+
// The mutant sandbox's package directory: Go prints failure-frame files as
|
|
560
|
+
// basenames relative to it, so FIX 2 resolves the test source there.
|
|
561
|
+
const pkgDirAbs = path.join(mutantCopy.repoRoot, path.dirname(targetRel));
|
|
562
|
+
try {
|
|
563
|
+
const baseline = runGoTest({ repoRoot: baselineCopy.repoRoot, testRun: args.testRun, timeoutMs, cacheRoot: baselineCopy.tmpRoot, pkgPath });
|
|
564
|
+
|
|
565
|
+
const mutation = mutateFunc({
|
|
566
|
+
targetAbs: path.join(mutantCopy.repoRoot, targetRel),
|
|
567
|
+
func: args.func,
|
|
568
|
+
mode: args.mode,
|
|
569
|
+
cacheRoot: mutantCopy.tmpRoot,
|
|
570
|
+
timeoutMs
|
|
571
|
+
});
|
|
572
|
+
|
|
573
|
+
let verdict;
|
|
574
|
+
let mutant = null;
|
|
575
|
+
if (!mutation.ok) {
|
|
576
|
+
// Baseline still reported so a caller can see it built/passed; the refusal
|
|
577
|
+
// classifies unrunnable without ever running a mutant.
|
|
578
|
+
verdict = {
|
|
579
|
+
status: "unrunnable",
|
|
580
|
+
proven: false,
|
|
581
|
+
reason: mutateErrorReason(mutation.code)
|
|
582
|
+
};
|
|
583
|
+
} else {
|
|
584
|
+
mutant = runGoTest({ repoRoot: mutantCopy.repoRoot, testRun: args.testRun, timeoutMs, cacheRoot: mutantCopy.tmpRoot, pkgPath });
|
|
585
|
+
verdict = classify({ baseline, mutant, name, pkgDirAbs, assertionLine });
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
const output = {
|
|
589
|
+
...verdict,
|
|
590
|
+
mode: args.mode,
|
|
591
|
+
testRun: args.testRun,
|
|
592
|
+
target: targetRel,
|
|
593
|
+
func: args.func,
|
|
594
|
+
baseline: {
|
|
595
|
+
exitCode: baseline.exitCode,
|
|
596
|
+
timedOut: baseline.timedOut,
|
|
597
|
+
elapsedMs: baseline.elapsedMs,
|
|
598
|
+
buildFailure: hasBuildFailure(baseline),
|
|
599
|
+
targetTestPassed: targetTestPassed(baseline, name),
|
|
600
|
+
failureSummary: failureSummary(baseline, name)
|
|
601
|
+
},
|
|
602
|
+
mutant: mutant
|
|
603
|
+
? {
|
|
604
|
+
exitCode: mutant.exitCode,
|
|
605
|
+
timedOut: mutant.timedOut,
|
|
606
|
+
elapsedMs: mutant.elapsedMs,
|
|
607
|
+
buildFailure: hasBuildFailure(mutant),
|
|
608
|
+
targetTestFailed: targetTestFailed(mutant, name),
|
|
609
|
+
panicked: targetTestPanicked(mutant, name, typeof assertionLine === "number"),
|
|
610
|
+
// MUST pass assertionLine so this reported field matches classify's verdict —
|
|
611
|
+
// `mapGoOracle` reads THIS `trustedAssertion` to close the cert.
|
|
612
|
+
trustedAssertion: mutantFailedAtTrustedAssertion(mutant, name, pkgDirAbs, assertionLine),
|
|
613
|
+
failureSummary: failureSummary(mutant, name)
|
|
614
|
+
}
|
|
615
|
+
: { skipped: true, reason: mutation.message ?? null },
|
|
616
|
+
medianProofMs: mutant ? Math.round((baseline.elapsedMs + mutant.elapsedMs) / 2) : baseline.elapsedMs
|
|
617
|
+
};
|
|
618
|
+
|
|
619
|
+
if (args.json) {
|
|
620
|
+
process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);
|
|
621
|
+
} else {
|
|
622
|
+
process.stdout.write(`${output.status}: ${output.reason}\n`);
|
|
623
|
+
process.stdout.write(`baseline=${baseline.exitCode} mutant=${mutant ? mutant.exitCode : "skipped"} median_ms=${output.medianProofMs}\n`);
|
|
624
|
+
}
|
|
625
|
+
process.exitCode = output.status === "unrunnable" ? 2 : 0;
|
|
626
|
+
} finally {
|
|
627
|
+
rmSync(baselineCopy.tmpRoot, { recursive: true, force: true });
|
|
628
|
+
rmSync(mutantCopy.tmpRoot, { recursive: true, force: true });
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
try {
|
|
633
|
+
main();
|
|
634
|
+
} catch (error) {
|
|
635
|
+
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n\n${usage()}\n`);
|
|
636
|
+
process.exitCode = 1;
|
|
637
|
+
}
|