@orangepro/orangepro-mcp 0.1.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +195 -251
- package/dist/local/analyze/analyzer.js +62 -17
- package/dist/local/analyze/treeSitter/engine.js +185 -22
- package/dist/local/autoProve.js +433 -38
- package/dist/local/cli.js +93 -10
- package/dist/local/cliArgs.js +1 -0
- package/dist/local/generate/runHints.js +9 -4
- package/dist/local/graph/factories.js +5 -2
- package/dist/local/ledger.js +1 -1
- package/dist/local/mcp.js +26 -13
- package/dist/local/operations.js +454 -52
- package/dist/local/pack/coverageReport.js +3 -3
- package/dist/local/proofDoctor.js +312 -0
- package/dist/local/rtm.js +40 -10
- package/dist/local/viz/behaviorReportData.js +79 -7
- package/dist/local/viz/behaviorReportHtml.js +526 -615
- package/dist/local/viz/html.js +1 -1
- package/docs/agent-workflow.md +10 -38
- package/docs/agents/claude-code.md +3 -9
- package/docs/agents/codex.md +3 -20
- package/docs/agents/cursor.md +2 -2
- package/docs/agents/opencode.md +2 -2
- package/docs/agents/vscode.md +2 -2
- package/docs/local-proof-kit.md +52 -19
- package/package.json +39 -6
- package/scripts/spikes/go-dynamic-proof-spike.mjs +637 -0
- package/scripts/spikes/go-mutate.go +182 -0
- package/scripts/spikes/java-dynamic-proof-spike.mjs +571 -0
- package/scripts/spikes/java-mutate.mjs +264 -0
- package/scripts/spikes/python-dynamic-proof-spike.mjs +245 -0
- package/scripts/spikes/python-mutate.py +89 -0
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* G1 — `opro doctor --proof`: explain WHY top targets are not Dynamically Proven.
|
|
3
|
+
*
|
|
4
|
+
* Read-only diagnostics over already-produced, already-redacted data:
|
|
5
|
+
* - `.orangepro/proof-attempts.json` — a distilled sidecar of the last run's
|
|
6
|
+
* auto-prove attempt classifications (written by opStart AFTER autoProve
|
|
7
|
+
* returns; autoProve itself and the proof oracle are untouched).
|
|
8
|
+
* - the graph + ledger via the CANONICAL judge (buildRtm) for proven counts —
|
|
9
|
+
* this module never re-derives or re-scores proof.
|
|
10
|
+
*
|
|
11
|
+
* Trust invariants (load-bearing):
|
|
12
|
+
* - Mints nothing, mutates no ledger, never writes on the doctor path.
|
|
13
|
+
* - Reasons are the upstream single-line redacted summaries; re-redacted at
|
|
14
|
+
* sidecar-write time as belt-and-braces. No raw stderr/stdout, ever.
|
|
15
|
+
* - A survived mutant is a proven NEGATIVE ("not proven, possibly equivalent"),
|
|
16
|
+
* never blamed on the user and never nudged toward weakening a test.
|
|
17
|
+
* - Stale sidecar (graph re-analyzed / git moved since) ⇒ fail closed: reasons
|
|
18
|
+
* are labeled stale and the headline says re-run, never presented as current.
|
|
19
|
+
*/
|
|
20
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
21
|
+
import { dirname, join } from "node:path";
|
|
22
|
+
import { workspacePaths } from "./workspace.js";
|
|
23
|
+
import { redactSecrets } from "./util/redact.js";
|
|
24
|
+
import { targetLanguage } from "./ledger.js";
|
|
25
|
+
import { readEnginesNode, satisfiesNodeRange } from "./proofRunnability.js";
|
|
26
|
+
export const PROOF_ATTEMPTS_SCHEMA_VERSION = "orangepro.proof_attempts.v1";
|
|
27
|
+
export const PROOF_ATTEMPTS_FILE = "proof-attempts.json";
|
|
28
|
+
export const PROOF_DOCTOR_SCHEMA_VERSION = "orangepro.proof_doctor.v1";
|
|
29
|
+
/** Languages with a shipped dynamic-proof profile. Everything else is honest "not yet". */
|
|
30
|
+
const PROVABLE_LANGUAGES = new Set(["typescript", "javascript", "go", "java", "python"]);
|
|
31
|
+
/**
|
|
32
|
+
* One label + one smallest-next-step per category. Copy discipline: honest
|
|
33
|
+
* pointers only — never nudge toward weakening a test, never call a test bad.
|
|
34
|
+
*/
|
|
35
|
+
export const PROOF_BLOCKER_GUIDE = {
|
|
36
|
+
module_not_found: {
|
|
37
|
+
label: "a module or dependency is missing in the proof sandbox",
|
|
38
|
+
next_step: "Install the package's dependencies (npm ci / go mod download / pip install) so imports resolve, then re-run `opro start`."
|
|
39
|
+
},
|
|
40
|
+
tsconfig_missing: {
|
|
41
|
+
label: "the package extends a monorepo tsconfig the sandbox cannot resolve",
|
|
42
|
+
next_step: "Run `opro start` from the monorepo root so parent tsconfigs are mirrored into the proof sandbox."
|
|
43
|
+
},
|
|
44
|
+
experimental_builtin: {
|
|
45
|
+
label: "the target needs an experimental Node builtin runtime flag",
|
|
46
|
+
next_step: "Re-run with the flag named in the attempt reason (e.g. NODE_OPTIONS=--experimental-sqlite) or a Node version where the builtin is stable."
|
|
47
|
+
},
|
|
48
|
+
engine_mismatch: {
|
|
49
|
+
label: "the runner Node is outside the package's declared engines range",
|
|
50
|
+
next_step: "Switch Node versions (e.g. `nvm use`) to satisfy engines.node, then re-run `opro start`."
|
|
51
|
+
},
|
|
52
|
+
db_or_external: {
|
|
53
|
+
label: "the test needs a database or external service the sandbox lacks",
|
|
54
|
+
next_step: "Provide the service locally (or a test double) via `orangepro_prove_loop` setup_commands, then re-run."
|
|
55
|
+
},
|
|
56
|
+
runner_missing: {
|
|
57
|
+
label: "no supported test runner is available for the target's package",
|
|
58
|
+
next_step: "Install the package's test framework (vitest / jest / mocha; go, mvn, pytest on PATH), then re-run `opro start`."
|
|
59
|
+
},
|
|
60
|
+
module_root_missing: {
|
|
61
|
+
label: "no module root (go.mod / pom.xml / build.gradle) was found under the analyzed path",
|
|
62
|
+
next_step: "Run `opro start` from the module root that owns this target (the directory containing go.mod / pom.xml)."
|
|
63
|
+
},
|
|
64
|
+
assertion_binding: {
|
|
65
|
+
label: "the proof could not bind exactly one test to the target (ambiguous test identity)",
|
|
66
|
+
next_step: "Give the covering test a unique title/name so the oracle can bind exactly one test to the mutant; ambiguity fails closed."
|
|
67
|
+
},
|
|
68
|
+
unsupported_language: {
|
|
69
|
+
label: "no dynamic-proof profile exists for this language yet",
|
|
70
|
+
next_step: "TS/JS, Go, Java and Python are dynamically provable today; other languages stay honestly in their static tiers."
|
|
71
|
+
},
|
|
72
|
+
setup_failed: {
|
|
73
|
+
label: "proof setup failed before the oracle could run",
|
|
74
|
+
next_step: "Read the attempt reason, fix the named setup step, then re-run `opro start`."
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
/** Wording is load-bearing: a survivor is a proven negative, never a user failure. */
|
|
78
|
+
export const NON_KILLING_NOTE = "Not proven: the test still passed while the target was mutated (possibly an equivalent mutation). " +
|
|
79
|
+
"The mutant surviving is a proven negative about assertion strength — it is never counted as Dynamically Proven.";
|
|
80
|
+
export function proofAttemptsPath(root) {
|
|
81
|
+
return join(workspacePaths(root).dir, PROOF_ATTEMPTS_FILE);
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Distill an AutoProveResult into the persistable sidecar. Reasons arrive
|
|
85
|
+
* already redacted (autoProve/classifyBaselineFailure discipline); redactSecrets
|
|
86
|
+
* is applied again here so the WRITER enforces the no-secrets guarantee even if
|
|
87
|
+
* an upstream path regresses.
|
|
88
|
+
*/
|
|
89
|
+
export function distillProofAttempts(auto, meta) {
|
|
90
|
+
const manifest = meta.graph.manifest;
|
|
91
|
+
return {
|
|
92
|
+
schema_version: PROOF_ATTEMPTS_SCHEMA_VERSION,
|
|
93
|
+
generated_at: meta.generatedAt,
|
|
94
|
+
graph_generated_at: manifest?.generated_at ?? null,
|
|
95
|
+
git_commit: manifest?.git?.commit ?? null,
|
|
96
|
+
git_dirty: manifest?.git?.dirty ?? null,
|
|
97
|
+
attempted: auto.attempted,
|
|
98
|
+
proven: auto.proven,
|
|
99
|
+
attempts: auto.attempts.map((a) => ({
|
|
100
|
+
target_symbol: a.target_symbol,
|
|
101
|
+
test_path: a.test_path || undefined,
|
|
102
|
+
classification: a.classification,
|
|
103
|
+
category: a.category,
|
|
104
|
+
reason: a.reason ? redactSecrets(a.reason) : undefined,
|
|
105
|
+
deduped: a.deduped,
|
|
106
|
+
language: targetLanguage(a.target_symbol)
|
|
107
|
+
})),
|
|
108
|
+
skipped: auto.skipped.map((s) => ({
|
|
109
|
+
target_symbol: s.target_symbol,
|
|
110
|
+
title: s.title,
|
|
111
|
+
reason: redactSecrets(s.reason)
|
|
112
|
+
}))
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
export function writeProofAttempts(root, file) {
|
|
116
|
+
const path = proofAttemptsPath(root);
|
|
117
|
+
writeFileSync(path, JSON.stringify(file, null, 2) + "\n", "utf8");
|
|
118
|
+
return path;
|
|
119
|
+
}
|
|
120
|
+
/** True when the sidecar anchors to the CURRENT graph generation + commit. */
|
|
121
|
+
export function proofAttemptsFresh(attempts, graph) {
|
|
122
|
+
const manifest = graph.manifest;
|
|
123
|
+
return (attempts.graph_generated_at === (manifest?.generated_at ?? null) &&
|
|
124
|
+
attempts.git_commit === (manifest?.git?.commit ?? null));
|
|
125
|
+
}
|
|
126
|
+
export function loadProofAttempts(root) {
|
|
127
|
+
const path = proofAttemptsPath(root);
|
|
128
|
+
if (!existsSync(path))
|
|
129
|
+
return null;
|
|
130
|
+
// Fail closed on ANY unreadable sidecar — malformed JSON, wrong schema, or a
|
|
131
|
+
// non-object shape. Unreadable evidence is no evidence; doctor must never crash.
|
|
132
|
+
let parsed;
|
|
133
|
+
try {
|
|
134
|
+
parsed = JSON.parse(readFileSync(path, "utf8"));
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
return null;
|
|
138
|
+
}
|
|
139
|
+
if (typeof parsed !== "object" || parsed === null)
|
|
140
|
+
return null;
|
|
141
|
+
const file = parsed;
|
|
142
|
+
if (file.schema_version !== PROOF_ATTEMPTS_SCHEMA_VERSION)
|
|
143
|
+
return null;
|
|
144
|
+
if (!Array.isArray(file.attempts) || !Array.isArray(file.skipped))
|
|
145
|
+
return null;
|
|
146
|
+
return file;
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Map one recorded attempt to a doctor blocker category. Uses the R-1 category
|
|
150
|
+
* when present; otherwise a CONSERVATIVE pattern match on the (redacted) reason.
|
|
151
|
+
* Anything unrecognized stays "setup_failed" — never guess a specific cause.
|
|
152
|
+
*/
|
|
153
|
+
export function blockerCategoryFor(attempt) {
|
|
154
|
+
const reason = attempt.reason ?? "";
|
|
155
|
+
if (/runner binary not found|unsupported or unknown test runner/i.test(reason))
|
|
156
|
+
return "runner_missing";
|
|
157
|
+
if (/no go\.mod found|no pom\.xml|no maven or gradle|build\.gradle/i.test(reason))
|
|
158
|
+
return "module_root_missing";
|
|
159
|
+
if (/ambiguous|appears more than once|uniquely passed|not uniquely/i.test(reason))
|
|
160
|
+
return "assertion_binding";
|
|
161
|
+
const cat = attempt.category;
|
|
162
|
+
if (cat && cat in PROOF_BLOCKER_GUIDE)
|
|
163
|
+
return cat;
|
|
164
|
+
return "setup_failed";
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Cheap static preflight for targets with NO attempt record: names blockers
|
|
168
|
+
* WITHOUT running any test. Checks are deliberately few and certain:
|
|
169
|
+
* unsupported language, missing go/java module root, engines mismatch.
|
|
170
|
+
*/
|
|
171
|
+
function preflightBlockers(rtm, root, io, limit) {
|
|
172
|
+
const rows = rtm.rows.filter((r) => r.evidence_tier !== "proven" && r.code_symbol.startsWith("sym:")).slice(0, limit);
|
|
173
|
+
const found = [];
|
|
174
|
+
for (const row of rows) {
|
|
175
|
+
const lang = targetLanguage(row.code_symbol);
|
|
176
|
+
const fileRel = row.code_symbol.match(/^sym:(.+)#/)?.[1] ?? "";
|
|
177
|
+
if (!PROVABLE_LANGUAGES.has(lang)) {
|
|
178
|
+
found.push({ category: "unsupported_language", target: row.code_symbol });
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
if (lang === "go" || lang === "java") {
|
|
182
|
+
const markers = lang === "go" ? ["go.mod"] : ["pom.xml", "build.gradle", "build.gradle.kts"];
|
|
183
|
+
if (!nearestMarker(root, fileRel, markers, io)) {
|
|
184
|
+
found.push({ category: "module_root_missing", target: row.code_symbol });
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
if (lang === "typescript" || lang === "javascript") {
|
|
189
|
+
const range = readEnginesNode(root, fileRel);
|
|
190
|
+
if (range && satisfiesNodeRange(io.nodeVersion, range) === false) {
|
|
191
|
+
found.push({ category: "engine_mismatch", target: row.code_symbol });
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return groupBlockers(found.map((f) => ({
|
|
196
|
+
target_symbol: f.target,
|
|
197
|
+
test_path: undefined,
|
|
198
|
+
category: f.category,
|
|
199
|
+
reason: undefined
|
|
200
|
+
})), "preflight");
|
|
201
|
+
}
|
|
202
|
+
/** Walk from the target file's dir UP to root (inclusive) looking for a marker file. */
|
|
203
|
+
function nearestMarker(root, fileRel, markers, io) {
|
|
204
|
+
let dir = join(root, dirname(fileRel));
|
|
205
|
+
const stop = root;
|
|
206
|
+
for (;;) {
|
|
207
|
+
for (const m of markers)
|
|
208
|
+
if (io.exists(join(dir, m)))
|
|
209
|
+
return true;
|
|
210
|
+
if (dir === stop)
|
|
211
|
+
return false;
|
|
212
|
+
const parent = dirname(dir);
|
|
213
|
+
if (parent === dir || !parent.startsWith(stop))
|
|
214
|
+
return false;
|
|
215
|
+
dir = parent;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
function groupBlockers(blocked, source) {
|
|
219
|
+
const groups = new Map();
|
|
220
|
+
for (const b of blocked) {
|
|
221
|
+
const category = source === "preflight" && b.category && b.category in PROOF_BLOCKER_GUIDE
|
|
222
|
+
? b.category
|
|
223
|
+
: blockerCategoryFor(b);
|
|
224
|
+
const guide = PROOF_BLOCKER_GUIDE[category];
|
|
225
|
+
const existing = groups.get(category);
|
|
226
|
+
if (existing) {
|
|
227
|
+
existing.count += 1;
|
|
228
|
+
if (existing.targets.length < 5)
|
|
229
|
+
existing.targets.push(b.target_symbol);
|
|
230
|
+
}
|
|
231
|
+
else {
|
|
232
|
+
groups.set(category, {
|
|
233
|
+
category,
|
|
234
|
+
label: guide.label,
|
|
235
|
+
count: 1,
|
|
236
|
+
targets: [b.target_symbol],
|
|
237
|
+
representative: { target_symbol: b.target_symbol, test_path: b.test_path, reason: b.reason },
|
|
238
|
+
next_step: guide.next_step,
|
|
239
|
+
source
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
return [...groups.values()].sort((a, b) => b.count - a.count);
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Pure assembly: graph + canonical RTM result + optional attempts sidecar →
|
|
247
|
+
* deduped blocker report. Never writes; never mints; never recomputes proof.
|
|
248
|
+
*/
|
|
249
|
+
export function buildProofDoctor(graph, rtm, attempts, opts = {}) {
|
|
250
|
+
const io = opts.io ?? { exists: existsSync, nodeVersion: process.version };
|
|
251
|
+
const proven = rtm.summary.proven;
|
|
252
|
+
const denominator = rtm.summary.total;
|
|
253
|
+
// Freshness: the sidecar must anchor to the CURRENT graph generation + commit.
|
|
254
|
+
const stale = Boolean(attempts && !proofAttemptsFresh(attempts, graph));
|
|
255
|
+
const currentAttempts = attempts && !stale ? attempts : null;
|
|
256
|
+
const blocked = (currentAttempts?.attempts ?? []).filter((a) => a.classification === "needs_setup");
|
|
257
|
+
const survivors = (currentAttempts?.attempts ?? []).filter((a) => a.classification === "non_killing");
|
|
258
|
+
let blockers = groupBlockers(blocked, "attempt");
|
|
259
|
+
if (blockers.length === 0 && !currentAttempts) {
|
|
260
|
+
blockers = preflightBlockers(rtm, graph.workspace?.root ?? "", io, opts.preflightLimit ?? 10);
|
|
261
|
+
}
|
|
262
|
+
// Dedupe identical (target, test) survivor pairs — repeat attempts add noise,
|
|
263
|
+
// not information. Attribution stays exact: one row per target+test pair.
|
|
264
|
+
const seenSurvivors = new Set();
|
|
265
|
+
const non_killing = [];
|
|
266
|
+
for (const a of survivors) {
|
|
267
|
+
const key = `${a.target_symbol}\u0000${a.test_path ?? ""}`;
|
|
268
|
+
if (seenSurvivors.has(key))
|
|
269
|
+
continue;
|
|
270
|
+
seenSurvivors.add(key);
|
|
271
|
+
non_killing.push({ target_symbol: a.target_symbol, test_path: a.test_path, note: NON_KILLING_NOTE });
|
|
272
|
+
}
|
|
273
|
+
let status;
|
|
274
|
+
let headline;
|
|
275
|
+
if (stale) {
|
|
276
|
+
status = "stale";
|
|
277
|
+
headline =
|
|
278
|
+
"Proof-attempt data is stale (the graph or commit changed since the last run) — re-run `opro start` for current blocker reasons.";
|
|
279
|
+
}
|
|
280
|
+
else if (proven > 0) {
|
|
281
|
+
status = "proven";
|
|
282
|
+
headline = `${proven} of ${denominator} behaviors are Dynamically Proven (RTM). Blockers below explain the rest.`;
|
|
283
|
+
}
|
|
284
|
+
else if (blockers.length > 0) {
|
|
285
|
+
const top = blockers[0];
|
|
286
|
+
status = "blocked";
|
|
287
|
+
headline =
|
|
288
|
+
top.count > 1
|
|
289
|
+
? `0 Dynamically Proven — one root cause blocked ${top.count} target${top.count === 1 ? "" : "s"}: ${top.label}.`
|
|
290
|
+
: `0 Dynamically Proven — top blocker: ${top.label}.`;
|
|
291
|
+
}
|
|
292
|
+
else if (non_killing.length > 0) {
|
|
293
|
+
status = "blocked";
|
|
294
|
+
headline = `0 Dynamically Proven — ${non_killing.length} attempt(s) ran but the mutant survived (see non_killing).`;
|
|
295
|
+
}
|
|
296
|
+
else {
|
|
297
|
+
status = "no_data";
|
|
298
|
+
headline = "No proof-attempt data yet — run `opro start` to attempt dynamic proof and record blockers.";
|
|
299
|
+
}
|
|
300
|
+
return {
|
|
301
|
+
schema_version: PROOF_DOCTOR_SCHEMA_VERSION,
|
|
302
|
+
status,
|
|
303
|
+
proven,
|
|
304
|
+
denominator,
|
|
305
|
+
attempted: currentAttempts?.attempted ?? null,
|
|
306
|
+
stale,
|
|
307
|
+
headline,
|
|
308
|
+
blockers,
|
|
309
|
+
non_killing,
|
|
310
|
+
generated_at: attempts?.generated_at ?? null
|
|
311
|
+
};
|
|
312
|
+
}
|
package/dist/local/rtm.js
CHANGED
|
@@ -29,9 +29,22 @@ export function buildRtm(graph, ledger, opts = {}) {
|
|
|
29
29
|
.filter((node) => inScope(node, targetSet, fileSet))
|
|
30
30
|
.map((node) => toRtmRow(indexes, node, ledgerBySymbol.get(node.external_id)))
|
|
31
31
|
.sort((a, b) => STATUS_ORDER[a.status] - STATUS_ORDER[b.status] || a.area.localeCompare(b.area) || a.file.localeCompare(b.file) || a.behavior.localeCompare(b.behavior));
|
|
32
|
-
|
|
32
|
+
// Display-union: surface CodeSymbols with a CURRENT valid dynamic-proof cert that fall
|
|
33
|
+
// OUTSIDE the deterministic denominator (e.g. a Formatter SPI method below the
|
|
34
|
+
// entry-point-adjacent bar). `ledgerBySymbol.get(...).proven` is already gated by
|
|
35
|
+
// isDynamicProofRecord + fingerprint-match + closed, so a stale/invalid/unclosed cert
|
|
36
|
+
// is never `true` here. These rows are counted in `summary.proven` but NOT in the
|
|
37
|
+
// denominator math (total/coverage_total/coverage_pct) — the denominator stays stable.
|
|
38
|
+
const denominatorIds = new Set(baseRows.map((row) => row.behavior_id));
|
|
39
|
+
const unionRows = graph.nodes
|
|
40
|
+
.filter((node) => node.kind === "CodeSymbol" && !denominatorIds.has(node.external_id) && ledgerBySymbol.get(node.external_id)?.proven === true)
|
|
41
|
+
.filter((node) => inScope(node, targetSet, fileSet))
|
|
42
|
+
.map((node) => ({ ...toRtmRow(indexes, node, ledgerBySymbol.get(node.external_id)), off_denominator: true }))
|
|
43
|
+
.sort((a, b) => a.area.localeCompare(b.area) || a.file.localeCompare(b.file) || a.behavior.localeCompare(b.behavior));
|
|
44
|
+
const displayRows = [...baseRows, ...unionRows];
|
|
45
|
+
const filteredRows = displayRows.filter((row) => !statusSet || statusSet.has(row.status));
|
|
33
46
|
const rows = opts.limit && opts.limit > 0 ? filteredRows.slice(0, opts.limit) : filteredRows;
|
|
34
|
-
return { summary: summarizeRows(baseRows), rows, ...(opts.scope ? { scope: opts.scope } : {}) };
|
|
47
|
+
return { summary: summarizeRows(baseRows, unionRows), rows, ...(opts.scope ? { scope: opts.scope } : {}) };
|
|
35
48
|
}
|
|
36
49
|
function inScope(node, targetSet, fileSet) {
|
|
37
50
|
if (!targetSet && !fileSet)
|
|
@@ -43,6 +56,11 @@ function inScope(node, targetSet, fileSet) {
|
|
|
43
56
|
}
|
|
44
57
|
export function renderRtmMarkdown(result) {
|
|
45
58
|
const s = result.summary;
|
|
59
|
+
// Off-denominator proven symbols (display-union): counted in `proven` but not in `total`.
|
|
60
|
+
const offDenominatorProven = result.rows.filter((row) => row.off_denominator === true && row.evidence_tier === "proven").length;
|
|
61
|
+
const provenValue = offDenominatorProven > 0
|
|
62
|
+
? `${s.proven} (${s.proven - offDenominatorProven} in denominator + ${offDenominatorProven} off-denominator)`
|
|
63
|
+
: `${s.proven}`;
|
|
46
64
|
const lines = [
|
|
47
65
|
"# OrangePro Traceability Matrix",
|
|
48
66
|
"",
|
|
@@ -51,7 +69,7 @@ export function renderRtmMarkdown(result) {
|
|
|
51
69
|
"| Metric | Value |",
|
|
52
70
|
"|---|---:|",
|
|
53
71
|
`| Total denominator behaviors | ${s.total} |`,
|
|
54
|
-
`| Dynamically Proven | ${
|
|
72
|
+
`| Dynamically Proven | ${provenValue} |`,
|
|
55
73
|
`| Runtime-covered | ${s.runtime_covered} |`,
|
|
56
74
|
`| Associated signal | ${s.associated} |`,
|
|
57
75
|
`| No integration signal | ${s.no_link} |`,
|
|
@@ -64,6 +82,9 @@ export function renderRtmMarkdown(result) {
|
|
|
64
82
|
if (result.scope?.guidance) {
|
|
65
83
|
lines.push(`> ${escapeMarkdown(result.scope.guidance)}`, "");
|
|
66
84
|
}
|
|
85
|
+
if (offDenominatorProven > 0) {
|
|
86
|
+
lines.push(`> ${offDenominatorProven} dynamically-proven symbol(s) sit OUTSIDE the deterministic denominator (marked \`off-denominator\` below). They are counted in Dynamically Proven but NOT in the denominator or Dynamic Proven source ratio.`, "");
|
|
87
|
+
}
|
|
67
88
|
if (result.rows.length < s.total) {
|
|
68
89
|
lines.push(`> Showing ${result.rows.length} row(s) from ${s.total} scoped denominator row(s). This can reflect \`--limit\` and/or \`--status\` filters. Use \`opro rtm --format json --out .orangepro/rtm-full.json\` for a full machine-readable RTM.`, "");
|
|
69
90
|
}
|
|
@@ -77,7 +98,7 @@ export function renderRtmMarkdown(result) {
|
|
|
77
98
|
lines.push([
|
|
78
99
|
row.behavior,
|
|
79
100
|
row.code_symbol || row.behavior_id,
|
|
80
|
-
row.area,
|
|
101
|
+
row.off_denominator ? `${row.area} (off-denominator)` : row.area,
|
|
81
102
|
row.language,
|
|
82
103
|
row.test_signal,
|
|
83
104
|
row.status,
|
|
@@ -93,7 +114,7 @@ export function renderRtmMarkdown(result) {
|
|
|
93
114
|
return lines.join("\n");
|
|
94
115
|
}
|
|
95
116
|
export function renderRtmCsv(result) {
|
|
96
|
-
const header = ["behavior", "behavior_id", "kind", "code_symbol", "file", "area", "language", "evidence_tier", "test_signal", "status", "suggested_next_test", "ledger_outcome", "ledger_run_id"];
|
|
117
|
+
const header = ["behavior", "behavior_id", "kind", "code_symbol", "file", "area", "language", "evidence_tier", "test_signal", "status", "suggested_next_test", "ledger_outcome", "ledger_run_id", "off_denominator"];
|
|
97
118
|
const rows = result.rows.map((row) => [
|
|
98
119
|
row.behavior,
|
|
99
120
|
row.behavior_id,
|
|
@@ -107,7 +128,8 @@ export function renderRtmCsv(result) {
|
|
|
107
128
|
row.status,
|
|
108
129
|
row.suggested_next_test,
|
|
109
130
|
row.ledger_outcome,
|
|
110
|
-
row.ledger_run_id
|
|
131
|
+
row.ledger_run_id,
|
|
132
|
+
row.off_denominator ? "true" : "false"
|
|
111
133
|
]);
|
|
112
134
|
return [header, ...rows].map((row) => row.map(csvCell).join(",")).join("\n") + "\n";
|
|
113
135
|
}
|
|
@@ -157,10 +179,18 @@ function statusFor(evidence, ledgerRecord, dynamicProof) {
|
|
|
157
179
|
return "Associated signal";
|
|
158
180
|
return "No integration signal";
|
|
159
181
|
}
|
|
160
|
-
|
|
182
|
+
/**
|
|
183
|
+
* `rows` = the deterministic denominator rows (drives ALL denominator math). `unionRows`
|
|
184
|
+
* = off-denominator symbols with a current valid dynamic-proof cert; they add ONLY to
|
|
185
|
+
* `proven` (the honest "Dynamically Proven" headline), never to total/coverage_total/
|
|
186
|
+
* coverage_pct. Splitting the two keeps the denominator stable while the headline counts
|
|
187
|
+
* every genuinely-proven symbol.
|
|
188
|
+
*/
|
|
189
|
+
function summarizeRows(rows, unionRows = []) {
|
|
161
190
|
const attempted = rows.filter((row) => row.ledger_outcome.startsWith("reproven") || row.ledger_outcome.startsWith("unproven")).length;
|
|
162
191
|
const reproven = rows.filter((row) => row.status === "Reproven (this run)").length;
|
|
163
|
-
const
|
|
192
|
+
const denominatorProven = rows.filter((row) => row.evidence_tier === "proven").length;
|
|
193
|
+
const proven = denominatorProven + unionRows.filter((row) => row.evidence_tier === "proven").length;
|
|
164
194
|
return {
|
|
165
195
|
total: rows.length,
|
|
166
196
|
proven,
|
|
@@ -171,9 +201,9 @@ function summarizeRows(rows) {
|
|
|
171
201
|
generated_unverifiable: rows.filter((row) => row.status === "Generated-unverifiable").length,
|
|
172
202
|
attempted,
|
|
173
203
|
kept_rate: attempted > 0 ? Number(((reproven / attempted) * 100).toFixed(2)) : 0,
|
|
174
|
-
coverage_confirmed:
|
|
204
|
+
coverage_confirmed: denominatorProven,
|
|
175
205
|
coverage_total: rows.length,
|
|
176
|
-
coverage_pct: rows.length > 0 ? Number(((
|
|
206
|
+
coverage_pct: rows.length > 0 ? Number(((denominatorProven / rows.length) * 100).toFixed(1)) : 0
|
|
177
207
|
};
|
|
178
208
|
}
|
|
179
209
|
/**
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
import { buildRtm } from "../rtm.js";
|
|
3
3
|
import { rankRiskGaps } from "../score/risk.js";
|
|
4
|
+
import { PROOF_BLOCKER_GUIDE } from "../proofDoctor.js";
|
|
4
5
|
/** Short human phrase per R-1 needs_setup category, for the "blocked because: …" panel copy. */
|
|
5
6
|
const BLOCK_CATEGORY_LABEL = {
|
|
6
7
|
module_not_found: "a missing module or dependency in the sandbox",
|
|
@@ -30,7 +31,19 @@ export function dominantBlockReason(needsSetup) {
|
|
|
30
31
|
topKey = k;
|
|
31
32
|
}
|
|
32
33
|
}
|
|
33
|
-
return { label: BLOCK_CATEGORY_LABEL[topKey] ?? "setup/runnability of the test in the sandbox", count: top, total: needsSetup.length };
|
|
34
|
+
return { label: BLOCK_CATEGORY_LABEL[topKey] ?? "setup/runnability of the test in the sandbox", count: top, total: needsSetup.length, category: topKey };
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* G1: category-specific smallest next step for the dominant blocker (display
|
|
38
|
+
* copy only; the guide lives in proofDoctor so doctor/CLI/report share one
|
|
39
|
+
* source). Unknown/uncategorized falls back to the generic handoff action.
|
|
40
|
+
*/
|
|
41
|
+
function nextStepFor(dom) {
|
|
42
|
+
if (!dom)
|
|
43
|
+
return null;
|
|
44
|
+
const key = dom.category === "runnability" ? "setup_failed" : dom.category;
|
|
45
|
+
const guide = PROOF_BLOCKER_GUIDE[key];
|
|
46
|
+
return guide ? `Next: ${guide.next_step}` : null;
|
|
34
47
|
}
|
|
35
48
|
function nodeFile(node) {
|
|
36
49
|
if (!node)
|
|
@@ -158,7 +171,7 @@ function proofGuidance(ledger, summary, dyn) {
|
|
|
158
171
|
state: "attempted",
|
|
159
172
|
title: `0 Dynamically Proven — top ${dyn.attempted} attempted, all setup-blocked`,
|
|
160
173
|
body: `Dynamic proof attempted ${dyn.attempted} target${plural}; all were blocked by ${dom.label} (${dom.count}/${dom.total}). This is a sandbox setup gap, not a static-test failure — the Statically Linked signals are still shown.`,
|
|
161
|
-
action: HANDOFF_ACTION
|
|
174
|
+
action: nextStepFor(dom) ?? HANDOFF_ACTION
|
|
162
175
|
};
|
|
163
176
|
}
|
|
164
177
|
const because = dom ? ` Blocked because: ${dom.label} (${dom.count}/${dom.total}).` : "";
|
|
@@ -166,7 +179,7 @@ function proofGuidance(ledger, summary, dyn) {
|
|
|
166
179
|
state: "attempted",
|
|
167
180
|
title: `0 Dynamically Proven — top ${dyn.attempted} attempted, 0 closed`,
|
|
168
181
|
body: `OrangePro mapped this repo statically. Dynamic proof is a targeted verification pass: it runs existing or generated tests, mutates the exact behavior, and promotes only tests that fail at an assertion. This run attempted the top ${dyn.attempted} eligible behavior${plural} and closed 0.${because} Static test signals stay Statically Linked.`,
|
|
169
|
-
action: HANDOFF_ACTION
|
|
182
|
+
action: nextStepFor(dom) ?? HANDOFF_ACTION
|
|
170
183
|
};
|
|
171
184
|
}
|
|
172
185
|
// Standalone report regen (no THIS-RUN data) → derive from the ledger only.
|
|
@@ -313,7 +326,53 @@ function candidateFlows(graph) {
|
|
|
313
326
|
}))
|
|
314
327
|
};
|
|
315
328
|
}
|
|
316
|
-
|
|
329
|
+
/** Tier sort rank: Proven first, then Test signal, Reachable, No signal. */
|
|
330
|
+
function tierRank(b) {
|
|
331
|
+
if (b.tier === "proven")
|
|
332
|
+
return 0;
|
|
333
|
+
if (b.tier === "assoc")
|
|
334
|
+
return 1;
|
|
335
|
+
return b.reachable ? 2 : 3;
|
|
336
|
+
}
|
|
337
|
+
/**
|
|
338
|
+
* Link REAL generated tests (graph.generated_tests) to a risk row: exact
|
|
339
|
+
* target-symbol match first, then same-file. Metadata shown is honest and
|
|
340
|
+
* verbatim (title, test_type, framework hint, weak-evidence disclosure, body)
|
|
341
|
+
* — nothing is fabricated; no tests ⇒ the template hides the section.
|
|
342
|
+
*/
|
|
343
|
+
function riskGeneratedTests(graph, gap, riskIds, isFirstRowForFile) {
|
|
344
|
+
const all = graph.generated_tests ?? [];
|
|
345
|
+
const fileOf = (sym) => sym?.match(/^sym:(.+)#/)?.[1];
|
|
346
|
+
// Exact target wins. The same-file fallback (test targets an UNLISTED symbol
|
|
347
|
+
// in this file) attaches to exactly ONE deterministic row — the file's first
|
|
348
|
+
// (highest-ranked) risk row — and is labeled "same-file target" so a real
|
|
349
|
+
// generated test never reads as generated FOR a sibling behavior.
|
|
350
|
+
const linked = all.flatMap((t) => {
|
|
351
|
+
if (!t.target_symbol_external_id)
|
|
352
|
+
return [];
|
|
353
|
+
if (t.target_symbol_external_id === gap.id)
|
|
354
|
+
return [{ t, sameFile: false }];
|
|
355
|
+
if (riskIds.has(t.target_symbol_external_id))
|
|
356
|
+
return [];
|
|
357
|
+
if (!isFirstRowForFile)
|
|
358
|
+
return [];
|
|
359
|
+
return fileOf(t.target_symbol_external_id) === gap.file ? [{ t, sameFile: true }] : [];
|
|
360
|
+
});
|
|
361
|
+
return linked.slice(0, 2).map(({ t, sameFile }) => ({
|
|
362
|
+
name: t.title,
|
|
363
|
+
concern: t.test_type && t.test_type !== "unknown" ? t.test_type : undefined,
|
|
364
|
+
assertion: [sameFile ? "same-file target" : "", t.framework_hint, t.weak_evidence_used ? "weak evidence disclosed" : ""]
|
|
365
|
+
.filter(Boolean)
|
|
366
|
+
.join(" · "),
|
|
367
|
+
code: t.body
|
|
368
|
+
}));
|
|
369
|
+
}
|
|
370
|
+
function riskRows(risks, graph) {
|
|
371
|
+
const riskIds = new Set(risks.map((r) => r.id));
|
|
372
|
+
const firstRowForFile = new Map();
|
|
373
|
+
for (const r of risks)
|
|
374
|
+
if (!firstRowForFile.has(r.file))
|
|
375
|
+
firstRowForFile.set(r.file, r.id);
|
|
317
376
|
return risks.map((risk, idx) => {
|
|
318
377
|
const methodMatch = risk.title.match(/^(GET|POST|PUT|PATCH|DELETE)\s+(.+)$/i);
|
|
319
378
|
const pathMatch = risk.file.match(/\/api\/(.+)$/);
|
|
@@ -326,6 +385,15 @@ function riskRows(risks) {
|
|
|
326
385
|
tags.push(["Entry point", "entry"]);
|
|
327
386
|
return {
|
|
328
387
|
rank: idx + 1,
|
|
388
|
+
...(() => {
|
|
389
|
+
const generatedTests = riskGeneratedTests(graph, risk, riskIds, firstRowForFile.get(risk.file) === risk.id);
|
|
390
|
+
return {
|
|
391
|
+
generatedTests,
|
|
392
|
+
// Honest category strip: only the concerns of REAL attached tests —
|
|
393
|
+
// every pill renders as shown ("n of n"), none fabricated as locked.
|
|
394
|
+
applicableCategories: [...new Set(generatedTests.map((t) => t.concern).filter((c) => Boolean(c)))]
|
|
395
|
+
};
|
|
396
|
+
})(),
|
|
329
397
|
verb: methodMatch?.[1]?.toUpperCase() ?? (risk.entry_point ? "ENTRY" : "CODE"),
|
|
330
398
|
path: methodMatch?.[2] ?? (pathMatch ? `/${pathMatch[1]}` : risk.file),
|
|
331
399
|
desc: risk.reasons.join(" · "),
|
|
@@ -348,6 +416,7 @@ export function buildBehaviorReportData(graph, ledger, opts = {}) {
|
|
|
348
416
|
const repoRoot = opts.repoRoot ?? graph.workspace.root;
|
|
349
417
|
const riskGaps = rankRiskGaps(graph, { repoRoot, limit: opts.riskLimit ?? 20 });
|
|
350
418
|
const lists = behaviorLists(rows, flowIds);
|
|
419
|
+
const risks = riskRows(riskGaps, graph);
|
|
351
420
|
return {
|
|
352
421
|
repo: path.basename(repoRoot || graph.workspace.name || "repo"),
|
|
353
422
|
scanned: (graph.updated_at || graph.created_at || new Date(0).toISOString()).slice(0, 10),
|
|
@@ -358,10 +427,13 @@ export function buildBehaviorReportData(graph, ledger, opts = {}) {
|
|
|
358
427
|
pipeline: pipeline(graph, ledger, summary),
|
|
359
428
|
scan: scanBlock(graph, rows),
|
|
360
429
|
behaviorGroups: lists.behaviorGroups,
|
|
361
|
-
|
|
430
|
+
// Proven first so the strongest evidence leads the grid (stable within tiers).
|
|
431
|
+
behaviors: [...lists.behaviors].sort((a, b) => tierRank(a) - tierRank(b)),
|
|
362
432
|
flows: flows(graph, rows, riskGaps),
|
|
363
433
|
candidateFlows: candidateFlows(graph),
|
|
364
|
-
risks
|
|
365
|
-
zeroProofExplainer: summary.proven === 0 ? { title: ZERO_PROOF_EXPLAINER.title, body: [...ZERO_PROOF_EXPLAINER.body] } : null
|
|
434
|
+
risks,
|
|
435
|
+
zeroProofExplainer: summary.proven === 0 ? { title: ZERO_PROOF_EXPLAINER.title, body: [...ZERO_PROOF_EXPLAINER.body] } : null,
|
|
436
|
+
generatedTotal: graph.generated_tests?.length ?? 0,
|
|
437
|
+
shownCount: risks.reduce((acc, r) => acc + r.generatedTests.length, 0)
|
|
366
438
|
};
|
|
367
439
|
}
|