@orangepro/orangepro-mcp 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -12
- package/dist/local/analyze/analyzer.js +61 -2
- package/dist/local/analyze/behaviorContracts.js +78 -5
- package/dist/local/analyze/parseCache.js +17 -3
- package/dist/local/analyze/treeSitter/engine.js +81 -4
- package/dist/local/autoProve.js +39 -21
- package/dist/local/cli.js +13 -3
- package/dist/local/enrich/markdown.js +11 -0
- package/dist/local/flows/flowWalker.js +11 -6
- package/dist/local/generate/generator.js +54 -3
- package/dist/local/generate/promptV5.js +12 -1
- package/dist/local/graph/ontology.js +3 -1
- package/dist/local/operations.js +61 -10
- package/dist/local/proofDoctor.js +28 -3
- package/dist/local/rtm.js +30 -26
- package/dist/local/score/risk.js +128 -37
- package/dist/local/util/walk.js +5 -4
- package/dist/local/viz/behaviorReportData.js +393 -23
- package/dist/local/viz/behaviorReportHtml.js +225 -10
- package/dist/local/viz/coverageReveal.js +29 -0
- package/dist/local/viz/payload.js +12 -5
- package/dist/local/workspace.js +25 -2
- package/package.json +7 -2
- package/scripts/spikes/go-dynamic-proof-spike.mjs +17 -13
- package/scripts/spikes/go-mutate.go +60 -21
- package/scripts/spikes/java-dynamic-proof-spike.mjs +6 -4
- package/scripts/spikes/java-mutate.mjs +3 -2
package/dist/local/cli.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import { parseArgs, collectSetupCommands } from "./cliArgs.js";
|
|
3
3
|
import { opAnalyze, opAiFlows, opAiLinks, opChanged, opCompare, opDoctor, opProofDoctor, opDynamicProof, opExplain, opGaps, opGenerate, opBehaviorCoverageHtml, opCoverageReport, opInit, opProveLoop, opRuntimeCoverage, opScore, opRecordRun, opRtm, opStats, opStatus, opUpdate, opSetModelDefault, opStart, getModelDefault, resolveDiffTargets, resolvePrCheckout, writeCompareReport } from "./operations.js";
|
|
4
4
|
import { dominantBlockReason } from "./viz/behaviorReportData.js";
|
|
5
|
+
import { coverageRevealLine } from "./viz/coverageReveal.js";
|
|
5
6
|
import { autoProve, isRoastSurvivor } from "./autoProve.js";
|
|
6
7
|
import { opRecipeDbSqljs } from "./recipe/dbSqljs.js";
|
|
7
8
|
import { runExportCli } from "./exportCli.js";
|
|
@@ -236,6 +237,11 @@ async function main() {
|
|
|
236
237
|
out(` skipped: ${skip.target_symbol ?? skip.title} — ${skip.reason}`);
|
|
237
238
|
}
|
|
238
239
|
out(` Runtime-covered: ${res.rtm.summary.runtime_covered}`);
|
|
240
|
+
// G6: same-denominator coverage-vs-proof reveal — renders only when runtime
|
|
241
|
+
// coverage was ingested; percentages share summary.total (never mixed scopes).
|
|
242
|
+
const reveal = coverageRevealLine(res.rtm.summary);
|
|
243
|
+
if (reveal)
|
|
244
|
+
out(` ${reveal}`);
|
|
239
245
|
out(` Statically Linked: ${res.rtm.summary.associated} (static test link, not dynamic proof)`);
|
|
240
246
|
out(` No integration signal: ${res.rtm.summary.no_link}`);
|
|
241
247
|
out(` AI-linked: ${res.ai_linked.behaviors} behavior(s), ${res.ai_linked.symbols} symbol(s), ${res.ai_linked.links} weak link(s) — not coverage`);
|
|
@@ -272,7 +278,10 @@ async function main() {
|
|
|
272
278
|
out(" - opro agent --client claude-code");
|
|
273
279
|
out(" - opro agent --client cursor");
|
|
274
280
|
out(" - opro agent --client opencode");
|
|
275
|
-
|
|
281
|
+
// opStart aggregates warnings from ai-flows generate AND apply — the same
|
|
282
|
+
// message (e.g. the prompt entry cap) can legitimately arrive twice.
|
|
283
|
+
// Dedupe at print time; JSON output keeps the raw array.
|
|
284
|
+
for (const w of [...new Set(res.warnings)])
|
|
276
285
|
out(` warning: ${w}`);
|
|
277
286
|
}
|
|
278
287
|
return 0;
|
|
@@ -462,7 +471,7 @@ async function main() {
|
|
|
462
471
|
}
|
|
463
472
|
if (coverageReport)
|
|
464
473
|
out(` coverage report: ${coverageReport}`);
|
|
465
|
-
for (const w of [...res.warnings, ...aiFlowWarnings, ...htmlWarnings])
|
|
474
|
+
for (const w of [...new Set([...res.warnings, ...aiFlowWarnings, ...htmlWarnings])])
|
|
466
475
|
out(` warning: ${w}`);
|
|
467
476
|
const sugg = res.analysis.exclude_suggestions ?? [];
|
|
468
477
|
if (sugg.length) {
|
|
@@ -573,7 +582,8 @@ async function main() {
|
|
|
573
582
|
out(` next: ${b.next_step}`);
|
|
574
583
|
}
|
|
575
584
|
for (const nk of res.non_killing) {
|
|
576
|
-
|
|
585
|
+
const nkLabel = nk.mutant_status === "associated_non_assertion_failure" ? "mutant failed (non-assertion)" : "mutant survived";
|
|
586
|
+
out(` ${nkLabel}: ${nk.target_symbol}${nk.test_path ? ` (test: ${nk.test_path})` : ""}`);
|
|
577
587
|
out(` ${nk.note}`);
|
|
578
588
|
}
|
|
579
589
|
}
|
|
@@ -4,6 +4,14 @@ import { slugify } from "../util/ids.js";
|
|
|
4
4
|
import { redactSecrets } from "../util/redact.js";
|
|
5
5
|
const DETECTOR = "markdown_docs";
|
|
6
6
|
const MAX_REQUIREMENTS = 60;
|
|
7
|
+
/**
|
|
8
|
+
* Repo-governance and template markdown must never mint Requirement nodes.
|
|
9
|
+
* Hint words like "should"/"must" are ubiquitous in CONTRIBUTING files and
|
|
10
|
+
* PR/issue templates — on Hono, ".github/PULL_REQUEST_TEMPLATE.md" produced
|
|
11
|
+
* REQ-md-the-author-should-do-the-following-if-applicable and surfaced as the
|
|
12
|
+
* report's top suggested next action. Product docs (README, docs/) still count.
|
|
13
|
+
*/
|
|
14
|
+
const GOVERNANCE_MD_RE = /(^|\/)\.github\/|(^|\/)(CONTRIBUTING|CODE_OF_CONDUCT|PULL_REQUEST_TEMPLATE|ISSUE_TEMPLATE|SECURITY|SUPPORT|CHANGELOG|LICENSE|GOVERNANCE|MAINTAINERS|CODEOWNERS|AUTHORS)[^\/]*$|(^|\/)\.changeset\/|\/templates?\/|(^|\/)(AGENTS|CLAUDE|GEMINI|COPILOT)\.md$|(^|\/)\.cursor(rules)?\//i;
|
|
7
15
|
/** Words that suggest a heading describes a requirement/feature behavior. */
|
|
8
16
|
const REQUIREMENT_HINTS = [
|
|
9
17
|
"requirement",
|
|
@@ -53,6 +61,9 @@ function parseBullet(line) {
|
|
|
53
61
|
* Bounded to ~60 requirements; all captured text is secret-redacted.
|
|
54
62
|
*/
|
|
55
63
|
export function enrichFromMarkdown(relPath, content) {
|
|
64
|
+
if (GOVERNANCE_MD_RE.test(relPath)) {
|
|
65
|
+
return { nodes: [], edges: [], candidate_edges: [], sources: [], warnings: [] };
|
|
66
|
+
}
|
|
56
67
|
const nodes = [];
|
|
57
68
|
const edges = [];
|
|
58
69
|
const warnings = [];
|
|
@@ -3,7 +3,7 @@ import { LOCAL_GRAPH_SCHEMA_VERSION } from "../graph/ontology.js";
|
|
|
3
3
|
import { rankRiskGaps } from "../score/risk.js";
|
|
4
4
|
import { stableId } from "../util/ids.js";
|
|
5
5
|
const DEFAULT_MAX_DEPTH = 8;
|
|
6
|
-
const DEFAULT_MAX_FLOWS_PER_ENTRY =
|
|
6
|
+
const DEFAULT_MAX_FLOWS_PER_ENTRY = 5;
|
|
7
7
|
const DEFAULT_GLOBAL_CAP = 500;
|
|
8
8
|
const HIGH_ROUTE_RE = /payment|refund|checkout|cart|order|auth|login|token|customer|user|tax|fulfillment|ship/i;
|
|
9
9
|
const MUTATION_METHOD_RE = /^(POST|PUT|PATCH|DELETE)\b/i;
|
|
@@ -103,11 +103,16 @@ export function rankEntries(graph, entries, adjacency) {
|
|
|
103
103
|
gap.id,
|
|
104
104
|
gap.risk_score
|
|
105
105
|
]));
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
106
|
+
// Endpoint-anchored flows first. An Endpoint entry IS the definition of a
|
|
107
|
+
// user-triggerable behavior (June 27 agreement); orphan call-graph roots are
|
|
108
|
+
// useful but must never crowd endpoints out of the global cap — on Twenty,
|
|
109
|
+
// saturated risk ties let ~25 internal orphan methods consume all 500 flow
|
|
110
|
+
// slots while every HTTP/GraphQL entry point went unrendered.
|
|
111
|
+
const score = (e) => Math.max(riskScores.get(e.start) ?? 0, fallbackScore(e, adjacency));
|
|
112
|
+
const byScore = (a, b) => score(b) - score(a) || a.external_id.localeCompare(b.external_id) || a.start.localeCompare(b.start);
|
|
113
|
+
const endpoints = entries.filter((e) => e.kind === "Endpoint").sort(byScore);
|
|
114
|
+
const behaviors = entries.filter((e) => e.kind !== "Endpoint").sort(byScore);
|
|
115
|
+
return [...endpoints, ...behaviors];
|
|
111
116
|
}
|
|
112
117
|
function prunePrefixSubsumed(flows) {
|
|
113
118
|
const sorted = [...flows].sort((a, b) => b.path.length - a.path.length || a.id.localeCompare(b.id));
|
|
@@ -552,7 +552,7 @@ export function gatherContext(graph, behavior, framework, fileReader) {
|
|
|
552
552
|
acceptance_criteria: acceptance,
|
|
553
553
|
workflow_steps: workflow,
|
|
554
554
|
framework,
|
|
555
|
-
test_layer: inferLayer(behavior, framework),
|
|
555
|
+
test_layer: inferLayer(behavior, framework, graph),
|
|
556
556
|
code_context: dedupe(codeContext),
|
|
557
557
|
source_excerpts: excerpts,
|
|
558
558
|
weak_context: dedupe(weakContext),
|
|
@@ -602,7 +602,7 @@ function flowChainFor(graph, behavior) {
|
|
|
602
602
|
};
|
|
603
603
|
});
|
|
604
604
|
}
|
|
605
|
-
function inferLayer(behavior, framework) {
|
|
605
|
+
function inferLayer(behavior, framework, graph) {
|
|
606
606
|
const fw = framework.toLowerCase();
|
|
607
607
|
if (fw.includes("playwright") || fw.includes("cypress"))
|
|
608
608
|
return "e2e";
|
|
@@ -613,6 +613,21 @@ function inferLayer(behavior, framework) {
|
|
|
613
613
|
const hint = String(behavior.properties.test_layer ?? "");
|
|
614
614
|
if (hint)
|
|
615
615
|
return hint;
|
|
616
|
+
// Graph-aware default (hop-count methodology): a behavior that an endpoint
|
|
617
|
+
// implements, or that participates in a multi-step call chain, is an
|
|
618
|
+
// INTEGRATION target — the code→flows→behaviors journey is the product;
|
|
619
|
+
// "unit" is only for genuinely 0-hop leaf functions. The old blanket
|
|
620
|
+
// "unit" default stamped every vitest/jest repo unit-first.
|
|
621
|
+
if (graph) {
|
|
622
|
+
const id = behavior.external_id;
|
|
623
|
+
const isEntryHandler = graph.edges.some((e) => e.relationship_type === "IMPLEMENTED_IN" && e.to_external_id === id);
|
|
624
|
+
if (isEntryHandler)
|
|
625
|
+
return "integration";
|
|
626
|
+
const inChain = graph.edges.some((e) => e.relationship_type === "CALLS" && (e.from_external_id === id || e.to_external_id === id)) ||
|
|
627
|
+
(graph.analysis?.flows?.flows ?? []).some((f) => f.entry_point.external_id === id || f.hops.some((h) => h.from === id || h.to === id));
|
|
628
|
+
if (inChain)
|
|
629
|
+
return "integration";
|
|
630
|
+
}
|
|
616
631
|
return "unit";
|
|
617
632
|
}
|
|
618
633
|
function tooThin(ctx) {
|
|
@@ -2366,13 +2381,49 @@ export async function generateTests(graph, opts, provider, fileReader, clock = s
|
|
|
2366
2381
|
const runnable = isRunnable(body, framework, import_provenance, importErrors) && !compileIssue;
|
|
2367
2382
|
if (!runnable) {
|
|
2368
2383
|
const reason = unresolved_reason ?? compileIssue ?? runnableFailureReason(body, framework, import_provenance, importErrors, declaredDeps);
|
|
2369
|
-
warnings.push(`
|
|
2384
|
+
warnings.push(`Non-runnable v5 generated test for "${gc.ctx.behavior_title}" / "${scenario.title}" kept as an English intent (no run command): ${reason}`);
|
|
2370
2385
|
missing.push({
|
|
2371
2386
|
external_id: behavior.external_id,
|
|
2372
2387
|
title: gc.ctx.behavior_title,
|
|
2373
2388
|
reason,
|
|
2374
2389
|
needed: ["a compiling generated test with a real assertion and resolvable subject import"]
|
|
2375
2390
|
});
|
|
2391
|
+
// Preserve the grounded INTENT in English, never the rejected code.
|
|
2392
|
+
// The scenario fields were authored by a model that saw source
|
|
2393
|
+
// excerpts — scrub the composed body with the same guard as code.
|
|
2394
|
+
// The scenario plan (title / assertion targets / rationale) is the
|
|
2395
|
+
// reviewable half of the draft; withholding the body entirely also
|
|
2396
|
+
// removes any residual source-echo risk. runnable:false + the reason
|
|
2397
|
+
// keep this honestly a draft — it ships with no run command and can
|
|
2398
|
+
// never enter the proof-ready set.
|
|
2399
|
+
const manualBody = sanitizeGeneratedBody([
|
|
2400
|
+
`Scenario: ${scenario.title}`,
|
|
2401
|
+
...(scenario.steps && scenario.steps.length
|
|
2402
|
+
? ["Steps:", ...scenario.steps.map((st, n) => ` ${n + 1}. ${st}`)]
|
|
2403
|
+
: []),
|
|
2404
|
+
...(scenario.test_data ? [`Test data: ${scenario.test_data}`] : []),
|
|
2405
|
+
...(scenario.assertion_targets.length ? [`Expected: ${scenario.assertion_targets.join("; ")}`] : []),
|
|
2406
|
+
...(scenario.rationale ? [`Why this test: ${scenario.rationale}`] : []),
|
|
2407
|
+
"",
|
|
2408
|
+
// Concise blocker: first clause only — the full remedy is one line.
|
|
2409
|
+
`Blocked by: ${reason.split(" — ")[0]}`,
|
|
2410
|
+
"Fix: install this repo's dependencies / configure the test runner, then re-run \`opro start\`."
|
|
2411
|
+
].join("\n"), gc.ctx.source_excerpts, "//").body;
|
|
2412
|
+
generated.push({
|
|
2413
|
+
id: `${run_id}-t${generated.length + 1}`,
|
|
2414
|
+
run_id,
|
|
2415
|
+
title: `${gc.ctx.behavior_title} — ${scenario.title}`,
|
|
2416
|
+
test_type: gc.ctx.test_layer,
|
|
2417
|
+
framework_hint: framework,
|
|
2418
|
+
body: manualBody,
|
|
2419
|
+
bucket: bucketForV5Scenario(scenario),
|
|
2420
|
+
prompt_version: PROMPT_VERSION_V5,
|
|
2421
|
+
grounding: { entity_ids: gc.entityIds, source_refs: [], weak_relationships_used: [] },
|
|
2422
|
+
weak_evidence_used: false,
|
|
2423
|
+
target_symbol_external_id: behavior.external_id,
|
|
2424
|
+
runnable: false,
|
|
2425
|
+
unresolved_reason: reason
|
|
2426
|
+
});
|
|
2376
2427
|
continue;
|
|
2377
2428
|
}
|
|
2378
2429
|
generated.push({
|
|
@@ -75,7 +75,7 @@ export function buildPlanningSystemPromptV5() {
|
|
|
75
75
|
"- Return a raw JSON array only. No prose, no markdown fences, no heading, no explanation.",
|
|
76
76
|
"- If no missing scenario is justified, return [] exactly.",
|
|
77
77
|
"- The first character of your response must be [ and the last character must be ].",
|
|
78
|
-
'[{"id":1,"title":"...","concern":"...","technique":"...","rationale":"...","assertion_targets":["..."],"complexity":"basic|intermediate|advanced","risk_rank":1}]'
|
|
78
|
+
'[{"id":1,"title":"...","concern":"...","technique":"...","rationale":"...","assertion_targets":["..."],"steps":["Given ...","When ...","Then ..."],"test_data":"concrete example input values (synthetic, showing the edge case)","complexity":"basic|intermediate|advanced","risk_rank":1}]'
|
|
79
79
|
].join("\n");
|
|
80
80
|
}
|
|
81
81
|
export function buildPlanningUserPromptV5(ctx) {
|
|
@@ -303,6 +303,15 @@ function validatePlannedScenario(v) {
|
|
|
303
303
|
const riskRank = toFinite(v.risk_rank);
|
|
304
304
|
if (riskRank === null)
|
|
305
305
|
return { ok: false, reason: "non-finite risk_rank" };
|
|
306
|
+
// Optional human-readable fields (tolerant: absent/malformed → omitted, never a rejection).
|
|
307
|
+
const steps = Array.isArray(v.steps)
|
|
308
|
+
? v.steps.filter((x) => typeof x === "string" && x.trim().length > 0).slice(0, 6).map((x) => x.slice(0, 240))
|
|
309
|
+
: undefined;
|
|
310
|
+
const test_data = typeof v.test_data === "string" && v.test_data.trim()
|
|
311
|
+
? v.test_data.slice(0, 400)
|
|
312
|
+
: v.test_data && typeof v.test_data === "object"
|
|
313
|
+
? JSON.stringify(v.test_data).slice(0, 400)
|
|
314
|
+
: undefined;
|
|
306
315
|
return {
|
|
307
316
|
ok: true,
|
|
308
317
|
value: {
|
|
@@ -312,6 +321,8 @@ function validatePlannedScenario(v) {
|
|
|
312
321
|
technique: technique,
|
|
313
322
|
rationale: typeof v.rationale === "string" ? v.rationale : "",
|
|
314
323
|
assertion_targets: targets,
|
|
324
|
+
...(steps && steps.length ? { steps } : {}),
|
|
325
|
+
...(test_data ? { test_data } : {}),
|
|
315
326
|
complexity,
|
|
316
327
|
risk_rank: riskRank
|
|
317
328
|
}
|
|
@@ -9,7 +9,9 @@
|
|
|
9
9
|
* The graph is built directly by OrangePro; it does not depend on any
|
|
10
10
|
* third-party graph product or format.
|
|
11
11
|
*/
|
|
12
|
-
|
|
12
|
+
// v2: Go method symbol ids are receiver-qualified (`sym:file.go#Recv.M`) — old
|
|
13
|
+
// graphs hold bare-name method ids and must force-rebuild (loadGraph hard-fails).
|
|
14
|
+
export const LOCAL_GRAPH_SCHEMA_VERSION = "orangepro.local_graph.v2";
|
|
13
15
|
/** Node kinds that map to behaviors/requirements for scoring + gaps + generation. */
|
|
14
16
|
export const BEHAVIOR_KINDS = new Set([
|
|
15
17
|
"Requirement",
|
package/dist/local/operations.js
CHANGED
|
@@ -35,7 +35,7 @@ import { changedImpact } from "./freshness/changed.js";
|
|
|
35
35
|
import { explainTest } from "./explain/explain.js";
|
|
36
36
|
import { buildVizPayload } from "./viz/payload.js";
|
|
37
37
|
import { renderVizHtml } from "./viz/html.js";
|
|
38
|
-
import { buildBehaviorReportData, dominantBlockReason } from "./viz/behaviorReportData.js";
|
|
38
|
+
import { buildBehaviorReportData, computeReportDelta, reportBaselineOf, dominantBlockReason } from "./viz/behaviorReportData.js";
|
|
39
39
|
import { renderBehaviorReport } from "./viz/behaviorReportHtml.js";
|
|
40
40
|
import { renderCoverageReport } from "./pack/coverageReport.js";
|
|
41
41
|
import { confirmedCoverageByLayer } from "./score/coverage.js";
|
|
@@ -273,11 +273,14 @@ function symbolTargetParts(symExtId) {
|
|
|
273
273
|
throw new Error(`Cannot derive dynamic proof target from symbol id: ${symExtId}`);
|
|
274
274
|
}
|
|
275
275
|
const [, file, symbolName] = match;
|
|
276
|
-
const
|
|
276
|
+
const segments = symbolName.split(".").filter(Boolean);
|
|
277
|
+
const method = segments.pop();
|
|
277
278
|
if (!file || !method) {
|
|
278
279
|
throw new Error(`Cannot derive dynamic proof target from symbol id: ${symExtId}`);
|
|
279
280
|
}
|
|
280
|
-
|
|
281
|
+
// The owner qualifier of a member id (TS `Class.method`, Go `Recv.M`). The Go
|
|
282
|
+
// lane passes it as --recv so the mutator matches the exact receiver.
|
|
283
|
+
return { file, method, ...(segments.length ? { memberQualifier: segments.join(".") } : {}) };
|
|
281
284
|
}
|
|
282
285
|
function assertProofTargetMatchesSymbol(opts, symbolTarget) {
|
|
283
286
|
if (opts.target_path !== undefined && opts.target_path !== "") {
|
|
@@ -560,8 +563,9 @@ export function opAnalyze(root, opts = {}, deps = defaultDeps()) {
|
|
|
560
563
|
? (!opts.suppressProgress && reportProgress("coverage: generating local runtime coverage before graph build", { current: 2, total: 4 }),
|
|
561
564
|
prepareRuntimeCoverage(scanRoot, { generate: true, timeoutMs: opts.coverageTimeoutMs, runner: deps.coverageRunner }))
|
|
562
565
|
: undefined;
|
|
563
|
-
|
|
564
|
-
|
|
566
|
+
// Idempotent for existing workspaces and also applies conservative migrations
|
|
567
|
+
// to untouched generated workspace files (for example .orangeproignore).
|
|
568
|
+
initWorkspace(root, now);
|
|
565
569
|
if (!opts.suppressProgress) {
|
|
566
570
|
reportProgress("analyze: parsing source and building deterministic graph", {
|
|
567
571
|
current: opts.generateCoverage ? 3 : 2,
|
|
@@ -800,8 +804,9 @@ export function opDoctor(root) {
|
|
|
800
804
|
*/
|
|
801
805
|
export function opProofDoctor(root) {
|
|
802
806
|
const graph = loadGraph(workspacePaths(root).graphPath);
|
|
803
|
-
const
|
|
804
|
-
|
|
807
|
+
const ledger = loadLedger(root);
|
|
808
|
+
const rtm = buildRtm(graph, ledger);
|
|
809
|
+
return buildProofDoctor(graph, rtm, loadProofAttempts(root), {}, ledger);
|
|
805
810
|
}
|
|
806
811
|
export function opGaps(root, opts = {}) {
|
|
807
812
|
const graph = loadGraph(workspacePaths(root).graphPath);
|
|
@@ -1007,6 +1012,10 @@ export function opDynamicProof(root, opts, deps = defaultDeps()) {
|
|
|
1007
1012
|
// Slice 2: bind a runtime-named subtest's mutant failure to the exact assertion line.
|
|
1008
1013
|
if (opts.go_assertion_line !== undefined)
|
|
1009
1014
|
args.push("--go-assertion-line", String(opts.go_assertion_line));
|
|
1015
|
+
// Receiver-qualified method target (`sym:file.go#Recv.M`) → the mutator must
|
|
1016
|
+
// match the exact receiver, never a same-named decl on another type.
|
|
1017
|
+
if (symbolTarget.memberQualifier)
|
|
1018
|
+
args.push("--recv", symbolTarget.memberQualifier);
|
|
1010
1019
|
const run = (deps.dynamicProofRunner ?? defaultDynamicProofRunner)(args, {
|
|
1011
1020
|
cwd: goRoot,
|
|
1012
1021
|
scriptPath: dynamicProofSpikePathFor("go")
|
|
@@ -1458,6 +1467,28 @@ export function autoProveChangedScope(graph, changed, baseRef) {
|
|
|
1458
1467
|
});
|
|
1459
1468
|
return hasEligibleTarget ? meaningful : undefined; // no eligible provable target in scope → global top-5
|
|
1460
1469
|
}
|
|
1470
|
+
function writeStartStaticSnapshot(root, baseRef, warnings) {
|
|
1471
|
+
let behaviorCoveragePath;
|
|
1472
|
+
try {
|
|
1473
|
+
reportProgress("artifacts: writing static behavior view (proof still running)", { current: 4, total: 8 });
|
|
1474
|
+
behaviorCoveragePath = opBehaviorCoverageHtml(root, `${WORKSPACE_DIR}/behavior-coverage.html`, {
|
|
1475
|
+
attempted: 0,
|
|
1476
|
+
proven: 0,
|
|
1477
|
+
needsSetup: []
|
|
1478
|
+
}).behavior_coverage_path;
|
|
1479
|
+
}
|
|
1480
|
+
catch (error) {
|
|
1481
|
+
warnings.push(`static behavior view not written: ${error instanceof Error ? error.message : String(error)}`);
|
|
1482
|
+
}
|
|
1483
|
+
try {
|
|
1484
|
+
reportProgress("artifacts: writing static RTM (proof still running)", { current: 4, total: 8 });
|
|
1485
|
+
opRtm(root, { format: "md", baseRef, limit: START_RTM_LIMIT });
|
|
1486
|
+
}
|
|
1487
|
+
catch (error) {
|
|
1488
|
+
warnings.push(`static RTM not written: ${error instanceof Error ? error.message : String(error)}`);
|
|
1489
|
+
}
|
|
1490
|
+
return behaviorCoveragePath ? { behaviorCoveragePath } : {};
|
|
1491
|
+
}
|
|
1461
1492
|
export async function opStart(root, opts = {}, deps = defaultDeps()) {
|
|
1462
1493
|
const providerOpts = startProviderOverride(root, opts);
|
|
1463
1494
|
const scanRoot = opts.source ? resolve(opts.source) : resolve(root);
|
|
@@ -1475,6 +1506,7 @@ export async function opStart(root, opts = {}, deps = defaultDeps()) {
|
|
|
1475
1506
|
}, deps);
|
|
1476
1507
|
const warnings = [...analyze.warnings];
|
|
1477
1508
|
reportProgress("start: deterministic graph is ready", { current: 4, total: 8 });
|
|
1509
|
+
const staticSnapshot = writeStartStaticSnapshot(root, opts.baseRef, warnings);
|
|
1478
1510
|
const providerConfigured = deps.aiProvider !== undefined || resolveProviderConfig(providerEnv, providerOpts) !== null;
|
|
1479
1511
|
let aiLinks = { status: "skipped", reason: "AI candidate links disabled for this run." };
|
|
1480
1512
|
if (opts.ai !== false) {
|
|
@@ -1578,7 +1610,7 @@ export async function opStart(root, opts = {}, deps = defaultDeps()) {
|
|
|
1578
1610
|
limit: batch.length,
|
|
1579
1611
|
prompt_version: opts.promptVersion ?? "v5"
|
|
1580
1612
|
}, providerDeps);
|
|
1581
|
-
accepted += generated.generated_tests.length;
|
|
1613
|
+
accepted += generated.generated_tests.filter((t) => t.runnable !== false).length;
|
|
1582
1614
|
warnings.push(...generated.warnings.map((w) => `generate: ${w}`));
|
|
1583
1615
|
}
|
|
1584
1616
|
if (accepted === 0)
|
|
@@ -1611,7 +1643,7 @@ export async function opStart(root, opts = {}, deps = defaultDeps()) {
|
|
|
1611
1643
|
catch (error) {
|
|
1612
1644
|
warnings.push(`coverage report not written: ${error instanceof Error ? error.message : String(error)}`);
|
|
1613
1645
|
}
|
|
1614
|
-
let coverageHtml;
|
|
1646
|
+
let coverageHtml = staticSnapshot.behaviorCoveragePath;
|
|
1615
1647
|
try {
|
|
1616
1648
|
reportProgress("artifacts: writing behavior coverage view", { current: 7, total: 8 });
|
|
1617
1649
|
// Forward THIS-RUN dynamic-proof outcome so the report can name the dominant setup/runnability
|
|
@@ -1969,9 +2001,28 @@ export function opBehaviorCoverageHtml(root, outputPath = "orangepro-behavior-co
|
|
|
1969
2001
|
// proof-attempts sidecar ONLY when it anchors to the current graph+commit
|
|
1970
2002
|
// (stale evidence is dropped — fail closed; display copy only, no tier math).
|
|
1971
2003
|
const dyn = dynamicProof ?? sidecarDynamicProof(root, graph);
|
|
1972
|
-
const
|
|
2004
|
+
const data = buildBehaviorReportData(graph, loadLedger(root), { repoRoot: root, dynamicProof: dyn });
|
|
2005
|
+
// Delta-since-last-run: best-effort read of the previous snapshot; a missing
|
|
2006
|
+
// or unreadable baseline means first run (banner hidden). Display-only —
|
|
2007
|
+
// the delta never touches tiers, ranks, or counts.
|
|
2008
|
+
const baselinePath = resolve(root, `${WORKSPACE_DIR}/report-baseline.json`);
|
|
2009
|
+
try {
|
|
2010
|
+
const prev = JSON.parse(readFileSync(baselinePath, "utf8"));
|
|
2011
|
+
if (prev && prev.summary && Array.isArray(prev.riskPaths))
|
|
2012
|
+
data.delta = computeReportDelta(prev, data);
|
|
2013
|
+
}
|
|
2014
|
+
catch {
|
|
2015
|
+
data.delta = null;
|
|
2016
|
+
}
|
|
2017
|
+
const html = renderBehaviorReport(data);
|
|
1973
2018
|
const htmlPath = resolve(root, outputPath);
|
|
1974
2019
|
writeFileSync(htmlPath, html, "utf8");
|
|
2020
|
+
try {
|
|
2021
|
+
writeFileSync(baselinePath, JSON.stringify(reportBaselineOf(data, new Date().toISOString())), "utf8");
|
|
2022
|
+
}
|
|
2023
|
+
catch {
|
|
2024
|
+
// baseline write is advisory
|
|
2025
|
+
}
|
|
1975
2026
|
return { behavior_coverage_path: htmlPath };
|
|
1976
2027
|
}
|
|
1977
2028
|
/** Fresh-only sidecar view for report regens; unreadable or stale ⇒ undefined. */
|
|
@@ -77,6 +77,16 @@ export const PROOF_BLOCKER_GUIDE = {
|
|
|
77
77
|
/** Wording is load-bearing: a survivor is a proven negative, never a user failure. */
|
|
78
78
|
export const NON_KILLING_NOTE = "Not proven: the test still passed while the target was mutated (possibly an equivalent mutation). " +
|
|
79
79
|
"The mutant surviving is a proven negative about assertion strength — it is never counted as Dynamically Proven.";
|
|
80
|
+
/** Opposite failure mode: the mutant DID make the test fail, but via a runtime
|
|
81
|
+
* crash rather than a trusted assertion. The test exercises the target; the
|
|
82
|
+
* proof standard (assertion failure) was not met. Misreporting this as
|
|
83
|
+
* "mutant survived" sends users to fix the wrong thing. */
|
|
84
|
+
export const NON_ASSERTION_NOTE = "Not proven: the mutant made the test FAIL, but with a runtime error instead of a trusted assertion failure. " +
|
|
85
|
+
"The test does exercise the target; strengthen the assertion to check the target's returned value directly, or re-run — " +
|
|
86
|
+
"whether the crash or the assertion is hit first can vary between runs.";
|
|
87
|
+
export function nonKillingNoteFor(mutantStatus) {
|
|
88
|
+
return mutantStatus === "associated_non_assertion_failure" ? NON_ASSERTION_NOTE : NON_KILLING_NOTE;
|
|
89
|
+
}
|
|
80
90
|
export function proofAttemptsPath(root) {
|
|
81
91
|
return join(workspacePaths(root).dir, PROOF_ATTEMPTS_FILE);
|
|
82
92
|
}
|
|
@@ -100,6 +110,7 @@ export function distillProofAttempts(auto, meta) {
|
|
|
100
110
|
target_symbol: a.target_symbol,
|
|
101
111
|
test_path: a.test_path || undefined,
|
|
102
112
|
classification: a.classification,
|
|
113
|
+
mutant_status: a.mutant_status,
|
|
103
114
|
category: a.category,
|
|
104
115
|
reason: a.reason ? redactSecrets(a.reason) : undefined,
|
|
105
116
|
deduped: a.deduped,
|
|
@@ -246,13 +257,26 @@ function groupBlockers(blocked, source) {
|
|
|
246
257
|
* Pure assembly: graph + canonical RTM result + optional attempts sidecar →
|
|
247
258
|
* deduped blocker report. Never writes; never mints; never recomputes proof.
|
|
248
259
|
*/
|
|
249
|
-
export function buildProofDoctor(graph, rtm, attempts, opts = {}) {
|
|
260
|
+
export function buildProofDoctor(graph, rtm, attempts, opts = {}, ledger) {
|
|
250
261
|
const io = opts.io ?? { exists: existsSync, nodeVersion: process.version };
|
|
251
262
|
const proven = rtm.summary.proven;
|
|
252
263
|
const denominator = rtm.summary.total;
|
|
253
264
|
// Freshness: the sidecar must anchor to the CURRENT graph generation + commit.
|
|
254
265
|
const stale = Boolean(attempts && !proofAttemptsFresh(attempts, graph));
|
|
255
266
|
const currentAttempts = attempts && !stale ? attempts : null;
|
|
267
|
+
// Legacy-sidecar backfill: proof-attempts files written before mutant_status
|
|
268
|
+
// was persisted carry no outcome detail, which forced every non-close into
|
|
269
|
+
// the "mutant survived" diagnosis. The ledger next to it is ground truth —
|
|
270
|
+
// recover mutant_status from the latest ledger record per target so the
|
|
271
|
+
// doctor self-heals without requiring a fresh `opro start`.
|
|
272
|
+
const ledgerStatusByTarget = new Map();
|
|
273
|
+
for (const r of ledger?.records ?? []) {
|
|
274
|
+
const ms = r.dynamic_proof?.mutant_status;
|
|
275
|
+
if (!ms)
|
|
276
|
+
continue;
|
|
277
|
+
ledgerStatusByTarget.set(r.target_symbol, ms); // records are append-ordered; last wins
|
|
278
|
+
}
|
|
279
|
+
const mutantStatusFor = (a) => a.mutant_status ?? ledgerStatusByTarget.get(a.target_symbol);
|
|
256
280
|
const blocked = (currentAttempts?.attempts ?? []).filter((a) => a.classification === "needs_setup");
|
|
257
281
|
const survivors = (currentAttempts?.attempts ?? []).filter((a) => a.classification === "non_killing");
|
|
258
282
|
let blockers = groupBlockers(blocked, "attempt");
|
|
@@ -268,7 +292,8 @@ export function buildProofDoctor(graph, rtm, attempts, opts = {}) {
|
|
|
268
292
|
if (seenSurvivors.has(key))
|
|
269
293
|
continue;
|
|
270
294
|
seenSurvivors.add(key);
|
|
271
|
-
|
|
295
|
+
const ms = mutantStatusFor(a);
|
|
296
|
+
non_killing.push({ target_symbol: a.target_symbol, test_path: a.test_path, mutant_status: ms, note: nonKillingNoteFor(ms) });
|
|
272
297
|
}
|
|
273
298
|
let status;
|
|
274
299
|
let headline;
|
|
@@ -291,7 +316,7 @@ export function buildProofDoctor(graph, rtm, attempts, opts = {}) {
|
|
|
291
316
|
}
|
|
292
317
|
else if (non_killing.length > 0) {
|
|
293
318
|
status = "blocked";
|
|
294
|
-
headline = `0 Dynamically Proven — ${non_killing.length} attempt(s) ran but
|
|
319
|
+
headline = `0 Dynamically Proven — ${non_killing.length} attempt(s) ran but did not close (see non_killing for the per-target outcome).`;
|
|
295
320
|
}
|
|
296
321
|
else {
|
|
297
322
|
status = "no_data";
|
package/dist/local/rtm.js
CHANGED
|
@@ -3,11 +3,12 @@ import { languageOf } from "./analyze/classify.js";
|
|
|
3
3
|
import { targetFingerprint } from "./ledger.js";
|
|
4
4
|
const STATUS_ORDER = {
|
|
5
5
|
"No integration signal": 0,
|
|
6
|
-
"
|
|
7
|
-
"
|
|
8
|
-
"
|
|
9
|
-
"
|
|
10
|
-
"
|
|
6
|
+
"Candidate signal (unconfirmed)": 1,
|
|
7
|
+
"Associated signal": 2,
|
|
8
|
+
"Generated-unverifiable": 3,
|
|
9
|
+
"Runtime-covered": 4,
|
|
10
|
+
"Proven": 5,
|
|
11
|
+
"Reproven (this run)": 6
|
|
11
12
|
};
|
|
12
13
|
const GENERIC_TEST_CATEGORIES = [
|
|
13
14
|
"happy-path",
|
|
@@ -72,6 +73,7 @@ export function renderRtmMarkdown(result) {
|
|
|
72
73
|
`| Dynamically Proven | ${provenValue} |`,
|
|
73
74
|
`| Runtime-covered | ${s.runtime_covered} |`,
|
|
74
75
|
`| Associated signal | ${s.associated} |`,
|
|
76
|
+
`| Candidate signal (unconfirmed) | ${s.candidate} |`,
|
|
75
77
|
`| No integration signal | ${s.no_link} |`,
|
|
76
78
|
`| Reproven this run | ${s.reproven_this_run} |`,
|
|
77
79
|
`| Generated unverifiable | ${s.generated_unverifiable} |`,
|
|
@@ -164,8 +166,13 @@ function evidenceTierFor(indexes, node, file, dynamicProof) {
|
|
|
164
166
|
return "proven";
|
|
165
167
|
if (node.kind === "CodeSymbol" && node.properties.runtime_covered === true)
|
|
166
168
|
return "runtime";
|
|
167
|
-
|
|
169
|
+
// Epistemic tiers (mirrors the platform's Proof/Candidate/Weak model):
|
|
170
|
+
// associated = a hard static test link (COVERS/TESTED_BY via real import) exists;
|
|
171
|
+
// candidate = only a lexical/Jaccard candidate edge exists — a lead, not evidence.
|
|
172
|
+
if (indexes.staticSignalsById.has(node.external_id))
|
|
168
173
|
return "associated";
|
|
174
|
+
if (indexes.candidateSignalsById.has(node.external_id) || (file !== "" && indexes.candidateSignalsByFile.has(file)))
|
|
175
|
+
return "candidate";
|
|
169
176
|
return "none";
|
|
170
177
|
}
|
|
171
178
|
function statusFor(evidence, ledgerRecord, dynamicProof) {
|
|
@@ -177,6 +184,8 @@ function statusFor(evidence, ledgerRecord, dynamicProof) {
|
|
|
177
184
|
return "Generated-unverifiable";
|
|
178
185
|
if (evidence === "associated")
|
|
179
186
|
return "Associated signal";
|
|
187
|
+
if (evidence === "candidate")
|
|
188
|
+
return "Candidate signal (unconfirmed)";
|
|
180
189
|
return "No integration signal";
|
|
181
190
|
}
|
|
182
191
|
/**
|
|
@@ -196,6 +205,7 @@ function summarizeRows(rows, unionRows = []) {
|
|
|
196
205
|
proven,
|
|
197
206
|
runtime_covered: rows.filter((row) => row.evidence_tier === "runtime").length,
|
|
198
207
|
associated: rows.filter((row) => row.evidence_tier === "associated").length,
|
|
208
|
+
candidate: rows.filter((row) => row.evidence_tier === "candidate").length,
|
|
199
209
|
no_link: rows.filter((row) => row.evidence_tier === "none").length,
|
|
200
210
|
reproven_this_run: reproven,
|
|
201
211
|
generated_unverifiable: rows.filter((row) => row.status === "Generated-unverifiable").length,
|
|
@@ -252,8 +262,8 @@ function compareLedgerRecords(a, aIndex, b, bIndex) {
|
|
|
252
262
|
function buildRtmIndexes(graph) {
|
|
253
263
|
const nodeById = new Map(graph.nodes.map((n) => [n.external_id, n]));
|
|
254
264
|
const staticSignalsById = new Map();
|
|
255
|
-
const
|
|
256
|
-
const
|
|
265
|
+
const candidateSignalsById = new Map();
|
|
266
|
+
const candidateSignalsByFile = new Map();
|
|
257
267
|
const add = (map, key, value) => {
|
|
258
268
|
const set = map.get(key);
|
|
259
269
|
if (set)
|
|
@@ -261,21 +271,16 @@ function buildRtmIndexes(graph) {
|
|
|
261
271
|
else
|
|
262
272
|
map.set(key, new Set([value]));
|
|
263
273
|
};
|
|
264
|
-
|
|
274
|
+
// Candidate (lexical/Jaccard) signals attach ONLY to the matched file itself.
|
|
275
|
+
// They must never propagate through imports: one lexical match on a barrel file
|
|
276
|
+
// previously marked its whole import subtree as "associated" — that inflated
|
|
277
|
+
// 93% of Twenty behaviors into the test-signal tier with zero real evidence.
|
|
265
278
|
const addFileAssociation = (file, value) => {
|
|
266
|
-
add(
|
|
267
|
-
const imports = importTargetsByFile.get(file) ?? [];
|
|
268
|
-
if (imports.length > ASSOCIATED_IMPORT_PROPAGATION_LIMIT)
|
|
269
|
-
return;
|
|
270
|
-
for (const importedFile of imports)
|
|
271
|
-
add(associatedSignalsByFile, importedFile, value);
|
|
279
|
+
add(candidateSignalsByFile, file, value);
|
|
272
280
|
};
|
|
273
281
|
for (const e of graph.edges) {
|
|
274
282
|
if (e.evidence_strength !== "hard")
|
|
275
283
|
continue;
|
|
276
|
-
if (e.relationship_type === "IMPORTS" && isFileSignalKey(e.from_external_id) && isFileSignalKey(e.to_external_id)) {
|
|
277
|
-
importTargetsByFile.set(e.from_external_id, [...(importTargetsByFile.get(e.from_external_id) ?? []), e.to_external_id]);
|
|
278
|
-
}
|
|
279
284
|
if (e.relationship_type !== "COVERS" && e.relationship_type !== "TESTED_BY")
|
|
280
285
|
continue;
|
|
281
286
|
const from = nodeById.get(e.from_external_id);
|
|
@@ -290,8 +295,8 @@ function buildRtmIndexes(graph) {
|
|
|
290
295
|
continue;
|
|
291
296
|
if (e.relationship_type !== "MAY_RELATE_TO" && e.relationship_type !== "MAY_BE_TESTED_BY" && e.relationship_type !== "MAY_COVER")
|
|
292
297
|
continue;
|
|
293
|
-
add(
|
|
294
|
-
add(
|
|
298
|
+
add(candidateSignalsById, e.from_external_id, e.to_external_id);
|
|
299
|
+
add(candidateSignalsById, e.to_external_id, e.from_external_id);
|
|
295
300
|
for (const [key, value] of [[e.from_external_id, e.to_external_id], [e.to_external_id, e.from_external_id]]) {
|
|
296
301
|
if (isFileSignalKey(key))
|
|
297
302
|
addFileAssociation(key, value);
|
|
@@ -300,21 +305,18 @@ function buildRtmIndexes(graph) {
|
|
|
300
305
|
const freeze = (map) => new Map([...map.entries()].map(([key, values]) => [key, [...values].sort()]));
|
|
301
306
|
return {
|
|
302
307
|
staticSignalsById: freeze(staticSignalsById),
|
|
303
|
-
|
|
304
|
-
|
|
308
|
+
candidateSignalsById: freeze(candidateSignalsById),
|
|
309
|
+
candidateSignalsByFile: freeze(candidateSignalsByFile)
|
|
305
310
|
};
|
|
306
311
|
}
|
|
307
312
|
function isFileSignalKey(id) {
|
|
308
313
|
return !id.startsWith("sym:") && !id.startsWith("test:") && !id.startsWith("flow:");
|
|
309
314
|
}
|
|
310
|
-
function hasAssociatedSignal(indexes, externalId, file) {
|
|
311
|
-
return indexes.staticSignalsById.has(externalId) || indexes.associatedSignalsById.has(externalId) || (file !== "" && indexes.associatedSignalsByFile.has(file));
|
|
312
|
-
}
|
|
313
315
|
function testSignalFor(indexes, externalId, file) {
|
|
314
316
|
const staticSignals = indexes.staticSignalsById.get(externalId) ?? [];
|
|
315
317
|
if (staticSignals.length > 0)
|
|
316
318
|
return `static candidate: ${staticSignals.slice(0, 3).join("; ")}`;
|
|
317
|
-
const associated = [...(indexes.
|
|
319
|
+
const associated = [...(indexes.candidateSignalsById.get(externalId) ?? []), ...(file ? indexes.candidateSignalsByFile.get(file) ?? [] : [])]
|
|
318
320
|
.filter((id) => id !== externalId && id !== file)
|
|
319
321
|
.sort();
|
|
320
322
|
if (associated.length > 0)
|
|
@@ -366,6 +368,8 @@ function normalizeStatusFilter(statuses) {
|
|
|
366
368
|
out.add("Runtime-covered");
|
|
367
369
|
else if (s === "associated")
|
|
368
370
|
out.add("Associated signal");
|
|
371
|
+
else if (s === "candidate" || s === "candidate-signal")
|
|
372
|
+
out.add("Candidate signal (unconfirmed)");
|
|
369
373
|
else if (s === "no-link" || s === "nolink" || s === "none")
|
|
370
374
|
out.add("No integration signal");
|
|
371
375
|
else if (s === "reproven" || s === "reproven-this-run")
|