@orangepro/orangepro-mcp 0.2.30 → 0.2.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -1
- package/dist/local/analyze/coverage.js +93 -12
- package/dist/local/analyze/coverageArtifacts.js +8 -1
- package/dist/local/autoProve.js +8 -6
- package/dist/local/cli.js +10 -2
- package/dist/local/cliArgs.js +2 -0
- package/dist/local/generate/draftGuidance.js +30 -0
- package/dist/local/generate/generator.js +7 -3
- package/dist/local/generate/providers.js +28 -15
- package/dist/local/operations.js +93 -13
- package/dist/local/pack/coverageReport.js +13 -3
- package/dist/local/viz/behaviorReportData.js +40 -3
- package/dist/local/viz/behaviorReportHtml.js +43 -5
- package/docs/local-proof-kit.md +8 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -199,6 +199,50 @@ Static mapping works across many languages via tree-sitter. Dynamic proof is del
|
|
|
199
199
|
|
|
200
200
|
---
|
|
201
201
|
|
|
202
|
+
## Highest-value local run
|
|
203
|
+
|
|
204
|
+
Use the repository's own setup and test commands first, and keep unit and integration
|
|
205
|
+
coverage in separate artifacts. Then run `opro start`; it performs analysis, ingests
|
|
206
|
+
the artifacts, attempts targeted proof, generates report-visible drafts, and writes the
|
|
207
|
+
final report. A separate `opro analyze` is unnecessary when `opro start` follows it.
|
|
208
|
+
|
|
209
|
+
```bash
|
|
210
|
+
# 1. Install/build exactly as the repository documents.
|
|
211
|
+
# 2. Run the repository's unit and integration coverage commands separately.
|
|
212
|
+
# 3. Record artifact provenance (example paths and commands):
|
|
213
|
+
mkdir -p .orangepro
|
|
214
|
+
# create .orangepro/coverage-suites.json using the schema below
|
|
215
|
+
|
|
216
|
+
opro coverage . # optional preflight: discover/generate artifacts
|
|
217
|
+
opro start . --proof-limit 5 --generate-limit 20
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
```json
|
|
221
|
+
{
|
|
222
|
+
"artifacts": {
|
|
223
|
+
".orangepro/coverage/unit.coverprofile": {
|
|
224
|
+
"suite": "unit",
|
|
225
|
+
"command": "make unit-test-coverage"
|
|
226
|
+
},
|
|
227
|
+
".orangepro/coverage/integration.coverprofile": {
|
|
228
|
+
"suite": "integration",
|
|
229
|
+
"command": "make integration-test-coverage"
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
Without this manifest, OrangePro conservatively infers clear `unit`/`integration` names
|
|
236
|
+
and labels everything else `unclassified`; it never guesses that an aggregate profile is
|
|
237
|
+
unit-only. The report shows unit, integration, their overlap, unclassified coverage, and
|
|
238
|
+
the combined union separately. `--proof-limit` controls dynamic proof attempts (which
|
|
239
|
+
may draft a test for proof); `--generate-limit` independently controls the additional
|
|
240
|
+
report-visible risk-gap drafting lane. A generation run
|
|
241
|
+
also records its terminal status and exact reason, so a compiler/import failure is not
|
|
242
|
+
misreported as a generic dependency problem.
|
|
243
|
+
|
|
244
|
+
---
|
|
245
|
+
|
|
202
246
|
## Privacy
|
|
203
247
|
|
|
204
248
|
- **No stored source.** Reads code in-process. Never uploads to an OrangePro server.
|
|
@@ -224,7 +268,7 @@ opro rtm # traceability matrix
|
|
|
224
268
|
opro export # metadata-only evidence pack
|
|
225
269
|
opro mcp # run as MCP server (stdio)
|
|
226
270
|
opro doctor # what evidence to add next
|
|
227
|
-
opro coverage #
|
|
271
|
+
opro coverage # discover/generate artifacts; analyze or start ingests them
|
|
228
272
|
```
|
|
229
273
|
|
|
230
274
|
Add `--json` to any read command for machine output. Run `opro help` for the full reference.
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
+
const COVERAGE_SUITE_MANIFEST = ".orangepro/coverage-suites.json";
|
|
3
4
|
const GO_COVERAGE_CANDIDATES = new Set([
|
|
4
5
|
"coverage.out",
|
|
5
6
|
"cover.out",
|
|
@@ -47,6 +48,48 @@ function symbolLanguage(n) {
|
|
|
47
48
|
return "java";
|
|
48
49
|
return "other";
|
|
49
50
|
}
|
|
51
|
+
function normalizedArtifactPath(value) {
|
|
52
|
+
const normalized = value.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
53
|
+
if (!normalized || path.posix.isAbsolute(normalized) || normalized.split("/").includes(".."))
|
|
54
|
+
return null;
|
|
55
|
+
return normalized;
|
|
56
|
+
}
|
|
57
|
+
function coverageSuiteManifest(root) {
|
|
58
|
+
const out = new Map();
|
|
59
|
+
try {
|
|
60
|
+
const raw = JSON.parse(readFileSync(path.join(root, COVERAGE_SUITE_MANIFEST), "utf8"));
|
|
61
|
+
for (const [artifactPath, value] of Object.entries(raw.artifacts ?? {})) {
|
|
62
|
+
const rel = normalizedArtifactPath(artifactPath);
|
|
63
|
+
if (!rel || !value || typeof value !== "object")
|
|
64
|
+
continue;
|
|
65
|
+
const suite = value.suite;
|
|
66
|
+
if (suite !== "unit" && suite !== "integration")
|
|
67
|
+
continue;
|
|
68
|
+
const command = value.command;
|
|
69
|
+
out.set(rel, { suite, ...(typeof command === "string" && command.trim() ? { command: command.trim() } : {}) });
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
// Missing or malformed provenance is fail-visible as unclassified coverage.
|
|
74
|
+
}
|
|
75
|
+
return out;
|
|
76
|
+
}
|
|
77
|
+
function inferredCoverageSuite(rel) {
|
|
78
|
+
const normalized = `/${rel.toLowerCase().replace(/[^a-z0-9/._-]+/g, "-")}/`;
|
|
79
|
+
if (/(?:^|[/_.-])(integration|e2e|end-to-end)(?:[/_.-]|$)/.test(normalized))
|
|
80
|
+
return "integration";
|
|
81
|
+
if (/(?:^|[/_.-])unit(?:[/_.-]|$)/.test(normalized))
|
|
82
|
+
return "unit";
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
export function coverageSuiteForArtifact(root, rel) {
|
|
86
|
+
const normalized = normalizedArtifactPath(rel) ?? rel.replace(/\\/g, "/");
|
|
87
|
+
const declared = coverageSuiteManifest(root).get(normalized);
|
|
88
|
+
if (declared)
|
|
89
|
+
return { suite: declared.suite, suite_source: "manifest", ...(declared.command ? { command: declared.command } : {}) };
|
|
90
|
+
const inferred = inferredCoverageSuite(normalized);
|
|
91
|
+
return inferred ? { suite: inferred, suite_source: "inferred" } : { suite: "unclassified", suite_source: "unclassified" };
|
|
92
|
+
}
|
|
50
93
|
function artifactCandidates(root, files) {
|
|
51
94
|
const out = new Map();
|
|
52
95
|
const add = (candidate) => {
|
|
@@ -79,7 +122,11 @@ function artifactCandidates(root, files) {
|
|
|
79
122
|
catch {
|
|
80
123
|
/* no generated coverage dir */
|
|
81
124
|
}
|
|
82
|
-
return [...out.values()]
|
|
125
|
+
return [...out.values()]
|
|
126
|
+
.map((candidate) => {
|
|
127
|
+
return { ...candidate, ...coverageSuiteForArtifact(root, candidate.path) };
|
|
128
|
+
})
|
|
129
|
+
.sort((a, b) => a.path.localeCompare(b.path));
|
|
83
130
|
}
|
|
84
131
|
function discoverNestedCoverageArtifacts(root) {
|
|
85
132
|
const out = [];
|
|
@@ -109,7 +156,7 @@ function discoverNestedCoverageArtifacts(root) {
|
|
|
109
156
|
}
|
|
110
157
|
};
|
|
111
158
|
visit("");
|
|
112
|
-
return out;
|
|
159
|
+
return out.map((candidate) => ({ ...candidate, ...coverageSuiteForArtifact(root, candidate.path) }));
|
|
113
160
|
}
|
|
114
161
|
function goModuleForDir(dir, goModulesByDir) {
|
|
115
162
|
for (;;) {
|
|
@@ -424,13 +471,14 @@ function parseJacocoXml(root, rel, codeFiles) {
|
|
|
424
471
|
};
|
|
425
472
|
}
|
|
426
473
|
function parseCoverageArtifact(root, candidate, goModulesByDir, codeFiles) {
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
474
|
+
const parsed = candidate.format === "go-coverprofile"
|
|
475
|
+
? parseGoCoverprofile(root, candidate.path, goModulesByDir, codeFiles)
|
|
476
|
+
: candidate.format === "lcov"
|
|
477
|
+
? parseLcov(root, candidate.path, codeFiles)
|
|
478
|
+
: candidate.format === "coverage-py"
|
|
479
|
+
? parseCoveragePyXml(root, candidate.path, codeFiles)
|
|
480
|
+
: parseJacocoXml(root, candidate.path, codeFiles);
|
|
481
|
+
return { ...parsed, ranges: parsed.ranges.map((range) => ({ ...range, suite: candidate.suite })) };
|
|
434
482
|
}
|
|
435
483
|
function overlaps(aStart, aEnd, bStart, bEnd) {
|
|
436
484
|
return aStart <= bEnd && bStart <= aEnd;
|
|
@@ -451,7 +499,14 @@ export function applyRuntimeCoverage(root, files, nodes, goModulesByDir) {
|
|
|
451
499
|
skippedArtifacts.push(skipped);
|
|
452
500
|
if (ranges.length === 0)
|
|
453
501
|
continue;
|
|
454
|
-
const stat = artifactStats.get(artifact.path) ?? {
|
|
502
|
+
const stat = artifactStats.get(artifact.path) ?? {
|
|
503
|
+
files: new Set(),
|
|
504
|
+
covered_ranges: 0,
|
|
505
|
+
format: artifact.format,
|
|
506
|
+
suite: artifact.suite,
|
|
507
|
+
suite_source: artifact.suite_source,
|
|
508
|
+
...(artifact.command ? { command: artifact.command } : {})
|
|
509
|
+
};
|
|
455
510
|
for (const range of ranges) {
|
|
456
511
|
const list = rangesByFile.get(range.file);
|
|
457
512
|
if (list)
|
|
@@ -467,6 +522,9 @@ export function applyRuntimeCoverage(root, files, nodes, goModulesByDir) {
|
|
|
467
522
|
return undefined;
|
|
468
523
|
const byLanguage = {};
|
|
469
524
|
const coveredSymbols = new Set();
|
|
525
|
+
const unitSymbols = new Set();
|
|
526
|
+
const integrationSymbols = new Set();
|
|
527
|
+
const unclassifiedSymbols = new Set();
|
|
470
528
|
let totalEligible = 0;
|
|
471
529
|
let symbolsWithSpans = 0;
|
|
472
530
|
const ingestedLanguages = new Set([...artifactStats.values()].map((s) => languageForFormat(s.format)));
|
|
@@ -491,12 +549,20 @@ export function applyRuntimeCoverage(root, files, nodes, goModulesByDir) {
|
|
|
491
549
|
if (overlapping.length === 0)
|
|
492
550
|
continue;
|
|
493
551
|
const formats = [...new Set(overlapping.map((r) => r.format))].sort();
|
|
552
|
+
const suites = [...new Set(overlapping.map((r) => r.suite ?? "unclassified"))].sort();
|
|
494
553
|
coveredSymbols.add(n.external_id);
|
|
554
|
+
if (suites.includes("unit"))
|
|
555
|
+
unitSymbols.add(n.external_id);
|
|
556
|
+
if (suites.includes("integration"))
|
|
557
|
+
integrationSymbols.add(n.external_id);
|
|
558
|
+
if (suites.includes("unclassified"))
|
|
559
|
+
unclassifiedSymbols.add(n.external_id);
|
|
495
560
|
bucket.covered++;
|
|
496
561
|
n.properties = {
|
|
497
562
|
...n.properties,
|
|
498
563
|
runtime_covered: true,
|
|
499
|
-
runtime_coverage_formats: formats
|
|
564
|
+
runtime_coverage_formats: formats,
|
|
565
|
+
runtime_coverage_suites: suites
|
|
500
566
|
};
|
|
501
567
|
}
|
|
502
568
|
for (const bucket of Object.values(byLanguage))
|
|
@@ -505,6 +571,9 @@ export function applyRuntimeCoverage(root, files, nodes, goModulesByDir) {
|
|
|
505
571
|
artifacts: [...artifactStats.entries()].map(([path, s]) => ({
|
|
506
572
|
path,
|
|
507
573
|
format: s.format,
|
|
574
|
+
suite: s.suite,
|
|
575
|
+
suite_source: s.suite_source,
|
|
576
|
+
...(s.command ? { command: s.command } : {}),
|
|
508
577
|
files: s.files.size,
|
|
509
578
|
covered_ranges: s.covered_ranges
|
|
510
579
|
})),
|
|
@@ -513,6 +582,18 @@ export function applyRuntimeCoverage(root, files, nodes, goModulesByDir) {
|
|
|
513
582
|
symbols_with_spans: symbolsWithSpans,
|
|
514
583
|
covered_symbols: coveredSymbols.size,
|
|
515
584
|
covered_pct: pct(coveredSymbols.size, totalEligible),
|
|
516
|
-
by_language: byLanguage
|
|
585
|
+
by_language: byLanguage,
|
|
586
|
+
by_suite: {
|
|
587
|
+
unit: unitSymbols.size,
|
|
588
|
+
integration: integrationSymbols.size,
|
|
589
|
+
overlap: [...unitSymbols].filter((id) => integrationSymbols.has(id)).length,
|
|
590
|
+
unclassified: unclassifiedSymbols.size,
|
|
591
|
+
union: coveredSymbols.size,
|
|
592
|
+
unit_pct: pct(unitSymbols.size, totalEligible),
|
|
593
|
+
integration_pct: pct(integrationSymbols.size, totalEligible),
|
|
594
|
+
overlap_pct: pct([...unitSymbols].filter((id) => integrationSymbols.has(id)).length, totalEligible),
|
|
595
|
+
unclassified_pct: pct(unclassifiedSymbols.size, totalEligible),
|
|
596
|
+
union_pct: pct(coveredSymbols.size, totalEligible)
|
|
597
|
+
}
|
|
517
598
|
};
|
|
518
599
|
}
|
|
@@ -2,6 +2,7 @@ import { spawnSync } from "node:child_process";
|
|
|
2
2
|
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync } from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { reportProgress } from "../util/progress.js";
|
|
5
|
+
import { coverageSuiteForArtifact } from "./coverage.js";
|
|
5
6
|
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
6
7
|
const DEFAULT_BUDGET_MS = 600_000;
|
|
7
8
|
const GENERATED_COVERAGE_DIR = ".orangepro/coverage";
|
|
@@ -129,6 +130,10 @@ export function prepareRuntimeCoverage(root, opts = {}) {
|
|
|
129
130
|
warnings.push("No local coverage generators found, so OrangePro did not generate runtime coverage.");
|
|
130
131
|
}
|
|
131
132
|
const artifacts = detectCoverageArtifacts(absRoot);
|
|
133
|
+
const unclassified = artifacts.filter((artifact) => artifact.ingestible && artifact.suite === "unclassified");
|
|
134
|
+
if (unclassified.length > 0) {
|
|
135
|
+
warnings.push(`${unclassified.length} ingestible coverage artifact(s) have no unit/integration provenance. Add ${COVERAGE_SUITE_MANIFEST_HELP} so OrangePro reports the suites separately.`);
|
|
136
|
+
}
|
|
132
137
|
const suggested_commands = suggestCoverageCommands(absRoot);
|
|
133
138
|
return {
|
|
134
139
|
root: absRoot,
|
|
@@ -139,10 +144,12 @@ export function prepareRuntimeCoverage(root, opts = {}) {
|
|
|
139
144
|
warnings
|
|
140
145
|
};
|
|
141
146
|
}
|
|
147
|
+
const COVERAGE_SUITE_MANIFEST_HELP = ".orangepro/coverage-suites.json with an artifacts map, for example {\"coverage/unit.coverprofile\":{\"suite\":\"unit\",\"command\":\"make unit-test-coverage\"}}";
|
|
142
148
|
export function detectCoverageArtifacts(root) {
|
|
143
149
|
const absRoot = path.resolve(root);
|
|
144
150
|
const out = new Map();
|
|
145
|
-
const add = (
|
|
151
|
+
const add = (raw) => {
|
|
152
|
+
const info = { ...raw, ...coverageSuiteForArtifact(absRoot, raw.path) };
|
|
146
153
|
const abs = path.join(absRoot, info.path);
|
|
147
154
|
if (!existsSync(abs))
|
|
148
155
|
return;
|
package/dist/local/autoProve.js
CHANGED
|
@@ -18,7 +18,7 @@ import { generateTests, readDeclaredDeps, unresolvedLocalImports } from "./gener
|
|
|
18
18
|
import { GENERATED_DIR, runHintsFor } from "./generate/runHints.js";
|
|
19
19
|
import { rankRiskGaps } from "./score/risk.js";
|
|
20
20
|
import { resolveProviderConfig } from "./localConfig.js";
|
|
21
|
-
import { buildProvider } from "./generate/providers.js";
|
|
21
|
+
import { buildProvider, DeterministicProvider } from "./generate/providers.js";
|
|
22
22
|
import { resolveContained } from "./reprove/paths.js";
|
|
23
23
|
import { buildRtm } from "./rtm.js";
|
|
24
24
|
import { loadLedger } from "./ledger.js";
|
|
@@ -834,10 +834,12 @@ export async function autoProve(root, opts, deps) {
|
|
|
834
834
|
attempts: ex.attempts
|
|
835
835
|
};
|
|
836
836
|
}
|
|
837
|
-
//
|
|
838
|
-
//
|
|
839
|
-
const
|
|
840
|
-
|
|
837
|
+
// Provider gate applies ONLY to the generation lane. Explicit deterministic mode is a
|
|
838
|
+
// valid offline provider; otherwise a real provider key/config is required.
|
|
839
|
+
const deterministic = opts.provider === "deterministic"
|
|
840
|
+
|| /^(1|true|yes)$/i.test(String(deps.env.ORANGEPRO_ALLOW_DETERMINISTIC ?? ""));
|
|
841
|
+
const providerConfig = deterministic ? null : resolveProviderConfig(deps.env, { provider: opts.provider, model: opts.model });
|
|
842
|
+
if (!providerConfig && !deterministic) {
|
|
841
843
|
const status = ex.proven > 0 ? "proven-run" : ex.attempted > 0 ? "ran-no-proof" : "skipped-no-key";
|
|
842
844
|
return {
|
|
843
845
|
ran: ex.attempted > 0,
|
|
@@ -851,7 +853,7 @@ export async function autoProve(root, opts, deps) {
|
|
|
851
853
|
attempts: ex.attempts
|
|
852
854
|
};
|
|
853
855
|
}
|
|
854
|
-
const provider = buildProvider(providerConfig);
|
|
856
|
+
const provider = deterministic ? new DeterministicProvider() : buildProvider(providerConfig);
|
|
855
857
|
const generate = deps.generate ?? generateTests;
|
|
856
858
|
const reader = fileReaderFor(sourceRoot);
|
|
857
859
|
// Generation gets only the budget the existing-tests lane left unspent, so existing +
|
package/dist/local/cli.js
CHANGED
|
@@ -78,7 +78,7 @@ const HELP = `opro — OrangePro (local-first, BYOK, metadata-only artifacts)
|
|
|
78
78
|
|
|
79
79
|
Usage:
|
|
80
80
|
opro # one-command start: analyze, optional AI links + flows, report, RTM, agent handoff
|
|
81
|
-
opro start [path] [--base <ref>] [--no-ai] [--no-ai-flows] [--generate-coverage] [--prompt-version v5] [--json]
|
|
81
|
+
opro start [path] [--base <ref>] [--no-ai] [--no-ai-flows] [--generate-coverage] [--proof-limit 5] [--generate-limit 20] [--prompt-version v5] [--json]
|
|
82
82
|
opro roast [path] [--limit 5] [--json] # keyless: find passing tests whose targeted mutant still survives
|
|
83
83
|
opro init
|
|
84
84
|
opro setup # interactive: choose a default model provider + model (saved locally)
|
|
@@ -199,6 +199,8 @@ async function main() {
|
|
|
199
199
|
aiAll: asBool(flags["ai-all"], false),
|
|
200
200
|
aiFlows: !asBool(flags["no-ai-flows"], false),
|
|
201
201
|
autoLimit: numericFlag(flags["auto-limit"]),
|
|
202
|
+
proofLimit: numericFlag(flags["proof-limit"]),
|
|
203
|
+
generateLimit: numericFlag(flags["generate-limit"]),
|
|
202
204
|
noAuto: asBool(flags["no-auto"], false),
|
|
203
205
|
promptVersion: flags["prompt-version"] === "v5" ? "v5" : undefined,
|
|
204
206
|
provider: typeof flags.provider === "string" ? flags.provider : undefined,
|
|
@@ -225,6 +227,10 @@ async function main() {
|
|
|
225
227
|
out(" Proof next: no behavior is dynamically proven yet; run from a coding agent with a model key and follow the generated-test proof handoff.");
|
|
226
228
|
}
|
|
227
229
|
const ap = res.auto_prove;
|
|
230
|
+
const gen = res.generation;
|
|
231
|
+
out(` Test generation: ${gen.status} — ${gen.runnable}/${gen.generated} runnable, ${Math.max(0, gen.drafts - gen.runnable)} blocked draft(s)`);
|
|
232
|
+
if (gen.reason)
|
|
233
|
+
out(` generation reason: ${gen.reason}`);
|
|
228
234
|
if (ap.status === "skipped-no-key") {
|
|
229
235
|
out(` Dynamic proof: skipped — ${ap.reason ?? "no provider key"}`);
|
|
230
236
|
}
|
|
@@ -527,7 +533,9 @@ async function main() {
|
|
|
527
533
|
if (res.artifacts.length) {
|
|
528
534
|
out(" found:");
|
|
529
535
|
for (const a of res.artifacts) {
|
|
530
|
-
out(` ${a.path} (${a.language}, ${a.format}${a.ingestible ? ", ingestible now" : ", detected only"})`);
|
|
536
|
+
out(` ${a.path} (${a.language}, ${a.format}, suite=${a.suite} [${a.suite_source}]${a.ingestible ? ", ingestible now" : ", detected only"})`);
|
|
537
|
+
if (a.command)
|
|
538
|
+
out(` command: ${a.command}`);
|
|
531
539
|
}
|
|
532
540
|
}
|
|
533
541
|
else {
|
package/dist/local/cliArgs.js
CHANGED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/** Classify only high-confidence generator/validator diagnostics. Ambiguous
|
|
2
|
+
* failures stay unknown rather than being mislabeled as a repo setup problem. */
|
|
3
|
+
export function classifyGeneratedDraftBlocker(reason) {
|
|
4
|
+
const text = reason ?? "";
|
|
5
|
+
if (/timed?\s*out|timeout|ETIMEDOUT|signal:\s*killed/i.test(text))
|
|
6
|
+
return "validation_timeout";
|
|
7
|
+
if (/(?:^|\b)(?:go|gofmt|python3|pytest|mvn|gradle|node|npm|vitest|jest|mocha)(?:\b[^.\n]*)?(?:not found|ENOENT|not installed)/i.test(text) ||
|
|
8
|
+
/(?:test runner|toolchain).*(?:missing|not found|not installed)/i.test(text))
|
|
9
|
+
return "toolchain_or_runner";
|
|
10
|
+
if (/no required module provides package|cannot find module|module not found|unresolved import|imports module-path package/i.test(text)) {
|
|
11
|
+
return "unresolved_import";
|
|
12
|
+
}
|
|
13
|
+
if (/syntax check failed|undefined:|unknown field|redeclared in this block|expected (?:declaration|operand|'[^']+'|"[^"]+"|[^ ]+),? found|literal not terminated|cannot assign/i.test(text))
|
|
14
|
+
return "generated_code";
|
|
15
|
+
return "unknown";
|
|
16
|
+
}
|
|
17
|
+
export function generatedDraftRemediation(reason) {
|
|
18
|
+
switch (classifyGeneratedDraftBlocker(reason)) {
|
|
19
|
+
case "generated_code":
|
|
20
|
+
return "Regenerate or repair the draft using symbols and APIs that exist in this package; installing dependencies will not fix this compiler error.";
|
|
21
|
+
case "unresolved_import":
|
|
22
|
+
return "Verify that the generated import path exists and is already declared by this repo; install dependencies only when the repo expects that import, then regenerate.";
|
|
23
|
+
case "toolchain_or_runner":
|
|
24
|
+
return "Install or configure the named toolchain/test runner, then re-run `opro start`.";
|
|
25
|
+
case "validation_timeout":
|
|
26
|
+
return "Warm the dependency cache or raise the relevant OrangePro validation timeout, then re-run `opro start`.";
|
|
27
|
+
default:
|
|
28
|
+
return "Review the named blocker, repair or regenerate the draft as needed, then re-run `opro start`; do not assume dependencies are missing.";
|
|
29
|
+
}
|
|
30
|
+
}
|
|
@@ -19,6 +19,7 @@ import { languageOf } from "../analyze/classify.js";
|
|
|
19
19
|
import { buildGroundedUserPrompt, buildRawUserPrompt, buildSystemPrompt, PROMPT_VERSION } from "./prompt.js";
|
|
20
20
|
import { buildBatchGenerationSystemPromptV5, buildBatchGenerationUserPromptV5, buildPlanningRepairSystemPromptV5, buildPlanningRepairUserPromptV5, buildPlanningSystemPromptV5, buildPlanningUserPromptV5, hasRepairableScenarioStructure, parseBatchGeneratedTests, parsePlannedScenariosStrict, scenarioTiesBackToRaw, PROMPT_VERSION_V5 } from "./promptV5.js";
|
|
21
21
|
import { BUCKET_LABEL, deriveBucketSignals, selectLocalBuckets } from "./buckets.js";
|
|
22
|
+
import { generatedDraftRemediation } from "./draftGuidance.js";
|
|
22
23
|
const MAX_LIMIT = 5;
|
|
23
24
|
const DEFAULT_LIMIT = 3;
|
|
24
25
|
const MAX_RELATED_FILES = 4;
|
|
@@ -30,6 +31,7 @@ const STATIC_CHECK_TIMEOUT_MS = 3000;
|
|
|
30
31
|
// before the target package is compiled. Keep the check authoritative instead
|
|
31
32
|
// of downgrading valid generated tests while dependencies are still downloading.
|
|
32
33
|
const GO_COMPILE_CHECK_TIMEOUT_MS = 120000;
|
|
34
|
+
const V5_GENERATION_MAX_TOKENS = 4000;
|
|
33
35
|
/** A source ref that names a test file (used to place a generated test next to it). */
|
|
34
36
|
const TEST_REF_RE = /(\.(test|spec)\.[cm]?[jt]sx?$)|((^|\/)test\.[cm]?[jt]sx?$)|(_test\.[a-z]+$)|(_spec\.[a-z]+$)|((^|\/)test_[^/]+\.[a-z]+$)/i;
|
|
35
37
|
function areaOf(relPath) {
|
|
@@ -2382,7 +2384,8 @@ export async function generateTests(graph, opts, provider, fileReader, clock = s
|
|
|
2382
2384
|
reportProgress(`Generating "${gc.ctx.behavior_title}" [v5 batch: ${selected.length}]…`);
|
|
2383
2385
|
completions.push(await provider.complete({
|
|
2384
2386
|
system: buildBatchGenerationSystemPromptV5(),
|
|
2385
|
-
user: buildBatchGenerationUserPromptV5({ ...gc.ctx, scenarios: selected })
|
|
2387
|
+
user: buildBatchGenerationUserPromptV5({ ...gc.ctx, scenarios: selected }),
|
|
2388
|
+
maxTokens: V5_GENERATION_MAX_TOKENS
|
|
2386
2389
|
}));
|
|
2387
2390
|
}
|
|
2388
2391
|
catch (e) {
|
|
@@ -2392,7 +2395,8 @@ export async function generateTests(graph, opts, provider, fileReader, clock = s
|
|
|
2392
2395
|
try {
|
|
2393
2396
|
completions.push(await provider.complete({
|
|
2394
2397
|
system: buildBatchGenerationSystemPromptV5(),
|
|
2395
|
-
user: buildBatchGenerationUserPromptV5({ ...gc.ctx, scenarios: [scenario] })
|
|
2398
|
+
user: buildBatchGenerationUserPromptV5({ ...gc.ctx, scenarios: [scenario] }),
|
|
2399
|
+
maxTokens: V5_GENERATION_MAX_TOKENS
|
|
2396
2400
|
}));
|
|
2397
2401
|
}
|
|
2398
2402
|
catch (singleErr) {
|
|
@@ -2541,7 +2545,7 @@ export async function generateTests(graph, opts, provider, fileReader, clock = s
|
|
|
2541
2545
|
"",
|
|
2542
2546
|
// Concise blocker: first clause only — the full remedy is one line.
|
|
2543
2547
|
`Blocked by: ${reason.split(" — ")[0]}`,
|
|
2544
|
-
|
|
2548
|
+
`Fix: ${generatedDraftRemediation(reason)}`
|
|
2545
2549
|
].join("\n"), gc.ctx.source_excerpts, "//").body;
|
|
2546
2550
|
generated.push({
|
|
2547
2551
|
id: `${run_id}-t${generated.length + 1}`,
|
|
@@ -108,16 +108,17 @@ export class OpenAICompatibleProvider {
|
|
|
108
108
|
const data = (await postJson(this.fetchImpl, `${this.cfg.baseUrl}/chat/completions`, { Authorization: `Bearer ${this.cfg.apiKey ?? ""}` }, body, providerTimeoutMs(this.cfg.model)));
|
|
109
109
|
const choice = data.choices?.[0];
|
|
110
110
|
const content = choice?.message?.content ?? "";
|
|
111
|
-
//
|
|
112
|
-
//
|
|
113
|
-
//
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
if (!content.trim() && choice?.finish_reason === "length" && !tried.has("starved")) {
|
|
117
|
-
tried.add("starved");
|
|
111
|
+
// A length stop means the completion is truncated even when it contains
|
|
112
|
+
// visible text. Returning that prefix turns cut-off source into a fake
|
|
113
|
+
// compiler/setup problem. Retry once with more room, then fail closed.
|
|
114
|
+
if (choice?.finish_reason === "length" && !tried.has("length-retry")) {
|
|
115
|
+
tried.add("length-retry");
|
|
118
116
|
maxTokens = maxTokens * 4;
|
|
119
117
|
continue;
|
|
120
118
|
}
|
|
119
|
+
if (choice?.finish_reason === "length") {
|
|
120
|
+
throw new Error("Model output was truncated at the token limit after one retry; no generated code was accepted.");
|
|
121
|
+
}
|
|
121
122
|
return content;
|
|
122
123
|
}
|
|
123
124
|
catch (e) {
|
|
@@ -176,14 +177,26 @@ export class AnthropicProvider {
|
|
|
176
177
|
return this.cfg.model;
|
|
177
178
|
}
|
|
178
179
|
async complete(req) {
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
180
|
+
let maxTokens = req.maxTokens ?? 900;
|
|
181
|
+
let retriedLength = false;
|
|
182
|
+
for (;;) {
|
|
183
|
+
const data = (await postJson(this.fetchImpl, `${this.cfg.baseUrl}/messages`, { "x-api-key": this.cfg.apiKey ?? "", "anthropic-version": "2023-06-01" }, {
|
|
184
|
+
model: this.cfg.model,
|
|
185
|
+
max_tokens: maxTokens,
|
|
186
|
+
temperature: req.temperature ?? 0.2,
|
|
187
|
+
system: req.system,
|
|
188
|
+
messages: [{ role: "user", content: req.user }]
|
|
189
|
+
}, providerTimeoutMs(this.cfg.model)));
|
|
190
|
+
if (data.stop_reason === "max_tokens" && !retriedLength) {
|
|
191
|
+
retriedLength = true;
|
|
192
|
+
maxTokens *= 4;
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
if (data.stop_reason === "max_tokens") {
|
|
196
|
+
throw new Error("Model output was truncated at the token limit after one retry; no generated code was accepted.");
|
|
197
|
+
}
|
|
198
|
+
return (data.content ?? []).map((c) => c.text ?? "").join("");
|
|
199
|
+
}
|
|
187
200
|
}
|
|
188
201
|
}
|
|
189
202
|
/**
|
package/dist/local/operations.js
CHANGED
|
@@ -21,6 +21,7 @@ import { doctorGraph } from "./score/doctor.js";
|
|
|
21
21
|
import { findGaps } from "./gaps/gaps.js";
|
|
22
22
|
import { rankRiskGaps } from "./score/risk.js";
|
|
23
23
|
import { generateTests } from "./generate/generator.js";
|
|
24
|
+
import { classifyGeneratedDraftBlocker } from "./generate/draftGuidance.js";
|
|
24
25
|
import { autoProve, NO_KEY_MESSAGE, isEligibleProvableTarget } from "./autoProve.js";
|
|
25
26
|
import { AGENT_RUN_WORKFLOW, GROUNDING_CONTRACT, runnableRunHintsFor } from "./generate/runHints.js";
|
|
26
27
|
import { buildProvider, DeterministicProvider } from "./generate/providers.js";
|
|
@@ -1514,10 +1515,13 @@ export async function opStart(root, opts = {}, deps = defaultDeps()) {
|
|
|
1514
1515
|
const warnings = [...analyze.warnings];
|
|
1515
1516
|
reportProgress("start: deterministic graph is ready", { current: 4, total: 8 });
|
|
1516
1517
|
const staticSnapshot = writeStartStaticSnapshot(root, opts.baseRef, warnings);
|
|
1517
|
-
const
|
|
1518
|
+
const aiProviderConfigured = deps.aiProvider !== undefined || resolveProviderConfig(providerEnv, providerOpts) !== null;
|
|
1519
|
+
const resolvedGenerationProvider = deps.aiProvider ?? resolveGenerationProvider(providerEnv, providerOpts);
|
|
1520
|
+
const generationProviderConfigured = resolvedGenerationProvider !== null;
|
|
1521
|
+
const deterministicGeneration = resolvedGenerationProvider?.providerName === "deterministic";
|
|
1518
1522
|
let aiLinks = { status: "skipped", reason: "AI candidate links disabled for this run." };
|
|
1519
1523
|
if (opts.ai !== false) {
|
|
1520
|
-
if (!
|
|
1524
|
+
if (!aiProviderConfigured) {
|
|
1521
1525
|
reportProgress("ai: skipped — no local provider key/base URL found", { current: 5, total: 8 });
|
|
1522
1526
|
aiLinks = {
|
|
1523
1527
|
status: "skipped",
|
|
@@ -1542,7 +1546,7 @@ export async function opStart(root, opts = {}, deps = defaultDeps()) {
|
|
|
1542
1546
|
}
|
|
1543
1547
|
let aiFlows = { status: "skipped", reason: "AI candidate flows disabled for this run." };
|
|
1544
1548
|
if (opts.ai !== false && opts.aiFlows !== false) {
|
|
1545
|
-
if (!
|
|
1549
|
+
if (!aiProviderConfigured) {
|
|
1546
1550
|
aiFlows = {
|
|
1547
1551
|
status: "skipped",
|
|
1548
1552
|
reason: "No model provider configured; set OPENAI_API_KEY, ANTHROPIC_API_KEY, or OLLAMA_BASE_URL to auto-apply AI candidate flows."
|
|
@@ -1574,7 +1578,7 @@ export async function opStart(root, opts = {}, deps = defaultDeps()) {
|
|
|
1574
1578
|
let autoProveResult;
|
|
1575
1579
|
try {
|
|
1576
1580
|
autoProveResult = await autoProve(root, {
|
|
1577
|
-
autoLimit: opts.autoLimit,
|
|
1581
|
+
autoLimit: opts.proofLimit ?? opts.autoLimit,
|
|
1578
1582
|
noAuto: opts.noAuto,
|
|
1579
1583
|
provider: providerOpts.provider,
|
|
1580
1584
|
model: providerOpts.model,
|
|
@@ -1599,40 +1603,115 @@ export async function opStart(root, opts = {}, deps = defaultDeps()) {
|
|
|
1599
1603
|
};
|
|
1600
1604
|
warnings.push(`auto-prove skipped: ${reason}`);
|
|
1601
1605
|
}
|
|
1602
|
-
|
|
1606
|
+
const generationLimit = Math.max(0, Math.min(50, Math.floor(opts.generateLimit ?? START_GENERATE_RISK_LIMIT)));
|
|
1607
|
+
let generationResult = {
|
|
1608
|
+
status: "disabled",
|
|
1609
|
+
requested: generationLimit,
|
|
1610
|
+
generated: 0,
|
|
1611
|
+
runnable: 0,
|
|
1612
|
+
drafts: 0,
|
|
1613
|
+
blockers: {},
|
|
1614
|
+
reason: opts.noAuto
|
|
1615
|
+
? "Test generation was disabled by --no-auto."
|
|
1616
|
+
: opts.ai === false
|
|
1617
|
+
? "Test generation was disabled by --no-ai."
|
|
1618
|
+
: generationLimit === 0
|
|
1619
|
+
? "Test generation budget is 0 (--generate-limit 0)."
|
|
1620
|
+
: undefined
|
|
1621
|
+
};
|
|
1622
|
+
if (!opts.noAuto && opts.ai !== false && generationLimit > 0 && !generationProviderConfigured) {
|
|
1623
|
+
generationResult = {
|
|
1624
|
+
...generationResult,
|
|
1625
|
+
status: "no_provider",
|
|
1626
|
+
reason: NO_PROVIDER_MESSAGE
|
|
1627
|
+
};
|
|
1628
|
+
warnings.push(`generate: ${NO_PROVIDER_MESSAGE}`);
|
|
1629
|
+
}
|
|
1630
|
+
else if (!opts.noAuto && generationProviderConfigured && opts.ai !== false && generationLimit > 0) {
|
|
1603
1631
|
try {
|
|
1604
1632
|
const graphForGeneration = loadGraph(workspacePaths(root).graphPath);
|
|
1605
1633
|
const generatedTargets = new Set((graphForGeneration.generated_tests ?? []).map((t) => t.target_symbol_external_id).filter((id) => Boolean(id)));
|
|
1606
1634
|
const targetIds = rankRiskGaps(graphForGeneration, {
|
|
1607
1635
|
repoRoot: root,
|
|
1608
|
-
limit:
|
|
1636
|
+
limit: generationLimit,
|
|
1609
1637
|
provenIds: provenSymbolIds(graphForGeneration, loadLedger(root))
|
|
1610
1638
|
})
|
|
1611
1639
|
.map((gap) => gap.id)
|
|
1612
1640
|
.filter((id) => !generatedTargets.has(id));
|
|
1613
|
-
if (targetIds.length) {
|
|
1641
|
+
if (!targetIds.length) {
|
|
1642
|
+
generationResult = {
|
|
1643
|
+
...generationResult,
|
|
1644
|
+
status: "no_targets",
|
|
1645
|
+
reason: "No eligible ungenerated risk targets were found."
|
|
1646
|
+
};
|
|
1647
|
+
}
|
|
1648
|
+
else {
|
|
1614
1649
|
reportProgress(`generate: drafting tests for top ${targetIds.length} risk target(s)`, { current: 6, total: 8 });
|
|
1615
|
-
|
|
1650
|
+
const generatedDrafts = [];
|
|
1616
1651
|
for (let i = 0; i < targetIds.length; i += START_GENERATE_BATCH_LIMIT) {
|
|
1617
1652
|
const batch = targetIds.slice(i, i + START_GENERATE_BATCH_LIMIT);
|
|
1618
1653
|
const generated = await opGenerate(root, {
|
|
1619
1654
|
...providerOpts,
|
|
1620
1655
|
target_ids: batch,
|
|
1621
1656
|
limit: batch.length,
|
|
1622
|
-
|
|
1657
|
+
// The offline deterministic stand-in emits the established v2 scaffold; v5 is
|
|
1658
|
+
// a two-phase model planning protocol and must not be selected implicitly for it.
|
|
1659
|
+
prompt_version: opts.promptVersion ?? (deterministicGeneration ? "v2" : "v5")
|
|
1623
1660
|
}, providerDeps);
|
|
1624
|
-
|
|
1661
|
+
generatedDrafts.push(...generated.generated_tests);
|
|
1625
1662
|
warnings.push(...generated.warnings.map((w) => `generate: ${w}`));
|
|
1626
1663
|
}
|
|
1627
|
-
|
|
1628
|
-
|
|
1664
|
+
const blockers = {};
|
|
1665
|
+
for (const draft of generatedDrafts.filter((test) => test.runnable === false)) {
|
|
1666
|
+
const blocker = classifyGeneratedDraftBlocker(draft.unresolved_reason);
|
|
1667
|
+
blockers[blocker] = (blockers[blocker] ?? 0) + 1;
|
|
1668
|
+
}
|
|
1669
|
+
const runnable = generatedDrafts.filter((test) => test.runnable !== false).length;
|
|
1670
|
+
generationResult = {
|
|
1671
|
+
status: generatedDrafts.length === 0
|
|
1672
|
+
? "no_results"
|
|
1673
|
+
: generatedDrafts.some((test) => test.runnable === false)
|
|
1674
|
+
? "completed_with_blockers"
|
|
1675
|
+
: "completed",
|
|
1676
|
+
requested: targetIds.length,
|
|
1677
|
+
generated: generatedDrafts.length,
|
|
1678
|
+
runnable,
|
|
1679
|
+
drafts: generatedDrafts.length - runnable,
|
|
1680
|
+
blockers,
|
|
1681
|
+
...(generatedDrafts.length === 0
|
|
1682
|
+
? { reason: "The provider returned no generated-test drafts for the selected risk targets." }
|
|
1683
|
+
: {})
|
|
1684
|
+
};
|
|
1685
|
+
if (generatedDrafts.length === 0)
|
|
1686
|
+
warnings.push(`generate: ${generationResult.reason}`);
|
|
1629
1687
|
}
|
|
1630
1688
|
}
|
|
1631
1689
|
catch (err) {
|
|
1632
|
-
const reason = err instanceof Error ? err.message : String(err);
|
|
1690
|
+
const reason = redactSecrets(err instanceof Error ? err.message : String(err));
|
|
1691
|
+
generationResult = {
|
|
1692
|
+
...generationResult,
|
|
1693
|
+
status: "failed",
|
|
1694
|
+
reason
|
|
1695
|
+
};
|
|
1633
1696
|
warnings.push(`generate skipped: ${reason}`);
|
|
1634
1697
|
}
|
|
1635
1698
|
}
|
|
1699
|
+
try {
|
|
1700
|
+
const graphWithGeneration = loadGraph(workspacePaths(root).graphPath);
|
|
1701
|
+
if (!graphWithGeneration.analysis)
|
|
1702
|
+
throw new Error("analysis metadata is missing after analyze");
|
|
1703
|
+
saveGraph(workspacePaths(root).graphPath, {
|
|
1704
|
+
...graphWithGeneration,
|
|
1705
|
+
updated_at: deps.clock(),
|
|
1706
|
+
analysis: {
|
|
1707
|
+
...graphWithGeneration.analysis,
|
|
1708
|
+
start_generation: generationResult
|
|
1709
|
+
}
|
|
1710
|
+
});
|
|
1711
|
+
}
|
|
1712
|
+
catch (error) {
|
|
1713
|
+
warnings.push(`generation outcome not persisted: ${error instanceof Error ? error.message : String(error)}`);
|
|
1714
|
+
}
|
|
1636
1715
|
// G1: persist the distilled, already-redacted attempt classifications so
|
|
1637
1716
|
// `opro doctor --proof` and standalone report regens can explain blockers
|
|
1638
1717
|
// after this process exits. Sidecar only — never read by the oracle, RTM,
|
|
@@ -1767,6 +1846,7 @@ export async function opStart(root, opts = {}, deps = defaultDeps()) {
|
|
|
1767
1846
|
changed,
|
|
1768
1847
|
gaps,
|
|
1769
1848
|
auto_prove: autoProveResult,
|
|
1849
|
+
generation: generationResult,
|
|
1770
1850
|
next_actions: nextActions,
|
|
1771
1851
|
agent_workflow: AGENT_RUN_WORKFLOW,
|
|
1772
1852
|
grounding_contract: GROUNDING_CONTRACT,
|
|
@@ -74,12 +74,22 @@ export function renderCoverageReport(graph, ledger = emptyLedger()) {
|
|
|
74
74
|
lines.push(`| ${language} | ${row.covered} / ${row.eligible} | ${row.covered_pct}% | ${row.symbols_with_spans} |`);
|
|
75
75
|
}
|
|
76
76
|
lines.push("");
|
|
77
|
+
lines.push("Coverage by test suite (inclusive counts; unit + integration must not be added because overlap is reported separately):");
|
|
78
|
+
lines.push("");
|
|
79
|
+
lines.push("| suite | covered symbols | eligible % |");
|
|
80
|
+
lines.push("| --- | ---: | ---: |");
|
|
81
|
+
lines.push(`| unit | ${runtime.by_suite.unit} | ${runtime.by_suite.unit_pct}% |`);
|
|
82
|
+
lines.push(`| integration | ${runtime.by_suite.integration} | ${runtime.by_suite.integration_pct}% |`);
|
|
83
|
+
lines.push(`| unit + integration overlap | ${runtime.by_suite.overlap} | ${runtime.by_suite.overlap_pct}% |`);
|
|
84
|
+
lines.push(`| unclassified | ${runtime.by_suite.unclassified} | ${runtime.by_suite.unclassified_pct}% |`);
|
|
85
|
+
lines.push(`| combined union | ${runtime.by_suite.union} | ${runtime.by_suite.union_pct}% |`);
|
|
86
|
+
lines.push("");
|
|
77
87
|
lines.push("Artifacts:");
|
|
78
88
|
lines.push("");
|
|
79
|
-
lines.push("| artifact | format | files | covered ranges |");
|
|
80
|
-
lines.push("| --- | --- | ---: | ---: |");
|
|
89
|
+
lines.push("| artifact | suite | provenance | command | format | files | covered ranges |");
|
|
90
|
+
lines.push("| --- | --- | --- | --- | --- | ---: | ---: |");
|
|
81
91
|
for (const artifact of runtime.artifacts) {
|
|
82
|
-
lines.push(`| ${escapeMd(artifact.path)} | ${artifact.format} | ${artifact.files} | ${artifact.covered_ranges} |`);
|
|
92
|
+
lines.push(`| ${escapeMd(artifact.path)} | ${artifact.suite} | ${artifact.suite_source} | ${escapeMd(artifact.command ?? "-")} | ${artifact.format} | ${artifact.files} | ${artifact.covered_ranges} |`);
|
|
83
93
|
}
|
|
84
94
|
lines.push("");
|
|
85
95
|
if (runtime.skipped_artifacts?.length) {
|
|
@@ -4,6 +4,7 @@ import { buildRtm } from "../rtm.js";
|
|
|
4
4
|
import { inspectRiskInputHealth, isEntryPoint, rankRiskGaps } from "../score/risk.js";
|
|
5
5
|
import { ORANGEPRO_VERSION } from "../version.js";
|
|
6
6
|
import { PROOF_BLOCKER_GUIDE } from "../proofDoctor.js";
|
|
7
|
+
import { classifyGeneratedDraftBlocker } from "../generate/draftGuidance.js";
|
|
7
8
|
/** Short human phrase per R-1 needs_setup category, for the "blocked because: …" panel copy. */
|
|
8
9
|
const BLOCK_CATEGORY_LABEL = {
|
|
9
10
|
module_not_found: "a missing module or dependency in the sandbox",
|
|
@@ -229,6 +230,7 @@ function scanBlock(graph, rows) {
|
|
|
229
230
|
const integration = tests.filter((n) => n.properties.test_layer === "integration" || n.properties.test_layer === "api" || n.properties.test_layer === "e2e").length;
|
|
230
231
|
const unit = tests.filter((n) => n.properties.test_layer === "unit" || n.properties.test_layer === "component").length;
|
|
231
232
|
const denominator = graph.analysis?.denominator;
|
|
233
|
+
const runtime = graph.analysis?.runtime_coverage;
|
|
232
234
|
const excludedCount = (denominator?.excluded_boilerplate ?? 0) +
|
|
233
235
|
(denominator?.excluded_infra ?? 0) +
|
|
234
236
|
(denominator?.excluded_generated ?? 0) +
|
|
@@ -236,7 +238,22 @@ function scanBlock(graph, rows) {
|
|
|
236
238
|
return {
|
|
237
239
|
services: [...services.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 50),
|
|
238
240
|
serviceTotal: services.size,
|
|
239
|
-
tests: { total: tests.length, integration, unit },
|
|
241
|
+
tests: { total: tests.length, integration, unit, unclassified: Math.max(0, tests.length - integration - unit) },
|
|
242
|
+
runtimeCoverage: runtime
|
|
243
|
+
? {
|
|
244
|
+
eligible: runtime.total_eligible_symbols,
|
|
245
|
+
unit: runtime.by_suite?.unit ?? 0,
|
|
246
|
+
integration: runtime.by_suite?.integration ?? 0,
|
|
247
|
+
overlap: runtime.by_suite?.overlap ?? 0,
|
|
248
|
+
unclassified: runtime.by_suite?.unclassified ?? runtime.covered_symbols,
|
|
249
|
+
union: runtime.by_suite?.union ?? runtime.covered_symbols,
|
|
250
|
+
unitPct: runtime.by_suite?.unit_pct ?? 0,
|
|
251
|
+
integrationPct: runtime.by_suite?.integration_pct ?? 0,
|
|
252
|
+
overlapPct: runtime.by_suite?.overlap_pct ?? 0,
|
|
253
|
+
unclassifiedPct: runtime.by_suite?.unclassified_pct ?? runtime.covered_pct,
|
|
254
|
+
unionPct: runtime.by_suite?.union_pct ?? runtime.covered_pct
|
|
255
|
+
}
|
|
256
|
+
: null,
|
|
240
257
|
excluded: {
|
|
241
258
|
count: excludedCount > 0 ? String(excludedCount) : "0",
|
|
242
259
|
text: "non-behavior symbols were excluded from the behavior count — generated code, framework internals, test-inferred flows, and infrastructure plumbing."
|
|
@@ -415,7 +432,8 @@ function riskGeneratedTests(graph, gap, riskIds, isFirstRowForFile) {
|
|
|
415
432
|
.filter(Boolean)
|
|
416
433
|
.join(" · "),
|
|
417
434
|
code: t.body,
|
|
418
|
-
runnable: t.runnable !== false
|
|
435
|
+
runnable: t.runnable !== false,
|
|
436
|
+
...(t.runnable === false ? { blocker: classifyGeneratedDraftBlocker(t.unresolved_reason) } : {})
|
|
419
437
|
}));
|
|
420
438
|
}
|
|
421
439
|
/** Incoming refs are method-attributed and can be fractional when a file-level
|
|
@@ -758,7 +776,23 @@ function riskTodo(risk, verb, path, generatedTests) {
|
|
|
758
776
|
return "Run the generated test below in your repo; follow its prove handoff so a mutation failure can mint Dynamically Proven.";
|
|
759
777
|
}
|
|
760
778
|
if (generatedTests.length) {
|
|
761
|
-
|
|
779
|
+
const blockers = new Set(generatedTests.map((t) => t.blocker ?? "unknown"));
|
|
780
|
+
if (blockers.size === 1) {
|
|
781
|
+
const blocker = [...blockers][0];
|
|
782
|
+
if (blocker === "generated_code") {
|
|
783
|
+
return "OrangePro withheld the generated code because compile validation found invalid or invented generated code. Review the blocker below, then regenerate or repair the draft; installing dependencies is not the fix.";
|
|
784
|
+
}
|
|
785
|
+
if (blocker === "unresolved_import") {
|
|
786
|
+
return "OrangePro withheld the generated code because an import path did not resolve. Verify that the import exists and is declared by this repo; install dependencies only when the repo expects it, then regenerate.";
|
|
787
|
+
}
|
|
788
|
+
if (blocker === "toolchain_or_runner") {
|
|
789
|
+
return "OrangePro could not validate the generated test because the named toolchain or test runner is unavailable. Install/configure it, then re-run `opro start`.";
|
|
790
|
+
}
|
|
791
|
+
if (blocker === "validation_timeout") {
|
|
792
|
+
return "OrangePro's generated-test validation timed out. Warm the dependency cache or raise the named validation timeout, then re-run `opro start`.";
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
return "OrangePro withheld generated code for the reasons shown below. Review each blocker, then repair or regenerate the drafts; do not assume repository dependencies are missing.";
|
|
762
796
|
}
|
|
763
797
|
const call = verb !== "BEHAVIOR"
|
|
764
798
|
? `issues ${verb} ${path}`
|
|
@@ -908,6 +942,9 @@ export function buildBehaviorReportData(graph, ledger, opts = {}) {
|
|
|
908
942
|
}
|
|
909
943
|
},
|
|
910
944
|
generatedTotal: graph.generated_tests?.length ?? 0,
|
|
945
|
+
generatedRunnableTotal: (graph.generated_tests ?? []).filter((t) => t.runnable !== false).length,
|
|
946
|
+
generatedDraftTotal: (graph.generated_tests ?? []).filter((t) => t.runnable === false).length,
|
|
947
|
+
generationOutcome: graph.analysis?.start_generation ?? null,
|
|
911
948
|
shownCount: risks.reduce((acc, r) => acc + r.generatedTests.length, 0)
|
|
912
949
|
};
|
|
913
950
|
}
|
|
@@ -366,9 +366,26 @@ body[data-mode="expert"] .simple-only{display:none!important}
|
|
|
366
366
|
<div class="split2">
|
|
367
367
|
<div><div class="v vb" id="test-int">—</div><div class="k">Integration</div></div>
|
|
368
368
|
<div><div class="v vp" id="test-unit">—</div><div class="k">Unit</div></div>
|
|
369
|
+
<div><div class="v" id="test-unknown">—</div><div class="k">Unclassified</div></div>
|
|
369
370
|
</div>
|
|
370
371
|
</div>
|
|
371
372
|
</div>
|
|
373
|
+
<div class="card" id="runtime-suite-card" hidden style="margin-top:16px">
|
|
374
|
+
<p class="card-lbl">Runtime coverage by suite — inclusive symbol counts; unit + integration overlap is shown separately and must not be added.</p>
|
|
375
|
+
<div class="split2">
|
|
376
|
+
<div><div class="v vp" id="runtime-unit">—</div><div class="k">Unit</div></div>
|
|
377
|
+
<div><div class="v vb" id="runtime-integration">—</div><div class="k">Integration</div></div>
|
|
378
|
+
<div><div class="v" id="runtime-overlap">—</div><div class="k">Both</div></div>
|
|
379
|
+
<div><div class="v" id="runtime-unclassified">—</div><div class="k">Unclassified</div></div>
|
|
380
|
+
<div><div class="v vg" id="runtime-union">—</div><div class="k">Combined union</div></div>
|
|
381
|
+
</div>
|
|
382
|
+
</div>
|
|
383
|
+
<div class="card" id="generation-outcome-card" hidden style="margin-top:16px">
|
|
384
|
+
<p class="card-lbl">Latest automated test-generation run</p>
|
|
385
|
+
<div class="v" id="generation-outcome-status">—</div>
|
|
386
|
+
<p class="card-sub" id="generation-outcome-counts"></p>
|
|
387
|
+
<p class="card-sub" id="generation-outcome-reason"></p>
|
|
388
|
+
</div>
|
|
372
389
|
</section>
|
|
373
390
|
|
|
374
391
|
<!-- TAB 2: BEHAVIORS — bridge from "methods" to "behaviors" -->
|
|
@@ -697,6 +714,23 @@ D.scan.services.forEach(([nm,ct])=>$("#svc-list").append(el("div","svc",\`<span
|
|
|
697
714
|
$("#test-total").textContent=D.scan.tests.total;
|
|
698
715
|
$("#test-int").textContent=D.scan.tests.integration;
|
|
699
716
|
$("#test-unit").textContent=D.scan.tests.unit;
|
|
717
|
+
$("#test-unknown").textContent=D.scan.tests.unclassified;
|
|
718
|
+
if(D.scan.runtimeCoverage){
|
|
719
|
+
const R=D.scan.runtimeCoverage;
|
|
720
|
+
$("#runtime-suite-card").hidden=false;
|
|
721
|
+
$("#runtime-unit").textContent=\`\${R.unit} (\${R.unitPct}%)\`;
|
|
722
|
+
$("#runtime-integration").textContent=\`\${R.integration} (\${R.integrationPct}%)\`;
|
|
723
|
+
$("#runtime-overlap").textContent=\`\${R.overlap} (\${R.overlapPct}%)\`;
|
|
724
|
+
$("#runtime-unclassified").textContent=\`\${R.unclassified} (\${R.unclassifiedPct}%)\`;
|
|
725
|
+
$("#runtime-union").textContent=\`\${R.union} (\${R.unionPct}%)\`;
|
|
726
|
+
}
|
|
727
|
+
if(D.generationOutcome){
|
|
728
|
+
const G=D.generationOutcome;
|
|
729
|
+
$("#generation-outcome-card").hidden=false;
|
|
730
|
+
$("#generation-outcome-status").textContent=String(G.status||"unknown").replaceAll("_"," ");
|
|
731
|
+
$("#generation-outcome-counts").textContent=G.generated+" draft(s): "+G.runnable+" runnable, "+Math.max(0,G.drafts-G.runnable)+" blocked; "+G.requested+" target(s) requested.";
|
|
732
|
+
$("#generation-outcome-reason").textContent=G.reason||"Generation completed; inspect each draft for its terminal validation result.";
|
|
733
|
+
}
|
|
700
734
|
|
|
701
735
|
// behaviors
|
|
702
736
|
function humanizeSig(sig){
|
|
@@ -883,6 +917,10 @@ const riskTopBanner=el("div","platform-top-banner",
|
|
|
883
917
|
<a class="platform-footer-btn" href="https://orangepro.ai/get-started" target="_blank">Unlock Full Analysis →</a>\`);
|
|
884
918
|
riskList.before(riskTopBanner);
|
|
885
919
|
const generatedRiskCount=D.risks.filter(r=>r.generatedTests&&r.generatedTests.length).length;
|
|
920
|
+
const generatedOutputCopy=[
|
|
921
|
+
D.generatedRunnableTotal?\`\${D.generatedRunnableTotal} runnable generated test\${D.generatedRunnableTotal===1?"":"s"}\`:'',
|
|
922
|
+
D.generatedDraftTotal?\`\${D.generatedDraftTotal} grounded draft\${D.generatedDraftTotal===1?"":"s"} with code withheld\`:'',
|
|
923
|
+
].filter(Boolean).join(' and ');
|
|
886
924
|
let activeRiskFilter=generatedRiskCount?"generated":"all";
|
|
887
925
|
function riskMatchesFilter(r){
|
|
888
926
|
const hasGenerated=Boolean(r.generatedTests&&r.generatedTests.length);
|
|
@@ -893,7 +931,7 @@ function riskMatchesFilter(r){
|
|
|
893
931
|
function renderRiskFilters(){
|
|
894
932
|
const options=[
|
|
895
933
|
["all","All",D.risks.length],
|
|
896
|
-
["generated","Flows with
|
|
934
|
+
["generated","Flows with generated output",generatedRiskCount],
|
|
897
935
|
["missing","No generated tests",D.risks.filter(r=>!(r.generatedTests&&r.generatedTests.length)).length]
|
|
898
936
|
];
|
|
899
937
|
riskTools.innerHTML=options.map(([key,label,count])=>\`<button class="risk-filter" type="button" data-risk-filter="\${key}" aria-pressed="\${key===activeRiskFilter}">\${label} <span class="gc">\${count}</span></button>\`).join("");
|
|
@@ -922,7 +960,7 @@ function riskCardHtml(r){
|
|
|
922
960
|
// per-test chips whenever the generator produced unit tests.
|
|
923
961
|
const allIntent=r.generatedTests.every(t=>t.runnable===false);
|
|
924
962
|
const kinds=[...new Set(r.generatedTests.map(t=>t.concern).filter(Boolean))];
|
|
925
|
-
const kindLbl=allIntent?" (
|
|
963
|
+
const kindLbl=allIntent?" (Generated drafts — code withheld)":kinds.length===1?" ("+kinds[0].replace(/_/g," ")+")":kinds.length>1?" (mixed)":"";
|
|
926
964
|
testsHtml=\`<div class="gen-tests"><div class="gen-tests-lbl">Generated tests\${esc(kindLbl)}</div>\`;
|
|
927
965
|
r.generatedTests.forEach(t=>{
|
|
928
966
|
const cBadge=t.concern?\`<span class="badge b-info" style="margin-left:6px;font-size:9px">\${esc(t.concern.replace('_',' '))}</span>\`:'';
|
|
@@ -946,12 +984,12 @@ function renderRisks(){
|
|
|
946
984
|
const remainingRiskFlows=Math.max(0,D.risks.length-generatedRiskCount);
|
|
947
985
|
riskList.append(el("div","paywall",
|
|
948
986
|
hiddenGeneratedFlows
|
|
949
|
-
? \`<div class="paywall-num">\${hiddenGeneratedFlows} more flows with
|
|
950
|
-
<div class="paywall-txt">OrangePro
|
|
987
|
+
? \`<div class="paywall-num">\${hiddenGeneratedFlows} more flows with generated output</div>
|
|
988
|
+
<div class="paywall-txt">OrangePro produced \${generatedOutputCopy} across \${generatedRiskCount} high-risk flow\${generatedRiskCount===1?"":"s"}. Use the “Flows with generated output” filter to review them first.</div>
|
|
951
989
|
<a class="paywall-btn" href="https://orangepro.ai/get-started" target="_blank">View all on OrangePro Platform →</a>\`
|
|
952
990
|
: remainingRiskFlows
|
|
953
991
|
? \`<div class="paywall-num">\${remainingRiskFlows} high-risk flows left</div>
|
|
954
|
-
<div class="paywall-txt">The local MCP
|
|
992
|
+
<div class="paywall-txt">The local MCP produced \${generatedOutputCopy} across \${generatedRiskCount} high-risk flow\${generatedRiskCount===1?"":"s"}. Generate the remaining high-risk flow tests on OrangePro Platform.</div>
|
|
955
993
|
<a class="paywall-btn" href="https://orangepro.ai/get-started" target="_blank">Generate remaining tests on Platform →</a>\`
|
|
956
994
|
: \`<div class="paywall-num">All generated tests are shown</div>
|
|
957
995
|
<div class="paywall-txt">OrangePro generated tests for every high-risk flow in this report, and every generated test is visible here.</div>\`));
|
package/docs/local-proof-kit.md
CHANGED
|
@@ -264,6 +264,14 @@ local kit ingests Go coverprofiles, JS/TS `lcov.info`, Python coverage.py XML, a
|
|
|
264
264
|
XML. These artifacts show executed lines inside symbols; they are not assertion-level proof and
|
|
265
265
|
never promote Associated signal or No integration signal rows to Proven.
|
|
266
266
|
|
|
267
|
+
For the highest-signal report in a repository with its own build/test workflow, bootstrap the
|
|
268
|
+
repository first, run its native coverage target, optionally use `opro coverage .` to confirm the
|
|
269
|
+
artifact is detected, and then run `opro start .`. The `coverage` command is discovery/generation
|
|
270
|
+
only; `analyze` and `start` ingest detected artifacts while rebuilding the graph. Running
|
|
271
|
+
`opro analyze .` immediately before `opro start .` is redundant. `opro start . --generate-coverage`
|
|
272
|
+
is a convenience for conventional repositories; prefer the repository's native coverage command
|
|
273
|
+
when it needs custom build tags, generated assets, services, or test-runner flags.
|
|
274
|
+
|
|
267
275
|
## Readiness score
|
|
268
276
|
|
|
269
277
|
A 0–100 **readiness** signal (not a proof of lift), in bands: `thin` (0–39),
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@orangepro/orangepro-mcp",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.31",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "OrangePro (`opro`) — a local-first, BYOK CLI + MCP server that builds an evidence graph from a local checkout, ingests runtime coverage, and generates grounded tests. Metadata-only exports; no source upload; generated tests stay local.",
|
|
6
6
|
"license": "MIT",
|