@orangepro/orangepro-mcp 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -3
- package/dist/local/analyze/behaviorContracts.js +78 -5
- package/dist/local/analyze/parseCache.js +16 -2
- package/dist/local/cli.js +7 -3
- package/dist/local/enrich/markdown.js +11 -0
- package/dist/local/flows/flowWalker.js +11 -6
- package/dist/local/generate/generator.js +54 -3
- package/dist/local/generate/promptV5.js +12 -1
- package/dist/local/operations.js +25 -5
- package/dist/local/proofDoctor.js +28 -3
- package/dist/local/rtm.js +30 -26
- package/dist/local/score/risk.js +128 -37
- package/dist/local/viz/behaviorReportData.js +393 -23
- package/dist/local/viz/behaviorReportHtml.js +225 -10
- package/dist/local/viz/payload.js +12 -5
- package/package.json +7 -2
package/README.md
CHANGED
|
@@ -23,7 +23,7 @@ The command writes:
|
|
|
23
23
|
|
|
24
24
|
```
|
|
25
25
|
.orangepro/
|
|
26
|
-
├── behavior-coverage.html ← open this:
|
|
26
|
+
├── behavior-coverage.html ← open this: system map, risks, flows, behaviors
|
|
27
27
|
├── graph.json ← deterministic evidence graph
|
|
28
28
|
├── COVERAGE_REPORT.md ← coverage and gap summary
|
|
29
29
|
├── rtm.md ← requirements traceability matrix
|
|
@@ -32,6 +32,8 @@ The command writes:
|
|
|
32
32
|
orangepro_generated/ ← contained generated tests; existing source files are untouched
|
|
33
33
|
```
|
|
34
34
|
|
|
35
|
+
The report opens on a **system map** of your repo — entry lanes (GraphQL/HTTP/Jobs) flowing into the services they reach, sized by traffic, colored by evidence tier, risk-ringed — identical on every run. Each rerun shows a **delta banner**: what changed since last run, or "No changes — identical graph, identical ranking." Every one of the ~N behaviors gets a plain-English description; every top risk gets a deterministic context line and a state-aware next step.
|
|
36
|
+
|
|
35
37
|
Run `opro export` when you want a machine-readable evidence pack.
|
|
36
38
|
<img width="895" height="960" alt="Screenshot 2026-07-08 at 1 18 01 AM" src="https://github.com/user-attachments/assets/73b8a812-2eab-43a1-8aed-4289545e630b" />
|
|
37
39
|
|
|
@@ -181,11 +183,13 @@ Each generated test includes:
|
|
|
181
183
|
- **Run hints** — where to write it, how to run it
|
|
182
184
|
- **Scenario bucket + technique** — what failure mode it targets and how
|
|
183
185
|
|
|
186
|
+
If the environment can't run tests yet (dependencies not installed, runner unconfigured), rejected drafts are kept as **Manual tests** — scenario, Given/When/Then steps, synthetic test data, and expected outcome in plain English, with the exact blocker named. Install dependencies and re-run `opro start` to turn them into runnable tests. Runnable tests always replace Manual tests for the same behavior; the two are never mixed.
|
|
187
|
+
|
|
184
188
|
---
|
|
185
189
|
|
|
186
190
|
## Test categories
|
|
187
191
|
|
|
188
|
-
Generation is evidence-gated. A category is produced only when the graph has supporting evidence — never padded with generic filler. These are the
|
|
192
|
+
Generation is evidence-gated. A category is produced only when the graph has supporting evidence — never padded with generic filler. These are the local generation buckets. The report additionally shows each risk's **applicable testing categories** (contract, boundary limits, integration flow, state lifecycle, failure recovery, …), derived deterministically from graph facts — covered categories from real attached tests render normally; the rest render locked, meaning "warranted here, generated on the platform," never "hidden tests exist." Neither taxonomy changes evidence tiers.
|
|
189
193
|
|
|
190
194
|
| Category | What it targets |
|
|
191
195
|
|----------|-----------------|
|
|
@@ -206,7 +210,8 @@ Every behavior gets exactly one tier. Nothing is labeled "tested" on faith.
|
|
|
206
210
|
|------|---------------|------------------|
|
|
207
211
|
| **Dynamically Proven** | A real test kills a targeted mutant of this behavior | `opro prove` after writing/running a test |
|
|
208
212
|
| **Runtime-covered** | Coverage tool executed this code | `opro start --generate-coverage` |
|
|
209
|
-
| **Statically Linked** |
|
|
213
|
+
| **Statically Linked** | A test **imports and calls** this code — a hard structural link | Automatic during analysis |
|
|
214
|
+
| **Unconfirmed Candidate** | A lexically similar test file exists, but nothing links it — a lead, **not evidence** | Automatic; upgrade it by writing the linking test |
|
|
210
215
|
| **No Signal** | Nothing tests this behavior yet | — |
|
|
211
216
|
|
|
212
217
|
> **"Dynamically Proven 0" is normal on first run.** Static analysis always runs. Dynamic proof requires running tests against targeted mutations. That's the trust model — nothing is Dynamically Proven until a real test kills a real mutant.
|
|
@@ -284,6 +289,8 @@ OrangePro separates **analysis** (what your code does) from **proof** (whether t
|
|
|
284
289
|
| **Generate** | Grounded tests for top gaps, per-behavior | Yes (BYOK) |
|
|
285
290
|
| **Prove** | Mutation-kill oracle confirms test actually breaks if behavior changes | No |
|
|
286
291
|
|
|
292
|
+
Reruns are cache-accelerated: unchanged files skip re-parsing, BYOK stages don't re-spend tokens on unchanged inputs, and proof certificates persist in a local ledger until the certified file changes. Upgrading the tool auto-invalidates caches.
|
|
293
|
+
|
|
287
294
|
---
|
|
288
295
|
|
|
289
296
|
## Privacy
|
|
@@ -6,8 +6,15 @@ const HTTP_METHODS = new Set(["get", "post", "put", "delete", "patch", "options"
|
|
|
6
6
|
// - computed router methods/paths are ignored.
|
|
7
7
|
// That is intentional while Endpoint nodes are informational only and excluded
|
|
8
8
|
// from the coverage denominator.
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
// Tolerates intermediate decorators (@UseGuards, @UseFilters, @HttpCode, …)
|
|
10
|
+
// between the route decorator and the method name — decorator-stacked NestJS
|
|
11
|
+
// controllers (Twenty: every route) previously produced ZERO endpoint contracts.
|
|
12
|
+
const NEST_METHOD_DECORATOR = /@(Get|Post|Put|Delete|Patch|Options|Head|All)\s*\(\s*(?:(["'`])([^"'`]*)\2|\[\s*(["'`])([^"'`]*)\4\s*\])?\s*\)(?:\s|@[A-Za-z_$][A-Za-z0-9_$]*(?:\s*\((?:[^()]|\([^()]*\))*\))?|\/\*[\s\S]*?\*\/|\/\/[^\n]*\n)*?(?:public\s+|private\s+|protected\s+|async\s+)*([A-Za-z_$][A-Za-z0-9_$]*)\s*\(/g;
|
|
13
|
+
const CLASS_DECLARATION = /(?:export\s+)?(?:abstract\s+)?class\s+([A-Za-z_$][A-Za-z0-9_$]*)/g;
|
|
14
|
+
const NEST_GQL_METHOD_DECORATOR = /@(Query|Mutation|Subscription|ResolveField)\s*\((?:[^()]|\([^()]*\))*\)(?:\s|@[A-Za-z_$][A-Za-z0-9_$]*(?:\s*\((?:[^()]|\([^()]*\))*\))?|\/\*[\s\S]*?\*\/|\/\/[^\n]*\n)*?(?:public\s+|private\s+|protected\s+|async\s+)*([A-Za-z_$][A-Za-z0-9_$]*)\s*\(/g;
|
|
15
|
+
const NEST_PROCESSOR_DECORATOR = /@Processor\s*\(\s*(?:(["'`])([^"'`]*)\1|[A-Za-z_$][A-Za-z0-9_$.]*)?\s*\)(?:\s|@[A-Za-z_$][A-Za-z0-9_$]*(?:\s*\((?:[^()]|\([^()]*\))*\))?|\/\*[\s\S]*?\*\/|\/\/[^\n]*\n)*?(?:export\s+)?class\s+([A-Za-z_$][A-Za-z0-9_$]*)/g;
|
|
16
|
+
const NEST_PROCESS_METHOD_DECORATOR = /@Process\s*\((?:[^()]|\([^()]*\))*\)(?:\s|@[A-Za-z_$][A-Za-z0-9_$]*(?:\s*\((?:[^()]|\([^()]*\))*\))?|\/\*[\s\S]*?\*\/|\/\/[^\n]*\n)*?(?:public\s+|private\s+|protected\s+|async\s+)*([A-Za-z_$][A-Za-z0-9_$]*)\s*\(/g;
|
|
17
|
+
const NEST_CONTROLLER_DECORATOR = /@Controller\s*\(\s*(?:(["'`])([^"'`]*)\1)?\s*\)(?:\s|@[A-Za-z_$][A-Za-z0-9_$]*(?:\s*\((?:[^()]|\([^()]*\))*\))?|\/\*[\s\S]*?\*\/|\/\/[^\n]*\n)*?(?:export\s+)?class\s+([A-Za-z_$][A-Za-z0-9_$]*)/g;
|
|
11
18
|
const EXPRESS_ROUTER_CALL = /\b(?:router|app)\s*\.\s*(get|post|put|delete|patch|options|head|all)\s*\(\s*(["'`])([^"'`]*)\2\s*,\s*([A-Za-z_$][A-Za-z0-9_$]*(?:\.[A-Za-z_$][A-Za-z0-9_$]*)?|\([^)]*\)\s*=>|async\s+\([^)]*\)\s*=>|function\s+[A-Za-z_$][A-Za-z0-9_$]*)/gi;
|
|
12
19
|
const EXPRESS_ROUTE_CHAIN = /\b(?:router|app)\s*\.\s*route\s*\(\s*(["'`])([^"'`]*)\1\s*\)((?:\s*\.\s*(?:get|post|put|delete|patch|options|head|all)\s*\([^)]*\))+)/gi;
|
|
13
20
|
const CHAINED_METHOD_CALL = /\.\s*(get|post|put|delete|patch|options|head|all)\s*\(\s*([A-Za-z_$][A-Za-z0-9_$]*(?:\.[A-Za-z_$][A-Za-z0-9_$]*)?|\([^)]*\)\s*=>|async\s+\([^)]*\)\s*=>|function\s+[A-Za-z_$][A-Za-z0-9_$]*)/gi;
|
|
@@ -17,6 +24,8 @@ export function extractBehaviorContracts(content, file) {
|
|
|
17
24
|
return dedupeContracts([
|
|
18
25
|
...extractFileRouteContracts(content, file),
|
|
19
26
|
...extractNestContracts(content, file),
|
|
27
|
+
...extractNestGraphqlContracts(content, file),
|
|
28
|
+
...extractNestProcessorContracts(content, file),
|
|
20
29
|
...extractRouterContracts(content, file, EXPRESS_ROUTER_CALL, "express"),
|
|
21
30
|
...extractExpressRouteChains(content, file),
|
|
22
31
|
...extractRouterContracts(content, file, FASTIFY_CALL, "fastify")
|
|
@@ -52,8 +61,8 @@ function extractNestContracts(content, file) {
|
|
|
52
61
|
const index = match.index ?? 0;
|
|
53
62
|
const controller = nearestController(controllers, index);
|
|
54
63
|
const method = httpMethod(match[1]);
|
|
55
|
-
const routePath = joinRoutePaths(controller?.path ?? "", match[3] ?? "");
|
|
56
|
-
const handler = match[
|
|
64
|
+
const routePath = joinRoutePaths(controller?.path ?? "", match[3] ?? match[5] ?? "");
|
|
65
|
+
const handler = match[6];
|
|
57
66
|
contracts.push(makeContract({
|
|
58
67
|
file,
|
|
59
68
|
framework: "nestjs",
|
|
@@ -65,6 +74,70 @@ function extractNestContracts(content, file) {
|
|
|
65
74
|
}
|
|
66
75
|
return contracts;
|
|
67
76
|
}
|
|
77
|
+
function extractNestGraphqlContracts(content, file) {
|
|
78
|
+
// GraphQL resolvers are first-class user-triggerable entry points. In
|
|
79
|
+
// NestJS-heavy monorepos (Twenty: 415 @Query/@Mutation methods vs 139 HTTP
|
|
80
|
+
// routes) skipping them left almost every user behavior without an Endpoint
|
|
81
|
+
// anchor, so static flows rooted at internal orphan methods instead.
|
|
82
|
+
if (!/@(Query|Mutation|Subscription)\s*\(/.test(content))
|
|
83
|
+
return [];
|
|
84
|
+
const resolvers = [...content.matchAll(CLASS_DECLARATION)].map((match) => ({
|
|
85
|
+
index: match.index ?? 0,
|
|
86
|
+
path: "",
|
|
87
|
+
name: match[1]
|
|
88
|
+
}));
|
|
89
|
+
if (resolvers.length === 0)
|
|
90
|
+
return [];
|
|
91
|
+
const contracts = [];
|
|
92
|
+
for (const match of content.matchAll(NEST_GQL_METHOD_DECORATOR)) {
|
|
93
|
+
const resolver = nearestController(resolvers, match.index ?? 0);
|
|
94
|
+
if (!resolver)
|
|
95
|
+
continue;
|
|
96
|
+
const opKind = match[1].toUpperCase();
|
|
97
|
+
if (opKind === "RESOLVEFIELD")
|
|
98
|
+
continue; // field resolvers are not user-triggerable operations
|
|
99
|
+
const handler = match[2];
|
|
100
|
+
contracts.push(makeContract({
|
|
101
|
+
file,
|
|
102
|
+
framework: "nestjs",
|
|
103
|
+
kind: "graphql_operation",
|
|
104
|
+
method: opKind,
|
|
105
|
+
path: `graphql:${handler}`,
|
|
106
|
+
handler,
|
|
107
|
+
controller: resolver.name
|
|
108
|
+
}));
|
|
109
|
+
}
|
|
110
|
+
return contracts;
|
|
111
|
+
}
|
|
112
|
+
function extractNestProcessorContracts(content, file) {
|
|
113
|
+
// Background jobs and crons are behaviors (June 27 definition: user- or
|
|
114
|
+
// system-triggerable, cross-layer, observable outcome). Anchoring them lets
|
|
115
|
+
// the flow walker show queue-driven chains instead of orphan roots.
|
|
116
|
+
const processors = [...content.matchAll(NEST_PROCESSOR_DECORATOR)].map((match) => ({
|
|
117
|
+
index: match.index ?? 0,
|
|
118
|
+
path: normalizeRoutePath(match[2] ?? ""),
|
|
119
|
+
name: match[3]
|
|
120
|
+
}));
|
|
121
|
+
if (processors.length === 0)
|
|
122
|
+
return [];
|
|
123
|
+
const contracts = [];
|
|
124
|
+
for (const match of content.matchAll(NEST_PROCESS_METHOD_DECORATOR)) {
|
|
125
|
+
const processor = nearestController(processors, match.index ?? 0);
|
|
126
|
+
if (!processor)
|
|
127
|
+
continue;
|
|
128
|
+
const handler = match[1];
|
|
129
|
+
contracts.push(makeContract({
|
|
130
|
+
file,
|
|
131
|
+
framework: "nestjs",
|
|
132
|
+
kind: "queue_processor",
|
|
133
|
+
method: "JOB",
|
|
134
|
+
path: `queue:${processor.path || processor.name}`,
|
|
135
|
+
handler,
|
|
136
|
+
controller: processor.name
|
|
137
|
+
}));
|
|
138
|
+
}
|
|
139
|
+
return contracts;
|
|
140
|
+
}
|
|
68
141
|
function nearestController(controllers, index) {
|
|
69
142
|
let current;
|
|
70
143
|
for (const controller of controllers) {
|
|
@@ -111,7 +184,7 @@ function makeContract(input) {
|
|
|
111
184
|
return {
|
|
112
185
|
id: `endpoint:${slugify(`${input.method}-${input.path}-${input.file}-${input.handler ?? ""}`)}`,
|
|
113
186
|
title,
|
|
114
|
-
kind: "http_endpoint",
|
|
187
|
+
kind: input.kind ?? "http_endpoint",
|
|
115
188
|
framework: input.framework,
|
|
116
189
|
method: input.method,
|
|
117
190
|
path: input.path,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
1
2
|
/**
|
|
2
3
|
* Phase 5.4.2 — persistent PARSE cache.
|
|
3
4
|
*
|
|
@@ -55,6 +56,19 @@
|
|
|
55
56
|
// v16: symbol extraction now carries source line spans for runtime coverage
|
|
56
57
|
// report ingestion; warm v15 entries lack the ranges and cannot be mapped.
|
|
57
58
|
export const PARSER_VERSION = 17; // 17: Go method symbols receiver-qualified (Recv.M + member_of)
|
|
59
|
+
/** Tool package version, folded into the cache guard so UPGRADES auto-invalidate
|
|
60
|
+
* the cache — bumping PARSER_VERSION by hand is a discipline; this is a lock.
|
|
61
|
+
* (The stale-cache incident: upgraded binary served old per-file results.) */
|
|
62
|
+
export const TOOL_VERSION = (() => {
|
|
63
|
+
try {
|
|
64
|
+
// dist/local/analyze/ → ../../../package.json
|
|
65
|
+
const req = createRequire(import.meta.url);
|
|
66
|
+
return String(req("../../../package.json").version ?? "0");
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
return "0";
|
|
70
|
+
}
|
|
71
|
+
})();
|
|
58
72
|
const SYMBOL_KINDS = new Set(["function", "class", "const", "method"]);
|
|
59
73
|
/** Strict structural validation — persisted data is untrusted FOR SHAPE; reject anything off-shape. */
|
|
60
74
|
function validSymbols(v) {
|
|
@@ -100,7 +114,7 @@ export class ParseCache {
|
|
|
100
114
|
constructor(data) {
|
|
101
115
|
this.entries = new Map();
|
|
102
116
|
// A version mismatch (or no data) starts empty — never trust a stale schema.
|
|
103
|
-
if (!data || data.version !== PARSER_VERSION || !data.entries || typeof data.entries !== "object")
|
|
117
|
+
if (!data || data.version !== PARSER_VERSION || data.tool !== TOOL_VERSION || !data.entries || typeof data.entries !== "object")
|
|
104
118
|
return;
|
|
105
119
|
for (const [key, raw] of Object.entries(data.entries)) {
|
|
106
120
|
if (!raw || typeof raw !== "object")
|
|
@@ -159,6 +173,6 @@ export class ParseCache {
|
|
|
159
173
|
if (e)
|
|
160
174
|
entries[key] = e;
|
|
161
175
|
}
|
|
162
|
-
return { version: PARSER_VERSION, entries };
|
|
176
|
+
return { version: PARSER_VERSION, tool: TOOL_VERSION, entries };
|
|
163
177
|
}
|
|
164
178
|
}
|
package/dist/local/cli.js
CHANGED
|
@@ -278,7 +278,10 @@ async function main() {
|
|
|
278
278
|
out(" - opro agent --client claude-code");
|
|
279
279
|
out(" - opro agent --client cursor");
|
|
280
280
|
out(" - opro agent --client opencode");
|
|
281
|
-
|
|
281
|
+
// opStart aggregates warnings from ai-flows generate AND apply — the same
|
|
282
|
+
// message (e.g. the prompt entry cap) can legitimately arrive twice.
|
|
283
|
+
// Dedupe at print time; JSON output keeps the raw array.
|
|
284
|
+
for (const w of [...new Set(res.warnings)])
|
|
282
285
|
out(` warning: ${w}`);
|
|
283
286
|
}
|
|
284
287
|
return 0;
|
|
@@ -468,7 +471,7 @@ async function main() {
|
|
|
468
471
|
}
|
|
469
472
|
if (coverageReport)
|
|
470
473
|
out(` coverage report: ${coverageReport}`);
|
|
471
|
-
for (const w of [...res.warnings, ...aiFlowWarnings, ...htmlWarnings])
|
|
474
|
+
for (const w of [...new Set([...res.warnings, ...aiFlowWarnings, ...htmlWarnings])])
|
|
472
475
|
out(` warning: ${w}`);
|
|
473
476
|
const sugg = res.analysis.exclude_suggestions ?? [];
|
|
474
477
|
if (sugg.length) {
|
|
@@ -579,7 +582,8 @@ async function main() {
|
|
|
579
582
|
out(` next: ${b.next_step}`);
|
|
580
583
|
}
|
|
581
584
|
for (const nk of res.non_killing) {
|
|
582
|
-
|
|
585
|
+
const nkLabel = nk.mutant_status === "associated_non_assertion_failure" ? "mutant failed (non-assertion)" : "mutant survived";
|
|
586
|
+
out(` ${nkLabel}: ${nk.target_symbol}${nk.test_path ? ` (test: ${nk.test_path})` : ""}`);
|
|
583
587
|
out(` ${nk.note}`);
|
|
584
588
|
}
|
|
585
589
|
}
|
|
@@ -4,6 +4,14 @@ import { slugify } from "../util/ids.js";
|
|
|
4
4
|
import { redactSecrets } from "../util/redact.js";
|
|
5
5
|
const DETECTOR = "markdown_docs";
|
|
6
6
|
const MAX_REQUIREMENTS = 60;
|
|
7
|
+
/**
|
|
8
|
+
* Repo-governance and template markdown must never mint Requirement nodes.
|
|
9
|
+
* Hint words like "should"/"must" are ubiquitous in CONTRIBUTING files and
|
|
10
|
+
* PR/issue templates — on Hono, ".github/PULL_REQUEST_TEMPLATE.md" produced
|
|
11
|
+
* REQ-md-the-author-should-do-the-following-if-applicable and surfaced as the
|
|
12
|
+
* report's top suggested next action. Product docs (README, docs/) still count.
|
|
13
|
+
*/
|
|
14
|
+
const GOVERNANCE_MD_RE = /(^|\/)\.github\/|(^|\/)(CONTRIBUTING|CODE_OF_CONDUCT|PULL_REQUEST_TEMPLATE|ISSUE_TEMPLATE|SECURITY|SUPPORT|CHANGELOG|LICENSE|GOVERNANCE|MAINTAINERS|CODEOWNERS|AUTHORS)[^\/]*$|(^|\/)\.changeset\/|\/templates?\/|(^|\/)(AGENTS|CLAUDE|GEMINI|COPILOT)\.md$|(^|\/)\.cursor(rules)?\//i;
|
|
7
15
|
/** Words that suggest a heading describes a requirement/feature behavior. */
|
|
8
16
|
const REQUIREMENT_HINTS = [
|
|
9
17
|
"requirement",
|
|
@@ -53,6 +61,9 @@ function parseBullet(line) {
|
|
|
53
61
|
* Bounded to ~60 requirements; all captured text is secret-redacted.
|
|
54
62
|
*/
|
|
55
63
|
export function enrichFromMarkdown(relPath, content) {
|
|
64
|
+
if (GOVERNANCE_MD_RE.test(relPath)) {
|
|
65
|
+
return { nodes: [], edges: [], candidate_edges: [], sources: [], warnings: [] };
|
|
66
|
+
}
|
|
56
67
|
const nodes = [];
|
|
57
68
|
const edges = [];
|
|
58
69
|
const warnings = [];
|
|
@@ -3,7 +3,7 @@ import { LOCAL_GRAPH_SCHEMA_VERSION } from "../graph/ontology.js";
|
|
|
3
3
|
import { rankRiskGaps } from "../score/risk.js";
|
|
4
4
|
import { stableId } from "../util/ids.js";
|
|
5
5
|
const DEFAULT_MAX_DEPTH = 8;
|
|
6
|
-
const DEFAULT_MAX_FLOWS_PER_ENTRY =
|
|
6
|
+
const DEFAULT_MAX_FLOWS_PER_ENTRY = 5;
|
|
7
7
|
const DEFAULT_GLOBAL_CAP = 500;
|
|
8
8
|
const HIGH_ROUTE_RE = /payment|refund|checkout|cart|order|auth|login|token|customer|user|tax|fulfillment|ship/i;
|
|
9
9
|
const MUTATION_METHOD_RE = /^(POST|PUT|PATCH|DELETE)\b/i;
|
|
@@ -103,11 +103,16 @@ export function rankEntries(graph, entries, adjacency) {
|
|
|
103
103
|
gap.id,
|
|
104
104
|
gap.risk_score
|
|
105
105
|
]));
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
106
|
+
// Endpoint-anchored flows first. An Endpoint entry IS the definition of a
|
|
107
|
+
// user-triggerable behavior (June 27 agreement); orphan call-graph roots are
|
|
108
|
+
// useful but must never crowd endpoints out of the global cap — on Twenty,
|
|
109
|
+
// saturated risk ties let ~25 internal orphan methods consume all 500 flow
|
|
110
|
+
// slots while every HTTP/GraphQL entry point went unrendered.
|
|
111
|
+
const score = (e) => Math.max(riskScores.get(e.start) ?? 0, fallbackScore(e, adjacency));
|
|
112
|
+
const byScore = (a, b) => score(b) - score(a) || a.external_id.localeCompare(b.external_id) || a.start.localeCompare(b.start);
|
|
113
|
+
const endpoints = entries.filter((e) => e.kind === "Endpoint").sort(byScore);
|
|
114
|
+
const behaviors = entries.filter((e) => e.kind !== "Endpoint").sort(byScore);
|
|
115
|
+
return [...endpoints, ...behaviors];
|
|
111
116
|
}
|
|
112
117
|
function prunePrefixSubsumed(flows) {
|
|
113
118
|
const sorted = [...flows].sort((a, b) => b.path.length - a.path.length || a.id.localeCompare(b.id));
|
|
@@ -552,7 +552,7 @@ export function gatherContext(graph, behavior, framework, fileReader) {
|
|
|
552
552
|
acceptance_criteria: acceptance,
|
|
553
553
|
workflow_steps: workflow,
|
|
554
554
|
framework,
|
|
555
|
-
test_layer: inferLayer(behavior, framework),
|
|
555
|
+
test_layer: inferLayer(behavior, framework, graph),
|
|
556
556
|
code_context: dedupe(codeContext),
|
|
557
557
|
source_excerpts: excerpts,
|
|
558
558
|
weak_context: dedupe(weakContext),
|
|
@@ -602,7 +602,7 @@ function flowChainFor(graph, behavior) {
|
|
|
602
602
|
};
|
|
603
603
|
});
|
|
604
604
|
}
|
|
605
|
-
function inferLayer(behavior, framework) {
|
|
605
|
+
function inferLayer(behavior, framework, graph) {
|
|
606
606
|
const fw = framework.toLowerCase();
|
|
607
607
|
if (fw.includes("playwright") || fw.includes("cypress"))
|
|
608
608
|
return "e2e";
|
|
@@ -613,6 +613,21 @@ function inferLayer(behavior, framework) {
|
|
|
613
613
|
const hint = String(behavior.properties.test_layer ?? "");
|
|
614
614
|
if (hint)
|
|
615
615
|
return hint;
|
|
616
|
+
// Graph-aware default (hop-count methodology): a behavior that an endpoint
|
|
617
|
+
// implements, or that participates in a multi-step call chain, is an
|
|
618
|
+
// INTEGRATION target — the code→flows→behaviors journey is the product;
|
|
619
|
+
// "unit" is only for genuinely 0-hop leaf functions. The old blanket
|
|
620
|
+
// "unit" default stamped every vitest/jest repo unit-first.
|
|
621
|
+
if (graph) {
|
|
622
|
+
const id = behavior.external_id;
|
|
623
|
+
const isEntryHandler = graph.edges.some((e) => e.relationship_type === "IMPLEMENTED_IN" && e.to_external_id === id);
|
|
624
|
+
if (isEntryHandler)
|
|
625
|
+
return "integration";
|
|
626
|
+
const inChain = graph.edges.some((e) => e.relationship_type === "CALLS" && (e.from_external_id === id || e.to_external_id === id)) ||
|
|
627
|
+
(graph.analysis?.flows?.flows ?? []).some((f) => f.entry_point.external_id === id || f.hops.some((h) => h.from === id || h.to === id));
|
|
628
|
+
if (inChain)
|
|
629
|
+
return "integration";
|
|
630
|
+
}
|
|
616
631
|
return "unit";
|
|
617
632
|
}
|
|
618
633
|
function tooThin(ctx) {
|
|
@@ -2366,13 +2381,49 @@ export async function generateTests(graph, opts, provider, fileReader, clock = s
|
|
|
2366
2381
|
const runnable = isRunnable(body, framework, import_provenance, importErrors) && !compileIssue;
|
|
2367
2382
|
if (!runnable) {
|
|
2368
2383
|
const reason = unresolved_reason ?? compileIssue ?? runnableFailureReason(body, framework, import_provenance, importErrors, declaredDeps);
|
|
2369
|
-
warnings.push(`
|
|
2384
|
+
warnings.push(`Non-runnable v5 generated test for "${gc.ctx.behavior_title}" / "${scenario.title}" kept as an English intent (no run command): ${reason}`);
|
|
2370
2385
|
missing.push({
|
|
2371
2386
|
external_id: behavior.external_id,
|
|
2372
2387
|
title: gc.ctx.behavior_title,
|
|
2373
2388
|
reason,
|
|
2374
2389
|
needed: ["a compiling generated test with a real assertion and resolvable subject import"]
|
|
2375
2390
|
});
|
|
2391
|
+
// Preserve the grounded INTENT in English, never the rejected code.
|
|
2392
|
+
// The scenario fields were authored by a model that saw source
|
|
2393
|
+
// excerpts — scrub the composed body with the same guard as code.
|
|
2394
|
+
// The scenario plan (title / assertion targets / rationale) is the
|
|
2395
|
+
// reviewable half of the draft; withholding the body entirely also
|
|
2396
|
+
// removes any residual source-echo risk. runnable:false + the reason
|
|
2397
|
+
// keep this honestly a draft — it ships with no run command and can
|
|
2398
|
+
// never enter the proof-ready set.
|
|
2399
|
+
const manualBody = sanitizeGeneratedBody([
|
|
2400
|
+
`Scenario: ${scenario.title}`,
|
|
2401
|
+
...(scenario.steps && scenario.steps.length
|
|
2402
|
+
? ["Steps:", ...scenario.steps.map((st, n) => ` ${n + 1}. ${st}`)]
|
|
2403
|
+
: []),
|
|
2404
|
+
...(scenario.test_data ? [`Test data: ${scenario.test_data}`] : []),
|
|
2405
|
+
...(scenario.assertion_targets.length ? [`Expected: ${scenario.assertion_targets.join("; ")}`] : []),
|
|
2406
|
+
...(scenario.rationale ? [`Why this test: ${scenario.rationale}`] : []),
|
|
2407
|
+
"",
|
|
2408
|
+
// Concise blocker: first clause only — the full remedy is one line.
|
|
2409
|
+
`Blocked by: ${reason.split(" — ")[0]}`,
|
|
2410
|
+
"Fix: install this repo's dependencies / configure the test runner, then re-run \`opro start\`."
|
|
2411
|
+
].join("\n"), gc.ctx.source_excerpts, "//").body;
|
|
2412
|
+
generated.push({
|
|
2413
|
+
id: `${run_id}-t${generated.length + 1}`,
|
|
2414
|
+
run_id,
|
|
2415
|
+
title: `${gc.ctx.behavior_title} — ${scenario.title}`,
|
|
2416
|
+
test_type: gc.ctx.test_layer,
|
|
2417
|
+
framework_hint: framework,
|
|
2418
|
+
body: manualBody,
|
|
2419
|
+
bucket: bucketForV5Scenario(scenario),
|
|
2420
|
+
prompt_version: PROMPT_VERSION_V5,
|
|
2421
|
+
grounding: { entity_ids: gc.entityIds, source_refs: [], weak_relationships_used: [] },
|
|
2422
|
+
weak_evidence_used: false,
|
|
2423
|
+
target_symbol_external_id: behavior.external_id,
|
|
2424
|
+
runnable: false,
|
|
2425
|
+
unresolved_reason: reason
|
|
2426
|
+
});
|
|
2376
2427
|
continue;
|
|
2377
2428
|
}
|
|
2378
2429
|
generated.push({
|
|
@@ -75,7 +75,7 @@ export function buildPlanningSystemPromptV5() {
|
|
|
75
75
|
"- Return a raw JSON array only. No prose, no markdown fences, no heading, no explanation.",
|
|
76
76
|
"- If no missing scenario is justified, return [] exactly.",
|
|
77
77
|
"- The first character of your response must be [ and the last character must be ].",
|
|
78
|
-
'[{"id":1,"title":"...","concern":"...","technique":"...","rationale":"...","assertion_targets":["..."],"complexity":"basic|intermediate|advanced","risk_rank":1}]'
|
|
78
|
+
'[{"id":1,"title":"...","concern":"...","technique":"...","rationale":"...","assertion_targets":["..."],"steps":["Given ...","When ...","Then ..."],"test_data":"concrete example input values (synthetic, showing the edge case)","complexity":"basic|intermediate|advanced","risk_rank":1}]'
|
|
79
79
|
].join("\n");
|
|
80
80
|
}
|
|
81
81
|
export function buildPlanningUserPromptV5(ctx) {
|
|
@@ -303,6 +303,15 @@ function validatePlannedScenario(v) {
|
|
|
303
303
|
const riskRank = toFinite(v.risk_rank);
|
|
304
304
|
if (riskRank === null)
|
|
305
305
|
return { ok: false, reason: "non-finite risk_rank" };
|
|
306
|
+
// Optional human-readable fields (tolerant: absent/malformed → omitted, never a rejection).
|
|
307
|
+
const steps = Array.isArray(v.steps)
|
|
308
|
+
? v.steps.filter((x) => typeof x === "string" && x.trim().length > 0).slice(0, 6).map((x) => x.slice(0, 240))
|
|
309
|
+
: undefined;
|
|
310
|
+
const test_data = typeof v.test_data === "string" && v.test_data.trim()
|
|
311
|
+
? v.test_data.slice(0, 400)
|
|
312
|
+
: v.test_data && typeof v.test_data === "object"
|
|
313
|
+
? JSON.stringify(v.test_data).slice(0, 400)
|
|
314
|
+
: undefined;
|
|
306
315
|
return {
|
|
307
316
|
ok: true,
|
|
308
317
|
value: {
|
|
@@ -312,6 +321,8 @@ function validatePlannedScenario(v) {
|
|
|
312
321
|
technique: technique,
|
|
313
322
|
rationale: typeof v.rationale === "string" ? v.rationale : "",
|
|
314
323
|
assertion_targets: targets,
|
|
324
|
+
...(steps && steps.length ? { steps } : {}),
|
|
325
|
+
...(test_data ? { test_data } : {}),
|
|
315
326
|
complexity,
|
|
316
327
|
risk_rank: riskRank
|
|
317
328
|
}
|
package/dist/local/operations.js
CHANGED
|
@@ -35,7 +35,7 @@ import { changedImpact } from "./freshness/changed.js";
|
|
|
35
35
|
import { explainTest } from "./explain/explain.js";
|
|
36
36
|
import { buildVizPayload } from "./viz/payload.js";
|
|
37
37
|
import { renderVizHtml } from "./viz/html.js";
|
|
38
|
-
import { buildBehaviorReportData, dominantBlockReason } from "./viz/behaviorReportData.js";
|
|
38
|
+
import { buildBehaviorReportData, computeReportDelta, reportBaselineOf, dominantBlockReason } from "./viz/behaviorReportData.js";
|
|
39
39
|
import { renderBehaviorReport } from "./viz/behaviorReportHtml.js";
|
|
40
40
|
import { renderCoverageReport } from "./pack/coverageReport.js";
|
|
41
41
|
import { confirmedCoverageByLayer } from "./score/coverage.js";
|
|
@@ -804,8 +804,9 @@ export function opDoctor(root) {
|
|
|
804
804
|
*/
|
|
805
805
|
export function opProofDoctor(root) {
|
|
806
806
|
const graph = loadGraph(workspacePaths(root).graphPath);
|
|
807
|
-
const
|
|
808
|
-
|
|
807
|
+
const ledger = loadLedger(root);
|
|
808
|
+
const rtm = buildRtm(graph, ledger);
|
|
809
|
+
return buildProofDoctor(graph, rtm, loadProofAttempts(root), {}, ledger);
|
|
809
810
|
}
|
|
810
811
|
export function opGaps(root, opts = {}) {
|
|
811
812
|
const graph = loadGraph(workspacePaths(root).graphPath);
|
|
@@ -1609,7 +1610,7 @@ export async function opStart(root, opts = {}, deps = defaultDeps()) {
|
|
|
1609
1610
|
limit: batch.length,
|
|
1610
1611
|
prompt_version: opts.promptVersion ?? "v5"
|
|
1611
1612
|
}, providerDeps);
|
|
1612
|
-
accepted += generated.generated_tests.length;
|
|
1613
|
+
accepted += generated.generated_tests.filter((t) => t.runnable !== false).length;
|
|
1613
1614
|
warnings.push(...generated.warnings.map((w) => `generate: ${w}`));
|
|
1614
1615
|
}
|
|
1615
1616
|
if (accepted === 0)
|
|
@@ -2000,9 +2001,28 @@ export function opBehaviorCoverageHtml(root, outputPath = "orangepro-behavior-co
|
|
|
2000
2001
|
// proof-attempts sidecar ONLY when it anchors to the current graph+commit
|
|
2001
2002
|
// (stale evidence is dropped — fail closed; display copy only, no tier math).
|
|
2002
2003
|
const dyn = dynamicProof ?? sidecarDynamicProof(root, graph);
|
|
2003
|
-
const
|
|
2004
|
+
const data = buildBehaviorReportData(graph, loadLedger(root), { repoRoot: root, dynamicProof: dyn });
|
|
2005
|
+
// Delta-since-last-run: best-effort read of the previous snapshot; a missing
|
|
2006
|
+
// or unreadable baseline means first run (banner hidden). Display-only —
|
|
2007
|
+
// the delta never touches tiers, ranks, or counts.
|
|
2008
|
+
const baselinePath = resolve(root, `${WORKSPACE_DIR}/report-baseline.json`);
|
|
2009
|
+
try {
|
|
2010
|
+
const prev = JSON.parse(readFileSync(baselinePath, "utf8"));
|
|
2011
|
+
if (prev && prev.summary && Array.isArray(prev.riskPaths))
|
|
2012
|
+
data.delta = computeReportDelta(prev, data);
|
|
2013
|
+
}
|
|
2014
|
+
catch {
|
|
2015
|
+
data.delta = null;
|
|
2016
|
+
}
|
|
2017
|
+
const html = renderBehaviorReport(data);
|
|
2004
2018
|
const htmlPath = resolve(root, outputPath);
|
|
2005
2019
|
writeFileSync(htmlPath, html, "utf8");
|
|
2020
|
+
try {
|
|
2021
|
+
writeFileSync(baselinePath, JSON.stringify(reportBaselineOf(data, new Date().toISOString())), "utf8");
|
|
2022
|
+
}
|
|
2023
|
+
catch {
|
|
2024
|
+
// baseline write is advisory
|
|
2025
|
+
}
|
|
2006
2026
|
return { behavior_coverage_path: htmlPath };
|
|
2007
2027
|
}
|
|
2008
2028
|
/** Fresh-only sidecar view for report regens; unreadable or stale ⇒ undefined. */
|
|
@@ -77,6 +77,16 @@ export const PROOF_BLOCKER_GUIDE = {
|
|
|
77
77
|
/** Wording is load-bearing: a survivor is a proven negative, never a user failure. */
|
|
78
78
|
export const NON_KILLING_NOTE = "Not proven: the test still passed while the target was mutated (possibly an equivalent mutation). " +
|
|
79
79
|
"The mutant surviving is a proven negative about assertion strength — it is never counted as Dynamically Proven.";
|
|
80
|
+
/** Opposite failure mode: the mutant DID make the test fail, but via a runtime
|
|
81
|
+
* crash rather than a trusted assertion. The test exercises the target; the
|
|
82
|
+
* proof standard (assertion failure) was not met. Misreporting this as
|
|
83
|
+
* "mutant survived" sends users to fix the wrong thing. */
|
|
84
|
+
export const NON_ASSERTION_NOTE = "Not proven: the mutant made the test FAIL, but with a runtime error instead of a trusted assertion failure. " +
|
|
85
|
+
"The test does exercise the target; strengthen the assertion to check the target's returned value directly, or re-run — " +
|
|
86
|
+
"whether the crash or the assertion is hit first can vary between runs.";
|
|
87
|
+
export function nonKillingNoteFor(mutantStatus) {
|
|
88
|
+
return mutantStatus === "associated_non_assertion_failure" ? NON_ASSERTION_NOTE : NON_KILLING_NOTE;
|
|
89
|
+
}
|
|
80
90
|
export function proofAttemptsPath(root) {
|
|
81
91
|
return join(workspacePaths(root).dir, PROOF_ATTEMPTS_FILE);
|
|
82
92
|
}
|
|
@@ -100,6 +110,7 @@ export function distillProofAttempts(auto, meta) {
|
|
|
100
110
|
target_symbol: a.target_symbol,
|
|
101
111
|
test_path: a.test_path || undefined,
|
|
102
112
|
classification: a.classification,
|
|
113
|
+
mutant_status: a.mutant_status,
|
|
103
114
|
category: a.category,
|
|
104
115
|
reason: a.reason ? redactSecrets(a.reason) : undefined,
|
|
105
116
|
deduped: a.deduped,
|
|
@@ -246,13 +257,26 @@ function groupBlockers(blocked, source) {
|
|
|
246
257
|
* Pure assembly: graph + canonical RTM result + optional attempts sidecar →
|
|
247
258
|
* deduped blocker report. Never writes; never mints; never recomputes proof.
|
|
248
259
|
*/
|
|
249
|
-
export function buildProofDoctor(graph, rtm, attempts, opts = {}) {
|
|
260
|
+
export function buildProofDoctor(graph, rtm, attempts, opts = {}, ledger) {
|
|
250
261
|
const io = opts.io ?? { exists: existsSync, nodeVersion: process.version };
|
|
251
262
|
const proven = rtm.summary.proven;
|
|
252
263
|
const denominator = rtm.summary.total;
|
|
253
264
|
// Freshness: the sidecar must anchor to the CURRENT graph generation + commit.
|
|
254
265
|
const stale = Boolean(attempts && !proofAttemptsFresh(attempts, graph));
|
|
255
266
|
const currentAttempts = attempts && !stale ? attempts : null;
|
|
267
|
+
// Legacy-sidecar backfill: proof-attempts files written before mutant_status
|
|
268
|
+
// was persisted carry no outcome detail, which forced every non-close into
|
|
269
|
+
// the "mutant survived" diagnosis. The ledger next to it is ground truth —
|
|
270
|
+
// recover mutant_status from the latest ledger record per target so the
|
|
271
|
+
// doctor self-heals without requiring a fresh `opro start`.
|
|
272
|
+
const ledgerStatusByTarget = new Map();
|
|
273
|
+
for (const r of ledger?.records ?? []) {
|
|
274
|
+
const ms = r.dynamic_proof?.mutant_status;
|
|
275
|
+
if (!ms)
|
|
276
|
+
continue;
|
|
277
|
+
ledgerStatusByTarget.set(r.target_symbol, ms); // records are append-ordered; last wins
|
|
278
|
+
}
|
|
279
|
+
const mutantStatusFor = (a) => a.mutant_status ?? ledgerStatusByTarget.get(a.target_symbol);
|
|
256
280
|
const blocked = (currentAttempts?.attempts ?? []).filter((a) => a.classification === "needs_setup");
|
|
257
281
|
const survivors = (currentAttempts?.attempts ?? []).filter((a) => a.classification === "non_killing");
|
|
258
282
|
let blockers = groupBlockers(blocked, "attempt");
|
|
@@ -268,7 +292,8 @@ export function buildProofDoctor(graph, rtm, attempts, opts = {}) {
|
|
|
268
292
|
if (seenSurvivors.has(key))
|
|
269
293
|
continue;
|
|
270
294
|
seenSurvivors.add(key);
|
|
271
|
-
|
|
295
|
+
const ms = mutantStatusFor(a);
|
|
296
|
+
non_killing.push({ target_symbol: a.target_symbol, test_path: a.test_path, mutant_status: ms, note: nonKillingNoteFor(ms) });
|
|
272
297
|
}
|
|
273
298
|
let status;
|
|
274
299
|
let headline;
|
|
@@ -291,7 +316,7 @@ export function buildProofDoctor(graph, rtm, attempts, opts = {}) {
|
|
|
291
316
|
}
|
|
292
317
|
else if (non_killing.length > 0) {
|
|
293
318
|
status = "blocked";
|
|
294
|
-
headline = `0 Dynamically Proven — ${non_killing.length} attempt(s) ran but
|
|
319
|
+
headline = `0 Dynamically Proven — ${non_killing.length} attempt(s) ran but did not close (see non_killing for the per-target outcome).`;
|
|
295
320
|
}
|
|
296
321
|
else {
|
|
297
322
|
status = "no_data";
|