@orangepro/orangepro-mcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +328 -0
- package/dist/local/agentWorkflow.js +81 -0
- package/dist/local/aiGraph/links.js +635 -0
- package/dist/local/analyze/analyzer.js +2129 -0
- package/dist/local/analyze/behaviorContracts.js +169 -0
- package/dist/local/analyze/boilerplate.js +42 -0
- package/dist/local/analyze/callGraph.js +458 -0
- package/dist/local/analyze/classify.js +219 -0
- package/dist/local/analyze/clustering.js +357 -0
- package/dist/local/analyze/confirm.js +2422 -0
- package/dist/local/analyze/coverage.js +518 -0
- package/dist/local/analyze/coverageArtifacts.js +607 -0
- package/dist/local/analyze/frameworks.js +115 -0
- package/dist/local/analyze/linkage/conventions.js +160 -0
- package/dist/local/analyze/parseCache.js +164 -0
- package/dist/local/analyze/selfAssert.js +53 -0
- package/dist/local/analyze/symbols.js +430 -0
- package/dist/local/analyze/testLayer.js +135 -0
- package/dist/local/analyze/treeSitter/engine.js +1253 -0
- package/dist/local/analyze/treeSitter/languages.js +101 -0
- package/dist/local/autoProve.js +620 -0
- package/dist/local/cli.js +1468 -0
- package/dist/local/cliArgs.js +112 -0
- package/dist/local/corpusScope.js +162 -0
- package/dist/local/enrich/csv.js +348 -0
- package/dist/local/enrich/index.js +43 -0
- package/dist/local/enrich/markdown.js +193 -0
- package/dist/local/explain/explain.js +91 -0
- package/dist/local/exportCli.js +26 -0
- package/dist/local/flows/flowWalker.js +215 -0
- package/dist/local/flows/llmFlowDiscovery.js +567 -0
- package/dist/local/freshness/changed.js +280 -0
- package/dist/local/freshness/manifest.js +35 -0
- package/dist/local/freshness/status.js +30 -0
- package/dist/local/gaps/gaps.js +114 -0
- package/dist/local/generate/buckets.js +73 -0
- package/dist/local/generate/compareJudge.js +124 -0
- package/dist/local/generate/compareReport.js +538 -0
- package/dist/local/generate/compareScore.js +105 -0
- package/dist/local/generate/deriveImports.js +91 -0
- package/dist/local/generate/generator.js +2586 -0
- package/dist/local/generate/prompt.js +144 -0
- package/dist/local/generate/promptV5.js +438 -0
- package/dist/local/generate/providers.js +400 -0
- package/dist/local/generate/runHints.js +304 -0
- package/dist/local/graph/citations.js +73 -0
- package/dist/local/graph/confirmable.js +72 -0
- package/dist/local/graph/factories.js +210 -0
- package/dist/local/graph/ontology.js +18 -0
- package/dist/local/interactive.js +53 -0
- package/dist/local/jobs/jobStore.js +80 -0
- package/dist/local/jobs/notify.js +29 -0
- package/dist/local/jobs/runner.js +75 -0
- package/dist/local/ledger.js +117 -0
- package/dist/local/localConfig.js +112 -0
- package/dist/local/mcp.js +548 -0
- package/dist/local/operations.js +1749 -0
- package/dist/local/pack/coverageReport.js +192 -0
- package/dist/local/pack/exporter.js +195 -0
- package/dist/local/pack/schema.js +128 -0
- package/dist/local/pack/summary.js +127 -0
- package/dist/local/pack/validate.js +25 -0
- package/dist/local/proofRunnability.js +366 -0
- package/dist/local/recipe/dbSqljs.js +255 -0
- package/dist/local/reprove/paths.js +13 -0
- package/dist/local/reprove/scoped.js +136 -0
- package/dist/local/resolve/barrelWalker.js +178 -0
- package/dist/local/resolve/exportIndex.js +270 -0
- package/dist/local/resolve/importGraph.js +347 -0
- package/dist/local/resolve/resolver.js +122 -0
- package/dist/local/resolve/resolverCache.js +117 -0
- package/dist/local/rtm.js +413 -0
- package/dist/local/score/coverage.js +99 -0
- package/dist/local/score/doctor.js +67 -0
- package/dist/local/score/risk.js +362 -0
- package/dist/local/score/score.js +182 -0
- package/dist/local/types.js +1 -0
- package/dist/local/util/hash.js +16 -0
- package/dist/local/util/ids.js +16 -0
- package/dist/local/util/progress.js +8 -0
- package/dist/local/util/redact.js +39 -0
- package/dist/local/util/time.js +1 -0
- package/dist/local/util/walk.js +174 -0
- package/dist/local/viz/behaviorReportData.js +367 -0
- package/dist/local/viz/behaviorReportHtml.js +664 -0
- package/dist/local/viz/d3.bundle.js +3 -0
- package/dist/local/viz/html.js +1152 -0
- package/dist/local/viz/payload.js +525 -0
- package/dist/local/workspace.js +99 -0
- package/docs/agent-workflow.md +167 -0
- package/docs/agents/claude-code.md +43 -0
- package/docs/agents/codex.md +52 -0
- package/docs/agents/cursor.md +39 -0
- package/docs/agents/opencode.md +43 -0
- package/docs/agents/vscode.md +34 -0
- package/docs/local-proof-kit.md +269 -0
- package/package.json +92 -0
- package/scripts/spikes/dynamic-proof-jest-reporter.cjs +66 -0
- package/scripts/spikes/dynamic-proof-mocha-reporter.cjs +105 -0
- package/scripts/spikes/dynamic-proof-spike.mjs +2335 -0
- package/scripts/spikes/dynamic-proof-vitest-reporter.mjs +81 -0
- package/scripts/spikes/failure-summary.mjs +29 -0
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
import { looksJson, testsFileExt } from "./compareReport.js";
|
|
2
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
/** Directory (inside the developer's repo) the agent should write generated tests to. */
|
|
5
|
+
export const GENERATED_DIR = "orangepro_generated";
|
|
6
|
+
function slug(s) {
|
|
7
|
+
return (s
|
|
8
|
+
.toLowerCase()
|
|
9
|
+
.replace(/[^a-z0-9]+/g, "_")
|
|
10
|
+
.replace(/^_+|_+$/g, "")
|
|
11
|
+
.slice(0, 60) || "test");
|
|
12
|
+
}
|
|
13
|
+
function javaPackage(body) {
|
|
14
|
+
return body.match(/^\s*package\s+([A-Za-z_][A-Za-z0-9_.]*)\s*;/m)?.[1] ?? null;
|
|
15
|
+
}
|
|
16
|
+
function javaClassName(body, fallback) {
|
|
17
|
+
return body.match(/\bclass\s+([A-Za-z_][A-Za-z0-9_]*)\b/)?.[1] ?? fallback;
|
|
18
|
+
}
|
|
19
|
+
/** The linked existing test file this generated test was grounded on, if any. */
|
|
20
|
+
function linkedTestRef(test) {
|
|
21
|
+
for (const r of test.grounding?.source_refs ?? []) {
|
|
22
|
+
if (/(\.(test|spec)\.[cm]?[jt]sx?$)|((^|\/)test\.[cm]?[jt]sx?$)|(_test\.[a-z]+$)|(_spec\.[a-z]+$)|((^|\/)test_[^/]+\.py$)/.test(r))
|
|
23
|
+
return r;
|
|
24
|
+
}
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
function isTsJsFramework(framework) {
|
|
28
|
+
const f = framework.toLowerCase();
|
|
29
|
+
return f.includes("playwright") || f.includes("cypress") || f.includes("vitest") || f.includes("jest") || f.includes("mocha") || f.includes("ava");
|
|
30
|
+
}
|
|
31
|
+
function linkedTsJsTestExt(linked) {
|
|
32
|
+
const topLevel = linked.match(/(^|\/)test\.([cm]?[jt]sx?)$/i);
|
|
33
|
+
if (topLevel)
|
|
34
|
+
return `test.${topLevel[2]}`;
|
|
35
|
+
const m = linked.match(/\.(test|spec|cy)\.([cm]?[jt]sx?)$/i);
|
|
36
|
+
return m ? `${m[1]}.${m[2]}` : null;
|
|
37
|
+
}
|
|
38
|
+
function groundingFileRef(test, ext) {
|
|
39
|
+
const refs = [...(test.grounding?.source_refs ?? []), ...(test.grounding?.entity_ids ?? [])];
|
|
40
|
+
for (const ref of refs) {
|
|
41
|
+
const rel = ref.startsWith("sym:") ? ref.slice("sym:".length).split("#")[0] : ref;
|
|
42
|
+
if (ext.test(rel))
|
|
43
|
+
return rel;
|
|
44
|
+
}
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Suggested path inside the repo for the agent to write this test to.
|
|
49
|
+
*
|
|
50
|
+
* Grounded tests reuse the linked existing test's imports, which are often
|
|
51
|
+
* RELATIVE to that test's directory (`./component`) — and monorepo test configs
|
|
52
|
+
* resolve module roots per package. So the generated test goes NEXT TO the
|
|
53
|
+
* linked test (same directory, `orangepro_generated_` prefix). Go is also
|
|
54
|
+
* package-directory sensitive, so Go tests fall back to the related Go file's
|
|
55
|
+
* directory rather than a new repo-root subpackage.
|
|
56
|
+
*/
|
|
57
|
+
export function suggestedTestPath(test, index) {
|
|
58
|
+
const linked = linkedTestRef(test);
|
|
59
|
+
const ext = (linked && isTsJsFramework(test.framework_hint) && linkedTsJsTestExt(linked)) || testsFileExt(test.framework_hint, [test.body]);
|
|
60
|
+
const stem = `${String(index + 1).padStart(2, "0")}_${slug(test.title)}`;
|
|
61
|
+
if (ext === "java") {
|
|
62
|
+
const className = javaClassName(test.body, `${stem.replace(/(^|_)([a-z0-9])/g, (_, _sep, c) => c.toUpperCase())}Test`);
|
|
63
|
+
const pkg = javaPackage(test.body);
|
|
64
|
+
return pkg ? `src/test/java/${pkg.replace(/\./g, "/")}/${className}.java` : `${GENERATED_DIR}/${className}.java`;
|
|
65
|
+
}
|
|
66
|
+
const file = ext === "go" ? `${stem}_test.go` : `${stem}.${ext}`;
|
|
67
|
+
if (linked && linked.includes("/")) {
|
|
68
|
+
return `${linked.slice(0, linked.lastIndexOf("/"))}/${GENERATED_DIR}_${file}`;
|
|
69
|
+
}
|
|
70
|
+
if (linked)
|
|
71
|
+
return `${GENERATED_DIR}_${file}`;
|
|
72
|
+
if (ext === "go") {
|
|
73
|
+
const goRef = groundingFileRef(test, /\.go$/i);
|
|
74
|
+
if (goRef && goRef.includes("/"))
|
|
75
|
+
return `${goRef.slice(0, goRef.lastIndexOf("/"))}/${GENERATED_DIR}_${file}`;
|
|
76
|
+
return `${GENERATED_DIR}_${file}`;
|
|
77
|
+
}
|
|
78
|
+
return `${GENERATED_DIR}/${file}`;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Suggested command to run a written test. A hint — the agent should prefer the
|
|
82
|
+
* repo's own configured command (package.json scripts / pyproject.toml) when it
|
|
83
|
+
* differs.
|
|
84
|
+
*/
|
|
85
|
+
function packageManager(root) {
|
|
86
|
+
try {
|
|
87
|
+
const pkg = JSON.parse(readFileSync(join(root, "package.json"), "utf8"));
|
|
88
|
+
if (pkg.packageManager?.startsWith("pnpm@"))
|
|
89
|
+
return "pnpm";
|
|
90
|
+
if (pkg.packageManager?.startsWith("yarn@"))
|
|
91
|
+
return "yarn";
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
/* fall through to lockfiles/default */
|
|
95
|
+
}
|
|
96
|
+
if (existsSync(join(root, "pnpm-lock.yaml")))
|
|
97
|
+
return "pnpm";
|
|
98
|
+
if (existsSync(join(root, "yarn.lock")))
|
|
99
|
+
return "yarn";
|
|
100
|
+
return "npm";
|
|
101
|
+
}
|
|
102
|
+
function frameworkScript(root, framework, path) {
|
|
103
|
+
let scripts = {};
|
|
104
|
+
try {
|
|
105
|
+
const pkg = JSON.parse(readFileSync(join(root, "package.json"), "utf8"));
|
|
106
|
+
scripts = pkg.scripts ?? {};
|
|
107
|
+
}
|
|
108
|
+
catch {
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
const fw = framework.toLowerCase();
|
|
112
|
+
const candidates = Object.entries(scripts)
|
|
113
|
+
.filter(([, command]) => command.toLowerCase().includes(fw))
|
|
114
|
+
.map(([name, command]) => {
|
|
115
|
+
const n = name.toLowerCase();
|
|
116
|
+
const c = command.toLowerCase();
|
|
117
|
+
if (n.includes("watch") || c.includes("watch"))
|
|
118
|
+
return { name, score: -1 };
|
|
119
|
+
if (fw === "vitest" && /\bvitest\b/.test(c) && !/\brun\b/.test(c))
|
|
120
|
+
return { name, score: -1 };
|
|
121
|
+
let score = 0;
|
|
122
|
+
if (path.includes("/unit/") && n.includes("unit"))
|
|
123
|
+
score += 40;
|
|
124
|
+
if (/\brun\b/.test(c))
|
|
125
|
+
score += 20;
|
|
126
|
+
if (n === "test")
|
|
127
|
+
score += 15;
|
|
128
|
+
if (n.startsWith(`test:${fw}`))
|
|
129
|
+
score += 12;
|
|
130
|
+
if (n.includes(fw))
|
|
131
|
+
score += 8;
|
|
132
|
+
if (n.startsWith("test"))
|
|
133
|
+
score += 5;
|
|
134
|
+
return { name, score };
|
|
135
|
+
})
|
|
136
|
+
.filter((c) => c.score >= 0)
|
|
137
|
+
.sort((a, b) => b.score - a.score || a.name.localeCompare(b.name));
|
|
138
|
+
return candidates[0]?.name ?? null;
|
|
139
|
+
}
|
|
140
|
+
function packageScriptCommand(root, framework, path) {
|
|
141
|
+
const script = frameworkScript(root, framework, path);
|
|
142
|
+
if (!script)
|
|
143
|
+
return null;
|
|
144
|
+
const pm = packageManager(root);
|
|
145
|
+
if (pm === "yarn")
|
|
146
|
+
return `yarn ${script} ${path}`;
|
|
147
|
+
return `${pm} run ${script} -- ${path}`;
|
|
148
|
+
}
|
|
149
|
+
function pytestMarkerFromBody(body) {
|
|
150
|
+
if (!body)
|
|
151
|
+
return null;
|
|
152
|
+
const markers = [...body.matchAll(/@pytest\.mark\.([A-Za-z_][A-Za-z0-9_]*)\b/g)]
|
|
153
|
+
.map((m) => m[1])
|
|
154
|
+
.filter((m) => !["skip", "skipif", "xfail", "parametrize", "usefixtures", "filterwarnings", "timeout"].includes(m));
|
|
155
|
+
return markers[0] ?? null;
|
|
156
|
+
}
|
|
157
|
+
function javaRunCommand(path, repoRoot) {
|
|
158
|
+
const javaClass = path.match(/\/([^/]+)\.java$/)?.[1] ?? "GeneratedTest";
|
|
159
|
+
if (!repoRoot)
|
|
160
|
+
return `mvn test -Dtest=${javaClass}`;
|
|
161
|
+
const testRoot = "/src/test/java/";
|
|
162
|
+
const moduleRel = path.includes(testRoot) ? path.slice(0, path.indexOf(testRoot)) : "";
|
|
163
|
+
const moduleRoot = join(repoRoot, moduleRel);
|
|
164
|
+
if (existsSync(join(moduleRoot, "pom.xml"))) {
|
|
165
|
+
const cmd = existsSync(join(moduleRoot, "mvnw")) ? "./mvnw" : "mvn";
|
|
166
|
+
return `${cmd} test -Dtest=${javaClass}`;
|
|
167
|
+
}
|
|
168
|
+
if (existsSync(join(moduleRoot, "build.gradle")) || existsSync(join(moduleRoot, "build.gradle.kts"))) {
|
|
169
|
+
const cmd = existsSync(join(moduleRoot, "gradlew")) ? "./gradlew" : "gradle";
|
|
170
|
+
return `${cmd} test --tests ${javaClass}`;
|
|
171
|
+
}
|
|
172
|
+
const cmd = existsSync(join(repoRoot, "mvnw")) ? "./mvnw" : "mvn";
|
|
173
|
+
return `${cmd} test -Dtest=${javaClass}`;
|
|
174
|
+
}
|
|
175
|
+
export function suggestedRunCommand(framework, path, repoRoot, body) {
|
|
176
|
+
const f = (framework || "").toLowerCase();
|
|
177
|
+
if (repoRoot && (f.includes("playwright") || f.includes("cypress") || f.includes("vitest") || f.includes("jest") || f.includes("mocha") || f.includes("ava"))) {
|
|
178
|
+
const scripted = packageScriptCommand(repoRoot, f.includes("playwright") ? "playwright" : f.includes("cypress") ? "cypress" : f.includes("vitest") ? "vitest" : f.includes("jest") ? "jest" : f.includes("mocha") ? "mocha" : "ava", path);
|
|
179
|
+
if (scripted)
|
|
180
|
+
return scripted;
|
|
181
|
+
}
|
|
182
|
+
if (f.includes("playwright"))
|
|
183
|
+
return `npx playwright test ${path}`;
|
|
184
|
+
if (f.includes("cypress"))
|
|
185
|
+
return `npx cypress run --spec ${path}`;
|
|
186
|
+
if (f.includes("vitest"))
|
|
187
|
+
return `npx vitest run ${path}`;
|
|
188
|
+
if (f.includes("jest"))
|
|
189
|
+
return `npx jest ${path}`;
|
|
190
|
+
if (f.includes("mocha"))
|
|
191
|
+
return `npx mocha ${path}`;
|
|
192
|
+
if (f.includes("ava"))
|
|
193
|
+
return `npx ava ${path}`;
|
|
194
|
+
if (f.includes("pytest") || f.includes("python")) {
|
|
195
|
+
const marker = pytestMarkerFromBody(body);
|
|
196
|
+
return marker ? `pytest ${path} -m ${marker}` : `pytest ${path}`;
|
|
197
|
+
}
|
|
198
|
+
if (f.includes("go"))
|
|
199
|
+
return "go test ./...";
|
|
200
|
+
if (f.includes("junit") || f.includes("java"))
|
|
201
|
+
return javaRunCommand(path, repoRoot);
|
|
202
|
+
return `<your repo's test command> ${path}`;
|
|
203
|
+
}
|
|
204
|
+
function runnerForDynamicProof(framework) {
|
|
205
|
+
const f = framework.toLowerCase();
|
|
206
|
+
if (f.includes("vitest"))
|
|
207
|
+
return "vitest";
|
|
208
|
+
if (f.includes("jest"))
|
|
209
|
+
return "jest";
|
|
210
|
+
if (f.includes("mocha"))
|
|
211
|
+
return "mocha";
|
|
212
|
+
return undefined;
|
|
213
|
+
}
|
|
214
|
+
function symbolFile(target) {
|
|
215
|
+
return target.slice("sym:".length).split("#")[0] ?? "";
|
|
216
|
+
}
|
|
217
|
+
function supportsDynamicProof(target) {
|
|
218
|
+
return /\.[cm]?[jt]sx?$/i.test(symbolFile(target));
|
|
219
|
+
}
|
|
220
|
+
export function runHintsFor(tests, repoRoot, startIndex = 0) {
|
|
221
|
+
return tests.map((t, i) => {
|
|
222
|
+
const suggested_path = suggestedTestPath(t, i + startIndex);
|
|
223
|
+
const target = t.target_symbol_external_id?.startsWith("sym:") ? t.target_symbol_external_id : undefined;
|
|
224
|
+
const canProve = target ? supportsDynamicProof(target) : false;
|
|
225
|
+
return {
|
|
226
|
+
generated_test_id: t.id,
|
|
227
|
+
title: t.title,
|
|
228
|
+
framework: t.framework_hint,
|
|
229
|
+
...(target
|
|
230
|
+
? {
|
|
231
|
+
target_symbol_external_id: target,
|
|
232
|
+
...(canProve
|
|
233
|
+
? {
|
|
234
|
+
prove_run: {
|
|
235
|
+
tool: "orangepro_prove",
|
|
236
|
+
args: {
|
|
237
|
+
target_symbol: target,
|
|
238
|
+
test_path: suggested_path,
|
|
239
|
+
replacement: "return null;",
|
|
240
|
+
...(runnerForDynamicProof(t.framework_hint) ? { runner: runnerForDynamicProof(t.framework_hint) } : {})
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
: {
|
|
245
|
+
handoff_note: "Dynamic public Proven currently supports TS/JS CodeSymbol targets only; use record_run for static diagnostics."
|
|
246
|
+
}),
|
|
247
|
+
record_run: {
|
|
248
|
+
tool: "orangepro_record_run",
|
|
249
|
+
args: { target_symbol: target, test_path: suggested_path }
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
: {
|
|
253
|
+
handoff_note: "No deterministic re-prove target (target is not a code symbol); write/run the test, then record the outcome manually."
|
|
254
|
+
}),
|
|
255
|
+
suggested_path,
|
|
256
|
+
run_command: suggestedRunCommand(t.framework_hint, suggested_path, repoRoot, t.body)
|
|
257
|
+
};
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
/**
|
|
261
|
+
* A body that is a JSON/XML test-case spec rather than runnable framework code.
|
|
262
|
+
* Run hints would mislead an agent into
|
|
263
|
+
* "running" a spec file, so these are excluded. Reuses the SAME JSON detection as
|
|
264
|
+
* the file-extension chooser (compareTestsExt → looksJson) so the two can never
|
|
265
|
+
* disagree — including the prose-wrapped ```json case (gpt-5/reasoning models).
|
|
266
|
+
*/
|
|
267
|
+
function isSpecBody(body) {
|
|
268
|
+
return looksJson(body) || (body || "").trimStart().startsWith("<");
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Run hints for the RUNNABLE (framework-code) tests only. Excludes:
|
|
272
|
+
* - non-runnable grounded drafts (`runnable === false`): no validated import /
|
|
273
|
+
* assertion, so a run_command would mislead (PLAN 6.5);
|
|
274
|
+
* - spec-mode bodies (JSON/XML eval artifacts from comparison mode).
|
|
275
|
+
* `runnable === undefined` is a legacy record, treated as runnable.
|
|
276
|
+
*/
|
|
277
|
+
export function runnableRunHintsFor(tests, repoRoot) {
|
|
278
|
+
return runHintsFor(tests.filter((t) => t.runnable !== false && !isSpecBody(t.body)), repoRoot);
|
|
279
|
+
}
|
|
280
|
+
/**
|
|
281
|
+
* Workflow for the calling agent. Returned alongside generated tests so the agent
|
|
282
|
+
* knows to write → run → report using its own shell tools.
|
|
283
|
+
*/
|
|
284
|
+
export const AGENT_RUN_WORKFLOW = [
|
|
285
|
+
"OrangePro returns the test code; YOUR agent runs it. Using your shell tools:",
|
|
286
|
+
"1. Write each test's `body` to its `suggested_path` in the repo.",
|
|
287
|
+
"2. Run its `run_command` from the package that owns the test's directory (monorepos: cd into that package first so the framework's module roots resolve) — or use the repo's own test command (npm test, pytest, npx playwright test).",
|
|
288
|
+
"3. For public Proven, call `prove_run`/`orangepro_prove` after the test passes; it reruns baseline + sentinel-mutant and closes only on a dynamic assertion kill. `record_run` is static diagnostics only.",
|
|
289
|
+
"4. Report pass/fail, error messages, and stack traces to the developer; propose a fix for failures.",
|
|
290
|
+
"Runnable Python and Go drafts require local validation tools on PATH (`python3` for pytest syntax checks, `gofmt` for Go syntax checks). If those tools are missing, OrangePro safely returns drafts without run commands.",
|
|
291
|
+
"Requires the repo's test framework + dependencies to already be installed. If a test fails to run for environment reasons (missing framework/deps), that's a local setup issue, not the generated test.",
|
|
292
|
+
"Single-repo, local, free-tier. Clean reproducible CI runs (GitHub Actions) and multi-repo lifecycle Knowledge Graph are part of the hosted/paid OrangePro platform — upgrade for those."
|
|
293
|
+
];
|
|
294
|
+
/**
|
|
295
|
+
* The keyless grounding contract, returned alongside generated tests. Tells the
|
|
296
|
+
* agent that each test ships with VALIDATED evidence (real graph entities, not
|
|
297
|
+
* generic guesses) and that it should ground its work in that evidence and report
|
|
298
|
+
* back which entities it actually used.
|
|
299
|
+
*/
|
|
300
|
+
export const GROUNDING_CONTRACT = [
|
|
301
|
+
"Each test carries `evidence`: grounding citations already VALIDATED against OrangePro's local graph (every `evidence_id` resolves to a real entity, with its kind, evidence_strength, and source_ref).",
|
|
302
|
+
"Treat hard/reviewed evidence as proof; weak/candidate evidence is a hint to verify, not trust. `evidence_summary` reports proof coverage and any broken citations.",
|
|
303
|
+
"Ground the test you keep in this evidence; when you accept or write a test, note which `evidence_id`s you used so OrangePro can record grounding-used vs grounding-ignored."
|
|
304
|
+
];
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { redactSecrets } from "../util/redact.js";
|
|
2
|
+
const PROOF_STRENGTHS = new Set(["hard", "reviewed"]);
|
|
3
|
+
function isProof(strength) {
|
|
4
|
+
return strength !== "unknown" && PROOF_STRENGTHS.has(strength);
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Build an O(1) external_id -> node index for the loaded graph. First-write wins,
|
|
8
|
+
* mirroring `findNode`'s first-match semantics, so a duplicate external_id
|
|
9
|
+
* resolves to the same node the rest of the kit would pick.
|
|
10
|
+
*/
|
|
11
|
+
export function buildCitationIndex(graph) {
|
|
12
|
+
const nodeById = new Map();
|
|
13
|
+
for (const node of graph.nodes) {
|
|
14
|
+
if (!nodeById.has(node.external_id))
|
|
15
|
+
nodeById.set(node.external_id, node);
|
|
16
|
+
}
|
|
17
|
+
return { nodeById };
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Validate a generated test's grounding citations against the graph index:
|
|
21
|
+
* resolve each cited entity_id, attach kind/strength/source_ref, and count how
|
|
22
|
+
* many are genuine proof (hard/reviewed).
|
|
23
|
+
*/
|
|
24
|
+
export function validateEvidence(index, grounding) {
|
|
25
|
+
const evidence = grounding.entity_ids.map((id) => {
|
|
26
|
+
const node = index.nodeById.get(id);
|
|
27
|
+
if (!node) {
|
|
28
|
+
return { evidence_id: id, validated: false, kind: "unknown", title: id, evidence_strength: "unknown" };
|
|
29
|
+
}
|
|
30
|
+
const cited = {
|
|
31
|
+
evidence_id: node.external_id,
|
|
32
|
+
validated: true,
|
|
33
|
+
kind: node.kind,
|
|
34
|
+
// Defense-in-depth: titles/source_refs are metadata, but re-scrub at this exit
|
|
35
|
+
// boundary so the "no secrets" guarantee holds regardless of how the node was
|
|
36
|
+
// populated (mirrors the evidence-pack exporter's redactDeep). external_id is
|
|
37
|
+
// structural (path / req id) and stays verbatim so the agent can reconcile it.
|
|
38
|
+
title: redactSecrets(node.title ?? node.external_id),
|
|
39
|
+
evidence_strength: node.evidence_strength
|
|
40
|
+
};
|
|
41
|
+
if (node.provenance?.source_ref)
|
|
42
|
+
cited.source_ref = redactSecrets(node.provenance.source_ref);
|
|
43
|
+
return cited;
|
|
44
|
+
});
|
|
45
|
+
const validated_count = evidence.reduce((n, e) => (e.validated ? n + 1 : n), 0);
|
|
46
|
+
const proof_count = evidence.reduce((n, e) => (e.validated && isProof(e.evidence_strength) ? n + 1 : n), 0);
|
|
47
|
+
return {
|
|
48
|
+
evidence,
|
|
49
|
+
validated_count,
|
|
50
|
+
invalid_count: evidence.length - validated_count,
|
|
51
|
+
proof_count,
|
|
52
|
+
has_proof: proof_count > 0
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Validate every generated test's citations against the graph and roll the
|
|
57
|
+
* results into a per-test list plus a run-level summary. Builds the index once.
|
|
58
|
+
*/
|
|
59
|
+
export function summarizeTestEvidence(graph, tests) {
|
|
60
|
+
const index = buildCitationIndex(graph);
|
|
61
|
+
const per_test = tests.map((t) => ({
|
|
62
|
+
generated_test_id: t.id,
|
|
63
|
+
title: t.title,
|
|
64
|
+
...validateEvidence(index, t.grounding)
|
|
65
|
+
}));
|
|
66
|
+
const summary = {
|
|
67
|
+
tests: per_test.length,
|
|
68
|
+
tests_with_proof: per_test.reduce((n, p) => (p.has_proof ? n + 1 : n), 0),
|
|
69
|
+
tests_without_validated_evidence: per_test.reduce((n, p) => (p.validated_count === 0 ? n + 1 : n), 0),
|
|
70
|
+
invalid_citations: per_test.reduce((n, p) => n + p.invalid_count, 0)
|
|
71
|
+
};
|
|
72
|
+
return { per_test, summary };
|
|
73
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// Structural-confirmability deferral (Phase 4.5 / Gate 7b).
|
|
2
|
+
//
|
|
3
|
+
// Some behaviors can never be proven by static TS import resolution + symbol
|
|
4
|
+
// evidence because their real coverage lives at a layer the resolver does not
|
|
5
|
+
// traverse (e2e / api). These are NOT gaps (a real test may exist) and NOT
|
|
6
|
+
// confirmed (we cannot prove it). Forcing them into either side breaks trust, so
|
|
7
|
+
// they get a third state: `not_structurally_confirmable` (nsc), excluded from the
|
|
8
|
+
// confirmed-% denominator and recorded with a `defer_reason`.
|
|
9
|
+
//
|
|
10
|
+
// v1 deferral signal: a behavior with NO hard TESTED_BY/COVERS edge whose linked
|
|
11
|
+
// candidate tests are ALL e2e- or api-layer (the AST classifier's authoritative
|
|
12
|
+
// layer). Precedence: a hard edge → confirmed BEFORE nsc is ever considered.
|
|
13
|
+
// (dynamic-import / DI-container / route-registration deferral is a documented
|
|
14
|
+
// follow-up; under-deferring keeps behaviors visible, the safe direction.)
|
|
15
|
+
/**
|
|
16
|
+
* Map of behavior external_id → defer reason for every behavior that is
|
|
17
|
+
* not-structurally-confirmable. O(edges + nodes), computed once per consumer.
|
|
18
|
+
*/
|
|
19
|
+
export function structurallyUnconfirmable(graph) {
|
|
20
|
+
// Behaviors touched by a hard coverage edge (either end) are confirmed — never nsc.
|
|
21
|
+
const hard = new Set();
|
|
22
|
+
for (const e of graph.edges) {
|
|
23
|
+
if (e.relationship_type === "TESTED_BY" || e.relationship_type === "COVERS") {
|
|
24
|
+
hard.add(e.from_external_id);
|
|
25
|
+
hard.add(e.to_external_id);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
// TestCase id → its authoritative layer (Phase 4.6).
|
|
29
|
+
const layerOf = new Map();
|
|
30
|
+
for (const n of graph.nodes) {
|
|
31
|
+
if (n.kind === "TestCase") {
|
|
32
|
+
layerOf.set(n.external_id, typeof n.properties.test_layer === "string" ? n.properties.test_layer : "unknown");
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
// behavior id → set of linked TestCase ids (via the weak candidate edges).
|
|
36
|
+
const linkedTests = new Map();
|
|
37
|
+
const link = (behavior, test) => {
|
|
38
|
+
if (!layerOf.has(test))
|
|
39
|
+
return;
|
|
40
|
+
let s = linkedTests.get(behavior);
|
|
41
|
+
if (!s) {
|
|
42
|
+
s = new Set();
|
|
43
|
+
linkedTests.set(behavior, s);
|
|
44
|
+
}
|
|
45
|
+
s.add(test);
|
|
46
|
+
};
|
|
47
|
+
for (const e of graph.candidate_edges) {
|
|
48
|
+
if (e.relationship_type !== "MAY_BE_TESTED_BY" && e.relationship_type !== "MAY_COVER")
|
|
49
|
+
continue;
|
|
50
|
+
// The TestCase is whichever end is a known TestCase; the other (NON-test) end
|
|
51
|
+
// is the behavior. Mutually exclusive so an edge between two TestCases never
|
|
52
|
+
// mis-attributes a TestCase as a behavior.
|
|
53
|
+
const toIsTest = layerOf.has(e.to_external_id);
|
|
54
|
+
const fromIsTest = layerOf.has(e.from_external_id);
|
|
55
|
+
if (toIsTest && !fromIsTest)
|
|
56
|
+
link(e.from_external_id, e.to_external_id);
|
|
57
|
+
else if (fromIsTest && !toIsTest)
|
|
58
|
+
link(e.to_external_id, e.from_external_id);
|
|
59
|
+
}
|
|
60
|
+
const out = new Map();
|
|
61
|
+
for (const [behavior, tests] of linkedTests) {
|
|
62
|
+
if (hard.has(behavior))
|
|
63
|
+
continue; // confirmed precedence
|
|
64
|
+
const layers = [...tests].map((t) => layerOf.get(t) ?? "unknown");
|
|
65
|
+
if (layers.length === 0)
|
|
66
|
+
continue;
|
|
67
|
+
if (!layers.every((l) => l === "e2e" || l === "api"))
|
|
68
|
+
continue;
|
|
69
|
+
out.set(behavior, layers.includes("e2e") ? "layer_e2e" : "layer_api");
|
|
70
|
+
}
|
|
71
|
+
return out;
|
|
72
|
+
}
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import { stableId } from "../util/ids.js";
|
|
2
|
+
import { BOILERPLATE_REASON } from "../analyze/boilerplate.js";
|
|
3
|
+
import { GENERATED_CODE_REASON, NON_PRODUCT_REASON } from "../analyze/classify.js";
|
|
4
|
+
import { BEHAVIOR_KINDS } from "./ontology.js";
|
|
5
|
+
/**
|
|
6
|
+
* Kind-based denominator defaults for nodes whose producer did not set the
|
|
7
|
+
* fields explicitly (hand-built graphs, fixtures). The four real producers
|
|
8
|
+
* (analyzer UserFlow/CodeSymbol, csv, markdown) always set them explicitly —
|
|
9
|
+
* these defaults encode the same Gate-3 policy: explicit requirements count,
|
|
10
|
+
* test-inferred flows never do, and a bare CodeSymbol is NOT counted unless
|
|
11
|
+
* the analyzer proves it eligible.
|
|
12
|
+
*/
|
|
13
|
+
function denominatorDefaults(kind) {
|
|
14
|
+
if (kind === "Requirement" || kind === "BusinessRule") {
|
|
15
|
+
return {
|
|
16
|
+
behavior_source: "requirement_template",
|
|
17
|
+
denominator_eligible: true,
|
|
18
|
+
denominator_reason: "Explicit requirement — always countable."
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
if (kind === "UserFlow") {
|
|
22
|
+
return {
|
|
23
|
+
behavior_source: "test_inferred",
|
|
24
|
+
denominator_eligible: false,
|
|
25
|
+
denominator_reason: "Inferred from test names — a test cannot witness its own requirement."
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
if (kind === "CodeSymbol") {
|
|
29
|
+
return {
|
|
30
|
+
behavior_source: "code_export",
|
|
31
|
+
denominator_eligible: false,
|
|
32
|
+
denominator_reason: "Code export not proven countable (analyzer decides eligibility)."
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
return {};
|
|
36
|
+
}
|
|
37
|
+
export function makeNode(input) {
|
|
38
|
+
const defaults = denominatorDefaults(input.kind);
|
|
39
|
+
return {
|
|
40
|
+
id: stableId(input.kind, input.external_id),
|
|
41
|
+
external_id: input.external_id,
|
|
42
|
+
kind: input.kind,
|
|
43
|
+
title: input.title,
|
|
44
|
+
properties: input.properties ?? {},
|
|
45
|
+
evidence_strength: input.evidence_strength,
|
|
46
|
+
review_status: input.review_status,
|
|
47
|
+
confidence: input.confidence,
|
|
48
|
+
provenance: input.provenance,
|
|
49
|
+
content_hash: input.content_hash,
|
|
50
|
+
behavior_source: input.behavior_source ?? defaults.behavior_source,
|
|
51
|
+
denominator_eligible: input.denominator_eligible ?? defaults.denominator_eligible,
|
|
52
|
+
denominator_reason: input.denominator_reason ?? defaults.denominator_reason
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
export function makeTestCaseNode(input) {
|
|
56
|
+
return makeNode({
|
|
57
|
+
kind: "TestCase",
|
|
58
|
+
external_id: `test:${input.testRel}`,
|
|
59
|
+
title: input.title,
|
|
60
|
+
properties: {
|
|
61
|
+
test_layer: input.testLayer,
|
|
62
|
+
layer_confidence: input.layerConfidence,
|
|
63
|
+
layer_signals: input.layerSignals,
|
|
64
|
+
file: input.testRel,
|
|
65
|
+
test_names: input.testNames
|
|
66
|
+
},
|
|
67
|
+
evidence_strength: "hard",
|
|
68
|
+
review_status: "auto_detected",
|
|
69
|
+
confidence: 1,
|
|
70
|
+
provenance: input.provenance,
|
|
71
|
+
content_hash: input.contentHash
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
export function makeEdge(input) {
|
|
75
|
+
return {
|
|
76
|
+
id: stableId("edge", `${input.from_external_id}|${input.relationship_type}|${input.to_external_id}`),
|
|
77
|
+
...input
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
export function makeProofEdges(input) {
|
|
81
|
+
const testExternalId = `test:${input.testRel}`;
|
|
82
|
+
return [
|
|
83
|
+
makeEdge({
|
|
84
|
+
from_external_id: input.symId,
|
|
85
|
+
to_external_id: testExternalId,
|
|
86
|
+
relationship_type: "TESTED_BY",
|
|
87
|
+
evidence_strength: "hard",
|
|
88
|
+
review_status: "auto_detected",
|
|
89
|
+
provenance: input.provenance,
|
|
90
|
+
last_verified: input.lastVerified
|
|
91
|
+
}),
|
|
92
|
+
makeEdge({
|
|
93
|
+
from_external_id: testExternalId,
|
|
94
|
+
to_external_id: input.symId,
|
|
95
|
+
relationship_type: "COVERS",
|
|
96
|
+
evidence_strength: "hard",
|
|
97
|
+
review_status: "auto_detected",
|
|
98
|
+
provenance: input.provenance,
|
|
99
|
+
last_verified: input.lastVerified
|
|
100
|
+
})
|
|
101
|
+
];
|
|
102
|
+
}
|
|
103
|
+
export function makeCandidateEdge(input) {
|
|
104
|
+
return {
|
|
105
|
+
id: stableId("cand", `${input.from_external_id}|${input.relationship_type}|${input.to_external_id}`),
|
|
106
|
+
...input
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
// ── Graph query helpers ──────────────────────────────────────────────
|
|
110
|
+
export function nodesByKind(graph, kind) {
|
|
111
|
+
return graph.nodes.filter((n) => n.kind === kind);
|
|
112
|
+
}
|
|
113
|
+
export function findNode(graph, externalId) {
|
|
114
|
+
return graph.nodes.find((n) => n.external_id === externalId);
|
|
115
|
+
}
|
|
116
|
+
export function behaviorNodes(graph) {
|
|
117
|
+
return graph.nodes.filter((n) => BEHAVIOR_KINDS.has(n.kind));
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* SOLE source of truth for what counts in the coverage denominator (Gate 3).
|
|
121
|
+
* Nothing else may decide eligibility — not kind sets, not evidence strength.
|
|
122
|
+
* Stale nodes never count: incremental update keeps deleted files' exports as
|
|
123
|
+
* `stale: true` ghosts, and a deleted export is not witnessable by the repo.
|
|
124
|
+
*/
|
|
125
|
+
export function isDenominatorEligible(n) {
|
|
126
|
+
return n.denominator_eligible === true && n.stale !== true;
|
|
127
|
+
}
|
|
128
|
+
/** The behaviors the coverage denominator is computed over. */
|
|
129
|
+
export function denominatorBehaviors(graph) {
|
|
130
|
+
return graph.nodes.filter(isDenominatorEligible);
|
|
131
|
+
}
|
|
132
|
+
/** Auditable composition of the denominator (shared by score + analyzer). */
|
|
133
|
+
export function denominatorComposition(graph) {
|
|
134
|
+
const comp = {
|
|
135
|
+
total: 0,
|
|
136
|
+
code_export: 0,
|
|
137
|
+
requirement_template: 0,
|
|
138
|
+
markdown_requirement: 0,
|
|
139
|
+
excluded_test_inferred: 0,
|
|
140
|
+
excluded_boilerplate: 0,
|
|
141
|
+
excluded_infra: 0,
|
|
142
|
+
excluded_generated: 0,
|
|
143
|
+
code_symbols_total: 0,
|
|
144
|
+
unattributed: 0
|
|
145
|
+
};
|
|
146
|
+
for (const n of graph.nodes) {
|
|
147
|
+
if (n.stale === true)
|
|
148
|
+
continue; // deleted-file ghosts are not witnessable
|
|
149
|
+
if (n.kind === "CodeSymbol")
|
|
150
|
+
comp.code_symbols_total++; // true "found" total (eligible + every excluded class)
|
|
151
|
+
if (isDenominatorEligible(n)) {
|
|
152
|
+
comp.total++;
|
|
153
|
+
if (n.behavior_source === "code_export")
|
|
154
|
+
comp.code_export++;
|
|
155
|
+
else if (n.behavior_source === "markdown_requirement")
|
|
156
|
+
comp.markdown_requirement++;
|
|
157
|
+
else if (n.behavior_source === "requirement_template")
|
|
158
|
+
comp.requirement_template++;
|
|
159
|
+
// An eligible node with a test_inferred/missing source is a producer-bug
|
|
160
|
+
// combination — surface it, never launder it into a requirement count.
|
|
161
|
+
else
|
|
162
|
+
comp.unattributed++;
|
|
163
|
+
}
|
|
164
|
+
else if (n.behavior_source === "test_inferred") {
|
|
165
|
+
comp.excluded_test_inferred++;
|
|
166
|
+
}
|
|
167
|
+
else if (n.denominator_reason === BOILERPLATE_REASON) {
|
|
168
|
+
// Recomputed from nodes (not persisted analysis) so the report disclosure
|
|
169
|
+
// can never disagree with the graph it is paired with.
|
|
170
|
+
comp.excluded_boilerplate++;
|
|
171
|
+
}
|
|
172
|
+
else if (n.denominator_reason === NON_PRODUCT_REASON) {
|
|
173
|
+
comp.excluded_infra++;
|
|
174
|
+
}
|
|
175
|
+
else if (n.denominator_reason === GENERATED_CODE_REASON) {
|
|
176
|
+
comp.excluded_generated++;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
return comp;
|
|
180
|
+
}
|
|
181
|
+
/** Hard/reviewed edges where the given external id is the source. */
|
|
182
|
+
export function outgoingEdges(graph, externalId) {
|
|
183
|
+
return graph.edges.filter((e) => e.from_external_id === externalId);
|
|
184
|
+
}
|
|
185
|
+
/** Hard/reviewed edges where the given external id is the target. */
|
|
186
|
+
export function incomingEdges(graph, externalId) {
|
|
187
|
+
return graph.edges.filter((e) => e.to_external_id === externalId);
|
|
188
|
+
}
|
|
189
|
+
export function edgesOfType(graph, externalId, type) {
|
|
190
|
+
return graph.edges.filter((e) => e.relationship_type === type && (e.from_external_id === externalId || e.to_external_id === externalId));
|
|
191
|
+
}
|
|
192
|
+
export function candidateEdgesFrom(graph, externalId) {
|
|
193
|
+
return graph.candidate_edges.filter((e) => e.from_external_id === externalId);
|
|
194
|
+
}
|
|
195
|
+
export function generatedTestById(graph, testId) {
|
|
196
|
+
return graph.generated_tests.find((t) => t.id === testId || t.title === testId);
|
|
197
|
+
}
|
|
198
|
+
/** Priority ranking helper used by gaps + generation target selection. */
|
|
199
|
+
export function priorityRank(value) {
|
|
200
|
+
const p = String(value ?? "").toLowerCase();
|
|
201
|
+
if (p.includes("crit") || p === "p0")
|
|
202
|
+
return 4;
|
|
203
|
+
if (p.includes("high") || p === "p1")
|
|
204
|
+
return 3;
|
|
205
|
+
if (p.includes("med") || p === "p2")
|
|
206
|
+
return 2;
|
|
207
|
+
if (p.includes("low") || p === "p3")
|
|
208
|
+
return 1;
|
|
209
|
+
return 0;
|
|
210
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OrangePro local evidence-graph ontology.
|
|
3
|
+
*
|
|
4
|
+
* This is an OrangePro-shaped, test-generation-oriented graph — NOT a generic
|
|
5
|
+
* code graph. Every node and edge carries evidence strength and provenance so
|
|
6
|
+
* the kit can prove *why* a generated test is grounded, and so a pack can later
|
|
7
|
+
* be promoted into a hosted tenant graph without changing the conceptual model.
|
|
8
|
+
*
|
|
9
|
+
* The graph is built directly by OrangePro; it does not depend on any
|
|
10
|
+
* third-party graph product or format.
|
|
11
|
+
*/
|
|
12
|
+
export const LOCAL_GRAPH_SCHEMA_VERSION = "orangepro.local_graph.v1";
|
|
13
|
+
/** Node kinds that map to behaviors/requirements for scoring + gaps + generation. */
|
|
14
|
+
export const BEHAVIOR_KINDS = new Set([
|
|
15
|
+
"Requirement",
|
|
16
|
+
"UserFlow",
|
|
17
|
+
"BusinessRule"
|
|
18
|
+
]);
|