@orangepro/orangepro-mcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +328 -0
- package/dist/local/agentWorkflow.js +81 -0
- package/dist/local/aiGraph/links.js +635 -0
- package/dist/local/analyze/analyzer.js +2129 -0
- package/dist/local/analyze/behaviorContracts.js +169 -0
- package/dist/local/analyze/boilerplate.js +42 -0
- package/dist/local/analyze/callGraph.js +458 -0
- package/dist/local/analyze/classify.js +219 -0
- package/dist/local/analyze/clustering.js +357 -0
- package/dist/local/analyze/confirm.js +2422 -0
- package/dist/local/analyze/coverage.js +518 -0
- package/dist/local/analyze/coverageArtifacts.js +607 -0
- package/dist/local/analyze/frameworks.js +115 -0
- package/dist/local/analyze/linkage/conventions.js +160 -0
- package/dist/local/analyze/parseCache.js +164 -0
- package/dist/local/analyze/selfAssert.js +53 -0
- package/dist/local/analyze/symbols.js +430 -0
- package/dist/local/analyze/testLayer.js +135 -0
- package/dist/local/analyze/treeSitter/engine.js +1253 -0
- package/dist/local/analyze/treeSitter/languages.js +101 -0
- package/dist/local/autoProve.js +620 -0
- package/dist/local/cli.js +1468 -0
- package/dist/local/cliArgs.js +112 -0
- package/dist/local/corpusScope.js +162 -0
- package/dist/local/enrich/csv.js +348 -0
- package/dist/local/enrich/index.js +43 -0
- package/dist/local/enrich/markdown.js +193 -0
- package/dist/local/explain/explain.js +91 -0
- package/dist/local/exportCli.js +26 -0
- package/dist/local/flows/flowWalker.js +215 -0
- package/dist/local/flows/llmFlowDiscovery.js +567 -0
- package/dist/local/freshness/changed.js +280 -0
- package/dist/local/freshness/manifest.js +35 -0
- package/dist/local/freshness/status.js +30 -0
- package/dist/local/gaps/gaps.js +114 -0
- package/dist/local/generate/buckets.js +73 -0
- package/dist/local/generate/compareJudge.js +124 -0
- package/dist/local/generate/compareReport.js +538 -0
- package/dist/local/generate/compareScore.js +105 -0
- package/dist/local/generate/deriveImports.js +91 -0
- package/dist/local/generate/generator.js +2586 -0
- package/dist/local/generate/prompt.js +144 -0
- package/dist/local/generate/promptV5.js +438 -0
- package/dist/local/generate/providers.js +400 -0
- package/dist/local/generate/runHints.js +304 -0
- package/dist/local/graph/citations.js +73 -0
- package/dist/local/graph/confirmable.js +72 -0
- package/dist/local/graph/factories.js +210 -0
- package/dist/local/graph/ontology.js +18 -0
- package/dist/local/interactive.js +53 -0
- package/dist/local/jobs/jobStore.js +80 -0
- package/dist/local/jobs/notify.js +29 -0
- package/dist/local/jobs/runner.js +75 -0
- package/dist/local/ledger.js +117 -0
- package/dist/local/localConfig.js +112 -0
- package/dist/local/mcp.js +548 -0
- package/dist/local/operations.js +1749 -0
- package/dist/local/pack/coverageReport.js +192 -0
- package/dist/local/pack/exporter.js +195 -0
- package/dist/local/pack/schema.js +128 -0
- package/dist/local/pack/summary.js +127 -0
- package/dist/local/pack/validate.js +25 -0
- package/dist/local/proofRunnability.js +366 -0
- package/dist/local/recipe/dbSqljs.js +255 -0
- package/dist/local/reprove/paths.js +13 -0
- package/dist/local/reprove/scoped.js +136 -0
- package/dist/local/resolve/barrelWalker.js +178 -0
- package/dist/local/resolve/exportIndex.js +270 -0
- package/dist/local/resolve/importGraph.js +347 -0
- package/dist/local/resolve/resolver.js +122 -0
- package/dist/local/resolve/resolverCache.js +117 -0
- package/dist/local/rtm.js +413 -0
- package/dist/local/score/coverage.js +99 -0
- package/dist/local/score/doctor.js +67 -0
- package/dist/local/score/risk.js +362 -0
- package/dist/local/score/score.js +182 -0
- package/dist/local/types.js +1 -0
- package/dist/local/util/hash.js +16 -0
- package/dist/local/util/ids.js +16 -0
- package/dist/local/util/progress.js +8 -0
- package/dist/local/util/redact.js +39 -0
- package/dist/local/util/time.js +1 -0
- package/dist/local/util/walk.js +174 -0
- package/dist/local/viz/behaviorReportData.js +367 -0
- package/dist/local/viz/behaviorReportHtml.js +664 -0
- package/dist/local/viz/d3.bundle.js +3 -0
- package/dist/local/viz/html.js +1152 -0
- package/dist/local/viz/payload.js +525 -0
- package/dist/local/workspace.js +99 -0
- package/docs/agent-workflow.md +167 -0
- package/docs/agents/claude-code.md +43 -0
- package/docs/agents/codex.md +52 -0
- package/docs/agents/cursor.md +39 -0
- package/docs/agents/opencode.md +43 -0
- package/docs/agents/vscode.md +34 -0
- package/docs/local-proof-kit.md +269 -0
- package/package.json +92 -0
- package/scripts/spikes/dynamic-proof-jest-reporter.cjs +66 -0
- package/scripts/spikes/dynamic-proof-mocha-reporter.cjs +105 -0
- package/scripts/spikes/dynamic-proof-spike.mjs +2335 -0
- package/scripts/spikes/dynamic-proof-vitest-reporter.mjs +81 -0
- package/scripts/spikes/failure-summary.mjs +29 -0
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import { LEDGER_SCHEMA_VERSION } from "../ledger.js";
|
|
2
|
+
import { resolveCoverage } from "../score/coverage.js";
|
|
3
|
+
import { rankRiskGaps } from "../score/risk.js";
|
|
4
|
+
import { buildRtm } from "../rtm.js";
|
|
5
|
+
/**
|
|
6
|
+
* Phase 5.2 — `COVERAGE_REPORT.md`, the human-readable third file of the
|
|
7
|
+
* 3-file contract (graph.html / COVERAGE_REPORT.md / graph.json). It states,
|
|
8
|
+
* auditably and in one place:
|
|
9
|
+
* - dynamic Proven % over the DENOMINATOR behaviors;
|
|
10
|
+
* - what the denominator is made of (Gate 3 composition line);
|
|
11
|
+
* - import-resolution coverage (the 7 resolver axes) — unresolved imports are
|
|
12
|
+
* test→source links the confirmer could not follow, so coverage can't be
|
|
13
|
+
* proven across them;
|
|
14
|
+
* - the graph schema version.
|
|
15
|
+
*/
|
|
16
|
+
const LAYER_ORDER = ["unit", "component", "integration", "api", "e2e", "manual", "unknown"];
|
|
17
|
+
// The 7 resolver axes that matter for static-association defensibility, in
|
|
18
|
+
// reading order; test→source is the gate axis.
|
|
19
|
+
const AXIS_LABELS = [
|
|
20
|
+
{ key: "test_to_source", label: "test → source (gate axis)" },
|
|
21
|
+
{ key: "test_to_test", label: "test → test" },
|
|
22
|
+
{ key: "test_internal", label: "test → internal" },
|
|
23
|
+
{ key: "source_to_source", label: "source → source" },
|
|
24
|
+
{ key: "barrel_terminal", label: "barrel → terminal" },
|
|
25
|
+
{ key: "workspace_package", label: "workspace package" },
|
|
26
|
+
{ key: "all_internal", label: "all internal" }
|
|
27
|
+
];
|
|
28
|
+
function emptyLedger() {
|
|
29
|
+
return { schema_version: LEDGER_SCHEMA_VERSION, records: [] };
|
|
30
|
+
}
|
|
31
|
+
export function renderCoverageReport(graph, ledger = emptyLedger()) {
|
|
32
|
+
const a = graph.analysis;
|
|
33
|
+
const lines = [];
|
|
34
|
+
lines.push("# Coverage Report");
|
|
35
|
+
lines.push("");
|
|
36
|
+
lines.push(`_Graph schema: ${graph.schema_version}_`);
|
|
37
|
+
lines.push(`_Generated: ${graph.updated_at || graph.created_at || "(unknown)"}_`);
|
|
38
|
+
lines.push("");
|
|
39
|
+
// ---- Public Proven + denominator: ONE atomic pair (shared with graph.html) ----
|
|
40
|
+
const { coverage: staticCov, denominator: comp } = resolveCoverage(graph);
|
|
41
|
+
const rtm = buildRtm(graph, ledger);
|
|
42
|
+
const s = rtm.summary;
|
|
43
|
+
lines.push("## Dynamic Proven");
|
|
44
|
+
lines.push("");
|
|
45
|
+
lines.push(`**${s.proven} of ${s.total} behaviors Proven (${s.coverage_pct}%)** — public Proven requires a closed dynamic targeted-proof ledger record. Static TESTED_BY/COVERS edges are Associated signal diagnostics, not proof.`);
|
|
46
|
+
const gate = a?.resolver_gate;
|
|
47
|
+
if (gate) {
|
|
48
|
+
lines.push("");
|
|
49
|
+
lines.push(gate.defensible
|
|
50
|
+
? `Defensible for static association diagnostics: import resolution on the gate axis (test → source) is ${gate.pct ?? "?"}% (≥ ${gate.threshold_pct}%).`
|
|
51
|
+
: `⚠ Static association diagnostics are not defensible repo-wide: test → source resolution is ${gate.pct ?? "?"}% (< ${gate.threshold_pct}%) or the scan was truncated.`);
|
|
52
|
+
}
|
|
53
|
+
lines.push("");
|
|
54
|
+
lines.push("Static assertion candidates by test layer (diagnostic only — never counted as Proven):");
|
|
55
|
+
lines.push("");
|
|
56
|
+
lines.push("| layer | static candidate behaviors |");
|
|
57
|
+
lines.push("| --- | ---: |");
|
|
58
|
+
for (const l of LAYER_ORDER)
|
|
59
|
+
lines.push(`| ${l} | ${staticCov.by_layer[l]} |`);
|
|
60
|
+
lines.push("");
|
|
61
|
+
if (staticCov.confirmed > 0 && staticCov.unknown_count > 0) {
|
|
62
|
+
lines.push(`${staticCov.unknown_count} static candidate behavior(s) (${staticCov.unknown_pct}%) have an undetermined test layer.`);
|
|
63
|
+
lines.push("");
|
|
64
|
+
}
|
|
65
|
+
const runtime = a?.runtime_coverage;
|
|
66
|
+
lines.push("## Runtime coverage");
|
|
67
|
+
lines.push("");
|
|
68
|
+
if (runtime) {
|
|
69
|
+
lines.push(`**${runtime.covered_symbols} of ${runtime.total_eligible_symbols} eligible symbols runtime-covered (${runtime.covered_pct}%)** — measured from local coverage-tool output, not name matching and not assertion-level proof.`);
|
|
70
|
+
lines.push("");
|
|
71
|
+
lines.push("| language | covered / eligible | covered % | symbols with spans |");
|
|
72
|
+
lines.push("| --- | ---: | ---: | ---: |");
|
|
73
|
+
for (const [language, row] of Object.entries(runtime.by_language).sort(([a], [b]) => a.localeCompare(b))) {
|
|
74
|
+
lines.push(`| ${language} | ${row.covered} / ${row.eligible} | ${row.covered_pct}% | ${row.symbols_with_spans} |`);
|
|
75
|
+
}
|
|
76
|
+
lines.push("");
|
|
77
|
+
lines.push("Artifacts:");
|
|
78
|
+
lines.push("");
|
|
79
|
+
lines.push("| artifact | format | files | covered ranges |");
|
|
80
|
+
lines.push("| --- | --- | ---: | ---: |");
|
|
81
|
+
for (const artifact of runtime.artifacts) {
|
|
82
|
+
lines.push(`| ${escapeMd(artifact.path)} | ${artifact.format} | ${artifact.files} | ${artifact.covered_ranges} |`);
|
|
83
|
+
}
|
|
84
|
+
lines.push("");
|
|
85
|
+
if (runtime.skipped_artifacts?.length) {
|
|
86
|
+
lines.push("Skipped artifacts:");
|
|
87
|
+
lines.push("");
|
|
88
|
+
lines.push("| artifact | format | reason |");
|
|
89
|
+
lines.push("| --- | --- | --- |");
|
|
90
|
+
for (const artifact of runtime.skipped_artifacts) {
|
|
91
|
+
lines.push(`| ${escapeMd(artifact.path)} | ${artifact.format} | ${escapeMd(artifact.reason)} |`);
|
|
92
|
+
}
|
|
93
|
+
lines.push("");
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
lines.push("No runtime coverage report was ingested. This report still shows dynamic Proven and static association diagnostics, but it does not claim actual executed coverage. Run `opro coverage .` for detected local coverage commands, or `opro analyze . --generate-coverage` where supported.");
|
|
98
|
+
lines.push("");
|
|
99
|
+
}
|
|
100
|
+
// ---- Denominator composition (Gate 3) — the SAME comp paired with cov above ----
|
|
101
|
+
const requirement = comp.requirement_template + comp.markdown_requirement;
|
|
102
|
+
lines.push("## Denominator");
|
|
103
|
+
lines.push("");
|
|
104
|
+
let compLine = `${comp.total} behaviors: ${comp.code_export} code_export, ${requirement} requirement`;
|
|
105
|
+
if (comp.excluded_test_inferred > 0)
|
|
106
|
+
compLine += `; ${comp.excluded_test_inferred} test-inferred excluded`;
|
|
107
|
+
if (comp.unattributed > 0)
|
|
108
|
+
compLine += `; ${comp.unattributed} unattributed`;
|
|
109
|
+
lines.push(compLine + ".");
|
|
110
|
+
lines.push("");
|
|
111
|
+
// Recomputed from nodes (comp), NOT read from persisted analysis — the
|
|
112
|
+
// disclosure is paired with the same graph the denominator was computed over.
|
|
113
|
+
// Complete accounting: total = counted + boilerplate + infra + generated + other (the
|
|
114
|
+
// buckets are disjoint and sum to every CodeSymbol node), so the "found" count
|
|
115
|
+
// is never a partial sum that drops non-callable consts / .d.ts / infra symbols.
|
|
116
|
+
const boilerplate = comp.excluded_boilerplate;
|
|
117
|
+
const infra = comp.excluded_infra;
|
|
118
|
+
const generated = comp.excluded_generated;
|
|
119
|
+
if (boilerplate > 0 || infra > 0 || generated > 0) {
|
|
120
|
+
const total = comp.code_symbols_total;
|
|
121
|
+
const other = Math.max(0, total - comp.code_export - boilerplate - infra - generated);
|
|
122
|
+
let line = `${total} code symbols found: ${comp.code_export} counted as behaviors`;
|
|
123
|
+
if (boilerplate > 0)
|
|
124
|
+
line += `, ${boilerplate} excluded as trivial accessors (getters/setters, toString/equals/hashCode, __repr__/__str__)`;
|
|
125
|
+
if (infra > 0)
|
|
126
|
+
line += `, ${infra} excluded as CI/test-infra (.github, e2e/playwright/cypress, fixtures/mocks)`;
|
|
127
|
+
if (generated > 0)
|
|
128
|
+
line += `, ${generated} excluded as generated code (Code generated ... DO NOT EDIT)`;
|
|
129
|
+
if (other > 0)
|
|
130
|
+
line += `, ${other} excluded as non-behavioral (type declarations, non-callable consts)`;
|
|
131
|
+
lines.push(line + ". Excluded symbols stay in the graph but carry no testable behavior, so counting them would distort coverage.");
|
|
132
|
+
lines.push("");
|
|
133
|
+
}
|
|
134
|
+
lines.push("Test-inferred behaviors (guessed from test describe-names) are inventoried but never counted — a test cannot prove its own requirement.");
|
|
135
|
+
lines.push("");
|
|
136
|
+
const riskGaps = rankRiskGaps(graph, { limit: 20 });
|
|
137
|
+
if (riskGaps.length > 0) {
|
|
138
|
+
lines.push("## Top gaps by risk");
|
|
139
|
+
lines.push("");
|
|
140
|
+
lines.push("Unconfirmed code behaviors ranked by structural impact and recent change activity. This is a prioritization list, not coverage proof.");
|
|
141
|
+
lines.push("");
|
|
142
|
+
lines.push("Risk score = Probability(1-10) × Impact(1-10) × DetectionDifficulty(1|5|10).");
|
|
143
|
+
lines.push("");
|
|
144
|
+
lines.push("| rank | behavior | file | risk | why |");
|
|
145
|
+
lines.push("| ---: | --- | --- | ---: | --- |");
|
|
146
|
+
riskGaps.forEach((g, i) => {
|
|
147
|
+
lines.push(`| ${i + 1} | ${escapeMd(g.title)} | ${escapeMd(g.file)} | ${g.risk_score} | ${escapeMd(g.reasons.join("; "))} |`);
|
|
148
|
+
});
|
|
149
|
+
lines.push("");
|
|
150
|
+
}
|
|
151
|
+
// ---- Import resolution (the 7 axes) ----
|
|
152
|
+
lines.push("## Import resolution");
|
|
153
|
+
lines.push("");
|
|
154
|
+
if (a?.resolver_metrics) {
|
|
155
|
+
const rm = a.resolver_metrics;
|
|
156
|
+
lines.push("Unresolved imports are links the confirmer cannot follow, so coverage can't be proven across them.");
|
|
157
|
+
lines.push("");
|
|
158
|
+
lines.push("| axis | resolved / eligible | resolved % | unresolved % |");
|
|
159
|
+
lines.push("| --- | ---: | ---: | ---: |");
|
|
160
|
+
for (const { key, label } of AXIS_LABELS) {
|
|
161
|
+
const m = rm[key];
|
|
162
|
+
if (!m)
|
|
163
|
+
continue;
|
|
164
|
+
const unresolved = m.n > 0 ? round1(100 - (m.resolved / m.n) * 100) : 0;
|
|
165
|
+
lines.push(`| ${label} | ${m.resolved} / ${m.n} | ${m.pct}% | ${unresolved}% |`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
else {
|
|
169
|
+
lines.push("_No TypeScript/JavaScript imports were scanned, so resolver metrics are not available._");
|
|
170
|
+
}
|
|
171
|
+
lines.push("");
|
|
172
|
+
const budget = a?.not_analyzed_due_to_budget;
|
|
173
|
+
if (a?.symbol_cap_hit || a?.files_cap_hit || budget) {
|
|
174
|
+
lines.push("## Caveats");
|
|
175
|
+
lines.push("");
|
|
176
|
+
if (budget) {
|
|
177
|
+
lines.push(`- ⚠ PARTIAL SCAN: the analyze budget (${budget.budget_ms}ms) stopped after ${budget.elapsed_ms}ms with ${budget.files_not_analyzed} file(s) NOT analyzed. The denominator is incomplete, so the Dynamic Proven headline is a partial view. Raise \`ORANGEPRO_MAX_ANALYZE_MS\` or scope with \`--base\`.`);
|
|
178
|
+
}
|
|
179
|
+
if (a?.symbol_cap_hit)
|
|
180
|
+
lines.push("- Symbol cap hit: some code symbols were omitted; the denominator understates the repo, so Dynamic Proven reads low. Raise `ORANGEPRO_MAX_SYMBOLS`.");
|
|
181
|
+
if (a?.files_cap_hit)
|
|
182
|
+
lines.push("- File cap hit: not all files were scanned; coverage measured an unknown fraction of the repo. Raise `ORANGEPRO_MAX_FILES`.");
|
|
183
|
+
lines.push("");
|
|
184
|
+
}
|
|
185
|
+
return lines.join("\n");
|
|
186
|
+
}
|
|
187
|
+
function round1(n) {
|
|
188
|
+
return Math.round(n * 10) / 10;
|
|
189
|
+
}
|
|
190
|
+
function escapeMd(s) {
|
|
191
|
+
return s.replace(/\|/g, "\\|").replace(/\n/g, " ");
|
|
192
|
+
}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import { systemClock } from "../util/time.js";
|
|
2
|
+
import { EVIDENCE_PACK_SCHEMA_VERSION } from "./schema.js";
|
|
3
|
+
import { redactSecrets } from "../util/redact.js";
|
|
4
|
+
/**
|
|
5
|
+
* A generated test's `body` is model output that can paraphrase the fed test
|
|
6
|
+
* source. The exported pack is the metadata-only IP boundary, so by default the
|
|
7
|
+
* body is NOT embedded — it stays in the local workspace graph. It is included
|
|
8
|
+
* only when `include_generated_bodies` is explicitly set. (This is distinct from
|
|
9
|
+
* the deferred raw-source-snippets policy, which never crosses the boundary.)
|
|
10
|
+
*/
|
|
11
|
+
const BODY_OMITTED = "[omitted: metadata-only export — generated test body kept local; set include_generated_bodies to embed]";
|
|
12
|
+
/**
|
|
13
|
+
* Build a strict, promotion-ready evidence pack from a local graph + score.
|
|
14
|
+
*
|
|
15
|
+
* The pack is the IP boundary: only metadata, provenance, and grounding cross
|
|
16
|
+
* it. Prompt text, scoring weights, traversal traces, and raw source NEVER do.
|
|
17
|
+
* The strict `evidencePackSchema` enforces this — extra keys would be rejected —
|
|
18
|
+
* so this builder deliberately constructs minimal, shape-exact objects.
|
|
19
|
+
*
|
|
20
|
+
* Everything is built immutably: input graph/nodes/edges are never mutated.
|
|
21
|
+
*/
|
|
22
|
+
export function buildPack(graph, score, opts, clock) {
|
|
23
|
+
const now = (clock ?? systemClock)();
|
|
24
|
+
const pack = {
|
|
25
|
+
schema_version: EVIDENCE_PACK_SCHEMA_VERSION,
|
|
26
|
+
created_at: now,
|
|
27
|
+
workspace: {
|
|
28
|
+
name: graph.workspace.name,
|
|
29
|
+
root_hash: graph.workspace.root_hash,
|
|
30
|
+
source_upload_policy: graph.workspace.source_upload_policy
|
|
31
|
+
},
|
|
32
|
+
sources: graph.sources.map(toSource),
|
|
33
|
+
entities: graph.nodes.map(toEntity),
|
|
34
|
+
// IMPORTS edges are local resolver substrate (they feed coverage metrics and
|
|
35
|
+
// the structural confirmer), not promotion evidence: at monorepo scale they
|
|
36
|
+
// would dominate the pack for zero consumer value, so they stay local-only.
|
|
37
|
+
relationships: graph.edges.filter(isPackRelationshipEdge).map(toRelationship),
|
|
38
|
+
candidate_relationships: graph.candidate_edges.map(toCandidateRelationship),
|
|
39
|
+
quality_score: {
|
|
40
|
+
overall: score.overall,
|
|
41
|
+
band: score.band,
|
|
42
|
+
breakdown: {
|
|
43
|
+
behavior_anchors: score.breakdown.behavior_anchors,
|
|
44
|
+
acceptance_criteria: score.breakdown.acceptance_criteria,
|
|
45
|
+
provenance: score.breakdown.provenance,
|
|
46
|
+
interface_mapping: score.breakdown.interface_mapping,
|
|
47
|
+
validation_evidence: score.breakdown.validation_evidence,
|
|
48
|
+
known_regressions: score.breakdown.known_regressions
|
|
49
|
+
},
|
|
50
|
+
missing_evidence: [...score.missing_evidence]
|
|
51
|
+
},
|
|
52
|
+
generation_runs: graph.generation_runs.map((run) => toGenerationRun(run, graph.generated_tests, opts?.include_generated_bodies ?? false))
|
|
53
|
+
};
|
|
54
|
+
return pack;
|
|
55
|
+
}
|
|
56
|
+
function toSource(source) {
|
|
57
|
+
return {
|
|
58
|
+
source_scope_id: source.source_scope_id,
|
|
59
|
+
source_system: source.source_system,
|
|
60
|
+
source_type: source.source_type,
|
|
61
|
+
display_name: source.display_name,
|
|
62
|
+
content_hash: source.content_hash,
|
|
63
|
+
metadata: { ...source.metadata }
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/** Map a node kind to the pack's snake_case entity_type. */
|
|
67
|
+
function entityTypeOf(kind) {
|
|
68
|
+
switch (kind) {
|
|
69
|
+
case "AcceptanceCriterion":
|
|
70
|
+
return "acceptance_criterion";
|
|
71
|
+
case "CodeSymbol":
|
|
72
|
+
return "code_symbol";
|
|
73
|
+
case "ConfigFile":
|
|
74
|
+
return "config_file";
|
|
75
|
+
case "TestCase":
|
|
76
|
+
return "test_case";
|
|
77
|
+
case "UserFlow":
|
|
78
|
+
return "user_flow";
|
|
79
|
+
case "TenantStub":
|
|
80
|
+
return "tenant_stub";
|
|
81
|
+
case "BusinessRule":
|
|
82
|
+
return "business_rule";
|
|
83
|
+
case "SourceScope":
|
|
84
|
+
return "source_scope";
|
|
85
|
+
case "EvidenceItem":
|
|
86
|
+
return "evidence_item";
|
|
87
|
+
default:
|
|
88
|
+
return kind.toLowerCase();
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Defense-in-depth secret scrub for any string crossing the pack boundary. Node
|
|
93
|
+
* properties are reviewed metadata (secret-redacted at ingest); re-redacting here
|
|
94
|
+
* makes the boundary's "no secrets" guarantee hold regardless of how a property
|
|
95
|
+
* was populated. Structure and non-secret text (e.g. acceptance criteria) are
|
|
96
|
+
* preserved — only secret patterns are replaced.
|
|
97
|
+
*/
|
|
98
|
+
function redactDeep(value) {
|
|
99
|
+
if (typeof value === "string")
|
|
100
|
+
return redactSecrets(value);
|
|
101
|
+
if (Array.isArray(value))
|
|
102
|
+
return value.map(redactDeep);
|
|
103
|
+
if (value && typeof value === "object") {
|
|
104
|
+
return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, redactDeep(v)]));
|
|
105
|
+
}
|
|
106
|
+
return value;
|
|
107
|
+
}
|
|
108
|
+
function toEntity(node) {
|
|
109
|
+
const properties = redactDeep({ ...node.properties });
|
|
110
|
+
if (node.content_hash !== undefined) {
|
|
111
|
+
properties.content_hash = node.content_hash;
|
|
112
|
+
}
|
|
113
|
+
return {
|
|
114
|
+
external_id: node.external_id,
|
|
115
|
+
entity_type: entityTypeOf(node.kind),
|
|
116
|
+
review_status: node.review_status,
|
|
117
|
+
confidence: node.confidence,
|
|
118
|
+
properties,
|
|
119
|
+
provenance: toProvenance(node.provenance)
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
/** Rebuild provenance into the strict shape, dropping any extra keys. */
|
|
123
|
+
function toProvenance(provenance) {
|
|
124
|
+
const out = {
|
|
125
|
+
source_scope_id: provenance.source_scope_id
|
|
126
|
+
};
|
|
127
|
+
if (provenance.source_ref !== undefined)
|
|
128
|
+
out.source_ref = provenance.source_ref;
|
|
129
|
+
if (provenance.quote_hash !== undefined)
|
|
130
|
+
out.quote_hash = provenance.quote_hash;
|
|
131
|
+
if (provenance.detector !== undefined)
|
|
132
|
+
out.detector = provenance.detector;
|
|
133
|
+
return out;
|
|
134
|
+
}
|
|
135
|
+
function isPackRelationshipEdge(edge) {
|
|
136
|
+
return edge.relationship_type !== "IMPORTS" && edge.evidence_strength !== "framework-derived";
|
|
137
|
+
}
|
|
138
|
+
function proofReviewStatus(status) {
|
|
139
|
+
if (status === "ai_suggested") {
|
|
140
|
+
throw new Error("AI-suggested review status is not valid on hard/reviewed evidence-pack relationships.");
|
|
141
|
+
}
|
|
142
|
+
return status;
|
|
143
|
+
}
|
|
144
|
+
function toRelationship(edge) {
|
|
145
|
+
const rel = {
|
|
146
|
+
from_external_id: edge.from_external_id,
|
|
147
|
+
to_external_id: edge.to_external_id,
|
|
148
|
+
relationship_type: edge.relationship_type,
|
|
149
|
+
evidence_strength: edge.evidence_strength,
|
|
150
|
+
review_status: proofReviewStatus(edge.review_status),
|
|
151
|
+
provenance: toProvenance(edge.provenance)
|
|
152
|
+
};
|
|
153
|
+
if (edge.confidence !== undefined)
|
|
154
|
+
rel.confidence = edge.confidence;
|
|
155
|
+
return rel;
|
|
156
|
+
}
|
|
157
|
+
function toCandidateRelationship(edge) {
|
|
158
|
+
return {
|
|
159
|
+
from_external_id: edge.from_external_id,
|
|
160
|
+
to_external_id: edge.to_external_id,
|
|
161
|
+
relationship_type: edge.relationship_type,
|
|
162
|
+
evidence_strength: edge.evidence_strength,
|
|
163
|
+
// Reasons can carry repo-derived text (e.g. a resolved import specifier):
|
|
164
|
+
// redact for defense-in-depth parity with toEntity's redactDeep.
|
|
165
|
+
reason: redactSecrets(edge.reason),
|
|
166
|
+
confidence: edge.confidence
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
function toGenerationRun(run, allTests, includeBody) {
|
|
170
|
+
const tests = allTests.filter((t) => t.run_id === run.run_id).map((t) => toGeneratedTest(t, includeBody));
|
|
171
|
+
return {
|
|
172
|
+
run_id: run.run_id,
|
|
173
|
+
model_provider: run.model_provider,
|
|
174
|
+
model_name: run.model_name,
|
|
175
|
+
input_mode: run.input_mode,
|
|
176
|
+
prompt_version: run.prompt_version,
|
|
177
|
+
generated_tests: tests
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
function toGeneratedTest(test, includeBody) {
|
|
181
|
+
return {
|
|
182
|
+
title: test.title,
|
|
183
|
+
test_type: test.test_type,
|
|
184
|
+
framework_hint: test.framework_hint,
|
|
185
|
+
body: includeBody ? test.body : BODY_OMITTED,
|
|
186
|
+
bucket: test.bucket,
|
|
187
|
+
prompt_version: test.prompt_version,
|
|
188
|
+
grounding: {
|
|
189
|
+
entity_ids: [...test.grounding.entity_ids],
|
|
190
|
+
source_refs: [...test.grounding.source_refs],
|
|
191
|
+
weak_relationships_used: [...test.grounding.weak_relationships_used]
|
|
192
|
+
},
|
|
193
|
+
weak_evidence_used: test.weak_evidence_used
|
|
194
|
+
};
|
|
195
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
/**
|
|
3
|
+
* Evidence pack schema (`orangepro.local_evidence_pack.v0`).
|
|
4
|
+
*
|
|
5
|
+
* Objects are `.strict()` on purpose: unknown keys FAIL validation. This is the
|
|
6
|
+
* IP boundary in code form — prompt text, prompt templates, scoring weights,
|
|
7
|
+
* ranking traces, traversal traces, or raw source would all be rejected, so the
|
|
8
|
+
* pack stays "promotion-ready but not reverse-engineering-ready".
|
|
9
|
+
*/
|
|
10
|
+
export const EVIDENCE_PACK_SCHEMA_VERSION = "orangepro.local_evidence_pack.v0";
|
|
11
|
+
const provenanceSchema = z
|
|
12
|
+
.object({
|
|
13
|
+
source_scope_id: z.string().min(1),
|
|
14
|
+
source_ref: z.string().optional(),
|
|
15
|
+
quote_hash: z.string().optional(),
|
|
16
|
+
detector: z.string().optional()
|
|
17
|
+
})
|
|
18
|
+
.strict();
|
|
19
|
+
const sourceSchema = z
|
|
20
|
+
.object({
|
|
21
|
+
source_scope_id: z.string().min(1),
|
|
22
|
+
source_system: z.string().min(1),
|
|
23
|
+
source_type: z.string().min(1),
|
|
24
|
+
display_name: z.string(),
|
|
25
|
+
content_hash: z.string(),
|
|
26
|
+
metadata: z.record(z.unknown())
|
|
27
|
+
})
|
|
28
|
+
.strict();
|
|
29
|
+
const entitySchema = z
|
|
30
|
+
.object({
|
|
31
|
+
external_id: z.string().min(1),
|
|
32
|
+
entity_type: z.string().min(1),
|
|
33
|
+
review_status: z.enum(["local_reviewed", "auto_detected", "inferred", "ai_suggested"]),
|
|
34
|
+
confidence: z.number().min(0).max(1),
|
|
35
|
+
properties: z.record(z.unknown()),
|
|
36
|
+
provenance: provenanceSchema
|
|
37
|
+
})
|
|
38
|
+
.strict();
|
|
39
|
+
const relationshipSchema = z
|
|
40
|
+
.object({
|
|
41
|
+
from_external_id: z.string().min(1),
|
|
42
|
+
to_external_id: z.string().min(1),
|
|
43
|
+
relationship_type: z.string().min(1),
|
|
44
|
+
evidence_strength: z.enum(["hard", "reviewed"]),
|
|
45
|
+
review_status: z.enum(["local_reviewed", "auto_detected", "inferred"]),
|
|
46
|
+
provenance: provenanceSchema,
|
|
47
|
+
confidence: z.number().min(0).max(1).optional()
|
|
48
|
+
})
|
|
49
|
+
.strict();
|
|
50
|
+
const candidateRelationshipSchema = z
|
|
51
|
+
.object({
|
|
52
|
+
from_external_id: z.string().min(1),
|
|
53
|
+
to_external_id: z.string().min(1),
|
|
54
|
+
relationship_type: z.string().min(1),
|
|
55
|
+
evidence_strength: z.enum(["candidate", "weak"]),
|
|
56
|
+
reason: z.string(),
|
|
57
|
+
confidence: z.number().min(0).max(1)
|
|
58
|
+
})
|
|
59
|
+
.strict();
|
|
60
|
+
const qualityScoreSchema = z
|
|
61
|
+
.object({
|
|
62
|
+
overall: z.number().min(0).max(100),
|
|
63
|
+
band: z.enum(["thin", "usable", "good", "strong"]),
|
|
64
|
+
breakdown: z
|
|
65
|
+
.object({
|
|
66
|
+
behavior_anchors: z.number().min(0).max(1),
|
|
67
|
+
acceptance_criteria: z.number().min(0).max(1),
|
|
68
|
+
provenance: z.number().min(0).max(1),
|
|
69
|
+
interface_mapping: z.number().min(0).max(1),
|
|
70
|
+
validation_evidence: z.number().min(0).max(1),
|
|
71
|
+
known_regressions: z.number().min(0).max(1)
|
|
72
|
+
})
|
|
73
|
+
.strict(),
|
|
74
|
+
missing_evidence: z.array(z.string())
|
|
75
|
+
})
|
|
76
|
+
.strict();
|
|
77
|
+
const groundingSchema = z
|
|
78
|
+
.object({
|
|
79
|
+
entity_ids: z.array(z.string()),
|
|
80
|
+
source_refs: z.array(z.string()),
|
|
81
|
+
weak_relationships_used: z.array(z.string())
|
|
82
|
+
})
|
|
83
|
+
.strict();
|
|
84
|
+
const generatedTestSchema = z
|
|
85
|
+
.object({
|
|
86
|
+
title: z.string().min(1),
|
|
87
|
+
test_type: z.string().min(1),
|
|
88
|
+
framework_hint: z.string(),
|
|
89
|
+
body: z.string(),
|
|
90
|
+
grounding: groundingSchema,
|
|
91
|
+
weak_evidence_used: z.boolean(),
|
|
92
|
+
// Optional per-artifact prompt lineage; old packs only carry this on the run.
|
|
93
|
+
prompt_version: z.string().optional(),
|
|
94
|
+
// Optional local scenario bucket (backward-compatible: old packs omit it).
|
|
95
|
+
bucket: z
|
|
96
|
+
.enum(["happy_path", "validation_error", "edge_case", "integration_flow", "security_privacy", "regression"])
|
|
97
|
+
.optional()
|
|
98
|
+
})
|
|
99
|
+
.strict();
|
|
100
|
+
const generationRunSchema = z
|
|
101
|
+
.object({
|
|
102
|
+
run_id: z.string().min(1),
|
|
103
|
+
model_provider: z.string(),
|
|
104
|
+
model_name: z.string(),
|
|
105
|
+
input_mode: z.enum(["graph_grounded", "raw_prompt"]),
|
|
106
|
+
prompt_version: z.string(),
|
|
107
|
+
generated_tests: z.array(generatedTestSchema)
|
|
108
|
+
})
|
|
109
|
+
.strict();
|
|
110
|
+
export const evidencePackSchema = z
|
|
111
|
+
.object({
|
|
112
|
+
schema_version: z.literal(EVIDENCE_PACK_SCHEMA_VERSION),
|
|
113
|
+
created_at: z.string(),
|
|
114
|
+
workspace: z
|
|
115
|
+
.object({
|
|
116
|
+
name: z.string(),
|
|
117
|
+
root_hash: z.string(),
|
|
118
|
+
source_upload_policy: z.enum(["metadata_only", "include_sources"])
|
|
119
|
+
})
|
|
120
|
+
.strict(),
|
|
121
|
+
sources: z.array(sourceSchema),
|
|
122
|
+
entities: z.array(entitySchema),
|
|
123
|
+
relationships: z.array(relationshipSchema),
|
|
124
|
+
candidate_relationships: z.array(candidateRelationshipSchema),
|
|
125
|
+
quality_score: qualityScoreSchema,
|
|
126
|
+
generation_runs: z.array(generationRunSchema)
|
|
127
|
+
})
|
|
128
|
+
.strict();
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Render an evidence pack as a human-readable Markdown summary.
|
|
3
|
+
*
|
|
4
|
+
* This is a trust artifact: it explains *what* evidence backs the pack and the
|
|
5
|
+
* generated tests WITHOUT exposing how anything was scored or prompted. It
|
|
6
|
+
* deliberately omits scoring weights, prompt text, and traversal traces — only
|
|
7
|
+
* the metadata already present in the pack is rendered.
|
|
8
|
+
*/
|
|
9
|
+
export function packToMarkdown(pack, analysis) {
|
|
10
|
+
const lines = [];
|
|
11
|
+
lines.push("# OrangePro Local Evidence Pack");
|
|
12
|
+
lines.push("");
|
|
13
|
+
lines.push(`- Created at: ${pack.created_at}`);
|
|
14
|
+
lines.push(`- Workspace: ${pack.workspace.name}`);
|
|
15
|
+
lines.push(`- Source upload policy: ${pack.workspace.source_upload_policy}`);
|
|
16
|
+
lines.push("");
|
|
17
|
+
if (analysis) {
|
|
18
|
+
// Phase 5.3: COVERAGE_REPORT.md is the SINGLE source of coverage truth
|
|
19
|
+
// (dynamic Proven %, denominator composition, resolver metrics). The pack defers
|
|
20
|
+
// to it and only surfaces completeness caveats that affect THIS pack — no
|
|
21
|
+
// competing coverage numbers live here.
|
|
22
|
+
lines.push("## Coverage");
|
|
23
|
+
lines.push("");
|
|
24
|
+
lines.push("Dynamic Proven, static association diagnostics, the denominator composition, and import-resolution metrics live in `COVERAGE_REPORT.md` (the 3-file contract: graph.html / COVERAGE_REPORT.md / graph.json). This pack does not restate them.");
|
|
25
|
+
if (analysis.flows_truncated > 0) {
|
|
26
|
+
lines.push(`- ⚠ Truncated: ${analysis.flows_truncated} test file(s) exceeded the inferred-behavior cap and are not represented. Raise ORANGEPRO_MAX_FLOWS to include them.`);
|
|
27
|
+
}
|
|
28
|
+
if (analysis.symbol_cap_hit) {
|
|
29
|
+
lines.push("- ⚠ Code-symbol extraction cap was reached; some symbols are omitted (the denominator understates the repo). Raise ORANGEPRO_MAX_SYMBOLS.");
|
|
30
|
+
}
|
|
31
|
+
if (analysis.not_analyzed_due_to_budget) {
|
|
32
|
+
const b = analysis.not_analyzed_due_to_budget;
|
|
33
|
+
lines.push(`- ⚠ PARTIAL SCAN: the analyze budget (${b.budget_ms}ms) stopped with ${b.files_not_analyzed} file(s) NOT analyzed — this pack came from a partial graph; coverage is a floor, not complete. Raise ORANGEPRO_MAX_ANALYZE_MS or scope with --base.`);
|
|
34
|
+
}
|
|
35
|
+
lines.push("");
|
|
36
|
+
}
|
|
37
|
+
lines.push("## Quality score");
|
|
38
|
+
lines.push("");
|
|
39
|
+
lines.push(`Overall: ${pack.quality_score.overall}/100 (band: ${pack.quality_score.band})`);
|
|
40
|
+
lines.push("");
|
|
41
|
+
for (const [dimension, value] of Object.entries(pack.quality_score.breakdown)) {
|
|
42
|
+
lines.push(`- ${labelDimension(dimension)}: ${formatFraction(value)}`);
|
|
43
|
+
}
|
|
44
|
+
lines.push("");
|
|
45
|
+
lines.push("## Why the score is not higher");
|
|
46
|
+
lines.push("");
|
|
47
|
+
if (pack.quality_score.missing_evidence.length === 0) {
|
|
48
|
+
lines.push("- No outstanding evidence gaps reported.");
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
for (const item of pack.quality_score.missing_evidence) {
|
|
52
|
+
lines.push(`- ${item}`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
lines.push("");
|
|
56
|
+
lines.push("## Sources");
|
|
57
|
+
lines.push("");
|
|
58
|
+
if (pack.sources.length === 0) {
|
|
59
|
+
lines.push("- No sources recorded.");
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
for (const source of pack.sources) {
|
|
63
|
+
lines.push(`- ${source.display_name} (${source.source_system} / ${source.source_type})`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
lines.push("");
|
|
67
|
+
lines.push("## Entities");
|
|
68
|
+
lines.push("");
|
|
69
|
+
lines.push(`Total entities: ${pack.entities.length}`);
|
|
70
|
+
lines.push("");
|
|
71
|
+
const counts = countByEntityType(pack.entities);
|
|
72
|
+
if (counts.length === 0) {
|
|
73
|
+
lines.push("- No entities recorded.");
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
for (const { entity_type, count } of counts) {
|
|
77
|
+
lines.push(`- ${entity_type}: ${count}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
lines.push("");
|
|
81
|
+
lines.push("## Generated tests");
|
|
82
|
+
lines.push("");
|
|
83
|
+
const tests = pack.generation_runs.flatMap((run) => run.generated_tests);
|
|
84
|
+
if (tests.length === 0) {
|
|
85
|
+
lines.push("- No generated tests in this pack.");
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
for (const test of tests) {
|
|
89
|
+
lines.push(`### ${test.title}`);
|
|
90
|
+
lines.push("");
|
|
91
|
+
if (test.bucket)
|
|
92
|
+
lines.push(`- Local bucket: ${test.bucket}`);
|
|
93
|
+
lines.push(`- Test type: ${test.test_type}`);
|
|
94
|
+
lines.push(`- Framework hint: ${test.framework_hint || "(none)"}`);
|
|
95
|
+
lines.push(`- Grounded by entities: ${formatList(test.grounding.entity_ids)}`);
|
|
96
|
+
lines.push(`- Source refs: ${formatList(test.grounding.source_refs)}`);
|
|
97
|
+
lines.push(`- Weak/candidate evidence used: ${test.weak_evidence_used ? "yes" : "no"}`);
|
|
98
|
+
lines.push("");
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
lines.push("## Provenance");
|
|
102
|
+
lines.push("");
|
|
103
|
+
lines.push("Every entity and relationship in this pack carries provenance (source scope, optional source reference and quote hash). The pack contains metadata, provenance, and grounding only — no raw source code, prompts, or scoring internals.");
|
|
104
|
+
lines.push("");
|
|
105
|
+
return lines.join("\n");
|
|
106
|
+
}
|
|
107
|
+
function labelDimension(dimension) {
|
|
108
|
+
return dimension
|
|
109
|
+
.split("_")
|
|
110
|
+
.map((part) => (part.length > 0 ? part[0].toUpperCase() + part.slice(1) : part))
|
|
111
|
+
.join(" ");
|
|
112
|
+
}
|
|
113
|
+
function formatFraction(value) {
|
|
114
|
+
return value.toFixed(2);
|
|
115
|
+
}
|
|
116
|
+
function formatList(values) {
|
|
117
|
+
return values.length > 0 ? values.join(", ") : "(none)";
|
|
118
|
+
}
|
|
119
|
+
function countByEntityType(entities) {
|
|
120
|
+
const counts = new Map();
|
|
121
|
+
for (const entity of entities) {
|
|
122
|
+
counts.set(entity.entity_type, (counts.get(entity.entity_type) ?? 0) + 1);
|
|
123
|
+
}
|
|
124
|
+
return [...counts.entries()]
|
|
125
|
+
.map(([entity_type, count]) => ({ entity_type, count }))
|
|
126
|
+
.sort((a, b) => a.entity_type.localeCompare(b.entity_type));
|
|
127
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { evidencePackSchema } from "./schema.js";
|
|
2
|
+
/** Validate an unknown object against the strict evidence-pack schema. */
|
|
3
|
+
export function validatePack(obj) {
|
|
4
|
+
const result = evidencePackSchema.safeParse(obj);
|
|
5
|
+
if (result.success) {
|
|
6
|
+
return { valid: true, errors: [] };
|
|
7
|
+
}
|
|
8
|
+
return { valid: false, errors: result.error.issues.map(formatIssue) };
|
|
9
|
+
}
|
|
10
|
+
/** Parse JSON text then validate; reports a JSON error rather than throwing. */
|
|
11
|
+
export function validatePackJson(text) {
|
|
12
|
+
let parsed;
|
|
13
|
+
try {
|
|
14
|
+
parsed = JSON.parse(text);
|
|
15
|
+
}
|
|
16
|
+
catch (error) {
|
|
17
|
+
const message = error instanceof Error ? error.message : "invalid JSON";
|
|
18
|
+
return { valid: false, errors: [`<root>: invalid JSON (${message})`] };
|
|
19
|
+
}
|
|
20
|
+
return validatePack(parsed);
|
|
21
|
+
}
|
|
22
|
+
function formatIssue(issue) {
|
|
23
|
+
const path = issue.path.length > 0 ? issue.path.join(".") : "<root>";
|
|
24
|
+
return `${path}: ${issue.message}`;
|
|
25
|
+
}
|