@enwithai/product-check 0.1.0-alpha.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ATTRIBUTION.md +2 -2
- package/CHANGELOG.md +45 -3
- package/NOTICE.md +5 -3
- package/README.md +8 -6
- package/dashboard/dashboard.css +6 -0
- package/dist/blueprint/build.d.ts +3 -1
- package/dist/blueprint/build.js +41 -5
- package/dist/cli/run.js +96 -5
- package/dist/contract/contract.d.ts +4 -0
- package/dist/contract/contract.js +16 -0
- package/dist/dashboard/render.d.ts +5 -0
- package/dist/dashboard/render.js +107 -3
- package/dist/discovery/pops.d.ts +2 -0
- package/dist/discovery/pops.js +7 -5
- package/dist/importers/operational.d.ts +56 -0
- package/dist/importers/operational.js +217 -0
- package/dist/importers/reports.d.ts +24 -0
- package/dist/importers/reports.js +37 -1
- package/dist/inspectors/source/surface.d.ts +30 -0
- package/dist/inspectors/source/surface.js +529 -0
- package/dist/parse/yaml.d.ts +4 -0
- package/dist/parse/yaml.js +39 -0
- package/dist/reporting/markdown.js +17 -0
- package/dist/reporting/spec-markdown.d.ts +9 -0
- package/dist/reporting/spec-markdown.js +144 -0
- package/dist/run.d.ts +2 -0
- package/dist/run.js +52 -2
- package/dist/schema/registry.js +1 -0
- package/dist/scoring/score.js +1 -1
- package/dist/spec/build.d.ts +28 -0
- package/dist/spec/build.js +278 -0
- package/dist/types.d.ts +78 -2
- package/dist/types.js +6 -1
- package/dist/validation/catalog.d.ts +1 -1
- package/dist/validation/catalog.js +125 -2
- package/dist/validation/context.d.ts +6 -1
- package/dist/validation/import-evaluators.js +37 -3
- package/dist/validation/integrity.js +4 -0
- package/dist/validation/operational-evaluators.d.ts +3 -0
- package/dist/validation/operational-evaluators.js +153 -0
- package/dist/validation/source-evaluators.js +1 -1
- package/package.json +5 -3
- package/schemas/as-built-spec.schema.json +356 -0
- package/schemas/blueprint.schema.json +3 -0
- package/schemas/scan-contract.schema.json +20 -0
- package/schemas/scorecard.schema.json +1 -1
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { code, md } from "./escape.js";
|
|
2
|
+
const TAG = {
|
|
3
|
+
observed: "observed",
|
|
4
|
+
inferred: "inferred",
|
|
5
|
+
hypothesis: "needs confirmation",
|
|
6
|
+
};
|
|
7
|
+
/** Renders the as-built spec. */
|
|
8
|
+
export function renderAsBuiltSpec(spec) {
|
|
9
|
+
const byId = new Map(spec.surface.map((s) => [s.id, s]));
|
|
10
|
+
const loc = (s) => code(`${s.location.path}${s.location.line ? `:${s.location.line}` : ""}`);
|
|
11
|
+
const sources = (r) => {
|
|
12
|
+
const items = r.surface_ids.map((id) => byId.get(id)).filter((s) => s !== undefined);
|
|
13
|
+
if (items.length === 0)
|
|
14
|
+
return r.evidence_ids.length ? ` Evidence: ${r.evidence_ids.join(", ")}.` : "";
|
|
15
|
+
const shown = items.slice(0, 3).map(loc).join(", ");
|
|
16
|
+
return ` Source: ${shown}${items.length > 3 ? ` and ${items.length - 3} more` : ""}.`;
|
|
17
|
+
};
|
|
18
|
+
// Statements are escaped where they are built (src/spec/build.ts), so they are printed as they are.
|
|
19
|
+
const text = (r) => r.statement;
|
|
20
|
+
const line = (r) => `- **${r.id}**: ${text(r)} _(${TAG[r.certainty]})_${sources(r)}`;
|
|
21
|
+
const date = spec.generated_at.slice(0, 10);
|
|
22
|
+
const out = [
|
|
23
|
+
`# Feature Specification: ${md(spec.target.name)} (as built)`,
|
|
24
|
+
"",
|
|
25
|
+
"**Feature Branch**: `000-as-built`",
|
|
26
|
+
"",
|
|
27
|
+
`**Created**: ${date}`,
|
|
28
|
+
"",
|
|
29
|
+
"**Status**: Draft (reverse-engineered)",
|
|
30
|
+
"",
|
|
31
|
+
`**Input**: Repository ${code(spec.target.repository)}${spec.target.commit ? ` at commit ${code(spec.target.commit.slice(0, 12))}` : " (no git commit recorded)"}, read by Product Check ${code(spec.tool.version)}`,
|
|
32
|
+
"",
|
|
33
|
+
"> **Reverse-engineered, not authored.** Product Check derived this specification from the repository's structure",
|
|
34
|
+
"> without running it. _observed_ statements point at a file and line; _inferred_ ones interpret what was found;",
|
|
35
|
+
"> anything the code cannot tell us is marked `NEEDS CLARIFICATION`. Confirm or correct each statement before",
|
|
36
|
+
"> treating this as the product's specification.",
|
|
37
|
+
"",
|
|
38
|
+
"## User Scenarios & Testing _(mandatory)_",
|
|
39
|
+
"",
|
|
40
|
+
];
|
|
41
|
+
if (spec.stories.length === 0)
|
|
42
|
+
out.push("No pages or endpoints were recognised, so no user stories could be derived.", "");
|
|
43
|
+
for (const s of spec.stories) {
|
|
44
|
+
out.push(`### User Story ${s.id.slice(3)} - ${md(s.title)} (Priority: ${s.critical_journey ? "P1" : "P2"})`, "", `**Purpose**: [NEEDS CLARIFICATION: who uses the ${md(s.title)} area, and what outcome must it deliver?]`, "");
|
|
45
|
+
if (s.critical_journey)
|
|
46
|
+
out.push(`Declared critical journey: ${md(s.critical_journey)}.`, "");
|
|
47
|
+
out.push(`Covers ${s.requirement_ids.join(", ")}.`, "");
|
|
48
|
+
if (s.test_behaviours.length > 0) {
|
|
49
|
+
out.push("**Behaviour described by tests** _(inferred; convert to Given/When/Then once confirmed)_:", "");
|
|
50
|
+
for (const t of s.test_behaviours.slice(0, 15))
|
|
51
|
+
out.push(`- ${md(t)}`);
|
|
52
|
+
if (s.test_behaviours.length > 15)
|
|
53
|
+
out.push(`- … and ${s.test_behaviours.length - 15} more`);
|
|
54
|
+
out.push("");
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
const storyTests = new Set(spec.stories.flatMap((s) => s.surface_ids));
|
|
58
|
+
const loose = spec.surface.filter((x) => x.kind === "test" && !storyTests.has(x.id));
|
|
59
|
+
if (loose.length > 0) {
|
|
60
|
+
out.push("### Other behaviour described by tests _(inferred)_", "");
|
|
61
|
+
for (const t of loose.slice(0, 40))
|
|
62
|
+
out.push(`- ${md(t.name)} (${loc(t)})`);
|
|
63
|
+
if (loose.length > 40)
|
|
64
|
+
out.push(`- … and ${loose.length - 40} more`);
|
|
65
|
+
out.push("");
|
|
66
|
+
}
|
|
67
|
+
out.push("### Edge Cases", "");
|
|
68
|
+
if (spec.edge_cases.length === 0)
|
|
69
|
+
out.push("- [NEEDS CLARIFICATION: no edge or error cases could be found in the repository]");
|
|
70
|
+
for (const e of spec.edge_cases)
|
|
71
|
+
out.push(`- ${text(e)} _(${TAG[e.certainty]})_`);
|
|
72
|
+
out.push("", "## Requirements _(mandatory)_", "", "### Functional Requirements", "");
|
|
73
|
+
if (spec.requirements.length === 0)
|
|
74
|
+
out.push("- [NEEDS CLARIFICATION: no functional surface was recognised]");
|
|
75
|
+
for (const r of spec.requirements)
|
|
76
|
+
out.push(line(r));
|
|
77
|
+
out.push("", "### Non-Functional Requirements", "");
|
|
78
|
+
for (const r of spec.non_functional)
|
|
79
|
+
out.push(line(r));
|
|
80
|
+
const entities = new Map();
|
|
81
|
+
for (const e of spec.surface.filter((s) => s.kind === "entity"))
|
|
82
|
+
entities.set(e.name.toLowerCase(), [...(entities.get(e.name.toLowerCase()) ?? []), e]);
|
|
83
|
+
if (entities.size > 0) {
|
|
84
|
+
out.push("", "### Key Entities", "");
|
|
85
|
+
for (const group of entities.values()) {
|
|
86
|
+
const best = group.find((g) => g.detail.startsWith("fields:")) ?? group[0];
|
|
87
|
+
out.push(`- **${md(best.name)}**: ${md(best.detail)} (${group.map(loc).join(", ")})`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
out.push("", "## Success Criteria _(mandatory)_", "");
|
|
91
|
+
for (const r of spec.success_criteria)
|
|
92
|
+
out.push(line(r));
|
|
93
|
+
out.push("", "## Clarifications needed", "");
|
|
94
|
+
for (const c of spec.clarifications)
|
|
95
|
+
out.push(`- ${md(c)}`);
|
|
96
|
+
out.push("", "## Uncertainties", "");
|
|
97
|
+
for (const u of spec.uncertainties)
|
|
98
|
+
out.push(`- ${md(u)}`);
|
|
99
|
+
out.push("", "## Appendix: as-built surface", "");
|
|
100
|
+
const table = (title, kind, cols, head) => {
|
|
101
|
+
const rows = spec.surface.filter((s) => s.kind === kind);
|
|
102
|
+
if (rows.length === 0)
|
|
103
|
+
return;
|
|
104
|
+
out.push(`### ${title}`, "", `| ${head.join(" | ")} |`, `|${head.map(() => "---").join("|")}|`);
|
|
105
|
+
for (const r of rows)
|
|
106
|
+
out.push(`| ${cols(r).join(" | ")} |`);
|
|
107
|
+
out.push("");
|
|
108
|
+
};
|
|
109
|
+
table("Pages", "page", (s) => [code(s.name), md(s.detail), loc(s), s.auth_hint ? "yes (inferred)" : "—"], [
|
|
110
|
+
"Route",
|
|
111
|
+
"Kind",
|
|
112
|
+
"Source",
|
|
113
|
+
"Auth helper",
|
|
114
|
+
]);
|
|
115
|
+
table("Endpoints", "endpoint", (s) => [
|
|
116
|
+
code(s.method ?? "ANY"),
|
|
117
|
+
code(s.name),
|
|
118
|
+
md(s.detail),
|
|
119
|
+
loc(s),
|
|
120
|
+
s.auth_hint ? "yes (inferred)" : "—",
|
|
121
|
+
], ["Method", "Path", "Kind", "Source", "Auth helper"]);
|
|
122
|
+
table("Data entities", "entity", (s) => [code(s.name), md(s.detail), loc(s)], [
|
|
123
|
+
"Entity",
|
|
124
|
+
"Detail",
|
|
125
|
+
"Source",
|
|
126
|
+
]);
|
|
127
|
+
table("Integrations", "integration", (s) => [code(s.name), md(s.detail), loc(s)], [
|
|
128
|
+
"Package",
|
|
129
|
+
"Category",
|
|
130
|
+
"Declared in",
|
|
131
|
+
]);
|
|
132
|
+
table("Configuration (names only)", "config", (s) => [code(s.name), md(s.detail), loc(s)], [
|
|
133
|
+
"Variable",
|
|
134
|
+
"Use",
|
|
135
|
+
"First reference",
|
|
136
|
+
]);
|
|
137
|
+
table("Authentication boundaries", "auth-boundary", (s) => [code(s.name), md(s.detail), loc(s)], [
|
|
138
|
+
"Boundary",
|
|
139
|
+
"Detail",
|
|
140
|
+
"Source",
|
|
141
|
+
]);
|
|
142
|
+
out.push(`Generated ${spec.generated_at} · ${spec.surface.length} surface item(s) · ${spec.evidence.length} evidence item(s) in as-built-spec.json`, "");
|
|
143
|
+
return out.join("\n");
|
|
144
|
+
}
|
package/dist/run.d.ts
CHANGED
|
@@ -20,5 +20,7 @@ export interface RunOutput {
|
|
|
20
20
|
results: Results;
|
|
21
21
|
observations: ObservationSet | null;
|
|
22
22
|
}
|
|
23
|
+
/** Run ID: UTC timestamp plus a short hash of the contract and start time. */
|
|
24
|
+
export declare function makeRunId(started: Date, contract: ScanContract): string;
|
|
23
25
|
/** Runs one assessment. */
|
|
24
26
|
export declare function runAssessment(options: RunOptions): Promise<RunOutput>;
|
package/dist/run.js
CHANGED
|
@@ -10,13 +10,17 @@ import { isAbsolute, relative, sep } from "node:path";
|
|
|
10
10
|
import { snapshotRepo } from "./discovery/repo.js";
|
|
11
11
|
import { buildPops, popCoverage } from "./discovery/pops.js";
|
|
12
12
|
import { inputError } from "./errors.js";
|
|
13
|
+
import { readYaml } from "./parse/yaml.js";
|
|
13
14
|
import { EvidenceCollector } from "./evidence/collector.js";
|
|
14
15
|
import { harToObservations } from "./importers/har.js";
|
|
15
|
-
import {
|
|
16
|
+
import { parseAlertRules, parseOpenSlo, parseServiceGraph } from "./importers/operational.js";
|
|
17
|
+
import { parseArchitectureCheck, parseImpactCheck, parseNpmAudit, parseOutcomeCheck, parseSpecCheck, } from "./importers/reports.js";
|
|
16
18
|
import { parseTestResults } from "./importers/tests.js";
|
|
17
19
|
import { probe } from "./inspectors/http/fetch.js";
|
|
18
20
|
import { deriveFacts } from "./inspectors/source/facts.js";
|
|
21
|
+
import { deriveSurface } from "./inspectors/source/surface.js";
|
|
19
22
|
import { buildBlueprint } from "./blueprint/build.js";
|
|
23
|
+
import { buildAsBuiltSpec } from "./spec/build.js";
|
|
20
24
|
import { formatErrors, validateAgainst } from "./schema/registry.js";
|
|
21
25
|
import { computeScorecard, isOpen } from "./scoring/score.js";
|
|
22
26
|
import { computeVerdict } from "./scoring/verdict.js";
|
|
@@ -57,8 +61,25 @@ function importReport(path, label, parse) {
|
|
|
57
61
|
const { data, sha } = readJson(path);
|
|
58
62
|
return { data: parse(data, label(path)), source: label(path), sha256: sha };
|
|
59
63
|
}
|
|
64
|
+
/**
|
|
65
|
+
* Imports a set of YAML/JSON files through one parser. The source lists every file; the checksum covers all of them
|
|
66
|
+
* in order.
|
|
67
|
+
*/
|
|
68
|
+
function importYamlSet(paths, label, parse) {
|
|
69
|
+
if (paths.length === 0)
|
|
70
|
+
return null;
|
|
71
|
+
const files = paths.map((path) => {
|
|
72
|
+
const { text, sha } = readInput(path);
|
|
73
|
+
return { source: label(path), docs: readYaml(text, label(path)), sha };
|
|
74
|
+
});
|
|
75
|
+
return {
|
|
76
|
+
data: parse(files),
|
|
77
|
+
source: files.map((f) => f.source).join(", "),
|
|
78
|
+
sha256: files.length === 1 ? (files[0]?.sha ?? "") : sha256(files.map((f) => f.sha).join("\n")),
|
|
79
|
+
};
|
|
80
|
+
}
|
|
60
81
|
/** Run ID: UTC timestamp plus a short hash of the contract and start time. */
|
|
61
|
-
function makeRunId(started, contract) {
|
|
82
|
+
export function makeRunId(started, contract) {
|
|
62
83
|
const stamp = started
|
|
63
84
|
.toISOString()
|
|
64
85
|
.replace(/[-:]/g, "")
|
|
@@ -193,6 +214,14 @@ export async function runAssessment(options) {
|
|
|
193
214
|
const spec = aborted ? null : importReport(contract.imports.spec_check, label, parseSpecCheck);
|
|
194
215
|
const impact = aborted ? null : importReport(contract.imports.impact_check, label, parseImpactCheck);
|
|
195
216
|
const outcome = aborted ? null : importReport(contract.imports.outcome_check, label, parseOutcomeCheck);
|
|
217
|
+
const architecture = aborted
|
|
218
|
+
? null
|
|
219
|
+
: importReport(contract.imports.architecture_check, label, parseArchitectureCheck);
|
|
220
|
+
const openslo = aborted ? null : importYamlSet(contract.imports.openslo, label, parseOpenSlo);
|
|
221
|
+
const alertRules = aborted ? null : importYamlSet(contract.imports.alert_rules, label, parseAlertRules);
|
|
222
|
+
const serviceGraph = aborted
|
|
223
|
+
? null
|
|
224
|
+
: importReport(contract.imports.service_graph, label, parseServiceGraph);
|
|
196
225
|
phases.push({
|
|
197
226
|
phase: "TEST",
|
|
198
227
|
status: aborted ? "SKIPPED" : "COMPLETED",
|
|
@@ -204,6 +233,10 @@ export async function runAssessment(options) {
|
|
|
204
233
|
spec && "Spec Check report imported",
|
|
205
234
|
impact && "Impact Check report imported",
|
|
206
235
|
outcome && "Outcome Check report imported",
|
|
236
|
+
architecture && "Architecture Check report imported",
|
|
237
|
+
openslo && `OpenSLO imported (${openslo.data.slos.length} SLO(s))`,
|
|
238
|
+
alertRules && `alert rules imported (${alertRules.data.alerting.length} alerting rule(s))`,
|
|
239
|
+
serviceGraph && `service graph imported (${serviceGraph.data.edges.length} edge(s))`,
|
|
207
240
|
].filter((x) => typeof x === "string"),
|
|
208
241
|
],
|
|
209
242
|
});
|
|
@@ -220,6 +253,7 @@ export async function runAssessment(options) {
|
|
|
220
253
|
auditImported: audit !== null,
|
|
221
254
|
specImported: spec !== null,
|
|
222
255
|
outcomeImported: outcome !== null,
|
|
256
|
+
operationalImported: openslo !== null || alertRules !== null || serviceGraph !== null,
|
|
223
257
|
aiInScope,
|
|
224
258
|
tier,
|
|
225
259
|
now: startedIso,
|
|
@@ -237,6 +271,10 @@ export async function runAssessment(options) {
|
|
|
237
271
|
spec,
|
|
238
272
|
impact,
|
|
239
273
|
outcome,
|
|
274
|
+
architecture,
|
|
275
|
+
openslo,
|
|
276
|
+
alertRules,
|
|
277
|
+
serviceGraph,
|
|
240
278
|
baseline: options.baseline?.results ?? null,
|
|
241
279
|
aiInScope,
|
|
242
280
|
};
|
|
@@ -294,6 +332,17 @@ export async function runAssessment(options) {
|
|
|
294
332
|
}
|
|
295
333
|
findings.sort((a, b) => ["P0", "P1", "P2", "P3"].indexOf(a.severity) - ["P0", "P1", "P2", "P3"].indexOf(b.severity) ||
|
|
296
334
|
(a.id < b.id ? -1 : 1));
|
|
335
|
+
// ------------------------------------------------------------------ AS-BUILT SPEC (same snapshot, same evidence)
|
|
336
|
+
const asBuilt = repo && contract.target.repository
|
|
337
|
+
? buildAsBuiltSpec({
|
|
338
|
+
contract,
|
|
339
|
+
facts: repo,
|
|
340
|
+
surface: deriveSurface(repo),
|
|
341
|
+
ev,
|
|
342
|
+
repository: label(contract.target.repository.path),
|
|
343
|
+
generatedAt: startedIso,
|
|
344
|
+
})
|
|
345
|
+
: null;
|
|
297
346
|
// ------------------------------------------------------------------ BLUEPRINT
|
|
298
347
|
const blueprint = buildBlueprint({
|
|
299
348
|
contract,
|
|
@@ -307,6 +356,7 @@ export async function runAssessment(options) {
|
|
|
307
356
|
hasBaseline: options.baseline !== null,
|
|
308
357
|
testsImported: tests.length > 0,
|
|
309
358
|
aiInScope,
|
|
359
|
+
asBuilt,
|
|
310
360
|
});
|
|
311
361
|
const previous = options.baseline?.results.history.at(-1) ?? null;
|
|
312
362
|
const scorecard = computeScorecard(validations, findings, contract.scoring, previous);
|
package/dist/schema/registry.js
CHANGED
|
@@ -16,6 +16,7 @@ export const KIND_SCHEMA = {
|
|
|
16
16
|
blueprint: "blueprint.schema.json",
|
|
17
17
|
"release-verdict": "release-verdict.schema.json",
|
|
18
18
|
observations: "observations.schema.json",
|
|
19
|
+
"as-built-spec": "as-built-spec.schema.json",
|
|
19
20
|
};
|
|
20
21
|
let cached = null;
|
|
21
22
|
/** Lazily loads the bundled schemas. */
|
package/dist/scoring/score.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// SPDX-License-Identifier: MIT
|
|
2
2
|
// SPDX-FileCopyrightText: 2026 EnWithAI Technologies
|
|
3
3
|
/**
|
|
4
|
-
* Scorecard computation (rubric pc-rubric-0.
|
|
4
|
+
* Scorecard computation (rubric pc-rubric-0.2). Pure and deterministic, so `product-check validate` can recompute
|
|
5
5
|
* it from validations and findings and detect hand-edited figures.
|
|
6
6
|
*
|
|
7
7
|
* - Customer, Technology and Trust are scored separately; there is no composite score.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Builds an as-built specification from the discovered product surface. The spec states what the repository
|
|
3
|
+
* structurally provides (pages, endpoints, entities, integrations, configuration) with evidence for each statement,
|
|
4
|
+
* marks interpretations as inferred, and leaves everything the code cannot tell us (purpose, users, priorities,
|
|
5
|
+
* measurable targets) as explicit `[NEEDS CLARIFICATION: …]` markers for a person to resolve. No intent is invented.
|
|
6
|
+
*
|
|
7
|
+
* Statements are Markdown: every repository-derived string inside them is escaped here (`code` or `md`), so the
|
|
8
|
+
* renderer can print them as they are.
|
|
9
|
+
*/
|
|
10
|
+
import type { EvidenceCollector } from "../evidence/collector.js";
|
|
11
|
+
import type { RepoFacts } from "../inspectors/source/facts.js";
|
|
12
|
+
import type { ProductSurface } from "../inspectors/source/surface.js";
|
|
13
|
+
import type { AsBuiltBody, AsBuiltSpec, Results, ScanContract } from "../types.js";
|
|
14
|
+
/** Inputs for the as-built spec. */
|
|
15
|
+
export interface SpecInput {
|
|
16
|
+
contract: ScanContract;
|
|
17
|
+
facts: RepoFacts;
|
|
18
|
+
surface: ProductSurface;
|
|
19
|
+
ev: EvidenceCollector;
|
|
20
|
+
repository: string;
|
|
21
|
+
generatedAt: string;
|
|
22
|
+
}
|
|
23
|
+
/** Builds the as-built specification. */
|
|
24
|
+
export declare function buildAsBuiltSpec(input: SpecInput): AsBuiltSpec;
|
|
25
|
+
/** The part of a spec that is embedded in the blueprint. */
|
|
26
|
+
export declare function asBuiltBody(spec: AsBuiltSpec): AsBuiltBody;
|
|
27
|
+
/** Rebuilds the standalone as-built spec from a results document (null when source was not inspected). */
|
|
28
|
+
export declare function specFromResults(r: Results): AsBuiltSpec | null;
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
import { SCHEMA_VERSION } from "../types.js";
|
|
2
|
+
import { code, md } from "../reporting/escape.js";
|
|
3
|
+
import { stableId } from "../util/hash.js";
|
|
4
|
+
import { TOOL_VERSION } from "../version.js";
|
|
5
|
+
const EDGE_HINT = /\b(invalid|reject|error|unauthori[sz]ed|forbidden|denied|expired|missing|empty|fail|not found|duplicate|limit|isolat)/i;
|
|
6
|
+
const RATE_LIMIT = /^(express-rate-limit|@upstash\/ratelimit|rate-limiter-flexible|@fastify\/rate-limit|slowapi)$/;
|
|
7
|
+
const cmp = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
|
|
8
|
+
const where = (s) => code(`${s.location.path}${s.location.line ? `:${s.location.line}` : ""}`);
|
|
9
|
+
/** Builds the as-built specification. */
|
|
10
|
+
export function buildAsBuiltSpec(input) {
|
|
11
|
+
const { facts, surface, ev, contract } = input;
|
|
12
|
+
const revision = facts.snapshot.git.commit;
|
|
13
|
+
// ------------------------------------------------------------- surface with evidence
|
|
14
|
+
const entryOf = (item) => {
|
|
15
|
+
const integration = item.kind === "integration";
|
|
16
|
+
const evidenceId = ev.add({
|
|
17
|
+
kind: integration ? "manifest-field" : "source-location",
|
|
18
|
+
pop: integration ? "dependency-manifests" : item.kind === "test" ? "automated-tests" : "repository",
|
|
19
|
+
strength: item.certainty === "observed" ? "static" : "inferred",
|
|
20
|
+
provenance: item.certainty,
|
|
21
|
+
locator: item.path,
|
|
22
|
+
line: item.line,
|
|
23
|
+
revision,
|
|
24
|
+
summary: `${item.kind} ${item.method ? `${item.method} ` : ""}${item.name}: ${item.detail}`,
|
|
25
|
+
excerpt: item.excerpt,
|
|
26
|
+
checksum: facts.snapshot.byPath.get(item.path)?.sha256 ?? null,
|
|
27
|
+
});
|
|
28
|
+
return {
|
|
29
|
+
id: stableId("SRF", [item.kind, item.method, item.name, item.path], 10),
|
|
30
|
+
kind: item.kind,
|
|
31
|
+
name: item.name,
|
|
32
|
+
method: item.method,
|
|
33
|
+
area: item.area,
|
|
34
|
+
detail: item.detail,
|
|
35
|
+
location: { path: item.path, line: item.line },
|
|
36
|
+
certainty: item.certainty,
|
|
37
|
+
auth_hint: item.auth_hint,
|
|
38
|
+
evidence_ids: [evidenceId],
|
|
39
|
+
};
|
|
40
|
+
};
|
|
41
|
+
const entries = surface.items.map(entryOf);
|
|
42
|
+
const ofKind = (k) => entries.filter((e) => e.kind === k);
|
|
43
|
+
const locEv = (path, line, summary, pop = "repository") => ev.add({
|
|
44
|
+
kind: "source-location",
|
|
45
|
+
pop,
|
|
46
|
+
strength: "static",
|
|
47
|
+
provenance: "observed",
|
|
48
|
+
locator: path,
|
|
49
|
+
line,
|
|
50
|
+
revision,
|
|
51
|
+
summary,
|
|
52
|
+
checksum: facts.snapshot.byPath.get(path)?.sha256 ?? null,
|
|
53
|
+
});
|
|
54
|
+
const requirements = [];
|
|
55
|
+
const clarifications = [];
|
|
56
|
+
const fr = (statement, certainty, items) => {
|
|
57
|
+
const id = `FR-${String(requirements.length + 1).padStart(3, "0")}`;
|
|
58
|
+
requirements.push({
|
|
59
|
+
id,
|
|
60
|
+
statement,
|
|
61
|
+
certainty,
|
|
62
|
+
surface_ids: items.map((i) => i.id),
|
|
63
|
+
evidence_ids: [...new Set(items.flatMap((i) => i.evidence_ids))].sort(),
|
|
64
|
+
});
|
|
65
|
+
return id;
|
|
66
|
+
};
|
|
67
|
+
// ------------------------------------------------------------- stories per route area
|
|
68
|
+
const routes = [...ofKind("page"), ...ofKind("endpoint")];
|
|
69
|
+
const areas = [...new Set(routes.map((r) => r.area))].sort((a, b) => a === "home" ? -1 : b === "home" ? 1 : cmp(a, b));
|
|
70
|
+
const tests = ofKind("test");
|
|
71
|
+
const usedTests = new Set();
|
|
72
|
+
const journeys = contract.critical_journeys;
|
|
73
|
+
const matchedJourneys = new Set();
|
|
74
|
+
const stories = [];
|
|
75
|
+
for (const area of areas) {
|
|
76
|
+
const members = routes
|
|
77
|
+
.filter((r) => r.area === area)
|
|
78
|
+
.sort((a, b) => (a.kind === b.kind ? 0 : a.kind === "page" ? -1 : 1) || cmp(a.name, b.name));
|
|
79
|
+
const ids = [];
|
|
80
|
+
for (const r of members) {
|
|
81
|
+
if (r.kind === "page")
|
|
82
|
+
ids.push(fr(`The system MUST serve the page ${code(r.name)}.${r.auth_hint ? " Its source references an authentication helper, so access is likely limited to signed-in users." : ""}`, r.auth_hint ? "inferred" : r.certainty, [r]));
|
|
83
|
+
else
|
|
84
|
+
ids.push(fr(`The system MUST handle ${r.method === "ANY" ? "requests of any method" : `${code(r.method ?? "ANY")} requests`} to ${code(r.name)}.${r.auth_hint ? " The handler references an authentication helper, so access is likely limited to signed-in users." : ""}`, r.auth_hint ? "inferred" : r.certainty, [r]));
|
|
85
|
+
}
|
|
86
|
+
// Match on a crude stem ("notes" → "note") so singular and plural mentions both count.
|
|
87
|
+
const stem = area.replace(/[^a-z0-9]/gi, "").replace(/(?<=.{3})s$/i, "");
|
|
88
|
+
const word = new RegExp(`\\b${stem}`, "i");
|
|
89
|
+
const matchable = area !== "home" && stem.length >= 3;
|
|
90
|
+
const behaviours = tests
|
|
91
|
+
.filter((t) => matchable && (word.test(t.name) || word.test(t.location.path)))
|
|
92
|
+
.map((t) => {
|
|
93
|
+
usedTests.add(t.id);
|
|
94
|
+
return t;
|
|
95
|
+
});
|
|
96
|
+
const journey = matchable ? (journeys.find((j) => word.test(j)) ?? null) : null;
|
|
97
|
+
if (journey)
|
|
98
|
+
matchedJourneys.add(journey);
|
|
99
|
+
const label = area === "home" ? "Home and top-level pages" : `${area.charAt(0).toUpperCase()}${area.slice(1)}`;
|
|
100
|
+
stories.push({
|
|
101
|
+
id: `US-${stories.length + 1}`,
|
|
102
|
+
area,
|
|
103
|
+
title: label,
|
|
104
|
+
critical_journey: journey,
|
|
105
|
+
surface_ids: [...members, ...behaviours].map((m) => m.id),
|
|
106
|
+
requirement_ids: ids,
|
|
107
|
+
test_behaviours: behaviours.map((b) => b.name),
|
|
108
|
+
});
|
|
109
|
+
clarifications.push(`Purpose of the ${label} area: who uses it, and what outcome it must deliver.`);
|
|
110
|
+
}
|
|
111
|
+
for (const j of journeys)
|
|
112
|
+
if (!matchedJourneys.has(j))
|
|
113
|
+
clarifications.push(`Critical journey "${j}" is declared in the scan contract but no discovered route matches it.`);
|
|
114
|
+
// ------------------------------------------------------------- entities, integrations, configuration, auth
|
|
115
|
+
const entities = new Map();
|
|
116
|
+
for (const e of ofKind("entity"))
|
|
117
|
+
entities.set(e.name.toLowerCase(), [...(entities.get(e.name.toLowerCase()) ?? []), e]);
|
|
118
|
+
for (const group of entities.values()) {
|
|
119
|
+
const first = group[0];
|
|
120
|
+
const fields = group.find((g) => g.detail.startsWith("fields:"));
|
|
121
|
+
fr(`The system MUST persist ${code(first.name)} records${fields || group.length > 1
|
|
122
|
+
? ` (${[fields ? md(fields.detail) : "", group.length > 1 ? `defined in ${group.length} places` : ""].filter(Boolean).join("; ")})`
|
|
123
|
+
: ""}.`, "observed", group);
|
|
124
|
+
}
|
|
125
|
+
for (const i of ofKind("integration"))
|
|
126
|
+
fr(`The system MUST integrate with ${code(i.name)} (${md(i.detail)}).`, "observed", [i]);
|
|
127
|
+
const config = ofKind("config");
|
|
128
|
+
if (config.length > 0) {
|
|
129
|
+
const browser = config.filter((c) => c.detail.startsWith("exposed"));
|
|
130
|
+
fr(`The system MUST be configured through ${config.length} environment variable(s): ${config.map((c) => code(c.name)).join(", ")}.${browser.length > 0 ? ` ${browser.map((c) => code(c.name)).join(", ")} ${browser.length === 1 ? "is" : "are"} exposed to the browser bundle and MUST NOT hold secrets.` : ""}`, "observed", config);
|
|
131
|
+
}
|
|
132
|
+
for (const a of ofKind("auth-boundary"))
|
|
133
|
+
fr(`Requests MUST pass through ${code(a.name)} before reaching application routes (${code(a.detail)}).`, "inferred", [a]);
|
|
134
|
+
// ------------------------------------------------------------- non-functional (observed only) + gaps
|
|
135
|
+
const nfr = [];
|
|
136
|
+
const addNfr = (statement, certainty, evidence) => {
|
|
137
|
+
nfr.push({
|
|
138
|
+
id: `NFR-${String(nfr.length + 1).padStart(3, "0")}`,
|
|
139
|
+
statement,
|
|
140
|
+
certainty,
|
|
141
|
+
surface_ids: [],
|
|
142
|
+
evidence_ids: evidence,
|
|
143
|
+
});
|
|
144
|
+
};
|
|
145
|
+
for (const r of facts.runtimeDecl)
|
|
146
|
+
addNfr(`The system MUST run on the declared runtime (${code(r.detail)}).`, "observed", [
|
|
147
|
+
locEv(r.path, r.line, `runtime declaration: ${r.detail}`),
|
|
148
|
+
]);
|
|
149
|
+
for (const h of facts.headerConfig)
|
|
150
|
+
addNfr(`Responses MUST carry the HTTP headers configured in ${code(h.path)} (${md(h.detail)}).`, "observed", [locEv(h.path, h.line, `header configuration: ${h.detail}`)]);
|
|
151
|
+
const rateLimit = [...facts.deps.keys()].filter((d) => RATE_LIMIT.test(d)).sort();
|
|
152
|
+
for (const d of rateLimit) {
|
|
153
|
+
const m = facts.deps.get(d);
|
|
154
|
+
addNfr(`Requests MUST be rate-limited (${code(d)}).`, "inferred", [
|
|
155
|
+
locEv(m?.manifest ?? "package.json", null, `dependency ${d}`, "dependency-manifests"),
|
|
156
|
+
]);
|
|
157
|
+
}
|
|
158
|
+
for (const gap of [
|
|
159
|
+
"response-time and throughput targets for the main pages and endpoints",
|
|
160
|
+
"availability target and acceptable downtime",
|
|
161
|
+
"how long personal and business data is kept, and how it is deleted",
|
|
162
|
+
])
|
|
163
|
+
addNfr(`[NEEDS CLARIFICATION: ${gap}]`, "hypothesis", []);
|
|
164
|
+
// ------------------------------------------------------------- edge cases
|
|
165
|
+
const edges = [];
|
|
166
|
+
const addEdge = (statement, certainty, items, evidence = []) => {
|
|
167
|
+
edges.push({
|
|
168
|
+
id: `EC-${String(edges.length + 1).padStart(3, "0")}`,
|
|
169
|
+
statement,
|
|
170
|
+
certainty,
|
|
171
|
+
surface_ids: items.map((i) => i.id),
|
|
172
|
+
evidence_ids: [...new Set([...evidence, ...items.flatMap((i) => i.evidence_ids)])].sort(),
|
|
173
|
+
});
|
|
174
|
+
};
|
|
175
|
+
for (const p of facts.notFoundPages)
|
|
176
|
+
addEdge("Requests for unknown routes show a dedicated not-found page.", "observed", [], [locEv(p, 1, "not-found page", "repository")]);
|
|
177
|
+
for (const t of tests.filter((x) => EDGE_HINT.test(x.name)))
|
|
178
|
+
addEdge(`A test describes this case: ${md(t.name)} (${where(t)}).`, "inferred", [t]);
|
|
179
|
+
// ------------------------------------------------------------- success criteria
|
|
180
|
+
const success = [
|
|
181
|
+
{
|
|
182
|
+
id: "SC-001",
|
|
183
|
+
statement: "[NEEDS CLARIFICATION: measurable outcomes cannot be derived from source code; state the targets this product must meet]",
|
|
184
|
+
certainty: "hypothesis",
|
|
185
|
+
surface_ids: [],
|
|
186
|
+
evidence_ids: [],
|
|
187
|
+
},
|
|
188
|
+
];
|
|
189
|
+
if (facts.e2eFiles.length > 0)
|
|
190
|
+
success.push({
|
|
191
|
+
id: "SC-002",
|
|
192
|
+
statement: `The ${facts.e2eFiles.length} end-to-end test file(s) pass on every release candidate (for example ${code(facts.e2eFiles[0] ?? "")}).`,
|
|
193
|
+
certainty: "inferred",
|
|
194
|
+
surface_ids: [],
|
|
195
|
+
evidence_ids: [locEv(facts.e2eFiles[0], null, "end-to-end test file", "automated-tests")],
|
|
196
|
+
});
|
|
197
|
+
// ------------------------------------------------------------- uncertainties
|
|
198
|
+
const uncertainties = [
|
|
199
|
+
"Purpose, users, priorities and business rules are not derivable from code; every story carries a clarification for them.",
|
|
200
|
+
"Nothing was executed: requirements describe what the code declares, not verified behaviour.",
|
|
201
|
+
...(tests.length > 0 && usedTests.size < tests.length
|
|
202
|
+
? [
|
|
203
|
+
`${tests.length - usedTests.size} test title(s) could not be matched to a route area and are listed separately.`,
|
|
204
|
+
]
|
|
205
|
+
: []),
|
|
206
|
+
...surface.notes,
|
|
207
|
+
];
|
|
208
|
+
const limitations = facts.snapshot.limitations;
|
|
209
|
+
return {
|
|
210
|
+
schema_version: SCHEMA_VERSION,
|
|
211
|
+
kind: "as-built-spec",
|
|
212
|
+
generated_at: input.generatedAt,
|
|
213
|
+
tool: { name: "product-check", version: TOOL_VERSION },
|
|
214
|
+
target: {
|
|
215
|
+
name: contract.target.name,
|
|
216
|
+
repository: input.repository,
|
|
217
|
+
commit: revision,
|
|
218
|
+
ref: facts.snapshot.git.ref ?? null,
|
|
219
|
+
},
|
|
220
|
+
surface: entries,
|
|
221
|
+
stories,
|
|
222
|
+
requirements,
|
|
223
|
+
non_functional: nfr,
|
|
224
|
+
success_criteria: success,
|
|
225
|
+
edge_cases: edges,
|
|
226
|
+
clarifications,
|
|
227
|
+
uncertainties,
|
|
228
|
+
evidence: referencedEvidence(ev.all(), {
|
|
229
|
+
surface: entries,
|
|
230
|
+
requirements,
|
|
231
|
+
non_functional: nfr,
|
|
232
|
+
success_criteria: success,
|
|
233
|
+
edge_cases: edges,
|
|
234
|
+
}),
|
|
235
|
+
limitations,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
/** The evidence items a spec references (a shared collector may hold others, for example during `run`). */
|
|
239
|
+
function referencedEvidence(all, parts) {
|
|
240
|
+
const ids = new Set([
|
|
241
|
+
parts.surface,
|
|
242
|
+
parts.requirements,
|
|
243
|
+
parts.non_functional,
|
|
244
|
+
parts.success_criteria,
|
|
245
|
+
parts.edge_cases,
|
|
246
|
+
].flatMap((list) => list.flatMap((x) => x.evidence_ids)));
|
|
247
|
+
return all.filter((e) => ids.has(e.id));
|
|
248
|
+
}
|
|
249
|
+
/** The part of a spec that is embedded in the blueprint. */
|
|
250
|
+
export function asBuiltBody(spec) {
|
|
251
|
+
return {
|
|
252
|
+
surface: spec.surface,
|
|
253
|
+
stories: spec.stories,
|
|
254
|
+
requirements: spec.requirements,
|
|
255
|
+
non_functional: spec.non_functional,
|
|
256
|
+
success_criteria: spec.success_criteria,
|
|
257
|
+
edge_cases: spec.edge_cases,
|
|
258
|
+
clarifications: spec.clarifications,
|
|
259
|
+
uncertainties: spec.uncertainties,
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
/** Rebuilds the standalone as-built spec from a results document (null when source was not inspected). */
|
|
263
|
+
export function specFromResults(r) {
|
|
264
|
+
const body = r.blueprint.as_is.as_built;
|
|
265
|
+
const repo = r.run.target_revision.repository;
|
|
266
|
+
if (!body || !repo)
|
|
267
|
+
return null;
|
|
268
|
+
return {
|
|
269
|
+
schema_version: SCHEMA_VERSION,
|
|
270
|
+
kind: "as-built-spec",
|
|
271
|
+
generated_at: r.run.started_at,
|
|
272
|
+
tool: { name: "product-check", version: r.run.tool_version },
|
|
273
|
+
target: { name: r.contract.target.name, repository: repo.label, commit: repo.commit, ref: repo.ref },
|
|
274
|
+
...body,
|
|
275
|
+
evidence: referencedEvidence(r.evidence, body),
|
|
276
|
+
limitations: r.limitations,
|
|
277
|
+
};
|
|
278
|
+
}
|