bmad-method-test-architecture-enterprise 2.0.4-next.2 → 2.0.4-next.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/cli/lib/evaluate/http-target.js +28 -8
- package/cli/lib/nfr-report.js +432 -0
- package/cli/lib/workflow-publication.js +170 -0
- package/cli/nfr.js +209 -0
- package/package.json +4 -2
- package/skills/bmod-tea/bmod.toml +1 -1
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"name": "bmad-method-test-architecture-enterprise",
|
|
32
32
|
"source": "./",
|
|
33
33
|
"description": "Master Test Architect module for quality strategy, test generation in red and expand modes with automatic run-and-heal, combined framework and CI setup, CI/CD quality gates, and structured testing education. The CI and ATDD commands remain available with existing customizations. Part of the BMad Method ecosystem.",
|
|
34
|
-
"version": "2.0.4-next.
|
|
34
|
+
"version": "2.0.4-next.3",
|
|
35
35
|
"author": {
|
|
36
36
|
"name": "Murat K Ozcan (TEA Creator) & Brian (BMad) Madison"
|
|
37
37
|
},
|
|
@@ -699,19 +699,38 @@ async function deploymentAccess({
|
|
|
699
699
|
return { origins, authorizations };
|
|
700
700
|
}
|
|
701
701
|
|
|
702
|
-
/**
|
|
703
|
-
function
|
|
702
|
+
/** Tries a port on one address, then releases it for the target; a busy candidate returns null. */
|
|
703
|
+
function availablePort(port, address, createServer) {
|
|
704
704
|
return new Promise((resolve, reject) => {
|
|
705
|
-
const server =
|
|
705
|
+
const server = createServer();
|
|
706
706
|
server.unref();
|
|
707
|
-
server.once('error', reject);
|
|
708
|
-
server.listen(
|
|
709
|
-
const
|
|
710
|
-
server.close(() => resolve(
|
|
707
|
+
server.once('error', (error) => (error.code === 'EADDRINUSE' ? resolve(null) : reject(error)));
|
|
708
|
+
server.listen(port, address, () => {
|
|
709
|
+
const selected = server.address().port;
|
|
710
|
+
server.close(() => resolve(selected));
|
|
711
711
|
});
|
|
712
712
|
});
|
|
713
713
|
}
|
|
714
714
|
|
|
715
|
+
/** A port available on every address the policy may select, released for the server to take. */
|
|
716
|
+
async function freePort(addresses, createServer = () => net.createServer()) {
|
|
717
|
+
const distinct = [...new Set(addresses.map((address) => canonicalAddress(address) ?? address))];
|
|
718
|
+
if (distinct.length === 0) throw new Error('a started server has no authorized address');
|
|
719
|
+
for (let attempt = 0; attempt < 32; attempt += 1) {
|
|
720
|
+
const candidate = await availablePort(0, distinct[0], createServer);
|
|
721
|
+
if (candidate === null) continue;
|
|
722
|
+
let available = true;
|
|
723
|
+
for (const address of distinct.slice(1)) {
|
|
724
|
+
if ((await availablePort(candidate, address, createServer)) === null) {
|
|
725
|
+
available = false;
|
|
726
|
+
break;
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
if (available) return candidate;
|
|
730
|
+
}
|
|
731
|
+
throw new Error(`could not find a free port on ${distinct.join(', ')}`);
|
|
732
|
+
}
|
|
733
|
+
|
|
715
734
|
/**
|
|
716
735
|
* Whether `address:port` accepts a connection now: `true`, or why the attempt
|
|
717
736
|
* failed (the socket error's code, `ECONNREFUSED` while nothing listens, or
|
|
@@ -1207,7 +1226,7 @@ function createApiPort({ entries, httpPort, cwd, targetOf, readEnvironment, mech
|
|
|
1207
1226
|
// A server behind a bridge gets a private directory for the Unix socket its shim serves, which only that call's
|
|
1208
1227
|
// server may write; it joins the scratch list like the port directory.
|
|
1209
1228
|
if (launched !== null && mechanism.bridges === true) bridgeDirectory = makeBridgeDirectory(scratch);
|
|
1210
|
-
const launchedPort = launched === null || reports ? null : await freePort();
|
|
1229
|
+
const launchedPort = launched === null || reports ? null : await freePort(launched.addresses);
|
|
1211
1230
|
const configurationAt = (port) =>
|
|
1212
1231
|
portConfiguration({
|
|
1213
1232
|
entries,
|
|
@@ -1397,6 +1416,7 @@ module.exports = {
|
|
|
1397
1416
|
defaultPortOf,
|
|
1398
1417
|
deploymentAccess,
|
|
1399
1418
|
deploymentAuthorizations,
|
|
1419
|
+
freePort,
|
|
1400
1420
|
httpPortFile,
|
|
1401
1421
|
isApiEntry,
|
|
1402
1422
|
makeBridgeDirectory,
|
|
@@ -0,0 +1,432 @@
|
|
|
1
|
+
/** Validate the actual NFR workflow context against supplied sources and its report. */
|
|
2
|
+
'use strict';
|
|
3
|
+
const fs = require('node:fs');
|
|
4
|
+
const path = require('node:path');
|
|
5
|
+
const yaml = require('js-yaml');
|
|
6
|
+
const MarkdownIt = require('markdown-it');
|
|
7
|
+
const { WorkflowError } = require('./workflow-cli');
|
|
8
|
+
const DOMAINS = ['security', 'performance', 'reliability', 'maintainability'];
|
|
9
|
+
const STATUSES = new Set(['PASS', 'CONCERNS', 'FAIL', 'N/A']);
|
|
10
|
+
const STEPS = [
|
|
11
|
+
'step-01-load-context',
|
|
12
|
+
'step-02-define-thresholds',
|
|
13
|
+
'step-03-gather-evidence',
|
|
14
|
+
'step-04e-aggregate-nfr',
|
|
15
|
+
'step-05-generate-report',
|
|
16
|
+
];
|
|
17
|
+
/** Reject a report field with the parser failure class used by the CLI. */
|
|
18
|
+
const bad = (message) => {
|
|
19
|
+
throw new WorkflowError('environment-parser', message);
|
|
20
|
+
};
|
|
21
|
+
/** Collapse whitespace so quoted observations survive line wrapping. */
|
|
22
|
+
const normalized = (value) => String(value).replaceAll(/\s+/g, ' ').trim();
|
|
23
|
+
/** Compare ADR category labels across spaces, hyphens and capitalization. */
|
|
24
|
+
const categoryLabel = (value) =>
|
|
25
|
+
value
|
|
26
|
+
.toLowerCase()
|
|
27
|
+
.replaceAll(/[-\s]+/g, ' ')
|
|
28
|
+
.trim();
|
|
29
|
+
/** Compare canonical context values after JSON serialization. */
|
|
30
|
+
const same = (left, right) => JSON.stringify(left) === JSON.stringify(right);
|
|
31
|
+
const markdown = new MarkdownIt({ html: true });
|
|
32
|
+
const DECORATIONS = { PASS: ['✅', '✔', '✔️'], CONCERNS: ['⚠', '⚠️'], FAIL: ['❌', '✖', '✖️'], 'N/A': ['➖'] };
|
|
33
|
+
/** Read visible inline text, preserving code excerpts while discarding formatting delimiters. */
|
|
34
|
+
function inlineText(token) {
|
|
35
|
+
return (token?.children || [])
|
|
36
|
+
.filter((child) => ['text', 'code_inline'].includes(child.type))
|
|
37
|
+
.map((child) => child.content)
|
|
38
|
+
.join('')
|
|
39
|
+
.trim();
|
|
40
|
+
}
|
|
41
|
+
/** Normalize one complete display enum with an optional matching approved decoration. */
|
|
42
|
+
function displayStatus(value) {
|
|
43
|
+
const match = String(value)
|
|
44
|
+
.trim()
|
|
45
|
+
.match(/^(PASS|CONCERNS|FAIL|N\/A)(?:[ \t]+(\S+))?$/i);
|
|
46
|
+
if (!match) bad('report status must contain one complete enum and an optional approved decoration');
|
|
47
|
+
const status = match[1].toUpperCase();
|
|
48
|
+
if (match[2] && !DECORATIONS[status].includes(match[2])) bad('report status decoration contradicts or ambiguously labels its enum');
|
|
49
|
+
return status;
|
|
50
|
+
}
|
|
51
|
+
/** Preserve a quoted source excerpt when Markdown renders it as a single inline code value. */
|
|
52
|
+
function literalValue(value) {
|
|
53
|
+
const blocks = markdown.parse(value, {});
|
|
54
|
+
if (blocks.length === 1 && blocks[0].type === 'fence') return blocks[0].content.trim();
|
|
55
|
+
const children = markdown.parseInline(value, {})[0]?.children || [];
|
|
56
|
+
return children.length === 1 && children[0].type === 'code_inline' ? children[0].content : value;
|
|
57
|
+
}
|
|
58
|
+
/** Preserve one or more complete fenced source quotations as separate actual observations. */
|
|
59
|
+
function actualValues(value) {
|
|
60
|
+
const blocks = markdown.parse(value, {});
|
|
61
|
+
if (blocks.length > 0 && blocks.every((token) => token.type === 'fence')) return blocks.map((token) => token.content.trim());
|
|
62
|
+
return [literalValue(value)];
|
|
63
|
+
}
|
|
64
|
+
/** The workflow's worst-status rollup excludes nonapplicable domains. */
|
|
65
|
+
function worst(values) {
|
|
66
|
+
if (values.includes('FAIL')) return 'FAIL';
|
|
67
|
+
if (values.includes('CONCERNS')) return 'CONCERNS';
|
|
68
|
+
return values.includes('PASS') ? 'PASS' : 'N/A';
|
|
69
|
+
}
|
|
70
|
+
/** Stable criterion identity defined by NFR step02. */
|
|
71
|
+
function criterionId(domain, label) {
|
|
72
|
+
return `${domain}-${label}`
|
|
73
|
+
.toLowerCase()
|
|
74
|
+
.replaceAll(/[^a-z0-9]+/g, '-')
|
|
75
|
+
.replaceAll(/^-+|-+$/g, '');
|
|
76
|
+
}
|
|
77
|
+
/** Parse YAML without silently accepting duplicate keys or extra documents. */
|
|
78
|
+
function parseYaml(text, label) {
|
|
79
|
+
try {
|
|
80
|
+
return yaml.load(text);
|
|
81
|
+
} catch (error) {
|
|
82
|
+
bad(`invalid ${label} YAML: ${error.message}`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/** Read canonical project-relative files from their explicit input role. */
|
|
86
|
+
function sourceText(reference, projectRoot, sources, label) {
|
|
87
|
+
if (
|
|
88
|
+
typeof reference !== 'string' ||
|
|
89
|
+
!reference ||
|
|
90
|
+
path.isAbsolute(reference) ||
|
|
91
|
+
reference.includes('\\') ||
|
|
92
|
+
path.posix.normalize(reference) !== reference ||
|
|
93
|
+
reference.startsWith('../') ||
|
|
94
|
+
reference.startsWith('./')
|
|
95
|
+
)
|
|
96
|
+
bad(`${label} must name a canonical project-relative supplied path`);
|
|
97
|
+
const file = path.join(projectRoot, reference);
|
|
98
|
+
if (!sources.includes(file)) bad(`${label} is outside its supplied input role: ${reference}`);
|
|
99
|
+
return normalized(fs.readFileSync(file, 'utf8'));
|
|
100
|
+
}
|
|
101
|
+
/** Require an actual source excerpt, while allowing normalized line wrapping. */
|
|
102
|
+
function excerpt(value, source, label) {
|
|
103
|
+
if (typeof value !== 'string' || !normalized(value) || !source.includes(normalized(value)))
|
|
104
|
+
bad(`${label} is not a quotation from its supplied source`);
|
|
105
|
+
}
|
|
106
|
+
/** Extract populated Markdown sections without matching headings inside fenced examples. */
|
|
107
|
+
function sections(text, level) {
|
|
108
|
+
const lines = text.split(/\r?\n/);
|
|
109
|
+
const tokens = markdown.parse(text, {});
|
|
110
|
+
const headings = tokens.flatMap((token, index) => {
|
|
111
|
+
if (token.type !== 'heading_open' || token.level !== 0 || !token.map) return [];
|
|
112
|
+
const label = inlineText(tokens[index + 1]);
|
|
113
|
+
return [{ level: Number(token.tag.slice(1)), label, start: token.map[0], end: token.map[1] }];
|
|
114
|
+
});
|
|
115
|
+
return headings
|
|
116
|
+
.filter((heading) => heading.level === level)
|
|
117
|
+
.map((heading) => {
|
|
118
|
+
const next = headings.find((other) => other.start > heading.start && other.level <= level);
|
|
119
|
+
return { label: heading.label, body: lines.slice(heading.end, next?.start ?? lines.length).join('\n') };
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
/** Remove literal block examples before interpreting bold report fields. */
|
|
123
|
+
function reportFields(text) {
|
|
124
|
+
const lines = text.split(/\r?\n/);
|
|
125
|
+
for (const token of markdown.parse(text, {})) {
|
|
126
|
+
if (['fence', 'code_block', 'html_block'].includes(token.type) && token.map)
|
|
127
|
+
for (let line = token.map[0]; line < token.map[1]; line++) lines[line] = lines[line].replaceAll(/[^\n]/g, ' ');
|
|
128
|
+
}
|
|
129
|
+
return lines.join('\n');
|
|
130
|
+
}
|
|
131
|
+
/** Read a unique bold report field, including multiline evidence lists. */
|
|
132
|
+
function field(body, name) {
|
|
133
|
+
const expression = new RegExp(
|
|
134
|
+
`^[ \\t]*(?:-[ \\t]+)?\\*\\*${name}:?\\*\\*[ \\t]*:?[ \\t]*([^]*?)(?=^[ \\t]*(?:-[ \\t]+)?\\*\\*[A-Za-z]|$(?![\\s\\S]))`,
|
|
135
|
+
'gim',
|
|
136
|
+
);
|
|
137
|
+
const values = [...reportFields(body).matchAll(expression)].map((match) => {
|
|
138
|
+
const start = match.index + match[0].length - match[1].length;
|
|
139
|
+
return body.slice(start, match.index + match[0].length).trim();
|
|
140
|
+
});
|
|
141
|
+
if (values.length !== 1 || !values[0]) bad(`criterion must contain exactly one populated ${name} field`);
|
|
142
|
+
return values[0];
|
|
143
|
+
}
|
|
144
|
+
/** Validate the fresh canonical context, exact ledger bindings, report and gate together. */
|
|
145
|
+
function readNfrReport(text, context, request) {
|
|
146
|
+
const { projectRoot, runScope, runKey, requestId, inputs, evidenceFiles } = request;
|
|
147
|
+
if (
|
|
148
|
+
!context ||
|
|
149
|
+
context.schemaVersion !== 1 ||
|
|
150
|
+
context.requestId !== requestId ||
|
|
151
|
+
context.runScope !== runScope ||
|
|
152
|
+
context.runKey !== runKey ||
|
|
153
|
+
context.supplied_project_root !== projectRoot
|
|
154
|
+
)
|
|
155
|
+
bad('the NFR context does not belong to the requested attempt and scope');
|
|
156
|
+
if (
|
|
157
|
+
!same(Object.keys(context.declared_nfr_criteria || {}).sort(), [...DOMAINS].sort()) ||
|
|
158
|
+
!same(Object.keys(context.domain_assessments || {}).sort(), [...DOMAINS].sort())
|
|
159
|
+
)
|
|
160
|
+
bad('context must carry exactly the four audited domains');
|
|
161
|
+
const criteria = new Map();
|
|
162
|
+
const validateCriterion = (criterion, domain, previousOrder) => {
|
|
163
|
+
if (
|
|
164
|
+
!criterion ||
|
|
165
|
+
typeof criterion.label !== 'string' ||
|
|
166
|
+
!criterion.label.trim() ||
|
|
167
|
+
criterion.id !== criterionId(domain, criterion.label) ||
|
|
168
|
+
!Number.isSafeInteger(criterion.order) ||
|
|
169
|
+
criterion.order <= previousOrder ||
|
|
170
|
+
criteria.has(criterion.id)
|
|
171
|
+
)
|
|
172
|
+
bad(`invalid criterion identity or source order in ${domain}`);
|
|
173
|
+
const source = sourceText(criterion.threshold_source, projectRoot, inputs, 'threshold source');
|
|
174
|
+
if (criterion.threshold !== 'UNKNOWN') excerpt(criterion.threshold, source, `${criterion.id} threshold`);
|
|
175
|
+
criteria.set(criterion.id, { ...criterion, domain });
|
|
176
|
+
return criterion.order;
|
|
177
|
+
};
|
|
178
|
+
for (const domain of DOMAINS) {
|
|
179
|
+
if (!Array.isArray(context.declared_nfr_criteria[domain])) bad(`invalid declared criteria for ${domain}`);
|
|
180
|
+
let order = -1;
|
|
181
|
+
for (const criterion of context.declared_nfr_criteria[domain]) order = validateCriterion(criterion, domain, order);
|
|
182
|
+
}
|
|
183
|
+
if (!Array.isArray(context.recorded_only_nfr_criteria))
|
|
184
|
+
bad('context must include recorded-only criteria, including an empty list when none were declared');
|
|
185
|
+
const recorded = new Set();
|
|
186
|
+
let recordedOrder = -1;
|
|
187
|
+
for (const criterion of context.recorded_only_nfr_criteria) {
|
|
188
|
+
if (DOMAINS.includes(criterion.domain) || typeof criterion.domain !== 'string' || criterion.assessment_mode !== 'recorded-only')
|
|
189
|
+
bad('recorded-only criteria cannot create audited domains');
|
|
190
|
+
recordedOrder = validateCriterion(criterion, criterion.domain, recordedOrder);
|
|
191
|
+
recorded.add(criterion.id);
|
|
192
|
+
}
|
|
193
|
+
if (!Array.isArray(context.supplied_evidence_ledger)) bad('context is missing the supplied evidence ledger');
|
|
194
|
+
const observations = new Map();
|
|
195
|
+
let lastPath = '';
|
|
196
|
+
for (const entry of context.supplied_evidence_ledger) {
|
|
197
|
+
if (
|
|
198
|
+
!entry ||
|
|
199
|
+
entry.source !== 'supplied' ||
|
|
200
|
+
entry.source_type !== 'implementation-evidence' ||
|
|
201
|
+
typeof entry.path !== 'string' ||
|
|
202
|
+
entry.path <= lastPath ||
|
|
203
|
+
!Array.isArray(entry.observations) ||
|
|
204
|
+
entry.observations.length === 0
|
|
205
|
+
)
|
|
206
|
+
bad('invalid or unsorted supplied evidence ledger');
|
|
207
|
+
lastPath = entry.path;
|
|
208
|
+
const source = sourceText(entry.path, projectRoot, evidenceFiles, 'implementation evidence');
|
|
209
|
+
let previous = '';
|
|
210
|
+
for (const observation of entry.observations) {
|
|
211
|
+
if (!observation || !criteria.has(observation.criterion_id) || recorded.has(observation.criterion_id))
|
|
212
|
+
bad('ledger observation has an undeclared audited criterion');
|
|
213
|
+
excerpt(observation.supports, source, `${observation.criterion_id} observation`);
|
|
214
|
+
const key = `${observation.criterion_id}\0${observation.supports}`;
|
|
215
|
+
if (key <= previous) bad('ledger observations must be deduplicated and sorted');
|
|
216
|
+
previous = key;
|
|
217
|
+
const bound = observations.get(observation.criterion_id) || [];
|
|
218
|
+
bound.push({ path: entry.path, supports: observation.supports });
|
|
219
|
+
observations.set(observation.criterion_id, bound);
|
|
220
|
+
}
|
|
221
|
+
const expectedDomains = DOMAINS.filter((domain) =>
|
|
222
|
+
entry.observations.some((observation) => criteria.get(observation.criterion_id).domain === domain),
|
|
223
|
+
);
|
|
224
|
+
if (!same(entry.domains, expectedDomains)) bad('ledger domain bindings contradict its criterion observations');
|
|
225
|
+
}
|
|
226
|
+
const frontmatter = text.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/);
|
|
227
|
+
if (!frontmatter) bad('the report is missing progress frontmatter');
|
|
228
|
+
const progress = parseYaml(frontmatter[1], 'progress');
|
|
229
|
+
const optionalSteps = [
|
|
230
|
+
'step-04-evaluate-and-score',
|
|
231
|
+
'step-04a-subagent-security',
|
|
232
|
+
'step-04b-subagent-performance',
|
|
233
|
+
'step-04c-subagent-reliability',
|
|
234
|
+
'step-04d-subagent-maintainability',
|
|
235
|
+
];
|
|
236
|
+
const completed = progress?.stepsCompleted;
|
|
237
|
+
const requiredSequence = Array.isArray(completed) ? completed.filter((step) => STEPS.includes(step)) : [];
|
|
238
|
+
const validSteps =
|
|
239
|
+
Array.isArray(completed) &&
|
|
240
|
+
new Set(completed).size === completed.length &&
|
|
241
|
+
completed.every((step) => STEPS.includes(step) || optionalSteps.includes(step)) &&
|
|
242
|
+
optionalSteps.every(
|
|
243
|
+
(step) =>
|
|
244
|
+
!completed.includes(step) ||
|
|
245
|
+
(completed.indexOf(step) > completed.indexOf(STEPS[2]) && completed.indexOf(step) < completed.indexOf(STEPS[3])),
|
|
246
|
+
);
|
|
247
|
+
if (
|
|
248
|
+
progress?.workflowStatus !== 'completed' ||
|
|
249
|
+
progress.runScope !== runScope ||
|
|
250
|
+
progress.runKey !== runKey ||
|
|
251
|
+
progress.lastStep !== STEPS.at(-1) ||
|
|
252
|
+
!validSteps ||
|
|
253
|
+
!same(requiredSequence, STEPS) ||
|
|
254
|
+
!(typeof progress.lastSaved === 'string' || progress.lastSaved instanceof Date) ||
|
|
255
|
+
!Number.isFinite(Date.parse(progress.lastSaved))
|
|
256
|
+
)
|
|
257
|
+
bad('the report checkpoint does not confirm this completed run');
|
|
258
|
+
const gates = [];
|
|
259
|
+
const reportBody = text.slice(frontmatter[0].length);
|
|
260
|
+
for (const token of markdown.parse(reportBody, {})) {
|
|
261
|
+
if (token.type !== 'fence' || token.level !== 0 || !['yaml', 'yml'].includes(token.info.trim().toLowerCase())) continue;
|
|
262
|
+
const value = parseYaml(token.content, 'gate');
|
|
263
|
+
if (value && Object.hasOwn(value, 'nfr_assessment')) gates.push(value.nfr_assessment);
|
|
264
|
+
}
|
|
265
|
+
if (gates.length !== 1) bad('report must carry exactly one nfr_assessment gate');
|
|
266
|
+
const gate = gates[0];
|
|
267
|
+
if (!gate || !same(Object.keys(gate.audited_domains || {}).sort(), [...DOMAINS].sort()) || !STATUSES.has(gate.overall_status))
|
|
268
|
+
bad('gate must declare exactly four audited domains and a valid overall status');
|
|
269
|
+
if (
|
|
270
|
+
typeof gate.blockers !== 'boolean' ||
|
|
271
|
+
['critical_issues', 'high_priority_issues', 'medium_priority_issues', 'concerns', 'evidence_gaps'].some(
|
|
272
|
+
(key) => !Number.isSafeInteger(gate[key]) || gate[key] < 0,
|
|
273
|
+
)
|
|
274
|
+
)
|
|
275
|
+
bad('gate issue counts and blockers flag must be typed');
|
|
276
|
+
const reportSections = sections(reportBody, 2);
|
|
277
|
+
for (const name of [
|
|
278
|
+
'Executive Summary',
|
|
279
|
+
'Quick Wins',
|
|
280
|
+
'Recommended Actions',
|
|
281
|
+
'Monitoring Hooks',
|
|
282
|
+
'Fail-Fast Mechanisms',
|
|
283
|
+
'Evidence Gaps',
|
|
284
|
+
'Findings Summary',
|
|
285
|
+
'Gate YAML Snippet',
|
|
286
|
+
'Related Artifacts',
|
|
287
|
+
'Recommendations Summary',
|
|
288
|
+
'Sign-Off',
|
|
289
|
+
]) {
|
|
290
|
+
const matches = reportSections.filter((section) => section.label === name);
|
|
291
|
+
if (
|
|
292
|
+
matches.length !== 1 ||
|
|
293
|
+
!matches[0].body
|
|
294
|
+
.replaceAll(/<!--[^]*?-->/g, '')
|
|
295
|
+
.replaceAll(/^#{1,6}[^\n]*$/gm, '')
|
|
296
|
+
.trim()
|
|
297
|
+
)
|
|
298
|
+
bad(`report requires one populated ${name} section`);
|
|
299
|
+
}
|
|
300
|
+
let concerns = 0;
|
|
301
|
+
let gaps = 0;
|
|
302
|
+
for (const domain of DOMAINS) {
|
|
303
|
+
const domainSections = reportSections.filter((section) => section.label.toLowerCase() === `${domain} assessment`);
|
|
304
|
+
if (domainSections.length !== 1) bad(`report requires exactly one ${domain} Assessment`);
|
|
305
|
+
const section = domainSections[0];
|
|
306
|
+
const expected = context.declared_nfr_criteria[domain];
|
|
307
|
+
const assessment = context.domain_assessments[domain];
|
|
308
|
+
if (
|
|
309
|
+
!assessment ||
|
|
310
|
+
!Array.isArray(assessment.findings) ||
|
|
311
|
+
!same(
|
|
312
|
+
assessment.findings.map((finding) => finding.criterion_id),
|
|
313
|
+
expected.map((criterion) => criterion.id),
|
|
314
|
+
)
|
|
315
|
+
)
|
|
316
|
+
bad(`${domain} findings omit or add declared criteria`);
|
|
317
|
+
const reportFindings = sections(section.body, 3);
|
|
318
|
+
if (
|
|
319
|
+
!same(
|
|
320
|
+
reportFindings.map((finding) => finding.label),
|
|
321
|
+
expected.map((criterion) => criterion.label),
|
|
322
|
+
)
|
|
323
|
+
)
|
|
324
|
+
bad(`${domain} report headings omit, duplicate or add declared criteria`);
|
|
325
|
+
const expectedGaps = [];
|
|
326
|
+
for (const [index, criterion] of expected.entries()) {
|
|
327
|
+
const finding = assessment.findings[index];
|
|
328
|
+
const evidence = observations.get(criterion.id) || [];
|
|
329
|
+
if (!STATUSES.has(finding.status) || !same(finding.evidence, evidence))
|
|
330
|
+
bad(`${criterion.id} status or evidence contradicts the canonical ledger`);
|
|
331
|
+
if ((evidence.length === 0 || criterion.threshold === 'UNKNOWN') && finding.status !== 'CONCERNS')
|
|
332
|
+
bad(`${criterion.id} requires CONCERNS for unknown thresholds or missing evidence`);
|
|
333
|
+
if (evidence.length === 0)
|
|
334
|
+
expectedGaps.push({ criterion_id: criterion.id, message: `${criterion.label}: no supplied implementation evidence` });
|
|
335
|
+
if (criterion.threshold === 'UNKNOWN' && evidence.length > 0)
|
|
336
|
+
expectedGaps.push({ criterion_id: criterion.id, message: `${criterion.label}: UNKNOWN threshold` });
|
|
337
|
+
const body = reportFindings[index].body;
|
|
338
|
+
if (
|
|
339
|
+
displayStatus(field(body, 'Status')) !== finding.status ||
|
|
340
|
+
normalized(literalValue(field(body, 'Threshold'))) !== normalized(criterion.threshold) ||
|
|
341
|
+
field(body, 'Threshold Source') !== `\`${criterion.threshold_source}\``
|
|
342
|
+
)
|
|
343
|
+
bad(`${criterion.id} report fields contradict its canonical criterion`);
|
|
344
|
+
const actual = actualValues(field(body, 'Actual'));
|
|
345
|
+
if (finding.status === 'PASS' && actual.includes('UNKNOWN')) bad(`${criterion.id} claims PASS without a known actual measurement`);
|
|
346
|
+
for (const observation of actual) {
|
|
347
|
+
if (
|
|
348
|
+
!normalized(observation) ||
|
|
349
|
+
(observation !== 'UNKNOWN' &&
|
|
350
|
+
!evidence.some((entry) =>
|
|
351
|
+
sourceText(entry.path, projectRoot, evidenceFiles, 'actual evidence').includes(normalized(observation)),
|
|
352
|
+
))
|
|
353
|
+
)
|
|
354
|
+
bad(`${criterion.id} actual is unsupported by supplied evidence`);
|
|
355
|
+
}
|
|
356
|
+
const evidenceText = field(body, 'Evidence');
|
|
357
|
+
const paths = [...evidenceText.matchAll(/`([^`]+)`/g)].map((match) => match[1]);
|
|
358
|
+
if (!same([...new Set(paths)], [...new Set(evidence.map((entry) => entry.path))]))
|
|
359
|
+
bad(`${criterion.id} report evidence paths contradict the canonical ledger`);
|
|
360
|
+
const supports = field(body, 'Supports');
|
|
361
|
+
for (const entry of evidence)
|
|
362
|
+
if (!normalized(supports).includes(normalized(entry.supports))) bad(`${criterion.id} report omits a bound observation`);
|
|
363
|
+
if (evidence.length === 0 && (evidenceText !== 'None' || supports !== 'None' || !same(actual, ['UNKNOWN'])))
|
|
364
|
+
bad(`${criterion.id} claims unsupported implementation evidence`);
|
|
365
|
+
concerns += finding.status === 'CONCERNS' ? 1 : 0;
|
|
366
|
+
}
|
|
367
|
+
const displayedGaps = Array.isArray(assessment.evidence_gaps)
|
|
368
|
+
? assessment.evidence_gaps.map((gap) => {
|
|
369
|
+
const criterion = criteria.get(gap.criterion_id);
|
|
370
|
+
if (criterion?.threshold === 'UNKNOWN' && gap.message === `${criterion.label}: UNKNOWN threshold gap`)
|
|
371
|
+
return { ...gap, message: `${criterion.label}: UNKNOWN threshold` };
|
|
372
|
+
return gap;
|
|
373
|
+
})
|
|
374
|
+
: assessment.evidence_gaps;
|
|
375
|
+
if (!same(displayedGaps, expectedGaps)) bad(`${domain} evidence gaps contradict its declared criteria`);
|
|
376
|
+
gaps += expectedGaps.length;
|
|
377
|
+
const status = worst(assessment.findings.map((finding) => finding.status));
|
|
378
|
+
const firstFinding = markdown
|
|
379
|
+
.parse(section.body, {})
|
|
380
|
+
.find((token) => token.type === 'heading_open' && token.level === 0 && token.tag === 'h3');
|
|
381
|
+
const prelude = reportFields(section.body.split(/\r?\n/).slice(0, firstFinding?.map?.[0]).join('\n'));
|
|
382
|
+
const domainStatuses = [...prelude.matchAll(/^[ \t]*(?:-[ \t]+)?\*\*(?:Domain )?Status:?\*\*[ \t]*:?[ \t]*([^\n]+)$/gim)].map((match) =>
|
|
383
|
+
displayStatus(match[1]),
|
|
384
|
+
);
|
|
385
|
+
const domainStatus = domainStatuses.length === 1 ? domainStatuses[0] : null;
|
|
386
|
+
if (assessment.status !== status || domainStatus !== status || gate.audited_domains[domain] !== status)
|
|
387
|
+
bad(`${domain} status contradicts its finding rollup`);
|
|
388
|
+
}
|
|
389
|
+
if (
|
|
390
|
+
gate.overall_status !== worst(Object.values(gate.audited_domains)) ||
|
|
391
|
+
gate.concerns !== concerns ||
|
|
392
|
+
gate.evidence_gaps !== gaps ||
|
|
393
|
+
gate.blockers !== (gate.overall_status === 'FAIL')
|
|
394
|
+
)
|
|
395
|
+
bad('gate overall status or counts contradict the normalized audit');
|
|
396
|
+
const tables = reportSections.filter((section) => section.label === 'Recorded-Only NFR Criteria');
|
|
397
|
+
if (tables.length > 1 || (recorded.size > 0 && tables.length !== 1)) bad('declared recorded-only criteria require one report table');
|
|
398
|
+
if (tables.length === 1) {
|
|
399
|
+
const tokens = markdown.parse(tables[0].body, {});
|
|
400
|
+
const rows = [];
|
|
401
|
+
let row;
|
|
402
|
+
for (const [index, token] of tokens.entries()) {
|
|
403
|
+
if (token.type === 'tr_open') {
|
|
404
|
+
row = [];
|
|
405
|
+
// Markdown silently drops cells beyond the header width. Keep those extras invalid.
|
|
406
|
+
const raw = tables[0].body
|
|
407
|
+
.split(/\r?\n/)
|
|
408
|
+
.slice(...token.map)
|
|
409
|
+
.join('\n')
|
|
410
|
+
.trim();
|
|
411
|
+
const cells = raw.split(/(?<!\\)(?:\\\\)*\|/);
|
|
412
|
+
const count = cells.length - (cells[0].trim() === '' ? 1 : 0) - (cells.at(-1).trim() === '' ? 1 : 0);
|
|
413
|
+
if (count !== 6) bad('recorded-only table must contain exactly six cells per row');
|
|
414
|
+
}
|
|
415
|
+
if (token.type === 'td_open') row?.push(inlineText(tokens[index + 1]));
|
|
416
|
+
if (token.type === 'tr_close' && row?.length > 0) rows.push(row);
|
|
417
|
+
}
|
|
418
|
+
// Categories are display labels; their identity remains the canonical criterion ID.
|
|
419
|
+
for (const row of rows) if (typeof row[1] === 'string') row[1] = categoryLabel(row[1]);
|
|
420
|
+
const expectedRows = context.recorded_only_nfr_criteria.map((criterion) => [
|
|
421
|
+
criterion.id,
|
|
422
|
+
categoryLabel(criterion.domain),
|
|
423
|
+
criterion.label,
|
|
424
|
+
criterion.threshold,
|
|
425
|
+
criterion.threshold_source,
|
|
426
|
+
'RECORDED ONLY',
|
|
427
|
+
]);
|
|
428
|
+
if (!same(rows, expectedRows)) bad('recorded-only table adds, omits, reorders or assesses criteria');
|
|
429
|
+
}
|
|
430
|
+
return { status: gate.overall_status, gate, progress, context };
|
|
431
|
+
}
|
|
432
|
+
module.exports = { readNfrReport, criterionId, worst, DOMAINS };
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
/** Source preservation and transactional publication for evidence audit workflows. */
|
|
2
|
+
'use strict';
|
|
3
|
+
const fs = require('node:fs');
|
|
4
|
+
const path = require('node:path');
|
|
5
|
+
const crypto = require('node:crypto');
|
|
6
|
+
const { WorkflowError, projectPath } = require('./workflow-cli');
|
|
7
|
+
|
|
8
|
+
/** Compare both canonical paths and existing inode aliases. */
|
|
9
|
+
function aliases(left, right) {
|
|
10
|
+
if (left === right) return true;
|
|
11
|
+
if (!fs.existsSync(left) || !fs.existsSync(right)) return false;
|
|
12
|
+
const a = fs.statSync(left);
|
|
13
|
+
const b = fs.statSync(right);
|
|
14
|
+
return a.dev === b.dev && a.ino === b.ino;
|
|
15
|
+
}
|
|
16
|
+
/** Test a canonical directory boundary. */
|
|
17
|
+
function within(root, candidate) {
|
|
18
|
+
const relative = path.relative(root, candidate);
|
|
19
|
+
return relative === '' || (relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative));
|
|
20
|
+
}
|
|
21
|
+
/** Inventory supplied directories, including symlink topology, without traversing cycles. */
|
|
22
|
+
function inventory(projectRoot, values) {
|
|
23
|
+
const files = new Set();
|
|
24
|
+
const directories = new Set();
|
|
25
|
+
const topology = [];
|
|
26
|
+
const visited = new Set();
|
|
27
|
+
function walk(value) {
|
|
28
|
+
const lexical = path.resolve(projectRoot, value);
|
|
29
|
+
const target = projectPath(projectRoot, value, 'supplied source');
|
|
30
|
+
const link = fs.lstatSync(lexical);
|
|
31
|
+
const stat = fs.statSync(target);
|
|
32
|
+
topology.push([lexical, link.isSymbolicLink() ? fs.readlinkSync(lexical) : '', target, stat.dev, stat.ino, link.mode, stat.mode]);
|
|
33
|
+
if (stat.isFile()) files.add(target);
|
|
34
|
+
else if (stat.isDirectory()) {
|
|
35
|
+
directories.add(target);
|
|
36
|
+
if (visited.has(target)) return;
|
|
37
|
+
visited.add(target);
|
|
38
|
+
for (const entry of fs.readdirSync(target).sort()) {
|
|
39
|
+
if (['.git', 'node_modules'].includes(entry)) continue;
|
|
40
|
+
walk(path.join(target, entry));
|
|
41
|
+
}
|
|
42
|
+
} else throw new WorkflowError('usage', 'supplied sources must contain regular files or directories');
|
|
43
|
+
}
|
|
44
|
+
try {
|
|
45
|
+
for (const value of values) walk(value);
|
|
46
|
+
} catch (error) {
|
|
47
|
+
throw error instanceof WorkflowError
|
|
48
|
+
? error
|
|
49
|
+
: new WorkflowError('usage', `unreadable supplied source: ${error.message}`, { cause: error });
|
|
50
|
+
}
|
|
51
|
+
return { files: [...files].sort(), directories: [...directories].sort(), topology: topology.sort((a, b) => a[0].localeCompare(b[0])) };
|
|
52
|
+
}
|
|
53
|
+
/** Freeze both source bytes and supplied directory membership. */
|
|
54
|
+
function protectSources(projectRoot, values) {
|
|
55
|
+
const state = inventory(projectRoot, values);
|
|
56
|
+
const digests = state.files.map((file) => [file, crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex')]);
|
|
57
|
+
return {
|
|
58
|
+
...state,
|
|
59
|
+
assertUnchanged() {
|
|
60
|
+
try {
|
|
61
|
+
const current = inventory(projectRoot, values);
|
|
62
|
+
if (JSON.stringify(current) !== JSON.stringify(state)) throw new Error('source topology changed');
|
|
63
|
+
for (const [file, digest] of digests) {
|
|
64
|
+
if (crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex') !== digest) throw new Error(file);
|
|
65
|
+
}
|
|
66
|
+
} catch (error) {
|
|
67
|
+
throw new WorkflowError('environment-parser', `the agent changed a supplied source: ${error.message}`, { cause: error });
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
/** Reject output destinations sharing source paths, inodes, or supplied directory descendants. */
|
|
73
|
+
function protectPublication(projectRoot, destinations, protection) {
|
|
74
|
+
const canonical = destinations.map((file) => projectPath(projectRoot, file, 'output destination'));
|
|
75
|
+
if (
|
|
76
|
+
canonical.some(
|
|
77
|
+
(file) => protection.files.some((input) => aliases(file, input)) || protection.directories.some((dir) => within(dir, file)),
|
|
78
|
+
)
|
|
79
|
+
)
|
|
80
|
+
throw new WorkflowError('usage', 'output destinations overlap supplied sources; select separate output and evidence directories');
|
|
81
|
+
if (canonical.some((file, index) => canonical.slice(0, index).some((other) => aliases(file, other))))
|
|
82
|
+
throw new WorkflowError('usage', 'output destinations alias each other');
|
|
83
|
+
return canonical;
|
|
84
|
+
}
|
|
85
|
+
/** Read only a new regular file from the current attempt. */
|
|
86
|
+
function freshArtifact(attemptDir, value) {
|
|
87
|
+
const lexical = path.resolve(attemptDir, value);
|
|
88
|
+
let stat;
|
|
89
|
+
try {
|
|
90
|
+
stat = fs.lstatSync(lexical);
|
|
91
|
+
} catch (error) {
|
|
92
|
+
throw new WorkflowError('environment-parser', `required artifact is unavailable: ${value}`, { cause: error });
|
|
93
|
+
}
|
|
94
|
+
const target = projectPath(attemptDir, lexical, 'generated artifact');
|
|
95
|
+
if (!stat.isFile() || stat.nlink !== 1 || target !== lexical)
|
|
96
|
+
throw new WorkflowError('environment-parser', `generated artifact must be a fresh regular file in this attempt: ${value}`);
|
|
97
|
+
const text = fs.readFileSync(target, 'utf8');
|
|
98
|
+
if (!text.trim()) throw new WorkflowError('environment-parser', `generated artifact is empty: ${value}`);
|
|
99
|
+
return { path: target, text };
|
|
100
|
+
}
|
|
101
|
+
/** Stage the complete artifact set before replacing files, and roll back partial publication. */
|
|
102
|
+
function publishArtifacts(artifacts, { projectRoot, runDir, protection }, io = fs) {
|
|
103
|
+
const targets = protectPublication(
|
|
104
|
+
projectRoot,
|
|
105
|
+
artifacts.map((artifact) => artifact.destination),
|
|
106
|
+
protection,
|
|
107
|
+
);
|
|
108
|
+
const staged = [];
|
|
109
|
+
let keepBackups = false;
|
|
110
|
+
try {
|
|
111
|
+
for (const [index, artifact] of artifacts.entries()) {
|
|
112
|
+
const target = targets[index];
|
|
113
|
+
io.mkdirSync(path.dirname(target), { recursive: true });
|
|
114
|
+
const directory = io.mkdtempSync(path.join(path.dirname(target), '.tea-publish-'));
|
|
115
|
+
const item = {
|
|
116
|
+
target,
|
|
117
|
+
directory,
|
|
118
|
+
next: path.join(directory, 'next'),
|
|
119
|
+
previous: path.join(directory, 'previous'),
|
|
120
|
+
existed: io.existsSync(target),
|
|
121
|
+
installed: false,
|
|
122
|
+
};
|
|
123
|
+
staged.push(item);
|
|
124
|
+
if (item.existed) {
|
|
125
|
+
if (!io.statSync(target).isFile()) throw new Error(`publication destination is not a regular file: ${target}`);
|
|
126
|
+
io.copyFileSync(target, item.previous);
|
|
127
|
+
}
|
|
128
|
+
if (typeof artifact.text === 'string') io.writeFileSync(item.next, artifact.text, { flag: 'wx' });
|
|
129
|
+
else io.copyFileSync(artifact.path, item.next);
|
|
130
|
+
if (item.existed) io.chmodSync(item.next, io.statSync(target).mode);
|
|
131
|
+
}
|
|
132
|
+
protection.assertUnchanged();
|
|
133
|
+
protectPublication(projectRoot, targets, protection);
|
|
134
|
+
for (const item of staged) {
|
|
135
|
+
io.renameSync(item.next, item.target);
|
|
136
|
+
item.installed = true;
|
|
137
|
+
}
|
|
138
|
+
return targets;
|
|
139
|
+
} catch (error_) {
|
|
140
|
+
const rollbackErrors = [];
|
|
141
|
+
for (const item of staged.filter((entry) => entry.installed).toReversed()) {
|
|
142
|
+
try {
|
|
143
|
+
if (item.existed) io.renameSync(item.previous, item.target);
|
|
144
|
+
else io.unlinkSync(item.target);
|
|
145
|
+
} catch (error) {
|
|
146
|
+
rollbackErrors.push(`${item.target}: ${error.message}`);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
keepBackups = rollbackErrors.length > 0;
|
|
150
|
+
const status = keepBackups
|
|
151
|
+
? `recovery backups: ${staged.map((item) => item.directory).join(', ')}; rollback errors: ${rollbackErrors.join('; ')}`
|
|
152
|
+
: 'previous artifacts restored';
|
|
153
|
+
const failureClass = error_ instanceof WorkflowError ? error_.failureClass : 'environment-configuration';
|
|
154
|
+
const error = new WorkflowError(failureClass, `could not publish artifacts: ${error_.message}; ${status}`, {
|
|
155
|
+
cause: error_,
|
|
156
|
+
});
|
|
157
|
+
error.runDir = runDir;
|
|
158
|
+
throw error;
|
|
159
|
+
} finally {
|
|
160
|
+
if (!keepBackups)
|
|
161
|
+
for (const item of staged) {
|
|
162
|
+
try {
|
|
163
|
+
io.rmSync(item.directory, { recursive: true, force: true });
|
|
164
|
+
} catch (error) {
|
|
165
|
+
process.stderr.write(`publication staging cleanup failed at ${item.directory}: ${error.message}\n`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
module.exports = { inventory, protectSources, protectPublication, freshArtifact, publishArtifacts };
|
package/cli/nfr.js
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/** Audit existing project NFR evidence with the packaged skill. */
|
|
3
|
+
'use strict';
|
|
4
|
+
const fs = require('node:fs');
|
|
5
|
+
const path = require('node:path');
|
|
6
|
+
const crypto = require('node:crypto');
|
|
7
|
+
const { Command } = require('commander');
|
|
8
|
+
const { resolveTeaConfig, CONFIG_LAYER_RELATIVE_PATHS, LEGACY_CONFIG_RELATIVE_PATH } = require('./lib/resolve-tea-config');
|
|
9
|
+
const { inventory, protectSources, protectPublication, freshArtifact, publishArtifacts } = require('./lib/workflow-publication');
|
|
10
|
+
const { readNfrReport } = require('./lib/nfr-report');
|
|
11
|
+
const {
|
|
12
|
+
WorkflowError,
|
|
13
|
+
addAgentOptions,
|
|
14
|
+
readInput,
|
|
15
|
+
projectPath,
|
|
16
|
+
resolveWorkflowSkill,
|
|
17
|
+
headlessPrompt,
|
|
18
|
+
runWithEvidence,
|
|
19
|
+
mainError,
|
|
20
|
+
} = require('./lib/workflow-cli');
|
|
21
|
+
/** Accumulate repeatable Commander path options without replacing earlier values. */
|
|
22
|
+
const collect = (value, previous) => [...previous, value];
|
|
23
|
+
/** Resolve a nonempty explicitly supplied source role. */
|
|
24
|
+
function sourceFiles(projectRoot, values, label) {
|
|
25
|
+
const { files } = inventory(projectRoot, values);
|
|
26
|
+
if (files.length === 0) throw new WorkflowError('usage', `${label} must supply at least one regular file`);
|
|
27
|
+
return files;
|
|
28
|
+
}
|
|
29
|
+
/** Verify unchanged sources and fresh actual workflow outputs. */
|
|
30
|
+
function validateAudit(request) {
|
|
31
|
+
request.protection.assertUnchanged();
|
|
32
|
+
const report = freshArtifact(request.attemptDir, request.reportFile);
|
|
33
|
+
const contextFile = freshArtifact(request.attemptDir, request.contextFile);
|
|
34
|
+
let context;
|
|
35
|
+
try {
|
|
36
|
+
context = JSON.parse(contextFile.text);
|
|
37
|
+
} catch (error) {
|
|
38
|
+
throw new WorkflowError('environment-parser', `invalid NFR context JSON: ${error.message}`, { cause: error });
|
|
39
|
+
}
|
|
40
|
+
return { ...readNfrReport(report.text, context, request), artifacts: [report, contextFile] };
|
|
41
|
+
}
|
|
42
|
+
/** Invoke the actual installed Create workflow and publish its validated audit. */
|
|
43
|
+
function run(argv) {
|
|
44
|
+
const program = addAgentOptions(
|
|
45
|
+
new Command().name('tea-nfr').description('Audit existing NFR evidence and publish a verified scope-specific report.'),
|
|
46
|
+
);
|
|
47
|
+
program
|
|
48
|
+
.option('--project-root <dir>', 'consuming project root', process.cwd())
|
|
49
|
+
.option('--skill-root <dir>', 'trusted skill override; packaged skill by default')
|
|
50
|
+
.requiredOption('--input <path>', 'requirements document (repeatable)', collect, [])
|
|
51
|
+
.requiredOption('--implementation <path>', 'accessible implementation file or directory (repeatable)', collect, [])
|
|
52
|
+
.requiredOption('--evidence <path>', 'existing evidence file or directory (repeatable)', collect, [])
|
|
53
|
+
.option('--scope <name>', 'system, epic or story', 'system')
|
|
54
|
+
.option('--scope-id <id>', 'epic or story stable lowercase ID; required for narrower scope')
|
|
55
|
+
.option('--output-dir <dir>', 'published artifact root inside the project; defaults to TEA test_artifacts')
|
|
56
|
+
.option('--evidence-dir <dir>', 'retained live prompts, streams and generated reports', '.tea-runs')
|
|
57
|
+
.option('--fail-on <status>', 'gate threshold: concerns, fail or none', 'concerns')
|
|
58
|
+
.setOptionValue('retries', '0')
|
|
59
|
+
.exitOverride()
|
|
60
|
+
.configureOutput({ writeErr: () => {} });
|
|
61
|
+
program.options.find((option) => option.long === '--retries').defaultValue = '0';
|
|
62
|
+
try {
|
|
63
|
+
program.parse(argv);
|
|
64
|
+
} catch (error) {
|
|
65
|
+
if (error.exitCode === 0) return 0;
|
|
66
|
+
throw new WorkflowError('usage', error.message);
|
|
67
|
+
}
|
|
68
|
+
const options = program.opts();
|
|
69
|
+
if (options.input.length === 0) throw new WorkflowError('usage', '--input requires at least one requirements document');
|
|
70
|
+
let projectRoot;
|
|
71
|
+
try {
|
|
72
|
+
projectRoot = fs.realpathSync(path.resolve(options.projectRoot));
|
|
73
|
+
} catch (error) {
|
|
74
|
+
throw new WorkflowError('usage', '--project-root must exist', { cause: error });
|
|
75
|
+
}
|
|
76
|
+
if (!['system', 'epic', 'story'].includes(options.scope)) throw new WorkflowError('usage', '--scope must be system, epic or story');
|
|
77
|
+
if (options.scope === 'system' && options.scopeId) throw new WorkflowError('usage', '--scope-id requires epic or story scope');
|
|
78
|
+
if (options.scope !== 'system' && (!options.scopeId || !/^[a-z0-9][a-z0-9-]{0,63}$/.test(options.scopeId)))
|
|
79
|
+
throw new WorkflowError('usage', 'epic or story scope requires a stable lowercase --scope-id');
|
|
80
|
+
if (!['concerns', 'fail', 'none'].includes(options.failOn)) throw new WorkflowError('usage', '--fail-on must be concerns, fail or none');
|
|
81
|
+
const configurationInputs = [...CONFIG_LAYER_RELATIVE_PATHS, LEGACY_CONFIG_RELATIVE_PATH]
|
|
82
|
+
.filter((file) => fs.existsSync(path.join(projectRoot, file)))
|
|
83
|
+
.map((file) => readInput(projectRoot, file, 'configuration'));
|
|
84
|
+
const inputs = [...new Set([...options.input.map((file) => readInput(projectRoot, file, '--input')), ...configurationInputs])];
|
|
85
|
+
const implementation = sourceFiles(projectRoot, options.implementation, '--implementation');
|
|
86
|
+
const evidenceFiles = sourceFiles(projectRoot, options.evidence, '--evidence');
|
|
87
|
+
if (
|
|
88
|
+
inputs.some((file) =>
|
|
89
|
+
evidenceFiles.some((evidence) => {
|
|
90
|
+
const a = fs.statSync(file),
|
|
91
|
+
b = fs.statSync(evidence);
|
|
92
|
+
return file === evidence || (a.dev === b.dev && a.ino === b.ino);
|
|
93
|
+
}),
|
|
94
|
+
)
|
|
95
|
+
)
|
|
96
|
+
throw new WorkflowError('usage', 'requirement inputs cannot also be implementation evidence');
|
|
97
|
+
const protection = protectSources(projectRoot, [...inputs, ...options.implementation, ...options.evidence]);
|
|
98
|
+
const skillRoot = resolveWorkflowSkill({ projectRoot, skillName: 'bmad-testarch-nfr', skillRoot: options.skillRoot });
|
|
99
|
+
const resolvedConfig = resolveTeaConfig({ projectRoot, skillRoot, skillName: 'bmad-testarch-nfr' });
|
|
100
|
+
const artifactRoot = options.outputDir ?? resolvedConfig.configSnapshot.modules.tea.test_artifacts;
|
|
101
|
+
if (typeof artifactRoot !== 'string' || !artifactRoot.trim())
|
|
102
|
+
throw new WorkflowError('environment-configuration', '--output-dir or TEA test_artifacts must be a nonempty string');
|
|
103
|
+
const outputDir = projectPath(projectRoot, artifactRoot.replaceAll('{project-root}', projectRoot), '--output-dir');
|
|
104
|
+
const runKey = options.scope === 'system' ? 'system' : `${options.scope}-${options.scopeId}`;
|
|
105
|
+
const filename = `nfr-assessment-${runKey}.md`;
|
|
106
|
+
const destination = projectPath(projectRoot, path.join(outputDir, 'nfr', filename), 'published report');
|
|
107
|
+
const contextFilename = `nfr-context-${runKey}.json`;
|
|
108
|
+
const contextDestination = projectPath(projectRoot, path.join(outputDir, 'nfr', contextFilename), 'published context');
|
|
109
|
+
const destinations = [destination, contextDestination];
|
|
110
|
+
const checkDestination = () => protectPublication(projectRoot, [...destinations, options.evidenceDir], protection);
|
|
111
|
+
checkDestination();
|
|
112
|
+
const result = runWithEvidence({
|
|
113
|
+
name: 'tea-nfr',
|
|
114
|
+
projectRoot,
|
|
115
|
+
evidenceRoot: options.evidenceDir,
|
|
116
|
+
options,
|
|
117
|
+
prepare({ attemptDir }) {
|
|
118
|
+
const artifactsRoot = path.join(attemptDir, 'artifacts');
|
|
119
|
+
const config = {
|
|
120
|
+
...resolvedConfig,
|
|
121
|
+
configSnapshot: {
|
|
122
|
+
core: resolvedConfig.configSnapshot.core,
|
|
123
|
+
modules: { tea: { ...resolvedConfig.configSnapshot.modules.tea, test_artifacts: artifactsRoot } },
|
|
124
|
+
},
|
|
125
|
+
};
|
|
126
|
+
const reportFile = path.join('artifacts', 'nfr', filename);
|
|
127
|
+
const contextFile = path.join('artifacts', 'nfr', contextFilename);
|
|
128
|
+
const requestId = crypto.randomUUID();
|
|
129
|
+
const contract = {
|
|
130
|
+
schemaVersion: 1,
|
|
131
|
+
requestId,
|
|
132
|
+
runScope: options.scope,
|
|
133
|
+
runKey,
|
|
134
|
+
supplied_project_root: projectRoot,
|
|
135
|
+
declared_nfr_criteria: Object.fromEntries(
|
|
136
|
+
['security', 'performance', 'reliability', 'maintainability'].map((domain) => [domain, []]),
|
|
137
|
+
),
|
|
138
|
+
recorded_only_nfr_criteria: [],
|
|
139
|
+
supplied_evidence_ledger: [],
|
|
140
|
+
domain_assessments: Object.fromEntries(
|
|
141
|
+
['security', 'performance', 'reliability', 'maintainability'].map((domain) => [
|
|
142
|
+
domain,
|
|
143
|
+
{ status: 'N/A', findings: [], evidence_gaps: [] },
|
|
144
|
+
]),
|
|
145
|
+
),
|
|
146
|
+
};
|
|
147
|
+
const requestLines = [
|
|
148
|
+
`Start Create mode at ${JSON.stringify(path.join(skillRoot, 'steps-c', 'step-01-load-context.md'))}.`,
|
|
149
|
+
`run_scope=${options.scope}; run_key=${runKey}; tea_browser_automation=none`,
|
|
150
|
+
`supplied_project_root=${JSON.stringify(projectRoot)}`,
|
|
151
|
+
`Requirement inputs are exactly this JSON list: ${JSON.stringify(inputs)}.`,
|
|
152
|
+
`Implementation files are accessible at exactly this JSON list: ${JSON.stringify(implementation)}.`,
|
|
153
|
+
`Existing implementation evidence is exactly this JSON list: ${JSON.stringify(evidenceFiles)}.`,
|
|
154
|
+
'Read and audit this existing implementation evidence. Preserve all supplied files. Read-only file inspection is allowed. Do not execute project tests, build scripts, deploy scripts, CI jobs or browser exploration.',
|
|
155
|
+
`Produce a completed scope-specific audit at ${JSON.stringify(path.join(attemptDir, reportFile))}.`,
|
|
156
|
+
`Persist your actual step02/03/04E canonical workflow context at ${JSON.stringify(path.join(attemptDir, contextFile))}. Required shape (empty arrays are shape examples, fill all actual declared criteria): ${JSON.stringify(contract)}.`,
|
|
157
|
+
'Each declared criterion has {id,order,label,threshold,threshold_source}; use actual stable criterion IDs and source order. Each recorded-only criterion also has domain and assessment_mode=recorded-only.',
|
|
158
|
+
'Use canonical project-relative threshold_source paths from requirement inputs. Copy the threshold as a literal source excerpt (line wrapping may normalize), or UNKNOWN. Do not invent threshold policy.',
|
|
159
|
+
'Each ledger entry has {path,source: supplied,source_type: implementation-evidence,domains,observations:[{criterion_id,supports}]}. Copy supports as literal factual excerpts from that exact supplied file. Sort paths and observations; deduplicate. Include all actual supported criterion bindings.',
|
|
160
|
+
'Each domain_assessments entry has {status,findings:[{criterion_id,status,evidence:[{path,supports}]}],evidence_gaps:[{criterion_id,message}]}. Match every declared criterion in source order, with all its bound ledger observations. Missing evidence has one gap with message "Label: no supplied implementation evidence". UNKNOWN threshold with evidence has one gap with message "Label: UNKNOWN threshold". Replace Label with the exact criterion label. Never manufacture implementation evidence from requirement documents.',
|
|
161
|
+
'Follow the actual workflow and checklist. Preserve completed steps and run identity in frontmatter. Include exactly one Gate YAML Snippet with nfr_assessment and the four audited_domains.',
|
|
162
|
+
'Under every audited domain, render each declared criterion as a ### subsection with separate bold Status, Threshold, Threshold Source, Actual, Evidence and Supports fields. Threshold Source is one backtick-quoted canonical requirement path. Actual is a copied implementation source excerpt, or UNKNOWN. Evidence lists all bound ledger paths in backticks. Supports includes each literal bound excerpt. With no evidence, use Evidence: None, Supports: None and Actual: UNKNOWN. Evidence paths must be backtick-quoted paths relative to the supplied project root and present in the supplied evidence list. Requirements are threshold sources.',
|
|
163
|
+
'Preserve the completed Create progress steps in exact workflow order and a valid lastSaved timestamp. Keep all populated report-template sections. Gate concerns/evidence_gaps counts equal normalized criterion concerns/gaps. Recorded-only declarations use a separate Recorded-Only NFR Criteria table with ID, category, label, threshold, source and RECORDED ONLY mode; they have no audited finding/status/gate impact.',
|
|
164
|
+
'Use only declared requirements and existing evidence. Record UNKNOWN thresholds and missing evidence as required by the skill. Complete all workflow steps without modifying source inputs.',
|
|
165
|
+
];
|
|
166
|
+
return {
|
|
167
|
+
prompt: headlessPrompt({ skillRoot, projectRoot, resolvedConfig: config, requestLines }),
|
|
168
|
+
reportFile,
|
|
169
|
+
projectRoot,
|
|
170
|
+
runScope: options.scope,
|
|
171
|
+
runKey,
|
|
172
|
+
requestId,
|
|
173
|
+
contextFile,
|
|
174
|
+
protection,
|
|
175
|
+
inputs,
|
|
176
|
+
evidenceFiles,
|
|
177
|
+
};
|
|
178
|
+
},
|
|
179
|
+
validate: validateAudit,
|
|
180
|
+
});
|
|
181
|
+
if (result.dryRun) {
|
|
182
|
+
process.stdout.write(
|
|
183
|
+
`${JSON.stringify({ mode: 'prompt-only', evidence: result.runDir, prompt: path.join(result.attemptDir, 'prompt.txt') })}\n`,
|
|
184
|
+
);
|
|
185
|
+
return 0;
|
|
186
|
+
}
|
|
187
|
+
checkDestination();
|
|
188
|
+
const published = publishArtifacts(
|
|
189
|
+
result.value.artifacts.map((artifact, index) => ({ ...artifact, destination: destinations[index] })),
|
|
190
|
+
{ projectRoot, runDir: result.runDir, protection },
|
|
191
|
+
);
|
|
192
|
+
const failed =
|
|
193
|
+
options.failOn !== 'none' && (result.value.status === 'FAIL' || (options.failOn === 'concerns' && result.value.status === 'CONCERNS'));
|
|
194
|
+
const gatePassed = !failed;
|
|
195
|
+
process.stdout.write(
|
|
196
|
+
`${JSON.stringify({ mode: 'live', runKey, status: result.value.status, gatePassed, evidence: result.runDir, report: published[0], context: published[1], gate: result.value.gate })}\n`,
|
|
197
|
+
);
|
|
198
|
+
return failed ? 1 : 0;
|
|
199
|
+
}
|
|
200
|
+
/** Run the public command with stable configuration and execution exit codes. */
|
|
201
|
+
function main(argv) {
|
|
202
|
+
try {
|
|
203
|
+
return run(argv);
|
|
204
|
+
} catch (error) {
|
|
205
|
+
return mainError('tea-nfr', error);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
if (require.main === module) process.exitCode = main(process.argv);
|
|
209
|
+
module.exports = { main, validateAudit, sourceFiles };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json.schemastore.org/package.json",
|
|
3
3
|
"name": "bmad-method-test-architecture-enterprise",
|
|
4
|
-
"version": "2.0.4-next.
|
|
4
|
+
"version": "2.0.4-next.3",
|
|
5
5
|
"description": "Master Test Architect for quality strategy, test automation, and release gates",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"bmad",
|
|
@@ -38,6 +38,7 @@
|
|
|
38
38
|
"tea-evaluate": "cli/evaluate.js",
|
|
39
39
|
"tea-fragment-selection-runner": "cli/fragment-selection-runner.js",
|
|
40
40
|
"tea-framework": "cli/framework.js",
|
|
41
|
+
"tea-nfr": "cli/nfr.js",
|
|
41
42
|
"tea-nfr-runner": "cli/nfr-runner.js",
|
|
42
43
|
"tea-routing-runner": "cli/routing-runner.js",
|
|
43
44
|
"tea-skill-runner": "cli/skill-runner.js",
|
|
@@ -92,7 +93,7 @@
|
|
|
92
93
|
"release:minor": "gh workflow run publish.yaml -f channel=latest -f bump=minor",
|
|
93
94
|
"release:next": "gh workflow run publish.yaml -f channel=next",
|
|
94
95
|
"release:patch": "gh workflow run publish.yaml -f channel=latest -f bump=patch",
|
|
95
|
-
"test": "npm run test:conflict-markers && npm run test:schemas && npm run test:install && npm run test:evaluate-guidance && npm run test:evaluate-authoring && npm run test:evaluate-gap-loop && npm run test:evaluate-dogfood && npm run test:evaluate-check && npm run test:evaluate-compare && npm run test:evaluate-before-state && npm run test:evaluate-ci-gates && npm run test:evaluate-ci-tiers && npm run test:evaluate-ci-repositories:tagged-release-pr && npm run test:evaluate-ci-repositories:tagged-release-merge && npm run test:evaluate-ci-repositories:tagged-release-release && npm run test:evaluate-ci-repositories:nightly-deploy-pr && npm run test:evaluate-ci-repositories:nightly-deploy-merge && npm run test:evaluate-ci-repositories:nightly-deploy-scheduled && npm run test:evaluate-ci-repositories:nightly-deploy-release && npm run test:evaluate-ci-render && npm run test:evaluate-pr-mcp && npm run test:evaluate-pr-api && npm run test:evaluate-pr-suite && npm run test:evaluate-pr-workflow && npm run test:evaluate-pr-tool-use && npm run test:evaluate-pr-promptfoo && npm run test:evaluate-pr-ai-feature && npm run test:evaluate-pr-test-review && npm run test:evaluate-pr-gap-loop && npm run test:evaluate-pr-learn && npm run test:evaluate-pr-tutorial && npm run test:evaluate-partitions && npm run test:evaluate-partition-plans-views-and-check && npm run test:evaluate-partition-plans-preflight-and-run && npm run test:evaluate-partition-plans-score && npm run test:evaluate-partition-plans-attempts-seal-and-mappings && npm run test:evaluate-partition-plans-attempts-records && npm run test:evaluate-partition-plans-attempts-gameability && npm run test:evaluate-interpret && npm run test:evaluate-calibration && npm run test:evaluate-boundaries && npm run test:evaluate-preflight && npm run test:evaluate-isolation-conformance && npm run test:evaluate-mutation:qualification-basics && npm run test:evaluate-mutation:qualification-guards && npm run test:evaluate-mutation:recovery && npm run test:evaluate-mutation:journals && npm run test:evaluate-run && npm run test:evaluate-progress && npm run test:evaluate-aggregate && npm run test:evaluate-confinement && npm run test:evaluate-confinement-audit && npm run test:evaluate-confinement-history && npm run test:evaluate-confinement-git-state && npm run test:evaluate-confinement-network && npm run test:evaluate-held-inputs && npm run test:evaluate-arms-historical && npm run test:evaluate-arms-deployments && npm run test:evaluate-evaluators && npm run test:evaluate-evaluators-agent-version && npm run test:evaluate-agents && npm run test:evaluate-agents-importers && npm run test:evaluate-agents-sealed-brief && npm run test:evaluate-held-attempts && npm run test:evaluate-private && npm run test:evaluate-records && npm run test:evaluate-mcp && npm run test:evaluate-api && npm run test:evaluate-workflow && npm run test:evaluate-tool-use && npm run test:evaluate-promptfoo && npm run test:evaluate-learned-framework && npm run test:docs-tutorial && npm run test:knowledge && npm run test:criteria-fragments && npm run test:doc-count-sources && npm run test:planning-doc-sources && npm run test:doc-counts && npm run test:doc-claim-sources && npm run test:doc-claims && npm run test:doc-invocation-entry && npm run test:doc-invocations && npm run test:doc-link-anchors && npm run test:docs-build-names && npm run test:docs-evaluate-sidebar && npm run test:docs-gap-reader && npm run test:enforce-hook && npm run test:framework-scaffold && npm run test:isolation-primitives && npm run test:atdd-isolation && npm run test:atdd-net-guard && npm run test:atdd-scoring && npm run test:atdd-workflow-guidance && npm run test:generation-healing && npm run test:framework-scaffold-install-isolation && npm run test:transcript-harness && npm run test:suite-manifest && npm run test:teach-me-testing-scoring && npm run test:eval-data && npm run test:eval-atdd-data && npm run test:eval-ci-data && npm run test:eval-framework-scaffold-data && npm run test:eval-nfr-data && npm run test:nfr-validate-corpus && npm run test:nfr-evidence-guidance && npm run test:eval-test-design-data && npm run test:eval-trace-data && npm run test:eval-routing-data && npm run test:eval-routing-boundaries && npm run test:eval-routing-evidence && npm run test:eval-transcript-data && npm run test:eval-teach-me-testing-data && npm run test:eval-schemas && npm run test:eval-diagnostics && npm run test:eval-replay && npm run test:eval-quality-corpus && npm run test:schema-versions && npm run test:contract-sources && npm run test:contracts && npm run test:contract-oracles && npm run test:probe-sources && npm run test:test-design-qualification && npm run test:test-review-qualification && npm run test:trace-qualification && npm run test:nfr-qualification && npm run test:ci-qualification && npm run test:probe-corpus && npm run test:compare-dominance && npm run test:compare-eval-runs && npm run test:trial-set-scoring && npm run test:automate-cli && npm run test:automate-generation-evidence && npm run test:automate-eval-fixture && npm run test:eval-automate-data && npm run test:clock-port && npm run test:probe-conformance && npm run test:corpus-conformance && npm run test:file-system-conformance && npm run test:file-system-port && npm run test:probe-targets && npm run test:port-totality && npm run test:ci-coverage && npm run test:ci-coverage-filters && npm run test:install-actionlint && npm run test:workflows-lint && npm run test:install-linux-sandbox && npm run test:shards && npm run test:groups && npm run test:cli && npm run test:workflow-cli && npm run test:framework-cli && npm run test:test-review-pr-scope && npm run test:test-review-report && npm run test:test-review-rubric && npm run test:bmad-output-gated && npm run test:release-metadata && npm run test:release-notes && npm run test:lockfile-age && npm run test:lockfile-age-cache && npm run test:licences && npm run test:supply-chain && npm run test:boundary && npm run test:direction && npm run test:lineage && npm run test:guard-publish && npm run test:layering-boundary-lineage && npm run test:changelog && npm run test:tea-workflow-descriptions && npm run test:docs-mermaid && npm run test:llms-headroom && npm run validate:schemas && npm run lint && npm run lint:md && npm run format:check",
|
|
96
|
+
"test": "npm run test:conflict-markers && npm run test:schemas && npm run test:install && npm run test:evaluate-guidance && npm run test:evaluate-authoring && npm run test:evaluate-gap-loop && npm run test:evaluate-dogfood && npm run test:evaluate-check && npm run test:evaluate-compare && npm run test:evaluate-before-state && npm run test:evaluate-ci-gates && npm run test:evaluate-ci-tiers && npm run test:evaluate-ci-repositories:tagged-release-pr && npm run test:evaluate-ci-repositories:tagged-release-merge && npm run test:evaluate-ci-repositories:tagged-release-release && npm run test:evaluate-ci-repositories:nightly-deploy-pr && npm run test:evaluate-ci-repositories:nightly-deploy-merge && npm run test:evaluate-ci-repositories:nightly-deploy-scheduled && npm run test:evaluate-ci-repositories:nightly-deploy-release && npm run test:evaluate-ci-render && npm run test:evaluate-pr-mcp && npm run test:evaluate-pr-api && npm run test:evaluate-pr-suite && npm run test:evaluate-pr-workflow && npm run test:evaluate-pr-tool-use && npm run test:evaluate-pr-promptfoo && npm run test:evaluate-pr-ai-feature && npm run test:evaluate-pr-test-review && npm run test:evaluate-pr-gap-loop && npm run test:evaluate-pr-learn && npm run test:evaluate-pr-tutorial && npm run test:evaluate-partitions && npm run test:evaluate-partition-plans-views-and-check && npm run test:evaluate-partition-plans-preflight-and-run && npm run test:evaluate-partition-plans-score && npm run test:evaluate-partition-plans-attempts-seal-and-mappings && npm run test:evaluate-partition-plans-attempts-records && npm run test:evaluate-partition-plans-attempts-gameability && npm run test:evaluate-interpret && npm run test:evaluate-calibration && npm run test:evaluate-boundaries && npm run test:evaluate-preflight && npm run test:evaluate-isolation-conformance && npm run test:evaluate-mutation:qualification-basics && npm run test:evaluate-mutation:qualification-guards && npm run test:evaluate-mutation:recovery && npm run test:evaluate-mutation:journals && npm run test:evaluate-run && npm run test:evaluate-progress && npm run test:evaluate-aggregate && npm run test:evaluate-confinement && npm run test:evaluate-confinement-audit && npm run test:evaluate-confinement-history && npm run test:evaluate-confinement-git-state && npm run test:evaluate-confinement-network && npm run test:evaluate-held-inputs && npm run test:evaluate-arms-historical && npm run test:evaluate-arms-deployments && npm run test:evaluate-evaluators && npm run test:evaluate-evaluators-agent-version && npm run test:evaluate-agents && npm run test:evaluate-agents-importers && npm run test:evaluate-agents-sealed-brief && npm run test:evaluate-held-attempts && npm run test:evaluate-private && npm run test:evaluate-records && npm run test:evaluate-mcp && npm run test:evaluate-api && npm run test:evaluate-workflow && npm run test:evaluate-tool-use && npm run test:evaluate-promptfoo && npm run test:evaluate-learned-framework && npm run test:docs-tutorial && npm run test:knowledge && npm run test:criteria-fragments && npm run test:doc-count-sources && npm run test:planning-doc-sources && npm run test:doc-counts && npm run test:doc-claim-sources && npm run test:doc-claims && npm run test:doc-invocation-entry && npm run test:doc-invocations && npm run test:doc-link-anchors && npm run test:docs-build-names && npm run test:docs-evaluate-sidebar && npm run test:docs-gap-reader && npm run test:enforce-hook && npm run test:framework-scaffold && npm run test:isolation-primitives && npm run test:atdd-isolation && npm run test:atdd-net-guard && npm run test:atdd-scoring && npm run test:atdd-workflow-guidance && npm run test:generation-healing && npm run test:framework-scaffold-install-isolation && npm run test:transcript-harness && npm run test:suite-manifest && npm run test:teach-me-testing-scoring && npm run test:eval-data && npm run test:eval-atdd-data && npm run test:eval-ci-data && npm run test:eval-framework-scaffold-data && npm run test:eval-nfr-data && npm run test:nfr-validate-corpus && npm run test:nfr-evidence-guidance && npm run test:nfr-cli && npm run test:eval-test-design-data && npm run test:eval-trace-data && npm run test:eval-routing-data && npm run test:eval-routing-boundaries && npm run test:eval-routing-evidence && npm run test:eval-transcript-data && npm run test:eval-teach-me-testing-data && npm run test:eval-schemas && npm run test:eval-diagnostics && npm run test:eval-replay && npm run test:eval-quality-corpus && npm run test:schema-versions && npm run test:contract-sources && npm run test:contracts && npm run test:contract-oracles && npm run test:probe-sources && npm run test:test-design-qualification && npm run test:test-review-qualification && npm run test:trace-qualification && npm run test:nfr-qualification && npm run test:ci-qualification && npm run test:probe-corpus && npm run test:compare-dominance && npm run test:compare-eval-runs && npm run test:trial-set-scoring && npm run test:automate-cli && npm run test:automate-generation-evidence && npm run test:automate-eval-fixture && npm run test:eval-automate-data && npm run test:clock-port && npm run test:probe-conformance && npm run test:corpus-conformance && npm run test:file-system-conformance && npm run test:file-system-port && npm run test:probe-targets && npm run test:port-totality && npm run test:ci-coverage && npm run test:ci-coverage-filters && npm run test:install-actionlint && npm run test:workflows-lint && npm run test:install-linux-sandbox && npm run test:shards && npm run test:groups && npm run test:cli && npm run test:workflow-cli && npm run test:framework-cli && npm run test:test-review-pr-scope && npm run test:test-review-report && npm run test:test-review-rubric && npm run test:bmad-output-gated && npm run test:release-metadata && npm run test:release-notes && npm run test:lockfile-age && npm run test:lockfile-age-cache && npm run test:licences && npm run test:supply-chain && npm run test:boundary && npm run test:direction && npm run test:lineage && npm run test:guard-publish && npm run test:layering-boundary-lineage && npm run test:changelog && npm run test:tea-workflow-descriptions && npm run test:docs-mermaid && npm run test:llms-headroom && npm run validate:schemas && npm run lint && npm run lint:md && npm run format:check",
|
|
96
97
|
"test:atdd-isolation": "node test/test-atdd-isolation.js",
|
|
97
98
|
"test:atdd-net-guard": "node test/test-atdd-net-guard.js",
|
|
98
99
|
"test:atdd-scoring": "node test/test-atdd-scoring.js",
|
|
@@ -239,6 +240,7 @@
|
|
|
239
240
|
"test:llms-headroom": "node test/test-llms-headroom.js",
|
|
240
241
|
"test:lockfile-age": "eval-quality-gates lockfile-age",
|
|
241
242
|
"test:lockfile-age-cache": "node test/test-generate-lockfile-age-cache.js",
|
|
243
|
+
"test:nfr-cli": "node --test test/test-nfr-cli.js",
|
|
242
244
|
"test:nfr-evidence-guidance": "node test/test-nfr-evidence-guidance.js",
|
|
243
245
|
"test:nfr-qualification": "node test/test-nfr-qualification.js",
|
|
244
246
|
"test:nfr-validate-corpus": "node test/test-nfr-validate-corpus.js",
|