bmad-method-test-architecture-enterprise 2.0.4-next.2 → 2.0.4-next.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +1 -0
- package/cli/lib/evaluate/http-target.js +28 -8
- package/cli/lib/nfr-report.js +432 -0
- package/cli/lib/resolve-tea-config.js +12 -4
- package/cli/lib/teach-turn.js +558 -0
- package/cli/lib/workflow-publication.js +170 -0
- package/cli/nfr.js +209 -0
- package/cli/teach.js +433 -0
- package/package.json +7 -3
- package/skills/bmad-teach-me-testing/steps-c/step-01-init.md +1 -2
- package/skills/bmad-teach-me-testing/steps-c/step-01b-continue.md +1 -1
- package/skills/bmad-teach-me-testing/steps-c/step-04-session-01.md +1 -1
- package/skills/bmad-teach-me-testing/steps-c/step-04-session-02.md +1 -1
- package/skills/bmod-tea/bmod.toml +1 -1
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"name": "bmad-method-test-architecture-enterprise",
|
|
32
32
|
"source": "./",
|
|
33
33
|
"description": "Master Test Architect module for quality strategy, test generation in red and expand modes with automatic run-and-heal, combined framework and CI setup, CI/CD quality gates, and structured testing education. The CI and ATDD commands remain available with existing customizations. Part of the BMad Method ecosystem.",
|
|
34
|
-
"version": "2.0.4-next.
|
|
34
|
+
"version": "2.0.4-next.4",
|
|
35
35
|
"author": {
|
|
36
36
|
"name": "Murat K Ozcan (TEA Creator) & Brian (BMad) Madison"
|
|
37
37
|
},
|
package/README.md
CHANGED
|
@@ -113,6 +113,7 @@ It requires Node.js 22.20.0 or later:
|
|
|
113
113
|
npm install --save-dev bmad-method-test-architecture-enterprise
|
|
114
114
|
```
|
|
115
115
|
|
|
116
|
+
- [`tea-teach`](https://bmad-code-org.github.io/bmad-method-test-architecture-enterprise/reference/tea-teach-cli/) teaches one learner turn and keeps conversation and session progress.
|
|
116
117
|
- [`tea-test-review`](https://bmad-code-org.github.io/bmad-method-test-architecture-enterprise/reference/tea-test-review-cli/) runs the review skill against changed tests and returns a gate verdict.
|
|
117
118
|
- [`tea-framework`](https://bmad-code-org.github.io/bmad-method-test-architecture-enterprise/reference/tea-framework-cli/) creates, resumes, validates, and edits test framework and CI setup with retained run evidence.
|
|
118
119
|
- [`tea-trace`](https://bmad-code-org.github.io/bmad-method-test-architecture-enterprise/reference/tea-trace-cli/) maps requirements to existing evidence and returns a coverage gate.
|
|
@@ -699,19 +699,38 @@ async function deploymentAccess({
|
|
|
699
699
|
return { origins, authorizations };
|
|
700
700
|
}
|
|
701
701
|
|
|
702
|
-
/**
|
|
703
|
-
function
|
|
702
|
+
/** Tries a port on one address, then releases it for the target; a busy candidate returns null. */
|
|
703
|
+
function availablePort(port, address, createServer) {
|
|
704
704
|
return new Promise((resolve, reject) => {
|
|
705
|
-
const server =
|
|
705
|
+
const server = createServer();
|
|
706
706
|
server.unref();
|
|
707
|
-
server.once('error', reject);
|
|
708
|
-
server.listen(
|
|
709
|
-
const
|
|
710
|
-
server.close(() => resolve(
|
|
707
|
+
server.once('error', (error) => (error.code === 'EADDRINUSE' ? resolve(null) : reject(error)));
|
|
708
|
+
server.listen(port, address, () => {
|
|
709
|
+
const selected = server.address().port;
|
|
710
|
+
server.close(() => resolve(selected));
|
|
711
711
|
});
|
|
712
712
|
});
|
|
713
713
|
}
|
|
714
714
|
|
|
715
|
+
/** A port available on every address the policy may select, released for the server to take. */
|
|
716
|
+
async function freePort(addresses, createServer = () => net.createServer()) {
|
|
717
|
+
const distinct = [...new Set(addresses.map((address) => canonicalAddress(address) ?? address))];
|
|
718
|
+
if (distinct.length === 0) throw new Error('a started server has no authorized address');
|
|
719
|
+
for (let attempt = 0; attempt < 32; attempt += 1) {
|
|
720
|
+
const candidate = await availablePort(0, distinct[0], createServer);
|
|
721
|
+
if (candidate === null) continue;
|
|
722
|
+
let available = true;
|
|
723
|
+
for (const address of distinct.slice(1)) {
|
|
724
|
+
if ((await availablePort(candidate, address, createServer)) === null) {
|
|
725
|
+
available = false;
|
|
726
|
+
break;
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
if (available) return candidate;
|
|
730
|
+
}
|
|
731
|
+
throw new Error(`could not find a free port on ${distinct.join(', ')}`);
|
|
732
|
+
}
|
|
733
|
+
|
|
715
734
|
/**
|
|
716
735
|
* Whether `address:port` accepts a connection now: `true`, or why the attempt
|
|
717
736
|
* failed (the socket error's code, `ECONNREFUSED` while nothing listens, or
|
|
@@ -1207,7 +1226,7 @@ function createApiPort({ entries, httpPort, cwd, targetOf, readEnvironment, mech
|
|
|
1207
1226
|
// A server behind a bridge gets a private directory for the Unix socket its shim serves, which only that call's
|
|
1208
1227
|
// server may write; it joins the scratch list like the port directory.
|
|
1209
1228
|
if (launched !== null && mechanism.bridges === true) bridgeDirectory = makeBridgeDirectory(scratch);
|
|
1210
|
-
const launchedPort = launched === null || reports ? null : await freePort();
|
|
1229
|
+
const launchedPort = launched === null || reports ? null : await freePort(launched.addresses);
|
|
1211
1230
|
const configurationAt = (port) =>
|
|
1212
1231
|
portConfiguration({
|
|
1213
1232
|
entries,
|
|
@@ -1397,6 +1416,7 @@ module.exports = {
|
|
|
1397
1416
|
defaultPortOf,
|
|
1398
1417
|
deploymentAccess,
|
|
1399
1418
|
deploymentAuthorizations,
|
|
1419
|
+
freePort,
|
|
1400
1420
|
httpPortFile,
|
|
1401
1421
|
isApiEntry,
|
|
1402
1422
|
makeBridgeDirectory,
|
|
@@ -0,0 +1,432 @@
|
|
|
1
|
+
/** Validate the actual NFR workflow context against supplied sources and its report. */
|
|
2
|
+
'use strict';
|
|
3
|
+
const fs = require('node:fs');
|
|
4
|
+
const path = require('node:path');
|
|
5
|
+
const yaml = require('js-yaml');
|
|
6
|
+
const MarkdownIt = require('markdown-it');
|
|
7
|
+
const { WorkflowError } = require('./workflow-cli');
|
|
8
|
+
const DOMAINS = ['security', 'performance', 'reliability', 'maintainability'];
|
|
9
|
+
const STATUSES = new Set(['PASS', 'CONCERNS', 'FAIL', 'N/A']);
|
|
10
|
+
const STEPS = [
|
|
11
|
+
'step-01-load-context',
|
|
12
|
+
'step-02-define-thresholds',
|
|
13
|
+
'step-03-gather-evidence',
|
|
14
|
+
'step-04e-aggregate-nfr',
|
|
15
|
+
'step-05-generate-report',
|
|
16
|
+
];
|
|
17
|
+
/** Reject a report field with the parser failure class used by the CLI. */
|
|
18
|
+
const bad = (message) => {
|
|
19
|
+
throw new WorkflowError('environment-parser', message);
|
|
20
|
+
};
|
|
21
|
+
/** Collapse whitespace so quoted observations survive line wrapping. */
|
|
22
|
+
const normalized = (value) => String(value).replaceAll(/\s+/g, ' ').trim();
|
|
23
|
+
/** Compare ADR category labels across spaces, hyphens and capitalization. */
|
|
24
|
+
const categoryLabel = (value) =>
|
|
25
|
+
value
|
|
26
|
+
.toLowerCase()
|
|
27
|
+
.replaceAll(/[-\s]+/g, ' ')
|
|
28
|
+
.trim();
|
|
29
|
+
/** Compare canonical context values after JSON serialization. */
|
|
30
|
+
const same = (left, right) => JSON.stringify(left) === JSON.stringify(right);
|
|
31
|
+
const markdown = new MarkdownIt({ html: true });
|
|
32
|
+
const DECORATIONS = { PASS: ['✅', '✔', '✔️'], CONCERNS: ['⚠', '⚠️'], FAIL: ['❌', '✖', '✖️'], 'N/A': ['➖'] };
|
|
33
|
+
/** Read visible inline text, preserving code excerpts while discarding formatting delimiters. */
|
|
34
|
+
function inlineText(token) {
|
|
35
|
+
return (token?.children || [])
|
|
36
|
+
.filter((child) => ['text', 'code_inline'].includes(child.type))
|
|
37
|
+
.map((child) => child.content)
|
|
38
|
+
.join('')
|
|
39
|
+
.trim();
|
|
40
|
+
}
|
|
41
|
+
/** Normalize one complete display enum with an optional matching approved decoration. */
|
|
42
|
+
function displayStatus(value) {
|
|
43
|
+
const match = String(value)
|
|
44
|
+
.trim()
|
|
45
|
+
.match(/^(PASS|CONCERNS|FAIL|N\/A)(?:[ \t]+(\S+))?$/i);
|
|
46
|
+
if (!match) bad('report status must contain one complete enum and an optional approved decoration');
|
|
47
|
+
const status = match[1].toUpperCase();
|
|
48
|
+
if (match[2] && !DECORATIONS[status].includes(match[2])) bad('report status decoration contradicts or ambiguously labels its enum');
|
|
49
|
+
return status;
|
|
50
|
+
}
|
|
51
|
+
/** Preserve a quoted source excerpt when Markdown renders it as a single inline code value. */
|
|
52
|
+
function literalValue(value) {
|
|
53
|
+
const blocks = markdown.parse(value, {});
|
|
54
|
+
if (blocks.length === 1 && blocks[0].type === 'fence') return blocks[0].content.trim();
|
|
55
|
+
const children = markdown.parseInline(value, {})[0]?.children || [];
|
|
56
|
+
return children.length === 1 && children[0].type === 'code_inline' ? children[0].content : value;
|
|
57
|
+
}
|
|
58
|
+
/** Preserve one or more complete fenced source quotations as separate actual observations. */
|
|
59
|
+
function actualValues(value) {
|
|
60
|
+
const blocks = markdown.parse(value, {});
|
|
61
|
+
if (blocks.length > 0 && blocks.every((token) => token.type === 'fence')) return blocks.map((token) => token.content.trim());
|
|
62
|
+
return [literalValue(value)];
|
|
63
|
+
}
|
|
64
|
+
/** The workflow's worst-status rollup excludes nonapplicable domains. */
|
|
65
|
+
function worst(values) {
|
|
66
|
+
if (values.includes('FAIL')) return 'FAIL';
|
|
67
|
+
if (values.includes('CONCERNS')) return 'CONCERNS';
|
|
68
|
+
return values.includes('PASS') ? 'PASS' : 'N/A';
|
|
69
|
+
}
|
|
70
|
+
/** Stable criterion identity defined by NFR step02. */
|
|
71
|
+
function criterionId(domain, label) {
|
|
72
|
+
return `${domain}-${label}`
|
|
73
|
+
.toLowerCase()
|
|
74
|
+
.replaceAll(/[^a-z0-9]+/g, '-')
|
|
75
|
+
.replaceAll(/^-+|-+$/g, '');
|
|
76
|
+
}
|
|
77
|
+
/** Parse YAML without silently accepting duplicate keys or extra documents. */
|
|
78
|
+
function parseYaml(text, label) {
|
|
79
|
+
try {
|
|
80
|
+
return yaml.load(text);
|
|
81
|
+
} catch (error) {
|
|
82
|
+
bad(`invalid ${label} YAML: ${error.message}`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/** Read canonical project-relative files from their explicit input role. */
|
|
86
|
+
function sourceText(reference, projectRoot, sources, label) {
|
|
87
|
+
if (
|
|
88
|
+
typeof reference !== 'string' ||
|
|
89
|
+
!reference ||
|
|
90
|
+
path.isAbsolute(reference) ||
|
|
91
|
+
reference.includes('\\') ||
|
|
92
|
+
path.posix.normalize(reference) !== reference ||
|
|
93
|
+
reference.startsWith('../') ||
|
|
94
|
+
reference.startsWith('./')
|
|
95
|
+
)
|
|
96
|
+
bad(`${label} must name a canonical project-relative supplied path`);
|
|
97
|
+
const file = path.join(projectRoot, reference);
|
|
98
|
+
if (!sources.includes(file)) bad(`${label} is outside its supplied input role: ${reference}`);
|
|
99
|
+
return normalized(fs.readFileSync(file, 'utf8'));
|
|
100
|
+
}
|
|
101
|
+
/** Require an actual source excerpt, while allowing normalized line wrapping. */
|
|
102
|
+
function excerpt(value, source, label) {
|
|
103
|
+
if (typeof value !== 'string' || !normalized(value) || !source.includes(normalized(value)))
|
|
104
|
+
bad(`${label} is not a quotation from its supplied source`);
|
|
105
|
+
}
|
|
106
|
+
/** Extract populated Markdown sections without matching headings inside fenced examples. */
|
|
107
|
+
function sections(text, level) {
|
|
108
|
+
const lines = text.split(/\r?\n/);
|
|
109
|
+
const tokens = markdown.parse(text, {});
|
|
110
|
+
const headings = tokens.flatMap((token, index) => {
|
|
111
|
+
if (token.type !== 'heading_open' || token.level !== 0 || !token.map) return [];
|
|
112
|
+
const label = inlineText(tokens[index + 1]);
|
|
113
|
+
return [{ level: Number(token.tag.slice(1)), label, start: token.map[0], end: token.map[1] }];
|
|
114
|
+
});
|
|
115
|
+
return headings
|
|
116
|
+
.filter((heading) => heading.level === level)
|
|
117
|
+
.map((heading) => {
|
|
118
|
+
const next = headings.find((other) => other.start > heading.start && other.level <= level);
|
|
119
|
+
return { label: heading.label, body: lines.slice(heading.end, next?.start ?? lines.length).join('\n') };
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
/** Remove literal block examples before interpreting bold report fields. */
|
|
123
|
+
function reportFields(text) {
|
|
124
|
+
const lines = text.split(/\r?\n/);
|
|
125
|
+
for (const token of markdown.parse(text, {})) {
|
|
126
|
+
if (['fence', 'code_block', 'html_block'].includes(token.type) && token.map)
|
|
127
|
+
for (let line = token.map[0]; line < token.map[1]; line++) lines[line] = lines[line].replaceAll(/[^\n]/g, ' ');
|
|
128
|
+
}
|
|
129
|
+
return lines.join('\n');
|
|
130
|
+
}
|
|
131
|
+
/** Read a unique bold report field, including multiline evidence lists. */
|
|
132
|
+
function field(body, name) {
|
|
133
|
+
const expression = new RegExp(
|
|
134
|
+
`^[ \\t]*(?:-[ \\t]+)?\\*\\*${name}:?\\*\\*[ \\t]*:?[ \\t]*([^]*?)(?=^[ \\t]*(?:-[ \\t]+)?\\*\\*[A-Za-z]|$(?![\\s\\S]))`,
|
|
135
|
+
'gim',
|
|
136
|
+
);
|
|
137
|
+
const values = [...reportFields(body).matchAll(expression)].map((match) => {
|
|
138
|
+
const start = match.index + match[0].length - match[1].length;
|
|
139
|
+
return body.slice(start, match.index + match[0].length).trim();
|
|
140
|
+
});
|
|
141
|
+
if (values.length !== 1 || !values[0]) bad(`criterion must contain exactly one populated ${name} field`);
|
|
142
|
+
return values[0];
|
|
143
|
+
}
|
|
144
|
+
/** Validate the fresh canonical context, exact ledger bindings, report and gate together. */
|
|
145
|
+
function readNfrReport(text, context, request) {
|
|
146
|
+
const { projectRoot, runScope, runKey, requestId, inputs, evidenceFiles } = request;
|
|
147
|
+
if (
|
|
148
|
+
!context ||
|
|
149
|
+
context.schemaVersion !== 1 ||
|
|
150
|
+
context.requestId !== requestId ||
|
|
151
|
+
context.runScope !== runScope ||
|
|
152
|
+
context.runKey !== runKey ||
|
|
153
|
+
context.supplied_project_root !== projectRoot
|
|
154
|
+
)
|
|
155
|
+
bad('the NFR context does not belong to the requested attempt and scope');
|
|
156
|
+
if (
|
|
157
|
+
!same(Object.keys(context.declared_nfr_criteria || {}).sort(), [...DOMAINS].sort()) ||
|
|
158
|
+
!same(Object.keys(context.domain_assessments || {}).sort(), [...DOMAINS].sort())
|
|
159
|
+
)
|
|
160
|
+
bad('context must carry exactly the four audited domains');
|
|
161
|
+
const criteria = new Map();
|
|
162
|
+
const validateCriterion = (criterion, domain, previousOrder) => {
|
|
163
|
+
if (
|
|
164
|
+
!criterion ||
|
|
165
|
+
typeof criterion.label !== 'string' ||
|
|
166
|
+
!criterion.label.trim() ||
|
|
167
|
+
criterion.id !== criterionId(domain, criterion.label) ||
|
|
168
|
+
!Number.isSafeInteger(criterion.order) ||
|
|
169
|
+
criterion.order <= previousOrder ||
|
|
170
|
+
criteria.has(criterion.id)
|
|
171
|
+
)
|
|
172
|
+
bad(`invalid criterion identity or source order in ${domain}`);
|
|
173
|
+
const source = sourceText(criterion.threshold_source, projectRoot, inputs, 'threshold source');
|
|
174
|
+
if (criterion.threshold !== 'UNKNOWN') excerpt(criterion.threshold, source, `${criterion.id} threshold`);
|
|
175
|
+
criteria.set(criterion.id, { ...criterion, domain });
|
|
176
|
+
return criterion.order;
|
|
177
|
+
};
|
|
178
|
+
for (const domain of DOMAINS) {
|
|
179
|
+
if (!Array.isArray(context.declared_nfr_criteria[domain])) bad(`invalid declared criteria for ${domain}`);
|
|
180
|
+
let order = -1;
|
|
181
|
+
for (const criterion of context.declared_nfr_criteria[domain]) order = validateCriterion(criterion, domain, order);
|
|
182
|
+
}
|
|
183
|
+
if (!Array.isArray(context.recorded_only_nfr_criteria))
|
|
184
|
+
bad('context must include recorded-only criteria, including an empty list when none were declared');
|
|
185
|
+
const recorded = new Set();
|
|
186
|
+
let recordedOrder = -1;
|
|
187
|
+
for (const criterion of context.recorded_only_nfr_criteria) {
|
|
188
|
+
if (DOMAINS.includes(criterion.domain) || typeof criterion.domain !== 'string' || criterion.assessment_mode !== 'recorded-only')
|
|
189
|
+
bad('recorded-only criteria cannot create audited domains');
|
|
190
|
+
recordedOrder = validateCriterion(criterion, criterion.domain, recordedOrder);
|
|
191
|
+
recorded.add(criterion.id);
|
|
192
|
+
}
|
|
193
|
+
if (!Array.isArray(context.supplied_evidence_ledger)) bad('context is missing the supplied evidence ledger');
|
|
194
|
+
const observations = new Map();
|
|
195
|
+
let lastPath = '';
|
|
196
|
+
for (const entry of context.supplied_evidence_ledger) {
|
|
197
|
+
if (
|
|
198
|
+
!entry ||
|
|
199
|
+
entry.source !== 'supplied' ||
|
|
200
|
+
entry.source_type !== 'implementation-evidence' ||
|
|
201
|
+
typeof entry.path !== 'string' ||
|
|
202
|
+
entry.path <= lastPath ||
|
|
203
|
+
!Array.isArray(entry.observations) ||
|
|
204
|
+
entry.observations.length === 0
|
|
205
|
+
)
|
|
206
|
+
bad('invalid or unsorted supplied evidence ledger');
|
|
207
|
+
lastPath = entry.path;
|
|
208
|
+
const source = sourceText(entry.path, projectRoot, evidenceFiles, 'implementation evidence');
|
|
209
|
+
let previous = '';
|
|
210
|
+
for (const observation of entry.observations) {
|
|
211
|
+
if (!observation || !criteria.has(observation.criterion_id) || recorded.has(observation.criterion_id))
|
|
212
|
+
bad('ledger observation has an undeclared audited criterion');
|
|
213
|
+
excerpt(observation.supports, source, `${observation.criterion_id} observation`);
|
|
214
|
+
const key = `${observation.criterion_id}\0${observation.supports}`;
|
|
215
|
+
if (key <= previous) bad('ledger observations must be deduplicated and sorted');
|
|
216
|
+
previous = key;
|
|
217
|
+
const bound = observations.get(observation.criterion_id) || [];
|
|
218
|
+
bound.push({ path: entry.path, supports: observation.supports });
|
|
219
|
+
observations.set(observation.criterion_id, bound);
|
|
220
|
+
}
|
|
221
|
+
const expectedDomains = DOMAINS.filter((domain) =>
|
|
222
|
+
entry.observations.some((observation) => criteria.get(observation.criterion_id).domain === domain),
|
|
223
|
+
);
|
|
224
|
+
if (!same(entry.domains, expectedDomains)) bad('ledger domain bindings contradict its criterion observations');
|
|
225
|
+
}
|
|
226
|
+
const frontmatter = text.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/);
|
|
227
|
+
if (!frontmatter) bad('the report is missing progress frontmatter');
|
|
228
|
+
const progress = parseYaml(frontmatter[1], 'progress');
|
|
229
|
+
const optionalSteps = [
|
|
230
|
+
'step-04-evaluate-and-score',
|
|
231
|
+
'step-04a-subagent-security',
|
|
232
|
+
'step-04b-subagent-performance',
|
|
233
|
+
'step-04c-subagent-reliability',
|
|
234
|
+
'step-04d-subagent-maintainability',
|
|
235
|
+
];
|
|
236
|
+
const completed = progress?.stepsCompleted;
|
|
237
|
+
const requiredSequence = Array.isArray(completed) ? completed.filter((step) => STEPS.includes(step)) : [];
|
|
238
|
+
const validSteps =
|
|
239
|
+
Array.isArray(completed) &&
|
|
240
|
+
new Set(completed).size === completed.length &&
|
|
241
|
+
completed.every((step) => STEPS.includes(step) || optionalSteps.includes(step)) &&
|
|
242
|
+
optionalSteps.every(
|
|
243
|
+
(step) =>
|
|
244
|
+
!completed.includes(step) ||
|
|
245
|
+
(completed.indexOf(step) > completed.indexOf(STEPS[2]) && completed.indexOf(step) < completed.indexOf(STEPS[3])),
|
|
246
|
+
);
|
|
247
|
+
if (
|
|
248
|
+
progress?.workflowStatus !== 'completed' ||
|
|
249
|
+
progress.runScope !== runScope ||
|
|
250
|
+
progress.runKey !== runKey ||
|
|
251
|
+
progress.lastStep !== STEPS.at(-1) ||
|
|
252
|
+
!validSteps ||
|
|
253
|
+
!same(requiredSequence, STEPS) ||
|
|
254
|
+
!(typeof progress.lastSaved === 'string' || progress.lastSaved instanceof Date) ||
|
|
255
|
+
!Number.isFinite(Date.parse(progress.lastSaved))
|
|
256
|
+
)
|
|
257
|
+
bad('the report checkpoint does not confirm this completed run');
|
|
258
|
+
const gates = [];
|
|
259
|
+
const reportBody = text.slice(frontmatter[0].length);
|
|
260
|
+
for (const token of markdown.parse(reportBody, {})) {
|
|
261
|
+
if (token.type !== 'fence' || token.level !== 0 || !['yaml', 'yml'].includes(token.info.trim().toLowerCase())) continue;
|
|
262
|
+
const value = parseYaml(token.content, 'gate');
|
|
263
|
+
if (value && Object.hasOwn(value, 'nfr_assessment')) gates.push(value.nfr_assessment);
|
|
264
|
+
}
|
|
265
|
+
if (gates.length !== 1) bad('report must carry exactly one nfr_assessment gate');
|
|
266
|
+
const gate = gates[0];
|
|
267
|
+
if (!gate || !same(Object.keys(gate.audited_domains || {}).sort(), [...DOMAINS].sort()) || !STATUSES.has(gate.overall_status))
|
|
268
|
+
bad('gate must declare exactly four audited domains and a valid overall status');
|
|
269
|
+
if (
|
|
270
|
+
typeof gate.blockers !== 'boolean' ||
|
|
271
|
+
['critical_issues', 'high_priority_issues', 'medium_priority_issues', 'concerns', 'evidence_gaps'].some(
|
|
272
|
+
(key) => !Number.isSafeInteger(gate[key]) || gate[key] < 0,
|
|
273
|
+
)
|
|
274
|
+
)
|
|
275
|
+
bad('gate issue counts and blockers flag must be typed');
|
|
276
|
+
const reportSections = sections(reportBody, 2);
|
|
277
|
+
for (const name of [
|
|
278
|
+
'Executive Summary',
|
|
279
|
+
'Quick Wins',
|
|
280
|
+
'Recommended Actions',
|
|
281
|
+
'Monitoring Hooks',
|
|
282
|
+
'Fail-Fast Mechanisms',
|
|
283
|
+
'Evidence Gaps',
|
|
284
|
+
'Findings Summary',
|
|
285
|
+
'Gate YAML Snippet',
|
|
286
|
+
'Related Artifacts',
|
|
287
|
+
'Recommendations Summary',
|
|
288
|
+
'Sign-Off',
|
|
289
|
+
]) {
|
|
290
|
+
const matches = reportSections.filter((section) => section.label === name);
|
|
291
|
+
if (
|
|
292
|
+
matches.length !== 1 ||
|
|
293
|
+
!matches[0].body
|
|
294
|
+
.replaceAll(/<!--[^]*?-->/g, '')
|
|
295
|
+
.replaceAll(/^#{1,6}[^\n]*$/gm, '')
|
|
296
|
+
.trim()
|
|
297
|
+
)
|
|
298
|
+
bad(`report requires one populated ${name} section`);
|
|
299
|
+
}
|
|
300
|
+
let concerns = 0;
|
|
301
|
+
let gaps = 0;
|
|
302
|
+
for (const domain of DOMAINS) {
|
|
303
|
+
const domainSections = reportSections.filter((section) => section.label.toLowerCase() === `${domain} assessment`);
|
|
304
|
+
if (domainSections.length !== 1) bad(`report requires exactly one ${domain} Assessment`);
|
|
305
|
+
const section = domainSections[0];
|
|
306
|
+
const expected = context.declared_nfr_criteria[domain];
|
|
307
|
+
const assessment = context.domain_assessments[domain];
|
|
308
|
+
if (
|
|
309
|
+
!assessment ||
|
|
310
|
+
!Array.isArray(assessment.findings) ||
|
|
311
|
+
!same(
|
|
312
|
+
assessment.findings.map((finding) => finding.criterion_id),
|
|
313
|
+
expected.map((criterion) => criterion.id),
|
|
314
|
+
)
|
|
315
|
+
)
|
|
316
|
+
bad(`${domain} findings omit or add declared criteria`);
|
|
317
|
+
const reportFindings = sections(section.body, 3);
|
|
318
|
+
if (
|
|
319
|
+
!same(
|
|
320
|
+
reportFindings.map((finding) => finding.label),
|
|
321
|
+
expected.map((criterion) => criterion.label),
|
|
322
|
+
)
|
|
323
|
+
)
|
|
324
|
+
bad(`${domain} report headings omit, duplicate or add declared criteria`);
|
|
325
|
+
const expectedGaps = [];
|
|
326
|
+
for (const [index, criterion] of expected.entries()) {
|
|
327
|
+
const finding = assessment.findings[index];
|
|
328
|
+
const evidence = observations.get(criterion.id) || [];
|
|
329
|
+
if (!STATUSES.has(finding.status) || !same(finding.evidence, evidence))
|
|
330
|
+
bad(`${criterion.id} status or evidence contradicts the canonical ledger`);
|
|
331
|
+
if ((evidence.length === 0 || criterion.threshold === 'UNKNOWN') && finding.status !== 'CONCERNS')
|
|
332
|
+
bad(`${criterion.id} requires CONCERNS for unknown thresholds or missing evidence`);
|
|
333
|
+
if (evidence.length === 0)
|
|
334
|
+
expectedGaps.push({ criterion_id: criterion.id, message: `${criterion.label}: no supplied implementation evidence` });
|
|
335
|
+
if (criterion.threshold === 'UNKNOWN' && evidence.length > 0)
|
|
336
|
+
expectedGaps.push({ criterion_id: criterion.id, message: `${criterion.label}: UNKNOWN threshold` });
|
|
337
|
+
const body = reportFindings[index].body;
|
|
338
|
+
if (
|
|
339
|
+
displayStatus(field(body, 'Status')) !== finding.status ||
|
|
340
|
+
normalized(literalValue(field(body, 'Threshold'))) !== normalized(criterion.threshold) ||
|
|
341
|
+
field(body, 'Threshold Source') !== `\`${criterion.threshold_source}\``
|
|
342
|
+
)
|
|
343
|
+
bad(`${criterion.id} report fields contradict its canonical criterion`);
|
|
344
|
+
const actual = actualValues(field(body, 'Actual'));
|
|
345
|
+
if (finding.status === 'PASS' && actual.includes('UNKNOWN')) bad(`${criterion.id} claims PASS without a known actual measurement`);
|
|
346
|
+
for (const observation of actual) {
|
|
347
|
+
if (
|
|
348
|
+
!normalized(observation) ||
|
|
349
|
+
(observation !== 'UNKNOWN' &&
|
|
350
|
+
!evidence.some((entry) =>
|
|
351
|
+
sourceText(entry.path, projectRoot, evidenceFiles, 'actual evidence').includes(normalized(observation)),
|
|
352
|
+
))
|
|
353
|
+
)
|
|
354
|
+
bad(`${criterion.id} actual is unsupported by supplied evidence`);
|
|
355
|
+
}
|
|
356
|
+
const evidenceText = field(body, 'Evidence');
|
|
357
|
+
const paths = [...evidenceText.matchAll(/`([^`]+)`/g)].map((match) => match[1]);
|
|
358
|
+
if (!same([...new Set(paths)], [...new Set(evidence.map((entry) => entry.path))]))
|
|
359
|
+
bad(`${criterion.id} report evidence paths contradict the canonical ledger`);
|
|
360
|
+
const supports = field(body, 'Supports');
|
|
361
|
+
for (const entry of evidence)
|
|
362
|
+
if (!normalized(supports).includes(normalized(entry.supports))) bad(`${criterion.id} report omits a bound observation`);
|
|
363
|
+
if (evidence.length === 0 && (evidenceText !== 'None' || supports !== 'None' || !same(actual, ['UNKNOWN'])))
|
|
364
|
+
bad(`${criterion.id} claims unsupported implementation evidence`);
|
|
365
|
+
concerns += finding.status === 'CONCERNS' ? 1 : 0;
|
|
366
|
+
}
|
|
367
|
+
const displayedGaps = Array.isArray(assessment.evidence_gaps)
|
|
368
|
+
? assessment.evidence_gaps.map((gap) => {
|
|
369
|
+
const criterion = criteria.get(gap.criterion_id);
|
|
370
|
+
if (criterion?.threshold === 'UNKNOWN' && gap.message === `${criterion.label}: UNKNOWN threshold gap`)
|
|
371
|
+
return { ...gap, message: `${criterion.label}: UNKNOWN threshold` };
|
|
372
|
+
return gap;
|
|
373
|
+
})
|
|
374
|
+
: assessment.evidence_gaps;
|
|
375
|
+
if (!same(displayedGaps, expectedGaps)) bad(`${domain} evidence gaps contradict its declared criteria`);
|
|
376
|
+
gaps += expectedGaps.length;
|
|
377
|
+
const status = worst(assessment.findings.map((finding) => finding.status));
|
|
378
|
+
const firstFinding = markdown
|
|
379
|
+
.parse(section.body, {})
|
|
380
|
+
.find((token) => token.type === 'heading_open' && token.level === 0 && token.tag === 'h3');
|
|
381
|
+
const prelude = reportFields(section.body.split(/\r?\n/).slice(0, firstFinding?.map?.[0]).join('\n'));
|
|
382
|
+
const domainStatuses = [...prelude.matchAll(/^[ \t]*(?:-[ \t]+)?\*\*(?:Domain )?Status:?\*\*[ \t]*:?[ \t]*([^\n]+)$/gim)].map((match) =>
|
|
383
|
+
displayStatus(match[1]),
|
|
384
|
+
);
|
|
385
|
+
const domainStatus = domainStatuses.length === 1 ? domainStatuses[0] : null;
|
|
386
|
+
if (assessment.status !== status || domainStatus !== status || gate.audited_domains[domain] !== status)
|
|
387
|
+
bad(`${domain} status contradicts its finding rollup`);
|
|
388
|
+
}
|
|
389
|
+
if (
|
|
390
|
+
gate.overall_status !== worst(Object.values(gate.audited_domains)) ||
|
|
391
|
+
gate.concerns !== concerns ||
|
|
392
|
+
gate.evidence_gaps !== gaps ||
|
|
393
|
+
gate.blockers !== (gate.overall_status === 'FAIL')
|
|
394
|
+
)
|
|
395
|
+
bad('gate overall status or counts contradict the normalized audit');
|
|
396
|
+
const tables = reportSections.filter((section) => section.label === 'Recorded-Only NFR Criteria');
|
|
397
|
+
if (tables.length > 1 || (recorded.size > 0 && tables.length !== 1)) bad('declared recorded-only criteria require one report table');
|
|
398
|
+
if (tables.length === 1) {
|
|
399
|
+
const tokens = markdown.parse(tables[0].body, {});
|
|
400
|
+
const rows = [];
|
|
401
|
+
let row;
|
|
402
|
+
for (const [index, token] of tokens.entries()) {
|
|
403
|
+
if (token.type === 'tr_open') {
|
|
404
|
+
row = [];
|
|
405
|
+
// Markdown silently drops cells beyond the header width. Keep those extras invalid.
|
|
406
|
+
const raw = tables[0].body
|
|
407
|
+
.split(/\r?\n/)
|
|
408
|
+
.slice(...token.map)
|
|
409
|
+
.join('\n')
|
|
410
|
+
.trim();
|
|
411
|
+
const cells = raw.split(/(?<!\\)(?:\\\\)*\|/);
|
|
412
|
+
const count = cells.length - (cells[0].trim() === '' ? 1 : 0) - (cells.at(-1).trim() === '' ? 1 : 0);
|
|
413
|
+
if (count !== 6) bad('recorded-only table must contain exactly six cells per row');
|
|
414
|
+
}
|
|
415
|
+
if (token.type === 'td_open') row?.push(inlineText(tokens[index + 1]));
|
|
416
|
+
if (token.type === 'tr_close' && row?.length > 0) rows.push(row);
|
|
417
|
+
}
|
|
418
|
+
// Categories are display labels; their identity remains the canonical criterion ID.
|
|
419
|
+
for (const row of rows) if (typeof row[1] === 'string') row[1] = categoryLabel(row[1]);
|
|
420
|
+
const expectedRows = context.recorded_only_nfr_criteria.map((criterion) => [
|
|
421
|
+
criterion.id,
|
|
422
|
+
categoryLabel(criterion.domain),
|
|
423
|
+
criterion.label,
|
|
424
|
+
criterion.threshold,
|
|
425
|
+
criterion.threshold_source,
|
|
426
|
+
'RECORDED ONLY',
|
|
427
|
+
]);
|
|
428
|
+
if (!same(rows, expectedRows)) bad('recorded-only table adds, omits, reorders or assesses criteria');
|
|
429
|
+
}
|
|
430
|
+
return { status: gate.overall_status, gate, progress, context };
|
|
431
|
+
}
|
|
432
|
+
module.exports = { readNfrReport, criterionId, worst, DOMAINS };
|
|
@@ -78,18 +78,22 @@ function configError(message) {
|
|
|
78
78
|
}
|
|
79
79
|
|
|
80
80
|
/** Read configuration from one pinned base commit, or from the working tree. */
|
|
81
|
-
function configSource(projectRoot, baseRef) {
|
|
81
|
+
function configSource(projectRoot, baseRef, inputPaths, inputPatterns) {
|
|
82
82
|
if (baseRef === undefined || baseRef === null) {
|
|
83
83
|
return {
|
|
84
|
+
recordInput: (file) => inputPaths?.add(file),
|
|
84
85
|
label: (relativePath) => path.join(projectRoot, relativePath),
|
|
85
86
|
list(pattern) {
|
|
86
87
|
try {
|
|
87
|
-
|
|
88
|
+
const matches = globSync(pattern, { cwd: projectRoot, dot: true, nodir: true, follow: false });
|
|
89
|
+
inputPatterns?.set(pattern, [...matches].sort());
|
|
90
|
+
return matches;
|
|
88
91
|
} catch (error) {
|
|
89
92
|
throw configError(`Failed to list policy files in ${projectRoot}: ${error.message}`);
|
|
90
93
|
}
|
|
91
94
|
},
|
|
92
95
|
read(relativePath) {
|
|
96
|
+
inputPaths?.add(path.join(projectRoot, relativePath));
|
|
93
97
|
try {
|
|
94
98
|
return fs.readFileSync(path.join(projectRoot, relativePath), 'utf8');
|
|
95
99
|
} catch (error) {
|
|
@@ -284,6 +288,7 @@ function parseToml(content, label) {
|
|
|
284
288
|
function readWorkflowCustomization(source, skillRoot, projectRoot, skillName = 'bmad-testarch-test-review') {
|
|
285
289
|
if (!skillRoot) return {};
|
|
286
290
|
const defaultsPath = path.join(skillRoot, 'customize.toml');
|
|
291
|
+
source.recordInput?.(defaultsPath);
|
|
287
292
|
let merged = {};
|
|
288
293
|
try {
|
|
289
294
|
merged = parseToml(fs.readFileSync(defaultsPath, 'utf8'), defaultsPath);
|
|
@@ -490,13 +495,16 @@ const KEY_TO_INSTALLED_FIELD = {
|
|
|
490
495
|
* @param {string} [options.skillRoot] - Trusted skill defaults for workflow customization.
|
|
491
496
|
* @param {object} [options.flags] - Parsed CLI options; only the keys in
|
|
492
497
|
* FLAG_TO_KEY are read, and only when not undefined.
|
|
498
|
+
* @param {Set<string>} [options.inputPaths] - Receives working-tree inputs, including missing optional layers.
|
|
499
|
+
* @param {Map<string, string[]>} [options.inputPatterns] - Receives policy glob membership used during resolution.
|
|
493
500
|
* @returns {{values: object, sources: object, installed: object, configSnapshot: object, workflowCustomization: object, configCommit: string|null, configPath: string, configPresent: boolean, configFormat: string|null}}
|
|
494
501
|
* `installed` carries one boolean per library gate, read from the project
|
|
495
502
|
* manifest rather than left to the agent.
|
|
496
503
|
* @throws {Error} With code TEA_CONFIG_INVALID on unusable config content.
|
|
497
504
|
*/
|
|
498
|
-
function resolveTeaConfig({ projectRoot, flags = {}, baseRef, skillRoot, skillName }) {
|
|
499
|
-
const source = configSource(projectRoot, baseRef);
|
|
505
|
+
function resolveTeaConfig({ projectRoot, flags = {}, baseRef, skillRoot, skillName, inputPaths, inputPatterns }) {
|
|
506
|
+
const source = configSource(projectRoot, baseRef, inputPaths, inputPatterns);
|
|
507
|
+
inputPaths?.add(path.join(projectRoot, 'package.json'));
|
|
500
508
|
const file = readTeaConfigFile(projectRoot, baseRef, source);
|
|
501
509
|
const workflowCustomization = readWorkflowCustomization(source, skillRoot, projectRoot, skillName);
|
|
502
510
|
|