bmad-method-test-architecture-enterprise 2.0.1 → 2.0.2-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +4 -3
- package/cli/lib/resolve-tea-config.js +6 -9
- package/cli/lib/trace-command.js +628 -0
- package/cli/lib/trace-live.js +101 -0
- package/cli/lib/workflow-cli.js +250 -0
- package/cli/trace.js +258 -0
- package/package.json +4 -2
- package/skills/bmad-testarch-trace/checklist.md +35 -38
- package/skills/bmad-testarch-trace/steps-c/step-01-load-context.md +5 -0
- package/skills/bmad-testarch-trace/steps-c/step-02-discover-tests.md +1 -0
- package/skills/bmad-testarch-trace/steps-c/step-03-map-criteria.md +7 -0
- package/skills/bmad-testarch-trace/steps-c/step-04-analyze-gaps.md +40 -1
- package/skills/bmad-testarch-trace/steps-c/step-05-gate-decision.md +22 -1
- package/skills/bmad-testarch-trace/trace-template.md +1 -0
- package/skills/bmad-testarch-trace/workflow.yaml +3 -3
- package/skills/bmod-tea/bmod.toml +1 -1
|
@@ -0,0 +1,628 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('node:fs');
|
|
4
|
+
const path = require('node:path');
|
|
5
|
+
const yaml = require('js-yaml');
|
|
6
|
+
const MarkdownIt = require('markdown-it');
|
|
7
|
+
const { readInput } = require('./workflow-cli');
|
|
8
|
+
const { liveReference } = require('./trace-live');
|
|
9
|
+
|
|
10
|
+
const GATE_TYPES = ['story', 'epic', 'release', 'hotfix'];
|
|
11
|
+
const COLLECTION_MODES = [
|
|
12
|
+
'contract_static',
|
|
13
|
+
'inventory_only',
|
|
14
|
+
'runtime_manifest',
|
|
15
|
+
'deferred_shared',
|
|
16
|
+
'waived',
|
|
17
|
+
'restricted',
|
|
18
|
+
'inaccessible',
|
|
19
|
+
];
|
|
20
|
+
const GATE_STATUSES = new Set(['PASS', 'CONCERNS', 'FAIL']);
|
|
21
|
+
const LEVELS = ['e2e', 'api', 'component', 'unit', 'live', 'other'];
|
|
22
|
+
|
|
23
|
+
function criterionText(value) {
|
|
24
|
+
const content = value.replace(/\s*\(P[0-3]\)\s*$/, '');
|
|
25
|
+
const children = new MarkdownIt().parseInline(content, {})[0]?.children ?? [];
|
|
26
|
+
return children
|
|
27
|
+
.map((token) =>
|
|
28
|
+
['softbreak', 'hardbreak'].includes(token.type) ? ' ' : ['text', 'code_inline'].includes(token.type) ? token.content : '',
|
|
29
|
+
)
|
|
30
|
+
.join('')
|
|
31
|
+
.replaceAll(/\s+/g, ' ')
|
|
32
|
+
.trim();
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function slug(value) {
|
|
36
|
+
return String(value)
|
|
37
|
+
.toLowerCase()
|
|
38
|
+
.replaceAll(/[^a-z0-9]+/g, '-')
|
|
39
|
+
.replaceAll(/^-|-$/g, '')
|
|
40
|
+
.slice(0, 64)
|
|
41
|
+
.replace(/-$/, '');
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function inputError(message) {
|
|
45
|
+
return Object.assign(new Error(message), { code: 'TRACE_INPUT_INVALID' });
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Resolve the scope once, before allocating outputs or invoking a vendor. */
|
|
49
|
+
function resolveTraceTarget({ projectRoot, target, targetId, gateType }) {
|
|
50
|
+
let document = null;
|
|
51
|
+
let text = '';
|
|
52
|
+
if (target) {
|
|
53
|
+
document = path.resolve(projectRoot, target);
|
|
54
|
+
try {
|
|
55
|
+
text = fs.readFileSync(document, 'utf8');
|
|
56
|
+
} catch (error) {
|
|
57
|
+
throw inputError(`Cannot read --target ${document}: ${error.message}`);
|
|
58
|
+
}
|
|
59
|
+
if (!text.trim()) throw inputError(`--target ${document} is empty.`);
|
|
60
|
+
}
|
|
61
|
+
const basename = document ? path.basename(document, path.extname(document)) : '';
|
|
62
|
+
const tokens = new MarkdownIt().parse(text, {});
|
|
63
|
+
const headingIndex = tokens.findIndex((token) => token.type === 'heading_open' && token.tag === 'h1');
|
|
64
|
+
const heading = headingIndex === -1 ? '' : tokens[headingIndex + 1].content;
|
|
65
|
+
const inferred =
|
|
66
|
+
/^story\b/i.test(heading) || /^\d+[.-]\d+[-.]/.test(basename) || /^story[-.]/i.test(basename)
|
|
67
|
+
? 'story'
|
|
68
|
+
: /^epic\b/i.test(heading) || /^epic[-.]/i.test(basename)
|
|
69
|
+
? 'epic'
|
|
70
|
+
: /^release\b/i.test(heading)
|
|
71
|
+
? 'release'
|
|
72
|
+
: /^hotfix\b/i.test(heading)
|
|
73
|
+
? 'hotfix'
|
|
74
|
+
: null;
|
|
75
|
+
const type = gateType ?? inferred ?? 'story';
|
|
76
|
+
if (!GATE_TYPES.includes(type)) throw inputError(`--gate-type must be one of ${GATE_TYPES.join(', ')}.`);
|
|
77
|
+
|
|
78
|
+
let id = targetId?.trim();
|
|
79
|
+
if (targetId !== undefined && !id) throw inputError('--target-id must be non-empty.');
|
|
80
|
+
if (!id && type === 'epic') {
|
|
81
|
+
id = heading.match(/^Epic\s+(\d+)\b/i)?.[1] ?? basename.match(/^epic[-.](\d+)(?:[-.]|$)/i)?.[1];
|
|
82
|
+
if (!id && document) id = slug(heading || basename);
|
|
83
|
+
}
|
|
84
|
+
if (!id && type === 'story') {
|
|
85
|
+
id = heading.match(/^Story\s+(\d+[.-]\d+)\b/i)?.[1] ?? basename.match(/^(?:story[-.])?(\d+[.-]\d+)(?:[-.]|$)/i)?.[1];
|
|
86
|
+
}
|
|
87
|
+
if (!id && type === 'release') id = heading.match(/^Release\s+(\S+)/i)?.[1];
|
|
88
|
+
if (!id && type === 'hotfix') id = heading.match(/^Hotfix\s+(\S+)/i)?.[1];
|
|
89
|
+
if (document && !id) throw inputError(`Cannot resolve a ${type} id from --target. Supply --target-id and --gate-type.`);
|
|
90
|
+
if (!id) return { type, id: '', label: '', runScope: 'system', runKey: 'system', document };
|
|
91
|
+
|
|
92
|
+
const keyPart = type === 'story' && document ? basename : slug(id);
|
|
93
|
+
if (!keyPart) throw inputError('--target-id must contain a letter or digit.');
|
|
94
|
+
return {
|
|
95
|
+
type,
|
|
96
|
+
id,
|
|
97
|
+
label: heading || `${type[0].toUpperCase()}${type.slice(1)} ${id}`,
|
|
98
|
+
runScope: type,
|
|
99
|
+
runKey: `${type}-${keyPart}`,
|
|
100
|
+
document,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function tracePaths(artifactRoot, runKey) {
|
|
105
|
+
const directory = path.join(artifactRoot, 'trace');
|
|
106
|
+
return {
|
|
107
|
+
matrix: path.join(directory, `traceability-matrix-${runKey}.md`),
|
|
108
|
+
summary: path.join(directory, `e2e-trace-summary-${runKey}.json`),
|
|
109
|
+
gate: path.join(directory, `gate-decision-${runKey}.json`),
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function readJson(file) {
|
|
114
|
+
const value = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
115
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`${file} must contain a JSON object.`);
|
|
116
|
+
return value;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Capture explicitly identified requirements from the caller's source before execution. */
|
|
120
|
+
function sourceOracleLedger(text, source) {
|
|
121
|
+
const tokens = new MarkdownIt().parse(text, {});
|
|
122
|
+
const ledger = new Map();
|
|
123
|
+
const add = (id, priority, requirement, line) => {
|
|
124
|
+
const existing = ledger.get(id);
|
|
125
|
+
if (existing) {
|
|
126
|
+
if (existing.priority && priority && existing.priority !== priority)
|
|
127
|
+
throw inputError(`Source oracle gives criterion ${id} conflicting priorities.`);
|
|
128
|
+
if (criterionText(existing.requirement) !== criterionText(requirement))
|
|
129
|
+
throw inputError(`Source oracle gives criterion ${id} conflicting requirement text.`);
|
|
130
|
+
existing.priority ||= priority || null;
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
ledger.set(id, { id, priority: priority || null, requirement, source: `${source}:${line}` });
|
|
134
|
+
};
|
|
135
|
+
for (let index = 0; index < tokens.length; index++) {
|
|
136
|
+
const token = tokens[index];
|
|
137
|
+
if (token.type === 'table_open') {
|
|
138
|
+
const rows = [];
|
|
139
|
+
let row;
|
|
140
|
+
let line;
|
|
141
|
+
while (++index < tokens.length && tokens[index].type !== 'table_close') {
|
|
142
|
+
if (tokens[index].type === 'tr_open') {
|
|
143
|
+
row = [];
|
|
144
|
+
line = (tokens[index].map?.[0] ?? token.map?.[0] ?? 0) + 1;
|
|
145
|
+
} else if (tokens[index].type === 'inline' && row) row.push(tokens[index].content.replaceAll(/[*`]/g, '').trim());
|
|
146
|
+
else if (tokens[index].type === 'tr_close') rows.push({ cells: row, line });
|
|
147
|
+
}
|
|
148
|
+
const header = rows.shift()?.cells ?? [];
|
|
149
|
+
const idIndex = header.findIndex((cell) => /^(?:id|criterion|criterion id|requirement id)$/i.test(cell));
|
|
150
|
+
const requirementIndex = header.findIndex((cell) => /^(?:requirement|criterion text)$/i.test(cell));
|
|
151
|
+
const priorityIndex = header.findIndex((cell) => /^priority$/i.test(cell));
|
|
152
|
+
if (idIndex === -1 || requirementIndex === -1) continue;
|
|
153
|
+
for (const { cells, line } of rows) {
|
|
154
|
+
const priority = priorityIndex === -1 ? '' : cells[priorityIndex];
|
|
155
|
+
if (priority && !/^P[0-3]$/.test(priority)) throw inputError('Source oracle has invalid priority.');
|
|
156
|
+
if (!cells[idIndex] || !cells[requirementIndex]) throw inputError('Source oracle has an incomplete criterion row.');
|
|
157
|
+
add(cells[idIndex], priority, cells[requirementIndex], line);
|
|
158
|
+
}
|
|
159
|
+
} else if (token.type === 'inline' && !(tokens[index - 1]?.type === 'heading_open' && tokens[index - 1].tag === 'h1')) {
|
|
160
|
+
let pending;
|
|
161
|
+
const flush = () => {
|
|
162
|
+
if (pending) add(pending.id, pending.priority, pending.requirement, pending.line);
|
|
163
|
+
};
|
|
164
|
+
for (const [offset, line] of token.content.split('\n').entries()) {
|
|
165
|
+
const claim = /^((?:AC|FR|NFR|REQ)[-_.]?\d+(?:[-_.]\d+)*)(?:\s*\((P[0-3])\))?\s*:\s*(.+)/.exec(line.replaceAll(/[*`]/g, ''));
|
|
166
|
+
if (claim) {
|
|
167
|
+
flush();
|
|
168
|
+
pending = {
|
|
169
|
+
id: claim[1],
|
|
170
|
+
priority: claim[2] ?? /\((P[0-3])\)\s*$/.exec(claim[3])?.[1] ?? /\b(P[0-3])\b/.exec(claim[3])?.[1],
|
|
171
|
+
requirement: claim[3],
|
|
172
|
+
line: (token.map?.[0] ?? 0) + offset + 1,
|
|
173
|
+
};
|
|
174
|
+
} else if (pending) pending.requirement += `\n${line}`;
|
|
175
|
+
}
|
|
176
|
+
flush();
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
return [...ledger.values()];
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** Reconcile the saved Step 1 ledger with the caller's frozen source claims. */
|
|
183
|
+
function oracleReference(progress, sourceLedger, required, allowLegacyExplicitOracle) {
|
|
184
|
+
const recorded = progress.oracleLedger;
|
|
185
|
+
if (
|
|
186
|
+
recorded === undefined &&
|
|
187
|
+
required &&
|
|
188
|
+
(!allowLegacyExplicitOracle || sourceLedger.length === 0 || sourceLedger.some((row) => !row.priority))
|
|
189
|
+
)
|
|
190
|
+
throw new Error('Trace progress must preserve the Step 1 oracleLedger.');
|
|
191
|
+
const reference = new Map();
|
|
192
|
+
if (recorded !== undefined) {
|
|
193
|
+
if (!Array.isArray(recorded) || recorded.length === 0) throw new Error('Trace Step 1 oracleLedger must be a nonempty array.');
|
|
194
|
+
for (const row of recorded) {
|
|
195
|
+
if (
|
|
196
|
+
!row ||
|
|
197
|
+
typeof row.id !== 'string' ||
|
|
198
|
+
!row.id.trim() ||
|
|
199
|
+
!/^P[0-3]$/.test(row.priority) ||
|
|
200
|
+
typeof row.requirement !== 'string' ||
|
|
201
|
+
!row.requirement.trim() ||
|
|
202
|
+
typeof row.source !== 'string' ||
|
|
203
|
+
!row.source.trim() ||
|
|
204
|
+
reference.has(row.id)
|
|
205
|
+
)
|
|
206
|
+
throw new Error('Trace Step 1 oracleLedger has an invalid or duplicate criterion.');
|
|
207
|
+
reference.set(row.id, row);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
if (sourceLedger.length > 0) {
|
|
211
|
+
if (recorded !== undefined && (reference.size !== sourceLedger.length || sourceLedger.some((row) => !reference.has(row.id))))
|
|
212
|
+
throw new Error('Trace Step 1 oracleLedger differs from the frozen source criterion identities.');
|
|
213
|
+
for (const row of sourceLedger) {
|
|
214
|
+
const saved = reference.get(row.id);
|
|
215
|
+
if (row.priority && saved && saved.priority !== row.priority)
|
|
216
|
+
throw new Error(`Trace Step 1 criterion ${row.id} changed its explicit source priority.`);
|
|
217
|
+
if (saved && (criterionText(saved.requirement) !== criterionText(row.requirement) || saved.source !== row.source))
|
|
218
|
+
throw new Error(`Trace Step 1 criterion ${row.id} changed its frozen requirement text or source binding.`);
|
|
219
|
+
reference.set(row.id, { ...row, priority: row.priority ?? saved?.priority, compareText: Boolean(saved) });
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
return reference;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/** Read criterion claims from Markdown headings and their coverage labels. */
|
|
226
|
+
function matrixInventory(matrix, reference) {
|
|
227
|
+
const criteria = new Map();
|
|
228
|
+
const tableClaims = new Map();
|
|
229
|
+
const ledgerPriorities = new Map();
|
|
230
|
+
let current = null;
|
|
231
|
+
let depth = 0;
|
|
232
|
+
const body = matrix.replace(/^---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/, '');
|
|
233
|
+
const tokens = new MarkdownIt().parse(body, {});
|
|
234
|
+
for (let index = 0; index < tokens.length; index++) {
|
|
235
|
+
if (tokens[index].type !== 'table_open') continue;
|
|
236
|
+
const rows = [];
|
|
237
|
+
let row = null;
|
|
238
|
+
while (++index < tokens.length && tokens[index].type !== 'table_close') {
|
|
239
|
+
if (tokens[index].type === 'tr_open') row = [];
|
|
240
|
+
else if (tokens[index].type === 'inline' && row) row.push(tokens[index].content.replaceAll(/[*`]/g, '').trim());
|
|
241
|
+
else if (tokens[index].type === 'tr_close') {
|
|
242
|
+
rows.push(row);
|
|
243
|
+
row = null;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
const header = rows.shift() ?? [];
|
|
247
|
+
const idIndex = header.findIndex((cell) => /^(?:id|criterion|criterion id)$/i.test(cell));
|
|
248
|
+
const requirementIndex = header.findIndex((cell) => /^requirement$/i.test(cell));
|
|
249
|
+
const priorityIndex = header.findIndex((cell) => /^priority$/i.test(cell));
|
|
250
|
+
const coverageIndex = header.findIndex((cell) => /^coverage$/i.test(cell));
|
|
251
|
+
if (idIndex === -1 || priorityIndex === -1) continue;
|
|
252
|
+
if (coverageIndex === -1 && !header.some((cell) => /^requirement$/i.test(cell))) continue;
|
|
253
|
+
for (const cells of rows) {
|
|
254
|
+
const id = cells[idIndex];
|
|
255
|
+
const priority = cells[priorityIndex];
|
|
256
|
+
if (!id || !/^P[0-3]$/.test(priority)) throw new Error('Trace criterion table has missing identity or priority.');
|
|
257
|
+
if (ledgerPriorities.has(id) && ledgerPriorities.get(id) !== priority)
|
|
258
|
+
throw new Error(`Trace criterion ${id} has conflicting priorities.`);
|
|
259
|
+
ledgerPriorities.set(id, priority);
|
|
260
|
+
if (coverageIndex === -1) continue;
|
|
261
|
+
if (tableClaims.has(id)) throw new Error(`Trace criterion ${id} is duplicated in the matrix table.`);
|
|
262
|
+
const status = /^(FULL|PARTIAL|NONE|UNIT-ONLY|INTEGRATION-ONLY)\b/.exec(cells[coverageIndex])?.[1];
|
|
263
|
+
if (!status) throw new Error(`Trace criterion ${id} has invalid table coverage.`);
|
|
264
|
+
tableClaims.set(id, { priority, status, requirement: requirementIndex === -1 ? undefined : cells[requirementIndex] });
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
for (let index = 0; index < tokens.length; index++) {
|
|
268
|
+
const token = tokens[index];
|
|
269
|
+
if (token.type === 'heading_open') {
|
|
270
|
+
const heading = tokens[index + 1].content;
|
|
271
|
+
const nextDepth = Number(token.tag.slice(1));
|
|
272
|
+
if (nextDepth <= depth) current = null;
|
|
273
|
+
const claim = /^\*{0,2}([A-Za-z0-9][A-Za-z0-9_.-]*)\*{0,2}(?:\s*\((P[0-3])\))?\s*:\s/.exec(heading);
|
|
274
|
+
if (claim && reference.size > 0 && !reference.has(claim[1]) && !ledgerPriorities.has(claim[1])) continue;
|
|
275
|
+
const priority =
|
|
276
|
+
claim?.[2] ??
|
|
277
|
+
/\((P[0-3])\)\s*$/.exec(heading)?.[1] ??
|
|
278
|
+
(claim ? ledgerPriorities.get(claim[1]) : null) ??
|
|
279
|
+
/\b(P[0-3])\b/.exec(heading)?.[1];
|
|
280
|
+
if (!claim || (!priority && !/^[A-Za-z]+-\d+$/.test(claim[1]))) continue;
|
|
281
|
+
if (!priority) throw new Error(`Trace criterion ${claim[1]} has no priority.`);
|
|
282
|
+
if (ledgerPriorities.has(claim[1]) && ledgerPriorities.get(claim[1]) !== priority)
|
|
283
|
+
throw new Error(`Trace criterion ${claim[1]} detail contradicts its oracle priority.`);
|
|
284
|
+
if (criteria.has(claim[1])) throw new Error(`Trace criterion ${claim[1]} is duplicated.`);
|
|
285
|
+
current = { priority, status: null, requirement: heading.slice(claim[0].length) };
|
|
286
|
+
depth = nextDepth;
|
|
287
|
+
criteria.set(claim[1], current);
|
|
288
|
+
} else if (current && token.type === 'inline') {
|
|
289
|
+
const coverage = /\*\*Coverage:?\*\*\s*:?\s*([A-Z-]+)/i.exec(token.content);
|
|
290
|
+
if (!coverage) continue;
|
|
291
|
+
if (current.status !== null) throw new Error('Trace criterion declares coverage more than once.');
|
|
292
|
+
current.status = coverage[1].toUpperCase();
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
for (const [id, criterion] of tableClaims) {
|
|
296
|
+
if (criteria.has(id)) {
|
|
297
|
+
const detail = criteria.get(id);
|
|
298
|
+
if (detail.priority !== criterion.priority || detail.status !== criterion.status)
|
|
299
|
+
throw new Error(`Trace criterion ${id} table and detail disagree.`);
|
|
300
|
+
if (criterion.requirement !== undefined && criterionText(detail.requirement) !== criterionText(criterion.requirement))
|
|
301
|
+
throw new Error(`Trace criterion ${id} table and detail requirement text disagree.`);
|
|
302
|
+
} else criteria.set(id, criterion);
|
|
303
|
+
}
|
|
304
|
+
for (const id of ledgerPriorities.keys()) {
|
|
305
|
+
if (!criteria.has(id)) throw new Error(`Trace oracle criterion ${id} is missing from the matrix.`);
|
|
306
|
+
}
|
|
307
|
+
for (const [id, expected] of reference) {
|
|
308
|
+
if (!criteria.has(id) || criteria.get(id).priority !== expected.priority)
|
|
309
|
+
throw new Error(`Trace criterion ${id} differs from its frozen oracle priority or is missing.`);
|
|
310
|
+
if (
|
|
311
|
+
expected.compareText !== false &&
|
|
312
|
+
(typeof criteria.get(id).requirement !== 'string' ||
|
|
313
|
+
criterionText(criteria.get(id).requirement) !== criterionText(expected.requirement))
|
|
314
|
+
)
|
|
315
|
+
throw new Error(`Trace criterion ${id} changed its frozen requirement text in the matrix.`);
|
|
316
|
+
}
|
|
317
|
+
if (reference.size > 0 && [...criteria.keys()].some((id) => !reference.has(id)))
|
|
318
|
+
throw new Error('Trace matrix has criteria absent from the Step 1 oracle ledger.');
|
|
319
|
+
const inventory = Object.fromEntries(['P0', 'P1', 'P2', 'P3'].map((priority) => [priority, { total: 0, covered: 0 }]));
|
|
320
|
+
for (const [id, criterion] of criteria) {
|
|
321
|
+
if (!['FULL', 'PARTIAL', 'NONE', 'UNIT-ONLY', 'INTEGRATION-ONLY'].includes(criterion.status))
|
|
322
|
+
throw new Error(`Trace criterion ${id} has missing or invalid coverage.`);
|
|
323
|
+
inventory[criterion.priority].total++;
|
|
324
|
+
if (criterion.status === 'FULL') inventory[criterion.priority].covered++;
|
|
325
|
+
}
|
|
326
|
+
return inventory;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/** Check the public artifact contract and Step 5's deterministic gate invariants. */
|
|
330
|
+
function validateTraceOutputs({
|
|
331
|
+
paths,
|
|
332
|
+
target,
|
|
333
|
+
collectionMode,
|
|
334
|
+
allowGate,
|
|
335
|
+
oracleLedger = [],
|
|
336
|
+
requireOracleLedger = false,
|
|
337
|
+
allowLegacyExplicitOracle = false,
|
|
338
|
+
liveCapture,
|
|
339
|
+
requireLiveManifest = false,
|
|
340
|
+
}) {
|
|
341
|
+
const attemptRoot = path.dirname(path.dirname(paths.matrix));
|
|
342
|
+
for (const [name, file] of Object.entries(paths)) {
|
|
343
|
+
if (name === 'gate' && !fs.existsSync(file)) continue;
|
|
344
|
+
try {
|
|
345
|
+
readInput(attemptRoot, file, `current attempt ${name}`);
|
|
346
|
+
} catch (error) {
|
|
347
|
+
throw new Error(error.message, { cause: error });
|
|
348
|
+
}
|
|
349
|
+
if (fs.statSync(file).nlink !== 1) throw new Error(`Current attempt ${name} must be an independent regular artifact.`);
|
|
350
|
+
}
|
|
351
|
+
const matrix = fs.readFileSync(paths.matrix, 'utf8');
|
|
352
|
+
const match = matrix.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/);
|
|
353
|
+
if (!match) throw new Error('Trace matrix is missing progress frontmatter.');
|
|
354
|
+
const progress = yaml.load(match[1]);
|
|
355
|
+
if (progress?.runKey !== target.runKey || progress?.runScope !== target.runScope)
|
|
356
|
+
throw new Error('Trace matrix has a different run identity.');
|
|
357
|
+
if (progress.workflowStatus !== 'completed' || progress.lastStep !== 'step-05-gate-decision')
|
|
358
|
+
throw new Error('Trace workflow has not completed its final step.');
|
|
359
|
+
const requiredSteps = [
|
|
360
|
+
'step-01-load-context',
|
|
361
|
+
'step-02-discover-tests',
|
|
362
|
+
'step-03-map-criteria',
|
|
363
|
+
'step-04-analyze-gaps',
|
|
364
|
+
'step-05-gate-decision',
|
|
365
|
+
];
|
|
366
|
+
if (!Array.isArray(progress.stepsCompleted) || requiredSteps.some((step) => !progress.stepsCompleted.includes(step)))
|
|
367
|
+
throw new Error('Trace matrix has incomplete workflow steps.');
|
|
368
|
+
|
|
369
|
+
const summary = readJson(paths.summary);
|
|
370
|
+
if (!/^0\.3\.\d+$/.test(summary.schema_version ?? '')) throw new Error('Trace summary must declare schema_version 0.3.x.');
|
|
371
|
+
if (summary.collection_mode !== collectionMode || summary.decision_mode !== 'deterministic')
|
|
372
|
+
throw new Error('Trace summary has different collection or decision settings.');
|
|
373
|
+
if (!['COLLECTED', 'WAIVED', 'RESTRICTED', 'INACCESSIBLE', 'DEFERRED_SHARED'].includes(summary.collection_status))
|
|
374
|
+
throw new Error('Trace summary has an unknown collection status.');
|
|
375
|
+
const fixedStatus = { waived: 'WAIVED', restricted: 'RESTRICTED', inaccessible: 'INACCESSIBLE', deferred_shared: 'DEFERRED_SHARED' }[
|
|
376
|
+
collectionMode
|
|
377
|
+
];
|
|
378
|
+
if (fixedStatus && summary.collection_status !== fixedStatus) throw new Error('Trace summary collection status contradicts its mode.');
|
|
379
|
+
if (collectionMode === 'contract_static' && summary.collection_status !== 'COLLECTED')
|
|
380
|
+
throw new Error('Trace summary collection status contradicts contract_static collection.');
|
|
381
|
+
if (requireLiveManifest && collectionMode === 'runtime_manifest') {
|
|
382
|
+
const expectedStatus = liveCapture?.manifest ? 'COLLECTED' : 'INACCESSIBLE';
|
|
383
|
+
if (summary.collection_status !== expectedStatus)
|
|
384
|
+
throw new Error('Trace runtime-manifest collection status contradicts the frozen supplied evidence.');
|
|
385
|
+
}
|
|
386
|
+
if (summary.target?.type !== target.type || String(summary.target?.id ?? '') !== target.id)
|
|
387
|
+
throw new Error('Trace summary has a different target.');
|
|
388
|
+
if (!Number.isFinite(Date.parse(summary.snapshot_at))) throw new Error('Trace summary has no valid snapshot_at timestamp.');
|
|
389
|
+
for (const field of ['repo', 'source_sha', 'evaluator'])
|
|
390
|
+
if (typeof summary[field] !== 'string') throw new Error(`Trace summary has no ${field} string.`);
|
|
391
|
+
if (!['high', 'medium', 'low'].includes(summary.confidence) || summary.oracle?.confidence !== summary.confidence)
|
|
392
|
+
throw new Error('Trace summary has inconsistent oracle confidence.');
|
|
393
|
+
if (
|
|
394
|
+
!['formal_requirements', 'spec_artifact', 'external_pointer', 'synthetic_source'].includes(summary.oracle?.resolution_mode) ||
|
|
395
|
+
!Array.isArray(summary.oracle?.sources) ||
|
|
396
|
+
typeof summary.oracle.synthetic !== 'boolean'
|
|
397
|
+
)
|
|
398
|
+
throw new Error('Trace summary has incomplete oracle metadata.');
|
|
399
|
+
if (!['acceptance_criteria', 'synthetic_requirements', 'openapi_endpoints', 'user_journeys'].includes(summary.inventory_basis))
|
|
400
|
+
throw new Error('Trace summary has no concrete inventory basis.');
|
|
401
|
+
for (const [name, bucket] of [
|
|
402
|
+
['inventory', summary.coverage?.inventory],
|
|
403
|
+
...['P0', 'P1', 'P2', 'P3'].map((key) => [key, summary.coverage?.priority_breakdown?.[key]]),
|
|
404
|
+
]) {
|
|
405
|
+
if (
|
|
406
|
+
!bucket ||
|
|
407
|
+
!Number.isInteger(bucket.total) ||
|
|
408
|
+
!Number.isInteger(bucket.covered) ||
|
|
409
|
+
bucket.total < 0 ||
|
|
410
|
+
bucket.covered < 0 ||
|
|
411
|
+
bucket.covered > bucket.total
|
|
412
|
+
) {
|
|
413
|
+
throw new Error(`Trace coverage ${name} has invalid counts.`);
|
|
414
|
+
}
|
|
415
|
+
const expected = bucket.total === 0 ? 100 : Math.round((bucket.covered / bucket.total) * 100);
|
|
416
|
+
if (bucket.pct !== expected) throw new Error(`Trace coverage ${name} percentage does not match its counts.`);
|
|
417
|
+
}
|
|
418
|
+
const priorities = ['P0', 'P1', 'P2', 'P3'].map((key) => summary.coverage.priority_breakdown[key]);
|
|
419
|
+
const reference = oracleReference(progress, oracleLedger, requireOracleLedger, allowLegacyExplicitOracle);
|
|
420
|
+
const declared = matrixInventory(matrix, reference);
|
|
421
|
+
for (const [priority, counts] of Object.entries(declared)) {
|
|
422
|
+
const reported = summary.coverage.priority_breakdown[priority];
|
|
423
|
+
if (reported.total !== counts.total || reported.covered !== counts.covered)
|
|
424
|
+
throw new Error(`Trace matrix ${priority} coverage contradicts its summary.`);
|
|
425
|
+
}
|
|
426
|
+
if (
|
|
427
|
+
priorities.reduce((total, bucket) => total + bucket.total, 0) !== summary.coverage.inventory.total ||
|
|
428
|
+
priorities.reduce((total, bucket) => total + bucket.covered, 0) !== summary.coverage.inventory.covered
|
|
429
|
+
)
|
|
430
|
+
throw new Error('Trace priority counts disagree with overall coverage.');
|
|
431
|
+
for (const level of LEVELS) {
|
|
432
|
+
const bucket = summary.coverage.by_level?.[level];
|
|
433
|
+
if (
|
|
434
|
+
!bucket ||
|
|
435
|
+
!Number.isInteger(bucket.tests) ||
|
|
436
|
+
bucket.tests < 0 ||
|
|
437
|
+
!Number.isInteger(bucket.criteria_covered) ||
|
|
438
|
+
bucket.criteria_covered < 0
|
|
439
|
+
)
|
|
440
|
+
throw new Error(`Trace summary has invalid ${level} inventory.`);
|
|
441
|
+
}
|
|
442
|
+
for (const field of ['files', 'cases', 'skipped_cases', 'fixme_cases', 'pending_cases']) {
|
|
443
|
+
if (!Number.isInteger(summary.tests?.[field]) || summary.tests[field] < 0) throw new Error(`Trace summary has invalid tests.${field}.`);
|
|
444
|
+
}
|
|
445
|
+
if (LEVELS.reduce((total, level) => total + summary.coverage.by_level[level].tests, 0) !== summary.tests.cases)
|
|
446
|
+
throw new Error('Trace test totals disagree with level inventory.');
|
|
447
|
+
for (const field of ['blockers', 'recommendations', 'rejected_evidence'])
|
|
448
|
+
if (!Array.isArray(summary[field])) throw new Error(`Trace summary has no ${field} array.`);
|
|
449
|
+
for (const field of ['critical_open', 'high_open', 'medium_open', 'low_open'])
|
|
450
|
+
if (!Number.isInteger(summary.risk_summary?.[field]) || summary.risk_summary[field] < 0)
|
|
451
|
+
throw new Error(`Trace summary has invalid risk_summary.${field}.`);
|
|
452
|
+
if (!summary.live_evidence || typeof summary.live_evidence !== 'object' || Array.isArray(summary.live_evidence))
|
|
453
|
+
throw new Error('Trace summary has no live_evidence metadata.');
|
|
454
|
+
const frozenLive = liveReference(liveCapture, new Set(reference.keys()));
|
|
455
|
+
if (frozenLive) {
|
|
456
|
+
for (const field of ['present', 'freshness', 'failed'])
|
|
457
|
+
if (summary.live_evidence[field] !== frozenLive[field])
|
|
458
|
+
throw new Error(`Trace live evidence ${field} differs from the frozen supplied manifest.`);
|
|
459
|
+
const reportedSha = summary.live_evidence.current_source_sha;
|
|
460
|
+
if (reportedSha !== frozenLive.current_source_sha && !(frozenLive.current_source_sha === '' && reportedSha === 'unknown'))
|
|
461
|
+
throw new Error('Trace live evidence current_source_sha differs from the frozen consuming revision.');
|
|
462
|
+
if ('fresh_failed' in summary.live_evidence && summary.live_evidence.fresh_failed !== frozenLive.freshFailed)
|
|
463
|
+
throw new Error('Trace live evidence fresh_failed differs from the frozen supplied manifest.');
|
|
464
|
+
}
|
|
465
|
+
// Step 4 counts every non-FULL P0 item as critical; P1-P3 gap buckets count NONE only.
|
|
466
|
+
for (const [index, field] of ['critical_open', 'high_open', 'medium_open', 'low_open'].entries()) {
|
|
467
|
+
const uncovered = priorities[index].total - priorities[index].covered;
|
|
468
|
+
if (index === 0 ? summary.risk_summary[field] !== uncovered : summary.risk_summary[field] > uncovered)
|
|
469
|
+
throw new Error(`Trace ${field} contradicts its uncovered priority inventory.`);
|
|
470
|
+
}
|
|
471
|
+
if (summary.links?.trace_report_path !== paths.matrix) throw new Error('Trace summary links to a different matrix.');
|
|
472
|
+
|
|
473
|
+
const eligible = allowGate && summary.collection_status === 'COLLECTED';
|
|
474
|
+
let gate = null;
|
|
475
|
+
if (eligible) {
|
|
476
|
+
if (!GATE_STATUSES.has(summary.gate_status) || summary.gate_basis !== 'priority_thresholds')
|
|
477
|
+
throw new Error('Eligible trace summary has no valid gate decision.');
|
|
478
|
+
const criteria = summary.gate_criteria;
|
|
479
|
+
for (const field of [
|
|
480
|
+
'p0_coverage_required',
|
|
481
|
+
'p0_coverage_actual',
|
|
482
|
+
'p1_coverage_target',
|
|
483
|
+
'p1_coverage_minimum',
|
|
484
|
+
'p1_coverage_actual',
|
|
485
|
+
'overall_coverage_minimum',
|
|
486
|
+
'overall_coverage_actual',
|
|
487
|
+
])
|
|
488
|
+
if (typeof criteria?.[field] !== 'string' || !/^\d+%$/.test(criteria[field])) throw new Error(`Trace gate has invalid ${field}.`);
|
|
489
|
+
for (const field of ['p0_status', 'p1_status', 'overall_status'])
|
|
490
|
+
if (!['MET', 'PARTIAL', 'NOT_MET'].includes(criteria?.[field])) throw new Error(`Trace gate has invalid ${field}.`);
|
|
491
|
+
// Step 5 defines these public gate fields. Validate their consistency with
|
|
492
|
+
// the inventory; criterion classification remains the skill's assessment.
|
|
493
|
+
const p0 = summary.coverage.priority_breakdown.P0.pct;
|
|
494
|
+
const p1 = summary.coverage.priority_breakdown.P1.pct;
|
|
495
|
+
const overall = summary.coverage.inventory.pct;
|
|
496
|
+
const expectedCriteria = {
|
|
497
|
+
p0_coverage_required: '100%',
|
|
498
|
+
p0_coverage_actual: `${p0}%`,
|
|
499
|
+
p0_status: p0 === 100 ? 'MET' : 'NOT_MET',
|
|
500
|
+
p1_coverage_target: '90%',
|
|
501
|
+
p1_coverage_minimum: '80%',
|
|
502
|
+
p1_coverage_actual: `${p1}%`,
|
|
503
|
+
p1_status: p1 >= 90 ? 'MET' : p1 >= 80 ? 'PARTIAL' : 'NOT_MET',
|
|
504
|
+
overall_coverage_minimum: '80%',
|
|
505
|
+
overall_coverage_actual: `${overall}%`,
|
|
506
|
+
overall_status: overall >= 80 ? 'MET' : 'NOT_MET',
|
|
507
|
+
};
|
|
508
|
+
for (const [field, expected] of Object.entries(expectedCriteria))
|
|
509
|
+
if (criteria[field] !== expected) throw new Error(`Trace gate ${field} contradicts the coverage inventory or Step 5 thresholds.`);
|
|
510
|
+
let expectedStatus = p0 < 100 || overall < 80 || p1 < 80 ? 'FAIL' : p1 < 90 ? 'CONCERNS' : 'PASS';
|
|
511
|
+
const liveOnly = summary.live_evidence.requirements_live_only;
|
|
512
|
+
if (!Number.isInteger(liveOnly) || liveOnly < 0) throw new Error('Trace summary has invalid live-only coverage metadata.');
|
|
513
|
+
const synthetic = summary.oracle.synthetic || ['synthetic_requirements', 'user_journeys'].includes(summary.inventory_basis);
|
|
514
|
+
const liveFailed = summary.live_evidence.failed;
|
|
515
|
+
if (summary.live_evidence.freshness === 'fresh' && (!Number.isInteger(liveFailed) || liveFailed < 0))
|
|
516
|
+
throw new Error('Trace fresh live evidence has invalid failed count.');
|
|
517
|
+
const freshFailed =
|
|
518
|
+
frozenLive?.freshFailed ?? summary.live_evidence.fresh_failed ?? (summary.live_evidence.freshness === 'fresh' ? liveFailed : 0);
|
|
519
|
+
if (freshFailed !== undefined && (!Number.isInteger(freshFailed) || freshFailed < 0))
|
|
520
|
+
throw new Error('Trace live evidence has invalid fresh_failed count.');
|
|
521
|
+
const freshFailure = freshFailed > 0;
|
|
522
|
+
if (expectedStatus === 'PASS' && ((synthetic && summary.confidence !== 'high') || liveOnly > 0 || freshFailure))
|
|
523
|
+
expectedStatus = 'CONCERNS';
|
|
524
|
+
if (summary.gate_status !== expectedStatus) throw new Error('Trace gate decision contradicts its coverage and confidence evidence.');
|
|
525
|
+
gate = readJson(paths.gate);
|
|
526
|
+
if (
|
|
527
|
+
gate.schema_version !== '0.1.0' ||
|
|
528
|
+
gate.gate_status !== summary.gate_status ||
|
|
529
|
+
gate.evaluated_at !== summary.snapshot_at ||
|
|
530
|
+
gate.target?.type !== summary.target.type ||
|
|
531
|
+
String(gate.target?.id ?? '') !== target.id
|
|
532
|
+
)
|
|
533
|
+
throw new Error('Trace gate artifact disagrees with its summary.');
|
|
534
|
+
if (
|
|
535
|
+
gate.collection_status !== summary.collection_status ||
|
|
536
|
+
gate.gate_basis !== summary.gate_basis ||
|
|
537
|
+
typeof gate.rationale !== 'string' ||
|
|
538
|
+
!gate.rationale.trim() ||
|
|
539
|
+
gate.critical_open !== summary.risk_summary.critical_open
|
|
540
|
+
)
|
|
541
|
+
throw new Error('Trace gate artifact has incomplete or inconsistent decision evidence.');
|
|
542
|
+
for (const field of ['p0_status', 'p1_status', 'overall_status'])
|
|
543
|
+
if (gate[field] !== criteria[field]) throw new Error(`Trace gate artifact disagrees with ${field}.`);
|
|
544
|
+
} else if ('gate_status' in summary || 'gate_criteria' in summary || fs.existsSync(paths.gate) || summary.gate_basis !== 'none') {
|
|
545
|
+
throw new Error('Trace collected no eligible gate, but emitted a gate signal.');
|
|
546
|
+
}
|
|
547
|
+
return { matrix, summary, gate };
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
/** Publish only the validated current attempt, updating the moved report link. */
|
|
551
|
+
function publishTraceOutputs(value, destinations, io = fs) {
|
|
552
|
+
const links = { ...value.summary.links, trace_report_path: destinations.matrix };
|
|
553
|
+
const summary = { ...value.summary, links };
|
|
554
|
+
const gate = value.gate ? { ...value.gate, links } : null;
|
|
555
|
+
const contents = {
|
|
556
|
+
matrix: value.matrix,
|
|
557
|
+
summary: `${JSON.stringify(summary, null, 2)}\n`,
|
|
558
|
+
gate: gate ? `${JSON.stringify(gate, null, 2)}\n` : null,
|
|
559
|
+
};
|
|
560
|
+
const staged = [];
|
|
561
|
+
let keepBackups = false;
|
|
562
|
+
try {
|
|
563
|
+
for (const [name, destination] of Object.entries(destinations)) {
|
|
564
|
+
io.mkdirSync(path.dirname(destination), { recursive: true });
|
|
565
|
+
const directory = io.mkdtempSync(path.join(path.dirname(destination), '.tea-trace-publish-'));
|
|
566
|
+
const item = {
|
|
567
|
+
destination,
|
|
568
|
+
directory,
|
|
569
|
+
previous: path.join(directory, 'previous'),
|
|
570
|
+
next: path.join(directory, 'next'),
|
|
571
|
+
existed: io.existsSync(destination),
|
|
572
|
+
installed: false,
|
|
573
|
+
remove: contents[name] === null,
|
|
574
|
+
};
|
|
575
|
+
staged.push(item);
|
|
576
|
+
if (item.existed) {
|
|
577
|
+
if (!io.statSync(destination).isFile()) throw new Error(`Trace destination is not a regular file: ${destination}`);
|
|
578
|
+
io.copyFileSync(destination, item.previous);
|
|
579
|
+
}
|
|
580
|
+
if (!item.remove) {
|
|
581
|
+
io.writeFileSync(item.next, contents[name], 'utf8');
|
|
582
|
+
if (item.existed) io.chmodSync(item.next, io.statSync(destination).mode);
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
for (const item of staged) {
|
|
586
|
+
if (item.remove) {
|
|
587
|
+
if (item.existed) io.unlinkSync(item.destination);
|
|
588
|
+
} else io.renameSync(item.next, item.destination);
|
|
589
|
+
item.installed = true;
|
|
590
|
+
}
|
|
591
|
+
} catch (error) {
|
|
592
|
+
const failures = [];
|
|
593
|
+
for (const item of staged.filter((entry) => entry.installed).toReversed()) {
|
|
594
|
+
try {
|
|
595
|
+
if (item.existed) io.renameSync(item.previous, item.destination);
|
|
596
|
+
else if (!item.remove) io.unlinkSync(item.destination);
|
|
597
|
+
} catch (error) {
|
|
598
|
+
failures.push(`${item.destination}: ${error.message}`);
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
keepBackups = failures.length > 0;
|
|
602
|
+
throw new Error(
|
|
603
|
+
`Trace publication failed: ${error.message}; ${keepBackups ? `recovery backups retained at ${staged.map((item) => item.directory).join(', ')}; rollback failures: ${failures.join('; ')}` : 'previous reports restored'}`,
|
|
604
|
+
{ cause: error },
|
|
605
|
+
);
|
|
606
|
+
} finally {
|
|
607
|
+
if (!keepBackups) {
|
|
608
|
+
for (const item of staged) {
|
|
609
|
+
try {
|
|
610
|
+
io.rmSync(item.directory, { recursive: true, force: true });
|
|
611
|
+
} catch (error) {
|
|
612
|
+
process.stderr.write(`Trace publication staging cleanup failed at ${item.directory}: ${error.message}\n`);
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
return summary;
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
module.exports = {
|
|
621
|
+
sourceOracleLedger,
|
|
622
|
+
COLLECTION_MODES,
|
|
623
|
+
GATE_TYPES,
|
|
624
|
+
resolveTraceTarget,
|
|
625
|
+
tracePaths,
|
|
626
|
+
validateTraceOutputs,
|
|
627
|
+
publishTraceOutputs,
|
|
628
|
+
};
|