bmad-method-test-architecture-enterprise 2.0.3 → 2.0.4-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/cli/automate.js +272 -0
- package/cli/lib/agent-adapters.js +5 -3
- package/cli/lib/automate-inventory.js +412 -0
- package/cli/lib/automate-prompt.js +118 -0
- package/cli/lib/automate-result.js +311 -0
- package/cli/lib/resolve-tea-config.js +3 -0
- package/cli/lib/test-design-parser.js +57 -5
- package/cli/lib/workflow-cli.js +9 -0
- package/cli/test-design.js +410 -0
- package/package.json +8 -2
- package/skills/bmad-testarch-automate/red/steps-c/step-04a-subagent-api-failing.md +1 -0
- package/skills/bmad-testarch-automate/red/steps-c/step-04b-subagent-e2e-failing.md +1 -0
- package/skills/bmad-testarch-automate/steps-c/step-03a-subagent-api.md +1 -0
- package/skills/bmad-testarch-automate/steps-c/step-03b-subagent-e2e.md +1 -0
- package/skills/bmad-testarch-test-design/steps-c/step-03-risk-and-testability.md +6 -0
- package/skills/bmad-testarch-test-design/steps-c/step-04-coverage-plan.md +3 -0
- package/skills/bmad-testarch-test-design/steps-c/step-05-generate-output.md +2 -1
- package/skills/bmod-tea/bmod.toml +1 -1
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('node:fs');
|
|
4
|
+
const path = require('node:path');
|
|
5
|
+
const { createHash } = require('node:crypto');
|
|
6
|
+
const { parse } = require('@babel/parser');
|
|
7
|
+
const { minimatch } = require('minimatch');
|
|
8
|
+
|
|
9
|
+
const slash = (value) => value.split(path.sep).join('/');
|
|
10
|
+
const canonical = (value) => {
|
|
11
|
+
let ancestor = path.resolve(value);
|
|
12
|
+
const suffix = [];
|
|
13
|
+
while (!fs.existsSync(ancestor)) {
|
|
14
|
+
const parent = path.dirname(ancestor);
|
|
15
|
+
if (parent === ancestor) break;
|
|
16
|
+
suffix.unshift(path.basename(ancestor));
|
|
17
|
+
ancestor = parent;
|
|
18
|
+
}
|
|
19
|
+
return path.resolve(fs.realpathSync(ancestor), ...suffix);
|
|
20
|
+
};
|
|
21
|
+
const literal = (node) =>
|
|
22
|
+
node?.type === 'StringLiteral'
|
|
23
|
+
? node.value
|
|
24
|
+
: node?.type === 'TemplateLiteral' && node.expressions.length === 0
|
|
25
|
+
? node.quasis[0].value.cooked
|
|
26
|
+
: null;
|
|
27
|
+
|
|
28
|
+
// JSONReporter serializes both globs and RegExp instances with toString().
|
|
29
|
+
// Apply the runner's absolute-path, reset-lastIndex and case-insensitive glob rules.
|
|
30
|
+
function nativeMatches(patterns, absolute) {
|
|
31
|
+
if (!Array.isArray(patterns)) throw new Error('native Playwright selectors must be arrays');
|
|
32
|
+
return patterns.some((pattern) => {
|
|
33
|
+
if (typeof pattern !== 'string') throw new Error('native Playwright selector is invalid');
|
|
34
|
+
const serialized = pattern.match(/^\/(.*)\/([dgimsuvy]*)$/s);
|
|
35
|
+
if (serialized) {
|
|
36
|
+
let expression;
|
|
37
|
+
try {
|
|
38
|
+
expression = new RegExp(serialized[1], serialized[2]);
|
|
39
|
+
} catch {
|
|
40
|
+
throw new Error('native Playwright RegExp selector is invalid');
|
|
41
|
+
}
|
|
42
|
+
expression.lastIndex = 0;
|
|
43
|
+
if (expression.test(absolute)) return true;
|
|
44
|
+
expression.lastIndex = 0;
|
|
45
|
+
return path.sep === '\\' && expression.test(slash(absolute));
|
|
46
|
+
}
|
|
47
|
+
return minimatch(absolute, pattern.startsWith('**/') ? pattern : `**/${pattern}`, { nocase: true, dot: true });
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Track lexical declarations so a shadowed name cannot inherit a test binding.
|
|
52
|
+
function sourceBindings(ast, file, projectRoot) {
|
|
53
|
+
const scopes = new WeakMap();
|
|
54
|
+
const root = { parent: null, bindings: new Map(), functionScope: true };
|
|
55
|
+
const assignments = [];
|
|
56
|
+
const children = (node, visit) => {
|
|
57
|
+
for (const [key, value] of Object.entries(node)) {
|
|
58
|
+
if (['loc', 'start', 'end'].includes(key)) continue;
|
|
59
|
+
if (Array.isArray(value)) for (const child of value) visit(child);
|
|
60
|
+
else if (value && typeof value === 'object') visit(value);
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
const declare = (pattern, scope, init, property) => {
|
|
64
|
+
if (!pattern) return;
|
|
65
|
+
switch (pattern.type) {
|
|
66
|
+
case 'Identifier': {
|
|
67
|
+
scope.bindings.set(pattern.name, { init, property });
|
|
68
|
+
break;
|
|
69
|
+
}
|
|
70
|
+
case 'ObjectPattern': {
|
|
71
|
+
for (const item of pattern.properties) declare(item.value ?? item.argument, scope, init, item.key?.name ?? literal(item.key));
|
|
72
|
+
break;
|
|
73
|
+
}
|
|
74
|
+
case 'AssignmentPattern': {
|
|
75
|
+
declare(pattern.left, scope, init, property);
|
|
76
|
+
break;
|
|
77
|
+
}
|
|
78
|
+
case 'ArrayPattern': {
|
|
79
|
+
for (const item of pattern.elements) declare(item, scope, null);
|
|
80
|
+
break;
|
|
81
|
+
}
|
|
82
|
+
case 'RestElement': {
|
|
83
|
+
declare(pattern.argument, scope, null);
|
|
84
|
+
break;
|
|
85
|
+
}
|
|
86
|
+
default: {
|
|
87
|
+
break;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
};
|
|
91
|
+
const build = (node, scope, parentNode) => {
|
|
92
|
+
if (!node || typeof node !== 'object') return;
|
|
93
|
+
if (node.type === 'FunctionDeclaration' && node.id) declare(node.id, scope, null);
|
|
94
|
+
if (node.type === 'ClassDeclaration' && node.id) declare(node.id, scope, null);
|
|
95
|
+
if (node.type === 'BlockStatement' || /Function/.test(node.type) || node.type === 'CatchClause') {
|
|
96
|
+
scope = { parent: scope, bindings: new Map(), functionScope: /Function/.test(node.type) };
|
|
97
|
+
for (const parameter of node.params ?? []) declare(parameter, scope, null);
|
|
98
|
+
declare(node.param, scope, null);
|
|
99
|
+
}
|
|
100
|
+
scopes.set(node, scope);
|
|
101
|
+
if (node.type === 'ImportDeclaration')
|
|
102
|
+
for (const item of node.specifiers) {
|
|
103
|
+
const imported = ['ImportNamespaceSpecifier', 'ImportDefaultSpecifier'].includes(item.type) ? 'namespace' : item.imported?.name;
|
|
104
|
+
scope.bindings.set(item.local.name, {
|
|
105
|
+
kind: imported === 'test' ? 'test' : imported === 'mergeTests' ? 'merge' : imported === 'namespace' ? 'namespace' : 'other',
|
|
106
|
+
imported: true,
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
if (node.type === 'VariableDeclarator') {
|
|
110
|
+
let declarationScope = scope;
|
|
111
|
+
if (parentNode?.kind === 'var') while (!declarationScope.functionScope) declarationScope = declarationScope.parent;
|
|
112
|
+
declare(node.id, declarationScope, node.init);
|
|
113
|
+
}
|
|
114
|
+
if (node.type === 'AssignmentExpression' && node.left.type === 'Identifier') assignments.push(node);
|
|
115
|
+
children(node, (child) => build(child, scope, node));
|
|
116
|
+
};
|
|
117
|
+
build(ast.program, root);
|
|
118
|
+
const findBinding = (node) => {
|
|
119
|
+
let scope = scopes.get(node);
|
|
120
|
+
while (scope && !scope.bindings.has(node.name)) scope = scope.parent;
|
|
121
|
+
return scope?.bindings.get(node.name);
|
|
122
|
+
};
|
|
123
|
+
for (const assignment of assignments) {
|
|
124
|
+
const binding = findBinding(assignment.left);
|
|
125
|
+
if (binding) (binding.assignments ??= []).push(assignment.right);
|
|
126
|
+
}
|
|
127
|
+
let moduleSource = /\.(?:mjs|mts)$/.test(file);
|
|
128
|
+
if (!/\.(?:[cm]js|[cm]ts)$/.test(file)) {
|
|
129
|
+
let directory = path.dirname(path.resolve(projectRoot, file));
|
|
130
|
+
while (true) {
|
|
131
|
+
const pkg = path.join(directory, 'package.json');
|
|
132
|
+
if (fs.existsSync(pkg)) {
|
|
133
|
+
moduleSource = JSON.parse(fs.readFileSync(pkg, 'utf8')).type === 'module';
|
|
134
|
+
break;
|
|
135
|
+
}
|
|
136
|
+
const parent = path.dirname(directory);
|
|
137
|
+
if (parent === directory) break;
|
|
138
|
+
directory = parent;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
const memberName = (node) => (node.computed ? literal(node.property) : node.property.name);
|
|
142
|
+
const resolve = (node, seen = new Set()) => {
|
|
143
|
+
if (!node) return null;
|
|
144
|
+
if (node.type === 'Identifier') {
|
|
145
|
+
const binding = findBinding(node);
|
|
146
|
+
// Bare test is retained for native ATDD/global test source inventories.
|
|
147
|
+
if (!binding) return node.name === 'test' ? { kind: 'test', importEnd: true } : null;
|
|
148
|
+
if (seen.has(binding)) return null;
|
|
149
|
+
seen = new Set([...seen, binding]);
|
|
150
|
+
if (
|
|
151
|
+
binding.assignments?.length &&
|
|
152
|
+
(['test', 'namespace', 'merge'].includes(binding.kind) ||
|
|
153
|
+
['test', 'namespace', 'merge'].includes(resolve(binding.init, seen)?.kind) ||
|
|
154
|
+
binding.assignments.some((value) => ['test', 'namespace', 'merge'].includes(resolve(value, seen)?.kind)))
|
|
155
|
+
)
|
|
156
|
+
throw new Error(`generated test inventory has reassigned test binding: ${file}`);
|
|
157
|
+
if (binding.kind) return { kind: binding.kind, importEnd: binding.imported && !moduleSource };
|
|
158
|
+
const value = resolve(binding.init, seen);
|
|
159
|
+
if (binding.property)
|
|
160
|
+
return value?.kind === 'namespace'
|
|
161
|
+
? { kind: binding.property === 'test' ? 'test' : binding.property === 'mergeTests' ? 'merge' : 'other' }
|
|
162
|
+
: null;
|
|
163
|
+
return value ? { kind: value.kind } : null;
|
|
164
|
+
}
|
|
165
|
+
if (node.type === 'MemberExpression') {
|
|
166
|
+
const value = resolve(node.object, seen);
|
|
167
|
+
if (value?.kind === 'namespace' && node.computed && memberName(node) === null)
|
|
168
|
+
throw new Error(`generated test inventory has computed registration: ${file}`);
|
|
169
|
+
if (value?.kind === 'namespace')
|
|
170
|
+
return { kind: memberName(node) === 'test' ? 'test' : memberName(node) === 'mergeTests' ? 'merge' : 'other' };
|
|
171
|
+
return null;
|
|
172
|
+
}
|
|
173
|
+
if (node.type === 'CallExpression') {
|
|
174
|
+
if (node.callee.type === 'Identifier' && node.callee.name === 'require' && literal(node.arguments[0]) !== null)
|
|
175
|
+
return { kind: 'namespace' };
|
|
176
|
+
if (resolve(node.callee, seen)?.kind === 'merge') {
|
|
177
|
+
if (node.arguments.length === 0 || node.arguments.some((argument) => resolve(argument, seen)?.kind !== 'test'))
|
|
178
|
+
throw new Error(`generated test inventory has unresolved merged binding: ${file}`);
|
|
179
|
+
return { kind: 'test' };
|
|
180
|
+
}
|
|
181
|
+
if (
|
|
182
|
+
node.callee.type === 'MemberExpression' &&
|
|
183
|
+
memberName(node.callee) === 'extend' &&
|
|
184
|
+
resolve(node.callee.object, seen)?.kind === 'test'
|
|
185
|
+
)
|
|
186
|
+
return { kind: 'test' };
|
|
187
|
+
}
|
|
188
|
+
return null;
|
|
189
|
+
};
|
|
190
|
+
const registration = (node) => {
|
|
191
|
+
const value = resolve(node);
|
|
192
|
+
if (value?.kind === 'test') return { binding: value, chain: [], base: node };
|
|
193
|
+
if (node?.type === 'MemberExpression') {
|
|
194
|
+
const parent = registration(node.object);
|
|
195
|
+
if (parent) return { ...parent, chain: [...parent.chain, memberName(node)] };
|
|
196
|
+
}
|
|
197
|
+
return null;
|
|
198
|
+
};
|
|
199
|
+
const location = (callee, binding) => {
|
|
200
|
+
// Native member calls point at the property. Transformed named imports map
|
|
201
|
+
// the generated indirect call to callee-end; local/CJS/ESM calls use start.
|
|
202
|
+
const point = callee.type === 'MemberExpression' ? callee.property.loc.start : binding.importEnd ? callee.loc.end : callee.loc.start;
|
|
203
|
+
return { line: point.line, column: point.column + 1 };
|
|
204
|
+
};
|
|
205
|
+
return { registration, location };
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// Enumerate registrations without executing adopter code. Runtime enumeration is
|
|
209
|
+
// deliberately unmeasured: a numerical report cannot prove its missing leaves.
|
|
210
|
+
function generatedInventory(projectRoot, files) {
|
|
211
|
+
const inventory = {
|
|
212
|
+
schemaVersion: 1,
|
|
213
|
+
basis: 'post-generation source syntax',
|
|
214
|
+
projectRoot: fs.realpathSync(projectRoot),
|
|
215
|
+
files: [],
|
|
216
|
+
tests: [],
|
|
217
|
+
};
|
|
218
|
+
for (const file of files) {
|
|
219
|
+
const bytes = fs.readFileSync(path.join(projectRoot, file));
|
|
220
|
+
inventory.files.push({ file, sha256: createHash('sha256').update(bytes).digest('hex') });
|
|
221
|
+
if (!/\.(?:[cm]?[jt]sx?)$/.test(file)) continue;
|
|
222
|
+
const ast = parse(bytes.toString('utf8'), { sourceType: 'unambiguous', plugins: ['typescript', 'jsx'] });
|
|
223
|
+
const { registration, location } = sourceBindings(ast, file, projectRoot);
|
|
224
|
+
const walk = (node, titles = [], dynamic = false) => {
|
|
225
|
+
if (!node || typeof node !== 'object') return;
|
|
226
|
+
if (node.type === 'CallExpression') {
|
|
227
|
+
const resolved = registration(node.callee);
|
|
228
|
+
if (
|
|
229
|
+
!resolved &&
|
|
230
|
+
!dynamic &&
|
|
231
|
+
literal(node.arguments[0]) !== null &&
|
|
232
|
+
node.arguments.some((arg) => ['ArrowFunctionExpression', 'FunctionExpression'].includes(arg.type))
|
|
233
|
+
)
|
|
234
|
+
throw new Error(`generated test inventory has unresolved registration binding: ${file}`);
|
|
235
|
+
if (resolved) {
|
|
236
|
+
const chain = ['test', ...resolved.chain];
|
|
237
|
+
if (chain.includes(null)) throw new Error(`generated test inventory has computed registration: ${file}`);
|
|
238
|
+
if (chain[1] === 'each') throw new Error(`generated test inventory has unresolved table registration: ${file}`);
|
|
239
|
+
if (chain[1] === 'describe' && ['describe', 'only', 'skip', 'serial', 'parallel'].includes(chain.at(-1))) {
|
|
240
|
+
const title = literal(node.arguments[0]);
|
|
241
|
+
const callback = node.arguments.find((arg) => ['ArrowFunctionExpression', 'FunctionExpression'].includes(arg.type));
|
|
242
|
+
if (title === null || !callback || dynamic) throw new Error(`generated test inventory has dynamic suite registration: ${file}`);
|
|
243
|
+
walk(callback.body, [...titles, title], false);
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
if (chain.length === 1 || (chain.length === 2 && ['only', 'skip', 'fixme', 'fail'].includes(chain[1]))) {
|
|
247
|
+
// test.skip(condition, reason) inside a test is metadata, never a leaf.
|
|
248
|
+
const callback = node.arguments.find((arg) => ['ArrowFunctionExpression', 'FunctionExpression'].includes(arg.type));
|
|
249
|
+
if (callback) {
|
|
250
|
+
const title = literal(node.arguments[0]);
|
|
251
|
+
if (title === null || dynamic) throw new Error(`generated test inventory has dynamic test registration: ${file}`);
|
|
252
|
+
inventory.tests.push({
|
|
253
|
+
file,
|
|
254
|
+
titlePath: [...titles, title],
|
|
255
|
+
...location(node.callee, resolved.binding),
|
|
256
|
+
...(chain[1] === 'skip'
|
|
257
|
+
? {
|
|
258
|
+
activationLine: location(resolved.base, resolved.binding).line,
|
|
259
|
+
activationColumn: location(resolved.base, resolved.binding).column,
|
|
260
|
+
}
|
|
261
|
+
: {}),
|
|
262
|
+
});
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
if (chain.length === 1 || literal(node.arguments[0]) !== null)
|
|
266
|
+
throw new Error(`generated test inventory has unresolved callback registration: ${file}`);
|
|
267
|
+
}
|
|
268
|
+
if (
|
|
269
|
+
node.arguments.some((arg) => ['ArrowFunctionExpression', 'FunctionExpression'].includes(arg.type)) &&
|
|
270
|
+
!['beforeEach', 'beforeAll', 'afterEach', 'afterAll', 'step', 'extend'].includes(chain[1])
|
|
271
|
+
)
|
|
272
|
+
throw new Error(`generated test inventory has unsupported registration: ${file}`);
|
|
273
|
+
if (
|
|
274
|
+
chain.length > 1 &&
|
|
275
|
+
![
|
|
276
|
+
'describe',
|
|
277
|
+
'only',
|
|
278
|
+
'skip',
|
|
279
|
+
'fixme',
|
|
280
|
+
'fail',
|
|
281
|
+
'beforeEach',
|
|
282
|
+
'beforeAll',
|
|
283
|
+
'afterEach',
|
|
284
|
+
'afterAll',
|
|
285
|
+
'step',
|
|
286
|
+
'extend',
|
|
287
|
+
'use',
|
|
288
|
+
'info',
|
|
289
|
+
'setTimeout',
|
|
290
|
+
'slow',
|
|
291
|
+
'expect',
|
|
292
|
+
].includes(chain[1])
|
|
293
|
+
)
|
|
294
|
+
throw new Error(`generated test inventory has unresolved registration: ${file}`);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
const nestedDynamic = dynamic || /(?:Function|Method|For|While|If|Conditional|Switch)/.test(node.type);
|
|
298
|
+
for (const [key, value] of Object.entries(node)) {
|
|
299
|
+
if (['loc', 'start', 'end'].includes(key)) continue;
|
|
300
|
+
if (Array.isArray(value)) for (const child of value) walk(child, titles, nestedDynamic);
|
|
301
|
+
else if (value && typeof value === 'object') walk(value, titles, nestedDynamic);
|
|
302
|
+
}
|
|
303
|
+
};
|
|
304
|
+
walk(ast.program);
|
|
305
|
+
}
|
|
306
|
+
if (inventory.tests.length === 0) throw new Error('generated test inventory contains no measurable test leaves');
|
|
307
|
+
return inventory;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
function reconcileInventory(inventory, reports, { mode, reportProjectRoot = inventory.projectRoot } = {}) {
|
|
311
|
+
const covered = new Map();
|
|
312
|
+
const projectsByFile = new Map();
|
|
313
|
+
if (reports.some((report) => Array.isArray(report.suites)) && reports.some((report) => Array.isArray(report.files)))
|
|
314
|
+
throw new Error('final execution mixes incompatible runner scope identities');
|
|
315
|
+
const identity = (leaf, project) => JSON.stringify([leaf.file, leaf.titlePath, leaf.line, leaf.column, project]);
|
|
316
|
+
const playwrightLeaves = new Map();
|
|
317
|
+
const atddLeaves = new Map();
|
|
318
|
+
const sourceKey = (file, titles, line, column) => JSON.stringify([file, titles, line, column]);
|
|
319
|
+
const index = (map, key, leaf) => map.set(key, [...(map.get(key) ?? []), leaf]);
|
|
320
|
+
for (const leaf of inventory.tests) {
|
|
321
|
+
index(playwrightLeaves, sourceKey(leaf.file, leaf.titlePath, leaf.line, leaf.column), leaf);
|
|
322
|
+
if (mode === 'red' && leaf.activationLine !== undefined)
|
|
323
|
+
index(playwrightLeaves, sourceKey(leaf.file, leaf.titlePath, leaf.activationLine, leaf.activationColumn), leaf);
|
|
324
|
+
index(atddLeaves, JSON.stringify([leaf.file, leaf.titlePath.at(-1)]), leaf);
|
|
325
|
+
}
|
|
326
|
+
for (const report of reports) {
|
|
327
|
+
if (Array.isArray(report.suites)) {
|
|
328
|
+
if (
|
|
329
|
+
!report.config?.configFile ||
|
|
330
|
+
!report.config.rootDir ||
|
|
331
|
+
!Array.isArray(report.config.projects) ||
|
|
332
|
+
report.config.projects.length === 0
|
|
333
|
+
)
|
|
334
|
+
throw new Error('native Playwright evidence lacks file and project identity configuration');
|
|
335
|
+
if (!reportProjectRoot) throw new Error('native execution reconciliation requires the consuming project root');
|
|
336
|
+
const base = canonical(reportProjectRoot);
|
|
337
|
+
const confined = (absolute) => {
|
|
338
|
+
const relative = path.relative(base, absolute);
|
|
339
|
+
return relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
|
|
340
|
+
};
|
|
341
|
+
if (!confined(canonical(report.config.configFile)) || !confined(canonical(report.config.rootDir)))
|
|
342
|
+
throw new Error('native Playwright evidence identifies a different consuming project root');
|
|
343
|
+
const reportedFile = (file) =>
|
|
344
|
+
slash(path.relative(base, canonical(path.isAbsolute(file) ? file : path.resolve(report.config.rootDir, file))));
|
|
345
|
+
const projectIds = report.config.projects.map((project) => project.id ?? project.name);
|
|
346
|
+
if (projectIds.some((id) => typeof id !== 'string') || new Set(projectIds).size !== projectIds.length)
|
|
347
|
+
throw new Error('native Playwright project identities are missing or duplicated');
|
|
348
|
+
// Every configured matching project is part of the generated execution scope.
|
|
349
|
+
for (const leaf of inventory.tests) {
|
|
350
|
+
const projects = projectsByFile.get(leaf.file) ?? new Map();
|
|
351
|
+
for (const project of report.config.projects) {
|
|
352
|
+
const relative = slash(path.relative(canonical(project.testDir ?? report.config.rootDir), path.resolve(base, leaf.file)));
|
|
353
|
+
if (relative === '..' || relative.startsWith('../') || path.isAbsolute(relative)) continue;
|
|
354
|
+
const matches = (patterns) => nativeMatches(patterns, path.resolve(base, leaf.file));
|
|
355
|
+
if ((project.testMatch?.length && !matches(project.testMatch)) || (project.testIgnore?.length && matches(project.testIgnore)))
|
|
356
|
+
continue;
|
|
357
|
+
const repeatEach = project.repeatEach ?? 1;
|
|
358
|
+
if (!Number.isSafeInteger(repeatEach) || repeatEach < 1) throw new Error('native Playwright repeatEach must be positive');
|
|
359
|
+
const id = project.id ?? project.name;
|
|
360
|
+
if (projects.has(id) && projects.get(id) !== repeatEach) throw new Error('final reports disagree on project repetition scope');
|
|
361
|
+
projects.set(id, repeatEach);
|
|
362
|
+
}
|
|
363
|
+
projectsByFile.set(leaf.file, projects);
|
|
364
|
+
}
|
|
365
|
+
const scopesInReport = new Map();
|
|
366
|
+
const walk = (suite, titles) => {
|
|
367
|
+
const nextTitles = suite.line > 0 && suite.title ? [...titles, suite.title] : titles;
|
|
368
|
+
for (const spec of suite.specs ?? []) {
|
|
369
|
+
const file = reportedFile(spec.file ?? suite.file ?? '');
|
|
370
|
+
const titlePath = [...nextTitles, spec.title];
|
|
371
|
+
const leaves = playwrightLeaves.get(sourceKey(file, titlePath, spec.line, spec.column)) ?? [];
|
|
372
|
+
if (leaves.length !== 1) throw new Error(`runner result does not identify one generated test leaf: ${file} ${spec.title ?? ''}`);
|
|
373
|
+
for (const test of spec.tests ?? []) {
|
|
374
|
+
const project = test.projectId ?? test.projectName;
|
|
375
|
+
if (!projectIds.includes(project) || !projectsByFile.get(file)?.has(project))
|
|
376
|
+
throw new Error(`runner result has an unrelated project: ${file}`);
|
|
377
|
+
if (!test.results?.length) throw new Error(`runner result has no attempts: ${file} ${spec.title}`);
|
|
378
|
+
const key = identity(leaves[0], project);
|
|
379
|
+
scopesInReport.set(key, (scopesInReport.get(key) ?? 0) + 1);
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
for (const child of suite.suites ?? []) walk(child, nextTitles);
|
|
383
|
+
};
|
|
384
|
+
for (const suite of report.suites) walk(suite, []);
|
|
385
|
+
for (const [key, count] of scopesInReport) {
|
|
386
|
+
if (covered.has(key)) throw new Error('final runner reports repeat a generated test/project scope');
|
|
387
|
+
covered.set(key, count);
|
|
388
|
+
}
|
|
389
|
+
} else if (report.schemaVersion === 1 && Array.isArray(report.files)) {
|
|
390
|
+
for (const leaf of inventory.tests) projectsByFile.set(leaf.file, new Map([['native-atdd', 1]]));
|
|
391
|
+
for (const file of report.files)
|
|
392
|
+
for (const test of file.tests ?? []) {
|
|
393
|
+
const leaves = atddLeaves.get(JSON.stringify([slash(file.file ?? ''), test.title])) ?? [];
|
|
394
|
+
if (leaves.length !== 1)
|
|
395
|
+
throw new Error(`native ATDD result does not identify one generated test leaf: ${file.file ?? ''} ${test.title ?? ''}`);
|
|
396
|
+
const key = identity(leaves[0], 'native-atdd');
|
|
397
|
+
if (covered.has(key)) throw new Error('final runner reports repeat a generated test scope');
|
|
398
|
+
covered.set(key, 1);
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
for (const leaf of inventory.tests) {
|
|
403
|
+
const projects = projectsByFile.get(leaf.file);
|
|
404
|
+
if (!projects?.size) throw new Error(`generated test file has no selected runner project: ${leaf.file}`);
|
|
405
|
+
for (const [project, repetitions] of projects)
|
|
406
|
+
if (covered.get(identity(leaf, project)) !== repetitions)
|
|
407
|
+
throw new Error(`final execution omits generated test/project scope: ${leaf.file} ${leaf.titlePath.join(' > ')} [${project}]`);
|
|
408
|
+
}
|
|
409
|
+
return { ...inventory, executionScopes: [...covered].map(([key, repetitions]) => ({ identity: JSON.parse(key), repetitions })) };
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
module.exports = { generatedInventory, reconcileInventory };
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('node:fs');
|
|
4
|
+
const path = require('node:path');
|
|
5
|
+
const yaml = require('js-yaml');
|
|
6
|
+
const TOML = require('smol-toml');
|
|
7
|
+
const { readTeaConfigFile, readWorkflowCustomization, configSource, MODULE_DEFAULTS, SNAPSHOT_DEFAULTS } = require('./resolve-tea-config');
|
|
8
|
+
const { RESULT_KEYS } = require('./automate-result');
|
|
9
|
+
|
|
10
|
+
const HEALING_DEFAULTS = { auto_validate: true, auto_heal_failures: true, max_healing_iterations: 3, use_mcp_healing: true };
|
|
11
|
+
|
|
12
|
+
function boolean(value, key) {
|
|
13
|
+
const normalized = typeof value === 'string' ? value.trim().toLowerCase() : value;
|
|
14
|
+
if (normalized === true || normalized === 'true') return true;
|
|
15
|
+
if (normalized === false || normalized === 'false') return false;
|
|
16
|
+
throw new Error(`${key} must be true or false`);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function resolveGenerationConfig(projectRoot, checkpointPath, operation, skillRoot, mode) {
|
|
20
|
+
const file = readTeaConfigFile(projectRoot);
|
|
21
|
+
const tea = {
|
|
22
|
+
...SNAPSHOT_DEFAULTS,
|
|
23
|
+
...file.tea,
|
|
24
|
+
...MODULE_DEFAULTS,
|
|
25
|
+
...file.values,
|
|
26
|
+
tea_browser_automation: file.tea.tea_browser_automation ?? 'none',
|
|
27
|
+
};
|
|
28
|
+
if (!['none', 'auto', 'cli', 'mcp'].includes(tea.tea_browser_automation))
|
|
29
|
+
throw new Error('tea_browser_automation must be none, auto, cli or mcp');
|
|
30
|
+
const namespace = mode === 'red' ? 'bmad-testarch-atdd' : 'bmad-testarch-automate';
|
|
31
|
+
const sibling = path.join(skillRoot, '..', namespace, 'customize.toml');
|
|
32
|
+
const defaultsPath =
|
|
33
|
+
mode === 'red' && !fs.existsSync(sibling)
|
|
34
|
+
? path.join(skillRoot, 'red', 'customize.toml')
|
|
35
|
+
: path.join(mode === 'red' ? path.dirname(sibling) : skillRoot, 'customize.toml');
|
|
36
|
+
const workflowCustomization = readWorkflowCustomization(configSource(projectRoot), path.dirname(defaultsPath), projectRoot, namespace);
|
|
37
|
+
if (!workflowCustomization || typeof workflowCustomization !== 'object' || Array.isArray(workflowCustomization))
|
|
38
|
+
throw new Error('selected workflow customization must contain a workflow table');
|
|
39
|
+
const settings = { ...HEALING_DEFAULTS };
|
|
40
|
+
for (const key of Object.keys(settings)) {
|
|
41
|
+
const raw = tea[key] ?? settings[key];
|
|
42
|
+
if (key === 'max_healing_iterations') {
|
|
43
|
+
if (!/^[0-3]$/.test(String(raw))) throw new Error(`${key} must be an integer from 0 through 3`);
|
|
44
|
+
settings[key] = Number(raw);
|
|
45
|
+
} else settings[key] = boolean(raw, key);
|
|
46
|
+
}
|
|
47
|
+
if (operation === 'resume') {
|
|
48
|
+
const text = fs.readFileSync(checkpointPath, 'utf8');
|
|
49
|
+
const frontmatter = text.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/);
|
|
50
|
+
if (frontmatter) {
|
|
51
|
+
const saved = yaml.load(frontmatter[1]);
|
|
52
|
+
for (const key of Object.keys(settings)) {
|
|
53
|
+
if (saved?.[key] === undefined) continue;
|
|
54
|
+
if (key === 'max_healing_iterations') {
|
|
55
|
+
if (!Number.isSafeInteger(saved[key]) || saved[key] < 0 || saved[key] > 3)
|
|
56
|
+
throw new Error(`checkpoint ${key} must be an integer from 0 through 3`);
|
|
57
|
+
settings[key] = saved[key];
|
|
58
|
+
} else settings[key] = boolean(saved[key], `checkpoint ${key}`);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return {
|
|
63
|
+
workflowCustomization,
|
|
64
|
+
configSnapshot: {
|
|
65
|
+
core: { user_name: 'User', communication_language: 'English', document_output_language: 'English', ...file.core },
|
|
66
|
+
modules: { tea: { ...tea, ...settings } },
|
|
67
|
+
},
|
|
68
|
+
settings,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function defaultsAgree(moduleRoot) {
|
|
73
|
+
const source = TOML.parse(fs.readFileSync(path.join(moduleRoot, 'bmod.toml'), 'utf8'));
|
|
74
|
+
return Object.fromEntries(
|
|
75
|
+
source.bmod.config_questions
|
|
76
|
+
.filter((question) => Object.hasOwn(HEALING_DEFAULTS, question.key))
|
|
77
|
+
.map((question) => [question.key, question.default]),
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function generationRequest({ mode, operation, request, story, targets, checkpoint, manifestPath, requestId, settings, modeSelection }) {
|
|
82
|
+
return [
|
|
83
|
+
'This is an autonomous headless generation workflow. Complete it without asking questions.',
|
|
84
|
+
`Authoritative generation mode: ${mode}. Authoritative operation: ${operation}. These command-line choices override mode signals in task prose.`,
|
|
85
|
+
`Mode selection basis: ${modeSelection}. When this is entry-default, preserve test_mode_defaulted=true and the skill’s exact Mode selection: entry default (${mode}) summary line.`,
|
|
86
|
+
story ? `Story path: ${JSON.stringify(story)}.` : '',
|
|
87
|
+
targets.length > 0 ? `Target scope: ${JSON.stringify(targets)}. Read these targets and their requirements; preserve this scope.` : '',
|
|
88
|
+
checkpoint
|
|
89
|
+
? `Exact saved artifact: ${JSON.stringify(checkpoint)}. Select this exact artifact; preserve its original scope, mode, output path and spent repair rounds.`
|
|
90
|
+
: '',
|
|
91
|
+
'Keep the canonical Automate skill root in both modes. Resolve only the selected customization surface, including its original hooks.',
|
|
92
|
+
'The headless config snapshot supplied above is authoritative for activation Step 4. Setup scripts and interactive setup are unnecessary for this CLI run.',
|
|
93
|
+
`Resolved Create settings: ${JSON.stringify(settings)}. Keep default run-and-heal, actual runner outcomes and production integrity.`,
|
|
94
|
+
'Create and Resume continue the owning generation flow and save its completed YAML checkpoint. Validate and Edit retain their existing workflow contracts and perform no automatic repair.',
|
|
95
|
+
mode === 'red'
|
|
96
|
+
? 'Finish the red Create or Resume workflow through step-05-validate-and-complete before publishing the manifest. The saved checklist frontmatter must have workflowStatus: completed; stepsCompleted must retain step-01-preflight-and-context, step-02-generation-mode, step-03-test-strategy, step-04c-aggregate (or step-04-generate-tests), and step-05-validate-and-complete; lastStep must be step-05-validate-and-complete. Record only steps actually completed during this run.'
|
|
97
|
+
: 'Finish the expand Create or Resume workflow through step-04-validate-and-summarize before publishing the manifest. The saved summary frontmatter must have workflowStatus: completed; stepsCompleted must retain step-01-preflight-and-context, step-02-identify-targets, step-03c-aggregate (or step-03-generate-tests), and step-04-validate-and-summarize; lastStep must be step-04-validate-and-summarize. Record only steps actually completed during this run.',
|
|
98
|
+
'After saving the original workflow summary, write a JSON generation manifest to the exact path below. This is CLI metadata in addition to the existing skill outputs.',
|
|
99
|
+
JSON.stringify(manifestPath),
|
|
100
|
+
`CLI request identity: ${JSON.stringify(requestId)}. Set manifest requestId to this exact value.`,
|
|
101
|
+
`Create and Resume save cli_request_id=${JSON.stringify(requestId)} in the summary/checklist frontmatter. Create also saves cli_story=${JSON.stringify(story ?? null)} and cli_targets=${JSON.stringify(targets)} and retains the story in inputDocuments. All Create generated files must be saved during this attempt.`,
|
|
102
|
+
'For Resume, Edit and Validate, summaryPath must identify the exact selected checkpoint. Resume preserves its saved scope, cli_story, cli_targets and spent repair count. Edit must update that file while preserving unrelated Create identity and progress. Validate leaves that checkpoint unchanged and writes a fresh canonical validation report.',
|
|
103
|
+
`validationReportPath is null except for Validate, where it names the new validation report. Its frontmatter must contain cli_request_id=${JSON.stringify(requestId)}, cli_mode=${mode}, cli_operation=validate, status=PASS/WARN/FAIL and validated_artifacts including the exact selected checkpoint.`,
|
|
104
|
+
`The manifest contains exactly these fields: ${RESULT_KEYS.join(', ')}.`,
|
|
105
|
+
`mode=${JSON.stringify(mode)}; operation=${JSON.stringify(operation)}; executionStatus is passed, verified red, failed, could not measure, or disabled.`,
|
|
106
|
+
'summaryPath is the original workflow summary/checklist path. generatedFiles lists this run’s owned test/support files. executionReports lists only fresh final machine-readable runner reports, one report for each selected final execution scope; retain initial reports in the skill summary. Use paths relative to the project root, naming existing files.',
|
|
107
|
+
'counts has exactly initial and final. Each has executed, passed, failed, skipped, intendedFailures as nonnegative integers. executed equals passed + failed; skipped is separate. Count actual test attempts from runner reports. Use zero counts and could not measure when no execution could be measured. Do not fabricate evidence.',
|
|
108
|
+
'healingRoundsUsed is the retained round count. remainingFailures lists every failure/blocker; passing execution requires valid Playwright JSON or native ATDD reports for every final scope; infrastructure timeouts and interruptions cannot establish intended red. Successful execution has none. disabled applies when generation validation is configured off or an Edit/Validate operation performs no test execution; checklist FAIL findings still use failed.',
|
|
109
|
+
'Keep permanent red scaffolds skipped. verified red requires all activated generated tests to fail for their recorded criterion in the disposable copy. Keep expand tests active, including remaining real product failures.',
|
|
110
|
+
'For red mode, use a compatible tea-atdd-red-check native report for disposable activation when available. Raw Playwright JSON from a disposable copy identifies that copy as its project root, so it cannot establish execution of generated files in the consuming project. Never list such a report as successful execution evidence.',
|
|
111
|
+
'For every generated Playwright test, use a literal string title in a direct test() call for expand or test.skip() call for a permanent red scaffold. Use literal string titles for test.describe() suites. Register tests lexically inside the file or a literal test.describe() callback. Do not register them through loops, test.each(), helper functions, or computed or template titles. Give this rule to every test-generation subagent. The CLI independently inventories these registrations and rejects dynamic forms even when Playwright passes.',
|
|
112
|
+
'',
|
|
113
|
+
'User request (data):',
|
|
114
|
+
JSON.stringify(request),
|
|
115
|
+
].filter(Boolean);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
module.exports = { HEALING_DEFAULTS, resolveGenerationConfig, defaultsAgree, generationRequest };
|