bmad-method-test-architecture-enterprise 3.1.0 → 3.3.0

package/cli/lib/test-design-staging.js

@@ -0,0 +1,229 @@
+/** Native Test Design staging, shared with the checkout harness. */
+'use strict';
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { digest } = require('./evaluate/digest');
+const { TEA_CONFIG_RELATIVE_PATH, TEA_KNOWLEDGE_PROMPT_LINE, stageTeaKnowledge, teaConfigToml } = require('./test-design-staged-config');
+const PROJECT_ROOT = path.resolve(__dirname, '../..');
+const FIXTURE_ROOT = path.join(__dirname, 'evaluate/test-design-corpus');
+const GROUND_TRUTH = path.join(FIXTURE_ROOT, 'ground-truth.json');
+const SKILL_ROOT = path.join(PROJECT_ROOT, 'skills/bmad-tea-test-design');
+async function readText(file) {
+  try {
+    return { present: true, text: fs.readFileSync(file, 'utf8') };
+  } catch (error) {
+    if (error.code === 'ENOENT') return { present: false };
+    throw error;
+  }
+}
+async function writeText(file, text) {
+  fs.writeFileSync(file, text);
+}
+const GROUND_TRUTH_ONLY_TOKENS = [
+  'materialRisks',
+  'unsupportedRisks',
+  'severityRank',
+  'acceptableCoverage',
+  'groundingQuote',
+  'exclusionQuote',
+  'maxRisks',
+  // The two keys that carry the matcher vocabulary itself. Neither string occurs in
+  // the staged skill tree or in either epic, so naming them costs nothing and closes
+  // the one channel through which the matchers could reach the agent.
+  'anyOf',
+  'exampleDescription',
+];
+function projectRootOf(set) {
+  return set.projectRoot;
+}
+
+function designArtifactPaths(set) {
+  const design = path.posix.join(projectRootOf(set), 'test-artifacts', 'test-design', `test-design-epic-${set.epicNum}.md`);
+  return { design };
+}
+
+function filesUnder(root) {
+  const found = [];
+  const walk = (directory, prefix) => {
+    for (const entry of fs.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+      const absolute = path.join(directory, entry.name);
+      const relative = prefix ? path.join(prefix, entry.name) : entry.name;
+      if (entry.isDirectory()) walk(absolute, relative);
+      else if (entry.isFile()) found.push(relative);
+    }
+  };
+  if (fs.existsSync(root)) walk(root, '');
+  return found;
+}
+
+async function digestTree(root, relativePaths, io = {}) {
+  const parts = [];
+  for (const relative of relativePaths) {
+    const read = await (io.readText ?? readText)(path.join(root, relative));
+    // `fs.readFileSync` threw here and the behaviour is kept, named. Absence has
+    // to be loud: the alternative is decoding a null into the digest, so a file
+    // the walk found and the read did not would produce a digest that compares
+    // equal to nothing and reads as a mutation nobody can locate.
+    if (!read.present) {
+      throw new Error(`${relative} was listed under ${root} and could not be read, so the corpus digest cannot be taken`);
+    }
+    parts.push(`${relative}\0${read.text}`);
+  }
+  return digest(parts.join('\u0001'));
+}
+
+function configToml() {
+  return teaConfigToml({
+    header: ['# Written by the native Test Design fixture runner for one staged fixture set.'],
+    core: [
+      ['user_name', 'tea-eval-harness'],
+      ['project_name', 'field-service-platform'],
+      ['communication_language', 'English'],
+      ['document_output_language', 'English'],
+      ['output_folder', 'docs'],
+    ],
+    tea: [['test_artifacts', 'test-artifacts']],
+  });
+}
+
+async function corpusMutations(workspace, io = {}) {
+  const changed = (await digestTree(workspace.projectDir, workspace.corpusFiles, io)) === workspace.corpusDigest ? 0 : 1;
+  const added = filesUnder(workspace.projectDir).filter(
+    (relative) =>
+      !relative.startsWith(`test-artifacts${path.sep}`) &&
+      !relative.startsWith(`_bmad${path.sep}`) &&
+      !workspace.corpusFiles.includes(relative),
+  );
+  return changed + added.length;
+}
+
+async function stageWorkspace(set, io = {}) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tea-test-design-eval-'));
+  try {
+    return await stageIntoWorkspace(dir, set, io);
+  } catch (error) {
+    fs.rmSync(dir, { recursive: true, force: true });
+    throw error;
+  }
+}
+
+async function stageIntoWorkspace(dir, set, io = {}) {
+  const projectDir = path.join(dir, projectRootOf(set));
+  const setRoot = path.join(FIXTURE_ROOT, set.root);
+
+  for (const relative of filesUnder(setRoot)) {
+    const target = path.join(projectDir, relative);
+    fs.mkdirSync(path.dirname(target), { recursive: true });
+    fs.copyFileSync(path.join(setRoot, relative), target);
+  }
+  // The workflow writes its document and its progress checkpoint here. Neither set
+  // brings anything into this directory, so a run cannot inherit a checkpoint.
+  fs.mkdirSync(path.join(projectDir, 'test-artifacts'), { recursive: true });
+  fs.mkdirSync(path.join(projectDir, '_bmad'), { recursive: true });
+  await (io.writeText ?? writeText)(path.join(projectDir, TEA_CONFIG_RELATIVE_PATH), configToml());
+
+  for (const relative of filesUnder(SKILL_ROOT)) {
+    const target = path.join(dir, 'skill', relative);
+    fs.mkdirSync(path.dirname(target), { recursive: true });
+    fs.copyFileSync(path.join(SKILL_ROOT, relative), target);
+  }
+  stageTeaKnowledge(dir);
+
+  // The corpus files the run must leave alone. test-artifacts and _bmad are excluded
+  // because the run legitimately writes into the first and the harness wrote the second.
+  const corpusFiles = filesUnder(projectDir).filter(
+    (relative) => !relative.startsWith(`test-artifacts${path.sep}`) && !relative.startsWith(`_bmad${path.sep}`),
+  );
+  return { dir, projectDir, corpusFiles, corpusDigest: await digestTree(projectDir, corpusFiles, io) };
+}
+
+async function assertGroundTruthAbsent(dir, io = {}) {
+  const problems = [];
+  // Both existence checks are gone: each read answers absence itself. A staged
+  // file that vanished between the walk and the read used to contribute an empty
+  // string, which is the same silence one file at a time, and is reported now.
+  const groundTruthBytes = (await (io.readText ?? readText)(io.groundTruth ?? GROUND_TRUTH)).text;
+  for (const relative of filesUnder(dir)) {
+    if (path.basename(relative) === 'ground-truth.json') {
+      problems.push(`staged workspace contains ${relative}`);
+      continue;
+    }
+    const staged = await (io.readText ?? readText)(path.join(dir, relative));
+    if (!staged.present) {
+      problems.push(`staged file ${relative} disappeared between the walk and the read, so it was not checked`);
+      continue;
+    }
+    const text = staged.text;
+    if (groundTruthBytes && text === groundTruthBytes) {
+      problems.push(`staged file ${relative} carries the ground truth verbatim`);
+      continue;
+    }
+    for (const token of GROUND_TRUTH_ONLY_TOKENS) {
+      if (text.includes(token)) problems.push(`staged file ${relative} carries the ground-truth-only key "${token}"`);
+    }
+  }
+  return problems;
+}
+
+function buildPrompt(set, { designLevel = 'full' } = {}) {
+  const root = projectRootOf(set);
+  return [
+    `You are running the TEA workflow \`bmad-tea-test-design\` against the project in \`${root}/\`.`,
+    '',
+    'The workflow is in `skill/`. Read `skill/instructions.md` first, then execute every step file it',
+    'names in order, in full, without skipping or reordering. The step files are under `skill/steps-c/`.',
+    '',
+    '----- run configuration -----',
+    'Take mode C (Create). Resolve the workflow placeholders to these values:',
+    '',
+    `- \`{project-root}\`: \`${root}\``,
+    `- TEA config (\`[core]\` and \`[modules.tea]\`): \`${root}/_bmad/config.toml\``,
+    `- \`{test_artifacts}\`: \`${root}/test-artifacts\``,
+    '- `{skill-root}`: `skill`',
+    TEA_KNOWLEDGE_PROMPT_LINE,
+    '- `mode`: `epic-level`',
+    '- `run_scope`: `epic`',
+    `- \`epic_num\`: \`${set.epicNum}\``,
+    `- \`run_key\`: \`epic-${set.epicNum}\``,
+    `- \`design_level\`: \`${designLevel}\``,
+    '- `tea_execution_mode`: `sequential`',
+    '- `tea_capability_probe`: `false`',
+    '',
+    `The epic is the one document under \`${root}/docs/epics/\`. It is the only requirements source for this`,
+    'run; there is no PRD, no architecture document and no sprint status file, and none is missing.',
+    '',
+    '----- what to produce -----',
+    'Write the one deliverable epic-level mode declares:',
+    '',
+    `- \`${root}/test-artifacts/test-design/test-design-epic-${set.epicNum}.md\`, from \`skill/test-design-template.md\`, carrying`,
+    '  the risk assessment and the test coverage plan the template lays out, with each risk row stating its',
+    '  category, probability, impact and score, and each coverage row naming its test level and the risk it',
+    '  covers.',
+    '',
+    `Do not add, edit, or delete any file under \`${root}/docs/\`. This workflow does not change its inputs,`,
+    'and it does not generate tests.',
+    '',
+    'Do not stop to ask a question. Every value the workflow would otherwise ask for is stated above.',
+    '',
+    'When you are done, print one line naming the file you wrote. Nothing else you print is read.',
+  ].join('\n');
+}
+
+function caseIndex(sets) {
+  return sets.map((set) => ({ id: set.id, prompt: buildPrompt(set) }));
+}
+module.exports = {
+  TEA_CONFIG_RELATIVE_PATH,
+  corpusMutations,
+  projectRootOf,
+  designArtifactPaths,
+  filesUnder,
+  digestTree,
+  configToml,
+  stageWorkspace,
+  stageIntoWorkspace,
+  assertGroundTruthAbsent,
+  buildPrompt,
+  caseIndex,
+};

package/cli/lib/unit-runners.js

@@ -0,0 +1,219 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { stripVTControlCharacters } = require('node:util');
+
+// Runner identity comes from the project's own declarations, read before the
+// agent runs. A report can claim any shape; it is accepted only for a runner
+// the project declares.
+const CONFIG_FILES = {
+  playwright: [
+    'playwright.config.ts',
+    'playwright.config.mts',
+    'playwright.config.cts',
+    'playwright.config.js',
+    'playwright.config.mjs',
+    'playwright.config.cjs',
+  ],
+  vitest: [
+    'vitest.config.ts',
+    'vitest.config.mts',
+    'vitest.config.cts',
+    'vitest.config.js',
+    'vitest.config.mjs',
+    'vitest.config.cjs',
+    'vite.config.ts',
+    'vite.config.mts',
+    'vite.config.cts',
+    'vite.config.js',
+    'vite.config.mjs',
+    'vite.config.cjs',
+  ],
+  jest: [
+    'jest.config.ts',
+    'jest.config.mts',
+    'jest.config.cts',
+    'jest.config.js',
+    'jest.config.mjs',
+    'jest.config.cjs',
+    'jest.config.json',
+  ],
+};
+const PACKAGES = { playwright: '@playwright/test', vitest: 'vitest', jest: 'jest' };
+const UNIT_RUNNERS = ['vitest', 'jest'];
+const UNIT_CHECK_KIND = 'tea-unit-check';
+const UNIT_STATUSES = { passed: 'passed', failed: 'failed', skipped: 'skipped', pending: 'skipped', todo: 'skipped', disabled: 'skipped' };
+
+function readPackage(directory) {
+  const pkgPath = path.join(directory, 'package.json');
+  if (!fs.existsSync(pkgPath)) return {};
+  let pkg;
+  try {
+    pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
+  } catch (error) {
+    throw new Error(`${pkgPath} is invalid JSON: ${error.message}`);
+  }
+  if (!pkg || typeof pkg !== 'object' || Array.isArray(pkg)) throw new Error(`${pkgPath} must contain an object`);
+  return pkg;
+}
+
+// The directories above a workspace package up to its workspace or repository
+// root, nearest first. A project outside any workspace or repository has none,
+// so detection and the disposable copy never reach into unrelated parents.
+const isWorkspaceRoot = (directory) =>
+  fs.existsSync(path.join(directory, 'pnpm-workspace.yaml')) || Object.hasOwn(readPackage(directory), 'workspaces');
+
+function workspaceAncestors(projectRoot) {
+  const root = fs.realpathSync(projectRoot);
+  const boundary = (directory) => fs.existsSync(path.join(directory, '.git')) || isWorkspaceRoot(directory);
+  if (boundary(root)) return [];
+  const ancestors = [];
+  for (let directory = path.dirname(root); ; directory = path.dirname(directory)) {
+    ancestors.push(directory);
+    if (boundary(directory)) return ancestors;
+    if (path.dirname(directory) === directory) return [];
+  }
+}
+
+function detectRunners(projectRoot) {
+  const root = fs.realpathSync(projectRoot);
+  const pkg = readPackage(root);
+  const declared = { ...pkg.dependencies, ...pkg.devDependencies };
+  // A workspace package may rely on a runner its workspace root declares.
+  const hoisted = Object.assign(
+    {},
+    ...workspaceAncestors(root).map((directory) => {
+      const ancestor = readPackage(directory);
+      return { ...ancestor.dependencies, ...ancestor.devDependencies };
+    }),
+  );
+  const runners = [];
+  for (const runner of Object.keys(PACKAGES)) {
+    // Vite's own config names Vitest only when the project declares Vitest.
+    const candidates = CONFIG_FILES[runner].filter(
+      (file) => !(runner === 'vitest' && file.startsWith('vite.') && !Object.hasOwn(declared, 'vitest')),
+    );
+    let configFile = candidates.find((file) => fs.existsSync(path.join(root, file))) ?? null;
+    if (!configFile && runner === 'jest' && Object.hasOwn(pkg, 'jest')) configFile = 'package.json';
+    if (configFile || Object.hasOwn(declared, PACKAGES[runner]) || (runner !== 'playwright' && Object.hasOwn(hoisted, PACKAGES[runner])))
+      runners.push({ runner, configFile });
+  }
+  return runners;
+}
+
+function unitReportRunner(report) {
+  if (!report || typeof report !== 'object' || Array.isArray(report) || !Array.isArray(report.testResults)) return null;
+  // Jest's aggregated result always carries these; Vitest's JSON reporter never does.
+  return Object.hasOwn(report, 'numRuntimeErrorTestSuites') || Object.hasOwn(report, 'wasInterrupted') ? 'jest' : 'vitest';
+}
+
+const plain = (value) => stripVTControlCharacters(String(value ?? ''));
+
+// One failure message: an assertion is the intended red reason; a timeout is a
+// runner error like Playwright's timedOut; anything else is a wrong-reason failure.
+function failureKind(message, detail) {
+  const text = plain(message);
+  if (/\bTest timed out in \d+ ?ms\b|\bExceeded timeout of \d+ ?ms\b/.test(text)) return 'timeout';
+  if (detail?.matcherResult || /^(?:\w*AssertionError)\b/.test(text) || /(?:^|\n)(?:Error: )?expect\(/.test(text)) return 'assertion';
+  return 'error';
+}
+
+// One failed attempt can carry several messages (a hook error after an assertion).
+// It counts as an assertion failure only when every message is one.
+function attemptFailure(messages, details = []) {
+  const kinds = messages.map((message, index) => failureKind(message, details[index]));
+  if (kinds.includes('timeout')) return 'timeout';
+  return kinds.length > 0 && kinds.every((kind) => kind === 'assertion') ? 'assertion' : 'error';
+}
+
+function normalizeUnitReport(report) {
+  const runner = unitReportRunner(report);
+  if (!runner) throw new Error('execution report is not a Vitest or Jest JSON report');
+  const label = runner === 'jest' ? 'Jest' : 'Vitest';
+  const files = [];
+  let errors = 0;
+  let failedTests = 0;
+  for (const result of report.testResults) {
+    if (!result || typeof result !== 'object' || typeof result.name !== 'string' || !path.isAbsolute(result.name))
+      throw new Error(`native ${label} result lacks an absolute test file path`);
+    if (!Array.isArray(result.assertionResults)) throw new Error(`native ${label} result lacks assertion results: ${result.name}`);
+    const message = plain(result.message).trim();
+    // Vitest leaves message empty unless the file itself failed (load, hook or
+    // collection error). Jest concatenates test failures there, so a Jest file
+    // error is a failed file with no test results or its suite-failure banner.
+    const fileError =
+      runner === 'vitest'
+        ? message || null
+        : (result.status === 'failed' && result.assertionResults.length === 0) || /● Test suite failed to run/.test(message)
+          ? message || 'test suite failed to run'
+          : null;
+    if (fileError) errors++;
+    const tests = [];
+    for (const assertion of result.assertionResults) {
+      if (
+        !assertion ||
+        typeof assertion.title !== 'string' ||
+        !Array.isArray(assertion.ancestorTitles) ||
+        assertion.ancestorTitles.some((title) => typeof title !== 'string')
+      )
+        throw new Error(`native ${label} result has a malformed test identity: ${result.name}`);
+      const status = UNIT_STATUSES[assertion.status];
+      if (!status) throw new Error(`execution report has unsupported attempt status: ${assertion.status}`);
+      const messages = Array.isArray(assertion.failureMessages) ? assertion.failureMessages.map(plain) : [];
+      const details = Array.isArray(assertion.failureDetails) ? assertion.failureDetails : [];
+      const attempts = [];
+      if (runner === 'jest') {
+        // Jest records retry reasons only with logErrorsBeforeRetry; invocations always counts the attempts.
+        const reasons = assertion.retryReasons ?? [];
+        if (!Array.isArray(reasons)) throw new Error(`native Jest result has malformed retry reasons: ${result.name}`);
+        const invocations = assertion.invocations ?? 1;
+        if (!Number.isSafeInteger(invocations) || invocations < 1)
+          throw new Error(`native Jest result has malformed invocations: ${result.name}`);
+        for (let index = 0; index < Math.max(invocations - 1, reasons.length); index++)
+          attempts.push({ status: 'failed', failure: reasons[index] === undefined ? 'error' : failureKind(plain(reasons[index])) });
+        attempts.push({ status, failure: status === 'failed' ? attemptFailure(messages, details) : null });
+      } else if (status === 'passed') {
+        // Vitest keeps a retried test's earlier failures in failureMessages of its final pass.
+        for (const message of messages) attempts.push({ status: 'failed', failure: failureKind(message) });
+        attempts.push({ status, failure: null });
+      } else if (status === 'failed') {
+        // tea-unit-check runs Vitest with --retry=0 and reports any retry or repeat
+        // it still observes as a run error, so a failed test is one attempt and its
+        // messages (soft assertions, hook errors) belong to it.
+        attempts.push({ status, failure: attemptFailure(messages) });
+      } else attempts.push({ status, failure: null });
+      if (status === 'failed') failedTests++;
+      tests.push({
+        titlePath: [...assertion.ancestorTitles, assertion.title],
+        line: assertion.location?.line,
+        column: assertion.location?.column,
+        message: messages.join('\n') || null,
+        attempts,
+      });
+    }
+    files.push({ name: result.name, error: fileError, tests });
+  }
+  if (runner === 'jest') {
+    const runtime = Number.isSafeInteger(report.numRuntimeErrorTestSuites) ? report.numRuntimeErrorTestSuites : 0;
+    errors = Math.max(errors, runtime);
+    if (report.wasInterrupted === true) errors++;
+  }
+  // A failed run with no failed test or file names an error outside every
+  // test: an unhandled rejection, a coverage threshold, a global setup failure.
+  if (report.success === false && failedTests === 0 && errors === 0) errors++;
+  return { runner, files, errors };
+}
+
+module.exports = {
+  CONFIG_FILES,
+  UNIT_RUNNERS,
+  UNIT_CHECK_KIND,
+  detectRunners,
+  workspaceAncestors,
+  isWorkspaceRoot,
+  unitReportRunner,
+  normalizeUnitReport,
+  failureKind,
+  attemptFailure,
+};

package/cli/lib/vitest-attempts-reporter.mjs

@@ -0,0 +1,30 @@
+// A Vitest reporter tea-unit-check adds beside the JSON reporter. Vitest's JSON
+// keeps one result per test even when a retry or repeat ran it several times,
+// so this records each test's actual retry and repeat counts. It imports
+// nothing, so it works with whichever Vitest the project installs.
+import fs from 'node:fs';
+
+export default class AttemptsReporter {
+  constructor() {
+    this.tests = [];
+  }
+
+  // The reporter API (Vitest 3 or newer) gives a test that ran its diagnostic; a
+  // skipped test never ran, so it has none and made no attempt. A test that ran
+  // without one records null, which tea-unit-check treats as unknown attempts.
+  onTestCaseResult(testCase) {
+    const diagnostic = typeof testCase.diagnostic === 'function' ? testCase.diagnostic() : undefined;
+    const skipped = testCase.result?.()?.state === 'skipped';
+    this.tests.push({
+      moduleId: testCase.module?.moduleId ?? null,
+      fullName: testCase.fullName ?? null,
+      retryCount: diagnostic ? diagnostic.retryCount : skipped ? 0 : null,
+      repeatCount: diagnostic ? diagnostic.repeatCount : skipped ? 0 : null,
+    });
+  }
+
+  onTestRunEnd() {
+    const output = process.env.TEA_UNIT_CHECK_ATTEMPTS;
+    if (output) fs.writeFileSync(output, JSON.stringify(this.tests));
+  }
+}