bmad-method-test-architecture-enterprise 3.2.0 → 3.3.0

package/cli/lib/test-design-staging.js

@@ -0,0 +1,229 @@
+/** Native Test Design staging, shared with the checkout harness. */
+'use strict';
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { digest } = require('./evaluate/digest');
+const { TEA_CONFIG_RELATIVE_PATH, TEA_KNOWLEDGE_PROMPT_LINE, stageTeaKnowledge, teaConfigToml } = require('./test-design-staged-config');
+const PROJECT_ROOT = path.resolve(__dirname, '../..');
+const FIXTURE_ROOT = path.join(__dirname, 'evaluate/test-design-corpus');
+const GROUND_TRUTH = path.join(FIXTURE_ROOT, 'ground-truth.json');
+const SKILL_ROOT = path.join(PROJECT_ROOT, 'skills/bmad-tea-test-design');
+async function readText(file) {
+  try {
+    return { present: true, text: fs.readFileSync(file, 'utf8') };
+  } catch (error) {
+    if (error.code === 'ENOENT') return { present: false };
+    throw error;
+  }
+}
+async function writeText(file, text) {
+  fs.writeFileSync(file, text);
+}
+const GROUND_TRUTH_ONLY_TOKENS = [
+  'materialRisks',
+  'unsupportedRisks',
+  'severityRank',
+  'acceptableCoverage',
+  'groundingQuote',
+  'exclusionQuote',
+  'maxRisks',
+  // The two keys that carry the matcher vocabulary itself. Neither string occurs in
+  // the staged skill tree or in either epic, so naming them costs nothing and closes
+  // the one channel through which the matchers could reach the agent.
+  'anyOf',
+  'exampleDescription',
+];
+function projectRootOf(set) {
+  return set.projectRoot;
+}
+
+function designArtifactPaths(set) {
+  const design = path.posix.join(projectRootOf(set), 'test-artifacts', 'test-design', `test-design-epic-${set.epicNum}.md`);
+  return { design };
+}
+
+function filesUnder(root) {
+  const found = [];
+  const walk = (directory, prefix) => {
+    for (const entry of fs.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+      const absolute = path.join(directory, entry.name);
+      const relative = prefix ? path.join(prefix, entry.name) : entry.name;
+      if (entry.isDirectory()) walk(absolute, relative);
+      else if (entry.isFile()) found.push(relative);
+    }
+  };
+  if (fs.existsSync(root)) walk(root, '');
+  return found;
+}
+
+async function digestTree(root, relativePaths, io = {}) {
+  const parts = [];
+  for (const relative of relativePaths) {
+    const read = await (io.readText ?? readText)(path.join(root, relative));
+    // `fs.readFileSync` threw here and the behaviour is kept, named. Absence has
+    // to be loud: the alternative is decoding a null into the digest, so a file
+    // the walk found and the read did not would produce a digest that compares
+    // equal to nothing and reads as a mutation nobody can locate.
+    if (!read.present) {
+      throw new Error(`${relative} was listed under ${root} and could not be read, so the corpus digest cannot be taken`);
+    }
+    parts.push(`${relative}\0${read.text}`);
+  }
+  return digest(parts.join('\u0001'));
+}
+
+function configToml() {
+  return teaConfigToml({
+    header: ['# Written by the native Test Design fixture runner for one staged fixture set.'],
+    core: [
+      ['user_name', 'tea-eval-harness'],
+      ['project_name', 'field-service-platform'],
+      ['communication_language', 'English'],
+      ['document_output_language', 'English'],
+      ['output_folder', 'docs'],
+    ],
+    tea: [['test_artifacts', 'test-artifacts']],
+  });
+}
+
+async function corpusMutations(workspace, io = {}) {
+  const changed = (await digestTree(workspace.projectDir, workspace.corpusFiles, io)) === workspace.corpusDigest ? 0 : 1;
+  const added = filesUnder(workspace.projectDir).filter(
+    (relative) =>
+      !relative.startsWith(`test-artifacts${path.sep}`) &&
+      !relative.startsWith(`_bmad${path.sep}`) &&
+      !workspace.corpusFiles.includes(relative),
+  );
+  return changed + added.length;
+}
+
+async function stageWorkspace(set, io = {}) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tea-test-design-eval-'));
+  try {
+    return await stageIntoWorkspace(dir, set, io);
+  } catch (error) {
+    fs.rmSync(dir, { recursive: true, force: true });
+    throw error;
+  }
+}
+
+async function stageIntoWorkspace(dir, set, io = {}) {
+  const projectDir = path.join(dir, projectRootOf(set));
+  const setRoot = path.join(FIXTURE_ROOT, set.root);
+
+  for (const relative of filesUnder(setRoot)) {
+    const target = path.join(projectDir, relative);
+    fs.mkdirSync(path.dirname(target), { recursive: true });
+    fs.copyFileSync(path.join(setRoot, relative), target);
+  }
+  // The workflow writes its document and its progress checkpoint here. Neither set
+  // brings anything into this directory, so a run cannot inherit a checkpoint.
+  fs.mkdirSync(path.join(projectDir, 'test-artifacts'), { recursive: true });
+  fs.mkdirSync(path.join(projectDir, '_bmad'), { recursive: true });
+  await (io.writeText ?? writeText)(path.join(projectDir, TEA_CONFIG_RELATIVE_PATH), configToml());
+
+  for (const relative of filesUnder(SKILL_ROOT)) {
+    const target = path.join(dir, 'skill', relative);
+    fs.mkdirSync(path.dirname(target), { recursive: true });
+    fs.copyFileSync(path.join(SKILL_ROOT, relative), target);
+  }
+  stageTeaKnowledge(dir);
+
+  // The corpus files the run must leave alone. test-artifacts and _bmad are excluded
+  // because the run legitimately writes into the first and the harness wrote the second.
+  const corpusFiles = filesUnder(projectDir).filter(
+    (relative) => !relative.startsWith(`test-artifacts${path.sep}`) && !relative.startsWith(`_bmad${path.sep}`),
+  );
+  return { dir, projectDir, corpusFiles, corpusDigest: await digestTree(projectDir, corpusFiles, io) };
+}
+
+async function assertGroundTruthAbsent(dir, io = {}) {
+  const problems = [];
+  // Both existence checks are gone: each read answers absence itself. A staged
+  // file that vanished between the walk and the read used to contribute an empty
+  // string, which is the same silence one file at a time, and is reported now.
+  const groundTruthBytes = (await (io.readText ?? readText)(io.groundTruth ?? GROUND_TRUTH)).text;
+  for (const relative of filesUnder(dir)) {
+    if (path.basename(relative) === 'ground-truth.json') {
+      problems.push(`staged workspace contains ${relative}`);
+      continue;
+    }
+    const staged = await (io.readText ?? readText)(path.join(dir, relative));
+    if (!staged.present) {
+      problems.push(`staged file ${relative} disappeared between the walk and the read, so it was not checked`);
+      continue;
+    }
+    const text = staged.text;
+    if (groundTruthBytes && text === groundTruthBytes) {
+      problems.push(`staged file ${relative} carries the ground truth verbatim`);
+      continue;
+    }
+    for (const token of GROUND_TRUTH_ONLY_TOKENS) {
+      if (text.includes(token)) problems.push(`staged file ${relative} carries the ground-truth-only key "${token}"`);
+    }
+  }
+  return problems;
+}
+
+function buildPrompt(set, { designLevel = 'full' } = {}) {
+  const root = projectRootOf(set);
+  return [
+    `You are running the TEA workflow \`bmad-tea-test-design\` against the project in \`${root}/\`.`,
+    '',
+    'The workflow is in `skill/`. Read `skill/instructions.md` first, then execute every step file it',
+    'names in order, in full, without skipping or reordering. The step files are under `skill/steps-c/`.',
+    '',
+    '----- run configuration -----',
+    'Take mode C (Create). Resolve the workflow placeholders to these values:',
+    '',
+    `- \`{project-root}\`: \`${root}\``,
+    `- TEA config (\`[core]\` and \`[modules.tea]\`): \`${root}/_bmad/config.toml\``,
+    `- \`{test_artifacts}\`: \`${root}/test-artifacts\``,
+    '- `{skill-root}`: `skill`',
+    TEA_KNOWLEDGE_PROMPT_LINE,
+    '- `mode`: `epic-level`',
+    '- `run_scope`: `epic`',
+    `- \`epic_num\`: \`${set.epicNum}\``,
+    `- \`run_key\`: \`epic-${set.epicNum}\``,
+    `- \`design_level\`: \`${designLevel}\``,
+    '- `tea_execution_mode`: `sequential`',
+    '- `tea_capability_probe`: `false`',
+    '',
+    `The epic is the one document under \`${root}/docs/epics/\`. It is the only requirements source for this`,
+    'run; there is no PRD, no architecture document and no sprint status file, and none is missing.',
+    '',
+    '----- what to produce -----',
+    'Write the one deliverable epic-level mode declares:',
+    '',
+    `- \`${root}/test-artifacts/test-design/test-design-epic-${set.epicNum}.md\`, from \`skill/test-design-template.md\`, carrying`,
+    '  the risk assessment and the test coverage plan the template lays out, with each risk row stating its',
+    '  category, probability, impact and score, and each coverage row naming its test level and the risk it',
+    '  covers.',
+    '',
+    `Do not add, edit, or delete any file under \`${root}/docs/\`. This workflow does not change its inputs,`,
+    'and it does not generate tests.',
+    '',
+    'Do not stop to ask a question. Every value the workflow would otherwise ask for is stated above.',
+    '',
+    'When you are done, print one line naming the file you wrote. Nothing else you print is read.',
+  ].join('\n');
+}
+
+function caseIndex(sets) {
+  return sets.map((set) => ({ id: set.id, prompt: buildPrompt(set) }));
+}
+module.exports = {
+  TEA_CONFIG_RELATIVE_PATH,
+  corpusMutations,
+  projectRootOf,
+  designArtifactPaths,
+  filesUnder,
+  digestTree,
+  configToml,
+  stageWorkspace,
+  stageIntoWorkspace,
+  assertGroundTruthAbsent,
+  buildPrompt,
+  caseIndex,
+};

package/cli/test-design-runner.js

@@ -52,9 +52,10 @@
 
 const fs = require('node:fs');
 const path = require('node:path');
+const { createHash, randomUUID } = require('node:crypto');
 const { Command } = require('commander');
 
-const { AGENT_ADAPTERS } = require('./lib/agent-adapters');
+const { AGENT_ADAPTERS, resolveModel, agentCaptureTelemetry } = require('./lib/agent-adapters');
 const { runAgent } = require('./lib/run-agent');
 const { EXIT_CODES, classOfAgentError, failureClassForExit, vendorEnvironmentNames } = require('./lib/runner-exit-codes');
 const { readDesign, scoredRiskProjection } = require('./lib/test-design-parser');
@@ -81,7 +82,8 @@ const TEST_DESIGN_REQUEST_KEYS = {
   argument: { required: [], permitted: [] },
   option: {
     required: ['agent', 'design-path'],
-    permitted: ['agent', 'agent-cmd', 'agent-arg', 'design-path', 'env-pass', 'model', 'timeout-ms'],
+    permitted: ['agent', 'agent-cmd', 'agent-arg', 'design-path', 'env-pass', 'model', 'timeout-ms', 'capture-path', 'capture-test-double'],
+    types: { 'capture-test-double': 'boolean' },
   },
   environment: { required: [], permitted: vendorEnvironmentNames() },
   stdin: { required: ['prompt'], permitted: ['prompt'] },
@@ -107,7 +109,7 @@ function readPrompt() {
 }
 
 /** Derive a projection only after the agent has finished its document. */
-function projectScoredRisks(cwd, relativeDesignPath) {
+function designText(cwd, relativeDesignPath) {
   const relative = path.normalize(relativeDesignPath);
   if (path.isAbsolute(relative) || relative.startsWith(`..${path.sep}`) || relative === '..' || !relative.endsWith('.md')) {
     throw new Error(`--design-path must name a markdown file inside the working directory: ${JSON.stringify(relativeDesignPath)}`);
@@ -120,8 +122,13 @@ function projectScoredRisks(cwd, relativeDesignPath) {
   if (fromRoot === '..' || fromRoot.startsWith(`..${path.sep}`) || path.isAbsolute(fromRoot) || fs.lstatSync(designPath).isSymbolicLink()) {
     throw new Error(`design path escapes the working directory: ${JSON.stringify(relativeDesignPath)}`);
   }
-  const text = fs.readFileSync(designPath, 'utf8');
+  return fs.readFileSync(designPath, 'utf8');
+}
+function projectScoredRisks(cwd, relativeDesignPath, { requireDeliverable = false } = {}) {
+  const text = designText(cwd, relativeDesignPath);
+  if (text === null) return null;
   const read = readDesign({ kind: 'text', value: text });
+  if (!read.ok && requireDeliverable) throw Object.assign(new Error(read.reason), { failureClass: read.failureClass });
   return scoredRiskProjection(read.ok ? read.design : { risks: [], text });
 }
 
@@ -136,6 +143,8 @@ function main(argv) {
     .option('--env-pass <NAME>', 'environment variable name allowed through to the agent (repeatable)', collect, [])
     .option('--model <name>', 'model to pin for this run; defaults to the adapter default')
     .option('--design-path <path>', 'staged design document path for a deterministic scored-risk projection')
+    .option('--capture-path <path>', 'opt-in source receipt and telemetry inside the workspace')
+    .option('--capture-test-double', 'label custom telemetry as a deterministic test double')
     .option('--timeout-ms <n>', 'agent wall-clock timeout in milliseconds', String(DEFAULT_TIMEOUT_MS));
 
   program.exitOverride();
@@ -151,6 +160,9 @@ function main(argv) {
   if (!Object.hasOwn(AGENT_ADAPTERS, options.agent)) {
     fail('environment-configuration', `unknown agent "${options.agent}"; expected one of ${Object.keys(AGENT_ADAPTERS).join(', ')}`);
   }
+  if (options.captureTestDouble && options.agent !== 'custom') {
+    fail('usage', '--capture-test-double requires --agent custom');
+  }
   // Tested on the raw string before the parse. Number.parseInt truncates, so
   // `--timeout-ms 20abc` became 20 and every run then timed out at twenty
   // milliseconds, and `--timeout-ms 1e9` became 1.
@@ -168,29 +180,251 @@ function main(argv) {
   }
 
   let stdout = '';
+  let source = null;
+  let captureFile = null;
+  let captureFd = null;
+  let captureIdentity = null;
+  let captureBytes = null;
+  let captureResult = null;
+  const reportCapture = () => {
+    if (captureResult) process.stdout.write(`${JSON.stringify({ sourceCapture: captureResult })}\n`);
+  };
+  const saveCapture = () => {
+    if (captureFd === null) return;
+    let integrityFailure = null;
+    try {
+      const held = fs.fstatSync(captureFd);
+      const named = fs.lstatSync(captureFile);
+      if (
+        !held.isFile() ||
+        held.nlink !== 1 ||
+        !named.isFile() ||
+        named.nlink !== 1 ||
+        held.dev !== captureIdentity.dev ||
+        held.ino !== captureIdentity.ino ||
+        named.dev !== held.dev ||
+        named.ino !== held.ino
+      )
+        integrityFailure = 'source capture path or link identity changed';
+      if (!integrityFailure && captureBytes !== null) {
+        if (held.size === captureBytes.length) {
+          const current = Buffer.alloc(held.size);
+          fs.readSync(captureFd, current, 0, current.length, 0);
+          if (!current.equals(captureBytes)) integrityFailure = 'source capture bytes changed';
+        } else integrityFailure = 'source capture bytes changed';
+      }
+    } catch (error) {
+      integrityFailure = `source capture identity could not be checked: ${error.message}`;
+    }
+    if (integrityFailure) {
+      const recovery = path.join(process.cwd(), `source-capture-recovery-${randomUUID()}.json`);
+      source.captureIntegrityViolation = { reason: integrityFailure, originalPath: captureFile, agentStatus: source.status };
+      source.status = 'failed';
+      const recoveryBytes = Buffer.from(`${JSON.stringify(source, null, 2)}\n`);
+      let recoveryFd = null;
+      let recoveryFailure = null;
+      try {
+        recoveryFd = fs.openSync(recovery, 'wx', 0o600);
+        fs.writeFileSync(recoveryFd, recoveryBytes);
+        fs.fsyncSync(recoveryFd);
+      } catch (error) {
+        recoveryFailure = error;
+      } finally {
+        for (const fd of [recoveryFd, captureFd]) {
+          if (fd === null) continue;
+          try {
+            fs.closeSync(fd);
+          } catch (error) {
+            recoveryFailure ??= error;
+          }
+        }
+        captureFd = null;
+      }
+      if (recoveryFailure) {
+        captureResult = {
+          status: 'unverified',
+          reason: integrityFailure,
+          recoveryError: recoveryFailure.message,
+          path: path.basename(recovery),
+        };
+        reportCapture();
+        fail(
+          'environment-configuration',
+          `${integrityFailure}; recovery receipt could not be preserved at ${recovery}: ${recoveryFailure.message}`,
+        );
+      }
+      captureResult = {
+        status: 'rejected',
+        path: path.basename(recovery),
+        sha256: `sha256:${createHash('sha256').update(recoveryBytes).digest('hex')}`,
+      };
+      reportCapture();
+      fail('environment-configuration', `${integrityFailure}; source evidence preserved at ${recovery}`);
+    }
+    if (options.designPath) {
+      try {
+        const text = designText(process.cwd(), options.designPath);
+        source.artifactSha256 = text === null ? null : `sha256:${createHash('sha256').update(text).digest('hex')}`;
+      } catch {
+        source.artifactSha256 = null;
+      }
+    }
+    captureBytes = Buffer.from(`${JSON.stringify(source, null, 2)}\n`);
+    fs.ftruncateSync(captureFd, 0);
+    fs.writeSync(captureFd, captureBytes, 0, captureBytes.length, 0);
+    fs.fsyncSync(captureFd);
+    captureResult = {
+      status: 'verified',
+      path: path.basename(captureFile),
+      sha256: `sha256:${createHash('sha256').update(captureBytes).digest('hex')}`,
+    };
+  };
+  const captureDoubleTelemetry = (stderr) => {
+    if (!source || !options.captureTestDouble || options.agent !== 'custom') return;
+    const line = stderr.split('\n').find((line) => line.startsWith('TEA_EXPERIMENT_TEST_TELEMETRY:'));
+    if (line) {
+      try {
+        const telemetry = JSON.parse(line.slice('TEA_EXPERIMENT_TEST_TELEMETRY:'.length));
+        source.observed = telemetry.observed ?? source.observed;
+        source.usage = telemetry.usage ?? null;
+        source.cost = telemetry.cost ?? null;
+      } catch (error) {
+        (source.telemetryErrors ??= []).push(`test-double telemetry unavailable: ${error.message}`);
+        source.observed = { resolvedModel: null, appliedEffort: null };
+        source.usage = null;
+        source.cost = null;
+      }
+    }
+  };
+  if (options.capturePath) {
+    const relative = path.normalize(options.capturePath);
+    if (path.isAbsolute(relative) || relative === '..' || relative.startsWith(`..${path.sep}`))
+      fail('usage', '--capture-path must stay inside the workspace');
+    captureFile = path.join(process.cwd(), relative);
+    if (fs.existsSync(captureFile)) fail('usage', '--capture-path already exists');
+    let captureParent;
+    let workspaceRoot;
+    try {
+      captureParent = fs.realpathSync(path.dirname(captureFile));
+      workspaceRoot = fs.realpathSync(process.cwd());
+    } catch (error) {
+      fail('usage', `--capture-path parent cannot be resolved: ${error.message}`);
+    }
+    if (captureParent !== workspaceRoot) fail('usage', '--capture-path must be directly inside the workspace');
+    let requestedModel;
+    try {
+      requestedModel = options.captureTestDouble ? (options.model ?? null) : resolveModel(options.agent, options.model, options.agentArg);
+    } catch (error) {
+      fail(classOfAgentError(error), error.message);
+    }
+    try {
+      captureFd = fs.openSync(captureFile, 'wx+', 0o600);
+      captureIdentity = fs.fstatSync(captureFd);
+      source = {
+        kind: 'test-design-source-capture',
+        label: options.captureTestDouble ? 'deterministic-test-double' : 'native',
+        workspaceId: randomUUID(),
+        workspace: process.cwd(),
+        requested: {
+          agent: options.agent,
+          model: requestedModel,
+        },
+        inputSha256: `sha256:${createHash('sha256').update(prompt).digest('hex')}`,
+        startedAt: new Date().toISOString(),
+        endedAt: null,
+        status: 'dispatched',
+        observed: { resolvedModel: null, appliedEffort: null },
+        usage: null,
+        cost: null,
+      };
+      saveCapture();
+    } catch (error) {
+      let message = error.message;
+      if (captureFd !== null) {
+        try {
+          fs.closeSync(captureFd);
+        } catch (closeError) {
+          message += `; capture close failed: ${closeError.message}`;
+        }
+        captureFd = null;
+      }
+      fail('environment-configuration', `could not initialize source capture: ${message}`);
+    }
+  }
   try {
-    ({ stdout } = runAgent(prompt, {
+    const answer = runAgent(prompt, {
       agent: options.agent,
       agentCommand: options.agentCmd,
       agentArgs: options.agentArg,
       envPass: options.envPass,
-      model: options.model,
+      model: options.captureTestDouble && options.agent === 'custom' ? undefined : options.model,
       timeout,
       cwd: process.cwd(),
       capabilities: RUNNER_CAPABILITIES,
-    }));
+      usageReport: Boolean(source) && options.agent !== 'custom',
+      captureTelemetry: Boolean(source) && options.agent !== 'custom',
+    });
+    stdout = answer.stdout;
+    if (source) {
+      source.observed = { resolvedModel: answer.telemetry?.resolvedModel ?? null, appliedEffort: answer.telemetry?.appliedEffort ?? null };
+      source.providerStdout = answer.sourceStdout ?? answer.stdout;
+      source.providerStderr = answer.sourceStderr ?? answer.stderr;
+      source.usage = answer.telemetry?.usage ?? null;
+      source.telemetryErrors = answer.telemetry?.errors ?? [];
+      const usd = source.usage?.costUsd;
+      if (usd !== undefined) source.cost = { currency: 'USD', components: { target: Number(usd), tools: null } };
+      captureDoubleTelemetry(answer.stderr);
+      source.status = 'completed';
+      source.processOutcome = { status: 'completed', exitCode: 0 };
+      source.endedAt = new Date().toISOString();
+      saveCapture();
+    }
   } catch (error) {
+    if (source) {
+      source.providerStdout = error.stdout ?? '';
+      source.providerStderr = error.stderr ?? '';
+      captureDoubleTelemetry(error.stderr ?? '');
+      if (options.agent !== 'custom') {
+        const parsed = agentCaptureTelemetry(options.agent, error.stdout ?? '', error.stderr ?? '');
+        source.usage = parsed.usage;
+        source.observed.resolvedModel = parsed.resolvedModel;
+        source.telemetryErrors = parsed.errors;
+        source.providerStdout = error.stdout ?? '';
+        source.providerStderr = error.stderr ?? '';
+        if (parsed.usage?.costUsd !== undefined)
+          source.cost = { currency: 'USD', components: { target: Number(parsed.usage.costUsd), tools: null } };
+      }
+      source.status = /timed out/.test(error.message) ? 'timeout' : /signal SIG(?:INT|TERM)/.test(error.message) ? 'canceled' : 'failed';
+      source.processOutcome = { status: source.status, exitCode: null };
+      source.endedAt = new Date().toISOString();
+      source.error = { code: error.code ?? null, message: error.message };
+      source.stdout = error.stdout ?? '';
+      source.stderr = error.stderr ?? '';
+      saveCapture();
+    }
+    reportCapture();
     fail(classOfAgentError(error), error.message);
   }
-
   if (options.designPath) {
     try {
-      const projection = projectScoredRisks(process.cwd(), options.designPath);
-      if (projection) process.stdout.write(`${JSON.stringify(projection)}\n`);
+      const projection = projectScoredRisks(process.cwd(), options.designPath, { requireDeliverable: Boolean(source) });
+      if (projection)
+        process.stdout.write(`${JSON.stringify(captureResult ? { ...projection, sourceCapture: captureResult } : projection)}\n`);
+      else if (source)
+        throw Object.assign(new Error('the native design deliverable is absent'), { failureClass: 'environment-missing-artifact' });
     } catch (error) {
+      if (source) {
+        source.agentStatus = source.status;
+        source.status = 'failed';
+        source.failureClass = error.failureClass ?? 'environment-parser';
+        source.error = { code: 'RUNNER_PROJECTION_FAILED', message: error.message };
+        saveCapture();
+      }
+      reportCapture();
       fail('environment-parser', `could not project scored risks: ${error.message}`);
     }
-  }
+  } else reportCapture();
+  if (captureFd !== null) fs.closeSync(captureFd);
 
   if (stdout.length > 0) process.stderr.write(stdout.endsWith('\n') ? stdout : `${stdout}\n`);
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/package.json",
   "name": "bmad-method-test-architecture-enterprise",
-  "version": "3.2.0",
+  "version": "3.3.0",
   "description": "Master Test Architect for quality strategy, test automation, and release gates",
   "keywords": [
     "bmad",
@@ -95,7 +95,7 @@
     "release:major": "gh workflow run publish.yaml -f bump=major",
     "release:minor": "gh workflow run publish.yaml -f bump=minor",
     "release:patch": "gh workflow run publish.yaml -f bump=patch",
-    "test": "npm run test:conflict-markers && npm run test:schemas && npm run test:install && npm run test:evaluate-guidance && npm run test:evaluate-authoring && npm run test:evaluate-gap-loop && npm run test:evaluate-dogfood && npm run test:evaluate-check && npm run test:evaluate-discovery && npm run test:evaluate-generator && npm run test:evaluate-candidate-review && npm run test:evaluate-compare && npm run test:evaluate-before-state && npm run test:evaluate-ci-gates && npm run test:evaluate-ci-tiers && npm run test:evaluate-ci-repositories:tagged-release-pr && npm run test:evaluate-ci-repositories:tagged-release-merge && npm run test:evaluate-ci-repositories:tagged-release-release && npm run test:evaluate-ci-repositories:nightly-deploy-pr && npm run test:evaluate-ci-repositories:nightly-deploy-merge && npm run test:evaluate-ci-repositories:nightly-deploy-scheduled && npm run test:evaluate-ci-repositories:nightly-deploy-release && npm run test:evaluate-ci-render && npm run test:evaluate-pr-mcp && npm run test:evaluate-pr-api && npm run test:evaluate-pr-suite && npm run test:evaluate-pr-workflow && npm run test:evaluate-pr-tool-use && npm run test:evaluate-pr-promptfoo && npm run test:evaluate-pr-ai-feature && npm run test:evaluate-pr-test-review && npm run test:evaluate-pr-gap-loop && npm run test:evaluate-pr-learn && npm run test:evaluate-pr-tutorial && npm run test:evaluate-partitions && npm run test:evaluate-partition-plans-views-and-check && npm run test:evaluate-partition-plans-preflight-and-run && npm run test:evaluate-partition-plans-score && npm run test:evaluate-partition-plans-attempts-seal-and-mappings && npm run test:evaluate-partition-plans-attempts-records && npm run test:evaluate-partition-plans-attempts-gameability && npm run test:evaluate-interpret && npm run test:evaluate-calibration && npm run test:evaluate-boundaries && npm run test:evaluate-preflight && npm run test:evaluate-isolation-conformance && npm run test:evaluate-mutation:qualification-basics && npm run test:evaluate-mutation:qualification-guards && npm run test:evaluate-mutation:recovery && npm run test:evaluate-mutation:journals && npm run test:evaluate-run && npm run test:evaluate-progress && npm run test:evaluate-aggregate && npm run test:evaluate-confinement && npm run test:evaluate-confinement-audit && npm run test:evaluate-confinement-history && npm run test:evaluate-confinement-git-state && npm run test:evaluate-confinement-network && npm run test:evaluate-held-inputs && npm run test:evaluate-arms-historical && npm run test:evaluate-arms-deployments && npm run test:evaluate-evaluators && npm run test:evaluate-evaluators-agent-version && npm run test:evaluate-agents && npm run test:evaluate-agents-importers && npm run test:evaluate-agents-sealed-brief && npm run test:evaluate-held-attempts && npm run test:evaluate-private && npm run test:evaluate-records && npm run test:evaluate-mcp && npm run test:evaluate-api && npm run test:evaluate-workflow && npm run test:evaluate-tool-use && npm run test:evaluate-promptfoo && npm run test:evaluate-learned-framework && npm run test:docs-tutorial && npm run test:knowledge && npm run test:criteria-fragments && npm run test:doc-count-sources && npm run test:planning-doc-sources && npm run test:doc-counts && npm run test:doc-claim-sources && npm run test:doc-claims && npm run test:doc-invocation-entry && npm run test:doc-invocations && npm run test:doc-link-anchors && npm run test:docs-build-names && npm run test:doc-shell-blocks && npm run test:doc-ci-samples && npm run test:doc-playwright-examples && npm run test:docs-evaluate-sidebar && npm run test:docs-general-claims && npm run test:docs-gap-reader && npm run test:enforce-hook && npm run test:framework-scaffold && npm run test:isolation-primitives && npm run test:atdd-isolation && npm run test:atdd-net-guard && npm run test:atdd-scoring && npm run test:atdd-workflow-guidance && npm run test:generation-healing && npm run test:framework-scaffold-install-isolation && npm run test:transcript-harness && npm run test:suite-manifest && npm run test:teach-me-testing-scoring && npm run test:teach-cli && npm run test:eval-data && npm run test:eval-atdd-data && npm run test:eval-ci-data && npm run test:eval-framework-scaffold-data && npm run test:eval-nfr-data && npm run test:nfr-validate-corpus && npm run test:nfr-evidence-guidance && npm run test:nfr-cli && npm run test:eval-test-design-data && npm run test:eval-trace-data && npm run test:eval-routing-data && npm run test:eval-routing-boundaries && npm run test:eval-routing-evidence && npm run test:eval-transcript-data && npm run test:eval-teach-me-testing-data && npm run test:eval-schemas && npm run test:eval-diagnostics && npm run test:eval-replay && npm run test:eval-quality-corpus && npm run test:schema-versions && npm run test:contract-sources && npm run test:contracts && npm run test:contract-oracles && npm run test:probe-sources && npm run test:test-design-qualification && npm run test:test-review-qualification && npm run test:trace-qualification && npm run test:nfr-qualification && npm run test:ci-qualification && npm run test:probe-corpus && npm run test:compare-dominance && npm run test:compare-eval-runs && npm run test:configuration-experiment && npm run test:trial-set-scoring && npm run test:automate-cli && npm run test:automate-unit-runners && npm run test:automate-generation-evidence && npm run test:automate-eval-fixture && npm run test:eval-automate-data && npm run test:clock-port && npm run test:probe-conformance && npm run test:corpus-conformance && npm run test:file-system-conformance && npm run test:file-system-port && npm run test:probe-targets && npm run test:port-totality && npm run test:ci-coverage && npm run test:ci-coverage-filters && npm run test:install-actionlint && npm run test:workflows-lint && npm run test:artifact-retry && npm run test:install-linux-sandbox && npm run test:shards && npm run test:groups && npm run test:cli && npm run test:workflow-cli && npm run test:migrate-customizations && npm run test:framework-cli && npm run test:test-review-pr-scope && npm run test:test-review-report && npm run test:test-review-rubric && npm run test:bmad-output-gated && npm run test:release-metadata && npm run test:release-notes && npm run test:lockfile-age && npm run test:lockfile-age-cache && npm run test:licences && npm run test:supply-chain && npm run test:boundary && npm run test:direction && npm run test:lineage && npm run test:guard-publish && npm run test:layering-boundary-lineage && npm run test:changelog && npm run test:tea-workflow-descriptions && npm run test:docs-mermaid && npm run test:llms-headroom && npm run validate:schemas && npm run lint && npm run lint:md && npm run format:check",
+    "test": "npm run test:conflict-markers && npm run test:schemas && npm run test:install && npm run test:evaluate-guidance && npm run test:evaluate-authoring && npm run test:evaluate-gap-loop && npm run test:evaluate-dogfood && npm run test:evaluate-check && npm run test:evaluate-discovery && npm run test:evaluate-generator && npm run test:evaluate-candidate-review && npm run test:evaluate-compare && npm run test:evaluate-before-state && npm run test:evaluate-ci-gates && npm run test:evaluate-ci-tiers && npm run test:evaluate-ci-repositories:tagged-release-pr && npm run test:evaluate-ci-repositories:tagged-release-merge && npm run test:evaluate-ci-repositories:tagged-release-release && npm run test:evaluate-ci-repositories:nightly-deploy-pr && npm run test:evaluate-ci-repositories:nightly-deploy-merge && npm run test:evaluate-ci-repositories:nightly-deploy-scheduled && npm run test:evaluate-ci-repositories:nightly-deploy-release && npm run test:evaluate-ci-render && npm run test:evaluate-pr-mcp && npm run test:evaluate-pr-api && npm run test:evaluate-pr-suite && npm run test:evaluate-pr-workflow && npm run test:evaluate-pr-tool-use && npm run test:evaluate-pr-promptfoo && npm run test:evaluate-pr-ai-feature && npm run test:evaluate-pr-test-review && npm run test:evaluate-pr-gap-loop && npm run test:evaluate-pr-learn && npm run test:evaluate-pr-tutorial && npm run test:evaluate-partitions && npm run test:evaluate-partition-plans-views-and-check && npm run test:evaluate-partition-plans-preflight-and-run && npm run test:evaluate-partition-plans-score && npm run test:evaluate-partition-plans-attempts-seal-and-mappings && npm run test:evaluate-partition-plans-attempts-records && npm run test:evaluate-partition-plans-attempts-gameability && npm run test:evaluate-interpret && npm run test:evaluate-calibration && npm run test:evaluate-boundaries && npm run test:evaluate-preflight && npm run test:evaluate-isolation-conformance && npm run test:evaluate-mutation:qualification-basics && npm run test:evaluate-mutation:qualification-guards && npm run test:evaluate-mutation:recovery && npm run test:evaluate-mutation:journals && npm run test:evaluate-run && npm run test:evaluate-progress && npm run test:evaluate-aggregate && npm run test:evaluate-confinement && npm run test:evaluate-confinement-audit && npm run test:evaluate-confinement-history && npm run test:evaluate-confinement-git-state && npm run test:evaluate-confinement-network && npm run test:evaluate-held-inputs && npm run test:evaluate-arms-historical && npm run test:evaluate-arms-deployments && npm run test:evaluate-evaluators && npm run test:evaluate-evaluators-agent-version && npm run test:evaluate-agents && npm run test:evaluate-agents-importers && npm run test:evaluate-agents-sealed-brief && npm run test:evaluate-held-attempts && npm run test:evaluate-private && npm run test:evaluate-records && npm run test:evaluate-mcp && npm run test:evaluate-api && npm run test:evaluate-workflow && npm run test:evaluate-tool-use && npm run test:evaluate-promptfoo && npm run test:evaluate-learned-framework && npm run test:docs-tutorial && npm run test:knowledge && npm run test:criteria-fragments && npm run test:doc-count-sources && npm run test:planning-doc-sources && npm run test:doc-counts && npm run test:doc-claim-sources && npm run test:doc-claims && npm run test:doc-invocation-entry && npm run test:doc-invocations && npm run test:doc-link-anchors && npm run test:docs-build-names && npm run test:doc-shell-blocks && npm run test:doc-ci-samples && npm run test:doc-playwright-examples && npm run test:docs-evaluate-sidebar && npm run test:docs-general-claims && npm run test:docs-gap-reader && npm run test:enforce-hook && npm run test:framework-scaffold && npm run test:isolation-primitives && npm run test:atdd-isolation && npm run test:atdd-net-guard && npm run test:atdd-scoring && npm run test:atdd-workflow-guidance && npm run test:generation-healing && npm run test:framework-scaffold-install-isolation && npm run test:transcript-harness && npm run test:suite-manifest && npm run test:teach-me-testing-scoring && npm run test:teach-cli && npm run test:eval-data && npm run test:eval-atdd-data && npm run test:eval-ci-data && npm run test:eval-framework-scaffold-data && npm run test:eval-nfr-data && npm run test:nfr-validate-corpus && npm run test:nfr-evidence-guidance && npm run test:nfr-cli && npm run test:eval-test-design-data && npm run test:eval-trace-data && npm run test:eval-routing-data && npm run test:eval-routing-boundaries && npm run test:eval-routing-evidence && npm run test:eval-transcript-data && npm run test:eval-teach-me-testing-data && npm run test:eval-schemas && npm run test:eval-diagnostics && npm run test:eval-replay && npm run test:eval-quality-corpus && npm run test:schema-versions && npm run test:contract-sources && npm run test:contracts && npm run test:contract-oracles && npm run test:probe-sources && npm run test:test-design-qualification && npm run test:test-review-qualification && npm run test:trace-qualification && npm run test:nfr-qualification && npm run test:ci-qualification && npm run test:probe-corpus && npm run test:compare-dominance && npm run test:compare-eval-runs && npm run test:configuration-experiment && npm run test:evaluate-experiments && npm run test:trial-set-scoring && npm run test:automate-cli && npm run test:automate-unit-runners && npm run test:automate-generation-evidence && npm run test:automate-eval-fixture && npm run test:eval-automate-data && npm run test:clock-port && npm run test:probe-conformance && npm run test:corpus-conformance && npm run test:file-system-conformance && npm run test:file-system-port && npm run test:probe-targets && npm run test:port-totality && npm run test:ci-coverage && npm run test:ci-coverage-filters && npm run test:install-actionlint && npm run test:workflows-lint && npm run test:artifact-retry && npm run test:install-linux-sandbox && npm run test:shards && npm run test:groups && npm run test:cli && npm run test:workflow-cli && npm run test:migrate-customizations && npm run test:framework-cli && npm run test:test-review-pr-scope && npm run test:test-review-report && npm run test:test-review-rubric && npm run test:bmad-output-gated && npm run test:release-metadata && npm run test:release-notes && npm run test:lockfile-age && npm run test:lockfile-age-cache && npm run test:licences && npm run test:supply-chain && npm run test:boundary && npm run test:direction && npm run test:lineage && npm run test:guard-publish && npm run test:layering-boundary-lineage && npm run test:changelog && npm run test:tea-workflow-descriptions && npm run test:docs-mermaid && npm run test:llms-headroom && npm run validate:schemas && npm run lint && npm run lint:md && npm run format:check",
     "test:artifact-retry": "node test/test-artifact-retry.js",
     "test:atdd-isolation": "node test/test-atdd-isolation.js",
     "test:atdd-net-guard": "node test/test-atdd-net-guard.js",
@@ -193,6 +193,7 @@
     "test:evaluate-dogfood": "node test/test-evaluate-dogfood.js",
     "test:evaluate-evaluators": "node test/test-evaluate-evaluators.js --group=evaluators",
     "test:evaluate-evaluators-agent-version": "node test/test-evaluate-evaluators.js --group=evaluators-agent-version",
+    "test:evaluate-experiments": "node test/test-evaluate-experiments.js",
     "test:evaluate-gap-loop": "node test/test-evaluate-gap-loop.js",
     "test:evaluate-generator": "node test/test-evaluate-generator.js",
     "test:evaluate-guidance": "node test/test-evaluate-guidance.js",

package/skills/bmad-tea-evaluate/SKILL.md

@@ -21,6 +21,17 @@ You will continue to operate with your given name, identity, and communication_s
 - `{skill-name}` resolves to the skill directory's basename.
 - Resolve sibling files such as `references/...` and `assets/...` from `{skill-root}`.
 
+## Execution Authority
+
+Resolve intent before activation, customization resolution or any hook. Ordinary evaluation keeps its current workflow unchanged. Native clean/mutated trials, run comparison and baseline acceptance retain their existing authority and meaning.
+
+An explicit native compare/accept request keeps that command's authority, including a request that mentions consistency while comparing saved runs. Saved-result consistency is an experiment route only when the user asks to analyze variability in existing results. Resolve contextual words such as “these results” from the conversation before choosing a route.
+
+- Saved-result consistency or explicitly requested offline experiment analysis: load `references/configuration-experiments.md` and take its offline route directly. Suppress prepend, append and completion hooks. Do not run configuration resolver scripts, model calls, judge calls or fresh trials. Return after reporting the saved evidence.
+- Fresh repeatability or a one-factor configuration experiment: load `references/configuration-experiments.md` directly. Suppress prepend, append and completion hooks throughout this route. No hook can grant execution or spending authority.
+- “Is it repeatable?” without a saved/fresh choice: clarify saved records versus fresh runs before executing anything.
+- Ordinary evaluation, ordinary offline record evaluation or native compare/accept (including comparing saved runs): proceed with On Activation below. Do not load the experiment guide, analyze experiments or add trials after the normal result.
+
 ## On Activation
 
 Before resolving customizations or running hooks, inspect `{project-root}/_bmad/custom/` for `bmad-testarch-evaluate.toml` and `bmad-testarch-evaluate.user.toml`. If either exists, stop and tell the user to run `npx --package=bmad-method-test-architecture-enterprise@3 tea-migrate-customizations --project-root {project-root}`. Resolve any reported manual merge, then start a fresh chat. Do not continue with default settings while legacy overrides remain.