bmad-method-test-architecture-enterprise 3.1.0 → 3.3.0

package/cli/test-design-runner.js

@@ -52,9 +52,10 @@
 
 const fs = require('node:fs');
 const path = require('node:path');
+const { createHash, randomUUID } = require('node:crypto');
 const { Command } = require('commander');
 
-const { AGENT_ADAPTERS } = require('./lib/agent-adapters');
+const { AGENT_ADAPTERS, resolveModel, agentCaptureTelemetry } = require('./lib/agent-adapters');
 const { runAgent } = require('./lib/run-agent');
 const { EXIT_CODES, classOfAgentError, failureClassForExit, vendorEnvironmentNames } = require('./lib/runner-exit-codes');
 const { readDesign, scoredRiskProjection } = require('./lib/test-design-parser');
@@ -81,7 +82,8 @@ const TEST_DESIGN_REQUEST_KEYS = {
   argument: { required: [], permitted: [] },
   option: {
     required: ['agent', 'design-path'],
-    permitted: ['agent', 'agent-cmd', 'agent-arg', 'design-path', 'env-pass', 'model', 'timeout-ms'],
+    permitted: ['agent', 'agent-cmd', 'agent-arg', 'design-path', 'env-pass', 'model', 'timeout-ms', 'capture-path', 'capture-test-double'],
+    types: { 'capture-test-double': 'boolean' },
   },
   environment: { required: [], permitted: vendorEnvironmentNames() },
   stdin: { required: ['prompt'], permitted: ['prompt'] },
@@ -107,7 +109,7 @@ function readPrompt() {
 }
 
 /** Derive a projection only after the agent has finished its document. */
-function projectScoredRisks(cwd, relativeDesignPath) {
+function designText(cwd, relativeDesignPath) {
   const relative = path.normalize(relativeDesignPath);
   if (path.isAbsolute(relative) || relative.startsWith(`..${path.sep}`) || relative === '..' || !relative.endsWith('.md')) {
     throw new Error(`--design-path must name a markdown file inside the working directory: ${JSON.stringify(relativeDesignPath)}`);
@@ -120,8 +122,13 @@ function projectScoredRisks(cwd, relativeDesignPath) {
   if (fromRoot === '..' || fromRoot.startsWith(`..${path.sep}`) || path.isAbsolute(fromRoot) || fs.lstatSync(designPath).isSymbolicLink()) {
     throw new Error(`design path escapes the working directory: ${JSON.stringify(relativeDesignPath)}`);
   }
-  const text = fs.readFileSync(designPath, 'utf8');
+  return fs.readFileSync(designPath, 'utf8');
+}
+function projectScoredRisks(cwd, relativeDesignPath, { requireDeliverable = false } = {}) {
+  const text = designText(cwd, relativeDesignPath);
+  if (text === null) return null;
   const read = readDesign({ kind: 'text', value: text });
+  if (!read.ok && requireDeliverable) throw Object.assign(new Error(read.reason), { failureClass: read.failureClass });
   return scoredRiskProjection(read.ok ? read.design : { risks: [], text });
 }
 
@@ -136,6 +143,8 @@ function main(argv) {
     .option('--env-pass <NAME>', 'environment variable name allowed through to the agent (repeatable)', collect, [])
     .option('--model <name>', 'model to pin for this run; defaults to the adapter default')
     .option('--design-path <path>', 'staged design document path for a deterministic scored-risk projection')
+    .option('--capture-path <path>', 'opt-in source receipt and telemetry inside the workspace')
+    .option('--capture-test-double', 'label custom telemetry as a deterministic test double')
     .option('--timeout-ms <n>', 'agent wall-clock timeout in milliseconds', String(DEFAULT_TIMEOUT_MS));
 
   program.exitOverride();
@@ -151,6 +160,9 @@ function main(argv) {
   if (!Object.hasOwn(AGENT_ADAPTERS, options.agent)) {
     fail('environment-configuration', `unknown agent "${options.agent}"; expected one of ${Object.keys(AGENT_ADAPTERS).join(', ')}`);
   }
+  if (options.captureTestDouble && options.agent !== 'custom') {
+    fail('usage', '--capture-test-double requires --agent custom');
+  }
   // Tested on the raw string before the parse. Number.parseInt truncates, so
   // `--timeout-ms 20abc` became 20 and every run then timed out at twenty
   // milliseconds, and `--timeout-ms 1e9` became 1.
@@ -168,29 +180,251 @@ function main(argv) {
   }
 
   let stdout = '';
+  let source = null;
+  let captureFile = null;
+  let captureFd = null;
+  let captureIdentity = null;
+  let captureBytes = null;
+  let captureResult = null;
+  const reportCapture = () => {
+    if (captureResult) process.stdout.write(`${JSON.stringify({ sourceCapture: captureResult })}\n`);
+  };
+  const saveCapture = () => {
+    if (captureFd === null) return;
+    let integrityFailure = null;
+    try {
+      const held = fs.fstatSync(captureFd);
+      const named = fs.lstatSync(captureFile);
+      if (
+        !held.isFile() ||
+        held.nlink !== 1 ||
+        !named.isFile() ||
+        named.nlink !== 1 ||
+        held.dev !== captureIdentity.dev ||
+        held.ino !== captureIdentity.ino ||
+        named.dev !== held.dev ||
+        named.ino !== held.ino
+      )
+        integrityFailure = 'source capture path or link identity changed';
+      if (!integrityFailure && captureBytes !== null) {
+        if (held.size === captureBytes.length) {
+          const current = Buffer.alloc(held.size);
+          fs.readSync(captureFd, current, 0, current.length, 0);
+          if (!current.equals(captureBytes)) integrityFailure = 'source capture bytes changed';
+        } else integrityFailure = 'source capture bytes changed';
+      }
+    } catch (error) {
+      integrityFailure = `source capture identity could not be checked: ${error.message}`;
+    }
+    if (integrityFailure) {
+      const recovery = path.join(process.cwd(), `source-capture-recovery-${randomUUID()}.json`);
+      source.captureIntegrityViolation = { reason: integrityFailure, originalPath: captureFile, agentStatus: source.status };
+      source.status = 'failed';
+      const recoveryBytes = Buffer.from(`${JSON.stringify(source, null, 2)}\n`);
+      let recoveryFd = null;
+      let recoveryFailure = null;
+      try {
+        recoveryFd = fs.openSync(recovery, 'wx', 0o600);
+        fs.writeFileSync(recoveryFd, recoveryBytes);
+        fs.fsyncSync(recoveryFd);
+      } catch (error) {
+        recoveryFailure = error;
+      } finally {
+        for (const fd of [recoveryFd, captureFd]) {
+          if (fd === null) continue;
+          try {
+            fs.closeSync(fd);
+          } catch (error) {
+            recoveryFailure ??= error;
+          }
+        }
+        captureFd = null;
+      }
+      if (recoveryFailure) {
+        captureResult = {
+          status: 'unverified',
+          reason: integrityFailure,
+          recoveryError: recoveryFailure.message,
+          path: path.basename(recovery),
+        };
+        reportCapture();
+        fail(
+          'environment-configuration',
+          `${integrityFailure}; recovery receipt could not be preserved at ${recovery}: ${recoveryFailure.message}`,
+        );
+      }
+      captureResult = {
+        status: 'rejected',
+        path: path.basename(recovery),
+        sha256: `sha256:${createHash('sha256').update(recoveryBytes).digest('hex')}`,
+      };
+      reportCapture();
+      fail('environment-configuration', `${integrityFailure}; source evidence preserved at ${recovery}`);
+    }
+    if (options.designPath) {
+      try {
+        const text = designText(process.cwd(), options.designPath);
+        source.artifactSha256 = text === null ? null : `sha256:${createHash('sha256').update(text).digest('hex')}`;
+      } catch {
+        source.artifactSha256 = null;
+      }
+    }
+    captureBytes = Buffer.from(`${JSON.stringify(source, null, 2)}\n`);
+    fs.ftruncateSync(captureFd, 0);
+    fs.writeSync(captureFd, captureBytes, 0, captureBytes.length, 0);
+    fs.fsyncSync(captureFd);
+    captureResult = {
+      status: 'verified',
+      path: path.basename(captureFile),
+      sha256: `sha256:${createHash('sha256').update(captureBytes).digest('hex')}`,
+    };
+  };
+  const captureDoubleTelemetry = (stderr) => {
+    if (!source || !options.captureTestDouble || options.agent !== 'custom') return;
+    const line = stderr.split('\n').find((line) => line.startsWith('TEA_EXPERIMENT_TEST_TELEMETRY:'));
+    if (line) {
+      try {
+        const telemetry = JSON.parse(line.slice('TEA_EXPERIMENT_TEST_TELEMETRY:'.length));
+        source.observed = telemetry.observed ?? source.observed;
+        source.usage = telemetry.usage ?? null;
+        source.cost = telemetry.cost ?? null;
+      } catch (error) {
+        (source.telemetryErrors ??= []).push(`test-double telemetry unavailable: ${error.message}`);
+        source.observed = { resolvedModel: null, appliedEffort: null };
+        source.usage = null;
+        source.cost = null;
+      }
+    }
+  };
+  if (options.capturePath) {
+    const relative = path.normalize(options.capturePath);
+    if (path.isAbsolute(relative) || relative === '..' || relative.startsWith(`..${path.sep}`))
+      fail('usage', '--capture-path must stay inside the workspace');
+    captureFile = path.join(process.cwd(), relative);
+    if (fs.existsSync(captureFile)) fail('usage', '--capture-path already exists');
+    let captureParent;
+    let workspaceRoot;
+    try {
+      captureParent = fs.realpathSync(path.dirname(captureFile));
+      workspaceRoot = fs.realpathSync(process.cwd());
+    } catch (error) {
+      fail('usage', `--capture-path parent cannot be resolved: ${error.message}`);
+    }
+    if (captureParent !== workspaceRoot) fail('usage', '--capture-path must be directly inside the workspace');
+    let requestedModel;
+    try {
+      requestedModel = options.captureTestDouble ? (options.model ?? null) : resolveModel(options.agent, options.model, options.agentArg);
+    } catch (error) {
+      fail(classOfAgentError(error), error.message);
+    }
+    try {
+      captureFd = fs.openSync(captureFile, 'wx+', 0o600);
+      captureIdentity = fs.fstatSync(captureFd);
+      source = {
+        kind: 'test-design-source-capture',
+        label: options.captureTestDouble ? 'deterministic-test-double' : 'native',
+        workspaceId: randomUUID(),
+        workspace: process.cwd(),
+        requested: {
+          agent: options.agent,
+          model: requestedModel,
+        },
+        inputSha256: `sha256:${createHash('sha256').update(prompt).digest('hex')}`,
+        startedAt: new Date().toISOString(),
+        endedAt: null,
+        status: 'dispatched',
+        observed: { resolvedModel: null, appliedEffort: null },
+        usage: null,
+        cost: null,
+      };
+      saveCapture();
+    } catch (error) {
+      let message = error.message;
+      if (captureFd !== null) {
+        try {
+          fs.closeSync(captureFd);
+        } catch (closeError) {
+          message += `; capture close failed: ${closeError.message}`;
+        }
+        captureFd = null;
+      }
+      fail('environment-configuration', `could not initialize source capture: ${message}`);
+    }
+  }
   try {
-    ({ stdout } = runAgent(prompt, {
+    const answer = runAgent(prompt, {
       agent: options.agent,
       agentCommand: options.agentCmd,
       agentArgs: options.agentArg,
       envPass: options.envPass,
-      model: options.model,
+      model: options.captureTestDouble && options.agent === 'custom' ? undefined : options.model,
       timeout,
       cwd: process.cwd(),
       capabilities: RUNNER_CAPABILITIES,
-    }));
+      usageReport: Boolean(source) && options.agent !== 'custom',
+      captureTelemetry: Boolean(source) && options.agent !== 'custom',
+    });
+    stdout = answer.stdout;
+    if (source) {
+      source.observed = { resolvedModel: answer.telemetry?.resolvedModel ?? null, appliedEffort: answer.telemetry?.appliedEffort ?? null };
+      source.providerStdout = answer.sourceStdout ?? answer.stdout;
+      source.providerStderr = answer.sourceStderr ?? answer.stderr;
+      source.usage = answer.telemetry?.usage ?? null;
+      source.telemetryErrors = answer.telemetry?.errors ?? [];
+      const usd = source.usage?.costUsd;
+      if (usd !== undefined) source.cost = { currency: 'USD', components: { target: Number(usd), tools: null } };
+      captureDoubleTelemetry(answer.stderr);
+      source.status = 'completed';
+      source.processOutcome = { status: 'completed', exitCode: 0 };
+      source.endedAt = new Date().toISOString();
+      saveCapture();
+    }
   } catch (error) {
+    if (source) {
+      source.providerStdout = error.stdout ?? '';
+      source.providerStderr = error.stderr ?? '';
+      captureDoubleTelemetry(error.stderr ?? '');
+      if (options.agent !== 'custom') {
+        const parsed = agentCaptureTelemetry(options.agent, error.stdout ?? '', error.stderr ?? '');
+        source.usage = parsed.usage;
+        source.observed.resolvedModel = parsed.resolvedModel;
+        source.telemetryErrors = parsed.errors;
+        source.providerStdout = error.stdout ?? '';
+        source.providerStderr = error.stderr ?? '';
+        if (parsed.usage?.costUsd !== undefined)
+          source.cost = { currency: 'USD', components: { target: Number(parsed.usage.costUsd), tools: null } };
+      }
+      source.status = /timed out/.test(error.message) ? 'timeout' : /signal SIG(?:INT|TERM)/.test(error.message) ? 'canceled' : 'failed';
+      source.processOutcome = { status: source.status, exitCode: null };
+      source.endedAt = new Date().toISOString();
+      source.error = { code: error.code ?? null, message: error.message };
+      source.stdout = error.stdout ?? '';
+      source.stderr = error.stderr ?? '';
+      saveCapture();
+    }
+    reportCapture();
     fail(classOfAgentError(error), error.message);
   }
-
   if (options.designPath) {
     try {
-      const projection = projectScoredRisks(process.cwd(), options.designPath);
-      if (projection) process.stdout.write(`${JSON.stringify(projection)}\n`);
+      const projection = projectScoredRisks(process.cwd(), options.designPath, { requireDeliverable: Boolean(source) });
+      if (projection)
+        process.stdout.write(`${JSON.stringify(captureResult ? { ...projection, sourceCapture: captureResult } : projection)}\n`);
+      else if (source)
+        throw Object.assign(new Error('the native design deliverable is absent'), { failureClass: 'environment-missing-artifact' });
     } catch (error) {
+      if (source) {
+        source.agentStatus = source.status;
+        source.status = 'failed';
+        source.failureClass = error.failureClass ?? 'environment-parser';
+        source.error = { code: 'RUNNER_PROJECTION_FAILED', message: error.message };
+        saveCapture();
+      }
+      reportCapture();
       fail('environment-parser', `could not project scored risks: ${error.message}`);
     }
-  }
+  } else reportCapture();
+  if (captureFd !== null) fs.closeSync(captureFd);
 
   if (stdout.length > 0) process.stderr.write(stdout.endsWith('\n') ? stdout : `${stdout}\n`);
 }