bmad-method-test-architecture-enterprise 3.2.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/cli/evaluate.js +20 -0
- package/cli/lib/agent-adapters.js +46 -0
- package/cli/lib/evaluate/configuration-experiment.js +3401 -0
- package/cli/lib/evaluate/experiments.js +1299 -0
- package/cli/lib/evaluate/retain-skill-artifacts.js +78 -0
- package/cli/lib/evaluate/test-design-corpus/clean/docs/epics/epic-9-last-sync-indicator.md +47 -0
- package/cli/lib/evaluate/test-design-corpus/ground-truth.json +222 -0
- package/cli/lib/evaluate/test-design-corpus/native-identity.json +4 -0
- package/cli/lib/evaluate/test-design-corpus/seeded/docs/epics/epic-7-offline-order-capture.md +75 -0
- package/cli/lib/evaluate/test-design-replay.js +452 -0
- package/cli/lib/run-agent.js +18 -2
- package/cli/lib/test-design-staged-config.js +60 -0
- package/cli/lib/test-design-staging.js +229 -0
- package/cli/test-design-runner.js +245 -11
- package/package.json +3 -2
- package/skills/bmad-tea-evaluate/SKILL.md +11 -0
- package/skills/bmad-tea-evaluate/references/configuration-experiments.md +37 -0
- package/skills/bmod-tea/bmod.toml +1 -1
- package/skills/bmod-tea/help/help.md +1 -1
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"name": "bmad-method-test-architecture-enterprise",
|
|
32
32
|
"source": "./",
|
|
33
33
|
"description": "Master Test Architect module for quality strategy, test generation in red and expand modes with automatic run-and-heal, combined framework and CI setup, CI/CD quality gates, and structured testing education. Existing CI and ATDD customizations can be migrated to the new skill names. Part of the BMad Method ecosystem.",
|
|
34
|
-
"version": "3.
|
|
34
|
+
"version": "3.3.0",
|
|
35
35
|
"author": {
|
|
36
36
|
"name": "Murat K Ozcan (TEA Creator) & Brian (BMad) Madison"
|
|
37
37
|
},
|
package/cli/evaluate.js
CHANGED
|
@@ -367,6 +367,26 @@ function buildProgram(run) {
|
|
|
367
367
|
.option('--evaluation <path>', 'the evaluation folder, or its evaluation.json')
|
|
368
368
|
.option('--tier <tier>', 'pr, merge, scheduled or release')
|
|
369
369
|
.action((options) => run(ciCommand, options));
|
|
370
|
+
program
|
|
371
|
+
.command('experiment')
|
|
372
|
+
.description('Opt-in saved Test Design consistency analysis or a bounded experiment proposal; no automatic adoption.')
|
|
373
|
+
.option('--saved <path>', 'exact saved run, retained attempt or experiment directory')
|
|
374
|
+
.option('--search-root <path>', 'find the latest compatible saved Test Design records')
|
|
375
|
+
.option('--propose <path>', 'prepare a proposal from an agent-owned declaration')
|
|
376
|
+
.option('--agent <name>', 'native execution adapter', 'codex')
|
|
377
|
+
.option('--max-usd <amount>', 'proposed spending cap; verified enforcement remains required')
|
|
378
|
+
.option('--recover <path>', 'inspect unresolved reservations without rerunning anything')
|
|
379
|
+
.option('--out <path>', 'new report or proposal directory')
|
|
380
|
+
.action((options) =>
|
|
381
|
+
run(async (opts) => {
|
|
382
|
+
try {
|
|
383
|
+
return await require('./lib/evaluate/experiments').experimentCommand(opts);
|
|
384
|
+
} catch (error) {
|
|
385
|
+
process.stderr.write(`${NAME} experiment: ${escapeUnprintable(error.message)}\n`);
|
|
386
|
+
return error.code === 'EXPERIMENT_USAGE' ? EXIT_CODES.usage : EXIT_CODES.authoring;
|
|
387
|
+
}
|
|
388
|
+
}, options),
|
|
389
|
+
);
|
|
370
390
|
return program;
|
|
371
391
|
}
|
|
372
392
|
|
|
@@ -710,11 +710,57 @@ function bridgedArgsRefused(agent, extra = []) {
|
|
|
710
710
|
return extra.filter((arg) => locked.some((flag) => arg === flag || arg.startsWith(`${flag}=`)));
|
|
711
711
|
}
|
|
712
712
|
|
|
713
|
+
/** Capture keeps partial native telemetry and parse errors without discarding raw streams. */
|
|
714
|
+
function agentCaptureTelemetry(agent, stdout, stderr) {
|
|
715
|
+
let parsed = { stdout: agentAnswerText(agent, stdout), usage: null };
|
|
716
|
+
const errors = [];
|
|
717
|
+
try {
|
|
718
|
+
parsed = agentReplyAndUsage(agent, stdout, stderr);
|
|
719
|
+
} catch (error) {
|
|
720
|
+
errors.push(error.message);
|
|
721
|
+
// Preserve independently present fields; absent or invalid money stays unknown.
|
|
722
|
+
try {
|
|
723
|
+
const report = agent === 'claude' ? JSON.parse(stdout) : null;
|
|
724
|
+
const usage = report?.usage;
|
|
725
|
+
if (usage) {
|
|
726
|
+
parsed.usage = {};
|
|
727
|
+
const input = usage.input_tokens + (usage.cache_creation_input_tokens ?? 0) + (usage.cache_read_input_tokens ?? 0);
|
|
728
|
+
if (Number.isFinite(input) && input >= 0) parsed.usage.inputTokens = input;
|
|
729
|
+
if (Number.isFinite(usage.output_tokens) && usage.output_tokens >= 0) parsed.usage.outputTokens = usage.output_tokens;
|
|
730
|
+
}
|
|
731
|
+
if (Number.isFinite(report?.total_cost_usd) && report.total_cost_usd >= 0) (parsed.usage ??= {}).costUsd = report.total_cost_usd;
|
|
732
|
+
} catch {
|
|
733
|
+
// Raw streams and parsing errors remain available even for malformed reports.
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
if (agent === 'codex' && parsed.usage === null) {
|
|
737
|
+
for (const line of stdout.split(/\r?\n/).filter(Boolean)) {
|
|
738
|
+
try {
|
|
739
|
+
const event = JSON.parse(line);
|
|
740
|
+
if (event.type !== 'turn.completed') continue;
|
|
741
|
+
const available = {};
|
|
742
|
+
if (Number.isSafeInteger(event.usage?.input_tokens) && event.usage.input_tokens >= 0)
|
|
743
|
+
available.inputTokens = event.usage.input_tokens;
|
|
744
|
+
if (Number.isSafeInteger(event.usage?.output_tokens) && event.usage.output_tokens >= 0)
|
|
745
|
+
available.outputTokens = event.usage.output_tokens;
|
|
746
|
+
if (Object.keys(available).length > 0) {
|
|
747
|
+
parsed.usage = available;
|
|
748
|
+
errors.push('codex verified money usage is unavailable; native token fields are retained');
|
|
749
|
+
}
|
|
750
|
+
} catch {
|
|
751
|
+
/* The raw event and any strict parsing error remain available. */
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
return { stdout: parsed.stdout, resolvedModel: resolvedModelFromAnswer(agent, stdout), appliedEffort: null, usage: parsed.usage, errors };
|
|
756
|
+
}
|
|
757
|
+
|
|
713
758
|
module.exports = {
|
|
714
759
|
AGENT_ADAPTERS,
|
|
715
760
|
AGENT_VERSION_CEILING_MS,
|
|
716
761
|
AGENT_VERSION_TIMEOUT_MS,
|
|
717
762
|
agentReplyAndUsage,
|
|
763
|
+
agentCaptureTelemetry,
|
|
718
764
|
resolvedModelFromAnswer,
|
|
719
765
|
agentAnswerText,
|
|
720
766
|
bridgedArgsRefused,
|