bmad-method 5.1.3 → 6.0.0-alpha.1

package/test/test-agent-schema.js

@@ -0,0 +1,387 @@
+/**
+ * Agent Schema Validation Test Runner
+ *
+ * Runs all test fixtures and verifies expected outcomes.
+ * Reports pass/fail for each test and overall coverage statistics.
+ *
+ * Usage: node test/test-agent-schema.js
+ * Exit codes: 0 = all tests pass, 1 = test failures
+ */
+
+const fs = require('node:fs');
+const path = require('node:path');
+const yaml = require('js-yaml');
+const { validateAgentFile } = require('../tools/schema/agent.js');
+const { glob } = require('glob');
+
+// ANSI color codes
+const colors = {
+  reset: '\u001B[0m',
+  green: '\u001B[32m',
+  red: '\u001B[31m',
+  yellow: '\u001B[33m',
+  blue: '\u001B[34m',
+  cyan: '\u001B[36m',
+  dim: '\u001B[2m',
+};
+
+/**
+ * Parse test metadata from YAML comments
+ * @param {string} filePath
+ * @returns {{shouldPass: boolean, errorExpectation?: object, pathContext?: string}}
+ */
+function parseTestMetadata(filePath) {
+  const content = fs.readFileSync(filePath, 'utf8');
+  const lines = content.split('\n');
+
+  let shouldPass = true;
+  let pathContext = null;
+  const errorExpectation = {};
+
+  for (const line of lines) {
+    if (line.includes('Expected: PASS')) {
+      shouldPass = true;
+    } else if (line.includes('Expected: FAIL')) {
+      shouldPass = false;
+    }
+
+    // Parse error metadata
+    const codeMatch = line.match(/^# Error code: (.+)$/);
+    if (codeMatch) {
+      errorExpectation.code = codeMatch[1].trim();
+    }
+
+    const pathMatch = line.match(/^# Error path: (.+)$/);
+    if (pathMatch) {
+      errorExpectation.path = pathMatch[1].trim();
+    }
+
+    const messageMatch = line.match(/^# Error message: (.+)$/);
+    if (messageMatch) {
+      errorExpectation.message = messageMatch[1].trim();
+    }
+
+    const minimumMatch = line.match(/^# Error minimum: (\d+)$/);
+    if (minimumMatch) {
+      errorExpectation.minimum = parseInt(minimumMatch[1], 10);
+    }
+
+    const expectedMatch = line.match(/^# Error expected: (.+)$/);
+    if (expectedMatch) {
+      errorExpectation.expected = expectedMatch[1].trim();
+    }
+
+    const receivedMatch = line.match(/^# Error received: (.+)$/);
+    if (receivedMatch) {
+      errorExpectation.received = receivedMatch[1].trim();
+    }
+
+    const keysMatch = line.match(/^# Error keys: \[(.+)\]$/);
+    if (keysMatch) {
+      errorExpectation.keys = keysMatch[1].split(',').map((k) => k.trim().replaceAll(/['"]/g, ''));
+    }
+
+    const contextMatch = line.match(/^# Path context: (.+)$/);
+    if (contextMatch) {
+      pathContext = contextMatch[1].trim();
+    }
+  }
+
+  return {
+    shouldPass,
+    errorExpectation: Object.keys(errorExpectation).length > 0 ? errorExpectation : null,
+    pathContext,
+  };
+}
+
+/**
+ * Convert dot-notation path string to array (handles array indices)
+ * e.g., "agent.menu[0].trigger" => ["agent", "menu", 0, "trigger"]
+ */
+function parsePathString(pathString) {
+  return pathString
+    .replaceAll(/\[(\d+)\]/g, '.$1') // Convert [0] to .0
+    .split('.')
+    .map((part) => {
+      const num = parseInt(part, 10);
+      return isNaN(num) ? part : num;
+    });
+}
+
+/**
+ * Validate error against expectations
+ * @param {object} error - Zod error issue
+ * @param {object} expectation - Expected error structure
+ * @returns {{valid: boolean, reason?: string}}
+ */
+function validateError(error, expectation) {
+  // Check error code
+  if (expectation.code && error.code !== expectation.code) {
+    return { valid: false, reason: `Expected code "${expectation.code}", got "${error.code}"` };
+  }
+
+  // Check error path
+  if (expectation.path) {
+    const expectedPath = parsePathString(expectation.path);
+    const actualPath = error.path;
+
+    if (JSON.stringify(expectedPath) !== JSON.stringify(actualPath)) {
+      return {
+        valid: false,
+        reason: `Expected path ${JSON.stringify(expectedPath)}, got ${JSON.stringify(actualPath)}`,
+      };
+    }
+  }
+
+  // For custom errors, strictly check message
+  if (expectation.code === 'custom' && expectation.message && error.message !== expectation.message) {
+    return {
+      valid: false,
+      reason: `Expected message "${expectation.message}", got "${error.message}"`,
+    };
+  }
+
+  // For Zod errors, check type-specific fields
+  if (expectation.minimum !== undefined && error.minimum !== expectation.minimum) {
+    return { valid: false, reason: `Expected minimum ${expectation.minimum}, got ${error.minimum}` };
+  }
+
+  if (expectation.expected && error.expected !== expectation.expected) {
+    return { valid: false, reason: `Expected type "${expectation.expected}", got "${error.expected}"` };
+  }
+
+  if (expectation.received && error.received !== expectation.received) {
+    return { valid: false, reason: `Expected received "${expectation.received}", got "${error.received}"` };
+  }
+
+  if (expectation.keys) {
+    const expectedKeys = expectation.keys.sort();
+    const actualKeys = (error.keys || []).sort();
+    if (JSON.stringify(expectedKeys) !== JSON.stringify(actualKeys)) {
+      return {
+        valid: false,
+        reason: `Expected keys ${JSON.stringify(expectedKeys)}, got ${JSON.stringify(actualKeys)}`,
+      };
+    }
+  }
+
+  return { valid: true };
+}
+
+/**
+ * Run a single test case
+ * @param {string} filePath
+ * @returns {{passed: boolean, message: string}}
+ */
+function runTest(filePath) {
+  const metadata = parseTestMetadata(filePath);
+  const { shouldPass, errorExpectation, pathContext } = metadata;
+
+  try {
+    const fileContent = fs.readFileSync(filePath, 'utf8');
+    let agentData;
+
+    try {
+      agentData = yaml.load(fileContent);
+    } catch (parseError) {
+      // YAML parse error
+      if (shouldPass) {
+        return {
+          passed: false,
+          message: `Expected PASS but got YAML parse error: ${parseError.message}`,
+        };
+      }
+      return {
+        passed: true,
+        message: 'Got expected YAML parse error',
+      };
+    }
+
+    // Determine validation path
+    // If pathContext is specified in comments, use it; otherwise derive from fixture location
+    let validationPath = pathContext;
+    if (!validationPath) {
+      // Map fixture location to simulated src/ path
+      const relativePath = path.relative(path.join(__dirname, 'fixtures/agent-schema'), filePath);
+      const parts = relativePath.split(path.sep);
+
+      if (parts.includes('metadata') && parts[0] === 'valid') {
+        // Valid metadata tests: check if filename suggests module or core
+        const filename = path.basename(filePath);
+        if (filename.includes('module')) {
+          validationPath = 'src/modules/bmm/agents/test.agent.yaml';
+        } else {
+          validationPath = 'src/core/agents/test.agent.yaml';
+        }
+      } else if (parts.includes('metadata') && parts[0] === 'invalid') {
+        // Invalid metadata tests: derive from filename
+        const filename = path.basename(filePath);
+        if (filename.includes('module') || filename.includes('wrong-module')) {
+          validationPath = 'src/modules/bmm/agents/test.agent.yaml';
+        } else if (filename.includes('core')) {
+          validationPath = 'src/core/agents/test.agent.yaml';
+        } else {
+          validationPath = 'src/core/agents/test.agent.yaml';
+        }
+      } else {
+        // Default to core agent path
+        validationPath = 'src/core/agents/test.agent.yaml';
+      }
+    }
+
+    const result = validateAgentFile(validationPath, agentData);
+
+    if (result.success && shouldPass) {
+      return {
+        passed: true,
+        message: 'Validation passed as expected',
+      };
+    }
+
+    if (!result.success && !shouldPass) {
+      const actualError = result.error.issues[0];
+
+      // If we have error expectations, validate strictly
+      if (errorExpectation) {
+        const validation = validateError(actualError, errorExpectation);
+
+        if (!validation.valid) {
+          return {
+            passed: false,
+            message: `Error validation failed: ${validation.reason}`,
+          };
+        }
+
+        return {
+          passed: true,
+          message: `Got expected error (${errorExpectation.code}): ${actualError.message}`,
+        };
+      }
+
+      // No specific expectations - just check that it failed
+      return {
+        passed: true,
+        message: `Got expected validation error: ${actualError?.message}`,
+      };
+    }
+
+    if (result.success && !shouldPass) {
+      return {
+        passed: false,
+        message: 'Expected validation to FAIL but it PASSED',
+      };
+    }
+
+    if (!result.success && shouldPass) {
+      return {
+        passed: false,
+        message: `Expected validation to PASS but it FAILED: ${result.error.issues[0]?.message}`,
+      };
+    }
+
+    return {
+      passed: false,
+      message: 'Unexpected test state',
+    };
+  } catch (error) {
+    return {
+      passed: false,
+      message: `Test execution error: ${error.message}`,
+    };
+  }
+}
+
+/**
+ * Main test runner
+ */
+async function main() {
+  console.log(`${colors.cyan}╔═══════════════════════════════════════════════════════════╗${colors.reset}`);
+  console.log(`${colors.cyan}║  Agent Schema Validation Test Suite                      ║${colors.reset}`);
+  console.log(`${colors.cyan}╚═══════════════════════════════════════════════════════════╝${colors.reset}\n`);
+
+  // Find all test fixtures
+  const testFiles = await glob('test/fixtures/agent-schema/**/*.agent.yaml', {
+    cwd: path.join(__dirname, '..'),
+    absolute: true,
+  });
+
+  if (testFiles.length === 0) {
+    console.log(`${colors.yellow}⚠️  No test fixtures found${colors.reset}`);
+    process.exit(0);
+  }
+
+  console.log(`Found ${colors.cyan}${testFiles.length}${colors.reset} test fixture(s)\n`);
+
+  // Group tests by category
+  const categories = {};
+  for (const testFile of testFiles) {
+    const relativePath = path.relative(path.join(__dirname, 'fixtures/agent-schema'), testFile);
+    const parts = relativePath.split(path.sep);
+    const validInvalid = parts[0]; // 'valid' or 'invalid'
+    const category = parts[1]; // 'top-level', 'metadata', etc.
+
+    const categoryKey = `${validInvalid}/${category}`;
+    if (!categories[categoryKey]) {
+      categories[categoryKey] = [];
+    }
+    categories[categoryKey].push(testFile);
+  }
+
+  // Run tests by category
+  let totalTests = 0;
+  let passedTests = 0;
+  const failures = [];
+
+  for (const [categoryKey, files] of Object.entries(categories).sort()) {
+    const [validInvalid, category] = categoryKey.split('/');
+    const categoryLabel = category.replaceAll('-', ' ').toUpperCase();
+    const validLabel = validInvalid === 'valid' ? '✅' : '❌';
+
+    console.log(`${colors.blue}${validLabel} ${categoryLabel} (${validInvalid})${colors.reset}`);
+
+    for (const testFile of files) {
+      totalTests++;
+      const testName = path.basename(testFile, '.agent.yaml');
+      const result = runTest(testFile);
+
+      if (result.passed) {
+        passedTests++;
+        console.log(`  ${colors.green}✓${colors.reset} ${testName} ${colors.dim}${result.message}${colors.reset}`);
+      } else {
+        console.log(`  ${colors.red}✗${colors.reset} ${testName} ${colors.red}${result.message}${colors.reset}`);
+        failures.push({
+          file: path.relative(process.cwd(), testFile),
+          message: result.message,
+        });
+      }
+    }
+    console.log('');
+  }
+
+  // Summary
+  console.log(`${colors.cyan}═══════════════════════════════════════════════════════════${colors.reset}`);
+  console.log(`${colors.cyan}Test Results:${colors.reset}`);
+  console.log(`  Total:  ${totalTests}`);
+  console.log(`  Passed: ${colors.green}${passedTests}${colors.reset}`);
+  console.log(`  Failed: ${passedTests === totalTests ? colors.green : colors.red}${totalTests - passedTests}${colors.reset}`);
+  console.log(`${colors.cyan}═══════════════════════════════════════════════════════════${colors.reset}\n`);
+
+  // Report failures
+  if (failures.length > 0) {
+    console.log(`${colors.red}❌ FAILED TESTS:${colors.reset}\n`);
+    for (const failure of failures) {
+      console.log(`${colors.red}✗${colors.reset} ${failure.file}`);
+      console.log(`  ${failure.message}\n`);
+    }
+    process.exit(1);
+  }
+
+  console.log(`${colors.green}✨ All tests passed!${colors.reset}\n`);
+  process.exit(0);
+}
+
+// Run
+main().catch((error) => {
+  console.error(`${colors.red}Fatal error:${colors.reset}`, error);
+  process.exit(1);
+});

package/test/test-cli-integration.sh

@@ -0,0 +1,159 @@
+#!/bin/bash
+# CLI Integration Tests for Agent Schema Validator
+# Tests the CLI wrapper (tools/validate-agent-schema.js) behavior and error handling
+# NOTE: Tests CLI functionality using temporary test fixtures
+
+echo "========================================"
+echo "CLI Integration Tests"
+echo "========================================"
+echo ""
+
+# Colors
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+PASSED=0
+FAILED=0
+
+# Get the repo root (assuming script is in test/ directory)
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+
+# Create temp directory for test fixtures
+TEMP_DIR=$(mktemp -d)
+cleanup() {
+  rm -rf "$TEMP_DIR"
+}
+trap cleanup EXIT
+
+# Test 1: CLI fails when no files found (exit 1)
+echo "Test 1: CLI fails when no agent files found (should exit 1)"
+mkdir -p "$TEMP_DIR/empty/src/core/agents"
+OUTPUT=$(node "$REPO_ROOT/tools/validate-agent-schema.js" "$TEMP_DIR/empty" 2>&1)
+EXIT_CODE=$?
+if [ $EXIT_CODE -eq 1 ] && echo "$OUTPUT" | grep -q "No agent files found"; then
+  echo -e "${GREEN}✓${NC} CLI fails correctly when no files found (exit 1)"
+  PASSED=$((PASSED + 1))
+else
+  echo -e "${RED}✗${NC} CLI failed to handle no files properly (exit code: $EXIT_CODE)"
+  FAILED=$((FAILED + 1))
+fi
+echo ""
+
+# Test 2: CLI reports validation errors with exit code 1
+echo "Test 2: CLI reports validation errors (should exit 1)"
+mkdir -p "$TEMP_DIR/invalid/src/core/agents"
+cat > "$TEMP_DIR/invalid/src/core/agents/bad.agent.yaml" << 'EOF'
+agent:
+  metadata:
+    id: bad
+    name: Bad
+    title: Bad
+    icon: 🧪
+  persona:
+    role: Test
+    identity: Test
+    communication_style: Test
+    principles: []
+  menu: []
+EOF
+OUTPUT=$(node "$REPO_ROOT/tools/validate-agent-schema.js" "$TEMP_DIR/invalid" 2>&1)
+EXIT_CODE=$?
+if [ $EXIT_CODE -eq 1 ] && echo "$OUTPUT" | grep -q "failed validation"; then
+  echo -e "${GREEN}✓${NC} CLI reports errors correctly (exit 1)"
+  PASSED=$((PASSED + 1))
+else
+  echo -e "${RED}✗${NC} CLI failed to report errors (exit code: $EXIT_CODE)"
+  FAILED=$((FAILED + 1))
+fi
+echo ""
+
+# Test 3: CLI discovers and counts agent files correctly
+echo "Test 3: CLI discovers and counts agent files"
+mkdir -p "$TEMP_DIR/valid/src/core/agents"
+cat > "$TEMP_DIR/valid/src/core/agents/test1.agent.yaml" << 'EOF'
+agent:
+  metadata:
+    id: test1
+    name: Test1
+    title: Test1
+    icon: 🧪
+  persona:
+    role: Test
+    identity: Test
+    communication_style: Test
+    principles: [Test]
+  menu:
+    - trigger: help
+      description: Help
+      action: help
+EOF
+cat > "$TEMP_DIR/valid/src/core/agents/test2.agent.yaml" << 'EOF'
+agent:
+  metadata:
+    id: test2
+    name: Test2
+    title: Test2
+    icon: 🧪
+  persona:
+    role: Test
+    identity: Test
+    communication_style: Test
+    principles: [Test]
+  menu:
+    - trigger: help
+      description: Help
+      action: help
+EOF
+OUTPUT=$(node "$REPO_ROOT/tools/validate-agent-schema.js" "$TEMP_DIR/valid" 2>&1)
+EXIT_CODE=$?
+if [ $EXIT_CODE -eq 0 ] && echo "$OUTPUT" | grep -q "Found 2 agent file"; then
+  echo -e "${GREEN}✓${NC} CLI discovers and counts files correctly"
+  PASSED=$((PASSED + 1))
+else
+  echo -e "${RED}✗${NC} CLI file discovery failed"
+  echo "Output: $OUTPUT"
+  FAILED=$((FAILED + 1))
+fi
+echo ""
+
+# Test 4: CLI provides detailed error messages
+echo "Test 4: CLI provides detailed error messages"
+OUTPUT=$(node "$REPO_ROOT/tools/validate-agent-schema.js" "$TEMP_DIR/invalid" 2>&1)
+if echo "$OUTPUT" | grep -q "Path:" && echo "$OUTPUT" | grep -q "Error:"; then
+  echo -e "${GREEN}✓${NC} CLI provides error details (Path and Error)"
+  PASSED=$((PASSED + 1))
+else
+  echo -e "${RED}✗${NC} CLI error details missing"
+  FAILED=$((FAILED + 1))
+fi
+echo ""
+
+# Test 5: CLI validates real BMAD agents (smoke test)
+echo "Test 5: CLI validates actual BMAD agents (smoke test)"
+OUTPUT=$(node "$REPO_ROOT/tools/validate-agent-schema.js" 2>&1)
+EXIT_CODE=$?
+if [ $EXIT_CODE -eq 0 ] && echo "$OUTPUT" | grep -qE "Found [0-9]+ agent file"; then
+  echo -e "${GREEN}✓${NC} CLI validates real BMAD agents successfully"
+  PASSED=$((PASSED + 1))
+else
+  echo -e "${RED}✗${NC} CLI failed on real BMAD agents (exit code: $EXIT_CODE)"
+  FAILED=$((FAILED + 1))
+fi
+echo ""
+
+# Summary
+echo "========================================"
+echo "Test Results:"
+echo "  Passed: ${GREEN}$PASSED${NC}"
+echo "  Failed: ${RED}$FAILED${NC}"
+echo "========================================"
+
+if [ $FAILED -eq 0 ]; then
+  echo -e "\n${GREEN}✨ All CLI integration tests passed!${NC}\n"
+  exit 0
+else
+  echo -e "\n${RED}❌ Some CLI integration tests failed${NC}\n"
+  exit 1
+fi

package/test/unit-test-schema.js

@@ -0,0 +1,133 @@
+/**
+ * Unit Tests for Agent Schema Edge Cases
+ *
+ * Tests internal functions to achieve 100% branch coverage
+ */
+
+const { validateAgentFile } = require('../tools/schema/agent.js');
+
+console.log('Running edge case unit tests...\n');
+
+let passed = 0;
+let failed = 0;
+
+// Test 1: Path with malformed module structure (no slash after module name)
+// This tests line 213: slashIndex === -1
+console.log('Test 1: Malformed module path (no slash after module name)');
+try {
+  const result = validateAgentFile('src/modules/bmm', {
+    agent: {
+      metadata: {
+        id: 'test',
+        name: 'Test',
+        title: 'Test',
+        icon: '🧪',
+      },
+      persona: {
+        role: 'Test',
+        identity: 'Test',
+        communication_style: 'Test',
+        principles: ['Test'],
+      },
+      menu: [{ trigger: 'help', description: 'Help', action: 'help' }],
+    },
+  });
+
+  if (result.success) {
+    console.log('✗ Should have failed - missing module field');
+    failed++;
+  } else {
+    console.log('✓ Correctly handled malformed path (treated as core agent)');
+    passed++;
+  }
+} catch (error) {
+  console.log('✗ Unexpected error:', error.message);
+  failed++;
+}
+console.log('');
+
+// Test 2: Module option with empty string
+// This tests line 222: trimmed.length > 0
+console.log('Test 2: Module agent with empty string in module field');
+try {
+  const result = validateAgentFile('src/modules/bmm/agents/test.agent.yaml', {
+    agent: {
+      metadata: {
+        id: 'test',
+        name: 'Test',
+        title: 'Test',
+        icon: '🧪',
+        module: '   ', // Empty after trimming
+      },
+      persona: {
+        role: 'Test',
+        identity: 'Test',
+        communication_style: 'Test',
+        principles: ['Test'],
+      },
+      menu: [{ trigger: 'help', description: 'Help', action: 'help' }],
+    },
+  });
+
+  if (result.success) {
+    console.log('✗ Should have failed - empty module string');
+    failed++;
+  } else {
+    console.log('✓ Correctly rejected empty module string');
+    passed++;
+  }
+} catch (error) {
+  console.log('✗ Unexpected error:', error.message);
+  failed++;
+}
+console.log('');
+
+// Test 3: Core agent path (src/core/agents/...) - tests the !filePath.startsWith(marker) branch
+console.log('Test 3: Core agent path returns null for module');
+try {
+  const result = validateAgentFile('src/core/agents/test.agent.yaml', {
+    agent: {
+      metadata: {
+        id: 'test',
+        name: 'Test',
+        title: 'Test',
+        icon: '🧪',
+        // No module field - correct for core agent
+      },
+      persona: {
+        role: 'Test',
+        identity: 'Test',
+        communication_style: 'Test',
+        principles: ['Test'],
+      },
+      menu: [{ trigger: 'help', description: 'Help', action: 'help' }],
+    },
+  });
+
+  if (result.success) {
+    console.log('✓ Core agent validated correctly (no module required)');
+    passed++;
+  } else {
+    console.log('✗ Core agent should pass without module field');
+    failed++;
+  }
+} catch (error) {
+  console.log('✗ Unexpected error:', error.message);
+  failed++;
+}
+console.log('');
+
+// Summary
+console.log('═══════════════════════════════════════');
+console.log('Edge Case Unit Test Results:');
+console.log(`  Passed: ${passed}`);
+console.log(`  Failed: ${failed}`);
+console.log('═══════════════════════════════════════\n');
+
+if (failed === 0) {
+  console.log('✨ All edge case tests passed!\n');
+  process.exit(0);
+} else {
+  console.log('❌ Some edge case tests failed\n');
+  process.exit(1);
+}