@ai-qa/workflow 2.0.10 → 2.0.13

package/scripts/healer.js

@@ -1,192 +1,114 @@
-const { DIRS, ensureDir, timestamp, log } = require('./utils');
+const { DIRS, log } = require('./utils');
+const context = require('./context-manager');
 const path = require('path');
 const fs = require('fs');
+const { execSync } = require('child_process');
 
-const KNOWN_BUG_PATTERNS = [
-  { pattern: /element\(s\) not found/i, type: 'selector' },
-  { pattern: /strict mode violation/i, type: 'selector' },
-  { pattern: /Target closed/i, type: 'environment' },
-  { pattern: /net::ERR_/i, type: 'environment' },
-  { pattern: /timeout/i, type: 'timing' },
-  { pattern: /page\.goto/i, type: 'navigation' },
-  { pattern: /Cannot find module/i, type: 'test-syntax' },
-];
-
-function classifyFailure(errorMessage) {
-  if (!errorMessage) return { type: 'unknown', fixable: false };
-  for (const rule of KNOWN_BUG_PATTERNS) {
-    if (rule.pattern.test(errorMessage)) {
-      return { type: rule.type, fixable: rule.type !== 'environment' };
-    }
-  }
-  return { type: 'unknown', fixable: false };
-}
-
-function selfHeal(runId, projectName, options = {}) {
-  const { maxAttempts = 2 } = options;
-
+function selfHeal(runId) {
   const runDir = runId
     ? path.join(DIRS.testResults, runId)
     : findLatestRunDir();
 
   if (!runDir || !fs.existsSync(runDir)) {
-    log('HEALER', 'No execution results found to heal');
-    return { healed: [], remaining: [], defects: [] };
+    log('RETRY', 'No execution results found');
+    return { rerun: [], stillFailing: [] };
   }
 
   const resultPath = path.join(runDir, 'execution-result.json');
   if (!fs.existsSync(resultPath)) {
-    log('HEALER', 'No execution result found');
-    return { healed: [], remaining: [], defects: [] };
+    log('RETRY', 'No execution result found');
+    return { rerun: [], stillFailing: [] };
   }
 
   const result = JSON.parse(fs.readFileSync(resultPath, 'utf-8'));
 
+  const storyName = result.story || null;
+
   if (result.success) {
-    log('HEALER', 'All tests passed - nothing to heal');
-    return { healed: [], remaining: [], defects: [] };
+    log('RETRY', 'All tests passed — nothing to retry');
+    return { rerun: [], stillFailing: [] };
   }
 
-  const screenshotDir = path.join(DIRS.testResults, 'screenshots', projectName || 'unknown');
-  ensureDir(screenshotDir);
-
-  const defects = [];
-  let remainingFailures = [...(result.failedTests || [])];
-
-  log('HEALER', `Found ${remainingFailures.length} failed test(s)`);
-  log('HEALER', `Screenshots will be saved to: test-results/screenshots/${projectName || 'unknown'}/`);
-
-  remainingFailures = remainingFailures.filter(failure => {
-    const classification = classifyFailure(failure.error);
-    const screenshotPath = path.join(screenshotDir, `${slugify(failure.test || 'failure')}.png`);
-
-    defects.push({
-      test: failure.test || failure.file,
-      error: failure.error ? failure.error.substring(0, 200) : 'Unknown',
-      classification: classification.type,
-      verdict: classification.fixable ? 'FIXABLE' : 'BUG',
-      action: classification.fixable
-        ? 'Attempting auto-heal'
-        : 'Cannot auto-fix — reporting as defect',
-      screenshot: screenshotPath,
-    });
-    log('HEALER', `  ${classification.fixable ? '⚠️' : '⛔'} ${failure.test || failure.file} — ${classification.type}${classification.fixable ? '' : ' (BUG)'}`);
-
-    if (!classification.fixable) return false;
-    return true;
-  });
-
-  const healingLog = [];
-  let attempt = 0;
-
-  while (attempt < maxAttempts && remainingFailures.length > 0) {
-    attempt++;
-    log('HEALER', `Healing attempt ${attempt}/${maxAttempts}`);
-
-    const healed = [];
-    const stillFailing = [];
-
-    for (const failure of remainingFailures) {
-      log('HEALER', `  Re-running: ${failure.test || failure.file}`);
-      const healedResult = attemptHealStrategy(failure, attempt, runDir);
-
-      if (healedResult.success) {
-        healed.push({ ...failure, healedOnAttempt: attempt, strategy: healedResult.strategy });
-        const defect = defects.find(d => d.test === failure.test);
-        if (defect) defect.healed = true;
-        log('HEALER', `  ✓ Healed: ${failure.test || failure.file}`);
-      } else {
-        stillFailing.push(failure);
-        log('HEALER', `  ✗ Still failing: ${failure.test || failure.file}`);
-        const defect = defects.find(d => d.test === failure.test);
-        if (defect) {
-          defect.verdict = 'UNSTABLE';
-          defect.action = 'Requires manual investigation';
-        }
-      }
-    }
+  const failedTests = result.failedTests || [];
+  log('RETRY', `Found ${failedTests.length} failed test(s)`);
 
-    healingLog.push({ attempt, healed, stillFailing, timestamp: new Date().toISOString() });
-    remainingFailures = stillFailing;
-  }
+  const rerun = [];
+  const stillFailing = [];
+  const healingAttempts = [];
+
+  for (const failure of failedTests) {
+    const testFile = failure.file || failure.test || '';
+    log('RETRY', `  Re-running: ${testFile}`);
 
-  remainingFailures.forEach(f => {
-    if (!defects.find(d => d.test === f.test)) {
-      defects.push({
-        test: f.test || f.file,
-        error: f.error ? f.error.substring(0, 200) : 'Failed after healing',
-        classification: 'unresolved',
-        verdict: 'UNSTABLE',
-        action: 'Requires manual investigation',
+    const startMs = Date.now();
+    const args = ['npx', 'playwright', 'test', testFile, '--reporter=list', '--timeout=60000'];
+
+    try {
+      execSync(args.join(' '), {
+        cwd: DIRS.ROOT,
+        encoding: 'utf-8',
+        timeout: 120000,
+        stdio: 'pipe',
       });
+      rerun.push({ test: testFile, status: 'passed' });
+      log('RETRY', `  ✓ Passed on retry: ${testFile}`);
+      healingAttempts.push({ test: testFile, status: 'passed', duration_ms: Date.now() - startMs, strategy: 'timeout_retry' });
+    } catch {
+      stillFailing.push({ test: testFile, status: 'failed', error: failure.error || '' });
+      log('RETRY', `  ✗ Still failing: ${testFile}`);
+      healingAttempts.push({ test: testFile, status: 'failed', duration_ms: Date.now() - startMs, strategy: 'timeout_retry' });
     }
-  });
+  }
 
   const healingReport = {
-    projectName: projectName || 'unknown',
-    screenshotDir: `test-results/screenshots/${projectName || 'unknown'}/`,
     runId: path.basename(runDir),
-    originalResult: { total: result.failedTests ? result.failedTests.length : 0 },
-    healingAttempts: healingLog,
-    totalHealed: healingLog.reduce((sum, a) => sum + a.healed.length, 0),
-    totalRemaining: remainingFailures.length,
-    totalDefects: defects.length,
-    defects,
+    story: storyName,
+    totalAttempts: healingAttempts.length,
+    totalHealed: rerun.length,
+    remainingFailures: stillFailing.length,
+    healingAttempts,
+    healedTests: rerun.map(r => ({ test: r.test, status: 'passed', healedOnAttempt: 1, strategy: 'timeout_retry' })),
+    stillFailingTests: stillFailing.map(f => ({ test: f.test, status: 'failed', error: f.error })),
     timestamp: new Date().toISOString(),
-    finalStatus: remainingFailures.length === 0 && defects.filter(d => d.verdict !== 'FIXABLE').length === 0 ? 'all-passed'
-      : remainingFailures.length === 0 ? 'all-passed-with-defects'
-      : 'partial',
   };
 
-  const reportPath = path.join(runDir, 'healing-report.json');
-  fs.writeFileSync(reportPath, JSON.stringify(healingReport, null, 2), 'utf-8');
-
-  log('HEALER', `Healing complete:`);
-  log('HEALER', `  Healed: ${healingReport.totalHealed}`);
-  log('HEALER', `  Defects found: ${defects.filter(d => d.verdict === 'BUG').length}`);
-  log('HEALER', `  Screenshots: test-results/screenshots/${projectName || 'unknown'}/`);
-
-  if (defects.length > 0) {
-    console.log('\n  Defects:');
-    defects.forEach(d => {
-      const icon = d.verdict === 'BUG' ? '⛔ BUG' : d.verdict === 'UNSTABLE' ? '⚠️ UNSTABLE' : '✓ FIXED';
-      console.log(`    ${icon} ${d.test}`);
-      console.log(`       → ${d.classification}: ${d.error.substring(0, 100)}`);
-      console.log(`       → Screenshot: ${d.screenshot}`);
-    });
-  }
+  writeHealingReport(runDir, healingReport);
 
-  return { healed: healingLog.flatMap(a => a.healed), remaining: remainingFailures, defects, reportPath };
-}
+  if (storyName) {
+    context.phaseComplete('heal', storyName);
+    if (rerun.length > 0) context.setHealed(storyName, path.basename(runDir));
+  }
 
-function attemptHealStrategy(failure, attempt, runDir) {
-  const strategies = [
-    { name: 'retry-standard' },
-    { name: 'retry-longer-timeout', timeout: 60000 },
-  ];
-
-  const strategy = strategies[Math.min(attempt - 1, strategies.length - 1)];
-  const testFile = failure.file || failure.test || '';
-
-  try {
-    const args = ['npx', 'playwright', 'test', testFile, '--reporter=list'];
-    if (strategy.timeout) args.push(`--timeout=${strategy.timeout}`);
-
-    require('child_process').execSync(args.join(' '), {
-      cwd: DIRS.ROOT,
-      encoding: 'utf-8',
-      timeout: 120000,
-      stdio: 'pipe',
-    });
-
-    return { success: true, strategy: strategy.name };
-  } catch {
-    return { success: false, strategy: strategy.name };
+  console.log('');
+  console.log('╔════════════════════════════════════════════════════════════════╗');
+  console.log('║  DEEPER DIAGNOSIS is handled by the AI agent                 ║');
+  console.log('║                                                               ║');
+  console.log('║  The AI reads playwright-test-healer.agent.md, debugs with   ║');
+  console.log('║  Playwright MCP (test_debug, browser_snapshot, etc.), and    ║');
+  console.log('║  fixes selectors/timing issues or marks real bugs.           ║');
+  console.log('║                                                               ║');
+  console.log('║  Tell your AI agent:                                          ║');
+  console.log('║  "Debug and heal the failing tests in test-results/' + path.basename(runDir) + '"');
+  console.log('╚════════════════════════════════════════════════════════════════╝');
+  console.log('');
+
+  if (rerun.length > 0) {
+    console.log(`  Re-ran ${rerun.length} test(s) with longer timeout`);
+    console.log(`  Passed on retry: ${rerun.length}`);
+  }
+  if (stillFailing.length > 0) {
+    console.log(`  Still failing: ${stillFailing.length}`);
+    stillFailing.forEach(f => console.log(`    - ${f.test}`));
   }
+  console.log('');
+
+  return { rerun, stillFailing };
 }
 
-function slugify(text) {
-  return text.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '').substring(0, 80);
+function writeHealingReport(runDir, report) {
+  const reportPath = path.join(runDir, 'healing-report.json');
+  fs.writeFileSync(reportPath, JSON.stringify(report, null, 2), 'utf-8');
+  log('RETRY', `Healing report saved: ${reportPath}`);
 }
 
 function findLatestRunDir() {
@@ -200,8 +122,7 @@ function findLatestRunDir() {
 
 if (require.main === module) {
   const runId = process.argv[2];
-  const projectName = process.argv[3];
-  selfHeal(runId, projectName);
+  selfHeal(runId);
 }
 
 module.exports = { selfHeal };

package/scripts/planner.js

@@ -1,142 +1,24 @@
-const { DIRS, CONFIG, parseUserStory, readMarkdown, writeMarkdown, slugify, log } = require('./utils');
-const path = require('path');
-
-function generateTestPlan(storyName) {
-  const storyPath = path.join(DIRS.userStory, storyName);
-  if (!require('fs').existsSync(storyPath)) {
-    console.error(`User story not found: ${storyPath}`);
-    console.log(`Available stories: ${require('./utils').listUserStories().join(', ')}`);
-    process.exit(1);
-  }
-
-  const story = parseUserStory(storyPath);
-  const planId = slugify(story.id || story.title || storyName.replace('.md', ''));
-  const planPath = path.join(DIRS.specs, `${planId}-test-plan.md`);
-
-  log('PLANNER', `Generating test plan from: ${storyName}`);
-  log('PLANNER', `Story: ${story.metaTitle || story.title}`);
-
-  const plan = generatePlanContent(story, planId);
-
-  writeMarkdown(planPath, plan);
-  log('PLANNER', `Test plan saved: specs/${planId}-test-plan.md`);
-
-  return { planId, planPath };
-}
-
-function generatePlanContent(story, planId) {
-  const date = new Date().toISOString().split('T')[0];
-
-  let plan = `# Test Plan: ${story.metaTitle || story.title}
-
-**ID**: PLAN-${(story.id || planId).toUpperCase()}
-**Feature**: ${story.title || story.metaTitle || 'N/A'}
-**Status**: Draft
-**User Story Ref**: ${story.id || 'N/A'}
-**Generated**: ${date}
-
----
-
-## 1. Overview
-
-${story.description || 'Automated test plan derived from user story.'}
-
----
-
-## 2. Preconditions
-
-`;
-
-  if (story.preconditions && story.preconditions.length > 0) {
-    story.preconditions.forEach(p => { plan += `- ${p}\n`; });
-  } else {
-    plan += `- Environment: ${CONFIG.project.url}\n`;
-    plan += `- Browser: ${CONFIG.browser.type === 'edge' ? 'Microsoft Edge via CDP' : 'Chromium'} (Port ${CONFIG.browser.cdpPort})\n`;
-    plan += `- User credentials as specified in user story\n`;
-  }
-
-  plan += `
-
----
-
-## 3. Test Scenarios
-
-`;
-
-  if (story.acceptanceCriteria) {
-    const sections = story.acceptanceCriteria.split(/(?=###\s+\d)/);
-    sections.forEach((section, idx) => {
-      const trimmed = section.trim();
-      if (!trimmed) return;
-
-      const titleMatch = trimmed.match(/###\s+\d+\.\s*(.+?)(?:\n|$)/);
-      const title = titleMatch ? titleMatch[1].trim() : `Scenario ${idx + 1}`;
-
-      const steps = parseSteps(trimmed);
-
-      plan += `### ${idx + 3}.1 Scenario: ${title}\n`;
-      plan += `**Steps**:\n`;
-      steps.forEach((s, i) => { plan += `${i + 1}. ${s}\n`; });
-      plan += `\n**Expected Results**:\n`;
-      plan += `- Feature behaves as described in acceptance criteria\n`;
-      plan += `- No errors or unexpected behavior\n`;
-      plan += `\n---\n\n`;
-    });
-  } else {
-    plan += `### 3.1 Scenario: Happy Path\n`;
-    plan += `**Steps**:\n`;
-    plan += `1. Login with the specified user credentials\n`;
-    plan += `2. Navigate to the feature\n`;
-    plan += `3. Execute the primary workflow\n`;
-    plan += `4. Verify success state\n\n`;
-    plan += `**Expected Results**:\n`;
-    plan += `- Workflow completes successfully\n\n`;
-    plan += `---\n\n`;
-  }
-
-  plan += `## 4. Success Criteria\n`;
-  plan += `- All scenarios pass without errors\n`;
-  plan += `- Screenshots are captured for key verification points\n`;
-  plan += `- Allure report is generated with test results\n\n`;
-
-  plan += `---\n\n`;
-  plan += `> **AI Instructions**: Review this test plan. Verify scenarios cover all acceptance criteria.\n`;
-  plan += `> Add edge cases, negative scenarios, and boundary tests where applicable.\n`;
-  plan += `> Update status from "Draft" to "Reviewed" once validated.\n`;
-
-  return plan;
-}
-
-function parseSteps(text) {
-  const steps = [];
-  const lines = text.split('\n');
-  let inSteps = false;
-
-  for (const line of lines) {
-    const trimmed = line.trim();
-    if (/^\d+\.\s+/.test(trimmed)) {
-      inSteps = true;
-      steps.push(trimmed.replace(/^\d+\.\s+/, ''));
-    } else if (inSteps && trimmed.startsWith('-')) {
-      steps.push(trimmed.replace(/^- /, ''));
-    } else if (inSteps && trimmed === '') {
-      continue;
-    } else if (inSteps) {
-      break;
-    }
-  }
-
-  return steps.length > 0 ? steps : ['Execute the workflow as described in the user story.'];
+function generateTestPlan() {
+  console.log('');
+  console.log('╔══════════════════════════════════════════════════════════╗');
+  console.log('║  PLANNING is handled by the AI agent                   ║');
+  console.log('║                                                         ║');
+  console.log('║  The AI reads playwright-test-planner.agent.md,         ║');
+  console.log('║  explores your app with Playwright MCP, and writes      ║');
+  console.log('║  the test plan directly to specs/.                      ║');
+  console.log('║                                                         ║');
+  console.log('║  The AI will STOP and ask for your approval before      ║');
+  console.log('║  proceeding to test generation.                         ║');
+  console.log('║                                                         ║');
+  console.log('║  Tell your AI agent:                                    ║');
+  console.log('║  "Read router.md and plan tests for my-story.md"        ║');
+  console.log('╚══════════════════════════════════════════════════════════╝');
+  console.log('');
+  process.exit(0);
 }
 
 if (require.main === module) {
-  const storyName = process.argv[2];
-  if (!storyName) {
-    console.log('Usage: node scripts/planner.js <user-story-file.md>');
-    console.log(`Available: ${require('./utils').listUserStories().join(', ')}`);
-    process.exit(1);
-  }
-  generateTestPlan(storyName);
+  generateTestPlan();
 }
 
 module.exports = { generateTestPlan };

package/scripts/reporter.js

@@ -1,4 +1,5 @@
 const { DIRS, ensureDir, readMarkdown, writeMarkdown, log } = require('./utils');
+const context = require('./context-manager');
 const path = require('path');
 const fs = require('fs');
 
@@ -26,6 +27,13 @@ function generateReport(runId) {
   const resultPath = path.join(runDir, 'execution-result.json');
   if (fs.existsSync(resultPath)) {
     report.execution = JSON.parse(fs.readFileSync(resultPath, 'utf-8'));
+    report.storyName = report.execution.story || null;
+  }
+
+  if (report.storyName) {
+    const trace = context.getTraceability();
+    const storyTrace = trace.stories[report.storyName];
+    if (storyTrace) report.traceability = storyTrace;
   }
 
   const healingPath = path.join(runDir, 'healing-report.json');
@@ -56,6 +64,10 @@ function generateReport(runId) {
 
   log('REPORTER', `Report saved: test-results/${path.basename(runDir)}/final-test-report.md`);
 
+  if (report.storyName) {
+    context.phaseComplete('report', report.storyName);
+  }
+
   return {
     reportPath,
     stats: report.stats,
@@ -77,13 +89,21 @@ function listTestFilesWithStatus() {
 }
 
 function generateMarkdownReport(report) {
-  const { runId, stats, execution, healing, testFiles } = report;
+  const { runId, stats, execution, healing, testFiles, storyName, traceability } = report;
 
   let md = `# AI QA Execution Report
 
 **Run ID**: ${runId}
 **Generated**: ${report.generatedAt}
 **Status**: ${stats.failed === 0 ? '✅ PASSED' : '❌ FAILED'}
+${storyName ? `**Story**: ${storyName}` : ''}
+
+---
+
+## Traceability
+
+${storyName && traceability ? `| Story | Plan | Spec | Status |\n|---|---|---|---|\n| ${storyName} | ${traceability.plan || '—'} | ${traceability.spec || '—'} | ${traceability.status || '—'} |\n` : `No traceability data available.\n`}
+${traceability && traceability.runs && traceability.runs.length > 1 ? `\n**Run History:** ${traceability.runs.length} run(s)\n` : ''}
 
 ---
 

package/scripts/utils.js

@@ -159,6 +159,8 @@ const DIRS = {
   scratch: path.join(ROOT, 'scratch'),
   docs: path.join(ROOT, 'docs'),
   scripts: path.join(ROOT, 'scripts'),
+  qaContext: path.join(ROOT, '.qa-context'),
+  auth: path.join(ROOT, '.auth'),
 };
 
 function ensureDir(dir) {