agentic-qe 3.3.3 → 3.3.5

package/scripts/fetch-content.js

@@ -0,0 +1,460 @@
+#!/usr/bin/env node
+/**
+ * Fetch Content CLI - 5-Tier Browser Cascade with Per-Tier Timeouts
+ *
+ * Single entry point for all browser content fetching operations.
+ * Automatically cascades through tiers with 30s timeout each.
+ *
+ * Usage:
+ *   node scripts/fetch-content.js <URL> <OUTPUT_DIR> [OPTIONS]
+ *
+ * Options:
+ *   --timeout <ms>      Per-tier timeout (default: 30000)
+ *   --skip-tiers <list> Comma-separated tiers to skip
+ *   --locale <locale>   Browser locale (default: en-US)
+ *
+ * Output:
+ *   - <OUTPUT_DIR>/content.html       Fetched HTML content
+ *   - <OUTPUT_DIR>/screenshot.png     Page screenshot
+ *   - <OUTPUT_DIR>/fetch-result.json  Structured result with metadata
+ *
+ * Exit codes:
+ *   0 - Success
+ *   1 - All tiers failed
+ *   2 - Invalid arguments
+ *
+ * @version 1.0.0
+ * @since v3
+ */
+
+const { exec, execSync, spawn } = require('child_process');
+const { promisify } = require('util');
+const fs = require('fs');
+const path = require('path');
+
+const execAsync = promisify(exec);
+
+// ============================================================================
+// Configuration
+// ============================================================================
+
+const DEFAULT_TIER_TIMEOUT = 30000; // 30 seconds per tier
+const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36';
+
+const TIERS = [
+  'vibium',
+  'playwright-stealth',
+  'http-fetch',
+  'websearch-fallback'
+];
+
+// ============================================================================
+// Argument Parsing
+// ============================================================================
+
+function parseArgs() {
+  const args = process.argv.slice(2);
+
+  if (args.length < 2) {
+    console.error('Usage: node fetch-content.js <URL> <OUTPUT_DIR> [OPTIONS]');
+    console.error('');
+    console.error('Options:');
+    console.error('  --timeout <ms>      Per-tier timeout (default: 30000)');
+    console.error('  --skip-tiers <list> Comma-separated tiers to skip');
+    console.error('  --locale <locale>   Browser locale (default: en-US)');
+    process.exit(2);
+  }
+
+  const options = {
+    url: args[0],
+    outputDir: args[1],
+    timeout: DEFAULT_TIER_TIMEOUT,
+    skipTiers: [],
+    locale: 'en-US',
+    userAgent: DEFAULT_USER_AGENT
+  };
+
+  for (let i = 2; i < args.length; i++) {
+    switch (args[i]) {
+      case '--timeout':
+        options.timeout = parseInt(args[++i], 10);
+        break;
+      case '--skip-tiers':
+        options.skipTiers = args[++i].split(',').map(t => t.trim());
+        break;
+      case '--locale':
+        options.locale = args[++i];
+        break;
+    }
+  }
+
+  return options;
+}
+
+// ============================================================================
+// Utility Functions
+// ============================================================================
+
+function log(tier, message, data = {}) {
+  const entry = {
+    timestamp: new Date().toISOString(),
+    tier,
+    message,
+    ...data
+  };
+  console.error(`[${entry.timestamp}] [${tier}] ${message}`);
+  return entry;
+}
+
+function withTimeout(promise, ms, tierName) {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      reject(new Error(`${tierName} timed out after ${ms}ms`));
+    }, ms);
+
+    promise
+      .then(result => {
+        clearTimeout(timer);
+        resolve(result);
+      })
+      .catch(err => {
+        clearTimeout(timer);
+        reject(err);
+      });
+  });
+}
+
+// ============================================================================
+// Tier Implementations
+// ============================================================================
+
+/**
+ * Tier 1: Vibium MCP Browser
+ * Note: Vibium requires MCP context - skip in CLI mode
+ */
+async function fetchWithVibium(options) {
+  log('vibium', 'Skipping - requires MCP context');
+  throw new Error('Vibium requires MCP context, not available in CLI');
+}
+
+/**
+ * Tier 2: Playwright + Stealth (PRIMARY for CLI)
+ */
+async function fetchWithPlaywrightStealth(options) {
+  log('playwright-stealth', 'Starting fetch', { url: options.url });
+
+  const workDir = path.join(options.outputDir, '.playwright-work');
+  fs.mkdirSync(workDir, { recursive: true });
+
+  // Check if playwright is installed globally or locally
+  const packageJsonPath = path.join(workDir, 'package.json');
+  if (!fs.existsSync(path.join(workDir, 'node_modules', 'playwright-extra'))) {
+    fs.writeFileSync(packageJsonPath, JSON.stringify({ name: 'pw-fetch', type: 'commonjs' }));
+
+    log('playwright-stealth', 'Installing dependencies...');
+    try {
+      execSync('npm install playwright-extra puppeteer-extra-plugin-stealth playwright 2>/dev/null', {
+        cwd: workDir,
+        stdio: 'pipe',
+        timeout: 60000
+      });
+    } catch (e) {
+      log('playwright-stealth', 'Install warning', { error: e.message });
+    }
+  }
+
+  const scriptPath = path.join(workDir, 'fetch.js');
+  const contentPath = path.join(options.outputDir, 'content.html');
+  const screenshotPath = path.join(options.outputDir, 'screenshot.png');
+
+  const script = `
+const { chromium } = require('playwright-extra');
+const stealth = require('puppeteer-extra-plugin-stealth')();
+chromium.use(stealth);
+
+(async () => {
+  const browser = await chromium.launch({
+    headless: true,
+    args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu']
+  });
+
+  const context = await browser.newContext({
+    userAgent: ${JSON.stringify(options.userAgent)},
+    viewport: { width: 1920, height: 1080 },
+    locale: ${JSON.stringify(options.locale)}
+  });
+
+  const page = await context.newPage();
+
+  try {
+    await page.goto(${JSON.stringify(options.url)}, {
+      waitUntil: 'domcontentloaded',
+      timeout: ${options.timeout - 5000}
+    });
+
+    await page.waitForTimeout(2000);
+
+    // Dismiss cookie banners
+    const cookieSelectors = [
+      '[data-testid="consent-accept-all"]',
+      '#onetrust-accept-btn-handler',
+      'button[id*="accept"]',
+      '[class*="cookie"] button',
+      'button:has-text("Accept")',
+      'button:has-text("Acceptă")'
+    ];
+
+    for (const sel of cookieSelectors) {
+      try {
+        const btn = await page.$(sel);
+        if (btn) {
+          await btn.click();
+          await page.waitForTimeout(1000);
+          break;
+        }
+      } catch (e) {}
+    }
+
+    const content = await page.content();
+    require('fs').writeFileSync(${JSON.stringify(contentPath)}, content);
+    await page.screenshot({ path: ${JSON.stringify(screenshotPath)} });
+
+    console.log(JSON.stringify({
+      success: true,
+      size: content.length,
+      title: await page.title(),
+      url: page.url()
+    }));
+  } finally {
+    await browser.close();
+  }
+})().catch(e => {
+  console.log(JSON.stringify({ success: false, error: e.message }));
+  process.exit(1);
+});
+`;
+
+  fs.writeFileSync(scriptPath, script);
+
+  const { stdout } = await execAsync(`node "${scriptPath}"`, {
+    cwd: workDir,
+    timeout: options.timeout
+  });
+
+  const result = JSON.parse(stdout.trim());
+  if (!result.success) {
+    throw new Error(result.error);
+  }
+
+  const content = fs.readFileSync(contentPath, 'utf8');
+
+  return {
+    success: true,
+    content,
+    contentSize: content.length,
+    screenshotPath,
+    metadata: { title: result.title, finalUrl: result.url }
+  };
+}
+
+/**
+ * Tier 3: HTTP Fetch (for simple/static sites)
+ */
+async function fetchWithHttp(options) {
+  log('http-fetch', 'Starting fetch', { url: options.url });
+
+  const response = await fetch(options.url, {
+    headers: {
+      'User-Agent': options.userAgent,
+      'Accept': 'text/html,application/xhtml+xml,*/*',
+      'Accept-Language': `${options.locale},en;q=0.5`
+    },
+    signal: AbortSignal.timeout(options.timeout)
+  });
+
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+  }
+
+  const content = await response.text();
+
+  if (content.length < 500) {
+    throw new Error('HTTP fetch returned insufficient content (bot blocked?)');
+  }
+
+  const contentPath = path.join(options.outputDir, 'content.html');
+  fs.writeFileSync(contentPath, content);
+
+  return {
+    success: true,
+    content,
+    contentSize: content.length,
+    metadata: { statusCode: response.status }
+  };
+}
+
+/**
+ * Tier 4: WebSearch Fallback (research-based, degraded mode)
+ */
+async function fetchWithWebSearch(options) {
+  log('websearch-fallback', 'Creating research-based content', { url: options.url });
+
+  // This tier creates a placeholder indicating manual research is needed
+  const urlObj = new URL(options.url);
+  const domain = urlObj.hostname;
+
+  const researchContent = `
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Research Required: ${options.url}</title>
+  <meta name="fetch-method" content="websearch-fallback">
+  <meta name="fetch-status" content="degraded">
+  <meta name="original-url" content="${options.url}">
+  <meta name="fetch-date" content="${new Date().toISOString()}">
+</head>
+<body>
+  <h1>Manual Research Required</h1>
+  <p><strong>Status:</strong> All automated fetch tiers failed.</p>
+  <p><strong>Target URL:</strong> ${options.url}</p>
+  <p><strong>Domain:</strong> ${domain}</p>
+  <hr>
+  <h2>Recommended Actions:</h2>
+  <ol>
+    <li>Use WebSearch tool to research "${domain} features functionality"</li>
+    <li>Check if site requires authentication or has strict bot protection</li>
+    <li>Consider manual browser inspection</li>
+  </ol>
+  <p><em>This is a degraded fallback. Analysis based on this content will be limited.</em></p>
+</body>
+</html>
+`;
+
+  const contentPath = path.join(options.outputDir, 'content.html');
+  fs.writeFileSync(contentPath, researchContent);
+
+  return {
+    success: true,
+    content: researchContent,
+    contentSize: researchContent.length,
+    metadata: {
+      note: 'Degraded mode - research-based content',
+      manualResearchRequired: true
+    }
+  };
+}
+
+// ============================================================================
+// Tier Dispatcher
+// ============================================================================
+
+const TIER_HANDLERS = {
+  'vibium': fetchWithVibium,
+  'playwright-stealth': fetchWithPlaywrightStealth,
+  'http-fetch': fetchWithHttp,
+  'websearch-fallback': fetchWithWebSearch
+};
+
+// ============================================================================
+// Main Cascade Logic
+// ============================================================================
+
+async function fetchWithCascade(options) {
+  fs.mkdirSync(options.outputDir, { recursive: true });
+
+  const result = {
+    success: false,
+    url: options.url,
+    tier: null,
+    status: 'failed',
+    content: null,
+    contentSize: 0,
+    screenshotPath: null,
+    tiersAttempted: [],
+    tierErrors: {},
+    metadata: {}
+  };
+
+  const tiersToTry = TIERS.filter(t => !options.skipTiers.includes(t));
+
+  for (const tier of tiersToTry) {
+    result.tiersAttempted.push(tier);
+    log(tier, `Attempting fetch (timeout: ${options.timeout}ms)`);
+
+    try {
+      const handler = TIER_HANDLERS[tier];
+      const tierResult = await withTimeout(
+        handler(options),
+        options.timeout,
+        tier
+      );
+
+      if (tierResult.success && tierResult.content) {
+        result.success = true;
+        result.tier = tier;
+        result.status = tier === 'websearch-fallback' ? 'degraded' : 'success';
+        result.content = tierResult.content;
+        result.contentSize = tierResult.contentSize;
+        result.screenshotPath = tierResult.screenshotPath || null;
+        result.metadata = tierResult.metadata || {};
+
+        log(tier, `SUCCESS: ${result.contentSize} bytes`);
+        break;
+      }
+    } catch (error) {
+      const errorMsg = error.message || String(error);
+      result.tierErrors[tier] = errorMsg;
+      log(tier, `FAILED: ${errorMsg}`);
+    }
+  }
+
+  // Save result metadata
+  const resultPath = path.join(options.outputDir, 'fetch-result.json');
+  const resultData = { ...result };
+  delete resultData.content; // Don't duplicate content in JSON
+  fs.writeFileSync(resultPath, JSON.stringify(resultData, null, 2));
+
+  return result;
+}
+
+// ============================================================================
+// Entry Point
+// ============================================================================
+
+async function main() {
+  const options = parseArgs();
+
+  console.error('');
+  console.error('╔════════════════════════════════════════════════════════════════╗');
+  console.error('║              FETCH CONTENT - 5-TIER CASCADE                    ║');
+  console.error('╠════════════════════════════════════════════════════════════════╣');
+  console.error(`║  URL: ${options.url.substring(0, 54).padEnd(54)} ║`);
+  console.error(`║  Output: ${options.outputDir.substring(0, 51).padEnd(51)} ║`);
+  console.error(`║  Timeout per tier: ${options.timeout}ms                                  ║`);
+  console.error('╚════════════════════════════════════════════════════════════════╝');
+  console.error('');
+
+  try {
+    const result = await fetchWithCascade(options);
+
+    // Output final result to stdout (for programmatic use)
+    console.log(JSON.stringify({
+      success: result.success,
+      tier: result.tier,
+      status: result.status,
+      contentSize: result.contentSize,
+      screenshotPath: result.screenshotPath,
+      contentPath: path.join(options.outputDir, 'content.html'),
+      resultPath: path.join(options.outputDir, 'fetch-result.json'),
+      tiersAttempted: result.tiersAttempted,
+      tierErrors: result.tierErrors
+    }, null, 2));
+
+    process.exit(result.success ? 0 : 1);
+  } catch (error) {
+    console.error(`Fatal error: ${error.message}`);
+    process.exit(1);
+  }
+}
+
+main();

package/scripts/merge-v3-to-root.sql

@@ -0,0 +1,48 @@
+-- Merge V3 unique data to Root database
+-- Run with: sqlite3 .agentic-qe/memory.db < scripts/merge-v3-to-root.sql
+
+-- Attach V3 database
+ATTACH DATABASE 'v3/.agentic-qe/memory.db' AS v3db;
+
+-- ============================================================================
+-- Merge SONA Patterns (schema compatible)
+-- ============================================================================
+INSERT OR IGNORE INTO sona_patterns
+SELECT * FROM v3db.sona_patterns
+WHERE id NOT IN (SELECT id FROM sona_patterns);
+
+-- ============================================================================
+-- Merge GOAP Actions (explicit columns due to schema difference)
+-- V3 has extra 'qe_domain' column, root has 'duration_estimate' vs 'estimated_duration_ms'
+-- ============================================================================
+INSERT OR IGNORE INTO goap_actions (
+  id, name, description, agent_type, preconditions, effects,
+  cost, duration_estimate, success_rate, execution_count, category,
+  created_at, updated_at
+)
+SELECT
+  id, name, description, agent_type, preconditions, effects,
+  cost, estimated_duration_ms, success_rate, execution_count, category,
+  created_at, updated_at
+FROM v3db.goap_actions
+WHERE id NOT IN (SELECT id FROM goap_actions);
+
+-- ============================================================================
+-- Merge KV Store entries (schema compatible - both have same columns)
+-- ============================================================================
+INSERT OR IGNORE INTO kv_store (key, namespace, value, created_at, expires_at)
+SELECT key, namespace, value, created_at, expires_at
+FROM v3db.kv_store
+WHERE namespace || ':' || key NOT IN (SELECT namespace || ':' || key FROM kv_store);
+
+-- ============================================================================
+-- Verify merge counts
+-- ============================================================================
+SELECT 'sona_patterns' as table_name, COUNT(*) as count FROM sona_patterns
+UNION ALL
+SELECT 'goap_actions', COUNT(*) FROM goap_actions
+UNION ALL
+SELECT 'kv_store', COUNT(*) FROM kv_store;
+
+-- Detach
+DETACH DATABASE v3db;

package/v3/CHANGELOG.md

@@ -5,6 +5,172 @@ All notable changes to Agentic QE will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.3.5] - 2026-01-30
+
+### 🎯 Highlights
+
+**QE Queen MCP-Powered Orchestration** - Complete rewrite of `qe-queen-coordinator` to use MCP tools for real fleet coordination. Queen now actually spawns agents via `mcp__agentic-qe__agent_spawn`, monitors task completion, and stores learnings - instead of just describing what agents would do.
+
+**Unified Database Architecture** - All databases consolidated to single `{project-root}/.agentic-qe/memory.db` with automatic project root detection. Eliminates scattered database files and ensures consistent data storage.
+
+**252 New Tests for Coordination Module** - Comprehensive test coverage for previously untested consensus providers, protocols, services, and cross-domain router.
+
+### Added
+
+#### QE Queen MCP-Powered Orchestration (v3.1.0)
+- **Mandatory 10-phase execution protocol** - fleet_init → memory_store
+- **Real agent spawning** via `mcp__agentic-qe__agent_spawn`
+- **Task monitoring loop** - polls `task_list` until completion
+- **Learning persistence** - stores patterns after each orchestration
+- **Task-to-domain routing table** - automatic agent selection by task type
+- **MCP tools reference** - fleet, agent, task, QE, and memory operations
+- **Execution examples** - comprehensive and coverage-specific
+
+#### Unified Database Architecture
+- **Project root detection** - finds nearest package.json/git root
+- **Single memory.db** - all tables in one SQLite database
+- **Automatic migration** - moves data from scattered locations
+- **Cross-phase memory hooks** - auto-installed on `aqe init`
+
+#### New Test Coverage (252 tests, all passing)
+- **consensus/providers/** - 6 provider test files
+  - `claude-provider.test.ts` (366 lines)
+  - `gemini-provider.test.ts` (391 lines)
+  - `native-learning-provider.test.ts` (500 lines)
+  - `ollama-provider.test.ts` (440 lines)
+  - `openai-provider.test.ts` (373 lines)
+  - `openrouter-provider.test.ts` (393 lines)
+- **protocols/** - 4 protocol test files
+  - `defect-investigation.test.ts` (618 lines)
+  - `learning-consolidation.test.ts` (594 lines)
+  - `morning-sync.test.ts` (853 lines)
+  - `quality-gate.test.ts` (727 lines)
+- **services/** - 2 service test files
+  - `task-audit-logger.test.ts` (611 lines)
+  - `index.test.ts` (103 lines)
+- **cross-domain-router.test.ts** (686 lines)
+
+### Changed
+
+#### QE Queen Coordinator
+- Upgraded from v3.0.0 to v3.1.0
+- Now uses MCP tools instead of descriptions
+- Added prohibited behaviors section
+- Added domain topology diagram
+
+#### CLI Hook Commands
+- Updated to use `aqe` binary instead of `npx`
+- Implemented missing CLI hook commands for Claude Code integration
+
+### Fixed
+
+- **CI timeout** - Increased Fast Tests timeout from 5m to 10m
+- **Workflow permissions** - Added permissions block to sauce-demo-e2e workflow
+- **Hook commands** - Fixed CLI hook commands to use correct binary
+
+---
+
+## [3.3.4] - 2026-01-29
+
+### 🎯 Highlights
+
+**QCSD Ideation Phase Complete** - Full Quality Conscious Software Delivery (QCSD) Ideation phase implementation with HTSM v6.3 quality criteria analysis, SFDIPOT product factors assessment, and cross-phase memory feedback loops.
+
+**Cross-Phase Memory System** - New persistent memory architecture enabling automated learning between QCSD phases (Production→Ideation, Production→Grooming, CI/CD→Development, Development→Grooming).
+
+**Comprehensive Test Coverage** - 358 files changed with 83,990+ lines of new tests across all 12 domains, kernel, MCP handlers, routing, and workers.
+
+### Added
+
+#### QCSD Ideation Phase Agents
+- **qe-quality-criteria-recommender** - HTSM v6.3 quality criteria analysis with 10 categories
+- **qe-product-factors-assessor** - SFDIPOT framework with 7 factors, 37 subcategories
+- **qe-risk-assessor** - Multi-factor risk scoring with mitigation recommendations
+- **qe-test-idea-rewriter** - Transform passive "Verify X" patterns to active test actions
+
+#### Cross-Phase Memory System
+- **CrossPhaseMemoryService** - File-based persistence for QCSD feedback loops
+- **Cross-phase MCP handlers** - 8 new tools for signal storage/retrieval
+- **Hook executor** - Automatic trigger of cross-phase hooks on agent completion
+- **4 feedback loops** - Strategic, Tactical, Operational, Quality Criteria
+
+#### New Skills
+- **a11y-ally** - Comprehensive WCAG accessibility audit with video caption generation
+- **qcsd-ideation-swarm** - Multi-agent swarm for QCSD Ideation phase
+- **skills-manifest.json** - Centralized skill registration
+
+#### Comprehensive Test Coverage (83,990+ lines)
+- **Kernel tests** - unified-memory, hybrid-backend, kernel, plugin-loader
+- **MCP handler tests** - All domain handlers, handler-factory, task-handlers
+- **Domain tests** - All 12 domain plugins with coordinator tests
+- **Learning tests** - pattern-store, experience-capture, v2-to-v3-migration
+- **Routing tests** - tiny-dancer-router, task-classifier, routing-config
+- **Sync tests** - claude-flow-bridge, sync-agent, json/sqlite readers
+- **Worker tests** - All 10 background workers
+
+#### E2E Test Framework
+- Moved e2e tests to v3/tests/e2e/
+- Sauce Demo test suite with accessibility, cart, checkout, security specs
+- Page Object Model with BasePage, CartPage, CheckoutPage, etc.
+
+#### Documentation Updates
+- Updated agent catalog with QCSD Ideation agents
+- Added HTSM v6.3 quality categories reference
+- Added SFDIPOT framework documentation
+- Updated skill counts: 61 → 63 QE Skills
+- Updated agent counts with new QCSD agents
+
+### Changed
+
+#### Handler Factory Migration
+- All 11 domain handlers now use centralized handler-factory.ts
+- Experience capture middleware wraps all domain operations
+- Consistent error handling across handlers
+
+#### Security Scanner Refactoring
+- Split monolithic security-scanner.ts into modular components
+- New scanner-orchestrator.ts for coordinating SAST/DAST scans
+- Separate sast-scanner.ts and dast-scanner.ts modules
+- security-patterns.ts for pattern definitions
+
+#### E2E Runner Modularization
+- Split e2e-runner.ts into 9 focused modules
+- browser-orchestrator.ts - Browser session management
+- step-executors.ts - Step execution logic
+- assertion-handlers.ts - Assertion processing
+- result-collector.ts - Test result aggregation
+
+### Fixed
+
+#### Test Timeout Fixes
+- Fixed 6 timeout failures in security-compliance/coordinator.test.ts
+- Added proper class-based mocks for SecurityScannerService
+- Added mocks for SecurityAuditorService and ComplianceValidatorService
+
+#### TypeScript Compilation
+- Fixed all TypeScript errors from PR #215 merge
+- Fixed history.length on unknown type casting
+- Fixed performTask payload access patterns
+- Fixed Map iteration with Array.from()
+
+#### Architecture Cleanup
+- Removed wrong-pattern TypeScript agent classes (QualityCriteriaRecommenderAgent, RiskAssessorAgent)
+- Removed orphaned QCSD agent tests
+- Moved n8n-validator to v3/packages/
+
+### Security
+
+- SSRF protection recommendations for DAST scanner (private IP blocking)
+- Path traversal edge case fix recommendation (startsWith + path.sep)
+- npm audit: 0 vulnerabilities
+
+### Deprecated
+
+- Root-level tests/e2e/ directory (moved to v3/tests/e2e/)
+- Root-level src/agents/ TypeScript classes (use .claude/agents/v3/*.md instead)
+
+---
+
 ## [3.3.3] - 2026-01-27
 
 ### 🎯 Highlights