agentic-qe 3.3.2 → 3.3.4

package/scripts/fetch-content.js

@@ -0,0 +1,460 @@
+#!/usr/bin/env node
+/**
+ * Fetch Content CLI - 5-Tier Browser Cascade with Per-Tier Timeouts
+ *
+ * Single entry point for all browser content fetching operations.
+ * Automatically cascades through tiers with 30s timeout each.
+ *
+ * Usage:
+ *   node scripts/fetch-content.js <URL> <OUTPUT_DIR> [OPTIONS]
+ *
+ * Options:
+ *   --timeout <ms>      Per-tier timeout (default: 30000)
+ *   --skip-tiers <list> Comma-separated tiers to skip
+ *   --locale <locale>   Browser locale (default: en-US)
+ *
+ * Output:
+ *   - <OUTPUT_DIR>/content.html       Fetched HTML content
+ *   - <OUTPUT_DIR>/screenshot.png     Page screenshot
+ *   - <OUTPUT_DIR>/fetch-result.json  Structured result with metadata
+ *
+ * Exit codes:
+ *   0 - Success
+ *   1 - All tiers failed
+ *   2 - Invalid arguments
+ *
+ * @version 1.0.0
+ * @since v3
+ */
+
+const { exec, execSync, spawn } = require('child_process');
+const { promisify } = require('util');
+const fs = require('fs');
+const path = require('path');
+
+const execAsync = promisify(exec);
+
+// ============================================================================
+// Configuration
+// ============================================================================
+
+const DEFAULT_TIER_TIMEOUT = 30000; // 30 seconds per tier
+const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36';
+
+const TIERS = [
+  'vibium',
+  'playwright-stealth',
+  'http-fetch',
+  'websearch-fallback'
+];
+
+// ============================================================================
+// Argument Parsing
+// ============================================================================
+
+function parseArgs() {
+  const args = process.argv.slice(2);
+
+  if (args.length < 2) {
+    console.error('Usage: node fetch-content.js <URL> <OUTPUT_DIR> [OPTIONS]');
+    console.error('');
+    console.error('Options:');
+    console.error('  --timeout <ms>      Per-tier timeout (default: 30000)');
+    console.error('  --skip-tiers <list> Comma-separated tiers to skip');
+    console.error('  --locale <locale>   Browser locale (default: en-US)');
+    process.exit(2);
+  }
+
+  const options = {
+    url: args[0],
+    outputDir: args[1],
+    timeout: DEFAULT_TIER_TIMEOUT,
+    skipTiers: [],
+    locale: 'en-US',
+    userAgent: DEFAULT_USER_AGENT
+  };
+
+  for (let i = 2; i < args.length; i++) {
+    switch (args[i]) {
+      case '--timeout':
+        options.timeout = parseInt(args[++i], 10);
+        break;
+      case '--skip-tiers':
+        options.skipTiers = args[++i].split(',').map(t => t.trim());
+        break;
+      case '--locale':
+        options.locale = args[++i];
+        break;
+    }
+  }
+
+  return options;
+}
+
+// ============================================================================
+// Utility Functions
+// ============================================================================
+
+function log(tier, message, data = {}) {
+  const entry = {
+    timestamp: new Date().toISOString(),
+    tier,
+    message,
+    ...data
+  };
+  console.error(`[${entry.timestamp}] [${tier}] ${message}`);
+  return entry;
+}
+
+function withTimeout(promise, ms, tierName) {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      reject(new Error(`${tierName} timed out after ${ms}ms`));
+    }, ms);
+
+    promise
+      .then(result => {
+        clearTimeout(timer);
+        resolve(result);
+      })
+      .catch(err => {
+        clearTimeout(timer);
+        reject(err);
+      });
+  });
+}
+
+// ============================================================================
+// Tier Implementations
+// ============================================================================
+
+/**
+ * Tier 1: Vibium MCP Browser
+ * Note: Vibium requires MCP context - skip in CLI mode
+ */
+async function fetchWithVibium(options) {
+  log('vibium', 'Skipping - requires MCP context');
+  throw new Error('Vibium requires MCP context, not available in CLI');
+}
+
+/**
+ * Tier 2: Playwright + Stealth (PRIMARY for CLI)
+ */
+async function fetchWithPlaywrightStealth(options) {
+  log('playwright-stealth', 'Starting fetch', { url: options.url });
+
+  const workDir = path.join(options.outputDir, '.playwright-work');
+  fs.mkdirSync(workDir, { recursive: true });
+
+  // Check if playwright is installed globally or locally
+  const packageJsonPath = path.join(workDir, 'package.json');
+  if (!fs.existsSync(path.join(workDir, 'node_modules', 'playwright-extra'))) {
+    fs.writeFileSync(packageJsonPath, JSON.stringify({ name: 'pw-fetch', type: 'commonjs' }));
+
+    log('playwright-stealth', 'Installing dependencies...');
+    try {
+      execSync('npm install playwright-extra puppeteer-extra-plugin-stealth playwright 2>/dev/null', {
+        cwd: workDir,
+        stdio: 'pipe',
+        timeout: 60000
+      });
+    } catch (e) {
+      log('playwright-stealth', 'Install warning', { error: e.message });
+    }
+  }
+
+  const scriptPath = path.join(workDir, 'fetch.js');
+  const contentPath = path.join(options.outputDir, 'content.html');
+  const screenshotPath = path.join(options.outputDir, 'screenshot.png');
+
+  const script = `
+const { chromium } = require('playwright-extra');
+const stealth = require('puppeteer-extra-plugin-stealth')();
+chromium.use(stealth);
+
+(async () => {
+  const browser = await chromium.launch({
+    headless: true,
+    args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu']
+  });
+
+  const context = await browser.newContext({
+    userAgent: ${JSON.stringify(options.userAgent)},
+    viewport: { width: 1920, height: 1080 },
+    locale: ${JSON.stringify(options.locale)}
+  });
+
+  const page = await context.newPage();
+
+  try {
+    await page.goto(${JSON.stringify(options.url)}, {
+      waitUntil: 'domcontentloaded',
+      timeout: ${options.timeout - 5000}
+    });
+
+    await page.waitForTimeout(2000);
+
+    // Dismiss cookie banners
+    const cookieSelectors = [
+      '[data-testid="consent-accept-all"]',
+      '#onetrust-accept-btn-handler',
+      'button[id*="accept"]',
+      '[class*="cookie"] button',
+      'button:has-text("Accept")',
+      'button:has-text("Acceptă")'
+    ];
+
+    for (const sel of cookieSelectors) {
+      try {
+        const btn = await page.$(sel);
+        if (btn) {
+          await btn.click();
+          await page.waitForTimeout(1000);
+          break;
+        }
+      } catch (e) {}
+    }
+
+    const content = await page.content();
+    require('fs').writeFileSync(${JSON.stringify(contentPath)}, content);
+    await page.screenshot({ path: ${JSON.stringify(screenshotPath)} });
+
+    console.log(JSON.stringify({
+      success: true,
+      size: content.length,
+      title: await page.title(),
+      url: page.url()
+    }));
+  } finally {
+    await browser.close();
+  }
+})().catch(e => {
+  console.log(JSON.stringify({ success: false, error: e.message }));
+  process.exit(1);
+});
+`;
+
+  fs.writeFileSync(scriptPath, script);
+
+  const { stdout } = await execAsync(`node "${scriptPath}"`, {
+    cwd: workDir,
+    timeout: options.timeout
+  });
+
+  const result = JSON.parse(stdout.trim());
+  if (!result.success) {
+    throw new Error(result.error);
+  }
+
+  const content = fs.readFileSync(contentPath, 'utf8');
+
+  return {
+    success: true,
+    content,
+    contentSize: content.length,
+    screenshotPath,
+    metadata: { title: result.title, finalUrl: result.url }
+  };
+}
+
+/**
+ * Tier 3: HTTP Fetch (for simple/static sites)
+ */
+async function fetchWithHttp(options) {
+  log('http-fetch', 'Starting fetch', { url: options.url });
+
+  const response = await fetch(options.url, {
+    headers: {
+      'User-Agent': options.userAgent,
+      'Accept': 'text/html,application/xhtml+xml,*/*',
+      'Accept-Language': `${options.locale},en;q=0.5`
+    },
+    signal: AbortSignal.timeout(options.timeout)
+  });
+
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+  }
+
+  const content = await response.text();
+
+  if (content.length < 500) {
+    throw new Error('HTTP fetch returned insufficient content (bot blocked?)');
+  }
+
+  const contentPath = path.join(options.outputDir, 'content.html');
+  fs.writeFileSync(contentPath, content);
+
+  return {
+    success: true,
+    content,
+    contentSize: content.length,
+    metadata: { statusCode: response.status }
+  };
+}
+
+/**
+ * Tier 4: WebSearch Fallback (research-based, degraded mode)
+ */
+async function fetchWithWebSearch(options) {
+  log('websearch-fallback', 'Creating research-based content', { url: options.url });
+
+  // This tier creates a placeholder indicating manual research is needed
+  const urlObj = new URL(options.url);
+  const domain = urlObj.hostname;
+
+  const researchContent = `
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Research Required: ${options.url}</title>
+  <meta name="fetch-method" content="websearch-fallback">
+  <meta name="fetch-status" content="degraded">
+  <meta name="original-url" content="${options.url}">
+  <meta name="fetch-date" content="${new Date().toISOString()}">
+</head>
+<body>
+  <h1>Manual Research Required</h1>
+  <p><strong>Status:</strong> All automated fetch tiers failed.</p>
+  <p><strong>Target URL:</strong> ${options.url}</p>
+  <p><strong>Domain:</strong> ${domain}</p>
+  <hr>
+  <h2>Recommended Actions:</h2>
+  <ol>
+    <li>Use WebSearch tool to research "${domain} features functionality"</li>
+    <li>Check if site requires authentication or has strict bot protection</li>
+    <li>Consider manual browser inspection</li>
+  </ol>
+  <p><em>This is a degraded fallback. Analysis based on this content will be limited.</em></p>
+</body>
+</html>
+`;
+
+  const contentPath = path.join(options.outputDir, 'content.html');
+  fs.writeFileSync(contentPath, researchContent);
+
+  return {
+    success: true,
+    content: researchContent,
+    contentSize: researchContent.length,
+    metadata: {
+      note: 'Degraded mode - research-based content',
+      manualResearchRequired: true
+    }
+  };
+}
+
+// ============================================================================
+// Tier Dispatcher
+// ============================================================================
+
+const TIER_HANDLERS = {
+  'vibium': fetchWithVibium,
+  'playwright-stealth': fetchWithPlaywrightStealth,
+  'http-fetch': fetchWithHttp,
+  'websearch-fallback': fetchWithWebSearch
+};
+
+// ============================================================================
+// Main Cascade Logic
+// ============================================================================
+
+async function fetchWithCascade(options) {
+  fs.mkdirSync(options.outputDir, { recursive: true });
+
+  const result = {
+    success: false,
+    url: options.url,
+    tier: null,
+    status: 'failed',
+    content: null,
+    contentSize: 0,
+    screenshotPath: null,
+    tiersAttempted: [],
+    tierErrors: {},
+    metadata: {}
+  };
+
+  const tiersToTry = TIERS.filter(t => !options.skipTiers.includes(t));
+
+  for (const tier of tiersToTry) {
+    result.tiersAttempted.push(tier);
+    log(tier, `Attempting fetch (timeout: ${options.timeout}ms)`);
+
+    try {
+      const handler = TIER_HANDLERS[tier];
+      const tierResult = await withTimeout(
+        handler(options),
+        options.timeout,
+        tier
+      );
+
+      if (tierResult.success && tierResult.content) {
+        result.success = true;
+        result.tier = tier;
+        result.status = tier === 'websearch-fallback' ? 'degraded' : 'success';
+        result.content = tierResult.content;
+        result.contentSize = tierResult.contentSize;
+        result.screenshotPath = tierResult.screenshotPath || null;
+        result.metadata = tierResult.metadata || {};
+
+        log(tier, `SUCCESS: ${result.contentSize} bytes`);
+        break;
+      }
+    } catch (error) {
+      const errorMsg = error.message || String(error);
+      result.tierErrors[tier] = errorMsg;
+      log(tier, `FAILED: ${errorMsg}`);
+    }
+  }
+
+  // Save result metadata
+  const resultPath = path.join(options.outputDir, 'fetch-result.json');
+  const resultData = { ...result };
+  delete resultData.content; // Don't duplicate content in JSON
+  fs.writeFileSync(resultPath, JSON.stringify(resultData, null, 2));
+
+  return result;
+}
+
+// ============================================================================
+// Entry Point
+// ============================================================================
+
+async function main() {
+  const options = parseArgs();
+
+  console.error('');
+  console.error('╔════════════════════════════════════════════════════════════════╗');
+  console.error('║              FETCH CONTENT - 5-TIER CASCADE                    ║');
+  console.error('╠════════════════════════════════════════════════════════════════╣');
+  console.error(`║  URL: ${options.url.substring(0, 54).padEnd(54)} ║`);
+  console.error(`║  Output: ${options.outputDir.substring(0, 51).padEnd(51)} ║`);
+  console.error(`║  Timeout per tier: ${options.timeout}ms                                  ║`);
+  console.error('╚════════════════════════════════════════════════════════════════╝');
+  console.error('');
+
+  try {
+    const result = await fetchWithCascade(options);
+
+    // Output final result to stdout (for programmatic use)
+    console.log(JSON.stringify({
+      success: result.success,
+      tier: result.tier,
+      status: result.status,
+      contentSize: result.contentSize,
+      screenshotPath: result.screenshotPath,
+      contentPath: path.join(options.outputDir, 'content.html'),
+      resultPath: path.join(options.outputDir, 'fetch-result.json'),
+      tiersAttempted: result.tiersAttempted,
+      tierErrors: result.tierErrors
+    }, null, 2));
+
+    process.exit(result.success ? 0 : 1);
+  } catch (error) {
+    console.error(`Fatal error: ${error.message}`);
+    process.exit(1);
+  }
+}
+
+main();

package/scripts/sync-claude-flow.cjs

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+/**
+ * Sync Claude Flow memories to AQE V3 database
+ */
+const fs = require('fs');
+const path = require('path');
+
+const projectRoot = process.cwd();
+const claudeFlowPath = path.join(projectRoot, '.claude-flow', 'memory', 'store.json');
+const aqeDbPath = path.join(projectRoot, 'v3', '.agentic-qe', 'memory.db');
+
+console.log('Claude Flow:', claudeFlowPath);
+console.log('AQE DB:', aqeDbPath);
+
+if (!fs.existsSync(claudeFlowPath)) {
+  console.error('Claude Flow store not found');
+  process.exit(1);
+}
+
+if (!fs.existsSync(aqeDbPath)) {
+  console.error('AQE DB not found');
+  process.exit(1);
+}
+
+const store = JSON.parse(fs.readFileSync(claudeFlowPath, 'utf-8'));
+const entries = store.entries || store;
+const keys = Object.keys(entries).filter(k => !k.startsWith('_') && k !== 'version');
+console.log('Entries to sync:', keys.length);
+
+const Database = require('better-sqlite3');
+const db = new Database(aqeDbPath);
+db.pragma('journal_mode = WAL');
+
+const insert = db.prepare(`
+  INSERT OR REPLACE INTO kv_store (key, namespace, value, created_at)
+  VALUES (?, ?, ?, ?)
+`);
+
+let count = 0;
+let learning = 0;
+
+// Also insert into sona_patterns for learning-related entries
+const insertSona = db.prepare(`
+  INSERT OR REPLACE INTO sona_patterns
+  (id, type, domain, action_type, outcome_reward, outcome_success, metadata, created_at)
+  VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'))
+`);
+
+for (const key of keys) {
+  const value = entries[key];
+
+  // Always store in kv_store
+  insert.run(
+    'cf:' + key,
+    'claude-flow',
+    JSON.stringify(value),
+    Date.now()
+  );
+  count++;
+
+  // Store learning-relevant entries in sona_patterns too
+  const keyLower = key.toLowerCase();
+  if (keyLower.includes('pattern') || keyLower.includes('analysis') ||
+      keyLower.includes('learning') || keyLower.includes('agent') ||
+      keyLower.includes('outcome') || keyLower.includes('quality')) {
+
+    // Determine domain
+    let domain = 'general';
+    const domains = ['test-generation', 'coverage-analysis', 'security-compliance',
+                     'quality-assessment', 'code-intelligence'];
+    for (const d of domains) {
+      if (keyLower.includes(d.replace('-', ''))) {
+        domain = d;
+        break;
+      }
+    }
+
+    const id = 'cf-' + key.replace(/[^a-zA-Z0-9-]/g, '-').slice(0, 60);
+
+    try {
+      insertSona.run(
+        id,
+        'claude-flow-import',
+        domain,
+        'learn-from-session',
+        0.7, // Assume successful since it was stored
+        1,
+        JSON.stringify({ sourceKey: key, importedAt: new Date().toISOString() })
+      );
+      learning++;
+    } catch (e) {
+      // Ignore duplicate errors
+    }
+  }
+}
+
+db.close();
+console.log('Synced to kv_store:', count, 'entries');
+console.log('Synced to sona_patterns:', learning, 'learning entries');