create-backlist 10.0.3 → 10.0.5

package/src/qa/browser/crawler.js

@@ -0,0 +1,305 @@
+// Smart crawler with HTTP fallback when browser unavailable
+import { URL }                  from 'node:url';
+import { shortId }              from '../qa-engine.js';
+import { getBrowserLaunchOptions } from './installer.js';
+
+const FETCH_TIMEOUT = 12_000;
+
+// ── HTTP-only crawler (no browser needed) ─────────────────────────────────
+export class HTTPCrawler {
+  #visited = new Set();
+  #baseUrl;
+
+  async crawl(baseUrl, { maxPages = 40, onRoute } = {}) {
+    this.#baseUrl = baseUrl;
+    this.#visited.clear();
+    const queue  = [{ url: baseUrl, depth: 0 }];
+    const routes = [];
+
+    while (queue.length > 0 && routes.length < maxPages) {
+      const { url, depth } = queue.shift();
+      const norm = this.#norm(url);
+      if (!norm || this.#visited.has(norm)) continue;
+      if (!this.#sameOrigin(norm, baseUrl))  continue;
+      if (depth > 3) continue;
+
+      this.#visited.add(norm);
+
+      const route = await this.#probeURL(norm, depth);
+      routes.push(route);
+      if (onRoute) onRoute(route);
+
+      // Extract links from HTML response
+      if (route.links) {
+        for (const link of route.links) {
+          const ln = this.#norm(link);
+          if (ln && !this.#visited.has(ln) && this.#sameOrigin(ln, baseUrl)) {
+            queue.push({ url: ln, depth: depth + 1 });
+          }
+        }
+      }
+    }
+
+    // Also probe common API paths
+    const apiPaths = [
+      '/api/health', '/api/status', '/api/v1/health',
+      '/api/v1/users', '/api/v1/products', '/health',
+      '/api/docs', '/sitemap.xml', '/robots.txt',
+    ];
+
+    for (const p of apiPaths) {
+      const url  = new URL(p, baseUrl).toString();
+      const norm = this.#norm(url);
+      if (this.#visited.has(norm)) continue;
+      this.#visited.add(norm);
+      const route = await this.#probeURL(url, 0);
+      if (route.status > 0 && route.status < 500) {
+        routes.push(route);
+        if (onRoute) onRoute(route);
+      }
+    }
+
+    return routes;
+  }
+
+  async #probeURL(url, depth) {
+    const t0 = Date.now();
+    try {
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT);
+
+      const res = await fetch(url, {
+        signal  : controller.signal,
+        headers : { 'User-Agent': 'Backlist-QA/12.0', Accept: 'text/html,application/json,*/*' },
+        redirect: 'follow',
+      });
+      clearTimeout(timer);
+
+      const ct      = res.headers.get('content-type') || '';
+      const duration = Date.now() - t0;
+      const headers  = {};
+      res.headers.forEach((v, k) => { headers[k] = v; });
+
+      let text  = '';
+      let links = [];
+      let forms = [];
+
+      if (ct.includes('text/html')) {
+        try { text = await res.text(); } catch {}
+        links = this.#extractLinks(text, url);
+        forms = this.#extractForms(text);
+      }
+
+      const type = this.#detectType(url, ct, res.status);
+
+      return {
+        id: shortId(), url, type, status: res.status,
+        duration, depth, links, forms, headers,
+        contentType: ct,
+        error: null,
+      };
+    } catch (err) {
+      return {
+        id: shortId(), url, type: 'error', status: 0,
+        duration: Date.now() - t0, depth,
+        links: [], forms: [], error: err.message,
+      };
+    }
+  }
+
+  #extractLinks(html, base) {
+    const links = [];
+    const re    = /href=["']([^"'#?][^"']*?)["']/gi;
+    let   m;
+    while ((m = re.exec(html)) !== null) {
+      try { links.push(new URL(m[1], base).toString()); } catch {}
+    }
+    return [...new Set(links)].slice(0, 30);
+  }
+
+  #extractForms(html) {
+    const forms = [];
+    const re    = /<form([^>]*)>([\s\S]*?)<\/form>/gi;
+    let   m;
+    while ((m = re.exec(html)) !== null) {
+      const attrs  = m[1];
+      const body   = m[2];
+      const action = (attrs.match(/action=["']([^"']+)["']/) || [])[1] || '';
+      const method = (attrs.match(/method=["']([^"']+)["']/) || [])[1] || 'GET';
+      const fields = [];
+      const inpRe  = /<input([^>]*)>/gi;
+      let   inp;
+      while ((inp = inpRe.exec(body)) !== null) {
+        const name     = (inp[1].match(/name=["']([^"']+)["']/)     || [])[1];
+        const type     = (inp[1].match(/type=["']([^"']+)["']/)     || [])[1] || 'text';
+        const required = /required/i.test(inp[1]);
+        if (name) fields.push({ name, type, required });
+      }
+      forms.push({ action, method, fields });
+    }
+    return forms;
+  }
+
+  #norm(url) {
+    try { const u = new URL(url); u.hash = ''; return u.toString(); }
+    catch { return null; }
+  }
+
+  #sameOrigin(url, base) {
+    try { return new URL(url).origin === new URL(base).origin; }
+    catch { return false; }
+  }
+
+  #detectType(url, ct, status) {
+    if (status >= 400) return 'error-page';
+    if (ct.includes('json') || url.includes('/api/')) return 'api';
+    if (url.endsWith('.xml') || url.endsWith('.txt')) return 'resource';
+    if (/\/(login|signin|auth)/i.test(url)) return 'auth';
+    if (/\/(admin)/i.test(url))             return 'admin';
+    if (/\/(dashboard)/i.test(url))         return 'dashboard';
+    return 'page';
+  }
+}
+
+// ── Browser-powered crawler (Playwright) ──────────────────────────────────
+export class SmartCrawler {
+  #visited = new Set();
+  #launchOpts = null;
+
+  constructor(_playwright) {}  // playwright passed but we resolve it ourselves
+
+  async crawl(baseUrl, { maxPages = 60, maxDepth = 4, onRoute } = {}) {
+    // Resolve launch options including auto-install
+    if (!this.#launchOpts) {
+      this.#launchOpts = await getBrowserLaunchOptions();
+    }
+
+    // If browser unavailable — delegate to HTTP crawler
+    if (!this.#launchOpts.available) {
+      const http = new HTTPCrawler();
+      return http.crawl(baseUrl, { maxPages, onRoute });
+    }
+
+    this.#visited.clear();
+
+    let playwright;
+    try { playwright = await import('playwright'); }
+    catch {
+      const http = new HTTPCrawler();
+      return http.crawl(baseUrl, { maxPages, onRoute });
+    }
+
+    const { executablePath, headless, args } = this.#launchOpts;
+
+    let browser;
+    try {
+      browser = await playwright.chromium.launch({ executablePath, headless, args });
+    } catch (err) {
+      console.log(chalk.yellow(`  ⚠  Browser launch failed: ${err.message}`));
+      console.log(chalk.gray('  Falling back to HTTP-only crawler...'));
+      const http = new HTTPCrawler();
+      return http.crawl(baseUrl, { maxPages, onRoute });
+    }
+
+    const context = await browser.newContext({
+      userAgent        : 'Backlist-QA-Crawler/12.0',
+      ignoreHTTPSErrors: true,
+    });
+
+    const routes = [];
+    const queue  = [{ url: baseUrl, depth: 0 }];
+
+    try {
+      while (queue.length > 0 && routes.length < maxPages) {
+        const { url, depth } = queue.shift();
+        const norm = this.#norm(url);
+        if (!norm || this.#visited.has(norm))       continue;
+        if (!this.#sameOrigin(norm, baseUrl))        continue;
+        if (depth > maxDepth)                        continue;
+        this.#visited.add(norm);
+
+        const route = await this.#probePage(context, norm, depth, baseUrl);
+        if (!route) continue;
+
+        routes.push(route);
+        if (onRoute) onRoute(route);
+
+        for (const link of (route.links || [])) {
+          const ln = this.#norm(link);
+          if (ln && !this.#visited.has(ln)) {
+            queue.push({ url: ln, depth: depth + 1 });
+          }
+        }
+      }
+    } finally {
+      await context.close().catch(() => {});
+      await browser.close().catch(() => {});
+    }
+
+    return routes;
+  }
+
+  async #probePage(context, url, depth, baseUrl) {
+    const page             = await context.newPage();
+    const networkRequests  = [];
+    const links            = new Set();
+
+    page.on('request', req => {
+      const u = req.url();
+      if ((u.includes('/api/') || req.resourceType() === 'fetch') && this.#sameOrigin(u, baseUrl)) {
+        networkRequests.push({ url: u, method: req.method(), type: 'api' });
+      }
+    });
+
+    try {
+      const response = await page.goto(url, { waitUntil: 'networkidle', timeout: 15_000 });
+      const status   = response?.status() || 0;
+      const ct       = response?.headers()['content-type'] || '';
+
+      if (!ct.includes('text/html') && !ct.includes('application/xhtml')) {
+        return { id: shortId(), url, type: this.#detectType(url, ct, status), status, depth, links: [], forms: [] };
+      }
+
+      const hrefs = await page.$$eval('a[href]', els => els.map(e => e.href).filter(Boolean)).catch(() => []);
+      hrefs.forEach(h => links.add(h));
+
+      const forms = await page.$$eval('form', els => els.map(f => ({
+        action: f.action, method: f.method || 'GET',
+        fields: Array.from(f.elements).map(el => ({
+          name: el.name, type: el.type, required: el.required,
+        })).filter(f => f.name),
+      }))).catch(() => []);
+
+      networkRequests.forEach(r => { if (this.#sameOrigin(r.url, baseUrl)) links.add(r.url); });
+
+      return {
+        id: shortId(), url, type: this.#detectType(url, ct, status),
+        status, depth, links: [...links], forms, contentType: ct,
+      };
+    } catch (err) {
+      return { id: shortId(), url, type: 'error', status: 0, depth, links: [], forms: [], error: err.message };
+    } finally {
+      await page.close().catch(() => {});
+    }
+  }
+
+  #norm(url) {
+    try { const u = new URL(url); u.hash = ''; return u.toString(); }
+    catch { return null; }
+  }
+
+  #sameOrigin(url, base) {
+    try { return new URL(url).origin === new URL(base).origin; }
+    catch { return false; }
+  }
+
+  #detectType(url, ct, status) {
+    if (status >= 400) return 'error-page';
+    if (ct.includes('json') || url.includes('/api/')) return 'api';
+    if (url.endsWith('.xml') || url.endsWith('.txt')) return 'resource';
+    if (/\/(login|signin|auth)/i.test(url)) return 'auth';
+    if (/\/(admin)/i.test(url))             return 'admin';
+    if (/\/(dashboard)/i.test(url))         return 'dashboard';
+    return 'page';
+  }
+}

package/src/qa/browser/installer.js

@@ -0,0 +1,209 @@
+// ═══════════════════════════════════════════════════════════════════════════
+//  Playwright Auto-Installer & Browser Manager
+//  Handles missing browsers gracefully with auto-install
+// ═══════════════════════════════════════════════════════════════════════════
+
+import { execSync, spawn }  from 'node:child_process';
+import path                 from 'node:path';
+import fs                   from 'fs-extra';
+import chalk                from 'chalk';
+import ora                  from 'ora';
+
+// ── Browser executable paths per platform ────────────────────────────────
+function getPlaywrightCacheDir() {
+  const home = process.env.HOME || process.env.USERPROFILE || '';
+  switch (process.platform) {
+    case 'win32':  return path.join(process.env.LOCALAPPDATA || path.join(home, 'AppData', 'Local'), 'ms-playwright');
+    case 'darwin': return path.join(home, 'Library', 'Caches', 'ms-playwright');
+    default:       return path.join(home, '.cache', 'ms-playwright');
+  }
+}
+
+function findChromiumExecutable() {
+  const cacheDir = getPlaywrightCacheDir();
+
+  try {
+    if (!fs.existsSync(cacheDir)) return null;
+
+    const entries = fs.readdirSync(cacheDir);
+
+    // Look for any chromium variant folder
+    const chromiumDirs = entries
+      .filter(e => e.startsWith('chromium') || e.startsWith('chrome'))
+      .sort()
+      .reverse(); // prefer newest
+
+    for (const dir of chromiumDirs) {
+      const candidates = [
+        // Headless shell (newer Playwright)
+        path.join(cacheDir, dir, 'chrome-headless-shell-win64', 'chrome-headless-shell.exe'),
+        path.join(cacheDir, dir, 'chrome-headless-shell-linux', 'chrome-headless-shell'),
+        path.join(cacheDir, dir, 'chrome-headless-shell-mac_arm', 'chrome-headless-shell'),
+        path.join(cacheDir, dir, 'chrome-headless-shell-mac_x64', 'chrome-headless-shell'),
+        // Regular chromium
+        path.join(cacheDir, dir, 'chrome-win64',       'chrome.exe'),
+        path.join(cacheDir, dir, 'chrome-linux',        'chrome'),
+        path.join(cacheDir, dir, 'chrome-mac',          'Chromium.app', 'Contents', 'MacOS', 'Chromium'),
+        path.join(cacheDir, dir, 'chrome-mac_arm',      'Chromium.app', 'Contents', 'MacOS', 'Chromium'),
+        path.join(cacheDir, dir, 'chromium-linux',      'chrome-linux', 'chrome'),
+        path.join(cacheDir, dir, 'chromium-win64',      'chrome.exe'),
+        path.join(cacheDir, dir, 'chromium-mac',        'chrome-mac',   'Chromium.app', 'Contents', 'MacOS', 'Chromium'),
+      ];
+
+      for (const candidate of candidates) {
+        if (fs.existsSync(candidate)) return candidate;
+      }
+    }
+  } catch {}
+
+  return null;
+}
+
+// ── Check system Chrome/Chromium ──────────────────────────────────────────
+function findSystemChrome() {
+  const candidates = {
+    win32: [
+      'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
+      'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
+      path.join(process.env.LOCALAPPDATA || '', 'Google\\Chrome\\Application\\chrome.exe'),
+      'C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe',
+      path.join(process.env.PROGRAMFILES || '', 'Microsoft\\Edge\\Application\\msedge.exe'),
+    ],
+    darwin: [
+      '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+      '/Applications/Chromium.app/Contents/MacOS/Chromium',
+      '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
+    ],
+    linux: [
+      '/usr/bin/google-chrome',
+      '/usr/bin/google-chrome-stable',
+      '/usr/bin/chromium-browser',
+      '/usr/bin/chromium',
+      '/snap/bin/chromium',
+    ],
+  };
+
+  const platform = process.platform;
+  const paths    = candidates[platform] || candidates.linux;
+
+  for (const p of paths) {
+    try {
+      if (fs.existsSync(p)) return p;
+    } catch {}
+  }
+
+  return null;
+}
+
+// ── Install Playwright browsers ───────────────────────────────────────────
+export async function installPlaywrightBrowsers() {
+  const spinner = ora({
+    text   : chalk.cyan('Installing Playwright Chromium browser...'),
+    spinner: 'dots12',
+    color  : 'cyan',
+  }).start();
+
+  return new Promise((resolve) => {
+    const proc = spawn(
+      process.execPath,        // use same node binary
+      ['node_modules/.bin/playwright', 'install', 'chromium', '--with-deps'],
+      {
+        cwd   : process.cwd(),
+        stdio : ['ignore', 'pipe', 'pipe'],
+        shell : process.platform === 'win32',
+        env   : { ...process.env },
+      }
+    );
+
+    let output = '';
+    proc.stdout?.on('data', d => { output += d.toString(); });
+    proc.stderr?.on('data', d => { output += d.toString(); });
+
+    proc.on('close', (code) => {
+      if (code === 0) {
+        spinner.succeed(chalk.green('Chromium installed successfully ✓'));
+        resolve({ success: true });
+      } else {
+        spinner.fail(chalk.red(`Chromium install failed (code ${code})`));
+        resolve({ success: false, output });
+      }
+    });
+
+    proc.on('error', (err) => {
+      spinner.fail(chalk.red(`Install process error: ${err.message}`));
+      resolve({ success: false, error: err.message });
+    });
+
+    // Timeout after 3 minutes
+    setTimeout(() => {
+      proc.kill();
+      spinner.fail(chalk.red('Browser install timed out (3 min)'));
+      resolve({ success: false, error: 'timeout' });
+    }, 180_000);
+  });
+}
+
+// ── Main browser check & setup ────────────────────────────────────────────
+export async function ensureBrowser() {
+  // 1. Check Playwright-managed Chromium
+  const playwrightChrome = findChromiumExecutable();
+  if (playwrightChrome) {
+    return { executablePath: playwrightChrome, source: 'playwright-managed' };
+  }
+
+  // 2. Check system Chrome/Edge/Chromium
+  const systemChrome = findSystemChrome();
+  if (systemChrome) {
+    console.log(chalk.gray(`  Using system browser: ${systemChrome}`));
+    return { executablePath: systemChrome, source: 'system' };
+  }
+
+  // 3. Try to auto-install
+  console.log('');
+  console.log(chalk.yellow('  ⚠  Playwright browser not found.'));
+  console.log(chalk.gray('  Attempting automatic installation...\n'));
+
+  const result = await installPlaywrightBrowsers();
+
+  if (result.success) {
+    const newPath = findChromiumExecutable();
+    if (newPath) {
+      return { executablePath: newPath, source: 'auto-installed' };
+    }
+  }
+
+  // 4. Return null — caller will use HTTP-only fallback
+  return { executablePath: null, source: 'unavailable', error: result?.error };
+}
+
+// ── Check if browser is available (fast check) ───────────────────────────
+export function isBrowserAvailable() {
+  return !!(findChromiumExecutable() || findSystemChrome());
+}
+
+// ── Get launch options with correct executable ────────────────────────────
+export async function getBrowserLaunchOptions() {
+  const browser = await ensureBrowser();
+
+  const baseArgs = [
+    '--no-sandbox',
+    '--disable-setuid-sandbox',
+    '--disable-dev-shm-usage',
+    '--disable-gpu',
+    '--no-first-run',
+    '--no-default-browser-check',
+    '--disable-extensions',
+  ];
+
+  if (!browser.executablePath) {
+    return { available: false, source: 'unavailable' };
+  }
+
+  return {
+    available      : true,
+    source         : browser.source,
+    executablePath : browser.executablePath,
+    headless       : true,
+    args           : baseArgs,
+  };
+}