create-backlist 10.0.4 → 10.0.5

package/src/qa/browser/crawler.js

@@ -1,223 +1,305 @@
-// Real smart crawler — discovers all routes via Playwright
-import { URL } from 'node:url';
-import { shortId } from '../qa-engine.js';
+// Smart crawler with HTTP fallback when browser unavailable
+import { URL }                  from 'node:url';
+import { shortId }              from '../qa-engine.js';
+import { getBrowserLaunchOptions } from './installer.js';
 
-export class SmartCrawler {
-  #playwright;
+const FETCH_TIMEOUT = 12_000;
+
+// ── HTTP-only crawler (no browser needed) ─────────────────────────────────
+export class HTTPCrawler {
   #visited = new Set();
-  #queue   = [];
-  #routes  = [];
+  #baseUrl;
+
+  async crawl(baseUrl, { maxPages = 40, onRoute } = {}) {
+    this.#baseUrl = baseUrl;
+    this.#visited.clear();
+    const queue  = [{ url: baseUrl, depth: 0 }];
+    const routes = [];
+
+    while (queue.length > 0 && routes.length < maxPages) {
+      const { url, depth } = queue.shift();
+      const norm = this.#norm(url);
+      if (!norm || this.#visited.has(norm)) continue;
+      if (!this.#sameOrigin(norm, baseUrl))  continue;
+      if (depth > 3) continue;
+
+      this.#visited.add(norm);
+
+      const route = await this.#probeURL(norm, depth);
+      routes.push(route);
+      if (onRoute) onRoute(route);
+
+      // Extract links from HTML response
+      if (route.links) {
+        for (const link of route.links) {
+          const ln = this.#norm(link);
+          if (ln && !this.#visited.has(ln) && this.#sameOrigin(ln, baseUrl)) {
+            queue.push({ url: ln, depth: depth + 1 });
+          }
+        }
+      }
+    }
+
+    // Also probe common API paths
+    const apiPaths = [
+      '/api/health', '/api/status', '/api/v1/health',
+      '/api/v1/users', '/api/v1/products', '/health',
+      '/api/docs', '/sitemap.xml', '/robots.txt',
+    ];
+
+    for (const p of apiPaths) {
+      const url  = new URL(p, baseUrl).toString();
+      const norm = this.#norm(url);
+      if (this.#visited.has(norm)) continue;
+      this.#visited.add(norm);
+      const route = await this.#probeURL(url, 0);
+      if (route.status > 0 && route.status < 500) {
+        routes.push(route);
+        if (onRoute) onRoute(route);
+      }
+    }
+
+    return routes;
+  }
+
+  async #probeURL(url, depth) {
+    const t0 = Date.now();
+    try {
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT);
+
+      const res = await fetch(url, {
+        signal  : controller.signal,
+        headers : { 'User-Agent': 'Backlist-QA/12.0', Accept: 'text/html,application/json,*/*' },
+        redirect: 'follow',
+      });
+      clearTimeout(timer);
+
+      const ct      = res.headers.get('content-type') || '';
+      const duration = Date.now() - t0;
+      const headers  = {};
+      res.headers.forEach((v, k) => { headers[k] = v; });
+
+      let text  = '';
+      let links = [];
+      let forms = [];
+
+      if (ct.includes('text/html')) {
+        try { text = await res.text(); } catch {}
+        links = this.#extractLinks(text, url);
+        forms = this.#extractForms(text);
+      }
 
-  constructor(playwright) {
-    this.#playwright = playwright;
+      const type = this.#detectType(url, ct, res.status);
+
+      return {
+        id: shortId(), url, type, status: res.status,
+        duration, depth, links, forms, headers,
+        contentType: ct,
+        error: null,
+      };
+    } catch (err) {
+      return {
+        id: shortId(), url, type: 'error', status: 0,
+        duration: Date.now() - t0, depth,
+        links: [], forms: [], error: err.message,
+      };
+    }
+  }
+
+  #extractLinks(html, base) {
+    const links = [];
+    const re    = /href=["']([^"'#?][^"']*?)["']/gi;
+    let   m;
+    while ((m = re.exec(html)) !== null) {
+      try { links.push(new URL(m[1], base).toString()); } catch {}
+    }
+    return [...new Set(links)].slice(0, 30);
+  }
+
+  #extractForms(html) {
+    const forms = [];
+    const re    = /<form([^>]*)>([\s\S]*?)<\/form>/gi;
+    let   m;
+    while ((m = re.exec(html)) !== null) {
+      const attrs  = m[1];
+      const body   = m[2];
+      const action = (attrs.match(/action=["']([^"']+)["']/) || [])[1] || '';
+      const method = (attrs.match(/method=["']([^"']+)["']/) || [])[1] || 'GET';
+      const fields = [];
+      const inpRe  = /<input([^>]*)>/gi;
+      let   inp;
+      while ((inp = inpRe.exec(body)) !== null) {
+        const name     = (inp[1].match(/name=["']([^"']+)["']/)     || [])[1];
+        const type     = (inp[1].match(/type=["']([^"']+)["']/)     || [])[1] || 'text';
+        const required = /required/i.test(inp[1]);
+        if (name) fields.push({ name, type, required });
+      }
+      forms.push({ action, method, fields });
+    }
+    return forms;
+  }
+
+  #norm(url) {
+    try { const u = new URL(url); u.hash = ''; return u.toString(); }
+    catch { return null; }
+  }
+
+  #sameOrigin(url, base) {
+    try { return new URL(url).origin === new URL(base).origin; }
+    catch { return false; }
+  }
+
+  #detectType(url, ct, status) {
+    if (status >= 400) return 'error-page';
+    if (ct.includes('json') || url.includes('/api/')) return 'api';
+    if (url.endsWith('.xml') || url.endsWith('.txt')) return 'resource';
+    if (/\/(login|signin|auth)/i.test(url)) return 'auth';
+    if (/\/(admin)/i.test(url))             return 'admin';
+    if (/\/(dashboard)/i.test(url))         return 'dashboard';
+    return 'page';
   }
+}
+
+// ── Browser-powered crawler (Playwright) ──────────────────────────────────
+export class SmartCrawler {
+  #visited = new Set();
+  #launchOpts = null;
+
+  constructor(_playwright) {}  // playwright passed but we resolve it ourselves
 
   async crawl(baseUrl, { maxPages = 60, maxDepth = 4, onRoute } = {}) {
+    // Resolve launch options including auto-install
+    if (!this.#launchOpts) {
+      this.#launchOpts = await getBrowserLaunchOptions();
+    }
+
+    // If browser unavailable — delegate to HTTP crawler
+    if (!this.#launchOpts.available) {
+      const http = new HTTPCrawler();
+      return http.crawl(baseUrl, { maxPages, onRoute });
+    }
+
     this.#visited.clear();
-    this.#queue = [{ url: baseUrl, depth: 0 }];
-    this.#routes = [];
 
-    const browser = await this.#playwright.chromium.launch({ headless: true });
-    const context = await browser.newContext({
-      userAgent: 'Backlist-QA-Crawler/12.0',
-      ignoreHTTPSErrors: true,
-    });
+    let playwright;
+    try { playwright = await import('playwright'); }
+    catch {
+      const http = new HTTPCrawler();
+      return http.crawl(baseUrl, { maxPages, onRoute });
+    }
+
+    const { executablePath, headless, args } = this.#launchOpts;
 
+    let browser;
     try {
-      while (this.#queue.length > 0 && this.#routes.length < maxPages) {
-        const { url, depth } = this.#queue.shift();
-        const normalized = this.#normalizeUrl(url);
+      browser = await playwright.chromium.launch({ executablePath, headless, args });
+    } catch (err) {
+      console.log(chalk.yellow(`  ⚠  Browser launch failed: ${err.message}`));
+      console.log(chalk.gray('  Falling back to HTTP-only crawler...'));
+      const http = new HTTPCrawler();
+      return http.crawl(baseUrl, { maxPages, onRoute });
+    }
 
-        if (!normalized || this.#visited.has(normalized)) continue;
-        if (!this.#isSameOrigin(normalized, baseUrl))      continue;
-        if (depth > maxDepth)                              continue;
+    const context = await browser.newContext({
+      userAgent        : 'Backlist-QA-Crawler/12.0',
+      ignoreHTTPSErrors: true,
+    });
 
-        this.#visited.add(normalized);
+    const routes = [];
+    const queue  = [{ url: baseUrl, depth: 0 }];
 
-        const route = await this.#probePage(context, normalized, depth, baseUrl);
+    try {
+      while (queue.length > 0 && routes.length < maxPages) {
+        const { url, depth } = queue.shift();
+        const norm = this.#norm(url);
+        if (!norm || this.#visited.has(norm))       continue;
+        if (!this.#sameOrigin(norm, baseUrl))        continue;
+        if (depth > maxDepth)                        continue;
+        this.#visited.add(norm);
+
+        const route = await this.#probePage(context, norm, depth, baseUrl);
         if (!route) continue;
 
-        this.#routes.push(route);
+        routes.push(route);
         if (onRoute) onRoute(route);
 
-        // Enqueue discovered links
-        for (const link of route.links || []) {
-          const linkUrl = this.#normalizeUrl(link);
-          if (linkUrl && !this.#visited.has(linkUrl)) {
-            this.#queue.push({ url: linkUrl, depth: depth + 1 });
+        for (const link of (route.links || [])) {
+          const ln = this.#norm(link);
+          if (ln && !this.#visited.has(ln)) {
+            queue.push({ url: ln, depth: depth + 1 });
           }
         }
       }
     } finally {
-      await context.close();
-      await browser.close();
-    }
-
-    // Also discover APIs from network intercepts
-    const apiRoutes = await this.#discoverAPIs(baseUrl);
-    for (const api of apiRoutes) {
-      if (!this.#routes.find(r => r.url === api.url)) {
-        this.#routes.push(api);
-        if (onRoute) onRoute(api);
-      }
+      await context.close().catch(() => {});
+      await browser.close().catch(() => {});
     }
 
-    return this.#routes;
+    return routes;
   }
 
   async #probePage(context, url, depth, baseUrl) {
-    const page = await context.newPage();
-    const networkRequests = [];
-    const links = new Set();
+    const page             = await context.newPage();
+    const networkRequests  = [];
+    const links            = new Set();
 
     page.on('request', req => {
-      if (req.url().includes('/api/') || req.resourceType() === 'fetch') {
-        networkRequests.push({
-          url   : req.url(),
-          method: req.method(),
-          type  : 'api',
-        });
+      const u = req.url();
+      if ((u.includes('/api/') || req.resourceType() === 'fetch') && this.#sameOrigin(u, baseUrl)) {
+        networkRequests.push({ url: u, method: req.method(), type: 'api' });
       }
     });
 
     try {
-      const response = await page.goto(url, {
-        waitUntil: 'networkidle',
-        timeout  : 15_000,
-      });
-
-      const status      = response?.status() || 0;
-      const contentType = response?.headers()['content-type'] || '';
+      const response = await page.goto(url, { waitUntil: 'networkidle', timeout: 15_000 });
+      const status   = response?.status() || 0;
+      const ct       = response?.headers()['content-type'] || '';
 
-      // Skip non-HTML
-      if (!contentType.includes('text/html') && !contentType.includes('application/xhtml')) {
-        const type = this.#detectResourceType(url, contentType);
-        return { id: shortId(), url, type, status, depth, links: [], forms: [], apis: [] };
+      if (!ct.includes('text/html') && !ct.includes('application/xhtml')) {
+        return { id: shortId(), url, type: this.#detectType(url, ct, status), status, depth, links: [], forms: [] };
       }
 
-      // Extract all links
-      const hrefs = await page.$$eval('a[href]', els =>
-        els.map(el => el.href).filter(Boolean)
-      ).catch(() => []);
+      const hrefs = await page.$$eval('a[href]', els => els.map(e => e.href).filter(Boolean)).catch(() => []);
       hrefs.forEach(h => links.add(h));
 
-      // Extract forms
       const forms = await page.$$eval('form', els => els.map(f => ({
-        action: f.action,
-        method: f.method || 'GET',
-        fields : Array.from(f.elements).map(el => ({
-          name    : el.name,
-          type    : el.type,
-          required: el.required,
+        action: f.action, method: f.method || 'GET',
+        fields: Array.from(f.elements).map(el => ({
+          name: el.name, type: el.type, required: el.required,
         })).filter(f => f.name),
       }))).catch(() => []);
 
-      // Detect page type
-      const type = this.#detectPageType(url, status);
-
-      // Add API routes discovered via network
-      for (const req of networkRequests) {
-        if (this.#isSameOrigin(req.url, baseUrl)) {
-          links.add(req.url);
-        }
-      }
+      networkRequests.forEach(r => { if (this.#sameOrigin(r.url, baseUrl)) links.add(r.url); });
 
       return {
-        id   : shortId(),
-        url,
-        type,
-        status,
-        depth,
-        links: [...links],
-        forms,
-        apis : networkRequests,
-        contentType,
+        id: shortId(), url, type: this.#detectType(url, ct, status),
+        status, depth, links: [...links], forms, contentType: ct,
       };
-
     } catch (err) {
-      return {
-        id   : shortId(),
-        url,
-        type : 'error',
-        status: 0,
-        depth,
-        links: [],
-        forms: [],
-        error: err.message,
-      };
+      return { id: shortId(), url, type: 'error', status: 0, depth, links: [], forms: [], error: err.message };
     } finally {
       await page.close().catch(() => {});
     }
   }
 
-  async #discoverAPIs(baseUrl) {
-    const browser = await this.#playwright.chromium.launch({ headless: true });
-    const context = await browser.newContext({ ignoreHTTPSErrors: true });
-    const page    = await context.newPage();
-    const apis    = new Set();
-
-    page.on('request', req => {
-      const url = req.url();
-      if (
-        (url.includes('/api/') || url.includes('/graphql') || url.includes('/rest/')) &&
-        this.#isSameOrigin(url, baseUrl)
-      ) {
-        apis.add(JSON.stringify({ url, method: req.method() }));
-      }
-    });
-
-    try {
-      await page.goto(baseUrl, { waitUntil: 'networkidle', timeout: 20_000 });
-      // Interact briefly to trigger more API calls
-      await page.waitForTimeout(2000);
-    } catch {}
-
-    await page.close();
-    await context.close();
-    await browser.close();
-
-    return [...apis].map(s => {
-      const parsed = JSON.parse(s);
-      return { id: shortId(), url: parsed.url, method: parsed.method, type: 'api' };
-    });
-  }
-
-  #normalizeUrl(url) {
-    try {
-      const u = new URL(url);
-      u.hash  = '';
-      return u.toString().replace(/\/$/, '') || '/';
-    } catch { return null; }
+  #norm(url) {
+    try { const u = new URL(url); u.hash = ''; return u.toString(); }
+    catch { return null; }
   }
 
-  #isSameOrigin(url, base) {
-    try {
-      return new URL(url).origin === new URL(base).origin;
-    } catch { return false; }
+  #sameOrigin(url, base) {
+    try { return new URL(url).origin === new URL(base).origin; }
+    catch { return false; }
   }
 
-  #detectPageType(url, status) {
-    if (status === 0)   return 'error';
-    if (status >= 400)  return 'error-page';
-    const u = url.toLowerCase();
-    if (u.includes('/api/'))       return 'api';
-    if (u.includes('/graphql'))    return 'api';
-    if (u.endsWith('.json'))       return 'api';
-    if (u.endsWith('.xml'))        return 'resource';
-    if (u.endsWith('.txt'))        return 'resource';
-    if (/\/(login|signin|auth)/i.test(u))    return 'auth';
-    if (/\/(register|signup)/i.test(u))      return 'auth';
-    if (/\/(admin)/i.test(u))                return 'admin';
-    if (/\/(dashboard)/i.test(u))            return 'dashboard';
+  #detectType(url, ct, status) {
+    if (status >= 400) return 'error-page';
+    if (ct.includes('json') || url.includes('/api/')) return 'api';
+    if (url.endsWith('.xml') || url.endsWith('.txt')) return 'resource';
+    if (/\/(login|signin|auth)/i.test(url)) return 'auth';
+    if (/\/(admin)/i.test(url))             return 'admin';
+    if (/\/(dashboard)/i.test(url))         return 'dashboard';
     return 'page';
   }
-
-  #detectResourceType(url, ct) {
-    if (ct.includes('json'))       return 'api';
-    if (ct.includes('javascript')) return 'script';
-    if (ct.includes('css'))        return 'style';
-    if (ct.includes('image'))      return 'image';
-    if (url.endsWith('.xml'))      return 'resource';
-    return 'unknown';
-  }
 }

package/src/qa/browser/installer.js

@@ -0,0 +1,209 @@
+// ═══════════════════════════════════════════════════════════════════════════
+//  Playwright Auto-Installer & Browser Manager
+//  Handles missing browsers gracefully with auto-install
+// ═══════════════════════════════════════════════════════════════════════════
+
+import { execSync, spawn }  from 'node:child_process';
+import path                 from 'node:path';
+import fs                   from 'fs-extra';
+import chalk                from 'chalk';
+import ora                  from 'ora';
+
+// ── Browser executable paths per platform ────────────────────────────────
+function getPlaywrightCacheDir() {
+  const home = process.env.HOME || process.env.USERPROFILE || '';
+  switch (process.platform) {
+    case 'win32':  return path.join(process.env.LOCALAPPDATA || path.join(home, 'AppData', 'Local'), 'ms-playwright');
+    case 'darwin': return path.join(home, 'Library', 'Caches', 'ms-playwright');
+    default:       return path.join(home, '.cache', 'ms-playwright');
+  }
+}
+
+function findChromiumExecutable() {
+  const cacheDir = getPlaywrightCacheDir();
+
+  try {
+    if (!fs.existsSync(cacheDir)) return null;
+
+    const entries = fs.readdirSync(cacheDir);
+
+    // Look for any chromium variant folder
+    const chromiumDirs = entries
+      .filter(e => e.startsWith('chromium') || e.startsWith('chrome'))
+      .sort()
+      .reverse(); // prefer newest
+
+    for (const dir of chromiumDirs) {
+      const candidates = [
+        // Headless shell (newer Playwright)
+        path.join(cacheDir, dir, 'chrome-headless-shell-win64', 'chrome-headless-shell.exe'),
+        path.join(cacheDir, dir, 'chrome-headless-shell-linux', 'chrome-headless-shell'),
+        path.join(cacheDir, dir, 'chrome-headless-shell-mac_arm', 'chrome-headless-shell'),
+        path.join(cacheDir, dir, 'chrome-headless-shell-mac_x64', 'chrome-headless-shell'),
+        // Regular chromium
+        path.join(cacheDir, dir, 'chrome-win64',       'chrome.exe'),
+        path.join(cacheDir, dir, 'chrome-linux',        'chrome'),
+        path.join(cacheDir, dir, 'chrome-mac',          'Chromium.app', 'Contents', 'MacOS', 'Chromium'),
+        path.join(cacheDir, dir, 'chrome-mac_arm',      'Chromium.app', 'Contents', 'MacOS', 'Chromium'),
+        path.join(cacheDir, dir, 'chromium-linux',      'chrome-linux', 'chrome'),
+        path.join(cacheDir, dir, 'chromium-win64',      'chrome.exe'),
+        path.join(cacheDir, dir, 'chromium-mac',        'chrome-mac',   'Chromium.app', 'Contents', 'MacOS', 'Chromium'),
+      ];
+
+      for (const candidate of candidates) {
+        if (fs.existsSync(candidate)) return candidate;
+      }
+    }
+  } catch {}
+
+  return null;
+}
+
+// ── Check system Chrome/Chromium ──────────────────────────────────────────
+function findSystemChrome() {
+  const candidates = {
+    win32: [
+      'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
+      'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
+      path.join(process.env.LOCALAPPDATA || '', 'Google\\Chrome\\Application\\chrome.exe'),
+      'C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe',
+      path.join(process.env.PROGRAMFILES || '', 'Microsoft\\Edge\\Application\\msedge.exe'),
+    ],
+    darwin: [
+      '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+      '/Applications/Chromium.app/Contents/MacOS/Chromium',
+      '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
+    ],
+    linux: [
+      '/usr/bin/google-chrome',
+      '/usr/bin/google-chrome-stable',
+      '/usr/bin/chromium-browser',
+      '/usr/bin/chromium',
+      '/snap/bin/chromium',
+    ],
+  };
+
+  const platform = process.platform;
+  const paths    = candidates[platform] || candidates.linux;
+
+  for (const p of paths) {
+    try {
+      if (fs.existsSync(p)) return p;
+    } catch {}
+  }
+
+  return null;
+}
+
+// ── Install Playwright browsers ───────────────────────────────────────────
+export async function installPlaywrightBrowsers() {
+  const spinner = ora({
+    text   : chalk.cyan('Installing Playwright Chromium browser...'),
+    spinner: 'dots12',
+    color  : 'cyan',
+  }).start();
+
+  return new Promise((resolve) => {
+    const proc = spawn(
+      process.execPath,        // use same node binary
+      ['node_modules/.bin/playwright', 'install', 'chromium', '--with-deps'],
+      {
+        cwd   : process.cwd(),
+        stdio : ['ignore', 'pipe', 'pipe'],
+        shell : process.platform === 'win32',
+        env   : { ...process.env },
+      }
+    );
+
+    let output = '';
+    proc.stdout?.on('data', d => { output += d.toString(); });
+    proc.stderr?.on('data', d => { output += d.toString(); });
+
+    proc.on('close', (code) => {
+      if (code === 0) {
+        spinner.succeed(chalk.green('Chromium installed successfully ✓'));
+        resolve({ success: true });
+      } else {
+        spinner.fail(chalk.red(`Chromium install failed (code ${code})`));
+        resolve({ success: false, output });
+      }
+    });
+
+    proc.on('error', (err) => {
+      spinner.fail(chalk.red(`Install process error: ${err.message}`));
+      resolve({ success: false, error: err.message });
+    });
+
+    // Timeout after 3 minutes
+    setTimeout(() => {
+      proc.kill();
+      spinner.fail(chalk.red('Browser install timed out (3 min)'));
+      resolve({ success: false, error: 'timeout' });
+    }, 180_000);
+  });
+}
+
+// ── Main browser check & setup ────────────────────────────────────────────
+export async function ensureBrowser() {
+  // 1. Check Playwright-managed Chromium
+  const playwrightChrome = findChromiumExecutable();
+  if (playwrightChrome) {
+    return { executablePath: playwrightChrome, source: 'playwright-managed' };
+  }
+
+  // 2. Check system Chrome/Edge/Chromium
+  const systemChrome = findSystemChrome();
+  if (systemChrome) {
+    console.log(chalk.gray(`  Using system browser: ${systemChrome}`));
+    return { executablePath: systemChrome, source: 'system' };
+  }
+
+  // 3. Try to auto-install
+  console.log('');
+  console.log(chalk.yellow('  ⚠  Playwright browser not found.'));
+  console.log(chalk.gray('  Attempting automatic installation...\n'));
+
+  const result = await installPlaywrightBrowsers();
+
+  if (result.success) {
+    const newPath = findChromiumExecutable();
+    if (newPath) {
+      return { executablePath: newPath, source: 'auto-installed' };
+    }
+  }
+
+  // 4. Return null — caller will use HTTP-only fallback
+  return { executablePath: null, source: 'unavailable', error: result?.error };
+}
+
+// ── Check if browser is available (fast check) ───────────────────────────
+export function isBrowserAvailable() {
+  return !!(findChromiumExecutable() || findSystemChrome());
+}
+
+// ── Get launch options with correct executable ────────────────────────────
+export async function getBrowserLaunchOptions() {
+  const browser = await ensureBrowser();
+
+  const baseArgs = [
+    '--no-sandbox',
+    '--disable-setuid-sandbox',
+    '--disable-dev-shm-usage',
+    '--disable-gpu',
+    '--no-first-run',
+    '--no-default-browser-check',
+    '--disable-extensions',
+  ];
+
+  if (!browser.executablePath) {
+    return { available: false, source: 'unavailable' };
+  }
+
+  return {
+    available      : true,
+    source         : browser.source,
+    executablePath : browser.executablePath,
+    headless       : true,
+    args           : baseArgs,
+  };
+}