@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/browser-pool.d.ts

@@ -0,0 +1,91 @@
+/**
+ * Browser lifecycle & page pool management.
+ * Handles Playwright loading, browser instances, and the idle page pool.
+ */
+import type { Browser, Page } from 'playwright';
+type ChromiumType = typeof import('playwright').chromium;
+import { closePool } from './http-fetch.js';
+export { closePool };
+/**
+ * Checks whether the Chromium binary bundled with our playwright alias exists.
+ * If missing, auto-installs it using `npx playwright install chromium` (which
+ * resolves to rebrowser-playwright because of the package.json alias).
+ *
+ * Call this once before the first browser launch so users get a helpful message
+ * instead of a confusing "Executable doesn't exist" stack trace.
+ */
+export declare function ensureChromiumInstalled(): Promise<void>;
+/** Whether Playwright has been loaded (for diagnostics). */
+export declare let playwrightLoaded: boolean;
+export declare function getStealthPlaywright(): Promise<ChromiumType>;
+/**
+ * Returns a realistic Chrome user agent.
+ * Delegates to the curated user-agents module so stealth mode never exposes
+ * the default "Chrome for Testing" UA which is a reliable bot-detection signal.
+ */
+export declare function getRandomUserAgent(): string;
+/**
+ * Common Chromium launch arguments for anti-bot-detection.
+ * Applied to BOTH regular and stealth browser instances.
+ * NOTE: --window-size is intentionally omitted here; it is added dynamically
+ * per browser launch using a random realistic viewport (see getRandomViewport()).
+ */
+export declare const ANTI_DETECTION_ARGS: readonly string[];
+/**
+ * Returns a random realistic viewport weighted by real-world market share.
+ * Used to avoid the telltale Playwright default of 1280×720.
+ */
+export declare function getRandomViewport(): {
+    width: number;
+    height: number;
+};
+/**
+ * Apply stealth init scripts to a page to reduce bot-detection signals:
+ * 1. Hides the `window.__pwInitScripts` Playwright leak.
+ * 2. Patches `navigator.userAgentData.brands` to include "Google Chrome"
+ *    (Chrome for Testing only ships "Chromium" which is a known detection signal).
+ */
+export declare function applyStealthScripts(page: Page, languages?: string[]): Promise<void>;
+/**
+ * Check current process memory usage against the pod limit.
+ * Returns { ok: true } when safe to launch a new browser context.
+ * Returns { ok: false } when memory is too high — caller should skip browser rendering.
+ */
+export declare function checkMemoryPressure(): {
+    ok: boolean;
+    rss: number;
+    heapUsed: number;
+    limit: number;
+};
+export declare const MAX_CONCURRENT_PAGES: number;
+export declare const PAGE_POOL_SIZE: number;
+export declare function removePooledPage(page: Page): void;
+export declare function takePooledPage(): Page | null;
+/** Returns the current number of pooled pages (for size checks in browser-fetch). */
+export declare function getPooledPagesCount(): number;
+export declare function ensurePagePool(browser?: Browser): Promise<void>;
+export declare function recyclePooledPage(page: Page): Promise<void>;
+export declare function getBrowser(): Promise<Browser>;
+export declare function getStealthBrowser(): Promise<Browser>;
+/**
+ * Get or create a browser instance with a persistent user data directory.
+ * Profile browsers bypass the shared browser pool so cookies/sessions survive
+ * between fetch calls.
+ *
+ * @param profileDir Absolute path to the Chrome user-data-dir directory
+ * @param headed     Whether to launch in headed (visible) mode
+ * @param stealth    Whether to use playwright-extra stealth instead of plain chromium
+ */
+export declare function getProfileBrowser(profileDir: string, headed?: boolean, stealth?: boolean): Promise<Browser>;
+export declare function warmup(): Promise<void>;
+/**
+ * Clean up browser resources (shared pool, stealth browser, and all profile browsers).
+ */
+export declare function cleanup(): Promise<void>;
+/**
+ * Close a specific persistent profile browser (e.g. when done with a session).
+ * Safe to call even if the browser has already been closed.
+ *
+ * @param profileDir Path to the profile directory used when launching
+ */
+export declare function closeProfileBrowser(profileDir: string): Promise<void>;

package/dist/core/browser-pool.js

@@ -0,0 +1,550 @@
+/**
+ * Browser lifecycle & page pool management.
+ * Handles Playwright loading, browser instances, and the idle page pool.
+ */
+import { getRealisticUserAgent } from './user-agents.js';
+import { startDnsWarmup } from './dns-cache.js';
+import { closePool } from './http-fetch.js';
+import { existsSync } from 'fs';
+import { execSync } from 'child_process';
+// Re-export closePool so fetcher.ts can barrel it from this module.
+export { closePool };
+// ── Browser auto-install ──────────────────────────────────────────────────────
+/**
+ * Checks whether the Chromium binary bundled with our playwright alias exists.
+ * If missing, auto-installs it using `npx playwright install chromium` (which
+ * resolves to rebrowser-playwright because of the package.json alias).
+ *
+ * Call this once before the first browser launch so users get a helpful message
+ * instead of a confusing "Executable doesn't exist" stack trace.
+ */
+export async function ensureChromiumInstalled() {
+    // Dynamically get the expected executable path from our playwright alias
+    let execPath;
+    try {
+        const pw = await import('playwright');
+        execPath = pw.chromium.executablePath();
+    }
+    catch {
+        // If playwright itself can't be imported we'll let the launch error surface
+        return;
+    }
+    if (existsSync(execPath)) {
+        return; // Already installed — fast path
+    }
+    // Binary is missing — auto-install with visible feedback
+    console.error('\x1b[33m⚙️  Installing browser for first-time use (this takes ~30s)...\x1b[0m');
+    console.error('\x1b[90m   Running: npx playwright install chromium\x1b[0m');
+    try {
+        execSync('npx playwright install chromium', {
+            stdio: 'inherit',
+            // Resolve npx relative to this package so we always use the aliased
+            // rebrowser-playwright, not whatever the user has globally.
+            cwd: new URL('../../..', import.meta.url).pathname,
+        });
+        console.error('\x1b[32m✅  Browser installed successfully.\x1b[0m');
+    }
+    catch (installErr) {
+        const msg = installErr instanceof Error ? installErr.message : String(installErr);
+        console.error(`\x1b[31m❌  Auto-install failed: ${msg}\x1b[0m`);
+        console.error('\x1b[31m   Please install the browser manually:\x1b[0m');
+        console.error('\x1b[36m   npx playwright install chromium\x1b[0m');
+        throw new Error('Chromium is not installed. Run: npx playwright install chromium');
+    }
+}
+// ── Playwright lazy loading ───────────────────────────────────────────────────
+let _chromium = null;
+let _stealthChromium = null;
+/** Whether Playwright has been loaded (for diagnostics). */
+export let playwrightLoaded = false;
+async function getPlaywright() {
+    if (!_chromium) {
+        await ensureChromiumInstalled();
+        const pw = await import('playwright');
+        _chromium = pw.chromium;
+        playwrightLoaded = true;
+    }
+    return _chromium;
+}
+export async function getStealthPlaywright() {
+    if (!_stealthChromium) {
+        const pwExtra = await import('playwright-extra');
+        const StealthPlugin = (await import('puppeteer-extra-plugin-stealth')).default;
+        _stealthChromium = pwExtra.chromium;
+        _stealthChromium.use(StealthPlugin());
+        playwrightLoaded = true;
+    }
+    return _stealthChromium;
+}
+// ── User agent & viewport helpers ─────────────────────────────────────────────
+/**
+ * Returns a realistic Chrome user agent.
+ * Delegates to the curated user-agents module so stealth mode never exposes
+ * the default "Chrome for Testing" UA which is a reliable bot-detection signal.
+ */
+export function getRandomUserAgent() {
+    return getRealisticUserAgent();
+}
+/**
+ * Common Chromium launch arguments for anti-bot-detection.
+ * Applied to BOTH regular and stealth browser instances.
+ * NOTE: --window-size is intentionally omitted here; it is added dynamically
+ * per browser launch using a random realistic viewport (see getRandomViewport()).
+ */
+export const ANTI_DETECTION_ARGS = [
+    '--disable-blink-features=AutomationControlled',
+    '--disable-infobars',
+    '--disable-dev-shm-usage',
+    '--no-sandbox',
+    '--disable-setuid-sandbox',
+    '--disable-gpu',
+    '--start-maximized',
+    // Chrome branding / stealth hardening
+    '--disable-features=ChromeUserAgentDataBranding,IsolateOrigins,site-per-process',
+    '--disable-component-extensions-with-background-pages',
+    '--disable-default-apps',
+    '--disable-extensions',
+    '--disable-hang-monitor',
+    '--disable-popup-blocking',
+    '--disable-prompt-on-repost',
+    '--disable-sync',
+    '--metrics-recording-only',
+    '--no-first-run',
+];
+/**
+ * Returns a random realistic viewport weighted by real-world market share.
+ * Used to avoid the telltale Playwright default of 1280×720.
+ */
+export function getRandomViewport() {
+    // Common real-world resolutions weighted by market share
+    const viewports = [
+        { width: 1920, height: 1080, weight: 35 }, // Full HD
+        { width: 1366, height: 768, weight: 20 }, // Laptop
+        { width: 1536, height: 864, weight: 15 }, // Scaled laptop
+        { width: 1440, height: 900, weight: 10 }, // MacBook
+        { width: 1680, height: 1050, weight: 8 }, // Large laptop
+        { width: 2560, height: 1440, weight: 7 }, // QHD
+        { width: 1280, height: 800, weight: 5 }, // Older laptop
+    ];
+    const total = viewports.reduce((s, v) => s + v.weight, 0);
+    let r = Math.random() * total;
+    for (const v of viewports) {
+        r -= v.weight;
+        if (r <= 0)
+            return { width: v.width, height: v.height };
+    }
+    return { width: 1920, height: 1080 };
+}
+/**
+ * Apply stealth init scripts to a page to reduce bot-detection signals:
+ * 1. Hides the `window.__pwInitScripts` Playwright leak.
+ * 2. Patches `navigator.userAgentData.brands` to include "Google Chrome"
+ *    (Chrome for Testing only ships "Chromium" which is a known detection signal).
+ */
+export async function applyStealthScripts(page, languages) {
+    const langList = languages && languages.length > 0 ? languages : ['en-US', 'en'];
+    // 1. Hide Playwright's __pwInitScripts marker
+    // Uses string form to avoid TypeScript DOM-lib requirements (tsconfig has no DOM lib).
+    await page.addInitScript(`
+    Object.defineProperty(window, '__pwInitScripts', {
+      get: () => undefined,
+      set: () => {},
+      configurable: true,
+    });
+  `);
+    // 2. Patch userAgentData brands to include "Google Chrome"
+    // Chrome for Testing only ships "Chromium" — a well-known bot-detection signal.
+    await page.addInitScript(`
+    (function () {
+      var uad = navigator.userAgentData;
+      if (!uad) return;
+      var originalBrands = uad.brands || [];
+      var hasChromeEntry = originalBrands.some(function(b) { return b.brand === 'Google Chrome'; });
+      if (hasChromeEntry) return;
+
+      var chromiumEntry = originalBrands.find(function(b) { return b.brand === 'Chromium'; });
+      var version = (chromiumEntry && chromiumEntry.version) || '136';
+      var patchedBrands = [
+        { brand: 'Chromium', version: version },
+        { brand: 'Google Chrome', version: version },
+        { brand: 'Not=A?Brand', version: '99' },
+      ];
+
+      Object.defineProperty(navigator, 'userAgentData', {
+        get: function() {
+          return {
+            brands: patchedBrands,
+            mobile: false,
+            platform: uad.platform || 'Windows',
+            getHighEntropyValues: uad.getHighEntropyValues ? uad.getHighEntropyValues.bind(uad) : undefined,
+            toJSON: function() {
+              return {
+                brands: patchedBrands,
+                mobile: false,
+                platform: uad.platform || 'Windows',
+              };
+            },
+          };
+        },
+        configurable: true,
+      });
+    })();
+  `);
+    // 3. Hide navigator.webdriver (THE #1 BOT SIGNAL)
+    await page.addInitScript(`
+    Object.defineProperty(navigator, 'webdriver', {
+      get: () => false,
+      configurable: true,
+    });
+    try { delete Object.getPrototypeOf(navigator).webdriver; } catch (e) {}
+  `);
+    // 4. Fake navigator.plugins (empty = bot signal, real Chrome has plugins)
+    await page.addInitScript(`
+    Object.defineProperty(navigator, 'plugins', {
+      get: () => {
+        var arr = [
+          { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
+          { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
+          { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' },
+        ];
+        arr.item = function(i) { return arr[i] || null; };
+        arr.namedItem = function(n) { return arr.find(function(p) { return p.name === n; }) || null; };
+        arr.refresh = function() {};
+        return arr;
+      },
+      configurable: true,
+    });
+  `);
+    // 5. Fake navigator.languages (use provided language preferences or default to en-US)
+    const langJson = JSON.stringify(langList);
+    await page.addInitScript(`Object.defineProperty(navigator, 'languages', { get: () => ${langJson}, configurable: true });`);
+    // 6. Fake window.chrome object (missing in headless = detected)
+    await page.addInitScript(`
+    if (!window.chrome) {
+      window.chrome = {
+        app: {
+          isInstalled: false,
+          InstallState: { INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' },
+          RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' }
+        },
+        runtime: {
+          OnInstalledReason: {}, OnRestartRequiredReason: {}, PlatformArch: {},
+          PlatformNaclArch: {}, PlatformOs: {}, RequestUpdateCheckStatus: {},
+          connect: function() {}, sendMessage: function() {}
+        },
+      };
+    }
+  `);
+    // 7. Fix permissions query (notifications should be 'prompt' not 'denied')
+    await page.addInitScript(`
+    try {
+      var originalQuery = window.Permissions && window.Permissions.prototype && window.Permissions.prototype.query;
+      if (originalQuery) {
+        window.Permissions.prototype.query = function(params) {
+          if (params && params.name === 'notifications') {
+            return Promise.resolve({ state: Notification.permission });
+          }
+          return originalQuery.call(this, params);
+        };
+      }
+    } catch (e) {}
+  `);
+    // 8. WebGL vendor/renderer spoofing (headless shows "Google SwiftShader")
+    await page.addInitScript(`
+    try {
+      var getParameter = WebGLRenderingContext.prototype.getParameter;
+      WebGLRenderingContext.prototype.getParameter = function(parameter) {
+        if (parameter === 37445) return 'Intel Inc.';
+        if (parameter === 37446) return 'Intel Iris OpenGL Engine';
+        return getParameter.call(this, parameter);
+      };
+      if (typeof WebGL2RenderingContext !== 'undefined') {
+        var getParameter2 = WebGL2RenderingContext.prototype.getParameter;
+        WebGL2RenderingContext.prototype.getParameter = function(parameter) {
+          if (parameter === 37445) return 'Intel Inc.';
+          if (parameter === 37446) return 'Intel Iris OpenGL Engine';
+          return getParameter2.call(this, parameter);
+        };
+      }
+    } catch (e) {}
+  `);
+    // 9. Hide automation-related properties
+    await page.addInitScript(`
+    try { Object.defineProperty(document, '$cdc_asdjflasutopfhvcZLmcfl_', { get: () => undefined }); } catch (e) {}
+    try { delete window.callPhantom; } catch (e) {}
+    try { delete window._phantom; } catch (e) {}
+    try { delete window.__nightmare; } catch (e) {}
+  `);
+}
+// ── Memory pressure guard ─────────────────────────────────────────────────────
+const MEMORY_LIMIT_MB = parseInt(process.env.MEMORY_LIMIT_MB || '768', 10); // Env-tunable guardrail for production pods
+/**
+ * Check current process memory usage against the pod limit.
+ * Returns { ok: true } when safe to launch a new browser context.
+ * Returns { ok: false } when memory is too high — caller should skip browser rendering.
+ */
+export function checkMemoryPressure() {
+    const mem = process.memoryUsage();
+    const rssMB = Math.round(mem.rss / 1024 / 1024);
+    const heapMB = Math.round(mem.heapUsed / 1024 / 1024);
+    return {
+        ok: rssMB < MEMORY_LIMIT_MB,
+        rss: rssMB,
+        heapUsed: heapMB,
+        limit: MEMORY_LIMIT_MB,
+    };
+}
+// ── Page pool constants & state ───────────────────────────────────────────────
+export const MAX_CONCURRENT_PAGES = parseInt(process.env.MAX_CONCURRENT_PAGES || '5', 10);
+export const PAGE_POOL_SIZE = parseInt(process.env.PAGE_POOL_SIZE || '3', 10);
+let sharedBrowser = null;
+let sharedStealthBrowser = null;
+const pooledPages = new Set();
+const idlePagePool = [];
+let pagePoolFillPromise = null;
+// ── Profile browser instances ─────────────────────────────────────────────────
+// Profile browsers are NOT shared — each profileDir gets its own instance.
+// These are keyed by profile path and kept alive between fetches in the same process.
+const profileBrowsers = new Map();
+// ── Pool helpers ──────────────────────────────────────────────────────────────
+export function removePooledPage(page) {
+    pooledPages.delete(page);
+    const idleIndex = idlePagePool.indexOf(page);
+    if (idleIndex >= 0) {
+        idlePagePool.splice(idleIndex, 1);
+    }
+}
+export function takePooledPage() {
+    while (idlePagePool.length > 0) {
+        const page = idlePagePool.shift();
+        if (page.isClosed()) {
+            removePooledPage(page);
+            continue;
+        }
+        return page;
+    }
+    return null;
+}
+/** Returns the current number of pooled pages (for size checks in browser-fetch). */
+export function getPooledPagesCount() {
+    return pooledPages.size;
+}
+export async function ensurePagePool(browser) {
+    const activeBrowser = browser ?? sharedBrowser;
+    if (!activeBrowser || !activeBrowser.isConnected()) {
+        return;
+    }
+    if (pagePoolFillPromise) {
+        await pagePoolFillPromise;
+        return;
+    }
+    pagePoolFillPromise = (async () => {
+        while (pooledPages.size < PAGE_POOL_SIZE) {
+            const pooledPage = await activeBrowser.newPage({
+                userAgent: getRandomUserAgent(),
+                viewport: null, // Use browser window size (set via --window-size at launch)
+            });
+            await applyStealthScripts(pooledPage);
+            pooledPages.add(pooledPage);
+            idlePagePool.push(pooledPage);
+        }
+    })().finally(() => {
+        pagePoolFillPromise = null;
+    });
+    await pagePoolFillPromise;
+}
+export async function recyclePooledPage(page) {
+    if (!pooledPages.has(page)) {
+        await page.close().catch(() => { });
+        return;
+    }
+    if (page.isClosed()) {
+        removePooledPage(page);
+        if (sharedBrowser?.isConnected()) {
+            void ensurePagePool(sharedBrowser).catch(() => { });
+        }
+        return;
+    }
+    try {
+        await page.unroute('**/*').catch(() => { });
+        await page.context().clearCookies().catch(() => { });
+        await page.setExtraHTTPHeaders({});
+        await page.goto('about:blank', { waitUntil: 'domcontentloaded', timeout: 5000 }).catch(() => { });
+        if (!idlePagePool.includes(page)) {
+            idlePagePool.push(page);
+        }
+    }
+    catch (e) {
+        // Non-fatal: page reset failed, removing from pool and closing
+        if (process.env.DEBUG)
+            console.debug('[webpeel]', 'page reset failed:', e instanceof Error ? e.message : e);
+        removePooledPage(page);
+        await page.close().catch(() => { });
+    }
+    if (sharedBrowser?.isConnected() && pooledPages.size < PAGE_POOL_SIZE) {
+        void ensurePagePool(sharedBrowser).catch(() => { });
+    }
+}
+// ── Browser getters ───────────────────────────────────────────────────────────
+export async function getBrowser() {
+    // SECURITY: Check if browser is still connected and healthy
+    if (sharedBrowser) {
+        try {
+            if (sharedBrowser.isConnected()) {
+                if (pooledPages.size < PAGE_POOL_SIZE) {
+                    void ensurePagePool(sharedBrowser).catch(() => { });
+                }
+                return sharedBrowser;
+            }
+        }
+        catch (e) {
+            // Browser is dead, recreate
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', 'shared browser health check failed, recreating:', e instanceof Error ? e.message : e);
+            sharedBrowser = null;
+        }
+    }
+    // Memory guard: check pressure before launching a new browser instance
+    const memPressure = checkMemoryPressure();
+    if (!memPressure.ok) {
+        // High memory — attempt to free existing resources first
+        console.warn(`[webpeel] Memory pressure detected (${memPressure.rss}MB RSS / ${memPressure.limit}MB limit). Cleaning up browser resources before launch.`);
+        // Close pooled pages to free memory
+        const pagesToClose = Array.from(pooledPages);
+        pooledPages.clear();
+        idlePagePool.length = 0;
+        pagePoolFillPromise = null;
+        await Promise.all(pagesToClose.map((page) => page.close().catch(() => { })));
+        // Close shared browser if it exists
+        if (sharedBrowser) {
+            await sharedBrowser.close().catch(() => { });
+            sharedBrowser = null;
+        }
+        // Re-check after cleanup
+        const memAfterCleanup = checkMemoryPressure();
+        if (!memAfterCleanup.ok) {
+            throw new Error(`[webpeel] Memory still too high after cleanup (${memAfterCleanup.rss}MB / ${memAfterCleanup.limit}MB). Skipping browser rendering to avoid OOM.`);
+        }
+        console.log(`[webpeel] Memory freed: ${memPressure.rss}MB → ${memAfterCleanup.rss}MB. Proceeding with browser launch.`);
+    }
+    pooledPages.clear();
+    idlePagePool.length = 0;
+    pagePoolFillPromise = null;
+    const vp = getRandomViewport();
+    const pw = await getPlaywright();
+    sharedBrowser = await pw.launch({
+        headless: true,
+        args: [...ANTI_DETECTION_ARGS, `--window-size=${vp.width},${vp.height}`],
+    });
+    void ensurePagePool(sharedBrowser).catch(() => { });
+    return sharedBrowser;
+}
+export async function getStealthBrowser() {
+    // SECURITY: Check if stealth browser is still connected and healthy
+    if (sharedStealthBrowser) {
+        try {
+            if (sharedStealthBrowser.isConnected()) {
+                return sharedStealthBrowser;
+            }
+        }
+        catch (e) {
+            // Browser is dead, recreate
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', 'stealth browser health check failed, recreating:', e instanceof Error ? e.message : e);
+            sharedStealthBrowser = null;
+        }
+    }
+    const stealthVp = getRandomViewport();
+    const stealthPw = await getStealthPlaywright();
+    const stealthBrowser = await stealthPw.launch({
+        headless: true,
+        args: [...ANTI_DETECTION_ARGS, `--window-size=${stealthVp.width},${stealthVp.height}`],
+    });
+    if (!stealthBrowser)
+        throw new Error('Failed to launch stealth browser');
+    sharedStealthBrowser = stealthBrowser;
+    return stealthBrowser;
+}
+/**
+ * Get or create a browser instance with a persistent user data directory.
+ * Profile browsers bypass the shared browser pool so cookies/sessions survive
+ * between fetch calls.
+ *
+ * @param profileDir Absolute path to the Chrome user-data-dir directory
+ * @param headed     Whether to launch in headed (visible) mode
+ * @param stealth    Whether to use playwright-extra stealth instead of plain chromium
+ */
+export async function getProfileBrowser(profileDir, headed = false, stealth = false) {
+    const existing = profileBrowsers.get(profileDir);
+    if (existing) {
+        try {
+            if (existing.isConnected())
+                return existing;
+        }
+        catch (e) {
+            // Profile browser is dead, recreate
+            if (process.env.DEBUG)
+                console.debug('[webpeel]', 'profile browser health check failed, recreating:', e instanceof Error ? e.message : e);
+        }
+        profileBrowsers.delete(profileDir);
+    }
+    const profileVp = getRandomViewport();
+    const launchOptions = {
+        headless: !headed,
+        args: [
+            ...ANTI_DETECTION_ARGS,
+            `--window-size=${profileVp.width},${profileVp.height}`,
+            `--user-data-dir=${profileDir}`,
+        ],
+    };
+    const launched = stealth
+        ? await (await getStealthPlaywright()).launch(launchOptions)
+        : await (await getPlaywright()).launch(launchOptions);
+    if (!launched)
+        throw new Error('Failed to launch profile browser');
+    profileBrowsers.set(profileDir, launched);
+    return launched;
+}
+// ── Warmup ────────────────────────────────────────────────────────────────────
+export async function warmup() {
+    startDnsWarmup();
+    const browser = await getBrowser();
+    await ensurePagePool(browser);
+}
+// ── Cleanup ───────────────────────────────────────────────────────────────────
+/**
+ * Clean up browser resources (shared pool, stealth browser, and all profile browsers).
+ */
+export async function cleanup() {
+    const pagesToClose = Array.from(pooledPages);
+    pooledPages.clear();
+    idlePagePool.length = 0;
+    pagePoolFillPromise = null;
+    await Promise.all(pagesToClose.map((page) => page.close().catch(() => { })));
+    if (sharedBrowser) {
+        await sharedBrowser.close();
+        sharedBrowser = null;
+    }
+    if (sharedStealthBrowser) {
+        await sharedStealthBrowser.close();
+        sharedStealthBrowser = null;
+    }
+    // Close all persistent profile browsers
+    const profileBrowserList = Array.from(profileBrowsers.values());
+    profileBrowsers.clear();
+    await Promise.all(profileBrowserList.map(b => b.close().catch(() => { })));
+    await closePool().catch(() => { });
+}
+/**
+ * Close a specific persistent profile browser (e.g. when done with a session).
+ * Safe to call even if the browser has already been closed.
+ *
+ * @param profileDir Path to the profile directory used when launching
+ */
+export async function closeProfileBrowser(profileDir) {
+    const browser = profileBrowsers.get(profileDir);
+    if (browser) {
+        profileBrowsers.delete(profileDir);
+        await browser.close().catch(() => { });
+    }
+}

package/dist/core/budget.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Smart content distillation for WebPeel
+ *
+ * Intelligently compresses content to fit within a token budget using
+ * heuristic-based techniques — no LLM required.
+ *
+ * This is NOT simple truncation: it prioritises information-dense content
+ * and progressively removes lower-value sections while preserving structure.
+ *
+ * @module budget
+ */
+/** Tokens per listing item used for budget estimation in extract-all mode */
+export declare const TOKENS_PER_LISTING_ITEM = 50;
+/**
+ * Distill content to fit within a token budget using smart compression.
+ *
+ * Strategy (applied progressively until within budget):
+ * 1. Remove image markdown — ![alt](url) → keep meaningful alt text
+ * 2. Remove boilerplate sections (cookie banners, nav headings, etc.)
+ * 3. Compress tables to MAX_TABLE_ROWS data rows
+ * 4. Collapse redundant whitespace
+ * 5. Remove low information-density paragraphs
+ * 6. Hard-truncate with notice as last resort
+ *
+ * @param content  The content string to distill
+ * @param budget   Maximum token budget (rough: 1 token ≈ 4 chars)
+ * @param format   Content format: 'markdown' | 'text' | 'json'
+ * @returns        Distilled content within the budget
+ */
+export declare function distillToBudget(content: string, budget: number, format: 'markdown' | 'text' | 'json'): string;
+/**
+ * Calculate how many listing items fit within a token budget.
+ *
+ * @param totalItems  Total available items
+ * @param budget      Token budget
+ * @returns           { maxItems, truncated, totalAvailable }
+ */
+export declare function budgetListings(totalItems: number, budget: number): {
+    maxItems: number;
+    truncated: boolean;
+    totalAvailable: number;
+};