@purepageio/fetch-engines 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Purepage
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE. 

package/README.md

@@ -0,0 +1,205 @@
+# @purepageio/fetch-engines
+
+A collection of configurable engines for fetching HTML content using plain `fetch` or Playwright.
+
+This package provides robust and customizable ways to retrieve web page content, handling retries, caching, user agents, and optional browser automation via Playwright for complex JavaScript-driven sites.
+
+## Installation
+
+```bash
+pnpm add @purepageio/fetch-engines
+# or with npm
+npm install @purepageio/fetch-engines
+# or with yarn
+yarn add @purepageio/fetch-engines
+```
+
+If you plan to use the `PlaywrightEngine`, you also need to install Playwright's browser binaries:
+
+```bash
+pnpm exec playwright install
+# or
+npx playwright install
+```
+
+## Engines
+
+- **`FetchEngine`**: Uses the standard `fetch` API. Suitable for simple HTML pages or APIs returning HTML. Lightweight and fast.
+- **`PlaywrightEngine`**: Uses Playwright to control a headless browser (Chromium, Firefox, WebKit). Handles JavaScript rendering, complex interactions (if needed), and provides options for stealth and anti-bot detection measures. More resource-intensive but necessary for dynamic websites.
+
+## Basic Usage
+
+### FetchEngine
+
+```typescript
+import { FetchEngine } from "@purepageio/fetch-engines";
+
+const engine = new FetchEngine();
+
+async function main() {
+  try {
+    const url = "https://example.com";
+    const result = await engine.fetchHTML(url);
+    console.log(`Fetched ${result.url}`);
+    console.log(`Title: ${result.title}`);
+    // console.log(`HTML: ${result.html.substring(0, 200)}...`);
+  } catch (error) {
+    console.error("Fetch failed:", error);
+  }
+}
+
+main();
+```
+
+### PlaywrightEngine
+
+```typescript
+import { PlaywrightEngine } from "@purepageio/fetch-engines";
+
+// Configure engine options (optional)
+const engine = new PlaywrightEngine({
+  maxRetries: 2, // Number of retry attempts
+  useHttpFallback: true, // Try simple HTTP fetch first
+  cacheTTL: 5 * 60 * 1000, // Cache results for 5 minutes (in milliseconds)
+});
+
+async function main() {
+  try {
+    const url = "https://quotes.toscrape.com/"; // A site that might benefit from JS rendering
+    const result = await engine.fetchHTML(url);
+    console.log(`Fetched ${result.url}`);
+    console.log(`Title: ${result.title}`);
+    // console.log(`HTML: ${result.html.substring(0, 200)}...`);
+  } catch (error) {
+    console.error("Playwright fetch failed:", error);
+  } finally {
+    // Important: Clean up browser resources when done
+    await engine.cleanup();
+  }
+}
+
+main();
+```
+
+## Configuration
+
+Engines accept an optional configuration object in their constructor to customize behavior.
+
+### FetchEngine
+
+The `FetchEngine` currently has **no configurable options** via its constructor. It uses standard `fetch` with default browser/Node.js retry/timeout behavior and a fixed set of browser-like headers.
+
+### PlaywrightEngine
+
+The `PlaywrightEngine` offers more extensive configuration:
+
+**General Options:**
+
+- `concurrentPages` (`number`, default: `3`)
+  - Maximum number of Playwright pages to process concurrently across all browser instances.
+- `maxRetries` (`number`, default: `3`)
+  - Maximum number of retry attempts for a failed fetch operation (excluding initial attempt).
+- `retryDelay` (`number`, default: `5000`)
+  - Delay in milliseconds between retry attempts.
+- `cacheTTL` (`number`, default: `900000` (15 minutes))
+  - Time-to-live for cached results in milliseconds. Set to `0` to disable the in-memory cache.
+- `useHttpFallback` (`boolean`, default: `true`)
+  - If `true`, the engine first attempts a simple, fast HTTP GET request. If this fails or appears to receive a challenge/CAPTCHA page, it then proceeds with a full Playwright browser request.
+- `useHeadedModeFallback` (`boolean`, default: `false`)
+  - If `true` and a Playwright request fails (potentially due to bot detection), subsequent requests _to that specific domain_ will automatically use a headed (visible) browser instance, which can sometimes bypass stricter checks. This requires the pool to potentially manage both headless and headed instances.
+- `defaultFastMode` (`boolean`, default: `true`)
+  - If `true`, requests initially run in "fast mode", blocking non-essential resources (images, fonts, stylesheets) and skipping human behavior simulation. This can significantly speed up fetches but may break some sites or increase detection risk. This can be overridden per-request via the `fetchHTML` options.
+- `simulateHumanBehavior` (`boolean`, default: `true`)
+  - If `true` and the request is _not_ in `fastMode`, the engine attempts basic human-like interactions (e.g., slight delays, mouse movements). _Note: This simulation is currently basic and may not defeat advanced bot detection._
+
+**Browser Pool Options:**
+
+These options are passed down to configure the underlying `PlaywrightBrowserPool` that manages browser instances.
+
+- `maxBrowsers` (`number`, default: `2`)
+  - Maximum number of concurrent browser instances (e.g., Chrome processes) the pool will manage.
+- `maxPagesPerContext` (`number`, default: `6`)
+  - Maximum number of pages that can be opened within a single browser context (like an isolated browser profile) before the pool prefers using a different context or browser instance. Helps isolate sessions.
+- `maxBrowserAge` (`number`, default: `1200000` (20 minutes))
+  - Maximum age in milliseconds a browser instance can live before the pool proactively closes and replaces it. Helps mitigate memory leaks or state issues.
+- `healthCheckInterval` (`number`, default: `60000` (1 minute))
+  - How often (in milliseconds) the pool checks the health of its browser instances (e.g., checking connectivity, age).
+- `useHeadedMode` (`boolean`, default: `false`)
+  - Forces the _entire_ browser pool to launch browsers in headed (visible) mode instead of the default headless mode. Primarily useful for debugging purposes.
+- `poolBlockedDomains` (`string[]`, default: `[]` - uses pool's internal defaults)
+  - List of domain _glob patterns_ (e.g., `*.google-analytics.com`, `*.doubleclick.net`) for requests that the browser should block. An empty array uses the pool's built-in default blocklist (recommended).
+- `poolBlockedResourceTypes` (`string[]`, default: `[]` - uses pool's internal defaults)
+  - List of Playwright resource types (e.g., `image`, `stylesheet`, `font`, `media`, `websocket`) to block. Blocking unnecessary resources can speed up page loads. An empty array uses the pool's built-in default blocklist (recommended).
+- `proxy` (`object | undefined`, default: `undefined`)
+  - Proxy configuration to be used by the browser instances.
+    - `server` (`string`): Proxy URL (e.g., `http://host:port`, `socks5://user:pass@host:port`).
+    - `username` (`string`, optional): Proxy username.
+    - `password` (`string`, optional): Proxy password.
+
+## Return Value
+
+Both `FetchEngine.fetchHTML()` and `PlaywrightEngine.fetchHTML()` return a Promise that resolves to a `FetchResult` object with the following properties:
+
+- `html` (`string`): The full HTML content of the fetched page.
+- `title` (`string | null`): The extracted `<title>` tag content, or `null` if no title is found.
+- `url` (`string`): The final URL after any redirects.
+- `isFromCache` (`boolean`): `true` if the result was served from the engine's cache, `false` otherwise.
+- `statusCode` (`number | undefined`): The HTTP status code of the final response. This is typically available for `FetchEngine` and the HTTP fallback in `PlaywrightEngine`, but might be `undefined` for some Playwright navigation scenarios if the primary response wasn't directly captured.
+- `error` (`FetchError | Error | undefined`): If an error occurred during the _final_ fetch attempt (after retries), this property will contain the error object. It might be a specific `FetchError` (see Error Handling) or a generic `Error`.
+
+## API Reference
+
+### `engine.fetchHTML(url, options?)`
+
+- `url` (`string`): The URL of the page to fetch.
+- `options` (`object`, optional): Per-request options to override engine defaults.
+  - For `PlaywrightEngine`, you can override `fastMode` (`boolean`) to force or disable fast mode for this specific request.
+  - _(Other per-request options may be added in the future)._
+- **Returns:** `Promise<FetchResult>`
+
+Fetches the HTML content for the given URL using the engine's configured strategy (plain fetch or Playwright).
+
+### `engine.cleanup()` (PlaywrightEngine only)
+
+- **Returns:** `Promise<void>`
+
+Gracefully shuts down all browser instances managed by the `PlaywrightEngine`'s browser pool. **It is crucial to call `await engine.cleanup()` when you are finished using a `PlaywrightEngine` instance** to release system resources.
+
+## Stealth / Anti-Detection (`PlaywrightEngine`)
+
+The `PlaywrightEngine` automatically integrates `playwright-extra` and its powerful stealth plugin (`puppeteer-extra-plugin-stealth`). This plugin applies various techniques to make the headless browser controlled by Playwright appear more like a regular human-operated browser, helping to bypass many common bot detection systems.
+
+There are **no manual configuration options** for stealth; it is enabled by default when using `PlaywrightEngine`. The previous options (`useStealthMode`, `randomizeFingerprint`, `evasionLevel`) have been removed.
+
+While effective, be aware that no stealth technique is foolproof, and sophisticated websites may still detect automated browsing.
+
+## Error Handling
+
+Errors during fetching are typically thrown as instances of `FetchError` (or its subclasses like `FetchEngineHttpError`), providing more context than standard `Error` objects.
+
+- `FetchError` properties:
+  - `message` (`string`): Description of the error.
+  - `code` (`string | undefined`): A specific error code (e.g., `ERR_NAVIGATION_TIMEOUT`, `ERR_HTTP_ERROR`, `ERR_NON_HTML_CONTENT`).
+  - `originalError` (`Error | undefined`): The underlying error that caused this fetch error (e.g., a Playwright error object).
+
+Common error scenarios include:
+
+- Network issues (DNS resolution failure, connection refused).
+- HTTP errors (4xx client errors, 5xx server errors).
+- Non-HTML content type received (for `FetchEngine`).
+- Playwright navigation timeouts.
+- Proxy connection errors.
+- Page crashes within Playwright.
+- Errors thrown by the browser pool (e.g., failure to launch browser).
+
+The `FetchResult` object may also contain an `error` property if the final fetch attempt failed after all retries.
+
+## Logging
+
+## Contributing
+
+Contributions are welcome! Please open an issue or submit a pull request on the [GitHub repository](https://github.com/purepageio/fetch-engines).
+
+## License
+
+MIT

package/dist/FetchEngine.d.ts

@@ -0,0 +1,46 @@
+import type { HTMLFetchResult, BrowserMetrics } from "./types.js";
+import type { IEngine } from "./IEngine.js";
+/**
+ * Custom error class for HTTP errors from FetchEngine.
+ */
+export declare class FetchEngineHttpError extends Error {
+    readonly statusCode: number;
+    constructor(message: string, statusCode: number);
+}
+/**
+ * FetchEngine - A lightweight engine for fetching HTML content using the standard `fetch` API.
+ *
+ * Ideal for fetching content from static websites or APIs where JavaScript execution is not required.
+ * It does not support advanced configurations like retries, caching, or proxies directly.
+ */
+export declare class FetchEngine implements IEngine {
+    private readonly headers;
+    /**
+     * Creates an instance of FetchEngine.
+     * Note: This engine currently does not accept configuration options.
+     */
+    constructor();
+    /**
+     * Fetches HTML content from the specified URL using the `fetch` API.
+     *
+     * @param url The URL to fetch.
+     * @returns A Promise resolving to an HTMLFetchResult object.
+     * @throws {FetchEngineHttpError} If the HTTP response status is not ok (e.g., 404, 500).
+     * @throws {Error} If the content type is not HTML or for other network errors.
+     */
+    fetchHTML(url: string): Promise<HTMLFetchResult>;
+    private detectSPA;
+    /**
+     * Cleans up resources used by the engine.
+     * For FetchEngine, this is a no-op as it doesn't manage persistent resources.
+     * @returns A Promise that resolves when cleanup is complete.
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Retrieves metrics for the engine.
+     * FetchEngine does not manage browsers, so it returns an empty array.
+     * @returns An empty array.
+     */
+    getMetrics(): BrowserMetrics[];
+}
+//# sourceMappingURL=FetchEngine.d.ts.map

package/dist/FetchEngine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"FetchEngine.d.ts","sourceRoot":"","sources":["../src/FetchEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAClE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAG5C;;GAEG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,SAAgB,UAAU,EAAE,MAAM,CAAC;gBAEvB,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM;CAShD;AAED;;;;;GAKG;AACH,qBAAa,WAAY,YAAW,OAAO;IACzC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAyB;IAEjD;;;OAGG;;IAeH;;;;;;;OAOG;IACG,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAgDtD,OAAO,CAAC,SAAS;IA+BjB;;;;OAIG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAK9B;;;;OAIG;IACH,UAAU,IAAI,cAAc,EAAE;CAI/B"}

package/dist/FetchEngine.js

@@ -0,0 +1,137 @@
+import { JSDOM } from "jsdom";
+/**
+ * Custom error class for HTTP errors from FetchEngine.
+ */
+export class FetchEngineHttpError extends Error {
+    statusCode;
+    constructor(message, statusCode) {
+        super(message);
+        this.name = "FetchEngineHttpError";
+        this.statusCode = statusCode;
+        // Maintain proper stack trace (requires target ES2015+ in tsconfig)
+        if (Error.captureStackTrace) {
+            Error.captureStackTrace(this, FetchEngineHttpError);
+        }
+    }
+}
+/**
+ * FetchEngine - A lightweight engine for fetching HTML content using the standard `fetch` API.
+ *
+ * Ideal for fetching content from static websites or APIs where JavaScript execution is not required.
+ * It does not support advanced configurations like retries, caching, or proxies directly.
+ */
+export class FetchEngine {
+    headers;
+    /**
+     * Creates an instance of FetchEngine.
+     * Note: This engine currently does not accept configuration options.
+     */
+    constructor() {
+        this.headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+            Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Language": "en-US,en;q=0.5",
+            "Upgrade-Insecure-Requests": "1",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "none",
+            "Sec-Fetch-User": "?1",
+        };
+    }
+    /**
+     * Fetches HTML content from the specified URL using the `fetch` API.
+     *
+     * @param url The URL to fetch.
+     * @returns A Promise resolving to an HTMLFetchResult object.
+     * @throws {FetchEngineHttpError} If the HTTP response status is not ok (e.g., 404, 500).
+     * @throws {Error} If the content type is not HTML or for other network errors.
+     */
+    async fetchHTML(url) {
+        try {
+            const response = await fetch(url, {
+                headers: this.headers,
+                redirect: "follow",
+            });
+            if (!response.ok) {
+                // Throw the custom error with status code
+                throw new FetchEngineHttpError(`HTTP error! status: ${response.status}`, response.status);
+            }
+            const contentType = response.headers.get("content-type") || "";
+            if (!contentType.includes("text/html")) {
+                throw new Error("Not an HTML page");
+            }
+            const html = await response.text();
+            // Use JSDOM to parse HTML and extract title
+            const dom = new JSDOM(html);
+            const title = dom.window.document.title || "";
+            // Check for potential SPA markers
+            const isSPA = this.detectSPA(dom.window.document);
+            if (isSPA) {
+                // Removed throwing error here, as the calling code should decide how to handle this.
+                // Consider adding a flag to the result instead.
+                console.warn(`SPA detected for ${url}, content might be incomplete without JavaScript rendering.`);
+                // Example: return { html, title, url: response.url, isSPA: true };
+            }
+            return {
+                html,
+                title,
+                url: response.url,
+                isFromCache: false, // FetchEngine doesn't cache
+                statusCode: response.status,
+                error: undefined,
+            };
+        }
+        catch (error) {
+            // console.error(`FetchEngine failed for ${url}:`, error); // Optional: Keep logging if desired
+            // Re-throw the original error to preserve its type (e.g., FetchEngineHttpError)
+            // Ensure the result conforms to HTMLFetchResult even on error (for consistency? No, spec says throw)
+            throw error;
+        }
+    }
+    detectSPA(document) {
+        // Check for common SPA frameworks and patterns
+        const spaMarkers = [
+            // React
+            "[data-reactroot]",
+            "#root",
+            "#app",
+            // Vue
+            "[data-v-app]",
+            "#app[data-v-]",
+            // Angular
+            "[ng-version]",
+            "[ng-app]",
+            // Common SPA patterns
+            'script[type="application/json+ld"]', // Less reliable marker
+            'meta[name="fragment"]',
+        ];
+        // Check if the body is nearly empty but has JS (More reliable)
+        const bodyContent = document.body?.textContent?.trim() || "";
+        const hasScripts = document.scripts.length > 0;
+        if (bodyContent.length < 150 && hasScripts) {
+            // Increased threshold slightly
+            return true;
+        }
+        // Check for SPA markers (Less reliable)
+        return spaMarkers.some((selector) => document.querySelector(selector) !== null);
+    }
+    /**
+     * Cleans up resources used by the engine.
+     * For FetchEngine, this is a no-op as it doesn't manage persistent resources.
+     * @returns A Promise that resolves when cleanup is complete.
+     */
+    async cleanup() {
+        // No resources to clean up for fetch engine
+        return Promise.resolve(); // Explicitly return resolved promise
+    }
+    /**
+     * Retrieves metrics for the engine.
+     * FetchEngine does not manage browsers, so it returns an empty array.
+     * @returns An empty array.
+     */
+    getMetrics() {
+        // Fetch engine doesn't maintain browser pool metrics
+        return [];
+    }
+}
+//# sourceMappingURL=FetchEngine.js.map

package/dist/FetchEngine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"FetchEngine.js","sourceRoot":"","sources":["../src/FetchEngine.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAE9B;;GAEG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC7B,UAAU,CAAS;IAEnC,YAAY,OAAe,EAAE,UAAkB;QAC7C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;QACnC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,oEAAoE;QACpE,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,WAAW;IACL,OAAO,CAAyB;IAEjD;;;OAGG;IACH;QACE,IAAI,CAAC,OAAO,GAAG;YACb,YAAY,EACV,iHAAiH;YACnH,MAAM,EAAE,4EAA4E;YACpF,iBAAiB,EAAE,gBAAgB;YACnC,2BAA2B,EAAE,GAAG;YAChC,gBAAgB,EAAE,UAAU;YAC5B,gBAAgB,EAAE,UAAU;YAC5B,gBAAgB,EAAE,MAAM;YACxB,gBAAgB,EAAE,IAAI;SACvB,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,SAAS,CAAC,GAAW;QACzB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,QAAQ,EAAE,QAAQ;aACnB,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,0CAA0C;gBAC1C,MAAM,IAAI,oBAAoB,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC5F,CAAC;YAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC/D,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;YACtC,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEnC,4CAA4C;YAC5C,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5B,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;YAE9C,kCAAkC;YAClC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAClD,IAAI,KAAK,EAAE,CAAC;gBACV,qFAAqF;gBACrF,gDAAgD;gBAChD,OAAO,CAAC,IAAI,CAAC,oBAAoB,GAAG,6DAA6D,CAAC,CAAC;gBACnG,mEAAmE;YACrE,CAAC;YAED,OAAO;gBACL,IAAI;gBACJ,KAAK;gBACL,GAAG,EAAE,QAAQ,CAAC,GAAG;gBACjB,WAAW,EAAE,KAAK,EAAE,4BAA4B;gBAChD,UAAU,EAAE,QAAQ,CAAC,MAAM;gBAC3B,KAAK,EAAE,SAAS;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,+FAA+F;YAC/F,gFAAgF;YAChF,qGAAqG;YACrG,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,SAAS,CAAC,QAAkB;QAClC,+CAA+C;QAC/C,MAAM,UAAU,GAAG;YACjB,QAAQ;YACR,kBAAkB;YAClB,OAAO;YACP,MAAM;YACN,MAAM;YACN,cAAc;YACd,eAAe;YACf,UAAU;YACV,cAAc;YACd,UAAU;YACV,sBAAsB;YACtB,oCAAoC,EAAE,uBAAuB;YAC7D,uBAAuB;SACxB,CAAC;QAEF,+DAA+D;QAC/D,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC7D,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QAE/C,IAAI,WAAW,CAAC,MAAM,GAAG,GAAG,IAAI,UAAU,EAAE,CAAC;YAC3C,+BAA+B;YAC/B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,wCAAwC;QACxC,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;IAClF,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,4CAA4C;QAC5C,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,qCAAqC;IACjE,CAAC;IAED;;;;OAIG;IACH,UAAU;QACR,qDAAqD;QACrD,OAAO,EAAE,CAAC;IACZ,CAAC;CACF"}

package/dist/FetchEngine.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=FetchEngine.test.d.ts.map

package/dist/FetchEngine.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"FetchEngine.test.d.ts","sourceRoot":"","sources":["../src/FetchEngine.test.ts"],"names":[],"mappings":""}

package/dist/FetchEngine.test.js

@@ -0,0 +1,44 @@
+import { describe, it, expect } from "vitest";
+import { FetchEngine } from "./FetchEngine.js";
+describe("FetchEngine", () => {
+    it("should fetch HTML and extract title from a static page", async () => {
+        const engine = new FetchEngine();
+        const url = "http://example.com";
+        const expectedUrl = "http://example.com/"; // Expect trailing slash
+        try {
+            const result = await engine.fetchHTML(url);
+            expect(result).toBeDefined();
+            expect(result.url).toBe(expectedUrl); // Use expectedUrl
+            expect(result.title).toBe("Example Domain");
+            expect(result.html).toContain("<title>Example Domain</title>");
+            expect(result.html).toContain("<h1>Example Domain</h1>");
+        }
+        catch (error) {
+            // If the test environment doesn't have fetch or network access, this might fail.
+            // In a real CI/CD, ensure network access or mock fetch.
+            console.warn("FetchEngine test failed, potentially due to network issues or missing fetch API:", error);
+            // Re-throw to fail the test if fetch was expected to work
+            throw error;
+        }
+    });
+    it("should throw an error for non-HTML content", async () => {
+        const engine = new FetchEngine();
+        // Use a URL known to return non-HTML content, e.g., a JSON endpoint or an image
+        const url = "https://httpbin.org/json";
+        // Expect the fetchHTML method to reject
+        await expect(engine.fetchHTML(url)).rejects.toThrow("Not an HTML page");
+    });
+    it("should throw an error for non-existent domains", async () => {
+        const engine = new FetchEngine();
+        const url = "http://domain-that-does-not-exist-fdsahjkl.xyz";
+        // Expect the fetchHTML method to reject (error message might vary)
+        await expect(engine.fetchHTML(url)).rejects.toThrow();
+    });
+    it("should handle http errors", async () => {
+        const engine = new FetchEngine();
+        const url = "https://httpbin.org/status/404"; // URL that returns 404
+        await expect(engine.fetchHTML(url)).rejects.toThrow(/HTTP error! status: 404/);
+    });
+    // Add more tests: SPA detection warning, etc.
+});
+//# sourceMappingURL=FetchEngine.test.js.map

package/dist/FetchEngine.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"FetchEngine.test.js","sourceRoot":"","sources":["../src/FetchEngine.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE/C,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;QACtE,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,oBAAoB,CAAC;QACjC,MAAM,WAAW,GAAG,qBAAqB,CAAC,CAAC,wBAAwB;QAEnE,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,kBAAkB;YACxD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,+BAA+B,CAAC,CAAC;YAC/D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,yBAAyB,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,iFAAiF;YACjF,wDAAwD;YACxD,OAAO,CAAC,IAAI,CAAC,kFAAkF,EAAE,KAAK,CAAC,CAAC;YACxG,0DAA0D;YAC1D,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QACjC,gFAAgF;QAChF,MAAM,GAAG,GAAG,0BAA0B,CAAC;QAEvC,wCAAwC;QACxC,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,gDAAgD,CAAC;QAE7D,mEAAmE;QACnE,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;QACzC,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,gCAAgC,CAAC,CAAC,uBAAuB;QAErE,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;IACjF,CAAC,CAAC,CAAC;IAEH,8CAA8C;AAChD,CAAC,CAAC,CAAC"}

package/dist/HybridEngine.d.ts

@@ -0,0 +1,15 @@
+import type { HTMLFetchResult, BrowserMetrics, PlaywrightEngineConfig } from "./types.js";
+import { IEngine } from "./IEngine.js";
+/**
+ * HybridEngine - Attempts fetching with FetchEngine first for speed,
+ * then falls back to PlaywrightEngine for complex sites or specific errors.
+ */
+export declare class HybridEngine implements IEngine {
+    private readonly fetchEngine;
+    private readonly playwrightEngine;
+    constructor(playwrightConfig?: PlaywrightEngineConfig);
+    fetchHTML(url: string): Promise<HTMLFetchResult>;
+    cleanup(): Promise<void>;
+    getMetrics(): BrowserMetrics[];
+}
+//# sourceMappingURL=HybridEngine.d.ts.map

package/dist/HybridEngine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"HybridEngine.d.ts","sourceRoot":"","sources":["../src/HybridEngine.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,eAAe,EACf,cAAc,EACd,sBAAsB,EACvB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC;;;GAGG;AACH,qBAAa,YAAa,YAAW,OAAO;IAC1C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;gBAExC,gBAAgB,GAAE,sBAA2B;IAKnD,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAkBhD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ9B,UAAU,IAAI,cAAc,EAAE;CAI/B"}

package/dist/HybridEngine.js

@@ -0,0 +1,45 @@
+import { FetchEngine } from "./FetchEngine.js";
+import { PlaywrightEngine } from "./PlaywrightEngine.js";
+/**
+ * HybridEngine - Attempts fetching with FetchEngine first for speed,
+ * then falls back to PlaywrightEngine for complex sites or specific errors.
+ */
+export class HybridEngine {
+    fetchEngine;
+    playwrightEngine;
+    constructor(playwrightConfig = {}) {
+        this.fetchEngine = new FetchEngine();
+        this.playwrightEngine = new PlaywrightEngine(playwrightConfig);
+    }
+    async fetchHTML(url) {
+        try {
+            // Attempt 1: Use the fast FetchEngine
+            const fetchResult = await this.fetchEngine.fetchHTML(url);
+            return fetchResult;
+        }
+        catch (_fetchError) {
+            // Prefixed unused error
+            // If FetchEngine fails (e.g., 403, network error, non-html), try Playwright
+            try {
+                const playwrightResult = await this.playwrightEngine.fetchHTML(url);
+                return playwrightResult;
+            }
+            catch (playwrightError) {
+                // If Playwright also fails, throw its error (potentially more informative)
+                throw playwrightError;
+            }
+        }
+    }
+    async cleanup() {
+        // Cleanup both engines concurrently
+        await Promise.allSettled([
+            this.fetchEngine.cleanup(),
+            this.playwrightEngine.cleanup(),
+        ]);
+    }
+    getMetrics() {
+        // FetchEngine doesn't produce metrics, only PlaywrightEngine does
+        return this.playwrightEngine.getMetrics();
+    }
+}
+//# sourceMappingURL=HybridEngine.js.map

package/dist/HybridEngine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"HybridEngine.js","sourceRoot":"","sources":["../src/HybridEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAQzD;;;GAGG;AACH,MAAM,OAAO,YAAY;IACN,WAAW,CAAc;IACzB,gBAAgB,CAAmB;IAEpD,YAAY,mBAA2C,EAAE;QACvD,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,gBAAgB,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAW;QACzB,IAAI,CAAC;YACH,sCAAsC;YACtC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC1D,OAAO,WAAW,CAAC;QACrB,CAAC;QAAC,OAAO,WAAgB,EAAE,CAAC;YAC1B,wBAAwB;YACxB,4EAA4E;YAC5E,IAAI,CAAC;gBACH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBACpE,OAAO,gBAAgB,CAAC;YAC1B,CAAC;YAAC,OAAO,eAAe,EAAE,CAAC;gBACzB,2EAA2E;gBAC3E,MAAM,eAAe,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,oCAAoC;QACpC,MAAM,OAAO,CAAC,UAAU,CAAC;YACvB,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE;YAC1B,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE;SAChC,CAAC,CAAC;IACL,CAAC;IAED,UAAU;QACR,kEAAkE;QAClE,OAAO,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;IAC5C,CAAC;CACF"}

package/dist/IEngine.d.ts

@@ -0,0 +1,22 @@
+import type { HTMLFetchResult, BrowserMetrics } from "./types.js";
+/**
+ * Interface for browser engines that can fetch HTML content from URLs
+ */
+export interface IEngine {
+    /**
+     * Fetches HTML content from a URL
+     * @param url The URL to fetch
+     * @returns A promise that resolves to an HTMLFetchResult
+     */
+    fetchHTML(url: string): Promise<HTMLFetchResult>;
+    /**
+     * Cleans up resources used by the engine
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Gets metrics about the engine's performance
+     * @returns An array of BrowserMetrics
+     */
+    getMetrics(): BrowserMetrics[];
+}
+//# sourceMappingURL=IEngine.d.ts.map

package/dist/IEngine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"IEngine.d.ts","sourceRoot":"","sources":["../src/IEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAElE;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAEjD;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEzB;;;OAGG;IACH,UAAU,IAAI,cAAc,EAAE,CAAC;CAChC"}

package/dist/IEngine.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=IEngine.js.map

package/dist/IEngine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IEngine.js","sourceRoot":"","sources":["../src/IEngine.ts"],"names":[],"mappings":""}