@purepageio/fetch-engines 0.2.11 → 0.3.0

package/README.md

@@ -32,6 +32,7 @@ This package provides a high-level abstraction, letting you focus on using the w
 - [API Reference](#api-reference)
 - [Stealth / Anti-Detection (`PlaywrightEngine`)](#stealth--anti-detection-playwrightengine)
 - [Error Handling](#error-handling)
+- [Logging](#logging)
 - [Contributing](#contributing)
 - [License](#license)
 
@@ -106,8 +107,11 @@ main();
 ```typescript
 import { PlaywrightEngine } from "@purepageio/fetch-engines";
 
-// Engine configured to fetch HTML by default
-const engine = new PlaywrightEngine({ markdown: false });
+// Engine configured to fetch HTML by default and pass custom launch arguments
+const engine = new PlaywrightEngine({
+  markdown: false,
+  playwrightLaunchOptions: { args: ["--disable-gpu"] },
+});
 
 async function main() {
   try {
@@ -191,17 +195,20 @@ The `PlaywrightEngine` accepts a `PlaywrightEngineConfig` object with the follow
 
 **General Options:**
 
-| Option                  | Type      | Default  | Description                                                                                                                               |
-| ----------------------- | --------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------- |
-| `markdown`              | `boolean` | `false`  | If `true`, converts content (from Playwright or fallback) to Markdown. `contentType` will be `'markdown'`. Can be overridden per-request. |
-| `useHttpFallback`       | `boolean` | `true`   | If `true`, attempts a fast HTTP fetch before using Playwright.                                                                            |
-| `useHeadedModeFallback` | `boolean` | `false`  | If `true`, automatically retries specific failed domains in headed (visible) mode.                                                        |
-| `defaultFastMode`       | `boolean` | `true`   | If `true`, initially blocks non-essential resources and skips human simulation. Can be overridden per-request.                            |
-| `simulateHumanBehavior` | `boolean` | `true`   | If `true` (and not `fastMode`), attempts basic human-like interactions.                                                                   |
-| `concurrentPages`       | `number`  | `3`      | Max number of pages to process concurrently within the engine queue.                                                                      |
-| `maxRetries`            | `number`  | `3`      | Max retry attempts for a failed fetch (excluding initial try).                                                                            |
-| `retryDelay`            | `number`  | `5000`   | Delay (ms) between retries.                                                                                                               |
-| `cacheTTL`              | `number`  | `900000` | Cache Time-To-Live (ms). `0` disables caching. (15 mins default)                                                                          |
+| Option                    | Type            | Default     | Description                                                                                                                                                                                                                                                           |
+| ------------------------- | --------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `markdown`                | `boolean`       | `false`     | If `true`, converts content (from Playwright or its internal HTTP fallback) to Markdown. `contentType` will be `'markdown'`. Can be overridden per-request.                                                                                                           |
+| `useHttpFallback`         | `boolean`       | `true`      | If `true`, attempts a fast HTTP fetch before using Playwright. Ineffective if `spaMode` is `true`.                                                                                                                                                                    |
+| `useHeadedModeFallback`   | `boolean`       | `false`     | If `true`, automatically retries specific failed Playwright attempts in headed (visible) mode.                                                                                                                                                                        |
+| `defaultFastMode`         | `boolean`       | `true`      | If `true`, initially blocks non-essential resources and skips human simulation. Can be overridden per-request. Effectively `false` if `spaMode` is `true`.                                                                                                            |
+| `simulateHumanBehavior`   | `boolean`       | `true`      | If `true` (and not `fastMode` or `spaMode`), attempts basic human-like interactions.                                                                                                                                                                                  |
+| `concurrentPages`         | `number`        | `3`         | Max number of pages to process concurrently within the engine queue.                                                                                                                                                                                                  |
+| `maxRetries`              | `number`        | `3`         | Max retry attempts for a failed fetch (excluding initial try).                                                                                                                                                                                                        |
+| `retryDelay`              | `number`        | `5000`      | Delay (ms) between retries.                                                                                                                                                                                                                                           |
+| `cacheTTL`                | `number`        | `900000`    | Cache Time-To-Live (ms). `0` disables caching. (15 mins default)                                                                                                                                                                                                      |
+| `spaMode`                 | `boolean`       | `false`     | If `true`, enables Single Page Application mode. This typically bypasses `useHttpFallback`, effectively sets `fastMode` to `false`, uses more patient load conditions (e.g., network idle), and may apply `spaRenderDelayMs`. Recommended for JavaScript-heavy sites. |
+| `spaRenderDelayMs`        | `number`        | `0`         | Explicit delay (ms) after page load events in `spaMode` to allow for client-side rendering. Only applies if `spaMode` is `true`.                                                                                                                                      |
+| `playwrightLaunchOptions` | `LaunchOptions` | `undefined` | Optional Playwright launch options (from `playwright` package, e.g., `{ args: ['--some-flag'] }`) passed when a browser instance is created. Merged with internal defaults.                                                                                           |
 
 **Browser Pool Options (Passed to internal `PlaywrightBrowserPool`):**
 
@@ -218,14 +225,42 @@ The `PlaywrightEngine` accepts a `PlaywrightEngineConfig` object with the follow
 
 ### HybridEngine
 
-The `HybridEngine` constructor accepts a single optional argument which uses the **`PlaywrightEngineConfig`** structure (see the `PlaywrightEngine` tables above). These options configure the underlying engines where applicable:
+The `HybridEngine` constructor accepts `PlaywrightEngineConfig` options. These settings configure the underlying engines and the hybrid strategy:
 
-- Options like `maxRetries`, `cacheTTL`, `proxy`, `maxBrowsers`, etc., are primarily passed to the internal `PlaywrightEngine`.
-- The `markdown` setting in the constructor (`boolean`, default: `false`) applies to **both** internal engines by default.
-- If you provide `markdown: true` in the `options` object when calling `fetchHTML`, this override **only applies if a fallback to `PlaywrightEngine` is necessary**. The `FetchEngine` part will always use the `markdown` setting provided in the `HybridEngine` constructor.
+- **Constructor `markdown` option:**
+  - Sets the default Markdown conversion for the internal `FetchEngine`. This `FetchEngine` instance **does not** react to per-request `markdown` overrides.
+  - Sets the default for the internal `PlaywrightEngine`.
+- **Constructor `spaMode` option:**
+  - Sets the default SPA mode for `HybridEngine`. If `true`, `HybridEngine` checks `FetchEngine`'s output for SPA shell characteristics. If an SPA shell is detected, it forces a fallback to `PlaywrightEngine` (which will also run in SPA mode).
+  - Sets the default for the internal `PlaywrightEngine`.
+- **Other `PlaywrightEngineConfig` options** (e.g., `maxRetries`, `cacheTTL`, `playwrightLaunchOptions`, pool settings) are primarily passed to and used by the internal `PlaywrightEngine`.
+
+**Per-request `options` in `HybridEngine.fetchHTML(url, options)`:**
+
+- **`options.markdown` (`boolean`):**
+  - If `FetchEngine` succeeds and its content is used (i.e., not an SPA shell when `spaMode` is active), this per-request `markdown` option is **ignored**. The content's format is determined by the `FetchEngine`'s constructor `markdown` setting.
+  - If `HybridEngine` falls back to `PlaywrightEngine` (due to `FetchEngine` failure or SPA shell detection), this per-request `markdown` option **overrides** the `PlaywrightEngine`'s default and determines if its output is Markdown.
+- **`options.spaMode` (`boolean`):**
+  - Overrides the `HybridEngine`'s default SPA mode behavior for this specific request (affecting SPA shell detection and potential fallback to `PlaywrightEngine`).
+  - If `PlaywrightEngine` is used, this option also overrides its default SPA mode.
+- **`options.fastMode` (`boolean`):**
+  - If `PlaywrightEngine` is used, this option overrides its `defaultFastMode` setting. It has no effect on `FetchEngine`.
 
 ```typescript
-// ... (HybridEngine examples remain the same) ...
+// Example: HybridEngine with SPA mode enabled by default
+const spaHybridEngine = new HybridEngine({ spaMode: true, spaRenderDelayMs: 2000 });
+
+async function fetchSpaSite() {
+  try {
+    // This will use PlaywrightEngine directly if smallblackdots is an SPA shell
+    const result = await spaHybridEngine.fetchHTML(
+      "https://www.smallblackdots.net/release/16109/corrina-joseph-wish-tonite-lonely"
+    );
+    console.log(`Title: ${result.title}`);
+  } catch (e) {
+    console.error(e);
+  }
+}
 ```
 
 ## Return Value
@@ -248,6 +283,7 @@ All `fetchHTML()` methods return a Promise that resolves to an `HTMLFetchResult`
 - `options?` (`FetchOptions`): Optional per-request overrides.
   - `markdown?: boolean`: (Playwright/Hybrid only) Request Markdown conversion. For Hybrid, only applies on fallback to Playwright.
   - `fastMode?: boolean`: (Playwright/Hybrid only) Override fast mode.
+  - `spaMode?: boolean`: (Playwright/Hybrid only) Override SPA mode behavior for this request.
 - **Returns:** `Promise<HTMLFetchResult>`
 
 Fetches content, returning HTML or Markdown based on configuration/options in `result.content` with `result.contentType` indicating the format.
@@ -276,67 +312,72 @@ Errors during fetching are typically thrown as instances of `FetchError` (or its
   - `originalError` (`Error | undefined`): The underlying error that caused this fetch error (e.g., a Playwright error object).
   - `statusCode` (`number | undefined`): The HTTP status code, if relevant (especially for `FetchEngineHttpError`).
 
-Common error scenarios include:
-
-- Network issues (DNS resolution failure, connection refused).
-- HTTP errors (4xx client errors, 5xx server errors) -> `FetchEngineHttpError` from `FetchEngine` or potentially wrapped `FetchError` from `PlaywrightEngine`.
-- Non-HTML content type received -> `FetchError` with code `ERR_NON_HTML_CONTENT` from `FetchEngine`.
-- Playwright navigation timeouts -> `FetchError` wrapping Playwright error, often with code `ERR_NAVIGATION_TIMEOUT`.
-- Proxy connection errors.
-- Page crashes within Playwright.
-- Errors thrown by the browser pool (e.g., failure to launch browser).
+Common `FetchError` codes and scenarios:
+
+- **`ERR_HTTP_ERROR`**: Thrown by `FetchEngine` for HTTP status codes >= 400. `error.statusCode` will be set.
+- **`ERR_NON_HTML_CONTENT`**: Thrown by `FetchEngine` if the content type is not HTML and `markdown` conversion is not requested.
+- **`ERR_PLAYWRIGHT_OPERATION`**: A general error from `PlaywrightEngine` indicating a failure during a Playwright operation (e.g., page acquisition, navigation, interaction). The `originalError` property will often contain the specific Playwright error.
+- **`ERR_NAVIGATION`**: Often seen as part of `ERR_PLAYWRIGHT_OPERATION`'s message or in `originalError` when a Playwright navigation fails (e.g., timeout, SSL error).
+- **`ERR_MARKDOWN_CONVERSION_NON_HTML`**: Thrown by `PlaywrightEngine` (or `HybridEngine` if falling back to Playwright) if `markdown: true` is requested for a non-HTML content type (e.g., XML, JSON).
+- **`ERR_UNSUPPORTED_RAW_CONTENT_TYPE`**: Thrown by `PlaywrightEngine` if `markdown: false` is requested for a content type it doesn't support for direct fetching (e.g., images, applications). Currently, it primarily supports `text/*` and `application/json`, `application/xml` like types when `markdown: false`.
+- **`ERR_CACHE_ERROR`**: Indicates an issue with cache read/write operations.
+- **`ERR_PROXY_CONFIG_ERROR`**: Problem with proxy configuration.
+- **`ERR_BROWSER_POOL_EXHAUSTED`**: If the browser pool cannot provide a page (e.g. max browsers reached and all are busy beyond timeout).
+- **Other Scenarios (often wrapped by `ERR_PLAYWRIGHT_OPERATION` or a generic `FetchError`):**
+  - Network issues (DNS resolution, connection refused).
+  - Proxy connection failures.
+  - Page crashes or context/browser disconnections within Playwright.
+  - Failures during browser launch or management by the pool.
 
 The `HTMLFetchResult` object may also contain an `error` property if the final fetch attempt failed after all retries but an earlier attempt (within retries) might have produced some intermediate (potentially unusable) result data. It's generally best to rely on the thrown error for failure handling.
 
 **Example:**
 
 ```typescript
-import { FetchEngine, FetchError } from "@purepageio/fetch-engines";
+import { PlaywrightEngine, FetchError } from "@purepageio/fetch-engines";
 
-const engine = new FetchEngine();
+// Example using PlaywrightEngine to illustrate more complex error handling
+const engine = new PlaywrightEngine({ useHttpFallback: false, maxRetries: 1 });
 
 async function fetchWithHandling(url: string) {
   try {
     const result = await engine.fetchHTML(url);
-    // Note: result.error is less common, primary errors are thrown.
     if (result.error) {
-      console.error(`Fetch for ${url} reported error after retries: ${result.error.message}`);
-    } else {
-      console.log(`Success for ${url}! Content type: ${result.contentType}`);
-      // Use result.content
+      console.warn(`Fetch for ${url} included non-critical error after retries: ${result.error.message}`);
     }
+    console.log(`Success for ${url}! Title: ${result.title}, Content type: ${result.contentType}`);
+    // Use result.content
   } catch (error) {
-    console.error(`Fetch failed entirely for ${url}:`);
+    console.error(`Fetch failed for ${url}:`);
     if (error instanceof FetchError) {
-      // Handle specific FetchError codes
-      switch (error.code) {
-        case "ERR_HTTP_ERROR":
-          console.error(`  HTTP Error: Status ${error.statusCode} - ${error.message}`);
-          break;
-        case "ERR_NON_HTML_CONTENT":
-          console.error(`  Wrong Content Type: ${error.message}`);
-          break;
-        // Add other specific codes as needed
-        default:
-          console.error(`  FetchError (${error.code || "UNKNOWN"}): ${error.message}`);
-          break;
+      console.error(`  Error Code: ${error.code || "N/A"}`);
+      console.error(`  Message: ${error.message}`);
+      if (error.statusCode) {
+        console.error(`  Status Code: ${error.statusCode}`);
       }
       if (error.originalError) {
-        console.error(`  Original Error: ${error.originalError.message}`);
+        console.error(`  Original Error: ${error.originalError.name} - ${error.originalError.message}`);
+      }
+      // Example of specific handling:
+      if (error.code === "ERR_PLAYWRIGHT_OPERATION") {
+        console.error("  Hint: This was a Playwright operation failure. Check Playwright logs or originalError.");
       }
     } else if (error instanceof Error) {
-      // Handle generic JavaScript errors
       console.error(`  Generic Error: ${error.message}`);
     } else {
-      // Handle unexpected throw types
-      console.error(`  Unknown error occurred.`);
+      console.error(`  Unknown error occurred: ${String(error)}`);
     }
   }
 }
 
-fetchWithHandling("https://example.com");
-fetchWithHandling("https://httpbin.org/status/404"); // Example causing HTTP error
-fetchWithHandling("https://httpbin.org/image/png"); // Example causing non-HTML error
+async function runExamples() {
+  await fetchWithHandling("https://nonexistentdomain.example.com"); // Likely DNS or navigation error
+  await fetchWithHandling("https://example.com/non_html_resource.json"); // Test with actual JSON URL if available
+  // or a site known to cause Playwright issues for a demo.
+  await engine.cleanup(); // Important for PlaywrightEngine
+}
+
+runExamples();
 ```
 
 ## Logging

package/dist/HybridEngine.d.ts

@@ -7,7 +7,9 @@ export declare class HybridEngine implements IEngine {
     private readonly fetchEngine;
     private readonly playwrightEngine;
     private readonly config;
+    private readonly playwrightOnlyPatterns;
     constructor(config?: PlaywrightEngineConfig);
+    private _isSpaShell;
     fetchHTML(url: string, options?: FetchOptions): Promise<HTMLFetchResult>;
     /**
      * Delegates getMetrics to the PlaywrightEngine.

package/dist/HybridEngine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"HybridEngine.d.ts","sourceRoot":"","sources":["../src/HybridEngine.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,sBAAsB,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAExG;;GAEG;AACH,qBAAa,YAAa,YAAW,OAAO;IAC1C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IACpD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAyB;gBAEpC,MAAM,GAAE,sBAA2B;IAQzC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,eAAe,CAAC;IA8BlF;;OAEG;IACH,UAAU,IAAI,cAAc,EAAE;IAI9B;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAM/B"}
+{"version":3,"file":"HybridEngine.d.ts","sourceRoot":"","sources":["../src/HybridEngine.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,sBAAsB,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAExG;;GAEG;AACH,qBAAa,YAAa,YAAW,OAAO;IAC1C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IACpD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAyB;IAChD,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAsB;gBAEjD,MAAM,GAAE,sBAA2B;IAU/C,OAAO,CAAC,WAAW;IAkBb,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,eAAe,CAAC;IA+DlF;;OAEG;IACH,UAAU,IAAI,cAAc,EAAE;IAI9B;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAM/B"}

package/dist/HybridEngine.js

@@ -7,39 +7,84 @@ export class HybridEngine {
     fetchEngine;
     playwrightEngine;
     config; // Store config for potential per-request PW overrides
+    playwrightOnlyPatterns;
     constructor(config = {}) {
         // Pass relevant config parts to each engine
         // FetchEngine only takes markdown option from the shared config
+        // spaMode from config is primarily for PlaywrightEngine, but HybridEngine uses it for decision making.
         this.fetchEngine = new FetchEngine({ markdown: config.markdown });
         this.playwrightEngine = new PlaywrightEngine(config);
         this.config = config; // Store for merging later
+        this.playwrightOnlyPatterns = config.playwrightOnlyPatterns || [];
+    }
+    _isSpaShell(htmlContent) {
+        if (!htmlContent || htmlContent.length < 150) {
+            // Very short content might be a shell or error
+            // Heuristic: if it's very short AND contains noscript, good chance it's a shell.
+            if (htmlContent.includes("<noscript>"))
+                return true;
+        }
+        // Check for <noscript> tag
+        if (htmlContent.includes("<noscript>"))
+            return true;
+        // Check for common empty root divs
+        if (/<div id=(?:"|')?(root|app)(?:"|')?[^>]*>\s*<\/div>/i.test(htmlContent))
+            return true;
+        // Check for empty title tag or no title tag at all
+        if (/<title>\s*<\/title>/i.test(htmlContent) || !/<title[^>]*>/i.test(htmlContent))
+            return true;
+        return false;
     }
     async fetchHTML(url, options = {}) {
-        // FetchEngine uses its constructor config; it doesn't accept per-request options here.
+        // Determine effective SPA mode and markdown options
+        // HybridEngine defaults to false for these if not otherwise specified in its own config or per-request options.
+        const effectiveSpaMode = options.spaMode !== undefined ? options.spaMode : this.config.spaMode !== undefined ? this.config.spaMode : false;
+        const effectiveMarkdown = options.markdown !== undefined
+            ? options.markdown
+            : this.config.markdown !== undefined
+                ? this.config.markdown
+                : false;
+        // Prepare options for PlaywrightEngine, to be used in fallback scenarios or direct calls
+        const playwrightOptions = {
+            ...this.config, // Start with base config given to HybridEngine (e.g. spaRenderDelayMs)
+            ...options, // Apply all per-request overrides first
+            markdown: effectiveMarkdown, // Then ensure HybridEngine's resolved markdown is set
+            spaMode: effectiveSpaMode, // Then ensure HybridEngine's resolved spaMode is set
+        };
+        // Check playwrightOnlyPatterns first
+        for (const pattern of this.playwrightOnlyPatterns) {
+            if (typeof pattern === "string" && url.includes(pattern)) {
+                console.warn(`HybridEngine: URL ${url} matches string pattern "${pattern}". Using PlaywrightEngine directly.`);
+                return this.playwrightEngine.fetchHTML(url, playwrightOptions);
+            }
+            else if (pattern instanceof RegExp && pattern.test(url)) {
+                console.warn(`HybridEngine: URL ${url} matches regex pattern "${pattern.toString()}". Using PlaywrightEngine directly.`);
+                return this.playwrightEngine.fetchHTML(url, playwrightOptions);
+            }
+        }
         try {
             const fetchResult = await this.fetchEngine.fetchHTML(url);
-            // If fetch succeeded, return its result directly (it handles its own markdown config)
-            // No need to check contentType here, FetchEngine handles it based on its constructor.
+            // If FetchEngine succeeded AND spaMode is active, check if it's just a shell
+            if (effectiveSpaMode && fetchResult && fetchResult.content) {
+                if (this._isSpaShell(fetchResult.content)) {
+                    console.warn(`HybridEngine: FetchEngine returned likely SPA shell for ${url} in spaMode. Forcing PlaywrightEngine.`);
+                    // Fallback to PlaywrightEngine, passing the determined effective options
+                    return this.playwrightEngine.fetchHTML(url, playwrightOptions);
+                }
+            }
+            // If not spaMode, or if spaMode but content is not a shell, return FetchEngine's result
             return fetchResult;
         }
         catch (fetchError) {
-            console.warn(`FetchEngine failed for ${url}: ${fetchError.message}. Falling back to PlaywrightEngine.`);
-            // Merge constructor config with per-request options for Playwright fallback
-            const playwrightOptions = {
-                ...this.config, // Start with base config given to HybridEngine
-                ...options, // Override with per-request options
-            };
+            console.warn(`HybridEngine: FetchEngine failed for ${url}: ${fetchError.message}. Falling back to PlaywrightEngine.`);
             try {
-                // Pass merged options to PlaywrightEngine
+                // Fallback to PlaywrightEngine, passing the determined effective options
                 const playwrightResult = await this.playwrightEngine.fetchHTML(url, playwrightOptions);
                 return playwrightResult;
             }
             catch (playwrightError) {
-                // Catch potential Playwright error
-                console.error(`PlaywrightEngine fallback failed for ${url}: ${playwrightError.message}`);
-                // Optionally, wrap or prioritize which error to throw
-                // Throwing the Playwright error as it's the last one encountered
-                throw playwrightError;
+                console.error(`HybridEngine: PlaywrightEngine fallback also failed for ${url}: ${playwrightError.message}`);
+                throw playwrightError; // Throw the Playwright error as it's the last one encountered
             }
         }
     }

package/dist/HybridEngine.js.map

@@ -1 +1 @@
-{"version":3,"file":"HybridEngine.js","sourceRoot":"","sources":["../src/HybridEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAIzD;;GAEG;AACH,MAAM,OAAO,YAAY;IACN,WAAW,CAAc;IACzB,gBAAgB,CAAmB;IACnC,MAAM,CAAyB,CAAC,sDAAsD;IAEvG,YAAY,SAAiC,EAAE;QAC7C,4CAA4C;QAC5C,gEAAgE;QAChE,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,0BAA0B;IAClD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,UAAwB,EAAE;QACrD,uFAAuF;QACvF,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC1D,sFAAsF;YACtF,sFAAsF;YACtF,OAAO,WAAW,CAAC;QACrB,CAAC;QAAC,OAAO,UAAe,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,0BAA0B,GAAG,KAAK,UAAU,CAAC,OAAO,qCAAqC,CAAC,CAAC;YAExG,4EAA4E;YAC5E,MAAM,iBAAiB,GAAiB;gBACtC,GAAG,IAAI,CAAC,MAAM,EAAE,+CAA+C;gBAC/D,GAAG,OAAO,EAAE,oCAAoC;aACjD,CAAC;YAEF,IAAI,CAAC;gBACH,0CAA0C;gBAC1C,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;gBACvF,OAAO,gBAAgB,CAAC;YAC1B,CAAC;YAAC,OAAO,eAAoB,EAAE,CAAC;gBAC9B,mCAAmC;gBACnC,OAAO,CAAC,KAAK,CAAC,wCAAwC,GAAG,KAAK,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC;gBACzF,sDAAsD;gBACtD,iEAAiE;gBACjE,MAAM,eAAe,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,MAAM,OAAO,CAAC,UAAU,CAAC;YACvB,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,yCAAyC;YACrE,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE;SAChC,CAAC,CAAC;IACL,CAAC;CACF"}
+{"version":3,"file":"HybridEngine.js","sourceRoot":"","sources":["../src/HybridEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAIzD;;GAEG;AACH,MAAM,OAAO,YAAY;IACN,WAAW,CAAc;IACzB,gBAAgB,CAAmB;IACnC,MAAM,CAAyB,CAAC,sDAAsD;IACtF,sBAAsB,CAAsB;IAE7D,YAAY,SAAiC,EAAE;QAC7C,4CAA4C;QAC5C,gEAAgE;QAChE,uGAAuG;QACvG,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,0BAA0B;QAChD,IAAI,CAAC,sBAAsB,GAAG,MAAM,CAAC,sBAAsB,IAAI,EAAE,CAAC;IACpE,CAAC;IAEO,WAAW,CAAC,WAAmB;QACrC,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC7C,+CAA+C;YAC/C,iFAAiF;YACjF,IAAI,WAAW,CAAC,QAAQ,CAAC,YAAY,CAAC;gBAAE,OAAO,IAAI,CAAC;QACtD,CAAC;QACD,2BAA2B;QAC3B,IAAI,WAAW,CAAC,QAAQ,CAAC,YAAY,CAAC;YAAE,OAAO,IAAI,CAAC;QAEpD,mCAAmC;QACnC,IAAI,qDAAqD,CAAC,IAAI,CAAC,WAAW,CAAC;YAAE,OAAO,IAAI,CAAC;QAEzF,mDAAmD;QACnD,IAAI,sBAAsB,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,WAAW,CAAC;YAAE,OAAO,IAAI,CAAC;QAEhG,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,UAAwB,EAAE;QACrD,oDAAoD;QACpD,gHAAgH;QAChH,MAAM,gBAAgB,GACpB,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;QACpH,MAAM,iBAAiB,GACrB,OAAO,CAAC,QAAQ,KAAK,SAAS;YAC5B,CAAC,CAAC,OAAO,CAAC,QAAQ;YAClB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS;gBAClC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ;gBACtB,CAAC,CAAC,KAAK,CAAC;QAEd,yFAAyF;QACzF,MAAM,iBAAiB,GAA6D;YAClF,GAAG,IAAI,CAAC,MAAM,EAAE,uEAAuE;YACvF,GAAG,OAAO,EAAE,wCAAwC;YACpD,QAAQ,EAAE,iBAAiB,EAAE,sDAAsD;YACnF,OAAO,EAAE,gBAAgB,EAAE,qDAAqD;SACjF,CAAC;QAEF,qCAAqC;QACrC,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAClD,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzD,OAAO,CAAC,IAAI,CAAC,qBAAqB,GAAG,4BAA4B,OAAO,qCAAqC,CAAC,CAAC;gBAC/G,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;YACjE,CAAC;iBAAM,IAAI,OAAO,YAAY,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1D,OAAO,CAAC,IAAI,CACV,qBAAqB,GAAG,2BAA2B,OAAO,CAAC,QAAQ,EAAE,qCAAqC,CAC3G,CAAC;gBACF,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;YACjE,CAAC;QACH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAE1D,6EAA6E;YAC7E,IAAI,gBAAgB,IAAI,WAAW,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;gBAC3D,IAAI,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC1C,OAAO,CAAC,IAAI,CACV,2DAA2D,GAAG,wCAAwC,CACvG,CAAC;oBACF,yEAAyE;oBACzE,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;gBACjE,CAAC;YACH,CAAC;YACD,wFAAwF;YACxF,OAAO,WAAW,CAAC;QACrB,CAAC;QAAC,OAAO,UAAe,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CACV,wCAAwC,GAAG,KAAK,UAAU,CAAC,OAAO,qCAAqC,CACxG,CAAC;YACF,IAAI,CAAC;gBACH,yEAAyE;gBACzE,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;gBACvF,OAAO,gBAAgB,CAAC;YAC1B,CAAC;YAAC,OAAO,eAAoB,EAAE,CAAC;gBAC9B,OAAO,CAAC,KAAK,CAAC,2DAA2D,GAAG,KAAK,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC5G,MAAM,eAAe,CAAC,CAAC,8DAA8D;YACvF,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,MAAM,OAAO,CAAC,UAAU,CAAC;YACvB,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,yCAAyC;YACrE,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE;SAChC,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/PlaywrightEngine.d.ts

@@ -51,19 +51,48 @@ export declare class PlaywrightEngine implements IEngine {
      * @param url The URL to fetch.
      * @param options Optional settings for this specific fetch operation.
      * @param options.fastMode Overrides the engine's `defaultFastMode` configuration for this request.
+     * @param options.spaMode Overrides the engine's `spaMode` configuration for this request.
      * @returns A Promise resolving to an HTMLFetchResult object.
      * @throws {FetchError} If the fetch fails after all retries or encounters critical errors.
      */
     fetchHTML(url: string, options?: FetchOptions & {
         markdown?: boolean;
+        spaMode?: boolean;
     }): Promise<HTMLFetchResult>;
+    /**
+     * Helper to check cache and potentially return a cached result.
+     * Handles logic for re-fetching if cache is stale or content type mismatch for markdown.
+     *
+     * @param url URL to check in cache
+     * @param currentConfig Current fetch configuration
+     * @returns Cached result or null if not found/needs re-fetch.
+     */
+    private _handleCacheCheck;
+    /**
+     * Attempts to fetch the URL using a simple HTTP GET request as a fallback.
+     *
+     * @param url The URL to fetch.
+     * @param currentConfig The current fetch configuration.
+     * @returns A Promise resolving to an HTMLFetchResult if successful, or null if fallback is skipped or a challenge page is encountered.
+     * @throws {FetchError} If the HTTP fallback itself fails with an unrecoverable error.
+     */
+    private _attemptHttpFallback;
+    /**
+     * Ensures the browser pool is initialized with the correct mode (headed/headless).
+     * Handles one retry attempt if the initial pool initialization fails.
+     *
+     * @param useHeadedMode Whether to initialize the pool in headed mode.
+     * @param currentConfig The current fetch configuration (for retryDelay).
+     * @returns A Promise that resolves when the pool is initialized, or rejects if initialization fails after retries.
+     * @throws {FetchError} If pool initialization fails after retries or if the pool is unavailable.
+     */
+    private _ensureBrowserPoolInitialized;
     /**
      * Internal recursive method to handle fetching with retries.
      *
      * @param url URL to fetch
      * @param currentConfig The merged configuration including markdown option
      * @param retryAttempt Current retry attempt number (starts at 0)
-     * @param parentRetryCount Tracks retries related to pool initialization errors (starts at 0)
      * @returns Promise resolving to HTMLFetchResult
      */
     private _fetchRecursive;

package/dist/PlaywrightEngine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"PlaywrightEngine.d.ts","sourceRoot":"","sources":["../src/PlaywrightEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,sBAAsB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AACxG,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAmB5C;;;;;;GAMG;AACH,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsC;IAC5D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmC;IAG1D,OAAO,CAAC,uBAAuB,CAAkB;IACjD,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,mBAAmB,CAA0B;IAGrD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAkBpC;IAEF;;;;;OAKG;gBACS,MAAM,GAAE,sBAA2B;IAM/C;;OAEG;YACW,qBAAqB;IAuCnC;;;OAGG;YACW,yBAAyB;IAmFvC,OAAO,CAAC,UAAU;IAalB;;OAEG;YACW,WAAW;IAazB;;OAEG;YACW,qBAAqB;IAqCnC;;OAEG;IACH,OAAO,CAAC,UAAU;IAUlB;;;;;;;;OAQG;IACG,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAY,GAAG;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,OAAO,CAAC,eAAe,CAAC;IAU3G;;;;;;;;OAQG;YACW,eAAe;IAsH7B;;;OAGG;YACW,mBAAmB;YAmJnB,kBAAkB;IAmChC;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAe9B;;;OAGG;IACH,UAAU,IAAI,cAAc,EAAE;IAQ9B,OAAO,CAAC,mBAAmB;CAS5B"}
+{"version":3,"file":"PlaywrightEngine.d.ts","sourceRoot":"","sources":["../src/PlaywrightEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,sBAAsB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AACxG,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAyC5C;;;;;;GAMG;AACH,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsC;IAC5D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiC;IAGxD,OAAO,CAAC,uBAAuB,CAAkB;IACjD,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,mBAAmB,CAA0B;IAGrD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAsBpC;IAEF;;;;;OAKG;gBACS,MAAM,GAAE,sBAA2B;IAM/C;;OAEG;YACW,qBAAqB;IAwCnC;;;OAGG;YACW,yBAAyB;IAiEvC,OAAO,CAAC,UAAU;IAalB;;OAEG;YACW,WAAW;IAazB;;OAEG;YACW,qBAAqB;IAwCnC;;OAEG;IACH,OAAO,CAAC,UAAU;IAUlB;;;;;;;;;OASG;IACG,SAAS,CACb,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,YAAY,GAAG;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,CAAA;KAAO,GACrE,OAAO,CAAC,eAAe,CAAC;IAc3B;;;;;;;OAOG;IACH,OAAO,CAAC,iBAAiB;IAmDzB;;;;;;;OAOG;YACW,oBAAoB;IAiClC;;;;;;;;OAQG;YACW,6BAA6B;IAmC3C;;;;;;;OAOG;YACW,eAAe;IAgH7B;;;OAGG;YACW,mBAAmB;YAqKnB,kBAAkB;IAyChC;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAoB9B;;;OAGG;IACH,UAAU,IAAI,cAAc,EAAE;IAQ9B,OAAO,CAAC,mBAAmB;CAS5B"}