@purepageio/fetch-engines 0.1.4 → 0.2.1

package/dist/FetchEngine.js

@@ -1,17 +1,14 @@
-import { JSDOM } from "jsdom";
+import { MarkdownConverter } from "./utils/markdown-converter.js"; // Import the converter
+import { FetchError } from "./errors.js"; // Only import FetchError
 /**
  * Custom error class for HTTP errors from FetchEngine.
  */
-export class FetchEngineHttpError extends Error {
+export class FetchEngineHttpError extends FetchError {
     statusCode;
     constructor(message, statusCode) {
-        super(message);
-        this.name = "FetchEngineHttpError";
+        super(message, "ERR_HTTP_ERROR", undefined, statusCode);
         this.statusCode = statusCode;
-        // Maintain proper stack trace (requires target ES2015+ in tsconfig)
-        if (Error.captureStackTrace) {
-            Error.captureStackTrace(this, FetchEngineHttpError);
-        }
+        this.name = "FetchEngineHttpError";
     }
 }
 /**
@@ -21,99 +18,81 @@ export class FetchEngineHttpError extends Error {
  * It does not support advanced configurations like retries, caching, or proxies directly.
  */
 export class FetchEngine {
-    headers;
+    options;
+    static DEFAULT_OPTIONS = {
+        markdown: false,
+    };
     /**
      * Creates an instance of FetchEngine.
-     * Note: This engine currently does not accept configuration options.
+     * @param options Configuration options for the FetchEngine.
      */
-    constructor() {
-        this.headers = {
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-            Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-            "Accept-Language": "en-US,en;q=0.5",
-            "Upgrade-Insecure-Requests": "1",
-            "Sec-Fetch-Dest": "document",
-            "Sec-Fetch-Mode": "navigate",
-            "Sec-Fetch-Site": "none",
-            "Sec-Fetch-User": "?1",
-        };
+    constructor(options = {}) {
+        this.options = { ...FetchEngine.DEFAULT_OPTIONS, ...options };
     }
     /**
-     * Fetches HTML content from the specified URL using the `fetch` API.
+     * Fetches HTML or converts to Markdown from the specified URL.
      *
      * @param url The URL to fetch.
      * @returns A Promise resolving to an HTMLFetchResult object.
      * @throws {FetchEngineHttpError} If the HTTP response status is not ok (e.g., 404, 500).
      * @throws {Error} If the content type is not HTML or for other network errors.
      */
-    async fetchHTML(url) {
+    async fetchHTML(url, options) {
+        const effectiveOptions = { ...this.options, ...options }; // Combine constructor and call options
+        let response;
         try {
-            const response = await fetch(url, {
-                headers: this.headers,
+            response = await fetch(url, {
                 redirect: "follow",
+                headers: {
+                    // Standard browser-like headers
+                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+                    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+                    "Accept-Language": "en-US,en;q=0.9",
+                },
             });
             if (!response.ok) {
-                // Throw the custom error with status code
                 throw new FetchEngineHttpError(`HTTP error! status: ${response.status}`, response.status);
             }
-            const contentType = response.headers.get("content-type") || "";
-            if (!contentType.includes("text/html")) {
-                throw new Error("Not an HTML page");
+            const contentTypeHeader = response.headers.get("content-type");
+            if (!contentTypeHeader || !contentTypeHeader.includes("text/html")) {
+                throw new FetchError("Content-Type is not text/html", "ERR_NON_HTML_CONTENT");
             }
             const html = await response.text();
-            // Use JSDOM to parse HTML and extract title
-            const dom = new JSDOM(html);
-            const title = dom.window.document.title || "";
-            // Check for potential SPA markers
-            const isSPA = this.detectSPA(dom.window.document);
-            if (isSPA) {
-                // Removed throwing error here, as the calling code should decide how to handle this.
-                // Consider adding a flag to the result instead.
-                console.warn(`SPA detected for ${url}, content might be incomplete without JavaScript rendering.`);
-                // Example: return { html, title, url: response.url, isSPA: true };
+            const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+            const title = titleMatch ? titleMatch[1].trim() : null;
+            let finalContent = html;
+            let finalContentType = "html";
+            if (effectiveOptions.markdown) {
+                try {
+                    const converter = new MarkdownConverter();
+                    finalContent = converter.convert(html);
+                    finalContentType = "markdown";
+                }
+                catch (conversionError) {
+                    console.error(`Markdown conversion failed for ${url} (FetchEngine):`, conversionError);
+                    // Fallback to original HTML on conversion error
+                }
             }
             return {
-                html,
-                title,
-                url: response.url,
-                isFromCache: false, // FetchEngine doesn't cache
+                content: finalContent,
+                contentType: finalContentType,
+                title: title,
+                url: response.url, // Use the final URL after redirects
+                isFromCache: false,
                 statusCode: response.status,
                 error: undefined,
             };
         }
         catch (error) {
-            // console.error(`FetchEngine failed for ${url}:`, error); // Optional: Keep logging if desired
-            // Re-throw the original error to preserve its type (e.g., FetchEngineHttpError)
-            // Ensure the result conforms to HTMLFetchResult even on error (for consistency? No, spec says throw)
-            throw error;
-        }
-    }
-    detectSPA(document) {
-        // Check for common SPA frameworks and patterns
-        const spaMarkers = [
-            // React
-            "[data-reactroot]",
-            "#root",
-            "#app",
-            // Vue
-            "[data-v-app]",
-            "#app[data-v-]",
-            // Angular
-            "[ng-version]",
-            "[ng-app]",
-            // Common SPA patterns
-            'script[type="application/json+ld"]', // Less reliable marker
-            'meta[name="fragment"]',
-        ];
-        // Check if the body is nearly empty but has JS (More reliable)
-        const bodyContent = document.body?.textContent?.trim() || "";
-        const hasScripts = document.scripts.length > 0;
-        if (bodyContent.length < 150 && hasScripts) {
-            // Increased threshold slightly
-            return true;
+            // Re-throw specific known errors directly
+            if (error instanceof FetchEngineHttpError ||
+                (error instanceof FetchError && error.code === "ERR_NON_HTML_CONTENT")) {
+                throw error;
+            }
+            // Wrap other/unexpected errors
+            const message = error instanceof Error ? error.message : "Unknown fetch error";
+            throw new FetchError(`Fetch failed: ${message}`, "ERR_FETCH_FAILED", error instanceof Error ? error : undefined);
         }
-        // Check for SPA markers (Less reliable)
-        return spaMarkers.some((selector) => document.querySelector(selector) !== null);
     }
     /**
      * Cleans up resources used by the engine.
@@ -121,8 +100,7 @@ export class FetchEngine {
      * @returns A Promise that resolves when cleanup is complete.
      */
     async cleanup() {
-        // No resources to clean up for fetch engine
-        return Promise.resolve(); // Explicitly return resolved promise
+        return Promise.resolve();
     }
     /**
      * Retrieves metrics for the engine.
@@ -130,7 +108,6 @@ export class FetchEngine {
      * @returns An empty array.
      */
     getMetrics() {
-        // Fetch engine doesn't maintain browser pool metrics
         return [];
     }
 }

package/dist/FetchEngine.js.map

@@ -1 +1 @@
-{"version":3,"file":"FetchEngine.js","sourceRoot":"","sources":["../src/FetchEngine.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAE9B;;GAEG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC7B,UAAU,CAAS;IAEnC,YAAY,OAAe,EAAE,UAAkB;QAC7C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;QACnC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,oEAAoE;QACpE,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,WAAW;IACL,OAAO,CAAyB;IAEjD;;;OAGG;IACH;QACE,IAAI,CAAC,OAAO,GAAG;YACb,YAAY,EACV,iHAAiH;YACnH,MAAM,EAAE,4EAA4E;YACpF,iBAAiB,EAAE,gBAAgB;YACnC,2BAA2B,EAAE,GAAG;YAChC,gBAAgB,EAAE,UAAU;YAC5B,gBAAgB,EAAE,UAAU;YAC5B,gBAAgB,EAAE,MAAM;YACxB,gBAAgB,EAAE,IAAI;SACvB,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,SAAS,CAAC,GAAW;QACzB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,QAAQ,EAAE,QAAQ;aACnB,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,0CAA0C;gBAC1C,MAAM,IAAI,oBAAoB,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC5F,CAAC;YAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC/D,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;YACtC,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEnC,4CAA4C;YAC5C,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5B,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;YAE9C,kCAAkC;YAClC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAClD,IAAI,KAAK,EAAE,CAAC;gBACV,qFAAqF;gBACrF,gDAAgD;gBAChD,OAAO,CAAC,IAAI,CAAC,oBAAoB,GAAG,6DAA6D,CAAC,CAAC;gBACnG,mEAAmE;YACrE,CAAC;YAED,OAAO;gBACL,IAAI;gBACJ,KAAK;gBACL,GAAG,EAAE,QAAQ,CAAC,GAAG;gBACjB,WAAW,EAAE,KAAK,EAAE,4BAA4B;gBAChD,UAAU,EAAE,QAAQ,CAAC,MAAM;gBAC3B,KAAK,EAAE,SAAS;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,+FAA+F;YAC/F,gFAAgF;YAChF,qGAAqG;YACrG,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,SAAS,CAAC,QAAkB;QAClC,+CAA+C;QAC/C,MAAM,UAAU,GAAG;YACjB,QAAQ;YACR,kBAAkB;YAClB,OAAO;YACP,MAAM;YACN,MAAM;YACN,cAAc;YACd,eAAe;YACf,UAAU;YACV,cAAc;YACd,UAAU;YACV,sBAAsB;YACtB,oCAAoC,EAAE,uBAAuB;YAC7D,uBAAuB;SACxB,CAAC;QAEF,+DAA+D;QAC/D,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC7D,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QAE/C,IAAI,WAAW,CAAC,MAAM,GAAG,GAAG,IAAI,UAAU,EAAE,CAAC;YAC3C,+BAA+B;YAC/B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,wCAAwC;QACxC,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;IAClF,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,4CAA4C;QAC5C,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,qCAAqC;IACjE,CAAC;IAED;;;;OAIG;IACH,UAAU;QACR,qDAAqD;QACrD,OAAO,EAAE,CAAC;IACZ,CAAC;CACF"}
+{"version":3,"file":"FetchEngine.js","sourceRoot":"","sources":["../src/FetchEngine.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC,CAAC,uBAAuB;AAC1F,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC,CAAC,yBAAyB;AAEnE;;GAEG;AACH,MAAM,OAAO,oBAAqB,SAAQ,UAAU;IAGhC;IAFlB,YACE,OAAe,EACC,UAAkB;QAElC,KAAK,CAAC,OAAO,EAAE,gBAAgB,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;QAFxC,eAAU,GAAV,UAAU,CAAQ;QAGlC,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,WAAW;IACL,OAAO,CAA+B;IAE/C,MAAM,CAAU,eAAe,GAAiC;QACtE,QAAQ,EAAE,KAAK;KAChB,CAAC;IAEF;;;OAGG;IACH,YAAY,UAA8B,EAAE;QAC1C,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,WAAW,CAAC,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;IAChE,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,OAA4B;QACvD,MAAM,gBAAgB,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,uCAAuC;QACjG,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC1B,QAAQ,EAAE,QAAQ;gBAClB,OAAO,EAAE;oBACP,gCAAgC;oBAChC,YAAY,EACV,iHAAiH;oBACnH,MAAM,EAAE,kGAAkG;oBAC1G,iBAAiB,EAAE,gBAAgB;iBACpC;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,oBAAoB,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC5F,CAAC;YAED,MAAM,iBAAiB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YAC/D,IAAI,CAAC,iBAAiB,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBACnE,MAAM,IAAI,UAAU,CAAC,+BAA+B,EAAE,sBAAsB,CAAC,CAAC;YAChF,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;YAC/D,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YAEvD,IAAI,YAAY,GAAG,IAAI,CAAC;YACxB,IAAI,gBAAgB,GAAwB,MAAM,CAAC;YAEnD,IAAI,gBAAgB,CAAC,QAAQ,EAAE,CAAC;gBAC9B,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;oBAC1C,YAAY,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;oBACvC,gBAAgB,GAAG,UAAU,CAAC;gBAChC,CAAC;gBAAC,OAAO,eAAoB,EAAE,CAAC;oBAC9B,OAAO,CAAC,KAAK,CAAC,kCAAkC,GAAG,iBAAiB,EAAE,eAAe,CAAC,CAAC;oBACvF,gDAAgD;gBAClD,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,YAAY;gBACrB,WAAW,EAAE,gBAAgB;gBAC7B,KAAK,EAAE,KAAK;gBACZ,GAAG,EAAE,QAAQ,CAAC,GAAG,EAAE,oCAAoC;gBACvD,WAAW,EAAE,KAAK;gBAClB,UAAU,EAAE,QAAQ,CAAC,MAAM;gBAC3B,KAAK,EAAE,SAAS;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,0CAA0C;YAC1C,IACE,KAAK,YAAY,oBAAoB;gBACrC,CAAC,KAAK,YAAY,UAAU,IAAI,KAAK,CAAC,IAAI,KAAK,sBAAsB,CAAC,EACtE,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YACD,+BAA+B;YAC/B,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,CAAC;YAC/E,MAAM,IAAI,UAAU,CAAC,iBAAiB,OAAO,EAAE,EAAE,kBAAkB,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACnH,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACH,UAAU;QACR,OAAO,EAAE,CAAC;IACZ,CAAC"}

package/dist/HybridEngine.d.ts

@@ -1,15 +1,21 @@
-import type { HTMLFetchResult, BrowserMetrics, PlaywrightEngineConfig } from "./types.js";
-import { IEngine } from "./IEngine.js";
+import type { IEngine } from "./IEngine.js";
+import type { HTMLFetchResult, PlaywrightEngineConfig, FetchOptions, BrowserMetrics } from "./types.js";
 /**
- * HybridEngine - Attempts fetching with FetchEngine first for speed,
- * then falls back to PlaywrightEngine for complex sites or specific errors.
+ * HybridEngine - Tries FetchEngine first, falls back to PlaywrightEngine on failure.
  */
 export declare class HybridEngine implements IEngine {
     private readonly fetchEngine;
     private readonly playwrightEngine;
-    constructor(playwrightConfig?: PlaywrightEngineConfig);
-    fetchHTML(url: string): Promise<HTMLFetchResult>;
-    cleanup(): Promise<void>;
+    private readonly config;
+    constructor(config?: PlaywrightEngineConfig);
+    fetchHTML(url: string, options?: FetchOptions): Promise<HTMLFetchResult>;
+    /**
+     * Delegates getMetrics to the PlaywrightEngine.
+     */
     getMetrics(): BrowserMetrics[];
+    /**
+     * Calls cleanup on both underlying engines.
+     */
+    cleanup(): Promise<void>;
 }
 //# sourceMappingURL=HybridEngine.d.ts.map

package/dist/HybridEngine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"HybridEngine.d.ts","sourceRoot":"","sources":["../src/HybridEngine.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAC1F,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC;;;GAGG;AACH,qBAAa,YAAa,YAAW,OAAO;IAC1C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;gBAExC,gBAAgB,GAAE,sBAA2B;IAKnD,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAkBhD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAK9B,UAAU,IAAI,cAAc,EAAE;CAI/B"}
+{"version":3,"file":"HybridEngine.d.ts","sourceRoot":"","sources":["../src/HybridEngine.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,sBAAsB,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAExG;;GAEG;AACH,qBAAa,YAAa,YAAW,OAAO;IAC1C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IACpD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAyB;gBAEpC,MAAM,GAAE,sBAA2B;IAQzC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,eAAe,CAAC;IA8BlF;;OAEG;IACH,UAAU,IAAI,cAAc,EAAE;IAI9B;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAM/B"}

package/dist/HybridEngine.js

@@ -1,42 +1,62 @@
 import { FetchEngine } from "./FetchEngine.js";
 import { PlaywrightEngine } from "./PlaywrightEngine.js";
 /**
- * HybridEngine - Attempts fetching with FetchEngine first for speed,
- * then falls back to PlaywrightEngine for complex sites or specific errors.
+ * HybridEngine - Tries FetchEngine first, falls back to PlaywrightEngine on failure.
  */
 export class HybridEngine {
     fetchEngine;
     playwrightEngine;
-    constructor(playwrightConfig = {}) {
-        this.fetchEngine = new FetchEngine();
-        this.playwrightEngine = new PlaywrightEngine(playwrightConfig);
+    config; // Store config for potential per-request PW overrides
+    constructor(config = {}) {
+        // Pass relevant config parts to each engine
+        // FetchEngine only takes markdown option from the shared config
+        this.fetchEngine = new FetchEngine({ markdown: config.markdown });
+        this.playwrightEngine = new PlaywrightEngine(config);
+        this.config = config; // Store for merging later
     }
-    async fetchHTML(url) {
+    async fetchHTML(url, options = {}) {
+        // FetchEngine uses its constructor config; it doesn't accept per-request options here.
         try {
-            // Attempt 1: Use the fast FetchEngine
             const fetchResult = await this.fetchEngine.fetchHTML(url);
+            // If fetch succeeded, return its result directly (it handles its own markdown config)
+            // No need to check contentType here, FetchEngine handles it based on its constructor.
             return fetchResult;
         }
-        catch (_fetchError) {
-            // Prefixed unused error
-            // If FetchEngine fails (e.g., 403, network error, non-html), try Playwright
+        catch (fetchError) {
+            console.warn(`FetchEngine failed for ${url}: ${fetchError.message}. Falling back to PlaywrightEngine.`);
+            // Merge constructor config with per-request options for Playwright fallback
+            const playwrightOptions = {
+                ...this.config, // Start with base config given to HybridEngine
+                ...options, // Override with per-request options
+            };
             try {
-                const playwrightResult = await this.playwrightEngine.fetchHTML(url);
+                // Pass merged options to PlaywrightEngine
+                const playwrightResult = await this.playwrightEngine.fetchHTML(url, playwrightOptions);
                 return playwrightResult;
             }
             catch (playwrightError) {
-                // If Playwright also fails, throw its error (potentially more informative)
+                // Catch potential Playwright error
+                console.error(`PlaywrightEngine fallback failed for ${url}: ${playwrightError.message}`);
+                // Optionally, wrap or prioritize which error to throw
+                // Throwing the Playwright error as it's the last one encountered
                 throw playwrightError;
             }
         }
     }
-    async cleanup() {
-        // Cleanup both engines concurrently
-        await Promise.allSettled([this.fetchEngine.cleanup(), this.playwrightEngine.cleanup()]);
-    }
+    /**
+     * Delegates getMetrics to the PlaywrightEngine.
+     */
     getMetrics() {
-        // FetchEngine doesn't produce metrics, only PlaywrightEngine does
         return this.playwrightEngine.getMetrics();
     }
+    /**
+     * Calls cleanup on both underlying engines.
+     */
+    async cleanup() {
+        await Promise.allSettled([
+            this.fetchEngine.cleanup(), // Although a no-op, call for consistency
+            this.playwrightEngine.cleanup(),
+        ]);
+    }
 }
 //# sourceMappingURL=HybridEngine.js.map

package/dist/HybridEngine.js.map

@@ -1 +1 @@
-{"version":3,"file":"HybridEngine.js","sourceRoot":"","sources":["../src/HybridEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAIzD;;;GAGG;AACH,MAAM,OAAO,YAAY;IACN,WAAW,CAAc;IACzB,gBAAgB,CAAmB;IAEpD,YAAY,mBAA2C,EAAE;QACvD,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,gBAAgB,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAW;QACzB,IAAI,CAAC;YACH,sCAAsC;YACtC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC1D,OAAO,WAAW,CAAC;QACrB,CAAC;QAAC,OAAO,WAAgB,EAAE,CAAC;YAC1B,wBAAwB;YACxB,4EAA4E;YAC5E,IAAI,CAAC;gBACH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBACpE,OAAO,gBAAgB,CAAC;YAC1B,CAAC;YAAC,OAAO,eAAe,EAAE,CAAC;gBACzB,2EAA2E;gBAC3E,MAAM,eAAe,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,oCAAoC;QACpC,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC1F,CAAC;IAED,UAAU;QACR,kEAAkE;QAClE,OAAO,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;IAC5C,CAAC;CACF"}
+{"version":3,"file":"HybridEngine.js","sourceRoot":"","sources":["../src/HybridEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAIzD;;GAEG;AACH,MAAM,OAAO,YAAY;IACN,WAAW,CAAc;IACzB,gBAAgB,CAAmB;IACnC,MAAM,CAAyB,CAAC,sDAAsD;IAEvG,YAAY,SAAiC,EAAE;QAC7C,4CAA4C;QAC5C,gEAAgE;QAChE,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,0BAA0B;IAClD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,UAAwB,EAAE;QACrD,uFAAuF;QACvF,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC1D,sFAAsF;YACtF,sFAAsF;YACtF,OAAO,WAAW,CAAC;QACrB,CAAC;QAAC,OAAO,UAAe,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,0BAA0B,GAAG,KAAK,UAAU,CAAC,OAAO,qCAAqC,CAAC,CAAC;YAExG,4EAA4E;YAC5E,MAAM,iBAAiB,GAAiB;gBACtC,GAAG,IAAI,CAAC,MAAM,EAAE,+CAA+C;gBAC/D,GAAG,OAAO,EAAE,oCAAoC;aACjD,CAAC;YAEF,IAAI,CAAC;gBACH,0CAA0C;gBAC1C,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;gBACvF,OAAO,gBAAgB,CAAC;YAC1B,CAAC;YAAC,OAAO,eAAoB,EAAE,CAAC;gBAC9B,mCAAmC;gBACnC,OAAO,CAAC,KAAK,CAAC,wCAAwC,GAAG,KAAK,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC;gBACzF,sDAAsD;gBACtD,iEAAiE;gBACjE,MAAM,eAAe,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,MAAM,OAAO,CAAC,UAAU,CAAC;YACvB,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,yCAAyC;YACrE,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE;SAChC,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/PlaywrightEngine.d.ts

@@ -54,12 +54,14 @@ export declare class PlaywrightEngine implements IEngine {
      * @returns A Promise resolving to an HTMLFetchResult object.
      * @throws {FetchError} If the fetch fails after all retries or encounters critical errors.
      */
-    fetchHTML(url: string, options?: FetchOptions): Promise<HTMLFetchResult>;
+    fetchHTML(url: string, options?: FetchOptions & {
+        markdown?: boolean;
+    }): Promise<HTMLFetchResult>;
     /**
      * Internal recursive method to handle fetching with retries.
      *
      * @param url URL to fetch
-     * @param options Original fetch options (e.g., fastMode override)
+     * @param currentConfig The merged configuration including markdown option
      * @param retryAttempt Current retry attempt number (starts at 0)
      * @param parentRetryCount Tracks retries related to pool initialization errors (starts at 0)
      * @returns Promise resolving to HTMLFetchResult

package/dist/PlaywrightEngine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"PlaywrightEngine.d.ts","sourceRoot":"","sources":["../src/PlaywrightEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,sBAAsB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AACxG,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAkB5C;;;;;;GAMG;AACH,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsC;IAC5D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAIE;IAGzB,OAAO,CAAC,uBAAuB,CAAkB;IACjD,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,mBAAmB,CAA0B;IAGrD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAoBpC;IAEF;;;;;OAKG;gBACS,MAAM,GAAE,sBAA2B;IAmB/C;;OAEG;YACW,qBAAqB;IAuCnC;;;OAGG;YACW,yBAAyB;IAiEvC,OAAO,CAAC,UAAU;IAWlB;;OAEG;YACW,WAAW;IAazB;;OAEG;YACW,qBAAqB;IAqCnC;;OAEG;IACH,OAAO,CAAC,UAAU;IAUlB;;;;;;;;OAQG;IACG,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,eAAe,CAAC;IAKlF;;;;;;;;OAQG;YACW,eAAe;IA2G7B;;;OAGG;YACW,mBAAmB;YAiEnB,kBAAkB;IAmChC;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAe9B;;;OAGG;IACH,UAAU,IAAI,cAAc,EAAE;IAQ9B,OAAO,CAAC,mBAAmB;CAS5B"}
+{"version":3,"file":"PlaywrightEngine.d.ts","sourceRoot":"","sources":["../src/PlaywrightEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,sBAAsB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AACxG,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAmB5C;;;;;;GAMG;AACH,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsC;IAC5D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmC;IAG1D,OAAO,CAAC,uBAAuB,CAAkB;IACjD,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,mBAAmB,CAA0B;IAGrD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAkBpC;IAEF;;;;;OAKG;gBACS,MAAM,GAAE,sBAA2B;IAM/C;;OAEG;YACW,qBAAqB;IAuCnC;;;OAGG;YACW,yBAAyB;IAmFvC,OAAO,CAAC,UAAU;IAalB;;OAEG;YACW,WAAW;IAazB;;OAEG;YACW,qBAAqB;IAqCnC;;OAEG;IACH,OAAO,CAAC,UAAU;IAUlB;;;;;;;;OAQG;IACG,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAY,GAAG;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,OAAO,CAAC,eAAe,CAAC;IAU3G;;;;;;;;OAQG;YACW,eAAe;IAsH7B;;;OAGG;YACW,mBAAmB;YAmFnB,kBAAkB;IAmChC;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAe9B;;;OAGG;IACH,UAAU,IAAI,cAAc,EAAE;IAQ9B,OAAO,CAAC,mBAAmB;CAS5B"}