@purepageio/fetch-engines 0.2.9 → 0.2.11

package/dist/FetchEngine.d.ts

@@ -0,0 +1,47 @@
+import type { HTMLFetchResult, BrowserMetrics, FetchEngineOptions } from "./types.js";
+import type { IEngine } from "./IEngine.js";
+import { FetchError } from "./errors.js";
+/**
+ * Custom error class for HTTP errors from FetchEngine.
+ */
+export declare class FetchEngineHttpError extends FetchError {
+    readonly statusCode: number;
+    constructor(message: string, statusCode: number);
+}
+/**
+ * FetchEngine - A lightweight engine for fetching HTML content using the standard `fetch` API.
+ *
+ * Ideal for fetching content from static websites or APIs where JavaScript execution is not required.
+ * It does not support advanced configurations like retries, caching, or proxies directly.
+ */
+export declare class FetchEngine implements IEngine {
+    private readonly options;
+    private static readonly DEFAULT_OPTIONS;
+    /**
+     * Creates an instance of FetchEngine.
+     * @param options Configuration options for the FetchEngine.
+     */
+    constructor(options?: FetchEngineOptions);
+    /**
+     * Fetches HTML or converts to Markdown from the specified URL.
+     *
+     * @param url The URL to fetch.
+     * @returns A Promise resolving to an HTMLFetchResult object.
+     * @throws {FetchEngineHttpError} If the HTTP response status is not ok (e.g., 404, 500).
+     * @throws {Error} If the content type is not HTML or for other network errors.
+     */
+    fetchHTML(url: string, options?: FetchEngineOptions): Promise<HTMLFetchResult>;
+    /**
+     * Cleans up resources used by the engine.
+     * For FetchEngine, this is a no-op as it doesn't manage persistent resources.
+     * @returns A Promise that resolves when cleanup is complete.
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Retrieves metrics for the engine.
+     * FetchEngine does not manage browsers, so it returns an empty array.
+     * @returns An empty array.
+     */
+    getMetrics(): BrowserMetrics[];
+}
+//# sourceMappingURL=FetchEngine.d.ts.map

package/dist/FetchEngine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"FetchEngine.d.ts","sourceRoot":"","sources":["../src/FetchEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AACtF,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAG5C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC;;GAEG;AACH,qBAAa,oBAAqB,SAAQ,UAAU;aAGhC,UAAU,EAAE,MAAM;gBADlC,OAAO,EAAE,MAAM,EACC,UAAU,EAAE,MAAM;CAKrC;AAED;;;;;GAKG;AACH,qBAAa,WAAY,YAAW,OAAO;IACzC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA+B;IAEvD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAErC;IAEF;;;OAGG;gBACS,OAAO,GAAE,kBAAuB;IAI5C;;;;;;;OAOG;IACG,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,eAAe,CAAC;IAiEpF;;;;OAIG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAI9B;;;;OAIG;IACH,UAAU,IAAI,cAAc,EAAE;CAG/B"}

package/dist/FetchEngine.js

@@ -0,0 +1,114 @@
+import { MarkdownConverter } from "./utils/markdown-converter.js"; // Import the converter
+import { FetchError } from "./errors.js"; // Only import FetchError
+/**
+ * Custom error class for HTTP errors from FetchEngine.
+ */
+export class FetchEngineHttpError extends FetchError {
+    statusCode;
+    constructor(message, statusCode) {
+        super(message, "ERR_HTTP_ERROR", undefined, statusCode);
+        this.statusCode = statusCode;
+        this.name = "FetchEngineHttpError";
+    }
+}
+/**
+ * FetchEngine - A lightweight engine for fetching HTML content using the standard `fetch` API.
+ *
+ * Ideal for fetching content from static websites or APIs where JavaScript execution is not required.
+ * It does not support advanced configurations like retries, caching, or proxies directly.
+ */
+export class FetchEngine {
+    options;
+    static DEFAULT_OPTIONS = {
+        markdown: false,
+    };
+    /**
+     * Creates an instance of FetchEngine.
+     * @param options Configuration options for the FetchEngine.
+     */
+    constructor(options = {}) {
+        this.options = { ...FetchEngine.DEFAULT_OPTIONS, ...options };
+    }
+    /**
+     * Fetches HTML or converts to Markdown from the specified URL.
+     *
+     * @param url The URL to fetch.
+     * @returns A Promise resolving to an HTMLFetchResult object.
+     * @throws {FetchEngineHttpError} If the HTTP response status is not ok (e.g., 404, 500).
+     * @throws {Error} If the content type is not HTML or for other network errors.
+     */
+    async fetchHTML(url, options) {
+        const effectiveOptions = { ...this.options, ...options }; // Combine constructor and call options
+        let response;
+        try {
+            response = await fetch(url, {
+                redirect: "follow",
+                headers: {
+                    // Standard browser-like headers
+                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+                    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+                    "Accept-Language": "en-US,en;q=0.9",
+                },
+            });
+            if (!response.ok) {
+                throw new FetchEngineHttpError(`HTTP error! status: ${response.status}`, response.status);
+            }
+            const contentTypeHeader = response.headers.get("content-type");
+            if (!contentTypeHeader || !contentTypeHeader.includes("text/html")) {
+                throw new FetchError("Content-Type is not text/html", "ERR_NON_HTML_CONTENT");
+            }
+            const html = await response.text();
+            const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+            const title = titleMatch ? titleMatch[1].trim() : null;
+            let finalContent = html;
+            let finalContentType = "html";
+            if (effectiveOptions.markdown) {
+                try {
+                    const converter = new MarkdownConverter();
+                    finalContent = converter.convert(html);
+                    finalContentType = "markdown";
+                }
+                catch (conversionError) {
+                    console.error(`Markdown conversion failed for ${url} (FetchEngine):`, conversionError);
+                    // Fallback to original HTML on conversion error
+                }
+            }
+            return {
+                content: finalContent,
+                contentType: finalContentType,
+                title: title,
+                url: response.url, // Use the final URL after redirects
+                isFromCache: false,
+                statusCode: response.status,
+                error: undefined,
+            };
+        }
+        catch (error) {
+            // Re-throw specific known errors directly
+            if (error instanceof FetchEngineHttpError ||
+                (error instanceof FetchError && error.code === "ERR_NON_HTML_CONTENT")) {
+                throw error;
+            }
+            // Wrap other/unexpected errors
+            const message = error instanceof Error ? error.message : "Unknown fetch error";
+            throw new FetchError(`Fetch failed: ${message}`, "ERR_FETCH_FAILED", error instanceof Error ? error : undefined);
+        }
+    }
+    /**
+     * Cleans up resources used by the engine.
+     * For FetchEngine, this is a no-op as it doesn't manage persistent resources.
+     * @returns A Promise that resolves when cleanup is complete.
+     */
+    async cleanup() {
+        return Promise.resolve();
+    }
+    /**
+     * Retrieves metrics for the engine.
+     * FetchEngine does not manage browsers, so it returns an empty array.
+     * @returns An empty array.
+     */
+    getMetrics() {
+        return [];
+    }
+}
+//# sourceMappingURL=FetchEngine.js.map

package/dist/FetchEngine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"FetchEngine.js","sourceRoot":"","sources":["../src/FetchEngine.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC,CAAC,uBAAuB;AAC1F,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC,CAAC,yBAAyB;AAEnE;;GAEG;AACH,MAAM,OAAO,oBAAqB,SAAQ,UAAU;IAGhC;IAFlB,YACE,OAAe,EACC,UAAkB;QAElC,KAAK,CAAC,OAAO,EAAE,gBAAgB,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;QAFxC,eAAU,GAAV,UAAU,CAAQ;QAGlC,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,WAAW;IACL,OAAO,CAA+B;IAE/C,MAAM,CAAU,eAAe,GAAiC;QACtE,QAAQ,EAAE,KAAK;KAChB,CAAC;IAEF;;;OAGG;IACH,YAAY,UAA8B,EAAE;QAC1C,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,WAAW,CAAC,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;IAChE,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,OAA4B;QACvD,MAAM,gBAAgB,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,uCAAuC;QACjG,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC1B,QAAQ,EAAE,QAAQ;gBAClB,OAAO,EAAE;oBACP,gCAAgC;oBAChC,YAAY,EACV,iHAAiH;oBACnH,MAAM,EAAE,kGAAkG;oBAC1G,iBAAiB,EAAE,gBAAgB;iBACpC;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,oBAAoB,CAAC,uBAAuB,QAAQ,CAAC,MAAM,EAAE,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC5F,CAAC;YAED,MAAM,iBAAiB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YAC/D,IAAI,CAAC,iBAAiB,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBACnE,MAAM,IAAI,UAAU,CAAC,+BAA+B,EAAE,sBAAsB,CAAC,CAAC;YAChF,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;YAC/D,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YAEvD,IAAI,YAAY,GAAG,IAAI,CAAC;YACxB,IAAI,gBAAgB,GAAwB,MAAM,CAAC;YAEnD,IAAI,gBAAgB,CAAC,QAAQ,EAAE,CAAC;gBAC9B,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;oBAC1C,YAAY,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;oBACvC,gBAAgB,GAAG,UAAU,CAAC;gBAChC,CAAC;gBAAC,OAAO,eAAoB,EAAE,CAAC;oBAC9B,OAAO,CAAC,KAAK,CAAC,kCAAkC,GAAG,iBAAiB,EAAE,eAAe,CAAC,CAAC;oBACvF,gDAAgD;gBAClD,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE,YAAY;gBACrB,WAAW,EAAE,gBAAgB;gBAC7B,KAAK,EAAE,KAAK;gBACZ,GAAG,EAAE,QAAQ,CAAC,GAAG,EAAE,oCAAoC;gBACvD,WAAW,EAAE,KAAK;gBAClB,UAAU,EAAE,QAAQ,CAAC,MAAM;gBAC3B,KAAK,EAAE,SAAS;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,0CAA0C;YAC1C,IACE,KAAK,YAAY,oBAAoB;gBACrC,CAAC,KAAK,YAAY,UAAU,IAAI,KAAK,CAAC,IAAI,KAAK,sBAAsB,CAAC,EACtE,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YACD,+BAA+B;YAC/B,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,CAAC;YAC/E,MAAM,IAAI,UAAU,CAAC,iBAAiB,OAAO,EAAE,EAAE,kBAAkB,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACnH,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACH,UAAU;QACR,OAAO,EAAE,CAAC;IACZ,CAAC"}

package/dist/FetchEngine.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=FetchEngine.test.d.ts.map

package/dist/FetchEngine.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"FetchEngine.test.d.ts","sourceRoot":"","sources":["../src/FetchEngine.test.ts"],"names":[],"mappings":""}

package/dist/FetchEngine.test.js

@@ -0,0 +1,44 @@
+import { describe, it, expect } from "vitest";
+import { FetchEngine } from "./FetchEngine.js";
+describe("FetchEngine", () => {
+    it("should fetch HTML and extract title from a static page", async () => {
+        const engine = new FetchEngine();
+        const url = "http://example.com";
+        const expectedUrl = "http://example.com/"; // Expect trailing slash
+        try {
+            const result = await engine.fetchHTML(url);
+            expect(result).toBeDefined();
+            expect(result.url).toBe(expectedUrl); // Use expectedUrl
+            expect(result.title).toBe("Example Domain");
+            expect(result.html).toContain("<title>Example Domain</title>");
+            expect(result.html).toContain("<h1>Example Domain</h1>");
+        }
+        catch (error) {
+            // If the test environment doesn't have fetch or network access, this might fail.
+            // In a real CI/CD, ensure network access or mock fetch.
+            console.warn("FetchEngine test failed, potentially due to network issues or missing fetch API:", error);
+            // Re-throw to fail the test if fetch was expected to work
+            throw error;
+        }
+    });
+    it("should throw an error for non-HTML content", async () => {
+        const engine = new FetchEngine();
+        // Use a URL known to return non-HTML content, e.g., a JSON endpoint or an image
+        const url = "https://httpbin.org/json";
+        // Expect the fetchHTML method to reject
+        await expect(engine.fetchHTML(url)).rejects.toThrow("Not an HTML page");
+    });
+    it("should throw an error for non-existent domains", async () => {
+        const engine = new FetchEngine();
+        const url = "http://domain-that-does-not-exist-fdsahjkl.xyz";
+        // Expect the fetchHTML method to reject (error message might vary)
+        await expect(engine.fetchHTML(url)).rejects.toThrow();
+    });
+    it("should handle http errors", async () => {
+        const engine = new FetchEngine();
+        const url = "https://httpbin.org/status/404"; // URL that returns 404
+        await expect(engine.fetchHTML(url)).rejects.toThrow(/HTTP error! status: 404/);
+    });
+    // Add more tests: SPA detection warning, etc.
+});
+//# sourceMappingURL=FetchEngine.test.js.map

package/dist/FetchEngine.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"FetchEngine.test.js","sourceRoot":"","sources":["../src/FetchEngine.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAE/C,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;QACtE,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,oBAAoB,CAAC;QACjC,MAAM,WAAW,GAAG,qBAAqB,CAAC,CAAC,wBAAwB;QAEnE,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAE3C,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,kBAAkB;YACxD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,+BAA+B,CAAC,CAAC;YAC/D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,yBAAyB,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,iFAAiF;YACjF,wDAAwD;YACxD,OAAO,CAAC,IAAI,CAAC,kFAAkF,EAAE,KAAK,CAAC,CAAC;YACxG,0DAA0D;YAC1D,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QACjC,gFAAgF;QAChF,MAAM,GAAG,GAAG,0BAA0B,CAAC;QAEvC,wCAAwC;QACxC,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,gDAAgD,CAAC;QAE7D,mEAAmE;QACnE,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;QACzC,MAAM,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,gCAAgC,CAAC,CAAC,uBAAuB;QAErE,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;IACjF,CAAC,CAAC,CAAC;IAEH,8CAA8C;AAChD,CAAC,CAAC,CAAC"}

package/dist/HybridEngine.d.ts

@@ -0,0 +1,21 @@
+import type { IEngine } from "./IEngine.js";
+import type { HTMLFetchResult, PlaywrightEngineConfig, FetchOptions, BrowserMetrics } from "./types.js";
+/**
+ * HybridEngine - Tries FetchEngine first, falls back to PlaywrightEngine on failure.
+ */
+export declare class HybridEngine implements IEngine {
+    private readonly fetchEngine;
+    private readonly playwrightEngine;
+    private readonly config;
+    constructor(config?: PlaywrightEngineConfig);
+    fetchHTML(url: string, options?: FetchOptions): Promise<HTMLFetchResult>;
+    /**
+     * Delegates getMetrics to the PlaywrightEngine.
+     */
+    getMetrics(): BrowserMetrics[];
+    /**
+     * Calls cleanup on both underlying engines.
+     */
+    cleanup(): Promise<void>;
+}
+//# sourceMappingURL=HybridEngine.d.ts.map

package/dist/HybridEngine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"HybridEngine.d.ts","sourceRoot":"","sources":["../src/HybridEngine.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,sBAAsB,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAExG;;GAEG;AACH,qBAAa,YAAa,YAAW,OAAO;IAC1C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IACpD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAyB;gBAEpC,MAAM,GAAE,sBAA2B;IAQzC,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,eAAe,CAAC;IA8BlF;;OAEG;IACH,UAAU,IAAI,cAAc,EAAE;IAI9B;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAM/B"}

package/dist/HybridEngine.js

@@ -0,0 +1,62 @@
+import { FetchEngine } from "./FetchEngine.js";
+import { PlaywrightEngine } from "./PlaywrightEngine.js";
+/**
+ * HybridEngine - Tries FetchEngine first, falls back to PlaywrightEngine on failure.
+ */
+export class HybridEngine {
+    fetchEngine;
+    playwrightEngine;
+    config; // Store config for potential per-request PW overrides
+    constructor(config = {}) {
+        // Pass relevant config parts to each engine
+        // FetchEngine only takes markdown option from the shared config
+        this.fetchEngine = new FetchEngine({ markdown: config.markdown });
+        this.playwrightEngine = new PlaywrightEngine(config);
+        this.config = config; // Store for merging later
+    }
+    async fetchHTML(url, options = {}) {
+        // FetchEngine uses its constructor config; it doesn't accept per-request options here.
+        try {
+            const fetchResult = await this.fetchEngine.fetchHTML(url);
+            // If fetch succeeded, return its result directly (it handles its own markdown config)
+            // No need to check contentType here, FetchEngine handles it based on its constructor.
+            return fetchResult;
+        }
+        catch (fetchError) {
+            console.warn(`FetchEngine failed for ${url}: ${fetchError.message}. Falling back to PlaywrightEngine.`);
+            // Merge constructor config with per-request options for Playwright fallback
+            const playwrightOptions = {
+                ...this.config, // Start with base config given to HybridEngine
+                ...options, // Override with per-request options
+            };
+            try {
+                // Pass merged options to PlaywrightEngine
+                const playwrightResult = await this.playwrightEngine.fetchHTML(url, playwrightOptions);
+                return playwrightResult;
+            }
+            catch (playwrightError) {
+                // Catch potential Playwright error
+                console.error(`PlaywrightEngine fallback failed for ${url}: ${playwrightError.message}`);
+                // Optionally, wrap or prioritize which error to throw
+                // Throwing the Playwright error as it's the last one encountered
+                throw playwrightError;
+            }
+        }
+    }
+    /**
+     * Delegates getMetrics to the PlaywrightEngine.
+     */
+    getMetrics() {
+        return this.playwrightEngine.getMetrics();
+    }
+    /**
+     * Calls cleanup on both underlying engines.
+     */
+    async cleanup() {
+        await Promise.allSettled([
+            this.fetchEngine.cleanup(), // Although a no-op, call for consistency
+            this.playwrightEngine.cleanup(),
+        ]);
+    }
+}
+//# sourceMappingURL=HybridEngine.js.map

package/dist/HybridEngine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"HybridEngine.js","sourceRoot":"","sources":["../src/HybridEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAIzD;;GAEG;AACH,MAAM,OAAO,YAAY;IACN,WAAW,CAAc;IACzB,gBAAgB,CAAmB;IACnC,MAAM,CAAyB,CAAC,sDAAsD;IAEvG,YAAY,SAAiC,EAAE;QAC7C,4CAA4C;QAC5C,gEAAgE;QAChE,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,0BAA0B;IAClD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,UAAwB,EAAE;QACrD,uFAAuF;QACvF,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC1D,sFAAsF;YACtF,sFAAsF;YACtF,OAAO,WAAW,CAAC;QACrB,CAAC;QAAC,OAAO,UAAe,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,0BAA0B,GAAG,KAAK,UAAU,CAAC,OAAO,qCAAqC,CAAC,CAAC;YAExG,4EAA4E;YAC5E,MAAM,iBAAiB,GAAiB;gBACtC,GAAG,IAAI,CAAC,MAAM,EAAE,+CAA+C;gBAC/D,GAAG,OAAO,EAAE,oCAAoC;aACjD,CAAC;YAEF,IAAI,CAAC;gBACH,0CAA0C;gBAC1C,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;gBACvF,OAAO,gBAAgB,CAAC;YAC1B,CAAC;YAAC,OAAO,eAAoB,EAAE,CAAC;gBAC9B,mCAAmC;gBACnC,OAAO,CAAC,KAAK,CAAC,wCAAwC,GAAG,KAAK,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC;gBACzF,sDAAsD;gBACtD,iEAAiE;gBACjE,MAAM,eAAe,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,MAAM,OAAO,CAAC,UAAU,CAAC;YACvB,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,yCAAyC;YACrE,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE;SAChC,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/IEngine.d.ts

@@ -0,0 +1,22 @@
+import type { HTMLFetchResult, BrowserMetrics } from "./types.js";
+/**
+ * Interface for browser engines that can fetch HTML content from URLs
+ */
+export interface IEngine {
+    /**
+     * Fetches HTML content from a URL
+     * @param url The URL to fetch
+     * @returns A promise that resolves to an HTMLFetchResult
+     */
+    fetchHTML(url: string): Promise<HTMLFetchResult>;
+    /**
+     * Cleans up resources used by the engine
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Gets metrics about the engine's performance
+     * @returns An array of BrowserMetrics
+     */
+    getMetrics(): BrowserMetrics[];
+}
+//# sourceMappingURL=IEngine.d.ts.map

package/dist/IEngine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"IEngine.d.ts","sourceRoot":"","sources":["../src/IEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAElE;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAEjD;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEzB;;;OAGG;IACH,UAAU,IAAI,cAAc,EAAE,CAAC;CAChC"}

package/dist/IEngine.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=IEngine.js.map

package/dist/IEngine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IEngine.js","sourceRoot":"","sources":["../src/IEngine.ts"],"names":[],"mappings":""}

package/dist/PlaywrightEngine.d.ts

@@ -0,0 +1,90 @@
+import type { HTMLFetchResult, BrowserMetrics, PlaywrightEngineConfig, FetchOptions } from "./types.js";
+import type { IEngine } from "./IEngine.js";
+/**
+ * PlaywrightEngine - Fetches HTML using a managed pool of headless Playwright browser instances.
+ *
+ * This engine is suitable for dynamic websites that require JavaScript execution.
+ * It incorporates `playwright-extra` with the stealth plugin for enhanced anti-detection capabilities.
+ * Features include caching, retries, HTTP fallback, and configurable browser pooling.
+ */
+export declare class PlaywrightEngine implements IEngine {
+    private browserPool;
+    private readonly queue;
+    private readonly cache;
+    private readonly config;
+    private initializingBrowserPool;
+    private isUsingHeadedMode;
+    private headedFallbackSites;
+    private static readonly DEFAULT_CONFIG;
+    /**
+     * Creates an instance of PlaywrightEngine.
+     *
+     * @param config Configuration options for the engine and its browser pool.
+     *               See `PlaywrightEngineConfig` for details.
+     */
+    constructor(config?: PlaywrightEngineConfig);
+    /**
+     * Initialize the browser pool with improved error handling and mode switching.
+     */
+    private initializeBrowserPool;
+    /**
+     * Fallback method using simple HTTP requests via Axios.
+     * Ensures return type matches HTMLFetchResult.
+     */
+    private fetchHTMLWithHttpFallback;
+    private checkCache;
+    /**
+     * Safely check if a page is still usable and connected.
+     */
+    private isPageValid;
+    /**
+     * Simulate human-like interactions on the page.
+     */
+    private simulateHumanBehavior;
+    /**
+     * Adds a result to the in-memory cache.
+     */
+    private addToCache;
+    /**
+     * Public method to fetch HTML. Delegates to the internal recursive fetch method.
+     *
+     * @param url The URL to fetch.
+     * @param options Optional settings for this specific fetch operation.
+     * @param options.fastMode Overrides the engine's `defaultFastMode` configuration for this request.
+     * @returns A Promise resolving to an HTMLFetchResult object.
+     * @throws {FetchError} If the fetch fails after all retries or encounters critical errors.
+     */
+    fetchHTML(url: string, options?: FetchOptions & {
+        markdown?: boolean;
+    }): Promise<HTMLFetchResult>;
+    /**
+     * Internal recursive method to handle fetching with retries.
+     *
+     * @param url URL to fetch
+     * @param currentConfig The merged configuration including markdown option
+     * @param retryAttempt Current retry attempt number (starts at 0)
+     * @param parentRetryCount Tracks retries related to pool initialization errors (starts at 0)
+     * @returns Promise resolving to HTMLFetchResult
+     */
+    private _fetchRecursive;
+    /**
+     * Performs the actual page fetch using a Playwright page from the pool.
+     * Ensures return type matches HTMLFetchResult.
+     */
+    private fetchWithPlaywright;
+    private applyBlockingRules;
+    /**
+     * Cleans up resources used by the engine, primarily closing browser instances in the pool.
+     *
+     * It is crucial to call this method when finished with the engine instance to release resources.
+     * @returns A Promise that resolves when cleanup is complete.
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Retrieves metrics from the underlying browser pool.
+     * @returns An array of BrowserMetrics objects, one for each active browser instance, or an empty array if the pool is not initialized.
+     */
+    getMetrics(): BrowserMetrics[];
+    private shouldUseHeadedMode;
+}
+//# sourceMappingURL=PlaywrightEngine.d.ts.map

package/dist/PlaywrightEngine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PlaywrightEngine.d.ts","sourceRoot":"","sources":["../src/PlaywrightEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,sBAAsB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AACxG,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAmB5C;;;;;;GAMG;AACH,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsC;IAC5D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmC;IAG1D,OAAO,CAAC,uBAAuB,CAAkB;IACjD,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,mBAAmB,CAA0B;IAGrD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAkBpC;IAEF;;;;;OAKG;gBACS,MAAM,GAAE,sBAA2B;IAM/C;;OAEG;YACW,qBAAqB;IAuCnC;;;OAGG;YACW,yBAAyB;IAmFvC,OAAO,CAAC,UAAU;IAalB;;OAEG;YACW,WAAW;IAazB;;OAEG;YACW,qBAAqB;IAqCnC;;OAEG;IACH,OAAO,CAAC,UAAU;IAUlB;;;;;;;;OAQG;IACG,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAY,GAAG;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,OAAO,CAAC,eAAe,CAAC;IAU3G;;;;;;;;OAQG;YACW,eAAe;IAsH7B;;;OAGG;YACW,mBAAmB;YAmJnB,kBAAkB;IAmChC;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAe9B;;;OAGG;IACH,UAAU,IAAI,cAAc,EAAE;IAQ9B,OAAO,CAAC,mBAAmB;CAS5B"}