@purepageio/fetch-engines 0.2.5 → 0.2.6

package/dist/HybridEngine.js

@@ -1,62 +0,0 @@
-import { FetchEngine } from "./FetchEngine.js";
-import { PlaywrightEngine } from "./PlaywrightEngine.js";
-/**
- * HybridEngine - Tries FetchEngine first, falls back to PlaywrightEngine on failure.
- */
-export class HybridEngine {
-    fetchEngine;
-    playwrightEngine;
-    config; // Store config for potential per-request PW overrides
-    constructor(config = {}) {
-        // Pass relevant config parts to each engine
-        // FetchEngine only takes markdown option from the shared config
-        this.fetchEngine = new FetchEngine({ markdown: config.markdown });
-        this.playwrightEngine = new PlaywrightEngine(config);
-        this.config = config; // Store for merging later
-    }
-    async fetchHTML(url, options = {}) {
-        // FetchEngine uses its constructor config; it doesn't accept per-request options here.
-        try {
-            const fetchResult = await this.fetchEngine.fetchHTML(url);
-            // If fetch succeeded, return its result directly (it handles its own markdown config)
-            // No need to check contentType here, FetchEngine handles it based on its constructor.
-            return fetchResult;
-        }
-        catch (fetchError) {
-            console.warn(`FetchEngine failed for ${url}: ${fetchError.message}. Falling back to PlaywrightEngine.`);
-            // Merge constructor config with per-request options for Playwright fallback
-            const playwrightOptions = {
-                ...this.config, // Start with base config given to HybridEngine
-                ...options, // Override with per-request options
-            };
-            try {
-                // Pass merged options to PlaywrightEngine
-                const playwrightResult = await this.playwrightEngine.fetchHTML(url, playwrightOptions);
-                return playwrightResult;
-            }
-            catch (playwrightError) {
-                // Catch potential Playwright error
-                console.error(`PlaywrightEngine fallback failed for ${url}: ${playwrightError.message}`);
-                // Optionally, wrap or prioritize which error to throw
-                // Throwing the Playwright error as it's the last one encountered
-                throw playwrightError;
-            }
-        }
-    }
-    /**
-     * Delegates getMetrics to the PlaywrightEngine.
-     */
-    getMetrics() {
-        return this.playwrightEngine.getMetrics();
-    }
-    /**
-     * Calls cleanup on both underlying engines.
-     */
-    async cleanup() {
-        await Promise.allSettled([
-            this.fetchEngine.cleanup(), // Although a no-op, call for consistency
-            this.playwrightEngine.cleanup(),
-        ]);
-    }
-}
-//# sourceMappingURL=HybridEngine.js.map

package/dist/HybridEngine.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"HybridEngine.js","sourceRoot":"","sources":["../src/HybridEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAIzD;;GAEG;AACH,MAAM,OAAO,YAAY;IACN,WAAW,CAAc;IACzB,gBAAgB,CAAmB;IACnC,MAAM,CAAyB,CAAC,sDAAsD;IAEvG,YAAY,SAAiC,EAAE;QAC7C,4CAA4C;QAC5C,gEAAgE;QAChE,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,0BAA0B;IAClD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAW,EAAE,UAAwB,EAAE;QACrD,uFAAuF;QACvF,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC1D,sFAAsF;YACtF,sFAAsF;YACtF,OAAO,WAAW,CAAC;QACrB,CAAC;QAAC,OAAO,UAAe,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,0BAA0B,GAAG,KAAK,UAAU,CAAC,OAAO,qCAAqC,CAAC,CAAC;YAExG,4EAA4E;YAC5E,MAAM,iBAAiB,GAAiB;gBACtC,GAAG,IAAI,CAAC,MAAM,EAAE,+CAA+C;gBAC/D,GAAG,OAAO,EAAE,oCAAoC;aACjD,CAAC;YAEF,IAAI,CAAC;gBACH,0CAA0C;gBAC1C,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;gBACvF,OAAO,gBAAgB,CAAC;YAC1B,CAAC;YAAC,OAAO,eAAoB,EAAE,CAAC;gBAC9B,mCAAmC;gBACnC,OAAO,CAAC,KAAK,CAAC,wCAAwC,GAAG,KAAK,eAAe,CAAC,OAAO,EAAE,CAAC,CAAC;gBACzF,sDAAsD;gBACtD,iEAAiE;gBACjE,MAAM,eAAe,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,CAAC;IAC5C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,MAAM,OAAO,CAAC,UAAU,CAAC;YACvB,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,yCAAyC;YACrE,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE;SAChC,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/IEngine.d.ts

@@ -1,22 +0,0 @@
-import type { HTMLFetchResult, BrowserMetrics } from "./types.js";
-/**
- * Interface for browser engines that can fetch HTML content from URLs
- */
-export interface IEngine {
-    /**
-     * Fetches HTML content from a URL
-     * @param url The URL to fetch
-     * @returns A promise that resolves to an HTMLFetchResult
-     */
-    fetchHTML(url: string): Promise<HTMLFetchResult>;
-    /**
-     * Cleans up resources used by the engine
-     */
-    cleanup(): Promise<void>;
-    /**
-     * Gets metrics about the engine's performance
-     * @returns An array of BrowserMetrics
-     */
-    getMetrics(): BrowserMetrics[];
-}
-//# sourceMappingURL=IEngine.d.ts.map

package/dist/IEngine.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"IEngine.d.ts","sourceRoot":"","sources":["../src/IEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAElE;;GAEG;AACH,MAAM,WAAW,OAAO;IACtB;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAEjD;;OAEG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEzB;;;OAGG;IACH,UAAU,IAAI,cAAc,EAAE,CAAC;CAChC"}

package/dist/IEngine.js

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=IEngine.js.map

package/dist/IEngine.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"IEngine.js","sourceRoot":"","sources":["../src/IEngine.ts"],"names":[],"mappings":""}

package/dist/PlaywrightEngine.d.ts

@@ -1,90 +0,0 @@
-import type { HTMLFetchResult, BrowserMetrics, PlaywrightEngineConfig, FetchOptions } from "./types.js";
-import type { IEngine } from "./IEngine.js";
-/**
- * PlaywrightEngine - Fetches HTML using a managed pool of headless Playwright browser instances.
- *
- * This engine is suitable for dynamic websites that require JavaScript execution.
- * It incorporates `playwright-extra` with the stealth plugin for enhanced anti-detection capabilities.
- * Features include caching, retries, HTTP fallback, and configurable browser pooling.
- */
-export declare class PlaywrightEngine implements IEngine {
-    private browserPool;
-    private readonly queue;
-    private readonly cache;
-    private readonly config;
-    private initializingBrowserPool;
-    private isUsingHeadedMode;
-    private headedFallbackSites;
-    private static readonly DEFAULT_CONFIG;
-    /**
-     * Creates an instance of PlaywrightEngine.
-     *
-     * @param config Configuration options for the engine and its browser pool.
-     *               See `PlaywrightEngineConfig` for details.
-     */
-    constructor(config?: PlaywrightEngineConfig);
-    /**
-     * Initialize the browser pool with improved error handling and mode switching.
-     */
-    private initializeBrowserPool;
-    /**
-     * Fallback method using simple HTTP requests via Axios.
-     * Ensures return type matches HTMLFetchResult.
-     */
-    private fetchHTMLWithHttpFallback;
-    private checkCache;
-    /**
-     * Safely check if a page is still usable and connected.
-     */
-    private isPageValid;
-    /**
-     * Simulate human-like interactions on the page.
-     */
-    private simulateHumanBehavior;
-    /**
-     * Adds a result to the in-memory cache.
-     */
-    private addToCache;
-    /**
-     * Public method to fetch HTML. Delegates to the internal recursive fetch method.
-     *
-     * @param url The URL to fetch.
-     * @param options Optional settings for this specific fetch operation.
-     * @param options.fastMode Overrides the engine's `defaultFastMode` configuration for this request.
-     * @returns A Promise resolving to an HTMLFetchResult object.
-     * @throws {FetchError} If the fetch fails after all retries or encounters critical errors.
-     */
-    fetchHTML(url: string, options?: FetchOptions & {
-        markdown?: boolean;
-    }): Promise<HTMLFetchResult>;
-    /**
-     * Internal recursive method to handle fetching with retries.
-     *
-     * @param url URL to fetch
-     * @param currentConfig The merged configuration including markdown option
-     * @param retryAttempt Current retry attempt number (starts at 0)
-     * @param parentRetryCount Tracks retries related to pool initialization errors (starts at 0)
-     * @returns Promise resolving to HTMLFetchResult
-     */
-    private _fetchRecursive;
-    /**
-     * Performs the actual page fetch using a Playwright page from the pool.
-     * Ensures return type matches HTMLFetchResult.
-     */
-    private fetchWithPlaywright;
-    private applyBlockingRules;
-    /**
-     * Cleans up resources used by the engine, primarily closing browser instances in the pool.
-     *
-     * It is crucial to call this method when finished with the engine instance to release resources.
-     * @returns A Promise that resolves when cleanup is complete.
-     */
-    cleanup(): Promise<void>;
-    /**
-     * Retrieves metrics from the underlying browser pool.
-     * @returns An array of BrowserMetrics objects, one for each active browser instance, or an empty array if the pool is not initialized.
-     */
-    getMetrics(): BrowserMetrics[];
-    private shouldUseHeadedMode;
-}
-//# sourceMappingURL=PlaywrightEngine.d.ts.map

package/dist/PlaywrightEngine.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"PlaywrightEngine.d.ts","sourceRoot":"","sources":["../src/PlaywrightEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,sBAAsB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AACxG,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAmB5C;;;;;;GAMG;AACH,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsC;IAC5D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmC;IAG1D,OAAO,CAAC,uBAAuB,CAAkB;IACjD,OAAO,CAAC,iBAAiB,CAAkB;IAC3C,OAAO,CAAC,mBAAmB,CAA0B;IAGrD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAkBpC;IAEF;;;;;OAKG;gBACS,MAAM,GAAE,sBAA2B;IAM/C;;OAEG;YACW,qBAAqB;IAuCnC;;;OAGG;YACW,yBAAyB;IAmFvC,OAAO,CAAC,UAAU;IAalB;;OAEG;YACW,WAAW;IAazB;;OAEG;YACW,qBAAqB;IAqCnC;;OAEG;IACH,OAAO,CAAC,UAAU;IAUlB;;;;;;;;OAQG;IACG,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,YAAY,GAAG;QAAE,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,OAAO,CAAC,eAAe,CAAC;IAU3G;;;;;;;;OAQG;YACW,eAAe;IAsH7B;;;OAGG;YACW,mBAAmB;YAmFnB,kBAAkB;IAmChC;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAe9B;;;OAGG;IACH,UAAU,IAAI,cAAc,EAAE;IAQ9B,OAAO,CAAC,mBAAmB;CAS5B"}