@purepageio/fetch-engines 0.2.4 → 0.2.6

package/dist/index.d.ts

@@ -1,8 +1,323 @@
-import type { IEngine } from "./IEngine.js";
-import { FetchEngine } from "./FetchEngine.js";
-import { PlaywrightEngine } from "./PlaywrightEngine.js";
-import type { HTMLFetchResult, BrowserMetrics } from "./types.js";
-export type { IEngine, HTMLFetchResult, BrowserMetrics };
-export { FetchEngine, PlaywrightEngine };
-export * from "./HybridEngine.js";
-//# sourceMappingURL=index.d.ts.map
+/**
+ * Defines the structure for the result of fetching HTML content.
+ */
+interface HTMLFetchResult {
+    /** The fetched HTML content OR the converted Markdown content. */
+    content: string;
+    /** Indicates the type of content in the 'content' field. */
+    contentType: "html" | "markdown";
+    /** The extracted title of the page, if available. */
+    title: string | null;
+    /** The final URL after any redirects. */
+    url: string;
+    /** Indicates if the result came from the cache. */
+    isFromCache: boolean;
+    /** The HTTP status code of the final response. */
+    statusCode: number | undefined;
+    /** Any error encountered during the fetch process. */
+    error: Error | undefined;
+}
+/**
+ * Metrics related to browser pool performance and status.
+ */
+interface BrowserMetrics {
+    id: string;
+    engine?: "playwright" | string;
+    pagesCreated: number;
+    activePages: number;
+    lastUsed: Date;
+    errors: number;
+    totalRequests?: number;
+    avgResponseTime?: number;
+    createdAt: Date;
+    isHealthy: boolean;
+}
+/**
+ * Configuration options for the PlaywrightEngine.
+ */
+interface PlaywrightEngineConfig {
+    /**
+     * Maximum number of Playwright pages to process concurrently.
+     * @default 3
+     */
+    concurrentPages?: number;
+    /**
+     * Maximum number of retry attempts for a failed fetch operation (excluding initial attempt).
+     * @default 3
+     */
+    maxRetries?: number;
+    /**
+     * Delay in milliseconds between retry attempts.
+     * @default 5000
+     */
+    retryDelay?: number;
+    /**
+     * Time-to-live for cached results in milliseconds. Set to 0 to disable.
+     * @default 900000 (15 minutes)
+     */
+    cacheTTL?: number;
+    /**
+     * If true, attempts a fast HTTP GET first before using Playwright.
+     * @default true
+     */
+    useHttpFallback?: boolean;
+    /**
+     * If true, automatically retries failed requests for a domain in headed mode.
+     * @default false
+     */
+    useHeadedModeFallback?: boolean;
+    /**
+     * If true, requests initially block non-essential resources and skip human simulation.
+     * Can be overridden per-request via fetchHTML options.
+     * @default true
+     */
+    defaultFastMode?: boolean;
+    /**
+     * If true (and not in fastMode), attempts basic human-like interactions.
+     * @default true
+     */
+    simulateHumanBehavior?: boolean;
+    /**
+     * Maximum number of concurrent browser instances the pool manages.
+     * Passed to PlaywrightBrowserPool.
+     * @default 2
+     */
+    maxBrowsers?: number;
+    /**
+     * Maximum number of pages per browser context before recycling.
+     * Passed to PlaywrightBrowserPool.
+     * @default 6
+     */
+    maxPagesPerContext?: number;
+    /**
+     * Maximum age in ms a browser instance lives before recycling.
+     * Passed to PlaywrightBrowserPool.
+     * @default 1200000 (20 minutes)
+     */
+    maxBrowserAge?: number;
+    /**
+     * How often (in ms) the pool checks browser health.
+     * Passed to PlaywrightBrowserPool.
+     * @default 60000 (1 minute)
+     */
+    healthCheckInterval?: number;
+    /**
+     * List of domain glob patterns to block requests to. Overrides pool default.
+     * Passed to PlaywrightBrowserPool.
+     * @default [] (uses pool's defaults)
+     */
+    poolBlockedDomains?: string[];
+    /**
+     * List of Playwright resource types (e.g., 'image', 'font') to block. Overrides pool default.
+     * Passed to PlaywrightBrowserPool.
+     * @default [] (uses pool's defaults)
+     */
+    poolBlockedResourceTypes?: string[];
+    /**
+     * Proxy configuration for browser instances.
+     * Passed to PlaywrightBrowserPool.
+     * @default undefined
+     */
+    proxy?: {
+        /** Proxy server URL (e.g., "http://host:port", "socks5://user:pass@host:port"). */
+        server: string;
+        /** Optional proxy username. */
+        username?: string;
+        /** Optional proxy password. */
+        password?: string;
+    };
+    /**
+     * Forces the entire pool to launch browsers in headed (visible) mode.
+     * Passed to PlaywrightBrowserPool.
+     * @default false
+     */
+    useHeadedMode?: boolean;
+    /**
+     * If true, the fetched HTML content will be converted to Markdown.
+     * @default false
+     */
+    markdown?: boolean;
+}
+/**
+ * Options that can be passed per-request to engine.fetchHTML().
+ */
+interface FetchOptions {
+    /** Overrides the engine's defaultFastMode for this specific request. (Playwright/Hybrid only) */
+    fastMode?: boolean;
+    /** Overrides the engine's markdown setting for this specific request. (Playwright/Hybrid only) */
+    markdown?: boolean;
+}
+/**
+ * Configuration options specifically for the FetchEngine.
+ */
+interface FetchEngineOptions {
+    /** If true, convert the fetched HTML to Markdown. Default: false */
+    markdown?: boolean;
+}
+
+/**
+ * Interface for browser engines that can fetch HTML content from URLs
+ */
+interface IEngine {
+    /**
+     * Fetches HTML content from a URL
+     * @param url The URL to fetch
+     * @returns A promise that resolves to an HTMLFetchResult
+     */
+    fetchHTML(url: string): Promise<HTMLFetchResult>;
+    /**
+     * Cleans up resources used by the engine
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Gets metrics about the engine's performance
+     * @returns An array of BrowserMetrics
+     */
+    getMetrics(): BrowserMetrics[];
+}
+
+/**
+ * FetchEngine - A lightweight engine for fetching HTML content using the standard `fetch` API.
+ *
+ * Ideal for fetching content from static websites or APIs where JavaScript execution is not required.
+ * It does not support advanced configurations like retries, caching, or proxies directly.
+ */
+declare class FetchEngine implements IEngine {
+    private readonly options;
+    private static readonly DEFAULT_OPTIONS;
+    /**
+     * Creates an instance of FetchEngine.
+     * @param options Configuration options for the FetchEngine.
+     */
+    constructor(options?: FetchEngineOptions);
+    /**
+     * Fetches HTML or converts to Markdown from the specified URL.
+     *
+     * @param url The URL to fetch.
+     * @returns A Promise resolving to an HTMLFetchResult object.
+     * @throws {FetchEngineHttpError} If the HTTP response status is not ok (e.g., 404, 500).
+     * @throws {Error} If the content type is not HTML or for other network errors.
+     */
+    fetchHTML(url: string, options?: FetchEngineOptions): Promise<HTMLFetchResult>;
+    /**
+     * Cleans up resources used by the engine.
+     * For FetchEngine, this is a no-op as it doesn't manage persistent resources.
+     * @returns A Promise that resolves when cleanup is complete.
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Retrieves metrics for the engine.
+     * FetchEngine does not manage browsers, so it returns an empty array.
+     * @returns An empty array.
+     */
+    getMetrics(): BrowserMetrics[];
+}
+
+/**
+ * PlaywrightEngine - Fetches HTML using a managed pool of headless Playwright browser instances.
+ *
+ * This engine is suitable for dynamic websites that require JavaScript execution.
+ * It incorporates `playwright-extra` with the stealth plugin for enhanced anti-detection capabilities.
+ * Features include caching, retries, HTTP fallback, and configurable browser pooling.
+ */
+declare class PlaywrightEngine implements IEngine {
+    private browserPool;
+    private readonly queue;
+    private readonly cache;
+    private readonly config;
+    private initializingBrowserPool;
+    private isUsingHeadedMode;
+    private headedFallbackSites;
+    private static readonly DEFAULT_CONFIG;
+    /**
+     * Creates an instance of PlaywrightEngine.
+     *
+     * @param config Configuration options for the engine and its browser pool.
+     *               See `PlaywrightEngineConfig` for details.
+     */
+    constructor(config?: PlaywrightEngineConfig);
+    /**
+     * Initialize the browser pool with improved error handling and mode switching.
+     */
+    private initializeBrowserPool;
+    /**
+     * Fallback method using simple HTTP requests via Axios.
+     * Ensures return type matches HTMLFetchResult.
+     */
+    private fetchHTMLWithHttpFallback;
+    private checkCache;
+    /**
+     * Safely check if a page is still usable and connected.
+     */
+    private isPageValid;
+    /**
+     * Simulate human-like interactions on the page.
+     */
+    private simulateHumanBehavior;
+    /**
+     * Adds a result to the in-memory cache.
+     */
+    private addToCache;
+    /**
+     * Public method to fetch HTML. Delegates to the internal recursive fetch method.
+     *
+     * @param url The URL to fetch.
+     * @param options Optional settings for this specific fetch operation.
+     * @param options.fastMode Overrides the engine's `defaultFastMode` configuration for this request.
+     * @returns A Promise resolving to an HTMLFetchResult object.
+     * @throws {FetchError} If the fetch fails after all retries or encounters critical errors.
+     */
+    fetchHTML(url: string, options?: FetchOptions & {
+        markdown?: boolean;
+    }): Promise<HTMLFetchResult>;
+    /**
+     * Internal recursive method to handle fetching with retries.
+     *
+     * @param url URL to fetch
+     * @param currentConfig The merged configuration including markdown option
+     * @param retryAttempt Current retry attempt number (starts at 0)
+     * @param parentRetryCount Tracks retries related to pool initialization errors (starts at 0)
+     * @returns Promise resolving to HTMLFetchResult
+     */
+    private _fetchRecursive;
+    /**
+     * Performs the actual page fetch using a Playwright page from the pool.
+     * Ensures return type matches HTMLFetchResult.
+     */
+    private fetchWithPlaywright;
+    private applyBlockingRules;
+    /**
+     * Cleans up resources used by the engine, primarily closing browser instances in the pool.
+     *
+     * It is crucial to call this method when finished with the engine instance to release resources.
+     * @returns A Promise that resolves when cleanup is complete.
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Retrieves metrics from the underlying browser pool.
+     * @returns An array of BrowserMetrics objects, one for each active browser instance, or an empty array if the pool is not initialized.
+     */
+    getMetrics(): BrowserMetrics[];
+    private shouldUseHeadedMode;
+}
+
+/**
+ * HybridEngine - Tries FetchEngine first, falls back to PlaywrightEngine on failure.
+ */
+declare class HybridEngine implements IEngine {
+    private readonly fetchEngine;
+    private readonly playwrightEngine;
+    private readonly config;
+    constructor(config?: PlaywrightEngineConfig);
+    fetchHTML(url: string, options?: FetchOptions): Promise<HTMLFetchResult>;
+    /**
+     * Delegates getMetrics to the PlaywrightEngine.
+     */
+    getMetrics(): BrowserMetrics[];
+    /**
+     * Calls cleanup on both underlying engines.
+     */
+    cleanup(): Promise<void>;
+}
+
+export { type BrowserMetrics, FetchEngine, type HTMLFetchResult, HybridEngine, type IEngine, PlaywrightEngine };