@purepageio/fetch-engines 0.2.5 → 0.2.6

package/dist/types.d.ts

@@ -1,167 +0,0 @@
-import type { Browser as PlaywrightBrowser, BrowserContext } from "playwright";
-/**
- * Defines the structure for the result of fetching HTML content.
- */
-export interface HTMLFetchResult {
-    /** The fetched HTML content OR the converted Markdown content. */
-    content: string;
-    /** Indicates the type of content in the 'content' field. */
-    contentType: "html" | "markdown";
-    /** The extracted title of the page, if available. */
-    title: string | null;
-    /** The final URL after any redirects. */
-    url: string;
-    /** Indicates if the result came from the cache. */
-    isFromCache: boolean;
-    /** The HTTP status code of the final response. */
-    statusCode: number | undefined;
-    /** Any error encountered during the fetch process. */
-    error: Error | undefined;
-}
-/**
- * Metrics related to browser pool performance and status.
- */
-export interface BrowserMetrics {
-    id: string;
-    engine?: "playwright" | string;
-    pagesCreated: number;
-    activePages: number;
-    lastUsed: Date;
-    errors: number;
-    totalRequests?: number;
-    avgResponseTime?: number;
-    createdAt: Date;
-    isHealthy: boolean;
-}
-/**
- * Internal representation of a Playwright browser instance within the pool.
- */
-export interface BrowserInstance {
-    browser: PlaywrightBrowser;
-    context: BrowserContext;
-    metrics: BrowserMetrics;
-    isHealthy: boolean;
-}
-/**
- * Configuration options for the PlaywrightEngine.
- */
-export interface PlaywrightEngineConfig {
-    /**
-     * Maximum number of Playwright pages to process concurrently.
-     * @default 3
-     */
-    concurrentPages?: number;
-    /**
-     * Maximum number of retry attempts for a failed fetch operation (excluding initial attempt).
-     * @default 3
-     */
-    maxRetries?: number;
-    /**
-     * Delay in milliseconds between retry attempts.
-     * @default 5000
-     */
-    retryDelay?: number;
-    /**
-     * Time-to-live for cached results in milliseconds. Set to 0 to disable.
-     * @default 900000 (15 minutes)
-     */
-    cacheTTL?: number;
-    /**
-     * If true, attempts a fast HTTP GET first before using Playwright.
-     * @default true
-     */
-    useHttpFallback?: boolean;
-    /**
-     * If true, automatically retries failed requests for a domain in headed mode.
-     * @default false
-     */
-    useHeadedModeFallback?: boolean;
-    /**
-     * If true, requests initially block non-essential resources and skip human simulation.
-     * Can be overridden per-request via fetchHTML options.
-     * @default true
-     */
-    defaultFastMode?: boolean;
-    /**
-     * If true (and not in fastMode), attempts basic human-like interactions.
-     * @default true
-     */
-    simulateHumanBehavior?: boolean;
-    /**
-     * Maximum number of concurrent browser instances the pool manages.
-     * Passed to PlaywrightBrowserPool.
-     * @default 2
-     */
-    maxBrowsers?: number;
-    /**
-     * Maximum number of pages per browser context before recycling.
-     * Passed to PlaywrightBrowserPool.
-     * @default 6
-     */
-    maxPagesPerContext?: number;
-    /**
-     * Maximum age in ms a browser instance lives before recycling.
-     * Passed to PlaywrightBrowserPool.
-     * @default 1200000 (20 minutes)
-     */
-    maxBrowserAge?: number;
-    /**
-     * How often (in ms) the pool checks browser health.
-     * Passed to PlaywrightBrowserPool.
-     * @default 60000 (1 minute)
-     */
-    healthCheckInterval?: number;
-    /**
-     * List of domain glob patterns to block requests to. Overrides pool default.
-     * Passed to PlaywrightBrowserPool.
-     * @default [] (uses pool's defaults)
-     */
-    poolBlockedDomains?: string[];
-    /**
-     * List of Playwright resource types (e.g., 'image', 'font') to block. Overrides pool default.
-     * Passed to PlaywrightBrowserPool.
-     * @default [] (uses pool's defaults)
-     */
-    poolBlockedResourceTypes?: string[];
-    /**
-     * Proxy configuration for browser instances.
-     * Passed to PlaywrightBrowserPool.
-     * @default undefined
-     */
-    proxy?: {
-        /** Proxy server URL (e.g., "http://host:port", "socks5://user:pass@host:port"). */
-        server: string;
-        /** Optional proxy username. */
-        username?: string;
-        /** Optional proxy password. */
-        password?: string;
-    };
-    /**
-     * Forces the entire pool to launch browsers in headed (visible) mode.
-     * Passed to PlaywrightBrowserPool.
-     * @default false
-     */
-    useHeadedMode?: boolean;
-    /**
-     * If true, the fetched HTML content will be converted to Markdown.
-     * @default false
-     */
-    markdown?: boolean;
-}
-/**
- * Options that can be passed per-request to engine.fetchHTML().
- */
-export interface FetchOptions {
-    /** Overrides the engine's defaultFastMode for this specific request. (Playwright/Hybrid only) */
-    fastMode?: boolean;
-    /** Overrides the engine's markdown setting for this specific request. (Playwright/Hybrid only) */
-    markdown?: boolean;
-}
-/**
- * Configuration options specifically for the FetchEngine.
- */
-export interface FetchEngineOptions {
-    /** If true, convert the fetched HTML to Markdown. Default: false */
-    markdown?: boolean;
-}
-//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,IAAI,iBAAiB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE/E;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,kEAAkE;IAClE,OAAO,EAAE,MAAM,CAAC;IAChB,4DAA4D;IAC5D,WAAW,EAAE,MAAM,GAAG,UAAU,CAAC;IACjC,qDAAqD;IACrD,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,yCAAyC;IACzC,GAAG,EAAE,MAAM,CAAC;IACZ,mDAAmD;IACnD,WAAW,EAAE,OAAO,CAAC;IACrB,kDAAkD;IAClD,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,sDAAsD;IACtD,KAAK,EAAE,KAAK,GAAG,SAAS,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,CAAC,EAAE,YAAY,GAAG,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,IAAI,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,EAAE,cAAc,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC;CACpB;AAKD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC;;;;OAIG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAIhC;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,EAAE,CAAC;IACpC;;;;OAIG;IACH,KAAK,CAAC,EAAE;QACN,mFAAmF;QACnF,MAAM,EAAE,MAAM,CAAC;QACf,+BAA+B;QAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,+BAA+B;QAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,iGAAiG;IACjG,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,kGAAkG;IAClG,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,oEAAoE;IACpE,QAAQ,CAAC,EAAE,OAAO,CAAC;CAEpB"}

package/dist/types.js

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/dist/utils/markdown-converter.d.ts

@@ -1,31 +0,0 @@
-export interface ConversionOptions {
-    /** Maximum length of the final Markdown content. Defaults to Infinity. */
-    maxContentLength?: number;
-}
-export declare class MarkdownConverter {
-    private turndownService;
-    constructor();
-    /**
-     * Converts HTML string to Markdown.
-     * @param html The HTML string to convert.
-     * @param options Conversion options.
-     * @returns The converted Markdown string.
-     */
-    convert(html: string, options?: ConversionOptions): string;
-    private setupPrioritizedRules;
-    private addContentExtractionRules;
-    private addStructureRules;
-    private addBlockRules;
-    private addInlineRules;
-    private preprocessHTML;
-    private cleanupHtml;
-    private cleanupContentHtml;
-    private removeHighLinkDensityElements;
-    private extractDocumentMetadata;
-    private detectForumPage;
-    private extractArticleContentElement;
-    private extractForumContentElement;
-    private hasHighLinkDensity;
-    private postprocessMarkdown;
-}
-//# sourceMappingURL=markdown-converter.d.ts.map

package/dist/utils/markdown-converter.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"markdown-converter.d.ts","sourceRoot":"","sources":["../../src/utils/markdown-converter.ts"],"names":[],"mappings":"AAmEA,MAAM,WAAW,iBAAiB;IAChC,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AASD,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,eAAe,CAAkB;;IA+BzC;;;;;OAKG;IACI,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB,GAAG,MAAM;IAerE,OAAO,CAAC,qBAAqB;IAS7B,OAAO,CAAC,yBAAyB;IA4CjC,OAAO,CAAC,iBAAiB;IAiBzB,OAAO,CAAC,aAAa;IAmErB,OAAO,CAAC,cAAc;IAqLtB,OAAO,CAAC,cAAc;IAmDtB,OAAO,CAAC,WAAW;IAanB,OAAO,CAAC,kBAAkB;IAqB1B,OAAO,CAAC,6BAA6B;IAkDrC,OAAO,CAAC,uBAAuB;IAwE/B,OAAO,CAAC,eAAe;IAoDvB,OAAO,CAAC,4BAA4B;IA8DpC,OAAO,CAAC,0BAA0B;IAqElC,OAAO,CAAC,kBAAkB;IAwB1B,OAAO,CAAC,mBAAmB;CAkD5B"}