@purepageio/fetch-engines 0.1.0

package/dist/types.d.ts

@@ -0,0 +1,151 @@
+import type { Browser as PlaywrightBrowser, BrowserContext } from "playwright";
+/**
+ * Result object returned by engine's fetchHTML method.
+ */
+export interface HTMLFetchResult {
+    /** The full HTML content of the fetched page. */
+    html: string;
+    /** The extracted content of the <title> tag, or an empty string if not found. */
+    title: string;
+    /** The final URL after any redirects. */
+    url: string;
+    /** Indicates if the result was served from the engine's cache. */
+    isFromCache: boolean;
+    /** The HTTP status code of the final response, if available. */
+    statusCode?: number;
+    /** Error object if the fetch failed after all retries. */
+    error?: Error;
+}
+/**
+ * Metrics related to browser pool performance and status.
+ */
+export interface BrowserMetrics {
+    id: string;
+    engine?: "playwright" | string;
+    pagesCreated: number;
+    activePages: number;
+    lastUsed: Date;
+    errors: number;
+    totalRequests?: number;
+    avgResponseTime?: number;
+    createdAt: Date;
+    isHealthy: boolean;
+}
+/**
+ * Internal representation of a Playwright browser instance within the pool.
+ */
+export interface BrowserInstance {
+    browser: PlaywrightBrowser;
+    context: BrowserContext;
+    metrics: BrowserMetrics;
+    isHealthy: boolean;
+}
+/**
+ * Configuration options for the PlaywrightEngine.
+ */
+export interface PlaywrightEngineConfig {
+    /**
+     * Maximum number of Playwright pages to process concurrently.
+     * @default 3
+     */
+    concurrentPages?: number;
+    /**
+     * Maximum number of retry attempts for a failed fetch operation (excluding initial attempt).
+     * @default 3
+     */
+    maxRetries?: number;
+    /**
+     * Delay in milliseconds between retry attempts.
+     * @default 5000
+     */
+    retryDelay?: number;
+    /**
+     * Time-to-live for cached results in milliseconds. Set to 0 to disable.
+     * @default 900000 (15 minutes)
+     */
+    cacheTTL?: number;
+    /**
+     * If true, attempts a fast HTTP GET first before using Playwright.
+     * @default true
+     */
+    useHttpFallback?: boolean;
+    /**
+     * If true, automatically retries failed requests for a domain in headed mode.
+     * @default false
+     */
+    useHeadedModeFallback?: boolean;
+    /**
+     * If true, requests initially block non-essential resources and skip human simulation.
+     * Can be overridden per-request via fetchHTML options.
+     * @default true
+     */
+    defaultFastMode?: boolean;
+    /**
+     * If true (and not in fastMode), attempts basic human-like interactions.
+     * @default true
+     */
+    simulateHumanBehavior?: boolean;
+    /**
+     * Maximum number of concurrent browser instances the pool manages.
+     * Passed to PlaywrightBrowserPool.
+     * @default 2
+     */
+    maxBrowsers?: number;
+    /**
+     * Maximum number of pages per browser context before recycling.
+     * Passed to PlaywrightBrowserPool.
+     * @default 6
+     */
+    maxPagesPerContext?: number;
+    /**
+     * Maximum age in ms a browser instance lives before recycling.
+     * Passed to PlaywrightBrowserPool.
+     * @default 1200000 (20 minutes)
+     */
+    maxBrowserAge?: number;
+    /**
+     * How often (in ms) the pool checks browser health.
+     * Passed to PlaywrightBrowserPool.
+     * @default 60000 (1 minute)
+     */
+    healthCheckInterval?: number;
+    /**
+     * List of domain glob patterns to block requests to. Overrides pool default.
+     * Passed to PlaywrightBrowserPool.
+     * @default [] (uses pool's defaults)
+     */
+    poolBlockedDomains?: string[];
+    /**
+     * List of Playwright resource types (e.g., 'image', 'font') to block. Overrides pool default.
+     * Passed to PlaywrightBrowserPool.
+     * @default [] (uses pool's defaults)
+     */
+    poolBlockedResourceTypes?: string[];
+    /**
+     * Proxy configuration for browser instances.
+     * Passed to PlaywrightBrowserPool.
+     * @default undefined
+     */
+    proxy?: {
+        /** Proxy server URL (e.g., "http://host:port", "socks5://user:pass@host:port"). */
+        server: string;
+        /** Optional proxy username. */
+        username?: string;
+        /** Optional proxy password. */
+        password?: string;
+    };
+    /**
+     * Forces the entire pool to launch browsers in headed (visible) mode.
+     * Passed to PlaywrightBrowserPool.
+     * @default false
+     */
+    useHeadedMode?: boolean;
+}
+/**
+ * Options that can be passed per-request to engine.fetchHTML().
+ */
+export interface FetchOptions {
+    /** Overrides the engine's defaultFastMode for this specific request. */
+    fastMode?: boolean;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,IAAI,iBAAiB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE/E;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,iDAAiD;IACjD,IAAI,EAAE,MAAM,CAAC;IACb,iFAAiF;IACjF,KAAK,EAAE,MAAM,CAAC;IACd,yCAAyC;IACzC,GAAG,EAAE,MAAM,CAAC;IACZ,kEAAkE;IAClE,WAAW,EAAE,OAAO,CAAC;IACrB,gEAAgE;IAChE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0DAA0D;IAC1D,KAAK,CAAC,EAAE,KAAK,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,CAAC,EAAE,YAAY,GAAG,MAAM,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,IAAI,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,EAAE,cAAc,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC;CACpB;AAKD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC;;;;OAIG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;OAGG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAIhC;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC9B;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,EAAE,CAAC;IACpC;;;;OAIG;IACH,KAAK,CAAC,EAAE;QACN,mFAAmF;QACnF,MAAM,EAAE,MAAM,CAAC;QACf,+BAA+B;QAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,+BAA+B;QAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,CAAC;IACF;;;;OAIG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,wEAAwE;IACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB"}

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "@purepageio/fetch-engines",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "A collection of configurable engines for fetching HTML content using fetch or Playwright.",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "dependencies": {
+    "axios": "^1.6.8",
+    "jsdom": "^24.0.0",
+    "p-queue": "^7.4.1",
+    "playwright": "^1.43.0",
+    "playwright-extra": "^4.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "user-agents": "^1.1.208",
+    "uuid": "^11.1.0"
+  },
+  "devDependencies": {
+    "@types/axios": "^0.14.0",
+    "@types/jsdom": "^21.1.6",
+    "@types/node": "^18.0.0",
+    "@types/user-agents": "^1.0.4",
+    "@types/uuid": "^10.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.29.0",
+    "@typescript-eslint/parser": "^8.29.0",
+    "eslint": "^9.23.0",
+    "eslint-config-prettier": "^10.1.1",
+    "eslint-plugin-prettier": "^5.2.5",
+    "globals": "^16.0.0",
+    "prettier": "^3.5.3",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.3.3",
+    "typescript-eslint": "^8.29.0",
+    "vitest": "^3.1.1"
+  },
+  "engines": {
+    "node": ">=22.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/purepageio/fetch-engines.git"
+  },
+  "keywords": [
+    "fetch",
+    "scrape",
+    "playwright",
+    "headless",
+    "browser",
+    "automation",
+    "engine"
+  ],
+  "author": "Purepage",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/purepageio/fetch-engines/issues"
+  },
+  "homepage": "https://github.com/purepageio/fetch-engines#readme",
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsc",
+    "lint": "eslint \"src/**/*.ts\" \"examples/**/*.ts\"",
+    "format": "prettier --write \"src/**/*.ts\" \"examples/**/*.ts\" \"*.{js,cjs,json,md}\"",
+    "test": "vitest run"
+  }
+}