@crawlee/impit-client 4.0.0-beta.25 → 4.0.0-beta.251

package/README.md

@@ -1,23 +1,23 @@
 <h1 align="center">
     <a href="https://crawlee.dev">
         <picture>
-          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
-          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
         </picture>
     </a>
-    <br>
+    <br />
     <small>A web scraping and browser automation library</small>
 </h1>
 
-<p align=center>
-    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+<p align="center">
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
 </p>
 
-<p align=center>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
-    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+<p align="center">
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 
@@ -89,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
 We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
 
 ```bash
-npm install crawlee@3.12.3-beta.13
+npm install crawlee@next
 ```
 
 If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -98,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
 {
     "overrides": {
        "apify": {
-           "@crawlee/core": "3.12.3-beta.13",
-           "@crawlee/types": "3.12.3-beta.13",
-           "@crawlee/utils": "3.12.3-beta.13"
+           "@crawlee/core": "$crawlee",
+           "@crawlee/types": "$crawlee",
+           "@crawlee/utils": "$crawlee"
        }
     }
 }

package/index.d.ts

@@ -1,27 +1,25 @@
 import type { CustomFetchOptions } from '@crawlee/http-client';
 import { BaseHttpClient } from '@crawlee/http-client';
+import type { CrawleeLogger } from '@crawlee/types';
 import { type ImpitOptions } from 'impit';
 export declare const Browser: {
-    readonly Chrome: "chrome";
-    readonly Firefox: "firefox";
+    readonly Chrome: 'chrome';
+    readonly Firefox: 'firefox';
 };
 /**
  * A HTTP client implementation based on the `impit` library.
  */
 export declare class ImpitHttpClient extends BaseHttpClient {
-    private impitOptions;
+    #private;
     /**
-     * Enables reuse of `impit` clients for the same set of options.
-     * This is useful for performance reasons, as creating
-     * a new client for each request breaks TCP connection
-     * (and other resources) reuse.
+     * @param options.cacheClients Whether to cache `impit` clients between requests. Defaults to `true`.
      */
-    private clientCache;
-    private getClient;
-    constructor(options?: Omit<ImpitOptions, 'proxyUrl' | 'timeout'>);
+    constructor(options?: Omit<ImpitOptions, 'proxyUrl' | 'timeout'> & {
+        cacheClients?: boolean;
+        logger?: CrawleeLogger;
+    });
     /**
      * @inheritDoc
      */
-    fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
+    protected fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
 }
-//# sourceMappingURL=index.d.ts.map

package/index.js

@@ -1,6 +1,27 @@
+import { serviceLocator } from '@crawlee/core';
 import { BaseHttpClient, ResponseWithUrl } from '@crawlee/http-client';
 import { Impit } from 'impit';
 import { LruCache } from '@apify/datastructures';
+// Concrete impit impersonation profiles per browser family. The plain `chrome` /
+// `firefox` aliases fall back to the oldest available version, which is a
+// fingerprint giveaway — we pick one of these explicitly instead. Keep in sync
+// with impit's `Browser` type when bumping the dependency.
+const IMPIT_VERSIONS_BY_BROWSER = {
+    chrome: [
+        'chrome100',
+        'chrome101',
+        'chrome104',
+        'chrome107',
+        'chrome110',
+        'chrome116',
+        'chrome124',
+        'chrome125',
+        'chrome131',
+        'chrome136',
+        'chrome142',
+    ],
+    firefox: ['firefox128', 'firefox133', 'firefox135', 'firefox144'],
+};
 export const Browser = {
     'Chrome': 'chrome',
     'Firefox': 'firefox',
@@ -9,42 +30,87 @@ export const Browser = {
  * A HTTP client implementation based on the `impit` library.
  */
 export class ImpitHttpClient extends BaseHttpClient {
-    impitOptions;
+    #impitOptions;
+    #cacheClients;
     /**
      * Enables reuse of `impit` clients for the same set of options.
      * This is useful for performance reasons, as creating
      * a new client for each request breaks TCP connection
      * (and other resources) reuse.
      */
-    clientCache = new LruCache({ maxLength: 10 });
-    getClient(options) {
+    #clientCache = new LruCache({ maxLength: 10 });
+    /**
+     * Stable impit impersonation version per fingerprint object, so the same
+     * session keeps impersonating the same browser version across requests
+     * instead of rerolling on every call.
+     */
+    #impitBrowserByFingerprint = new WeakMap();
+    #getClient(options) {
+        if (!this.#cacheClients) {
+            return new Impit(options);
+        }
         const { cookieJar, ...rest } = options;
         const cacheKey = JSON.stringify(rest);
-        const existingClient = this.clientCache.get(cacheKey);
+        const existingClient = this.#clientCache.get(cacheKey);
         if (existingClient && (!cookieJar || existingClient.cookieJar === cookieJar)) {
             return existingClient.client;
         }
         const client = new Impit(options);
-        this.clientCache.add(cacheKey, { client, cookieJar: cookieJar });
+        this.#clientCache.add(cacheKey, { client, cookieJar: cookieJar });
         return client;
     }
+    /**
+     * @param options.cacheClients Whether to cache `impit` clients between requests. Defaults to `true`.
+     */
     constructor(options) {
-        super();
-        this.impitOptions = options ?? {};
+        super({ logger: options?.logger });
+        const { cacheClients = true, logger, ...impitOptions } = options ?? {};
+        if (impitOptions.browser) {
+            (logger ?? serviceLocator.getLogger()).warningOnce('The `browser` option of ImpitHttpClient overrides the browser fingerprint of the session. ' +
+                'Using it with Crawlee crawlers can lead to incompatible Session setups.');
+        }
+        this.#impitOptions = impitOptions;
+        this.#cacheClients = cacheClients;
     }
     /**
      * @inheritDoc
      */
     async fetch(request, options) {
-        const { proxyUrl, redirect, signal } = options ?? {};
-        const impit = this.getClient({
-            ...this.impitOptions,
+        const { proxyUrl, redirect, signal, fingerprint, ignoreTlsErrors } = options ?? {};
+        const impitBrowser = this.#impitOptions.browser ? undefined : this.#resolveImpitBrowser(fingerprint);
+        const impit = this.#getClient({
+            ...this.#impitOptions,
+            ...(impitBrowser ? { browser: impitBrowser } : {}),
+            // The per-request flag (from the crawler option or a MITM proxy session)
+            // can only enable ignoring, never override a constructor-level `true`.
+            ...(ignoreTlsErrors ? { ignoreTlsErrors: true } : {}),
             proxyUrl,
             followRedirects: redirect === 'follow',
         });
-        const response = await impit.fetch(request, { signal: signal ?? undefined });
+        // The caller's signal carries the deadline (for crawlers, the navigation window that `extendTimeout()` can
+        // push out), so impit's 30 s default must not cut the request short. Without a signal, the default stays.
+        const response = await impit.fetch(request, {
+            signal: signal ?? undefined,
+            ...(signal ? { timeout: null } : {}),
+        });
+        // Once the headers are in, an aborted signal only stops impit's body reads, while the download itself keeps
+        // going. Piping the body through the signal cancels the download too.
+        const body = signal && response.body ? response.body.pipeThrough(new TransformStream(), { signal }) : response.body;
         // todo - cast shouldn't be needed here, impit returns `Uint8Array`
-        return new ResponseWithUrl(response.body, response);
+        return new ResponseWithUrl(body, response);
+    }
+    #resolveImpitBrowser(fingerprint) {
+        if (!fingerprint?.browser)
+            return undefined;
+        const cached = this.#impitBrowserByFingerprint.get(fingerprint);
+        if (cached)
+            return cached;
+        // impit can only impersonate Chrome and Firefox. Map other (Chromium-based or
+        // unsupported) families like `edge`/`safari` onto Chrome so the request still
+        // carries realistic browser headers instead of impit's bare `*/*` defaults.
+        const versions = IMPIT_VERSIONS_BY_BROWSER[fingerprint.browser] ?? IMPIT_VERSIONS_BY_BROWSER.chrome;
+        const picked = versions[Math.floor(Math.random() * versions.length)];
+        this.#impitBrowserByFingerprint.set(fingerprint, picked);
+        return picked;
     }
 }
-//# sourceMappingURL=index.js.map

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "@crawlee/impit-client",
-    "version": "4.0.0-beta.25",
+    "version": "4.0.0-beta.251",
     "description": "impit-based HTTP client implementation for Crawlee. Impersonates browser requests to avoid bot detection.",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
@@ -38,7 +38,7 @@
     },
     "homepage": "https://crawlee.dev",
     "scripts": {
-        "build": "yarn clean && yarn compile && yarn copy",
+        "build": "pnpm clean && pnpm compile && pnpm copy",
         "clean": "rimraf ./dist",
         "compile": "tsc -p tsconfig.build.json",
         "copy": "tsx ../../scripts/copy.ts"
@@ -47,10 +47,12 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/datastructures": "^2.0.3",
-        "@crawlee/http-client": "4.0.0-beta.25",
-        "impit": "^0.8.2",
-        "tough-cookie": "^6.0.0"
+        "@apify/datastructures": "^3.0.1",
+        "@crawlee/core": "4.0.0-beta.251",
+        "@crawlee/http-client": "4.0.0-beta.251",
+        "@crawlee/types": "4.0.0-beta.251",
+        "impit": "^0.14.8",
+        "tough-cookie": "^6.0.2"
     },
     "lerna": {
         "command": {
@@ -59,5 +61,5 @@
             }
         }
     },
-    "gitHead": "17411c1cb92d9829cf4179cc9c0a7d46a17942a4"
+    "gitHead": "e8a0d8a538fabf624c73b6354dbf7194882300a5"
 }

package/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAmB,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAS,KAAK,YAAY,EAAE,MAAM,OAAO,CAAC;AAKjD,eAAO,MAAM,OAAO;;;CAGV,CAAC;AAEX;;GAEG;AACH,qBAAa,eAAgB,SAAQ,cAAc;IAC/C,OAAO,CAAC,YAAY,CAAe;IAEnC;;;;;OAKG;IACH,OAAO,CAAC,WAAW,CAA2F;IAE9G,OAAO,CAAC,SAAS;gBAgBL,OAAO,CAAC,EAAE,IAAI,CAAC,YAAY,EAAE,UAAU,GAAG,SAAS,CAAC;IAKhE;;OAEG;IACG,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,kBAAkB,GAAG,OAAO,CAAC,QAAQ,CAAC;CAc/F"}

package/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,EAAE,KAAK,EAAqB,MAAM,OAAO,CAAC;AAGjD,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEjD,MAAM,CAAC,MAAM,OAAO,GAAG;IACnB,QAAQ,EAAE,QAAQ;IAClB,SAAS,EAAE,SAAS;CACd,CAAC;AAEX;;GAEG;AACH,MAAM,OAAO,eAAgB,SAAQ,cAAc;IACvC,YAAY,CAAe;IAEnC;;;;;OAKG;IACK,WAAW,GAA2D,IAAI,QAAQ,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;IAEtG,SAAS,CAAC,OAAqB;QACnC,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;QAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEtD,IAAI,cAAc,IAAI,CAAC,CAAC,SAAS,IAAI,cAAc,CAAC,SAAS,KAAK,SAAS,CAAC,EAAE,CAAC;YAC3E,OAAO,cAAc,CAAC,MAAM,CAAC;QACjC,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,SAA2B,EAAE,CAAC,CAAC;QAEnF,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,YAAY,OAAoD;QAC5D,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,YAAY,GAAG,OAAO,IAAI,EAAE,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,OAAgB,EAAE,OAA0C;QACpE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;QAErD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC;YACzB,GAAG,IAAI,CAAC,YAAY;YACpB,QAAQ;YACR,eAAe,EAAE,QAAQ,KAAK,QAAQ;SACzC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC,CAAC;QAE7E,mEAAmE;QACnE,OAAO,IAAI,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACxD,CAAC;CACJ"}