@crawlee/impit-client 3.17.1-beta.9 → 3.18.0

package/index.d.ts

@@ -1,4 +1,4 @@
-import type { BaseHttpClient, HttpRequest, HttpResponse, ResponseTypes, StreamingHttpResponse } from '@crawlee/core';
+import type { BaseHttpClient, HttpRequest, HttpResponse, RedirectHandler, ResponseTypes, StreamingHttpResponse } from '@crawlee/core';
 import type { ImpitOptions } from 'impit';
 export declare const Browser: {
     readonly Chrome: "chrome";
@@ -11,6 +11,7 @@ export declare class ImpitHttpClient implements BaseHttpClient {
     private impitOptions;
     private maxRedirects;
     private followRedirects;
+    private cacheClients;
     /**
      * Enables reuse of `impit` clients for the same set of options.
      * This is useful for performance reasons, as creating
@@ -19,8 +20,12 @@ export declare class ImpitHttpClient implements BaseHttpClient {
      */
     private clientCache;
     private getClient;
+    /**
+     * @param options.cacheClients Whether to cache `impit` clients between requests. Defaults to `true`.
+     */
     constructor(options?: Omit<ImpitOptions, 'proxyUrl'> & {
         maxRedirects?: number;
+        cacheClients?: boolean;
     });
     /**
      * Flattens the headers of a `HttpRequest` to a format that can be passed to `impit`.
@@ -30,6 +35,11 @@ export declare class ImpitHttpClient implements BaseHttpClient {
     private intoHeaders;
     private intoImpitBody;
     private shouldRewriteRedirectToGet;
+    /**
+     * Converts Fetch/Impit headers into a simple header map.
+     * `Object.fromEntries` would keep only the last `set-cookie` value, so those are collected separately.
+     */
+    private intoSimpleHeaders;
     /**
      * Common implementation for `sendRequest` and `stream` methods.
      * @param request `HttpRequest` object
@@ -44,5 +54,5 @@ export declare class ImpitHttpClient implements BaseHttpClient {
     /**
      * @inheritDoc
      */
-    stream(request: HttpRequest): Promise<StreamingHttpResponse>;
+    stream(request: HttpRequest, onRedirect?: RedirectHandler): Promise<StreamingHttpResponse>;
 }

package/index.js

@@ -14,6 +14,9 @@ exports.Browser = {
  */
 class ImpitHttpClient {
     getClient(options) {
+        if (!this.cacheClients) {
+            return new impit_1.Impit(options);
+        }
         const { cookieJar, ...rest } = options;
         const cacheKey = JSON.stringify(rest);
         const existingClient = this.clientCache.get(cacheKey);
@@ -24,6 +27,9 @@ class ImpitHttpClient {
         this.clientCache.add(cacheKey, { client, cookieJar: cookieJar });
         return client;
     }
+    /**
+     * @param options.cacheClients Whether to cache `impit` clients between requests. Defaults to `true`.
+     */
     constructor(options) {
         Object.defineProperty(this, "impitOptions", {
             enumerable: true,
@@ -43,6 +49,12 @@ class ImpitHttpClient {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "cacheClients", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         /**
          * Enables reuse of `impit` clients for the same set of options.
          * This is useful for performance reasons, as creating
@@ -55,9 +67,11 @@ class ImpitHttpClient {
             writable: true,
             value: new datastructures_1.LruCache({ maxLength: 10 })
         });
-        this.impitOptions = options ?? {};
-        this.maxRedirects = options?.maxRedirects ?? 10;
-        this.followRedirects = options?.followRedirects ?? true;
+        const { maxRedirects = 10, followRedirects = true, cacheClients = true, ...impitOptions } = options ?? {};
+        this.impitOptions = impitOptions;
+        this.maxRedirects = maxRedirects;
+        this.followRedirects = followRedirects;
+        this.cacheClients = cacheClients;
     }
     /**
      * Flattens the headers of a `HttpRequest` to a format that can be passed to `impit`.
@@ -97,12 +111,29 @@ class ImpitHttpClient {
             return method !== 'HEAD';
         return false;
     }
+    /**
+     * Converts Fetch/Impit headers into a simple header map.
+     * `Object.fromEntries` would keep only the last `set-cookie` value, so those are collected separately.
+     */
+    intoSimpleHeaders(headers) {
+        const result = {};
+        for (const [key, value] of headers.entries()) {
+            if (key === 'set-cookie')
+                continue;
+            result[key] = value;
+        }
+        const setCookies = headers.getSetCookie();
+        if (setCookies.length > 0) {
+            result['set-cookie'] = setCookies.length === 1 ? setCookies[0] : setCookies;
+        }
+        return result;
+    }
     /**
      * Common implementation for `sendRequest` and `stream` methods.
      * @param request `HttpRequest` object
      * @returns `HttpResponse` object
      */
-    async getResponse(request, redirects) {
+    async getResponse(request, redirects, onRedirect) {
         if ((redirects?.redirectCount ?? 0) > this.maxRedirects) {
             throw new Error(`Too many redirects, maximum is ${this.maxRedirects}.`);
         }
@@ -125,14 +156,34 @@ class ImpitHttpClient {
             if (!location) {
                 throw new Error('Redirect response missing location header.');
             }
+            const nextRedirectUrls = [...(redirects?.redirectUrls ?? []), redirectUrl];
+            const updatedRequest = {
+                url: redirectUrl.href,
+                headers: { ...(request.headers ?? {}) },
+            };
+            // Match GotScrapingHttpClient: allow HttpCrawler to persist redirect cookies into the session
+            // and mutate Cookie / URL for the next hop.
+            onRedirect?.({
+                redirectUrls: nextRedirectUrls,
+                url,
+                statusCode: response.status,
+                statusMessage: response.statusText,
+                headers: this.intoSimpleHeaders(response.headers),
+                trailers: {},
+                complete: true,
+            }, updatedRequest);
+            const nextUrl = typeof updatedRequest.url === 'string'
+                ? updatedRequest.url
+                : (updatedRequest.url?.href ?? redirectUrl.href);
             return this.getResponse({
                 ...request,
                 method: this.shouldRewriteRedirectToGet(response.status, request.method) ? 'GET' : request.method,
-                url: redirectUrl.href,
+                url: nextUrl,
+                headers: updatedRequest.headers,
             }, {
                 redirectCount: (redirects?.redirectCount ?? 0) + 1,
-                redirectUrls: [...(redirects?.redirectUrls ?? []), redirectUrl],
-            });
+                redirectUrls: nextRedirectUrls,
+            }, onRedirect);
         }
         return {
             response,
@@ -159,7 +210,7 @@ class ImpitHttpClient {
                 throw new Error('Unsupported response type.');
         }
         return {
-            headers: Object.fromEntries(response.headers.entries()),
+            headers: this.intoSimpleHeaders(response.headers),
             statusCode: response.status,
             url: response.url,
             request,
@@ -193,8 +244,8 @@ class ImpitHttpClient {
     /**
      * @inheritDoc
      */
-    async stream(request) {
-        const { response, redirectUrls } = await this.getResponse(request);
+    async stream(request, onRedirect) {
+        const { response, redirectUrls } = await this.getResponse(request, undefined, onRedirect);
         const [stream, getDownloadProgress] = this.getStreamWithProgress(response);
         return {
             request,
@@ -207,7 +258,7 @@ class ImpitHttpClient {
             },
             uploadProgress: { percent: 100, transferred: 0 },
             redirectUrls,
-            headers: Object.fromEntries(response.headers.entries()),
+            headers: this.intoSimpleHeaders(response.headers),
             trailers: {},
         };
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/impit-client",
-    "version": "3.17.1-beta.9",
+    "version": "3.18.0",
     "description": "impit-based HTTP client implementation for Crawlee. Impersonates browser requests to avoid bot detection.",
     "engines": {
         "node": ">=20.0.0"
@@ -56,7 +56,7 @@
         "@crawlee/core": "^3.12.1"
     },
     "devDependencies": {
-        "@crawlee/core": "^3.17.0"
+        "@crawlee/core": "^3.18.0"
     },
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "c547ce4489d8617620a51516cc885187503f3136"
+    "gitHead": "49c115e1ce3b3fbf2bef61f16bffeadfdcf5bd19"
 }