@crawlee/impit-client 3.17.1-beta.71 → 3.17.1-beta.72

package/index.d.ts

@@ -1,4 +1,4 @@
-import type { BaseHttpClient, HttpRequest, HttpResponse, ResponseTypes, StreamingHttpResponse } from '@crawlee/core';
+import type { BaseHttpClient, HttpRequest, HttpResponse, RedirectHandler, ResponseTypes, StreamingHttpResponse } from '@crawlee/core';
 import type { ImpitOptions } from 'impit';
 export declare const Browser: {
     readonly Chrome: "chrome";
@@ -35,6 +35,11 @@ export declare class ImpitHttpClient implements BaseHttpClient {
     private intoHeaders;
     private intoImpitBody;
     private shouldRewriteRedirectToGet;
+    /**
+     * Converts Fetch/Impit headers into a simple header map.
+     * `Object.fromEntries` would keep only the last `set-cookie` value, so those are collected separately.
+     */
+    private intoSimpleHeaders;
     /**
      * Common implementation for `sendRequest` and `stream` methods.
      * @param request `HttpRequest` object
@@ -49,5 +54,5 @@ export declare class ImpitHttpClient implements BaseHttpClient {
     /**
      * @inheritDoc
      */
-    stream(request: HttpRequest): Promise<StreamingHttpResponse>;
+    stream(request: HttpRequest, onRedirect?: RedirectHandler): Promise<StreamingHttpResponse>;
 }

package/index.js

@@ -111,12 +111,29 @@ class ImpitHttpClient {
             return method !== 'HEAD';
         return false;
     }
+    /**
+     * Converts Fetch/Impit headers into a simple header map.
+     * `Object.fromEntries` would keep only the last `set-cookie` value, so those are collected separately.
+     */
+    intoSimpleHeaders(headers) {
+        const result = {};
+        for (const [key, value] of headers.entries()) {
+            if (key === 'set-cookie')
+                continue;
+            result[key] = value;
+        }
+        const setCookies = headers.getSetCookie();
+        if (setCookies.length > 0) {
+            result['set-cookie'] = setCookies.length === 1 ? setCookies[0] : setCookies;
+        }
+        return result;
+    }
     /**
      * Common implementation for `sendRequest` and `stream` methods.
      * @param request `HttpRequest` object
      * @returns `HttpResponse` object
      */
-    async getResponse(request, redirects) {
+    async getResponse(request, redirects, onRedirect) {
         if ((redirects?.redirectCount ?? 0) > this.maxRedirects) {
             throw new Error(`Too many redirects, maximum is ${this.maxRedirects}.`);
         }
@@ -139,14 +156,34 @@ class ImpitHttpClient {
             if (!location) {
                 throw new Error('Redirect response missing location header.');
             }
+            const nextRedirectUrls = [...(redirects?.redirectUrls ?? []), redirectUrl];
+            const updatedRequest = {
+                url: redirectUrl.href,
+                headers: { ...(request.headers ?? {}) },
+            };
+            // Match GotScrapingHttpClient: allow HttpCrawler to persist redirect cookies into the session
+            // and mutate Cookie / URL for the next hop.
+            onRedirect?.({
+                redirectUrls: nextRedirectUrls,
+                url,
+                statusCode: response.status,
+                statusMessage: response.statusText,
+                headers: this.intoSimpleHeaders(response.headers),
+                trailers: {},
+                complete: true,
+            }, updatedRequest);
+            const nextUrl = typeof updatedRequest.url === 'string'
+                ? updatedRequest.url
+                : (updatedRequest.url?.href ?? redirectUrl.href);
             return this.getResponse({
                 ...request,
                 method: this.shouldRewriteRedirectToGet(response.status, request.method) ? 'GET' : request.method,
-                url: redirectUrl.href,
+                url: nextUrl,
+                headers: updatedRequest.headers,
             }, {
                 redirectCount: (redirects?.redirectCount ?? 0) + 1,
-                redirectUrls: [...(redirects?.redirectUrls ?? []), redirectUrl],
-            });
+                redirectUrls: nextRedirectUrls,
+            }, onRedirect);
         }
         return {
             response,
@@ -173,7 +210,7 @@ class ImpitHttpClient {
                 throw new Error('Unsupported response type.');
         }
         return {
-            headers: Object.fromEntries(response.headers.entries()),
+            headers: this.intoSimpleHeaders(response.headers),
             statusCode: response.status,
             url: response.url,
             request,
@@ -207,8 +244,8 @@ class ImpitHttpClient {
     /**
      * @inheritDoc
      */
-    async stream(request) {
-        const { response, redirectUrls } = await this.getResponse(request);
+    async stream(request, onRedirect) {
+        const { response, redirectUrls } = await this.getResponse(request, undefined, onRedirect);
         const [stream, getDownloadProgress] = this.getStreamWithProgress(response);
         return {
             request,
@@ -221,7 +258,7 @@ class ImpitHttpClient {
             },
             uploadProgress: { percent: 100, transferred: 0 },
             redirectUrls,
-            headers: Object.fromEntries(response.headers.entries()),
+            headers: this.intoSimpleHeaders(response.headers),
             trailers: {},
         };
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/impit-client",
-    "version": "3.17.1-beta.71",
+    "version": "3.17.1-beta.72",
     "description": "impit-based HTTP client implementation for Crawlee. Impersonates browser requests to avoid bot detection.",
     "engines": {
         "node": ">=20.0.0"
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "94c4c029140e4068184ad3f03d0a71fd8d6984dc"
+    "gitHead": "b28c3cd14324b806ed2d4691790a423fb1760ea0"
 }