@crawlee/basic 4.0.0-beta.21 → 4.0.0-beta.22

package/internals/send-request.d.ts

@@ -1,5 +1,5 @@
-import type { Request, Session } from '@crawlee/core';
-import type { BaseHttpClient, HttpRequestOptions } from '@crawlee/types';
+import type { Request as CrawleeRequest, Session } from '@crawlee/core';
+import type { BaseHttpClient, HttpRequestOptions, SendRequestOptions } from '@crawlee/types';
 /**
  * Prepares a function to be used as the `sendRequest` context helper.
  *
@@ -7,7 +7,6 @@ import type { BaseHttpClient, HttpRequestOptions } from '@crawlee/types';
  * @param httpClient The HTTP client that will perform the requests.
  * @param originRequest The crawling request being processed.
  * @param session The user session associated with the current request.
- * @param getProxyUrl A function that will return the proxy URL that should be used for handling the request.
  */
-export declare function createSendRequest(httpClient: BaseHttpClient, originRequest: Request, session: Session | undefined): (overrideOptions?: Partial<HttpRequestOptions>) => Promise<Response>;
+export declare function createSendRequest(httpClient: BaseHttpClient, originRequest: CrawleeRequest, session: Session | undefined): (overrideRequest?: Partial<HttpRequestOptions>, overrideOptions?: SendRequestOptions) => Promise<Response>;
 //# sourceMappingURL=send-request.d.ts.map

package/internals/send-request.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"send-request.d.ts","sourceRoot":"","sources":["../../src/internals/send-request.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACtD,OAAO,KAAK,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAEzE;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,GAAG,SAAS,IAChG,kBAAiB,OAAO,CAAC,kBAAkB,CAAM,KAAG,OAAO,CAAC,QAAQ,CAAC,CAWtF"}
+{"version":3,"file":"send-request.d.ts","sourceRoot":"","sources":["../../src/internals/send-request.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,IAAI,cAAc,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxE,OAAO,KAAK,EAAE,cAAc,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAE7F;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC7B,UAAU,EAAE,cAAc,EAC1B,aAAa,EAAE,cAAc,EAC7B,OAAO,EAAE,OAAO,GAAG,SAAS,IAGxB,kBAAiB,OAAO,CAAC,kBAAkB,CAAM,EACjD,kBAAiB,kBAAuB,KACzC,OAAO,CAAC,QAAQ,CAAC,CA6BvB"}

package/internals/send-request.js

@@ -5,18 +5,32 @@
  * @param httpClient The HTTP client that will perform the requests.
  * @param originRequest The crawling request being processed.
  * @param session The user session associated with the current request.
- * @param getProxyUrl A function that will return the proxy URL that should be used for handling the request.
  */
 export function createSendRequest(httpClient, originRequest, session) {
-    return async (overrideOptions = {}) => {
-        const cookieJar = session
-            ? {
-                getCookieString: async (url) => session.getCookieString(url),
-                setCookie: async (rawCookie, url) => session.setCookie(rawCookie, url),
-                ...overrideOptions?.cookieJar,
-            }
-            : overrideOptions?.cookieJar;
-        return httpClient.sendRequest(originRequest.intoFetchAPIRequest(), { session, cookieJar: cookieJar });
+    return async (overrideRequest = {}, overrideOptions = {}) => {
+        const sessionCookieJar = {
+            getCookieString: async (url) => session?.getCookieString(url),
+            setCookie: async (rawCookie, url) => session?.setCookie(rawCookie, url),
+        };
+        const baseRequest = originRequest.intoFetchAPIRequest();
+        const mergedUrl = overrideRequest.url ?? baseRequest.url;
+        const mergedMethod = overrideRequest.method ?? baseRequest.method;
+        const mergedHeaders = new Headers(baseRequest.headers);
+        if (overrideRequest.headers) {
+            overrideRequest.headers.forEach((value, key) => {
+                mergedHeaders.set(key, value);
+            });
+        }
+        const request = new Request(mergedUrl, {
+            method: mergedMethod,
+            headers: mergedHeaders,
+            body: overrideRequest.body ?? baseRequest.body,
+        });
+        return httpClient.sendRequest(request, {
+            session,
+            cookieJar: overrideOptions?.cookieJar ?? sessionCookieJar,
+            timeout: overrideOptions.timeout,
+        });
     };
 }
 //# sourceMappingURL=send-request.js.map

package/internals/send-request.js.map

@@ -1 +1 @@
-{"version":3,"file":"send-request.js","sourceRoot":"","sources":["../../src/internals/send-request.ts"],"names":[],"mappings":"AAGA;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB,CAAC,UAA0B,EAAE,aAAsB,EAAE,OAA4B;IAC9G,OAAO,KAAK,EAAE,kBAA+C,EAAE,EAAqB,EAAE;QAClF,MAAM,SAAS,GAAG,OAAO;YACrB,CAAC,CAAC;gBACI,eAAe,EAAE,KAAK,EAAE,GAAW,EAAE,EAAE,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC;gBACpE,SAAS,EAAE,KAAK,EAAE,SAAiB,EAAE,GAAW,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,EAAE,GAAG,CAAC;gBACtF,GAAG,eAAe,EAAE,SAAS;aAChC;YACH,CAAC,CAAC,eAAe,EAAE,SAAS,CAAC;QAEjC,OAAO,UAAU,CAAC,WAAW,CAAC,aAAa,CAAC,mBAAmB,EAAE,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,SAAgB,EAAE,CAAC,CAAC;IACjH,CAAC,CAAC;AACN,CAAC"}
+{"version":3,"file":"send-request.js","sourceRoot":"","sources":["../../src/internals/send-request.ts"],"names":[],"mappings":"AAGA;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAC7B,UAA0B,EAC1B,aAA6B,EAC7B,OAA4B;IAE5B,OAAO,KAAK,EACR,kBAA+C,EAAE,EACjD,kBAAsC,EAAE,EACvB,EAAE;QACnB,MAAM,gBAAgB,GAAG;YACrB,eAAe,EAAE,KAAK,EAAE,GAAW,EAAE,EAAE,CAAC,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC;YACrE,SAAS,EAAE,KAAK,EAAE,SAAiB,EAAE,GAAW,EAAE,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC,SAAS,EAAE,GAAG,CAAC;SAC1F,CAAC;QAEF,MAAM,WAAW,GAAG,aAAa,CAAC,mBAAmB,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,eAAe,CAAC,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC;QACzD,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM,CAAC;QAElE,MAAM,aAAa,GAAG,IAAI,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QACvD,IAAI,eAAe,CAAC,OAAO,EAAE,CAAC;YAC1B,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBAC3C,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC;QACP,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,SAAS,EAAE;YACnC,MAAM,EAAE,YAAY;YACpB,OAAO,EAAE,aAAa;YACtB,IAAI,EAAE,eAAe,CAAC,IAAI,IAAI,WAAW,CAAC,IAAI;SAClC,CAAC,CAAC;QAElB,OAAO,UAAU,CAAC,WAAW,CAAC,OAAO,EAAE;YACnC,OAAO;YACP,SAAS,EAAE,eAAe,EAAE,SAAS,IAAK,gBAAwB;YAClE,OAAO,EAAE,eAAe,CAAC,OAAO;SACnC,CAAC,CAAC;IACP,CAAC,CAAC;AACN,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.21",
+    "version": "4.0.0-beta.22",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,9 +42,9 @@
         "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.21",
-        "@crawlee/types": "4.0.0-beta.21",
-        "@crawlee/utils": "4.0.0-beta.21",
+        "@crawlee/core": "4.0.0-beta.22",
+        "@crawlee/types": "4.0.0-beta.22",
+        "@crawlee/utils": "4.0.0-beta.22",
         "csv-stringify": "^6.5.2",
         "fs-extra": "^11.3.0",
         "got-scraping": "^4.1.1",
@@ -60,5 +60,5 @@
             }
         }
     },
-    "gitHead": "e370ab2f4ffcf4f63b52a61c9b6e97081d525e64"
+    "gitHead": "74e5a8dae2b5c919ca40cc3f9135744448aca0ac"
 }