@crawlee/core 4.0.0-beta.193 → 4.0.0-beta.195

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.193",
+    "version": "4.0.0-beta.195",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,10 +53,10 @@
         "@apify/log": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/fs-storage": "4.0.0-beta.193",
-        "@crawlee/http-client": "4.0.0-beta.193",
-        "@crawlee/types": "4.0.0-beta.193",
-        "@crawlee/utils": "4.0.0-beta.193",
+        "@crawlee/fs-storage": "4.0.0-beta.195",
+        "@crawlee/http-client": "4.0.0-beta.195",
+        "@crawlee/types": "4.0.0-beta.195",
+        "@crawlee/utils": "4.0.0-beta.195",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -76,5 +76,5 @@
             }
         }
     },
-    "gitHead": "db10ed8633118f3d53dcef58118f408ce578d5d5"
+    "gitHead": "7ef1df353c43614acc1275a53ef8ca96e850fe3e"
 }

package/request.js

@@ -2,11 +2,11 @@ import crypto from 'node:crypto';
 import { z } from 'zod';
 import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
 import { serviceLocator } from './service_locator.js';
-import { keys } from './typedefs.js';
 import { parseArgument, schemas } from './validators.js';
-const requestUrlSchema = z.object({ url: z.string() });
 // new properties on the Request object breaks serialization
-const requestOptionalSchemaShapes = {
+// Compiled once: every `Request` runs it, and the generated fast path skips zod's interpreter.
+const requestOptionsSchema = z.compile(z.looseObject({
+    url: z.string(),
     uniqueKey: z.string().optional(),
     method: z.string().optional(),
     payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
@@ -23,9 +23,7 @@ const requestOptionalSchemaShapes = {
     crawlDepth: schemas.anyNumber
         .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
         .optional(),
-};
-// Each schema is wrapped in a single-key object so validation errors carry the property name.
-const requestOptionalSchemas = Object.fromEntries(Object.entries(requestOptionalSchemaShapes).map(([key, schema]) => [key, z.object({ [key]: schema })]));
+}));
 /**
  * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata, together
  * with the processing state a request queue keeps about it (`retryCount`, `errorMessages`, `handledAt`, ...).
@@ -110,22 +108,7 @@ class CrawleeRequest {
                 'Did you mean `new Request({ url })`?');
         }
         parseArgument(options, schemas.anyObject, 'RequestOptions');
-        parseArgument(options, requestUrlSchema, 'RequestOptions');
-        // Full-shape validation is slow, because it checks all predicates
-        // even if the validated object has only 1 property.
-        // This custom validation loop iterates only over existing
-        // properties and speeds up the validation cca 3-fold.
-        keys(options).forEach((prop) => {
-            // skip url, because it is validated above
-            if (prop === 'url') {
-                return;
-            }
-            const schema = requestOptionalSchemas[prop];
-            const value = options[prop];
-            if (schema) {
-                parseArgument({ [prop]: value }, schema, 'RequestOptions');
-            }
-        });
+        parseArgument(options, requestOptionsSchema, 'RequestOptions');
         const { url, uniqueKey, payload, noRetry = false, sessionId, maxRetries, headers = {}, userData = {}, label, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
         let { method = 'GET' } = options;
         method = method.toUpperCase();

package/storages/request_manager.d.ts

@@ -41,6 +41,16 @@ export interface IRequestManager extends IRequestLoader {
      * @returns `true` if anything in the composition took responsibility for the signal.
      */
     recordPacingSignal(signal: PacingSignal): boolean;
+    /**
+     * Extends the lock on a request previously handed out by `fetchNextRequest()` and still being
+     * processed, on storage backends that reserve requests via locking (e.g. via
+     * {@link CrawlingContext.extendTimeout|`context.extendTimeout`}).
+     *
+     * @returns `true` when the lock was prolonged, `false` when this manager does not lock
+     * requests or no longer holds this one; non-locking implementations may leave it `undefined`,
+     * which callers treat as `false`.
+     */
+    extendRequestProcessingTimeSecs?(request: Request, secs: number): Promise<boolean>;
 }
 /**
  * How much of the URL space a {@link PacingSignal} covers.

package/storages/request_manager_tandem.d.ts

@@ -92,4 +92,5 @@ export declare class RequestManagerTandem implements IRequestManager {
      * @inheritdoc
      */
     recordPacingSignal(signal: PacingSignal): boolean;
+    extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean>;
 }

package/storages/request_manager_tandem.js

@@ -196,4 +196,7 @@ export class RequestManagerTandem {
     recordPacingSignal(signal) {
         return this.#resolvedRequestManager?.recordPacingSignal(signal) ?? false;
     }
+    async extendRequestProcessingTimeSecs(request, secs) {
+        return (await this.getRequestManager()).extendRequestProcessingTimeSecs?.(request, secs) ?? false;
+    }
 }

package/storages/request_queue.d.ts

@@ -195,6 +195,12 @@ export declare class RequestQueue implements IStorage, IRequestManager {
      * short the reservation of a long-lived one and have its in-flight request stolen.
      */
     setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
+    /**
+     * @inheritdoc
+     * Unlike {@link RequestQueue.setExpectedRequestProcessingTimeSecs}, which sizes every future lock,
+     * this only touches the one request it is given.
+     */
+    extendRequestProcessingTimeSecs(request: Request, secs: number): Promise<boolean>;
     /**
      * Caches information about request to beware of unneeded addRequest() calls.
      */

package/storages/request_queue.js

@@ -38,19 +38,20 @@ const addRequestsBatchedOptionsSchema = z.strictObject({
     waitBetweenBatchesMillis: schemas.anyNumber.default(1000),
     maxNewRequests: schemas.anyNumber.optional(),
 });
-const newRequestLikeSchema = z.looseObject({
+// Compiled: these run once per request.
+const newRequestLikeSchema = z.compile(z.looseObject({
     url: z.string(),
     id: z.undefined().optional(),
-});
-const handledRequestSchema = z.looseObject({
+}));
+const handledRequestSchema = z.compile(z.looseObject({
     id: z.string(),
     uniqueKey: z.string(),
     handledAt: z.string().optional(),
-});
-const reclaimedRequestSchema = z.looseObject({
+}));
+const reclaimedRequestSchema = z.compile(z.looseObject({
     id: z.string(),
     uniqueKey: z.string(),
-});
+}));
 const uniqueKeySchema = z.string();
 const openOptionsSchema = z.strictObject({
     configuration: z.instanceof(Configuration).optional(),
@@ -674,6 +675,17 @@ export class RequestQueue {
         this.#expectedRequestProcessingSecs = secs;
         await this.backend.setExpectedRequestProcessingTimeSecs?.(secs);
     }
+    /**
+     * @inheritdoc
+     * Unlike {@link RequestQueue.setExpectedRequestProcessingTimeSecs}, which sizes every future lock,
+     * this only touches the one request it is given.
+     */
+    async extendRequestProcessingTimeSecs(request, secs) {
+        if (!request.id) {
+            return false;
+        }
+        return (await this.backend.extendRequestProcessingTimeSecs?.(request.id, secs)) ?? false;
+    }
     /**
      * Caches information about request to beware of unneeded addRequest() calls.
      */