@crawlee/core 4.0.0-beta.194 → 4.0.0-beta.196

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.194",
+    "version": "4.0.0-beta.196",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,10 +53,10 @@
         "@apify/log": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/fs-storage": "4.0.0-beta.194",
-        "@crawlee/http-client": "4.0.0-beta.194",
-        "@crawlee/types": "4.0.0-beta.194",
-        "@crawlee/utils": "4.0.0-beta.194",
+        "@crawlee/fs-storage": "4.0.0-beta.196",
+        "@crawlee/http-client": "4.0.0-beta.196",
+        "@crawlee/types": "4.0.0-beta.196",
+        "@crawlee/utils": "4.0.0-beta.196",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -76,5 +76,5 @@
             }
         }
     },
-    "gitHead": "c1f1ba3f6d24d5de59360c0616c49ce557ceb290"
+    "gitHead": "769227e112c5e1694b3e80b5a56dc4c007407d3d"
 }

package/request.js

@@ -2,11 +2,11 @@ import crypto from 'node:crypto';
 import { z } from 'zod';
 import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
 import { serviceLocator } from './service_locator.js';
-import { keys } from './typedefs.js';
 import { parseArgument, schemas } from './validators.js';
-const requestUrlSchema = z.object({ url: z.string() });
 // new properties on the Request object breaks serialization
-const requestOptionalSchemaShapes = {
+// Compiled once: every `Request` runs it, and the generated fast path skips zod's interpreter.
+const requestOptionsSchema = z.compile(z.looseObject({
+    url: z.string(),
     uniqueKey: z.string().optional(),
     method: z.string().optional(),
     payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
@@ -23,9 +23,7 @@ const requestOptionalSchemaShapes = {
     crawlDepth: schemas.anyNumber
         .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
         .optional(),
-};
-// Each schema is wrapped in a single-key object so validation errors carry the property name.
-const requestOptionalSchemas = Object.fromEntries(Object.entries(requestOptionalSchemaShapes).map(([key, schema]) => [key, z.object({ [key]: schema })]));
+}));
 /**
  * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata, together
  * with the processing state a request queue keeps about it (`retryCount`, `errorMessages`, `handledAt`, ...).
@@ -110,22 +108,7 @@ class CrawleeRequest {
                 'Did you mean `new Request({ url })`?');
         }
         parseArgument(options, schemas.anyObject, 'RequestOptions');
-        parseArgument(options, requestUrlSchema, 'RequestOptions');
-        // Full-shape validation is slow, because it checks all predicates
-        // even if the validated object has only 1 property.
-        // This custom validation loop iterates only over existing
-        // properties and speeds up the validation cca 3-fold.
-        keys(options).forEach((prop) => {
-            // skip url, because it is validated above
-            if (prop === 'url') {
-                return;
-            }
-            const schema = requestOptionalSchemas[prop];
-            const value = options[prop];
-            if (schema) {
-                parseArgument({ [prop]: value }, schema, 'RequestOptions');
-            }
-        });
+        parseArgument(options, requestOptionsSchema, 'RequestOptions');
         const { url, uniqueKey, payload, noRetry = false, sessionId, maxRetries, headers = {}, userData = {}, label, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
         let { method = 'GET' } = options;
         method = method.toUpperCase();

package/storages/request_queue.js

@@ -38,19 +38,20 @@ const addRequestsBatchedOptionsSchema = z.strictObject({
     waitBetweenBatchesMillis: schemas.anyNumber.default(1000),
     maxNewRequests: schemas.anyNumber.optional(),
 });
-const newRequestLikeSchema = z.looseObject({
+// Compiled: these run once per request.
+const newRequestLikeSchema = z.compile(z.looseObject({
     url: z.string(),
     id: z.undefined().optional(),
-});
-const handledRequestSchema = z.looseObject({
+}));
+const handledRequestSchema = z.compile(z.looseObject({
     id: z.string(),
     uniqueKey: z.string(),
     handledAt: z.string().optional(),
-});
-const reclaimedRequestSchema = z.looseObject({
+}));
+const reclaimedRequestSchema = z.compile(z.looseObject({
     id: z.string(),
     uniqueKey: z.string(),
-});
+}));
 const uniqueKeySchema = z.string();
 const openOptionsSchema = z.strictObject({
     configuration: z.instanceof(Configuration).optional(),