@crawlee/core 4.0.0-beta.242 → 4.0.0-beta.244

package/configuration.js

@@ -1,5 +1,6 @@
 import { readFileSync } from 'node:fs';
 import { join } from 'node:path';
+import JSON5 from 'json5';
 import { z } from 'zod';
 import { log, LogLevel } from './log.js';
 import { serviceLocator } from './service-locator.js';
@@ -222,8 +223,9 @@ export class Configuration {
      */
     static #loadFileOptions() {
         try {
-            const file = readFileSync(join(process.cwd(), 'crawlee.json'));
-            return JSON.parse(file.toString());
+            const file = readFileSync(join(process.cwd(), 'crawlee.json'), 'utf8');
+            // JSON5 also handles a leading UTF-8 byte order mark, which some Windows tools write
+            return JSON5.parse(file);
         }
         catch {
             return {};

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.242",
+    "version": "4.0.0-beta.244",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.13.0"
@@ -53,10 +53,10 @@
         "@apify/log": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/fs-storage": "4.0.0-beta.242",
-        "@crawlee/http-client": "4.0.0-beta.242",
-        "@crawlee/types": "4.0.0-beta.242",
-        "@crawlee/utils": "4.0.0-beta.242",
+        "@crawlee/fs-storage": "4.0.0-beta.244",
+        "@crawlee/http-client": "4.0.0-beta.244",
+        "@crawlee/types": "4.0.0-beta.244",
+        "@crawlee/utils": "4.0.0-beta.244",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -76,5 +76,5 @@
             }
         }
     },
-    "gitHead": "9bd381d862366b6b874938fb0717f21fd430b9de"
+    "gitHead": "e26d10df9c1d550b7237ac27b25999432544d6a1"
 }

package/request.js

@@ -24,6 +24,27 @@ const requestOptionsSchema = z.compile(z.looseObject({
         .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
         .optional(),
 }));
+/** Storages return `null` for fields left unset by other clients, e.g. the Python SDK. */
+const unset = (schema) => schema.nullish().transform((value) => value ?? undefined);
+/** A request as storages return it. Accepts no less than the constructor does for the same fields. */
+const storedRequestSchema = z.compile(z.object({
+    id: z.string().optional(),
+    url: z.string(),
+    uniqueKey: z.string(),
+    method: z.custom((value) => typeof value === 'string').optional(),
+    payload: unset(z.custom((value) => typeof value === 'string' || value instanceof Uint8Array)),
+    noRetry: unset(z.boolean()),
+    retryCount: z.number().optional(),
+    // Entries are not checked, persisted requests can hold `null`s there (#4087).
+    errorMessages: unset(z.custom(Array.isArray)),
+    headers: unset(z.custom((value) => typeof value === 'object' && value !== null)),
+    userData: schemas.anyObject.optional(),
+    // `apify-client` parses `*At` fields into `Date`s, whatever the schema says.
+    handledAt: unset(z.custom((value) => typeof value === 'string' || value instanceof Date)),
+    loadedUrl: unset(z.string()),
+}));
+// `@crawlee/types` cannot depend on `zod`, so this keeps `RequestSchema` in sync with the schema above.
+true;
 /**
  * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata, together
  * with the processing state a request queue keeps about it (`retryCount`, `errorMessages`, `handledAt`, ...).
@@ -191,13 +212,13 @@ class CrawleeRequest {
      * {@link RequestOptions} object cannot carry. Subclasses get instances of themselves.
      */
     static fromSchema(schema) {
-        const { id, retryCount = 0, errorMessages = [], handledAt, loadedUrl, ...options } = schema;
+        const stored = parseArgument(schema, storedRequestSchema, 'RequestSchema');
+        const { id, retryCount = 0, errorMessages = [], handledAt, loadedUrl, ...options } = stored;
         const request = new this(options);
         request.id = id;
         request.retryCount = retryCount;
         request.errorMessages = [...errorMessages];
         request.loadedUrl = loadedUrl;
-        // `apify-client` parses `*At` fields into `Date`s, whatever the schema says.
         request.handledAt =
             handledAt instanceof Date ? handledAt.toISOString() : handledAt;
         return request;