@crawlee/core 4.0.0-beta.27 → 4.0.0-beta.270

package/request.js

@@ -1,46 +1,55 @@
 import crypto from 'node:crypto';
-import util from 'node:util';
-import ow from 'ow';
-import { normalizeUrl } from '@apify/utilities';
-import { log as defaultLog } from './log.js';
-import { keys } from './typedefs.js';
+import { z } from 'zod';
+import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
+import { serviceLocator } from './service-locator.js';
+import { snapshotValue } from './storages/transaction.js';
+import { parseArgument, schemas } from './validators.js';
 // new properties on the Request object breaks serialization
-const log = defaultLog.child({ prefix: 'Request' });
-const requestOptionalPredicates = {
-    id: ow.optional.string,
-    loadedUrl: ow.optional.string.url,
-    uniqueKey: ow.optional.string,
-    method: ow.optional.string,
-    payload: ow.optional.any(ow.string, ow.uint8Array),
-    noRetry: ow.optional.boolean,
-    retryCount: ow.optional.number,
-    sessionRotationCount: ow.optional.number,
-    maxRetries: ow.optional.number,
-    errorMessages: ow.optional.array.ofType(ow.string),
-    headers: ow.optional.object,
-    userData: ow.optional.object,
-    label: ow.optional.string,
-    handledAt: ow.optional.any(ow.string.date, ow.date),
-    keepUrlFragment: ow.optional.boolean,
-    useExtendedUniqueKey: ow.optional.boolean,
-    skipNavigation: ow.optional.boolean,
-    crawlDepth: ow.optional.number.greaterThanOrEqual(0),
-    state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
-};
-export var RequestState;
-(function (RequestState) {
-    RequestState[RequestState["UNPROCESSED"] = 0] = "UNPROCESSED";
-    RequestState[RequestState["BEFORE_NAV"] = 1] = "BEFORE_NAV";
-    RequestState[RequestState["AFTER_NAV"] = 2] = "AFTER_NAV";
-    RequestState[RequestState["REQUEST_HANDLER"] = 3] = "REQUEST_HANDLER";
-    RequestState[RequestState["DONE"] = 4] = "DONE";
-    RequestState[RequestState["ERROR_HANDLER"] = 5] = "ERROR_HANDLER";
-    RequestState[RequestState["ERROR"] = 6] = "ERROR";
-    RequestState[RequestState["SKIPPED"] = 7] = "SKIPPED";
-})(RequestState || (RequestState = {}));
+// Compiled once: every `Request` runs it, and the generated fast path skips zod's interpreter.
+const requestOptionsSchema = z.compile(z.looseObject({
+    url: z.string(),
+    uniqueKey: z.string().optional(),
+    method: z.string().optional(),
+    payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
+    noRetry: z.boolean().optional(),
+    sessionId: z.string().optional(),
+    maxRetries: schemas.anyNumber.optional(),
+    headers: z.looseObject({}).optional(),
+    userData: z.looseObject({}).optional(),
+    label: z.string().optional(),
+    keepUrlFragment: z.boolean().optional(),
+    useExtendedUniqueKey: z.boolean().optional(),
+    alwaysEnqueue: z.boolean().optional(),
+    skipNavigation: z.boolean().optional(),
+    crawlDepth: schemas.anyNumber
+        .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
+        .optional(),
+}));
+/** Storages return `null` for fields left unset by other clients, e.g. the Python SDK. */
+const unset = (schema) => schema.nullish().transform((value) => value ?? undefined);
+/** A request as storages return it. Accepts no less than the constructor does for the same fields. */
+const storedRequestSchema = z.compile(z.object({
+    id: z.string().optional(),
+    url: z.string(),
+    uniqueKey: z.string(),
+    method: z.custom((value) => typeof value === 'string').optional(),
+    payload: unset(z.custom((value) => typeof value === 'string' || value instanceof Uint8Array)),
+    noRetry: unset(z.boolean()),
+    retryCount: z.number().optional(),
+    // Entries are not checked, persisted requests can hold `null`s there (#4087).
+    errorMessages: unset(z.custom(Array.isArray)),
+    headers: unset(z.custom((value) => typeof value === 'object' && value !== null)),
+    userData: schemas.anyObject.optional(),
+    // `apify-client` parses `*At` fields into `Date`s, whatever the schema says.
+    handledAt: unset(z.custom((value) => typeof value === 'string' || value instanceof Date)),
+    loadedUrl: unset(z.string()),
+}));
+// `@crawlee/types` cannot depend on `zod`, so this keeps `RequestSchema` in sync with the schema above.
+true;
 /**
- * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
- * The `Request` object also stores information about errors that occurred during processing of the request.
+ * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata, together
+ * with the processing state a request queue keeps about it (`retryCount`, `errorMessages`, `handledAt`, ...).
+ * Inside a crawler, requests are {@link CrawlingRequest}s, which add the crawler's own per-request settings.
  *
  * Each `Request` instance has the `uniqueKey` property, which can be either specified
  * manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
@@ -56,21 +65,15 @@ export var RequestState;
  * const request = new Request({
  *     url: 'http://www.example.com',
  *     headers: { Accept: 'application/json' },
+ *     userData: { foo: 'bar' },
  * });
  *
- * ...
- *
- * request.userData.foo = 'bar';
- * request.pushErrorMessage(new Error('Request failed!'));
- *
- * ...
- *
- * const foo = request.userData.foo;
+ * await requestQueue.addRequest(request);
  * ```
  * @category Sources
  */
 class CrawleeRequest {
-    /** Request ID */
+    /** Storage-assigned request ID. Only present on requests that went through a {@link RequestQueue}. */
     id;
     /** URL of the web page to crawl. */
     url;
@@ -101,8 +104,13 @@ class CrawleeRequest {
     /** Object with HTTP headers. Key is header name, value is the value. */
     headers;
     /** Private store for the custom user data assigned to the request. */
-    _userData = {};
-    /** Custom user data assigned to the request. */
+    #userData = {};
+    /**
+     * Custom user data assigned to the request.
+     *
+     * All data stored in `userData` must be JSON-serializable.
+     * Storing non-serializable values (e.g. functions, symbols) may result in unexpected results.
+     */
     userData = {};
     /**
      * ISO datetime string that indicates the time when the request has been processed.
@@ -111,94 +119,113 @@ class CrawleeRequest {
     handledAt;
     /**
      * `Request` parameters including the URL, HTTP method and headers, and others.
+     *
+     * Processing state (`retryCount`, `errorMessages`, `handledAt`, `loadedUrl`, `id`) is not an option;
+     * requests coming back from a storage are rebuilt with {@link Request.fromSchema}.
      */
     constructor(options) {
-        ow(options, 'RequestOptions', ow.object);
-        ow(options.url, 'RequestOptions.url', ow.string);
-        // 'ow' validation is slow, because it checks all predicates
-        // even if the validated object has only 1 property.
-        // This custom validation loop iterates only over existing
-        // properties and speeds up the validation cca 3-fold.
-        // See https://github.com/sindresorhus/ow/issues/193
-        keys(options).forEach((prop) => {
-            // skip url, because it is validated above
-            if (prop === 'url') {
-                return;
-            }
-            const predicate = requestOptionalPredicates[prop];
-            const value = options[prop];
-            if (predicate) {
-                ow(value, `RequestOptions.${prop}`, predicate);
-            }
-        });
-        const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionRotationCount = 0, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
+        // A bare URL is a common slip — point at the object form instead of a generic type error.
+        if (typeof options === 'string') {
+            throw new TypeError(`\`Request\` options must be an object, got the string '${options}'. ` +
+                'Did you mean `new Request({ url })`?');
+        }
+        parseArgument(options, schemas.anyObject, 'RequestOptions');
+        parseArgument(options, requestOptionsSchema, 'RequestOptions');
+        const { url, uniqueKey, payload, noRetry = false, sessionId, maxRetries, headers = {}, userData = {}, label, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
         let { method = 'GET' } = options;
         method = method.toUpperCase();
         if (method === 'GET' && payload)
             throw new Error('Request with GET method cannot have a payload.');
-        this.id = id;
+        if (uniqueKey && alwaysEnqueue) {
+            throw new Error('`alwaysEnqueue` cannot be used together with a custom `uniqueKey`.');
+        }
         this.url = url;
-        this.loadedUrl = loadedUrl;
         this.uniqueKey =
             uniqueKey ||
-                CrawleeRequest.computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
+                CrawleeRequest.computeUniqueKey({
+                    url,
+                    method,
+                    payload,
+                    keepUrlFragment,
+                    useExtendedUniqueKey,
+                    alwaysEnqueue,
+                });
         this.method = method;
         this.payload = payload;
         this.noRetry = noRetry;
-        this.retryCount = retryCount;
-        this.sessionRotationCount = sessionRotationCount;
-        this.errorMessages = [...errorMessages];
+        this.retryCount = 0;
+        this.errorMessages = [];
         this.headers = { ...headers };
-        this.handledAt = handledAt instanceof Date ? handledAt.toISOString() : handledAt;
         if (label) {
             userData.label = label;
         }
+        // Read `__crawlee` explicitly - on a `userData` coming from another Request instance the
+        // bag is non-enumerable, so the spread alone would silently drop the internal state
+        // (e.g. `skipNavigation`) when a request is re-wrapped after a storage round trip.
+        this.#userData = {
+            ...snapshotValue(userData),
+            __crawlee: snapshotValue(userData.__crawlee ?? {}),
+        };
+        // `userData` must stay an enumerable own accessor — serialization in the storages relies on it
         Object.defineProperties(this, {
-            _userData: {
-                value: { __crawlee: {}, ...userData },
-                enumerable: false,
-                writable: true,
-            },
             userData: {
-                get: () => this._userData,
+                get: () => this.#userData,
                 set: (value) => {
                     Object.defineProperties(value, {
                         __crawlee: {
-                            value: this._userData.__crawlee,
+                            value: this.#userData.__crawlee,
                             enumerable: false,
                             writable: true,
                         },
                         toJSON: {
                             value: () => {
-                                if (Object.keys(this._userData.__crawlee).length > 0) {
+                                if (Object.keys(this.#userData.__crawlee).length > 0) {
                                     return {
-                                        ...this._userData,
-                                        __crawlee: this._userData.__crawlee,
+                                        ...this.#userData,
+                                        __crawlee: this.#userData.__crawlee,
                                     };
                                 }
-                                return this._userData;
+                                return this.#userData;
                             },
                             enumerable: false,
                             writable: true,
                         },
                     });
-                    this._userData = value;
+                    this.#userData = value;
                 },
                 enumerable: true,
             },
         });
-        // reassign userData to ensure internal `__crawlee` object is non-enumerable
-        this.userData = userData;
+        // reassign the copy, not the possibly shared `userData`, so that internal `__crawlee` object is non-enumerable
+        this.userData = this.#userData;
+        const crawlee = (this.userData.__crawlee ??= {});
         if (skipNavigation != null)
-            this.skipNavigation = skipNavigation;
+            crawlee.skipNavigation = skipNavigation;
         if (maxRetries != null)
-            this.maxRetries = maxRetries;
+            crawlee.maxRetries = maxRetries;
         if (crawlDepth != null)
-            this.userData.__crawlee.crawlDepth ??= crawlDepth;
-        // If it's already set, don't override it (for instance when fetching from storage)
-        if (enqueueStrategy) {
-            this.enqueueStrategy ??= enqueueStrategy;
-        }
+            crawlee.crawlDepth ??= crawlDepth;
+        if (sessionId)
+            crawlee.sessionId = sessionId;
+        // A stored request keeps the strategy it was enqueued under.
+        if (enqueueStrategy)
+            crawlee.enqueueStrategy ??= enqueueStrategy;
+    }
+    /**
+     * Rebuilds a request from its stored form, including the processing state a
+     * {@link RequestOptions} object cannot carry. Subclasses get instances of themselves.
+     */
+    static fromSchema(schema) {
+        const stored = parseArgument(schema, storedRequestSchema, 'RequestSchema');
+        const { id, retryCount = 0, errorMessages = [], handledAt, loadedUrl, ...options } = stored;
+        const request = new this(options);
+        request.id = id;
+        request.retryCount = retryCount;
+        request.errorMessages = [...errorMessages];
+        request.loadedUrl = loadedUrl;
+        request.handledAt =
+            handledAt instanceof Date ? handledAt.toISOString() : handledAt;
+        return request;
     }
     /**
      * Converts the Crawlee Request object to a `fetch` API Request object.
@@ -211,47 +238,6 @@ class CrawleeRequest {
             body: this.payload,
         });
     }
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
-    get skipNavigation() {
-        return this.userData.__crawlee?.skipNavigation ?? false;
-    }
-    /** Tells the crawler processing this request to skip the navigation and process the request directly. */
-    set skipNavigation(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { skipNavigation: value };
-        }
-        else {
-            this.userData.__crawlee.skipNavigation = value;
-        }
-    }
-    /**
-     * Depth of the request in the current crawl tree.
-     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
-     */
-    get crawlDepth() {
-        return this.userData.__crawlee?.crawlDepth ?? 0;
-    }
-    /**
-     * Depth of the request in the current crawl tree.
-     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
-     */
-    set crawlDepth(value) {
-        this.userData.__crawlee ??= {};
-        this.userData.__crawlee.crawlDepth = value;
-    }
-    /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
-    get sessionRotationCount() {
-        return this.userData.__crawlee?.sessionRotationCount ?? 0;
-    }
-    /** Indicates the number of times the crawling of the request has rotated the session due to a session or a proxy error. */
-    set sessionRotationCount(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { sessionRotationCount: value };
-        }
-        else {
-            this.userData.__crawlee.sessionRotationCount = value;
-        }
-    }
     /** shortcut for getting `request.userData.label` */
     get label() {
         return this.userData.label;
@@ -260,124 +246,29 @@ class CrawleeRequest {
     set label(value) {
         this.userData.label = value;
     }
-    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
-    get maxRetries() {
-        return this.userData.__crawlee?.maxRetries;
-    }
-    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
-    set maxRetries(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { maxRetries: value };
-        }
-        else {
-            this.userData.__crawlee.maxRetries = value;
-        }
-    }
-    /** Describes the request's current lifecycle state. */
-    get state() {
-        return this.userData.__crawlee?.state ?? RequestState.UNPROCESSED;
-    }
-    /** Describes the request's current lifecycle state. */
-    set state(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { state: value };
-        }
-        else {
-            this.userData.__crawlee.state = value;
-        }
-    }
-    /**
-     * Reason for skipping this request.
-     */
-    get skippedReason() {
-        return this.userData.__crawlee?.skippedReason;
-    }
-    /**
-     * Reason for skipping this request.
-     */
-    set skippedReason(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { skippedReason: value };
-        }
-        else {
-            this.userData.__crawlee.skippedReason = value;
-        }
-    }
-    get enqueueStrategy() {
-        return this.userData.__crawlee?.enqueueStrategy;
-    }
-    set enqueueStrategy(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { enqueueStrategy: value };
-        }
-        else {
-            this.userData.__crawlee.enqueueStrategy = value;
-        }
-    }
-    /**
-     * Stores information about an error that occurred during processing of this request.
-     *
-     * You should always use Error instances when throwing errors in JavaScript.
-     *
-     * Nevertheless, to improve the debugging experience when using third party libraries
-     * that may not always throw an Error instance, the function performs a type
-     * inspection of the passed argument and attempts to extract as much information
-     * as possible, since just throwing a bad type error makes any debugging rather difficult.
-     *
-     * @param errorOrMessage Error object or error message to be stored in the request.
-     * @param [options]
-     */
-    pushErrorMessage(errorOrMessage, options = {}) {
-        const { omitStack } = options;
-        let message;
-        const type = typeof errorOrMessage;
-        if (type === 'object') {
-            if (!errorOrMessage) {
-                message = 'null';
-            }
-            else if (errorOrMessage instanceof Error) {
-                message = omitStack
-                    ? errorOrMessage.message
-                    : // .stack includes the message
-                        errorOrMessage.stack;
-            }
-            else if (Reflect.has(Object(errorOrMessage), 'message')) {
-                message = Reflect.get(Object(errorOrMessage), 'message');
-            }
-            else if (errorOrMessage.toString() !== '[object Object]') {
-                message = errorOrMessage.toString();
-            }
-            else {
-                try {
-                    message = util.inspect(errorOrMessage);
-                }
-                catch (err) {
-                    message = 'Unable to extract any message from the received object.';
-                }
-            }
-        }
-        else if (type === 'undefined') {
-            message = 'undefined';
-        }
-        else {
-            message = errorOrMessage.toString();
-        }
-        this.errorMessages.push(message);
-    }
     /** @internal */
-    static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, }) {
+    static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, }) {
         const normalizedMethod = method.toUpperCase();
         const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
+        let uniqueKey;
         if (!useExtendedUniqueKey) {
             if (normalizedMethod !== 'GET' && payload) {
-                log.warningOnce(`We've encountered a ${normalizedMethod} Request with a payload. ` +
+                serviceLocator
+                    .getLogger()
+                    .warningOnce(`We've encountered a ${normalizedMethod} Request with a payload. ` +
                     'This is fine. Just letting you know that if your requests point to the same URL ' +
                     'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.');
             }
-            return normalizedUrl;
+            uniqueKey = normalizedUrl;
+        }
+        else {
+            const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : '';
+            uniqueKey = `${normalizedMethod}|${payloadHash}|${normalizedUrl}`;
+        }
+        if (alwaysEnqueue) {
+            uniqueKey = `${cryptoRandomObjectId(17)}|${uniqueKey}`;
         }
-        const payloadHash = payload ? CrawleeRequest.hashPayload(payload) : '';
-        return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
+        return uniqueKey;
     }
     /** @internal */
     static hashPayload(payload) {
@@ -385,4 +276,3 @@ class CrawleeRequest {
     }
 }
 export { CrawleeRequest as Request };
-//# sourceMappingURL=request.js.map

package/serialization.d.ts

@@ -29,4 +29,3 @@ export declare function deserializeArray<T extends string | Buffer>(compressedDa
  * @internal
  */
 export declare function createDeserialize(compressedData: Buffer | Uint8Array): Readable;
-//# sourceMappingURL=serialization.d.ts.map

package/serialization.js

@@ -1,9 +1,11 @@
 import { pipeline as streamPipeline, Readable, Writable } from 'node:stream';
 import util from 'node:util';
 import zlib from 'node:zlib';
-import ow from 'ow';
-import StreamArray from 'stream-json/streamers/StreamArray.js';
+import StreamArray from 'stream-json/streamers/stream-array.js';
+import { z } from 'zod';
+import { parseArgument, schemas } from './validators.js';
 const pipeline = util.promisify(streamPipeline);
+const uint8ArraySchema = z.instanceof(Uint8Array);
 /**
  * Transforms an array of items to a JSON in a streaming
  * fashion to save memory. It operates in batches to speed
@@ -11,32 +13,31 @@ const pipeline = util.promisify(streamPipeline);
  * @internal
  */
 class ArrayToJson extends Readable {
-    data;
-    offset = 0;
-    batchSize;
+    #offset = 0;
+    #batchSize;
+    #data;
     constructor(data, options = {}) {
         super({
             ...options,
             autoDestroy: true,
             emitClose: true,
         });
-        this.data = data;
         const { batchSize = 10000 } = options;
-        this.batchSize = batchSize;
-        this.data = data;
+        this.#batchSize = batchSize;
+        this.#data = data;
         this.push('[');
     }
     _read() {
         try {
-            const items = this.data.slice(this.offset, this.offset + this.batchSize);
+            const items = this.#data.slice(this.#offset, this.#offset + this.#batchSize);
             if (items.length) {
                 const json = JSON.stringify(items);
                 // Strip brackets to flatten the batch.
                 const itemString = json.substring(1, json.length - 1);
-                if (this.offset > 0)
+                if (this.#offset > 0)
                     this.push(',', 'utf8');
                 this.push(itemString, 'utf8');
-                this.offset += this.batchSize;
+                this.#offset += this.#batchSize;
             }
             else {
                 this.push(']');
@@ -58,7 +59,7 @@ class ArrayToJson extends Readable {
  * @internal
  */
 export async function serializeArray(data) {
-    ow(data, ow.array);
+    parseArgument(data, schemas.anyArray);
     const { chunks, collector } = createChunkCollector();
     await pipeline(new ArrayToJson(data), zlib.createGzip(), collector);
     return Buffer.concat(chunks);
@@ -73,9 +74,9 @@ export async function serializeArray(data) {
  * @internal
  */
 export async function deserializeArray(compressedData) {
-    ow(compressedData, ow.uint8Array);
+    parseArgument(compressedData, uint8ArraySchema);
     const { chunks, collector } = createChunkCollector({ fromValuesStream: true });
-    await pipeline(Readable.from([compressedData]), zlib.createGunzip(), StreamArray.withParser(), collector);
+    await pipeline(Readable.from([compressedData]), zlib.createGunzip(), StreamArray.withParserAsStream(), collector);
     return chunks;
 }
 /**
@@ -88,8 +89,8 @@ export async function deserializeArray(compressedData) {
  * @internal
  */
 export function createDeserialize(compressedData) {
-    ow(compressedData, ow.uint8Array);
-    const streamArray = StreamArray.withParser();
+    parseArgument(compressedData, uint8ArraySchema);
+    const streamArray = StreamArray.withParserAsStream();
     const destination = pluckValue(streamArray);
     streamPipeline(Readable.from([compressedData]), zlib.createGunzip(), destination, (err) => destination.emit(err));
     return destination;
@@ -116,7 +117,6 @@ function createChunkCollector(options = {}) {
 }
 function pluckValue(streamArray) {
     const realPush = streamArray.push.bind(streamArray);
-    streamArray.push = (obj) => realPush(obj && obj.value);
+    streamArray.push = (obj) => realPush(obj?.value ?? null);
     return streamArray;
 }
-//# sourceMappingURL=serialization.js.map