@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1

package/request.js

@@ -1,46 +1,33 @@
 import crypto from 'node:crypto';
-import util from 'node:util';
-import ow from 'ow';
+import { z } from 'zod';
 import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
 import { serviceLocator } from './service_locator.js';
-import { keys } from './typedefs.js';
+import { parseArgument, schemas } from './validators.js';
 // new properties on the Request object breaks serialization
-const requestOptionalPredicates = {
-    id: ow.optional.string,
-    loadedUrl: ow.optional.string.url,
-    uniqueKey: ow.optional.string,
-    method: ow.optional.string,
-    payload: ow.optional.any(ow.string, ow.uint8Array),
-    noRetry: ow.optional.boolean,
-    retryCount: ow.optional.number,
-    sessionId: ow.optional.string,
-    maxRetries: ow.optional.number,
-    errorMessages: ow.optional.array.ofType(ow.string),
-    headers: ow.optional.object,
-    userData: ow.optional.object,
-    label: ow.optional.string,
-    handledAt: ow.optional.any(ow.string.date, ow.date),
-    keepUrlFragment: ow.optional.boolean,
-    useExtendedUniqueKey: ow.optional.boolean,
-    alwaysEnqueue: ow.optional.boolean,
-    skipNavigation: ow.optional.boolean,
-    crawlDepth: ow.optional.number.greaterThanOrEqual(0),
-    state: ow.optional.number.greaterThanOrEqual(0).lessThanOrEqual(6),
-};
-export var RequestState;
-(function (RequestState) {
-    RequestState[RequestState["UNPROCESSED"] = 0] = "UNPROCESSED";
-    RequestState[RequestState["BEFORE_NAV"] = 1] = "BEFORE_NAV";
-    RequestState[RequestState["AFTER_NAV"] = 2] = "AFTER_NAV";
-    RequestState[RequestState["REQUEST_HANDLER"] = 3] = "REQUEST_HANDLER";
-    RequestState[RequestState["DONE"] = 4] = "DONE";
-    RequestState[RequestState["ERROR_HANDLER"] = 5] = "ERROR_HANDLER";
-    RequestState[RequestState["ERROR"] = 6] = "ERROR";
-    RequestState[RequestState["SKIPPED"] = 7] = "SKIPPED";
-})(RequestState || (RequestState = {}));
+// Compiled once: every `Request` runs it, and the generated fast path skips zod's interpreter.
+const requestOptionsSchema = z.compile(z.looseObject({
+    url: z.string(),
+    uniqueKey: z.string().optional(),
+    method: z.string().optional(),
+    payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
+    noRetry: z.boolean().optional(),
+    sessionId: z.string().optional(),
+    maxRetries: schemas.anyNumber.optional(),
+    headers: z.looseObject({}).optional(),
+    userData: z.looseObject({}).optional(),
+    label: z.string().optional(),
+    keepUrlFragment: z.boolean().optional(),
+    useExtendedUniqueKey: z.boolean().optional(),
+    alwaysEnqueue: z.boolean().optional(),
+    skipNavigation: z.boolean().optional(),
+    crawlDepth: schemas.anyNumber
+        .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
+        .optional(),
+}));
 /**
- * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
- * The `Request` object also stores information about errors that occurred during processing of the request.
+ * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata, together
+ * with the processing state a request queue keeps about it (`retryCount`, `errorMessages`, `handledAt`, ...).
+ * Inside a crawler, requests are {@link CrawlingRequest}s, which add the crawler's own per-request settings.
  *
  * Each `Request` instance has the `uniqueKey` property, which can be either specified
  * manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
@@ -56,21 +43,15 @@ export var RequestState;
  * const request = new Request({
  *     url: 'http://www.example.com',
  *     headers: { Accept: 'application/json' },
+ *     userData: { foo: 'bar' },
  * });
  *
- * ...
- *
- * request.userData.foo = 'bar';
- * request.pushErrorMessage(new Error('Request failed!'));
- *
- * ...
- *
- * const foo = request.userData.foo;
+ * await requestQueue.addRequest(request);
  * ```
  * @category Sources
  */
 class CrawleeRequest {
-    /** Request ID */
+    /** Storage-assigned request ID. Only present on requests that went through a {@link RequestQueue}. */
     id;
     /** URL of the web page to crawl. */
     url;
@@ -101,7 +82,7 @@ class CrawleeRequest {
     /** Object with HTTP headers. Key is header name, value is the value. */
     headers;
     /** Private store for the custom user data assigned to the request. */
-    _userData = {};
+    #userData = {};
     /**
      * Custom user data assigned to the request.
      *
@@ -116,27 +97,19 @@ class CrawleeRequest {
     handledAt;
     /**
      * `Request` parameters including the URL, HTTP method and headers, and others.
+     *
+     * Processing state (`retryCount`, `errorMessages`, `handledAt`, `loadedUrl`, `id`) is not an option;
+     * requests coming back from a storage are rebuilt with {@link Request.fromSchema}.
      */
     constructor(options) {
-        ow(options, 'RequestOptions', ow.object);
-        ow(options.url, 'RequestOptions.url', ow.string);
-        // 'ow' validation is slow, because it checks all predicates
-        // even if the validated object has only 1 property.
-        // This custom validation loop iterates only over existing
-        // properties and speeds up the validation cca 3-fold.
-        // See https://github.com/sindresorhus/ow/issues/193
-        keys(options).forEach((prop) => {
-            // skip url, because it is validated above
-            if (prop === 'url') {
-                return;
-            }
-            const predicate = requestOptionalPredicates[prop];
-            const value = options[prop];
-            if (predicate) {
-                ow(value, `RequestOptions.${prop}`, predicate);
-            }
-        });
-        const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionId, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
+        // A bare URL is a common slip — point at the object form instead of a generic type error.
+        if (typeof options === 'string') {
+            throw new TypeError(`\`Request\` options must be an object, got the string '${options}'. ` +
+                'Did you mean `new Request({ url })`?');
+        }
+        parseArgument(options, schemas.anyObject, 'RequestOptions');
+        parseArgument(options, requestOptionsSchema, 'RequestOptions');
+        const { url, uniqueKey, payload, noRetry = false, sessionId, maxRetries, headers = {}, userData = {}, label, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
         let { method = 'GET' } = options;
         method = method.toUpperCase();
         if (method === 'GET' && payload)
@@ -144,9 +117,7 @@ class CrawleeRequest {
         if (uniqueKey && alwaysEnqueue) {
             throw new Error('`alwaysEnqueue` cannot be used together with a custom `uniqueKey`.');
         }
-        this.id = id;
         this.url = url;
-        this.loadedUrl = loadedUrl;
         this.uniqueKey =
             uniqueKey ||
                 CrawleeRequest.computeUniqueKey({
@@ -160,61 +131,76 @@ class CrawleeRequest {
         this.method = method;
         this.payload = payload;
         this.noRetry = noRetry;
-        this.retryCount = retryCount;
-        this.errorMessages = [...errorMessages];
+        this.retryCount = 0;
+        this.errorMessages = [];
         this.headers = { ...headers };
-        this.handledAt = handledAt instanceof Date ? handledAt.toISOString() : handledAt;
         if (label) {
             userData.label = label;
         }
+        // Read `__crawlee` explicitly - on a `userData` coming from another Request instance the
+        // bag is non-enumerable, so the spread alone would silently drop the internal state
+        // (e.g. `skipNavigation`) when a request is re-wrapped after a storage round trip.
+        this.#userData = { __crawlee: userData.__crawlee ?? {}, ...userData };
+        // `userData` must stay an enumerable own accessor — serialization in the storages relies on it
         Object.defineProperties(this, {
-            _userData: {
-                value: { __crawlee: {}, ...userData },
-                enumerable: false,
-                writable: true,
-            },
             userData: {
-                get: () => this._userData,
+                get: () => this.#userData,
                 set: (value) => {
                     Object.defineProperties(value, {
                         __crawlee: {
-                            value: this._userData.__crawlee,
+                            value: this.#userData.__crawlee,
                             enumerable: false,
                             writable: true,
                         },
                         toJSON: {
                             value: () => {
-                                if (Object.keys(this._userData.__crawlee).length > 0) {
+                                if (Object.keys(this.#userData.__crawlee).length > 0) {
                                     return {
-                                        ...this._userData,
-                                        __crawlee: this._userData.__crawlee,
+                                        ...this.#userData,
+                                        __crawlee: this.#userData.__crawlee,
                                     };
                                 }
-                                return this._userData;
+                                return this.#userData;
                             },
                             enumerable: false,
                             writable: true,
                         },
                     });
-                    this._userData = value;
+                    this.#userData = value;
                 },
                 enumerable: true,
             },
         });
         // reassign userData to ensure internal `__crawlee` object is non-enumerable
         this.userData = userData;
+        const crawlee = (this.userData.__crawlee ??= {});
         if (skipNavigation != null)
-            this.skipNavigation = skipNavigation;
+            crawlee.skipNavigation = skipNavigation;
         if (maxRetries != null)
-            this.maxRetries = maxRetries;
+            crawlee.maxRetries = maxRetries;
         if (crawlDepth != null)
-            this.userData.__crawlee.crawlDepth ??= crawlDepth;
+            crawlee.crawlDepth ??= crawlDepth;
         if (sessionId)
-            this.sessionId = sessionId;
-        // If it's already set, don't override it (for instance when fetching from storage)
-        if (enqueueStrategy) {
-            this.enqueueStrategy ??= enqueueStrategy;
-        }
+            crawlee.sessionId = sessionId;
+        // A stored request keeps the strategy it was enqueued under.
+        if (enqueueStrategy)
+            crawlee.enqueueStrategy ??= enqueueStrategy;
+    }
+    /**
+     * Rebuilds a request from its stored form, including the processing state a
+     * {@link RequestOptions} object cannot carry. Subclasses get instances of themselves.
+     */
+    static fromSchema(schema) {
+        const { id, retryCount = 0, errorMessages = [], handledAt, loadedUrl, ...options } = schema;
+        const request = new this(options);
+        request.id = id;
+        request.retryCount = retryCount;
+        request.errorMessages = [...errorMessages];
+        request.loadedUrl = loadedUrl;
+        // `apify-client` parses `*At` fields into `Date`s, whatever the schema says.
+        request.handledAt =
+            handledAt instanceof Date ? handledAt.toISOString() : handledAt;
+        return request;
     }
     /**
      * Converts the Crawlee Request object to a `fetch` API Request object.
@@ -227,54 +213,6 @@ class CrawleeRequest {
             body: this.payload,
         });
     }
-    /**
-     * Tells the crawler processing this request to skip the navigation and process the request directly.
-     *
-     * When this is set to `true`, the crawling context will not contain the results of the navigation
-     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
-     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
-     */
-    get skipNavigation() {
-        return this.userData.__crawlee?.skipNavigation ?? false;
-    }
-    /**
-     * Tells the crawler processing this request to skip the navigation and process the request directly.
-     *
-     * When this is set to `true`, the crawling context will not contain the results of the navigation
-     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
-     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
-     */
-    set skipNavigation(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { skipNavigation: value };
-        }
-        else {
-            this.userData.__crawlee.skipNavigation = value;
-        }
-    }
-    /**
-     * Depth of the request in the current crawl tree.
-     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
-     */
-    get crawlDepth() {
-        return this.userData.__crawlee?.crawlDepth ?? 0;
-    }
-    /**
-     * Depth of the request in the current crawl tree.
-     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
-     */
-    set crawlDepth(value) {
-        this.userData.__crawlee ??= {};
-        this.userData.__crawlee.crawlDepth = value;
-    }
-    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
-    get sessionId() {
-        return this.userData.__crawlee?.sessionId;
-    }
-    set sessionId(value) {
-        this.userData.__crawlee ??= {};
-        this.userData.__crawlee.sessionId = value;
-    }
     /** shortcut for getting `request.userData.label` */
     get label() {
         return this.userData.label;
@@ -283,110 +221,6 @@ class CrawleeRequest {
     set label(value) {
         this.userData.label = value;
     }
-    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
-    get maxRetries() {
-        return this.userData.__crawlee?.maxRetries;
-    }
-    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
-    set maxRetries(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { maxRetries: value };
-        }
-        else {
-            this.userData.__crawlee.maxRetries = value;
-        }
-    }
-    /** Describes the request's current lifecycle state. */
-    get state() {
-        return this.userData.__crawlee?.state ?? RequestState.UNPROCESSED;
-    }
-    /** Describes the request's current lifecycle state. */
-    set state(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { state: value };
-        }
-        else {
-            this.userData.__crawlee.state = value;
-        }
-    }
-    /**
-     * Reason for skipping this request.
-     */
-    get skippedReason() {
-        return this.userData.__crawlee?.skippedReason;
-    }
-    /**
-     * Reason for skipping this request.
-     */
-    set skippedReason(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { skippedReason: value };
-        }
-        else {
-            this.userData.__crawlee.skippedReason = value;
-        }
-    }
-    get enqueueStrategy() {
-        return this.userData.__crawlee?.enqueueStrategy;
-    }
-    set enqueueStrategy(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { enqueueStrategy: value };
-        }
-        else {
-            this.userData.__crawlee.enqueueStrategy = value;
-        }
-    }
-    /**
-     * Stores information about an error that occurred during processing of this request.
-     *
-     * You should always use Error instances when throwing errors in JavaScript.
-     *
-     * Nevertheless, to improve the debugging experience when using third party libraries
-     * that may not always throw an Error instance, the function performs a type
-     * inspection of the passed argument and attempts to extract as much information
-     * as possible, since just throwing a bad type error makes any debugging rather difficult.
-     *
-     * @param errorOrMessage Error object or error message to be stored in the request.
-     * @param [options]
-     */
-    pushErrorMessage(errorOrMessage, options = {}) {
-        const { omitStack } = options;
-        let message;
-        const type = typeof errorOrMessage;
-        if (type === 'object') {
-            if (!errorOrMessage) {
-                message = 'null';
-            }
-            else if (errorOrMessage instanceof Error) {
-                message = omitStack
-                    ? errorOrMessage.message
-                    : // .stack includes the message
-                        errorOrMessage.stack;
-            }
-            else if (Reflect.has(Object(errorOrMessage), 'message')) {
-                message = Reflect.get(Object(errorOrMessage), 'message');
-            }
-            else if (errorOrMessage.toString() !== '[object Object]') {
-                message = errorOrMessage.toString();
-            }
-            else {
-                try {
-                    message = util.inspect(errorOrMessage);
-                }
-                catch (err) {
-                    message = 'Unable to extract any message from the received object.';
-                }
-            }
-        }
-        else if (type === 'undefined') {
-            message = 'undefined';
-        }
-        else {
-            message = errorOrMessage.toString();
-        }
-        this.errorMessages.push(message);
-    }
     /** @internal */
     static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, }) {
         const normalizedMethod = method.toUpperCase();

package/serialization.js

@@ -1,9 +1,11 @@
 import { pipeline as streamPipeline, Readable, Writable } from 'node:stream';
 import util from 'node:util';
 import zlib from 'node:zlib';
-import ow from 'ow';
-import StreamArray from 'stream-json/streamers/StreamArray.js';
+import StreamArray from 'stream-json/streamers/stream-array.js';
+import { z } from 'zod';
+import { parseArgument, schemas } from './validators.js';
 const pipeline = util.promisify(streamPipeline);
+const uint8ArraySchema = z.instanceof(Uint8Array);
 /**
  * Transforms an array of items to a JSON in a streaming
  * fashion to save memory. It operates in batches to speed
@@ -11,32 +13,31 @@ const pipeline = util.promisify(streamPipeline);
  * @internal
  */
 class ArrayToJson extends Readable {
-    data;
-    offset = 0;
-    batchSize;
+    #offset = 0;
+    #batchSize;
+    #data;
     constructor(data, options = {}) {
         super({
             ...options,
             autoDestroy: true,
             emitClose: true,
         });
-        this.data = data;
         const { batchSize = 10000 } = options;
-        this.batchSize = batchSize;
-        this.data = data;
+        this.#batchSize = batchSize;
+        this.#data = data;
         this.push('[');
     }
     _read() {
         try {
-            const items = this.data.slice(this.offset, this.offset + this.batchSize);
+            const items = this.#data.slice(this.#offset, this.#offset + this.#batchSize);
             if (items.length) {
                 const json = JSON.stringify(items);
                 // Strip brackets to flatten the batch.
                 const itemString = json.substring(1, json.length - 1);
-                if (this.offset > 0)
+                if (this.#offset > 0)
                     this.push(',', 'utf8');
                 this.push(itemString, 'utf8');
-                this.offset += this.batchSize;
+                this.#offset += this.#batchSize;
             }
             else {
                 this.push(']');
@@ -58,7 +59,7 @@ class ArrayToJson extends Readable {
  * @internal
  */
 export async function serializeArray(data) {
-    ow(data, ow.array);
+    parseArgument(data, schemas.anyArray);
     const { chunks, collector } = createChunkCollector();
     await pipeline(new ArrayToJson(data), zlib.createGzip(), collector);
     return Buffer.concat(chunks);
@@ -73,9 +74,9 @@ export async function serializeArray(data) {
  * @internal
  */
 export async function deserializeArray(compressedData) {
-    ow(compressedData, ow.uint8Array);
+    parseArgument(compressedData, uint8ArraySchema);
     const { chunks, collector } = createChunkCollector({ fromValuesStream: true });
-    await pipeline(Readable.from([compressedData]), zlib.createGunzip(), StreamArray.withParser(), collector);
+    await pipeline(Readable.from([compressedData]), zlib.createGunzip(), StreamArray.withParserAsStream(), collector);
     return chunks;
 }
 /**
@@ -88,8 +89,8 @@ export async function deserializeArray(compressedData) {
  * @internal
  */
 export function createDeserialize(compressedData) {
-    ow(compressedData, ow.uint8Array);
-    const streamArray = StreamArray.withParser();
+    parseArgument(compressedData, uint8ArraySchema);
+    const streamArray = StreamArray.withParserAsStream();
     const destination = pluckValue(streamArray);
     streamPipeline(Readable.from([compressedData]), zlib.createGunzip(), destination, (err) => destination.emit(err));
     return destination;

package/service_locator.d.ts

@@ -60,8 +60,20 @@ interface ServiceLocatorInterface {
     getChildLog(prefix: string): CrawleeLogger;
     /**
      * Get the storage instance manager (shared across all storage types).
+     * @internal
      */
     getStorageInstanceManager(): StorageInstanceManager;
+    /**
+     * Returns the currently set services without triggering the implicit creation of defaults.
+     * Used to inherit already-materialized services into crawler-scoped service locators.
+     * @internal
+     */
+    getServicesIfSet(): {
+        configuration?: Configuration;
+        eventManager?: EventManager;
+        storageBackend?: StorageBackend;
+        logger?: CrawleeLogger;
+    };
     /**
      * Resets the service locator to its initial state.
      * Used mainly for testing purposes.
@@ -101,16 +113,7 @@ interface ServiceLocatorInterface {
  * ```
  */
 export declare class ServiceLocator implements ServiceLocatorInterface {
-    private configuration?;
-    private eventManager?;
-    private storageBackend?;
-    private logger?;
-    /**
-     * Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
-     * Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
-     * Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
-     */
-    private static storageInstanceManager?;
+    #private;
     /**
      * Creates a new ServiceLocator instance.
      *
@@ -120,6 +123,13 @@ export declare class ServiceLocator implements ServiceLocatorInterface {
      * @param logger Optional logger instance to use
      */
     constructor(configuration?: Configuration, eventManager?: EventManager, storageBackend?: StorageBackend, logger?: CrawleeLogger);
+    /** @internal */
+    getServicesIfSet(): {
+        configuration?: Configuration;
+        eventManager?: EventManager;
+        storageBackend?: StorageBackend;
+        logger?: CrawleeLogger;
+    };
     getConfiguration(): Configuration;
     setConfiguration(configuration: Configuration): void;
     getEventManager(): EventManager;
@@ -129,7 +139,9 @@ export declare class ServiceLocator implements ServiceLocatorInterface {
     getLogger(): CrawleeLogger;
     setLogger(logger: CrawleeLogger): void;
     getChildLog(prefix: string): CrawleeLogger;
+    /** @internal */
     getStorageInstanceManager(): StorageInstanceManager;
+    /** @internal */
     reset(): void;
 }
 /**