@crawlee/core 4.0.0-rc.0 → 4.0.0-rc.1

package/recoverable_state.js

@@ -1,5 +1,9 @@
+import { Readable } from 'node:stream';
 import { addTimeoutToPromise, storage as timeoutStorage } from '@apify/timeout';
-import { EventType, KeyValueStore, serviceLocator, StateValidationError } from '@crawlee/core';
+import { StateValidationError } from './errors.js';
+import { EventType } from './events/event_manager.js';
+import { serviceLocator } from './service_locator.js';
+import { KeyValueStore } from './storages/key_value_store.js';
 const DEFAULT_PERSISTENCE_TIMEOUT_MILLIS = 60_000;
 /**
  * Applies a {@link SyncStateConversion}, throwing a {@link StateValidationError} for a schema that rejects
@@ -34,6 +38,7 @@ export class RecoverableState {
     #defaultState;
     #state = null;
     #initialized = false;
+    #recordLoaded = false;
     #listening = false;
     #persistenceEnabled;
     #persistStateKey;
@@ -43,6 +48,7 @@ export class RecoverableState {
     #log;
     #serialize;
     #deserialize;
+    #contentType;
     #persistStateQuietly;
     /**
      * Initialize a new recoverable state object.
@@ -61,8 +67,18 @@ export class RecoverableState {
         this.#configuration = options.configuration;
         this.#keyValueStore = options.keyValueStore ?? null;
         this.#log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
+        this.#contentType = options.contentType;
+        if (this.#contentType !== undefined && (options.serialize === undefined || options.deserialize === undefined)) {
+            throw new Error(`A 'contentType' for the state persisted under key '${this.#persistStateKey}' requires both 'serialize' and 'deserialize' - the record is no longer JSON the default codec can handle.`);
+        }
         this.#serialize = this.#toConversion(options.serialize);
         this.#deserialize = this.#toConversion(options.deserialize);
+        // A pending open() may reject before anything here awaits it. Observe the rejection so it surfaces on
+        // first use rather than as an unhandled rejection - most immediately when persistence is disabled and
+        // nothing ever awaits the store.
+        if (this.#keyValueStore !== null) {
+            Promise.resolve(this.#keyValueStore).then(undefined, () => { });
+        }
         // The automatic persists, where a rejection has nowhere useful to go - the event manager does not catch
         // listener errors, and throwing from teardown would bury the outcome of the work it cleans up after.
         this.#persistStateQuietly = async (eventData) => this.persistState(eventData).catch((error) => this.#log.warning(`Failed to persist the state under key '${this.#persistStateKey}'.`, { error }));
@@ -91,7 +107,8 @@ export class RecoverableState {
      * is no record to restore.
      *
      * Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
-     * listener is registered again and the record reloaded.
+     * listener is registered again. The record is not reloaded: the in-memory state is what the teardown wrote, and
+     * reloading it would only drop whatever the deserialization leaves out.
      *
      * @returns The loaded state object
      */
@@ -100,10 +117,7 @@ export class RecoverableState {
             return this.currentValue;
         }
         if (this.#persistenceEnabled) {
-            this.#keyValueStore ??= KeyValueStore.open(null, {
-                configuration: this.#configuration ?? serviceLocator.getConfiguration(),
-            });
-            await this.#resolveKeyValueStore();
+            await this.#openKeyValueStore();
             serviceLocator.getEventManager().on(EventType.PERSIST_STATE, this.#persistStateQuietly);
             this.#listening = true;
         }
@@ -111,7 +125,10 @@ export class RecoverableState {
         // fully wired object running on the default state rather than a half-initialized one.
         this.#initialized = true;
         this.#state ??= this.#defaultState();
-        await this.#loadSavedState();
+        if (!this.#recordLoaded) {
+            this.#recordLoaded = true;
+            await this.#loadSavedState();
+        }
         return this.currentValue;
     }
     /**
@@ -161,7 +178,7 @@ export class RecoverableState {
      * one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
      * or {@link RecoverableState.teardown} before clearing the record.
      *
-     * A no-op if persistence is disabled or no KeyValueStore is available yet.
+     * A no-op if persistence is disabled.
      */
     async resetStore() {
         if (this.#listening) {
@@ -170,10 +187,7 @@ export class RecoverableState {
         if (!this.#persistenceEnabled) {
             return;
         }
-        const keyValueStore = await this.#resolveKeyValueStore();
-        if (keyValueStore === null) {
-            return;
-        }
+        const keyValueStore = await this.#openKeyValueStore();
         await this.#withTimeout(async () => keyValueStore.setValue(this.#persistStateKey, null), 'Clearing the persisted state');
     }
     /**
@@ -195,7 +209,18 @@ export class RecoverableState {
         }
         this.#log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
         const serializedState = await this.#serialize(this.currentValue);
-        await this.#withTimeout(async () => keyValueStore.setValue(this.#persistStateKey, serializedState), 'Persisting the state');
+        await this.#withTimeout(async () => this.#contentType === undefined
+            ? keyValueStore.setValue(this.#persistStateKey, serializedState)
+            : keyValueStore.setValue(this.#persistStateKey, serializedState, {
+                contentType: this.#contentType,
+            }), 'Persisting the state');
+    }
+    /** Falls back to the default store when none was supplied - persistence has to write somewhere. */
+    async #openKeyValueStore() {
+        this.#keyValueStore ??= KeyValueStore.open(null, {
+            configuration: this.#configuration ?? serviceLocator.getConfiguration(),
+        });
+        return (await this.#resolveKeyValueStore());
     }
     /** Awaits a store handed over as a pending `open()`, keeping the resolved instance for later calls. */
     async #resolveKeyValueStore() {
@@ -216,7 +241,19 @@ export class RecoverableState {
         if (keyValueStore === null) {
             return;
         }
-        const storedState = await this.#withTimeout(async () => keyValueStore.getValue(this.#persistStateKey), 'Loading the persisted state');
+        // With a content type, the record is whatever `serialize` produced, so the codec's parse is skipped and
+        // `deserialize` gets the bytes as a stream - the shape a streaming parser wants.
+        // TODO: read the record as a stream instead of buffering it and wrapping (https://github.com/apify/crawlee/issues/2929).
+        const storedState = await this.#withTimeout(async () => {
+            if (this.#contentType === undefined) {
+                return keyValueStore.getValue(this.#persistStateKey);
+            }
+            const record = await keyValueStore.getRecord(this.#persistStateKey);
+            if (record === null) {
+                return null;
+            }
+            return Readable.from(Buffer.isBuffer(record.value) ? record.value : Buffer.from(record.value));
+        }, 'Loading the persisted state');
         if (storedState === null || storedState === undefined) {
             return;
         }

package/request.d.ts

@@ -1,20 +1,12 @@
 import type { BinaryLike } from 'node:crypto';
-import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
-import type { EnqueueStrategyOption } from './enqueue_links/enqueue_links.js';
-import type { SkippedRequestReason } from './enqueue_links/shared.js';
-export declare enum RequestState {
-    UNPROCESSED = 0,
-    BEFORE_NAV = 1,
-    AFTER_NAV = 2,
-    REQUEST_HANDLER = 3,
-    DONE = 4,
-    ERROR_HANDLER = 5,
-    ERROR = 6,
-    SKIPPED = 7
-}
+import type { AllowedHttpMethods, Dictionary, RequestSchema } from '@crawlee/types';
+import type { EnqueueStrategy } from '@crawlee/utils';
+/** The `strategy` option accepted by {@link ExtractLinksOptions} and {@link EnqueueUrlsOptions}. */
+export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
 /**
- * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
- * The `Request` object also stores information about errors that occurred during processing of the request.
+ * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata, together
+ * with the processing state a request queue keeps about it (`retryCount`, `errorMessages`, `handledAt`, ...).
+ * Inside a crawler, requests are {@link CrawlingRequest}s, which add the crawler's own per-request settings.
  *
  * Each `Request` instance has the `uniqueKey` property, which can be either specified
  * manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
@@ -30,22 +22,16 @@ export declare enum RequestState {
  * const request = new Request({
  *     url: 'http://www.example.com',
  *     headers: { Accept: 'application/json' },
+ *     userData: { foo: 'bar' },
  * });
  *
- * ...
- *
- * request.userData.foo = 'bar';
- * request.pushErrorMessage(new Error('Request failed!'));
- *
- * ...
- *
- * const foo = request.userData.foo;
+ * await requestQueue.addRequest(request);
  * ```
  * @category Sources
  */
 declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
     #private;
-    /** Request ID */
+    /** Storage-assigned request ID. Only present on requests that went through a {@link RequestQueue}. */
     id?: string;
     /** URL of the web page to crawl. */
     url: string;
@@ -89,78 +75,25 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
     handledAt?: string;
     /**
      * `Request` parameters including the URL, HTTP method and headers, and others.
+     *
+     * Processing state (`retryCount`, `errorMessages`, `handledAt`, `loadedUrl`, `id`) is not an option;
+     * requests coming back from a storage are rebuilt with {@link Request.fromSchema}.
      */
     constructor(options: RequestOptions<UserData>);
+    /**
+     * Rebuilds a request from its stored form, including the processing state a
+     * {@link RequestOptions} object cannot carry. Subclasses get instances of themselves.
+     */
+    static fromSchema<UserData extends Dictionary = Dictionary>(schema: RequestSchema): CrawleeRequest<UserData>;
     /**
      * Converts the Crawlee Request object to a `fetch` API Request object.
      * @returns The native `fetch` API Request object.
      */
     intoFetchAPIRequest(): Request;
-    /**
-     * Tells the crawler processing this request to skip the navigation and process the request directly.
-     *
-     * When this is set to `true`, the crawling context will not contain the results of the navigation
-     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
-     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
-     */
-    get skipNavigation(): boolean;
-    /**
-     * Tells the crawler processing this request to skip the navigation and process the request directly.
-     *
-     * When this is set to `true`, the crawling context will not contain the results of the navigation
-     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
-     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
-     */
-    set skipNavigation(value: boolean);
-    /**
-     * Depth of the request in the current crawl tree.
-     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
-     */
-    get crawlDepth(): number;
-    /**
-     * Depth of the request in the current crawl tree.
-     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
-     */
-    set crawlDepth(value: number);
-    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
-    get sessionId(): string | undefined;
-    set sessionId(value: string | undefined);
     /** shortcut for getting `request.userData.label` */
     get label(): string | undefined;
     /** shortcut for setting `request.userData.label` */
     set label(value: string | undefined);
-    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
-    get maxRetries(): number | undefined;
-    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
-    set maxRetries(value: number | undefined);
-    /** Describes the request's current lifecycle state. */
-    get state(): RequestState;
-    /** Describes the request's current lifecycle state. */
-    set state(value: RequestState);
-    /**
-     * Reason for skipping this request.
-     */
-    get skippedReason(): SkippedRequestReason | undefined;
-    /**
-     * Reason for skipping this request.
-     */
-    set skippedReason(value: SkippedRequestReason | undefined);
-    private get enqueueStrategy();
-    private set enqueueStrategy(value);
-    /**
-     * Stores information about an error that occurred during processing of this request.
-     *
-     * You should always use Error instances when throwing errors in JavaScript.
-     *
-     * Nevertheless, to improve the debugging experience when using third party libraries
-     * that may not always throw an Error instance, the function performs a type
-     * inspection of the passed argument and attempts to extract as much information
-     * as possible, since just throwing a bad type error makes any debugging rather difficult.
-     *
-     * @param errorOrMessage Error object or error message to be stored in the request.
-     * @param [options]
-     */
-    pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
     /** @internal */
     static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, alwaysEnqueue, }: ComputeUniqueKeyOptions): string;
     /** @internal */
@@ -264,32 +197,13 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
      * @default 0
      */
     crawlDepth?: number;
-    /**
-     * Reason for skipping this request.
-     * This is used to provide more information about why the request was skipped.
-     * @internal
-     */
-    skippedReason?: SkippedRequestReason;
     /**
      * Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`.
      */
     maxRetries?: number;
     /** @internal */
-    id?: string;
-    /** @internal */
-    handledAt?: string;
-    /** @internal */
-    lockExpiresAt?: Date;
-    /** @internal */
     enqueueStrategy?: EnqueueStrategyOption;
 }
-export interface PushErrorMessageOptions {
-    /**
-     * Only push the error message without stack trace when true.
-     * @default false
-     */
-    omitStack?: boolean;
-}
 interface ComputeUniqueKeyOptions {
     url: string;
     method: AllowedHttpMethods;

package/request.js

@@ -1,41 +1,21 @@
 import crypto from 'node:crypto';
-import util from 'node:util';
 import { z } from 'zod';
 import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
 import { serviceLocator } from './service_locator.js';
-import { keys } from './typedefs.js';
 import { parseArgument, schemas } from './validators.js';
-const dateString = z.string().refine((value) => !Number.isNaN(Date.parse(value)), {
-    message: 'Invalid input: expected a date string',
-});
-export var RequestState;
-(function (RequestState) {
-    RequestState[RequestState["UNPROCESSED"] = 0] = "UNPROCESSED";
-    RequestState[RequestState["BEFORE_NAV"] = 1] = "BEFORE_NAV";
-    RequestState[RequestState["AFTER_NAV"] = 2] = "AFTER_NAV";
-    RequestState[RequestState["REQUEST_HANDLER"] = 3] = "REQUEST_HANDLER";
-    RequestState[RequestState["DONE"] = 4] = "DONE";
-    RequestState[RequestState["ERROR_HANDLER"] = 5] = "ERROR_HANDLER";
-    RequestState[RequestState["ERROR"] = 6] = "ERROR";
-    RequestState[RequestState["SKIPPED"] = 7] = "SKIPPED";
-})(RequestState || (RequestState = {}));
-const requestUrlSchema = z.object({ url: z.string() });
 // new properties on the Request object breaks serialization
-const requestOptionalSchemaShapes = {
-    id: z.string().optional(),
-    loadedUrl: z.url().optional(),
+// Compiled once: every `Request` runs it, and the generated fast path skips zod's interpreter.
+const requestOptionsSchema = z.compile(z.looseObject({
+    url: z.string(),
     uniqueKey: z.string().optional(),
     method: z.string().optional(),
     payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
     noRetry: z.boolean().optional(),
-    retryCount: schemas.anyNumber.optional(),
     sessionId: z.string().optional(),
     maxRetries: schemas.anyNumber.optional(),
-    errorMessages: schemas.arrayOf(z.string(), 'strings').optional(),
     headers: z.looseObject({}).optional(),
     userData: z.looseObject({}).optional(),
     label: z.string().optional(),
-    handledAt: z.union([dateString, z.date()]).optional(),
     keepUrlFragment: z.boolean().optional(),
     useExtendedUniqueKey: z.boolean().optional(),
     alwaysEnqueue: z.boolean().optional(),
@@ -43,13 +23,11 @@ const requestOptionalSchemaShapes = {
     crawlDepth: schemas.anyNumber
         .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
         .optional(),
-    state: z.enum(RequestState).optional(),
-};
-// Each schema is wrapped in a single-key object so validation errors carry the property name.
-const requestOptionalSchemas = Object.fromEntries(Object.entries(requestOptionalSchemaShapes).map(([key, schema]) => [key, z.object({ [key]: schema })]));
+}));
 /**
- * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
- * The `Request` object also stores information about errors that occurred during processing of the request.
+ * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata, together
+ * with the processing state a request queue keeps about it (`retryCount`, `errorMessages`, `handledAt`, ...).
+ * Inside a crawler, requests are {@link CrawlingRequest}s, which add the crawler's own per-request settings.
  *
  * Each `Request` instance has the `uniqueKey` property, which can be either specified
  * manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
@@ -65,21 +43,15 @@ const requestOptionalSchemas = Object.fromEntries(Object.entries(requestOptional
  * const request = new Request({
  *     url: 'http://www.example.com',
  *     headers: { Accept: 'application/json' },
+ *     userData: { foo: 'bar' },
  * });
  *
- * ...
- *
- * request.userData.foo = 'bar';
- * request.pushErrorMessage(new Error('Request failed!'));
- *
- * ...
- *
- * const foo = request.userData.foo;
+ * await requestQueue.addRequest(request);
  * ```
  * @category Sources
  */
 class CrawleeRequest {
-    /** Request ID */
+    /** Storage-assigned request ID. Only present on requests that went through a {@link RequestQueue}. */
     id;
     /** URL of the web page to crawl. */
     url;
@@ -125,6 +97,9 @@ class CrawleeRequest {
     handledAt;
     /**
      * `Request` parameters including the URL, HTTP method and headers, and others.
+     *
+     * Processing state (`retryCount`, `errorMessages`, `handledAt`, `loadedUrl`, `id`) is not an option;
+     * requests coming back from a storage are rebuilt with {@link Request.fromSchema}.
      */
     constructor(options) {
         // A bare URL is a common slip — point at the object form instead of a generic type error.
@@ -133,23 +108,8 @@ class CrawleeRequest {
                 'Did you mean `new Request({ url })`?');
         }
         parseArgument(options, schemas.anyObject, 'RequestOptions');
-        parseArgument(options, requestUrlSchema, 'RequestOptions');
-        // Full-shape validation is slow, because it checks all predicates
-        // even if the validated object has only 1 property.
-        // This custom validation loop iterates only over existing
-        // properties and speeds up the validation cca 3-fold.
-        keys(options).forEach((prop) => {
-            // skip url, because it is validated above
-            if (prop === 'url') {
-                return;
-            }
-            const schema = requestOptionalSchemas[prop];
-            const value = options[prop];
-            if (schema) {
-                parseArgument({ [prop]: value }, schema, 'RequestOptions');
-            }
-        });
-        const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionId, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
+        parseArgument(options, requestOptionsSchema, 'RequestOptions');
+        const { url, uniqueKey, payload, noRetry = false, sessionId, maxRetries, headers = {}, userData = {}, label, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
         let { method = 'GET' } = options;
         method = method.toUpperCase();
         if (method === 'GET' && payload)
@@ -157,9 +117,7 @@ class CrawleeRequest {
         if (uniqueKey && alwaysEnqueue) {
             throw new Error('`alwaysEnqueue` cannot be used together with a custom `uniqueKey`.');
         }
-        this.id = id;
         this.url = url;
-        this.loadedUrl = loadedUrl;
         this.uniqueKey =
             uniqueKey ||
                 CrawleeRequest.computeUniqueKey({
@@ -173,10 +131,9 @@ class CrawleeRequest {
         this.method = method;
         this.payload = payload;
         this.noRetry = noRetry;
-        this.retryCount = retryCount;
-        this.errorMessages = [...errorMessages];
+        this.retryCount = 0;
+        this.errorMessages = [];
         this.headers = { ...headers };
-        this.handledAt = handledAt instanceof Date ? handledAt.toISOString() : handledAt;
         if (label) {
             userData.label = label;
         }
@@ -216,18 +173,34 @@ class CrawleeRequest {
         });
         // reassign userData to ensure internal `__crawlee` object is non-enumerable
         this.userData = userData;
+        const crawlee = (this.userData.__crawlee ??= {});
         if (skipNavigation != null)
-            this.skipNavigation = skipNavigation;
+            crawlee.skipNavigation = skipNavigation;
         if (maxRetries != null)
-            this.maxRetries = maxRetries;
+            crawlee.maxRetries = maxRetries;
         if (crawlDepth != null)
-            this.userData.__crawlee.crawlDepth ??= crawlDepth;
+            crawlee.crawlDepth ??= crawlDepth;
         if (sessionId)
-            this.sessionId = sessionId;
-        // If it's already set, don't override it (for instance when fetching from storage)
-        if (enqueueStrategy) {
-            this.enqueueStrategy ??= enqueueStrategy;
-        }
+            crawlee.sessionId = sessionId;
+        // A stored request keeps the strategy it was enqueued under.
+        if (enqueueStrategy)
+            crawlee.enqueueStrategy ??= enqueueStrategy;
+    }
+    /**
+     * Rebuilds a request from its stored form, including the processing state a
+     * {@link RequestOptions} object cannot carry. Subclasses get instances of themselves.
+     */
+    static fromSchema(schema) {
+        const { id, retryCount = 0, errorMessages = [], handledAt, loadedUrl, ...options } = schema;
+        const request = new this(options);
+        request.id = id;
+        request.retryCount = retryCount;
+        request.errorMessages = [...errorMessages];
+        request.loadedUrl = loadedUrl;
+        // `apify-client` parses `*At` fields into `Date`s, whatever the schema says.
+        request.handledAt =
+            handledAt instanceof Date ? handledAt.toISOString() : handledAt;
+        return request;
     }
     /**
      * Converts the Crawlee Request object to a `fetch` API Request object.
@@ -240,54 +213,6 @@ class CrawleeRequest {
             body: this.payload,
         });
     }
-    /**
-     * Tells the crawler processing this request to skip the navigation and process the request directly.
-     *
-     * When this is set to `true`, the crawling context will not contain the results of the navigation
-     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
-     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
-     */
-    get skipNavigation() {
-        return this.userData.__crawlee?.skipNavigation ?? false;
-    }
-    /**
-     * Tells the crawler processing this request to skip the navigation and process the request directly.
-     *
-     * When this is set to `true`, the crawling context will not contain the results of the navigation
-     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
-     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
-     */
-    set skipNavigation(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { skipNavigation: value };
-        }
-        else {
-            this.userData.__crawlee.skipNavigation = value;
-        }
-    }
-    /**
-     * Depth of the request in the current crawl tree.
-     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
-     */
-    get crawlDepth() {
-        return this.userData.__crawlee?.crawlDepth ?? 0;
-    }
-    /**
-     * Depth of the request in the current crawl tree.
-     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
-     */
-    set crawlDepth(value) {
-        this.userData.__crawlee ??= {};
-        this.userData.__crawlee.crawlDepth = value;
-    }
-    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
-    get sessionId() {
-        return this.userData.__crawlee?.sessionId;
-    }
-    set sessionId(value) {
-        this.userData.__crawlee ??= {};
-        this.userData.__crawlee.sessionId = value;
-    }
     /** shortcut for getting `request.userData.label` */
     get label() {
         return this.userData.label;
@@ -296,110 +221,6 @@ class CrawleeRequest {
     set label(value) {
         this.userData.label = value;
     }
-    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
-    get maxRetries() {
-        return this.userData.__crawlee?.maxRetries;
-    }
-    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
-    set maxRetries(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { maxRetries: value };
-        }
-        else {
-            this.userData.__crawlee.maxRetries = value;
-        }
-    }
-    /** Describes the request's current lifecycle state. */
-    get state() {
-        return this.userData.__crawlee?.state ?? RequestState.UNPROCESSED;
-    }
-    /** Describes the request's current lifecycle state. */
-    set state(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { state: value };
-        }
-        else {
-            this.userData.__crawlee.state = value;
-        }
-    }
-    /**
-     * Reason for skipping this request.
-     */
-    get skippedReason() {
-        return this.userData.__crawlee?.skippedReason;
-    }
-    /**
-     * Reason for skipping this request.
-     */
-    set skippedReason(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { skippedReason: value };
-        }
-        else {
-            this.userData.__crawlee.skippedReason = value;
-        }
-    }
-    get enqueueStrategy() {
-        return this.userData.__crawlee?.enqueueStrategy;
-    }
-    set enqueueStrategy(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { enqueueStrategy: value };
-        }
-        else {
-            this.userData.__crawlee.enqueueStrategy = value;
-        }
-    }
-    /**
-     * Stores information about an error that occurred during processing of this request.
-     *
-     * You should always use Error instances when throwing errors in JavaScript.
-     *
-     * Nevertheless, to improve the debugging experience when using third party libraries
-     * that may not always throw an Error instance, the function performs a type
-     * inspection of the passed argument and attempts to extract as much information
-     * as possible, since just throwing a bad type error makes any debugging rather difficult.
-     *
-     * @param errorOrMessage Error object or error message to be stored in the request.
-     * @param [options]
-     */
-    pushErrorMessage(errorOrMessage, options = {}) {
-        const { omitStack } = options;
-        let message;
-        const type = typeof errorOrMessage;
-        if (type === 'object') {
-            if (!errorOrMessage) {
-                message = 'null';
-            }
-            else if (errorOrMessage instanceof Error) {
-                message = omitStack
-                    ? errorOrMessage.message
-                    : // .stack includes the message
-                        errorOrMessage.stack;
-            }
-            else if (Reflect.has(Object(errorOrMessage), 'message')) {
-                message = Reflect.get(Object(errorOrMessage), 'message');
-            }
-            else if (errorOrMessage.toString() !== '[object Object]') {
-                message = errorOrMessage.toString();
-            }
-            else {
-                try {
-                    message = util.inspect(errorOrMessage);
-                }
-                catch (err) {
-                    message = 'Unable to extract any message from the received object.';
-                }
-            }
-        }
-        else if (type === 'undefined') {
-            message = 'undefined';
-        }
-        else {
-            message = errorOrMessage.toString();
-        }
-        this.errorMessages.push(message);
-    }
     /** @internal */
     static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, }) {
         const normalizedMethod = method.toUpperCase();