@crawlee/core 4.0.0-beta.183 → 4.0.0-beta.185

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.183",
+    "version": "4.0.0-beta.185",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,10 +53,10 @@
         "@apify/log": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/fs-storage": "4.0.0-beta.183",
-        "@crawlee/http-client": "4.0.0-beta.183",
-        "@crawlee/types": "4.0.0-beta.183",
-        "@crawlee/utils": "4.0.0-beta.183",
+        "@crawlee/fs-storage": "4.0.0-beta.185",
+        "@crawlee/http-client": "4.0.0-beta.185",
+        "@crawlee/types": "4.0.0-beta.185",
+        "@crawlee/utils": "4.0.0-beta.185",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -76,5 +76,5 @@
             }
         }
     },
-    "gitHead": "3832a8d3392bba9e3e7cc6d06b9be0be4ee1ab98"
+    "gitHead": "642c670956a5a60384d5ed5e67ee809927cc419b"
 }

package/request.d.ts

@@ -1,22 +1,12 @@
 import type { BinaryLike } from 'node:crypto';
-import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';
+import type { AllowedHttpMethods, Dictionary, RequestSchema } from '@crawlee/types';
 import type { EnqueueStrategy } from '@crawlee/utils';
 /** The `strategy` option accepted by {@link ExtractLinksOptions} and {@link EnqueueUrlsOptions}. */
 export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
-export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'transform' | 'redirect' | 'depth';
-export declare enum RequestState {
-    UNPROCESSED = 0,
-    BEFORE_NAV = 1,
-    AFTER_NAV = 2,
-    REQUEST_HANDLER = 3,
-    DONE = 4,
-    ERROR_HANDLER = 5,
-    ERROR = 6,
-    SKIPPED = 7
-}
 /**
- * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
- * The `Request` object also stores information about errors that occurred during processing of the request.
+ * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata, together
+ * with the processing state a request queue keeps about it (`retryCount`, `errorMessages`, `handledAt`, ...).
+ * Inside a crawler, requests are {@link CrawlingRequest}s, which add the crawler's own per-request settings.
  *
  * Each `Request` instance has the `uniqueKey` property, which can be either specified
  * manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
@@ -32,22 +22,16 @@ export declare enum RequestState {
  * const request = new Request({
  *     url: 'http://www.example.com',
  *     headers: { Accept: 'application/json' },
+ *     userData: { foo: 'bar' },
  * });
  *
- * ...
- *
- * request.userData.foo = 'bar';
- * request.pushErrorMessage(new Error('Request failed!'));
- *
- * ...
- *
- * const foo = request.userData.foo;
+ * await requestQueue.addRequest(request);
  * ```
  * @category Sources
  */
 declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
     #private;
-    /** Request ID */
+    /** Storage-assigned request ID. Only present on requests that went through a {@link RequestQueue}. */
     id?: string;
     /** URL of the web page to crawl. */
     url: string;
@@ -91,78 +75,25 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
     handledAt?: string;
     /**
      * `Request` parameters including the URL, HTTP method and headers, and others.
+     *
+     * Processing state (`retryCount`, `errorMessages`, `handledAt`, `loadedUrl`, `id`) is not an option;
+     * requests coming back from a storage are rebuilt with {@link Request.fromSchema}.
      */
     constructor(options: RequestOptions<UserData>);
+    /**
+     * Rebuilds a request from its stored form, including the processing state a
+     * {@link RequestOptions} object cannot carry. Subclasses get instances of themselves.
+     */
+    static fromSchema<UserData extends Dictionary = Dictionary>(schema: RequestSchema): CrawleeRequest<UserData>;
     /**
      * Converts the Crawlee Request object to a `fetch` API Request object.
      * @returns The native `fetch` API Request object.
      */
     intoFetchAPIRequest(): Request;
-    /**
-     * Tells the crawler processing this request to skip the navigation and process the request directly.
-     *
-     * When this is set to `true`, the crawling context will not contain the results of the navigation
-     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
-     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
-     */
-    get skipNavigation(): boolean;
-    /**
-     * Tells the crawler processing this request to skip the navigation and process the request directly.
-     *
-     * When this is set to `true`, the crawling context will not contain the results of the navigation
-     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
-     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
-     */
-    set skipNavigation(value: boolean);
-    /**
-     * Depth of the request in the current crawl tree.
-     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
-     */
-    get crawlDepth(): number;
-    /**
-     * Depth of the request in the current crawl tree.
-     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
-     */
-    set crawlDepth(value: number);
-    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
-    get sessionId(): string | undefined;
-    set sessionId(value: string | undefined);
     /** shortcut for getting `request.userData.label` */
     get label(): string | undefined;
     /** shortcut for setting `request.userData.label` */
     set label(value: string | undefined);
-    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
-    get maxRetries(): number | undefined;
-    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
-    set maxRetries(value: number | undefined);
-    /** Describes the request's current lifecycle state. */
-    get state(): RequestState;
-    /** Describes the request's current lifecycle state. */
-    set state(value: RequestState);
-    /**
-     * Reason for skipping this request.
-     */
-    get skippedReason(): SkippedRequestReason | undefined;
-    /**
-     * Reason for skipping this request.
-     */
-    set skippedReason(value: SkippedRequestReason | undefined);
-    private get enqueueStrategy();
-    private set enqueueStrategy(value);
-    /**
-     * Stores information about an error that occurred during processing of this request.
-     *
-     * You should always use Error instances when throwing errors in JavaScript.
-     *
-     * Nevertheless, to improve the debugging experience when using third party libraries
-     * that may not always throw an Error instance, the function performs a type
-     * inspection of the passed argument and attempts to extract as much information
-     * as possible, since just throwing a bad type error makes any debugging rather difficult.
-     *
-     * @param errorOrMessage Error object or error message to be stored in the request.
-     * @param [options]
-     */
-    pushErrorMessage(errorOrMessage: unknown, options?: PushErrorMessageOptions): void;
     /** @internal */
     static computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey, alwaysEnqueue, }: ComputeUniqueKeyOptions): string;
     /** @internal */
@@ -266,32 +197,13 @@ export interface RequestOptions<UserData extends Dictionary = Dictionary> {
      * @default 0
      */
     crawlDepth?: number;
-    /**
-     * Reason for skipping this request.
-     * This is used to provide more information about why the request was skipped.
-     * @internal
-     */
-    skippedReason?: SkippedRequestReason;
     /**
      * Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`.
      */
     maxRetries?: number;
     /** @internal */
-    id?: string;
-    /** @internal */
-    handledAt?: string;
-    /** @internal */
-    lockExpiresAt?: Date;
-    /** @internal */
     enqueueStrategy?: EnqueueStrategyOption;
 }
-export interface PushErrorMessageOptions {
-    /**
-     * Only push the error message without stack trace when true.
-     * @default false
-     */
-    omitStack?: boolean;
-}
 interface ComputeUniqueKeyOptions {
     url: string;
     method: AllowedHttpMethods;

package/request.js

@@ -1,41 +1,21 @@
 import crypto from 'node:crypto';
-import util from 'node:util';
 import { z } from 'zod';
 import { cryptoRandomObjectId, normalizeUrl } from '@apify/utilities';
 import { serviceLocator } from './service_locator.js';
 import { keys } from './typedefs.js';
 import { parseArgument, schemas } from './validators.js';
-const dateString = z.string().refine((value) => !Number.isNaN(Date.parse(value)), {
-    message: 'Invalid input: expected a date string',
-});
-export var RequestState;
-(function (RequestState) {
-    RequestState[RequestState["UNPROCESSED"] = 0] = "UNPROCESSED";
-    RequestState[RequestState["BEFORE_NAV"] = 1] = "BEFORE_NAV";
-    RequestState[RequestState["AFTER_NAV"] = 2] = "AFTER_NAV";
-    RequestState[RequestState["REQUEST_HANDLER"] = 3] = "REQUEST_HANDLER";
-    RequestState[RequestState["DONE"] = 4] = "DONE";
-    RequestState[RequestState["ERROR_HANDLER"] = 5] = "ERROR_HANDLER";
-    RequestState[RequestState["ERROR"] = 6] = "ERROR";
-    RequestState[RequestState["SKIPPED"] = 7] = "SKIPPED";
-})(RequestState || (RequestState = {}));
 const requestUrlSchema = z.object({ url: z.string() });
 // new properties on the Request object breaks serialization
 const requestOptionalSchemaShapes = {
-    id: z.string().optional(),
-    loadedUrl: z.url().optional(),
     uniqueKey: z.string().optional(),
     method: z.string().optional(),
     payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
     noRetry: z.boolean().optional(),
-    retryCount: schemas.anyNumber.optional(),
     sessionId: z.string().optional(),
     maxRetries: schemas.anyNumber.optional(),
-    errorMessages: schemas.arrayOf(z.string(), 'strings').optional(),
     headers: z.looseObject({}).optional(),
     userData: z.looseObject({}).optional(),
     label: z.string().optional(),
-    handledAt: z.union([dateString, z.date()]).optional(),
     keepUrlFragment: z.boolean().optional(),
     useExtendedUniqueKey: z.boolean().optional(),
     alwaysEnqueue: z.boolean().optional(),
@@ -43,13 +23,13 @@ const requestOptionalSchemaShapes = {
     crawlDepth: schemas.anyNumber
         .refine((value) => value >= 0, 'Expected a number greater than or equal to 0')
         .optional(),
-    state: z.enum(RequestState).optional(),
 };
 // Each schema is wrapped in a single-key object so validation errors carry the property name.
 const requestOptionalSchemas = Object.fromEntries(Object.entries(requestOptionalSchemaShapes).map(([key, schema]) => [key, z.object({ [key]: schema })]));
 /**
- * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata.
- * The `Request` object also stores information about errors that occurred during processing of the request.
+ * Represents a URL to be crawled, optionally including HTTP method, headers, payload and other metadata, together
+ * with the processing state a request queue keeps about it (`retryCount`, `errorMessages`, `handledAt`, ...).
+ * Inside a crawler, requests are {@link CrawlingRequest}s, which add the crawler's own per-request settings.
  *
  * Each `Request` instance has the `uniqueKey` property, which can be either specified
  * manually in the constructor or generated automatically from the URL. Two requests with the same `uniqueKey`
@@ -65,21 +45,15 @@ const requestOptionalSchemas = Object.fromEntries(Object.entries(requestOptional
  * const request = new Request({
  *     url: 'http://www.example.com',
  *     headers: { Accept: 'application/json' },
+ *     userData: { foo: 'bar' },
  * });
  *
- * ...
- *
- * request.userData.foo = 'bar';
- * request.pushErrorMessage(new Error('Request failed!'));
- *
- * ...
- *
- * const foo = request.userData.foo;
+ * await requestQueue.addRequest(request);
  * ```
  * @category Sources
  */
 class CrawleeRequest {
-    /** Request ID */
+    /** Storage-assigned request ID. Only present on requests that went through a {@link RequestQueue}. */
     id;
     /** URL of the web page to crawl. */
     url;
@@ -125,6 +99,9 @@ class CrawleeRequest {
     handledAt;
     /**
      * `Request` parameters including the URL, HTTP method and headers, and others.
+     *
+     * Processing state (`retryCount`, `errorMessages`, `handledAt`, `loadedUrl`, `id`) is not an option;
+     * requests coming back from a storage are rebuilt with {@link Request.fromSchema}.
      */
     constructor(options) {
         // A bare URL is a common slip — point at the object form instead of a generic type error.
@@ -149,7 +126,7 @@ class CrawleeRequest {
                 parseArgument({ [prop]: value }, schema, 'RequestOptions');
             }
         });
-        const { id, url, loadedUrl, uniqueKey, payload, noRetry = false, retryCount = 0, sessionId, maxRetries, errorMessages = [], headers = {}, userData = {}, label, handledAt, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
+        const { url, uniqueKey, payload, noRetry = false, sessionId, maxRetries, headers = {}, userData = {}, label, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, skipNavigation, enqueueStrategy, crawlDepth, } = options;
         let { method = 'GET' } = options;
         method = method.toUpperCase();
         if (method === 'GET' && payload)
@@ -157,9 +134,7 @@ class CrawleeRequest {
         if (uniqueKey && alwaysEnqueue) {
             throw new Error('`alwaysEnqueue` cannot be used together with a custom `uniqueKey`.');
         }
-        this.id = id;
         this.url = url;
-        this.loadedUrl = loadedUrl;
         this.uniqueKey =
             uniqueKey ||
                 CrawleeRequest.computeUniqueKey({
@@ -173,10 +148,9 @@ class CrawleeRequest {
         this.method = method;
         this.payload = payload;
         this.noRetry = noRetry;
-        this.retryCount = retryCount;
-        this.errorMessages = [...errorMessages];
+        this.retryCount = 0;
+        this.errorMessages = [];
         this.headers = { ...headers };
-        this.handledAt = handledAt instanceof Date ? handledAt.toISOString() : handledAt;
         if (label) {
             userData.label = label;
         }
@@ -216,18 +190,34 @@ class CrawleeRequest {
         });
         // reassign userData to ensure internal `__crawlee` object is non-enumerable
         this.userData = userData;
+        const crawlee = (this.userData.__crawlee ??= {});
         if (skipNavigation != null)
-            this.skipNavigation = skipNavigation;
+            crawlee.skipNavigation = skipNavigation;
         if (maxRetries != null)
-            this.maxRetries = maxRetries;
+            crawlee.maxRetries = maxRetries;
         if (crawlDepth != null)
-            this.userData.__crawlee.crawlDepth ??= crawlDepth;
+            crawlee.crawlDepth ??= crawlDepth;
         if (sessionId)
-            this.sessionId = sessionId;
-        // If it's already set, don't override it (for instance when fetching from storage)
-        if (enqueueStrategy) {
-            this.enqueueStrategy ??= enqueueStrategy;
-        }
+            crawlee.sessionId = sessionId;
+        // A stored request keeps the strategy it was enqueued under.
+        if (enqueueStrategy)
+            crawlee.enqueueStrategy ??= enqueueStrategy;
+    }
+    /**
+     * Rebuilds a request from its stored form, including the processing state a
+     * {@link RequestOptions} object cannot carry. Subclasses get instances of themselves.
+     */
+    static fromSchema(schema) {
+        const { id, retryCount = 0, errorMessages = [], handledAt, loadedUrl, ...options } = schema;
+        const request = new this(options);
+        request.id = id;
+        request.retryCount = retryCount;
+        request.errorMessages = [...errorMessages];
+        request.loadedUrl = loadedUrl;
+        // `apify-client` parses `*At` fields into `Date`s, whatever the schema says.
+        request.handledAt =
+            handledAt instanceof Date ? handledAt.toISOString() : handledAt;
+        return request;
     }
     /**
      * Converts the Crawlee Request object to a `fetch` API Request object.
@@ -240,54 +230,6 @@ class CrawleeRequest {
             body: this.payload,
         });
     }
-    /**
-     * Tells the crawler processing this request to skip the navigation and process the request directly.
-     *
-     * When this is set to `true`, the crawling context will not contain the results of the navigation
-     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
-     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
-     */
-    get skipNavigation() {
-        return this.userData.__crawlee?.skipNavigation ?? false;
-    }
-    /**
-     * Tells the crawler processing this request to skip the navigation and process the request directly.
-     *
-     * When this is set to `true`, the crawling context will not contain the results of the navigation
-     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
-     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
-     */
-    set skipNavigation(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { skipNavigation: value };
-        }
-        else {
-            this.userData.__crawlee.skipNavigation = value;
-        }
-    }
-    /**
-     * Depth of the request in the current crawl tree.
-     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
-     */
-    get crawlDepth() {
-        return this.userData.__crawlee?.crawlDepth ?? 0;
-    }
-    /**
-     * Depth of the request in the current crawl tree.
-     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
-     */
-    set crawlDepth(value) {
-        this.userData.__crawlee ??= {};
-        this.userData.__crawlee.crawlDepth = value;
-    }
-    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
-    get sessionId() {
-        return this.userData.__crawlee?.sessionId;
-    }
-    set sessionId(value) {
-        this.userData.__crawlee ??= {};
-        this.userData.__crawlee.sessionId = value;
-    }
     /** shortcut for getting `request.userData.label` */
     get label() {
         return this.userData.label;
@@ -296,110 +238,6 @@ class CrawleeRequest {
     set label(value) {
         this.userData.label = value;
     }
-    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
-    get maxRetries() {
-        return this.userData.__crawlee?.maxRetries;
-    }
-    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
-    set maxRetries(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { maxRetries: value };
-        }
-        else {
-            this.userData.__crawlee.maxRetries = value;
-        }
-    }
-    /** Describes the request's current lifecycle state. */
-    get state() {
-        return this.userData.__crawlee?.state ?? RequestState.UNPROCESSED;
-    }
-    /** Describes the request's current lifecycle state. */
-    set state(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { state: value };
-        }
-        else {
-            this.userData.__crawlee.state = value;
-        }
-    }
-    /**
-     * Reason for skipping this request.
-     */
-    get skippedReason() {
-        return this.userData.__crawlee?.skippedReason;
-    }
-    /**
-     * Reason for skipping this request.
-     */
-    set skippedReason(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { skippedReason: value };
-        }
-        else {
-            this.userData.__crawlee.skippedReason = value;
-        }
-    }
-    get enqueueStrategy() {
-        return this.userData.__crawlee?.enqueueStrategy;
-    }
-    set enqueueStrategy(value) {
-        if (!this.userData.__crawlee) {
-            this.userData.__crawlee = { enqueueStrategy: value };
-        }
-        else {
-            this.userData.__crawlee.enqueueStrategy = value;
-        }
-    }
-    /**
-     * Stores information about an error that occurred during processing of this request.
-     *
-     * You should always use Error instances when throwing errors in JavaScript.
-     *
-     * Nevertheless, to improve the debugging experience when using third party libraries
-     * that may not always throw an Error instance, the function performs a type
-     * inspection of the passed argument and attempts to extract as much information
-     * as possible, since just throwing a bad type error makes any debugging rather difficult.
-     *
-     * @param errorOrMessage Error object or error message to be stored in the request.
-     * @param [options]
-     */
-    pushErrorMessage(errorOrMessage, options = {}) {
-        const { omitStack } = options;
-        let message;
-        const type = typeof errorOrMessage;
-        if (type === 'object') {
-            if (!errorOrMessage) {
-                message = 'null';
-            }
-            else if (errorOrMessage instanceof Error) {
-                message = omitStack
-                    ? errorOrMessage.message
-                    : // .stack includes the message
-                        errorOrMessage.stack;
-            }
-            else if (Reflect.has(Object(errorOrMessage), 'message')) {
-                message = Reflect.get(Object(errorOrMessage), 'message');
-            }
-            else if (errorOrMessage.toString() !== '[object Object]') {
-                message = errorOrMessage.toString();
-            }
-            else {
-                try {
-                    message = util.inspect(errorOrMessage);
-                }
-                catch (err) {
-                    message = 'Unable to extract any message from the received object.';
-                }
-            }
-        }
-        else if (type === 'undefined') {
-            message = 'undefined';
-        }
-        else {
-            message = errorOrMessage.toString();
-        }
-        this.errorMessages.push(message);
-    }
     /** @internal */
     static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false, alwaysEnqueue = false, }) {
         const normalizedMethod = method.toUpperCase();

package/storages/request_queue.js

@@ -336,7 +336,7 @@ export class RequestQueue {
                 ? // Requests without a snapshot were deduplicated or written through; nothing to replay.
                     entry.requests
                         .filter((journaled) => journaled.snapshot !== undefined)
-                        .map((journaled) => new Request(journaled.snapshot))
+                        .map((journaled) => Request.fromSchema(journaled.snapshot))
                 : []);
             if (requests.length === 0)
                 continue;
@@ -473,7 +473,7 @@ export class RequestQueue {
                     if (opts.url !== undefined && typeof opts.url !== 'string') {
                         throw new Error(`Request options are not valid, the 'url' property is not a string. Input: ${inspect(opts)}`);
                     }
-                    if (opts.id !== undefined) {
+                    if ('id' in opts && opts.id !== undefined) {
                         throw new Error(`Request options are not valid, the 'id' property must not be present. Input: ${inspect(opts)}`);
                     }
                     if (opts.requestsFromUrl !== undefined &&
@@ -534,12 +534,12 @@ export class RequestQueue {
         // Requests buffered by the active transaction (under the `deferred` write policy) are visible to it.
         const buffered = transaction && this.bufferedRequests(transaction).get(uniqueKey);
         if (buffered) {
-            return new Request(buffered);
+            return Request.fromSchema(buffered);
         }
-        const requestOptions = await this.backend.getRequest(uniqueKey);
-        if (!requestOptions)
+        const schema = await this.backend.getRequest(uniqueKey);
+        if (!schema)
             return null;
-        return new Request(requestOptions);
+        return Request.fromSchema(schema);
     }
     /**
      * Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
@@ -564,10 +564,10 @@ export class RequestQueue {
             return null;
         }
         this.#statsTracker.add('headItemReadCount');
-        const requestOptions = await this.backend.fetchNextRequest();
-        if (!requestOptions)
+        const schema = await this.backend.fetchNextRequest();
+        if (!schema)
             return null;
-        return new Request(requestOptions);
+        return Request.fromSchema(schema);
     }
     /**
      * Marks a request that was previously returned by the
@@ -685,7 +685,6 @@ export class RequestQueue {
             isHandled: queueOperationInfo.wasAlreadyHandled,
             uniqueKey: queueOperationInfo.uniqueKey,
             hydrated: null,
-            lockExpiresAt: null,
             forefront: queueOperationInfo.forefront,
         });
     }

package/storages/transaction.d.ts

@@ -1,4 +1,4 @@
-import type { Awaitable, Dictionary } from '@crawlee/types';
+import type { Awaitable, Dictionary, RequestSchema } from '@crawlee/types';
 import type { RecordOptions } from './key_value_store.js';
 /**
  * Governs whether writes of a given storage type performed inside a {@link StorageTransaction} are
@@ -72,7 +72,7 @@ export interface JournaledRequest {
      * A full JSON snapshot of the request for the commit replay. Only present for buffered additions —
      * deduplicated and write-through ones are journaled for introspection only.
      */
-    snapshot?: Dictionary;
+    snapshot?: RequestSchema;
 }
 /**
  * A batch of request queue additions recorded in a transaction journal.