@crawlee/core 4.0.0-rc.0 → 4.0.0-rc.1

package/memory-storage/resource-clients/request-queue.js

@@ -19,11 +19,11 @@ export class RequestQueueBackend extends BaseClient {
     pendingRequestCount = 0;
     /**
      * Serializes every operation that reads-then-writes this backend's shared queue state — the
-     * `requests` map, the `forefrontRequestIds` array, the `inProgressRequestIds` set and the request
-     * counts. Those mutations span `await` points, so without this mutex a concurrent operation could
-     * interleave and corrupt them (e.g. a head scan pruning `forefrontRequestIds` while
-     * `addBatchOfRequests` pushes to it). Held by every mutating method as well as by `isEmpty`/
-     * `isFinished`, whose head scan also prunes `forefrontRequestIds`.
+     * `requests` map, the `pendingRequestIds` set, the `forefrontRequestIds` array, the
+     * `inProgressRequestIds` set and the request counts. Those mutations span `await` points, so
+     * without this mutex a concurrent operation could interleave and corrupt them (e.g. a head scan
+     * pruning `forefrontRequestIds` while `addBatchOfRequests` pushes to it). Held by every mutating
+     * method as well as by `isEmpty`/`isFinished`, whose head scan also prunes `forefrontRequestIds`.
      */
     #queueStateMutex = new AsyncQueue();
     #forefrontRequestIds = [];
@@ -37,16 +37,21 @@ export class RequestQueueBackend extends BaseClient {
      */
     #inProgressRequestIds = new Set();
     #requests = new Map();
-    // kept as TS-private: storage-backend tests read this field at runtime
-    storageBackend;
+    /**
+     * IDs of requests that are not yet handled (pending or in progress), in insertion order. Handled
+     * requests stay in `requests` for deduplication but are removed from here, so head scans only
+     * ever walk the unhandled tail instead of every request the queue has ever seen.
+     */
+    #pendingRequestIds = new Set();
+    #storageBackend;
     constructor(options) {
         super(options.id ?? randomUUID());
         this.name = options.name;
         this.cacheKey = options.cacheKey ?? this.name ?? this.id;
-        this.storageBackend = options.storageBackend;
+        this.#storageBackend = options.storageBackend;
     }
     async getMetadata() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         return this.toRequestQueueInfo();
     }
     async drop() {
@@ -55,16 +60,15 @@ export class RequestQueueBackend extends BaseClient {
         // removed, which `listPendingHead` would then dereference as `undefined`.
         await this.#queueStateMutex.wait();
         try {
-            const storeIndex = this.storageBackend.requestQueueBackendCache.findIndex((queue) => queue.id === this.id);
-            if (storeIndex !== -1) {
-                const [oldBackend] = this.storageBackend.requestQueueBackendCache.splice(storeIndex, 1);
-                oldBackend.pendingRequestCount = 0;
+            if (this.#storageBackend.evictBackend('RequestQueue', this.id)) {
+                this.pendingRequestCount = 0;
                 // Clear all in-memory state, consistent with `purge`. Clearing `requests` alone would
                 // leave dangling ids in `forefrontRequestIds`/`inProgressRequestIds`, which a later head
                 // scan would resolve to a missing request and dereference.
-                oldBackend.#requests.clear();
-                oldBackend.#forefrontRequestIds = [];
-                oldBackend.#inProgressRequestIds.clear();
+                this.#requests.clear();
+                this.#pendingRequestIds.clear();
+                this.#forefrontRequestIds = [];
+                this.#inProgressRequestIds.clear();
             }
         }
         finally {
@@ -78,23 +82,22 @@ export class RequestQueueBackend extends BaseClient {
         try {
             // Clear all in-memory state
             this.#requests.clear();
+            this.#pendingRequestIds.clear();
             this.#forefrontRequestIds = [];
             this.#inProgressRequestIds.clear();
             this.handledRequestCount = 0;
             this.pendingRequestCount = 0;
-            this.updateTimestamps(true);
+            this.#updateTimestamps(true);
         }
         finally {
             this.#queueStateMutex.shift();
         }
     }
-    *requestKeyIterator() {
+    *#requestKeyIterator() {
         for (let i = this.#forefrontRequestIds.length - 1; i >= 0; i--) {
             yield this.#forefrontRequestIds[i];
         }
-        for (const key of this.#requests.keys()) {
-            yield key;
-        }
+        yield* this.#pendingRequestIds;
     }
     /**
      * Scans the queue and returns the pending head — requests that are neither handled nor currently
@@ -109,16 +112,16 @@ export class RequestQueueBackend extends BaseClient {
      * Computing the flag is expensive: because an in-progress request may sit anywhere in the queue, it
      * forces a scan of every pending entry even when only `limit` items are wanted. Callers that only
      * need the head (e.g. {@link fetchNextRequest}, {@link isEmpty}) leave it off so the scan can stop as
-     * soon as the page is filled, keeping those calls O(head) instead of O(N).
+     * soon as the page is filled, keeping those calls O(head) instead of O(pending).
      */
-    async listPendingHead(limit, detectInProgressRequests = false) {
+    async #listPendingHead(limit, detectInProgressRequests = false) {
         const items = [];
         let hasInProgressRequests = false;
         // Tracks processed request IDs to avoid duplicates (request in both `forefrontRequestIds` and `requests`).
         const seenRequestIds = new Set();
         // Tracks handled request IDs from `forefrontRequestIds` to be removed.
         const handledForefrontIds = new Set();
-        for (const requestId of this.requestKeyIterator()) {
+        for (const requestId of this.#requestKeyIterator()) {
             // Once the requested page is filled we can stop — unless the caller asked us to detect
             // in-progress requests and we have not yet seen one, in which case we must keep scanning.
             if (items.length >= limit && (!detectInProgressRequests || hasInProgressRequests)) {
@@ -129,11 +132,10 @@ export class RequestQueueBackend extends BaseClient {
             }
             seenRequestIds.add(requestId);
             const request = this.#requests.get(requestId);
-            // Permanently-handled requests (`orderNo === null`) are in a terminal state and can be skipped.
+            // Only `forefrontRequestIds` can still reference a handled request (`orderNo === null`);
+            // `pendingRequestIds` drops them on handling. Remember the id so the list gets pruned below.
             if (request.orderNo === null) {
-                if (this.#forefrontRequestIds.includes(requestId)) {
-                    handledForefrontIds.add(requestId);
-                }
+                handledForefrontIds.add(requestId);
                 continue;
             }
             // In progress (fetched but not yet handled or reclaimed) — skip it, but remember that the
@@ -153,17 +155,17 @@ export class RequestQueueBackend extends BaseClient {
         };
     }
     async fetchNextRequest() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         await this.#queueStateMutex.wait();
         try {
-            const { items: [head], } = await this.listPendingHead(1);
+            const { items: [head], } = await this.#listPendingHead(1);
             if (!head) {
                 return undefined;
             }
             // Mark the request as in progress so it is not handed out again until it is handled or
             // reclaimed. The request keeps its `orderNo` (and thus its forefront / normal ordering).
             this.#inProgressRequestIds.add(head.id);
-            return this.jsonToRequest(head.json) ?? undefined;
+            return this.#jsonToRequest(head.json) ?? undefined;
         }
         finally {
             this.#queueStateMutex.shift();
@@ -182,7 +184,7 @@ export class RequestQueueBackend extends BaseClient {
                 unprocessedRequests: [],
             };
             for (const model of requests) {
-                const requestModel = this.createInternalRequest(model, options.forefront);
+                const requestModel = this.#createInternalRequest(model, options.forefront);
                 const existingRequestWithId = this.#requests.get(requestModel.id);
                 if (existingRequestWithId) {
                     result.processedRequests.push({
@@ -195,6 +197,7 @@ export class RequestQueueBackend extends BaseClient {
                 }
                 this.#requests.set(requestModel.id, requestModel);
                 if (requestModel.orderNo) {
+                    this.#pendingRequestIds.add(requestModel.id);
                     this.pendingRequestCount += 1;
                 }
                 else {
@@ -212,7 +215,7 @@ export class RequestQueueBackend extends BaseClient {
                     wasAlreadyPresent: false,
                 });
             }
-            this.updateTimestamps(true);
+            this.#updateTimestamps(true);
             return result;
         }
         finally {
@@ -221,14 +224,14 @@ export class RequestQueueBackend extends BaseClient {
     }
     async getRequest(uniqueKey) {
         parseArgument(uniqueKey, uniqueKeySchema);
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         const id = uniqueKeyToRequestId(uniqueKey);
         const json = this.#requests.get(id)?.json;
-        return this.jsonToRequest(json);
+        return this.#jsonToRequest(json);
     }
     async markRequestAsHandled(request) {
         parseArgument(request, schemas.storageRequest);
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
         // `requests` map, `inProgressRequestIds` set and request counts stay consistent across the
         // `await` points below.
@@ -245,15 +248,16 @@ export class RequestQueueBackend extends BaseClient {
             // A handled request has `orderNo === null`. Marking it again is an idempotent no-op.
             const wasAlreadyHandled = existingRequest.orderNo === null;
             const handledAt = request.handledAt ?? new Date().toISOString();
-            const requestModel = this.createInternalRequest({ ...request, handledAt }, false);
+            const requestModel = this.#createInternalRequest({ ...request, handledAt }, false);
             this.#requests.set(id, requestModel);
+            this.#pendingRequestIds.delete(id);
             // The request is no longer in progress for this client.
             this.#inProgressRequestIds.delete(id);
             if (!wasAlreadyHandled) {
                 this.pendingRequestCount -= 1;
                 this.handledRequestCount += 1;
             }
-            this.updateTimestamps(true);
+            this.#updateTimestamps(true);
             return {
                 requestId: id,
                 wasAlreadyHandled,
@@ -267,7 +271,7 @@ export class RequestQueueBackend extends BaseClient {
     async reclaimRequest(request, options = {}) {
         parseArgument(request, schemas.storageRequest);
         parseArgument(options, schemas.requestQueueOperationOptions);
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
         // `requests` map, `forefrontRequestIds` array and `inProgressRequestIds` set stay consistent
         // across the `await` points below.
@@ -284,14 +288,14 @@ export class RequestQueueBackend extends BaseClient {
             }
             // Reclaiming resets the `orderNo` to a fresh timestamp, restoring the request to the queue
             // (at the front if `forefront`).
-            const requestModel = this.createInternalRequest(request, options.forefront);
+            const requestModel = this.#createInternalRequest(request, options.forefront);
             this.#requests.set(id, requestModel);
             // The request is no longer in progress for this client.
             this.#inProgressRequestIds.delete(id);
             if (options.forefront) {
                 this.#forefrontRequestIds.push(id);
             }
-            this.updateTimestamps(true);
+            this.#updateTimestamps(true);
             return {
                 requestId: id,
                 wasAlreadyHandled: false,
@@ -303,7 +307,7 @@ export class RequestQueueBackend extends BaseClient {
         }
     }
     async isEmpty() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         // "Empty" means there is nothing left to fetch right now — i.e. the next `fetchNextRequest`
         // would return `null`. Requests that are currently in progress are intentionally NOT counted
         // here: they are not fetchable, so the queue is empty from a consumer's point of view. Whether
@@ -314,7 +318,7 @@ export class RequestQueueBackend extends BaseClient {
         // racing a concurrent mutator (e.g. `addBatchOfRequests`) at its `await` points.
         await this.#queueStateMutex.wait();
         try {
-            const { items } = await this.listPendingHead(1);
+            const { items } = await this.#listPendingHead(1);
             return items.length === 0;
         }
         finally {
@@ -322,7 +326,7 @@ export class RequestQueueBackend extends BaseClient {
         }
     }
     async isFinished() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         // The queue is finished only when there is nothing left to fetch AND nothing currently in
         // progress. Counting in-progress requests is what allows a crawler with concurrency to keep
         // waiting while it still holds the last requests, instead of finishing prematurely.
@@ -334,7 +338,7 @@ export class RequestQueueBackend extends BaseClient {
         // racing a concurrent mutator (e.g. `addBatchOfRequests`) at its `await` points.
         await this.#queueStateMutex.wait();
         try {
-            const { items, hasInProgressRequests } = await this.listPendingHead(1, true);
+            const { items, hasInProgressRequests } = await this.#listPendingHead(1, true);
             return items.length === 0 && !hasInProgressRequests;
         }
         finally {
@@ -347,13 +351,13 @@ export class RequestQueueBackend extends BaseClient {
      * nothing is marked in progress.
      */
     async listItems() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         // `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state
         // mutex to avoid racing a concurrent mutator at its `await` points.
         await this.#queueStateMutex.wait();
         try {
-            const { items } = await this.listPendingHead(Number.POSITIVE_INFINITY);
-            return items.map((request) => this.jsonToRequest(request.json));
+            const { items } = await this.#listPendingHead(Number.POSITIVE_INFINITY);
+            return items.map((request) => this.#jsonToRequest(request.json));
         }
         finally {
             this.#queueStateMutex.shift();
@@ -371,20 +375,20 @@ export class RequestQueueBackend extends BaseClient {
             totalRequestCount: this.#requests.size,
         };
     }
-    updateTimestamps(hasBeenModified) {
+    #updateTimestamps(hasBeenModified) {
         this.accessedAt = new Date();
         if (hasBeenModified) {
             this.modifiedAt = new Date();
         }
     }
-    jsonToRequest(requestJson) {
+    #jsonToRequest(requestJson) {
         if (!requestJson)
             return undefined;
         const request = JSON.parse(requestJson);
         return purgeNullsFromObject(request);
     }
-    createInternalRequest(request, forefront) {
-        const orderNo = this.calculateOrderNo(request, forefront);
+    #createInternalRequest(request, forefront) {
+        const orderNo = this.#calculateOrderNo(request, forefront);
         const id = uniqueKeyToRequestId(request.uniqueKey);
         if (request.id && request.id !== id) {
             throw new Error('Request ID does not match its uniqueKey.');
@@ -400,7 +404,7 @@ export class RequestQueueBackend extends BaseClient {
             url: request.url,
         };
     }
-    calculateOrderNo(request, forefront) {
+    #calculateOrderNo(request, forefront) {
         if (request.handledAt)
             return null;
         const timestamp = Date.now();

package/package.json

@@ -1,13 +1,14 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-rc.0",
+    "version": "4.0.0-rc.1",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
         ".": "./index.js",
+        "./internal": "./internal.js",
         "./package.json": "./package.json"
     },
     "keywords": [
@@ -47,29 +48,26 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/consts": "^2.41.0",
-        "@apify/datastructures": "^2.0.3",
-        "@apify/log": "^2.5.18",
-        "@apify/timeout": "^0.4.4",
-        "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-rc.0",
-        "@crawlee/http-client": "4.0.0-rc.0",
-        "@crawlee/types": "4.0.0-rc.0",
-        "@crawlee/utils": "4.0.0-rc.0",
+        "@apify/consts": "^3.0.1",
+        "@apify/datastructures": "^3.0.1",
+        "@apify/log": "^3.0.1",
+        "@apify/timeout": "^1.0.1",
+        "@apify/utilities": "^3.0.1",
+        "@crawlee/fs-storage": "4.0.0-rc.1",
+        "@crawlee/http-client": "4.0.0-rc.1",
+        "@crawlee/types": "4.0.0-rc.1",
+        "@crawlee/utils": "4.0.0-rc.1",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
-        "content-type": "^1.0.5",
+        "content-type": "^3.1.1",
         "csv-stringify": "^6.5.2",
         "json5": "^2.2.3",
         "mime-types": "^3.0.1",
-        "minimatch": "^10.0.1",
-        "stream-json": "^1.9.1",
-        "tldts": "^7.0.6",
-        "tough-cookie": "^6.0.0",
+        "stream-json": "^3.7.0",
         "tslib": "^2.8.1",
         "type-fest": "^4.41.0",
-        "zod": "^4.4.3"
+        "zod": "^4.5.4"
     },
     "lerna": {
         "command": {
@@ -78,5 +76,5 @@
             }
         }
     },
-    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }

package/proxy_configuration.d.ts

@@ -1,28 +1,28 @@
 import type { ProxyInfo } from '@crawlee/types';
-import type { Request } from './request.js';
 export interface ProxyConfigurationFunction {
-    (options?: {
-        request?: Request;
-    }): string | null | Promise<string | null>;
+    (): string | null | Promise<string | null>;
 }
-type UrlList = (string | null)[];
 export interface ProxyConfigurationOptions {
     /**
      * An array of custom proxy URLs to be rotated.
      * Custom proxies are not compatible with Apify Proxy and an attempt to use both
      * configuration options will cause an error to be thrown on initialize.
      */
-    proxyUrls?: UrlList;
+    proxyUrls?: (string | null)[];
     /**
-     * Custom function that allows you to generate the new proxy URL dynamically. It gets an optional parameter with the `Request` object when applicable.
+     * Custom function that allows you to generate the new proxy URL dynamically.
      * Can return either stringified proxy URL or `null` if the proxy should not be used. Can be asynchronous.
      *
      * This function is used to generate the URL when {@link ProxyConfiguration.newUrl} or {@link ProxyConfiguration.newProxyInfo} is called.
      */
     newUrlFunction?: ProxyConfigurationFunction;
-}
-interface NewUrlOptions {
-    request?: Request;
+    /**
+     * When truthy, the constructor throws unless one of `proxyUrls` / `newUrlFunction` was given. Falsy by
+     * default, so a bare `ProxyConfiguration` can be constructed. Set by the Apify SDK, which builds the options
+     * object itself; declared here only so that it stays type-checkable.
+     * @internal
+     */
+    validateRequired?: boolean;
 }
 /**
  * Minimal contract that any object passed to a crawler as its `proxyConfiguration`
@@ -36,10 +36,14 @@ interface NewUrlOptions {
  */
 export interface IProxyConfiguration {
     /**
-     * Creates a new {@link ProxyInfo} object describing the proxy to use for the given
-     * request. Returns `undefined` when no proxy should be used.
+     * Creates a new {@link ProxyInfo} object describing the proxy to use.
+     * Returns `undefined` when no proxy should be used.
+     *
+     * @param proxyInfo A previously created `ProxyInfo`, e.g. one restored with a persisted session. Implementations
+     *  should return an equivalent `ProxyInfo` that is usable in the current environment, or the argument itself when
+     *  there is nothing to refresh.
      */
-    newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
+    newProxyInfo(proxyInfo?: ProxyInfo): Promise<ProxyInfo | undefined>;
 }
 /**
  * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
@@ -100,22 +104,15 @@ export declare class ProxyConfiguration implements IProxyConfiguration {
      * Use it if you want to work with a rich representation of a proxy URL.
      * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
      *
+     * @param proxyInfo A previously created `ProxyInfo`, returned unchanged.
      * @return Represents information about used proxy and its configuration.
      */
-    newProxyInfo(options?: NewUrlOptions): Promise<ProxyInfo | undefined>;
+    newProxyInfo(proxyInfo?: ProxyInfo): Promise<ProxyInfo | undefined>;
     /**
      * Returns a new proxy URL based on provided configuration options.
      *
      * @return A string with a proxy URL, including authentication credentials and port number.
      *  For example, `http://bob:password123@proxy.example.com:8000`
      */
-    newUrl(options?: NewUrlOptions): Promise<string | undefined>;
-    private handleProxyUrlsList;
-    /**
-     * Calls the custom newUrlFunction and checks format of its return value
-     */
-    private callNewUrlFunction;
-    private throwCannotCombineCustomMethods;
-    private throwNoOptionsProvided;
+    newUrl(): Promise<string | undefined>;
 }
-export {};

package/proxy_configuration.js

@@ -61,6 +61,9 @@ export class ProxyConfiguration {
      * ```
      */
     constructor(options = {}) {
+        // `validateRequired` is destructured off before the strict-object parse on purpose: the Apify SDK passes it
+        // through a computed key (`['validateRequired' as string]: false`), and leaving it in `rest` would make
+        // `Actor.createProxyConfiguration()` fail the `z.strictObject` check with a `ZodError`.
         const { validateRequired, ...rest } = options;
         if ('tieredProxyUrls' in rest) {
             throw new Error('The `tieredProxyUrls` option has been removed in Crawlee v4. ' +
@@ -68,9 +71,9 @@ export class ProxyConfiguration {
         }
         const { proxyUrls, newUrlFunction } = parseArgument(rest, proxyConfigurationOptionsSchema);
         if (proxyUrls && newUrlFunction)
-            this.throwCannotCombineCustomMethods();
+            this.#throwCannotCombineCustomMethods();
         if (!proxyUrls && !newUrlFunction && validateRequired)
-            this.throwNoOptionsProvided();
+            this.#throwNoOptionsProvided();
         this.#proxyUrls = proxyUrls;
         this.#newUrlFunction = newUrlFunction;
     }
@@ -81,10 +84,13 @@ export class ProxyConfiguration {
      * Use it if you want to work with a rich representation of a proxy URL.
      * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
      *
+     * @param proxyInfo A previously created `ProxyInfo`, returned unchanged.
      * @return Represents information about used proxy and its configuration.
      */
-    async newProxyInfo(options) {
-        const url = await this.newUrl(options);
+    async newProxyInfo(proxyInfo) {
+        if (proxyInfo)
+            return proxyInfo;
+        const url = await this.newUrl();
         if (!url)
             return undefined;
         const { username, password, port, hostname } = new URL(url);
@@ -102,20 +108,20 @@ export class ProxyConfiguration {
      * @return A string with a proxy URL, including authentication credentials and port number.
      *  For example, `http://bob:password123@proxy.example.com:8000`
      */
-    async newUrl(options) {
+    async newUrl() {
         if (this.#newUrlFunction) {
-            return (await this.callNewUrlFunction({ request: options?.request })) ?? undefined;
+            return (await this.#callNewUrlFunction()) ?? undefined;
         }
-        return this.handleProxyUrlsList() ?? undefined;
+        return this.#handleProxyUrlsList() ?? undefined;
     }
-    handleProxyUrlsList() {
+    #handleProxyUrlsList() {
         return this.#proxyUrls[this.#nextCustomUrlIndex++ % this.#proxyUrls.length];
     }
     /**
      * Calls the custom newUrlFunction and checks format of its return value
      */
-    async callNewUrlFunction(options) {
-        const proxyUrl = await this.#newUrlFunction(options);
+    async #callNewUrlFunction() {
+        const proxyUrl = await this.#newUrlFunction();
         try {
             if (proxyUrl) {
                 new URL(proxyUrl); // eslint-disable-line no-new
@@ -126,10 +132,10 @@ export class ProxyConfiguration {
             throw new Error(`The provided newUrlFunction did not return a valid URL.\nCause: ${err.message}`);
         }
     }
-    throwCannotCombineCustomMethods() {
+    #throwCannotCombineCustomMethods() {
         throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".');
     }
-    throwNoOptionsProvided() {
+    #throwNoOptionsProvided() {
         throw new Error('One of "options.proxyUrls" or "options.newUrlFunction" needs to be provided.');
     }
 }

package/recoverable_state.d.ts

@@ -1,5 +1,6 @@
-import type { Configuration, CrawleeLogger } from '@crawlee/core';
-import { KeyValueStore } from '@crawlee/core';
+import type { Configuration } from './configuration.js';
+import type { CrawleeLogger } from './log.js';
+import { KeyValueStore } from './storages/key_value_store.js';
 import type { Awaitable } from '@crawlee/types';
 import type { StandardSchemaV1 } from '@standard-schema/spec';
 /**
@@ -47,9 +48,9 @@ export interface RecoverableStatePersistenceOptions {
     persistenceTimeoutMillis?: number;
 }
 /**
- * Options for configuring the RecoverableState
+ * The fields of {@link RecoverableStateOptions}, without the constraint tying `contentType` to the conversions.
  */
-export interface RecoverableStateOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> extends RecoverableStatePersistenceOptions {
+export interface RecoverableStateBaseOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> extends RecoverableStatePersistenceOptions {
     /**
      * The state used when no persisted state is found, and the state {@link RecoverableState.reset} restores.
      *
@@ -68,14 +69,34 @@ export interface RecoverableStateOptions<TStateModel = Record<string, unknown>,
     /**
      * Optional conversion of the state to a plain JSON-serializable value before it is persisted.
      * If not provided, the state is persisted as is.
+     *
+     * With {@link RecoverableStateBaseOptions.contentType} set, it has to produce what
+     * {@link KeyValueStore.setValue} accepts alongside an explicit content type - a `string`, a `Buffer` or a
+     * stream.
      */
     serialize?: StateConversion<TStateModel, TPersistedState>;
     /**
      * Optional conversion of a persisted value back to the state model, and the place to validate a record before
      * trusting it. If not provided, the persisted value is used as is.
+     *
+     * With {@link RecoverableStateBaseOptions.contentType} set, it receives a `Readable` of the record bytes
+     * instead of a parsed value.
      */
     deserialize?: StateConversion<TPersistedState, TStateModel>;
+    /**
+     * Content type of the persisted record. Setting it hands the record encoding over to
+     * {@link RecoverableStateBaseOptions.serialize} and {@link RecoverableStateBaseOptions.deserialize}, both of
+     * which are then required - the default JSON codec is bypassed in both directions. Meant for a state too large
+     * for `JSON.stringify`, which `serialize` can then stream out instead.
+     */
+    contentType?: string;
 }
+/**
+ * Options for configuring the RecoverableState
+ */
+export type RecoverableStateOptions<TStateModel = Record<string, unknown>, TPersistedState = TStateModel> = RecoverableStateBaseOptions<TStateModel, TPersistedState> & ({
+    contentType?: undefined;
+} | Required<Pick<RecoverableStateBaseOptions<TStateModel, TPersistedState>, 'serialize' | 'deserialize' | 'contentType'>>);
 /**
  * A class for managing persistent recoverable state using a plain JavaScript object.
  *
@@ -102,7 +123,8 @@ export declare class RecoverableState<TStateModel = Record<string, unknown>, TPe
      * is no record to restore.
      *
      * Calling this again after a {@link RecoverableState.teardown} starts a new persistence window - the
-     * listener is registered again and the record reloaded.
+     * listener is registered again. The record is not reloaded: the in-memory state is what the teardown wrote, and
+     * reloading it would only drop whatever the deserialization leaves out.
      *
      * @returns The loaded state object
      */
@@ -139,7 +161,7 @@ export declare class RecoverableState<TStateModel = Record<string, unknown>, TPe
      * one would write the record straight back. Use {@link RecoverableState.reset} to reset the state itself,
      * or {@link RecoverableState.teardown} before clearing the record.
      *
-     * A no-op if persistence is disabled or no KeyValueStore is available yet.
+     * A no-op if persistence is disabled.
      */
     resetStore(): Promise<void>;
     /**