@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1

package/memory-storage/resource-clients/request-queue.js

@@ -1,23 +1,10 @@
 import { randomUUID } from 'node:crypto';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
 import { AsyncQueue } from '@sapphire/async-queue';
-import { s } from '@sapphire/shapeshift';
+import { z } from 'zod';
 import { purgeNullsFromObject, uniqueKeyToRequestId } from '../utils.js';
 import { BaseClient } from './common/base-client.js';
-const requestShape = s
-    .object({
-    id: s.string(),
-    url: s.string().url({ allowedProtocols: ['http:', 'https:'] }),
-    uniqueKey: s.string(),
-    method: s.string().optional(),
-    retryCount: s.number().int().optional(),
-    handledAt: s.union([s.string(), s.date().valid()]).optional(),
-})
-    .passthrough();
-const requestShapeWithoutId = requestShape.omit(['id']);
-const batchRequestShapeWithoutId = requestShapeWithoutId.array();
-const requestOptionsShape = s.object({
-    forefront: s.boolean().optional(),
-});
+const uniqueKeySchema = z.string();
 export class RequestQueueBackend extends BaseClient {
     name;
     /**
@@ -32,14 +19,14 @@ export class RequestQueueBackend extends BaseClient {
     pendingRequestCount = 0;
     /**
      * Serializes every operation that reads-then-writes this backend's shared queue state — the
-     * `requests` map, the `forefrontRequestIds` array, the `inProgressRequestIds` set and the request
-     * counts. Those mutations span `await` points, so without this mutex a concurrent operation could
-     * interleave and corrupt them (e.g. a head scan pruning `forefrontRequestIds` while
-     * `addBatchOfRequests` pushes to it). Held by every mutating method as well as by `isEmpty`/
-     * `isFinished`, whose head scan also prunes `forefrontRequestIds`.
+     * `requests` map, the `pendingRequestIds` set, the `forefrontRequestIds` array, the
+     * `inProgressRequestIds` set and the request counts. Those mutations span `await` points, so
+     * without this mutex a concurrent operation could interleave and corrupt them (e.g. a head scan
+     * pruning `forefrontRequestIds` while `addBatchOfRequests` pushes to it). Held by every mutating
+     * method as well as by `isEmpty`/`isFinished`, whose head scan also prunes `forefrontRequestIds`.
      */
-    queueStateMutex = new AsyncQueue();
-    forefrontRequestIds = [];
+    #queueStateMutex = new AsyncQueue();
+    #forefrontRequestIds = [];
     /**
      * IDs of requests currently fetched but not yet handled or reclaimed. A request in this set is
      * "in progress" and will not be handed out again by {@link fetchNextRequest}.
@@ -48,65 +35,69 @@ export class RequestQueueBackend extends BaseClient {
      * process and is never shared with another consumer, so there is no need for an expiring,
      * cross-process-visible lock — tracking in-progress requests in this set is enough.
      */
-    inProgressRequestIds = new Set();
-    requests = new Map();
-    storageBackend;
+    #inProgressRequestIds = new Set();
+    #requests = new Map();
+    /**
+     * IDs of requests that are not yet handled (pending or in progress), in insertion order. Handled
+     * requests stay in `requests` for deduplication but are removed from here, so head scans only
+     * ever walk the unhandled tail instead of every request the queue has ever seen.
+     */
+    #pendingRequestIds = new Set();
+    #storageBackend;
     constructor(options) {
         super(options.id ?? randomUUID());
         this.name = options.name;
         this.cacheKey = options.cacheKey ?? this.name ?? this.id;
-        this.storageBackend = options.storageBackend;
+        this.#storageBackend = options.storageBackend;
     }
     async getMetadata() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         return this.toRequestQueueInfo();
     }
     async drop() {
         // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so a concurrent
         // operation cannot observe half-cleared state — e.g. a forefront id whose request has already been
         // removed, which `listPendingHead` would then dereference as `undefined`.
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
-            const storeIndex = this.storageBackend.requestQueueBackendCache.findIndex((queue) => queue.id === this.id);
-            if (storeIndex !== -1) {
-                const [oldBackend] = this.storageBackend.requestQueueBackendCache.splice(storeIndex, 1);
-                oldBackend.pendingRequestCount = 0;
+            if (this.#storageBackend.evictBackend('RequestQueue', this.id)) {
+                this.pendingRequestCount = 0;
                 // Clear all in-memory state, consistent with `purge`. Clearing `requests` alone would
                 // leave dangling ids in `forefrontRequestIds`/`inProgressRequestIds`, which a later head
                 // scan would resolve to a missing request and dereference.
-                oldBackend.requests.clear();
-                oldBackend.forefrontRequestIds = [];
-                oldBackend.inProgressRequestIds.clear();
+                this.#requests.clear();
+                this.#pendingRequestIds.clear();
+                this.#forefrontRequestIds = [];
+                this.#inProgressRequestIds.clear();
             }
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     async purge() {
         // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so a concurrent
         // operation cannot observe or repopulate half-cleared state across the `await` below.
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
             // Clear all in-memory state
-            this.requests.clear();
-            this.forefrontRequestIds = [];
-            this.inProgressRequestIds.clear();
+            this.#requests.clear();
+            this.#pendingRequestIds.clear();
+            this.#forefrontRequestIds = [];
+            this.#inProgressRequestIds.clear();
             this.handledRequestCount = 0;
             this.pendingRequestCount = 0;
-            this.updateTimestamps(true);
+            this.#updateTimestamps(true);
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
-    *requestKeyIterator() {
-        for (let i = this.forefrontRequestIds.length - 1; i >= 0; i--) {
-            yield this.forefrontRequestIds[i];
-        }
-        for (const key of this.requests.keys()) {
-            yield key;
+    *#requestKeyIterator() {
+        for (let i = this.#forefrontRequestIds.length - 1; i >= 0; i--) {
+            yield this.#forefrontRequestIds[i];
         }
+        yield* this.#pendingRequestIds;
     }
     /**
      * Scans the queue and returns the pending head — requests that are neither handled nor currently
@@ -121,16 +112,16 @@ export class RequestQueueBackend extends BaseClient {
      * Computing the flag is expensive: because an in-progress request may sit anywhere in the queue, it
      * forces a scan of every pending entry even when only `limit` items are wanted. Callers that only
      * need the head (e.g. {@link fetchNextRequest}, {@link isEmpty}) leave it off so the scan can stop as
-     * soon as the page is filled, keeping those calls O(head) instead of O(N).
+     * soon as the page is filled, keeping those calls O(head) instead of O(pending).
      */
-    async listPendingHead(limit, detectInProgressRequests = false) {
+    async #listPendingHead(limit, detectInProgressRequests = false) {
         const items = [];
         let hasInProgressRequests = false;
         // Tracks processed request IDs to avoid duplicates (request in both `forefrontRequestIds` and `requests`).
         const seenRequestIds = new Set();
         // Tracks handled request IDs from `forefrontRequestIds` to be removed.
         const handledForefrontIds = new Set();
-        for (const requestId of this.requestKeyIterator()) {
+        for (const requestId of this.#requestKeyIterator()) {
             // Once the requested page is filled we can stop — unless the caller asked us to detect
             // in-progress requests and we have not yet seen one, in which case we must keep scanning.
             if (items.length >= limit && (!detectInProgressRequests || hasInProgressRequests)) {
@@ -140,17 +131,16 @@ export class RequestQueueBackend extends BaseClient {
                 continue;
             }
             seenRequestIds.add(requestId);
-            const request = this.requests.get(requestId);
-            // Permanently-handled requests (`orderNo === null`) are in a terminal state and can be skipped.
+            const request = this.#requests.get(requestId);
+            // Only `forefrontRequestIds` can still reference a handled request (`orderNo === null`);
+            // `pendingRequestIds` drops them on handling. Remember the id so the list gets pruned below.
             if (request.orderNo === null) {
-                if (this.forefrontRequestIds.includes(requestId)) {
-                    handledForefrontIds.add(requestId);
-                }
+                handledForefrontIds.add(requestId);
                 continue;
             }
             // In progress (fetched but not yet handled or reclaimed) — skip it, but remember that the
             // queue is not truly empty.
-            if (this.inProgressRequestIds.has(requestId)) {
+            if (this.#inProgressRequestIds.has(requestId)) {
                 hasInProgressRequests = true;
                 continue;
             }
@@ -158,44 +148,44 @@ export class RequestQueueBackend extends BaseClient {
                 items.push(request);
             }
         }
-        this.forefrontRequestIds = this.forefrontRequestIds.filter((id) => !handledForefrontIds.has(id));
+        this.#forefrontRequestIds = this.#forefrontRequestIds.filter((id) => !handledForefrontIds.has(id));
         return {
             items: items.sort((a, b) => a.orderNo - b.orderNo),
             hasInProgressRequests: detectInProgressRequests ? hasInProgressRequests : undefined,
         };
     }
     async fetchNextRequest() {
-        this.updateTimestamps(false);
-        await this.queueStateMutex.wait();
+        this.#updateTimestamps(false);
+        await this.#queueStateMutex.wait();
         try {
-            const { items: [head], } = await this.listPendingHead(1);
+            const { items: [head], } = await this.#listPendingHead(1);
             if (!head) {
                 return undefined;
             }
             // Mark the request as in progress so it is not handed out again until it is handled or
             // reclaimed. The request keeps its `orderNo` (and thus its forefront / normal ordering).
-            this.inProgressRequestIds.add(head.id);
-            return this._jsonToRequest(head.json) ?? undefined;
+            this.#inProgressRequestIds.add(head.id);
+            return this.#jsonToRequest(head.json) ?? undefined;
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     async addBatchOfRequests(requests, options = {}) {
-        batchRequestShapeWithoutId.parse(requests);
-        requestOptionsShape.parse(options);
+        parseArgument(requests, schemas.storageRequestBatch);
+        parseArgument(options, schemas.requestQueueOperationOptions);
         // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so that the
         // shared `requests` map, `forefrontRequestIds` array and request counts are not corrupted by a
         // concurrent operation interleaving at one of the `await` points below.
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
             const result = {
                 processedRequests: [],
                 unprocessedRequests: [],
             };
             for (const model of requests) {
-                const requestModel = this._createInternalRequest(model, options.forefront);
-                const existingRequestWithId = this.requests.get(requestModel.id);
+                const requestModel = this.#createInternalRequest(model, options.forefront);
+                const existingRequestWithId = this.#requests.get(requestModel.id);
                 if (existingRequestWithId) {
                     result.processedRequests.push({
                         requestId: existingRequestWithId.id,
@@ -205,15 +195,16 @@ export class RequestQueueBackend extends BaseClient {
                     });
                     continue;
                 }
-                this.requests.set(requestModel.id, requestModel);
+                this.#requests.set(requestModel.id, requestModel);
                 if (requestModel.orderNo) {
+                    this.#pendingRequestIds.add(requestModel.id);
                     this.pendingRequestCount += 1;
                 }
                 else {
                     this.handledRequestCount += 1;
                 }
                 if (options.forefront) {
-                    this.forefrontRequestIds.push(requestModel.id);
+                    this.#forefrontRequestIds.push(requestModel.id);
                 }
                 result.processedRequests.push({
                     requestId: requestModel.id,
@@ -224,30 +215,30 @@ export class RequestQueueBackend extends BaseClient {
                     wasAlreadyPresent: false,
                 });
             }
-            this.updateTimestamps(true);
+            this.#updateTimestamps(true);
             return result;
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     async getRequest(uniqueKey) {
-        s.string().parse(uniqueKey);
-        this.updateTimestamps(false);
+        parseArgument(uniqueKey, uniqueKeySchema);
+        this.#updateTimestamps(false);
         const id = uniqueKeyToRequestId(uniqueKey);
-        const json = this.requests.get(id)?.json;
-        return this._jsonToRequest(json);
+        const json = this.#requests.get(id)?.json;
+        return this.#jsonToRequest(json);
     }
     async markRequestAsHandled(request) {
-        requestShape.parse(request);
-        this.updateTimestamps(false);
+        parseArgument(request, schemas.storageRequest);
+        this.#updateTimestamps(false);
         // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
         // `requests` map, `inProgressRequestIds` set and request counts stay consistent across the
         // `await` points below.
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
             const id = uniqueKeyToRequestId(request.uniqueKey);
-            const existingRequest = this.requests.get(id);
+            const existingRequest = this.#requests.get(id);
             // The request must exist to be marked as handled. We intentionally do NOT require it to still
             // be in progress: marking an already-released request handled must still succeed, otherwise
             // the request could be handed out again and the queue would never finish.
@@ -257,15 +248,16 @@ export class RequestQueueBackend extends BaseClient {
             // A handled request has `orderNo === null`. Marking it again is an idempotent no-op.
             const wasAlreadyHandled = existingRequest.orderNo === null;
             const handledAt = request.handledAt ?? new Date().toISOString();
-            const requestModel = this._createInternalRequest({ ...request, handledAt }, false);
-            this.requests.set(id, requestModel);
+            const requestModel = this.#createInternalRequest({ ...request, handledAt }, false);
+            this.#requests.set(id, requestModel);
+            this.#pendingRequestIds.delete(id);
             // The request is no longer in progress for this client.
-            this.inProgressRequestIds.delete(id);
+            this.#inProgressRequestIds.delete(id);
             if (!wasAlreadyHandled) {
                 this.pendingRequestCount -= 1;
                 this.handledRequestCount += 1;
             }
-            this.updateTimestamps(true);
+            this.#updateTimestamps(true);
             return {
                 requestId: id,
                 wasAlreadyHandled,
@@ -273,20 +265,20 @@ export class RequestQueueBackend extends BaseClient {
             };
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     async reclaimRequest(request, options = {}) {
-        requestShape.parse(request);
-        requestOptionsShape.parse(options);
-        this.updateTimestamps(false);
+        parseArgument(request, schemas.storageRequest);
+        parseArgument(options, schemas.requestQueueOperationOptions);
+        this.#updateTimestamps(false);
         // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
         // `requests` map, `forefrontRequestIds` array and `inProgressRequestIds` set stay consistent
         // across the `await` points below.
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
             const id = uniqueKeyToRequestId(request.uniqueKey);
-            const existingRequest = this.requests.get(id);
+            const existingRequest = this.#requests.get(id);
             // The request must exist and not already be handled to be reclaimed. As with
             // `markRequestAsHandled`, we do NOT require it to still be in progress — returning an
             // already-released request to the queue (e.g. to honor a `forefront` reorder) must still
@@ -296,14 +288,14 @@ export class RequestQueueBackend extends BaseClient {
             }
             // Reclaiming resets the `orderNo` to a fresh timestamp, restoring the request to the queue
             // (at the front if `forefront`).
-            const requestModel = this._createInternalRequest(request, options.forefront);
-            this.requests.set(id, requestModel);
+            const requestModel = this.#createInternalRequest(request, options.forefront);
+            this.#requests.set(id, requestModel);
             // The request is no longer in progress for this client.
-            this.inProgressRequestIds.delete(id);
+            this.#inProgressRequestIds.delete(id);
             if (options.forefront) {
-                this.forefrontRequestIds.push(id);
+                this.#forefrontRequestIds.push(id);
             }
-            this.updateTimestamps(true);
+            this.#updateTimestamps(true);
             return {
                 requestId: id,
                 wasAlreadyHandled: false,
@@ -311,11 +303,11 @@ export class RequestQueueBackend extends BaseClient {
             };
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     async isEmpty() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         // "Empty" means there is nothing left to fetch right now — i.e. the next `fetchNextRequest`
         // would return `null`. Requests that are currently in progress are intentionally NOT counted
         // here: they are not fetchable, so the queue is empty from a consumer's point of view. Whether
@@ -324,17 +316,17 @@ export class RequestQueueBackend extends BaseClient {
         //
         // `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state mutex to avoid
         // racing a concurrent mutator (e.g. `addBatchOfRequests`) at its `await` points.
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
-            const { items } = await this.listPendingHead(1);
+            const { items } = await this.#listPendingHead(1);
             return items.length === 0;
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     async isFinished() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         // The queue is finished only when there is nothing left to fetch AND nothing currently in
         // progress. Counting in-progress requests is what allows a crawler with concurrency to keep
         // waiting while it still holds the last requests, instead of finishing prematurely.
@@ -344,13 +336,13 @@ export class RequestQueueBackend extends BaseClient {
         //
         // `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state mutex to avoid
         // racing a concurrent mutator (e.g. `addBatchOfRequests`) at its `await` points.
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
-            const { items, hasInProgressRequests } = await this.listPendingHead(1, true);
+            const { items, hasInProgressRequests } = await this.#listPendingHead(1, true);
             return items.length === 0 && !hasInProgressRequests;
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     /**
@@ -359,16 +351,16 @@ export class RequestQueueBackend extends BaseClient {
      * nothing is marked in progress.
      */
     async listItems() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         // `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state
         // mutex to avoid racing a concurrent mutator at its `await` points.
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
-            const { items } = await this.listPendingHead(Number.POSITIVE_INFINITY);
-            return items.map((request) => this._jsonToRequest(request.json));
+            const { items } = await this.#listPendingHead(Number.POSITIVE_INFINITY);
+            return items.map((request) => this.#jsonToRequest(request.json));
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     toRequestQueueInfo() {
@@ -380,23 +372,23 @@ export class RequestQueueBackend extends BaseClient {
             modifiedAt: this.modifiedAt,
             name: this.name,
             pendingRequestCount: this.pendingRequestCount,
-            totalRequestCount: this.requests.size,
+            totalRequestCount: this.#requests.size,
         };
     }
-    updateTimestamps(hasBeenModified) {
+    #updateTimestamps(hasBeenModified) {
         this.accessedAt = new Date();
         if (hasBeenModified) {
             this.modifiedAt = new Date();
         }
     }
-    _jsonToRequest(requestJson) {
+    #jsonToRequest(requestJson) {
         if (!requestJson)
             return undefined;
         const request = JSON.parse(requestJson);
         return purgeNullsFromObject(request);
     }
-    _createInternalRequest(request, forefront) {
-        const orderNo = this._calculateOrderNo(request, forefront);
+    #createInternalRequest(request, forefront) {
+        const orderNo = this.#calculateOrderNo(request, forefront);
         const id = uniqueKeyToRequestId(request.uniqueKey);
         if (request.id && request.id !== id) {
             throw new Error('Request ID does not match its uniqueKey.');
@@ -412,7 +404,7 @@ export class RequestQueueBackend extends BaseClient {
             url: request.url,
         };
     }
-    _calculateOrderNo(request, forefront) {
+    #calculateOrderNo(request, forefront) {
         if (request.handledAt)
             return null;
         const timestamp = Date.now();

package/owned_or_injected.d.ts

@@ -10,9 +10,7 @@
  * @internal
  */
 export declare class OwnedOrInjected<Injected, Owned extends Injected = Injected> {
-    private _value;
-    private readonly _owned;
-    private _present;
+    #private;
     private constructor();
     /**
      * Resolves a collaborator from an optionally-injected instance. `Injected` is the public/borrowed type exposed via

package/owned_or_injected.js

@@ -10,13 +10,13 @@
  * @internal
  */
 export class OwnedOrInjected {
-    _value;
-    _owned;
-    _present;
+    #value;
+    #owned;
+    #present;
     constructor(value, owned, present) {
-        this._value = value;
-        this._owned = owned;
-        this._present = present;
+        this.#value = value;
+        this.#owned = owned;
+        this.#present = present;
     }
     /**
      * Resolves a collaborator from an optionally-injected instance. `Injected` is the public/borrowed type exposed via
@@ -42,24 +42,24 @@ export class OwnedOrInjected {
      * crawler-built defaults, `false` for user-injected instances.
      */
     get isOwned() {
-        return this._owned;
+        return this.#owned;
     }
     /**
      * Whether a value is currently available. `false` for an owned slot whose default hasn't been built yet
      * (e.g. a lazily-opened request queue before its first use).
      */
     get isPresent() {
-        return this._present;
+        return this.#present;
     }
     /**
      * The resolved instance, typed as the public `Injected` type. Throws if the value is not present yet — callers that
      * expect a lazily-filled owned slot should read {@link OwnedOrInjected.maybeValue|`maybeValue`} instead.
      */
     get value() {
-        if (!this._present) {
+        if (!this.#present) {
             throw new Error('OwnedOrInjected value is not initialized yet');
         }
-        return this._value;
+        return this.#value;
     }
     /**
      * The resolved instance, or `undefined` when a lazily-filled owned slot hasn't been built yet. The non-throwing
@@ -67,7 +67,7 @@ export class OwnedOrInjected {
      * a possibly-empty slot without the `isPresent ? value : …` dance.
      */
     get maybeValue() {
-        return this._present ? this._value : undefined;
+        return this.#present ? this.#value : undefined;
     }
     /**
      * Fills the (owned) slot with the crawler-built default, returning it for convenience. Only valid on an owned,
@@ -75,14 +75,14 @@ export class OwnedOrInjected {
      * would silently orphan the previous instance's lifecycle).
      */
     set(value) {
-        if (!this._owned) {
+        if (!this.#owned) {
             throw new Error('Cannot set() a borrowed OwnedOrInjected value');
         }
-        if (this._present) {
+        if (this.#present) {
             throw new Error('OwnedOrInjected value is already initialized');
         }
-        this._value = value;
-        this._present = true;
+        this.#value = value;
+        this.#present = true;
         return value;
     }
     /**
@@ -90,9 +90,9 @@ export class OwnedOrInjected {
      * owns a present instance — a no-op for a borrowed instance or an owned-but-not-yet-built slot.
      */
     async ifOwned(fn) {
-        if (!this._owned || !this._present) {
+        if (!this.#owned || !this.#present) {
             return undefined;
         }
-        return fn(this._value);
+        return fn(this.#value);
     }
 }

package/package.json

@@ -1,13 +1,14 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.99",
+    "version": "4.0.0-rc.1",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
         ".": "./index.js",
+        "./internal": "./internal.js",
         "./package.json": "./package.json"
     },
     "keywords": [
@@ -47,30 +48,26 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/consts": "^2.41.0",
-        "@apify/datastructures": "^2.0.3",
-        "@apify/log": "^2.5.18",
-        "@apify/pseudo_url": "^2.0.59",
-        "@apify/timeout": "^0.4.4",
-        "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.99",
-        "@crawlee/types": "4.0.0-beta.99",
-        "@crawlee/utils": "4.0.0-beta.99",
+        "@apify/consts": "^3.0.1",
+        "@apify/datastructures": "^3.0.1",
+        "@apify/log": "^3.0.1",
+        "@apify/timeout": "^1.0.1",
+        "@apify/utilities": "^3.0.1",
+        "@crawlee/fs-storage": "4.0.0-rc.1",
+        "@crawlee/http-client": "4.0.0-rc.1",
+        "@crawlee/types": "4.0.0-rc.1",
+        "@crawlee/utils": "4.0.0-rc.1",
         "@sapphire/async-queue": "^1.5.5",
-        "@sapphire/shapeshift": "^4.0.0",
+        "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
-        "content-type": "^1.0.5",
+        "content-type": "^3.1.1",
         "csv-stringify": "^6.5.2",
         "json5": "^2.2.3",
         "mime-types": "^3.0.1",
-        "minimatch": "^10.0.1",
-        "ow": "^2.0.0",
-        "stream-json": "^1.9.1",
-        "tldts": "^7.0.6",
-        "tough-cookie": "^6.0.0",
+        "stream-json": "^3.7.0",
         "tslib": "^2.8.1",
         "type-fest": "^4.41.0",
-        "zod": "^3.24.0 || ^4.0.0"
+        "zod": "^4.5.4"
     },
     "lerna": {
         "command": {
@@ -79,5 +76,5 @@
             }
         }
     },
-    "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }