@crawlee/core 4.0.0-beta.104 → 4.0.0-beta.106

package/memory-storage/resource-clients/request-queue.js

@@ -38,8 +38,8 @@ export class RequestQueueBackend extends BaseClient {
      * `addBatchOfRequests` pushes to it). Held by every mutating method as well as by `isEmpty`/
      * `isFinished`, whose head scan also prunes `forefrontRequestIds`.
      */
-    queueStateMutex = new AsyncQueue();
-    forefrontRequestIds = [];
+    #queueStateMutex = new AsyncQueue();
+    #forefrontRequestIds = [];
     /**
      * IDs of requests currently fetched but not yet handled or reclaimed. A request in this set is
      * "in progress" and will not be handed out again by {@link fetchNextRequest}.
@@ -48,8 +48,9 @@ export class RequestQueueBackend extends BaseClient {
      * process and is never shared with another consumer, so there is no need for an expiring,
      * cross-process-visible lock — tracking in-progress requests in this set is enough.
      */
-    inProgressRequestIds = new Set();
-    requests = new Map();
+    #inProgressRequestIds = new Set();
+    #requests = new Map();
+    // kept as TS-private: storage-backend tests read this field at runtime
     storageBackend;
     constructor(options) {
         super(options.id ?? randomUUID());
@@ -65,7 +66,7 @@ export class RequestQueueBackend extends BaseClient {
         // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so a concurrent
         // operation cannot observe half-cleared state — e.g. a forefront id whose request has already been
         // removed, which `listPendingHead` would then dereference as `undefined`.
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
             const storeIndex = this.storageBackend.requestQueueBackendCache.findIndex((queue) => queue.id === this.id);
             if (storeIndex !== -1) {
@@ -74,37 +75,37 @@ export class RequestQueueBackend extends BaseClient {
                 // Clear all in-memory state, consistent with `purge`. Clearing `requests` alone would
                 // leave dangling ids in `forefrontRequestIds`/`inProgressRequestIds`, which a later head
                 // scan would resolve to a missing request and dereference.
-                oldBackend.requests.clear();
-                oldBackend.forefrontRequestIds = [];
-                oldBackend.inProgressRequestIds.clear();
+                oldBackend.#requests.clear();
+                oldBackend.#forefrontRequestIds = [];
+                oldBackend.#inProgressRequestIds.clear();
             }
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     async purge() {
         // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so a concurrent
         // operation cannot observe or repopulate half-cleared state across the `await` below.
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
             // Clear all in-memory state
-            this.requests.clear();
-            this.forefrontRequestIds = [];
-            this.inProgressRequestIds.clear();
+            this.#requests.clear();
+            this.#forefrontRequestIds = [];
+            this.#inProgressRequestIds.clear();
             this.handledRequestCount = 0;
             this.pendingRequestCount = 0;
             this.updateTimestamps(true);
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     *requestKeyIterator() {
-        for (let i = this.forefrontRequestIds.length - 1; i >= 0; i--) {
-            yield this.forefrontRequestIds[i];
+        for (let i = this.#forefrontRequestIds.length - 1; i >= 0; i--) {
+            yield this.#forefrontRequestIds[i];
         }
-        for (const key of this.requests.keys()) {
+        for (const key of this.#requests.keys()) {
             yield key;
         }
     }
@@ -140,17 +141,17 @@ export class RequestQueueBackend extends BaseClient {
                 continue;
             }
             seenRequestIds.add(requestId);
-            const request = this.requests.get(requestId);
+            const request = this.#requests.get(requestId);
             // Permanently-handled requests (`orderNo === null`) are in a terminal state and can be skipped.
             if (request.orderNo === null) {
-                if (this.forefrontRequestIds.includes(requestId)) {
+                if (this.#forefrontRequestIds.includes(requestId)) {
                     handledForefrontIds.add(requestId);
                 }
                 continue;
             }
             // In progress (fetched but not yet handled or reclaimed) — skip it, but remember that the
             // queue is not truly empty.
-            if (this.inProgressRequestIds.has(requestId)) {
+            if (this.#inProgressRequestIds.has(requestId)) {
                 hasInProgressRequests = true;
                 continue;
             }
@@ -158,7 +159,7 @@ export class RequestQueueBackend extends BaseClient {
                 items.push(request);
             }
         }
-        this.forefrontRequestIds = this.forefrontRequestIds.filter((id) => !handledForefrontIds.has(id));
+        this.#forefrontRequestIds = this.#forefrontRequestIds.filter((id) => !handledForefrontIds.has(id));
         return {
             items: items.sort((a, b) => a.orderNo - b.orderNo),
             hasInProgressRequests: detectInProgressRequests ? hasInProgressRequests : undefined,
@@ -166,7 +167,7 @@ export class RequestQueueBackend extends BaseClient {
     }
     async fetchNextRequest() {
         this.updateTimestamps(false);
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
             const { items: [head], } = await this.listPendingHead(1);
             if (!head) {
@@ -174,11 +175,11 @@ export class RequestQueueBackend extends BaseClient {
             }
             // Mark the request as in progress so it is not handed out again until it is handled or
             // reclaimed. The request keeps its `orderNo` (and thus its forefront / normal ordering).
-            this.inProgressRequestIds.add(head.id);
-            return this._jsonToRequest(head.json) ?? undefined;
+            this.#inProgressRequestIds.add(head.id);
+            return this.jsonToRequest(head.json) ?? undefined;
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     async addBatchOfRequests(requests, options = {}) {
@@ -187,15 +188,15 @@ export class RequestQueueBackend extends BaseClient {
         // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so that the
         // shared `requests` map, `forefrontRequestIds` array and request counts are not corrupted by a
         // concurrent operation interleaving at one of the `await` points below.
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
             const result = {
                 processedRequests: [],
                 unprocessedRequests: [],
             };
             for (const model of requests) {
-                const requestModel = this._createInternalRequest(model, options.forefront);
-                const existingRequestWithId = this.requests.get(requestModel.id);
+                const requestModel = this.createInternalRequest(model, options.forefront);
+                const existingRequestWithId = this.#requests.get(requestModel.id);
                 if (existingRequestWithId) {
                     result.processedRequests.push({
                         requestId: existingRequestWithId.id,
@@ -205,7 +206,7 @@ export class RequestQueueBackend extends BaseClient {
                     });
                     continue;
                 }
-                this.requests.set(requestModel.id, requestModel);
+                this.#requests.set(requestModel.id, requestModel);
                 if (requestModel.orderNo) {
                     this.pendingRequestCount += 1;
                 }
@@ -213,7 +214,7 @@ export class RequestQueueBackend extends BaseClient {
                     this.handledRequestCount += 1;
                 }
                 if (options.forefront) {
-                    this.forefrontRequestIds.push(requestModel.id);
+                    this.#forefrontRequestIds.push(requestModel.id);
                 }
                 result.processedRequests.push({
                     requestId: requestModel.id,
@@ -228,15 +229,15 @@ export class RequestQueueBackend extends BaseClient {
             return result;
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     async getRequest(uniqueKey) {
         s.string().parse(uniqueKey);
         this.updateTimestamps(false);
         const id = uniqueKeyToRequestId(uniqueKey);
-        const json = this.requests.get(id)?.json;
-        return this._jsonToRequest(json);
+        const json = this.#requests.get(id)?.json;
+        return this.jsonToRequest(json);
     }
     async markRequestAsHandled(request) {
         requestShape.parse(request);
@@ -244,10 +245,10 @@ export class RequestQueueBackend extends BaseClient {
         // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
         // `requests` map, `inProgressRequestIds` set and request counts stay consistent across the
         // `await` points below.
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
             const id = uniqueKeyToRequestId(request.uniqueKey);
-            const existingRequest = this.requests.get(id);
+            const existingRequest = this.#requests.get(id);
             // The request must exist to be marked as handled. We intentionally do NOT require it to still
             // be in progress: marking an already-released request handled must still succeed, otherwise
             // the request could be handed out again and the queue would never finish.
@@ -257,10 +258,10 @@ export class RequestQueueBackend extends BaseClient {
             // A handled request has `orderNo === null`. Marking it again is an idempotent no-op.
             const wasAlreadyHandled = existingRequest.orderNo === null;
             const handledAt = request.handledAt ?? new Date().toISOString();
-            const requestModel = this._createInternalRequest({ ...request, handledAt }, false);
-            this.requests.set(id, requestModel);
+            const requestModel = this.createInternalRequest({ ...request, handledAt }, false);
+            this.#requests.set(id, requestModel);
             // The request is no longer in progress for this client.
-            this.inProgressRequestIds.delete(id);
+            this.#inProgressRequestIds.delete(id);
             if (!wasAlreadyHandled) {
                 this.pendingRequestCount -= 1;
                 this.handledRequestCount += 1;
@@ -273,7 +274,7 @@ export class RequestQueueBackend extends BaseClient {
             };
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     async reclaimRequest(request, options = {}) {
@@ -283,10 +284,10 @@ export class RequestQueueBackend extends BaseClient {
         // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
         // `requests` map, `forefrontRequestIds` array and `inProgressRequestIds` set stay consistent
         // across the `await` points below.
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
             const id = uniqueKeyToRequestId(request.uniqueKey);
-            const existingRequest = this.requests.get(id);
+            const existingRequest = this.#requests.get(id);
             // The request must exist and not already be handled to be reclaimed. As with
             // `markRequestAsHandled`, we do NOT require it to still be in progress — returning an
             // already-released request to the queue (e.g. to honor a `forefront` reorder) must still
@@ -296,12 +297,12 @@ export class RequestQueueBackend extends BaseClient {
             }
             // Reclaiming resets the `orderNo` to a fresh timestamp, restoring the request to the queue
             // (at the front if `forefront`).
-            const requestModel = this._createInternalRequest(request, options.forefront);
-            this.requests.set(id, requestModel);
+            const requestModel = this.createInternalRequest(request, options.forefront);
+            this.#requests.set(id, requestModel);
             // The request is no longer in progress for this client.
-            this.inProgressRequestIds.delete(id);
+            this.#inProgressRequestIds.delete(id);
             if (options.forefront) {
-                this.forefrontRequestIds.push(id);
+                this.#forefrontRequestIds.push(id);
             }
             this.updateTimestamps(true);
             return {
@@ -311,7 +312,7 @@ export class RequestQueueBackend extends BaseClient {
             };
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     async isEmpty() {
@@ -324,13 +325,13 @@ export class RequestQueueBackend extends BaseClient {
         //
         // `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state mutex to avoid
         // racing a concurrent mutator (e.g. `addBatchOfRequests`) at its `await` points.
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
             const { items } = await this.listPendingHead(1);
             return items.length === 0;
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     async isFinished() {
@@ -344,13 +345,13 @@ export class RequestQueueBackend extends BaseClient {
         //
         // `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state mutex to avoid
         // racing a concurrent mutator (e.g. `addBatchOfRequests`) at its `await` points.
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
             const { items, hasInProgressRequests } = await this.listPendingHead(1, true);
             return items.length === 0 && !hasInProgressRequests;
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     /**
@@ -362,13 +363,13 @@ export class RequestQueueBackend extends BaseClient {
         this.updateTimestamps(false);
         // `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state
         // mutex to avoid racing a concurrent mutator at its `await` points.
-        await this.queueStateMutex.wait();
+        await this.#queueStateMutex.wait();
         try {
             const { items } = await this.listPendingHead(Number.POSITIVE_INFINITY);
-            return items.map((request) => this._jsonToRequest(request.json));
+            return items.map((request) => this.jsonToRequest(request.json));
         }
         finally {
-            this.queueStateMutex.shift();
+            this.#queueStateMutex.shift();
         }
     }
     toRequestQueueInfo() {
@@ -380,7 +381,7 @@ export class RequestQueueBackend extends BaseClient {
             modifiedAt: this.modifiedAt,
             name: this.name,
             pendingRequestCount: this.pendingRequestCount,
-            totalRequestCount: this.requests.size,
+            totalRequestCount: this.#requests.size,
         };
     }
     updateTimestamps(hasBeenModified) {
@@ -389,14 +390,14 @@ export class RequestQueueBackend extends BaseClient {
             this.modifiedAt = new Date();
         }
     }
-    _jsonToRequest(requestJson) {
+    jsonToRequest(requestJson) {
         if (!requestJson)
             return undefined;
         const request = JSON.parse(requestJson);
         return purgeNullsFromObject(request);
     }
-    _createInternalRequest(request, forefront) {
-        const orderNo = this._calculateOrderNo(request, forefront);
+    createInternalRequest(request, forefront) {
+        const orderNo = this.calculateOrderNo(request, forefront);
         const id = uniqueKeyToRequestId(request.uniqueKey);
         if (request.id && request.id !== id) {
             throw new Error('Request ID does not match its uniqueKey.');
@@ -412,7 +413,7 @@ export class RequestQueueBackend extends BaseClient {
             url: request.url,
         };
     }
-    _calculateOrderNo(request, forefront) {
+    calculateOrderNo(request, forefront) {
         if (request.handledAt)
             return null;
         const timestamp = Date.now();

package/owned_or_injected.d.ts

@@ -10,9 +10,7 @@
  * @internal
  */
 export declare class OwnedOrInjected<Injected, Owned extends Injected = Injected> {
-    private _value;
-    private readonly _owned;
-    private _present;
+    #private;
     private constructor();
     /**
      * Resolves a collaborator from an optionally-injected instance. `Injected` is the public/borrowed type exposed via

package/owned_or_injected.js

@@ -10,13 +10,13 @@
  * @internal
  */
 export class OwnedOrInjected {
-    _value;
-    _owned;
-    _present;
+    #value;
+    #owned;
+    #present;
     constructor(value, owned, present) {
-        this._value = value;
-        this._owned = owned;
-        this._present = present;
+        this.#value = value;
+        this.#owned = owned;
+        this.#present = present;
     }
     /**
      * Resolves a collaborator from an optionally-injected instance. `Injected` is the public/borrowed type exposed via
@@ -42,24 +42,24 @@ export class OwnedOrInjected {
      * crawler-built defaults, `false` for user-injected instances.
      */
     get isOwned() {
-        return this._owned;
+        return this.#owned;
     }
     /**
      * Whether a value is currently available. `false` for an owned slot whose default hasn't been built yet
      * (e.g. a lazily-opened request queue before its first use).
      */
     get isPresent() {
-        return this._present;
+        return this.#present;
     }
     /**
      * The resolved instance, typed as the public `Injected` type. Throws if the value is not present yet — callers that
      * expect a lazily-filled owned slot should read {@link OwnedOrInjected.maybeValue|`maybeValue`} instead.
      */
     get value() {
-        if (!this._present) {
+        if (!this.#present) {
             throw new Error('OwnedOrInjected value is not initialized yet');
         }
-        return this._value;
+        return this.#value;
     }
     /**
      * The resolved instance, or `undefined` when a lazily-filled owned slot hasn't been built yet. The non-throwing
@@ -67,7 +67,7 @@ export class OwnedOrInjected {
      * a possibly-empty slot without the `isPresent ? value : …` dance.
      */
     get maybeValue() {
-        return this._present ? this._value : undefined;
+        return this.#present ? this.#value : undefined;
     }
     /**
      * Fills the (owned) slot with the crawler-built default, returning it for convenience. Only valid on an owned,
@@ -75,14 +75,14 @@ export class OwnedOrInjected {
      * would silently orphan the previous instance's lifecycle).
      */
     set(value) {
-        if (!this._owned) {
+        if (!this.#owned) {
             throw new Error('Cannot set() a borrowed OwnedOrInjected value');
         }
-        if (this._present) {
+        if (this.#present) {
             throw new Error('OwnedOrInjected value is already initialized');
         }
-        this._value = value;
-        this._present = true;
+        this.#value = value;
+        this.#present = true;
         return value;
     }
     /**
@@ -90,9 +90,9 @@ export class OwnedOrInjected {
      * owns a present instance — a no-op for a borrowed instance or an owned-but-not-yet-built slot.
      */
     async ifOwned(fn) {
-        if (!this._owned || !this._present) {
+        if (!this.#owned || !this.#present) {
             return undefined;
         }
-        return fn(this._value);
+        return fn(this.#value);
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.104",
+    "version": "4.0.0-beta.106",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,9 +52,9 @@
         "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.104",
-        "@crawlee/types": "4.0.0-beta.104",
-        "@crawlee/utils": "4.0.0-beta.104",
+        "@crawlee/fs-storage": "4.0.0-beta.106",
+        "@crawlee/types": "4.0.0-beta.106",
+        "@crawlee/utils": "4.0.0-beta.106",
         "@sapphire/async-queue": "^1.5.5",
         "@sapphire/shapeshift": "^4.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -78,5 +78,5 @@
             }
         }
     },
-    "gitHead": "1c761e689e56ac06b67fc4f1dc091f693e024d12"
+    "gitHead": "c622f1fc65e65221ea245817c58ecc0ffb4a5cb0"
 }

package/proxy_configuration.d.ts

@@ -70,10 +70,8 @@ export interface IProxyConfiguration {
  * @category Scaling
  */
 export declare class ProxyConfiguration implements IProxyConfiguration {
+    #private;
     readonly isManInTheMiddle = false;
-    private nextCustomUrlIndex;
-    private proxyUrls?;
-    private newUrlFunction?;
     /**
      * Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
      * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures

package/proxy_configuration.js

@@ -29,9 +29,9 @@ import ow from 'ow';
  */
 export class ProxyConfiguration {
     isManInTheMiddle = false;
-    nextCustomUrlIndex = 0;
-    proxyUrls;
-    newUrlFunction;
+    #nextCustomUrlIndex = 0;
+    #proxyUrls;
+    #newUrlFunction;
     /**
      * Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
      * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
@@ -67,8 +67,8 @@ export class ProxyConfiguration {
             this.throwCannotCombineCustomMethods();
         if (!proxyUrls && !newUrlFunction && validateRequired)
             this.throwNoOptionsProvided();
-        this.proxyUrls = proxyUrls;
-        this.newUrlFunction = newUrlFunction;
+        this.#proxyUrls = proxyUrls;
+        this.#newUrlFunction = newUrlFunction;
     }
     /**
      * This function creates a new {@link ProxyInfo} info object.
@@ -99,19 +99,19 @@ export class ProxyConfiguration {
      *  For example, `http://bob:password123@proxy.example.com:8000`
      */
     async newUrl(options) {
-        if (this.newUrlFunction) {
+        if (this.#newUrlFunction) {
             return (await this.callNewUrlFunction({ request: options?.request })) ?? undefined;
         }
         return this.handleProxyUrlsList() ?? undefined;
     }
     handleProxyUrlsList() {
-        return this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
+        return this.#proxyUrls[this.#nextCustomUrlIndex++ % this.#proxyUrls.length];
     }
     /**
      * Calls the custom newUrlFunction and checks format of its return value
      */
     async callNewUrlFunction(options) {
-        const proxyUrl = await this.newUrlFunction(options);
+        const proxyUrl = await this.#newUrlFunction(options);
         try {
             if (proxyUrl) {
                 new URL(proxyUrl); // eslint-disable-line no-new

package/recoverable_state.d.ts

@@ -59,16 +59,7 @@ export interface RecoverableStateOptions<TStateModel = Record<string, unknown>>
  * The class automatically hooks into the event system to persist state when needed.
  */
 export declare class RecoverableState<TStateModel = Record<string, unknown>> {
-    private readonly defaultState;
-    private state;
-    private readonly persistenceEnabled;
-    private readonly persistStateKey;
-    private readonly persistStateKvsName?;
-    private readonly persistStateKvsId?;
-    private keyValueStore;
-    private readonly log;
-    private readonly serialize;
-    private readonly deserialize;
+    #private;
     /**
      * Initialize a new recoverable state object.
      *