@crawlee/core 4.0.0-beta.211 → 4.0.0-beta.212

package/configuration.d.ts

@@ -129,23 +129,4 @@ export declare class Configuration {
      * Delegates to the global ServiceLocator, making it the single source of truth for service management.
      */
     static getGlobalConfiguration(): Configuration;
-    /**
-     * Resolves all field values once using the priority chain:
-     * constructor options > env vars > crawlee.json > schema defaults.
-     */
-    private static resolveAll;
-    /**
-     * Registers getters (and throwing setters) on the instance for each field.
-     */
-    private registerAccessors;
-    /**
-     * Reads the first defined env var value for a field definition.
-     * Empty strings are treated as unset, falling through to crawlee.json or schema defaults.
-     * (Crawlee v3 coerced `''` to `false`/`0`/`''` per type — v4 drops that for consistency.)
-     */
-    private static readEnvVar;
-    /**
-     * Loads config options from crawlee.json in the current working directory.
-     */
-    private static loadFileOptions;
 }

package/configuration.js

@@ -143,9 +143,9 @@ export class Configuration {
      */
     constructor(options = {}) {
         const fields = this.constructor.fields;
-        const fileOptions = Configuration.loadFileOptions();
-        this.#resolvedValues = Configuration.resolveAll(fields, options, fileOptions);
-        this.registerAccessors();
+        const fileOptions = Configuration.#loadFileOptions();
+        this.#resolvedValues = Configuration.#resolveAll(fields, options, fileOptions);
+        this.#registerAccessors();
         // Set the log level
         const logLevel = this.logLevel;
         if (logLevel != null) {
@@ -164,7 +164,7 @@ export class Configuration {
      * Resolves all field values once using the priority chain:
      * constructor options > env vars > crawlee.json > schema defaults.
      */
-    static resolveAll(fields, userOptions, fileOptions) {
+    static #resolveAll(fields, userOptions, fileOptions) {
         const values = {};
         for (const [key, fieldDef] of Object.entries(fields)) {
             // 1. Constructor options (highest priority)
@@ -173,7 +173,7 @@ export class Configuration {
                 continue;
             }
             // 2. Environment variables
-            const envValue = Configuration.readEnvVar(fieldDef);
+            const envValue = Configuration.#readEnvVar(fieldDef);
             if (envValue != null) {
                 values[key] = fieldDef.schema.parse(envValue);
                 continue;
@@ -192,7 +192,7 @@ export class Configuration {
     /**
      * Registers getters (and throwing setters) on the instance for each field.
      */
-    registerAccessors() {
+    #registerAccessors() {
         const fields = this.constructor.fields;
         const descriptors = {};
         for (const key of Object.keys(fields)) {
@@ -212,7 +212,7 @@ export class Configuration {
      * Empty strings are treated as unset, falling through to crawlee.json or schema defaults.
      * (Crawlee v3 coerced `''` to `false`/`0`/`''` per type — v4 drops that for consistency.)
      */
-    static readEnvVar(fieldDef) {
+    static #readEnvVar(fieldDef) {
         if (!fieldDef.envVar)
             return undefined;
         const envVars = Array.isArray(fieldDef.envVar) ? fieldDef.envVar : [fieldDef.envVar];
@@ -226,7 +226,7 @@ export class Configuration {
     /**
      * Loads config options from crawlee.json in the current working directory.
      */
-    static loadFileOptions() {
+    static #loadFileOptions() {
         try {
             const file = readFileSync(join(process.cwd(), 'crawlee.json'));
             return JSON.parse(file.toString());

package/events/local_event_manager.d.ts

@@ -29,11 +29,4 @@ export declare class LocalEventManager extends EventManager {
      * @internal
      */
     isContainerizedWrapper(): Promise<boolean>;
-    /**
-     * Creates a SystemInfo object based on local metrics.
-     */
-    private createSystemInfo;
-    private createCpuInfo;
-    private createMemoryInfo;
-    private getMemoryInfo;
 }

package/events/local_event_manager.js

@@ -44,7 +44,7 @@ export class LocalEventManager extends EventManager {
      * @internal
      */
     async emitSystemInfoEvent(intervalCallback) {
-        const info = await this.createSystemInfo({
+        const info = await this.#createSystemInfo({
             maxUsedCpuRatio: serviceLocator.getConfiguration().maxUsedCpuRatio,
         });
         this.events.emit(EventType.SYSTEM_INFO, info);
@@ -60,14 +60,14 @@ export class LocalEventManager extends EventManager {
     /**
      * Creates a SystemInfo object based on local metrics.
      */
-    async createSystemInfo(options) {
+    async #createSystemInfo(options) {
         return {
             createdAt: new Date(),
-            ...(await this.createCpuInfo(options)),
-            ...(await this.createMemoryInfo()),
+            ...(await this.#createCpuInfo(options)),
+            ...(await this.#createMemoryInfo()),
         };
     }
-    async createCpuInfo(options) {
+    async #createCpuInfo(options) {
         const { getCurrentCpuTicksV2 } = await import('../system-info/cpu-info.js');
         const usedCpuRatio = await getCurrentCpuTicksV2({
             containerized: await this.isContainerizedWrapper(),
@@ -78,9 +78,9 @@ export class LocalEventManager extends EventManager {
             isCpuOverloaded: usedCpuRatio > options.maxUsedCpuRatio,
         };
     }
-    async createMemoryInfo() {
+    async #createMemoryInfo() {
         try {
-            const memInfo = await this.getMemoryInfo();
+            const memInfo = await this.#getMemoryInfo();
             return {
                 memTotalBytes: memInfo.totalBytes,
                 memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
@@ -91,7 +91,7 @@ export class LocalEventManager extends EventManager {
             return {};
         }
     }
-    async getMemoryInfo() {
+    async #getMemoryInfo() {
         const { getMemoryInfo } = await import('../system-info/memory-info.js');
         return getMemoryInfo({
             containerized: await this.isContainerizedWrapper(),

package/memory-storage/resource-clients/dataset.d.ts

@@ -25,16 +25,11 @@ export declare class DatasetBackend<Data extends Dictionary = Dictionary> extend
     accessedAt: Date;
     modifiedAt: Date;
     itemCount: number;
-    private readonly storageBackend;
     constructor(options: DatasetBackendOptions);
     getMetadata(): Promise<storage.DatasetInfo>;
     drop(): Promise<void>;
     purge(): Promise<void>;
     getData(options?: storage.DatasetBackendListOptions): Promise<storage.PaginatedList<Data>>;
-    private getDataPage;
     pushData(items: Data[]): Promise<void>;
     toDatasetInfo(): storage.DatasetInfo;
-    private generateLocalEntryName;
-    private getStartAndEndIndexes;
-    private updateTimestamps;
 }

package/memory-storage/resource-clients/dataset.js

@@ -23,20 +23,19 @@ export class DatasetBackend extends BaseClient {
     modifiedAt = new Date();
     itemCount = 0;
     #datasetEntries = new Map();
-    // kept as TS-private: storage-backend tests read this field at runtime
-    storageBackend;
+    #storageBackend;
     constructor(options) {
         super(options.id ?? randomUUID());
         this.name = options.name;
         this.cacheKey = options.cacheKey ?? this.name ?? this.id;
-        this.storageBackend = options.storageBackend;
+        this.#storageBackend = options.storageBackend;
     }
     async getMetadata() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         return this.toDatasetInfo();
     }
     async drop() {
-        if (this.storageBackend.evictBackend('Dataset', this.id)) {
+        if (this.#storageBackend.evictBackend('Dataset', this.id)) {
             this.itemCount = 0;
             this.#datasetEntries.clear();
         }
@@ -44,25 +43,25 @@ export class DatasetBackend extends BaseClient {
     async purge() {
         this.itemCount = 0;
         this.#datasetEntries.clear();
-        this.updateTimestamps(true);
+        this.#updateTimestamps(true);
     }
     getData(options = {}) {
         const { desc, limit, offset } = parseArgument(options, schemas.datasetListItemsOptions);
-        return this.getDataPage({
+        return this.#getDataPage({
             desc,
             offset: offset ?? 0,
             limit: Math.min(limit ?? LIST_ITEMS_LIMIT, LIST_ITEMS_LIMIT),
         });
     }
-    async getDataPage(options = {}) {
+    async #getDataPage(options = {}) {
         const { limit = LIST_ITEMS_LIMIT, offset = 0, desc } = options;
-        const [start, end] = this.getStartAndEndIndexes(desc ? Math.max(this.itemCount - offset - limit, 0) : offset, limit);
+        const [start, end] = this.#getStartAndEndIndexes(desc ? Math.max(this.itemCount - offset - limit, 0) : offset, limit);
         const items = [];
         for (let idx = start; idx < end; idx++) {
-            const entryNumber = this.generateLocalEntryName(idx);
+            const entryNumber = this.#generateLocalEntryName(idx);
             items.push(this.#datasetEntries.get(entryNumber));
         }
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         return {
             count: items.length,
             desc: desc ?? false,
@@ -74,10 +73,10 @@ export class DatasetBackend extends BaseClient {
     }
     async pushData(items) {
         for (const entry of items) {
-            const idx = this.generateLocalEntryName(++this.itemCount);
+            const idx = this.#generateLocalEntryName(++this.itemCount);
             this.#datasetEntries.set(idx, JSON.parse(JSON.stringify(entry)));
         }
-        this.updateTimestamps(true);
+        this.#updateTimestamps(true);
     }
     toDatasetInfo() {
         return {
@@ -89,15 +88,15 @@ export class DatasetBackend extends BaseClient {
             name: this.name,
         };
     }
-    generateLocalEntryName(idx) {
+    #generateLocalEntryName(idx) {
         return idx.toString().padStart(LOCAL_ENTRY_NAME_DIGITS, '0');
     }
-    getStartAndEndIndexes(offset, limit = this.itemCount) {
+    #getStartAndEndIndexes(offset, limit = this.itemCount) {
         const start = offset + 1;
         const end = Math.min(offset + limit, this.itemCount) + 1;
         return [start, end];
     }
-    updateTimestamps(hasBeenModified) {
+    #updateTimestamps(hasBeenModified) {
         this.accessedAt = new Date();
         if (hasBeenModified) {
             this.modifiedAt = new Date();

package/memory-storage/resource-clients/key-value-store.d.ts

@@ -29,7 +29,6 @@ export declare class KeyValueStoreBackend extends BaseClient implements storage.
     createdAt: Date;
     accessedAt: Date;
     modifiedAt: Date;
-    private readonly storageBackend;
     constructor(options: KeyValueStoreBackendOptions);
     getMetadata(): Promise<storage.KeyValueStoreInfo>;
     drop(): Promise<void>;
@@ -52,5 +51,4 @@ export declare class KeyValueStoreBackend extends BaseClient implements storage.
     setValue(record: storage.KeyValueStoreInputRecord): Promise<void>;
     deleteValue(key: string): Promise<void>;
     toKeyValueStoreInfo(): storage.KeyValueStoreInfo;
-    private updateTimestamps;
 }

package/memory-storage/resource-clients/key-value-store.js

@@ -31,26 +31,25 @@ export class KeyValueStoreBackend extends BaseClient {
     accessedAt = new Date();
     modifiedAt = new Date();
     #keyValueEntries = new Map();
-    // kept as TS-private: storage-backend tests read this field at runtime
-    storageBackend;
+    #storageBackend;
     constructor(options) {
         super(options.id ?? randomUUID());
         this.name = options.name;
         this.cacheKey = options.cacheKey ?? this.name ?? this.id;
-        this.storageBackend = options.storageBackend;
+        this.#storageBackend = options.storageBackend;
     }
     async getMetadata() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         return this.toKeyValueStoreInfo();
     }
     async drop() {
-        if (this.storageBackend.evictBackend('KeyValueStore', this.id)) {
+        if (this.#storageBackend.evictBackend('KeyValueStore', this.id)) {
             this.#keyValueEntries.clear();
         }
     }
     async purge() {
         this.#keyValueEntries.clear();
-        this.updateTimestamps(true);
+        this.#updateTimestamps(true);
     }
     async listKeys(options = {}) {
         const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions);
@@ -77,7 +76,7 @@ export class KeyValueStoreBackend extends BaseClient {
         const isTruncated = limit !== undefined && filteredItems.length > limit;
         const pageItems = isTruncated ? filteredItems.slice(0, limit) : filteredItems;
         const nextExclusiveStartKey = isTruncated ? pageItems[pageItems.length - 1].key : undefined;
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         return {
             items: pageItems,
             count: pageItems.length,
@@ -121,7 +120,7 @@ export class KeyValueStoreBackend extends BaseClient {
             // frontend treats it as "no content type" rather than a bogus value.
             contentType: entry.contentType ?? (mime.contentType(entry.extension) || undefined),
         };
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         return record;
     }
     async setValue(record) {
@@ -153,13 +152,13 @@ export class KeyValueStoreBackend extends BaseClient {
             contentType,
         };
         this.#keyValueEntries.set(key, normalizedRecord);
-        this.updateTimestamps(true);
+        this.#updateTimestamps(true);
     }
     async deleteValue(key) {
         parseArgument(key, keySchema);
         if (this.#keyValueEntries.has(key)) {
             this.#keyValueEntries.delete(key);
-            this.updateTimestamps(true);
+            this.#updateTimestamps(true);
         }
     }
     toKeyValueStoreInfo() {
@@ -171,7 +170,7 @@ export class KeyValueStoreBackend extends BaseClient {
             modifiedAt: this.modifiedAt,
         };
     }
-    updateTimestamps(hasBeenModified) {
+    #updateTimestamps(hasBeenModified) {
         this.accessedAt = new Date();
         if (hasBeenModified) {
             this.modifiedAt = new Date();

package/memory-storage/resource-clients/request-queue.d.ts

@@ -34,28 +34,10 @@ export declare class RequestQueueBackend extends BaseClient implements storage.R
     modifiedAt: Date;
     handledRequestCount: number;
     pendingRequestCount: number;
-    private readonly storageBackend;
     constructor(options: RequestQueueBackendOptions);
     getMetadata(): Promise<storage.RequestQueueInfo>;
     drop(): Promise<void>;
     purge(): Promise<void>;
-    private requestKeyIterator;
-    /**
-     * Scans the queue and returns the pending head — requests that are neither handled nor currently
-     * in progress — ordered by `orderNo`, deduplicated.
-     *
-     * When `detectInProgressRequests` is set, the result also carries an `hasInProgressRequests` flag
-     * telling whether any unhandled-but-in-progress request was skipped along the way. It lets
-     * {@link isFinished} distinguish "no work left at all" from "work remains, but it is currently being
-     * processed". Without it, a consumer with concurrency could consider the queue finished and shut the
-     * crawler down while it is still handling the last requests.
-     *
-     * Computing the flag is expensive: because an in-progress request may sit anywhere in the queue, it
-     * forces a scan of every pending entry even when only `limit` items are wanted. Callers that only
-     * need the head (e.g. {@link fetchNextRequest}, {@link isEmpty}) leave it off so the scan can stop as
-     * soon as the page is filled, keeping those calls O(head) instead of O(pending).
-     */
-    private listPendingHead;
     fetchNextRequest(): Promise<storage.UpdateRequestSchema | undefined>;
     addBatchOfRequests(requests: storage.RequestSchema[], options?: storage.RequestQueueOperationOptions): Promise<storage.BatchAddRequestsResult>;
     getRequest(uniqueKey: string): Promise<storage.UpdateRequestSchema | undefined>;
@@ -70,8 +52,4 @@ export declare class RequestQueueBackend extends BaseClient implements storage.R
      */
     listItems(): Promise<storage.UpdateRequestSchema[]>;
     toRequestQueueInfo(): storage.RequestQueueInfo;
-    private updateTimestamps;
-    private jsonToRequest;
-    private createInternalRequest;
-    private calculateOrderNo;
 }

package/memory-storage/resource-clients/request-queue.js

@@ -43,16 +43,15 @@ export class RequestQueueBackend extends BaseClient {
      * ever walk the unhandled tail instead of every request the queue has ever seen.
      */
     #pendingRequestIds = new Set();
-    // kept as TS-private: storage-backend tests read this field at runtime
-    storageBackend;
+    #storageBackend;
     constructor(options) {
         super(options.id ?? randomUUID());
         this.name = options.name;
         this.cacheKey = options.cacheKey ?? this.name ?? this.id;
-        this.storageBackend = options.storageBackend;
+        this.#storageBackend = options.storageBackend;
     }
     async getMetadata() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         return this.toRequestQueueInfo();
     }
     async drop() {
@@ -61,7 +60,7 @@ export class RequestQueueBackend extends BaseClient {
         // removed, which `listPendingHead` would then dereference as `undefined`.
         await this.#queueStateMutex.wait();
         try {
-            if (this.storageBackend.evictBackend('RequestQueue', this.id)) {
+            if (this.#storageBackend.evictBackend('RequestQueue', this.id)) {
                 this.pendingRequestCount = 0;
                 // Clear all in-memory state, consistent with `purge`. Clearing `requests` alone would
                 // leave dangling ids in `forefrontRequestIds`/`inProgressRequestIds`, which a later head
@@ -88,13 +87,13 @@ export class RequestQueueBackend extends BaseClient {
             this.#inProgressRequestIds.clear();
             this.handledRequestCount = 0;
             this.pendingRequestCount = 0;
-            this.updateTimestamps(true);
+            this.#updateTimestamps(true);
         }
         finally {
             this.#queueStateMutex.shift();
         }
     }
-    *requestKeyIterator() {
+    *#requestKeyIterator() {
         for (let i = this.#forefrontRequestIds.length - 1; i >= 0; i--) {
             yield this.#forefrontRequestIds[i];
         }
@@ -115,14 +114,14 @@ export class RequestQueueBackend extends BaseClient {
      * need the head (e.g. {@link fetchNextRequest}, {@link isEmpty}) leave it off so the scan can stop as
      * soon as the page is filled, keeping those calls O(head) instead of O(pending).
      */
-    async listPendingHead(limit, detectInProgressRequests = false) {
+    async #listPendingHead(limit, detectInProgressRequests = false) {
         const items = [];
         let hasInProgressRequests = false;
         // Tracks processed request IDs to avoid duplicates (request in both `forefrontRequestIds` and `requests`).
         const seenRequestIds = new Set();
         // Tracks handled request IDs from `forefrontRequestIds` to be removed.
         const handledForefrontIds = new Set();
-        for (const requestId of this.requestKeyIterator()) {
+        for (const requestId of this.#requestKeyIterator()) {
             // Once the requested page is filled we can stop — unless the caller asked us to detect
             // in-progress requests and we have not yet seen one, in which case we must keep scanning.
             if (items.length >= limit && (!detectInProgressRequests || hasInProgressRequests)) {
@@ -156,17 +155,17 @@ export class RequestQueueBackend extends BaseClient {
         };
     }
     async fetchNextRequest() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         await this.#queueStateMutex.wait();
         try {
-            const { items: [head], } = await this.listPendingHead(1);
+            const { items: [head], } = await this.#listPendingHead(1);
             if (!head) {
                 return undefined;
             }
             // Mark the request as in progress so it is not handed out again until it is handled or
             // reclaimed. The request keeps its `orderNo` (and thus its forefront / normal ordering).
             this.#inProgressRequestIds.add(head.id);
-            return this.jsonToRequest(head.json) ?? undefined;
+            return this.#jsonToRequest(head.json) ?? undefined;
         }
         finally {
             this.#queueStateMutex.shift();
@@ -185,7 +184,7 @@ export class RequestQueueBackend extends BaseClient {
                 unprocessedRequests: [],
             };
             for (const model of requests) {
-                const requestModel = this.createInternalRequest(model, options.forefront);
+                const requestModel = this.#createInternalRequest(model, options.forefront);
                 const existingRequestWithId = this.#requests.get(requestModel.id);
                 if (existingRequestWithId) {
                     result.processedRequests.push({
@@ -216,7 +215,7 @@ export class RequestQueueBackend extends BaseClient {
                     wasAlreadyPresent: false,
                 });
             }
-            this.updateTimestamps(true);
+            this.#updateTimestamps(true);
             return result;
         }
         finally {
@@ -225,14 +224,14 @@ export class RequestQueueBackend extends BaseClient {
     }
     async getRequest(uniqueKey) {
         parseArgument(uniqueKey, uniqueKeySchema);
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         const id = uniqueKeyToRequestId(uniqueKey);
         const json = this.#requests.get(id)?.json;
-        return this.jsonToRequest(json);
+        return this.#jsonToRequest(json);
     }
     async markRequestAsHandled(request) {
         parseArgument(request, schemas.storageRequest);
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
         // `requests` map, `inProgressRequestIds` set and request counts stay consistent across the
         // `await` points below.
@@ -249,7 +248,7 @@ export class RequestQueueBackend extends BaseClient {
             // A handled request has `orderNo === null`. Marking it again is an idempotent no-op.
             const wasAlreadyHandled = existingRequest.orderNo === null;
             const handledAt = request.handledAt ?? new Date().toISOString();
-            const requestModel = this.createInternalRequest({ ...request, handledAt }, false);
+            const requestModel = this.#createInternalRequest({ ...request, handledAt }, false);
             this.#requests.set(id, requestModel);
             this.#pendingRequestIds.delete(id);
             // The request is no longer in progress for this client.
@@ -258,7 +257,7 @@ export class RequestQueueBackend extends BaseClient {
                 this.pendingRequestCount -= 1;
                 this.handledRequestCount += 1;
             }
-            this.updateTimestamps(true);
+            this.#updateTimestamps(true);
             return {
                 requestId: id,
                 wasAlreadyHandled,
@@ -272,7 +271,7 @@ export class RequestQueueBackend extends BaseClient {
     async reclaimRequest(request, options = {}) {
         parseArgument(request, schemas.storageRequest);
         parseArgument(options, schemas.requestQueueOperationOptions);
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         // Serialize against other mutators (and the head scans in `isEmpty`/`isFinished`) so the shared
         // `requests` map, `forefrontRequestIds` array and `inProgressRequestIds` set stay consistent
         // across the `await` points below.
@@ -289,14 +288,14 @@ export class RequestQueueBackend extends BaseClient {
             }
             // Reclaiming resets the `orderNo` to a fresh timestamp, restoring the request to the queue
             // (at the front if `forefront`).
-            const requestModel = this.createInternalRequest(request, options.forefront);
+            const requestModel = this.#createInternalRequest(request, options.forefront);
             this.#requests.set(id, requestModel);
             // The request is no longer in progress for this client.
             this.#inProgressRequestIds.delete(id);
             if (options.forefront) {
                 this.#forefrontRequestIds.push(id);
             }
-            this.updateTimestamps(true);
+            this.#updateTimestamps(true);
             return {
                 requestId: id,
                 wasAlreadyHandled: false,
@@ -308,7 +307,7 @@ export class RequestQueueBackend extends BaseClient {
         }
     }
     async isEmpty() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         // "Empty" means there is nothing left to fetch right now — i.e. the next `fetchNextRequest`
         // would return `null`. Requests that are currently in progress are intentionally NOT counted
         // here: they are not fetchable, so the queue is empty from a consumer's point of view. Whether
@@ -319,7 +318,7 @@ export class RequestQueueBackend extends BaseClient {
         // racing a concurrent mutator (e.g. `addBatchOfRequests`) at its `await` points.
         await this.#queueStateMutex.wait();
         try {
-            const { items } = await this.listPendingHead(1);
+            const { items } = await this.#listPendingHead(1);
             return items.length === 0;
         }
         finally {
@@ -327,7 +326,7 @@ export class RequestQueueBackend extends BaseClient {
         }
     }
     async isFinished() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         // The queue is finished only when there is nothing left to fetch AND nothing currently in
         // progress. Counting in-progress requests is what allows a crawler with concurrency to keep
         // waiting while it still holds the last requests, instead of finishing prematurely.
@@ -339,7 +338,7 @@ export class RequestQueueBackend extends BaseClient {
         // racing a concurrent mutator (e.g. `addBatchOfRequests`) at its `await` points.
         await this.#queueStateMutex.wait();
         try {
-            const { items, hasInProgressRequests } = await this.listPendingHead(1, true);
+            const { items, hasInProgressRequests } = await this.#listPendingHead(1, true);
             return items.length === 0 && !hasInProgressRequests;
         }
         finally {
@@ -352,13 +351,13 @@ export class RequestQueueBackend extends BaseClient {
      * nothing is marked in progress.
      */
     async listItems() {
-        this.updateTimestamps(false);
+        this.#updateTimestamps(false);
         // `listPendingHead` prunes `forefrontRequestIds` as it scans, so we must hold the queue-state
         // mutex to avoid racing a concurrent mutator at its `await` points.
         await this.#queueStateMutex.wait();
         try {
-            const { items } = await this.listPendingHead(Number.POSITIVE_INFINITY);
-            return items.map((request) => this.jsonToRequest(request.json));
+            const { items } = await this.#listPendingHead(Number.POSITIVE_INFINITY);
+            return items.map((request) => this.#jsonToRequest(request.json));
         }
         finally {
             this.#queueStateMutex.shift();
@@ -376,20 +375,20 @@ export class RequestQueueBackend extends BaseClient {
             totalRequestCount: this.#requests.size,
         };
     }
-    updateTimestamps(hasBeenModified) {
+    #updateTimestamps(hasBeenModified) {
         this.accessedAt = new Date();
         if (hasBeenModified) {
             this.modifiedAt = new Date();
         }
     }
-    jsonToRequest(requestJson) {
+    #jsonToRequest(requestJson) {
         if (!requestJson)
             return undefined;
         const request = JSON.parse(requestJson);
         return purgeNullsFromObject(request);
     }
-    createInternalRequest(request, forefront) {
-        const orderNo = this.calculateOrderNo(request, forefront);
+    #createInternalRequest(request, forefront) {
+        const orderNo = this.#calculateOrderNo(request, forefront);
         const id = uniqueKeyToRequestId(request.uniqueKey);
         if (request.id && request.id !== id) {
             throw new Error('Request ID does not match its uniqueKey.');
@@ -405,7 +404,7 @@ export class RequestQueueBackend extends BaseClient {
             url: request.url,
         };
     }
-    calculateOrderNo(request, forefront) {
+    #calculateOrderNo(request, forefront) {
         if (request.handledAt)
             return null;
         const timestamp = Date.now();

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.211",
+    "version": "4.0.0-beta.212",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -53,10 +53,10 @@
         "@apify/log": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/fs-storage": "4.0.0-beta.211",
-        "@crawlee/http-client": "4.0.0-beta.211",
-        "@crawlee/types": "4.0.0-beta.211",
-        "@crawlee/utils": "4.0.0-beta.211",
+        "@crawlee/fs-storage": "4.0.0-beta.212",
+        "@crawlee/http-client": "4.0.0-beta.212",
+        "@crawlee/types": "4.0.0-beta.212",
+        "@crawlee/utils": "4.0.0-beta.212",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -76,5 +76,5 @@
             }
         }
     },
-    "gitHead": "17bab8ab5b98554ca2db1d09bc5f61c59194a760"
+    "gitHead": "171a63a5d7f2a6d74cbce58479204ac20d30b87e"
 }

package/proxy_configuration.d.ts

@@ -110,11 +110,4 @@ export declare class ProxyConfiguration implements IProxyConfiguration {
      *  For example, `http://bob:password123@proxy.example.com:8000`
      */
     newUrl(): Promise<string | undefined>;
-    private handleProxyUrlsList;
-    /**
-     * Calls the custom newUrlFunction and checks format of its return value
-     */
-    private callNewUrlFunction;
-    private throwCannotCombineCustomMethods;
-    private throwNoOptionsProvided;
 }