@crawlee/core 4.0.0-beta.142 → 4.0.0-beta.144

package/autoscaling/autoscaled_pool.d.ts

@@ -20,8 +20,15 @@ export interface TaskLoopPredicates {
      */
     isFinishedFunction?: () => Promise<boolean>;
 }
+export interface TaskLoopOptions extends TaskLoopPredicates {
+    /**
+     * How often the pool should check if a new task is ready, in seconds.
+     * @default 0.5
+     */
+    maybeRunIntervalSecs?: number;
+}
 /** @internal */
-export interface AutoscaledPoolOptions extends TaskLoopPredicates {
+export interface AutoscaledPoolOptions extends TaskLoopOptions {
     /**
      * The governor that decides whether there is free compute for one more task. Typically a
      * {@link ConcurrencySystem}, but any {@link IConcurrencySystem} works. Share a single instance across
@@ -41,12 +48,6 @@ export interface AutoscaledPoolOptions extends TaskLoopPredicates {
      * The function must either be labeled `async` or return a promise.
      */
     runTaskFunction?: () => Promise<unknown>;
-    /**
-     * Indicates how often the pool should call the `runTaskFunction()` to start a new task, in seconds.
-     * This has no effect on starting new tasks immediately after a task completes.
-     * @default 0.5
-     */
-    maybeRunIntervalSecs?: number;
     /**
      * Timeout in which the `runTaskFunction` needs to finish, given in seconds.
      * @default 0

package/autoscaling/autoscaled_pool.js

@@ -19,7 +19,7 @@ const autoscaledPoolOptionsSchema = z.strictObject({
         .default(0),
     log: validators.logger.default(() => serviceLocator.getLogger()),
     concurrencySystem: schemas.anyObject,
-    consumer: schemas.anyObject.refine((value) => typeof value.id === 'string' && value.id.length > 0, "Expected an object with a non-empty string 'id'"),
+    consumer: schemas.anyObject.refine((value) => typeof value?.id === 'string' && value.id.length > 0, "Expected an object with a non-empty string 'id'"),
 });
 /**
  * Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.

package/autoscaling/concurrency_system.d.ts

@@ -158,7 +158,6 @@ export declare class ConcurrencySystem implements IConcurrencySystem {
     private readonly scaleUpStepRatio;
     private readonly scaleDownStepRatio;
     private readonly maxTasksPerMinute;
-    private _currentConcurrency;
     private readonly snapshotter;
     private readonly systemStatus;
     constructor(options?: ConcurrencySystemOptions);

package/autoscaling/concurrency_system.js

@@ -54,8 +54,7 @@ export class ConcurrencySystem {
     #minConcurrency;
     #maxConcurrency;
     #desiredConcurrency;
-    // kept as TS-private _-prefixed: autoscaled_pool tests write this backing field directly
-    _currentConcurrency = 0;
+    #currentConcurrency = 0;
     #lastLoggingTime;
     #tasksPerMinute = Array.from({ length: 60 }, () => 0);
     snapshotter;
@@ -158,7 +157,7 @@ export class ConcurrencySystem {
         this.#desiredConcurrency = Math.min(atLeastMin, this.#maxConcurrency);
     }
     get currentConcurrency() {
-        return this._currentConcurrency;
+        return this.#currentConcurrency;
     }
     /** Whether the system is currently monitoring load and autoscaling the budget. */
     get isRunning() {
@@ -247,13 +246,13 @@ export class ConcurrencySystem {
      */
     hasCapacityForTask(_consumer) {
         this.warnIfNotRunning();
-        if (this._currentConcurrency >= this.#desiredConcurrency) {
+        if (this.#currentConcurrency >= this.#desiredConcurrency) {
             this.log.perf('Task will not run. Desired concurrency achieved.');
             return false;
         }
         const currentStatus = this.systemStatus.getCurrentStatus();
         const { isSystemIdle } = currentStatus;
-        if (!isSystemIdle && this._currentConcurrency >= this.#minConcurrency) {
+        if (!isSystemIdle && this.#currentConcurrency >= this.#minConcurrency) {
             this.log.perf('Task will not be run. System is overloaded.', currentStatus);
             return false;
         }
@@ -283,13 +282,13 @@ export class ConcurrencySystem {
             this.log.perf('Task will not run. Maximum tasks per minute reached.');
             return false;
         }
-        this._currentConcurrency++;
+        this.#currentConcurrency++;
         this.#tasksPerMinute[0]++;
         return true;
     }
     /** Returns a slot to the shared budget, whoever booked it. */
     registerTaskEnd(_consumer) {
-        this._currentConcurrency--;
+        this.#currentConcurrency--;
     }
     /**
      * What the system currently makes of the machine: the per-signal overload verdicts, evaluated over the
@@ -310,7 +309,7 @@ export class ConcurrencySystem {
         const { isSystemIdle } = systemStatus;
         const weAreNotAtMax = this.#desiredConcurrency < this.#maxConcurrency;
         const minCurrentConcurrency = Math.floor(this.#desiredConcurrency * this.desiredConcurrencyRatio);
-        const weAreReachingDesiredConcurrency = this._currentConcurrency >= minCurrentConcurrency;
+        const weAreReachingDesiredConcurrency = this.#currentConcurrency >= minCurrentConcurrency;
         if (isSystemIdle && weAreNotAtMax && weAreReachingDesiredConcurrency)
             this.scaleUp(systemStatus);
         const isSystemOverloaded = !isSystemIdle;
@@ -325,7 +324,7 @@ export class ConcurrencySystem {
             else if (now > this.#lastLoggingTime + this.#loggingIntervalMillis) {
                 this.#lastLoggingTime = now;
                 this.log.info('state', {
-                    currentConcurrency: this._currentConcurrency,
+                    currentConcurrency: this.#currentConcurrency,
                     desiredConcurrency: this.#desiredConcurrency,
                     systemStatus,
                 });

package/autoscaling/load_signal.d.ts

@@ -61,8 +61,6 @@ export interface LoadSignal {
  */
 export declare class SnapshotStore<T extends LoadSnapshot = LoadSnapshot> {
     #private;
-    /** Retention window in milliseconds. Unbounded until {@link SnapshotStore.useSampleWindow|`useSampleWindow()`}. */
-    private historyMillis;
     /**
      * Sizes retention to the window the signal will be sampled over, as handed to it in
      * {@link LoadSignal.start|`start()`}. Until this is called nothing is pruned at all, so a signal that ignores

package/autoscaling/load_signal.js

@@ -5,16 +5,14 @@ import { weightedAvg } from './weighted_avg.js';
  */
 export class SnapshotStore {
     #snapshots = [];
-    /** Retention window in milliseconds. Unbounded until {@link SnapshotStore.useSampleWindow|`useSampleWindow()`}. */
-    // kept as TS-private: concurrency_system tests read this retention window directly
-    historyMillis = Infinity;
+    #historyMillis = Infinity;
     /**
      * Sizes retention to the window the signal will be sampled over, as handed to it in
      * {@link LoadSignal.start|`start()`}. Until this is called nothing is pruned at all, so a signal that ignores
      * its start context grows unboundedly.
      */
     useSampleWindow(maxSampleWindowMillis) {
-        this.historyMillis = maxSampleWindowMillis;
+        this.#historyMillis = maxSampleWindowMillis;
     }
     /**
      * Add a snapshot and prune entries older than the history window.
@@ -24,7 +22,7 @@ export class SnapshotStore {
         let oldCount = 0;
         for (let i = 0; i < this.#snapshots.length; i++) {
             const { createdAt } = this.#snapshots[i];
-            if (now.getTime() - new Date(createdAt).getTime() > this.historyMillis)
+            if (now.getTime() - new Date(createdAt).getTime() > this.#historyMillis)
                 oldCount++;
             else
                 break;

package/crawlers/statistics.d.ts

@@ -89,7 +89,6 @@ export interface CalculatedStatistics {
  */
 export declare class Statistics<StateExtension extends object = {}, PersistedStateExtension extends object = StateExtension> implements IStatistics<StateExtension> {
     #private;
-    private static id;
     /**
      * An error tracker for final retry errors.
      */

package/crawlers/statistics.js

@@ -158,8 +158,7 @@ function buildStatisticStateCodec(statistics) {
  * @category Crawlers
  */
 export class Statistics {
-    // kept as TS-private: statistics tests read the static counter directly
-    static id = 0;
+    static #id = 0;
     /**
      * An error tracker for final retry errors.
      */
@@ -201,7 +200,7 @@ export class Statistics {
      */
     constructor(options = {}) {
         const { logIntervalSecs, logMessage, log, keyValueStore, persistenceOptions, saveErrorSnapshots, id, stateExtension, } = parseArgument(options, statisticsOptionsSchema);
-        this.id = id ?? String(Statistics.id++);
+        this.id = id ?? String(Statistics.#id++);
         this.#persistStateKey = `CRAWLEE_CRAWLER_STATISTICS_${this.id}`;
         this.log = (log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });
         this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });

package/log.d.ts

@@ -33,8 +33,7 @@ export type { CrawleeLogger, CrawleeLoggerOptions };
  * ```
  */
 export declare abstract class BaseCrawleeLogger implements CrawleeLogger {
-    private options;
-    private readonly warningsLogged;
+    #private;
     constructor(options?: Partial<CrawleeLoggerOptions>);
     /**
      * Core logging method. Subclasses must implement this to dispatch log messages
@@ -73,7 +72,7 @@ export declare abstract class BaseCrawleeLogger implements CrawleeLogger {
  * Users who want to use a different logging library should implement {@link BaseCrawleeLogger} directly.
  */
 export declare class ApifyLogAdapter extends BaseCrawleeLogger {
-    private readonly apifyLog;
+    #private;
     constructor(apifyLog: Log, options?: Partial<CrawleeLoggerOptions>);
     logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
     protected createChild(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;

package/log.js

@@ -30,18 +30,18 @@ import log, { Log, Logger, LoggerJson, LoggerText, LogLevel } from '@apify/log';
  * ```
  */
 export class BaseCrawleeLogger {
-    // kept as TS-private: the adaptive crawler's log proxy calls non-intercepted methods with `this === proxy`,
-    // where `#`-field access would throw at runtime
-    options;
-    warningsLogged = new Set();
+    // Note: If wrapping logger in a Proxy, unbound methods calling #-fields throw TypeError
+    // unless bound to the target (see createLogProxy in adaptive-playwright-crawler.ts).
+    #options;
+    #warningsLogged = new Set();
     constructor(options = {}) {
-        this.options = options;
+        this.#options = options;
     }
     getOptions() {
-        return this.options;
+        return this.#options;
     }
     setOptions(options) {
-        this.options = { ...this.options, ...options };
+        this.#options = { ...this.#options, ...options };
     }
     child(options) {
         return this.createChild(options);
@@ -63,8 +63,8 @@ export class BaseCrawleeLogger {
         this.logWithLevel(LogLevel.WARNING, message, data);
     }
     warningOnce(message) {
-        if (!this.warningsLogged.has(message)) {
-            this.warningsLogged.add(message);
+        if (!this.#warningsLogged.has(message)) {
+            this.#warningsLogged.add(message);
             this.warning(message);
         }
     }
@@ -88,18 +88,16 @@ export class BaseCrawleeLogger {
  * Users who want to use a different logging library should implement {@link BaseCrawleeLogger} directly.
  */
 export class ApifyLogAdapter extends BaseCrawleeLogger {
-    apifyLog;
-    constructor(
-    // kept as a TS-private parameter property: reached through the adaptive crawler's log proxy, see above
-    apifyLog, options) {
+    #apifyLog;
+    constructor(apifyLog, options) {
         super(options ?? {});
-        this.apifyLog = apifyLog;
+        this.#apifyLog = apifyLog;
     }
     logWithLevel(level, message, data) {
-        this.apifyLog.internal(level, message, data);
+        this.#apifyLog.internal(level, message, data);
     }
     createChild(options) {
-        return new ApifyLogAdapter(this.apifyLog.child({ prefix: options.prefix ?? null }), {
+        return new ApifyLogAdapter(this.#apifyLog.child({ prefix: options.prefix ?? null }), {
             ...this.getOptions(),
             ...options,
         });

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.142",
+    "version": "4.0.0-beta.144",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,10 +52,10 @@
         "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.142",
-        "@crawlee/http-client": "4.0.0-beta.142",
-        "@crawlee/types": "4.0.0-beta.142",
-        "@crawlee/utils": "4.0.0-beta.142",
+        "@crawlee/fs-storage": "4.0.0-beta.144",
+        "@crawlee/http-client": "4.0.0-beta.144",
+        "@crawlee/types": "4.0.0-beta.144",
+        "@crawlee/utils": "4.0.0-beta.144",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -78,5 +78,5 @@
             }
         }
     },
-    "gitHead": "d40a48fcb78865bd622a05027103155ac08a227f"
+    "gitHead": "4acf68a225909523047f872cbdeb87428bfc560d"
 }

package/session_pool/session.d.ts

@@ -28,7 +28,10 @@ export interface SessionOptions {
     errorScoreDecrement?: number;
     /** Date of creation. */
     createdAt?: Date;
-    /** Date of expiration. */
+    /**
+     * Date of expiration.
+     * @default createdAt + maxAgeSecs
+     */
     expiresAt?: Date;
     /**
      * Indicates how many times the session has been used.

package/session_pool/session.js

@@ -1,7 +1,6 @@
 import { CookieJar } from 'tough-cookie';
 import { z } from 'zod';
 import { cryptoRandomObjectId } from '@apify/utilities';
-import { getDefaultCookieExpirationDate } from '../cookie_utils.js';
 import { serviceLocator } from '../service_locator.js';
 import { parseArgument, schemas, validators } from '../validators.js';
 // `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
@@ -88,7 +87,9 @@ export class Session {
      * Session configuration.
      */
     constructor(options = {}) {
-        const { id, cookieJar, proxyInfo, maxAgeSecs, userData, maxErrorScore, errorScoreDecrement, createdAt, usageCount, errorScore, maxUsageCount, retired, log, fingerprint, expiresAt = getDefaultCookieExpirationDate(maxAgeSecs), } = parseArgument(options, sessionOptionsSchema);
+        const { id, cookieJar, proxyInfo, maxAgeSecs, userData, maxErrorScore, errorScoreDecrement, createdAt, usageCount, errorScore, maxUsageCount, retired, log, fingerprint, 
+        // Anchored to `createdAt` rather than to "now", so the documented `createdAt + maxAgeSecs` holds.
+        expiresAt = new Date(createdAt.getTime() + maxAgeSecs * 1000), } = parseArgument(options, sessionOptionsSchema);
         this.#log = log.child({ prefix: 'Session' });
         this.#cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
         this.#proxyInfo = proxyInfo;

package/session_pool/session_pool.d.ts

@@ -114,13 +114,6 @@ export interface SessionPoolOptions {
 export declare class SessionPool implements ISessionPool {
     #private;
     readonly id: string;
-    private maxPoolSize;
-    private createSessionFunction;
-    private keyValueStore?;
-    private sessionMap;
-    private sessionOptions;
-    private persistStateKeyValueStoreId?;
-    private persistStateKey;
     constructor(options?: SessionPoolOptions);
     /**
      * Gets count of usable sessions in the pool.

package/session_pool/session_pool.js

@@ -82,14 +82,13 @@ export class SessionPool {
     id;
     #log;
     #sessions = [];
-    // kept as TS-private: session_pool tests read/override the members below directly
-    maxPoolSize;
-    createSessionFunction;
-    keyValueStore;
-    sessionMap = new Map();
-    sessionOptions;
-    persistStateKeyValueStoreId;
-    persistStateKey;
+    #maxPoolSize;
+    #createSessionFunction;
+    #keyValueStore;
+    #sessionMap = new Map();
+    #sessionOptions;
+    #persistStateKeyValueStoreId;
+    #persistStateKey;
     #listener;
     #events;
     #persistenceOptions;
@@ -105,18 +104,18 @@ export class SessionPool {
         this.#log = log.child({ prefix: 'SessionPool' });
         this.#persistenceOptions = persistenceOptions;
         // Pool Configuration
-        this.maxPoolSize = maxPoolSize;
-        this.createSessionFunction = createSessionFunction || this.defaultCreateSessionFunction;
+        this.#maxPoolSize = maxPoolSize;
+        this.#createSessionFunction = createSessionFunction || this.defaultCreateSessionFunction;
         // Session configuration. The pool-scoped logger is merged into per-call sessionOptions inside
         // `invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions
         // having to know about it.
-        this.sessionOptions = {
+        this.#sessionOptions = {
             ...sessionOptions,
             log: this.#log,
         };
         // Session keyValueStore
-        this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
-        this.persistStateKey = persistStateKey ?? `${PERSIST_STATE_KEY}_${this.id}`;
+        this.#persistStateKeyValueStoreId = persistStateKeyValueStoreId;
+        this.#persistStateKey = persistStateKey ?? `${PERSIST_STATE_KEY}_${this.id}`;
     }
     /**
      * Gets count of usable sessions in the pool.
@@ -146,11 +145,11 @@ export class SessionPool {
         if (!this.#persistenceOptions.enable) {
             return;
         }
-        this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId ? { id: this.persistStateKeyValueStoreId } : null, {
+        this.#keyValueStore = await KeyValueStore.open(this.#persistStateKeyValueStoreId ? { id: this.#persistStateKeyValueStoreId } : null, {
             configuration: serviceLocator.getConfiguration(),
         });
-        if (!this.persistStateKeyValueStoreId) {
-            this.#log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
+        if (!this.#persistStateKeyValueStoreId) {
+            this.#log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.#keyValueStore.id}`);
         }
         // in case of migration happened and SessionPool state should be restored from the keyValueStore.
         await this.maybeLoadSessionPool();
@@ -167,7 +166,7 @@ export class SessionPool {
         await this.ensureInitialized();
         const { id } = options;
         if (id) {
-            const sessionExists = this.sessionMap.has(id);
+            const sessionExists = this.#sessionMap.has(id);
             if (sessionExists) {
                 throw new Error(`Cannot add session with id '${id}' as it already exists in the pool`);
             }
@@ -203,7 +202,7 @@ export class SessionPool {
         await this.#queue.wait();
         try {
             if (sessionId) {
-                const session = this.sessionMap.get(sessionId);
+                const session = this.#sessionMap.get(sessionId);
                 if (session?.isUsable())
                     return session;
                 return undefined;
@@ -229,7 +228,7 @@ export class SessionPool {
             return;
         }
         await this.ensureInitialized();
-        await this.keyValueStore?.setValue(this.persistStateKey, null);
+        await this.#keyValueStore?.setValue(this.#persistStateKey, null);
     }
     /**
      * Returns an object representing the internal state of the `SessionPool` instance.
@@ -254,12 +253,12 @@ export class SessionPool {
         }
         await this.ensureInitialized();
         this.#log.debug('Persisting state', {
-            persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
-            persistStateKey: this.persistStateKey,
+            persistStateKeyValueStoreId: this.#persistStateKeyValueStoreId,
+            persistStateKey: this.#persistStateKey,
         });
-        await this.keyValueStore
-            ?.setValue(this.persistStateKey, await this.getState())
-            .catch((error) => this.#log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }));
+        await this.#keyValueStore
+            ?.setValue(this.#persistStateKey, await this.getState())
+            .catch((error) => this.#log.warning(`Failed to persist the session pool stats to ${this.#persistStateKey}`, { error }));
     }
     async [Symbol.asyncDispose]() {
         await this.teardown({ persistState: true });
@@ -287,7 +286,7 @@ export class SessionPool {
         this.#sessions = this.#sessions.filter((storedSession) => {
             if (storedSession.isUsable())
                 return true;
-            this.sessionMap.delete(storedSession.id);
+            this.#sessionMap.delete(storedSession.id);
             this.#log.debug(`Removed Session - ${storedSession.id}`);
             return false;
         });
@@ -298,7 +297,7 @@ export class SessionPool {
      */
     registerSession(newSession) {
         this.#sessions.push(newSession);
-        this.sessionMap.set(newSession.id, newSession);
+        this.#sessionMap.set(newSession.id, newSession);
     }
     /**
      * Gets random index.
@@ -329,10 +328,10 @@ export class SessionPool {
     async invokeCreateSessionFunction(perCallOptions) {
         const sessionOptions = {
             fingerprint: createDefaultSessionFingerprint(),
-            ...this.sessionOptions,
+            ...this.#sessionOptions,
             ...perCallOptions,
         };
-        return this.createSessionFunction({ sessionOptions });
+        return this.#createSessionFunction({ sessionOptions });
     }
     /**
      * Creates new session and adds it to the pool.
@@ -348,7 +347,7 @@ export class SessionPool {
      * Decides whether there is enough space for creating new session.
      */
     hasSpaceForSession() {
-        return this.#sessions.length < this.maxPoolSize;
+        return this.#sessions.length < this.#maxPoolSize;
     }
     /**
      * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
@@ -376,13 +375,13 @@ export class SessionPool {
      * If the state was persisted it loads the `SessionPool` from the persisted state.
      */
     async maybeLoadSessionPool() {
-        const loadedSessionPool = await this.keyValueStore?.getValue(this.persistStateKey);
+        const loadedSessionPool = await this.#keyValueStore?.getValue(this.#persistStateKey);
         if (!loadedSessionPool)
             return;
         // Invalidate old sessions and load active sessions only
         this.#log.debug('Recreating state from KeyValueStore', {
-            persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
-            persistStateKey: this.persistStateKey,
+            persistStateKeyValueStoreId: this.#persistStateKeyValueStoreId,
+            persistStateKey: this.#persistStateKey,
         });
         for (const sessionObject of loadedSessionPool.sessions) {
             sessionObject.createdAt = new Date(sessionObject.createdAt);

package/storages/dataset.js

@@ -126,6 +126,7 @@ export class Dataset {
      *   The objects must be serializable to JSON.
      */
     async pushData(data) {
+        tryCancel();
         const transaction = activeStorageTransaction();
         parseArgument(data, schemas.anyObject);
         // Normalize to array and validate each item

package/storages/key_value_store.d.ts

@@ -65,7 +65,7 @@ export declare class KeyValueStore {
     readonly configuration: Configuration;
     readonly id: string;
     readonly name?: string;
-    private readonly backend;
+    readonly backend: KeyValueStoreBackend;
     /**
      * @internal
      */

package/storages/key_value_store.js

@@ -86,7 +86,6 @@ export class KeyValueStore {
     configuration;
     id;
     name;
-    // kept as TS-private: key_value_store tests spy on the backend directly
     backend;
     #persistStateEventStarted = false;
     /** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */

package/storages/request_list.d.ts

@@ -239,7 +239,6 @@ export declare class RequestList implements IRequestLoader {
      * @internal
      */
     areRequestsPersisted: boolean;
-    private sources;
     /**
      * To create new instance of `RequestList` we need to use `RequestList.open()` factory method.
      * @param options All `RequestList` configuration options

package/storages/request_list.js

@@ -131,8 +131,7 @@ export class RequestList {
     #initialState;
     #store;
     #keepDuplicateUrls;
-    // kept as TS-private: request_list tests read this field directly
-    sources;
+    #sources;
     #sourcesFunction;
     #proxyConfiguration;
     #httpClient;
@@ -153,7 +152,7 @@ export class RequestList {
         // If this option is set then all requests will get a pre-generated unique ID and duplicate URLs will be kept in the list.
         this.#keepDuplicateUrls = keepDuplicateUrls;
         // Will be empty after initialization to save memory.
-        this.sources = sources ? [...sources] : [];
+        this.#sources = sources ? [...sources] : [];
         this.#sourcesFunction = sourcesFunction;
         // The proxy configuration used for `requestsFromUrl` requests.
         this.#proxyConfiguration = proxyConfiguration;
@@ -195,11 +194,11 @@ export class RequestList {
     async addPersistedRequests(persistedRequests) {
         // We don't need the sources so we purge them to
         // prevent them from hanging in memory.
-        for (let i = 0; i < this.sources.length; i++) {
+        for (let i = 0; i < this.#sources.length; i++) {
             // oxlint-disable-next-line typescript/no-array-delete -- intentional, drop the slot so V8 can collect the object
-            delete this.sources[i];
+            delete this.#sources[i];
         }
-        this.sources = [];
+        this.#sources = [];
         this.areRequestsPersisted = true;
         const requestStream = createDeserialize(persistedRequests);
         for await (const request of requestStream) {
@@ -214,13 +213,13 @@ export class RequestList {
      */
     async addRequestsFromSources() {
         // We'll load all sources in sequence to ensure that they get loaded in the right order.
-        const sourcesCount = this.sources.length;
+        const sourcesCount = this.#sources.length;
         for (let i = 0; i < sourcesCount; i++) {
-            const source = this.sources[i];
+            const source = this.#sources[i];
             // Using delete here to drop the original object ASAP to free memory
             // .pop would reverse the array and .shift is SLOW.
             // oxlint-disable-next-line typescript/no-array-delete
-            delete this.sources[i];
+            delete this.#sources[i];
             if (typeof source === 'object' && source.requestsFromUrl) {
                 const fetchedRequests = await this.fetchRequestsFromUrl(source);
                 await this.addFetchedRequests(source, fetchedRequests);
@@ -230,7 +229,7 @@ export class RequestList {
             }
         }
         // Drop the original array full of empty indexes.
-        this.sources = [];
+        this.#sources = [];
         if (this.#sourcesFunction) {
             try {
                 const sourcesFromFunction = await this.#sourcesFunction();

package/storages/request_queue.d.ts

@@ -49,8 +49,6 @@ export declare class RequestQueue implements IStorage, IRequestManager {
     readonly name?: string;
     readonly backend: RequestQueueBackend;
     readonly log: CrawleeLogger;
-    private requestCache;
-    private inProgressRequestBatchCount;
     /**
      * Backend-independent usage counters tracked for this request queue (write operations and
      * queue-head reads issued to the underlying storage backend). Counted per backend call.

package/storages/request_queue.js

@@ -98,8 +98,7 @@ export class RequestQueue {
     backend;
     #proxyConfiguration;
     log;
-    // kept as TS-private: request_queue tests read this cache directly
-    requestCache;
+    #requestCache;
     /**
      * Remembers the `requestId` of every request already submitted to the client — including background
      * batches that `requestCache` skips — so overlapping URL sets aren't re-submitted.
@@ -107,8 +106,7 @@ export class RequestQueue {
      */
     #requestSeenCache;
     #queuePausedForMigration = false;
-    // kept as TS-private: packages/core/test request-queue tests write this counter directly
-    inProgressRequestBatchCount = 0;
+    #inProgressRequestBatchCount = 0;
     /**
      * The largest expected request-processing time (in seconds) seen so far via
      * {@link setExpectedRequestProcessingTimeSecs}. Used to ensure that value is only ever raised, never
@@ -137,7 +135,7 @@ export class RequestQueue {
         this.#events = serviceLocator.getEventManager();
         this.backend = options.backend;
         this.#proxyConfiguration = options.proxyConfiguration;
-        this.requestCache = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
+        this.#requestCache = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
         this.#requestSeenCache = new RequestDeduplicationCache();
         this.log = serviceLocator.getLogger().child({ prefix: `RequestQueue(${this.id}, ${this.name ?? 'no-name'})` });
         this.#events.on(EventType.MIGRATING, async () => {
@@ -191,7 +189,7 @@ export class RequestQueue {
             return this.addRequestDeferred(transaction, request, forefront);
         }
         const cacheKey = getRequestId(request.uniqueKey);
-        const cachedInfo = this.requestCache.get(cacheKey);
+        const cachedInfo = this.#requestCache.get(cacheKey);
         if (cachedInfo) {
             request.id = cachedInfo.id;
             this.recordRequestJournalEntry(transaction, [request], forefront, true);
@@ -275,7 +273,7 @@ export class RequestQueue {
         // The caches hold real backend ids. Only *writing* provisional ids to them would be wrong;
         // reading saves a probe. Same lookup as the write-through path.
         const cacheKey = getRequestId(request.uniqueKey);
-        const cachedInfo = this.requestCache.get(cacheKey);
+        const cachedInfo = this.#requestCache.get(cacheKey);
         const knownRequestId = cachedInfo?.id ?? this.#requestSeenCache.get(cacheKey);
         if (knownRequestId) {
             this.recordRequestJournalEntry(transaction, [request], forefront, false);
@@ -418,7 +416,7 @@ export class RequestQueue {
         for (const request of requests) {
             const cacheKey = getCachedRequestId(request.uniqueKey);
             // Prefer the full `requestCache` record; fall back to the dedup cache for background batches it skips.
-            const cachedInfo = this.requestCache.get(cacheKey);
+            const cachedInfo = this.#requestCache.get(cacheKey);
             const knownRequestId = cachedInfo?.id ?? this.#requestSeenCache.get(cacheKey);
             if (knownRequestId) {
                 request.id = knownRequestId;
@@ -517,9 +515,9 @@ export class RequestQueue {
                 return processedRequests;
             },
             trackBackgroundBatches: (batches) => {
-                this.inProgressRequestBatchCount += 1;
+                this.#inProgressRequestBatchCount += 1;
                 void batches.finally(() => {
-                    this.inProgressRequestBatchCount -= 1;
+                    this.#inProgressRequestBatchCount -= 1;
                 });
             },
         });
@@ -580,7 +578,7 @@ export class RequestQueue {
     async markRequestAsHandled(request) {
         rejectOperationInTransaction('RequestQueue.markRequestAsHandled()', 'it is part of the crawler request-processing bookkeeping, which a transaction must not affect.');
         parseArgument(request, handledRequestSchema);
-        const forefront = this.requestCache.get(getRequestId(request.uniqueKey))?.forefront ?? false;
+        const forefront = this.#requestCache.get(getRequestId(request.uniqueKey))?.forefront ?? false;
         const handledAt = request.handledAt ?? new Date().toISOString();
         this.#statsTracker.add('writeCount');
         const processedRequest = await this.backend.markRequestAsHandled({
@@ -655,7 +653,7 @@ export class RequestQueue {
     async isFinished() {
         const transaction = activeStorageTransaction();
         // We are not finished if we're still adding new requests in the background.
-        if (this.inProgressRequestBatchCount > 0) {
+        if (this.#inProgressRequestBatchCount > 0) {
             return false;
         }
         // Requests buffered by the active transaction count as pending from its point of view.
@@ -686,8 +684,8 @@ export class RequestQueue {
      */
     cacheRequest(cacheKey, queueOperationInfo) {
         // Remove the previous entry, as otherwise our cache will never update 👀
-        this.requestCache.remove(cacheKey);
-        this.requestCache.add(cacheKey, {
+        this.#requestCache.remove(cacheKey);
+        this.#requestCache.add(cacheKey, {
             id: queueOperationInfo.requestId,
             isHandled: queueOperationInfo.wasAlreadyHandled,
             uniqueKey: queueOperationInfo.uniqueKey,
@@ -713,9 +711,9 @@ export class RequestQueue {
         rejectOperationInTransaction('RequestQueue.purge()');
         await this.backend.purge();
         // Reset in-memory bookkeeping so the queue behaves as if freshly opened.
-        this.requestCache.clear();
+        this.#requestCache.clear();
         this.#requestSeenCache.clear();
-        this.inProgressRequestBatchCount = 0;
+        this.#inProgressRequestBatchCount = 0;
         // Reset the expected-processing-time high-water mark too, otherwise the monotonic-raise guard
         // in `setExpectedRequestProcessingTimeSecs` would let a value raised in an earlier run leak into a
         // later one and silently swallow a lower hint (the queue is meant to be reusable across runs).