@crawlee/core 4.0.0-beta.105 → 4.0.0-beta.107

package/session_pool/session_pool.js

@@ -60,24 +60,25 @@ const SESSION_REUSE_STRATEGIES = ['random', 'round-robin', 'use-until-failure'];
  * @category Scaling
  */
 export class SessionPool {
-    static nextId = 0;
+    static #nextId = 0;
     id;
-    log;
+    #log;
+    #sessions = [];
+    // kept as TS-private: session_pool tests read/override the members below directly
     maxPoolSize;
     createSessionFunction;
     keyValueStore;
-    sessions = [];
     sessionMap = new Map();
     sessionOptions;
     persistStateKeyValueStoreId;
     persistStateKey;
-    listener;
-    events;
-    persistenceOptions;
-    sessionReuseStrategy;
-    initPromise;
-    queue = new AsyncQueue();
-    roundRobinIndex = 0;
+    #listener;
+    #events;
+    #persistenceOptions;
+    #sessionReuseStrategy;
+    #initPromise;
+    #queue = new AsyncQueue();
+    #roundRobinIndex = 0;
     constructor(options = {}) {
         ow(options, ow.object.exactShape({
             id: ow.optional.any(ow.number, ow.string),
@@ -93,20 +94,20 @@ export class SessionPool {
         const { id, maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions = {}, log = serviceLocator.getLogger(), persistenceOptions = {
             enable: true,
         }, sessionReuseStrategy = 'random', } = options;
-        this.id = id != null ? String(id) : String(SessionPool.nextId++);
-        this.sessionReuseStrategy = sessionReuseStrategy;
-        this.events = serviceLocator.getEventManager();
-        this.log = log.child({ prefix: 'SessionPool' });
-        this.persistenceOptions = persistenceOptions;
+        this.id = id != null ? String(id) : String(SessionPool.#nextId++);
+        this.#sessionReuseStrategy = sessionReuseStrategy;
+        this.#events = serviceLocator.getEventManager();
+        this.#log = log.child({ prefix: 'SessionPool' });
+        this.#persistenceOptions = persistenceOptions;
         // Pool Configuration
         this.maxPoolSize = maxPoolSize;
         this.createSessionFunction = createSessionFunction || this.defaultCreateSessionFunction;
         // Session configuration. The pool-scoped logger is merged into per-call sessionOptions inside
-        // `_invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions
+        // `invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions
         // having to know about it.
         this.sessionOptions = {
             ...sessionOptions,
-            log: this.log,
+            log: this.#log,
         };
         // Session keyValueStore
         this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
@@ -117,39 +118,39 @@ export class SessionPool {
      */
     async usableSessionsCount() {
         await this.ensureInitialized();
-        return this.sessions.filter((session) => session.isUsable()).length;
+        return this.#sessions.filter((session) => session.isUsable()).length;
     }
     /**
      * Gets count of retired sessions in the pool.
      */
     async retiredSessionsCount() {
         await this.ensureInitialized();
-        return this.sessions.filter((session) => !session.isUsable()).length;
+        return this.#sessions.filter((session) => !session.isUsable()).length;
     }
     /**
      * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
      * Called automatically on first use of any public method.
      */
     async ensureInitialized() {
-        if (!this.initPromise) {
-            this.initPromise = this.setupPool();
+        if (!this.#initPromise) {
+            this.#initPromise = this.setupPool();
         }
-        return this.initPromise;
+        return this.#initPromise;
     }
     async setupPool() {
-        if (!this.persistenceOptions.enable) {
+        if (!this.#persistenceOptions.enable) {
             return;
         }
         this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId ? { id: this.persistStateKeyValueStoreId } : null, {
             configuration: serviceLocator.getConfiguration(),
         });
         if (!this.persistStateKeyValueStoreId) {
-            this.log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
+            this.#log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
         }
         // in case of migration happened and SessionPool state should be restored from the keyValueStore.
         await this.maybeLoadSessionPool();
-        this.listener = this.persistState.bind(this);
-        this.events.on("persistState" /* EventType.PERSIST_STATE */, this.listener);
+        this.#listener = this.persistState.bind(this);
+        this.#events.on("persistState" /* EventType.PERSIST_STATE */, this.#listener);
     }
     /**
      * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
@@ -169,8 +170,8 @@ export class SessionPool {
         if (!this.hasSpaceForSession()) {
             this.removeRetiredSessions();
         }
-        const newSession = options instanceof Session ? options : await this._invokeCreateSessionFunction(options);
-        this.log.debug(`Adding new Session - ${newSession.id}`);
+        const newSession = options instanceof Session ? options : await this.invokeCreateSessionFunction(options);
+        this.#log.debug(`Adding new Session - ${newSession.id}`);
         this.registerSession(newSession);
     }
     /**
@@ -181,7 +182,7 @@ export class SessionPool {
      */
     async newSession(sessionOptions) {
         await this.ensureInitialized();
-        const newSession = await this._invokeCreateSessionFunction(sessionOptions);
+        const newSession = await this.invokeCreateSessionFunction(sessionOptions);
         this.registerSession(newSession);
         return newSession;
     }
@@ -194,7 +195,7 @@ export class SessionPool {
      */
     async getSession(sessionId) {
         await this.ensureInitialized();
-        await this.queue.wait();
+        await this.#queue.wait();
         try {
             if (sessionId) {
                 const session = this.sessionMap.get(sessionId);
@@ -212,14 +213,14 @@ export class SessionPool {
             return await this.createSession();
         }
         finally {
-            this.queue.shift();
+            this.#queue.shift();
         }
     }
     /**
      * @param options - Override the persistence options provided in the constructor
      */
     async resetStore(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
+        if (!this.#persistenceOptions.enable && !options?.enable) {
             return;
         }
         await this.ensureInitialized();
@@ -234,7 +235,7 @@ export class SessionPool {
         return {
             usableSessionsCount: await this.usableSessionsCount(),
             retiredSessionsCount: await this.retiredSessionsCount(),
-            sessions: this.sessions.map((session) => session.getState()),
+            sessions: this.#sessions.map((session) => session.getState()),
         };
     }
     /**
@@ -243,28 +244,28 @@ export class SessionPool {
      * @param options - Override the persistence options provided in the constructor
      */
     async persistState(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
+        if (!this.#persistenceOptions.enable && !options?.enable) {
             return;
         }
         await this.ensureInitialized();
-        this.log.debug('Persisting state', {
+        this.#log.debug('Persisting state', {
             persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
             persistStateKey: this.persistStateKey,
         });
         await this.keyValueStore
             ?.setValue(this.persistStateKey, await this.getState())
-            .catch((error) => this.log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }));
+            .catch((error) => this.#log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }));
     }
     /**
      * Removes listener from `persistState` event.
      * This function should be called after you are done with using the `SessionPool` instance.
      */
     async teardown() {
-        if (!this.initPromise)
+        if (!this.#initPromise)
             return;
         await this.ensureInitialized();
-        if (this.listener) {
-            this.events.off("persistState" /* EventType.PERSIST_STATE */, this.listener);
+        if (this.#listener) {
+            this.#events.off("persistState" /* EventType.PERSIST_STATE */, this.#listener);
         }
         await this.persistState();
     }
@@ -272,11 +273,11 @@ export class SessionPool {
      * Removes retired `Session` instances from `SessionPool`.
      */
     removeRetiredSessions() {
-        this.sessions = this.sessions.filter((storedSession) => {
+        this.#sessions = this.#sessions.filter((storedSession) => {
             if (storedSession.isUsable())
                 return true;
             this.sessionMap.delete(storedSession.id);
-            this.log.debug(`Removed Session - ${storedSession.id}`);
+            this.#log.debug(`Removed Session - ${storedSession.id}`);
             return false;
         });
     }
@@ -285,14 +286,14 @@ export class SessionPool {
      * @param newSession `Session` instance to be added.
      */
     registerSession(newSession) {
-        this.sessions.push(newSession);
+        this.#sessions.push(newSession);
         this.sessionMap.set(newSession.id, newSession);
     }
     /**
      * Gets random index.
      */
     getRandomIndex() {
-        return Math.floor(Math.random() * this.sessions.length);
+        return Math.floor(Math.random() * this.#sessions.length);
     }
     /**
      * Creates new session without any extra behavior.
@@ -315,7 +316,7 @@ export class SessionPool {
      * through `maybeLoadSessionPool` naturally wins because it arrives in
      * `perCallOptions`.
      */
-    async _invokeCreateSessionFunction(perCallOptions) {
+    async invokeCreateSessionFunction(perCallOptions) {
         const sessionOptions = {
             fingerprint: createDefaultSessionFingerprint(),
             ...this.sessionOptions,
@@ -328,35 +329,35 @@ export class SessionPool {
      * @returns Newly created `Session` instance.
      */
     async createSession() {
-        const newSession = await this._invokeCreateSessionFunction();
+        const newSession = await this.invokeCreateSessionFunction();
         this.registerSession(newSession);
-        this.log.debug(`Created new Session - ${newSession.id}`);
+        this.#log.debug(`Created new Session - ${newSession.id}`);
         return newSession;
     }
     /**
      * Decides whether there is enough space for creating new session.
      */
     hasSpaceForSession() {
-        return this.sessions.length < this.maxPoolSize;
+        return this.#sessions.length < this.maxPoolSize;
     }
     /**
      * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
      * Returns `undefined` when no session should be reused and a new one should be created instead.
      */
     pickSession() {
-        if (this.sessionReuseStrategy !== 'use-until-failure' && this.hasSpaceForSession())
+        if (this.#sessionReuseStrategy !== 'use-until-failure' && this.hasSpaceForSession())
             return undefined;
-        if (this.sessionReuseStrategy === 'use-until-failure') {
-            return this.sessions.find((session) => session.isUsable());
+        if (this.#sessionReuseStrategy === 'use-until-failure') {
+            return this.#sessions.find((session) => session.isUsable());
         }
         let picked;
-        if (this.sessionReuseStrategy === 'round-robin') {
-            const index = this.roundRobinIndex % this.sessions.length;
-            this.roundRobinIndex = index + 1;
-            picked = this.sessions[index];
+        if (this.#sessionReuseStrategy === 'round-robin') {
+            const index = this.#roundRobinIndex % this.#sessions.length;
+            this.#roundRobinIndex = index + 1;
+            picked = this.#sessions[index];
         }
         else {
-            picked = this.sessions[this.getRandomIndex()];
+            picked = this.#sessions[this.getRandomIndex()];
         }
         return picked.isUsable() ? picked : undefined;
     }
@@ -369,18 +370,18 @@ export class SessionPool {
         if (!loadedSessionPool)
             return;
         // Invalidate old sessions and load active sessions only
-        this.log.debug('Recreating state from KeyValueStore', {
+        this.#log.debug('Recreating state from KeyValueStore', {
             persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
             persistStateKey: this.persistStateKey,
         });
         for (const sessionObject of loadedSessionPool.sessions) {
             sessionObject.createdAt = new Date(sessionObject.createdAt);
             sessionObject.expiresAt = new Date(sessionObject.expiresAt);
-            const recreatedSession = await this._invokeCreateSessionFunction(sessionObject);
+            const recreatedSession = await this.invokeCreateSessionFunction(sessionObject);
             if (recreatedSession.isUsable()) {
                 this.registerSession(recreatedSession);
             }
         }
-        this.log.debug(`${this.sessions.length} active sessions loaded from KeyValueStore`);
+        this.#log.debug(`${this.#sessions.length} active sessions loaded from KeyValueStore`);
     }
 }

package/storages/dataset.d.ts

@@ -140,12 +140,12 @@ export interface DatasetExportToOptions extends DatasetExportOptions {
  * @category Result Stores
  */
 export declare class Dataset<Data extends Dictionary = Dictionary> {
+    #private;
     readonly configuration: Configuration;
     id: string;
     name?: string;
     backend: DatasetBackend<Data>;
     log: CrawleeLogger;
-    private readonly statsTracker;
     /**
      * @internal
      */

package/storages/dataset.js

@@ -89,7 +89,7 @@ export class Dataset {
     name;
     backend;
     log;
-    statsTracker = new StorageStatsTracker({
+    #statsTracker = new StorageStatsTracker({
         readCount: 0,
         writeCount: 0,
     });
@@ -108,7 +108,7 @@ export class Dataset {
      * the underlying storage backend). Counted per backend call.
      */
     get stats() {
-        return this.statsTracker.current;
+        return this.#statsTracker.current;
     }
     /**
      * Stores an object or an array of objects to the dataset.
@@ -140,7 +140,7 @@ export class Dataset {
             });
             return;
         }
-        this.statsTracker.add('writeCount');
+        this.#statsTracker.add('writeCount');
         await this.backend.pushData(items);
     }
     /**
@@ -166,7 +166,7 @@ export class Dataset {
     async readPage(options) {
         const buffered = this.bufferedJournalEntries()?.flatMap((entry) => entry.items);
         // Every branch below hits the backend exactly once.
-        this.statsTracker.add('readCount');
+        this.#statsTracker.add('readCount');
         if (!buffered?.length) {
             return this.backend.getData(options);
         }
@@ -235,7 +235,7 @@ export class Dataset {
         // One backend call with all journaled items, in order - as close to atomic as the backend allows.
         // Straight to the backend: the items were validated and snapshotted at write time.
         if (items.length > 0) {
-            this.statsTracker.add('writeCount');
+            this.#statsTracker.add('writeCount');
             await this.backend.pushData(items);
         }
     }

package/storages/key_value_store.d.ts

@@ -61,14 +61,11 @@ import type { StorageIdentifier } from './storage_instance_manager.js';
  * @category Result Stores
  */
 export declare class KeyValueStore {
+    #private;
     readonly configuration: Configuration;
     readonly id: string;
     readonly name?: string;
     private readonly backend;
-    private persistStateEventStarted;
-    /** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
-    private readonly cache;
-    private readonly statsTracker;
     /**
      * @internal
      */

package/storages/key_value_store.js

@@ -71,11 +71,12 @@ export class KeyValueStore {
     configuration;
     id;
     name;
+    // kept as TS-private: key_value_store tests spy on the backend directly
     backend;
-    persistStateEventStarted = false;
+    #persistStateEventStarted = false;
     /** Cache for persistent (auto-saved) values. When we try to set such value, the cache will be updated automatically. */
-    cache = new Map();
-    statsTracker = new StorageStatsTracker({
+    #cache = new Map();
+    #statsTracker = new StorageStatsTracker({
         readCount: 0,
         writeCount: 0,
         deleteCount: 0,
@@ -95,7 +96,7 @@ export class KeyValueStore {
      * list operations issued to the underlying storage backend). Counted per backend call.
      */
     get stats() {
-        return this.statsTracker.current;
+        return this.#statsTracker.current;
     }
     /**
      * Gets a value from the key-value store.
@@ -178,7 +179,7 @@ export class KeyValueStore {
                 contentType: serialized.contentType ?? null,
             };
         }
-        this.statsTracker.add('readCount');
+        this.#statsTracker.add('readCount');
         const record = await this.backend.getValue(key);
         if (!record)
             return null;
@@ -235,8 +236,8 @@ export class KeyValueStore {
     }
     async getAutoSavedValue(key, defaultValue = {}) {
         tryCancel();
-        if (this.cache.has(key)) {
-            return this.cache.get(key);
+        if (this.#cache.has(key)) {
+            return this.#cache.get(key);
         }
         // Auto-saved state is deliberately *not* transactional. The direct read bypasses any active
         // transaction - a buffered value seeded into this shared cache would survive a rollback forever.
@@ -245,25 +246,25 @@ export class KeyValueStore {
         // the value will in cache at this point, and returning the new fetched value would introduce two different instances of
         // the auto-saved object, and only the latter one would be persisted.
         // Therefore we re-check the cache here, and if such race condition happened, we drop the fetched value and return the cached one.
-        if (this.cache.has(key)) {
-            return this.cache.get(key);
+        if (this.#cache.has(key)) {
+            return this.#cache.get(key);
         }
-        this.cache.set(key, value);
+        this.#cache.set(key, value);
         this.ensurePersistStateEvent();
         return value;
     }
     ensurePersistStateEvent() {
-        if (this.persistStateEventStarted) {
+        if (this.#persistStateEventStarted) {
             return;
         }
         serviceLocator.getEventManager().on('persistState', async () => {
             const promises = [];
-            for (const [key, value] of this.cache) {
+            for (const [key, value] of this.#cache) {
                 promises.push(this.setValue(key, value).catch((error) => serviceLocator.getLogger().warning(`Failed to persist the state value to ${key}`, { error })));
             }
             await Promise.all(promises);
         });
-        this.persistStateEventStarted = true;
+        this.#persistStateEventStarted = true;
     }
     async *fetchKeyValuePages(options, mapRecord) {
         // Reduce the journal once for the whole iteration, not once per key inside `readRecord`.
@@ -303,7 +304,7 @@ export class KeyValueStore {
         }
         let exclusiveStartKey;
         while (true) {
-            this.statsTracker.add('listCount');
+            this.#statsTracker.add('listCount');
             const { items, isTruncated, nextExclusiveStartKey } = await this.backend.listKeys({
                 ...options,
                 exclusiveStartKey,
@@ -401,28 +402,28 @@ export class KeyValueStore {
             return;
         }
         // If we try to set the value of a cached state to a different reference, we need to update the cache accordingly.
-        const cachedValue = this.cache.get(key);
+        const cachedValue = this.#cache.get(key);
         if (cachedValue && cachedValue !== value) {
             if (value === null) {
                 // Cached state can be only object, so a propagation of `null` means removing all its properties.
-                Object.keys(cachedValue).forEach((k) => this.cache.delete(k));
+                Object.keys(cachedValue).forEach((k) => this.#cache.delete(k));
             }
             else if (typeof value === 'object') {
                 // We need to remove the keys that are no longer present in the new value.
                 Object.keys(cachedValue)
                     .filter((k) => !(k in value))
-                    .forEach((k) => this.cache.delete(k));
+                    .forEach((k) => this.#cache.delete(k));
                 // And update the existing ones + add new ones.
                 Object.assign(cachedValue, value);
             }
         }
         // In this case delete the record.
         if (value === null) {
-            this.statsTracker.add('deleteCount');
+            this.#statsTracker.add('deleteCount');
             return this.backend.deleteValue(key);
         }
         const serialized = serializeValue(value, optionsCopy.contentType);
-        this.statsTracker.add('writeCount');
+        this.#statsTracker.add('writeCount');
         return this.backend.setValue({
             key,
             value: serialized.value,
@@ -454,7 +455,7 @@ export class KeyValueStore {
     /** @internal */
     clearCache() {
         rejectOperationInTransaction('KeyValueStore.clearCache()');
-        this.cache.clear();
+        this.#cache.clear();
     }
     /**
      * Iterates over key-value store keys, yielding each in turn to an `iteratee` function.

package/storages/request_dedup_cache.d.ts

@@ -12,9 +12,8 @@
  * @internal
  */
 export declare class RequestDeduplicationCache {
+    #private;
     private readonly size;
-    private keys;
-    private ids;
     constructor(size?: number);
     get(cacheKey: string): string | null;
     add(cacheKey: string, requestId: string): void;

package/storages/request_dedup_cache.js

@@ -13,26 +13,26 @@
  */
 export class RequestDeduplicationCache {
     size;
-    keys;
-    ids;
+    #keys;
+    #ids;
     // The slot count is the same for every queue, so it's a fixed default rather than a per-consumer option.
     constructor(size = 1_000_000) {
         this.size = size;
-        this.keys = new Array(size);
-        this.ids = new Array(size);
+        this.#keys = new Array(size);
+        this.#ids = new Array(size);
     }
     get(cacheKey) {
         const index = this.indexOf(cacheKey);
-        return this.keys[index] === cacheKey ? this.ids[index] : null;
+        return this.#keys[index] === cacheKey ? this.#ids[index] : null;
     }
     add(cacheKey, requestId) {
         const index = this.indexOf(cacheKey);
-        this.keys[index] = cacheKey;
-        this.ids[index] = requestId;
+        this.#keys[index] = cacheKey;
+        this.#ids[index] = requestId;
     }
     clear() {
-        this.keys = new Array(this.size);
-        this.ids = new Array(this.size);
+        this.#keys = new Array(this.size);
+        this.#ids = new Array(this.size);
     }
     // A cheap FNV-1a hash of the cache key — avoids pulling in a dedicated hashing dependency.
     indexOf(cacheKey) {

package/storages/request_list.d.ts

@@ -216,28 +216,18 @@ export interface RequestListOptions {
  * @category Sources
  */
 export declare class RequestList implements IRequestLoader {
-    private log;
+    #private;
     /**
      * Array of all requests from all sources, in the order as they appeared in sources.
      * All requests in the array have distinct uniqueKey!
      * @internal
      */
     readonly requests: (Request | RequestOptions)[];
-    /** Index to the next item in requests array to fetch. All previous requests are either handled or in progress. */
-    private nextIndex;
-    /** Dictionary, key is Request.uniqueKey, value is corresponding index in the requests array. */
-    private uniqueKeyToIndex;
     /**
      * Set of `uniqueKey`s of requests that were returned by fetchNextRequest().
      * @internal
      */
     inProgress: Set<string>;
-    /**
-     * `uniqueKey`s of requests that were in progress when the state was last persisted and thus need to be
-     * re-crawled after a restart. They are served before advancing through the rest of the sources.
-     * @internal
-     */
-    private requestsToRetry;
     /**
      * Starts as true because until we handle the first request, the list is effectively persisted by doing nothing.
      * @internal
@@ -248,17 +238,7 @@ export declare class RequestList implements IRequestLoader {
      * @internal
      */
     areRequestsPersisted: boolean;
-    private isLoading;
-    private isInitialized;
-    private persistStateKey?;
-    private persistRequestsKey?;
-    private initialState?;
-    private store?;
-    private keepDuplicateUrls;
     private sources;
-    private sourcesFunction?;
-    private proxyConfiguration?;
-    private httpClient?;
     /**
      * To create new instance of `RequestList` we need to use `RequestList.open()` factory method.
      * @param options All `RequestList` configuration options
@@ -437,7 +417,7 @@ export declare class RequestList implements IRequestLoader {
     /**
      * @internal wraps public utility for mocking purposes
      */
-    private _downloadListOfUrls;
+    private downloadListOfUrls;
 }
 /**
  * Represents state of a {@link RequestList}. It can be used to resume a {@link RequestList} which has been previously processed.