@crawlee/core 4.0.0-beta.105 → 4.0.0-beta.106

package/service_locator.js

@@ -39,16 +39,16 @@ import { StorageInstanceManager } from './storages/storage_instance_manager.js';
  * ```
  */
 export class ServiceLocator {
-    configuration;
-    eventManager;
-    storageBackend;
-    logger;
+    #configuration;
+    #eventManager;
+    #storageBackend;
+    #logger;
     /**
      * Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
      * Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
      * Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
      */
-    static storageInstanceManager;
+    static #storageInstanceManager;
     /**
      * Creates a new ServiceLocator instance.
      *
@@ -58,61 +58,61 @@ export class ServiceLocator {
      * @param logger Optional logger instance to use
      */
     constructor(configuration, eventManager, storageBackend, logger) {
-        this.configuration = configuration;
-        this.eventManager = eventManager;
-        this.storageBackend = storageBackend;
-        this.logger = logger;
+        this.#configuration = configuration;
+        this.#eventManager = eventManager;
+        this.#storageBackend = storageBackend;
+        this.#logger = logger;
     }
     getConfiguration() {
-        if (!this.configuration) {
+        if (!this.#configuration) {
             this.getLogger().debug('No configuration set, implicitly creating and using default Configuration.');
-            this.configuration = new Configuration();
+            this.#configuration = new Configuration();
         }
-        return this.configuration;
+        return this.#configuration;
     }
     setConfiguration(configuration) {
         // Same instance, no need to do anything
-        if (this.configuration === configuration) {
+        if (this.#configuration === configuration) {
             return;
         }
         // Already have a different configuration that was retrieved
-        if (this.configuration) {
-            throw new ServiceConflictError('Configuration', configuration, this.configuration);
+        if (this.#configuration) {
+            throw new ServiceConflictError('Configuration', configuration, this.#configuration);
         }
-        this.configuration = configuration;
+        this.#configuration = configuration;
     }
     getEventManager() {
-        if (!this.eventManager) {
+        if (!this.#eventManager) {
             this.getLogger().debug('No event manager set, implicitly creating and using default LocalEventManager.');
-            if (!this.configuration) {
+            if (!this.#configuration) {
                 this.getLogger().warning('Implicit creation of event manager will implicitly set configuration as side effect. ' +
                     'It is advised to explicitly first set the configuration instead.');
             }
-            this.eventManager = LocalEventManager.fromConfiguration(this.getConfiguration());
+            this.#eventManager = LocalEventManager.fromConfiguration(this.getConfiguration());
         }
-        return this.eventManager;
+        return this.#eventManager;
     }
     setEventManager(eventManager) {
         // Same instance, no need to do anything
-        if (this.eventManager === eventManager) {
+        if (this.#eventManager === eventManager) {
             return;
         }
         // Already have a different event manager that was retrieved
-        if (this.eventManager) {
-            throw new ServiceConflictError('EventManager', eventManager, this.eventManager);
+        if (this.#eventManager) {
+            throw new ServiceConflictError('EventManager', eventManager, this.#eventManager);
         }
-        this.eventManager = eventManager;
+        this.#eventManager = eventManager;
     }
     getStorageBackend() {
-        if (!this.storageBackend) {
+        if (!this.#storageBackend) {
             this.getLogger().debug('No storage backend set, implicitly creating and using the default storage backend ' +
                 '(FileSystemStorageBackend when persistStorage is enabled, MemoryStorageBackend otherwise).');
-            if (!this.configuration) {
+            if (!this.#configuration) {
                 this.getLogger().warning('Implicit creation of storage backend will implicitly set configuration as side effect. ' +
                     'It is advised to explicitly first set the configuration instead.');
             }
             const configuration = this.getConfiguration();
-            this.storageBackend = configuration.persistStorage
+            this.#storageBackend = configuration.persistStorage
                 ? new FileSystemStorageBackend({
                     localDataDirectory: configuration.storageDir,
                     logger: this.getLogger().child({ prefix: 'FileSystemStorageBackend' }),
@@ -121,50 +121,50 @@ export class ServiceLocator {
                     logger: this.getLogger().child({ prefix: 'MemoryStorageBackend' }),
                 });
         }
-        return this.storageBackend;
+        return this.#storageBackend;
     }
     setStorageBackend(storageBackend) {
         // Same instance, no need to do anything
-        if (this.storageBackend === storageBackend) {
+        if (this.#storageBackend === storageBackend) {
             return;
         }
         // Already have a different storage backend that was retrieved
-        if (this.storageBackend) {
-            throw new ServiceConflictError('StorageBackend', storageBackend, this.storageBackend);
+        if (this.#storageBackend) {
+            throw new ServiceConflictError('StorageBackend', storageBackend, this.#storageBackend);
         }
-        this.storageBackend = storageBackend;
+        this.#storageBackend = storageBackend;
     }
     getLogger() {
-        if (!this.logger) {
-            this.logger = new ApifyLogAdapter(log);
+        if (!this.#logger) {
+            this.#logger = new ApifyLogAdapter(log);
         }
-        return this.logger;
+        return this.#logger;
     }
     setLogger(logger) {
-        if (this.logger === logger) {
+        if (this.#logger === logger) {
             return;
         }
-        if (this.logger) {
-            throw new ServiceConflictError('Logger', logger, this.logger);
+        if (this.#logger) {
+            throw new ServiceConflictError('Logger', logger, this.#logger);
         }
-        this.logger = logger;
+        this.#logger = logger;
     }
     getChildLog(prefix) {
         return this.getLogger().child({ prefix });
     }
     getStorageInstanceManager() {
-        if (!ServiceLocator.storageInstanceManager) {
-            ServiceLocator.storageInstanceManager = new StorageInstanceManager();
+        if (!ServiceLocator.#storageInstanceManager) {
+            ServiceLocator.#storageInstanceManager = new StorageInstanceManager();
         }
-        return ServiceLocator.storageInstanceManager;
+        return ServiceLocator.#storageInstanceManager;
     }
     reset() {
-        this.configuration = undefined;
-        this.eventManager = undefined;
-        this.storageBackend = undefined;
-        this.logger = undefined;
-        ServiceLocator.storageInstanceManager?.clearCache();
-        ServiceLocator.storageInstanceManager = undefined;
+        this.#configuration = undefined;
+        this.#eventManager = undefined;
+        this.#storageBackend = undefined;
+        this.#logger = undefined;
+        ServiceLocator.#storageInstanceManager?.clearCache();
+        ServiceLocator.#storageInstanceManager = undefined;
     }
 }
 /**

package/session_pool/session.d.ts

@@ -65,20 +65,9 @@ export interface SessionOptions {
  * @category Scaling
  */
 export declare class Session implements ISession {
+    #private;
     readonly id: string;
     readonly userData: Dictionary;
-    private _maxErrorScore;
-    private _errorScoreDecrement;
-    private _createdAt;
-    private _expiresAt;
-    private _usageCount;
-    private _maxUsageCount;
-    private _errorScore;
-    private _retired;
-    private _proxyInfo?;
-    private _cookieJar;
-    private _fingerprint?;
-    private log;
     get errorScore(): number;
     get usageCount(): number;
     get maxErrorScore(): number;

package/session_pool/session.js

@@ -12,57 +12,57 @@ import { serviceLocator } from '../service_locator.js';
 export class Session {
     id;
     userData;
-    _maxErrorScore;
-    _errorScoreDecrement;
-    _createdAt;
-    _expiresAt;
-    _usageCount;
-    _maxUsageCount;
-    _errorScore;
-    _retired = false;
-    _proxyInfo;
-    _cookieJar;
-    _fingerprint;
-    log;
+    #maxErrorScore;
+    #errorScoreDecrement;
+    #createdAt;
+    #expiresAt;
+    #usageCount;
+    #maxUsageCount;
+    #errorScore;
+    #retired = false;
+    #proxyInfo;
+    #cookieJar;
+    #fingerprint;
+    #log;
     get errorScore() {
-        return this._errorScore;
+        return this.#errorScore;
     }
     get usageCount() {
-        return this._usageCount;
+        return this.#usageCount;
     }
     get maxErrorScore() {
-        return this._maxErrorScore;
+        return this.#maxErrorScore;
     }
     get errorScoreDecrement() {
-        return this._errorScoreDecrement;
+        return this.#errorScoreDecrement;
     }
     get expiresAt() {
-        return this._expiresAt;
+        return this.#expiresAt;
     }
     get createdAt() {
-        return this._createdAt;
+        return this.#createdAt;
     }
     get maxUsageCount() {
-        return this._maxUsageCount;
+        return this.#maxUsageCount;
     }
     get cookieJar() {
-        return this._cookieJar;
+        return this.#cookieJar;
     }
     get proxyInfo() {
-        return this._proxyInfo;
+        return this.#proxyInfo;
     }
     get fingerprint() {
-        return this._fingerprint;
+        return this.#fingerprint;
     }
     set fingerprint(fingerprint) {
-        this._fingerprint = fingerprint;
+        this.#fingerprint = fingerprint;
     }
     /**
      * `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
      * a retired session is never picked by the pool and cannot be revived via `markGood()`.
      */
     get retired() {
-        return this._retired;
+        return this.#retired;
     }
     /**
      * Session configuration.
@@ -87,21 +87,21 @@ export class Session {
         }));
         const { id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), proxyInfo = undefined, maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, retired = false, log = serviceLocator.getLogger(), fingerprint, } = options;
         const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
-        this.log = log.child({ prefix: 'Session' });
-        this._cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
-        this._proxyInfo = proxyInfo;
-        this._fingerprint = fingerprint;
+        this.#log = log.child({ prefix: 'Session' });
+        this.#cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
+        this.#proxyInfo = proxyInfo;
+        this.#fingerprint = fingerprint;
         this.id = id;
         this.userData = userData;
-        this._maxErrorScore = maxErrorScore;
-        this._errorScoreDecrement = errorScoreDecrement;
+        this.#maxErrorScore = maxErrorScore;
+        this.#errorScoreDecrement = errorScoreDecrement;
         // Internal
-        this._expiresAt = expiresAt;
-        this._createdAt = createdAt;
-        this._usageCount = usageCount; // indicates how many times the session has been used
-        this._errorScore = errorScore; // indicates number of markBaded request with the session
-        this._maxUsageCount = maxUsageCount;
-        this._retired = retired;
+        this.#expiresAt = expiresAt;
+        this.#createdAt = createdAt;
+        this.#usageCount = usageCount; // indicates how many times the session has been used
+        this.#errorScore = errorScore; // indicates number of markBaded request with the session
+        this.#maxUsageCount = maxUsageCount;
+        this.#retired = retired;
     }
     /**
      * Indicates whether the session is blocked.
@@ -130,16 +130,16 @@ export class Session {
      * Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
      */
     isUsable() {
-        return !this._retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
+        return !this.#retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
     }
     /**
      * This method should be called after a successful session usage.
      * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
      */
     markGood() {
-        this._usageCount += 1;
-        if (this._errorScore > 0) {
-            this._errorScore -= this._errorScoreDecrement;
+        this.#usageCount += 1;
+        if (this.#errorScore > 0) {
+            this.#errorScore -= this.#errorScoreDecrement;
         }
         this.maybeSelfRetire();
     }
@@ -151,9 +151,9 @@ export class Session {
         return {
             id: this.id,
             cookieJar: this.cookieJar.toJSON(),
-            proxyInfo: this._proxyInfo,
+            proxyInfo: this.#proxyInfo,
             userData: this.userData,
-            fingerprint: this._fingerprint,
+            fingerprint: this.#fingerprint,
             maxErrorScore: this.maxErrorScore,
             errorScoreDecrement: this.errorScoreDecrement,
             expiresAt: this.expiresAt.toISOString(),
@@ -161,7 +161,7 @@ export class Session {
             usageCount: this.usageCount,
             maxUsageCount: this.maxUsageCount,
             errorScore: this.errorScore,
-            retired: this._retired,
+            retired: this.#retired,
         };
     }
     /**
@@ -172,19 +172,19 @@ export class Session {
      * For transient external failures (such as `5XX` responses), use `markBad()` instead.
      */
     retire() {
-        if (this._retired)
+        if (this.#retired)
             return;
-        this._errorScore += this._maxErrorScore;
-        this._usageCount += 1;
-        this._retired = true;
+        this.#errorScore += this.#maxErrorScore;
+        this.#usageCount += 1;
+        this.#retired = true;
     }
     /**
      * Increases usage and error count.
      * Should be used when the session has been used unsuccessfully. For example because of timeouts.
      */
     markBad() {
-        this._errorScore += 1;
-        this._usageCount += 1;
+        this.#errorScore += 1;
+        this.#usageCount += 1;
         this.maybeSelfRetire();
     }
     /**
@@ -204,7 +204,7 @@ export class Session {
             this.cookieJar.setCookieSync(rawCookie, url);
         }
         catch (e) {
-            this.log.warning('Could not set cookie.', { url, error: e.message });
+            this.#log.warning('Could not set cookie.', { url, error: e.message });
         }
     }
     /**

package/session_pool/session_pool.d.ts

@@ -112,24 +112,15 @@ export interface SessionPoolOptions {
  * @category Scaling
  */
 export declare class SessionPool implements ISessionPool {
-    private static nextId;
+    #private;
     readonly id: string;
-    private log;
     private maxPoolSize;
     private createSessionFunction;
     private keyValueStore?;
-    private sessions;
     private sessionMap;
     private sessionOptions;
     private persistStateKeyValueStoreId?;
     private persistStateKey;
-    private listener?;
-    private events;
-    private persistenceOptions;
-    private sessionReuseStrategy;
-    private initPromise?;
-    private queue;
-    private roundRobinIndex;
     constructor(options?: SessionPoolOptions);
     /**
      * Gets count of usable sessions in the pool.
@@ -222,7 +213,7 @@ export declare class SessionPool implements ISessionPool {
      * through `maybeLoadSessionPool` naturally wins because it arrives in
      * `perCallOptions`.
      */
-    private _invokeCreateSessionFunction;
+    private invokeCreateSessionFunction;
     /**
      * Creates new session and adds it to the pool.
      * @returns Newly created `Session` instance.