@crawlee/core 4.0.0-beta.104 → 4.0.0-beta.106

package/recoverable_state.js

@@ -10,30 +10,30 @@ import { KeyValueStore, serviceLocator } from '@crawlee/core';
  * The class automatically hooks into the event system to persist state when needed.
  */
 export class RecoverableState {
-    defaultState;
-    state = null;
-    persistenceEnabled;
-    persistStateKey;
-    persistStateKvsName;
-    persistStateKvsId;
-    keyValueStore = null;
-    log;
-    serialize;
-    deserialize;
+    #defaultState;
+    #state = null;
+    #persistenceEnabled;
+    #persistStateKey;
+    #persistStateKvsName;
+    #persistStateKvsId;
+    #keyValueStore = null;
+    #log;
+    #serialize;
+    #deserialize;
     /**
      * Initialize a new recoverable state object.
      *
      * @param options Configuration options for the recoverable state
      */
     constructor(options) {
-        this.defaultState = options.defaultState;
-        this.persistStateKey = options.persistStateKey;
-        this.persistenceEnabled = options.persistenceEnabled ?? false;
-        this.persistStateKvsName = options.persistStateKvsName;
-        this.persistStateKvsId = options.persistStateKvsId;
-        this.log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
-        this.serialize = options.serialize ?? JSON.stringify;
-        this.deserialize = options.deserialize ?? JSON.parse;
+        this.#defaultState = options.defaultState;
+        this.#persistStateKey = options.persistStateKey;
+        this.#persistenceEnabled = options.persistenceEnabled ?? false;
+        this.#persistStateKvsName = options.persistStateKvsName;
+        this.#persistStateKvsId = options.persistStateKvsId;
+        this.#log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
+        this.#serialize = options.serialize ?? JSON.stringify;
+        this.#deserialize = options.deserialize ?? JSON.parse;
         this.persistState = this.persistState.bind(this);
     }
     /**
@@ -45,21 +45,21 @@ export class RecoverableState {
      * @returns The loaded state object
      */
     async initialize() {
-        if (this.state !== null && this.state !== undefined) {
+        if (this.#state !== null && this.#state !== undefined) {
             return this.currentValue;
         }
-        if (!this.persistenceEnabled) {
-            this.state = this.deserialize(this.serialize(this.defaultState));
+        if (!this.#persistenceEnabled) {
+            this.#state = this.#deserialize(this.#serialize(this.#defaultState));
             return this.currentValue;
         }
         let kvsIdentifier = null;
-        if (this.persistStateKvsName) {
-            kvsIdentifier = { name: this.persistStateKvsName };
+        if (this.#persistStateKvsName) {
+            kvsIdentifier = { name: this.#persistStateKvsName };
         }
-        else if (this.persistStateKvsId) {
-            kvsIdentifier = { id: this.persistStateKvsId };
+        else if (this.#persistStateKvsId) {
+            kvsIdentifier = { id: this.#persistStateKvsId };
         }
-        this.keyValueStore = await KeyValueStore.open(kvsIdentifier, {
+        this.#keyValueStore = await KeyValueStore.open(kvsIdentifier, {
             configuration: serviceLocator.getConfiguration(),
         });
         await this.loadSavedState();
@@ -75,7 +75,7 @@ export class RecoverableState {
      * and persists the current state one last time.
      */
     async teardown() {
-        if (!this.persistenceEnabled || !this.persistState) {
+        if (!this.#persistenceEnabled || !this.persistState) {
             return;
         }
         const eventManager = serviceLocator.getEventManager();
@@ -86,10 +86,10 @@ export class RecoverableState {
      * Get the current state.
      */
     get currentValue() {
-        if (this.state === null) {
+        if (this.#state === null) {
             throw new Error('Recoverable state has not yet been loaded');
         }
-        return this.state;
+        return this.#state;
     }
     /**
      * Reset the state to the default values and clear any persisted state.
@@ -98,12 +98,12 @@ export class RecoverableState {
      * clears the persisted state from the KeyValueStore.
      */
     async reset() {
-        this.state = this.deserialize(this.serialize(this.defaultState));
-        if (this.persistenceEnabled) {
-            if (this.keyValueStore === null) {
+        this.#state = this.#deserialize(this.#serialize(this.#defaultState));
+        if (this.#persistenceEnabled) {
+            if (this.#keyValueStore === null) {
                 throw new Error('Recoverable state has not yet been initialized');
             }
-            await this.keyValueStore.setValue(this.persistStateKey, null);
+            await this.#keyValueStore.setValue(this.#persistStateKey, null);
         }
     }
     /**
@@ -115,12 +115,12 @@ export class RecoverableState {
      * @param eventData Optional data associated with a PERSIST_STATE event
      */
     async persistState(eventData) {
-        this.log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
-        if (this.keyValueStore === null || this.state === null) {
+        this.#log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
+        if (this.#keyValueStore === null || this.#state === null) {
             throw new Error('Recoverable state has not yet been initialized');
         }
-        if (this.persistenceEnabled) {
-            await this.keyValueStore.setValue(this.persistStateKey, this.serialize(this.state), {
+        if (this.#persistenceEnabled) {
+            await this.#keyValueStore.setValue(this.#persistStateKey, this.#serialize(this.#state), {
                 contentType: 'text/plain', // HACK - the result is expected to be JSON, but we do this to avoid the implicit JSON.parse in `KeyValueStore.getValue`
             });
         }
@@ -129,15 +129,15 @@ export class RecoverableState {
      * Load the saved state from the KeyValueStore
      */
     async loadSavedState() {
-        if (this.keyValueStore === null) {
+        if (this.#keyValueStore === null) {
             throw new Error('Recoverable state has not yet been initialized');
         }
-        const storedState = await this.keyValueStore.getValue(this.persistStateKey);
+        const storedState = await this.#keyValueStore.getValue(this.#persistStateKey);
         if (storedState === null || storedState === undefined) {
-            this.state = this.deserialize(this.serialize(this.defaultState));
+            this.#state = this.#deserialize(this.#serialize(this.#defaultState));
         }
         else {
-            this.state = this.deserialize(storedState);
+            this.#state = this.#deserialize(storedState);
         }
     }
 }

package/request.d.ts

@@ -44,6 +44,7 @@ export declare enum RequestState {
  * @category Sources
  */
 declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
+    #private;
     /** Request ID */
     id?: string;
     /** URL of the web page to crawl. */
@@ -74,8 +75,6 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
     errorMessages: string[];
     /** Object with HTTP headers. Key is header name, value is the value. */
     headers?: Record<string, string>;
-    /** Private store for the custom user data assigned to the request. */
-    private _userData;
     /**
      * Custom user data assigned to the request.
      *

package/request.js

@@ -101,7 +101,7 @@ class CrawleeRequest {
     /** Object with HTTP headers. Key is header name, value is the value. */
     headers;
     /** Private store for the custom user data assigned to the request. */
-    _userData = {};
+    #userData = {};
     /**
      * Custom user data assigned to the request.
      *
@@ -167,36 +167,33 @@ class CrawleeRequest {
         if (label) {
             userData.label = label;
         }
+        this.#userData = { __crawlee: {}, ...userData };
+        // `userData` must stay an enumerable own accessor — serialization in the storages relies on it
         Object.defineProperties(this, {
-            _userData: {
-                value: { __crawlee: {}, ...userData },
-                enumerable: false,
-                writable: true,
-            },
             userData: {
-                get: () => this._userData,
+                get: () => this.#userData,
                 set: (value) => {
                     Object.defineProperties(value, {
                         __crawlee: {
-                            value: this._userData.__crawlee,
+                            value: this.#userData.__crawlee,
                             enumerable: false,
                             writable: true,
                         },
                         toJSON: {
                             value: () => {
-                                if (Object.keys(this._userData.__crawlee).length > 0) {
+                                if (Object.keys(this.#userData.__crawlee).length > 0) {
                                     return {
-                                        ...this._userData,
-                                        __crawlee: this._userData.__crawlee,
+                                        ...this.#userData,
+                                        __crawlee: this.#userData.__crawlee,
                                     };
                                 }
-                                return this._userData;
+                                return this.#userData;
                             },
                             enumerable: false,
                             writable: true,
                         },
                     });
-                    this._userData = value;
+                    this.#userData = value;
                 },
                 enumerable: true,
             },

package/router.d.ts

@@ -215,10 +215,7 @@ export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary
  * ```
  */
 export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
-    private readonly routes;
-    private readonly schemas;
-    private readonly timeouts;
-    private readonly middlewares;
+    #private;
     /**
      * use Router.create() instead!
      * @ignore

package/router.js

@@ -161,10 +161,10 @@ export async function validateUserData(label, schema, userData) {
  * ```
  */
 export class Router {
-    routes = new Map();
-    schemas = new Map();
-    timeouts = new Map();
-    middlewares = [];
+    #routes = new Map();
+    #schemas = new Map();
+    #timeouts = new Map();
+    #middlewares = [];
     /**
      * use Router.create() instead!
      * @ignore
@@ -172,9 +172,9 @@ export class Router {
     constructor() { }
     addHandler(label, handler, options = {}) {
         this.validate(label);
-        this.routes.set(label, handler);
+        this.#routes.set(label, handler);
         if (options.requestHandlerTimeoutSecs !== undefined) {
-            this.timeouts.set(label, options.requestHandlerTimeoutSecs);
+            this.#timeouts.set(label, options.requestHandlerTimeoutSecs);
         }
     }
     /**
@@ -187,9 +187,9 @@ export class Router {
      */
     addDefaultHandler(handler, options = {}) {
         this.validate(defaultRoute);
-        this.routes.set(defaultRoute, handler);
+        this.#routes.set(defaultRoute, handler);
         if (options.requestHandlerTimeoutSecs !== undefined) {
-            this.timeouts.set(defaultRoute, options.requestHandlerTimeoutSecs);
+            this.#timeouts.set(defaultRoute, options.requestHandlerTimeoutSecs);
         }
     }
     /**
@@ -199,25 +199,25 @@ export class Router {
      */
     getSchema(label) {
         if (label != null) {
-            const schema = this.schemas.get(label);
+            const schema = this.#schemas.get(label);
             if (schema) {
                 return schema;
             }
             // A label with its own route is fully specified; don't fall back to the default-route schema.
-            if (this.routes.has(label)) {
+            if (this.#routes.has(label)) {
                 return undefined;
             }
         }
         // Requests with no route of their own fall through to the default handler, so validate their
         // `userData` against the default-route schema, if one was registered.
-        return this.schemas.get(defaultRoute);
+        return this.#schemas.get(defaultRoute);
     }
     /**
      * Registers a middleware that will be fired before the matching route handler.
      * Multiple middlewares can be registered, they will be fired in the same order.
      */
     use(middleware) {
-        this.middlewares.push(middleware);
+        this.#middlewares.push(middleware);
     }
     /**
      * Returns the `requestHandlerTimeoutSecs` registered for a label, or `undefined` when the route did not
@@ -226,27 +226,27 @@ export class Router {
      * the default route asked for. Used by the crawler; not meant to be called directly.
      */
     getTimeoutSecs(label) {
-        if (label && this.routes.has(label)) {
-            return this.timeouts.get(label);
+        if (label && this.#routes.has(label)) {
+            return this.#timeouts.get(label);
         }
-        return this.timeouts.get(defaultRoute);
+        return this.#timeouts.get(defaultRoute);
     }
     /**
      * The longest `requestHandlerTimeoutSecs` any route asked for, or `undefined` when no route overrides it.
      * The crawler needs an upper bound up front, before it knows which routes a run will actually hit.
      */
     getMaxTimeoutSecs() {
-        return this.timeouts.size > 0 ? Math.max(...this.timeouts.values()) : undefined;
+        return this.#timeouts.size > 0 ? Math.max(...this.#timeouts.values()) : undefined;
     }
     /**
      * Returns route handler for given label. If no label is provided, the default request handler will be returned.
      */
     getHandler(label) {
-        if (label && this.routes.has(label)) {
-            return this.routes.get(label);
+        if (label && this.#routes.has(label)) {
+            return this.#routes.get(label);
         }
-        if (this.routes.has(defaultRoute)) {
-            return this.routes.get(defaultRoute);
+        if (this.#routes.has(defaultRoute)) {
+            return this.#routes.get(defaultRoute);
         }
         throw new MissingRouteError(`Route not found for label '${String(label)}'.` +
             ' You must set up a route for this label or a default route.' +
@@ -267,7 +267,7 @@ export class Router {
      * Throws when the label already exists in our registry.
      */
     validate(label) {
-        if (this.routes.has(label)) {
+        if (this.#routes.has(label)) {
             const message = label === defaultRoute
                 ? `Default route is already defined!`
                 : `Route for label '${String(label)}' is already defined!`;
@@ -291,14 +291,14 @@ export class Router {
                 router.addHandler(label, value);
             }
             else {
-                router.schemas.set(label, value);
+                router.#schemas.set(label, value);
             }
         }
         const func = async function (context) {
             const { url, loadedUrl, label } = context.request;
             context.log.debug('Page opened.', { label, url: loadedUrl ?? url });
             await router.validateRequest(context);
-            for (const middleware of router.middlewares) {
+            for (const middleware of router.#middlewares) {
                 await middleware(context);
             }
             return router.getHandler(label)(context);

package/serialization.js

@@ -11,32 +11,31 @@ const pipeline = util.promisify(streamPipeline);
  * @internal
  */
 class ArrayToJson extends Readable {
-    data;
-    offset = 0;
-    batchSize;
+    #offset = 0;
+    #batchSize;
+    #data;
     constructor(data, options = {}) {
         super({
             ...options,
             autoDestroy: true,
             emitClose: true,
         });
-        this.data = data;
         const { batchSize = 10000 } = options;
-        this.batchSize = batchSize;
-        this.data = data;
+        this.#batchSize = batchSize;
+        this.#data = data;
         this.push('[');
     }
     _read() {
         try {
-            const items = this.data.slice(this.offset, this.offset + this.batchSize);
+            const items = this.#data.slice(this.#offset, this.#offset + this.#batchSize);
             if (items.length) {
                 const json = JSON.stringify(items);
                 // Strip brackets to flatten the batch.
                 const itemString = json.substring(1, json.length - 1);
-                if (this.offset > 0)
+                if (this.#offset > 0)
                     this.push(',', 'utf8');
                 this.push(itemString, 'utf8');
-                this.offset += this.batchSize;
+                this.#offset += this.#batchSize;
             }
             else {
                 this.push(']');

package/service_locator.d.ts

@@ -101,16 +101,7 @@ interface ServiceLocatorInterface {
  * ```
  */
 export declare class ServiceLocator implements ServiceLocatorInterface {
-    private configuration?;
-    private eventManager?;
-    private storageBackend?;
-    private logger?;
-    /**
-     * Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
-     * Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
-     * Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
-     */
-    private static storageInstanceManager?;
+    #private;
     /**
      * Creates a new ServiceLocator instance.
      *

package/service_locator.js

@@ -39,16 +39,16 @@ import { StorageInstanceManager } from './storages/storage_instance_manager.js';
  * ```
  */
 export class ServiceLocator {
-    configuration;
-    eventManager;
-    storageBackend;
-    logger;
+    #configuration;
+    #eventManager;
+    #storageBackend;
+    #logger;
     /**
      * Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
      * Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
      * Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
      */
-    static storageInstanceManager;
+    static #storageInstanceManager;
     /**
      * Creates a new ServiceLocator instance.
      *
@@ -58,61 +58,61 @@ export class ServiceLocator {
      * @param logger Optional logger instance to use
      */
     constructor(configuration, eventManager, storageBackend, logger) {
-        this.configuration = configuration;
-        this.eventManager = eventManager;
-        this.storageBackend = storageBackend;
-        this.logger = logger;
+        this.#configuration = configuration;
+        this.#eventManager = eventManager;
+        this.#storageBackend = storageBackend;
+        this.#logger = logger;
     }
     getConfiguration() {
-        if (!this.configuration) {
+        if (!this.#configuration) {
             this.getLogger().debug('No configuration set, implicitly creating and using default Configuration.');
-            this.configuration = new Configuration();
+            this.#configuration = new Configuration();
         }
-        return this.configuration;
+        return this.#configuration;
     }
     setConfiguration(configuration) {
         // Same instance, no need to do anything
-        if (this.configuration === configuration) {
+        if (this.#configuration === configuration) {
             return;
         }
         // Already have a different configuration that was retrieved
-        if (this.configuration) {
-            throw new ServiceConflictError('Configuration', configuration, this.configuration);
+        if (this.#configuration) {
+            throw new ServiceConflictError('Configuration', configuration, this.#configuration);
         }
-        this.configuration = configuration;
+        this.#configuration = configuration;
     }
     getEventManager() {
-        if (!this.eventManager) {
+        if (!this.#eventManager) {
             this.getLogger().debug('No event manager set, implicitly creating and using default LocalEventManager.');
-            if (!this.configuration) {
+            if (!this.#configuration) {
                 this.getLogger().warning('Implicit creation of event manager will implicitly set configuration as side effect. ' +
                     'It is advised to explicitly first set the configuration instead.');
             }
-            this.eventManager = LocalEventManager.fromConfiguration(this.getConfiguration());
+            this.#eventManager = LocalEventManager.fromConfiguration(this.getConfiguration());
         }
-        return this.eventManager;
+        return this.#eventManager;
     }
     setEventManager(eventManager) {
         // Same instance, no need to do anything
-        if (this.eventManager === eventManager) {
+        if (this.#eventManager === eventManager) {
             return;
         }
         // Already have a different event manager that was retrieved
-        if (this.eventManager) {
-            throw new ServiceConflictError('EventManager', eventManager, this.eventManager);
+        if (this.#eventManager) {
+            throw new ServiceConflictError('EventManager', eventManager, this.#eventManager);
         }
-        this.eventManager = eventManager;
+        this.#eventManager = eventManager;
     }
     getStorageBackend() {
-        if (!this.storageBackend) {
+        if (!this.#storageBackend) {
             this.getLogger().debug('No storage backend set, implicitly creating and using the default storage backend ' +
                 '(FileSystemStorageBackend when persistStorage is enabled, MemoryStorageBackend otherwise).');
-            if (!this.configuration) {
+            if (!this.#configuration) {
                 this.getLogger().warning('Implicit creation of storage backend will implicitly set configuration as side effect. ' +
                     'It is advised to explicitly first set the configuration instead.');
             }
             const configuration = this.getConfiguration();
-            this.storageBackend = configuration.persistStorage
+            this.#storageBackend = configuration.persistStorage
                 ? new FileSystemStorageBackend({
                     localDataDirectory: configuration.storageDir,
                     logger: this.getLogger().child({ prefix: 'FileSystemStorageBackend' }),
@@ -121,50 +121,50 @@ export class ServiceLocator {
                     logger: this.getLogger().child({ prefix: 'MemoryStorageBackend' }),
                 });
         }
-        return this.storageBackend;
+        return this.#storageBackend;
     }
     setStorageBackend(storageBackend) {
         // Same instance, no need to do anything
-        if (this.storageBackend === storageBackend) {
+        if (this.#storageBackend === storageBackend) {
             return;
         }
         // Already have a different storage backend that was retrieved
-        if (this.storageBackend) {
-            throw new ServiceConflictError('StorageBackend', storageBackend, this.storageBackend);
+        if (this.#storageBackend) {
+            throw new ServiceConflictError('StorageBackend', storageBackend, this.#storageBackend);
         }
-        this.storageBackend = storageBackend;
+        this.#storageBackend = storageBackend;
     }
     getLogger() {
-        if (!this.logger) {
-            this.logger = new ApifyLogAdapter(log);
+        if (!this.#logger) {
+            this.#logger = new ApifyLogAdapter(log);
         }
-        return this.logger;
+        return this.#logger;
     }
     setLogger(logger) {
-        if (this.logger === logger) {
+        if (this.#logger === logger) {
             return;
         }
-        if (this.logger) {
-            throw new ServiceConflictError('Logger', logger, this.logger);
+        if (this.#logger) {
+            throw new ServiceConflictError('Logger', logger, this.#logger);
         }
-        this.logger = logger;
+        this.#logger = logger;
     }
     getChildLog(prefix) {
         return this.getLogger().child({ prefix });
     }
     getStorageInstanceManager() {
-        if (!ServiceLocator.storageInstanceManager) {
-            ServiceLocator.storageInstanceManager = new StorageInstanceManager();
+        if (!ServiceLocator.#storageInstanceManager) {
+            ServiceLocator.#storageInstanceManager = new StorageInstanceManager();
         }
-        return ServiceLocator.storageInstanceManager;
+        return ServiceLocator.#storageInstanceManager;
     }
     reset() {
-        this.configuration = undefined;
-        this.eventManager = undefined;
-        this.storageBackend = undefined;
-        this.logger = undefined;
-        ServiceLocator.storageInstanceManager?.clearCache();
-        ServiceLocator.storageInstanceManager = undefined;
+        this.#configuration = undefined;
+        this.#eventManager = undefined;
+        this.#storageBackend = undefined;
+        this.#logger = undefined;
+        ServiceLocator.#storageInstanceManager?.clearCache();
+        ServiceLocator.#storageInstanceManager = undefined;
     }
 }
 /**

package/session_pool/session.d.ts

@@ -65,20 +65,9 @@ export interface SessionOptions {
  * @category Scaling
  */
 export declare class Session implements ISession {
+    #private;
     readonly id: string;
     readonly userData: Dictionary;
-    private _maxErrorScore;
-    private _errorScoreDecrement;
-    private _createdAt;
-    private _expiresAt;
-    private _usageCount;
-    private _maxUsageCount;
-    private _errorScore;
-    private _retired;
-    private _proxyInfo?;
-    private _cookieJar;
-    private _fingerprint?;
-    private log;
     get errorScore(): number;
     get usageCount(): number;
     get maxErrorScore(): number;