@crawlee/core 4.0.0-beta.105 → 4.0.0-beta.106

package/owned_or_injected.js

@@ -10,13 +10,13 @@
  * @internal
  */
 export class OwnedOrInjected {
-    _value;
-    _owned;
-    _present;
+    #value;
+    #owned;
+    #present;
     constructor(value, owned, present) {
-        this._value = value;
-        this._owned = owned;
-        this._present = present;
+        this.#value = value;
+        this.#owned = owned;
+        this.#present = present;
     }
     /**
      * Resolves a collaborator from an optionally-injected instance. `Injected` is the public/borrowed type exposed via
@@ -42,24 +42,24 @@ export class OwnedOrInjected {
      * crawler-built defaults, `false` for user-injected instances.
      */
     get isOwned() {
-        return this._owned;
+        return this.#owned;
     }
     /**
      * Whether a value is currently available. `false` for an owned slot whose default hasn't been built yet
      * (e.g. a lazily-opened request queue before its first use).
      */
     get isPresent() {
-        return this._present;
+        return this.#present;
     }
     /**
      * The resolved instance, typed as the public `Injected` type. Throws if the value is not present yet — callers that
      * expect a lazily-filled owned slot should read {@link OwnedOrInjected.maybeValue|`maybeValue`} instead.
      */
     get value() {
-        if (!this._present) {
+        if (!this.#present) {
             throw new Error('OwnedOrInjected value is not initialized yet');
         }
-        return this._value;
+        return this.#value;
     }
     /**
      * The resolved instance, or `undefined` when a lazily-filled owned slot hasn't been built yet. The non-throwing
@@ -67,7 +67,7 @@ export class OwnedOrInjected {
      * a possibly-empty slot without the `isPresent ? value : …` dance.
      */
     get maybeValue() {
-        return this._present ? this._value : undefined;
+        return this.#present ? this.#value : undefined;
     }
     /**
      * Fills the (owned) slot with the crawler-built default, returning it for convenience. Only valid on an owned,
@@ -75,14 +75,14 @@ export class OwnedOrInjected {
      * would silently orphan the previous instance's lifecycle).
      */
     set(value) {
-        if (!this._owned) {
+        if (!this.#owned) {
             throw new Error('Cannot set() a borrowed OwnedOrInjected value');
         }
-        if (this._present) {
+        if (this.#present) {
             throw new Error('OwnedOrInjected value is already initialized');
         }
-        this._value = value;
-        this._present = true;
+        this.#value = value;
+        this.#present = true;
         return value;
     }
     /**
@@ -90,9 +90,9 @@ export class OwnedOrInjected {
      * owns a present instance — a no-op for a borrowed instance or an owned-but-not-yet-built slot.
      */
     async ifOwned(fn) {
-        if (!this._owned || !this._present) {
+        if (!this.#owned || !this.#present) {
             return undefined;
         }
-        return fn(this._value);
+        return fn(this.#value);
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.105",
+    "version": "4.0.0-beta.106",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,9 +52,9 @@
         "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.105",
-        "@crawlee/types": "4.0.0-beta.105",
-        "@crawlee/utils": "4.0.0-beta.105",
+        "@crawlee/fs-storage": "4.0.0-beta.106",
+        "@crawlee/types": "4.0.0-beta.106",
+        "@crawlee/utils": "4.0.0-beta.106",
         "@sapphire/async-queue": "^1.5.5",
         "@sapphire/shapeshift": "^4.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -78,5 +78,5 @@
             }
         }
     },
-    "gitHead": "26073a822c7699ac487931383a39192bc3daae7a"
+    "gitHead": "c622f1fc65e65221ea245817c58ecc0ffb4a5cb0"
 }

package/proxy_configuration.d.ts

@@ -70,10 +70,8 @@ export interface IProxyConfiguration {
  * @category Scaling
  */
 export declare class ProxyConfiguration implements IProxyConfiguration {
+    #private;
     readonly isManInTheMiddle = false;
-    private nextCustomUrlIndex;
-    private proxyUrls?;
-    private newUrlFunction?;
     /**
      * Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
      * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures

package/proxy_configuration.js

@@ -29,9 +29,9 @@ import ow from 'ow';
  */
 export class ProxyConfiguration {
     isManInTheMiddle = false;
-    nextCustomUrlIndex = 0;
-    proxyUrls;
-    newUrlFunction;
+    #nextCustomUrlIndex = 0;
+    #proxyUrls;
+    #newUrlFunction;
     /**
      * Creates a {@link ProxyConfiguration} instance based on the provided options. Proxy servers are used to prevent target websites from
      * blocking your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
@@ -67,8 +67,8 @@ export class ProxyConfiguration {
             this.throwCannotCombineCustomMethods();
         if (!proxyUrls && !newUrlFunction && validateRequired)
             this.throwNoOptionsProvided();
-        this.proxyUrls = proxyUrls;
-        this.newUrlFunction = newUrlFunction;
+        this.#proxyUrls = proxyUrls;
+        this.#newUrlFunction = newUrlFunction;
     }
     /**
      * This function creates a new {@link ProxyInfo} info object.
@@ -99,19 +99,19 @@ export class ProxyConfiguration {
      *  For example, `http://bob:password123@proxy.example.com:8000`
      */
     async newUrl(options) {
-        if (this.newUrlFunction) {
+        if (this.#newUrlFunction) {
             return (await this.callNewUrlFunction({ request: options?.request })) ?? undefined;
         }
         return this.handleProxyUrlsList() ?? undefined;
     }
     handleProxyUrlsList() {
-        return this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
+        return this.#proxyUrls[this.#nextCustomUrlIndex++ % this.#proxyUrls.length];
     }
     /**
      * Calls the custom newUrlFunction and checks format of its return value
      */
     async callNewUrlFunction(options) {
-        const proxyUrl = await this.newUrlFunction(options);
+        const proxyUrl = await this.#newUrlFunction(options);
         try {
             if (proxyUrl) {
                 new URL(proxyUrl); // eslint-disable-line no-new

package/recoverable_state.d.ts

@@ -59,16 +59,7 @@ export interface RecoverableStateOptions<TStateModel = Record<string, unknown>>
  * The class automatically hooks into the event system to persist state when needed.
  */
 export declare class RecoverableState<TStateModel = Record<string, unknown>> {
-    private readonly defaultState;
-    private state;
-    private readonly persistenceEnabled;
-    private readonly persistStateKey;
-    private readonly persistStateKvsName?;
-    private readonly persistStateKvsId?;
-    private keyValueStore;
-    private readonly log;
-    private readonly serialize;
-    private readonly deserialize;
+    #private;
     /**
      * Initialize a new recoverable state object.
      *

package/recoverable_state.js

@@ -10,30 +10,30 @@ import { KeyValueStore, serviceLocator } from '@crawlee/core';
  * The class automatically hooks into the event system to persist state when needed.
  */
 export class RecoverableState {
-    defaultState;
-    state = null;
-    persistenceEnabled;
-    persistStateKey;
-    persistStateKvsName;
-    persistStateKvsId;
-    keyValueStore = null;
-    log;
-    serialize;
-    deserialize;
+    #defaultState;
+    #state = null;
+    #persistenceEnabled;
+    #persistStateKey;
+    #persistStateKvsName;
+    #persistStateKvsId;
+    #keyValueStore = null;
+    #log;
+    #serialize;
+    #deserialize;
     /**
      * Initialize a new recoverable state object.
      *
      * @param options Configuration options for the recoverable state
      */
     constructor(options) {
-        this.defaultState = options.defaultState;
-        this.persistStateKey = options.persistStateKey;
-        this.persistenceEnabled = options.persistenceEnabled ?? false;
-        this.persistStateKvsName = options.persistStateKvsName;
-        this.persistStateKvsId = options.persistStateKvsId;
-        this.log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
-        this.serialize = options.serialize ?? JSON.stringify;
-        this.deserialize = options.deserialize ?? JSON.parse;
+        this.#defaultState = options.defaultState;
+        this.#persistStateKey = options.persistStateKey;
+        this.#persistenceEnabled = options.persistenceEnabled ?? false;
+        this.#persistStateKvsName = options.persistStateKvsName;
+        this.#persistStateKvsId = options.persistStateKvsId;
+        this.#log = options.logger ?? serviceLocator.getLogger().child({ prefix: 'RecoverableState' });
+        this.#serialize = options.serialize ?? JSON.stringify;
+        this.#deserialize = options.deserialize ?? JSON.parse;
         this.persistState = this.persistState.bind(this);
     }
     /**
@@ -45,21 +45,21 @@ export class RecoverableState {
      * @returns The loaded state object
      */
     async initialize() {
-        if (this.state !== null && this.state !== undefined) {
+        if (this.#state !== null && this.#state !== undefined) {
             return this.currentValue;
         }
-        if (!this.persistenceEnabled) {
-            this.state = this.deserialize(this.serialize(this.defaultState));
+        if (!this.#persistenceEnabled) {
+            this.#state = this.#deserialize(this.#serialize(this.#defaultState));
             return this.currentValue;
         }
         let kvsIdentifier = null;
-        if (this.persistStateKvsName) {
-            kvsIdentifier = { name: this.persistStateKvsName };
+        if (this.#persistStateKvsName) {
+            kvsIdentifier = { name: this.#persistStateKvsName };
         }
-        else if (this.persistStateKvsId) {
-            kvsIdentifier = { id: this.persistStateKvsId };
+        else if (this.#persistStateKvsId) {
+            kvsIdentifier = { id: this.#persistStateKvsId };
         }
-        this.keyValueStore = await KeyValueStore.open(kvsIdentifier, {
+        this.#keyValueStore = await KeyValueStore.open(kvsIdentifier, {
             configuration: serviceLocator.getConfiguration(),
         });
         await this.loadSavedState();
@@ -75,7 +75,7 @@ export class RecoverableState {
      * and persists the current state one last time.
      */
     async teardown() {
-        if (!this.persistenceEnabled || !this.persistState) {
+        if (!this.#persistenceEnabled || !this.persistState) {
             return;
         }
         const eventManager = serviceLocator.getEventManager();
@@ -86,10 +86,10 @@ export class RecoverableState {
      * Get the current state.
      */
     get currentValue() {
-        if (this.state === null) {
+        if (this.#state === null) {
             throw new Error('Recoverable state has not yet been loaded');
         }
-        return this.state;
+        return this.#state;
     }
     /**
      * Reset the state to the default values and clear any persisted state.
@@ -98,12 +98,12 @@ export class RecoverableState {
      * clears the persisted state from the KeyValueStore.
      */
     async reset() {
-        this.state = this.deserialize(this.serialize(this.defaultState));
-        if (this.persistenceEnabled) {
-            if (this.keyValueStore === null) {
+        this.#state = this.#deserialize(this.#serialize(this.#defaultState));
+        if (this.#persistenceEnabled) {
+            if (this.#keyValueStore === null) {
                 throw new Error('Recoverable state has not yet been initialized');
             }
-            await this.keyValueStore.setValue(this.persistStateKey, null);
+            await this.#keyValueStore.setValue(this.#persistStateKey, null);
         }
     }
     /**
@@ -115,12 +115,12 @@ export class RecoverableState {
      * @param eventData Optional data associated with a PERSIST_STATE event
      */
     async persistState(eventData) {
-        this.log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
-        if (this.keyValueStore === null || this.state === null) {
+        this.#log.debug(`Persisting state of the RecoverableState (eventData=${JSON.stringify(eventData)}).`);
+        if (this.#keyValueStore === null || this.#state === null) {
             throw new Error('Recoverable state has not yet been initialized');
         }
-        if (this.persistenceEnabled) {
-            await this.keyValueStore.setValue(this.persistStateKey, this.serialize(this.state), {
+        if (this.#persistenceEnabled) {
+            await this.#keyValueStore.setValue(this.#persistStateKey, this.#serialize(this.#state), {
                 contentType: 'text/plain', // HACK - the result is expected to be JSON, but we do this to avoid the implicit JSON.parse in `KeyValueStore.getValue`
             });
         }
@@ -129,15 +129,15 @@ export class RecoverableState {
      * Load the saved state from the KeyValueStore
      */
     async loadSavedState() {
-        if (this.keyValueStore === null) {
+        if (this.#keyValueStore === null) {
             throw new Error('Recoverable state has not yet been initialized');
         }
-        const storedState = await this.keyValueStore.getValue(this.persistStateKey);
+        const storedState = await this.#keyValueStore.getValue(this.#persistStateKey);
         if (storedState === null || storedState === undefined) {
-            this.state = this.deserialize(this.serialize(this.defaultState));
+            this.#state = this.#deserialize(this.#serialize(this.#defaultState));
         }
         else {
-            this.state = this.deserialize(storedState);
+            this.#state = this.#deserialize(storedState);
         }
     }
 }

package/request.d.ts

@@ -44,6 +44,7 @@ export declare enum RequestState {
  * @category Sources
  */
 declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
+    #private;
     /** Request ID */
     id?: string;
     /** URL of the web page to crawl. */
@@ -74,8 +75,6 @@ declare class CrawleeRequest<UserData extends Dictionary = Dictionary> {
     errorMessages: string[];
     /** Object with HTTP headers. Key is header name, value is the value. */
     headers?: Record<string, string>;
-    /** Private store for the custom user data assigned to the request. */
-    private _userData;
     /**
      * Custom user data assigned to the request.
      *

package/request.js

@@ -101,7 +101,7 @@ class CrawleeRequest {
     /** Object with HTTP headers. Key is header name, value is the value. */
     headers;
     /** Private store for the custom user data assigned to the request. */
-    _userData = {};
+    #userData = {};
     /**
      * Custom user data assigned to the request.
      *
@@ -167,36 +167,33 @@ class CrawleeRequest {
         if (label) {
             userData.label = label;
         }
+        this.#userData = { __crawlee: {}, ...userData };
+        // `userData` must stay an enumerable own accessor — serialization in the storages relies on it
         Object.defineProperties(this, {
-            _userData: {
-                value: { __crawlee: {}, ...userData },
-                enumerable: false,
-                writable: true,
-            },
             userData: {
-                get: () => this._userData,
+                get: () => this.#userData,
                 set: (value) => {
                     Object.defineProperties(value, {
                         __crawlee: {
-                            value: this._userData.__crawlee,
+                            value: this.#userData.__crawlee,
                             enumerable: false,
                             writable: true,
                         },
                         toJSON: {
                             value: () => {
-                                if (Object.keys(this._userData.__crawlee).length > 0) {
+                                if (Object.keys(this.#userData.__crawlee).length > 0) {
                                     return {
-                                        ...this._userData,
-                                        __crawlee: this._userData.__crawlee,
+                                        ...this.#userData,
+                                        __crawlee: this.#userData.__crawlee,
                                     };
                                 }
-                                return this._userData;
+                                return this.#userData;
                             },
                             enumerable: false,
                             writable: true,
                         },
                     });
-                    this._userData = value;
+                    this.#userData = value;
                 },
                 enumerable: true,
             },

package/router.d.ts

@@ -215,10 +215,7 @@ export type RouterRoutes<Context, Routes extends Record<keyof Routes, Dictionary
  * ```
  */
 export declare class Router<Context extends Omit<RestrictedCrawlingContext, 'enqueueLinks'>, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> {
-    private readonly routes;
-    private readonly schemas;
-    private readonly timeouts;
-    private readonly middlewares;
+    #private;
     /**
      * use Router.create() instead!
      * @ignore

package/router.js

@@ -161,10 +161,10 @@ export async function validateUserData(label, schema, userData) {
  * ```
  */
 export class Router {
-    routes = new Map();
-    schemas = new Map();
-    timeouts = new Map();
-    middlewares = [];
+    #routes = new Map();
+    #schemas = new Map();
+    #timeouts = new Map();
+    #middlewares = [];
     /**
      * use Router.create() instead!
      * @ignore
@@ -172,9 +172,9 @@ export class Router {
     constructor() { }
     addHandler(label, handler, options = {}) {
         this.validate(label);
-        this.routes.set(label, handler);
+        this.#routes.set(label, handler);
         if (options.requestHandlerTimeoutSecs !== undefined) {
-            this.timeouts.set(label, options.requestHandlerTimeoutSecs);
+            this.#timeouts.set(label, options.requestHandlerTimeoutSecs);
         }
     }
     /**
@@ -187,9 +187,9 @@ export class Router {
      */
     addDefaultHandler(handler, options = {}) {
         this.validate(defaultRoute);
-        this.routes.set(defaultRoute, handler);
+        this.#routes.set(defaultRoute, handler);
         if (options.requestHandlerTimeoutSecs !== undefined) {
-            this.timeouts.set(defaultRoute, options.requestHandlerTimeoutSecs);
+            this.#timeouts.set(defaultRoute, options.requestHandlerTimeoutSecs);
         }
     }
     /**
@@ -199,25 +199,25 @@ export class Router {
      */
     getSchema(label) {
         if (label != null) {
-            const schema = this.schemas.get(label);
+            const schema = this.#schemas.get(label);
             if (schema) {
                 return schema;
             }
             // A label with its own route is fully specified; don't fall back to the default-route schema.
-            if (this.routes.has(label)) {
+            if (this.#routes.has(label)) {
                 return undefined;
             }
         }
         // Requests with no route of their own fall through to the default handler, so validate their
         // `userData` against the default-route schema, if one was registered.
-        return this.schemas.get(defaultRoute);
+        return this.#schemas.get(defaultRoute);
     }
     /**
      * Registers a middleware that will be fired before the matching route handler.
      * Multiple middlewares can be registered, they will be fired in the same order.
      */
     use(middleware) {
-        this.middlewares.push(middleware);
+        this.#middlewares.push(middleware);
     }
     /**
      * Returns the `requestHandlerTimeoutSecs` registered for a label, or `undefined` when the route did not
@@ -226,27 +226,27 @@ export class Router {
      * the default route asked for. Used by the crawler; not meant to be called directly.
      */
     getTimeoutSecs(label) {
-        if (label && this.routes.has(label)) {
-            return this.timeouts.get(label);
+        if (label && this.#routes.has(label)) {
+            return this.#timeouts.get(label);
         }
-        return this.timeouts.get(defaultRoute);
+        return this.#timeouts.get(defaultRoute);
     }
     /**
      * The longest `requestHandlerTimeoutSecs` any route asked for, or `undefined` when no route overrides it.
      * The crawler needs an upper bound up front, before it knows which routes a run will actually hit.
      */
     getMaxTimeoutSecs() {
-        return this.timeouts.size > 0 ? Math.max(...this.timeouts.values()) : undefined;
+        return this.#timeouts.size > 0 ? Math.max(...this.#timeouts.values()) : undefined;
     }
     /**
      * Returns route handler for given label. If no label is provided, the default request handler will be returned.
      */
     getHandler(label) {
-        if (label && this.routes.has(label)) {
-            return this.routes.get(label);
+        if (label && this.#routes.has(label)) {
+            return this.#routes.get(label);
         }
-        if (this.routes.has(defaultRoute)) {
-            return this.routes.get(defaultRoute);
+        if (this.#routes.has(defaultRoute)) {
+            return this.#routes.get(defaultRoute);
         }
         throw new MissingRouteError(`Route not found for label '${String(label)}'.` +
             ' You must set up a route for this label or a default route.' +
@@ -267,7 +267,7 @@ export class Router {
      * Throws when the label already exists in our registry.
      */
     validate(label) {
-        if (this.routes.has(label)) {
+        if (this.#routes.has(label)) {
             const message = label === defaultRoute
                 ? `Default route is already defined!`
                 : `Route for label '${String(label)}' is already defined!`;
@@ -291,14 +291,14 @@ export class Router {
                 router.addHandler(label, value);
             }
             else {
-                router.schemas.set(label, value);
+                router.#schemas.set(label, value);
             }
         }
         const func = async function (context) {
             const { url, loadedUrl, label } = context.request;
             context.log.debug('Page opened.', { label, url: loadedUrl ?? url });
             await router.validateRequest(context);
-            for (const middleware of router.middlewares) {
+            for (const middleware of router.#middlewares) {
                 await middleware(context);
             }
             return router.getHandler(label)(context);

package/serialization.js

@@ -11,32 +11,31 @@ const pipeline = util.promisify(streamPipeline);
  * @internal
  */
 class ArrayToJson extends Readable {
-    data;
-    offset = 0;
-    batchSize;
+    #offset = 0;
+    #batchSize;
+    #data;
     constructor(data, options = {}) {
         super({
             ...options,
             autoDestroy: true,
             emitClose: true,
         });
-        this.data = data;
         const { batchSize = 10000 } = options;
-        this.batchSize = batchSize;
-        this.data = data;
+        this.#batchSize = batchSize;
+        this.#data = data;
         this.push('[');
     }
     _read() {
         try {
-            const items = this.data.slice(this.offset, this.offset + this.batchSize);
+            const items = this.#data.slice(this.#offset, this.#offset + this.#batchSize);
             if (items.length) {
                 const json = JSON.stringify(items);
                 // Strip brackets to flatten the batch.
                 const itemString = json.substring(1, json.length - 1);
-                if (this.offset > 0)
+                if (this.#offset > 0)
                     this.push(',', 'utf8');
                 this.push(itemString, 'utf8');
-                this.offset += this.batchSize;
+                this.#offset += this.#batchSize;
             }
             else {
                 this.push(']');

package/service_locator.d.ts

@@ -101,16 +101,7 @@ interface ServiceLocatorInterface {
  * ```
  */
 export declare class ServiceLocator implements ServiceLocatorInterface {
-    private configuration?;
-    private eventManager?;
-    private storageBackend?;
-    private logger?;
-    /**
-     * Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
-     * Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
-     * Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
-     */
-    private static storageInstanceManager?;
+    #private;
     /**
      * Creates a new ServiceLocator instance.
      *