@crawlee/core 4.0.0-beta.13 → 4.0.0-beta.131

package/service_locator.d.ts

@@ -0,0 +1,165 @@
+import type { StorageBackend } from '@crawlee/types';
+import { Configuration } from './configuration.js';
+import type { EventManager } from './events/event_manager.js';
+import type { CrawleeLogger } from './log.js';
+import { StorageInstanceManager } from './storages/storage_instance_manager.js';
+interface ServiceLocatorInterface {
+    /**
+     * Get the configuration.
+     * Creates a default Configuration instance if none has been set.
+     */
+    getConfiguration(): Configuration;
+    /**
+     * Set the configuration.
+     *
+     * @param configuration The configuration to set
+     * @throws {ServiceConflictError} If a different configuration has already been retrieved
+     */
+    setConfiguration(configuration: Configuration): void;
+    /**
+     * Get the event manager.
+     * Creates a default LocalEventManager instance if none has been set.
+     */
+    getEventManager(): EventManager;
+    /**
+     * Set the event manager.
+     *
+     * @param eventManager The event manager to set
+     * @throws {ServiceConflictError} If a different event manager has already been retrieved
+     */
+    setEventManager(eventManager: EventManager): void;
+    /**
+     * Get the storage backend.
+     * Creates a default storage backend if none has been set — `FileSystemStorageBackend` when
+     * `persistStorage` is enabled (the default), `MemoryStorageBackend` otherwise.
+     */
+    getStorageBackend(): StorageBackend;
+    /**
+     * Set the storage backend.
+     *
+     * @param storageBackend The storage backend to set
+     * @throws {ServiceConflictError} If a different storage backend has already been retrieved
+     */
+    setStorageBackend(storageBackend: StorageBackend): void;
+    /**
+     * Get the logger.
+     * Returns the default `@apify/log` logger if none has been set.
+     */
+    getLogger(): CrawleeLogger;
+    /**
+     * Set the logger.
+     *
+     * @param logger The logger to set
+     * @throws {ServiceConflictError} If a different logger has already been retrieved
+     */
+    setLogger(logger: CrawleeLogger): void;
+    /**
+     * Get a child logger with the given prefix.
+     * Equivalent to `getLogger().child({ prefix })`.
+     */
+    getChildLog(prefix: string): CrawleeLogger;
+    /**
+     * Get the storage instance manager (shared across all storage types).
+     */
+    getStorageInstanceManager(): StorageInstanceManager;
+    /**
+     * Returns the currently set services without triggering the implicit creation of defaults.
+     * Used to inherit already-materialized services into crawler-scoped service locators.
+     * @internal
+     */
+    getServicesIfSet(): {
+        configuration?: Configuration;
+        eventManager?: EventManager;
+        storageBackend?: StorageBackend;
+        logger?: CrawleeLogger;
+    };
+    /**
+     * Resets the service locator to its initial state.
+     * Used mainly for testing purposes.
+     * @internal
+     */
+    reset(): void;
+}
+/**
+ * Service locator for managing the services used by Crawlee.
+ *
+ * All services are initialized to their default value lazily.
+ *
+ * There are two primary usage patterns:
+ *
+ * **1. Global service locator (for default services):**
+ * ```typescript
+ * import { serviceLocator, BasicCrawler } from 'crawlee';
+ *
+ * // Optionally configure global services before creating crawlers
+ * serviceLocator.setStorageBackend(myCustomClient);
+ *
+ * // Crawler uses global services
+ * const crawler = new BasicCrawler({ ... });
+ * ```
+ *
+ * **2. Per-crawler services (recommended for isolation):**
+ * ```typescript
+ * import { BasicCrawler, Configuration, LocalEventManager, MemoryStorageBackend } from 'crawlee';
+ *
+ * const crawler = new BasicCrawler({
+ *     requestHandler: async ({ request }) => { ... },
+ *     configuration: new Configuration({ ... }),           // custom configuration
+ *     storageBackend: new MemoryStorageBackend(),          // custom storage
+ *     eventManager: LocalEventManager.fromConfiguration(), // custom events
+ * });
+ * // Crawler has its own isolated ServiceLocator instance
+ * ```
+ */
+export declare class ServiceLocator implements ServiceLocatorInterface {
+    #private;
+    /**
+     * Creates a new ServiceLocator instance.
+     *
+     * @param configuration Optional configuration instance to use
+     * @param eventManager Optional event manager instance to use
+     * @param storageBackend Optional storage backend instance to use
+     * @param logger Optional logger instance to use
+     */
+    constructor(configuration?: Configuration, eventManager?: EventManager, storageBackend?: StorageBackend, logger?: CrawleeLogger);
+    /** @internal */
+    getServicesIfSet(): {
+        configuration?: Configuration;
+        eventManager?: EventManager;
+        storageBackend?: StorageBackend;
+        logger?: CrawleeLogger;
+    };
+    getConfiguration(): Configuration;
+    setConfiguration(configuration: Configuration): void;
+    getEventManager(): EventManager;
+    setEventManager(eventManager: EventManager): void;
+    getStorageBackend(): StorageBackend;
+    setStorageBackend(storageBackend: StorageBackend): void;
+    getLogger(): CrawleeLogger;
+    setLogger(logger: CrawleeLogger): void;
+    getChildLog(prefix: string): CrawleeLogger;
+    getStorageInstanceManager(): StorageInstanceManager;
+    reset(): void;
+}
+/**
+ * Wraps all methods on `target` so that any code they invoke will see the given
+ * `serviceLocator` via `AsyncLocalStorage`, rather than the global one.
+ *
+ * Walks the prototype chain and replaces each method on the *instance* (not the prototype)
+ * with a wrapper that calls `serviceLocatorStorage.run(serviceLocator, originalMethod)`.
+ *
+ * The `AsyncLocalStorage` context propagates through the entire sync/async call tree of each
+ * wrapped method — including `super` calls, since the prototype methods execute within the
+ * context established by the instance-level wrapper.
+ *
+ * @internal
+ * @returns Scope control functions: `run` executes a callback within the scoped context,
+ *   `enterScope`/`exitScope` allow entering/leaving the scope imperatively (e.g., for constructor bodies).
+ */
+export declare function bindMethodsToServiceLocator(serviceLocator: ServiceLocator, target: {}): {
+    run: <T>(fn: () => T) => T;
+    enterScope: () => void;
+    exitScope: () => void;
+};
+export declare const serviceLocator: ServiceLocatorInterface;
+export {};

package/service_locator.js

@@ -0,0 +1,253 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { FileSystemStorageBackend } from '@crawlee/fs-storage';
+import log from '@apify/log';
+import { Configuration } from './configuration.js';
+import { ServiceConflictError } from './errors.js';
+import { LocalEventManager } from './events/local_event_manager.js';
+import { ApifyLogAdapter } from './log.js';
+import { MemoryStorageBackend } from './memory-storage/index.js';
+import { StorageInstanceManager } from './storages/storage_instance_manager.js';
+/**
+ * Service locator for managing the services used by Crawlee.
+ *
+ * All services are initialized to their default value lazily.
+ *
+ * There are two primary usage patterns:
+ *
+ * **1. Global service locator (for default services):**
+ * ```typescript
+ * import { serviceLocator, BasicCrawler } from 'crawlee';
+ *
+ * // Optionally configure global services before creating crawlers
+ * serviceLocator.setStorageBackend(myCustomClient);
+ *
+ * // Crawler uses global services
+ * const crawler = new BasicCrawler({ ... });
+ * ```
+ *
+ * **2. Per-crawler services (recommended for isolation):**
+ * ```typescript
+ * import { BasicCrawler, Configuration, LocalEventManager, MemoryStorageBackend } from 'crawlee';
+ *
+ * const crawler = new BasicCrawler({
+ *     requestHandler: async ({ request }) => { ... },
+ *     configuration: new Configuration({ ... }),           // custom configuration
+ *     storageBackend: new MemoryStorageBackend(),          // custom storage
+ *     eventManager: LocalEventManager.fromConfiguration(), // custom events
+ * });
+ * // Crawler has its own isolated ServiceLocator instance
+ * ```
+ */
+export class ServiceLocator {
+    #configuration;
+    #eventManager;
+    #storageBackend;
+    #logger;
+    /**
+     * Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
+     * Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
+     * Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
+     */
+    static #storageInstanceManager;
+    /**
+     * Creates a new ServiceLocator instance.
+     *
+     * @param configuration Optional configuration instance to use
+     * @param eventManager Optional event manager instance to use
+     * @param storageBackend Optional storage backend instance to use
+     * @param logger Optional logger instance to use
+     */
+    constructor(configuration, eventManager, storageBackend, logger) {
+        this.#configuration = configuration;
+        this.#eventManager = eventManager;
+        this.#storageBackend = storageBackend;
+        this.#logger = logger;
+    }
+    /** @internal */
+    getServicesIfSet() {
+        return {
+            configuration: this.#configuration,
+            eventManager: this.#eventManager,
+            storageBackend: this.#storageBackend,
+            logger: this.#logger,
+        };
+    }
+    getConfiguration() {
+        if (!this.#configuration) {
+            this.getLogger().debug('No configuration set, implicitly creating and using default Configuration.');
+            this.#configuration = new Configuration();
+        }
+        return this.#configuration;
+    }
+    setConfiguration(configuration) {
+        // Same instance, no need to do anything
+        if (this.#configuration === configuration) {
+            return;
+        }
+        // Already have a different configuration that was retrieved
+        if (this.#configuration) {
+            throw new ServiceConflictError('Configuration', configuration, this.#configuration);
+        }
+        this.#configuration = configuration;
+    }
+    getEventManager() {
+        if (!this.#eventManager) {
+            this.getLogger().debug('No event manager set, implicitly creating and using default LocalEventManager.');
+            if (!this.#configuration) {
+                this.getLogger().warning('Implicit creation of event manager will implicitly set configuration as side effect. ' +
+                    'It is advised to explicitly first set the configuration instead.');
+            }
+            this.#eventManager = LocalEventManager.fromConfiguration(this.getConfiguration());
+        }
+        return this.#eventManager;
+    }
+    setEventManager(eventManager) {
+        // Same instance, no need to do anything
+        if (this.#eventManager === eventManager) {
+            return;
+        }
+        // Already have a different event manager that was retrieved
+        if (this.#eventManager) {
+            throw new ServiceConflictError('EventManager', eventManager, this.#eventManager);
+        }
+        this.#eventManager = eventManager;
+    }
+    getStorageBackend() {
+        if (!this.#storageBackend) {
+            this.getLogger().debug('No storage backend set, implicitly creating and using the default storage backend ' +
+                '(FileSystemStorageBackend when persistStorage is enabled, MemoryStorageBackend otherwise).');
+            if (!this.#configuration) {
+                this.getLogger().warning('Implicit creation of storage backend will implicitly set configuration as side effect. ' +
+                    'It is advised to explicitly first set the configuration instead.');
+            }
+            const configuration = this.getConfiguration();
+            this.#storageBackend = configuration.persistStorage
+                ? new FileSystemStorageBackend({
+                    localDataDirectory: configuration.storageDir,
+                    logger: this.getLogger().child({ prefix: 'FileSystemStorageBackend' }),
+                })
+                : new MemoryStorageBackend({
+                    logger: this.getLogger().child({ prefix: 'MemoryStorageBackend' }),
+                });
+        }
+        return this.#storageBackend;
+    }
+    setStorageBackend(storageBackend) {
+        // Same instance, no need to do anything
+        if (this.#storageBackend === storageBackend) {
+            return;
+        }
+        // Already have a different storage backend that was retrieved
+        if (this.#storageBackend) {
+            throw new ServiceConflictError('StorageBackend', storageBackend, this.#storageBackend);
+        }
+        this.#storageBackend = storageBackend;
+    }
+    getLogger() {
+        if (!this.#logger) {
+            this.#logger = new ApifyLogAdapter(log);
+        }
+        return this.#logger;
+    }
+    setLogger(logger) {
+        if (this.#logger === logger) {
+            return;
+        }
+        if (this.#logger) {
+            throw new ServiceConflictError('Logger', logger, this.#logger);
+        }
+        this.#logger = logger;
+    }
+    getChildLog(prefix) {
+        return this.getLogger().child({ prefix });
+    }
+    getStorageInstanceManager() {
+        if (!ServiceLocator.#storageInstanceManager) {
+            ServiceLocator.#storageInstanceManager = new StorageInstanceManager();
+        }
+        return ServiceLocator.#storageInstanceManager;
+    }
+    reset() {
+        this.#configuration = undefined;
+        this.#eventManager = undefined;
+        this.#storageBackend = undefined;
+        this.#logger = undefined;
+        ServiceLocator.#storageInstanceManager?.clearCache();
+        ServiceLocator.#storageInstanceManager = undefined;
+    }
+}
+/**
+ * Used as the default service provider when crawlers don't specify custom services.
+ */
+const globalServiceLocator = new ServiceLocator();
+const serviceLocatorStorage = new AsyncLocalStorage();
+/**
+ * Wraps all methods on `target` so that any code they invoke will see the given
+ * `serviceLocator` via `AsyncLocalStorage`, rather than the global one.
+ *
+ * Walks the prototype chain and replaces each method on the *instance* (not the prototype)
+ * with a wrapper that calls `serviceLocatorStorage.run(serviceLocator, originalMethod)`.
+ *
+ * The `AsyncLocalStorage` context propagates through the entire sync/async call tree of each
+ * wrapped method — including `super` calls, since the prototype methods execute within the
+ * context established by the instance-level wrapper.
+ *
+ * @internal
+ * @returns Scope control functions: `run` executes a callback within the scoped context,
+ *   `enterScope`/`exitScope` allow entering/leaving the scope imperatively (e.g., for constructor bodies).
+ */
+export function bindMethodsToServiceLocator(serviceLocator, target) {
+    let proto = Object.getPrototypeOf(target);
+    const seenKeys = new Set();
+    while (proto !== null && proto !== Object.prototype) {
+        const propertyKeys = [...Object.getOwnPropertyNames(proto), ...Object.getOwnPropertySymbols(proto)];
+        for (const propertyKey of propertyKeys) {
+            // The chain is walked derived-first, so the first occurrence of a key is the one dynamic
+            // dispatch would pick — a subclass override must not be clobbered by its base version.
+            if (seenKeys.has(propertyKey))
+                continue;
+            seenKeys.add(propertyKey);
+            const descriptor = Object.getOwnPropertyDescriptor(proto, propertyKey);
+            // We use property descriptors rather than accessing target[propertyKey] directly,
+            // because that would trigger getters and cause unwanted side effects.
+            // Skip getters, setters, and constructors — only wrap regular methods.
+            if (propertyKey === 'constructor' ||
+                !descriptor ||
+                descriptor.get ||
+                descriptor.set ||
+                typeof descriptor.value !== 'function')
+                continue;
+            const original = descriptor.value;
+            target[propertyKey] = (...args) => {
+                return serviceLocatorStorage.run(serviceLocator, () => {
+                    return original.apply(target, args);
+                });
+            };
+        }
+        proto = Object.getPrototypeOf(proto);
+    }
+    let previousStore;
+    return {
+        run: (fn) => serviceLocatorStorage.run(serviceLocator, fn),
+        enterScope: () => {
+            previousStore = serviceLocatorStorage.getStore();
+            serviceLocatorStorage.enterWith(serviceLocator);
+        },
+        exitScope: () => {
+            serviceLocatorStorage.enterWith(previousStore); // casting to any so that `undefined` is accepted - this "unsets" the AsyncLocalStorage
+        },
+    };
+}
+export const serviceLocator = new Proxy({}, {
+    get(_target, prop) {
+        const active = serviceLocatorStorage.getStore() ?? globalServiceLocator;
+        const value = Reflect.get(active, prop, active);
+        if (typeof value === 'function') {
+            return value.bind(active);
+        }
+        return value;
+    },
+    set(_target, prop) {
+        throw new TypeError(`Cannot set property '${String(prop)}' on serviceLocator directly. Use the setter methods (e.g. setConfiguration(), setStorageBackend()) instead.`);
+    },
+});

package/session_pool/consts.d.ts

@@ -1,4 +1,3 @@
 export declare const BLOCKED_STATUS_CODES: number[];
-export declare const PERSIST_STATE_KEY = "SDK_SESSION_POOL_STATE";
+export declare const PERSIST_STATE_KEY = "CRAWLEE_SESSION_POOL_STATE";
 export declare const MAX_POOL_SIZE = 1000;
-//# sourceMappingURL=consts.d.ts.map

package/session_pool/consts.js

@@ -1,4 +1,3 @@
 export const BLOCKED_STATUS_CODES = [401, 403, 429];
-export const PERSIST_STATE_KEY = 'SDK_SESSION_POOL_STATE';
+export const PERSIST_STATE_KEY = 'CRAWLEE_SESSION_POOL_STATE';
 export const MAX_POOL_SIZE = 1000;
-//# sourceMappingURL=consts.js.map

package/session_pool/errors.d.ts

@@ -5,4 +5,3 @@ export declare class CookieParseError extends Error {
     readonly cookieHeaderString: unknown;
     constructor(cookieHeaderString: unknown);
 }
-//# sourceMappingURL=errors.d.ts.map

package/session_pool/errors.js

@@ -9,4 +9,3 @@ export class CookieParseError extends Error {
         Error.captureStackTrace(this, CookieParseError);
     }
 }
-//# sourceMappingURL=errors.js.map

package/session_pool/fingerprint.d.ts

@@ -0,0 +1,9 @@
+import type { SessionFingerprint } from '@crawlee/types';
+/**
+ * Build a {@link SessionFingerprint} whose `platform` matches the host OS
+ * and whose `browser`/`device` are randomized within the realistic profiles for
+ * that platform. Used by {@link SessionPool} as the default fingerprint for
+ * freshly created sessions; callers can override by passing their own
+ * `fingerprint` in `sessionOptions`.
+ */
+export declare function createDefaultSessionFingerprint(): SessionFingerprint;

package/session_pool/fingerprint.js

@@ -0,0 +1,30 @@
+/**
+ * (browser, platform, device) combinations that correspond to setups people
+ * actually run. Anything not listed here (e.g. `edge` on android, `safari` on
+ * windows, `desktop` mobile platforms) is left out so a randomized default
+ * never produces a fingerprint that would itself be a giveaway.
+ */
+const PROFILES_BY_PLATFORM = [
+    { browser: 'chrome', platform: 'windows', device: 'desktop' },
+    { browser: 'firefox', platform: 'windows', device: 'desktop' },
+    { browser: 'edge', platform: 'windows', device: 'desktop' },
+    { browser: 'chrome', platform: 'macos', device: 'desktop' },
+    { browser: 'firefox', platform: 'macos', device: 'desktop' },
+    { browser: 'safari', platform: 'macos', device: 'desktop' },
+    { browser: 'edge', platform: 'macos', device: 'desktop' },
+    { browser: 'chrome', platform: 'linux', device: 'desktop' },
+    { browser: 'firefox', platform: 'linux', device: 'desktop' },
+    { browser: 'chrome', platform: 'android', device: 'mobile' },
+    { browser: 'firefox', platform: 'android', device: 'mobile' },
+    { browser: 'safari', platform: 'ios', device: 'mobile' },
+];
+/**
+ * Build a {@link SessionFingerprint} whose `platform` matches the host OS
+ * and whose `browser`/`device` are randomized within the realistic profiles for
+ * that platform. Used by {@link SessionPool} as the default fingerprint for
+ * freshly created sessions; callers can override by passing their own
+ * `fingerprint` in `sessionOptions`.
+ */
+export function createDefaultSessionFingerprint() {
+    return { ...PROFILES_BY_PLATFORM[Math.floor(Math.random() * PROFILES_BY_PLATFORM.length)] };
+}

package/session_pool/index.d.ts

@@ -1,6 +1,4 @@
 export * from './errors.js';
-export * from './events.js';
 export * from './session.js';
 export * from './session_pool.js';
 export * from './consts.js';
-//# sourceMappingURL=index.d.ts.map

package/session_pool/index.js

@@ -1,6 +1,4 @@
 export * from './errors.js';
-export * from './events.js';
 export * from './session.js';
 export * from './session_pool.js';
 export * from './consts.js';
-//# sourceMappingURL=index.js.map