@crawlee/core 4.0.0-beta.14 → 4.0.0-beta.141

package/service_locator.js

@@ -0,0 +1,253 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { FileSystemStorageBackend } from '@crawlee/fs-storage';
+import log from '@apify/log';
+import { Configuration } from './configuration.js';
+import { ServiceConflictError } from './errors.js';
+import { LocalEventManager } from './events/local_event_manager.js';
+import { ApifyLogAdapter } from './log.js';
+import { MemoryStorageBackend } from './memory-storage/index.js';
+import { StorageInstanceManager } from './storages/storage_instance_manager.js';
+/**
+ * Service locator for managing the services used by Crawlee.
+ *
+ * All services are initialized to their default value lazily.
+ *
+ * There are two primary usage patterns:
+ *
+ * **1. Global service locator (for default services):**
+ * ```typescript
+ * import { serviceLocator, BasicCrawler } from 'crawlee';
+ *
+ * // Optionally configure global services before creating crawlers
+ * serviceLocator.setStorageBackend(myCustomClient);
+ *
+ * // Crawler uses global services
+ * const crawler = new BasicCrawler({ ... });
+ * ```
+ *
+ * **2. Per-crawler services (recommended for isolation):**
+ * ```typescript
+ * import { BasicCrawler, Configuration, LocalEventManager, MemoryStorageBackend } from 'crawlee';
+ *
+ * const crawler = new BasicCrawler({
+ *     requestHandler: async ({ request }) => { ... },
+ *     configuration: new Configuration({ ... }),           // custom configuration
+ *     storageBackend: new MemoryStorageBackend(),          // custom storage
+ *     eventManager: LocalEventManager.fromConfiguration(), // custom events
+ * });
+ * // Crawler has its own isolated ServiceLocator instance
+ * ```
+ */
+export class ServiceLocator {
+    #configuration;
+    #eventManager;
+    #storageBackend;
+    #logger;
+    /**
+     * Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
+     * Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
+     * Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
+     */
+    static #storageInstanceManager;
+    /**
+     * Creates a new ServiceLocator instance.
+     *
+     * @param configuration Optional configuration instance to use
+     * @param eventManager Optional event manager instance to use
+     * @param storageBackend Optional storage backend instance to use
+     * @param logger Optional logger instance to use
+     */
+    constructor(configuration, eventManager, storageBackend, logger) {
+        this.#configuration = configuration;
+        this.#eventManager = eventManager;
+        this.#storageBackend = storageBackend;
+        this.#logger = logger;
+    }
+    /** @internal */
+    getServicesIfSet() {
+        return {
+            configuration: this.#configuration,
+            eventManager: this.#eventManager,
+            storageBackend: this.#storageBackend,
+            logger: this.#logger,
+        };
+    }
+    getConfiguration() {
+        if (!this.#configuration) {
+            this.getLogger().debug('No configuration set, implicitly creating and using default Configuration.');
+            this.#configuration = new Configuration();
+        }
+        return this.#configuration;
+    }
+    setConfiguration(configuration) {
+        // Same instance, no need to do anything
+        if (this.#configuration === configuration) {
+            return;
+        }
+        // Already have a different configuration that was retrieved
+        if (this.#configuration) {
+            throw new ServiceConflictError('Configuration', configuration, this.#configuration);
+        }
+        this.#configuration = configuration;
+    }
+    getEventManager() {
+        if (!this.#eventManager) {
+            this.getLogger().debug('No event manager set, implicitly creating and using default LocalEventManager.');
+            if (!this.#configuration) {
+                this.getLogger().warning('Implicit creation of event manager will implicitly set configuration as side effect. ' +
+                    'It is advised to explicitly first set the configuration instead.');
+            }
+            this.#eventManager = LocalEventManager.fromConfiguration(this.getConfiguration());
+        }
+        return this.#eventManager;
+    }
+    setEventManager(eventManager) {
+        // Same instance, no need to do anything
+        if (this.#eventManager === eventManager) {
+            return;
+        }
+        // Already have a different event manager that was retrieved
+        if (this.#eventManager) {
+            throw new ServiceConflictError('EventManager', eventManager, this.#eventManager);
+        }
+        this.#eventManager = eventManager;
+    }
+    getStorageBackend() {
+        if (!this.#storageBackend) {
+            this.getLogger().debug('No storage backend set, implicitly creating and using the default storage backend ' +
+                '(FileSystemStorageBackend when persistStorage is enabled, MemoryStorageBackend otherwise).');
+            if (!this.#configuration) {
+                this.getLogger().warning('Implicit creation of storage backend will implicitly set configuration as side effect. ' +
+                    'It is advised to explicitly first set the configuration instead.');
+            }
+            const configuration = this.getConfiguration();
+            this.#storageBackend = configuration.persistStorage
+                ? new FileSystemStorageBackend({
+                    localDataDirectory: configuration.storageDir,
+                    logger: this.getLogger().child({ prefix: 'FileSystemStorageBackend' }),
+                })
+                : new MemoryStorageBackend({
+                    logger: this.getLogger().child({ prefix: 'MemoryStorageBackend' }),
+                });
+        }
+        return this.#storageBackend;
+    }
+    setStorageBackend(storageBackend) {
+        // Same instance, no need to do anything
+        if (this.#storageBackend === storageBackend) {
+            return;
+        }
+        // Already have a different storage backend that was retrieved
+        if (this.#storageBackend) {
+            throw new ServiceConflictError('StorageBackend', storageBackend, this.#storageBackend);
+        }
+        this.#storageBackend = storageBackend;
+    }
+    getLogger() {
+        if (!this.#logger) {
+            this.#logger = new ApifyLogAdapter(log);
+        }
+        return this.#logger;
+    }
+    setLogger(logger) {
+        if (this.#logger === logger) {
+            return;
+        }
+        if (this.#logger) {
+            throw new ServiceConflictError('Logger', logger, this.#logger);
+        }
+        this.#logger = logger;
+    }
+    getChildLog(prefix) {
+        return this.getLogger().child({ prefix });
+    }
+    getStorageInstanceManager() {
+        if (!ServiceLocator.#storageInstanceManager) {
+            ServiceLocator.#storageInstanceManager = new StorageInstanceManager();
+        }
+        return ServiceLocator.#storageInstanceManager;
+    }
+    reset() {
+        this.#configuration = undefined;
+        this.#eventManager = undefined;
+        this.#storageBackend = undefined;
+        this.#logger = undefined;
+        ServiceLocator.#storageInstanceManager?.clearCache();
+        ServiceLocator.#storageInstanceManager = undefined;
+    }
+}
+/**
+ * Used as the default service provider when crawlers don't specify custom services.
+ */
+const globalServiceLocator = new ServiceLocator();
+const serviceLocatorStorage = new AsyncLocalStorage();
+/**
+ * Wraps all methods on `target` so that any code they invoke will see the given
+ * `serviceLocator` via `AsyncLocalStorage`, rather than the global one.
+ *
+ * Walks the prototype chain and replaces each method on the *instance* (not the prototype)
+ * with a wrapper that calls `serviceLocatorStorage.run(serviceLocator, originalMethod)`.
+ *
+ * The `AsyncLocalStorage` context propagates through the entire sync/async call tree of each
+ * wrapped method — including `super` calls, since the prototype methods execute within the
+ * context established by the instance-level wrapper.
+ *
+ * @internal
+ * @returns Scope control functions: `run` executes a callback within the scoped context,
+ *   `enterScope`/`exitScope` allow entering/leaving the scope imperatively (e.g., for constructor bodies).
+ */
+export function bindMethodsToServiceLocator(serviceLocator, target) {
+    let proto = Object.getPrototypeOf(target);
+    const seenKeys = new Set();
+    while (proto !== null && proto !== Object.prototype) {
+        const propertyKeys = [...Object.getOwnPropertyNames(proto), ...Object.getOwnPropertySymbols(proto)];
+        for (const propertyKey of propertyKeys) {
+            // The chain is walked derived-first, so the first occurrence of a key is the one dynamic
+            // dispatch would pick — a subclass override must not be clobbered by its base version.
+            if (seenKeys.has(propertyKey))
+                continue;
+            seenKeys.add(propertyKey);
+            const descriptor = Object.getOwnPropertyDescriptor(proto, propertyKey);
+            // We use property descriptors rather than accessing target[propertyKey] directly,
+            // because that would trigger getters and cause unwanted side effects.
+            // Skip getters, setters, and constructors — only wrap regular methods.
+            if (propertyKey === 'constructor' ||
+                !descriptor ||
+                descriptor.get ||
+                descriptor.set ||
+                typeof descriptor.value !== 'function')
+                continue;
+            const original = descriptor.value;
+            target[propertyKey] = (...args) => {
+                return serviceLocatorStorage.run(serviceLocator, () => {
+                    return original.apply(target, args);
+                });
+            };
+        }
+        proto = Object.getPrototypeOf(proto);
+    }
+    let previousStore;
+    return {
+        run: (fn) => serviceLocatorStorage.run(serviceLocator, fn),
+        enterScope: () => {
+            previousStore = serviceLocatorStorage.getStore();
+            serviceLocatorStorage.enterWith(serviceLocator);
+        },
+        exitScope: () => {
+            serviceLocatorStorage.enterWith(previousStore); // casting to any so that `undefined` is accepted - this "unsets" the AsyncLocalStorage
+        },
+    };
+}
+export const serviceLocator = new Proxy({}, {
+    get(_target, prop) {
+        const active = serviceLocatorStorage.getStore() ?? globalServiceLocator;
+        const value = Reflect.get(active, prop, active);
+        if (typeof value === 'function') {
+            return value.bind(active);
+        }
+        return value;
+    },
+    set(_target, prop) {
+        throw new TypeError(`Cannot set property '${String(prop)}' on serviceLocator directly. Use the setter methods (e.g. setConfiguration(), setStorageBackend()) instead.`);
+    },
+});

package/session_pool/consts.d.ts

@@ -1,4 +1,3 @@
 export declare const BLOCKED_STATUS_CODES: number[];
-export declare const PERSIST_STATE_KEY = "SDK_SESSION_POOL_STATE";
+export declare const PERSIST_STATE_KEY = "CRAWLEE_SESSION_POOL_STATE";
 export declare const MAX_POOL_SIZE = 1000;
-//# sourceMappingURL=consts.d.ts.map

package/session_pool/consts.js

@@ -1,4 +1,3 @@
 export const BLOCKED_STATUS_CODES = [401, 403, 429];
-export const PERSIST_STATE_KEY = 'SDK_SESSION_POOL_STATE';
+export const PERSIST_STATE_KEY = 'CRAWLEE_SESSION_POOL_STATE';
 export const MAX_POOL_SIZE = 1000;
-//# sourceMappingURL=consts.js.map

package/session_pool/errors.d.ts

@@ -5,4 +5,3 @@ export declare class CookieParseError extends Error {
     readonly cookieHeaderString: unknown;
     constructor(cookieHeaderString: unknown);
 }
-//# sourceMappingURL=errors.d.ts.map

package/session_pool/errors.js

@@ -9,4 +9,3 @@ export class CookieParseError extends Error {
         Error.captureStackTrace(this, CookieParseError);
     }
 }
-//# sourceMappingURL=errors.js.map

package/session_pool/fingerprint.d.ts

@@ -0,0 +1,9 @@
+import type { SessionFingerprint } from '@crawlee/types';
+/**
+ * Build a {@link SessionFingerprint} whose `platform` matches the host OS
+ * and whose `browser`/`device` are randomized within the realistic profiles for
+ * that platform. Used by {@link SessionPool} as the default fingerprint for
+ * freshly created sessions; callers can override by passing their own
+ * `fingerprint` in `sessionOptions`.
+ */
+export declare function createDefaultSessionFingerprint(): SessionFingerprint;

package/session_pool/fingerprint.js

@@ -0,0 +1,30 @@
+/**
+ * (browser, platform, device) combinations that correspond to setups people
+ * actually run. Anything not listed here (e.g. `edge` on android, `safari` on
+ * windows, `desktop` mobile platforms) is left out so a randomized default
+ * never produces a fingerprint that would itself be a giveaway.
+ */
+const PROFILES_BY_PLATFORM = [
+    { browser: 'chrome', platform: 'windows', device: 'desktop' },
+    { browser: 'firefox', platform: 'windows', device: 'desktop' },
+    { browser: 'edge', platform: 'windows', device: 'desktop' },
+    { browser: 'chrome', platform: 'macos', device: 'desktop' },
+    { browser: 'firefox', platform: 'macos', device: 'desktop' },
+    { browser: 'safari', platform: 'macos', device: 'desktop' },
+    { browser: 'edge', platform: 'macos', device: 'desktop' },
+    { browser: 'chrome', platform: 'linux', device: 'desktop' },
+    { browser: 'firefox', platform: 'linux', device: 'desktop' },
+    { browser: 'chrome', platform: 'android', device: 'mobile' },
+    { browser: 'firefox', platform: 'android', device: 'mobile' },
+    { browser: 'safari', platform: 'ios', device: 'mobile' },
+];
+/**
+ * Build a {@link SessionFingerprint} whose `platform` matches the host OS
+ * and whose `browser`/`device` are randomized within the realistic profiles for
+ * that platform. Used by {@link SessionPool} as the default fingerprint for
+ * freshly created sessions; callers can override by passing their own
+ * `fingerprint` in `sessionOptions`.
+ */
+export function createDefaultSessionFingerprint() {
+    return { ...PROFILES_BY_PLATFORM[Math.floor(Math.random() * PROFILES_BY_PLATFORM.length)] };
+}

package/session_pool/index.d.ts

@@ -1,6 +1,4 @@
 export * from './errors.js';
-export * from './events.js';
 export * from './session.js';
 export * from './session_pool.js';
 export * from './consts.js';
-//# sourceMappingURL=index.d.ts.map

package/session_pool/index.js

@@ -1,6 +1,4 @@
 export * from './errors.js';
-export * from './events.js';
 export * from './session.js';
 export * from './session_pool.js';
 export * from './consts.js';
-//# sourceMappingURL=index.js.map

package/session_pool/session.d.ts

@@ -1,24 +1,6 @@
-import type { Cookie as CookieObject, Dictionary } from '@crawlee/types';
-import type { Cookie, SerializedCookieJar } from 'tough-cookie';
+import type { Dictionary, ISession, ProxyInfo, SessionFingerprint, SessionState } from '@crawlee/types';
 import { CookieJar } from 'tough-cookie';
-import type { Log } from '@apify/log';
-import type { ProxyInfo } from '../proxy_configuration.js';
-/**
- * Persistable {@link Session} state.
- */
-export interface SessionState {
-    id: string;
-    cookieJar: SerializedCookieJar;
-    proxyInfo?: ProxyInfo;
-    userData: object;
-    errorScore: number;
-    maxErrorScore: number;
-    errorScoreDecrement: number;
-    usageCount: number;
-    maxUsageCount: number;
-    expiresAt: string;
-    createdAt: string;
-}
+import type { CrawleeLogger } from '../log.js';
 export interface SessionOptions {
     /** Id of session used for generating fingerprints. It is used as proxy session name. */
     id?: string;
@@ -59,12 +41,22 @@ export interface SessionOptions {
      * @default 50
      */
     maxUsageCount?: number;
-    /** SessionPool instance. Session will emit the `sessionRetired` event on this instance. */
-    sessionPool?: import('./session_pool.js').SessionPool;
-    log?: Log;
+    /**
+     * Marks the session as already retired. Used when restoring a previously persisted session
+     * so that `isUsable()` reflects the terminal state regardless of error score or usage count.
+     * @default false
+     */
+    retired?: boolean;
+    log?: CrawleeLogger;
     errorScore?: number;
     cookieJar?: CookieJar;
     proxyInfo?: ProxyInfo;
+    /**
+     * Browser / HTTP client fingerprint tied to this session. Backends use this to make
+     * repeated requests with the same session look consistent (same user-agent, headers,
+     * TLS profile). See {@link SessionFingerprint}.
+     */
+    fingerprint?: SessionFingerprint;
 }
 /**
  * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
@@ -72,21 +64,10 @@ export interface SessionOptions {
  * Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
  * @category Scaling
  */
-export declare class Session {
+export declare class Session implements ISession {
+    #private;
     readonly id: string;
-    private maxAgeSecs;
-    userData: Dictionary;
-    private _maxErrorScore;
-    private _errorScoreDecrement;
-    private _createdAt;
-    private _expiresAt;
-    private _usageCount;
-    private _maxUsageCount;
-    private sessionPool;
-    private _errorScore;
-    private _proxyInfo?;
-    private _cookieJar;
-    private log;
+    readonly userData: Dictionary;
     get errorScore(): number;
     get usageCount(): number;
     get maxErrorScore(): number;
@@ -96,10 +77,17 @@ export declare class Session {
     get maxUsageCount(): number;
     get cookieJar(): CookieJar;
     get proxyInfo(): ProxyInfo | undefined;
+    get fingerprint(): SessionFingerprint | undefined;
+    set fingerprint(fingerprint: SessionFingerprint | undefined);
+    /**
+     * `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
+     * a retired session is never picked by the pool and cannot be revived via `markGood()`.
+     */
+    get retired(): boolean;
     /**
      * Session configuration.
      */
-    constructor(options: SessionOptions);
+    constructor(options?: SessionOptions);
     /**
      * Indicates whether the session is blocked.
      * Session is blocked once it reaches the `maxErrorScore`.
@@ -118,7 +106,7 @@ export declare class Session {
     isMaxUsageCountReached(): boolean;
     /**
      * Indicates whether the session can be used for next requests.
-     * Session is usable when it is not expired, not blocked and the maximum usage count has not be reached.
+     * Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
      */
     isUsable(): boolean;
     /**
@@ -132,11 +120,11 @@ export declare class Session {
      */
     getState(): SessionState;
     /**
-     * Marks session as blocked and emits event on the `SessionPool`
-     * This method should be used if the session usage was unsuccessful
-     * and you are sure that it is because of the session configuration and not any external matters.
-     * For example when server returns 403 status code.
-     * If the session does not work due to some external factors as server error such as 5XX you probably want to use `markBad` method.
+     * Permanently retires the session — `isUsable()` will return `false` from here on,
+     * and no `markGood()` / `markBad()` can revive it. Calling `retire()` again is a no-op.
+     *
+     * Use this when you're confident the session itself is the problem (e.g. a `403` response).
+     * For transient external failures (such as `5XX` responses), use `markBad()` instead.
      */
     retire(): void;
     /**
@@ -144,60 +132,19 @@ export declare class Session {
      * Should be used when the session has been used unsuccessfully. For example because of timeouts.
      */
     markBad(): void;
-    /**
-     * With certain status codes: `401`, `403` or `429` we can be certain
-     * that the target website is blocking us. This function helps to do this conveniently
-     * by retiring the session when such code is received. Optionally, the default status
-     * codes can be extended in the second parameter.
-     * @param statusCode HTTP status code.
-     * @returns Whether the session was retired.
-     */
-    retireOnBlockedStatusCodes(statusCode: number): boolean;
-    /**
-     * Saves cookies from an HTTP response to be used with the session.
-     * It expects an object with a `headers` property that's either an `Object`
-     * (typical Node.js responses) or a `Function` (Puppeteer Response).
-     *
-     * It then parses and saves the cookies from the `set-cookie` header, if available.
-     */
-    setCookiesFromResponse(response: Response): void;
-    /**
-     * Saves an array with cookie objects to be used with the session.
-     * The objects should be in the format that
-     * [Puppeteer uses](https://pptr.dev/#?product=Puppeteer&version=v2.0.0&show=api-pagecookiesurls),
-     * but you can also use this function to set cookies manually:
-     *
-     * ```
-     * [
-     *   { name: 'cookie1', value: 'my-cookie' },
-     *   { name: 'cookie2', value: 'your-cookie' }
-     * ]
-     * ```
-     */
-    setCookies(cookies: CookieObject[], url: string): void;
-    /**
-     * Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
-     * @param url website url. Only cookies stored for this url will be returned
-     */
-    getCookies(url: string): CookieObject[];
     /**
      * Returns cookies saved with the session in the typical
      * key1=value1; key2=value2 format, ready to be used in
      * a cookie header or elsewhere.
      * @returns Represents `Cookie` header.
      */
-    getCookieString(url: string): string;
+    getCookieString(url: string): Promise<string>;
     /**
      * Sets a cookie within this session for the specific URL.
      */
-    setCookie(rawCookie: string, url: string): void;
-    /**
-     * Sets cookies.
-     */
-    protected _setCookies(cookies: Cookie[], url: string): void;
+    setCookie(rawCookie: string, url: string): Promise<void>;
     /**
      * Checks if session is not usable. if it is not retires the session.
      */
-    protected _maybeSelfRetire(): void;
+    private maybeSelfRetire;
 }
-//# sourceMappingURL=session.d.ts.map