@crawlee/core 4.0.0-beta.23 → 4.0.0-beta.230

package/service-locator.d.ts

@@ -0,0 +1,168 @@
+import type { StorageBackend } from '@crawlee/types';
+import { Configuration } from './configuration.js';
+import type { EventManager } from './events/event-manager.js';
+import type { CrawleeLogger } from './log.js';
+import { StorageInstanceManager } from './storages/storage-instance-manager.js';
+interface ServiceLocatorInterface {
+    /**
+     * Get the configuration.
+     * Creates a default Configuration instance if none has been set.
+     */
+    getConfiguration(): Configuration;
+    /**
+     * Set the configuration.
+     *
+     * @param configuration The configuration to set
+     * @throws {ServiceConflictError} If a different configuration has already been retrieved
+     */
+    setConfiguration(configuration: Configuration): void;
+    /**
+     * Get the event manager.
+     * Creates a default LocalEventManager instance if none has been set.
+     */
+    getEventManager(): EventManager;
+    /**
+     * Set the event manager.
+     *
+     * @param eventManager The event manager to set
+     * @throws {ServiceConflictError} If a different event manager has already been retrieved
+     */
+    setEventManager(eventManager: EventManager): void;
+    /**
+     * Get the storage backend.
+     * Creates a default storage backend if none has been set — `FileSystemStorageBackend` when
+     * `persistStorage` is enabled (the default), `MemoryStorageBackend` otherwise.
+     */
+    getStorageBackend(): StorageBackend;
+    /**
+     * Set the storage backend.
+     *
+     * @param storageBackend The storage backend to set
+     * @throws {ServiceConflictError} If a different storage backend has already been retrieved
+     */
+    setStorageBackend(storageBackend: StorageBackend): void;
+    /**
+     * Get the logger.
+     * Returns the default `@apify/log` logger if none has been set.
+     */
+    getLogger(): CrawleeLogger;
+    /**
+     * Set the logger.
+     *
+     * @param logger The logger to set
+     * @throws {ServiceConflictError} If a different logger has already been retrieved
+     */
+    setLogger(logger: CrawleeLogger): void;
+    /**
+     * Get a child logger with the given prefix.
+     * Equivalent to `getLogger().child({ prefix })`.
+     */
+    getChildLog(prefix: string): CrawleeLogger;
+    /**
+     * Get the storage instance manager (shared across all storage types).
+     * @internal
+     */
+    getStorageInstanceManager(): StorageInstanceManager;
+    /**
+     * Returns the currently set services without triggering the implicit creation of defaults.
+     * Used to inherit already-materialized services into crawler-scoped service locators.
+     * @internal
+     */
+    getServicesIfSet(): {
+        configuration?: Configuration;
+        eventManager?: EventManager;
+        storageBackend?: StorageBackend;
+        logger?: CrawleeLogger;
+    };
+    /**
+     * Resets the service locator to its initial state.
+     * Used mainly for testing purposes.
+     * @internal
+     */
+    reset(): void;
+}
+/**
+ * Service locator for managing the services used by Crawlee.
+ *
+ * All services are initialized to their default value lazily.
+ *
+ * There are two primary usage patterns:
+ *
+ * **1. Global service locator (for default services):**
+ * ```typescript
+ * import { serviceLocator, BasicCrawler } from 'crawlee';
+ *
+ * // Optionally configure global services before creating crawlers
+ * serviceLocator.setStorageBackend(myCustomClient);
+ *
+ * // Crawler uses global services
+ * const crawler = new BasicCrawler({ ... });
+ * ```
+ *
+ * **2. Per-crawler services (recommended for isolation):**
+ * ```typescript
+ * import { BasicCrawler, Configuration, LocalEventManager, MemoryStorageBackend } from 'crawlee';
+ *
+ * const crawler = new BasicCrawler({
+ *     requestHandler: async ({ request }) => { ... },
+ *     configuration: new Configuration({ ... }),           // custom configuration
+ *     storageBackend: new MemoryStorageBackend(),          // custom storage
+ *     eventManager: LocalEventManager.fromConfiguration(), // custom events
+ * });
+ * // Crawler has its own isolated ServiceLocator instance
+ * ```
+ */
+export declare class ServiceLocator implements ServiceLocatorInterface {
+    #private;
+    /**
+     * Creates a new ServiceLocator instance.
+     *
+     * @param configuration Optional configuration instance to use
+     * @param eventManager Optional event manager instance to use
+     * @param storageBackend Optional storage backend instance to use
+     * @param logger Optional logger instance to use
+     */
+    constructor(configuration?: Configuration, eventManager?: EventManager, storageBackend?: StorageBackend, logger?: CrawleeLogger);
+    /** @internal */
+    getServicesIfSet(): {
+        configuration?: Configuration;
+        eventManager?: EventManager;
+        storageBackend?: StorageBackend;
+        logger?: CrawleeLogger;
+    };
+    getConfiguration(): Configuration;
+    setConfiguration(configuration: Configuration): void;
+    getEventManager(): EventManager;
+    setEventManager(eventManager: EventManager): void;
+    getStorageBackend(): StorageBackend;
+    setStorageBackend(storageBackend: StorageBackend): void;
+    getLogger(): CrawleeLogger;
+    setLogger(logger: CrawleeLogger): void;
+    getChildLog(prefix: string): CrawleeLogger;
+    /** @internal */
+    getStorageInstanceManager(): StorageInstanceManager;
+    /** @internal */
+    reset(): void;
+}
+/**
+ * Wraps all methods on `target` so that any code they invoke will see the given
+ * `serviceLocator` via `AsyncLocalStorage`, rather than the global one.
+ *
+ * Walks the prototype chain and replaces each method on the *instance* (not the prototype)
+ * with a wrapper that calls `serviceLocatorStorage.run(serviceLocator, originalMethod)`.
+ *
+ * The `AsyncLocalStorage` context propagates through the entire sync/async call tree of each
+ * wrapped method — including `super` calls, since the prototype methods execute within the
+ * context established by the instance-level wrapper.
+ *
+ * @internal
+ * @returns Scope control functions: `run` executes a callback within the scoped context,
+ *   `enterScope`/`exitScope` allow entering/leaving the scope imperatively (e.g., for constructor bodies).
+ */
+export declare function bindMethodsToServiceLocator(serviceLocator: ServiceLocator, target: {}): {
+    run: <T>(fn: () => T) => T;
+    enterScope: () => void;
+    exitScope: () => void;
+};
+export declare const serviceLocator: ServiceLocatorInterface;
+export {};

package/service-locator.js

@@ -0,0 +1,255 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { FileSystemStorageBackend } from '@crawlee/fs-storage';
+import log from '@apify/log';
+import { Configuration } from './configuration.js';
+import { ServiceConflictError } from './errors.js';
+import { LocalEventManager } from './events/local-event-manager.js';
+import { ApifyLogAdapter } from './log.js';
+import { MemoryStorageBackend } from './memory-storage/index.js';
+import { StorageInstanceManager } from './storages/storage-instance-manager.js';
+/**
+ * Service locator for managing the services used by Crawlee.
+ *
+ * All services are initialized to their default value lazily.
+ *
+ * There are two primary usage patterns:
+ *
+ * **1. Global service locator (for default services):**
+ * ```typescript
+ * import { serviceLocator, BasicCrawler } from 'crawlee';
+ *
+ * // Optionally configure global services before creating crawlers
+ * serviceLocator.setStorageBackend(myCustomClient);
+ *
+ * // Crawler uses global services
+ * const crawler = new BasicCrawler({ ... });
+ * ```
+ *
+ * **2. Per-crawler services (recommended for isolation):**
+ * ```typescript
+ * import { BasicCrawler, Configuration, LocalEventManager, MemoryStorageBackend } from 'crawlee';
+ *
+ * const crawler = new BasicCrawler({
+ *     requestHandler: async ({ request }) => { ... },
+ *     configuration: new Configuration({ ... }),           // custom configuration
+ *     storageBackend: new MemoryStorageBackend(),          // custom storage
+ *     eventManager: LocalEventManager.fromConfiguration(), // custom events
+ * });
+ * // Crawler has its own isolated ServiceLocator instance
+ * ```
+ */
+export class ServiceLocator {
+    #configuration;
+    #eventManager;
+    #storageBackend;
+    #logger;
+    /**
+     * Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
+     * Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
+     * Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
+     */
+    static #storageInstanceManager;
+    /**
+     * Creates a new ServiceLocator instance.
+     *
+     * @param configuration Optional configuration instance to use
+     * @param eventManager Optional event manager instance to use
+     * @param storageBackend Optional storage backend instance to use
+     * @param logger Optional logger instance to use
+     */
+    constructor(configuration, eventManager, storageBackend, logger) {
+        this.#configuration = configuration;
+        this.#eventManager = eventManager;
+        this.#storageBackend = storageBackend;
+        this.#logger = logger;
+    }
+    /** @internal */
+    getServicesIfSet() {
+        return {
+            configuration: this.#configuration,
+            eventManager: this.#eventManager,
+            storageBackend: this.#storageBackend,
+            logger: this.#logger,
+        };
+    }
+    getConfiguration() {
+        if (!this.#configuration) {
+            this.getLogger().debug('No configuration set, implicitly creating and using default Configuration.');
+            this.#configuration = new Configuration();
+        }
+        return this.#configuration;
+    }
+    setConfiguration(configuration) {
+        // Same instance, no need to do anything
+        if (this.#configuration === configuration) {
+            return;
+        }
+        // Already have a different configuration that was retrieved
+        if (this.#configuration) {
+            throw new ServiceConflictError('Configuration', configuration, this.#configuration);
+        }
+        this.#configuration = configuration;
+    }
+    getEventManager() {
+        if (!this.#eventManager) {
+            this.getLogger().debug('No event manager set, implicitly creating and using default LocalEventManager.');
+            if (!this.#configuration) {
+                this.getLogger().warning('Implicit creation of event manager will implicitly set configuration as side effect. ' +
+                    'It is advised to explicitly first set the configuration instead.');
+            }
+            this.#eventManager = LocalEventManager.fromConfiguration(this.getConfiguration());
+        }
+        return this.#eventManager;
+    }
+    setEventManager(eventManager) {
+        // Same instance, no need to do anything
+        if (this.#eventManager === eventManager) {
+            return;
+        }
+        // Already have a different event manager that was retrieved
+        if (this.#eventManager) {
+            throw new ServiceConflictError('EventManager', eventManager, this.#eventManager);
+        }
+        this.#eventManager = eventManager;
+    }
+    getStorageBackend() {
+        if (!this.#storageBackend) {
+            this.getLogger().debug('No storage backend set, implicitly creating and using the default storage backend ' +
+                '(FileSystemStorageBackend when persistStorage is enabled, MemoryStorageBackend otherwise).');
+            if (!this.#configuration) {
+                this.getLogger().warning('Implicit creation of storage backend will implicitly set configuration as side effect. ' +
+                    'It is advised to explicitly first set the configuration instead.');
+            }
+            const configuration = this.getConfiguration();
+            this.#storageBackend = configuration.persistStorage
+                ? new FileSystemStorageBackend({
+                    localDataDirectory: configuration.storageDir,
+                    logger: this.getLogger().child({ prefix: 'FileSystemStorageBackend' }),
+                })
+                : new MemoryStorageBackend({
+                    logger: this.getLogger().child({ prefix: 'MemoryStorageBackend' }),
+                });
+        }
+        return this.#storageBackend;
+    }
+    setStorageBackend(storageBackend) {
+        // Same instance, no need to do anything
+        if (this.#storageBackend === storageBackend) {
+            return;
+        }
+        // Already have a different storage backend that was retrieved
+        if (this.#storageBackend) {
+            throw new ServiceConflictError('StorageBackend', storageBackend, this.#storageBackend);
+        }
+        this.#storageBackend = storageBackend;
+    }
+    getLogger() {
+        if (!this.#logger) {
+            this.#logger = new ApifyLogAdapter(log);
+        }
+        return this.#logger;
+    }
+    setLogger(logger) {
+        if (this.#logger === logger) {
+            return;
+        }
+        if (this.#logger) {
+            throw new ServiceConflictError('Logger', logger, this.#logger);
+        }
+        this.#logger = logger;
+    }
+    getChildLog(prefix) {
+        return this.getLogger().child({ prefix });
+    }
+    /** @internal */
+    getStorageInstanceManager() {
+        if (!ServiceLocator.#storageInstanceManager) {
+            ServiceLocator.#storageInstanceManager = new StorageInstanceManager();
+        }
+        return ServiceLocator.#storageInstanceManager;
+    }
+    /** @internal */
+    reset() {
+        this.#configuration = undefined;
+        this.#eventManager = undefined;
+        this.#storageBackend = undefined;
+        this.#logger = undefined;
+        ServiceLocator.#storageInstanceManager?.clearCache();
+        ServiceLocator.#storageInstanceManager = undefined;
+    }
+}
+/**
+ * Used as the default service provider when crawlers don't specify custom services.
+ */
+const globalServiceLocator = new ServiceLocator();
+const serviceLocatorStorage = new AsyncLocalStorage();
+/**
+ * Wraps all methods on `target` so that any code they invoke will see the given
+ * `serviceLocator` via `AsyncLocalStorage`, rather than the global one.
+ *
+ * Walks the prototype chain and replaces each method on the *instance* (not the prototype)
+ * with a wrapper that calls `serviceLocatorStorage.run(serviceLocator, originalMethod)`.
+ *
+ * The `AsyncLocalStorage` context propagates through the entire sync/async call tree of each
+ * wrapped method — including `super` calls, since the prototype methods execute within the
+ * context established by the instance-level wrapper.
+ *
+ * @internal
+ * @returns Scope control functions: `run` executes a callback within the scoped context,
+ *   `enterScope`/`exitScope` allow entering/leaving the scope imperatively (e.g., for constructor bodies).
+ */
+export function bindMethodsToServiceLocator(serviceLocator, target) {
+    let proto = Object.getPrototypeOf(target);
+    const seenKeys = new Set();
+    while (proto !== null && proto !== Object.prototype) {
+        const propertyKeys = [...Object.getOwnPropertyNames(proto), ...Object.getOwnPropertySymbols(proto)];
+        for (const propertyKey of propertyKeys) {
+            // The chain is walked derived-first, so the first occurrence of a key is the one dynamic
+            // dispatch would pick — a subclass override must not be clobbered by its base version.
+            if (seenKeys.has(propertyKey))
+                continue;
+            seenKeys.add(propertyKey);
+            const descriptor = Object.getOwnPropertyDescriptor(proto, propertyKey);
+            // We use property descriptors rather than accessing target[propertyKey] directly,
+            // because that would trigger getters and cause unwanted side effects.
+            // Skip getters, setters, and constructors — only wrap regular methods.
+            if (propertyKey === 'constructor' ||
+                !descriptor ||
+                descriptor.get ||
+                descriptor.set ||
+                typeof descriptor.value !== 'function')
+                continue;
+            const original = descriptor.value;
+            target[propertyKey] = (...args) => {
+                return serviceLocatorStorage.run(serviceLocator, () => {
+                    return original.apply(target, args);
+                });
+            };
+        }
+        proto = Object.getPrototypeOf(proto);
+    }
+    let previousStore;
+    return {
+        run: (fn) => serviceLocatorStorage.run(serviceLocator, fn),
+        enterScope: () => {
+            previousStore = serviceLocatorStorage.getStore();
+            serviceLocatorStorage.enterWith(serviceLocator);
+        },
+        exitScope: () => {
+            serviceLocatorStorage.enterWith(previousStore); // casting to any so that `undefined` is accepted - this "unsets" the AsyncLocalStorage
+        },
+    };
+}
+export const serviceLocator = new Proxy({}, {
+    get(_target, prop) {
+        const active = serviceLocatorStorage.getStore() ?? globalServiceLocator;
+        const value = Reflect.get(active, prop, active);
+        if (typeof value === 'function') {
+            return value.bind(active);
+        }
+        return value;
+    },
+    set(_target, prop) {
+        throw new TypeError(`Cannot set property '${String(prop)}' on serviceLocator directly. Use the setter methods (e.g. setConfiguration(), setStorageBackend()) instead.`);
+    },
+});

package/storages/batched-adds.d.ts

@@ -0,0 +1,37 @@
+import type { ProcessedRequest } from '@crawlee/types';
+import type { Source } from '../request.js';
+import type { AddRequestsBatchedResult } from './request-queue.js';
+export interface DrainRequestBatchesOptions<TItem extends Source> {
+    /**
+     * The requests to add, already normalized by the caller. Consumed lazily: an unbounded or expensive
+     * iterable is only pulled from as far as the batching (and any `maxNewRequests` budget) requires.
+     */
+    items: AsyncGenerator<TItem>;
+    batchSize: number;
+    waitBetweenBatchesMillis: number;
+    waitForAllRequestsToBeAdded: boolean;
+    maxNewRequests?: number;
+    /**
+     * Adds a single chunk and reports what it processed.
+     *
+     * @param isInitial Whether this is the first chunk, which is added before this function returns. Later
+     *  chunks land in the background, which is why some callers cache only the first.
+     */
+    processChunk: (chunk: TItem[], isInitial: boolean) => Promise<ProcessedRequest[]>;
+    /**
+     * Called with the promise covering every chunk after the first, so the caller can keep its own
+     * `isFinished` honest while batches are still landing.
+     */
+    trackBackgroundBatches?: (batches: Promise<unknown>) => void;
+}
+/**
+ * Drives the chunk-by-chunk half of `addRequestsBatched`: the first chunk is added before returning and the
+ * rest continue in the background, paced by `waitBetweenBatchesMillis`.
+ *
+ * Callers differ only in how a chunk is added and how the input is normalized, so that is all
+ * {@link DrainRequestBatchesOptions} asks for - the budget arithmetic, the lazy chunking, the
+ * over-limit reporting and the transaction handling are identical for everyone and live here. In
+ * particular, every caller has to keep its background chunks out of a transaction they will outlive,
+ * so that is read from the ambient transaction rather than asked of the caller.
+ */
+export declare function drainRequestBatches<TItem extends Source>(options: DrainRequestBatchesOptions<TItem>): Promise<AddRequestsBatchedResult>;

package/storages/batched-adds.js

@@ -0,0 +1,73 @@
+import { setTimeout as sleep } from 'node:timers/promises';
+import { chunkedAsyncIterable, peekableAsyncIterable } from '../iterables.js';
+import { activeStorageTransaction, withDirectStorageAccess } from './transaction.js';
+/**
+ * Drives the chunk-by-chunk half of `addRequestsBatched`: the first chunk is added before returning and the
+ * rest continue in the background, paced by `waitBetweenBatchesMillis`.
+ *
+ * Callers differ only in how a chunk is added and how the input is normalized, so that is all
+ * {@link DrainRequestBatchesOptions} asks for - the budget arithmetic, the lazy chunking, the
+ * over-limit reporting and the transaction handling are identical for everyone and live here. In
+ * particular, every caller has to keep its background chunks out of a transaction they will outlive,
+ * so that is read from the ambient transaction rather than asked of the caller.
+ */
+export async function drainRequestBatches(options) {
+    const { items, batchSize, waitBetweenBatchesMillis, waitForAllRequestsToBeAdded, maxNewRequests, processChunk, trackBackgroundBatches, } = options;
+    const deferred = activeStorageTransaction()?.policy.requestQueue === 'deferred';
+    let remainingBudget = maxNewRequests ?? Infinity;
+    const requestsOverLimit = [];
+    // Never hand a chunk more than the budget allows, so an over-large final batch cannot overshoot.
+    const effectiveChunkSize = maxNewRequests !== undefined ? () => Math.min(batchSize, remainingBudget) : batchSize;
+    const chunks = peekableAsyncIterable(chunkedAsyncIterable(items, effectiveChunkSize));
+    const chunksIterator = chunks[Symbol.asyncIterator]();
+    const addChunk = async (chunk, isInitial) => {
+        const processedRequests = await processChunk(chunk, isInitial);
+        if (maxNewRequests !== undefined) {
+            remainingBudget -= processedRequests.filter((request) => !request.wasAlreadyPresent).length;
+        }
+        return processedRequests;
+    };
+    const buildResult = async (addedRequests, waitForAll) => {
+        if (maxNewRequests !== undefined) {
+            // `chunkedAsyncIterable` stops pulling once the budget-derived chunk size hits zero, so whatever
+            // is left is still sitting in `items` rather than in a chunk we have seen.
+            for await (const item of items) {
+                requestsOverLimit.push(item);
+            }
+        }
+        return { addedRequests, waitForAllRequestsToBeAdded: waitForAll, requestsOverLimit };
+    };
+    const initialChunk = await chunksIterator.peek();
+    if (initialChunk === undefined) {
+        return buildResult([], Promise.resolve([]));
+    }
+    const addedRequests = await addChunk(initialChunk, true);
+    await chunksIterator.next();
+    if ((await chunksIterator.peek()) === undefined) {
+        return buildResult(addedRequests, Promise.resolve([]));
+    }
+    const processRemainingChunks = async () => {
+        const added = [];
+        for await (const chunk of chunks) {
+            added.push(...(await addChunk(chunk, false)));
+            // Under `deferred` no chunk performs backend I/O, so pacing them would only stall the handler.
+            await sleep(deferred ? 0 : waitBetweenBatchesMillis);
+        }
+        return added;
+    };
+    // With a budget we must drain everything before we can report what went over it; under `deferred` a
+    // writer that finishes after commit would have nowhere to put its journal entries.
+    const awaitsRemainder = waitForAllRequestsToBeAdded || maxNewRequests !== undefined || deferred;
+    // An un-awaited writer outlives the transaction scope it inherits, so it must not record into a
+    // transaction that may already be closed. It writes directly - its write-through additions were never
+    // going to be rolled back anyway - which means the requests it adds are not journaled.
+    // See `StorageTransactionView.enqueuedUrls`.
+    const remainder = awaitsRemainder ? processRemainingChunks() : withDirectStorageAccess(processRemainingChunks);
+    // The caller is not obliged to await `remainder`, so give it a handler of its own - an unhandled
+    // rejection here would otherwise take the process down.
+    trackBackgroundBatches?.(remainder.catch(() => { }));
+    if (awaitsRemainder) {
+        addedRequests.push(...(await remainder));
+    }
+    return buildResult(addedRequests, remainder);
+}