@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1

package/service_locator.js

@@ -39,16 +39,16 @@ import { StorageInstanceManager } from './storages/storage_instance_manager.js';
  * ```
  */
 export class ServiceLocator {
-    configuration;
-    eventManager;
-    storageBackend;
-    logger;
+    #configuration;
+    #eventManager;
+    #storageBackend;
+    #logger;
     /**
      * Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
      * Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
      * Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
      */
-    static storageInstanceManager;
+    static #storageInstanceManager;
     /**
      * Creates a new ServiceLocator instance.
      *
@@ -58,61 +58,70 @@ export class ServiceLocator {
      * @param logger Optional logger instance to use
      */
     constructor(configuration, eventManager, storageBackend, logger) {
-        this.configuration = configuration;
-        this.eventManager = eventManager;
-        this.storageBackend = storageBackend;
-        this.logger = logger;
+        this.#configuration = configuration;
+        this.#eventManager = eventManager;
+        this.#storageBackend = storageBackend;
+        this.#logger = logger;
+    }
+    /** @internal */
+    getServicesIfSet() {
+        return {
+            configuration: this.#configuration,
+            eventManager: this.#eventManager,
+            storageBackend: this.#storageBackend,
+            logger: this.#logger,
+        };
     }
     getConfiguration() {
-        if (!this.configuration) {
+        if (!this.#configuration) {
             this.getLogger().debug('No configuration set, implicitly creating and using default Configuration.');
-            this.configuration = new Configuration();
+            this.#configuration = new Configuration();
         }
-        return this.configuration;
+        return this.#configuration;
     }
     setConfiguration(configuration) {
         // Same instance, no need to do anything
-        if (this.configuration === configuration) {
+        if (this.#configuration === configuration) {
             return;
         }
         // Already have a different configuration that was retrieved
-        if (this.configuration) {
-            throw new ServiceConflictError('Configuration', configuration, this.configuration);
+        if (this.#configuration) {
+            throw new ServiceConflictError('Configuration', configuration, this.#configuration);
         }
-        this.configuration = configuration;
+        this.#configuration = configuration;
     }
     getEventManager() {
-        if (!this.eventManager) {
+        if (!this.#eventManager) {
             this.getLogger().debug('No event manager set, implicitly creating and using default LocalEventManager.');
-            if (!this.configuration) {
+            if (!this.#configuration) {
                 this.getLogger().warning('Implicit creation of event manager will implicitly set configuration as side effect. ' +
                     'It is advised to explicitly first set the configuration instead.');
             }
-            this.eventManager = LocalEventManager.fromConfiguration(this.getConfiguration());
+            this.#eventManager = LocalEventManager.fromConfiguration(this.getConfiguration());
         }
-        return this.eventManager;
+        return this.#eventManager;
     }
     setEventManager(eventManager) {
         // Same instance, no need to do anything
-        if (this.eventManager === eventManager) {
+        if (this.#eventManager === eventManager) {
             return;
         }
         // Already have a different event manager that was retrieved
-        if (this.eventManager) {
-            throw new ServiceConflictError('EventManager', eventManager, this.eventManager);
+        if (this.#eventManager) {
+            throw new ServiceConflictError('EventManager', eventManager, this.#eventManager);
         }
-        this.eventManager = eventManager;
+        this.#eventManager = eventManager;
     }
     getStorageBackend() {
-        if (!this.storageBackend) {
+        if (!this.#storageBackend) {
             this.getLogger().debug('No storage backend set, implicitly creating and using the default storage backend ' +
                 '(FileSystemStorageBackend when persistStorage is enabled, MemoryStorageBackend otherwise).');
-            if (!this.configuration) {
+            if (!this.#configuration) {
                 this.getLogger().warning('Implicit creation of storage backend will implicitly set configuration as side effect. ' +
                     'It is advised to explicitly first set the configuration instead.');
             }
             const configuration = this.getConfiguration();
-            this.storageBackend = configuration.persistStorage
+            this.#storageBackend = configuration.persistStorage
                 ? new FileSystemStorageBackend({
                     localDataDirectory: configuration.storageDir,
                     logger: this.getLogger().child({ prefix: 'FileSystemStorageBackend' }),
@@ -121,50 +130,52 @@ export class ServiceLocator {
                     logger: this.getLogger().child({ prefix: 'MemoryStorageBackend' }),
                 });
         }
-        return this.storageBackend;
+        return this.#storageBackend;
     }
     setStorageBackend(storageBackend) {
         // Same instance, no need to do anything
-        if (this.storageBackend === storageBackend) {
+        if (this.#storageBackend === storageBackend) {
             return;
         }
         // Already have a different storage backend that was retrieved
-        if (this.storageBackend) {
-            throw new ServiceConflictError('StorageBackend', storageBackend, this.storageBackend);
+        if (this.#storageBackend) {
+            throw new ServiceConflictError('StorageBackend', storageBackend, this.#storageBackend);
         }
-        this.storageBackend = storageBackend;
+        this.#storageBackend = storageBackend;
     }
     getLogger() {
-        if (!this.logger) {
-            this.logger = new ApifyLogAdapter(log);
+        if (!this.#logger) {
+            this.#logger = new ApifyLogAdapter(log);
         }
-        return this.logger;
+        return this.#logger;
     }
     setLogger(logger) {
-        if (this.logger === logger) {
+        if (this.#logger === logger) {
             return;
         }
-        if (this.logger) {
-            throw new ServiceConflictError('Logger', logger, this.logger);
+        if (this.#logger) {
+            throw new ServiceConflictError('Logger', logger, this.#logger);
         }
-        this.logger = logger;
+        this.#logger = logger;
     }
     getChildLog(prefix) {
         return this.getLogger().child({ prefix });
     }
+    /** @internal */
     getStorageInstanceManager() {
-        if (!ServiceLocator.storageInstanceManager) {
-            ServiceLocator.storageInstanceManager = new StorageInstanceManager();
+        if (!ServiceLocator.#storageInstanceManager) {
+            ServiceLocator.#storageInstanceManager = new StorageInstanceManager();
         }
-        return ServiceLocator.storageInstanceManager;
+        return ServiceLocator.#storageInstanceManager;
     }
+    /** @internal */
     reset() {
-        this.configuration = undefined;
-        this.eventManager = undefined;
-        this.storageBackend = undefined;
-        this.logger = undefined;
-        ServiceLocator.storageInstanceManager?.clearCache();
-        ServiceLocator.storageInstanceManager = undefined;
+        this.#configuration = undefined;
+        this.#eventManager = undefined;
+        this.#storageBackend = undefined;
+        this.#logger = undefined;
+        ServiceLocator.#storageInstanceManager?.clearCache();
+        ServiceLocator.#storageInstanceManager = undefined;
     }
 }
 /**

package/storages/batched_adds.d.ts

@@ -0,0 +1,37 @@
+import type { ProcessedRequest } from '@crawlee/types';
+import type { Source } from '../request.js';
+import type { AddRequestsBatchedResult } from './request_queue.js';
+export interface DrainRequestBatchesOptions<TItem extends Source> {
+    /**
+     * The requests to add, already normalized by the caller. Consumed lazily: an unbounded or expensive
+     * iterable is only pulled from as far as the batching (and any `maxNewRequests` budget) requires.
+     */
+    items: AsyncGenerator<TItem>;
+    batchSize: number;
+    waitBetweenBatchesMillis: number;
+    waitForAllRequestsToBeAdded: boolean;
+    maxNewRequests?: number;
+    /**
+     * Adds a single chunk and reports what it processed.
+     *
+     * @param isInitial Whether this is the first chunk, which is added before this function returns. Later
+     *  chunks land in the background, which is why some callers cache only the first.
+     */
+    processChunk: (chunk: TItem[], isInitial: boolean) => Promise<ProcessedRequest[]>;
+    /**
+     * Called with the promise covering every chunk after the first, so the caller can keep its own
+     * `isFinished` honest while batches are still landing.
+     */
+    trackBackgroundBatches?: (batches: Promise<unknown>) => void;
+}
+/**
+ * Drives the chunk-by-chunk half of `addRequestsBatched`: the first chunk is added before returning and the
+ * rest continue in the background, paced by `waitBetweenBatchesMillis`.
+ *
+ * Callers differ only in how a chunk is added and how the input is normalized, so that is all
+ * {@link DrainRequestBatchesOptions} asks for - the budget arithmetic, the lazy chunking, the
+ * over-limit reporting and the transaction handling are identical for everyone and live here. In
+ * particular, every caller has to keep its background chunks out of a transaction they will outlive,
+ * so that is read from the ambient transaction rather than asked of the caller.
+ */
+export declare function drainRequestBatches<TItem extends Source>(options: DrainRequestBatchesOptions<TItem>): Promise<AddRequestsBatchedResult>;

package/storages/batched_adds.js

@@ -0,0 +1,73 @@
+import { setTimeout as sleep } from 'node:timers/promises';
+import { chunkedAsyncIterable, peekableAsyncIterable } from '../iterables.js';
+import { activeStorageTransaction, withDirectStorageAccess } from './transaction.js';
+/**
+ * Drives the chunk-by-chunk half of `addRequestsBatched`: the first chunk is added before returning and the
+ * rest continue in the background, paced by `waitBetweenBatchesMillis`.
+ *
+ * Callers differ only in how a chunk is added and how the input is normalized, so that is all
+ * {@link DrainRequestBatchesOptions} asks for - the budget arithmetic, the lazy chunking, the
+ * over-limit reporting and the transaction handling are identical for everyone and live here. In
+ * particular, every caller has to keep its background chunks out of a transaction they will outlive,
+ * so that is read from the ambient transaction rather than asked of the caller.
+ */
+export async function drainRequestBatches(options) {
+    const { items, batchSize, waitBetweenBatchesMillis, waitForAllRequestsToBeAdded, maxNewRequests, processChunk, trackBackgroundBatches, } = options;
+    const deferred = activeStorageTransaction()?.policy.requestQueue === 'deferred';
+    let remainingBudget = maxNewRequests ?? Infinity;
+    const requestsOverLimit = [];
+    // Never hand a chunk more than the budget allows, so an over-large final batch cannot overshoot.
+    const effectiveChunkSize = maxNewRequests !== undefined ? () => Math.min(batchSize, remainingBudget) : batchSize;
+    const chunks = peekableAsyncIterable(chunkedAsyncIterable(items, effectiveChunkSize));
+    const chunksIterator = chunks[Symbol.asyncIterator]();
+    const addChunk = async (chunk, isInitial) => {
+        const processedRequests = await processChunk(chunk, isInitial);
+        if (maxNewRequests !== undefined) {
+            remainingBudget -= processedRequests.filter((request) => !request.wasAlreadyPresent).length;
+        }
+        return processedRequests;
+    };
+    const buildResult = async (addedRequests, waitForAll) => {
+        if (maxNewRequests !== undefined) {
+            // `chunkedAsyncIterable` stops pulling once the budget-derived chunk size hits zero, so whatever
+            // is left is still sitting in `items` rather than in a chunk we have seen.
+            for await (const item of items) {
+                requestsOverLimit.push(item);
+            }
+        }
+        return { addedRequests, waitForAllRequestsToBeAdded: waitForAll, requestsOverLimit };
+    };
+    const initialChunk = await chunksIterator.peek();
+    if (initialChunk === undefined) {
+        return buildResult([], Promise.resolve([]));
+    }
+    const addedRequests = await addChunk(initialChunk, true);
+    await chunksIterator.next();
+    if ((await chunksIterator.peek()) === undefined) {
+        return buildResult(addedRequests, Promise.resolve([]));
+    }
+    const processRemainingChunks = async () => {
+        const added = [];
+        for await (const chunk of chunks) {
+            added.push(...(await addChunk(chunk, false)));
+            // Under `deferred` no chunk performs backend I/O, so pacing them would only stall the handler.
+            await sleep(deferred ? 0 : waitBetweenBatchesMillis);
+        }
+        return added;
+    };
+    // With a budget we must drain everything before we can report what went over it; under `deferred` a
+    // writer that finishes after commit would have nowhere to put its journal entries.
+    const awaitsRemainder = waitForAllRequestsToBeAdded || maxNewRequests !== undefined || deferred;
+    // An un-awaited writer outlives the transaction scope it inherits, so it must not record into a
+    // transaction that may already be closed. It writes directly - its write-through additions were never
+    // going to be rolled back anyway - which means the requests it adds are not journaled.
+    // See `StorageTransactionView.enqueuedUrls`.
+    const remainder = awaitsRemainder ? processRemainingChunks() : withDirectStorageAccess(processRemainingChunks);
+    // The caller is not obliged to await `remainder`, so give it a handler of its own - an unhandled
+    // rejection here would otherwise take the process down.
+    trackBackgroundBatches?.(remainder.catch(() => { }));
+    if (awaitsRemainder) {
+        addedRequests.push(...(await remainder));
+    }
+    return buildResult(addedRequests, remainder);
+}

package/storages/dataset.d.ts

@@ -1,6 +1,6 @@
 import type { Awaitable, DatasetBackend, DatasetInfo, Dictionary } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
-import type { CrawleeLogger } from '../log.js';
+import type { JournalEntry } from './transaction.js';
 import type { DatasetStats } from './storage_stats.js';
 import type { StorageOpenOptions } from './utils.js';
 import type { StorageIdentifier } from './storage_instance_manager.js';
@@ -139,12 +139,11 @@ export interface DatasetExportToOptions extends DatasetExportOptions {
  * @category Result Stores
  */
 export declare class Dataset<Data extends Dictionary = Dictionary> {
+    #private;
     readonly configuration: Configuration;
-    id: string;
-    name?: string;
-    backend: DatasetBackend<Data>;
-    log: CrawleeLogger;
-    private readonly statsTracker;
+    readonly id: string;
+    readonly name?: string;
+    private readonly backend;
     /**
      * @internal
      */
@@ -170,6 +169,8 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      * Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
      */
     getData(options?: DatasetDataOptions): Promise<DatasetContent<Data>>;
+    /** @internal */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
      * Returns all the data from the dataset. This will iterate through the whole dataset
      * via the `listItems()` client method, which gives you only paginated results.
@@ -312,8 +313,6 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      * @param [options] An object containing extra options for `reduce()`
      */
     reduce<T>(iteratee: DatasetReducer<T, Data>, memo: T, options?: DatasetIteratorOptions): Promise<T>;
-    private fetchEntryPages;
-    private fetchPages;
     /**
      * Returns dataset items.
      *
@@ -380,6 +379,11 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      * depending on the mode of operation.
      */
     drop(): Promise<void>;
+    /**
+     * Removes all items from the dataset but keeps the dataset itself, along with its
+     * {@link Dataset.id|`id`} and {@link Dataset.name|`name`}.
+     */
+    purge(): Promise<void>;
     /**
      * Opens a dataset and returns a promise resolving to an instance of the {@link Dataset} class.
      *
@@ -458,6 +462,7 @@ export interface DatasetReducer<T, Data> {
      */
     (memo: T, item: Data, index: number): Awaitable<T>;
 }
+/** @internal */
 export interface DatasetOptions {
     /** Resolved metadata for the dataset, as returned by the backend's `getMetadata()`. */
     metadata: DatasetInfo;