@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.0

package/session_pool/session_pool.js

@@ -1,11 +1,29 @@
 import { AsyncQueue } from '@sapphire/async-queue';
-import ow from 'ow';
+import { z } from 'zod';
+import { EventType } from '../events/event_manager.js';
 import { serviceLocator } from '../service_locator.js';
 import { KeyValueStore } from '../storages/key_value_store.js';
+import { parseArgument, schemas, validators } from '../validators.js';
 import { MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
 import { createDefaultSessionFingerprint } from './fingerprint.js';
 import { Session } from './session.js';
 const SESSION_REUSE_STRATEGIES = ['random', 'round-robin', 'use-until-failure'];
+// `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
+// copy), so class instances like loggers keep their prototype.
+const sessionPoolOptionsSchema = z.strictObject({
+    id: z.union([schemas.anyNumber, z.string()]).optional(),
+    maxPoolSize: schemas.anyNumber.default(MAX_POOL_SIZE),
+    persistStateKeyValueStoreId: z.string().optional(),
+    persistStateKey: z.string().optional(),
+    createSessionFunction: schemas.anyFunction.optional(),
+    sessionOptions: schemas.anyObject.default(() => ({})),
+    log: validators.logger.default(() => serviceLocator.getLogger()),
+    persistenceOptions: schemas.anyObject.default(() => ({ enable: true })),
+    sessionReuseStrategy: z.enum(SESSION_REUSE_STRATEGIES).default('random'),
+});
+const createSessionOptionsSchema = z.strictObject({
+    sessionOptions: schemas.anyObject.default(() => ({})),
+});
 /**
  * Handles the rotation, creation and persistence of user-like sessions.
  * Creates a pool of {@link Session} instances, that are randomly rotated.
@@ -60,53 +78,41 @@ const SESSION_REUSE_STRATEGIES = ['random', 'round-robin', 'use-until-failure'];
  * @category Scaling
  */
 export class SessionPool {
-    static nextId = 0;
+    static #nextId = 0;
     id;
-    log;
+    #log;
+    #sessions = [];
+    // kept as TS-private: session_pool tests read/override the members below directly
     maxPoolSize;
     createSessionFunction;
     keyValueStore;
-    sessions = [];
     sessionMap = new Map();
     sessionOptions;
     persistStateKeyValueStoreId;
     persistStateKey;
-    listener;
-    events;
-    persistenceOptions;
-    sessionReuseStrategy;
-    initPromise;
-    queue = new AsyncQueue();
-    roundRobinIndex = 0;
+    #listener;
+    #events;
+    #persistenceOptions;
+    #sessionReuseStrategy;
+    #initPromise;
+    #queue = new AsyncQueue();
+    #roundRobinIndex = 0;
     constructor(options = {}) {
-        ow(options, ow.object.exactShape({
-            id: ow.optional.any(ow.number, ow.string),
-            maxPoolSize: ow.optional.number,
-            persistStateKeyValueStoreId: ow.optional.string,
-            persistStateKey: ow.optional.string,
-            createSessionFunction: ow.optional.function,
-            sessionOptions: ow.optional.object,
-            log: ow.optional.object,
-            persistenceOptions: ow.optional.object,
-            sessionReuseStrategy: ow.optional.string.oneOf([...SESSION_REUSE_STRATEGIES]),
-        }));
-        const { id, maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions = {}, log = serviceLocator.getLogger(), persistenceOptions = {
-            enable: true,
-        }, sessionReuseStrategy = 'random', } = options;
-        this.id = id != null ? String(id) : String(SessionPool.nextId++);
-        this.sessionReuseStrategy = sessionReuseStrategy;
-        this.events = serviceLocator.getEventManager();
-        this.log = log.child({ prefix: 'SessionPool' });
-        this.persistenceOptions = persistenceOptions;
+        const { id, maxPoolSize, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions, log, persistenceOptions, sessionReuseStrategy, } = parseArgument(options, sessionPoolOptionsSchema);
+        this.id = id != null ? String(id) : String(SessionPool.#nextId++);
+        this.#sessionReuseStrategy = sessionReuseStrategy;
+        this.#events = serviceLocator.getEventManager();
+        this.#log = log.child({ prefix: 'SessionPool' });
+        this.#persistenceOptions = persistenceOptions;
         // Pool Configuration
         this.maxPoolSize = maxPoolSize;
         this.createSessionFunction = createSessionFunction || this.defaultCreateSessionFunction;
         // Session configuration. The pool-scoped logger is merged into per-call sessionOptions inside
-        // `_invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions
+        // `invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions
         // having to know about it.
         this.sessionOptions = {
             ...sessionOptions,
-            log: this.log,
+            log: this.#log,
         };
         // Session keyValueStore
         this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
@@ -117,39 +123,39 @@ export class SessionPool {
      */
     async usableSessionsCount() {
         await this.ensureInitialized();
-        return this.sessions.filter((session) => session.isUsable()).length;
+        return this.#sessions.filter((session) => session.isUsable()).length;
     }
     /**
      * Gets count of retired sessions in the pool.
      */
     async retiredSessionsCount() {
         await this.ensureInitialized();
-        return this.sessions.filter((session) => !session.isUsable()).length;
+        return this.#sessions.filter((session) => !session.isUsable()).length;
     }
     /**
      * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
      * Called automatically on first use of any public method.
      */
     async ensureInitialized() {
-        if (!this.initPromise) {
-            this.initPromise = this.setupPool();
+        if (!this.#initPromise) {
+            this.#initPromise = this.setupPool();
         }
-        return this.initPromise;
+        return this.#initPromise;
     }
     async setupPool() {
-        if (!this.persistenceOptions.enable) {
+        if (!this.#persistenceOptions.enable) {
             return;
         }
         this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId ? { id: this.persistStateKeyValueStoreId } : null, {
             configuration: serviceLocator.getConfiguration(),
         });
         if (!this.persistStateKeyValueStoreId) {
-            this.log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
+            this.#log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
         }
         // in case of migration happened and SessionPool state should be restored from the keyValueStore.
         await this.maybeLoadSessionPool();
-        this.listener = this.persistState.bind(this);
-        this.events.on("persistState" /* EventType.PERSIST_STATE */, this.listener);
+        this.#listener = this.persistState.bind(this);
+        this.#events.on(EventType.PERSIST_STATE, this.#listener);
     }
     /**
      * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
@@ -169,8 +175,8 @@ export class SessionPool {
         if (!this.hasSpaceForSession()) {
             this.removeRetiredSessions();
         }
-        const newSession = options instanceof Session ? options : await this._invokeCreateSessionFunction(options);
-        this.log.debug(`Adding new Session - ${newSession.id}`);
+        const newSession = options instanceof Session ? options : await this.invokeCreateSessionFunction(options);
+        this.#log.debug(`Adding new Session - ${newSession.id}`);
         this.registerSession(newSession);
     }
     /**
@@ -181,7 +187,7 @@ export class SessionPool {
      */
     async newSession(sessionOptions) {
         await this.ensureInitialized();
-        const newSession = await this._invokeCreateSessionFunction(sessionOptions);
+        const newSession = await this.invokeCreateSessionFunction(sessionOptions);
         this.registerSession(newSession);
         return newSession;
     }
@@ -194,7 +200,7 @@ export class SessionPool {
      */
     async getSession(sessionId) {
         await this.ensureInitialized();
-        await this.queue.wait();
+        await this.#queue.wait();
         try {
             if (sessionId) {
                 const session = this.sessionMap.get(sessionId);
@@ -212,14 +218,14 @@ export class SessionPool {
             return await this.createSession();
         }
         finally {
-            this.queue.shift();
+            this.#queue.shift();
         }
     }
     /**
      * @param options - Override the persistence options provided in the constructor
      */
     async resetStore(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
+        if (!this.#persistenceOptions.enable && !options?.enable) {
             return;
         }
         await this.ensureInitialized();
@@ -234,7 +240,7 @@ export class SessionPool {
         return {
             usableSessionsCount: await this.usableSessionsCount(),
             retiredSessionsCount: await this.retiredSessionsCount(),
-            sessions: this.sessions.map((session) => session.getState()),
+            sessions: this.#sessions.map((session) => session.getState()),
         };
     }
     /**
@@ -243,40 +249,43 @@ export class SessionPool {
      * @param options - Override the persistence options provided in the constructor
      */
     async persistState(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
+        if (!this.#persistenceOptions.enable && !options?.enable) {
             return;
         }
         await this.ensureInitialized();
-        this.log.debug('Persisting state', {
+        this.#log.debug('Persisting state', {
             persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
             persistStateKey: this.persistStateKey,
         });
         await this.keyValueStore
             ?.setValue(this.persistStateKey, await this.getState())
-            .catch((error) => this.log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }));
+            .catch((error) => this.#log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }));
     }
     /**
      * Removes listener from `persistState` event.
      * This function should be called after you are done with using the `SessionPool` instance.
+     * @param options - Set `persistState` to false when the final state was already persisted by the event manager.
      */
-    async teardown() {
-        if (!this.initPromise)
+    async teardown({ persistState = true } = {}) {
+        if (!this.#initPromise)
             return;
         await this.ensureInitialized();
-        if (this.listener) {
-            this.events.off("persistState" /* EventType.PERSIST_STATE */, this.listener);
+        if (this.#listener) {
+            this.#events.off(EventType.PERSIST_STATE, this.#listener);
+        }
+        if (persistState) {
+            await this.persistState();
         }
-        await this.persistState();
     }
     /**
      * Removes retired `Session` instances from `SessionPool`.
      */
     removeRetiredSessions() {
-        this.sessions = this.sessions.filter((storedSession) => {
+        this.#sessions = this.#sessions.filter((storedSession) => {
             if (storedSession.isUsable())
                 return true;
             this.sessionMap.delete(storedSession.id);
-            this.log.debug(`Removed Session - ${storedSession.id}`);
+            this.#log.debug(`Removed Session - ${storedSession.id}`);
             return false;
         });
     }
@@ -285,14 +294,14 @@ export class SessionPool {
      * @param newSession `Session` instance to be added.
      */
     registerSession(newSession) {
-        this.sessions.push(newSession);
+        this.#sessions.push(newSession);
         this.sessionMap.set(newSession.id, newSession);
     }
     /**
      * Gets random index.
      */
     getRandomIndex() {
-        return Math.floor(Math.random() * this.sessions.length);
+        return Math.floor(Math.random() * this.#sessions.length);
     }
     /**
      * Creates new session without any extra behavior.
@@ -301,8 +310,7 @@ export class SessionPool {
      * @returns New session.
      */
     async defaultCreateSessionFunction(options = {}) {
-        ow(options, ow.object.exactShape({ sessionOptions: ow.optional.object }));
-        const { sessionOptions = {} } = options;
+        const { sessionOptions } = parseArgument(options, createSessionOptionsSchema);
         return new Session(sessionOptions);
     }
     /**
@@ -315,7 +323,7 @@ export class SessionPool {
      * through `maybeLoadSessionPool` naturally wins because it arrives in
      * `perCallOptions`.
      */
-    async _invokeCreateSessionFunction(perCallOptions) {
+    async invokeCreateSessionFunction(perCallOptions) {
         const sessionOptions = {
             fingerprint: createDefaultSessionFingerprint(),
             ...this.sessionOptions,
@@ -328,35 +336,35 @@ export class SessionPool {
      * @returns Newly created `Session` instance.
      */
     async createSession() {
-        const newSession = await this._invokeCreateSessionFunction();
+        const newSession = await this.invokeCreateSessionFunction();
         this.registerSession(newSession);
-        this.log.debug(`Created new Session - ${newSession.id}`);
+        this.#log.debug(`Created new Session - ${newSession.id}`);
         return newSession;
     }
     /**
      * Decides whether there is enough space for creating new session.
      */
     hasSpaceForSession() {
-        return this.sessions.length < this.maxPoolSize;
+        return this.#sessions.length < this.maxPoolSize;
     }
     /**
      * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
      * Returns `undefined` when no session should be reused and a new one should be created instead.
      */
     pickSession() {
-        if (this.sessionReuseStrategy !== 'use-until-failure' && this.hasSpaceForSession())
+        if (this.#sessionReuseStrategy !== 'use-until-failure' && this.hasSpaceForSession())
             return undefined;
-        if (this.sessionReuseStrategy === 'use-until-failure') {
-            return this.sessions.find((session) => session.isUsable());
+        if (this.#sessionReuseStrategy === 'use-until-failure') {
+            return this.#sessions.find((session) => session.isUsable());
         }
         let picked;
-        if (this.sessionReuseStrategy === 'round-robin') {
-            const index = this.roundRobinIndex % this.sessions.length;
-            this.roundRobinIndex = index + 1;
-            picked = this.sessions[index];
+        if (this.#sessionReuseStrategy === 'round-robin') {
+            const index = this.#roundRobinIndex % this.#sessions.length;
+            this.#roundRobinIndex = index + 1;
+            picked = this.#sessions[index];
         }
         else {
-            picked = this.sessions[this.getRandomIndex()];
+            picked = this.#sessions[this.getRandomIndex()];
         }
         return picked.isUsable() ? picked : undefined;
     }
@@ -369,18 +377,18 @@ export class SessionPool {
         if (!loadedSessionPool)
             return;
         // Invalidate old sessions and load active sessions only
-        this.log.debug('Recreating state from KeyValueStore', {
+        this.#log.debug('Recreating state from KeyValueStore', {
             persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
             persistStateKey: this.persistStateKey,
         });
         for (const sessionObject of loadedSessionPool.sessions) {
             sessionObject.createdAt = new Date(sessionObject.createdAt);
             sessionObject.expiresAt = new Date(sessionObject.expiresAt);
-            const recreatedSession = await this._invokeCreateSessionFunction(sessionObject);
+            const recreatedSession = await this.invokeCreateSessionFunction(sessionObject);
             if (recreatedSession.isUsable()) {
                 this.registerSession(recreatedSession);
             }
         }
-        this.log.debug(`${this.sessions.length} active sessions loaded from KeyValueStore`);
+        this.#log.debug(`${this.#sessions.length} active sessions loaded from KeyValueStore`);
     }
 }

package/storages/batched_adds.d.ts

@@ -0,0 +1,37 @@
+import type { ProcessedRequest } from '@crawlee/types';
+import type { Source } from '../request.js';
+import type { AddRequestsBatchedResult } from './request_queue.js';
+export interface DrainRequestBatchesOptions<TItem extends Source> {
+    /**
+     * The requests to add, already normalized by the caller. Consumed lazily: an unbounded or expensive
+     * iterable is only pulled from as far as the batching (and any `maxNewRequests` budget) requires.
+     */
+    items: AsyncGenerator<TItem>;
+    batchSize: number;
+    waitBetweenBatchesMillis: number;
+    waitForAllRequestsToBeAdded: boolean;
+    maxNewRequests?: number;
+    /**
+     * Adds a single chunk and reports what it processed.
+     *
+     * @param isInitial Whether this is the first chunk, which is added before this function returns. Later
+     *  chunks land in the background, which is why some callers cache only the first.
+     */
+    processChunk: (chunk: TItem[], isInitial: boolean) => Promise<ProcessedRequest[]>;
+    /**
+     * Called with the promise covering every chunk after the first, so the caller can keep its own
+     * `isFinished` honest while batches are still landing.
+     */
+    trackBackgroundBatches?: (batches: Promise<unknown>) => void;
+}
+/**
+ * Drives the chunk-by-chunk half of `addRequestsBatched`: the first chunk is added before returning and the
+ * rest continue in the background, paced by `waitBetweenBatchesMillis`.
+ *
+ * Callers differ only in how a chunk is added and how the input is normalized, so that is all
+ * {@link DrainRequestBatchesOptions} asks for - the budget arithmetic, the lazy chunking, the
+ * over-limit reporting and the transaction handling are identical for everyone and live here. In
+ * particular, every caller has to keep its background chunks out of a transaction they will outlive,
+ * so that is read from the ambient transaction rather than asked of the caller.
+ */
+export declare function drainRequestBatches<TItem extends Source>(options: DrainRequestBatchesOptions<TItem>): Promise<AddRequestsBatchedResult>;

package/storages/batched_adds.js

@@ -0,0 +1,73 @@
+import { setTimeout as sleep } from 'node:timers/promises';
+import { chunkedAsyncIterable, peekableAsyncIterable } from '../iterables.js';
+import { activeStorageTransaction, withDirectStorageAccess } from './transaction.js';
+/**
+ * Drives the chunk-by-chunk half of `addRequestsBatched`: the first chunk is added before returning and the
+ * rest continue in the background, paced by `waitBetweenBatchesMillis`.
+ *
+ * Callers differ only in how a chunk is added and how the input is normalized, so that is all
+ * {@link DrainRequestBatchesOptions} asks for - the budget arithmetic, the lazy chunking, the
+ * over-limit reporting and the transaction handling are identical for everyone and live here. In
+ * particular, every caller has to keep its background chunks out of a transaction they will outlive,
+ * so that is read from the ambient transaction rather than asked of the caller.
+ */
+export async function drainRequestBatches(options) {
+    const { items, batchSize, waitBetweenBatchesMillis, waitForAllRequestsToBeAdded, maxNewRequests, processChunk, trackBackgroundBatches, } = options;
+    const deferred = activeStorageTransaction()?.policy.requestQueue === 'deferred';
+    let remainingBudget = maxNewRequests ?? Infinity;
+    const requestsOverLimit = [];
+    // Never hand a chunk more than the budget allows, so an over-large final batch cannot overshoot.
+    const effectiveChunkSize = maxNewRequests !== undefined ? () => Math.min(batchSize, remainingBudget) : batchSize;
+    const chunks = peekableAsyncIterable(chunkedAsyncIterable(items, effectiveChunkSize));
+    const chunksIterator = chunks[Symbol.asyncIterator]();
+    const addChunk = async (chunk, isInitial) => {
+        const processedRequests = await processChunk(chunk, isInitial);
+        if (maxNewRequests !== undefined) {
+            remainingBudget -= processedRequests.filter((request) => !request.wasAlreadyPresent).length;
+        }
+        return processedRequests;
+    };
+    const buildResult = async (addedRequests, waitForAll) => {
+        if (maxNewRequests !== undefined) {
+            // `chunkedAsyncIterable` stops pulling once the budget-derived chunk size hits zero, so whatever
+            // is left is still sitting in `items` rather than in a chunk we have seen.
+            for await (const item of items) {
+                requestsOverLimit.push(item);
+            }
+        }
+        return { addedRequests, waitForAllRequestsToBeAdded: waitForAll, requestsOverLimit };
+    };
+    const initialChunk = await chunksIterator.peek();
+    if (initialChunk === undefined) {
+        return buildResult([], Promise.resolve([]));
+    }
+    const addedRequests = await addChunk(initialChunk, true);
+    await chunksIterator.next();
+    if ((await chunksIterator.peek()) === undefined) {
+        return buildResult(addedRequests, Promise.resolve([]));
+    }
+    const processRemainingChunks = async () => {
+        const added = [];
+        for await (const chunk of chunks) {
+            added.push(...(await addChunk(chunk, false)));
+            // Under `deferred` no chunk performs backend I/O, so pacing them would only stall the handler.
+            await sleep(deferred ? 0 : waitBetweenBatchesMillis);
+        }
+        return added;
+    };
+    // With a budget we must drain everything before we can report what went over it; under `deferred` a
+    // writer that finishes after commit would have nowhere to put its journal entries.
+    const awaitsRemainder = waitForAllRequestsToBeAdded || maxNewRequests !== undefined || deferred;
+    // An un-awaited writer outlives the transaction scope it inherits, so it must not record into a
+    // transaction that may already be closed. It writes directly - its write-through additions were never
+    // going to be rolled back anyway - which means the requests it adds are not journaled.
+    // See `StorageTransactionView.enqueuedUrls`.
+    const remainder = awaitsRemainder ? processRemainingChunks() : withDirectStorageAccess(processRemainingChunks);
+    // The caller is not obliged to await `remainder`, so give it a handler of its own - an unhandled
+    // rejection here would otherwise take the process down.
+    trackBackgroundBatches?.(remainder.catch(() => { }));
+    if (awaitsRemainder) {
+        addedRequests.push(...(await remainder));
+    }
+    return buildResult(addedRequests, remainder);
+}

package/storages/dataset.d.ts

@@ -1,6 +1,7 @@
 import type { Awaitable, DatasetBackend, DatasetInfo, Dictionary } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
 import type { CrawleeLogger } from '../log.js';
+import type { JournalEntry } from './transaction.js';
 import type { DatasetStats } from './storage_stats.js';
 import type { StorageOpenOptions } from './utils.js';
 import type { StorageIdentifier } from './storage_instance_manager.js';
@@ -139,12 +140,12 @@ export interface DatasetExportToOptions extends DatasetExportOptions {
  * @category Result Stores
  */
 export declare class Dataset<Data extends Dictionary = Dictionary> {
+    #private;
     readonly configuration: Configuration;
     id: string;
     name?: string;
     backend: DatasetBackend<Data>;
     log: CrawleeLogger;
-    private readonly statsTracker;
     /**
      * @internal
      */
@@ -170,6 +171,16 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      * Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
      */
     getData(options?: DatasetDataOptions): Promise<DatasetContent<Data>>;
+    /**
+     * The single transaction-aware page read all dataset read paths go through — both `getData()` and
+     * the private `fetchPages()`. Returns the real page concatenated with the current transaction's
+     * buffered items, with `offset` / `limit` / `desc` windowing applied across the concatenation.
+     */
+    private readPage;
+    /** The active transaction's buffered writes to this dataset, derived from its journal. */
+    private bufferedJournalEntries;
+    /** @internal */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
      * Returns all the data from the dataset. This will iterate through the whole dataset
      * via the `listItems()` client method, which gives you only paginated results.