@crawlee/core 4.0.0-beta.104 → 4.0.0-beta.106

package/session_pool/session.js

@@ -12,57 +12,57 @@ import { serviceLocator } from '../service_locator.js';
 export class Session {
     id;
     userData;
-    _maxErrorScore;
-    _errorScoreDecrement;
-    _createdAt;
-    _expiresAt;
-    _usageCount;
-    _maxUsageCount;
-    _errorScore;
-    _retired = false;
-    _proxyInfo;
-    _cookieJar;
-    _fingerprint;
-    log;
+    #maxErrorScore;
+    #errorScoreDecrement;
+    #createdAt;
+    #expiresAt;
+    #usageCount;
+    #maxUsageCount;
+    #errorScore;
+    #retired = false;
+    #proxyInfo;
+    #cookieJar;
+    #fingerprint;
+    #log;
     get errorScore() {
-        return this._errorScore;
+        return this.#errorScore;
     }
     get usageCount() {
-        return this._usageCount;
+        return this.#usageCount;
     }
     get maxErrorScore() {
-        return this._maxErrorScore;
+        return this.#maxErrorScore;
     }
     get errorScoreDecrement() {
-        return this._errorScoreDecrement;
+        return this.#errorScoreDecrement;
     }
     get expiresAt() {
-        return this._expiresAt;
+        return this.#expiresAt;
     }
     get createdAt() {
-        return this._createdAt;
+        return this.#createdAt;
     }
     get maxUsageCount() {
-        return this._maxUsageCount;
+        return this.#maxUsageCount;
     }
     get cookieJar() {
-        return this._cookieJar;
+        return this.#cookieJar;
     }
     get proxyInfo() {
-        return this._proxyInfo;
+        return this.#proxyInfo;
     }
     get fingerprint() {
-        return this._fingerprint;
+        return this.#fingerprint;
     }
     set fingerprint(fingerprint) {
-        this._fingerprint = fingerprint;
+        this.#fingerprint = fingerprint;
     }
     /**
      * `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
      * a retired session is never picked by the pool and cannot be revived via `markGood()`.
      */
     get retired() {
-        return this._retired;
+        return this.#retired;
     }
     /**
      * Session configuration.
@@ -87,21 +87,21 @@ export class Session {
         }));
         const { id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), proxyInfo = undefined, maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, retired = false, log = serviceLocator.getLogger(), fingerprint, } = options;
         const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
-        this.log = log.child({ prefix: 'Session' });
-        this._cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
-        this._proxyInfo = proxyInfo;
-        this._fingerprint = fingerprint;
+        this.#log = log.child({ prefix: 'Session' });
+        this.#cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
+        this.#proxyInfo = proxyInfo;
+        this.#fingerprint = fingerprint;
         this.id = id;
         this.userData = userData;
-        this._maxErrorScore = maxErrorScore;
-        this._errorScoreDecrement = errorScoreDecrement;
+        this.#maxErrorScore = maxErrorScore;
+        this.#errorScoreDecrement = errorScoreDecrement;
         // Internal
-        this._expiresAt = expiresAt;
-        this._createdAt = createdAt;
-        this._usageCount = usageCount; // indicates how many times the session has been used
-        this._errorScore = errorScore; // indicates number of markBaded request with the session
-        this._maxUsageCount = maxUsageCount;
-        this._retired = retired;
+        this.#expiresAt = expiresAt;
+        this.#createdAt = createdAt;
+        this.#usageCount = usageCount; // indicates how many times the session has been used
+        this.#errorScore = errorScore; // indicates number of markBaded request with the session
+        this.#maxUsageCount = maxUsageCount;
+        this.#retired = retired;
     }
     /**
      * Indicates whether the session is blocked.
@@ -130,16 +130,16 @@ export class Session {
      * Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
      */
     isUsable() {
-        return !this._retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
+        return !this.#retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
     }
     /**
      * This method should be called after a successful session usage.
      * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
      */
     markGood() {
-        this._usageCount += 1;
-        if (this._errorScore > 0) {
-            this._errorScore -= this._errorScoreDecrement;
+        this.#usageCount += 1;
+        if (this.#errorScore > 0) {
+            this.#errorScore -= this.#errorScoreDecrement;
         }
         this.maybeSelfRetire();
     }
@@ -151,9 +151,9 @@ export class Session {
         return {
             id: this.id,
             cookieJar: this.cookieJar.toJSON(),
-            proxyInfo: this._proxyInfo,
+            proxyInfo: this.#proxyInfo,
             userData: this.userData,
-            fingerprint: this._fingerprint,
+            fingerprint: this.#fingerprint,
             maxErrorScore: this.maxErrorScore,
             errorScoreDecrement: this.errorScoreDecrement,
             expiresAt: this.expiresAt.toISOString(),
@@ -161,7 +161,7 @@ export class Session {
             usageCount: this.usageCount,
             maxUsageCount: this.maxUsageCount,
             errorScore: this.errorScore,
-            retired: this._retired,
+            retired: this.#retired,
         };
     }
     /**
@@ -172,19 +172,19 @@ export class Session {
      * For transient external failures (such as `5XX` responses), use `markBad()` instead.
      */
     retire() {
-        if (this._retired)
+        if (this.#retired)
             return;
-        this._errorScore += this._maxErrorScore;
-        this._usageCount += 1;
-        this._retired = true;
+        this.#errorScore += this.#maxErrorScore;
+        this.#usageCount += 1;
+        this.#retired = true;
     }
     /**
      * Increases usage and error count.
      * Should be used when the session has been used unsuccessfully. For example because of timeouts.
      */
     markBad() {
-        this._errorScore += 1;
-        this._usageCount += 1;
+        this.#errorScore += 1;
+        this.#usageCount += 1;
         this.maybeSelfRetire();
     }
     /**
@@ -204,7 +204,7 @@ export class Session {
             this.cookieJar.setCookieSync(rawCookie, url);
         }
         catch (e) {
-            this.log.warning('Could not set cookie.', { url, error: e.message });
+            this.#log.warning('Could not set cookie.', { url, error: e.message });
         }
     }
     /**

package/session_pool/session_pool.d.ts

@@ -112,24 +112,15 @@ export interface SessionPoolOptions {
  * @category Scaling
  */
 export declare class SessionPool implements ISessionPool {
-    private static nextId;
+    #private;
     readonly id: string;
-    private log;
     private maxPoolSize;
     private createSessionFunction;
     private keyValueStore?;
-    private sessions;
     private sessionMap;
     private sessionOptions;
     private persistStateKeyValueStoreId?;
     private persistStateKey;
-    private listener?;
-    private events;
-    private persistenceOptions;
-    private sessionReuseStrategy;
-    private initPromise?;
-    private queue;
-    private roundRobinIndex;
     constructor(options?: SessionPoolOptions);
     /**
      * Gets count of usable sessions in the pool.
@@ -222,7 +213,7 @@ export declare class SessionPool implements ISessionPool {
      * through `maybeLoadSessionPool` naturally wins because it arrives in
      * `perCallOptions`.
      */
-    private _invokeCreateSessionFunction;
+    private invokeCreateSessionFunction;
     /**
      * Creates new session and adds it to the pool.
      * @returns Newly created `Session` instance.

package/session_pool/session_pool.js

@@ -60,24 +60,25 @@ const SESSION_REUSE_STRATEGIES = ['random', 'round-robin', 'use-until-failure'];
  * @category Scaling
  */
 export class SessionPool {
-    static nextId = 0;
+    static #nextId = 0;
     id;
-    log;
+    #log;
+    #sessions = [];
+    // kept as TS-private: session_pool tests read/override the members below directly
     maxPoolSize;
     createSessionFunction;
     keyValueStore;
-    sessions = [];
     sessionMap = new Map();
     sessionOptions;
     persistStateKeyValueStoreId;
     persistStateKey;
-    listener;
-    events;
-    persistenceOptions;
-    sessionReuseStrategy;
-    initPromise;
-    queue = new AsyncQueue();
-    roundRobinIndex = 0;
+    #listener;
+    #events;
+    #persistenceOptions;
+    #sessionReuseStrategy;
+    #initPromise;
+    #queue = new AsyncQueue();
+    #roundRobinIndex = 0;
     constructor(options = {}) {
         ow(options, ow.object.exactShape({
             id: ow.optional.any(ow.number, ow.string),
@@ -93,20 +94,20 @@ export class SessionPool {
         const { id, maxPoolSize = MAX_POOL_SIZE, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions = {}, log = serviceLocator.getLogger(), persistenceOptions = {
             enable: true,
         }, sessionReuseStrategy = 'random', } = options;
-        this.id = id != null ? String(id) : String(SessionPool.nextId++);
-        this.sessionReuseStrategy = sessionReuseStrategy;
-        this.events = serviceLocator.getEventManager();
-        this.log = log.child({ prefix: 'SessionPool' });
-        this.persistenceOptions = persistenceOptions;
+        this.id = id != null ? String(id) : String(SessionPool.#nextId++);
+        this.#sessionReuseStrategy = sessionReuseStrategy;
+        this.#events = serviceLocator.getEventManager();
+        this.#log = log.child({ prefix: 'SessionPool' });
+        this.#persistenceOptions = persistenceOptions;
         // Pool Configuration
         this.maxPoolSize = maxPoolSize;
         this.createSessionFunction = createSessionFunction || this.defaultCreateSessionFunction;
         // Session configuration. The pool-scoped logger is merged into per-call sessionOptions inside
-        // `_invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions
+        // `invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions
         // having to know about it.
         this.sessionOptions = {
             ...sessionOptions,
-            log: this.log,
+            log: this.#log,
         };
         // Session keyValueStore
         this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
@@ -117,39 +118,39 @@ export class SessionPool {
      */
     async usableSessionsCount() {
         await this.ensureInitialized();
-        return this.sessions.filter((session) => session.isUsable()).length;
+        return this.#sessions.filter((session) => session.isUsable()).length;
     }
     /**
      * Gets count of retired sessions in the pool.
      */
     async retiredSessionsCount() {
         await this.ensureInitialized();
-        return this.sessions.filter((session) => !session.isUsable()).length;
+        return this.#sessions.filter((session) => !session.isUsable()).length;
     }
     /**
      * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
      * Called automatically on first use of any public method.
      */
     async ensureInitialized() {
-        if (!this.initPromise) {
-            this.initPromise = this.setupPool();
+        if (!this.#initPromise) {
+            this.#initPromise = this.setupPool();
         }
-        return this.initPromise;
+        return this.#initPromise;
     }
     async setupPool() {
-        if (!this.persistenceOptions.enable) {
+        if (!this.#persistenceOptions.enable) {
             return;
         }
         this.keyValueStore = await KeyValueStore.open(this.persistStateKeyValueStoreId ? { id: this.persistStateKeyValueStoreId } : null, {
             configuration: serviceLocator.getConfiguration(),
         });
         if (!this.persistStateKeyValueStoreId) {
-            this.log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
+            this.#log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
         }
         // in case of migration happened and SessionPool state should be restored from the keyValueStore.
         await this.maybeLoadSessionPool();
-        this.listener = this.persistState.bind(this);
-        this.events.on("persistState" /* EventType.PERSIST_STATE */, this.listener);
+        this.#listener = this.persistState.bind(this);
+        this.#events.on("persistState" /* EventType.PERSIST_STATE */, this.#listener);
     }
     /**
      * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
@@ -169,8 +170,8 @@ export class SessionPool {
         if (!this.hasSpaceForSession()) {
             this.removeRetiredSessions();
         }
-        const newSession = options instanceof Session ? options : await this._invokeCreateSessionFunction(options);
-        this.log.debug(`Adding new Session - ${newSession.id}`);
+        const newSession = options instanceof Session ? options : await this.invokeCreateSessionFunction(options);
+        this.#log.debug(`Adding new Session - ${newSession.id}`);
         this.registerSession(newSession);
     }
     /**
@@ -181,7 +182,7 @@ export class SessionPool {
      */
     async newSession(sessionOptions) {
         await this.ensureInitialized();
-        const newSession = await this._invokeCreateSessionFunction(sessionOptions);
+        const newSession = await this.invokeCreateSessionFunction(sessionOptions);
         this.registerSession(newSession);
         return newSession;
     }
@@ -194,7 +195,7 @@ export class SessionPool {
      */
     async getSession(sessionId) {
         await this.ensureInitialized();
-        await this.queue.wait();
+        await this.#queue.wait();
         try {
             if (sessionId) {
                 const session = this.sessionMap.get(sessionId);
@@ -212,14 +213,14 @@ export class SessionPool {
             return await this.createSession();
         }
         finally {
-            this.queue.shift();
+            this.#queue.shift();
         }
     }
     /**
      * @param options - Override the persistence options provided in the constructor
      */
     async resetStore(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
+        if (!this.#persistenceOptions.enable && !options?.enable) {
             return;
         }
         await this.ensureInitialized();
@@ -234,7 +235,7 @@ export class SessionPool {
         return {
             usableSessionsCount: await this.usableSessionsCount(),
             retiredSessionsCount: await this.retiredSessionsCount(),
-            sessions: this.sessions.map((session) => session.getState()),
+            sessions: this.#sessions.map((session) => session.getState()),
         };
     }
     /**
@@ -243,28 +244,28 @@ export class SessionPool {
      * @param options - Override the persistence options provided in the constructor
      */
     async persistState(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
+        if (!this.#persistenceOptions.enable && !options?.enable) {
             return;
         }
         await this.ensureInitialized();
-        this.log.debug('Persisting state', {
+        this.#log.debug('Persisting state', {
             persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
             persistStateKey: this.persistStateKey,
         });
         await this.keyValueStore
             ?.setValue(this.persistStateKey, await this.getState())
-            .catch((error) => this.log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }));
+            .catch((error) => this.#log.warning(`Failed to persist the session pool stats to ${this.persistStateKey}`, { error }));
     }
     /**
      * Removes listener from `persistState` event.
      * This function should be called after you are done with using the `SessionPool` instance.
      */
     async teardown() {
-        if (!this.initPromise)
+        if (!this.#initPromise)
             return;
         await this.ensureInitialized();
-        if (this.listener) {
-            this.events.off("persistState" /* EventType.PERSIST_STATE */, this.listener);
+        if (this.#listener) {
+            this.#events.off("persistState" /* EventType.PERSIST_STATE */, this.#listener);
         }
         await this.persistState();
     }
@@ -272,11 +273,11 @@ export class SessionPool {
      * Removes retired `Session` instances from `SessionPool`.
      */
     removeRetiredSessions() {
-        this.sessions = this.sessions.filter((storedSession) => {
+        this.#sessions = this.#sessions.filter((storedSession) => {
             if (storedSession.isUsable())
                 return true;
             this.sessionMap.delete(storedSession.id);
-            this.log.debug(`Removed Session - ${storedSession.id}`);
+            this.#log.debug(`Removed Session - ${storedSession.id}`);
             return false;
         });
     }
@@ -285,14 +286,14 @@ export class SessionPool {
      * @param newSession `Session` instance to be added.
      */
     registerSession(newSession) {
-        this.sessions.push(newSession);
+        this.#sessions.push(newSession);
         this.sessionMap.set(newSession.id, newSession);
     }
     /**
      * Gets random index.
      */
     getRandomIndex() {
-        return Math.floor(Math.random() * this.sessions.length);
+        return Math.floor(Math.random() * this.#sessions.length);
     }
     /**
      * Creates new session without any extra behavior.
@@ -315,7 +316,7 @@ export class SessionPool {
      * through `maybeLoadSessionPool` naturally wins because it arrives in
      * `perCallOptions`.
      */
-    async _invokeCreateSessionFunction(perCallOptions) {
+    async invokeCreateSessionFunction(perCallOptions) {
         const sessionOptions = {
             fingerprint: createDefaultSessionFingerprint(),
             ...this.sessionOptions,
@@ -328,35 +329,35 @@ export class SessionPool {
      * @returns Newly created `Session` instance.
      */
     async createSession() {
-        const newSession = await this._invokeCreateSessionFunction();
+        const newSession = await this.invokeCreateSessionFunction();
         this.registerSession(newSession);
-        this.log.debug(`Created new Session - ${newSession.id}`);
+        this.#log.debug(`Created new Session - ${newSession.id}`);
         return newSession;
     }
     /**
      * Decides whether there is enough space for creating new session.
      */
     hasSpaceForSession() {
-        return this.sessions.length < this.maxPoolSize;
+        return this.#sessions.length < this.maxPoolSize;
     }
     /**
      * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
      * Returns `undefined` when no session should be reused and a new one should be created instead.
      */
     pickSession() {
-        if (this.sessionReuseStrategy !== 'use-until-failure' && this.hasSpaceForSession())
+        if (this.#sessionReuseStrategy !== 'use-until-failure' && this.hasSpaceForSession())
             return undefined;
-        if (this.sessionReuseStrategy === 'use-until-failure') {
-            return this.sessions.find((session) => session.isUsable());
+        if (this.#sessionReuseStrategy === 'use-until-failure') {
+            return this.#sessions.find((session) => session.isUsable());
         }
         let picked;
-        if (this.sessionReuseStrategy === 'round-robin') {
-            const index = this.roundRobinIndex % this.sessions.length;
-            this.roundRobinIndex = index + 1;
-            picked = this.sessions[index];
+        if (this.#sessionReuseStrategy === 'round-robin') {
+            const index = this.#roundRobinIndex % this.#sessions.length;
+            this.#roundRobinIndex = index + 1;
+            picked = this.#sessions[index];
         }
         else {
-            picked = this.sessions[this.getRandomIndex()];
+            picked = this.#sessions[this.getRandomIndex()];
         }
         return picked.isUsable() ? picked : undefined;
     }
@@ -369,18 +370,18 @@ export class SessionPool {
         if (!loadedSessionPool)
             return;
         // Invalidate old sessions and load active sessions only
-        this.log.debug('Recreating state from KeyValueStore', {
+        this.#log.debug('Recreating state from KeyValueStore', {
             persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
             persistStateKey: this.persistStateKey,
         });
         for (const sessionObject of loadedSessionPool.sessions) {
             sessionObject.createdAt = new Date(sessionObject.createdAt);
             sessionObject.expiresAt = new Date(sessionObject.expiresAt);
-            const recreatedSession = await this._invokeCreateSessionFunction(sessionObject);
+            const recreatedSession = await this.invokeCreateSessionFunction(sessionObject);
             if (recreatedSession.isUsable()) {
                 this.registerSession(recreatedSession);
             }
         }
-        this.log.debug(`${this.sessions.length} active sessions loaded from KeyValueStore`);
+        this.#log.debug(`${this.#sessions.length} active sessions loaded from KeyValueStore`);
     }
 }

package/storages/dataset.d.ts

@@ -1,6 +1,7 @@
 import type { Awaitable, DatasetBackend, DatasetInfo, Dictionary } from '@crawlee/types';
 import { Configuration } from '../configuration.js';
 import type { CrawleeLogger } from '../log.js';
+import type { JournalEntry } from './transaction.js';
 import type { DatasetStats } from './storage_stats.js';
 import type { StorageOpenOptions } from './utils.js';
 import type { StorageIdentifier } from './storage_instance_manager.js';
@@ -139,12 +140,12 @@ export interface DatasetExportToOptions extends DatasetExportOptions {
  * @category Result Stores
  */
 export declare class Dataset<Data extends Dictionary = Dictionary> {
+    #private;
     readonly configuration: Configuration;
     id: string;
     name?: string;
     backend: DatasetBackend<Data>;
     log: CrawleeLogger;
-    private readonly statsTracker;
     /**
      * @internal
      */
@@ -170,6 +171,16 @@ export declare class Dataset<Data extends Dictionary = Dictionary> {
      * Returns {@link DatasetContent} object holding the items in the dataset based on the provided parameters.
      */
     getData(options?: DatasetDataOptions): Promise<DatasetContent<Data>>;
+    /**
+     * The single transaction-aware page read all dataset read paths go through — both `getData()` and
+     * the private `fetchPages()`. Returns the real page concatenated with the current transaction's
+     * buffered items, with `offset` / `limit` / `desc` windowing applied across the concatenation.
+     */
+    private readPage;
+    /** The active transaction's buffered writes to this dataset, derived from its journal. */
+    private bufferedJournalEntries;
+    /** @internal */
+    commitJournalEntries(entries: JournalEntry[]): Promise<void>;
     /**
      * Returns all the data from the dataset. This will iterate through the whole dataset
      * via the `listItems()` client method, which gives you only paginated results.