@crawlee/core 4.0.0-beta.98 → 4.0.0-rc.0

package/session_pool/session.d.ts

@@ -65,20 +65,9 @@ export interface SessionOptions {
  * @category Scaling
  */
 export declare class Session implements ISession {
+    #private;
     readonly id: string;
     readonly userData: Dictionary;
-    private _maxErrorScore;
-    private _errorScoreDecrement;
-    private _createdAt;
-    private _expiresAt;
-    private _usageCount;
-    private _maxUsageCount;
-    private _errorScore;
-    private _retired;
-    private _proxyInfo?;
-    private _cookieJar;
-    private _fingerprint?;
-    private log;
     get errorScore(): number;
     get usageCount(): number;
     get maxErrorScore(): number;
@@ -149,11 +138,11 @@ export declare class Session implements ISession {
      * a cookie header or elsewhere.
      * @returns Represents `Cookie` header.
      */
-    getCookieString(url: string): string;
+    getCookieString(url: string): Promise<string>;
     /**
      * Sets a cookie within this session for the specific URL.
      */
-    setCookie(rawCookie: string, url: string): void;
+    setCookie(rawCookie: string, url: string): Promise<void>;
     /**
      * Checks if session is not usable. if it is not retires the session.
      */

package/session_pool/session.js

@@ -1,8 +1,28 @@
-import ow from 'ow';
 import { CookieJar } from 'tough-cookie';
+import { z } from 'zod';
 import { cryptoRandomObjectId } from '@apify/utilities';
 import { getDefaultCookieExpirationDate } from '../cookie_utils.js';
 import { serviceLocator } from '../service_locator.js';
+import { parseArgument, schemas, validators } from '../validators.js';
+// `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
+// copy), so class instances like cookie jars and loggers keep their prototype.
+const sessionOptionsSchema = z.strictObject({
+    id: z.string().default(() => `session_${cryptoRandomObjectId(10)}`),
+    cookieJar: schemas.anyObject.default(() => new CookieJar()),
+    proxyInfo: schemas.anyObject.optional(),
+    maxAgeSecs: schemas.anyNumber.default(3000),
+    userData: schemas.anyObject.default(() => ({})),
+    maxErrorScore: schemas.anyNumber.default(3),
+    errorScoreDecrement: schemas.anyNumber.default(0.5),
+    createdAt: z.date().default(() => new Date()),
+    expiresAt: z.date().optional(),
+    usageCount: schemas.anyNumber.default(0),
+    errorScore: schemas.anyNumber.default(0),
+    maxUsageCount: schemas.anyNumber.default(50),
+    retired: z.boolean().default(false),
+    log: validators.logger.default(() => serviceLocator.getLogger()),
+    fingerprint: schemas.anyObject.optional(),
+});
 /**
  * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
  * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
@@ -12,96 +32,78 @@ import { serviceLocator } from '../service_locator.js';
 export class Session {
     id;
     userData;
-    _maxErrorScore;
-    _errorScoreDecrement;
-    _createdAt;
-    _expiresAt;
-    _usageCount;
-    _maxUsageCount;
-    _errorScore;
-    _retired = false;
-    _proxyInfo;
-    _cookieJar;
-    _fingerprint;
-    log;
+    #maxErrorScore;
+    #errorScoreDecrement;
+    #createdAt;
+    #expiresAt;
+    #usageCount;
+    #maxUsageCount;
+    #errorScore;
+    #retired = false;
+    #proxyInfo;
+    #cookieJar;
+    #fingerprint;
+    #log;
     get errorScore() {
-        return this._errorScore;
+        return this.#errorScore;
     }
     get usageCount() {
-        return this._usageCount;
+        return this.#usageCount;
     }
     get maxErrorScore() {
-        return this._maxErrorScore;
+        return this.#maxErrorScore;
     }
     get errorScoreDecrement() {
-        return this._errorScoreDecrement;
+        return this.#errorScoreDecrement;
     }
     get expiresAt() {
-        return this._expiresAt;
+        return this.#expiresAt;
     }
     get createdAt() {
-        return this._createdAt;
+        return this.#createdAt;
     }
     get maxUsageCount() {
-        return this._maxUsageCount;
+        return this.#maxUsageCount;
     }
     get cookieJar() {
-        return this._cookieJar;
+        return this.#cookieJar;
     }
     get proxyInfo() {
-        return this._proxyInfo;
+        return this.#proxyInfo;
     }
     get fingerprint() {
-        return this._fingerprint;
+        return this.#fingerprint;
     }
     set fingerprint(fingerprint) {
-        this._fingerprint = fingerprint;
+        this.#fingerprint = fingerprint;
     }
     /**
      * `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
      * a retired session is never picked by the pool and cannot be revived via `markGood()`.
      */
     get retired() {
-        return this._retired;
+        return this.#retired;
     }
     /**
      * Session configuration.
      */
     constructor(options = {}) {
-        ow(options, ow.object.exactShape({
-            id: ow.optional.string,
-            cookieJar: ow.optional.object,
-            proxyInfo: ow.optional.object,
-            maxAgeSecs: ow.optional.number,
-            userData: ow.optional.object,
-            maxErrorScore: ow.optional.number,
-            errorScoreDecrement: ow.optional.number,
-            createdAt: ow.optional.date,
-            expiresAt: ow.optional.date,
-            usageCount: ow.optional.number,
-            errorScore: ow.optional.number,
-            maxUsageCount: ow.optional.number,
-            retired: ow.optional.boolean,
-            log: ow.optional.object,
-            fingerprint: ow.optional.object,
-        }));
-        const { id = `session_${cryptoRandomObjectId(10)}`, cookieJar = new CookieJar(), proxyInfo = undefined, maxAgeSecs = 3000, userData = {}, maxErrorScore = 3, errorScoreDecrement = 0.5, createdAt = new Date(), usageCount = 0, errorScore = 0, maxUsageCount = 50, retired = false, log = serviceLocator.getLogger(), fingerprint, } = options;
-        const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
-        this.log = log.child({ prefix: 'Session' });
-        this._cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
-        this._proxyInfo = proxyInfo;
-        this._fingerprint = fingerprint;
+        const { id, cookieJar, proxyInfo, maxAgeSecs, userData, maxErrorScore, errorScoreDecrement, createdAt, usageCount, errorScore, maxUsageCount, retired, log, fingerprint, expiresAt = getDefaultCookieExpirationDate(maxAgeSecs), } = parseArgument(options, sessionOptionsSchema);
+        this.#log = log.child({ prefix: 'Session' });
+        this.#cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
+        this.#proxyInfo = proxyInfo;
+        this.#fingerprint = fingerprint;
         this.id = id;
         this.userData = userData;
-        this._maxErrorScore = maxErrorScore;
-        this._errorScoreDecrement = errorScoreDecrement;
+        this.#maxErrorScore = maxErrorScore;
+        this.#errorScoreDecrement = errorScoreDecrement;
         // Internal
-        this._expiresAt = expiresAt;
-        this._createdAt = createdAt;
-        this._usageCount = usageCount; // indicates how many times the session has been used
-        this._errorScore = errorScore; // indicates number of markBaded request with the session
-        this._maxUsageCount = maxUsageCount;
-        this._retired = retired;
+        this.#expiresAt = expiresAt;
+        this.#createdAt = createdAt;
+        this.#usageCount = usageCount; // indicates how many times the session has been used
+        this.#errorScore = errorScore; // indicates number of markBaded request with the session
+        this.#maxUsageCount = maxUsageCount;
+        this.#retired = retired;
     }
     /**
      * Indicates whether the session is blocked.
@@ -130,16 +132,16 @@ export class Session {
      * Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
      */
     isUsable() {
-        return !this._retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
+        return !this.#retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
     }
     /**
      * This method should be called after a successful session usage.
      * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
      */
     markGood() {
-        this._usageCount += 1;
-        if (this._errorScore > 0) {
-            this._errorScore -= this._errorScoreDecrement;
+        this.#usageCount += 1;
+        if (this.#errorScore > 0) {
+            this.#errorScore -= this.#errorScoreDecrement;
         }
         this.maybeSelfRetire();
     }
@@ -151,9 +153,9 @@ export class Session {
         return {
             id: this.id,
             cookieJar: this.cookieJar.toJSON(),
-            proxyInfo: this._proxyInfo,
+            proxyInfo: this.#proxyInfo,
             userData: this.userData,
-            fingerprint: this._fingerprint,
+            fingerprint: this.#fingerprint,
             maxErrorScore: this.maxErrorScore,
             errorScoreDecrement: this.errorScoreDecrement,
             expiresAt: this.expiresAt.toISOString(),
@@ -161,7 +163,7 @@ export class Session {
             usageCount: this.usageCount,
             maxUsageCount: this.maxUsageCount,
             errorScore: this.errorScore,
-            retired: this._retired,
+            retired: this.#retired,
         };
     }
     /**
@@ -172,19 +174,19 @@ export class Session {
      * For transient external failures (such as `5XX` responses), use `markBad()` instead.
      */
     retire() {
-        if (this._retired)
+        if (this.#retired)
             return;
-        this._errorScore += this._maxErrorScore;
-        this._usageCount += 1;
-        this._retired = true;
+        this.#errorScore += this.#maxErrorScore;
+        this.#usageCount += 1;
+        this.#retired = true;
     }
     /**
      * Increases usage and error count.
      * Should be used when the session has been used unsuccessfully. For example because of timeouts.
      */
     markBad() {
-        this._errorScore += 1;
-        this._usageCount += 1;
+        this.#errorScore += 1;
+        this.#usageCount += 1;
         this.maybeSelfRetire();
     }
     /**
@@ -193,18 +195,18 @@ export class Session {
      * a cookie header or elsewhere.
      * @returns Represents `Cookie` header.
      */
-    getCookieString(url) {
-        return this.cookieJar.getCookieStringSync(url, {});
+    async getCookieString(url) {
+        return this.cookieJar.getCookieString(url, {});
     }
     /**
      * Sets a cookie within this session for the specific URL.
      */
-    setCookie(rawCookie, url) {
+    async setCookie(rawCookie, url) {
         try {
-            this.cookieJar.setCookieSync(rawCookie, url);
+            await this.cookieJar.setCookie(rawCookie, url);
         }
         catch (e) {
-            this.log.warning('Could not set cookie.', { url, error: e.message });
+            this.#log.warning('Could not set cookie.', { url, error: e.message });
         }
     }
     /**

package/session_pool/session_pool.d.ts

@@ -112,24 +112,15 @@ export interface SessionPoolOptions {
  * @category Scaling
  */
 export declare class SessionPool implements ISessionPool {
-    private static nextId;
+    #private;
     readonly id: string;
-    private log;
     private maxPoolSize;
     private createSessionFunction;
     private keyValueStore?;
-    private sessions;
     private sessionMap;
     private sessionOptions;
     private persistStateKeyValueStoreId?;
     private persistStateKey;
-    private listener?;
-    private events;
-    private persistenceOptions;
-    private sessionReuseStrategy;
-    private initPromise?;
-    private queue;
-    private roundRobinIndex;
     constructor(options?: SessionPoolOptions);
     /**
      * Gets count of usable sessions in the pool.
@@ -190,8 +181,11 @@ export declare class SessionPool implements ISessionPool {
     /**
      * Removes listener from `persistState` event.
      * This function should be called after you are done with using the `SessionPool` instance.
+     * @param options - Set `persistState` to false when the final state was already persisted by the event manager.
      */
-    teardown(): Promise<void>;
+    teardown({ persistState }?: {
+        persistState?: boolean;
+    }): Promise<void>;
     /**
      * Removes retired `Session` instances from `SessionPool`.
      */
@@ -222,7 +216,7 @@ export declare class SessionPool implements ISessionPool {
      * through `maybeLoadSessionPool` naturally wins because it arrives in
      * `perCallOptions`.
      */
-    private _invokeCreateSessionFunction;
+    private invokeCreateSessionFunction;
     /**
      * Creates new session and adds it to the pool.
      * @returns Newly created `Session` instance.