@crawlee/core 4.0.0-beta.80 → 4.0.0-beta.82

package/session_pool/session.js

@@ -141,7 +141,7 @@ export class Session {
         if (this._errorScore > 0) {
             this._errorScore -= this._errorScoreDecrement;
         }
-        this._maybeSelfRetire();
+        this.maybeSelfRetire();
     }
     /**
      * Gets session state for persistence in KeyValueStore.
@@ -185,7 +185,7 @@ export class Session {
     markBad() {
         this._errorScore += 1;
         this._usageCount += 1;
-        this._maybeSelfRetire();
+        this.maybeSelfRetire();
     }
     /**
      * Returns cookies saved with the session in the typical
@@ -210,7 +210,7 @@ export class Session {
     /**
      * Checks if session is not usable. if it is not retires the session.
      */
-    _maybeSelfRetire() {
+    maybeSelfRetire() {
         if (!this.isUsable()) {
             this.retire();
         }

package/session_pool/session_pool.d.ts

@@ -1,8 +1,6 @@
 import type { ISessionPool } from '@crawlee/types';
 import type { PersistenceOptions } from '../crawlers/statistics.js';
-import type { EventManager } from '../events/event_manager.js';
 import type { CrawleeLogger } from '../log.js';
-import { KeyValueStore } from '../storages/key_value_store.js';
 import type { SessionOptions } from './session.js';
 import { Session } from './session.js';
 declare const SESSION_REUSE_STRATEGIES: readonly ["random", "round-robin", "use-until-failure"];
@@ -116,19 +114,19 @@ export interface SessionPoolOptions {
 export declare class SessionPool implements ISessionPool {
     private static nextId;
     readonly id: string;
-    protected log: CrawleeLogger;
-    protected maxPoolSize: number;
-    protected createSessionFunction: CreateSession;
-    protected keyValueStore?: KeyValueStore;
-    protected sessions: Session[];
-    protected sessionMap: Map<string, Session>;
-    protected sessionOptions: SessionOptions;
-    protected persistStateKeyValueStoreId?: string;
-    protected persistStateKey: string;
-    protected _listener?: () => Promise<void>;
-    protected events: EventManager;
-    protected persistenceOptions: PersistenceOptions;
-    protected sessionReuseStrategy: SessionReuseStrategy;
+    private log;
+    private maxPoolSize;
+    private createSessionFunction;
+    private keyValueStore?;
+    private sessions;
+    private sessionMap;
+    private sessionOptions;
+    private persistStateKeyValueStoreId?;
+    private persistStateKey;
+    private listener?;
+    private events;
+    private persistenceOptions;
+    private sessionReuseStrategy;
     private initPromise?;
     private queue;
     private roundRobinIndex;
@@ -145,7 +143,7 @@ export declare class SessionPool implements ISessionPool {
      * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
      * Called automatically on first use of any public method.
      */
-    protected ensureInitialized(): Promise<void>;
+    private ensureInitialized;
     private setupPool;
     /**
      * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
@@ -197,25 +195,23 @@ export declare class SessionPool implements ISessionPool {
     /**
      * Removes retired `Session` instances from `SessionPool`.
      */
-    protected _removeRetiredSessions(): void;
+    private removeRetiredSessions;
     /**
      * Adds `Session` instance to `SessionPool`.
      * @param newSession `Session` instance to be added.
      */
-    protected _addSession(newSession: Session): void;
+    private registerSession;
     /**
      * Gets random index.
      */
-    protected _getRandomIndex(): number;
+    private getRandomIndex;
     /**
      * Creates new session without any extra behavior.
      * @param [options]
      * @param [options.sessionOptions] The configuration options for the session being created.
      * @returns New session.
      */
-    protected _defaultCreateSessionFunction(options?: {
-        sessionOptions?: SessionOptions;
-    }): Promise<Session>;
+    private defaultCreateSessionFunction;
     /**
      * Invokes `createSessionFunction` with `sessionOptions` already merged from pool-wide defaults and
      * the supplied per-call overrides, so custom implementations don't need to spread `pool.sessionOptions` themselves.
@@ -223,7 +219,7 @@ export declare class SessionPool implements ISessionPool {
      * A default {@link SessionFingerprint} is generated up front (host OS as
      * `platform`, a random valid `browser`/`device` for that platform). Pool-wide
      * and per-call options override it, and a persisted fingerprint coming
-     * through `_maybeLoadSessionPool` naturally wins because it arrives in
+     * through `maybeLoadSessionPool` naturally wins because it arrives in
      * `perCallOptions`.
      */
     private _invokeCreateSessionFunction;
@@ -231,20 +227,20 @@ export declare class SessionPool implements ISessionPool {
      * Creates new session and adds it to the pool.
      * @returns Newly created `Session` instance.
      */
-    protected _createSession(): Promise<Session>;
+    private createSession;
     /**
      * Decides whether there is enough space for creating new session.
      */
-    protected _hasSpaceForSession(): boolean;
+    private hasSpaceForSession;
     /**
      * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
      * Returns `undefined` when no session should be reused and a new one should be created instead.
      */
-    protected _pickSession(): Session | undefined;
+    private pickSession;
     /**
      * Potentially loads `SessionPool`.
      * If the state was persisted it loads the `SessionPool` from the persisted state.
      */
-    protected _maybeLoadSessionPool(): Promise<void>;
+    private maybeLoadSessionPool;
 }
 export {};

package/session_pool/session_pool.js

@@ -71,7 +71,7 @@ export class SessionPool {
     sessionOptions;
     persistStateKeyValueStoreId;
     persistStateKey;
-    _listener;
+    listener;
     events;
     persistenceOptions;
     sessionReuseStrategy;
@@ -100,7 +100,7 @@ export class SessionPool {
         this.persistenceOptions = persistenceOptions;
         // Pool Configuration
         this.maxPoolSize = maxPoolSize;
-        this.createSessionFunction = createSessionFunction || this._defaultCreateSessionFunction;
+        this.createSessionFunction = createSessionFunction || this.defaultCreateSessionFunction;
         // Session configuration. The pool-scoped logger is merged into per-call sessionOptions inside
         // `_invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions
         // having to know about it.
@@ -147,9 +147,9 @@ export class SessionPool {
             this.log.debug(`No 'persistStateKeyValueStoreId' options specified, this session pool's data has been saved in the KeyValueStore with the id: ${this.keyValueStore.id}`);
         }
         // in case of migration happened and SessionPool state should be restored from the keyValueStore.
-        await this._maybeLoadSessionPool();
-        this._listener = this.persistState.bind(this);
-        this.events.on("persistState" /* EventType.PERSIST_STATE */, this._listener);
+        await this.maybeLoadSessionPool();
+        this.listener = this.persistState.bind(this);
+        this.events.on("persistState" /* EventType.PERSIST_STATE */, this.listener);
     }
     /**
      * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
@@ -166,12 +166,12 @@ export class SessionPool {
                 throw new Error(`Cannot add session with id '${id}' as it already exists in the pool`);
             }
         }
-        if (!this._hasSpaceForSession()) {
-            this._removeRetiredSessions();
+        if (!this.hasSpaceForSession()) {
+            this.removeRetiredSessions();
         }
         const newSession = options instanceof Session ? options : await this._invokeCreateSessionFunction(options);
         this.log.debug(`Adding new Session - ${newSession.id}`);
-        this._addSession(newSession);
+        this.registerSession(newSession);
     }
     /**
      * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
@@ -182,7 +182,7 @@ export class SessionPool {
     async newSession(sessionOptions) {
         await this.ensureInitialized();
         const newSession = await this._invokeCreateSessionFunction(sessionOptions);
-        this._addSession(newSession);
+        this.registerSession(newSession);
         return newSession;
     }
     /**
@@ -202,14 +202,14 @@ export class SessionPool {
                     return session;
                 return undefined;
             }
-            const pickedSession = this._pickSession();
+            const pickedSession = this.pickSession();
             if (pickedSession)
                 return pickedSession;
-            if (this._hasSpaceForSession()) {
-                return await this._createSession();
+            if (this.hasSpaceForSession()) {
+                return await this.createSession();
             }
-            this._removeRetiredSessions();
-            return await this._createSession();
+            this.removeRetiredSessions();
+            return await this.createSession();
         }
         finally {
             this.queue.shift();
@@ -263,15 +263,15 @@ export class SessionPool {
         if (!this.initPromise)
             return;
         await this.ensureInitialized();
-        if (this._listener) {
-            this.events.off("persistState" /* EventType.PERSIST_STATE */, this._listener);
+        if (this.listener) {
+            this.events.off("persistState" /* EventType.PERSIST_STATE */, this.listener);
         }
         await this.persistState();
     }
     /**
      * Removes retired `Session` instances from `SessionPool`.
      */
-    _removeRetiredSessions() {
+    removeRetiredSessions() {
         this.sessions = this.sessions.filter((storedSession) => {
             if (storedSession.isUsable())
                 return true;
@@ -284,14 +284,14 @@ export class SessionPool {
      * Adds `Session` instance to `SessionPool`.
      * @param newSession `Session` instance to be added.
      */
-    _addSession(newSession) {
+    registerSession(newSession) {
         this.sessions.push(newSession);
         this.sessionMap.set(newSession.id, newSession);
     }
     /**
      * Gets random index.
      */
-    _getRandomIndex() {
+    getRandomIndex() {
         return Math.floor(Math.random() * this.sessions.length);
     }
     /**
@@ -300,7 +300,7 @@ export class SessionPool {
      * @param [options.sessionOptions] The configuration options for the session being created.
      * @returns New session.
      */
-    async _defaultCreateSessionFunction(options = {}) {
+    async defaultCreateSessionFunction(options = {}) {
         ow(options, ow.object.exactShape({ sessionOptions: ow.optional.object }));
         const { sessionOptions = {} } = options;
         return new Session(sessionOptions);
@@ -312,7 +312,7 @@ export class SessionPool {
      * A default {@link SessionFingerprint} is generated up front (host OS as
      * `platform`, a random valid `browser`/`device` for that platform). Pool-wide
      * and per-call options override it, and a persisted fingerprint coming
-     * through `_maybeLoadSessionPool` naturally wins because it arrives in
+     * through `maybeLoadSessionPool` naturally wins because it arrives in
      * `perCallOptions`.
      */
     async _invokeCreateSessionFunction(perCallOptions) {
@@ -327,24 +327,24 @@ export class SessionPool {
      * Creates new session and adds it to the pool.
      * @returns Newly created `Session` instance.
      */
-    async _createSession() {
+    async createSession() {
         const newSession = await this._invokeCreateSessionFunction();
-        this._addSession(newSession);
+        this.registerSession(newSession);
         this.log.debug(`Created new Session - ${newSession.id}`);
         return newSession;
     }
     /**
      * Decides whether there is enough space for creating new session.
      */
-    _hasSpaceForSession() {
+    hasSpaceForSession() {
         return this.sessions.length < this.maxPoolSize;
     }
     /**
      * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
      * Returns `undefined` when no session should be reused and a new one should be created instead.
      */
-    _pickSession() {
-        if (this.sessionReuseStrategy !== 'use-until-failure' && this._hasSpaceForSession())
+    pickSession() {
+        if (this.sessionReuseStrategy !== 'use-until-failure' && this.hasSpaceForSession())
             return undefined;
         if (this.sessionReuseStrategy === 'use-until-failure') {
             return this.sessions.find((session) => session.isUsable());
@@ -356,7 +356,7 @@ export class SessionPool {
             picked = this.sessions[index];
         }
         else {
-            picked = this.sessions[this._getRandomIndex()];
+            picked = this.sessions[this.getRandomIndex()];
         }
         return picked.isUsable() ? picked : undefined;
     }
@@ -364,7 +364,7 @@ export class SessionPool {
      * Potentially loads `SessionPool`.
      * If the state was persisted it loads the `SessionPool` from the persisted state.
      */
-    async _maybeLoadSessionPool() {
+    async maybeLoadSessionPool() {
         const loadedSessionPool = await this.keyValueStore?.getValue(this.persistStateKey);
         if (!loadedSessionPool)
             return;
@@ -378,7 +378,7 @@ export class SessionPool {
             sessionObject.expiresAt = new Date(sessionObject.expiresAt);
             const recreatedSession = await this._invokeCreateSessionFunction(sessionObject);
             if (recreatedSession.isUsable()) {
-                this._addSession(recreatedSession);
+                this.registerSession(recreatedSession);
             }
         }
         this.log.debug(`${this.sessions.length} active sessions loaded from KeyValueStore`);

package/storages/request_list.d.ts

@@ -1,7 +1,7 @@
 import type { BaseHttpClient, Dictionary } from '@crawlee/types';
 import type { Configuration } from '../configuration.js';
-import type { ProxyConfiguration } from '../proxy_configuration.js';
-import { type InternalSource, Request, type RequestOptions, type Source } from '../request.js';
+import type { IProxyConfiguration } from '../proxy_configuration.js';
+import { Request, type RequestOptions, type Source } from '../request.js';
 import type { IRequestLoader } from './request_loader.js';
 import type { IRequestManager } from './request_manager.js';
 /** @internal */
@@ -87,7 +87,7 @@ export interface RequestListOptions {
      * Takes advantage of the internal address rotation and authentication process.
      * If undefined, the `requestsFromUrl` requests will be made without proxy.
      */
-    proxyConfiguration?: ProxyConfiguration;
+    proxyConfiguration?: IProxyConfiguration;
     /**
      * Identifies the key in the default key-value store under which `RequestList` periodically stores its
      * state (i.e. which URLs were crawled and which not).
@@ -222,7 +222,7 @@ export declare class RequestList implements IRequestLoader {
      * All requests in the array have distinct uniqueKey!
      * @internal
      */
-    requests: (Request | RequestOptions)[];
+    readonly requests: (Request | RequestOptions)[];
     /** Index to the next item in requests array to fetch. All previous requests are either handled or in progress. */
     private nextIndex;
     /** Dictionary, key is Request.uniqueKey, value is corresponding index in the requests array. */
@@ -275,14 +275,14 @@ export declare class RequestList implements IRequestLoader {
      * This needs to be done in a memory efficient way. We should update the input
      * to a Stream once apify-client supports streams.
      */
-    protected _addPersistedRequests(persistedRequests: Buffer): Promise<void>;
+    private addPersistedRequests;
     /**
      * Add Requests from both options.sources and options.sourcesFunction.
      * This function is called only when persisted sources were not loaded.
      * We need to avoid keeping both sources and requests in memory
      * to reduce memory footprint with very large sources.
      */
-    protected _addRequestsFromSources(): Promise<void>;
+    private addRequestsFromSources;
     /**
      * @inheritDoc
      */
@@ -298,16 +298,16 @@ export declare class RequestList implements IRequestLoader {
      * are automatically persisted at RequestList initialization (if the persistRequestsKey is set),
      * but there's no reason to persist it again afterwards, because RequestList is immutable.
      */
-    protected _persistRequests(): Promise<void>;
+    private persistRequests;
     /**
      * Restores RequestList state from a state object.
      */
-    protected _restoreState(state?: RequestListState): void;
+    private restoreState;
     /**
      * Attempts to load state and requests using the `RequestList` configuration
      * and returns a tuple of [state, requests] where each may be null if not loaded.
      */
-    protected _loadStateAndPersistedRequests(): Promise<[RequestListState, Buffer]>;
+    private loadStateAndPersistedRequests;
     /**
      * Returns an object representing the internal state of the `RequestList` instance.
      * Note that the object's fields can change in future releases.
@@ -337,31 +337,31 @@ export declare class RequestList implements IRequestLoader {
     /**
      * Adds all fetched requests from a URL from a remote resource.
      */
-    protected _addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[]): Promise<void>;
-    protected _getPersistedState<T>(key: string): Promise<T>;
+    private addFetchedRequests;
+    private getPersistedState;
     /**
      * Fetches URLs from requestsFromUrl and returns them in format of list of requests
      */
-    protected _fetchRequestsFromUrl(source: InternalSource): Promise<RequestOptions[]>;
+    private fetchRequestsFromUrl;
     /**
      * Adds given request.
      * If the `source` parameter is a string or plain object and not an instance
      * of a `Request`, then the function creates a `Request` instance.
      */
-    protected _addRequest(source: RequestListSource): void;
+    private addRequest;
     /**
      * Helper function that validates unique key.
      * Throws an error if uniqueKey is not a non-empty string.
      */
-    protected _ensureUniqueKeyValid(uniqueKey: string): void;
+    private ensureUniqueKeyValid;
     /**
      * Checks that a request is currently being processed and throws an error if not.
      */
-    protected _ensureInProgress(uniqueKey: string): void;
+    private ensureInProgress;
     /**
      * Throws an error if request list wasn't initialized.
      */
-    protected _ensureIsInitialized(): void;
+    private ensureIsInitialized;
     /**
      * Returns the total number of unique requests present in the `RequestList`.
      */