@crawlee/basic 4.0.0-beta.99 → 4.0.0-rc.1

package/internals/session_pool/session_pool.js

@@ -0,0 +1,453 @@
+import { KeyValueStore, RecoverableState, serviceLocator, validators } from '@crawlee/core';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { AsyncQueue } from '@sapphire/async-queue';
+import { z } from 'zod';
+import { createDefaultSessionFingerprint } from './fingerprint.js';
+import { Session } from './session.js';
+/** Default upper bound on the number of sessions the pool keeps around. */
+const MAX_POOL_SIZE = 1000;
+/** Prefix of the default key the pool persists its state under; the pool id is appended to it. */
+const PERSIST_STATE_KEY = 'CRAWLEE_SESSION_POOL_STATE';
+// Runtime mirror of `SessionReuseStrategy` for the zod schema below. Declared this way round - and not as
+// `typeof SESSION_REUSE_STRATEGIES[number]` - so the public type is a plain literal union that does not drag
+// the array into the API surface. `satisfies` keeps the array from naming a strategy the type does not have.
+const SESSION_REUSE_STRATEGIES = [
+    'random',
+    'round-robin',
+    'use-until-failure',
+];
+// `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
+// copy), so class instances like loggers keep their prototype.
+const sessionPoolOptionsSchema = z.strictObject({
+    id: z.union([schemas.anyNumber, z.string()]).optional(),
+    maxPoolSize: schemas.anyNumber.default(MAX_POOL_SIZE),
+    persistStateKeyValueStoreId: z.string().optional(),
+    persistStateKey: z.string().optional(),
+    createSessionFunction: schemas.anyFunction.optional(),
+    sessionOptions: schemas.anyObject.default(() => ({})),
+    log: validators.logger.default(() => serviceLocator.getLogger()),
+    persistenceOptions: schemas.anyObject.default(() => ({ enable: true })),
+    sessionReuseStrategy: z.enum(SESSION_REUSE_STRATEGIES).default('random'),
+});
+const createSessionOptionsSchema = z.strictObject({
+    sessionOptions: schemas.anyObject.default(() => ({})),
+});
+// Only what the recreation reads is pinned down - the rest of a session's state is the session's own business.
+const persistedSessionState = z.custom((value) => typeof value === 'object' &&
+    value !== null &&
+    typeof value.createdAt === 'string' &&
+    typeof value.expiresAt === 'string', 'Expected a persisted session with `createdAt` and `expiresAt` timestamps');
+const persistedSessionPoolState = z.object({
+    usableSessionsCount: z.number(),
+    retiredSessionsCount: z.number(),
+    sessions: z.array(persistedSessionState),
+});
+const sessionPoolInternalState = z.object({
+    sessions: z.array(z.instanceof(Session)),
+});
+/**
+ * The conversion between the pool's sessions and its persisted record. Decoding recreates the sessions through
+ * the pool's factory and keeps only the usable ones, which is why the codec is built per pool.
+ */
+function buildSessionPoolStateCodec(pool) {
+    return z.codec(persistedSessionPoolState, sessionPoolInternalState, {
+        decode: async (record) => {
+            const sessions = [];
+            for (const sessionState of record.sessions) {
+                const session = await pool.recreateSession({
+                    ...sessionState,
+                    createdAt: new Date(sessionState.createdAt),
+                    expiresAt: new Date(sessionState.expiresAt),
+                });
+                if (session.isUsable()) {
+                    sessions.push(session);
+                }
+            }
+            pool.log.debug(`${sessions.length} active sessions loaded from KeyValueStore`);
+            return { sessions };
+        },
+        encode: ({ sessions }) => pool.toRecord(sessions),
+    });
+}
+/**
+ * Handles the rotation, creation and persistence of user-like sessions.
+ * Creates a pool of {@link Session} instances, that are randomly rotated.
+ * When some session is marked as blocked, it is removed and new one is created instead (the pool never returns an unusable session).
+ * Learn more in the {@doclink guides/session-management | Session management guide}.
+ *
+ * Session pool is already integrated into crawlers and is always active.
+ * All public methods are lazy-initialized — the pool initializes itself on first use.
+ *
+ * You can configure the pool with many options. See the {@link SessionPoolOptions}.
+ * Session pool is by default persisted in default {@link KeyValueStore}.
+ * If you want to have one pool for all runs you have to specify
+ * {@link SessionPoolOptions.persistStateKeyValueStoreId}.
+ *
+ * **Advanced usage:**
+ *
+ * ```javascript
+ * const sessionPool = new SessionPool({
+ *     maxPoolSize: 25,
+ *     sessionOptions:{
+ *          maxAgeSecs: 10,
+ *          maxUsageCount: 150, // for example when you know that the site blocks after 150 requests.
+ *     },
+ *     persistStateKeyValueStoreId: 'my-key-value-store-for-sessions',
+ *     persistStateKey: 'my-session-pool',
+ * });
+ *
+ * // Get random session from the pool
+ * const session1 = await sessionPool.getSession();
+ * const session2 = await sessionPool.getSession();
+ * const session3 = await sessionPool.getSession();
+ *
+ * // Now you can mark the session either failed or successful
+ *
+ * // Marks session as bad after unsuccessful usage -> it increases error count (soft retire)
+ * session1.markBad()
+ *
+ * // Marks as successful.
+ * session2.markGood()
+ *
+ * // Retires session -> session is removed from the pool
+ * session3.retire()
+ *
+ * ```
+ *
+ * **Default session allocation flow:*
+ * 1. Until the `SessionPool` reaches `maxPoolSize`, new sessions are created, provided to the user and added to the pool
+ * 2. Blocked/retired sessions stay in the pool but are never provided to the user
+ * 3. Once the pool is full (live plus blocked session count reaches `maxPoolSize`), a random session from the pool is provided.
+ * 4. If a blocked session would be picked, instead all blocked sessions are evicted from the pool and a new session is created and provided
+ *
+ * @category Scaling
+ */
+export class SessionPool {
+    static #nextId = 0;
+    id;
+    #log;
+    #maxPoolSize;
+    #createSessionFunction;
+    #sessionMap = new Map();
+    #sessionOptions;
+    #persistStateKeyValueStoreId;
+    #persistStateKey;
+    #persistenceOptions;
+    #sessionReuseStrategy;
+    /**
+     * The sessions live in a {@link RecoverableState} that handles the loading, recreation and periodic
+     * persistence of the record under `persistStateKey`.
+     */
+    #state;
+    #stateCodec;
+    #initPromise;
+    #queue = new AsyncQueue();
+    #roundRobinIndex = 0;
+    constructor(options = {}) {
+        const { id, maxPoolSize, persistStateKeyValueStoreId, persistStateKey, createSessionFunction, sessionOptions, log, persistenceOptions, sessionReuseStrategy, } = parseArgument(options, sessionPoolOptionsSchema);
+        this.id = id != null ? String(id) : String(SessionPool.#nextId++);
+        this.#sessionReuseStrategy = sessionReuseStrategy;
+        this.#log = log.child({ prefix: 'SessionPool' });
+        this.#persistenceOptions = persistenceOptions;
+        // Pool Configuration
+        this.#maxPoolSize = maxPoolSize;
+        this.#createSessionFunction = createSessionFunction || this.#defaultCreateSessionFunction;
+        // Session configuration. The pool-scoped logger is merged into per-call sessionOptions inside
+        // `invokeCreateSessionFunction`, so every Session inherits it without custom createSessionFunctions
+        // having to know about it.
+        this.#sessionOptions = {
+            ...sessionOptions,
+            log: this.#log,
+        };
+        // Session keyValueStore
+        this.#persistStateKeyValueStoreId = persistStateKeyValueStoreId;
+        this.#persistStateKey = persistStateKey ?? `${PERSIST_STATE_KEY}_${this.id}`;
+        this.#stateCodec = buildSessionPoolStateCodec({
+            recreateSession: (sessionState) => this.#invokeCreateSessionFunction(sessionState),
+            toRecord: (sessions) => this.#buildPersistedState(sessions),
+            log: this.#log,
+        });
+        this.#state = new RecoverableState({
+            defaultState: () => ({ sessions: [] }),
+            persistStateKey: this.#persistStateKey,
+            persistenceEnabled: !!this.#persistenceOptions.enable,
+            logger: this.#log,
+            keyValueStore: this.#persistenceOptions.enable && persistStateKeyValueStoreId
+                ? KeyValueStore.open({ id: persistStateKeyValueStoreId }, { configuration: serviceLocator.getConfiguration() })
+                : undefined,
+            // The codec validates in the decode direction, so it is a Standard Schema as-is; encoding needs a call.
+            deserialize: this.#stateCodec,
+            serialize: (state) => this.#stateCodec.encode(state),
+        });
+        this.#state.reset();
+    }
+    /**
+     * Gets count of usable sessions in the pool.
+     */
+    async usableSessionsCount() {
+        await this.#ensureInitialized();
+        return this.#state.currentValue.sessions.filter((session) => session.isUsable()).length;
+    }
+    /**
+     * Gets count of retired sessions in the pool.
+     */
+    async retiredSessionsCount() {
+        await this.#ensureInitialized();
+        return this.#state.currentValue.sessions.filter((session) => !session.isUsable()).length;
+    }
+    /**
+     * Starts periodic state persistence and potentially loads SessionPool state from {@link KeyValueStore}.
+     * Called automatically on first use of any public method.
+     */
+    async #ensureInitialized() {
+        if (!this.#initPromise) {
+            this.#initPromise = this.#setupPool();
+        }
+        return this.#initPromise;
+    }
+    async #setupPool() {
+        await this.#state.initialize();
+        for (const session of this.#state.currentValue.sessions) {
+            this.#sessionMap.set(session.id, session);
+        }
+    }
+    /**
+     * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
+     * but this allows you to add more sessions once the max pool size is reached.
+     * This also allows you to add session with overridden session options (e.g. with specific session id).
+     *
+     * If the pool already contains a session with the same id, it is replaced when it is no longer usable
+     * (e.g. retired), and an error is thrown otherwise.
+     * @param [options] The configuration options for the session being added to the session pool.
+     */
+    async addSession(options = {}) {
+        await this.#ensureInitialized();
+        const { id } = options;
+        if (id) {
+            const existingSession = this.#sessionMap.get(id);
+            if (existingSession) {
+                if (existingSession.isUsable()) {
+                    throw new Error(`Cannot add session with id '${id}' as it already exists in the pool`);
+                }
+                this.#removeSession(existingSession);
+            }
+        }
+        if (!this.#hasSpaceForSession()) {
+            this.#removeRetiredSessions();
+        }
+        const newSession = options instanceof Session ? options : await this.#invokeCreateSessionFunction(options);
+        this.#log.debug(`Adding new Session - ${newSession.id}`);
+        this.#registerSession(newSession);
+    }
+    /**
+     * Adds a new session to the session pool. The pool automatically creates sessions up to the maximum size of the pool,
+     * but this allows you to add more sessions once the max pool size is reached.
+     * This also allows you to add session with overridden session options (e.g. with specific session id).
+     * @param [options] The configuration options for the session being added to the session pool.
+     */
+    async newSession(sessionOptions) {
+        await this.#ensureInitialized();
+        const newSession = await this.#invokeCreateSessionFunction(sessionOptions);
+        this.#registerSession(newSession);
+        return newSession;
+    }
+    /**
+     * Gets session.
+     * If there is space for new session, it creates and returns new session.
+     * If the session pool is full, it picks a session from the pool,
+     * If the picked session is usable it is returned, otherwise it creates and returns a new one.
+     * @param [sessionId] If provided, it returns the usable session with this id, `undefined` otherwise.
+     */
+    async getSession(sessionId) {
+        await this.#ensureInitialized();
+        await this.#queue.wait();
+        try {
+            if (sessionId) {
+                const session = this.#sessionMap.get(sessionId);
+                if (session?.isUsable())
+                    return session;
+                return undefined;
+            }
+            const pickedSession = this.#pickSession();
+            if (pickedSession)
+                return pickedSession;
+            if (this.#hasSpaceForSession()) {
+                return await this.#createSession();
+            }
+            this.#removeRetiredSessions();
+            return await this.#createSession();
+        }
+        finally {
+            this.#queue.shift();
+        }
+    }
+    /**
+     * Discards all sessions in the pool, both usable and retired, resetting it to a blank state.
+     *
+     * The persisted record is left alone - use {@link SessionPool.resetStore} to clear that as well.
+     */
+    reset() {
+        this.#state.reset();
+        this.#sessionMap.clear();
+        this.#roundRobinIndex = 0;
+    }
+    /**
+     * Clears the persisted pool record, leaving the in-memory sessions alone.
+     *
+     * Throws while the state is still being persisted periodically - the next PERSIST_STATE event would write
+     * the record straight back. Call {@link SessionPool.teardown} first, or use {@link SessionPool.reset}
+     * to discard the sessions themselves. A no-op if persistence is disabled.
+     */
+    async resetStore() {
+        await this.#state.resetStore();
+    }
+    /**
+     * Returns an object representing the internal state of the `SessionPool` instance.
+     * @internal
+     */
+    async getState() {
+        await this.#ensureInitialized();
+        return this.#buildPersistedState(this.#state.currentValue.sessions);
+    }
+    /**
+     * Persists the current state of the `SessionPool` into the default {@link KeyValueStore}.
+     * The state is persisted automatically in regular intervals.
+     */
+    async persistState() {
+        if (!this.#persistenceOptions.enable) {
+            return;
+        }
+        await this.#ensureInitialized();
+        this.#log.debug('Persisting state', {
+            persistStateKeyValueStoreId: this.#persistStateKeyValueStoreId,
+            persistStateKey: this.#persistStateKey,
+        });
+        await this.#state
+            .persistState()
+            .catch((error) => this.#log.warning(`Failed to persist the session pool stats to ${this.#persistStateKey}`, { error }));
+    }
+    async [Symbol.asyncDispose]() {
+        await this.teardown();
+    }
+    /**
+     * Stops the periodic state persistence and persists the state one last time.
+     * This function should be called after you are done with using the `SessionPool` instance.
+     * Using the pool again afterwards initializes it anew.
+     */
+    async teardown() {
+        if (!this.#initPromise)
+            return;
+        await this.#ensureInitialized();
+        await this.#state.teardown();
+        // The next use initializes the pool again, restoring the record and resuming the periodic persistence -
+        // a crawler that is run twice gets a persisting pool both times.
+        this.#initPromise = undefined;
+    }
+    /**
+     * Removes retired `Session` instances from `SessionPool`.
+     */
+    #removeRetiredSessions() {
+        this.#state.currentValue.sessions = this.#state.currentValue.sessions.filter((storedSession) => {
+            if (storedSession.isUsable())
+                return true;
+            this.#sessionMap.delete(storedSession.id);
+            this.#log.debug(`Removed Session - ${storedSession.id}`);
+            return false;
+        });
+    }
+    /**
+     * Removes a `Session` instance from `SessionPool`.
+     */
+    #removeSession(session) {
+        this.#state.currentValue.sessions = this.#state.currentValue.sessions.filter((storedSession) => storedSession !== session);
+        this.#sessionMap.delete(session.id);
+        this.#log.debug(`Removed Session - ${session.id}`);
+    }
+    /**
+     * Adds `Session` instance to `SessionPool`.
+     * @param newSession `Session` instance to be added.
+     */
+    #registerSession(newSession) {
+        this.#state.currentValue.sessions.push(newSession);
+        this.#sessionMap.set(newSession.id, newSession);
+    }
+    /**
+     * Gets random index.
+     */
+    #getRandomIndex() {
+        return Math.floor(Math.random() * this.#state.currentValue.sessions.length);
+    }
+    /**
+     * Creates new session without any extra behavior.
+     * @param [options]
+     * @param [options.sessionOptions] The configuration options for the session being created.
+     * @returns New session.
+     */
+    async #defaultCreateSessionFunction(options = {}) {
+        const { sessionOptions } = parseArgument(options, createSessionOptionsSchema);
+        return new Session(sessionOptions);
+    }
+    /**
+     * Invokes `createSessionFunction` with `sessionOptions` already merged from pool-wide defaults and
+     * the supplied per-call overrides, so custom implementations don't need to spread `pool.sessionOptions` themselves.
+     *
+     * A default {@link SessionFingerprint} is generated up front (host OS as
+     * `platform`, a random valid `browser`/`device` for that platform). Pool-wide
+     * and per-call options override it, and a persisted fingerprint coming
+     * through `maybeLoadSessionPool` naturally wins because it arrives in
+     * `perCallOptions`.
+     */
+    async #invokeCreateSessionFunction(perCallOptions) {
+        const sessionOptions = {
+            fingerprint: createDefaultSessionFingerprint(),
+            ...this.#sessionOptions,
+            ...perCallOptions,
+        };
+        return this.#createSessionFunction({ sessionOptions });
+    }
+    /**
+     * Creates new session and adds it to the pool.
+     * @returns Newly created `Session` instance.
+     */
+    async #createSession() {
+        const newSession = await this.#invokeCreateSessionFunction();
+        this.#registerSession(newSession);
+        this.#log.debug(`Created new Session - ${newSession.id}`);
+        return newSession;
+    }
+    /**
+     * Decides whether there is enough space for creating new session.
+     */
+    #hasSpaceForSession() {
+        return this.#state.currentValue.sessions.length < this.#maxPoolSize;
+    }
+    /**
+     * Picks a session from the `SessionPool` according to the configured `sessionReuseStrategy`.
+     * Returns `undefined` when no session should be reused and a new one should be created instead.
+     */
+    #pickSession() {
+        if (this.#sessionReuseStrategy !== 'use-until-failure' && this.#hasSpaceForSession())
+            return undefined;
+        if (this.#sessionReuseStrategy === 'use-until-failure') {
+            return this.#state.currentValue.sessions.find((session) => session.isUsable());
+        }
+        let picked;
+        if (this.#sessionReuseStrategy === 'round-robin') {
+            const index = this.#roundRobinIndex % this.#state.currentValue.sessions.length;
+            this.#roundRobinIndex = index + 1;
+            picked = this.#state.currentValue.sessions[index];
+        }
+        else {
+            picked = this.#state.currentValue.sessions[this.#getRandomIndex()];
+        }
+        return picked.isUsable() ? picked : undefined;
+    }
+    /** The persisted form of the pool, which is also what {@link SessionPool.getState} returns. */
+    #buildPersistedState(sessions) {
+        return {
+            usableSessionsCount: sessions.filter((session) => session.isUsable()).length,
+            retiredSessionsCount: sessions.filter((session) => !session.isUsable()).length,
+            sessions: sessions.map((session) => session.getState()),
+        };
+    }
+}

package/internals/sitemap_request_loader.d.ts

@@ -0,0 +1,158 @@
+import type { BaseHttpClient } from '@crawlee/http-client';
+import type { ParseSitemapOptions } from '@crawlee/utils';
+import { EnqueueStrategy } from '@crawlee/utils';
+import type { IRequestLoader, IRequestManager, RequestLoaderStatus } from '@crawlee/core';
+import { Request } from '@crawlee/core';
+import type { UrlPatternInput } from './enqueue_links/index.js';
+interface UrlConstraints {
+    /**
+     * An array of URL patterns that URLs must match to be included.
+     *
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
+     *
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, use a `RegExp`.
+     *
+     * If `include` is an empty array or `undefined`, then the `SitemapRequestLoader`
+     * includes all the URLs from the sitemap.
+     */
+    include?: readonly UrlPatternInput[];
+    /**
+     * An array of URL patterns. Matching URLs will **not** be included.
+     *
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
+     *
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, use a `RegExp`.
+     */
+    exclude?: readonly UrlPatternInput[];
+}
+export interface SitemapRequestLoaderOptions extends UrlConstraints {
+    /**
+     * List of sitemap URLs to parse.
+     */
+    sitemapUrls: string[];
+    /**
+     * Proxy URL to be used for sitemap loading.
+     */
+    proxyUrl?: string;
+    /**
+     * Key for persisting the state of the request list in the `KeyValueStore`.
+     */
+    persistStateKey?: string;
+    /**
+     * Persistence-related options to control how and when crawler's data gets persisted.
+     */
+    persistenceOptions?: {
+        /**
+         * Use this flag to disable or enable periodic persistence to key value store.
+         * @default true
+         */
+        enable?: boolean;
+    };
+    /**
+     * Abort signal to be used for sitemap loading.
+     */
+    signal?: AbortSignal;
+    /**
+     * Timeout for sitemap loading in milliseconds. If both `signal` and `timeoutMillis` are provided, either of them can abort the loading.
+     */
+    timeoutMillis?: number;
+    /**
+     * Maximum number of buffered URLs for the sitemap loading stream.
+     * If the buffer is full, the stream will pause until the buffer is drained.
+     *
+     * @default 200
+     */
+    maxBufferSize?: number;
+    /**
+     * Keep only sitemap-derived URLs matching this strategy relative to the parent sitemap URL; non-`http(s)`
+     * schemes are always dropped. The filtering stays enforced after navigation (e.g. across redirects).
+     * Pass `'all'` to disable host filtering.
+     * @default EnqueueStrategy.SameHostname
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
+    /**
+     * Advanced options for the underlying `parseSitemap` call.
+     */
+    parseSitemapOptions?: Omit<ParseSitemapOptions, 'emitNestedSitemaps' | 'maxDepth'>;
+    /**
+     * Custom HTTP client to be used for sitemap loading.
+     */
+    httpClient?: BaseHttpClient;
+}
+/**
+ * A list of URLs to crawl parsed from a sitemap.
+ *
+ * The loading of the sitemap is performed in the background so that crawling can start before the sitemap is fully loaded.
+ */
+export declare class SitemapRequestLoader implements IRequestLoader {
+    #private;
+    /**
+     * Set of URLs that were returned by `fetchNextRequest()` and not marked as handled yet.
+     * @internal
+     */
+    inProgress: Set<string>;
+    /** @internal */
+    private constructor();
+    /**
+     * Indicates whether the background processing of sitemap contents has successfully finished.
+     *
+     * If this is `false`, the background processing is either still in progress or was aborted.
+     */
+    isSitemapFullyLoaded(): boolean;
+    /**
+     * Open a sitemap and start processing it.
+     *
+     * Resolves to a new instance of `SitemapRequestLoader`, which **might not be fully loaded yet** - i.e. the sitemap might still be loading in the background.
+     *
+     * Track the loading progress using the `isSitemapFullyLoaded` property.
+     */
+    static open(options: SitemapRequestLoaderOptions): Promise<SitemapRequestLoader>;
+    /**
+     * @inheritDoc
+     */
+    getTotalCount(): Promise<number>;
+    /**
+     * @inheritDoc
+     */
+    getPendingCount(): Promise<number>;
+    /**
+     * Combines this list with a request manager (a {@link RequestQueue} by default) into a
+     * {@link RequestManagerTandem}, allowing requests to be added and reclaimed while still
+     * being read from this list first.
+     */
+    toTandem(requestManager?: IRequestManager): Promise<IRequestManager>;
+    /**
+     * @inheritDoc
+     */
+    checkReadiness(): Promise<RequestLoaderStatus>;
+    /**
+     * @inheritDoc
+     */
+    getHandledCount(): Promise<number>;
+    /**
+     * @inheritDoc
+     */
+    persistState(): Promise<void>;
+    /**
+     * @inheritDoc
+     */
+    fetchNextRequest(): Promise<Request | null>;
+    /**
+     * @inheritDoc
+     */
+// @ts-ignore optional peer dependency or compatibility with es2022
+    [Symbol.asyncIterator](): AsyncGenerator<Request<import("@crawlee/types").Dictionary>, void, unknown>;
+    /**
+     * Aborts the internal sitemap loading, stops the processing of the sitemap contents and drops all the pending URLs.
+     *
+     * Calling `fetchNextRequest()` after this method will always return `null`.
+     */
+    teardown(): Promise<void>;
+    /**
+     * @inheritDoc
+     */
+    markRequestAsHandled(request: Request): Promise<void>;
+}
+export {};