@crawlee/basic 4.0.0-beta.23 → 4.0.0-beta.230

package/internals/session-pool/session.d.ts

@@ -0,0 +1,151 @@
+import type { CrawleeLogger } from '@crawlee/core';
+import type { Dictionary, ISession, ProxyInfo, SessionFingerprint, SessionState } from '@crawlee/types';
+import { CookieJar } from 'tough-cookie';
+export interface SessionOptions {
+    /** Id of session used for generating fingerprints. It is used as proxy session name. */
+    id?: string;
+    /**
+     * Number of seconds after which the session is considered as expired.
+     * @default 3000
+     */
+    maxAgeSecs?: number;
+    /** Object where custom user data can be stored. For example custom headers. */
+    userData?: Dictionary;
+    /**
+     * Maximum number of marking session as blocked usage.
+     * If the `errorScore` reaches the `maxErrorScore` session is marked as block and it is thrown away.
+     * It starts at 0. Calling the `markBad` function increases the `errorScore` by 1.
+     * Calling the `markGood` will decrease the `errorScore` by `errorScoreDecrement`
+     * @default 3
+     */
+    maxErrorScore?: number;
+    /**
+     * It is used for healing the session.
+     * For example: if your session is marked bad two times, but it is successful on the third attempt it's errorScore
+     * is decremented by this number.
+     * @default 0.5
+     */
+    errorScoreDecrement?: number;
+    /** Date of creation. */
+    createdAt?: Date;
+    /**
+     * Date of expiration.
+     * @default createdAt + maxAgeSecs
+     */
+    expiresAt?: Date;
+    /**
+     * Indicates how many times the session has been used.
+     * @default 0
+     */
+    usageCount?: number;
+    /**
+     * Session should be used only a limited amount of times.
+     * This number indicates how many times the session is going to be used, before it is thrown away.
+     * @default 50
+     */
+    maxUsageCount?: number;
+    /**
+     * Marks the session as already retired. Used when restoring a previously persisted session
+     * so that `isUsable()` reflects the terminal state regardless of error score or usage count.
+     * @default false
+     */
+    retired?: boolean;
+    /** @internal */
+    log?: CrawleeLogger;
+    errorScore?: number;
+    cookieJar?: CookieJar;
+    proxyInfo?: ProxyInfo;
+    /**
+     * Browser / HTTP client fingerprint tied to this session. Backends use this to make
+     * repeated requests with the same session look consistent (same user-agent, headers,
+     * TLS profile). See {@link SessionFingerprint}.
+     */
+    fingerprint?: SessionFingerprint;
+}
+/**
+ * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
+ * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
+ * Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
+ * @category Scaling
+ */
+export declare class Session implements ISession {
+    #private;
+    readonly id: string;
+    readonly userData: Dictionary;
+    get errorScore(): number;
+    get usageCount(): number;
+    get maxErrorScore(): number;
+    get errorScoreDecrement(): number;
+    get expiresAt(): Date;
+    get createdAt(): Date;
+    get maxUsageCount(): number;
+    get cookieJar(): CookieJar;
+    get proxyInfo(): ProxyInfo | undefined;
+    get fingerprint(): SessionFingerprint | undefined;
+    set fingerprint(fingerprint: SessionFingerprint | undefined);
+    /**
+     * `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
+     * a retired session is never picked by the pool and cannot be revived via `markGood()`.
+     */
+    get retired(): boolean;
+    /**
+     * Session configuration.
+     */
+    constructor(options?: SessionOptions);
+    /**
+     * Indicates whether the session is blocked.
+     * Session is blocked once it reaches the `maxErrorScore`.
+     */
+    isBlocked(): boolean;
+    /**
+     * Indicates whether the session is expired.
+     * Session expiration is determined by the `maxAgeSecs`.
+     * Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
+     */
+    isExpired(): boolean;
+    /**
+     * Indicates whether the session is used maximum number of times.
+     * Session maximum usage count can be changed by `maxUsageCount` parameter.
+     */
+    isMaxUsageCountReached(): boolean;
+    /**
+     * Indicates whether the session can be used for next requests.
+     * Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
+     */
+    isUsable(): boolean;
+    /**
+     * This method should be called after a successful session usage.
+     * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
+     */
+    markGood(): void;
+    /**
+     * Gets session state for persistence in KeyValueStore.
+     *
+     * @internal
+     */
+    getState(): SessionState;
+    /**
+     * Permanently retires the session — `isUsable()` will return `false` from here on,
+     * and no `markGood()` / `markBad()` can revive it. Calling `retire()` again is a no-op.
+     *
+     * Use this when you're confident the session itself is the problem (e.g. a `403` response).
+     * For transient external failures (such as `5XX` responses), use `markBad()` instead.
+     */
+    retire(): void;
+    /**
+     * Increases usage and error count.
+     * Should be used when the session has been used unsuccessfully. For example because of timeouts.
+     */
+    markBad(): void;
+    /**
+     * Returns cookies saved with the session in the typical
+     * key1=value1; key2=value2 format, ready to be used in
+     * a cookie header or elsewhere.
+     * @returns Represents `Cookie` header.
+     */
+    getCookieString(url: string): Promise<string>;
+    /**
+     * Sets a cookie within this session for the specific URL.
+     */
+    setCookie(rawCookie: string, url: string): Promise<void>;
+}

package/internals/session-pool/session.js

@@ -0,0 +1,225 @@
+import { serviceLocator, validators } from '@crawlee/core';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { CookieJar } from 'tough-cookie';
+import { z } from 'zod';
+import { cryptoRandomObjectId } from '@apify/utilities';
+// `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
+// copy), so class instances like cookie jars and loggers keep their prototype.
+const sessionOptionsSchema = z.strictObject({
+    id: z.string().default(() => `session_${cryptoRandomObjectId(10)}`),
+    cookieJar: schemas.anyObject.default(() => new CookieJar()),
+    proxyInfo: schemas.anyObject.optional(),
+    maxAgeSecs: schemas.anyNumber.default(3000),
+    userData: schemas.anyObject.default(() => ({})),
+    maxErrorScore: schemas.anyNumber.default(3),
+    errorScoreDecrement: schemas.anyNumber.default(0.5),
+    createdAt: z.date().default(() => new Date()),
+    expiresAt: z.date().optional(),
+    usageCount: schemas.anyNumber.default(0),
+    errorScore: schemas.anyNumber.default(0),
+    maxUsageCount: schemas.anyNumber.default(50),
+    retired: z.boolean().default(false),
+    log: validators.logger.default(() => serviceLocator.getLogger()),
+    fingerprint: schemas.anyObject.optional(),
+});
+/**
+ * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
+ * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
+ * Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
+ * @category Scaling
+ */
+export class Session {
+    id;
+    userData;
+    #maxErrorScore;
+    #errorScoreDecrement;
+    #createdAt;
+    #expiresAt;
+    #usageCount;
+    #maxUsageCount;
+    #errorScore;
+    #retired = false;
+    #proxyInfo;
+    #cookieJar;
+    #fingerprint;
+    #log;
+    get errorScore() {
+        return this.#errorScore;
+    }
+    get usageCount() {
+        return this.#usageCount;
+    }
+    get maxErrorScore() {
+        return this.#maxErrorScore;
+    }
+    get errorScoreDecrement() {
+        return this.#errorScoreDecrement;
+    }
+    get expiresAt() {
+        return this.#expiresAt;
+    }
+    get createdAt() {
+        return this.#createdAt;
+    }
+    get maxUsageCount() {
+        return this.#maxUsageCount;
+    }
+    get cookieJar() {
+        return this.#cookieJar;
+    }
+    get proxyInfo() {
+        return this.#proxyInfo;
+    }
+    get fingerprint() {
+        return this.#fingerprint;
+    }
+    set fingerprint(fingerprint) {
+        this.#fingerprint = fingerprint;
+    }
+    /**
+     * `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
+     * a retired session is never picked by the pool and cannot be revived via `markGood()`.
+     */
+    get retired() {
+        return this.#retired;
+    }
+    /**
+     * Session configuration.
+     */
+    constructor(options = {}) {
+        const { id, cookieJar, proxyInfo, maxAgeSecs, userData, maxErrorScore, errorScoreDecrement, createdAt, usageCount, errorScore, maxUsageCount, retired, log, fingerprint, 
+        // Anchored to `createdAt` rather than to "now", so the documented `createdAt + maxAgeSecs` holds.
+        expiresAt = new Date(createdAt.getTime() + maxAgeSecs * 1000), } = parseArgument(options, sessionOptionsSchema);
+        this.#log = log.child({ prefix: 'Session' });
+        this.#cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
+        this.#proxyInfo = proxyInfo;
+        this.#fingerprint = fingerprint;
+        this.id = id;
+        this.userData = userData;
+        this.#maxErrorScore = maxErrorScore;
+        this.#errorScoreDecrement = errorScoreDecrement;
+        // Internal
+        this.#expiresAt = expiresAt;
+        this.#createdAt = createdAt;
+        this.#usageCount = usageCount; // indicates how many times the session has been used
+        this.#errorScore = errorScore; // indicates number of markBaded request with the session
+        this.#maxUsageCount = maxUsageCount;
+        this.#retired = retired;
+    }
+    /**
+     * Indicates whether the session is blocked.
+     * Session is blocked once it reaches the `maxErrorScore`.
+     */
+    isBlocked() {
+        return this.errorScore >= this.maxErrorScore;
+    }
+    /**
+     * Indicates whether the session is expired.
+     * Session expiration is determined by the `maxAgeSecs`.
+     * Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
+     */
+    isExpired() {
+        return this.expiresAt <= new Date();
+    }
+    /**
+     * Indicates whether the session is used maximum number of times.
+     * Session maximum usage count can be changed by `maxUsageCount` parameter.
+     */
+    isMaxUsageCountReached() {
+        return this.usageCount >= this.maxUsageCount;
+    }
+    /**
+     * Indicates whether the session can be used for next requests.
+     * Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
+     */
+    isUsable() {
+        return !this.#retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
+    }
+    /**
+     * This method should be called after a successful session usage.
+     * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
+     */
+    markGood() {
+        if (this.#retired)
+            return;
+        this.#usageCount += 1;
+        if (this.#errorScore > 0) {
+            this.#errorScore -= this.#errorScoreDecrement;
+        }
+        this.#maybeSelfRetire();
+    }
+    /**
+     * Gets session state for persistence in KeyValueStore.
+     *
+     * @internal
+     */
+    getState() {
+        return {
+            id: this.id,
+            cookieJar: this.cookieJar.toJSON(),
+            proxyInfo: this.#proxyInfo,
+            userData: this.userData,
+            fingerprint: this.#fingerprint,
+            maxErrorScore: this.maxErrorScore,
+            errorScoreDecrement: this.errorScoreDecrement,
+            expiresAt: this.expiresAt.toISOString(),
+            createdAt: this.createdAt.toISOString(),
+            usageCount: this.usageCount,
+            maxUsageCount: this.maxUsageCount,
+            errorScore: this.errorScore,
+            retired: this.#retired,
+        };
+    }
+    /**
+     * Permanently retires the session — `isUsable()` will return `false` from here on,
+     * and no `markGood()` / `markBad()` can revive it. Calling `retire()` again is a no-op.
+     *
+     * Use this when you're confident the session itself is the problem (e.g. a `403` response).
+     * For transient external failures (such as `5XX` responses), use `markBad()` instead.
+     */
+    retire() {
+        if (this.#retired && this.isBlocked())
+            return;
+        this.#errorScore += this.#maxErrorScore;
+        this.#usageCount += 1;
+        this.#retired = true;
+    }
+    /**
+     * Increases usage and error count.
+     * Should be used when the session has been used unsuccessfully. For example because of timeouts.
+     */
+    markBad() {
+        this.#errorScore += 1;
+        this.#usageCount += 1;
+        this.#maybeSelfRetire();
+    }
+    /**
+     * Returns cookies saved with the session in the typical
+     * key1=value1; key2=value2 format, ready to be used in
+     * a cookie header or elsewhere.
+     * @returns Represents `Cookie` header.
+     */
+    async getCookieString(url) {
+        return this.cookieJar.getCookieString(url, {});
+    }
+    /**
+     * Sets a cookie within this session for the specific URL.
+     */
+    async setCookie(rawCookie, url) {
+        try {
+            await this.cookieJar.setCookie(rawCookie, url);
+        }
+        catch (e) {
+            this.#log.warning('Could not set cookie.', { url, error: e.message });
+        }
+    }
+    /**
+     * Makes an unusable session's state terminal. Unlike `retire()`, this leaves the error score alone so that
+     * `isBlocked()` keeps telling a burned identity from one that merely ran out of uses.
+     */
+    #maybeSelfRetire() {
+        if (!this.isUsable()) {
+            this.#retired = true;
+        }
+    }
+}

package/internals/sitemap-request-loader.d.ts

@@ -0,0 +1,158 @@
+import type { BaseHttpClient } from '@crawlee/http-client';
+import type { ParseSitemapOptions } from '@crawlee/utils';
+import { EnqueueStrategy } from '@crawlee/utils';
+import type { IRequestLoader, IRequestManager, RequestLoaderStatus } from '@crawlee/core';
+import { Request } from '@crawlee/core';
+import type { UrlPatternInput } from './enqueue-links/index.js';
+interface UrlConstraints {
+    /**
+     * An array of URL patterns that URLs must match to be included.
+     *
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
+     *
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, use a `RegExp`.
+     *
+     * If `include` is an empty array or `undefined`, then the `SitemapRequestLoader`
+     * includes all the URLs from the sitemap.
+     */
+    include?: readonly UrlPatternInput[];
+    /**
+     * An array of URL patterns. Matching URLs will **not** be included.
+     *
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
+     *
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, use a `RegExp`.
+     */
+    exclude?: readonly UrlPatternInput[];
+}
+export interface SitemapRequestLoaderOptions extends UrlConstraints {
+    /**
+     * List of sitemap URLs to parse.
+     */
+    sitemapUrls: string[];
+    /**
+     * Proxy URL to be used for sitemap loading.
+     */
+    proxyUrl?: string;
+    /**
+     * Key for persisting the state of the request list in the `KeyValueStore`.
+     */
+    persistStateKey?: string;
+    /**
+     * Persistence-related options to control how and when crawler's data gets persisted.
+     */
+    persistenceOptions?: {
+        /**
+         * Use this flag to disable or enable periodic persistence to key value store.
+         * @default true
+         */
+        enable?: boolean;
+    };
+    /**
+     * Abort signal to be used for sitemap loading.
+     */
+    signal?: AbortSignal;
+    /**
+     * Timeout for sitemap loading in milliseconds. If both `signal` and `timeoutMillis` are provided, either of them can abort the loading.
+     */
+    timeoutMillis?: number;
+    /**
+     * Maximum number of buffered URLs for the sitemap loading stream.
+     * If the buffer is full, the stream will pause until the buffer is drained.
+     *
+     * @default 200
+     */
+    maxBufferSize?: number;
+    /**
+     * Keep only sitemap-derived URLs matching this strategy relative to the parent sitemap URL; non-`http(s)`
+     * schemes are always dropped. The filtering stays enforced after navigation (e.g. across redirects).
+     * Pass `'all'` to disable host filtering.
+     * @default EnqueueStrategy.SameHostname
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
+    /**
+     * Advanced options for the underlying `parseSitemap` call.
+     */
+    parseSitemapOptions?: Omit<ParseSitemapOptions, 'emitNestedSitemaps' | 'maxDepth'>;
+    /**
+     * Custom HTTP client to be used for sitemap loading.
+     */
+    httpClient?: BaseHttpClient;
+}
+/**
+ * A list of URLs to crawl parsed from a sitemap.
+ *
+ * The loading of the sitemap is performed in the background so that crawling can start before the sitemap is fully loaded.
+ */
+export declare class SitemapRequestLoader implements IRequestLoader {
+    #private;
+    /**
+     * Set of URLs that were returned by `fetchNextRequest()` and not marked as handled yet.
+     * @internal
+     */
+    inProgress: Set<string>;
+    /** @internal */
+    private constructor();
+    /**
+     * Indicates whether the background processing of sitemap contents has successfully finished.
+     *
+     * If this is `false`, the background processing is either still in progress or was aborted.
+     */
+    isSitemapFullyLoaded(): boolean;
+    /**
+     * Open a sitemap and start processing it.
+     *
+     * Resolves to a new instance of `SitemapRequestLoader`, which **might not be fully loaded yet** - i.e. the sitemap might still be loading in the background.
+     *
+     * Track the loading progress using the `isSitemapFullyLoaded` property.
+     */
+    static open(options: SitemapRequestLoaderOptions): Promise<SitemapRequestLoader>;
+    /**
+     * @inheritDoc
+     */
+    getTotalCount(): Promise<number>;
+    /**
+     * @inheritDoc
+     */
+    getPendingCount(): Promise<number>;
+    /**
+     * Combines this list with a request manager (a {@link RequestQueue} by default) into a
+     * {@link RequestManagerTandem}, allowing requests to be added and reclaimed while still
+     * being read from this list first.
+     */
+    toTandem(requestManager?: IRequestManager): Promise<IRequestManager>;
+    /**
+     * @inheritDoc
+     */
+    checkReadiness(): Promise<RequestLoaderStatus>;
+    /**
+     * @inheritDoc
+     */
+    getHandledCount(): Promise<number>;
+    /**
+     * @inheritDoc
+     */
+    persistState(): Promise<void>;
+    /**
+     * @inheritDoc
+     */
+    fetchNextRequest(): Promise<Request | null>;
+    /**
+     * @inheritDoc
+     */
+// @ts-ignore optional peer dependency or compatibility with es2022
+    [Symbol.asyncIterator](): AsyncGenerator<Request<import("@crawlee/types").Dictionary>, void, unknown>;
+    /**
+     * Aborts the internal sitemap loading, stops the processing of the sitemap contents and drops all the pending URLs.
+     *
+     * Calling `fetchNextRequest()` after this method will always return `null`.
+     */
+    teardown(): Promise<void>;
+    /**
+     * @inheritDoc
+     */
+    markRequestAsHandled(request: Request): Promise<void>;
+}
+export {};