@crawlee/basic 4.0.0-beta.18 → 4.0.0-beta.180

package/internals/send-request.d.ts

@@ -1,4 +1,6 @@
-import { type BaseHttpClient, type HttpRequestOptions, type Request, type Session } from '@crawlee/core';
+import type { Request as CrawleeRequest } from '@crawlee/core';
+import type { BaseHttpClient } from '@crawlee/http-client';
+import type { HttpRequestOptions, ISession, SendRequestOptions } from '@crawlee/types';
 /**
  * Prepares a function to be used as the `sendRequest` context helper.
  *
@@ -6,7 +8,5 @@ import { type BaseHttpClient, type HttpRequestOptions, type Request, type Sessio
  * @param httpClient The HTTP client that will perform the requests.
  * @param originRequest The crawling request being processed.
  * @param session The user session associated with the current request.
- * @param getProxyUrl A function that will return the proxy URL that should be used for handling the request.
  */
-export declare function createSendRequest(httpClient: BaseHttpClient, originRequest: Request, session: Session | undefined): (overrideOptions?: Partial<HttpRequestOptions>) => Promise<Response>;
-//# sourceMappingURL=send-request.d.ts.map
+export declare function createSendRequest(httpClient: BaseHttpClient, originRequest: CrawleeRequest, session: ISession): (overrideRequest?: Partial<HttpRequestOptions>, overrideOptions?: SendRequestOptions) => Promise<Response>;

package/internals/send-request.js

@@ -5,18 +5,29 @@
  * @param httpClient The HTTP client that will perform the requests.
  * @param originRequest The crawling request being processed.
  * @param session The user session associated with the current request.
- * @param getProxyUrl A function that will return the proxy URL that should be used for handling the request.
  */
 export function createSendRequest(httpClient, originRequest, session) {
-    return async (overrideOptions = {}) => {
-        const cookieJar = session
-            ? {
-                getCookieString: async (url) => session.getCookieString(url),
-                setCookie: async (rawCookie, url) => session.setCookie(rawCookie, url),
-                ...overrideOptions?.cookieJar,
-            }
-            : overrideOptions?.cookieJar;
-        return httpClient.sendRequest(originRequest.intoFetchAPIRequest(), { session, cookieJar: cookieJar });
+    return async (overrideRequest = {}, overrideOptions = {}) => {
+        const baseRequest = originRequest.intoFetchAPIRequest();
+        const mergedUrl = overrideRequest.url ?? baseRequest.url;
+        const mergedMethod = overrideRequest.method ?? baseRequest.method;
+        const mergedHeaders = new Headers(baseRequest.headers);
+        if (overrideRequest.headers) {
+            overrideRequest.headers.forEach((value, key) => {
+                mergedHeaders.set(key, value);
+            });
+        }
+        const request = new Request(mergedUrl, {
+            method: mergedMethod,
+            headers: mergedHeaders,
+            body: overrideRequest.body ?? baseRequest.body,
+        });
+        return httpClient.sendRequest(request, {
+            session,
+            cookieJar: overrideOptions?.cookieJar ?? session.cookieJar,
+            timeoutMillis: overrideOptions.timeoutMillis,
+            signal: overrideOptions.signal,
+            ignoreTlsErrors: overrideOptions.ignoreTlsErrors,
+        });
     };
 }
-//# sourceMappingURL=send-request.js.map

package/internals/session_pool/consts.d.ts

@@ -0,0 +1,3 @@
+export declare const BLOCKED_STATUS_CODES: number[];
+export declare const PERSIST_STATE_KEY = "CRAWLEE_SESSION_POOL_STATE";
+export declare const MAX_POOL_SIZE = 1000;

package/internals/session_pool/consts.js

@@ -0,0 +1,3 @@
+export const BLOCKED_STATUS_CODES = [401, 403, 429];
+export const PERSIST_STATE_KEY = 'CRAWLEE_SESSION_POOL_STATE';
+export const MAX_POOL_SIZE = 1000;

package/internals/session_pool/errors.d.ts

@@ -0,0 +1,7 @@
+/**
+ * @ignore
+ */
+export declare class CookieParseError extends Error {
+    readonly cookieHeaderString: unknown;
+    constructor(cookieHeaderString: unknown);
+}

package/internals/session_pool/errors.js

@@ -0,0 +1,11 @@
+/**
+ * @ignore
+ */
+export class CookieParseError extends Error {
+    cookieHeaderString;
+    constructor(cookieHeaderString) {
+        super(`Could not parse cookie header string: ${cookieHeaderString}`);
+        this.cookieHeaderString = cookieHeaderString;
+        Error.captureStackTrace(this, CookieParseError);
+    }
+}

package/internals/session_pool/fingerprint.d.ts

@@ -0,0 +1,9 @@
+import type { SessionFingerprint } from '@crawlee/types';
+/**
+ * Build a {@link SessionFingerprint} whose `platform` matches the host OS
+ * and whose `browser`/`device` are randomized within the realistic profiles for
+ * that platform. Used by {@link SessionPool} as the default fingerprint for
+ * freshly created sessions; callers can override by passing their own
+ * `fingerprint` in `sessionOptions`.
+ */
+export declare function createDefaultSessionFingerprint(): SessionFingerprint;

package/internals/session_pool/fingerprint.js

@@ -0,0 +1,30 @@
+/**
+ * (browser, platform, device) combinations that correspond to setups people
+ * actually run. Anything not listed here (e.g. `edge` on android, `safari` on
+ * windows, `desktop` mobile platforms) is left out so a randomized default
+ * never produces a fingerprint that would itself be a giveaway.
+ */
+const PROFILES_BY_PLATFORM = [
+    { browser: 'chrome', platform: 'windows', device: 'desktop' },
+    { browser: 'firefox', platform: 'windows', device: 'desktop' },
+    { browser: 'edge', platform: 'windows', device: 'desktop' },
+    { browser: 'chrome', platform: 'macos', device: 'desktop' },
+    { browser: 'firefox', platform: 'macos', device: 'desktop' },
+    { browser: 'safari', platform: 'macos', device: 'desktop' },
+    { browser: 'edge', platform: 'macos', device: 'desktop' },
+    { browser: 'chrome', platform: 'linux', device: 'desktop' },
+    { browser: 'firefox', platform: 'linux', device: 'desktop' },
+    { browser: 'chrome', platform: 'android', device: 'mobile' },
+    { browser: 'firefox', platform: 'android', device: 'mobile' },
+    { browser: 'safari', platform: 'ios', device: 'mobile' },
+];
+/**
+ * Build a {@link SessionFingerprint} whose `platform` matches the host OS
+ * and whose `browser`/`device` are randomized within the realistic profiles for
+ * that platform. Used by {@link SessionPool} as the default fingerprint for
+ * freshly created sessions; callers can override by passing their own
+ * `fingerprint` in `sessionOptions`.
+ */
+export function createDefaultSessionFingerprint() {
+    return { ...PROFILES_BY_PLATFORM[Math.floor(Math.random() * PROFILES_BY_PLATFORM.length)] };
+}

package/internals/session_pool/index.d.ts

@@ -0,0 +1,4 @@
+export * from './errors.js';
+export * from './session.js';
+export * from './session_pool.js';
+export * from './consts.js';

package/internals/session_pool/index.js

@@ -0,0 +1,4 @@
+export * from './errors.js';
+export * from './session.js';
+export * from './session_pool.js';
+export * from './consts.js';

package/internals/session_pool/session.d.ts

@@ -0,0 +1,153 @@
+import type { CrawleeLogger } from '@crawlee/core';
+import type { Dictionary, ISession, ProxyInfo, SessionFingerprint, SessionState } from '@crawlee/types';
+import { CookieJar } from 'tough-cookie';
+export interface SessionOptions {
+    /** Id of session used for generating fingerprints. It is used as proxy session name. */
+    id?: string;
+    /**
+     * Number of seconds after which the session is considered as expired.
+     * @default 3000
+     */
+    maxAgeSecs?: number;
+    /** Object where custom user data can be stored. For example custom headers. */
+    userData?: Dictionary;
+    /**
+     * Maximum number of marking session as blocked usage.
+     * If the `errorScore` reaches the `maxErrorScore` session is marked as block and it is thrown away.
+     * It starts at 0. Calling the `markBad` function increases the `errorScore` by 1.
+     * Calling the `markGood` will decrease the `errorScore` by `errorScoreDecrement`
+     * @default 3
+     */
+    maxErrorScore?: number;
+    /**
+     * It is used for healing the session.
+     * For example: if your session is marked bad two times, but it is successful on the third attempt it's errorScore
+     * is decremented by this number.
+     * @default 0.5
+     */
+    errorScoreDecrement?: number;
+    /** Date of creation. */
+    createdAt?: Date;
+    /**
+     * Date of expiration.
+     * @default createdAt + maxAgeSecs
+     */
+    expiresAt?: Date;
+    /**
+     * Indicates how many times the session has been used.
+     * @default 0
+     */
+    usageCount?: number;
+    /**
+     * Session should be used only a limited amount of times.
+     * This number indicates how many times the session is going to be used, before it is thrown away.
+     * @default 50
+     */
+    maxUsageCount?: number;
+    /**
+     * Marks the session as already retired. Used when restoring a previously persisted session
+     * so that `isUsable()` reflects the terminal state regardless of error score or usage count.
+     * @default false
+     */
+    retired?: boolean;
+    log?: CrawleeLogger;
+    errorScore?: number;
+    cookieJar?: CookieJar;
+    proxyInfo?: ProxyInfo;
+    /**
+     * Browser / HTTP client fingerprint tied to this session. Backends use this to make
+     * repeated requests with the same session look consistent (same user-agent, headers,
+     * TLS profile). See {@link SessionFingerprint}.
+     */
+    fingerprint?: SessionFingerprint;
+}
+/**
+ * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
+ * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
+ * Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
+ * @category Scaling
+ */
+export declare class Session implements ISession {
+    #private;
+    readonly id: string;
+    readonly userData: Dictionary;
+    get errorScore(): number;
+    get usageCount(): number;
+    get maxErrorScore(): number;
+    get errorScoreDecrement(): number;
+    get expiresAt(): Date;
+    get createdAt(): Date;
+    get maxUsageCount(): number;
+    get cookieJar(): CookieJar;
+    get proxyInfo(): ProxyInfo | undefined;
+    get fingerprint(): SessionFingerprint | undefined;
+    set fingerprint(fingerprint: SessionFingerprint | undefined);
+    /**
+     * `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
+     * a retired session is never picked by the pool and cannot be revived via `markGood()`.
+     */
+    get retired(): boolean;
+    /**
+     * Session configuration.
+     */
+    constructor(options?: SessionOptions);
+    /**
+     * Indicates whether the session is blocked.
+     * Session is blocked once it reaches the `maxErrorScore`.
+     */
+    isBlocked(): boolean;
+    /**
+     * Indicates whether the session is expired.
+     * Session expiration is determined by the `maxAgeSecs`.
+     * Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
+     */
+    isExpired(): boolean;
+    /**
+     * Indicates whether the session is used maximum number of times.
+     * Session maximum usage count can be changed by `maxUsageCount` parameter.
+     */
+    isMaxUsageCountReached(): boolean;
+    /**
+     * Indicates whether the session can be used for next requests.
+     * Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
+     */
+    isUsable(): boolean;
+    /**
+     * This method should be called after a successful session usage.
+     * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
+     */
+    markGood(): void;
+    /**
+     * Gets session state for persistence in KeyValueStore.
+     * @internal
+     */
+    getState(): SessionState;
+    /**
+     * Permanently retires the session — `isUsable()` will return `false` from here on,
+     * and no `markGood()` / `markBad()` can revive it. Calling `retire()` again is a no-op.
+     *
+     * Use this when you're confident the session itself is the problem (e.g. a `403` response).
+     * For transient external failures (such as `5XX` responses), use `markBad()` instead.
+     */
+    retire(): void;
+    /**
+     * Increases usage and error count.
+     * Should be used when the session has been used unsuccessfully. For example because of timeouts.
+     */
+    markBad(): void;
+    /**
+     * Returns cookies saved with the session in the typical
+     * key1=value1; key2=value2 format, ready to be used in
+     * a cookie header or elsewhere.
+     * @returns Represents `Cookie` header.
+     */
+    getCookieString(url: string): Promise<string>;
+    /**
+     * Sets a cookie within this session for the specific URL.
+     */
+    setCookie(rawCookie: string, url: string): Promise<void>;
+    /**
+     * Checks if session is not usable. if it is not retires the session.
+     */
+    private maybeSelfRetire;
+}

package/internals/session_pool/session.js

@@ -0,0 +1,221 @@
+import { serviceLocator, validators } from '@crawlee/core';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { CookieJar } from 'tough-cookie';
+import { z } from 'zod';
+import { cryptoRandomObjectId } from '@apify/utilities';
+// `schemas.anyObject` passes values through by reference (object schemas return a pruned plain
+// copy), so class instances like cookie jars and loggers keep their prototype.
+const sessionOptionsSchema = z.strictObject({
+    id: z.string().default(() => `session_${cryptoRandomObjectId(10)}`),
+    cookieJar: schemas.anyObject.default(() => new CookieJar()),
+    proxyInfo: schemas.anyObject.optional(),
+    maxAgeSecs: schemas.anyNumber.default(3000),
+    userData: schemas.anyObject.default(() => ({})),
+    maxErrorScore: schemas.anyNumber.default(3),
+    errorScoreDecrement: schemas.anyNumber.default(0.5),
+    createdAt: z.date().default(() => new Date()),
+    expiresAt: z.date().optional(),
+    usageCount: schemas.anyNumber.default(0),
+    errorScore: schemas.anyNumber.default(0),
+    maxUsageCount: schemas.anyNumber.default(50),
+    retired: z.boolean().default(false),
+    log: validators.logger.default(() => serviceLocator.getLogger()),
+    fingerprint: schemas.anyObject.optional(),
+});
+/**
+ * Sessions are used to store information such as cookies and can be used for generating fingerprints and proxy sessions.
+ * You can imagine each session as a specific user, with its own cookies, IP (via proxy) and potentially a unique browser fingerprint.
+ * Session internal state can be enriched with custom user data for example some authorization tokens and specific headers in general.
+ * @category Scaling
+ */
+export class Session {
+    id;
+    userData;
+    #maxErrorScore;
+    #errorScoreDecrement;
+    #createdAt;
+    #expiresAt;
+    #usageCount;
+    #maxUsageCount;
+    #errorScore;
+    #retired = false;
+    #proxyInfo;
+    #cookieJar;
+    #fingerprint;
+    #log;
+    get errorScore() {
+        return this.#errorScore;
+    }
+    get usageCount() {
+        return this.#usageCount;
+    }
+    get maxErrorScore() {
+        return this.#maxErrorScore;
+    }
+    get errorScoreDecrement() {
+        return this.#errorScoreDecrement;
+    }
+    get expiresAt() {
+        return this.#expiresAt;
+    }
+    get createdAt() {
+        return this.#createdAt;
+    }
+    get maxUsageCount() {
+        return this.#maxUsageCount;
+    }
+    get cookieJar() {
+        return this.#cookieJar;
+    }
+    get proxyInfo() {
+        return this.#proxyInfo;
+    }
+    get fingerprint() {
+        return this.#fingerprint;
+    }
+    set fingerprint(fingerprint) {
+        this.#fingerprint = fingerprint;
+    }
+    /**
+     * `true` once {@link Session.retire|`retire()`} has been called. Retirement is terminal:
+     * a retired session is never picked by the pool and cannot be revived via `markGood()`.
+     */
+    get retired() {
+        return this.#retired;
+    }
+    /**
+     * Session configuration.
+     */
+    constructor(options = {}) {
+        const { id, cookieJar, proxyInfo, maxAgeSecs, userData, maxErrorScore, errorScoreDecrement, createdAt, usageCount, errorScore, maxUsageCount, retired, log, fingerprint, 
+        // Anchored to `createdAt` rather than to "now", so the documented `createdAt + maxAgeSecs` holds.
+        expiresAt = new Date(createdAt.getTime() + maxAgeSecs * 1000), } = parseArgument(options, sessionOptionsSchema);
+        this.#log = log.child({ prefix: 'Session' });
+        this.#cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
+        this.#proxyInfo = proxyInfo;
+        this.#fingerprint = fingerprint;
+        this.id = id;
+        this.userData = userData;
+        this.#maxErrorScore = maxErrorScore;
+        this.#errorScoreDecrement = errorScoreDecrement;
+        // Internal
+        this.#expiresAt = expiresAt;
+        this.#createdAt = createdAt;
+        this.#usageCount = usageCount; // indicates how many times the session has been used
+        this.#errorScore = errorScore; // indicates number of markBaded request with the session
+        this.#maxUsageCount = maxUsageCount;
+        this.#retired = retired;
+    }
+    /**
+     * Indicates whether the session is blocked.
+     * Session is blocked once it reaches the `maxErrorScore`.
+     */
+    isBlocked() {
+        return this.errorScore >= this.maxErrorScore;
+    }
+    /**
+     * Indicates whether the session is expired.
+     * Session expiration is determined by the `maxAgeSecs`.
+     * Once the session is older than `createdAt + maxAgeSecs` the session is considered expired.
+     */
+    isExpired() {
+        return this.expiresAt <= new Date();
+    }
+    /**
+     * Indicates whether the session is used maximum number of times.
+     * Session maximum usage count can be changed by `maxUsageCount` parameter.
+     */
+    isMaxUsageCountReached() {
+        return this.usageCount >= this.maxUsageCount;
+    }
+    /**
+     * Indicates whether the session can be used for next requests.
+     * Session is usable when it is not retired, not expired, not blocked and the maximum usage count has not be reached.
+     */
+    isUsable() {
+        return !this.#retired && !this.isBlocked() && !this.isExpired() && !this.isMaxUsageCountReached();
+    }
+    /**
+     * This method should be called after a successful session usage.
+     * It increases `usageCount` and potentially lowers the `errorScore` by the `errorScoreDecrement`.
+     */
+    markGood() {
+        this.#usageCount += 1;
+        if (this.#errorScore > 0) {
+            this.#errorScore -= this.#errorScoreDecrement;
+        }
+        this.maybeSelfRetire();
+    }
+    /**
+     * Gets session state for persistence in KeyValueStore.
+     * @internal
+     */
+    getState() {
+        return {
+            id: this.id,
+            cookieJar: this.cookieJar.toJSON(),
+            proxyInfo: this.#proxyInfo,
+            userData: this.userData,
+            fingerprint: this.#fingerprint,
+            maxErrorScore: this.maxErrorScore,
+            errorScoreDecrement: this.errorScoreDecrement,
+            expiresAt: this.expiresAt.toISOString(),
+            createdAt: this.createdAt.toISOString(),
+            usageCount: this.usageCount,
+            maxUsageCount: this.maxUsageCount,
+            errorScore: this.errorScore,
+            retired: this.#retired,
+        };
+    }
+    /**
+     * Permanently retires the session — `isUsable()` will return `false` from here on,
+     * and no `markGood()` / `markBad()` can revive it. Calling `retire()` again is a no-op.
+     *
+     * Use this when you're confident the session itself is the problem (e.g. a `403` response).
+     * For transient external failures (such as `5XX` responses), use `markBad()` instead.
+     */
+    retire() {
+        if (this.#retired)
+            return;
+        this.#errorScore += this.#maxErrorScore;
+        this.#usageCount += 1;
+        this.#retired = true;
+    }
+    /**
+     * Increases usage and error count.
+     * Should be used when the session has been used unsuccessfully. For example because of timeouts.
+     */
+    markBad() {
+        this.#errorScore += 1;
+        this.#usageCount += 1;
+        this.maybeSelfRetire();
+    }
+    /**
+     * Returns cookies saved with the session in the typical
+     * key1=value1; key2=value2 format, ready to be used in
+     * a cookie header or elsewhere.
+     * @returns Represents `Cookie` header.
+     */
+    async getCookieString(url) {
+        return this.cookieJar.getCookieString(url, {});
+    }
+    /**
+     * Sets a cookie within this session for the specific URL.
+     */
+    async setCookie(rawCookie, url) {
+        try {
+            await this.cookieJar.setCookie(rawCookie, url);
+        }
+        catch (e) {
+            this.#log.warning('Could not set cookie.', { url, error: e.message });
+        }
+    }
+    /**
+     * Checks if session is not usable. if it is not retires the session.
+     */
+    maybeSelfRetire() {
+        if (!this.isUsable()) {
+            this.retire();
+        }
+    }
+}