@crawlee/basic 4.0.0-beta.99 → 4.0.0-rc.1

package/internals/cookie_utils.d.ts

@@ -0,0 +1,40 @@
+import type { Cookie as CookieObject } from '@crawlee/types';
+import { Cookie } from 'tough-cookie';
+/**
+ * @internal
+ */
+export declare function getCookiesFromResponse(response: Response): Cookie[];
+/**
+ * Calculate cookie expiration date
+ * @param maxAgeSecs
+ * @returns Calculated date by session max age seconds.
+ * @internal
+ */
+export declare function getDefaultCookieExpirationDate(maxAgeSecs: number): Date;
+/**
+ * Transforms tough-cookie to puppeteer cookie.
+ * @param toughCookie Cookie from CookieJar
+ * @return Cookie compatible with browser pool
+ * @internal
+ */
+export declare function toughCookieToBrowserPoolCookie(toughCookie: Cookie): CookieObject;
+/**
+ * Transforms browser-pool cookie to tough-cookie.
+ * @param cookieObject Cookie object (for instance from the `page.cookies` method).
+ * @param maxAgeSecs Fallback expiration in seconds when the cookie itself has no `expires`.
+ * When omitted, such a cookie is stored as a session cookie (no automatic expiration).
+ * @internal
+ */
+export declare function browserPoolCookieToToughCookie(cookieObject: CookieObject, maxAgeSecs?: number): Cookie;
+/**
+ * @internal
+ * @param cookieString The cookie string to attempt parsing
+ * @returns Browser pool compatible cookie, or null if cookie cannot be parsed
+ */
+export declare function cookieStringToToughCookie(cookieString: string): CookieObject | null;
+/**
+ * Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
+ * if we see two cookies with same keys but different casing.
+ * @internal
+ */
+export declare function mergeCookies(url: string, sourceCookies: string[]): string;

package/internals/cookie_utils.js

@@ -0,0 +1,122 @@
+import { serviceLocator } from '@crawlee/core';
+import { Cookie, CookieJar } from 'tough-cookie';
+import { CookieParseError } from './session_pool/errors.js';
+/**
+ * @internal
+ */
+export function getCookiesFromResponse(response) {
+    const headers = response.headers;
+    const cookieHeaders = headers.getSetCookie();
+    try {
+        return cookieHeaders.map((cookie) => Cookie.parse(cookie));
+    }
+    catch (e) {
+        throw new CookieParseError(cookieHeaders);
+    }
+}
+/**
+ * Calculate cookie expiration date
+ * @param maxAgeSecs
+ * @returns Calculated date by session max age seconds.
+ * @internal
+ */
+export function getDefaultCookieExpirationDate(maxAgeSecs) {
+    return new Date(Date.now() + maxAgeSecs * 1000);
+}
+/**
+ * Transforms tough-cookie to puppeteer cookie.
+ * @param toughCookie Cookie from CookieJar
+ * @return Cookie compatible with browser pool
+ * @internal
+ */
+export function toughCookieToBrowserPoolCookie(toughCookie) {
+    return {
+        name: toughCookie.key,
+        value: toughCookie.value,
+        // Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms
+        // If there is no expires date (so defaults to Infinity), we don't provide it to the browsers
+        expires: toughCookie.expires == null || toughCookie.expires === 'Infinity'
+            ? undefined
+            : toughCookie.expires.getTime() / 1000,
+        domain: toughCookie.domain ? `${toughCookie.hostOnly ? '' : '.'}${toughCookie.domain}` : undefined,
+        path: toughCookie.path ?? undefined,
+        secure: toughCookie.secure,
+        httpOnly: toughCookie.httpOnly,
+    };
+}
+/**
+ * Transforms browser-pool cookie to tough-cookie.
+ * @param cookieObject Cookie object (for instance from the `page.cookies` method).
+ * @param maxAgeSecs Fallback expiration in seconds when the cookie itself has no `expires`.
+ * When omitted, such a cookie is stored as a session cookie (no automatic expiration).
+ * @internal
+ */
+export function browserPoolCookieToToughCookie(cookieObject, maxAgeSecs) {
+    const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
+    let expires;
+    if (isExpiresValid) {
+        expires = new Date(cookieObject.expires * 1000);
+    }
+    else if (maxAgeSecs != null) {
+        expires = getDefaultCookieExpirationDate(maxAgeSecs);
+    }
+    const domainHasLeadingDot = cookieObject.domain?.startsWith?.('.');
+    const domain = domainHasLeadingDot ? cookieObject.domain?.slice?.(1) : cookieObject.domain;
+    return new Cookie({
+        key: cookieObject.name,
+        value: cookieObject.value,
+        ...(expires !== undefined && { expires }),
+        domain,
+        path: cookieObject.path,
+        secure: cookieObject.secure,
+        httpOnly: cookieObject.httpOnly,
+        hostOnly: !domainHasLeadingDot,
+    });
+}
+/**
+ * @internal
+ * @param cookieString The cookie string to attempt parsing
+ * @returns Browser pool compatible cookie, or null if cookie cannot be parsed
+ */
+export function cookieStringToToughCookie(cookieString) {
+    const parsed = Cookie.parse(cookieString);
+    if (parsed) {
+        return toughCookieToBrowserPoolCookie(parsed);
+    }
+    return null;
+}
+/**
+ * Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
+ * if we see two cookies with same keys but different casing.
+ * @internal
+ */
+export function mergeCookies(url, sourceCookies) {
+    const jar = new CookieJar();
+    // ignore empty cookies
+    for (const sourceCookieString of sourceCookies) {
+        // ignore empty cookies
+        if (!sourceCookieString)
+            continue;
+        const cookies = sourceCookieString.split(/ *; */);
+        for (const cookieString of cookies) {
+            // ignore extra spaces
+            if (!cookieString)
+                continue;
+            const cookie = Cookie.parse(cookieString);
+            if (!cookie) {
+                serviceLocator.getLogger().warning(`Skipping malformed cookie fragment: '${cookieString}'`);
+                continue;
+            }
+            const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
+                return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
+            });
+            if (similarKeyCookie) {
+                serviceLocator
+                    .getLogger()
+                    .warningOnce(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
+            }
+            jar.setCookieSync(cookie, url);
+        }
+    }
+    return jar.getCookieStringSync(url);
+}

package/internals/crawler-run.d.ts

@@ -0,0 +1,76 @@
+import type { CrawleeLogger } from '@crawlee/core';
+import type { AutoscaledPoolOptions } from './autoscaling/autoscaled_pool.js';
+import type { IConcurrencySystem } from './autoscaling/concurrency_system.js';
+/** Everything a run needs to exist. Whatever a crawler resolves per run is resolved before the run is built. */
+export interface CrawlerRunSetup {
+    log: CrawleeLogger;
+    /** The governor this run books its requests against, started and stopped by whoever owns it. */
+    concurrencySystem: IConcurrencySystem;
+    taskLoopOptions: Omit<AutoscaledPoolOptions, 'concurrencySystem' | 'consumer'>;
+    consumer: AutoscaledPoolOptions['consumer'];
+}
+/**
+ * Everything that belongs to a single {@link BasicCrawler.run|`crawler.run()`} — the task loop dispatching its
+ * requests, the concurrency governor those requests are booked against, and the bookkeeping that must not survive
+ * into a later run.
+ *
+ * The crawler builds a fresh one for every run, which is what keeps per-run state from leaking across runs: there
+ * is no reset checklist to keep in sync. A finished run stays readable as the history of the last crawl, so
+ * liveness is asked about explicitly (`isLive`) rather than inferred from presence.
+ *
+ * The lifecycle is `dispatchRequests()` and `finish()`. Everything a run owns exists from construction, so there
+ * is no half-built state to guard against in between.
+ *
+ * @internal
+ */
+export declare class CrawlerRun {
+    #private;
+    /** Whether this run initialized the ambient event manager, and is therefore the one that closes it. */
+    ownsEventManager: boolean;
+    constructor(setup: CrawlerRunSetup);
+    /**
+     * Runs the task loop until the crawl is over or {@link CrawlerRun.abort|aborted}. The governor has to be
+     * running by now — the task loop refuses to book anything against one that is not.
+     */
+    dispatchRequests(): Promise<void>;
+    /** Whether the `run()` call this belongs to is still executing. */
+    get isLive(): boolean;
+    /**
+     * Whether the task loop is dispatching requests. Pausing one that has not started would not suspend a crawl,
+     * it would keep it from ever starting.
+     */
+    get isDispatching(): boolean;
+    /** Whether a graceful shutdown of this run has been requested. */
+    get stopRequested(): boolean;
+    /** Asks the run to stop taking new requests. Only the first call logs `reason`. */
+    stop(reason: string): void;
+    /** Stops dispatching new requests, resolving once the ones in flight have settled. */
+    pause(timeoutSecs?: number): Promise<void>;
+    /** Resumes dispatching after a {@link CrawlerRun.pause|`pause()`}. */
+    resume(): void;
+    /** Ends the crawl without waiting for the requests in flight, so a pending `crawl()` resolves. */
+    abort(): Promise<void>;
+    /**
+     * Logs `message` the first time this run asks for it under `key`. Keyed rather than deduped by text, so that
+     * two messages describing the same shutdown say it once between them.
+     */
+    logOnce(key: string, message: string): void;
+    /**
+     * Nudges the task loop at `readyAt`, when the request manager says it will have work again. The loop polls on
+     * its own anyway, so this only shortens the wait — hence a single timer that only an earlier wake-up replaces,
+     * `unref`'d so it never keeps the process alive. A run that has ended arms nothing.
+     */
+    scheduleWake(readyAt: number): void;
+    /**
+     * Ends the run: drops a pending wake-up and aborts the task loop. Idempotent, so aborting a run through
+     * `teardown()` and then finishing it does the work once. The governor is left alone — it belongs to whoever
+     * started it.
+     */
+    end(): Promise<void>;
+    /**
+     * Ends the run and marks it as history: the crawler stops reporting itself as running, and what is left is
+     * readable but no longer drivable. Ending here too means a subclass `teardown()` that forgets `super` cannot
+     * leave the task loop dispatching.
+     */
+    finish(): Promise<void>;
+}

package/internals/crawler-run.js

@@ -0,0 +1,142 @@
+import { AutoscaledPool } from './autoscaling/autoscaled_pool.js';
+/**
+ * Everything that belongs to a single {@link BasicCrawler.run|`crawler.run()`} — the task loop dispatching its
+ * requests, the concurrency governor those requests are booked against, and the bookkeeping that must not survive
+ * into a later run.
+ *
+ * The crawler builds a fresh one for every run, which is what keeps per-run state from leaking across runs: there
+ * is no reset checklist to keep in sync. A finished run stays readable as the history of the last crawl, so
+ * liveness is asked about explicitly (`isLive`) rather than inferred from presence.
+ *
+ * The lifecycle is `dispatchRequests()` and `finish()`. Everything a run owns exists from construction, so there
+ * is no half-built state to guard against in between.
+ *
+ * @internal
+ */
+export class CrawlerRun {
+    #log;
+    #autoscaledPool;
+    #loggedOnce = new Set();
+    #stopRequested = false;
+    #dispatching = false;
+    #ended = false;
+    #finished = false;
+    #wakeTimer;
+    #wakeAt = 0;
+    /** Whether this run initialized the ambient event manager, and is therefore the one that closes it. */
+    ownsEventManager = false;
+    constructor(setup) {
+        this.#log = setup.log;
+        this.#autoscaledPool = new AutoscaledPool({
+            ...setup.taskLoopOptions,
+            concurrencySystem: setup.concurrencySystem,
+            consumer: setup.consumer,
+        });
+    }
+    /**
+     * Runs the task loop until the crawl is over or {@link CrawlerRun.abort|aborted}. The governor has to be
+     * running by now — the task loop refuses to book anything against one that is not.
+     */
+    async dispatchRequests() {
+        this.#dispatching = true;
+        try {
+            await this.#autoscaledPool.run();
+        }
+        finally {
+            this.#dispatching = false;
+        }
+    }
+    /** Whether the `run()` call this belongs to is still executing. */
+    get isLive() {
+        return !this.#finished;
+    }
+    /**
+     * Whether the task loop is dispatching requests. Pausing one that has not started would not suspend a crawl,
+     * it would keep it from ever starting.
+     */
+    get isDispatching() {
+        return this.#dispatching;
+    }
+    /** Whether a graceful shutdown of this run has been requested. */
+    get stopRequested() {
+        return this.#stopRequested;
+    }
+    /** Asks the run to stop taking new requests. Only the first call logs `reason`. */
+    stop(reason) {
+        if (this.#stopRequested) {
+            return;
+        }
+        this.#log.info(reason);
+        this.#stopRequested = true;
+    }
+    /** Stops dispatching new requests, resolving once the ones in flight have settled. */
+    async pause(timeoutSecs) {
+        await this.#autoscaledPool.pause(timeoutSecs);
+    }
+    /** Resumes dispatching after a {@link CrawlerRun.pause|`pause()`}. */
+    resume() {
+        this.#autoscaledPool.resume();
+    }
+    /** Ends the crawl without waiting for the requests in flight, so a pending `crawl()` resolves. */
+    async abort() {
+        await this.#autoscaledPool.abort();
+    }
+    /**
+     * Logs `message` the first time this run asks for it under `key`. Keyed rather than deduped by text, so that
+     * two messages describing the same shutdown say it once between them.
+     */
+    logOnce(key, message) {
+        if (this.#loggedOnce.has(key)) {
+            return;
+        }
+        this.#log.info(message);
+        this.#loggedOnce.add(key);
+    }
+    /**
+     * Nudges the task loop at `readyAt`, when the request manager says it will have work again. The loop polls on
+     * its own anyway, so this only shortens the wait — hence a single timer that only an earlier wake-up replaces,
+     * `unref`'d so it never keeps the process alive. A run that has ended arms nothing.
+     */
+    scheduleWake(readyAt) {
+        if (this.#ended) {
+            return;
+        }
+        if (this.#wakeTimer !== undefined) {
+            if (this.#wakeAt <= readyAt) {
+                return;
+            }
+            clearTimeout(this.#wakeTimer);
+        }
+        this.#wakeAt = readyAt;
+        this.#wakeTimer = setTimeout(() => {
+            this.#wakeTimer = undefined;
+            void this.#autoscaledPool.notify();
+        }, Math.max(0, readyAt - Date.now()));
+        this.#wakeTimer.unref();
+    }
+    /**
+     * Ends the run: drops a pending wake-up and aborts the task loop. Idempotent, so aborting a run through
+     * `teardown()` and then finishing it does the work once. The governor is left alone — it belongs to whoever
+     * started it.
+     */
+    async end() {
+        if (this.#ended) {
+            return;
+        }
+        this.#ended = true;
+        if (this.#wakeTimer !== undefined) {
+            clearTimeout(this.#wakeTimer);
+            this.#wakeTimer = undefined;
+        }
+        await this.#autoscaledPool.abort();
+    }
+    /**
+     * Ends the run and marks it as history: the crawler stops reporting itself as running, and what is left is
+     * readable but no longer drivable. Ending here too means a subclass `teardown()` that forgets `super` cannot
+     * leave the task loop dispatching.
+     */
+    async finish() {
+        await this.end();
+        this.#finished = true;
+    }
+}

package/internals/crawlers/context_pipeline.d.ts

@@ -0,0 +1,75 @@
+import type { Awaitable } from '@crawlee/types';
+/**
+ * Registers a callback that runs after the final context consumer finishes. Cleanups run in reverse registration
+ * order and receive the same `error` as the `onError` parameter of {@link ContextPipeline.call}: `undefined` on
+ * success, otherwise a {@link ContextPipelineInitializationError} (a middleware failed) or a
+ * {@link RequestHandlerError} (the consumer failed), with the original in `cause`. A cleanup that throws fails the
+ * whole call with a {@link ContextPipelineCleanupError}, which is critical.
+ */
+export type CleanupRegistrar = (cleanup: (error?: unknown) => Awaitable<void>) => void;
+/**
+ * A middleware step in the context pipeline: receives the context built so far and returns an extension that gets
+ * merged into it. Anything that needs tearing down is registered via `onCleanup`.
+ *
+ * @template TCrawlingContext - The input context type for this middleware
+ * @template TCrawlingContextExtension - The enhanced output context type
+ */
+export type ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> = (context: TCrawlingContext, onCleanup: CleanupRegistrar) => Awaitable<TCrawlingContextExtension>;
+/**
+ * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
+ *
+ * The enhancement is done by a chain of middlewares that are added to the pipeline after its creation.
+ * This class provides a type-safe way to build a pipeline of context transformations where each step
+ * can enhance the context with additional properties or utilities.
+ *
+ * @template TContextBase - The base context type that serves as the starting point
+ * @template TCrawlingContext - The final context type after all middleware transformations
+ */
+export declare abstract class ContextPipeline<TContextBase, TCrawlingContext extends TContextBase> {
+    /**
+     * Creates a new empty context pipeline.
+     *
+     * @template TContextBase - The base context type for the pipeline
+     * @returns A new ContextPipeline instance with no transformations
+     */
+    static create<TContextBase>(): ContextPipeline<TContextBase, TContextBase>;
+    /**
+     * Adds a middleware to the pipeline, creating a new pipeline instance.
+     *
+     * This method provides a fluent interface for building context transformation pipelines.
+     * Each middleware can enhance the context with additional properties or utilities.
+     *
+     * @template TCrawlingContextExtension - The enhanced context type produced by this middleware
+     * @param middleware - The middleware to add to the pipeline
+     * @returns A new ContextPipeline instance with the added middleware
+     */
+    abstract compose<TCrawlingContextExtension>(middleware: ContextMiddleware<TCrawlingContext, TCrawlingContextExtension>): ContextPipeline<TContextBase, TCrawlingContext & TCrawlingContextExtension>;
+    /**
+     * Chains another pipeline onto this one. The other pipeline's base context must match
+     * this pipeline's output context. Returns a new pipeline that runs this pipeline's
+     * middlewares first, then the other pipeline's middlewares.
+     *
+     * @template TFinalContext - The final context type after the chained pipeline's transformations
+     * @param other - The pipeline to append after this one
+     * @returns A new ContextPipeline combining both pipelines' middlewares
+     */
+    abstract chain<TFinalContext extends TCrawlingContext>(other: ContextPipeline<TCrawlingContext, TFinalContext>): ContextPipeline<TContextBase, TFinalContext>;
+    /**
+     * Executes the middleware pipeline and passes the final context to a consumer function.
+     *
+     * This method runs the crawling context through the entire middleware chain, enhancing it
+     * at each step, and then passes the final enhanced context to the provided consumer function.
+     * Proper cleanup is performed even if exceptions occur during processing.
+     *
+     * @param crawlingContext - The initial context to process through the pipeline
+     * @param finalContextConsumer - The function that will receive the final enhanced context
+     * @param onError - Receives a middleware failure wrapped in a {@link ContextPipelineInitializationError}, or a
+     *   consumer failure wrapped in a {@link RequestHandlerError}, with the original in `cause`. After a middleware
+     *   failure, the rest of the chain and the consumer are skipped. Runs before the cleanups, so it still sees
+     *   whatever the middlewares set up.
+     *
+     * @throws {ContextPipelineCleanupError} When cleanup operations fail
+     * @throws Whatever `onError` throws
+     */
+    abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>, onError: (error: unknown) => Awaitable<void>): Promise<void>;
+}

package/internals/crawlers/context_pipeline.js

@@ -0,0 +1,109 @@
+import { serviceLocator } from '@crawlee/core';
+import { ContextPipelineCleanupError, ContextPipelineInitializationError, RequestHandlerError } from '../errors.js';
+/**
+ * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
+ *
+ * The enhancement is done by a chain of middlewares that are added to the pipeline after its creation.
+ * This class provides a type-safe way to build a pipeline of context transformations where each step
+ * can enhance the context with additional properties or utilities.
+ *
+ * @template TContextBase - The base context type that serves as the starting point
+ * @template TCrawlingContext - The final context type after all middleware transformations
+ */
+export class ContextPipeline {
+    /**
+     * Creates a new empty context pipeline.
+     *
+     * @template TContextBase - The base context type for the pipeline
+     * @returns A new ContextPipeline instance with no transformations
+     */
+    static create() {
+        return new ContextPipelineImpl(async (context) => context);
+    }
+}
+/**
+ * Implementation of the `ContextPipeline` logic. This hides implementation details such as the `middleware` and `parent`
+ * properties from the `ContextPipeline` interface, making type checking more reliable.
+ */
+class ContextPipelineImpl extends ContextPipeline {
+    #middleware;
+    #parent;
+    constructor(middleware, parent) {
+        super();
+        this.#middleware = middleware;
+        this.#parent = parent;
+    }
+    /**
+     * @inheritdoc
+     */
+    compose(middleware) {
+        return new ContextPipelineImpl(middleware, this);
+    }
+    chain(other) {
+        const otherMiddlewares = Array.from(other.#middlewareChain()).reverse();
+        let result = this;
+        for (const middleware of otherMiddlewares) {
+            result = result.compose(middleware);
+        }
+        return result;
+    }
+    *#middlewareChain() {
+        let step = this;
+        while (step !== undefined) {
+            yield step.#middleware;
+            step = step.#parent;
+        }
+    }
+    /**
+     * @inheritdoc
+     */
+    async call(crawlingContext, finalContextConsumer, onError) {
+        const middlewares = Array.from(this.#middlewareChain()).reverse();
+        const cleanupStack = [];
+        const onCleanup = (cleanup) => {
+            cleanupStack.push(cleanup);
+        };
+        let failure;
+        let consumerStarted = false;
+        try {
+            for (const middleware of middlewares) {
+                const contextExtension = await middleware(crawlingContext, onCleanup);
+                const extensionNames = [
+                    ...Object.getOwnPropertyNames(contextExtension),
+                    ...Object.getOwnPropertySymbols(contextExtension),
+                ];
+                for (const key of extensionNames) {
+                    try {
+                        if (Object.getOwnPropertyDescriptor(crawlingContext, key)?.configurable !== false) {
+                            Object.defineProperty(crawlingContext, key, Object.getOwnPropertyDescriptor(contextExtension, key));
+                        }
+                    }
+                    catch (error) {
+                        serviceLocator
+                            .getLogger()
+                            .debug(`Context pipeline failed to define property ${key.toString()}:`, error);
+                    }
+                }
+            }
+            consumerStarted = true;
+            await finalContextConsumer(crawlingContext);
+        }
+        catch (exception) {
+            failure = consumerStarted
+                ? new RequestHandlerError(exception)
+                : new ContextPipelineInitializationError(exception);
+            await onError(failure);
+        }
+        finally {
+            try {
+                for (const cleanup of cleanupStack.reverse()) {
+                    await cleanup(failure);
+                }
+            }
+            catch (exception) {
+                // eslint-disable-next-line no-unsafe-finally
+                throw new ContextPipelineCleanupError(exception);
+            }
+        }
+    }
+}