@crawlee/core 4.0.0-beta.99 → 4.0.0-rc.1

package/autoscaling/system_status.js

@@ -1,139 +0,0 @@
-import { evaluateLoadSignalSample } from './load_signal.js';
-/**
- * How far back the *current* system status looks by default — the window that gates task dispatch.
- * @internal
- */
-export const DEFAULT_CURRENT_HISTORY_SECS = 5;
-/**
- * How far back the *historical* system status looks by default — the window autoscaling decisions are based on, and
- * therefore how much history the signals are asked to retain.
- * @internal
- */
-export const DEFAULT_SNAPSHOT_HISTORY_SECS = 30;
-/** The four built-in signal names that map to typed `SystemInfo` fields, and the option that switches each off. */
-const BUILTIN_SIGNAL_OPTION_KEYS = {
-    memInfo: 'memory',
-    eventLoopInfo: 'eventLoop',
-    cpuInfo: 'cpu',
-    clientInfo: 'client',
-};
-const BUILTIN_SIGNAL_NAMES = new Set(Object.keys(BUILTIN_SIGNAL_OPTION_KEYS));
-/**
- * Reads the overload verdict of every signal — the {@link Snapshotter}'s built-in four plus any custom ones — and
- * combines them into a {@link SystemInfo}: each signal is a time-weighted average of its snapshots, and the system
- * is overloaded whenever at least one of them is.
- *
- * Evaluated over two windows, both requested explicitly from every signal so that a signal's private retention cannot
- * widen what it contributes: a short `currentHistorySecs` one ({@link SystemStatus.getCurrentStatus}, gating task
- * dispatch) and a longer `historySecs` one ({@link SystemStatus.getHistoricalStatus}, driving autoscaling).
- *
- * An implementation detail of the {@link ConcurrencySystem}, configured through
- * {@link ConcurrencySystemOptions}.
- * @internal
- */
-export class SystemStatus {
-    currentHistoryMillis;
-    historyMillis;
-    signals;
-    constructor(options) {
-        const { currentHistorySecs = DEFAULT_CURRENT_HISTORY_SECS, historySecs = DEFAULT_SNAPSHOT_HISTORY_SECS, snapshotter, loadSignals = [], } = options;
-        this.currentHistoryMillis = currentHistorySecs * 1000;
-        this.historyMillis = historySecs * 1000;
-        this.signals = [...snapshotter.getLoadSignals(), ...loadSignals];
-        this.assertUniqueSignalNames();
-    }
-    /**
-     * The widest window any signal will be queried with, and therefore exactly how much history the signals are asked
-     * to retain when they start. Derived here, where the windows are resolved, so nothing has to reapply their
-     * defaults.
-     */
-    get maxSampleWindowMillis() {
-        return Math.max(this.currentHistoryMillis, this.historyMillis);
-    }
-    /**
-     * Signal names are the keys of the reported {@link SystemInfo}, so a duplicate would leave a status object that
-     * contradicts actual behavior: both signals are still evaluated (any overloaded one holds concurrency down), but
-     * only the last is reported.
-     */
-    assertUniqueSignalNames() {
-        const seen = new Set();
-        for (const { name } of this.signals) {
-            if (!seen.has(name)) {
-                seen.add(name);
-                continue;
-            }
-            const hint = BUILTIN_SIGNAL_NAMES.has(name)
-                ? `it is the name of a built-in signal. To replace that signal, switch it off with \`loadSignals: { ${BUILTIN_SIGNAL_OPTION_KEYS[name]}: false }\` and keep your implementation in \`loadSignals.custom\`; to run yours alongside it, give it a different name.`
-                : 'two custom signals cannot share a name - rename one of them.';
-            throw new Error(`Duplicate load signal name ${JSON.stringify(name)}: ${hint}`);
-        }
-    }
-    /**
-     * Returns an {@link SystemInfo} object with the following structure:
-     *
-     * ```javascript
-     * {
-     *     isSystemIdle: Boolean,
-     *     memInfo: Object,
-     *     eventLoopInfo: Object,
-     *     cpuInfo: Object
-     * }
-     * ```
-     *
-     * Where the `isSystemIdle` property is set to `false` if the system
-     * has been overloaded in the last `options.currentHistorySecs` seconds,
-     * and `true` otherwise.
-     */
-    getCurrentStatus() {
-        return this.isSystemIdle(this.currentHistoryMillis);
-    }
-    /**
-     * Returns an {@link SystemInfo} object with the following structure:
-     *
-     * ```javascript
-     * {
-     *     isSystemIdle: Boolean,
-     *     memInfo: Object,
-     *     eventLoopInfo: Object,
-     *     cpuInfo: Object
-     * }
-     * ```
-     *
-     * Where the `isSystemIdle` property is set to `false` if the system has been overloaded within the last
-     * `historySecs` seconds and `true` otherwise.
-     */
-    getHistoricalStatus() {
-        return this.isSystemIdle(this.historyMillis);
-    }
-    /**
-     * Returns a system status object.
-     */
-    isSystemIdle(sampleDurationMillis) {
-        const result = {
-            isSystemIdle: true,
-            memInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
-            eventLoopInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
-            cpuInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
-            clientInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
-        };
-        let loadSignalInfo;
-        for (const signal of this.signals) {
-            const sample = signal.getSample(sampleDurationMillis);
-            const info = evaluateLoadSignalSample(sample, signal.overloadedRatio);
-            if (info.isOverloaded) {
-                result.isSystemIdle = false;
-            }
-            if (BUILTIN_SIGNAL_NAMES.has(signal.name)) {
-                result[signal.name] = info;
-            }
-            else {
-                loadSignalInfo ??= {};
-                loadSignalInfo[signal.name] = info;
-            }
-        }
-        if (loadSignalInfo) {
-            result.loadSignalInfo = loadSignalInfo;
-        }
-        return result;
-    }
-}

package/autoscaling/weighted_avg.d.ts

@@ -1,5 +0,0 @@
-/**
- * Computes a weighted average of an array of numbers, complemented by an array of weights.
- * @ignore
- */
-export declare function weightedAvg(arrValues: number[], arrWeights: number[]): number;

package/autoscaling/weighted_avg.js

@@ -1,14 +0,0 @@
-/**
- * Computes a weighted average of an array of numbers, complemented by an array of weights.
- * @ignore
- */
-export function weightedAvg(arrValues, arrWeights) {
-    const result = arrValues
-        .map((value, i) => {
-        const weight = arrWeights[i];
-        const sum = value * weight;
-        return [sum, weight];
-    })
-        .reduce((p, c) => [p[0] + c[0], p[1] + c[1]], [0, 0]);
-    return result[0] / result[1];
-}

package/cookie_utils.d.ts

@@ -1,44 +0,0 @@
-import type { Cookie as CookieObject } from '@crawlee/types';
-import { Cookie } from 'tough-cookie';
-export interface ResponseLike {
-    url?: string | (() => string);
-    headers?: Record<string, string | string[] | undefined> | (() => Record<string, string | string[] | undefined>);
-}
-/**
- * @internal
- */
-export declare function getCookiesFromResponse(response: Response): Cookie[];
-/**
- * Calculate cookie expiration date
- * @param maxAgeSecs
- * @returns Calculated date by session max age seconds.
- * @internal
- */
-export declare function getDefaultCookieExpirationDate(maxAgeSecs: number): Date;
-/**
- * Transforms tough-cookie to puppeteer cookie.
- * @param toughCookie Cookie from CookieJar
- * @return Cookie compatible with browser pool
- * @internal
- */
-export declare function toughCookieToBrowserPoolCookie(toughCookie: Cookie): CookieObject;
-/**
- * Transforms browser-pool cookie to tough-cookie.
- * @param cookieObject Cookie object (for instance from the `page.cookies` method).
- * @param maxAgeSecs Fallback expiration in seconds when the cookie itself has no `expires`.
- * When omitted, such a cookie is stored as a session cookie (no automatic expiration).
- * @internal
- */
-export declare function browserPoolCookieToToughCookie(cookieObject: CookieObject, maxAgeSecs?: number): Cookie;
-/**
- * @internal
- * @param cookieString The cookie string to attempt parsing
- * @returns Browser pool compatible cookie, or null if cookie cannot be parsed
- */
-export declare function cookieStringToToughCookie(cookieString: string): CookieObject | null;
-/**
- * Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
- * if we see two cookies with same keys but different casing.
- * @internal
- */
-export declare function mergeCookies(url: string, sourceCookies: string[]): string;

package/cookie_utils.js

@@ -1,122 +0,0 @@
-import { Cookie, CookieJar } from 'tough-cookie';
-import { serviceLocator } from './service_locator.js';
-import { CookieParseError } from './session_pool/errors.js';
-/**
- * @internal
- */
-export function getCookiesFromResponse(response) {
-    const headers = response.headers;
-    const cookieHeaders = headers.getSetCookie();
-    try {
-        return cookieHeaders.map((cookie) => Cookie.parse(cookie));
-    }
-    catch (e) {
-        throw new CookieParseError(cookieHeaders);
-    }
-}
-/**
- * Calculate cookie expiration date
- * @param maxAgeSecs
- * @returns Calculated date by session max age seconds.
- * @internal
- */
-export function getDefaultCookieExpirationDate(maxAgeSecs) {
-    return new Date(Date.now() + maxAgeSecs * 1000);
-}
-/**
- * Transforms tough-cookie to puppeteer cookie.
- * @param toughCookie Cookie from CookieJar
- * @return Cookie compatible with browser pool
- * @internal
- */
-export function toughCookieToBrowserPoolCookie(toughCookie) {
-    return {
-        name: toughCookie.key,
-        value: toughCookie.value,
-        // Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms
-        // If there is no expires date (so defaults to Infinity), we don't provide it to the browsers
-        expires: toughCookie.expires == null || toughCookie.expires === 'Infinity'
-            ? undefined
-            : toughCookie.expires.getTime() / 1000,
-        domain: toughCookie.domain ? `${toughCookie.hostOnly ? '' : '.'}${toughCookie.domain}` : undefined,
-        path: toughCookie.path ?? undefined,
-        secure: toughCookie.secure,
-        httpOnly: toughCookie.httpOnly,
-    };
-}
-/**
- * Transforms browser-pool cookie to tough-cookie.
- * @param cookieObject Cookie object (for instance from the `page.cookies` method).
- * @param maxAgeSecs Fallback expiration in seconds when the cookie itself has no `expires`.
- * When omitted, such a cookie is stored as a session cookie (no automatic expiration).
- * @internal
- */
-export function browserPoolCookieToToughCookie(cookieObject, maxAgeSecs) {
-    const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
-    let expires;
-    if (isExpiresValid) {
-        expires = new Date(cookieObject.expires * 1000);
-    }
-    else if (maxAgeSecs != null) {
-        expires = getDefaultCookieExpirationDate(maxAgeSecs);
-    }
-    const domainHasLeadingDot = cookieObject.domain?.startsWith?.('.');
-    const domain = domainHasLeadingDot ? cookieObject.domain?.slice?.(1) : cookieObject.domain;
-    return new Cookie({
-        key: cookieObject.name,
-        value: cookieObject.value,
-        ...(expires !== undefined && { expires }),
-        domain,
-        path: cookieObject.path,
-        secure: cookieObject.secure,
-        httpOnly: cookieObject.httpOnly,
-        hostOnly: !domainHasLeadingDot,
-    });
-}
-/**
- * @internal
- * @param cookieString The cookie string to attempt parsing
- * @returns Browser pool compatible cookie, or null if cookie cannot be parsed
- */
-export function cookieStringToToughCookie(cookieString) {
-    const parsed = Cookie.parse(cookieString);
-    if (parsed) {
-        return toughCookieToBrowserPoolCookie(parsed);
-    }
-    return null;
-}
-/**
- * Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
- * if we see two cookies with same keys but different casing.
- * @internal
- */
-export function mergeCookies(url, sourceCookies) {
-    const jar = new CookieJar();
-    // ignore empty cookies
-    for (const sourceCookieString of sourceCookies) {
-        // ignore empty cookies
-        if (!sourceCookieString)
-            continue;
-        const cookies = sourceCookieString.split(/ *; */);
-        for (const cookieString of cookies) {
-            // ignore extra spaces
-            if (!cookieString)
-                continue;
-            const cookie = Cookie.parse(cookieString);
-            if (!cookie) {
-                serviceLocator.getLogger().warning(`Skipping malformed cookie fragment: '${cookieString}'`);
-                continue;
-            }
-            const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
-                return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
-            });
-            if (similarKeyCookie) {
-                serviceLocator
-                    .getLogger()
-                    .warningOnce(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
-            }
-            jar.setCookieSync(cookie, url);
-        }
-    }
-    return jar.getCookieStringSync(url);
-}

package/crawlers/context_pipeline.d.ts

@@ -1,70 +0,0 @@
-import type { Awaitable } from '@crawlee/types';
-/**
- * Represents a middleware step in the context pipeline.
- *
- * @template TCrawlingContext - The input context type for this middleware
- * @template TCrawlingContextExtension - The enhanced output context type
- */
-export interface ContextMiddleware<TCrawlingContext, TCrawlingContextExtension> {
-    /** The main middleware function that enhances the context */
-    action: (context: TCrawlingContext) => Awaitable<TCrawlingContextExtension>;
-    /** Optional cleanup function called after the consumer finishes or fails */
-    cleanup?: (context: TCrawlingContext & TCrawlingContextExtension, error?: unknown) => Awaitable<void>;
-}
-/**
- * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
- *
- * The enhancement is done by a chain of middlewares that are added to the pipeline after its creation.
- * This class provides a type-safe way to build a pipeline of context transformations where each step
- * can enhance the context with additional properties or utilities.
- *
- * @template TContextBase - The base context type that serves as the starting point
- * @template TCrawlingContext - The final context type after all middleware transformations
- */
-export declare abstract class ContextPipeline<TContextBase, TCrawlingContext extends TContextBase> {
-    /**
-     * Creates a new empty context pipeline.
-     *
-     * @template TContextBase - The base context type for the pipeline
-     * @returns A new ContextPipeline instance with no transformations
-     */
-    static create<TContextBase>(): ContextPipeline<TContextBase, TContextBase>;
-    /**
-     * Adds a middleware to the pipeline, creating a new pipeline instance.
-     *
-     * This method provides a fluent interface for building context transformation pipelines.
-     * Each middleware can enhance the context with additional properties or utilities.
-     *
-     * @template TCrawlingContextExtension - The enhanced context type produced by this middleware
-     * @param middleware - The middleware to add to the pipeline
-     * @returns A new ContextPipeline instance with the added middleware
-     */
-    abstract compose<TCrawlingContextExtension>(middleware: ContextMiddleware<TCrawlingContext, TCrawlingContextExtension>): ContextPipeline<TContextBase, TCrawlingContext & TCrawlingContextExtension>;
-    /**
-     * Chains another pipeline onto this one. The other pipeline's base context must match
-     * this pipeline's output context. Returns a new pipeline that runs this pipeline's
-     * middlewares first, then the other pipeline's middlewares.
-     *
-     * @template TFinalContext - The final context type after the chained pipeline's transformations
-     * @param other - The pipeline to append after this one
-     * @returns A new ContextPipeline combining both pipelines' middlewares
-     */
-    abstract chain<TFinalContext extends TCrawlingContext>(other: ContextPipeline<TCrawlingContext, TFinalContext>): ContextPipeline<TContextBase, TFinalContext>;
-    /**
-     * Executes the middleware pipeline and passes the final context to a consumer function.
-     *
-     * This method runs the crawling context through the entire middleware chain, enhancing it
-     * at each step, and then passes the final enhanced context to the provided consumer function.
-     * Proper cleanup is performed even if exceptions occur during processing.
-     *
-     * @param crawlingContext - The initial context to process through the pipeline
-     * @param finalContextConsumer - The function that will receive the final enhanced context
-     *
-     * @throws {ContextPipelineInitializationError} When a middleware fails during initialization
-     * @throws {ContextPipelineInterruptedError} When the pipeline is intentionally interrupted during initialization
-     * @throws {RequestHandlerError} When the final context consumer throws an exception
-     * @throws {ContextPipelineCleanupError} When cleanup operations fail
-     * @throws {SessionError} Session errors are re-thrown as-is for special handling
-     */
-    abstract call(crawlingContext: TContextBase, finalContextConsumer: (finalContext: TCrawlingContext) => Awaitable<unknown>): Promise<void>;
-}

package/crawlers/context_pipeline.js

@@ -1,122 +0,0 @@
-import { ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, RequestHandlerError, SessionError, } from '../errors.js';
-import { serviceLocator } from '../service_locator.js';
-/**
- * Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.
- *
- * The enhancement is done by a chain of middlewares that are added to the pipeline after its creation.
- * This class provides a type-safe way to build a pipeline of context transformations where each step
- * can enhance the context with additional properties or utilities.
- *
- * @template TContextBase - The base context type that serves as the starting point
- * @template TCrawlingContext - The final context type after all middleware transformations
- */
-export class ContextPipeline {
-    /**
-     * Creates a new empty context pipeline.
-     *
-     * @template TContextBase - The base context type for the pipeline
-     * @returns A new ContextPipeline instance with no transformations
-     */
-    static create() {
-        return new ContextPipelineImpl({ action: async (context) => context });
-    }
-}
-/**
- * Implementation of the `ContextPipeline` logic. This hides implementation details such as the `middleware` and `parent`
- * properties from the `ContextPipeline` interface, making type checking more reliable.
- */
-class ContextPipelineImpl extends ContextPipeline {
-    middleware;
-    parent;
-    constructor(middleware, parent) {
-        super();
-        this.middleware = middleware;
-        this.parent = parent;
-    }
-    /**
-     * @inheritdoc
-     */
-    compose(middleware) {
-        return new ContextPipelineImpl(middleware, this);
-    }
-    chain(other) {
-        const otherMiddlewares = Array.from(other.middlewareChain()).reverse();
-        let result = this;
-        for (const middleware of otherMiddlewares) {
-            result = result.compose(middleware);
-        }
-        return result;
-    }
-    *middlewareChain() {
-        let step = this;
-        while (step !== undefined) {
-            yield step.middleware;
-            step = step.parent;
-        }
-    }
-    /**
-     * @inheritdoc
-     */
-    async call(crawlingContext, finalContextConsumer) {
-        const middlewares = Array.from(this.middlewareChain()).reverse();
-        const cleanupStack = [];
-        let consumerException;
-        try {
-            for (const { action, cleanup } of middlewares) {
-                try {
-                    const contextExtension = await action(crawlingContext);
-                    const extensionNames = [
-                        ...Object.getOwnPropertyNames(contextExtension),
-                        ...Object.getOwnPropertySymbols(contextExtension),
-                    ];
-                    for (const key of extensionNames) {
-                        try {
-                            if (Object.getOwnPropertyDescriptor(crawlingContext, key)?.configurable !== false) {
-                                Object.defineProperty(crawlingContext, key, Object.getOwnPropertyDescriptor(contextExtension, key));
-                            }
-                        }
-                        catch (error) {
-                            serviceLocator
-                                .getLogger()
-                                .debug(`Context pipeline failed to define property ${key.toString()}:`, error);
-                        }
-                    }
-                    if (cleanup) {
-                        cleanupStack.push(cleanup);
-                    }
-                }
-                catch (exception) {
-                    if (exception instanceof SessionError) {
-                        throw exception; // Session errors are re-thrown as-is
-                    }
-                    if (exception instanceof ContextPipelineInterruptedError) {
-                        throw exception;
-                    }
-                    throw new ContextPipelineInitializationError(exception);
-                }
-            }
-            try {
-                await finalContextConsumer(crawlingContext);
-            }
-            catch (exception) {
-                if (exception instanceof SessionError) {
-                    consumerException = exception;
-                    throw exception; // Session errors are re-thrown as-is
-                }
-                consumerException = exception;
-                throw new RequestHandlerError(exception);
-            }
-        }
-        finally {
-            try {
-                for (const cleanup of cleanupStack.reverse()) {
-                    await cleanup(crawlingContext, consumerException);
-                }
-            }
-            catch (exception) {
-                // eslint-disable-next-line no-unsafe-finally
-                throw new ContextPipelineCleanupError(exception);
-            }
-        }
-    }
-}