@crawlee/basic 4.0.0-beta.99 → 4.0.0-rc.1

package/internals/enqueue_links/shared.js

@@ -0,0 +1,258 @@
+import { URL } from 'node:url';
+import { Minimatch } from 'minimatch';
+import { z } from 'zod';
+import { Request } from '@crawlee/core';
+import { schemas } from '@crawlee/utils/internal';
+const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
+/**
+ * To keep high performance when the same patterns are passed on every `enqueueLinks()` call,
+ * each glob/regexp is only compiled once and then kept in a cache.
+ * @ignore
+ */
+const enqueueLinksPatternCache = new Map();
+/**
+ * Accepts one {@link UrlPatternInput} — a glob string, a RegExp instance, or a `{ glob }` / `{ regexp }` object.
+ * @internal
+ */
+export const urlPatternSchema = z.union([
+    z.string(),
+    z.instanceof(RegExp),
+    schemas.objectWithKeys(['glob']),
+    schemas.objectWithKeys(['regexp']),
+]);
+/**
+ * Builds the `{ request, reason }` argument passed to a {@link SkippedRequestCallback}, constructing the
+ * `Request` lazily on first access to `request` (and caching it) since most skips are never observed by a
+ * real callback and building a `Request` isn't free.
+ * @ignore
+ */
+export function createSkippedRequestArgs(source, reason) {
+    let request;
+    return {
+        reason,
+        get request() {
+            request ??=
+                source instanceof Request
+                    ? source
+                    : new Request(typeof source === 'string' ? { url: source } : source);
+            return request;
+        },
+    };
+}
+/**
+ * @ignore
+ */
+export function updateEnqueueLinksPatternCache(item, pattern) {
+    enqueueLinksPatternCache.set(item, pattern);
+    if (enqueueLinksPatternCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {
+        const key = enqueueLinksPatternCache.keys().next().value;
+        enqueueLinksPatternCache.delete(key);
+    }
+}
+/**
+ * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
+ * to construct Glob objects from Glob pattern strings.
+ * @ignore
+ */
+export function constructGlobObjectsFromGlobs(globs) {
+    return globs
+        .filter((glob) => {
+        // Skip possibly nullish, empty strings
+        if (!glob) {
+            return false;
+        }
+        if (typeof glob === 'string') {
+            return glob.trim().length > 0;
+        }
+        if (glob.glob) {
+            return glob.glob.trim().length > 0;
+        }
+        return false;
+    })
+        .map((item) => {
+        // Get glob object from cache.
+        let globObject = enqueueLinksPatternCache.get(item);
+        if (globObject)
+            return globObject;
+        if (typeof item === 'string') {
+            globObject = { glob: validateGlobPattern(item) };
+        }
+        else {
+            globObject = { glob: validateGlobPattern(item.glob) };
+        }
+        updateEnqueueLinksPatternCache(item, globObject);
+        return globObject;
+    });
+}
+/**
+ * @internal
+ */
+export function validateGlobPattern(glob) {
+    const globTrimmed = glob.trim();
+    if (globTrimmed.length === 0)
+        throw new Error(`Cannot parse Glob pattern '${globTrimmed}': it must be an non-empty string`);
+    return globTrimmed;
+}
+/**
+ * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
+ * to check RegExps input and return valid RegExps.
+ * @ignore
+ */
+export function constructRegExpObjectsFromRegExps(regexps) {
+    return regexps.map((item) => {
+        // Get regexp object from cache.
+        let regexpObject = enqueueLinksPatternCache.get(item);
+        if (regexpObject)
+            return regexpObject;
+        if (item instanceof RegExp) {
+            regexpObject = { regexp: item };
+        }
+        else {
+            regexpObject = { regexp: item.regexp };
+        }
+        updateEnqueueLinksPatternCache(item, regexpObject);
+        return regexpObject;
+    });
+}
+/**
+ * Helper factory used in the `enqueueLinks()` function to construct UrlPatternObjects
+ * from a mixed array of glob strings, glob objects, RegExp instances, and regexp objects.
+ * @ignore
+ */
+export function constructUrlPatternObjects(patterns) {
+    const result = [];
+    for (const item of patterns) {
+        if (typeof item === 'string' || 'glob' in item) {
+            result.push(...constructGlobObjectsFromGlobs([item]));
+        }
+        else if (item instanceof RegExp || 'regexp' in item) {
+            result.push(...constructRegExpObjectsFromRegExps([item]));
+        }
+    }
+    return result;
+}
+/**
+ * Filters request options by URL patterns.
+ *
+ * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
+ * @ignore
+ */
+export function filterRequestOptionsByPatterns(requestOptions, includePatterns, excludePatterns = [], strategy, onSkippedRequestOptions) {
+    const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
+    const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
+    return requestOptions
+        .filter((opts) => {
+        const matchesExclude = excludeMatchers.some(({ match }) => match(opts.url));
+        if (matchesExclude) {
+            onSkippedRequestOptions?.(opts);
+        }
+        return !matchesExclude;
+    })
+        .map((opts) => {
+        if (!includeMatchers) {
+            return { ...opts, enqueueStrategy: strategy };
+        }
+        for (const { match } of includeMatchers) {
+            if (match(opts.url)) {
+                return { ...opts, enqueueStrategy: strategy };
+            }
+        }
+        // didn't match any positive pattern
+        onSkippedRequestOptions?.(opts);
+        return null;
+    })
+        .filter((opts) => opts !== null);
+}
+function isAbsoluteUrl(url) {
+    try {
+        // eslint-disable-next-line no-new
+        new URL(url);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * @ignore
+ */
+export function createRequestOptions(sources, options = {}) {
+    return sources
+        .map((src) => typeof src === 'string'
+        ? { url: src, enqueueStrategy: options.strategy }
+        : { ...src, enqueueStrategy: options.strategy })
+        .filter(({ url }) => {
+        try {
+            return new URL(url, options.baseUrl).href;
+        }
+        catch (err) {
+            return false;
+        }
+    })
+        .map((requestOptions) => {
+        // Leave already-absolute URLs untouched - re-deriving them via `new URL()` would normalize them
+        // (e.g. adding a trailing slash to a bare domain), which is surprising for URLs that didn't need
+        // resolving against `baseUrl` in the first place.
+        if (!isAbsoluteUrl(requestOptions.url)) {
+            requestOptions.url = new URL(requestOptions.url, options.baseUrl).href;
+        }
+        requestOptions.userData ??= options.userData ?? {};
+        if (typeof options.label === 'string') {
+            requestOptions.userData = {
+                ...requestOptions.userData,
+                label: options.label,
+            };
+        }
+        if (options.skipNavigation) {
+            requestOptions.skipNavigation = true;
+        }
+        if (options.sessionId) {
+            requestOptions.sessionId = options.sessionId;
+        }
+        return requestOptions;
+    });
+}
+/**
+ * @ignore
+ */
+function createPatternObjectMatcher(urlPatternObject) {
+    const { regexp, glob } = urlPatternObject;
+    let match;
+    if (regexp) {
+        match = (url) => regexp.test(url);
+    }
+    else if (glob) {
+        const m = new Minimatch(glob, { nocase: true });
+        match = (url) => m.match(url);
+    }
+    else {
+        match = () => false;
+    }
+    return { match };
+}
+/**
+ * Applies a {@link RequestTransform} function to a list of request options.
+ * Options for which the transform returns a falsy value are removed from the list.
+ * @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
+ * @ignore
+ * @internal
+ */
+export function applyRequestTransform(requestOptions, transformFn, onSkipped) {
+    return requestOptions
+        .map((opts) => {
+        const transformed = transformFn(opts);
+        if (transformed === 'skip') {
+            onSkipped?.(opts);
+            return null;
+        }
+        if (transformed === 'unchanged') {
+            return opts;
+        }
+        if (!transformed) {
+            onSkipped?.(opts);
+            return null;
+        }
+        return transformed;
+    })
+        .filter((r) => r !== null);
+}

package/internals/errors.d.ts

@@ -0,0 +1,68 @@
+import { CriticalError, NonRetryableError } from '@crawlee/core';
+import type { SkippedRequestReason } from './crawling_request.js';
+/**
+ * @ignore
+ */
+export declare class MissingRouteError extends CriticalError {
+}
+/**
+ * Errors of `RetryRequestError` type will always be retried by the crawler.
+ *
+ * *This error overrides the `maxRequestRetries` option, i.e. the request can be retried indefinitely until it succeeds.*
+ */
+export declare class RetryRequestError extends Error {
+    constructor(message?: string);
+}
+/**
+ * Thrown when a domain has rate-limited us and the request should simply be attempted again later.
+ *
+ * The request is reclaimed without recording a failure: it costs neither a retry nor session reputation, because
+ * nothing about the request or the session was at fault. A {@link ThrottlingRequestManager} holds it back until
+ * the domain's backoff expires, so retries are paced rather than immediate.
+ */
+export declare class RequestThrottledError extends RetryRequestError {
+    constructor(message?: string);
+}
+/**
+ * Thrown when a domain has rate-limited us for so long that no request has got through, and the crawl is
+ * abandoned rather than kept waiting.
+ *
+ * Waiting longer will not help: at this point the concurrency is too high for the domain, or it has blocked us.
+ * The affected requests are deliberately left in their queue, so re-running the crawl without purging storages
+ * resumes them once the domain recovers.
+ */
+export declare class PersistentRateLimitError extends CriticalError {
+}
+/**
+ * Thrown when a requested session is not found in the referenced SessionPool.
+ */
+export declare class MissingSessionError extends Error {
+    constructor(sessionId?: string);
+}
+/**
+ * Settles the request as skipped rather than failed: no retry, no failure, and its storage writes are rolled back.
+ * Use {@link RestrictedCrawlingContext.skipRequest|`context.skipRequest()`} rather than throwing this directly.
+ * @internal
+ */
+export declare class ContextPipelineInterruptedError extends Error {
+    readonly reason: SkippedRequestReason;
+    readonly skipMessage?: string | undefined;
+    constructor(reason: SkippedRequestReason, skipMessage?: string | undefined);
+}
+export declare class ContextPipelineCleanupError extends CriticalError {
+    constructor(error: unknown, options?: ErrorOptions);
+}
+/** Wraps a failure of a context pipeline middleware, telling it apart from a {@link RequestHandlerError}. */
+export declare class ContextPipelineInitializationError extends Error {
+    constructor(error: unknown, options?: ErrorOptions);
+}
+/** Wraps a failure of the context pipeline's final consumer (the request handler), telling it apart from a middleware failure. */
+export declare class RequestHandlerError extends Error {
+    constructor(error: unknown, options?: ErrorOptions);
+}
+/**
+ * Thrown by crawlers when `skipNavigation` is used on a request.
+ * Subclasses can catch this error to skip their own navigation-dependent logic.
+ */
+export declare class NavigationSkippedError extends NonRetryableError {
+}

package/internals/errors.js

@@ -0,0 +1,83 @@
+import { CriticalError, NonRetryableError } from '@crawlee/core';
+/**
+ * @ignore
+ */
+export class MissingRouteError extends CriticalError {
+}
+/**
+ * Errors of `RetryRequestError` type will always be retried by the crawler.
+ *
+ * *This error overrides the `maxRequestRetries` option, i.e. the request can be retried indefinitely until it succeeds.*
+ */
+export class RetryRequestError extends Error {
+    constructor(message) {
+        super(message ?? "Request is being retried at the user's request");
+    }
+}
+/**
+ * Thrown when a domain has rate-limited us and the request should simply be attempted again later.
+ *
+ * The request is reclaimed without recording a failure: it costs neither a retry nor session reputation, because
+ * nothing about the request or the session was at fault. A {@link ThrottlingRequestManager} holds it back until
+ * the domain's backoff expires, so retries are paced rather than immediate.
+ */
+export class RequestThrottledError extends RetryRequestError {
+    constructor(message) {
+        super(message ?? 'Request is being retried later because its domain is rate-limiting us');
+    }
+}
+/**
+ * Thrown when a domain has rate-limited us for so long that no request has got through, and the crawl is
+ * abandoned rather than kept waiting.
+ *
+ * Waiting longer will not help: at this point the concurrency is too high for the domain, or it has blocked us.
+ * The affected requests are deliberately left in their queue, so re-running the crawl without purging storages
+ * resumes them once the domain recovers.
+ */
+export class PersistentRateLimitError extends CriticalError {
+}
+/**
+ * Thrown when a requested session is not found in the referenced SessionPool.
+ */
+export class MissingSessionError extends Error {
+    constructor(sessionId) {
+        super(`The current SessionPool instance couldn't find a valid session${sessionId ? ` for the following id: ${sessionId}.` : '.'}`);
+    }
+}
+/**
+ * Settles the request as skipped rather than failed: no retry, no failure, and its storage writes are rolled back.
+ * Use {@link RestrictedCrawlingContext.skipRequest|`context.skipRequest()`} rather than throwing this directly.
+ * @internal
+ */
+export class ContextPipelineInterruptedError extends Error {
+    reason;
+    skipMessage;
+    constructor(reason, skipMessage) {
+        super(skipMessage ?? 'Request was skipped');
+        this.reason = reason;
+        this.skipMessage = skipMessage;
+    }
+}
+export class ContextPipelineCleanupError extends CriticalError {
+    constructor(error, options) {
+        super(undefined, { cause: error, ...options });
+    }
+}
+/** Wraps a failure of a context pipeline middleware, telling it apart from a {@link RequestHandlerError}. */
+export class ContextPipelineInitializationError extends Error {
+    constructor(error, options) {
+        super(undefined, { cause: error, ...options });
+    }
+}
+/** Wraps a failure of the context pipeline's final consumer (the request handler), telling it apart from a middleware failure. */
+export class RequestHandlerError extends Error {
+    constructor(error, options) {
+        super(undefined, { cause: error, ...options });
+    }
+}
+/**
+ * Thrown by crawlers when `skipNavigation` is used on a request.
+ * Subclasses can catch this error to skip their own navigation-dependent logic.
+ */
+export class NavigationSkippedError extends NonRetryableError {
+}

package/internals/http.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Parses a `Retry-After` response header into a delay in milliseconds.
+ *
+ * The header holds either a non-negative number of seconds or an HTTP-date.
+ * See [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After).
+ *
+ * @returns The delay in milliseconds, or `null` if the header is absent, unparseable, or already elapsed.
+ */
+export declare function parseRetryAfterHeader(value?: string | null): number | null;

package/internals/http.js

@@ -0,0 +1,28 @@
+/**
+ * Parses a `Retry-After` response header into a delay in milliseconds.
+ *
+ * The header holds either a non-negative number of seconds or an HTTP-date.
+ * See [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After).
+ *
+ * @returns The delay in milliseconds, or `null` if the header is absent, unparseable, or already elapsed.
+ */
+export function parseRetryAfterHeader(value) {
+    if (!value) {
+        return null;
+    }
+    const trimmed = value.trim();
+    // Per the spec this is a `delay-seconds`: digits only, so a negative or fractional value is not one.
+    if (/^\d+$/.test(trimmed)) {
+        // `Retry-After: 0` names no future deadline, same as an HTTP-date that has already passed. Reporting it
+        // as a zero delay would leave the domain unthrottled while still counting as a rate-limit event, so the
+        // caller would defer the request for free and re-send it immediately.
+        const delayMs = Number(trimmed) * 1000;
+        return delayMs > 0 ? delayMs : null;
+    }
+    const date = Date.parse(trimmed);
+    if (!Number.isNaN(date)) {
+        const delayMs = date - Date.now();
+        return delayMs > 0 ? delayMs : null;
+    }
+    return null;
+}

package/internals/request-timeout.d.ts

@@ -16,10 +16,16 @@ export declare const extendTimeoutKey: unique symbol;
  * bail, so a request whose timeout already elapsed does not carry on after the crawler moved past it.
  */
 export declare const timeoutExpiredKey: unique symbol;
+/**
+ * Stops the internal request timeout's clock until the returned callback is called. Error handling has a timeout
+ * of its own, and must not be cut short (and handled again) by the request timeout.
+ */
+export declare const suspendTimeoutKey: unique symbol;
 /** The slots the internal request timeout hangs on the in-flight crawling context. */
 export interface RequestTimeoutContext {
     [extendTimeoutKey]?: (extraMillis: number) => void;
     [timeoutExpiredKey]?: () => boolean;
+    [suspendTimeoutKey]?: () => () => void;
     [navigationDeadlineKey]?: number;
 }
 /**

package/internals/request-timeout.js

@@ -17,6 +17,11 @@ export const extendTimeoutKey = Symbol('extendTimeout');
  * bail, so a request whose timeout already elapsed does not carry on after the crawler moved past it.
  */
 export const timeoutExpiredKey = Symbol('timeoutExpired');
+/**
+ * Stops the internal request timeout's clock until the returned callback is called. Error handling has a timeout
+ * of its own, and must not be cut short (and handled again) by the request timeout.
+ */
+export const suspendTimeoutKey = Symbol('suspendTimeout');
 /**
  * Milliseconds left in the shared navigation window for `ctx`, lazily starting the window on first use.
  * @internal
@@ -46,21 +51,41 @@ export async function raceWithTimeout(context, work, { timeoutMillis, requestId
     let deadline = Date.now() + timeoutMillis;
     let settled = false;
     let firedByTimeout = false;
+    let suspendedAt;
     const timeout = new Promise((_, reject) => {
         const fire = () => {
             settled = true;
             firedByTimeout = true;
             reject(new TimeoutError(`Request timed out after ${timeoutMillis / 1e3} seconds (${requestId}).`));
         };
+        const schedule = () => {
+            clearTimeout(timer);
+            timer = setTimeout(fire, Math.max(deadline - Date.now(), 0));
+        };
         timer = setTimeout(fire, timeoutMillis);
         context[timeoutExpiredKey] = () => firedByTimeout;
         context[extendTimeoutKey] = (extraMillis) => {
             if (settled) {
                 return;
             }
-            clearTimeout(timer);
             deadline += extraMillis;
-            timer = setTimeout(fire, Math.max(deadline - Date.now(), 0));
+            if (suspendedAt === undefined) {
+                schedule();
+            }
+        };
+        context[suspendTimeoutKey] = () => {
+            if (settled || suspendedAt !== undefined) {
+                return () => { };
+            }
+            clearTimeout(timer);
+            suspendedAt = Date.now();
+            return () => {
+                deadline += Date.now() - suspendedAt;
+                suspendedAt = undefined;
+                if (!settled) {
+                    schedule();
+                }
+            };
         };
     });
     try {