@crawlee/basic 4.0.0-beta.170 → 4.0.0-beta.171

package/internals/enqueue_links/enqueue_links.js

@@ -0,0 +1,76 @@
+import { EnqueueStrategy } from '@crawlee/utils';
+import { getDomain } from 'tldts';
+/**
+ * @internal
+ * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
+ * - If a user provides a base url, we always return it
+ * - If a user specifies {@link EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original
+ *   request domain, or a redirected one
+ * - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
+ */
+export function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }) {
+    // User provided base url takes priority
+    if (userProvidedBaseUrl) {
+        return userProvidedBaseUrl;
+    }
+    const originalUrlOrigin = new URL(originalRequestUrl).origin;
+    const finalUrlOrigin = new URL(finalRequestUrl ?? originalRequestUrl).origin;
+    // We can assume users want to go off the domain in this case
+    if (enqueueStrategy === EnqueueStrategy.All) {
+        return finalUrlOrigin;
+    }
+    // If the user wants to ensure the same domain is accessed, regardless of subdomains, we check to ensure the domains match
+    // If they don't (we went off domain via a redirect), we keep filtering against the original domain - returning undefined would disable the filtering entirely
+    if (enqueueStrategy === EnqueueStrategy.SameDomain) {
+        const originalHostname = getDomain(originalUrlOrigin, { mixedInputs: false });
+        const finalHostname = getDomain(finalUrlOrigin, { mixedInputs: false });
+        if (originalHostname === finalHostname) {
+            return finalUrlOrigin;
+        }
+        return originalUrlOrigin;
+    }
+    // Always enqueue urls that are from the same origin in all other cases, as the filtering happens on the original request url, even if there was a redirect
+    // before actually finding the urls
+    return originalUrlOrigin;
+}
+/**
+ * @internal
+ * Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
+ */
+export function buildEnqueueStrategyPatterns(baseUrl, strategy) {
+    const url = new URL(baseUrl);
+    switch (strategy) {
+        case EnqueueStrategy.SameHostname:
+            // We need to get the origin of the passed in domain in the event someone sets baseUrl
+            // to an url like https://example.com/deep/default/path and one of the found urls is an
+            // absolute relative path (/path/to/page)
+            return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
+        case EnqueueStrategy.SameDomain: {
+            // Get the actual hostname from the base url
+            const baseUrlHostname = getDomain(url.hostname, { mixedInputs: false });
+            if (baseUrlHostname) {
+                // We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
+                url.hostname = baseUrlHostname;
+                return [
+                    { glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) },
+                    { glob: ignoreHttpSchema(`${url.origin}/**`) },
+                ];
+            }
+            // We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
+            // as SameDomainAndSubdomain
+            return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
+        }
+        case EnqueueStrategy.SameOrigin:
+            // The same behavior as SameHostname, but respecting the protocol of the URL
+            return [{ glob: `${url.origin}/**` }];
+        case EnqueueStrategy.All:
+        default:
+            return [{ glob: `http{s,}://**` }];
+    }
+}
+/**
+ * Internal function that changes the enqueue glob patterns to match both http and https
+ */
+function ignoreHttpSchema(pattern) {
+    return pattern.replace(/^(https?):\/\//, 'http{s,}://');
+}

package/internals/enqueue_links/index.d.ts

@@ -0,0 +1,2 @@
+export * from './enqueue_links.js';
+export * from './shared.js';

package/internals/enqueue_links/index.js

@@ -0,0 +1,2 @@
+export * from './enqueue_links.js';
+export * from './shared.js';

package/internals/enqueue_links/shared.d.ts

@@ -0,0 +1,98 @@
+import type { Awaitable, Dictionary } from '@crawlee/types';
+import { z } from 'zod';
+import type { EnqueueStrategyOption, RequestOptions, SkippedRequestReason, Source } from '@crawlee/core';
+import { Request } from '@crawlee/core';
+export interface UrlPatternObject {
+    glob?: string;
+    regexp?: RegExp;
+}
+export interface GlobObject {
+    glob: string;
+}
+export type GlobInput = string | GlobObject;
+export interface RegExpObject {
+    regexp: RegExp;
+}
+export type RegExpInput = RegExp | RegExpObject;
+/** Unified URL pattern input — accepts glob strings, glob objects, RegExp instances, or regexp objects. */
+export type UrlPatternInput = GlobInput | RegExpInput;
+/**
+ * Accepts one {@link UrlPatternInput} — a glob string, a RegExp instance, or a `{ glob }` / `{ regexp }` object.
+ * @internal
+ */
+export declare const urlPatternSchema: z.ZodType<UrlPatternInput>;
+export type SkippedRequestCallback = (args: {
+    request: Request;
+    reason: SkippedRequestReason;
+}) => Awaitable<void>;
+/**
+ * Builds the `{ request, reason }` argument passed to a {@link SkippedRequestCallback}, constructing the
+ * `Request` lazily on first access to `request` (and caching it) since most skips are never observed by a
+ * real callback and building a `Request` isn't free.
+ * @ignore
+ */
+export declare function createSkippedRequestArgs(source: string | Source, reason: SkippedRequestReason): Parameters<SkippedRequestCallback>[0];
+/**
+ * @ignore
+ */
+export declare function updateEnqueueLinksPatternCache(item: GlobInput | RegExpInput, pattern: RegExpObject | GlobObject): void;
+/**
+ * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
+ * to construct Glob objects from Glob pattern strings.
+ * @ignore
+ */
+export declare function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): GlobObject[];
+/**
+ * @internal
+ */
+export declare function validateGlobPattern(glob: string): string;
+/**
+ * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
+ * to check RegExps input and return valid RegExps.
+ * @ignore
+ */
+export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): RegExpObject[];
+/**
+ * Helper factory used in the `enqueueLinks()` function to construct UrlPatternObjects
+ * from a mixed array of glob strings, glob objects, RegExp instances, and regexp objects.
+ * @ignore
+ */
+export declare function constructUrlPatternObjects(patterns: readonly UrlPatternInput[]): UrlPatternObject[];
+/**
+ * Filters request options by URL patterns.
+ *
+ * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
+ * @ignore
+ */
+export declare function filterRequestOptionsByPatterns(requestOptions: RequestOptions[], includePatterns: UrlPatternObject[] | undefined, excludePatterns?: UrlPatternObject[], strategy?: EnqueueStrategyOption, onSkippedRequestOptions?: (options: RequestOptions) => void): RequestOptions[];
+/**
+ * @ignore
+ */
+export declare function createRequestOptions(sources: readonly (string | Record<string, unknown>)[], options?: {
+    label?: string;
+    userData?: Dictionary;
+    baseUrl?: string;
+    skipNavigation?: boolean;
+    sessionId?: string;
+    strategy?: EnqueueStrategyOption;
+}): RequestOptions[];
+/**
+ * Takes a {@link RequestOptions} object and changes its attributes in a desired way. This user-function is used
+ * by {@link enqueueLinks} to modify request options before they are converted to {@link Request} instances.
+ */
+export interface RequestTransform {
+    /**
+     * @param original Request options to be modified.
+     * @returns The modified request options to enqueue, `'unchanged'` to keep the original options as-is,
+     *   or a falsy value / `'skip'` to exclude the request from the queue.
+     */
+    (original: RequestOptions): RequestOptions | false | undefined | null | 'skip' | 'unchanged';
+}
+/**
+ * Applies a {@link RequestTransform} function to a list of request options.
+ * Options for which the transform returns a falsy value are removed from the list.
+ * @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
+ * @ignore
+ * @internal
+ */
+export declare function applyRequestTransform(requestOptions: RequestOptions[], transformFn: RequestTransform, onSkipped?: (requestOptions: RequestOptions) => void): RequestOptions[];

package/internals/enqueue_links/shared.js

@@ -0,0 +1,259 @@
+import { URL } from 'node:url';
+import { Minimatch } from 'minimatch';
+import { z } from 'zod';
+import { Request } from '@crawlee/core';
+import { schemas } from '@crawlee/utils/internal';
+const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
+/**
+ * To keep high performance when the same patterns are passed on every `enqueueLinks()` call,
+ * each glob/regexp is only compiled once and then kept in a cache.
+ * @ignore
+ */
+const enqueueLinksPatternCache = new Map();
+/**
+ * Accepts one {@link UrlPatternInput} — a glob string, a RegExp instance, or a `{ glob }` / `{ regexp }` object.
+ * @internal
+ */
+export const urlPatternSchema = z.union([
+    z.string(),
+    z.instanceof(RegExp),
+    schemas.objectWithKeys(['glob']),
+    schemas.objectWithKeys(['regexp']),
+]);
+/**
+ * Builds the `{ request, reason }` argument passed to a {@link SkippedRequestCallback}, constructing the
+ * `Request` lazily on first access to `request` (and caching it) since most skips are never observed by a
+ * real callback and building a `Request` isn't free.
+ * @ignore
+ */
+export function createSkippedRequestArgs(source, reason) {
+    const sourceReason = typeof source === 'string' ? undefined : source.skippedReason;
+    let request;
+    return {
+        reason: sourceReason ?? reason,
+        get request() {
+            request ??=
+                source instanceof Request
+                    ? source
+                    : new Request(typeof source === 'string' ? { url: source } : source);
+            return request;
+        },
+    };
+}
+/**
+ * @ignore
+ */
+export function updateEnqueueLinksPatternCache(item, pattern) {
+    enqueueLinksPatternCache.set(item, pattern);
+    if (enqueueLinksPatternCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {
+        const key = enqueueLinksPatternCache.keys().next().value;
+        enqueueLinksPatternCache.delete(key);
+    }
+}
+/**
+ * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
+ * to construct Glob objects from Glob pattern strings.
+ * @ignore
+ */
+export function constructGlobObjectsFromGlobs(globs) {
+    return globs
+        .filter((glob) => {
+        // Skip possibly nullish, empty strings
+        if (!glob) {
+            return false;
+        }
+        if (typeof glob === 'string') {
+            return glob.trim().length > 0;
+        }
+        if (glob.glob) {
+            return glob.glob.trim().length > 0;
+        }
+        return false;
+    })
+        .map((item) => {
+        // Get glob object from cache.
+        let globObject = enqueueLinksPatternCache.get(item);
+        if (globObject)
+            return globObject;
+        if (typeof item === 'string') {
+            globObject = { glob: validateGlobPattern(item) };
+        }
+        else {
+            globObject = { glob: validateGlobPattern(item.glob) };
+        }
+        updateEnqueueLinksPatternCache(item, globObject);
+        return globObject;
+    });
+}
+/**
+ * @internal
+ */
+export function validateGlobPattern(glob) {
+    const globTrimmed = glob.trim();
+    if (globTrimmed.length === 0)
+        throw new Error(`Cannot parse Glob pattern '${globTrimmed}': it must be an non-empty string`);
+    return globTrimmed;
+}
+/**
+ * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
+ * to check RegExps input and return valid RegExps.
+ * @ignore
+ */
+export function constructRegExpObjectsFromRegExps(regexps) {
+    return regexps.map((item) => {
+        // Get regexp object from cache.
+        let regexpObject = enqueueLinksPatternCache.get(item);
+        if (regexpObject)
+            return regexpObject;
+        if (item instanceof RegExp) {
+            regexpObject = { regexp: item };
+        }
+        else {
+            regexpObject = { regexp: item.regexp };
+        }
+        updateEnqueueLinksPatternCache(item, regexpObject);
+        return regexpObject;
+    });
+}
+/**
+ * Helper factory used in the `enqueueLinks()` function to construct UrlPatternObjects
+ * from a mixed array of glob strings, glob objects, RegExp instances, and regexp objects.
+ * @ignore
+ */
+export function constructUrlPatternObjects(patterns) {
+    const result = [];
+    for (const item of patterns) {
+        if (typeof item === 'string' || 'glob' in item) {
+            result.push(...constructGlobObjectsFromGlobs([item]));
+        }
+        else if (item instanceof RegExp || 'regexp' in item) {
+            result.push(...constructRegExpObjectsFromRegExps([item]));
+        }
+    }
+    return result;
+}
+/**
+ * Filters request options by URL patterns.
+ *
+ * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
+ * @ignore
+ */
+export function filterRequestOptionsByPatterns(requestOptions, includePatterns, excludePatterns = [], strategy, onSkippedRequestOptions) {
+    const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
+    const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
+    return requestOptions
+        .filter((opts) => {
+        const matchesExclude = excludeMatchers.some(({ match }) => match(opts.url));
+        if (matchesExclude) {
+            onSkippedRequestOptions?.(opts);
+        }
+        return !matchesExclude;
+    })
+        .map((opts) => {
+        if (!includeMatchers) {
+            return { ...opts, enqueueStrategy: strategy };
+        }
+        for (const { match } of includeMatchers) {
+            if (match(opts.url)) {
+                return { ...opts, enqueueStrategy: strategy };
+            }
+        }
+        // didn't match any positive pattern
+        onSkippedRequestOptions?.(opts);
+        return null;
+    })
+        .filter((opts) => opts !== null);
+}
+function isAbsoluteUrl(url) {
+    try {
+        // eslint-disable-next-line no-new
+        new URL(url);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * @ignore
+ */
+export function createRequestOptions(sources, options = {}) {
+    return sources
+        .map((src) => typeof src === 'string'
+        ? { url: src, enqueueStrategy: options.strategy }
+        : { ...src, enqueueStrategy: options.strategy })
+        .filter(({ url }) => {
+        try {
+            return new URL(url, options.baseUrl).href;
+        }
+        catch (err) {
+            return false;
+        }
+    })
+        .map((requestOptions) => {
+        // Leave already-absolute URLs untouched - re-deriving them via `new URL()` would normalize them
+        // (e.g. adding a trailing slash to a bare domain), which is surprising for URLs that didn't need
+        // resolving against `baseUrl` in the first place.
+        if (!isAbsoluteUrl(requestOptions.url)) {
+            requestOptions.url = new URL(requestOptions.url, options.baseUrl).href;
+        }
+        requestOptions.userData ??= options.userData ?? {};
+        if (typeof options.label === 'string') {
+            requestOptions.userData = {
+                ...requestOptions.userData,
+                label: options.label,
+            };
+        }
+        if (options.skipNavigation) {
+            requestOptions.skipNavigation = true;
+        }
+        if (options.sessionId) {
+            requestOptions.sessionId = options.sessionId;
+        }
+        return requestOptions;
+    });
+}
+/**
+ * @ignore
+ */
+function createPatternObjectMatcher(urlPatternObject) {
+    const { regexp, glob } = urlPatternObject;
+    let match;
+    if (regexp) {
+        match = (url) => regexp.test(url);
+    }
+    else if (glob) {
+        const m = new Minimatch(glob, { nocase: true });
+        match = (url) => m.match(url);
+    }
+    else {
+        match = () => false;
+    }
+    return { match };
+}
+/**
+ * Applies a {@link RequestTransform} function to a list of request options.
+ * Options for which the transform returns a falsy value are removed from the list.
+ * @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
+ * @ignore
+ * @internal
+ */
+export function applyRequestTransform(requestOptions, transformFn, onSkipped) {
+    return requestOptions
+        .map((opts) => {
+        const transformed = transformFn(opts);
+        if (transformed === 'skip') {
+            onSkipped?.(opts);
+            return null;
+        }
+        if (transformed === 'unchanged') {
+            return opts;
+        }
+        if (!transformed) {
+            onSkipped?.(opts);
+            return null;
+        }
+        return transformed;
+    })
+        .filter((r) => r !== null);
+}

package/internals/errors.d.ts

@@ -0,0 +1,58 @@
+import { CriticalError, NonRetryableError } from '@crawlee/core';
+/**
+ * @ignore
+ */
+export declare class MissingRouteError extends CriticalError {
+}
+/**
+ * Errors of `RetryRequestError` type will always be retried by the crawler.
+ *
+ * *This error overrides the `maxRequestRetries` option, i.e. the request can be retried indefinitely until it succeeds.*
+ */
+export declare class RetryRequestError extends Error {
+    constructor(message?: string);
+}
+/**
+ * Thrown when a domain has rate-limited us and the request should simply be attempted again later.
+ *
+ * The request is reclaimed without recording a failure: it costs neither a retry nor session reputation, because
+ * nothing about the request or the session was at fault. A {@link ThrottlingRequestManager} holds it back until
+ * the domain's backoff expires, so retries are paced rather than immediate.
+ */
+export declare class RequestThrottledError extends RetryRequestError {
+    constructor(message?: string);
+}
+/**
+ * Thrown when a domain has rate-limited us for so long that no request has got through, and the crawl is
+ * abandoned rather than kept waiting.
+ *
+ * Waiting longer will not help: at this point the concurrency is too high for the domain, or it has blocked us.
+ * The affected requests are deliberately left in their queue, so re-running the crawl without purging storages
+ * resumes them once the domain recovers.
+ */
+export declare class PersistentRateLimitError extends CriticalError {
+}
+/**
+ * Thrown when a requested session is not found in the referenced SessionPool.
+ */
+export declare class MissingSessionError extends Error {
+    constructor(sessionId?: string);
+}
+export declare class ContextPipelineInterruptedError extends Error {
+    constructor(message?: string);
+}
+export declare class ContextPipelineInitializationError extends Error {
+    constructor(error: unknown, options?: ErrorOptions);
+}
+export declare class ContextPipelineCleanupError extends CriticalError {
+    constructor(error: unknown, options?: ErrorOptions);
+}
+export declare class RequestHandlerError extends Error {
+    constructor(error: unknown, options?: ErrorOptions);
+}
+/**
+ * Thrown by crawlers when `skipNavigation` is used on a request.
+ * Subclasses can catch this error to skip their own navigation-dependent logic.
+ */
+export declare class NavigationSkippedError extends NonRetryableError {
+}

package/internals/errors.js

@@ -0,0 +1,72 @@
+import { CriticalError, NonRetryableError } from '@crawlee/core';
+/**
+ * @ignore
+ */
+export class MissingRouteError extends CriticalError {
+}
+/**
+ * Errors of `RetryRequestError` type will always be retried by the crawler.
+ *
+ * *This error overrides the `maxRequestRetries` option, i.e. the request can be retried indefinitely until it succeeds.*
+ */
+export class RetryRequestError extends Error {
+    constructor(message) {
+        super(message ?? "Request is being retried at the user's request");
+    }
+}
+/**
+ * Thrown when a domain has rate-limited us and the request should simply be attempted again later.
+ *
+ * The request is reclaimed without recording a failure: it costs neither a retry nor session reputation, because
+ * nothing about the request or the session was at fault. A {@link ThrottlingRequestManager} holds it back until
+ * the domain's backoff expires, so retries are paced rather than immediate.
+ */
+export class RequestThrottledError extends RetryRequestError {
+    constructor(message) {
+        super(message ?? 'Request is being retried later because its domain is rate-limiting us');
+    }
+}
+/**
+ * Thrown when a domain has rate-limited us for so long that no request has got through, and the crawl is
+ * abandoned rather than kept waiting.
+ *
+ * Waiting longer will not help: at this point the concurrency is too high for the domain, or it has blocked us.
+ * The affected requests are deliberately left in their queue, so re-running the crawl without purging storages
+ * resumes them once the domain recovers.
+ */
+export class PersistentRateLimitError extends CriticalError {
+}
+/**
+ * Thrown when a requested session is not found in the referenced SessionPool.
+ */
+export class MissingSessionError extends Error {
+    constructor(sessionId) {
+        super(`The current SessionPool instance couldn't find a valid session${sessionId ? ` for the following id: ${sessionId}.` : '.'}`);
+    }
+}
+export class ContextPipelineInterruptedError extends Error {
+    constructor(message) {
+        super(`Request handling was interrupted during context initialization ${message ? ` - ${message}` : ''}`);
+    }
+}
+export class ContextPipelineInitializationError extends Error {
+    constructor(error, options) {
+        super(undefined, { cause: error, ...options });
+    }
+}
+export class ContextPipelineCleanupError extends CriticalError {
+    constructor(error, options) {
+        super(undefined, { cause: error, ...options });
+    }
+}
+export class RequestHandlerError extends Error {
+    constructor(error, options) {
+        super(undefined, { cause: error, ...options });
+    }
+}
+/**
+ * Thrown by crawlers when `skipNavigation` is used on a request.
+ * Subclasses can catch this error to skip their own navigation-dependent logic.
+ */
+export class NavigationSkippedError extends NonRetryableError {
+}

package/internals/http.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Parses a `Retry-After` response header into a delay in milliseconds.
+ *
+ * The header holds either a non-negative number of seconds or an HTTP-date.
+ * See [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After).
+ *
+ * @returns The delay in milliseconds, or `null` if the header is absent, unparseable, or already elapsed.
+ */
+export declare function parseRetryAfterHeader(value?: string | null): number | null;

package/internals/http.js

@@ -0,0 +1,28 @@
+/**
+ * Parses a `Retry-After` response header into a delay in milliseconds.
+ *
+ * The header holds either a non-negative number of seconds or an HTTP-date.
+ * See [MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Retry-After).
+ *
+ * @returns The delay in milliseconds, or `null` if the header is absent, unparseable, or already elapsed.
+ */
+export function parseRetryAfterHeader(value) {
+    if (!value) {
+        return null;
+    }
+    const trimmed = value.trim();
+    // Per the spec this is a `delay-seconds`: digits only, so a negative or fractional value is not one.
+    if (/^\d+$/.test(trimmed)) {
+        // `Retry-After: 0` names no future deadline, same as an HTTP-date that has already passed. Reporting it
+        // as a zero delay would leave the domain unthrottled while still counting as a rate-limit event, so the
+        // caller would defer the request for free and re-send it immediately.
+        const delayMs = Number(trimmed) * 1000;
+        return delayMs > 0 ? delayMs : null;
+    }
+    const date = Date.parse(trimmed);
+    if (!Number.isNaN(date)) {
+        const delayMs = date - Date.now();
+        return delayMs > 0 ? delayMs : null;
+    }
+    return null;
+}