@crawlee/basic 4.0.0-beta.99 → 4.0.0-rc.1

package/internals/crawling_request.js

@@ -0,0 +1,127 @@
+import util from 'node:util';
+import { Request } from '@crawlee/core';
+export var RequestState;
+(function (RequestState) {
+    RequestState[RequestState["UNPROCESSED"] = 0] = "UNPROCESSED";
+    RequestState[RequestState["BEFORE_NAV"] = 1] = "BEFORE_NAV";
+    RequestState[RequestState["AFTER_NAV"] = 2] = "AFTER_NAV";
+    RequestState[RequestState["REQUEST_HANDLER"] = 3] = "REQUEST_HANDLER";
+    RequestState[RequestState["DONE"] = 4] = "DONE";
+    RequestState[RequestState["ERROR_HANDLER"] = 5] = "ERROR_HANDLER";
+    RequestState[RequestState["ERROR"] = 6] = "ERROR";
+    RequestState[RequestState["SKIPPED"] = 7] = "SKIPPED";
+})(RequestState || (RequestState = {}));
+/**
+ * A {@link Request} as seen by a crawler: the same stored record, with the crawler's own per-request state
+ * (`userData.__crawlee`) exposed as properties. This is what `request` is in every request handler.
+ *
+ * Requests fetched from a request manager are plain {@link Request}s; {@link BasicCrawler} rebuilds them
+ * as `CrawlingRequest`s before handing them to the pipeline.
+ * @category Sources
+ */
+export class CrawlingRequest extends Request {
+    static fromSchema(schema) {
+        return super.fromSchema(schema);
+    }
+    get #crawleeData() {
+        return (this.userData.__crawlee ??= {});
+    }
+    /**
+     * Tells the crawler processing this request to skip the navigation and process the request directly.
+     *
+     * When this is set to `true`, the crawling context will not contain the results of the navigation
+     * (e.g. `response`, `body`, `contentType`, `$` or `request.loadedUrl`).
+     * Accessing these properties will throw a {@link NavigationSkippedError} at runtime.
+     */
+    get skipNavigation() {
+        return this.#crawleeData.skipNavigation ?? false;
+    }
+    set skipNavigation(value) {
+        this.#crawleeData.skipNavigation = value;
+    }
+    /**
+     * Depth of the request in the current crawl tree.
+     * Note that this is dependent on the crawler setup and might produce unexpected results when used with multiple crawlers.
+     */
+    get crawlDepth() {
+        return this.#crawleeData.crawlDepth ?? 0;
+    }
+    set crawlDepth(value) {
+        this.#crawleeData.crawlDepth = value;
+    }
+    /** ID of a session to use for this request. When set, the crawler will fetch this session from the session pool instead of creating a new one. */
+    get sessionId() {
+        return this.#crawleeData.sessionId;
+    }
+    set sessionId(value) {
+        this.#crawleeData.sessionId = value;
+    }
+    /** Maximum number of retries for this request. Allows to override the global `maxRequestRetries` option of `BasicCrawler`. */
+    get maxRetries() {
+        return this.#crawleeData.maxRetries;
+    }
+    set maxRetries(value) {
+        this.#crawleeData.maxRetries = value;
+    }
+    /** Describes the request's current lifecycle state. */
+    get state() {
+        return this.#crawleeData.state ?? RequestState.UNPROCESSED;
+    }
+    set state(value) {
+        this.#crawleeData.state = value;
+    }
+    /** The strategy the request was enqueued under, if it came from `enqueueLinks`. @internal */
+    get enqueueStrategy() {
+        return this.#crawleeData.enqueueStrategy;
+    }
+    /**
+     * Stores information about an error that occurred during processing of this request.
+     *
+     * You should always use Error instances when throwing errors in JavaScript.
+     *
+     * Nevertheless, to improve the debugging experience when using third party libraries
+     * that may not always throw an Error instance, the function performs a type
+     * inspection of the passed argument and attempts to extract as much information
+     * as possible, since just throwing a bad type error makes any debugging rather difficult.
+     *
+     * @param errorOrMessage Error object or error message to be stored in the request.
+     * @param [options]
+     */
+    pushErrorMessage(errorOrMessage, options = {}) {
+        const { omitStack } = options;
+        let message;
+        const type = typeof errorOrMessage;
+        if (type === 'object') {
+            if (!errorOrMessage) {
+                message = 'null';
+            }
+            else if (errorOrMessage instanceof Error) {
+                message = omitStack
+                    ? errorOrMessage.message
+                    : // .stack includes the message
+                        errorOrMessage.stack;
+            }
+            else if (Reflect.has(Object(errorOrMessage), 'message')) {
+                message = Reflect.get(Object(errorOrMessage), 'message');
+            }
+            else if (errorOrMessage.toString() !== '[object Object]') {
+                message = errorOrMessage.toString();
+            }
+            else {
+                try {
+                    message = util.inspect(errorOrMessage);
+                }
+                catch (err) {
+                    message = 'Unable to extract any message from the received object.';
+                }
+            }
+        }
+        else if (type === 'undefined') {
+            message = 'undefined';
+        }
+        else {
+            message = errorOrMessage.toString();
+        }
+        this.errorMessages.push(message);
+    }
+}

package/internals/enqueue_links/enqueue_links.d.ts

@@ -0,0 +1,152 @@
+import type { Dictionary } from '@crawlee/types';
+import type { EnqueueStrategyOption, RequestQueueOperationOptions } from '@crawlee/core';
+import type { RequestTransform, SkippedRequestCallback, UrlPatternInput, UrlPatternObject } from './shared.js';
+/**
+ * Options shared by the `extractLinks()` context helper across crawler types.
+ */
+export interface ExtractLinksOptions {
+    /** A CSS selector matching links to be extracted. */
+    selector?: string;
+    /**
+     * A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
+     * since the relative URL resolution is done inside the browser automatically.
+     */
+    baseUrl?: string;
+}
+/**
+ * Options accepted by the `enqueueUrls()` context helper exposed by `BasicCrawler`.
+ */
+export interface EnqueueUrlsOptions extends RequestQueueOperationOptions {
+    /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
+    limit?: number;
+    /** Sets {@link Request.userData} for newly enqueued requests. */
+    userData?: Dictionary;
+    /**
+     * Sets {@link Request.label} for newly enqueued requests.
+     *
+     * Can be overwritten by `transformRequestFunction`.
+     */
+    label?: string;
+    /** Sets {@link Request.sessionId} for newly enqueued requests. */
+    sessionId?: string;
+    /**
+     * If set to `true`, tells the crawler to skip navigation and process the request directly.
+     * @default false
+     */
+    skipNavigation?: boolean;
+    /**
+     * A base URL that will be used to resolve relative URLs.
+     */
+    baseUrl?: string;
+    /**
+     * An array of URL patterns that URLs must match to be enqueued.
+     *
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
+     *
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, use a `RegExp`.
+     *
+     * The patterns are combined with the {@link EnqueueUrlsOptions.strategy|`strategy`} using AND logic - a URL
+     * must match at least one `include` pattern **and** satisfy the strategy to be enqueued. To match URLs across
+     * hostnames, pass an explicit {@link EnqueueStrategy.All} strategy.
+     *
+     * If `undefined`, the links are enqueued based on the {@link EnqueueUrlsOptions.strategy|`strategy`} alone.
+     * Passing an empty array is not allowed.
+     */
+    include?: readonly UrlPatternInput[];
+    /**
+     * An array of URL patterns. Matching URLs will **not** be enqueued.
+     *
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
+     *
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, use a `RegExp`.
+     */
+    exclude?: readonly UrlPatternInput[];
+    /**
+     * After request options are filtered by `include`/`exclude` patterns, this function can be used
+     * to remove them or modify their contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
+     * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
+     * or to dynamically update or create `userData`.
+     *
+     * For example: by adding `keepUrlFragment: true` to the request options, URL fragments will not be removed
+     * when `uniqueKey` is computed.
+     *
+     * **Example:**
+     * ```javascript
+     * {
+     *     transformRequestFunction: (request) => {
+     *         request.userData.foo = 'bar';
+     *         request.keepUrlFragment = true;
+     *         return request;
+     *     }
+     * }
+     * ```
+     *
+     * Note that `transformRequestFunction` has the highest priority and can overwrite
+     * the global `label` option.
+     *
+     * The function receives a {@link RequestOptions} object and can return either:
+     * - The modified {@link RequestOptions} object
+     * - `'unchanged'` to keep the original options as-is
+     * - A falsy value or `'skip'` to exclude the request from the queue
+     */
+    transformRequestFunction?: RequestTransform;
+    /**
+     * The strategy to use when enqueueing the urls.
+     *
+     * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
+     *
+     * ```md
+     * Protocol          Domain
+     * ┌────┐          ┌─────────┐
+     * https://example.crawlee.dev/...
+     * │       └─────────────────┤
+     * │             Hostname    │
+     * │                         │
+     * └─────────────────────────┘
+     *          Origin
+     *```
+     *
+     * @default EnqueueStrategy.SameHostname
+     */
+    strategy?: EnqueueStrategyOption;
+    /**
+     * By default, only the first batch (1000) of found requests will be added to the queue before resolving the call.
+     * You can use this option to wait for adding all of them.
+     */
+    waitForAllRequestsToBeAdded?: boolean;
+    /**
+     * When a request is skipped for some reason, you can use this callback to act on it.
+     * This is currently fired for requests skipped
+     * 1. based on robots.txt file,
+     * 2. because they don't match enqueueLinks filters,
+     * 3. or because the maxRequestsPerCrawl limit has been reached
+     */
+    onSkippedRequest?: SkippedRequestCallback;
+}
+/** The combined options accepted by a crawler context's `enqueueLinks()` helper: `extractLinks()` + `enqueueUrls()`. */
+export type EnqueueLinksOptions = ExtractLinksOptions & EnqueueUrlsOptions;
+/**
+ * @internal
+ * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
+ * - If a user provides a base url, we always return it
+ * - If a user specifies {@link EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original
+ *   request domain, or a redirected one
+ * - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
+ */
+export declare function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }: ResolveBaseUrl): string;
+/**
+ * @internal
+ */
+export interface ResolveBaseUrl {
+    userProvidedBaseUrl?: string;
+    enqueueStrategy?: EnqueueStrategyOption;
+    originalRequestUrl: string;
+    finalRequestUrl?: string;
+}
+/**
+ * @internal
+ * Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
+ */
+export declare function buildEnqueueStrategyPatterns(baseUrl: string, strategy: EnqueueStrategyOption): UrlPatternObject[];

package/internals/enqueue_links/enqueue_links.js

@@ -0,0 +1,76 @@
+import { EnqueueStrategy } from '@crawlee/utils';
+import { getDomain } from 'tldts';
+/**
+ * @internal
+ * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
+ * - If a user provides a base url, we always return it
+ * - If a user specifies {@link EnqueueStrategy.All} strategy, they do not care if the newly found urls are on the original
+ *   request domain, or a redirected one
+ * - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
+ */
+export function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }) {
+    // User provided base url takes priority
+    if (userProvidedBaseUrl) {
+        return userProvidedBaseUrl;
+    }
+    const originalUrlOrigin = new URL(originalRequestUrl).origin;
+    const finalUrlOrigin = new URL(finalRequestUrl ?? originalRequestUrl).origin;
+    // We can assume users want to go off the domain in this case
+    if (enqueueStrategy === EnqueueStrategy.All) {
+        return finalUrlOrigin;
+    }
+    // If the user wants to ensure the same domain is accessed, regardless of subdomains, we check to ensure the domains match
+    // If they don't (we went off domain via a redirect), we keep filtering against the original domain - returning undefined would disable the filtering entirely
+    if (enqueueStrategy === EnqueueStrategy.SameDomain) {
+        const originalHostname = getDomain(originalUrlOrigin, { mixedInputs: false });
+        const finalHostname = getDomain(finalUrlOrigin, { mixedInputs: false });
+        if (originalHostname === finalHostname) {
+            return finalUrlOrigin;
+        }
+        return originalUrlOrigin;
+    }
+    // Always enqueue urls that are from the same origin in all other cases, as the filtering happens on the original request url, even if there was a redirect
+    // before actually finding the urls
+    return originalUrlOrigin;
+}
+/**
+ * @internal
+ * Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
+ */
+export function buildEnqueueStrategyPatterns(baseUrl, strategy) {
+    const url = new URL(baseUrl);
+    switch (strategy) {
+        case EnqueueStrategy.SameHostname:
+            // We need to get the origin of the passed in domain in the event someone sets baseUrl
+            // to an url like https://example.com/deep/default/path and one of the found urls is an
+            // absolute relative path (/path/to/page)
+            return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
+        case EnqueueStrategy.SameDomain: {
+            // Get the actual hostname from the base url
+            const baseUrlHostname = getDomain(url.hostname, { mixedInputs: false });
+            if (baseUrlHostname) {
+                // We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
+                url.hostname = baseUrlHostname;
+                return [
+                    { glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) },
+                    { glob: ignoreHttpSchema(`${url.origin}/**`) },
+                ];
+            }
+            // We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
+            // as SameDomainAndSubdomain
+            return [{ glob: ignoreHttpSchema(`${url.origin}/**`) }];
+        }
+        case EnqueueStrategy.SameOrigin:
+            // The same behavior as SameHostname, but respecting the protocol of the URL
+            return [{ glob: `${url.origin}/**` }];
+        case EnqueueStrategy.All:
+        default:
+            return [{ glob: `http{s,}://**` }];
+    }
+}
+/**
+ * Internal function that changes the enqueue glob patterns to match both http and https
+ */
+function ignoreHttpSchema(pattern) {
+    return pattern.replace(/^(https?):\/\//, 'http{s,}://');
+}

package/internals/enqueue_links/index.d.ts

@@ -0,0 +1,3 @@
+export * from './enqueue_links.js';
+export { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, updateEnqueueLinksPatternCache, urlPatternSchema, validateGlobPattern, } from './shared.js';
+export type { GlobInput, GlobObject, RegExpInput, RegExpObject, RequestTransform, SkippedRequestCallback, UrlPatternInput, } from './shared.js';

package/internals/enqueue_links/index.js

@@ -0,0 +1,3 @@
+export * from './enqueue_links.js';
+// Not `export *`: keeps the internal `UrlPatternObject` off the public surface.
+export { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, updateEnqueueLinksPatternCache, urlPatternSchema, validateGlobPattern, } from './shared.js';

package/internals/enqueue_links/shared.d.ts

@@ -0,0 +1,106 @@
+import type { Awaitable, Dictionary } from '@crawlee/types';
+import { z } from 'zod';
+import type { EnqueueStrategyOption, RequestOptions, Source } from '@crawlee/core';
+import { Request } from '@crawlee/core';
+import type { SkippedRequestReason } from '../crawling_request.js';
+export interface GlobObject {
+    glob: string;
+}
+export type GlobInput = string | GlobObject;
+export interface RegExpObject {
+    regexp: RegExp;
+}
+export type RegExpInput = RegExp | RegExpObject;
+/** Unified URL pattern input — accepts glob strings, glob objects, RegExp instances, or regexp objects. */
+export type UrlPatternInput = GlobInput | RegExpInput;
+/**
+ * Accepts one {@link UrlPatternInput} — a glob string, a RegExp instance, or a `{ glob }` / `{ regexp }` object.
+ * @internal
+ */
+export declare const urlPatternSchema: z.ZodType<UrlPatternInput>;
+export type SkippedRequestCallback = (args: {
+    request: Request;
+    reason: SkippedRequestReason;
+    /** The `message` passed to {@link RestrictedCrawlingContext.skipRequest|`context.skipRequest()`}, if any. */
+    message?: string;
+}) => Awaitable<void>;
+/**
+ * Builds the `{ request, reason }` argument passed to a {@link SkippedRequestCallback}, constructing the
+ * `Request` lazily on first access to `request` (and caching it) since most skips are never observed by a
+ * real callback and building a `Request` isn't free.
+ * @ignore
+ */
+export declare function createSkippedRequestArgs(source: string | Source, reason: SkippedRequestReason): Parameters<SkippedRequestCallback>[0];
+/**
+ * @ignore
+ */
+export declare function updateEnqueueLinksPatternCache(item: GlobInput | RegExpInput, pattern: RegExpObject | GlobObject): void;
+/**
+ * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
+ * to construct Glob objects from Glob pattern strings.
+ * @ignore
+ */
+export declare function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): GlobObject[];
+/**
+ * @internal
+ */
+export declare function validateGlobPattern(glob: string): string;
+/**
+ * Helper factory used in the `enqueueLinks()` and enqueueLinksByClickingElements() function
+ * to check RegExps input and return valid RegExps.
+ * @ignore
+ */
+export declare function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): RegExpObject[];
+/**
+ * The compiled form of a {@link UrlPatternInput}, produced by {@link constructUrlPatternObjects}. Deliberately
+ * not re-exported from the package entry point — it is an implementation detail of the pattern matching helpers.
+ * @internal
+ */
+export interface UrlPatternObject {
+    glob?: string;
+    regexp?: RegExp;
+}
+/**
+ * Helper factory used in the `enqueueLinks()` function to construct UrlPatternObjects
+ * from a mixed array of glob strings, glob objects, RegExp instances, and regexp objects.
+ * @ignore
+ */
+export declare function constructUrlPatternObjects(patterns: readonly UrlPatternInput[]): UrlPatternObject[];
+/**
+ * Filters request options by URL patterns.
+ *
+ * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
+ * @ignore
+ */
+export declare function filterRequestOptionsByPatterns(requestOptions: RequestOptions[], includePatterns: UrlPatternObject[] | undefined, excludePatterns?: UrlPatternObject[], strategy?: EnqueueStrategyOption, onSkippedRequestOptions?: (options: RequestOptions) => void): RequestOptions[];
+/**
+ * @ignore
+ */
+export declare function createRequestOptions(sources: readonly (string | Record<string, unknown>)[], options?: {
+    label?: string;
+    userData?: Dictionary;
+    baseUrl?: string;
+    skipNavigation?: boolean;
+    sessionId?: string;
+    strategy?: EnqueueStrategyOption;
+}): RequestOptions[];
+/**
+ * Takes a {@link RequestOptions} object and changes its attributes in a desired way. This user-function is used
+ * by {@link enqueueLinks} to modify request options before they are converted to {@link Request} instances.
+ */
+export interface RequestTransform {
+    /**
+     * @param original Request options to be modified.
+     * @returns The modified request options to enqueue, `'unchanged'` to keep the original options as-is,
+     *   or a falsy value / `'skip'` to exclude the request from the queue.
+     */
+    (original: RequestOptions): RequestOptions | false | undefined | null | 'skip' | 'unchanged';
+}
+/**
+ * Applies a {@link RequestTransform} function to a list of request options.
+ * Options for which the transform returns a falsy value are removed from the list.
+ * @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
+ * @ignore
+ * @internal
+ */
+export declare function applyRequestTransform(requestOptions: RequestOptions[], transformFn: RequestTransform, onSkipped?: (requestOptions: RequestOptions) => void): RequestOptions[];