@crawlee/core 4.0.0-beta.15 → 4.0.0-beta.150

package/debug.d.ts

@@ -0,0 +1,36 @@
+import type { IncomingMessage } from 'node:http';
+import type { Dictionary } from '@crawlee/types';
+import type { Request } from './request.js';
+interface BrowserResponseLike {
+    status(): number;
+}
+/**
+ * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
+ *
+ * @param request [Request](https://sdk.apify.com/docs/api/request) object.
+ * @param [response]
+ *   Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
+ *   or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
+ * @param [additionalFields] Object containing additional fields to be added.
+ *
+ * @internal
+ */
+export declare function createRequestDebugInfo(request: Request, response?: IncomingMessage | Partial<BrowserResponseLike>, additionalFields?: Dictionary): Dictionary;
+/**
+ * Returns a human-readable label for an unknown value,
+ * suitable for embedding in error messages and log output.
+ *
+ * Returns `constructor.name` when available (e.g. `"Configuration"`, `"Number"`),
+ * otherwise falls back to `util.inspect` (e.g. for `null`, `undefined`).
+ *
+ * @internal
+ */
+export declare function inspectValue(value: unknown): string;
+/**
+ * Returns the type of a value as a lowercase string, with `Date`, `Buffer` and `RegExp` reported
+ * by their constructor name. Used for building validation error messages.
+ *
+ * @internal
+ */
+export declare function getObjectType(value: unknown): string;
+export {};

package/debug.js

@@ -0,0 +1,70 @@
+import { inspect } from 'node:util';
+import { parseArgument, schemas } from './validators.js';
+/**
+ * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
+ *
+ * @param request [Request](https://sdk.apify.com/docs/api/request) object.
+ * @param [response]
+ *   Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
+ *   or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
+ * @param [additionalFields] Object containing additional fields to be added.
+ *
+ * @internal
+ */
+export function createRequestDebugInfo(request, response = {}, additionalFields = {}) {
+    parseArgument(request, schemas.anyObject);
+    parseArgument(response, schemas.anyObject);
+    parseArgument(additionalFields, schemas.anyObject);
+    return {
+        requestId: request.id,
+        url: request.url,
+        loadedUrl: request.loadedUrl,
+        method: request.method,
+        retryCount: request.retryCount,
+        errorMessages: request.errorMessages,
+        // Puppeteer response has .status() function and NodeJS response, statusCode property.
+        statusCode: 'status' in response && response.status instanceof Function
+            ? response.status()
+            : response.statusCode,
+        ...additionalFields,
+    };
+}
+/**
+ * Returns a human-readable label for an unknown value,
+ * suitable for embedding in error messages and log output.
+ *
+ * Returns `constructor.name` when available (e.g. `"Configuration"`, `"Number"`),
+ * otherwise falls back to `util.inspect` (e.g. for `null`, `undefined`).
+ *
+ * @internal
+ */
+export function inspectValue(value) {
+    if (typeof value === 'object' && value !== null && value.constructor?.name) {
+        return value.constructor.name;
+    }
+    return inspect(value, {
+        depth: 0,
+        compact: true,
+        maxStringLength: 64,
+        breakLength: Infinity,
+        colors: false,
+    });
+}
+/**
+ * Returns the type of a value as a lowercase string, with `Date`, `Buffer` and `RegExp` reported
+ * by their constructor name. Used for building validation error messages.
+ *
+ * @internal
+ */
+export function getObjectType(value) {
+    const simple = typeof value;
+    if (['string', 'number', 'boolean', 'bigint'].includes(simple)) {
+        return simple;
+    }
+    const objectType = Object.prototype.toString.call(value);
+    const type = /\[object (\w+)]/.exec(objectType)[1];
+    if (type === 'Uint8Array') {
+        return 'Buffer';
+    }
+    return ['Date', 'Buffer', 'RegExp'].includes(type) ? type : type.toLowerCase();
+}

package/enqueue_links/enqueue_links.d.ts

@@ -1,98 +1,76 @@
-import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type RobotsTxtFile } from '@crawlee/utils';
-import type { SetRequired } from 'type-fest';
-import type { Request } from '../request.js';
-import type { AddRequestsBatchedOptions, AddRequestsBatchedResult, RequestProvider, RequestQueueOperationOptions } from '../storages/request_provider.js';
-import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, SkippedRequestCallback } from './shared.js';
-export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
+import type { Dictionary } from '@crawlee/types';
+import { EnqueueStrategy } from '@crawlee/utils';
+import type { RequestQueueOperationOptions } from '../storages/request_queue.js';
+import type { RequestTransform, SkippedRequestCallback, UrlPatternInput, UrlPatternObject } from './shared.js';
+/**
+ * Options shared by the `extractLinks()` context helper across crawler types.
+ */
+export interface ExtractLinksOptions {
+    /** A CSS selector matching links to be extracted. */
+    selector?: string;
+    /**
+     * A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
+     * since the relative URL resolution is done inside the browser automatically.
+     */
+    baseUrl?: string;
+}
+/**
+ * Options accepted by the `enqueueUrls()` context helper exposed by `BasicCrawler`.
+ */
+export interface EnqueueUrlsOptions extends RequestQueueOperationOptions {
     /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
     limit?: number;
-    /** An array of URLs to enqueue. */
-    urls?: readonly string[];
-    /** A request queue to which the URLs will be enqueued. */
-    requestQueue?: RequestProvider;
-    /** A CSS selector matching links to be enqueued. */
-    selector?: string;
     /** Sets {@link Request.userData} for newly enqueued requests. */
     userData?: Dictionary;
     /**
      * Sets {@link Request.label} for newly enqueued requests.
      *
-     * Note that the request options specified in `globs`, `regexps`, or `pseudoUrls` objects
-     * have priority over this option.
+     * Can be overwritten by `transformRequestFunction`.
      */
     label?: string;
+    /** Sets {@link Request.sessionId} for newly enqueued requests. */
+    sessionId?: string;
     /**
      * If set to `true`, tells the crawler to skip navigation and process the request directly.
      * @default false
      */
     skipNavigation?: boolean;
     /**
-     * A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
-     * since the relative URL resolution is done inside the browser automatically.
+     * A base URL that will be used to resolve relative URLs.
      */
     baseUrl?: string;
     /**
-     * An array of glob pattern strings or plain objects
-     * containing glob pattern strings matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `glob` property, which holds the glob pattern string.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
-     *
-     * The matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
-     *
-     * If `globs` is an empty array or `undefined`, and `regexps` are also not defined, then the function
-     * enqueues the links with the same subdomain.
-     */
-    globs?: readonly GlobInput[];
-    /**
-     * An array of glob pattern strings, regexp patterns or plain objects
-     * containing patterns matching URLs that will **never** be enqueued.
+     * An array of URL patterns that URLs must match to be enqueued.
      *
-     * The plain objects must include either the `glob` property or the `regexp` property.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
      * Glob matching is always case-insensitive.
-     * If you need case-sensitive matching, provide a regexp.
-     */
-    exclude?: readonly (GlobInput | RegExpInput)[];
-    /**
-     * An array of regular expressions or plain objects
-     * containing regular expressions matching the URLs to be enqueued.
+     * If you need case-sensitive matching, use a `RegExp`.
      *
-     * The plain objects must include at least the `regexp` property, which holds the regular expression.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
+     * The patterns are combined with the {@link EnqueueUrlsOptions.strategy|`strategy`} using AND logic - a URL
+     * must match at least one `include` pattern **and** satisfy the strategy to be enqueued. To match URLs across
+     * hostnames, pass an explicit {@link EnqueueStrategy.All} strategy.
      *
-     * If `regexps` is an empty array or `undefined`, and `globs` are also not defined, then the function
-     * enqueues the links with the same subdomain.
+     * If `undefined`, the links are enqueued based on the {@link EnqueueUrlsOptions.strategy|`strategy`} alone.
+     * Passing an empty array is not allowed.
      */
-    regexps?: readonly RegExpInput[];
+    include?: readonly UrlPatternInput[];
     /**
-     * *NOTE:* In future versions of SDK the options will be removed.
-     * Please use `globs` or `regexps` instead.
-     *
-     * An array of {@link PseudoUrl} strings or plain objects
-     * containing {@link PseudoUrl} strings matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `purl` property, which holds the pseudo-URL string.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
-     *
-     * With a pseudo-URL string, the matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
+     * An array of URL patterns. Matching URLs will **not** be enqueued.
      *
-     * If `pseudoUrls` is an empty array or `undefined`, then the function
-     * enqueues the links with the same subdomain.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
-     * @deprecated prefer using `globs` or `regexps` instead
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, use a `RegExp`.
      */
-    pseudoUrls?: readonly PseudoUrlInput[];
+    exclude?: readonly UrlPatternInput[];
     /**
-     * Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used
-     * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
+     * After request options are filtered by `include`/`exclude` patterns, this function can be used
+     * to remove them or modify their contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
      * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
      * or to dynamically update or create `userData`.
      *
-     * For example: by adding `keepUrlFragment: true` to the `request` object, URL fragments will not be removed
+     * For example: by adding `keepUrlFragment: true` to the request options, URL fragments will not be removed
      * when `uniqueKey` is computed.
      *
      * **Example:**
@@ -106,8 +84,13 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      * }
      * ```
      *
-     * Note that the request options specified in `globs`, `regexps`, or `pseudoUrls` objects
-     * have priority over this function. Some request options returned by `transformRequestFunction` may be overwritten by pattern-based options from `globs`, `regexps`, or `pseudoUrls`.
+     * Note that `transformRequestFunction` has the highest priority and can overwrite
+     * the global `label` option.
+     *
+     * The function receives a {@link RequestOptions} object and can return either:
+     * - The modified {@link RequestOptions} object
+     * - `'unchanged'` to keep the original options as-is
+     * - A falsy value or `'skip'` to exclude the request from the queue
      */
     transformRequestFunction?: RequestTransform;
     /**
@@ -128,17 +111,12 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      *
      * @default EnqueueStrategy.SameHostname
      */
-    strategy?: EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
+    strategy?: EnqueueStrategyOption;
     /**
      * By default, only the first batch (1000) of found requests will be added to the queue before resolving the call.
      * You can use this option to wait for adding all of them.
      */
     waitForAllRequestsToBeAdded?: boolean;
-    /**
-     * RobotsTxtFile instance for the current request that triggered the `enqueueLinks`.
-     * If provided, disallowed URLs will be ignored.
-     */
-    robotsTxtFile?: Pick<RobotsTxtFile, 'isAllowed'>;
     /**
      * When a request is skipped for some reason, you can use this callback to act on it.
      * This is currently fired for requests skipped
@@ -148,86 +126,11 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
      */
     onSkippedRequest?: SkippedRequestCallback;
 }
-/**
- * The different enqueueing strategies available.
- *
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
- *
- * ```md
- * Protocol          Domain
- * ┌────┐          ┌─────────┐
- * https://example.crawlee.dev/...
- * │       └─────────────────┤
- * │             Hostname    │
- * │                         │
- * └─────────────────────────┘
- *          Origin
- *```
- *
- * - The `Protocol` is usually `http` or `https`
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
- */
-export declare enum EnqueueStrategy {
-    /**
-     * Matches any URLs found
-     */
-    All = "all",
-    /**
-     * Matches any URLs that have the same hostname.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `https://example.com/hello` will not be matched.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    SameHostname = "same-hostname",
-    /**
-     * Matches any URLs that have the same domain as the base URL.
-     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
-     * `https://example.com`.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    SameDomain = "same-domain",
-    /**
-     * Matches any URLs that have the same hostname and protocol.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `http://wow.example.com/hello` will not be matched.
-     *
-     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
-     */
-    SameOrigin = "same-origin"
-}
-/**
- * This function enqueues the urls provided to the {@link RequestQueue} provided. If you want to automatically find and enqueue links,
- * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.
- *
- * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions
- * and override settings of the enqueued {@link Request} objects.
- *
- * **Example usage**
- *
- * ```javascript
- * await enqueueLinks({
- *   urls: aListOfFoundUrls,
- *   requestQueue,
- *   selector: 'a.product-detail',
- *   globs: [
- *       'https://www.example.com/handbags/*',
- *       'https://www.example.com/purses/*'
- *   ],
- * });
- * ```
- *
- * @param options All `enqueueLinks()` parameters are passed via an options object.
- * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
- */
-export declare function enqueueLinks(options: SetRequired<Omit<EnqueueLinksOptions, 'requestQueue'>, 'urls'> & {
-    requestQueue: {
-        addRequestsBatched: (requests: Request<Dictionary>[], options: AddRequestsBatchedOptions) => Promise<AddRequestsBatchedResult>;
-    };
-}): Promise<BatchAddRequestsResult>;
+/** The combined options accepted by a crawler context's `enqueueLinks()` helper: `extractLinks()` + `enqueueUrls()`. */
+export type EnqueueLinksOptions = ExtractLinksOptions & EnqueueUrlsOptions;
+export { EnqueueStrategy };
+/** The `strategy` option accepted by {@link ExtractLinksOptions} and {@link EnqueueUrlsOptions}. */
+export type EnqueueStrategyOption = EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
 /**
  * @internal
  * This method helps resolve the baseUrl that will be used for filtering in {@link enqueueLinks}.
@@ -236,14 +139,18 @@ export declare function enqueueLinks(options: SetRequired<Omit<EnqueueLinksOptio
  *   request domain, or a redirected one
  * - In all other cases, we return the domain of the original request as that's the one we need to use for filtering
  */
-export declare function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }: ResolveBaseUrl): string | undefined;
+export declare function resolveBaseUrlForEnqueueLinksFiltering({ enqueueStrategy, finalRequestUrl, originalRequestUrl, userProvidedBaseUrl, }: ResolveBaseUrl): string;
 /**
  * @internal
  */
 export interface ResolveBaseUrl {
     userProvidedBaseUrl?: string;
-    enqueueStrategy?: EnqueueLinksOptions['strategy'];
+    enqueueStrategy?: EnqueueStrategyOption;
     originalRequestUrl: string;
     finalRequestUrl?: string;
 }
-//# sourceMappingURL=enqueue_links.d.ts.map
+/**
+ * @internal
+ * Builds the glob patterns a URL must match to satisfy the given enqueue `strategy`, anchored at `baseUrl`.
+ */
+export declare function buildEnqueueStrategyPatterns(baseUrl: string, strategy: EnqueueStrategyOption): UrlPatternObject[];