@crawlee/basic 4.0.0-beta.99 → 4.0.0-rc.1

package/internals/crawlers/crawler_commons.d.ts

@@ -0,0 +1,216 @@
+import type { Awaitable, Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
+import type { ReadonlyDeep } from 'type-fest';
+import type { AddRequestsBatchedResult, CrawleeLogger, Dataset, KeyValueStore, Request, RequestOptions, Source, StorageIdentifier } from '@crawlee/core';
+import type { EnqueueUrlsOptions } from '../enqueue_links/index.js';
+import type { CrawlingRequest } from '../crawling_request.js';
+/** @internal */
+export type IsAny<T> = 0 extends 1 & T ? true : false;
+/**
+ * A request input (URL string, request-options object, or {@link Request}) whose `userData` is typed
+ * according to its `label`, based on a router's route map.
+ *
+ * When the route map is open (the default `Record<string, ...>`), this is just the regular loose
+ * {@link Source} input. When the map declares concrete labels, providing a `label` requires the matching
+ * `userData` shape and rejects labels not present in the map; unlabeled requests keep loose `userData`.
+ */
+export type LabeledSource<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | Source : string | Request | ({
+    requestsFromUrl?: string;
+    regex?: RegExp;
+} & ({
+    [Label in keyof Routes & string]: Omit<Partial<RequestOptions<Routes[Label]>>, 'label'> & {
+        label: Label;
+    };
+}[keyof Routes & string] | (Omit<Partial<RequestOptions>, 'label'> & {
+    label?: undefined;
+})));
+/**
+ * The iterable/array of {@link LabeledSource} inputs accepted by the label-aware `addRequests`/`run`
+ * methods of a crawler bound to a typed router.
+ */
+export type TypedRequestsLike<Routes extends Record<keyof Routes, Dictionary>> = AsyncIterable<LabeledSource<Routes>> | Iterable<LabeledSource<Routes>> | LabeledSource<Routes>[];
+/**
+ * The label-aware `addRequests` method signature exposed on a request handler's context when the crawler is
+ * bound to a typed router. Mirrors {@link RestrictedCrawlingContext.addRequests} with typed sources.
+ */
+export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
+/**
+ * An `enqueueLinks`-options object with its `label`/`userData` retyped according to a router's route map: a
+ * declared `label` requires the matching `userData` shape (unknown labels are rejected), while unlabeled
+ * calls keep loose `userData`. Returns the options unchanged when the route map is open (the default).
+ */
+type TypedEnqueueLinksOptions<Options, Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? Options : Omit<Options, 'label' | 'userData'> & ({
+    [Label in keyof Routes & string]: {
+        label: Label;
+        userData?: Routes[Label];
+    };
+}[keyof Routes & string] | {
+    label?: undefined;
+    userData?: Dictionary;
+});
+/**
+ * Transforms a context's existing `enqueueLinks` method so that the `label`/`userData` in its options follow
+ * the router's route map, while preserving everything else about the signature (argument optionality and
+ * return type, which differ between crawler types).
+ */
+export type TypedContextEnqueueLinks<EnqueueLinks, Routes extends Record<keyof Routes, Dictionary>> = EnqueueLinks extends (options?: infer Options) => infer Result ? (options?: TypedEnqueueLinksOptions<Options, Routes>) => Result : EnqueueLinks extends (options: infer Options) => infer Result ? (options: TypedEnqueueLinksOptions<Options, Routes>) => Result : EnqueueLinks;
+/** A {@link Request} that has been dispatched, so its `id` and `loadedUrl` are guaranteed to be present. */
+export type LoadedRequest<R extends Request> = R & Required<Pick<R, 'id' | 'loadedUrl'>>;
+/** @internal */
+export type LoadedContext<Context extends RestrictedCrawlingContext> = IsAny<Context> extends true ? Context : {
+    request: LoadedRequest<Context['request']>;
+} & Omit<Context, 'request'>;
+export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> {
+    /** @internal */
+    id: string;
+    session: ISession;
+    /**
+     * An object with information about currently used proxy by the crawler
+     * and configured by the {@link ProxyConfiguration} class.
+     */
+    proxyInfo?: ProxyInfo;
+    /**
+     * The request being processed.
+     */
+    request: CrawlingRequest<UserData>;
+    /**
+     * This function allows you to push data to a {@link Dataset} specified by name, or the one currently used by the crawler.
+     *
+     * Shortcut for `crawler.pushData()`.
+     *
+     * @param [data] Data to be pushed to the default dataset.
+     */
+    pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdentifier?: string | StorageIdentifier): Promise<void>;
+    /**
+     * Add requests directly to the request queue currently used by the crawler.
+     *
+     * Optionally, the function allows you to filter the target URLs using an array of glob or regexp patterns,
+     * the same way {@link CrawlingContext.enqueueLinks|`enqueueLinks`} does for extracted links.
+     *
+     * @param requests The requests to add
+     * @param options Options for the request queue
+     */
+    addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
+    /**
+     * Returns the state - a piece of mutable persistent data shared across all the request handler runs.
+     */
+    useState: <State extends Dictionary = Dictionary>(defaultValue?: State) => Promise<State>;
+    /**
+     * Get a key-value store with given name or id, or the default one for the crawler.
+     */
+    getKeyValueStore: (identifier?: string | StorageIdentifier) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue' | 'getPublicUrl'>>;
+    /**
+     * A preconfigured logger for the request handler.
+     */
+    log: CrawleeLogger;
+    /**
+     * Stops processing the current request and marks it as handled without counting it as a failure: no retry, no
+     * `failedRequestHandler`, and no error log. The request's `state` becomes {@link RequestState.SKIPPED|`SKIPPED`}
+     * and {@link BasicCrawlerOptions.onSkippedRequest|`onSkippedRequest`} fires with the `'manual'` reason.
+     *
+     * Storage writes made for the request so far are rolled back, as with a failure, unless
+     * {@link BasicCrawlerOptions.transactionalStorage|`transactionalStorage`} is disabled.
+     *
+     * Works in `errorHandler` too, skipping a request that would otherwise be retried. In `failedRequestHandler` it
+     * has no effect, since the request has already failed.
+     *
+     * ```ts
+     * preNavigationHooks: [
+     *     async ({ request, skipRequest }) => {
+     *         if (await alreadyScraped(request.url)) skipRequest('already scraped');
+     *     },
+     * ],
+     * ```
+     *
+     * @param message Passed to `onSkippedRequest`.
+     */
+    skipRequest(message?: string): never;
+}
+export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
+    /**
+     * Fires HTTP request via the internal HTTP client, allowing to override the request options on the fly.
+     *
+     * This is handy when you work with a browser crawler but want to execute some requests outside it (e.g. API requests).
+     * Check the [Skipping navigations for certain requests](https://crawlee.dev/js/docs/examples/skip-navigation) example for
+     * more detailed explanation of how to do that.
+     *
+     * ```ts
+     * async requestHandler({ sendRequest }) {
+     *     const { body } = await sendRequest({
+     *         // override headers only
+     *         headers: { ... },
+     *     });
+     * },
+     * ```
+     */
+    sendRequest: (requestOverrides?: Partial<HttpRequestOptions>, optionsOverrides?: SendRequestOptions) => Promise<Response>;
+    /**
+     * Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.
+     *
+     * The callback runs *outside* the request's storage transaction, so storage writes made here are
+     * applied immediately and are **not** rolled back when the request fails. In
+     * {@link AdaptivePlaywrightCrawler} it also runs once per request handler attempt, so a write
+     * here can land more than once for a single request. Push results from the request handler itself.
+     */
+    registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
+    /**
+     * Registers `callback` to run once the request's storage writes have been committed, or once
+     * committing them has failed - the point where a write made in the handler can finally be reacted
+     * to where it was made. Two things belong here: side effects that must agree with what actually
+     * landed (result counters, progress reporting), and handling of a write that did not land at all.
+     *
+     * ```ts
+     * async requestHandler({ pushData, afterStorageCommit, useState }) {
+     *     const state = await useState({ itemCount: 0 });
+     *
+     *     await pushData(item);
+     *     afterStorageCommit((error) => {
+     *         if (error) {
+     *             throw new NonRetryableError('The page is too big to store', { cause: error });
+     *         }
+     *         state.itemCount++;
+     *     });
+     * }
+     * ```
+     *
+     * `useState()` is deliberately not transactional, which is what makes the counter case necessary:
+     * incrementing in the handler would keep the increment for a request that pushes an item and then
+     * fails, while the item itself is rolled back.
+     *
+     * Callbacks run in registration order, after the commit and before the request is marked as handled.
+     * The first one to throw propagates and the rest do not run; on a failed commit its error replaces
+     * the commit error, so a callback can decide what the request fails with. After a *successful*
+     * commit, a callback that throws fails the request **without a retry** whatever it throws - the
+     * writes are already durable and retrying would duplicate them, so anything that is not already a
+     * {@link NonRetryableError} is wrapped in an {@link AfterCommitError}.
+     *
+     * Callbacks are *not* run when the request handler itself fails - nothing was written, and
+     * `errorHandler` / `failedRequestHandler` cover that.
+     *
+     * Throws when storage is not transactional (`transactionalStorage: false`) - writes are then applied
+     * as they are made and throw at the call site on their own. In {@link AdaptivePlaywrightCrawler}
+     * the callback belongs to the current request handler attempt, so it runs only for the attempt whose
+     * writes are the ones being committed.
+     */
+    afterStorageCommit(callback: (error?: Error) => Awaitable<void>): void;
+    /**
+     * Gives the current request `secs` more seconds to finish, for when how long it needs is only apparent
+     * once it is already running - a listing page that turns out to have far more to scroll through than
+     * usual, say. Prefer `requestHandlerTimeoutSecs`, or a per-route override via
+     * {@link Router.addHandler|`router.addHandler`}, whenever the time needed is known up front.
+     *
+     * ```ts
+     * router.addHandler('LIST', async ({ extendTimeout, page }) => {
+     *     const pageCount = await countPages(page);
+     *     extendTimeout(pageCount * 10);
+     *     await scrapeAllPages(page);
+     * });
+     * ```
+     *
+     * Extends the request handler's own timeout and the crawler's internal one together, so the extension
+     * is not immediately undone by the latter. On a locking storage backend the request's lock is
+     * extended by the same amount, so the extra time is not spent while the queue considers the request
+     * free to hand out again. Calling it from a handler that has already timed out does nothing.
+     */
+    extendTimeout(secs: number): void;
+}
+export {};

package/internals/crawlers/crawler_commons.js

@@ -0,0 +1 @@
+export {};

package/internals/crawlers/error_snapshotter.d.ts

@@ -0,0 +1,29 @@
+import type { CrawlingContext } from '../crawlers/crawler_commons.js';
+import type { ErrnoException } from './error_tracker.js';
+import type { SnapshottableProperties } from './internals/types.js';
+interface ErrorSnapshot {
+    screenshotFileName?: string;
+    screenshotFileUrl?: string;
+    htmlFileName?: string;
+    htmlFileUrl?: string;
+}
+/**
+ * ErrorSnapshotter class is used to capture a screenshot of the page and a snapshot of the HTML when an error occurs during web crawling.
+ *
+ * This functionality is opt-in, and can be enabled via the crawler options:
+ *
+ * ```ts
+ * const crawler = new BasicCrawler({
+ *   // ...
+ *   statistics: new Statistics({ saveErrorSnapshots: true }),
+ * });
+ * ```
+ */
+export declare class ErrorSnapshotter {
+    #private;
+    /**
+     * Capture a snapshot of the error context.
+     */
+    captureSnapshot(error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<ErrorSnapshot>;
+}
+export {};

package/internals/crawlers/error_snapshotter.js

@@ -0,0 +1,116 @@
+import crypto from 'node:crypto';
+const MAX_ERROR_CHARACTERS = 30;
+const MAX_HASH_LENGTH = 30;
+const MAX_FILENAME_LENGTH = 250;
+const BASE_MESSAGE = 'An error occurred';
+const SNAPSHOT_PREFIX = 'ERROR_SNAPSHOT';
+/**
+ * ErrorSnapshotter class is used to capture a screenshot of the page and a snapshot of the HTML when an error occurs during web crawling.
+ *
+ * This functionality is opt-in, and can be enabled via the crawler options:
+ *
+ * ```ts
+ * const crawler = new BasicCrawler({
+ *   // ...
+ *   statistics: new Statistics({ saveErrorSnapshots: true }),
+ * });
+ * ```
+ */
+export class ErrorSnapshotter {
+    /**
+     * Capture a snapshot of the error context.
+     */
+    async captureSnapshot(error, context) {
+        try {
+            const page = context?.page;
+            const body = context?.body;
+            const keyValueStore = await context?.getKeyValueStore();
+            // If the key-value store is not available, or the body and page are not available, return empty filenames
+            if (!keyValueStore || (!body && !page)) {
+                return {};
+            }
+            const fileName = this.#generateFilename(error);
+            let screenshotFileName;
+            let htmlFileName;
+            if (page) {
+                const capturedFiles = await this.#contextCaptureSnapshot(context, fileName);
+                if (capturedFiles) {
+                    screenshotFileName = capturedFiles.screenshotFileName;
+                    htmlFileName = capturedFiles.htmlFileName;
+                }
+                // If the snapshot for browsers failed to capture the HTML, try to capture it from the page content
+                if (!htmlFileName) {
+                    const html = await page.content();
+                    htmlFileName = html ? await this.#saveHTMLSnapshot(html, keyValueStore, fileName) : undefined;
+                }
+            }
+            else if (typeof body === 'string') {
+                // for non-browser contexts
+                htmlFileName = await this.#saveHTMLSnapshot(body, keyValueStore, fileName);
+            }
+            return {
+                screenshotFileName,
+                screenshotFileUrl: screenshotFileName && (await keyValueStore.getPublicUrl(screenshotFileName)),
+                htmlFileName,
+                htmlFileUrl: htmlFileName && (await keyValueStore.getPublicUrl(htmlFileName)),
+            };
+        }
+        catch {
+            return {};
+        }
+    }
+    /**
+     * Captures a snapshot of the current page using the context.saveSnapshot function.
+     * This function is applicable for browser contexts only.
+     * Returns an object containing the filenames of the screenshot and HTML file.
+     */
+    async #contextCaptureSnapshot(context, fileName) {
+        try {
+            await context.saveSnapshot({ key: fileName });
+            return {
+                screenshotFileName: `${fileName}.jpg`,
+                htmlFileName: `${fileName}.html`,
+            };
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /**
+     * Save the HTML snapshot of the page, and return the key it was stored under.
+     */
+    async #saveHTMLSnapshot(html, keyValueStore, fileName) {
+        try {
+            await keyValueStore.setValue(fileName, html, { contentType: 'text/html' });
+            // The record key is `fileName` - returning it with an `.html` suffix (as v3 did,
+            // where local storage put the extension in the key) would break `getPublicUrl`.
+            return fileName;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /**
+     * Generate a unique fileName for each error snapshot.
+     */
+    #generateFilename(error) {
+        // Create a hash of the error stack trace
+        const errorStackHash = crypto
+            .createHash('sha1')
+            .update(error.stack || error.message || '')
+            .digest('hex')
+            .slice(0, MAX_HASH_LENGTH);
+        const errorMessagePrefix = (error.message || BASE_MESSAGE).slice(0, MAX_ERROR_CHARACTERS).trim();
+        /**
+         * Remove non-word characters from the start and end of a string.
+         */
+        const sanitizeString = (str) => {
+            return str.replace(/^\W+|\W+$/g, '');
+        };
+        // Generate fileName and remove disallowed characters
+        const fileName = `${SNAPSHOT_PREFIX}_${sanitizeString(errorStackHash)}_${sanitizeString(errorMessagePrefix)}`
+            .replace(/\W+/g, '-') // Replace non-word characters with a dash
+            .slice(0, MAX_FILENAME_LENGTH);
+        return fileName;
+    }
+}

package/internals/crawlers/error_tracker.d.ts

@@ -0,0 +1,51 @@
+import type { CrawlingContext } from '../crawlers/crawler_commons.js';
+/**
+ * Node.js Error interface
+ */
+export interface ErrnoException extends Error {
+    errno?: number;
+    code?: string | number;
+    path?: string;
+    syscall?: string;
+    cause?: any;
+}
+export interface ErrorTrackerOptions {
+    showErrorCode: boolean;
+    showErrorName: boolean;
+    showStackTrace: boolean;
+    showFullStack: boolean;
+    showErrorMessage: boolean;
+    showFullMessage: boolean;
+    saveErrorSnapshots: boolean;
+}
+/**
+ * This class tracks errors and computes a summary of information like:
+ * - where the errors happened
+ * - what the error names are
+ * - what the error codes are
+ * - what is the general error message
+ *
+ * This is extremely useful when there are dynamic error messages, such as argument validation.
+ *
+ * Since the structure of the `tracker.result` object differs when using different options,
+ * it's typed as `Record<string, unknown>`. The most deep object has a `count` property, which is a number.
+ *
+ * It's possible to get the total amount of errors via the `tracker.total` property.
+ */
+export declare class ErrorTracker {
+    #private;
+    /** The grouped error tree the tracker accumulates. Read-only: the object is mutated in place as errors arrive. */
+    get result(): Record<string, unknown>;
+    /** The total number of errors passed to the tracker, including the same error seen repeatedly. */
+    get total(): number;
+    constructor(options?: Partial<ErrorTrackerOptions>);
+    add(error: ErrnoException): void;
+    /**
+     * This method is async, because it captures a snapshot of the error context.
+     * We added this new method to avoid breaking changes.
+     */
+    addAsync(error: ErrnoException, context?: CrawlingContext): Promise<void>;
+    getUniqueErrorCount(): number;
+    getMostPopularErrors(count: number): [number, string[]][];
+    reset(): void;
+}