@crawlee/core 4.0.0-rc.0 → 4.0.0-rc.1

package/crawlers/crawler_commons.d.ts

@@ -1,159 +0,0 @@
-import type { Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
-import type { ReadonlyDeep } from 'type-fest';
-import type { EnqueueUrlsOptions } from '../enqueue_links/enqueue_links.js';
-import type { CrawleeLogger } from '../log.js';
-import type { Request, RequestOptions, Source } from '../request.js';
-import type { StorageIdentifier } from '../storages/storage_instance_manager.js';
-import type { Dataset } from '../storages/dataset.js';
-import type { KeyValueStore } from '../storages/key_value_store.js';
-import type { AddRequestsBatchedResult } from '../storages/request_queue.js';
-/** @internal */
-export type IsAny<T> = 0 extends 1 & T ? true : false;
-/**
- * A request input (URL string, request-options object, or {@link Request}) whose `userData` is typed
- * according to its `label`, based on a router's route map.
- *
- * When the route map is open (the default `Record<string, ...>`), this is just the regular loose
- * {@link Source} input. When the map declares concrete labels, providing a `label` requires the matching
- * `userData` shape and rejects labels not present in the map; unlabeled requests keep loose `userData`.
- */
-export type LabeledSource<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | Source : string | Request | ({
-    requestsFromUrl?: string;
-    regex?: RegExp;
-} & ({
-    [Label in keyof Routes & string]: Omit<Partial<RequestOptions<Routes[Label]>>, 'label'> & {
-        label: Label;
-    };
-}[keyof Routes & string] | (Omit<Partial<RequestOptions>, 'label'> & {
-    label?: undefined;
-})));
-/**
- * The iterable/array of {@link LabeledSource} inputs accepted by the label-aware `addRequests`/`run`
- * methods of a crawler bound to a typed router.
- * @internal
- */
-export type TypedRequestsLike<Routes extends Record<keyof Routes, Dictionary>> = AsyncIterable<LabeledSource<Routes>> | Iterable<LabeledSource<Routes>> | LabeledSource<Routes>[];
-/**
- * The label-aware `addRequests` method signature exposed on a request handler's context when the crawler is
- * bound to a typed router. Mirrors {@link RestrictedCrawlingContext.addRequests} with typed sources.
- */
-export type TypedContextAddRequests<Routes extends Record<keyof Routes, Dictionary>> = (requestsLike: ReadonlyDeep<LabeledSource<Routes>[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
-/**
- * An `enqueueLinks`-options object with its `label`/`userData` retyped according to a router's route map: a
- * declared `label` requires the matching `userData` shape (unknown labels are rejected), while unlabeled
- * calls keep loose `userData`. Returns the options unchanged when the route map is open (the default).
- */
-type TypedEnqueueLinksOptions<Options, Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? Options : Omit<Options, 'label' | 'userData'> & ({
-    [Label in keyof Routes & string]: {
-        label: Label;
-        userData?: Routes[Label];
-    };
-}[keyof Routes & string] | {
-    label?: undefined;
-    userData?: Dictionary;
-});
-/**
- * Transforms a context's existing `enqueueLinks` method so that the `label`/`userData` in its options follow
- * the router's route map, while preserving everything else about the signature (argument optionality and
- * return type, which differ between crawler types).
- */
-export type TypedContextEnqueueLinks<EnqueueLinks, Routes extends Record<keyof Routes, Dictionary>> = EnqueueLinks extends (options?: infer Options) => infer Result ? (options?: TypedEnqueueLinksOptions<Options, Routes>) => Result : EnqueueLinks extends (options: infer Options) => infer Result ? (options: TypedEnqueueLinksOptions<Options, Routes>) => Result : EnqueueLinks;
-export type WithRequired<T, K extends keyof T> = T & {
-    [P in K]-?: T[P];
-};
-export type LoadedRequest<R extends Request> = WithRequired<R, 'id' | 'loadedUrl'>;
-/** @internal */
-export type LoadedContext<Context extends RestrictedCrawlingContext> = IsAny<Context> extends true ? Context : {
-    request: LoadedRequest<Context['request']>;
-} & Omit<Context, 'request'>;
-export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> {
-    id: string;
-    session: ISession;
-    /**
-     * An object with information about currently used proxy by the crawler
-     * and configured by the {@link ProxyConfiguration} class.
-     */
-    proxyInfo?: ProxyInfo;
-    /**
-     * The original {@link Request} object.
-     */
-    request: Request<UserData>;
-    /**
-     * This function allows you to push data to a {@link Dataset} specified by name, or the one currently used by the crawler.
-     *
-     * Shortcut for `crawler.pushData()`.
-     *
-     * @param [data] Data to be pushed to the default dataset.
-     */
-    pushData(data: ReadonlyDeep<Parameters<Dataset['pushData']>[0]>, datasetIdentifier?: string | StorageIdentifier): Promise<void>;
-    /**
-     * Add requests directly to the request queue currently used by the crawler.
-     *
-     * Optionally, the function allows you to filter the target URLs using an array of glob or regexp patterns,
-     * the same way {@link CrawlingContext.enqueueLinks|`enqueueLinks`} does for extracted links.
-     *
-     * @param requests The requests to add
-     * @param options Options for the request queue
-     */
-    addRequests: (requestsLike: ReadonlyDeep<(string | Source)[]>, options?: ReadonlyDeep<EnqueueUrlsOptions>) => Promise<AddRequestsBatchedResult>;
-    /**
-     * Returns the state - a piece of mutable persistent data shared across all the request handler runs.
-     */
-    useState: <State extends Dictionary = Dictionary>(defaultValue?: State) => Promise<State>;
-    /**
-     * Get a key-value store with given name or id, or the default one for the crawler.
-     */
-    getKeyValueStore: (identifier?: string | StorageIdentifier) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue' | 'getPublicUrl'>>;
-    /**
-     * A preconfigured logger for the request handler.
-     */
-    log: CrawleeLogger;
-}
-export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
-    /**
-     * Fires HTTP request via the internal HTTP client, allowing to override the request options on the fly.
-     *
-     * This is handy when you work with a browser crawler but want to execute some requests outside it (e.g. API requests).
-     * Check the [Skipping navigations for certain requests](https://crawlee.dev/js/docs/examples/skip-navigation) example for
-     * more detailed explanation of how to do that.
-     *
-     * ```ts
-     * async requestHandler({ sendRequest }) {
-     *     const { body } = await sendRequest({
-     *         // override headers only
-     *         headers: { ... },
-     *     });
-     * },
-     * ```
-     */
-    sendRequest: (requestOverrides?: Partial<HttpRequestOptions>, optionsOverrides?: SendRequestOptions) => Promise<Response>;
-    /**
-     * Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.
-     *
-     * The callback runs *outside* the request's storage transaction, so storage writes made here are
-     * applied immediately and are **not** rolled back when the request fails. In
-     * {@link AdaptivePlaywrightCrawler} it also runs once per request handler attempt, so a write
-     * here can land more than once for a single request. Push results from the request handler itself.
-     */
-    registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
-    /**
-     * Gives the current request `secs` more seconds to finish, for when how long it needs is only apparent
-     * once it is already running - a listing page that turns out to have far more to scroll through than
-     * usual, say. Prefer `requestHandlerTimeoutSecs`, or a per-route override via
-     * {@link Router.addHandler|`router.addHandler`}, whenever the time needed is known up front.
-     *
-     * ```ts
-     * router.addHandler('LIST', async ({ extendTimeout, page }) => {
-     *     const pageCount = await countPages(page);
-     *     extendTimeout(pageCount * 10);
-     *     await scrapeAllPages(page);
-     * });
-     * ```
-     *
-     * Extends the request handler's own timeout and the crawler's internal one together, so the extension
-     * is not immediately undone by the latter. Calling it from a handler that has already timed out does
-     * nothing.
-     */
-    extendTimeout(secs: number): void;
-}
-export {};

package/crawlers/error_snapshotter.d.ts

@@ -1,57 +0,0 @@
-import type { CrawlingContext } from '../crawlers/crawler_commons.js';
-import type { KeyValueStore } from '../storages/key_value_store.js';
-import type { ErrnoException } from './error_tracker.js';
-import type { SnapshottableProperties } from './internals/types.js';
-interface BrowserCrawlingContext {
-    saveSnapshot: (options: {
-        key: string;
-    }) => Promise<void>;
-}
-export interface SnapshotResult {
-    screenshotFileName?: string;
-    htmlFileName?: string;
-}
-interface ErrorSnapshot {
-    screenshotFileName?: string;
-    screenshotFileUrl?: string;
-    htmlFileName?: string;
-    htmlFileUrl?: string;
-}
-/**
- * ErrorSnapshotter class is used to capture a screenshot of the page and a snapshot of the HTML when an error occurs during web crawling.
- *
- * This functionality is opt-in, and can be enabled via the crawler options:
- *
- * ```ts
- * const crawler = new BasicCrawler({
- *   // ...
- *   statistics: new Statistics({ saveErrorSnapshots: true }),
- * });
- * ```
- */
-export declare class ErrorSnapshotter {
-    static readonly MAX_ERROR_CHARACTERS = 30;
-    static readonly MAX_HASH_LENGTH = 30;
-    static readonly MAX_FILENAME_LENGTH = 250;
-    static readonly BASE_MESSAGE = "An error occurred";
-    static readonly SNAPSHOT_PREFIX = "ERROR_SNAPSHOT";
-    /**
-     * Capture a snapshot of the error context.
-     */
-    captureSnapshot(error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<ErrorSnapshot>;
-    /**
-     * Captures a snapshot of the current page using the context.saveSnapshot function.
-     * This function is applicable for browser contexts only.
-     * Returns an object containing the filenames of the screenshot and HTML file.
-     */
-    contextCaptureSnapshot(context: BrowserCrawlingContext, fileName: string): Promise<SnapshotResult | undefined>;
-    /**
-     * Save the HTML snapshot of the page, and return the key it was stored under.
-     */
-    saveHTMLSnapshot(html: string, keyValueStore: Pick<KeyValueStore, 'setValue'>, fileName: string): Promise<string | undefined>;
-    /**
-     * Generate a unique fileName for each error snapshot.
-     */
-    generateFilename(error: ErrnoException): string;
-}
-export {};

package/crawlers/error_snapshotter.js

@@ -1,117 +0,0 @@
-import crypto from 'node:crypto';
-/**
- * ErrorSnapshotter class is used to capture a screenshot of the page and a snapshot of the HTML when an error occurs during web crawling.
- *
- * This functionality is opt-in, and can be enabled via the crawler options:
- *
- * ```ts
- * const crawler = new BasicCrawler({
- *   // ...
- *   statistics: new Statistics({ saveErrorSnapshots: true }),
- * });
- * ```
- */
-export class ErrorSnapshotter {
-    static MAX_ERROR_CHARACTERS = 30;
-    static MAX_HASH_LENGTH = 30;
-    static MAX_FILENAME_LENGTH = 250;
-    static BASE_MESSAGE = 'An error occurred';
-    static SNAPSHOT_PREFIX = 'ERROR_SNAPSHOT';
-    /**
-     * Capture a snapshot of the error context.
-     */
-    async captureSnapshot(error, context) {
-        try {
-            const page = context?.page;
-            const body = context?.body;
-            const keyValueStore = await context?.getKeyValueStore();
-            // If the key-value store is not available, or the body and page are not available, return empty filenames
-            if (!keyValueStore || (!body && !page)) {
-                return {};
-            }
-            const fileName = this.generateFilename(error);
-            let screenshotFileName;
-            let htmlFileName;
-            if (page) {
-                const capturedFiles = await this.contextCaptureSnapshot(context, fileName);
-                if (capturedFiles) {
-                    screenshotFileName = capturedFiles.screenshotFileName;
-                    htmlFileName = capturedFiles.htmlFileName;
-                }
-                // If the snapshot for browsers failed to capture the HTML, try to capture it from the page content
-                if (!htmlFileName) {
-                    const html = await page.content();
-                    htmlFileName = html ? await this.saveHTMLSnapshot(html, keyValueStore, fileName) : undefined;
-                }
-            }
-            else if (typeof body === 'string') {
-                // for non-browser contexts
-                htmlFileName = await this.saveHTMLSnapshot(body, keyValueStore, fileName);
-            }
-            return {
-                screenshotFileName,
-                screenshotFileUrl: screenshotFileName && (await keyValueStore.getPublicUrl(screenshotFileName)),
-                htmlFileName,
-                htmlFileUrl: htmlFileName && (await keyValueStore.getPublicUrl(htmlFileName)),
-            };
-        }
-        catch {
-            return {};
-        }
-    }
-    /**
-     * Captures a snapshot of the current page using the context.saveSnapshot function.
-     * This function is applicable for browser contexts only.
-     * Returns an object containing the filenames of the screenshot and HTML file.
-     */
-    async contextCaptureSnapshot(context, fileName) {
-        try {
-            await context.saveSnapshot({ key: fileName });
-            return {
-                screenshotFileName: `${fileName}.jpg`,
-                htmlFileName: `${fileName}.html`,
-            };
-        }
-        catch {
-            return undefined;
-        }
-    }
-    /**
-     * Save the HTML snapshot of the page, and return the key it was stored under.
-     */
-    async saveHTMLSnapshot(html, keyValueStore, fileName) {
-        try {
-            await keyValueStore.setValue(fileName, html, { contentType: 'text/html' });
-            // The record key is `fileName` - returning it with an `.html` suffix (as v3 did,
-            // where local storage put the extension in the key) would break `getPublicUrl`.
-            return fileName;
-        }
-        catch {
-            return undefined;
-        }
-    }
-    /**
-     * Generate a unique fileName for each error snapshot.
-     */
-    generateFilename(error) {
-        const { SNAPSHOT_PREFIX, BASE_MESSAGE, MAX_HASH_LENGTH, MAX_ERROR_CHARACTERS, MAX_FILENAME_LENGTH } = ErrorSnapshotter;
-        // Create a hash of the error stack trace
-        const errorStackHash = crypto
-            .createHash('sha1')
-            .update(error.stack || error.message || '')
-            .digest('hex')
-            .slice(0, MAX_HASH_LENGTH);
-        const errorMessagePrefix = (error.message || BASE_MESSAGE).slice(0, MAX_ERROR_CHARACTERS).trim();
-        /**
-         * Remove non-word characters from the start and end of a string.
-         */
-        const sanitizeString = (str) => {
-            return str.replace(/^\W+|\W+$/g, '');
-        };
-        // Generate fileName and remove disallowed characters
-        const fileName = `${SNAPSHOT_PREFIX}_${sanitizeString(errorStackHash)}_${sanitizeString(errorMessagePrefix)}`
-            .replace(/\W+/g, '-') // Replace non-word characters with a dash
-            .slice(0, MAX_FILENAME_LENGTH);
-        return fileName;
-    }
-}

package/crawlers/error_tracker.d.ts

@@ -1,54 +0,0 @@
-import type { CrawlingContext } from '../crawlers/crawler_commons.js';
-import { ErrorSnapshotter } from './error_snapshotter.js';
-import type { SnapshottableProperties } from './internals/types.js';
-/**
- * Node.js Error interface
- */
-export interface ErrnoException extends Error {
-    errno?: number;
-    code?: string | number;
-    path?: string;
-    syscall?: string;
-    cause?: any;
-}
-export interface ErrorTrackerOptions {
-    showErrorCode: boolean;
-    showErrorName: boolean;
-    showStackTrace: boolean;
-    showFullStack: boolean;
-    showErrorMessage: boolean;
-    showFullMessage: boolean;
-    saveErrorSnapshots: boolean;
-}
-/**
- * This class tracks errors and computes a summary of information like:
- * - where the errors happened
- * - what the error names are
- * - what the error codes are
- * - what is the general error message
- *
- * This is extremely useful when there are dynamic error messages, such as argument validation.
- *
- * Since the structure of the `tracker.result` object differs when using different options,
- * it's typed as `Record<string, unknown>`. The most deep object has a `count` property, which is a number.
- *
- * It's possible to get the total amount of errors via the `tracker.total` property.
- */
-export declare class ErrorTracker {
-    #private;
-    result: Record<string, unknown>;
-    total: number;
-    errorSnapshotter?: ErrorSnapshotter;
-    constructor(options?: Partial<ErrorTrackerOptions>);
-    private updateGroup;
-    add(error: ErrnoException): void;
-    /**
-     * This method is async, because it captures a snapshot of the error context.
-     * We added this new method to avoid breaking changes.
-     */
-    addAsync(error: ErrnoException, context?: CrawlingContext): Promise<void>;
-    getUniqueErrorCount(): number;
-    getMostPopularErrors(count: number): [number, string[]][];
-    captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<void>;
-    reset(): void;
-}