@crawlee/basic 4.0.0-beta.170 → 4.0.0-beta.172

package/internals/crawlers/error_snapshotter.js

@@ -0,0 +1,117 @@
+import crypto from 'node:crypto';
+/**
+ * ErrorSnapshotter class is used to capture a screenshot of the page and a snapshot of the HTML when an error occurs during web crawling.
+ *
+ * This functionality is opt-in, and can be enabled via the crawler options:
+ *
+ * ```ts
+ * const crawler = new BasicCrawler({
+ *   // ...
+ *   statistics: new Statistics({ saveErrorSnapshots: true }),
+ * });
+ * ```
+ */
+export class ErrorSnapshotter {
+    static MAX_ERROR_CHARACTERS = 30;
+    static MAX_HASH_LENGTH = 30;
+    static MAX_FILENAME_LENGTH = 250;
+    static BASE_MESSAGE = 'An error occurred';
+    static SNAPSHOT_PREFIX = 'ERROR_SNAPSHOT';
+    /**
+     * Capture a snapshot of the error context.
+     */
+    async captureSnapshot(error, context) {
+        try {
+            const page = context?.page;
+            const body = context?.body;
+            const keyValueStore = await context?.getKeyValueStore();
+            // If the key-value store is not available, or the body and page are not available, return empty filenames
+            if (!keyValueStore || (!body && !page)) {
+                return {};
+            }
+            const fileName = this.generateFilename(error);
+            let screenshotFileName;
+            let htmlFileName;
+            if (page) {
+                const capturedFiles = await this.contextCaptureSnapshot(context, fileName);
+                if (capturedFiles) {
+                    screenshotFileName = capturedFiles.screenshotFileName;
+                    htmlFileName = capturedFiles.htmlFileName;
+                }
+                // If the snapshot for browsers failed to capture the HTML, try to capture it from the page content
+                if (!htmlFileName) {
+                    const html = await page.content();
+                    htmlFileName = html ? await this.saveHTMLSnapshot(html, keyValueStore, fileName) : undefined;
+                }
+            }
+            else if (typeof body === 'string') {
+                // for non-browser contexts
+                htmlFileName = await this.saveHTMLSnapshot(body, keyValueStore, fileName);
+            }
+            return {
+                screenshotFileName,
+                screenshotFileUrl: screenshotFileName && (await keyValueStore.getPublicUrl(screenshotFileName)),
+                htmlFileName,
+                htmlFileUrl: htmlFileName && (await keyValueStore.getPublicUrl(htmlFileName)),
+            };
+        }
+        catch {
+            return {};
+        }
+    }
+    /**
+     * Captures a snapshot of the current page using the context.saveSnapshot function.
+     * This function is applicable for browser contexts only.
+     * Returns an object containing the filenames of the screenshot and HTML file.
+     */
+    async contextCaptureSnapshot(context, fileName) {
+        try {
+            await context.saveSnapshot({ key: fileName });
+            return {
+                screenshotFileName: `${fileName}.jpg`,
+                htmlFileName: `${fileName}.html`,
+            };
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /**
+     * Save the HTML snapshot of the page, and return the key it was stored under.
+     */
+    async saveHTMLSnapshot(html, keyValueStore, fileName) {
+        try {
+            await keyValueStore.setValue(fileName, html, { contentType: 'text/html' });
+            // The record key is `fileName` - returning it with an `.html` suffix (as v3 did,
+            // where local storage put the extension in the key) would break `getPublicUrl`.
+            return fileName;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /**
+     * Generate a unique fileName for each error snapshot.
+     */
+    generateFilename(error) {
+        const { SNAPSHOT_PREFIX, BASE_MESSAGE, MAX_HASH_LENGTH, MAX_ERROR_CHARACTERS, MAX_FILENAME_LENGTH } = ErrorSnapshotter;
+        // Create a hash of the error stack trace
+        const errorStackHash = crypto
+            .createHash('sha1')
+            .update(error.stack || error.message || '')
+            .digest('hex')
+            .slice(0, MAX_HASH_LENGTH);
+        const errorMessagePrefix = (error.message || BASE_MESSAGE).slice(0, MAX_ERROR_CHARACTERS).trim();
+        /**
+         * Remove non-word characters from the start and end of a string.
+         */
+        const sanitizeString = (str) => {
+            return str.replace(/^\W+|\W+$/g, '');
+        };
+        // Generate fileName and remove disallowed characters
+        const fileName = `${SNAPSHOT_PREFIX}_${sanitizeString(errorStackHash)}_${sanitizeString(errorMessagePrefix)}`
+            .replace(/\W+/g, '-') // Replace non-word characters with a dash
+            .slice(0, MAX_FILENAME_LENGTH);
+        return fileName;
+    }
+}

package/internals/crawlers/error_tracker.d.ts

@@ -0,0 +1,54 @@
+import type { CrawlingContext } from '../crawlers/crawler_commons.js';
+import { ErrorSnapshotter } from './error_snapshotter.js';
+import type { SnapshottableProperties } from './internals/types.js';
+/**
+ * Node.js Error interface
+ */
+export interface ErrnoException extends Error {
+    errno?: number;
+    code?: string | number;
+    path?: string;
+    syscall?: string;
+    cause?: any;
+}
+export interface ErrorTrackerOptions {
+    showErrorCode: boolean;
+    showErrorName: boolean;
+    showStackTrace: boolean;
+    showFullStack: boolean;
+    showErrorMessage: boolean;
+    showFullMessage: boolean;
+    saveErrorSnapshots: boolean;
+}
+/**
+ * This class tracks errors and computes a summary of information like:
+ * - where the errors happened
+ * - what the error names are
+ * - what the error codes are
+ * - what is the general error message
+ *
+ * This is extremely useful when there are dynamic error messages, such as argument validation.
+ *
+ * Since the structure of the `tracker.result` object differs when using different options,
+ * it's typed as `Record<string, unknown>`. The most deep object has a `count` property, which is a number.
+ *
+ * It's possible to get the total amount of errors via the `tracker.total` property.
+ */
+export declare class ErrorTracker {
+    #private;
+    result: Record<string, unknown>;
+    total: number;
+    errorSnapshotter?: ErrorSnapshotter;
+    constructor(options?: Partial<ErrorTrackerOptions>);
+    private updateGroup;
+    add(error: ErrnoException): void;
+    /**
+     * This method is async, because it captures a snapshot of the error context.
+     * We added this new method to avoid breaking changes.
+     */
+    addAsync(error: ErrnoException, context?: CrawlingContext): Promise<void>;
+    getUniqueErrorCount(): number;
+    getMostPopularErrors(count: number): [number, string[]][];
+    captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext & SnapshottableProperties): Promise<void>;
+    reset(): void;
+}

package/internals/crawlers/error_tracker.js

@@ -0,0 +1,308 @@
+import { inspect } from 'node:util';
+import { ErrorSnapshotter } from './error_snapshotter.js';
+const extractPathFromStackTraceLine = (line) => {
+    const lastStartingRoundBracketIndex = line.lastIndexOf('(');
+    if (lastStartingRoundBracketIndex !== -1) {
+        const closingRoundBracketIndex = line.indexOf(')', lastStartingRoundBracketIndex);
+        if (closingRoundBracketIndex !== -1) {
+            return line.slice(lastStartingRoundBracketIndex + 1, closingRoundBracketIndex);
+        }
+    }
+    return line;
+};
+// https://v8.dev/docs/stack-trace-api#appendix%3A-stack-trace-format
+const getPathFromStackTrace = (stack) => {
+    for (const line of stack) {
+        const path = extractPathFromStackTraceLine(line);
+        if (path.startsWith('node:') || path.includes('/node_modules/') || path.includes('\\node_modules\\')) {
+            continue;
+        }
+        return path;
+    }
+    return extractPathFromStackTraceLine(stack[0]);
+};
+const getStackTraceGroup = (error, storage, showFullStack) => {
+    const stack = error.stack?.split('\n').map((line) => line.trim());
+    let sliceAt = -1;
+    if (stack) {
+        for (let i = 0; i < stack.length; i++) {
+            if (stack[i].startsWith('at ') || stack[i].startsWith('eval at ')) {
+                sliceAt = i;
+                break;
+            }
+        }
+    }
+    let normalizedStackTrace = null;
+    if (sliceAt !== -1) {
+        normalizedStackTrace = showFullStack
+            ? stack
+                .slice(sliceAt)
+                .map((x) => x.trim())
+                .join('\n')
+            : getPathFromStackTrace(stack.slice(sliceAt));
+    }
+    if (!normalizedStackTrace) {
+        normalizedStackTrace = 'missing stack trace';
+    }
+    if (!(normalizedStackTrace in storage)) {
+        storage[normalizedStackTrace] = Object.create(null);
+    }
+    return storage[normalizedStackTrace];
+};
+const getErrorCodeGroup = (error, storage) => {
+    let { code } = error;
+    if (code === undefined) {
+        code = 'missing error code';
+    }
+    if (!(code in storage)) {
+        storage[code] = Object.create(null);
+    }
+    return storage[String(code)];
+};
+const getErrorNameGroup = (error, storage) => {
+    const { name } = error;
+    if (!(name in storage)) {
+        storage[name] = Object.create(null);
+    }
+    return storage[name];
+};
+const findBiggestWordIntersection = (a, b) => {
+    let maxStreak = 0;
+    let bStreakIndex = -1;
+    let aStreakIndex = -1;
+    for (let aIndex = 0; aIndex < a.length; aIndex++) {
+        let bIndex = -1;
+        do {
+            let aWalkIndex = aIndex;
+            bIndex = b.indexOf(a[aIndex], bIndex + 1);
+            let bWalkIndex = bIndex;
+            let streak = 0;
+            while (aWalkIndex < a.length && bWalkIndex < b.length && b[bWalkIndex++] === a[aWalkIndex++]) {
+                streak++;
+            }
+            if (streak > maxStreak) {
+                maxStreak = streak;
+                aStreakIndex = aIndex;
+                bStreakIndex = bIndex;
+            }
+        } while (bIndex !== -1);
+    }
+    return {
+        maxStreak,
+        aStreakIndex,
+        bStreakIndex,
+    };
+};
+const arrayCount = (array, target) => {
+    let result = 0;
+    for (const item of array) {
+        if (item === target) {
+            result++;
+        }
+    }
+    return result;
+};
+const calculatePlaceholder = (a, b) => {
+    const { maxStreak, aStreakIndex, bStreakIndex } = findBiggestWordIntersection(a, b);
+    if (maxStreak === 0) {
+        return ['_'];
+    }
+    const leftA = a.slice(0, aStreakIndex);
+    const leftB = b.slice(0, bStreakIndex);
+    const rightA = a.slice(aStreakIndex + maxStreak);
+    const rightB = b.slice(bStreakIndex + maxStreak);
+    const output = [];
+    if (leftA.length !== 0 || leftB.length !== 0) {
+        output.push(...calculatePlaceholder(leftA, leftB));
+    }
+    output.push(...a.slice(aStreakIndex, aStreakIndex + maxStreak));
+    if (rightA.length !== 0 || rightB.length !== 0) {
+        output.push(...calculatePlaceholder(rightA, rightB));
+    }
+    return output;
+};
+const normalizedCalculatePlaceholder = (a, b) => {
+    const output = calculatePlaceholder(a, b);
+    // We can't be too general
+    if (arrayCount(output, '_') / output.length >= 0.5) {
+        return ['_'];
+    }
+    return output;
+};
+// Merge A (missing placeholders) into B (can contain placeholders but does not have to)
+const mergeMessages = (a, b, storage) => {
+    const placeholder = normalizedCalculatePlaceholder(a.split(' '), b.split(' ')).join(' ');
+    if (placeholder === '_') {
+        return undefined;
+    }
+    const count = storage[a].count + storage[b].count;
+    delete storage[a];
+    delete storage[b];
+    storage[placeholder] = Object.assign(Object.create(null), {
+        count,
+    });
+    return placeholder;
+};
+const getErrorMessageGroup = (error, storage, showFullMessage) => {
+    let { message } = error;
+    if (!message) {
+        try {
+            message =
+                typeof error === 'string'
+                    ? error
+                    : `Unknown error message. Received non-error object: ${JSON.stringify(error)}`;
+        }
+        catch {
+            message = `Unknown error message. Received non-error object, and could not stringify it: ${inspect(error, {
+                depth: 0,
+            })}`;
+        }
+    }
+    if (!showFullMessage) {
+        const newLineIndex = message.indexOf('\n');
+        message = message.slice(0, newLineIndex === -1 ? undefined : newLineIndex);
+    }
+    if (!(message in storage)) {
+        storage[message] = Object.assign(Object.create(null), {
+            count: 0,
+        });
+        // This actually safe, since we Object.create(null) so no prototype pollution can happen.
+        // eslint-disable-next-line no-restricted-syntax, guard-for-in
+        for (const existingMessage in storage) {
+            const newMessage = mergeMessages(message, existingMessage, storage);
+            if (newMessage) {
+                message = newMessage;
+                break;
+            }
+        }
+    }
+    return storage[message];
+};
+const increaseCount = (group) => {
+    if (!('count' in group)) {
+        // In case users don't want to display error message
+        group.count = 0;
+    }
+    group.count++;
+};
+/**
+ * This class tracks errors and computes a summary of information like:
+ * - where the errors happened
+ * - what the error names are
+ * - what the error codes are
+ * - what is the general error message
+ *
+ * This is extremely useful when there are dynamic error messages, such as argument validation.
+ *
+ * Since the structure of the `tracker.result` object differs when using different options,
+ * it's typed as `Record<string, unknown>`. The most deep object has a `count` property, which is a number.
+ *
+ * It's possible to get the total amount of errors via the `tracker.total` property.
+ */
+export class ErrorTracker {
+    #options;
+    result;
+    total;
+    errorSnapshotter;
+    constructor(options = {}) {
+        this.#options = {
+            showErrorCode: true,
+            showErrorName: true,
+            showStackTrace: true,
+            showFullStack: false,
+            showErrorMessage: true,
+            showFullMessage: false,
+            saveErrorSnapshots: false,
+            ...options,
+        };
+        if (this.#options.saveErrorSnapshots) {
+            this.errorSnapshotter = new ErrorSnapshotter();
+        }
+        this.result = Object.create(null);
+        this.total = 0;
+    }
+    updateGroup(error) {
+        let group = this.result;
+        if (this.#options.showStackTrace) {
+            group = getStackTraceGroup(error, group, this.#options.showFullStack);
+        }
+        if (this.#options.showErrorCode) {
+            group = getErrorCodeGroup(error, group);
+        }
+        if (this.#options.showErrorName) {
+            group = getErrorNameGroup(error, group);
+        }
+        if (this.#options.showErrorMessage) {
+            group = getErrorMessageGroup(error, group, this.#options.showFullMessage);
+        }
+        increaseCount(group);
+        return group;
+    }
+    add(error) {
+        this.total++;
+        this.updateGroup(error);
+        if (typeof error.cause === 'object' && error.cause !== null) {
+            this.add(error.cause);
+        }
+    }
+    /**
+     * This method is async, because it captures a snapshot of the error context.
+     * We added this new method to avoid breaking changes.
+     */
+    async addAsync(error, context) {
+        this.total++;
+        const group = this.updateGroup(error);
+        // Capture a snapshot (screenshot and HTML) on the first occurrence of an error
+        if (group.count === 1 && context) {
+            await this.captureSnapshot(group, error, context).catch(() => { });
+        }
+        if (typeof error.cause === 'object' && error.cause !== null) {
+            await this.addAsync(error.cause);
+        }
+    }
+    getUniqueErrorCount() {
+        let count = 0;
+        const goDeeper = (group) => {
+            if ('count' in group) {
+                count++;
+                return;
+            }
+            // eslint-disable-next-line guard-for-in, no-restricted-syntax
+            for (const key in group) {
+                goDeeper(group[key]);
+            }
+        };
+        goDeeper(this.result);
+        return count;
+    }
+    getMostPopularErrors(count) {
+        const result = [];
+        const goDeeper = (group, path) => {
+            if ('count' in group) {
+                result.push([group.count, path]);
+                return;
+            }
+            // eslint-disable-next-line guard-for-in, no-restricted-syntax
+            for (const key in group) {
+                goDeeper(group[key], [...path, key]);
+            }
+        };
+        goDeeper(this.result, []);
+        return result.sort((a, b) => b[0] - a[0]).slice(0, count);
+    }
+    async captureSnapshot(storage, error, context) {
+        if (!this.errorSnapshotter) {
+            return;
+        }
+        const { screenshotFileUrl, htmlFileUrl } = await this.errorSnapshotter.captureSnapshot(error, context);
+        storage.firstErrorScreenshotUrl = screenshotFileUrl;
+        storage.firstErrorHtmlUrl = htmlFileUrl;
+    }
+    reset() {
+        // This actually safe, since we Object.create(null) so no prototype pollution can happen.
+        // eslint-disable-next-line no-restricted-syntax, guard-for-in
+        for (const key in this.result) {
+            delete this.result[key];
+        }
+    }
+}

package/internals/crawlers/index.d.ts

@@ -0,0 +1,5 @@
+export * from './context_pipeline.js';
+export type * from './crawler_commons.js';
+export * from './statistics.js';
+export * from './error_tracker.js';
+export * from './error_snapshotter.js';

package/internals/crawlers/index.js

@@ -0,0 +1,4 @@
+export * from './context_pipeline.js';
+export * from './statistics.js';
+export * from './error_tracker.js';
+export * from './error_snapshotter.js';

package/internals/crawlers/internals/types.d.ts

@@ -0,0 +1,7 @@
+export interface BrowserPage {
+    content: () => Promise<string>;
+}
+export interface SnapshottableProperties {
+    body?: unknown;
+    page?: BrowserPage;
+}

package/internals/crawlers/internals/types.js

@@ -0,0 +1 @@
+export {};