@crawlee/core 3.9.3-beta.47 → 3.9.3-beta.49

package/crawlers/error_snapshotter.d.ts

@@ -0,0 +1,56 @@
+import type { ErrnoException } from './error_tracker';
+import type { CrawlingContext } from '../crawlers/crawler_commons';
+import type { KeyValueStore } from '../storages';
+interface BrowserCrawlingContext {
+    saveSnapshot: (options: {
+        key: string;
+    }) => Promise<void>;
+}
+export interface SnapshotResult {
+    screenshotFileName?: string;
+    htmlFileName?: string;
+}
+/**
+ * ErrorSnapshotter class is used to capture a screenshot of the page and a snapshot of the HTML when an error occurs during web crawling.
+ *
+ * This functionality is opt-in, and can be enabled via the crawler options:
+ *
+ * ```ts
+ * const crawler = new BasicCrawler({
+ *   // ...
+ *   statisticsOptions: {
+ *     saveErrorSnapshots: true,
+ *   },
+ * });
+ * ```
+ */
+export declare class ErrorSnapshotter {
+    static readonly MAX_ERROR_CHARACTERS = 30;
+    static readonly MAX_HASH_LENGTH = 30;
+    static readonly MAX_FILENAME_LENGTH = 250;
+    static readonly BASE_MESSAGE = "An error occurred";
+    static readonly SNAPSHOT_PREFIX = "ERROR_SNAPSHOT";
+    /**
+     * Capture a snapshot of the error context.
+     */
+    captureSnapshot(error: ErrnoException, context: CrawlingContext): Promise<{
+        screenshotFileName?: string;
+        htmlFileName?: string;
+    }>;
+    /**
+     * Captures a snapshot of the current page using the context.saveSnapshot function.
+     * This function is applicable for browser contexts only.
+     * Returns an object containing the filenames of the screenshot and HTML file.
+     */
+    contextCaptureSnapshot(context: BrowserCrawlingContext, fileName: string): Promise<SnapshotResult | undefined>;
+    /**
+     * Save the HTML snapshot of the page, and return the fileName with the extension.
+     */
+    saveHTMLSnapshot(html: string, keyValueStore: KeyValueStore, fileName: string): Promise<string | undefined>;
+    /**
+     * Generate a unique fileName for each error snapshot.
+     */
+    generateFilename(error: ErrnoException): string;
+}
+export {};
+//# sourceMappingURL=error_snapshotter.d.ts.map

package/crawlers/error_snapshotter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"error_snapshotter.d.ts","sourceRoot":"","sources":["../../src/crawlers/error_snapshotter.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAGjD,UAAU,sBAAsB;IAC5B,YAAY,EAAE,CAAC,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7D;AAMD,MAAM,WAAW,cAAc;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,gBAAgB;IACzB,MAAM,CAAC,QAAQ,CAAC,oBAAoB,MAAM;IAC1C,MAAM,CAAC,QAAQ,CAAC,eAAe,MAAM;IACrC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,OAAO;IAC1C,MAAM,CAAC,QAAQ,CAAC,YAAY,uBAAuB;IACnD,MAAM,CAAC,QAAQ,CAAC,eAAe,oBAAoB;IAEnD;;OAEG;IACG,eAAe,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC;QAAE,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IA6CvI;;;;OAIG;IACG,sBAAsB,CAAC,OAAO,EAAE,sBAAsB,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC;IAYpH;;OAEG;IACG,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IASjH;;OAEG;IACH,gBAAgB,CAAC,KAAK,EAAE,cAAc,GAAG,MAAM;CAoBlD"}

package/crawlers/error_snapshotter.js

@@ -0,0 +1,141 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ErrorSnapshotter = void 0;
+const tslib_1 = require("tslib");
+const node_crypto_1 = tslib_1.__importDefault(require("node:crypto"));
+/**
+ * ErrorSnapshotter class is used to capture a screenshot of the page and a snapshot of the HTML when an error occurs during web crawling.
+ *
+ * This functionality is opt-in, and can be enabled via the crawler options:
+ *
+ * ```ts
+ * const crawler = new BasicCrawler({
+ *   // ...
+ *   statisticsOptions: {
+ *     saveErrorSnapshots: true,
+ *   },
+ * });
+ * ```
+ */
+class ErrorSnapshotter {
+    /**
+     * Capture a snapshot of the error context.
+     */
+    async captureSnapshot(error, context) {
+        try {
+            const page = context?.page;
+            const body = context?.body;
+            const keyValueStore = await context?.getKeyValueStore();
+            // If the key-value store is not available, or the body and page are not available, return empty filenames
+            if (!keyValueStore || (!body && !page)) {
+                return {};
+            }
+            const fileName = this.generateFilename(error);
+            let screenshotFileName;
+            let htmlFileName;
+            if (page) {
+                const capturedFiles = await this.contextCaptureSnapshot(context, fileName);
+                if (capturedFiles) {
+                    screenshotFileName = capturedFiles.screenshotFileName;
+                    htmlFileName = capturedFiles.htmlFileName;
+                }
+                // If the snapshot for browsers failed to capture the HTML, try to capture it from the page content
+                if (!htmlFileName) {
+                    const html = await page.content();
+                    htmlFileName = html ? await this.saveHTMLSnapshot(html, keyValueStore, fileName) : undefined;
+                }
+            }
+            else if (typeof body === 'string') { // for non-browser contexts
+                htmlFileName = await this.saveHTMLSnapshot(body, keyValueStore, fileName);
+            }
+            return {
+                screenshotFileName,
+                htmlFileName,
+            };
+        }
+        catch {
+            return {};
+        }
+    }
+    /**
+     * Captures a snapshot of the current page using the context.saveSnapshot function.
+     * This function is applicable for browser contexts only.
+     * Returns an object containing the filenames of the screenshot and HTML file.
+     */
+    async contextCaptureSnapshot(context, fileName) {
+        try {
+            await context.saveSnapshot({ key: fileName });
+            return {
+                screenshotFileName: `${fileName}.jpg`,
+                htmlFileName: `${fileName}.html`,
+            };
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /**
+     * Save the HTML snapshot of the page, and return the fileName with the extension.
+     */
+    async saveHTMLSnapshot(html, keyValueStore, fileName) {
+        try {
+            await keyValueStore.setValue(fileName, html, { contentType: 'text/html' });
+            return `${fileName}.html`;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /**
+     * Generate a unique fileName for each error snapshot.
+     */
+    generateFilename(error) {
+        const { SNAPSHOT_PREFIX, BASE_MESSAGE, MAX_HASH_LENGTH, MAX_ERROR_CHARACTERS, MAX_FILENAME_LENGTH } = ErrorSnapshotter;
+        // Create a hash of the error stack trace
+        const errorStackHash = node_crypto_1.default.createHash('sha1').update(error.stack || error.message || '').digest('hex').slice(0, MAX_HASH_LENGTH);
+        const errorMessagePrefix = (error.message || BASE_MESSAGE).slice(0, MAX_ERROR_CHARACTERS).trim();
+        /**
+         * Remove non-word characters from the start and end of a string.
+         */
+        const sanitizeString = (str) => {
+            return str.replace(/^\W+|\W+$/g, '');
+        };
+        // Generate fileName and remove disallowed characters
+        const fileName = `${SNAPSHOT_PREFIX}_${sanitizeString(errorStackHash)}_${sanitizeString(errorMessagePrefix)}`
+            .replace(/\W+/g, '-') // Replace non-word characters with a dash
+            .slice(0, MAX_FILENAME_LENGTH);
+        return fileName;
+    }
+}
+exports.ErrorSnapshotter = ErrorSnapshotter;
+Object.defineProperty(ErrorSnapshotter, "MAX_ERROR_CHARACTERS", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: 30
+});
+Object.defineProperty(ErrorSnapshotter, "MAX_HASH_LENGTH", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: 30
+});
+Object.defineProperty(ErrorSnapshotter, "MAX_FILENAME_LENGTH", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: 250
+});
+Object.defineProperty(ErrorSnapshotter, "BASE_MESSAGE", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: 'An error occurred'
+});
+Object.defineProperty(ErrorSnapshotter, "SNAPSHOT_PREFIX", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: 'ERROR_SNAPSHOT'
+});
+//# sourceMappingURL=error_snapshotter.js.map

package/crawlers/error_snapshotter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"error_snapshotter.js","sourceRoot":"","sources":["../../src/crawlers/error_snapshotter.ts"],"names":[],"mappings":";;;;AAAA,sEAAiC;AAoBjC;;;;;;;;;;;;;GAaG;AACH,MAAa,gBAAgB;IAOzB;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,KAAqB,EAAE,OAAwB;QACjE,IAAI,CAAC;YACD,MAAM,IAAI,GAAG,OAAO,EAAE,IAA+B,CAAC;YACtD,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,CAAC;YAE3B,MAAM,aAAa,GAAG,MAAM,OAAO,EAAE,gBAAgB,EAAE,CAAC;YACxD,0GAA0G;YAC1G,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrC,OAAO,EAAE,CAAC;YACd,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAE9C,IAAI,kBAAsC,CAAC;YAC3C,IAAI,YAAgC,CAAC;YAErC,IAAI,IAAI,EAAE,CAAC;gBACP,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,sBAAsB,CACnD,OAA4C,EAC5C,QAAQ,CACX,CAAC;gBAEF,IAAI,aAAa,EAAE,CAAC;oBAChB,kBAAkB,GAAG,aAAa,CAAC,kBAAkB,CAAC;oBACtD,YAAY,GAAG,aAAa,CAAC,YAAY,CAAC;gBAC9C,CAAC;gBAED,mGAAmG;gBACnG,IAAI,CAAC,YAAY,EAAE,CAAC;oBAChB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;oBAClC,YAAY,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACjG,CAAC;YACL,CAAC;iBAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC,CAAC,2BAA2B;gBAC9D,YAAY,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;YAC9E,CAAC;YAED,OAAO;gBACH,kBAAkB;gBAClB,YAAY;aACf,CAAC;QACN,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,sBAAsB,CAAC,OAA+B,EAAE,QAAgB;QAC1E,IAAI,CAAC;YACD,MAAM,OAAO,CAAC,YAAY,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC9C,OAAO;gBACH,kBAAkB,EAAE,GAAG,QAAQ,MAAM;gBACrC,YAAY,EAAE,GAAG,QAAQ,OAAO;aACnC,CAAC;QACN,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,SAAS,CAAC;QACrB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAC,IAAY,EAAE,aAA4B,EAAE,QAAgB;QAC/E,IAAI,CAAC;YACD,MAAM,aAAa,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC;YAC3E,OAAO,GAAG,QAAQ,OAAO,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,SAAS,CAAC;QACrB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,KAAqB;QAClC,MAAM,EAAE,eAAe,EAAE,YAAY,EAAE,eAAe,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,GAAG,gBAAgB,CAAC;QACvH,yCAAyC;QACzC,MAAM,cAAc,GAAG,qBAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;QACpI,MAAM,kBAAkB,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC,IAAI,EAAE,CAAC;QAEjG;;WAEG;QACH,MAAM,cAAc,GAAG,CAAC,GAAW,EAAU,EAAE;YAC3C,OAAO,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QACzC,CAAC,CAAC;QAEF,qDAAqD;QACrD,MAAM,QAAQ,GAAG,GAAG,eAAe,IAAI,cAAc,CAAC,cAAc,CAAC,IAAI,cAAc,CAAC,kBAAkB,CAAC,EAAE;aACxG,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,0CAA0C;aAC/D,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;QAEnC,OAAO,QAAQ,CAAC;IACpB,CAAC;;AA1GL,4CA2GC;AA1GmB;;;;WAAuB,EAAE;GAAC;AAC1B;;;;WAAkB,EAAE;GAAC;AACrB;;;;WAAsB,GAAG;GAAC;AAC1B;;;;WAAe,mBAAmB;GAAC;AACnC;;;;WAAkB,gBAAgB;GAAC"}

package/crawlers/error_tracker.d.ts

@@ -0,0 +1,54 @@
+import { ErrorSnapshotter } from './error_snapshotter';
+import type { CrawlingContext } from '../crawlers/crawler_commons';
+/**
+ * Node.js Error interface
+ */
+export interface ErrnoException extends Error {
+    errno?: number;
+    code?: string | number;
+    path?: string;
+    syscall?: string;
+    cause?: any;
+}
+export interface ErrorTrackerOptions {
+    showErrorCode: boolean;
+    showErrorName: boolean;
+    showStackTrace: boolean;
+    showFullStack: boolean;
+    showErrorMessage: boolean;
+    showFullMessage: boolean;
+    saveErrorSnapshots: boolean;
+}
+/**
+ * This class tracks errors and computes a summary of information like:
+ * - where the errors happened
+ * - what the error names are
+ * - what the error codes are
+ * - what is the general error message
+ *
+ * This is extremely useful when there are dynamic error messages, such as argument validation.
+ *
+ * Since the structure of the `tracker.result` object differs when using different options,
+ * it's typed as `Record<string, unknown>`. The most deep object has a `count` property, which is a number.
+ *
+ * It's possible to get the total amount of errors via the `tracker.total` property.
+ */
+export declare class ErrorTracker {
+    #private;
+    result: Record<string, unknown>;
+    total: number;
+    errorSnapshotter?: ErrorSnapshotter;
+    constructor(options?: Partial<ErrorTrackerOptions>);
+    private updateGroup;
+    add(error: ErrnoException): void;
+    /**
+     * This method is async, because it captures a snapshot of the error context.
+     * We added this new method to avoid breaking changes.
+     */
+    addAsync(error: ErrnoException, context?: CrawlingContext): Promise<void>;
+    getUniqueErrorCount(): number;
+    getMostPopularErrors(count: number): [number, string[]][];
+    captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext): Promise<void>;
+    reset(): void;
+}
+//# sourceMappingURL=error_tracker.d.ts.map

package/crawlers/error_tracker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"error_tracker.d.ts","sourceRoot":"","sources":["../../src/crawlers/error_tracker.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAEnE;;GAEG;AACH,MAAM,WAAW,cAAe,SAAQ,KAAK;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,GAAG,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IAChC,aAAa,EAAE,OAAO,CAAC;IACvB,aAAa,EAAE,OAAO,CAAC;IACvB,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,eAAe,EAAE,OAAO,CAAC;IACzB,kBAAkB,EAAE,OAAO,CAAC;CAC/B;AAoPD;;;;;;;;;;;;;GAaG;AACH,qBAAa,YAAY;;IAGrB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEhC,KAAK,EAAE,MAAM,CAAC;IAEd,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;gBAExB,OAAO,GAAE,OAAO,CAAC,mBAAmB,CAAM;IAoBtD,OAAO,CAAC,WAAW;IAwBnB,GAAG,CAAC,KAAK,EAAE,cAAc;IAUzB;;;OAGG;IACG,QAAQ,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,eAAe;IAe/D,mBAAmB;IAoBnB,oBAAoB,CAAC,KAAK,EAAE,MAAM;IAoB5B,eAAe,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,eAAe;IAWvG,KAAK;CAOR"}

package/crawlers/error_tracker.js

@@ -0,0 +1,323 @@
+"use strict";
+var _ErrorTracker_options;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ErrorTracker = void 0;
+const tslib_1 = require("tslib");
+const node_util_1 = require("node:util");
+const error_snapshotter_1 = require("./error_snapshotter");
+const extractPathFromStackTraceLine = (line) => {
+    const lastStartingRoundBracketIndex = line.lastIndexOf('(');
+    if (lastStartingRoundBracketIndex !== -1) {
+        const closingRoundBracketIndex = line.indexOf(')', lastStartingRoundBracketIndex);
+        if (closingRoundBracketIndex !== -1) {
+            return line.slice(lastStartingRoundBracketIndex + 1, closingRoundBracketIndex);
+        }
+    }
+    return line;
+};
+// https://v8.dev/docs/stack-trace-api#appendix%3A-stack-trace-format
+const getPathFromStackTrace = (stack) => {
+    for (const line of stack) {
+        const path = extractPathFromStackTraceLine(line);
+        if (path.startsWith('node:')
+            || path.includes('/node_modules/')
+            || path.includes('\\node_modules\\')) {
+            continue;
+        }
+        return path;
+    }
+    return extractPathFromStackTraceLine(stack[0]);
+};
+const getStackTraceGroup = (error, storage, showFullStack) => {
+    const stack = error.stack?.split('\n').map((line) => line.trim());
+    let sliceAt = -1;
+    if (stack) {
+        for (let i = 0; i < stack.length; i++) {
+            if (stack[i].startsWith('at ') || stack[i].startsWith('eval at ')) {
+                sliceAt = i;
+                break;
+            }
+        }
+    }
+    let normalizedStackTrace = null;
+    if (sliceAt !== -1) {
+        normalizedStackTrace = showFullStack ? stack.slice(sliceAt).map((x) => x.trim()).join('\n') : getPathFromStackTrace(stack.slice(sliceAt));
+    }
+    if (!normalizedStackTrace) {
+        normalizedStackTrace = 'missing stack trace';
+    }
+    if (!(normalizedStackTrace in storage)) {
+        storage[normalizedStackTrace] = Object.create(null);
+    }
+    return storage[normalizedStackTrace];
+};
+const getErrorCodeGroup = (error, storage) => {
+    let { code } = error;
+    if (code === undefined) {
+        code = 'missing error code';
+    }
+    if (!(code in storage)) {
+        storage[code] = Object.create(null);
+    }
+    return storage[String(code)];
+};
+const getErrorNameGroup = (error, storage) => {
+    const { name } = error;
+    if (!(name in storage)) {
+        storage[name] = Object.create(null);
+    }
+    return storage[name];
+};
+const findBiggestWordIntersection = (a, b) => {
+    let maxStreak = 0;
+    let bStreakIndex = -1;
+    let aStreakIndex = -1;
+    for (let aIndex = 0; aIndex < a.length; aIndex++) {
+        let bIndex = -1;
+        do {
+            let aWalkIndex = aIndex;
+            bIndex = b.indexOf(a[aIndex], bIndex + 1);
+            let bWalkIndex = bIndex;
+            let streak = 0;
+            while (aWalkIndex < a.length && bWalkIndex < b.length && b[bWalkIndex++] === a[aWalkIndex++]) {
+                streak++;
+            }
+            if (streak > maxStreak) {
+                maxStreak = streak;
+                aStreakIndex = aIndex;
+                bStreakIndex = bIndex;
+            }
+        } while (bIndex !== -1);
+    }
+    return {
+        maxStreak,
+        aStreakIndex,
+        bStreakIndex,
+    };
+};
+const arrayCount = (array, target) => {
+    let result = 0;
+    for (const item of array) {
+        if (item === target) {
+            result++;
+        }
+    }
+    return result;
+};
+const calculatePlaceholder = (a, b) => {
+    const { maxStreak, aStreakIndex, bStreakIndex } = findBiggestWordIntersection(a, b);
+    if (maxStreak === 0) {
+        return ['_'];
+    }
+    const leftA = a.slice(0, aStreakIndex);
+    const leftB = b.slice(0, bStreakIndex);
+    const rightA = a.slice(aStreakIndex + maxStreak);
+    const rightB = b.slice(bStreakIndex + maxStreak);
+    const output = [];
+    if (leftA.length !== 0 || leftB.length !== 0) {
+        output.push(...calculatePlaceholder(leftA, leftB));
+    }
+    output.push(...a.slice(aStreakIndex, aStreakIndex + maxStreak));
+    if (rightA.length !== 0 || rightB.length !== 0) {
+        output.push(...calculatePlaceholder(rightA, rightB));
+    }
+    return output;
+};
+const normalizedCalculatePlaceholder = (a, b) => {
+    const output = calculatePlaceholder(a, b);
+    // We can't be too general
+    if ((arrayCount(output, '_') / output.length) >= 0.5) {
+        return ['_'];
+    }
+    return output;
+};
+// Merge A (missing placeholders) into B (can contain placeholders but does not have to)
+const mergeMessages = (a, b, storage) => {
+    const placeholder = normalizedCalculatePlaceholder(a.split(' '), b.split(' ')).join(' ');
+    if (placeholder === '_') {
+        return undefined;
+    }
+    const count = storage[a].count + storage[b].count;
+    delete storage[a];
+    delete storage[b];
+    storage[placeholder] = Object.assign(Object.create(null), {
+        count,
+    });
+    return placeholder;
+};
+const getErrorMessageGroup = (error, storage, showFullMessage) => {
+    let { message } = error;
+    if (!message) {
+        try {
+            message = typeof error === 'string' ? error : `Unknown error message. Received non-error object: ${JSON.stringify(error)}`;
+        }
+        catch {
+            message = `Unknown error message. Received non-error object, and could not stringify it: ${(0, node_util_1.inspect)(error, { depth: 0 })}`;
+        }
+    }
+    if (!showFullMessage) {
+        const newLineIndex = message.indexOf('\n');
+        message = message.slice(0, newLineIndex === -1 ? undefined : newLineIndex);
+    }
+    if (!(message in storage)) {
+        storage[message] = Object.assign(Object.create(null), {
+            count: 0,
+        });
+        // This actually safe, since we Object.create(null) so no prototype pollution can happen.
+        // eslint-disable-next-line no-restricted-syntax, guard-for-in
+        for (const existingMessage in storage) {
+            const newMessage = mergeMessages(message, existingMessage, storage);
+            if (newMessage) {
+                message = newMessage;
+                break;
+            }
+        }
+    }
+    return storage[message];
+};
+const increaseCount = (group) => {
+    if (!('count' in group)) {
+        // In case users don't want to display error message
+        group.count = 0;
+    }
+    group.count++;
+};
+/**
+ * This class tracks errors and computes a summary of information like:
+ * - where the errors happened
+ * - what the error names are
+ * - what the error codes are
+ * - what is the general error message
+ *
+ * This is extremely useful when there are dynamic error messages, such as argument validation.
+ *
+ * Since the structure of the `tracker.result` object differs when using different options,
+ * it's typed as `Record<string, unknown>`. The most deep object has a `count` property, which is a number.
+ *
+ * It's possible to get the total amount of errors via the `tracker.total` property.
+ */
+class ErrorTracker {
+    constructor(options = {}) {
+        _ErrorTracker_options.set(this, void 0);
+        Object.defineProperty(this, "result", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "total", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "errorSnapshotter", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        tslib_1.__classPrivateFieldSet(this, _ErrorTracker_options, {
+            showErrorCode: true,
+            showErrorName: true,
+            showStackTrace: true,
+            showFullStack: false,
+            showErrorMessage: true,
+            showFullMessage: false,
+            saveErrorSnapshots: false,
+            ...options,
+        }, "f");
+        if (tslib_1.__classPrivateFieldGet(this, _ErrorTracker_options, "f").saveErrorSnapshots) {
+            this.errorSnapshotter = new error_snapshotter_1.ErrorSnapshotter();
+        }
+        this.result = Object.create(null);
+        this.total = 0;
+    }
+    updateGroup(error) {
+        let group = this.result;
+        if (tslib_1.__classPrivateFieldGet(this, _ErrorTracker_options, "f").showStackTrace) {
+            group = getStackTraceGroup(error, group, tslib_1.__classPrivateFieldGet(this, _ErrorTracker_options, "f").showFullStack);
+        }
+        if (tslib_1.__classPrivateFieldGet(this, _ErrorTracker_options, "f").showErrorCode) {
+            group = getErrorCodeGroup(error, group);
+        }
+        if (tslib_1.__classPrivateFieldGet(this, _ErrorTracker_options, "f").showErrorName) {
+            group = getErrorNameGroup(error, group);
+        }
+        if (tslib_1.__classPrivateFieldGet(this, _ErrorTracker_options, "f").showErrorMessage) {
+            group = getErrorMessageGroup(error, group, tslib_1.__classPrivateFieldGet(this, _ErrorTracker_options, "f").showFullMessage);
+        }
+        increaseCount(group);
+        return group;
+    }
+    add(error) {
+        this.total++;
+        this.updateGroup(error);
+        if (typeof error.cause === 'object' && error.cause !== null) {
+            this.add(error.cause);
+        }
+    }
+    /**
+     * This method is async, because it captures a snapshot of the error context.
+     * We added this new method to avoid breaking changes.
+     */
+    async addAsync(error, context) {
+        this.total++;
+        const group = this.updateGroup(error);
+        // Capture a snapshot (screenshot and HTML) on the first occurrence of an error
+        if (group.count === 1 && context) {
+            await this.captureSnapshot(group, error, context).catch(() => { });
+        }
+        if (typeof error.cause === 'object' && error.cause !== null) {
+            await this.addAsync(error.cause);
+        }
+    }
+    getUniqueErrorCount() {
+        let count = 0;
+        const goDeeper = (group) => {
+            if ('count' in group) {
+                count++;
+                return;
+            }
+            // eslint-disable-next-line guard-for-in, no-restricted-syntax
+            for (const key in group) {
+                goDeeper(group[key]);
+            }
+        };
+        goDeeper(this.result);
+        return count;
+    }
+    getMostPopularErrors(count) {
+        const result = [];
+        const goDeeper = (group, path) => {
+            if ('count' in group) {
+                result.push([group.count, path]);
+                return;
+            }
+            // eslint-disable-next-line guard-for-in, no-restricted-syntax
+            for (const key in group) {
+                goDeeper(group[key], [...path, key]);
+            }
+        };
+        goDeeper(this.result, []);
+        return result.sort((a, b) => b[0] - a[0]).slice(0, count);
+    }
+    async captureSnapshot(storage, error, context) {
+        if (!this.errorSnapshotter) {
+            return;
+        }
+        const { screenshotFileName, htmlFileName } = await this.errorSnapshotter.captureSnapshot(error, context);
+        storage.firstErrorScreenshot = screenshotFileName;
+        storage.firstErrorHtml = htmlFileName;
+    }
+    reset() {
+        // This actually safe, since we Object.create(null) so no prototype pollution can happen.
+        // eslint-disable-next-line no-restricted-syntax, guard-for-in
+        for (const key in this.result) {
+            delete this.result[key];
+        }
+    }
+}
+exports.ErrorTracker = ErrorTracker;
+_ErrorTracker_options = new WeakMap();
+//# sourceMappingURL=error_tracker.js.map

package/crawlers/error_tracker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"error_tracker.js","sourceRoot":"","sources":["../../src/crawlers/error_tracker.ts"],"names":[],"mappings":";;;;;AAAA,yCAAoC;AAEpC,2DAAuD;AAwBvD,MAAM,6BAA6B,GAAG,CAAC,IAAY,EAAE,EAAE;IACnD,MAAM,6BAA6B,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAE5D,IAAI,6BAA6B,KAAK,CAAC,CAAC,EAAE,CAAC;QACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,6BAA6B,CAAC,CAAC;QAElF,IAAI,wBAAwB,KAAK,CAAC,CAAC,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC,KAAK,CAAC,6BAA6B,GAAG,CAAC,EAAE,wBAAwB,CAAC,CAAC;QACnF,CAAC;IACL,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AAEF,qEAAqE;AACrE,MAAM,qBAAqB,GAAG,CAAC,KAAe,EAAE,EAAE;IAC9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,6BAA6B,CAAC,IAAI,CAAC,CAAC;QAEjD,IACI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;eACrB,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC;eAC/B,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EACtC,CAAC;YACC,SAAS;QACb,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACnD,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,CAAC,KAAqB,EAAE,OAAgC,EAAE,aAAsB,EAAE,EAAE;IAC3G,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAElE,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;IAEjB,IAAI,KAAK,EAAE,CAAC;QACR,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBAChE,OAAO,GAAG,CAAC,CAAC;gBACZ,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,IAAI,oBAAoB,GAAG,IAAI,CAAC;IAChC,IAAI,OAAO,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,oBAAoB,GAAG,aAAa,CAAC,CAAC,CAAC,KAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,KAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IAChJ,CAAC;IAED,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACxB,oBAAoB,GAAG,qBAAqB,CAAC;IACjD,CAAC;IAED,IAAI,CAAC,CAAC,oBAAoB,IAAI,OAAO,CAAC,EAAE,CAAC;QACrC,OAAO,CAAC,oBAAoB,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxD,CAAC;IAED,OAAO,OAAO,CAAC,oBAAoB,CAA4B,CAAC;AACpE,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,KAAqB,EAAE,OAAgC,EAAE,EAAE;IAClF,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAErB,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACrB,IAAI,GAAG,oBAAoB,CAAC;IAChC,CAAC;IAED,IAAI,CAAC,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAA4B,CAAC;AAC5D,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,KAAqB,EAAE,OAAgC,EAAE,EAAE;IAClF,MAAM,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAEvB,IAAI,CAAC,CAAC,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,OAAO,CAAC,IAAI,CAA4B,CAAC;AACpD,CAAC,CAAC;AAEF,MAAM,2BAA2B,GAAG,CAAC,CAAW,EAAE,CAAW,EAAE,EAAE;IAC7D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IACtB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QAC/C,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC;QAEhB,GAAG,CAAC;YACA,IAAI,UAAU,GAAG,MAAM,CAAC;YAExB,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;YAE1C,IAAI,UAAU,GAAG,MAAM,CAAC;YAExB,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,OAAO,UAAU,GAAG,CAAC,CAAC,MAAM,IAAI,UAAU,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC,EAAE,CAAC;gBAC3F,MAAM,EAAE,CAAC;YACb,CAAC;YAED,IAAI,MAAM,GAAG,SAAS,EAAE,CAAC;gBACrB,SAAS,GAAG,MAAM,CAAC;gBACnB,YAAY,GAAG,MAAM,CAAC;gBACtB,YAAY,GAAG,MAAM,CAAC;YAC1B,CAAC;QACL,CAAC,QAAQ,MAAM,KAAK,CAAC,CAAC,EAAE;IAC5B,CAAC;IAED,OAAO;QACH,SAAS;QACT,YAAY;QACZ,YAAY;KACf,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,KAAgB,EAAE,MAAe,EAAE,EAAE;IACrD,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YAClB,MAAM,EAAE,CAAC;QACb,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAW,EAAE,CAAW,EAAE,EAAE;IACtD,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,YAAY,EAAE,GAAG,2BAA2B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEpF,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,YAAY,GAAG,SAAS,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,YAAY,GAAG,SAAS,CAAC,CAAC;IAEjD,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3C,MAAM,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,YAAY,EAAE,YAAY,GAAG,SAAS,CAAC,CAAC,CAAC;IAEhE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7C,MAAM,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,8BAA8B,GAAG,CAAC,CAAW,EAAE,CAAW,EAAE,EAAE;IAChE,MAAM,MAAM,GAAG,oBAAoB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAE1C,0BAA0B;IAC1B,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,wFAAwF;AACxF,MAAM,aAAa,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,OAAgC,EAAE,EAAE;IAC7E,MAAM,WAAW,GAAG,8BAA8B,CAC9C,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EACZ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CACf,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEZ,IAAI,WAAW,KAAK,GAAG,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC;IACrB,CAAC;IAMD,MAAM,KAAK,GAAI,OAAO,CAAC,CAAC,CAAc,CAAC,KAAK,GAAI,OAAO,CAAC,CAAC,CAAc,CAAC,KAAK,CAAC;IAE9E,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IAClB,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IAElB,OAAO,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QACtD,KAAK;KACR,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC;AACvB,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,KAAqB,EAAE,OAAgC,EAAE,eAAwB,EAAE,EAAE;IAC/G,IAAI,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IAExB,IAAI,CAAC,OAAO,EAAE,CAAC;QACX,IAAI,CAAC;YACD,OAAO,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,qDAAqD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/H,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,GAAG,iFAAiF,IAAA,mBAAO,EAAC,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;QAC9H,CAAC;IACL,CAAC;IAED,IAAI,CAAC,eAAe,EAAE,CAAC;QACnB,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC3C,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IAC/E,CAAC;IAED,IAAI,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;YAClD,KAAK,EAAE,CAAC;SACX,CAAC,CAAC;QAEH,yFAAyF;QACzF,8DAA8D;QAC9D,KAAK,MAAM,eAAe,IAAI,OAAO,EAAE,CAAC;YACpC,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC;YACpE,IAAI,UAAU,EAAE,CAAC;gBACb,OAAO,GAAG,UAAU,CAAC;gBACrB,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC,OAAO,CAA4B,CAAC;AACvD,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,KAAyB,EAAE,EAAE;IAChD,IAAI,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,EAAE,CAAC;QACtB,oDAAoD;QACpD,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,KAAM,EAAE,CAAC;AACnB,CAAC,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,MAAa,YAAY;IASrB,YAAY,UAAwC,EAAE;QARtD,wCAA8B;QAE9B;;;;;WAAgC;QAEhC;;;;;WAAc;QAEd;;;;;WAAoC;QAGhC,+BAAA,IAAI,yBAAY;YACZ,aAAa,EAAE,IAAI;YACnB,aAAa,EAAE,IAAI;YACnB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,KAAK;YACpB,gBAAgB,EAAE,IAAI;YACtB,eAAe,EAAE,KAAK;YACtB,kBAAkB,EAAE,KAAK;YACzB,GAAG,OAAO;SACb,MAAA,CAAC;QAEF,IAAI,+BAAA,IAAI,6BAAS,CAAC,kBAAkB,EAAE,CAAC;YACnC,IAAI,CAAC,gBAAgB,GAAG,IAAI,oCAAgB,EAAE,CAAC;QACnD,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;IACnB,CAAC;IAEO,WAAW,CAAC,KAAqB;QACrC,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAExB,IAAI,+BAAA,IAAI,6BAAS,CAAC,cAAc,EAAE,CAAC;YAC/B,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,KAAK,EAAE,+BAAA,IAAI,6BAAS,CAAC,aAAa,CAAC,CAAC;QAC1E,CAAC;QAED,IAAI,+BAAA,IAAI,6BAAS,CAAC,aAAa,EAAE,CAAC;YAC9B,KAAK,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,+BAAA,IAAI,6BAAS,CAAC,aAAa,EAAE,CAAC;YAC9B,KAAK,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,+BAAA,IAAI,6BAAS,CAAC,gBAAgB,EAAE,CAAC;YACjC,KAAK,GAAG,oBAAoB,CAAC,KAAK,EAAE,KAAK,EAAE,+BAAA,IAAI,6BAAS,CAAC,eAAe,CAAC,CAAC;QAC9E,CAAC;QAED,aAAa,CAAC,KAA0B,CAAC,CAAC;QAE1C,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,GAAG,CAAC,KAAqB;QACrB,IAAI,CAAC,KAAK,EAAE,CAAC;QAEb,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAExB,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC1D,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ,CAAC,KAAqB,EAAE,OAAyB;QAC3D,IAAI,CAAC,KAAK,EAAE,CAAC;QAEb,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAEtC,+EAA+E;QAC/E,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QACvE,CAAC;QAED,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC1D,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;IACL,CAAC;IAED,mBAAmB;QACf,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,MAAM,QAAQ,GAAG,CAAC,KAA8B,EAAQ,EAAE;YACtD,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;gBACnB,KAAK,EAAE,CAAC;gBACR,OAAO;YACX,CAAC;YAED,8DAA8D;YAC9D,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,QAAQ,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC,CAAC;YACpD,CAAC;QACL,CAAC,CAAC;QAEF,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEtB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,oBAAoB,CAAC,KAAa;QAC9B,MAAM,MAAM,GAAyB,EAAE,CAAC;QAExC,MAAM,QAAQ,GAAG,CAAC,KAA8B,EAAE,IAAc,EAAQ,EAAE;YACtE,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;gBACnB,MAAM,CAAC,IAAI,CAAC,CAAE,KAAa,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;gBAC1C,OAAO;YACX,CAAC;YAED,8DAA8D;YAC9D,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACtB,QAAQ,CAAC,KAAK,CAAC,GAAG,CAA4B,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;YACpE,CAAC;QACL,CAAC,CAAC;QAEF,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAE1B,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAgC,EAAE,KAAqB,EAAE,OAAwB;QACnG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACzB,OAAO;QACX,CAAC;QAED,MAAM,EAAE,kBAAkB,EAAE,YAAY,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAEzG,OAAO,CAAC,oBAAoB,GAAG,kBAAkB,CAAC;QAClD,OAAO,CAAC,cAAc,GAAG,YAAY,CAAC;IAC1C,CAAC;IAED,KAAK;QACD,yFAAyF;QACzF,8DAA8D;QAC9D,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;CACJ;AA5ID,oCA4IC"}

package/crawlers/index.d.ts

@@ -2,4 +2,6 @@ export * from './crawler_commons';
 export * from './crawler_extension';
 export * from './crawler_utils';
 export * from './statistics';
+export * from './error_tracker';
+export * from './error_snapshotter';
 //# sourceMappingURL=index.d.ts.map

package/crawlers/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/crawlers/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/crawlers/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC"}

package/crawlers/index.js

@@ -5,4 +5,6 @@ tslib_1.__exportStar(require("./crawler_commons"), exports);
 tslib_1.__exportStar(require("./crawler_extension"), exports);
 tslib_1.__exportStar(require("./crawler_utils"), exports);
 tslib_1.__exportStar(require("./statistics"), exports);
+tslib_1.__exportStar(require("./error_tracker"), exports);
+tslib_1.__exportStar(require("./error_snapshotter"), exports);
 //# sourceMappingURL=index.js.map

package/crawlers/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/crawlers/index.ts"],"names":[],"mappings":";;;AAAA,4DAAkC;AAClC,8DAAoC;AACpC,0DAAgC;AAChC,uDAA6B"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/crawlers/index.ts"],"names":[],"mappings":";;;AAAA,4DAAkC;AAClC,8DAAoC;AACpC,0DAAgC;AAChC,uDAA6B;AAC7B,0DAAgC;AAChC,8DAAoC"}

package/crawlers/statistics.d.ts

@@ -1,4 +1,4 @@
-import { ErrorTracker } from '@crawlee/utils';
+import { ErrorTracker } from './error_tracker';
 import { Configuration } from '../configuration';
 import { KeyValueStore } from '../storages/key_value_store';
 /**
@@ -166,6 +166,11 @@ export interface StatisticsOptions {
      * Control how and when to persist the statistics.
      */
     persistenceOptions?: PersistenceOptions;
+    /**
+     * Save HTML snapshot (and a screenshot if possible) when an error occurs.
+     * @default false
+     */
+    saveErrorSnapshots?: boolean;
 }
 /**
  * Format of the persisted stats