@crawlee/core 4.0.0-beta.2 → 4.0.0-beta.200

package/crawlers/crawler_commons.js

@@ -1,116 +0,0 @@
-import { KeyValueStore } from '../storages/key_value_store.js';
-/**
- * A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
- *
- * @experimental
- */
-export class RequestHandlerResult {
-    config;
-    crawleeStateKey;
-    _keyValueStoreChanges = {};
-    pushDataCalls = [];
-    addRequestsCalls = [];
-    enqueueLinksCalls = [];
-    constructor(config, crawleeStateKey) {
-        this.config = config;
-        this.crawleeStateKey = crawleeStateKey;
-    }
-    /**
-     * A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
-     */
-    get calls() {
-        return {
-            pushData: this.pushDataCalls,
-            addRequests: this.addRequestsCalls,
-            enqueueLinks: this.enqueueLinksCalls,
-        };
-    }
-    /**
-     * A record of changes made to key-value stores by a request handler.
-     */
-    get keyValueStoreChanges() {
-        return this._keyValueStoreChanges;
-    }
-    /**
-     * Items added to datasets by a request handler.
-     */
-    get datasetItems() {
-        return this.pushDataCalls.flatMap(([data, datasetIdOrName]) => (Array.isArray(data) ? data : [data]).map((item) => ({ item, datasetIdOrName })));
-    }
-    /**
-     * URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
-     */
-    get enqueuedUrls() {
-        const result = [];
-        for (const [options] of this.enqueueLinksCalls) {
-            result.push(...(options?.urls?.map((url) => ({ url, label: options?.label })) ?? []));
-        }
-        for (const [requests] of this.addRequestsCalls) {
-            for (const request of requests) {
-                if (typeof request === 'object' &&
-                    (!('requestsFromUrl' in request) || request.requestsFromUrl !== undefined) &&
-                    request.url !== undefined) {
-                    result.push({ url: request.url, label: request.label });
-                }
-                else if (typeof request === 'string') {
-                    result.push({ url: request });
-                }
-            }
-        }
-        return result;
-    }
-    /**
-     * URL lists enqueued to the request queue by a request handler via {@link RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
-     */
-    get enqueuedUrlLists() {
-        const result = [];
-        for (const [requests] of this.addRequestsCalls) {
-            for (const request of requests) {
-                if (typeof request === 'object' &&
-                    'requestsFromUrl' in request &&
-                    request.requestsFromUrl !== undefined) {
-                    result.push({ listUrl: request.requestsFromUrl, label: request.label });
-                }
-            }
-        }
-        return result;
-    }
-    pushData = async (data, datasetIdOrName) => {
-        this.pushDataCalls.push([data, datasetIdOrName]);
-    };
-    enqueueLinks = async (options) => {
-        this.enqueueLinksCalls.push([options]);
-    };
-    addRequests = async (requests, options = {}) => {
-        this.addRequestsCalls.push([requests, options]);
-    };
-    useState = async (defaultValue) => {
-        const store = await this.getKeyValueStore(undefined);
-        return await store.getAutoSavedValue(this.crawleeStateKey, defaultValue);
-    };
-    getKeyValueStore = async (idOrName) => {
-        const store = await KeyValueStore.open(idOrName, { config: this.config });
-        return {
-            id: this.idOrDefault(idOrName),
-            name: idOrName,
-            getValue: async (key) => this.getKeyValueStoreChangedValue(idOrName, key) ?? (await store.getValue(key)),
-            setValue: async (key, value, options) => {
-                this.setKeyValueStoreChangedValue(idOrName, key, value, options);
-            },
-            getAutoSavedValue: store.getAutoSavedValue.bind(store),
-            getPublicUrl: store.getPublicUrl.bind(store),
-        };
-    };
-    idOrDefault = (idOrName) => idOrName ?? this.config.get('defaultKeyValueStoreId');
-    getKeyValueStoreChangedValue = (idOrName, key) => {
-        const id = this.idOrDefault(idOrName);
-        this._keyValueStoreChanges[id] ??= {};
-        return this.keyValueStoreChanges[id][key]?.changedValue ?? null;
-    };
-    setKeyValueStoreChangedValue = (idOrName, key, changedValue, options) => {
-        const id = this.idOrDefault(idOrName);
-        this._keyValueStoreChanges[id] ??= {};
-        this._keyValueStoreChanges[id][key] = { changedValue, options };
-    };
-}
-//# sourceMappingURL=crawler_commons.js.map

package/crawlers/crawler_commons.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"crawler_commons.js","sourceRoot":"","sources":["../../src/crawlers/crawler_commons.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,aAAa,EAAsB,MAAM,gCAAgC,CAAC;AA6JnF;;;;GAIG;AACH,MAAM,OAAO,oBAAoB;IAWjB;IACA;IAXJ,qBAAqB,GACzB,EAAE,CAAC;IAEC,aAAa,GAAwD,EAAE,CAAC;IAExE,gBAAgB,GAA2D,EAAE,CAAC;IAE9E,iBAAiB,GAA4D,EAAE,CAAC;IAExF,YACY,MAAqB,EACrB,eAAuB;QADvB,WAAM,GAAN,MAAM,CAAe;QACrB,oBAAe,GAAf,eAAe,CAAQ;IAChC,CAAC;IAEJ;;OAEG;IACH,IAAI,KAAK;QAKL,OAAO;YACH,QAAQ,EAAE,IAAI,CAAC,aAAa;YAC5B,WAAW,EAAE,IAAI,CAAC,gBAAgB;YAClC,YAAY,EAAE,IAAI,CAAC,iBAAiB;SACvC,CAAC;IACN,CAAC;IAED;;OAEG;IACH,IAAI,oBAAoB;QAGpB,OAAO,IAAI,CAAC,qBAAqB,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE,EAAE,CAC1D,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC,CACnF,CAAC;IACN,CAAC;IAED;;OAEG;IACH,IAAI,YAAY;QACZ,MAAM,MAAM,GAAsC,EAAE,CAAC;QAErD,KAAK,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1F,CAAC;QAED,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IACI,OAAO,OAAO,KAAK,QAAQ;oBAC3B,CAAC,CAAC,CAAC,iBAAiB,IAAI,OAAO,CAAC,IAAI,OAAO,CAAC,eAAe,KAAK,SAAS,CAAC;oBAC1E,OAAO,CAAC,GAAG,KAAK,SAAS,EAC3B,CAAC;oBACC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5D,CAAC;qBAAM,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACrC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;gBAClC,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,IAAI,gBAAgB;QAChB,MAAM,MAAM,GAA0C,EAAE,CAAC;QAEzD,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,IACI,OAAO,OAAO,KAAK,QAAQ;oBAC3B,iBAAiB,IAAI,OAAO;oBAC5B,OAAO,CAAC,eAAe,KAAK,SAAS,EACvC,CAAC;oBACC,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,eAAe,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5E,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,QAAQ,GAA0C,KAAK,EAAE,IAAI,EAAE,eAAe,EAAE,EAAE;QAC9E,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAC;IACrD,CAAC,CAAC;IAEF,YAAY,GAA8C,KAAK,EAAE,OAAO,EAAE,EAAE;QACxE,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3C,CAAC,CAAC;IAEF,WAAW,GAA6C,KAAK,EAAE,QAAQ,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;QACrF,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;IACpD,CAAC,CAAC;IAEF,QAAQ,GAA0C,KAAK,EAAE,YAAY,EAAE,EAAE;QACrE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QACrD,OAAO,MAAM,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;IAC7E,CAAC,CAAC;IAEF,gBAAgB,GAAkD,KAAK,EAAE,QAAQ,EAAE,EAAE;QACjF,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAE1E,OAAO;YACH,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;YAC9B,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACxG,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;gBACpC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;YACrE,CAAC;YACD,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC;YACtD,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;SAC/C,CAAC;IACN,CAAC,CAAC;IAEM,WAAW,GAAG,CAAC,QAAiB,EAAU,EAAE,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IAEnG,4BAA4B,GAAG,CAAC,QAA4B,EAAE,GAAW,EAAE,EAAE;QACjF,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QACtC,OAAO,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,YAAY,IAAI,IAAI,CAAC;IACpE,CAAC,CAAC;IAEM,4BAA4B,GAAG,CACnC,QAA4B,EAC5B,GAAW,EACX,YAAqB,EACrB,OAAuB,EACzB,EAAE;QACA,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QACtC,IAAI,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;IACpE,CAAC,CAAC;CACL"}

package/crawlers/crawler_extension.d.ts

@@ -1,12 +0,0 @@
-import { type Log } from '../log.js';
-/**
- * Abstract class with pre-defined method to connect to the Crawlers class by the "use" crawler method.
- * @category Crawlers
- * @ignore
- */
-export declare abstract class CrawlerExtension {
-    name: string;
-    log: Log;
-    getCrawlerOptions(): Record<string, unknown>;
-}
-//# sourceMappingURL=crawler_extension.d.ts.map

package/crawlers/crawler_extension.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"crawler_extension.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_extension.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,GAAG,EAAqB,MAAM,WAAW,CAAC;AAExD;;;;GAIG;AACH,8BAAsB,gBAAgB;IAClC,IAAI,SAAyB;IAC7B,GAAG,EAAE,GAAG,CAA2C;IAEnD,iBAAiB,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;CAG/C"}

package/crawlers/crawler_extension.js

@@ -1,14 +0,0 @@
-import { log as defaultLog } from '../log.js';
-/**
- * Abstract class with pre-defined method to connect to the Crawlers class by the "use" crawler method.
- * @category Crawlers
- * @ignore
- */
-export class CrawlerExtension {
-    name = this.constructor.name;
-    log = defaultLog.child({ prefix: this.name });
-    getCrawlerOptions() {
-        throw new Error(`${this.name} has not implemented "getCrawlerOptions" method.`);
-    }
-}
-//# sourceMappingURL=crawler_extension.js.map

package/crawlers/crawler_extension.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"crawler_extension.js","sourceRoot":"","sources":["../../src/crawlers/crawler_extension.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,GAAG,IAAI,UAAU,EAAE,MAAM,WAAW,CAAC;AAExD;;;;GAIG;AACH,MAAM,OAAgB,gBAAgB;IAClC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;IAC7B,GAAG,GAAQ,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAEnD,iBAAiB;QACb,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,kDAAkD,CAAC,CAAC;IACpF,CAAC;CACJ"}

package/crawlers/crawler_utils.d.ts

@@ -1,10 +0,0 @@
-import type { Session } from '../session_pool/session.js';
-/**
- * Handles timeout request
- * @internal
- */
-export declare function handleRequestTimeout({ session, errorMessage }: {
-    session?: Session;
-    errorMessage: string;
-}): void;
-//# sourceMappingURL=crawler_utils.d.ts.map

package/crawlers/crawler_utils.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"crawler_utils.d.ts","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAE1D;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,EAAE;IAAE,OAAO,CAAC,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,QAK1G"}

package/crawlers/crawler_utils.js

@@ -1,12 +0,0 @@
-import { TimeoutError } from '@apify/timeout';
-/**
- * Handles timeout request
- * @internal
- */
-export function handleRequestTimeout({ session, errorMessage }) {
-    session?.markBad();
-    const timeoutMillis = errorMessage.match(/(\d+)\s?ms/)?.[1]; // first capturing group
-    const timeoutSecs = Number(timeoutMillis) / 1000;
-    throw new TimeoutError(`Navigation timed out after ${timeoutSecs} seconds.`);
-}
-//# sourceMappingURL=crawler_utils.js.map

package/crawlers/crawler_utils.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"crawler_utils.js","sourceRoot":"","sources":["../../src/crawlers/crawler_utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAI9C;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,EAAE,OAAO,EAAE,YAAY,EAA+C;IACvG,OAAO,EAAE,OAAO,EAAE,CAAC;IACnB,MAAM,aAAa,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,wBAAwB;IACrF,MAAM,WAAW,GAAG,MAAM,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC;IACjD,MAAM,IAAI,YAAY,CAAC,8BAA8B,WAAW,WAAW,CAAC,CAAC;AACjF,CAAC"}

package/crawlers/error_snapshotter.d.ts

@@ -1,59 +0,0 @@
-import type { CrawlingContext } from '../crawlers/crawler_commons.js';
-import type { KeyValueStore } from '../storages/key_value_store.js';
-import type { ErrnoException } from './error_tracker.js';
-interface BrowserCrawlingContext {
-    saveSnapshot: (options: {
-        key: string;
-    }) => Promise<void>;
-}
-export interface SnapshotResult {
-    screenshotFileName?: string;
-    htmlFileName?: string;
-}
-interface ErrorSnapshot {
-    screenshotFileName?: string;
-    screenshotFileUrl?: string;
-    htmlFileName?: string;
-    htmlFileUrl?: string;
-}
-/**
- * ErrorSnapshotter class is used to capture a screenshot of the page and a snapshot of the HTML when an error occurs during web crawling.
- *
- * This functionality is opt-in, and can be enabled via the crawler options:
- *
- * ```ts
- * const crawler = new BasicCrawler({
- *   // ...
- *   statisticsOptions: {
- *     saveErrorSnapshots: true,
- *   },
- * });
- * ```
- */
-export declare class ErrorSnapshotter {
-    static readonly MAX_ERROR_CHARACTERS = 30;
-    static readonly MAX_HASH_LENGTH = 30;
-    static readonly MAX_FILENAME_LENGTH = 250;
-    static readonly BASE_MESSAGE = "An error occurred";
-    static readonly SNAPSHOT_PREFIX = "ERROR_SNAPSHOT";
-    /**
-     * Capture a snapshot of the error context.
-     */
-    captureSnapshot(error: ErrnoException, context: CrawlingContext): Promise<ErrorSnapshot>;
-    /**
-     * Captures a snapshot of the current page using the context.saveSnapshot function.
-     * This function is applicable for browser contexts only.
-     * Returns an object containing the filenames of the screenshot and HTML file.
-     */
-    contextCaptureSnapshot(context: BrowserCrawlingContext, fileName: string): Promise<SnapshotResult | undefined>;
-    /**
-     * Save the HTML snapshot of the page, and return the fileName with the extension.
-     */
-    saveHTMLSnapshot(html: string, keyValueStore: KeyValueStore, fileName: string): Promise<string | undefined>;
-    /**
-     * Generate a unique fileName for each error snapshot.
-     */
-    generateFilename(error: ErrnoException): string;
-}
-export {};
-//# sourceMappingURL=error_snapshotter.d.ts.map

package/crawlers/error_snapshotter.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"error_snapshotter.d.ts","sourceRoot":"","sources":["../../src/crawlers/error_snapshotter.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGzD,UAAU,sBAAsB;IAC5B,YAAY,EAAE,CAAC,OAAO,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7D;AAMD,MAAM,WAAW,cAAc;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,UAAU,aAAa;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,gBAAgB;IACzB,MAAM,CAAC,QAAQ,CAAC,oBAAoB,MAAM;IAC1C,MAAM,CAAC,QAAQ,CAAC,eAAe,MAAM;IACrC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,OAAO;IAC1C,MAAM,CAAC,QAAQ,CAAC,YAAY,uBAAuB;IACnD,MAAM,CAAC,QAAQ,CAAC,eAAe,oBAAoB;IAEnD;;OAEG;IACG,eAAe,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,aAAa,CAAC;IAgD9F;;;;OAIG;IACG,sBAAsB,CACxB,OAAO,EAAE,sBAAsB,EAC/B,QAAQ,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC;IAYtC;;OAEG;IACG,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IASjH;;OAEG;IACH,gBAAgB,CAAC,KAAK,EAAE,cAAc,GAAG,MAAM;CAyBlD"}

package/crawlers/error_snapshotter.js

@@ -1,118 +0,0 @@
-import crypto from 'node:crypto';
-/**
- * ErrorSnapshotter class is used to capture a screenshot of the page and a snapshot of the HTML when an error occurs during web crawling.
- *
- * This functionality is opt-in, and can be enabled via the crawler options:
- *
- * ```ts
- * const crawler = new BasicCrawler({
- *   // ...
- *   statisticsOptions: {
- *     saveErrorSnapshots: true,
- *   },
- * });
- * ```
- */
-export class ErrorSnapshotter {
-    static MAX_ERROR_CHARACTERS = 30;
-    static MAX_HASH_LENGTH = 30;
-    static MAX_FILENAME_LENGTH = 250;
-    static BASE_MESSAGE = 'An error occurred';
-    static SNAPSHOT_PREFIX = 'ERROR_SNAPSHOT';
-    /**
-     * Capture a snapshot of the error context.
-     */
-    async captureSnapshot(error, context) {
-        try {
-            const page = context?.page;
-            const body = context?.body;
-            const keyValueStore = await context?.getKeyValueStore();
-            // If the key-value store is not available, or the body and page are not available, return empty filenames
-            if (!keyValueStore || (!body && !page)) {
-                return {};
-            }
-            const fileName = this.generateFilename(error);
-            let screenshotFileName;
-            let htmlFileName;
-            if (page) {
-                const capturedFiles = await this.contextCaptureSnapshot(context, fileName);
-                if (capturedFiles) {
-                    screenshotFileName = capturedFiles.screenshotFileName;
-                    htmlFileName = capturedFiles.htmlFileName;
-                }
-                // If the snapshot for browsers failed to capture the HTML, try to capture it from the page content
-                if (!htmlFileName) {
-                    const html = await page.content();
-                    htmlFileName = html ? await this.saveHTMLSnapshot(html, keyValueStore, fileName) : undefined;
-                }
-            }
-            else if (typeof body === 'string') {
-                // for non-browser contexts
-                htmlFileName = await this.saveHTMLSnapshot(body, keyValueStore, fileName);
-            }
-            return {
-                screenshotFileName,
-                screenshotFileUrl: screenshotFileName && keyValueStore.getPublicUrl(screenshotFileName),
-                htmlFileName,
-                htmlFileUrl: htmlFileName && keyValueStore.getPublicUrl(htmlFileName),
-            };
-        }
-        catch {
-            return {};
-        }
-    }
-    /**
-     * Captures a snapshot of the current page using the context.saveSnapshot function.
-     * This function is applicable for browser contexts only.
-     * Returns an object containing the filenames of the screenshot and HTML file.
-     */
-    async contextCaptureSnapshot(context, fileName) {
-        try {
-            await context.saveSnapshot({ key: fileName });
-            return {
-                screenshotFileName: `${fileName}.jpg`,
-                htmlFileName: `${fileName}.html`,
-            };
-        }
-        catch {
-            return undefined;
-        }
-    }
-    /**
-     * Save the HTML snapshot of the page, and return the fileName with the extension.
-     */
-    async saveHTMLSnapshot(html, keyValueStore, fileName) {
-        try {
-            await keyValueStore.setValue(fileName, html, { contentType: 'text/html' });
-            return `${fileName}.html`;
-        }
-        catch {
-            return undefined;
-        }
-    }
-    /**
-     * Generate a unique fileName for each error snapshot.
-     */
-    generateFilename(error) {
-        const { SNAPSHOT_PREFIX, BASE_MESSAGE, MAX_HASH_LENGTH, MAX_ERROR_CHARACTERS, MAX_FILENAME_LENGTH } = ErrorSnapshotter;
-        // Create a hash of the error stack trace
-        const errorStackHash = crypto
-            .createHash('sha1')
-            .update(error.stack || error.message || '')
-            .digest('hex')
-            .slice(0, MAX_HASH_LENGTH);
-        const errorMessagePrefix = (error.message || BASE_MESSAGE).slice(0, MAX_ERROR_CHARACTERS).trim();
-        /**
-         * Remove non-word characters from the start and end of a string.
-         */
-        const sanitizeString = (str) => {
-            return str.replace(/^\W+|\W+$/g, '');
-        };
-        // Generate fileName and remove disallowed characters
-        const fileName = `${SNAPSHOT_PREFIX}_${sanitizeString(errorStackHash)}_${sanitizeString(errorMessagePrefix)}`
-            .replace(/\W+/g, '-') // Replace non-word characters with a dash
-            .slice(0, MAX_FILENAME_LENGTH);
-        return fileName;
-    }
-}
-//# sourceMappingURL=error_snapshotter.js.map

package/crawlers/error_snapshotter.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"error_snapshotter.js","sourceRoot":"","sources":["../../src/crawlers/error_snapshotter.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AA2BjC;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAO,gBAAgB;IACzB,MAAM,CAAU,oBAAoB,GAAG,EAAE,CAAC;IAC1C,MAAM,CAAU,eAAe,GAAG,EAAE,CAAC;IACrC,MAAM,CAAU,mBAAmB,GAAG,GAAG,CAAC;IAC1C,MAAM,CAAU,YAAY,GAAG,mBAAmB,CAAC;IACnD,MAAM,CAAU,eAAe,GAAG,gBAAgB,CAAC;IAEnD;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,KAAqB,EAAE,OAAwB;QACjE,IAAI,CAAC;YACD,MAAM,IAAI,GAAG,OAAO,EAAE,IAA+B,CAAC;YACtD,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,CAAC;YAE3B,MAAM,aAAa,GAAG,MAAM,OAAO,EAAE,gBAAgB,EAAE,CAAC;YACxD,0GAA0G;YAC1G,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrC,OAAO,EAAE,CAAC;YACd,CAAC;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAE9C,IAAI,kBAAsC,CAAC;YAC3C,IAAI,YAAgC,CAAC;YAErC,IAAI,IAAI,EAAE,CAAC;gBACP,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,sBAAsB,CACnD,OAA4C,EAC5C,QAAQ,CACX,CAAC;gBAEF,IAAI,aAAa,EAAE,CAAC;oBAChB,kBAAkB,GAAG,aAAa,CAAC,kBAAkB,CAAC;oBACtD,YAAY,GAAG,aAAa,CAAC,YAAY,CAAC;gBAC9C,CAAC;gBAED,mGAAmG;gBACnG,IAAI,CAAC,YAAY,EAAE,CAAC;oBAChB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;oBAClC,YAAY,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBACjG,CAAC;YACL,CAAC;iBAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAClC,2BAA2B;gBAC3B,YAAY,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;YAC9E,CAAC;YAED,OAAO;gBACH,kBAAkB;gBAClB,iBAAiB,EAAE,kBAAkB,IAAI,aAAa,CAAC,YAAY,CAAC,kBAAkB,CAAC;gBACvF,YAAY;gBACZ,WAAW,EAAE,YAAY,IAAI,aAAa,CAAC,YAAY,CAAC,YAAY,CAAC;aACxE,CAAC;QACN,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,sBAAsB,CACxB,OAA+B,EAC/B,QAAgB;QAEhB,IAAI,CAAC;YACD,MAAM,OAAO,CAAC,YAAY,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC9C,OAAO;gBACH,kBAAkB,EAAE,GAAG,QAAQ,MAAM;gBACrC,YAAY,EAAE,GAAG,QAAQ,OAAO;aACnC,CAAC;QACN,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,SAAS,CAAC;QACrB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAC,IAAY,EAAE,aAA4B,EAAE,QAAgB;QAC/E,IAAI,CAAC;YACD,MAAM,aAAa,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,CAAC;YAC3E,OAAO,GAAG,QAAQ,OAAO,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,SAAS,CAAC;QACrB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,KAAqB;QAClC,MAAM,EAAE,eAAe,EAAE,YAAY,EAAE,eAAe,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,GAC/F,gBAAgB,CAAC;QACrB,yCAAyC;QACzC,MAAM,cAAc,GAAG,MAAM;aACxB,UAAU,CAAC,MAAM,CAAC;aAClB,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;aAC1C,MAAM,CAAC,KAAK,CAAC;aACb,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;QAC/B,MAAM,kBAAkB,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC,IAAI,EAAE,CAAC;QAEjG;;WAEG;QACH,MAAM,cAAc,GAAG,CAAC,GAAW,EAAU,EAAE;YAC3C,OAAO,GAAG,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QACzC,CAAC,CAAC;QAEF,qDAAqD;QACrD,MAAM,QAAQ,GAAG,GAAG,eAAe,IAAI,cAAc,CAAC,cAAc,CAAC,IAAI,cAAc,CAAC,kBAAkB,CAAC,EAAE;aACxG,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,0CAA0C;aAC/D,KAAK,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;QAEnC,OAAO,QAAQ,CAAC;IACpB,CAAC"}

package/crawlers/error_tracker.d.ts

@@ -1,54 +0,0 @@
-import type { CrawlingContext } from '../crawlers/crawler_commons.js';
-import { ErrorSnapshotter } from './error_snapshotter.js';
-/**
- * Node.js Error interface
- */
-export interface ErrnoException extends Error {
-    errno?: number;
-    code?: string | number;
-    path?: string;
-    syscall?: string;
-    cause?: any;
-}
-export interface ErrorTrackerOptions {
-    showErrorCode: boolean;
-    showErrorName: boolean;
-    showStackTrace: boolean;
-    showFullStack: boolean;
-    showErrorMessage: boolean;
-    showFullMessage: boolean;
-    saveErrorSnapshots: boolean;
-}
-/**
- * This class tracks errors and computes a summary of information like:
- * - where the errors happened
- * - what the error names are
- * - what the error codes are
- * - what is the general error message
- *
- * This is extremely useful when there are dynamic error messages, such as argument validation.
- *
- * Since the structure of the `tracker.result` object differs when using different options,
- * it's typed as `Record<string, unknown>`. The most deep object has a `count` property, which is a number.
- *
- * It's possible to get the total amount of errors via the `tracker.total` property.
- */
-export declare class ErrorTracker {
-    #private;
-    result: Record<string, unknown>;
-    total: number;
-    errorSnapshotter?: ErrorSnapshotter;
-    constructor(options?: Partial<ErrorTrackerOptions>);
-    private updateGroup;
-    add(error: ErrnoException): void;
-    /**
-     * This method is async, because it captures a snapshot of the error context.
-     * We added this new method to avoid breaking changes.
-     */
-    addAsync(error: ErrnoException, context?: CrawlingContext): Promise<void>;
-    getUniqueErrorCount(): number;
-    getMostPopularErrors(count: number): [number, string[]][];
-    captureSnapshot(storage: Record<string, unknown>, error: ErrnoException, context: CrawlingContext): Promise<void>;
-    reset(): void;
-}
-//# sourceMappingURL=error_tracker.d.ts.map

package/crawlers/error_tracker.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"error_tracker.d.ts","sourceRoot":"","sources":["../../src/crawlers/error_tracker.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE1D;;GAEG;AACH,MAAM,WAAW,cAAe,SAAQ,KAAK;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,GAAG,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IAChC,aAAa,EAAE,OAAO,CAAC;IACvB,aAAa,EAAE,OAAO,CAAC;IACvB,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,eAAe,EAAE,OAAO,CAAC;IACzB,kBAAkB,EAAE,OAAO,CAAC;CAC/B;AAuPD;;;;;;;;;;;;;GAaG;AACH,qBAAa,YAAY;;IAGrB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEhC,KAAK,EAAE,MAAM,CAAC;IAEd,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;gBAExB,OAAO,GAAE,OAAO,CAAC,mBAAmB,CAAM;IAoBtD,OAAO,CAAC,WAAW;IAwBnB,GAAG,CAAC,KAAK,EAAE,cAAc;IAUzB;;;OAGG;IACG,QAAQ,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE,eAAe;IAe/D,mBAAmB;IAoBnB,oBAAoB,CAAC,KAAK,EAAE,MAAM;IAoB5B,eAAe,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,eAAe;IAWvG,KAAK;CAOR"}