@crawlee/core 3.0.0-alpha.2 → 3.0.0-alpha.3

package/dist/crawlers/statistics.d.ts

@@ -0,0 +1,147 @@
+import { Configuration } from '../configuration';
+/**
+ * @ignore
+ */
+declare class Job {
+    private lastRunAt;
+    private runs;
+    private durationMillis?;
+    run(): number;
+    finish(): number;
+    retryCount(): number;
+}
+/**
+ * The statistics class provides an interface to collecting and logging run
+ * statistics for requests.
+ *
+ * All statistic information is saved on key value store
+ * under the key `SDK_CRAWLER_STATISTICS_*`, persists between
+ * migrations and abort/resurrect
+ *
+ * @category Crawlers
+ */
+export declare class Statistics {
+    private static id;
+    /**
+     * Statistic instance id.
+     */
+    readonly id: number;
+    /**
+     * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
+     */
+    state: StatisticState;
+    /**
+     * Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on
+     */
+    readonly requestRetryHistogram: number[];
+    private keyValueStore;
+    private persistStateKey;
+    private logIntervalMillis;
+    private logMessage;
+    private listener;
+    private requestsInProgress;
+    private readonly log;
+    private instanceStart;
+    private logInterval;
+    private events;
+    /**
+     * @internal
+     */
+    constructor(options?: StatisticsOptions);
+    /**
+     * Set the current statistic instance to pristine values
+     */
+    reset(): void;
+    /**
+     * Starts a job
+     * @ignore
+     */
+    startJob(id: number | string): void;
+    /**
+     * Mark job as finished and sets the state
+     * @ignore
+     */
+    finishJob(id: number | string): void;
+    /**
+     * Mark job as failed and sets the state
+     * @ignore
+     */
+    failJob(id: number | string): void;
+    /**
+     * Calculate the current statistics
+     */
+    calculate(): {
+        requestAvgFailedDurationMillis: number;
+        requestAvgFinishedDurationMillis: number;
+        requestsFinishedPerMinute: number;
+        requestsFailedPerMinute: number;
+        requestTotalDurationMillis: number;
+        requestsTotal: number;
+        crawlerRuntimeMillis: number;
+    };
+    /**
+     * Initializes the key value store for persisting the statistics,
+     * displaying the current state in predefined intervals
+     */
+    startCapturing(): Promise<void>;
+    /**
+     * Stops logging and remove event listeners, then persist
+     */
+    stopCapturing(): Promise<void>;
+    protected _saveRetryCountForJob(job: Job): void;
+    /**
+     * Persist internal state to the key value store
+     */
+    persistState(): Promise<void>;
+    /**
+     * Loads the current statistic from the key value store if any
+     */
+    protected _maybeLoadStatistics(): Promise<void>;
+    protected _teardown(): void;
+    /**
+     * Make this class serializable when called with `JSON.stringify(statsInstance)` directly
+     * or through `keyValueStore.setValue('KEY', statsInstance)`
+     */
+    toJSON(): StatisticPersistedState;
+}
+interface StatisticsOptions {
+    logIntervalSecs?: number;
+    logMessage?: string;
+    config?: Configuration;
+}
+/**
+ * Format of the persisted stats
+ */
+export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt'> {
+    requestRetryHistogram: number[];
+    statsId: number;
+    requestAvgFailedDurationMillis: number;
+    requestAvgFinishedDurationMillis: number;
+    requestsFinishedPerMinute: number;
+    requestsFailedPerMinute: number;
+    requestTotalDurationMillis: number;
+    requestsTotal: number;
+    crawlerRuntimeMillis: number;
+    crawlerLastStartTimestamp: number;
+    statsPersistedAt: string;
+}
+/**
+ * Contains the statistics state
+ */
+export interface StatisticState {
+    requestsFinished: number;
+    requestsFailed: number;
+    requestsRetries: number;
+    requestsFailedPerMinute: number;
+    requestsFinishedPerMinute: number;
+    requestMinDurationMillis: number;
+    requestMaxDurationMillis: number;
+    requestTotalFailedDurationMillis: number;
+    requestTotalFinishedDurationMillis: number;
+    crawlerStartedAt: Date | string | null;
+    crawlerFinishedAt: Date | string | null;
+    crawlerRuntimeMillis: number;
+    statsPersistedAt: Date | string | null;
+}
+export {};
+//# sourceMappingURL=statistics.d.ts.map

package/dist/crawlers/statistics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD;;GAEG;AACH,cAAM,GAAG;IACL,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,IAAI,CAAK;IACjB,OAAO,CAAC,cAAc,CAAC,CAAS;IAEhC,GAAG;IAKH,MAAM;IAKN,UAAU;CAGb;AAED;;;;;;;;;GASG;AACH,qBAAa,UAAU;IACnB,OAAO,CAAC,MAAM,CAAC,EAAE,CAAK;IAEtB;;OAEG;IACH,QAAQ,CAAC,EAAE,SAAmB;IAE9B;;OAEG;IACI,KAAK,EAAG,cAAc,CAAC;IAE9B;;OAEG;IACH,QAAQ,CAAC,qBAAqB,EAAE,MAAM,EAAE,CAAM;IAE9C,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,eAAe,CAAuC;IAC9D,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA8C;IAClE,OAAO,CAAC,aAAa,CAAU;IAC/B,OAAO,CAAC,WAAW,CAAU;IAC7B,OAAO,CAAC,MAAM,CAAe;IAE7B;;OAEG;gBACS,OAAO,GAAE,iBAAsB;IAsB3C;;OAEG;IACH,KAAK;IAwBL;;;OAGG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAO5B;;;OAGG;IACH,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAY7B;;;OAGG;IACH,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM;IAS3B;;OAEG;IACH,SAAS;;;;;;;;;IAqBT;;;OAGG;IACG,cAAc;IAmBpB;;OAEG;IACG,aAAa;IAQnB,SAAS,CAAC,qBAAqB,CAAC,GAAG,EAAE,GAAG;IAQxC;;OAEG;IACG,YAAY;IAWlB;;OAEG;cACa,oBAAoB;IAwCpC,SAAS,CAAC,SAAS,IAAI,IAAI;IAU3B;;;OAGG;IACH,MAAM,IAAI,uBAAuB;CAepC;AAED,UAAU,iBAAiB;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,IAAI,CAAC,cAAc,EAAE,kBAAkB,CAAC;IACrF,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,8BAA8B,EAAE,MAAM,CAAC;IACvC,gCAAgC,EAAE,MAAM,CAAC;IACzC,yBAAyB,EAAE,MAAM,CAAC;IAClC,uBAAuB,EAAE,MAAM,CAAC;IAChC,0BAA0B,EAAE,MAAM,CAAC;IACnC,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,yBAAyB,EAAE,MAAM,CAAC;IAClC,gBAAgB,EAAE,MAAM,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,uBAAuB,EAAE,MAAM,CAAC;IAChC,yBAAyB,EAAE,MAAM,CAAC;IAClC,wBAAwB,EAAE,MAAM,CAAC;IACjC,wBAAwB,EAAE,MAAM,CAAC;IACjC,gCAAgC,EAAE,MAAM,CAAC;IACzC,kCAAkC,EAAE,MAAM,CAAC;IAC3C,gBAAgB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACvC,iBAAiB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;IACxC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAAC;CAC1C"}

package/dist/crawlers/statistics.js

@@ -0,0 +1,360 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Statistics = void 0;
+const tslib_1 = require("tslib");
+/* eslint-disable max-classes-per-file */
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const log_1 = require("../log");
+const key_value_store_1 = require("../storages/key_value_store");
+const configuration_1 = require("../configuration");
+/**
+ * @ignore
+ */
+class Job {
+    constructor() {
+        Object.defineProperty(this, "lastRunAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "runs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 0
+        });
+        Object.defineProperty(this, "durationMillis", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+    }
+    run() {
+        this.lastRunAt = Date.now();
+        return ++this.runs;
+    }
+    finish() {
+        this.durationMillis = Date.now() - this.lastRunAt;
+        return this.durationMillis;
+    }
+    retryCount() {
+        return Math.max(0, this.runs - 1);
+    }
+}
+/**
+ * The statistics class provides an interface to collecting and logging run
+ * statistics for requests.
+ *
+ * All statistic information is saved on key value store
+ * under the key `SDK_CRAWLER_STATISTICS_*`, persists between
+ * migrations and abort/resurrect
+ *
+ * @category Crawlers
+ */
+class Statistics {
+    /**
+     * @internal
+     */
+    constructor(options = {}) {
+        /**
+         * Statistic instance id.
+         */
+        Object.defineProperty(this, "id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: Statistics.id++
+        }); // assign an id while incrementing so it can be saved/restored from KV
+        /**
+         * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
+         */
+        Object.defineProperty(this, "state", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on
+         */
+        Object.defineProperty(this, "requestRetryHistogram", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "keyValueStore", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "persistStateKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: `SDK_CRAWLER_STATISTICS_${this.id}`
+        });
+        Object.defineProperty(this, "logIntervalMillis", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "logMessage", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "listener", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "requestsInProgress", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+        Object.defineProperty(this, "log", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: log_1.log.child({ prefix: 'Statistics' })
+        });
+        Object.defineProperty(this, "instanceStart", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "logInterval", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "events", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        (0, ow_1.default)(options, ow_1.default.object.exactShape({
+            logIntervalSecs: ow_1.default.optional.number,
+            logMessage: ow_1.default.optional.string,
+            config: ow_1.default.optional.object,
+        }));
+        const { logIntervalSecs = 60, logMessage = 'Statistics', config = configuration_1.Configuration.getGlobalConfig(), } = options;
+        this.logIntervalMillis = logIntervalSecs * 1000;
+        this.logMessage = logMessage;
+        this.listener = this.persistState.bind(this);
+        this.events = config.getEventManager();
+        // initialize by "resetting"
+        this.reset();
+    }
+    /**
+     * Set the current statistic instance to pristine values
+     */
+    reset() {
+        this.state = {
+            requestsFinished: 0,
+            requestsFailed: 0,
+            requestsRetries: 0,
+            requestsFailedPerMinute: 0,
+            requestsFinishedPerMinute: 0,
+            requestMinDurationMillis: Infinity,
+            requestMaxDurationMillis: 0,
+            requestTotalFailedDurationMillis: 0,
+            requestTotalFinishedDurationMillis: 0,
+            crawlerStartedAt: null,
+            crawlerFinishedAt: null,
+            statsPersistedAt: null,
+            crawlerRuntimeMillis: 0,
+        };
+        this.requestRetryHistogram.length = 0;
+        this.requestsInProgress.clear();
+        this.instanceStart = Date.now();
+        this._teardown();
+    }
+    /**
+     * Starts a job
+     * @ignore
+     */
+    startJob(id) {
+        let job = this.requestsInProgress.get(id);
+        if (!job)
+            job = new Job();
+        job.run();
+        this.requestsInProgress.set(id, job);
+    }
+    /**
+     * Mark job as finished and sets the state
+     * @ignore
+     */
+    finishJob(id) {
+        const job = this.requestsInProgress.get(id);
+        if (!job)
+            return;
+        const jobDurationMillis = job.finish();
+        this.state.requestsFinished++;
+        this.state.requestTotalFinishedDurationMillis += jobDurationMillis;
+        this._saveRetryCountForJob(job);
+        if (jobDurationMillis < this.state.requestMinDurationMillis)
+            this.state.requestMinDurationMillis = jobDurationMillis;
+        if (jobDurationMillis > this.state.requestMaxDurationMillis)
+            this.state.requestMaxDurationMillis = jobDurationMillis;
+        this.requestsInProgress.delete(id);
+    }
+    /**
+     * Mark job as failed and sets the state
+     * @ignore
+     */
+    failJob(id) {
+        const job = this.requestsInProgress.get(id);
+        if (!job)
+            return;
+        this.state.requestTotalFailedDurationMillis += job.finish();
+        this.state.requestsFailed++;
+        this._saveRetryCountForJob(job);
+        this.requestsInProgress.delete(id);
+    }
+    /**
+     * Calculate the current statistics
+     */
+    calculate() {
+        const { requestsFailed, requestsFinished, requestTotalFailedDurationMillis, requestTotalFinishedDurationMillis, } = this.state;
+        const totalMillis = Date.now() - this.instanceStart;
+        const totalMinutes = totalMillis / 1000 / 60;
+        return {
+            requestAvgFailedDurationMillis: Math.round(requestTotalFailedDurationMillis / requestsFailed) || Infinity,
+            requestAvgFinishedDurationMillis: Math.round(requestTotalFinishedDurationMillis / requestsFinished) || Infinity,
+            requestsFinishedPerMinute: Math.round(requestsFinished / totalMinutes) || 0,
+            requestsFailedPerMinute: Math.floor(requestsFailed / totalMinutes) || 0,
+            requestTotalDurationMillis: requestTotalFinishedDurationMillis + requestTotalFailedDurationMillis,
+            requestsTotal: requestsFailed + requestsFinished,
+            crawlerRuntimeMillis: totalMillis,
+        };
+    }
+    /**
+     * Initializes the key value store for persisting the statistics,
+     * displaying the current state in predefined intervals
+     */
+    async startCapturing() {
+        this.keyValueStore = await key_value_store_1.KeyValueStore.open();
+        await this._maybeLoadStatistics();
+        if (this.state.crawlerStartedAt === null) {
+            this.state.crawlerStartedAt = new Date();
+        }
+        this.events.on("persistState" /* PERSIST_STATE */, this.listener);
+        this.logInterval = setInterval(() => {
+            this.log.info(this.logMessage, {
+                ...this.calculate(),
+                retryHistogram: this.requestRetryHistogram,
+            });
+        }, this.logIntervalMillis);
+    }
+    /**
+     * Stops logging and remove event listeners, then persist
+     */
+    async stopCapturing() {
+        this._teardown();
+        this.state.crawlerFinishedAt = new Date();
+        await this.persistState();
+    }
+    _saveRetryCountForJob(job) {
+        const retryCount = job.retryCount();
+        if (retryCount > 0)
+            this.state.requestsRetries++;
+        this.requestRetryHistogram[retryCount] = this.requestRetryHistogram[retryCount]
+            ? this.requestRetryHistogram[retryCount] + 1
+            : 1;
+    }
+    /**
+     * Persist internal state to the key value store
+     */
+    async persistState() {
+        // this might be called before startCapturing was called without using await, should not crash
+        if (!this.keyValueStore) {
+            return;
+        }
+        this.log.debug('Persisting state', { persistStateKey: this.persistStateKey });
+        await this.keyValueStore.setValue(this.persistStateKey, this.toJSON());
+    }
+    /**
+     * Loads the current statistic from the key value store if any
+     */
+    async _maybeLoadStatistics() {
+        // this might be called before startCapturing was called without using await, should not crash
+        if (!this.keyValueStore) {
+            return;
+        }
+        const savedState = await this.keyValueStore.getValue(this.persistStateKey);
+        if (!savedState)
+            return;
+        // We saw a run where the requestRetryHistogram was not iterable and crashed
+        // the actor. Adding some logging to monitor this problem in the future.
+        if (!Array.isArray(savedState.requestRetryHistogram)) {
+            this.log.warning('Received invalid state from Key-value store.', {
+                persistStateKey: this.persistStateKey,
+                state: savedState,
+            });
+        }
+        this.log.debug('Recreating state from KeyValueStore', { persistStateKey: this.persistStateKey });
+        this.requestRetryHistogram.push(...savedState.requestRetryHistogram);
+        this.state.requestsFinished = savedState.requestsFinished;
+        this.state.requestsFailed = savedState.requestsFailed;
+        this.state.requestsRetries = savedState.requestsRetries;
+        this.state.requestTotalFailedDurationMillis = savedState.requestTotalFailedDurationMillis;
+        this.state.requestTotalFinishedDurationMillis = savedState.requestTotalFinishedDurationMillis;
+        this.state.requestMinDurationMillis = savedState.requestMinDurationMillis;
+        this.state.requestMaxDurationMillis = savedState.requestMaxDurationMillis;
+        // persisted state uses ISO date strings
+        this.state.crawlerFinishedAt = savedState.crawlerFinishedAt ? new Date(savedState.crawlerFinishedAt) : null;
+        this.state.crawlerStartedAt = savedState.crawlerStartedAt ? new Date(savedState.crawlerStartedAt) : null;
+        this.state.statsPersistedAt = savedState.statsPersistedAt ? new Date(savedState.statsPersistedAt) : null;
+        this.state.crawlerRuntimeMillis = savedState.crawlerRuntimeMillis;
+        this.instanceStart = Date.now() - (+this.state.statsPersistedAt - savedState.crawlerLastStartTimestamp);
+        this.log.debug('Loaded from KeyValueStore');
+    }
+    _teardown() {
+        // this can be called before a call to startCapturing happens (or in a 'finally' block)
+        this.events.off("persistState" /* PERSIST_STATE */, this.listener);
+        if (this.logInterval) {
+            clearInterval(this.logInterval);
+            this.logInterval = null;
+        }
+    }
+    /**
+     * Make this class serializable when called with `JSON.stringify(statsInstance)` directly
+     * or through `keyValueStore.setValue('KEY', statsInstance)`
+     */
+    toJSON() {
+        // merge all the current state information that can be used from the outside
+        // without the need to reconstruct for the sake of stats.calculate()
+        // omit duplicated information
+        return {
+            ...this.state,
+            crawlerLastStartTimestamp: this.instanceStart,
+            crawlerFinishedAt: this.state.crawlerFinishedAt ? new Date(this.state.crawlerFinishedAt).toISOString() : null,
+            crawlerStartedAt: this.state.crawlerStartedAt ? new Date(this.state.crawlerStartedAt).toISOString() : null,
+            requestRetryHistogram: this.requestRetryHistogram,
+            statsId: this.id,
+            statsPersistedAt: new Date().toISOString(),
+            ...this.calculate(),
+        };
+    }
+}
+exports.Statistics = Statistics;
+Object.defineProperty(Statistics, "id", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: 0
+});
+//# sourceMappingURL=statistics.js.map

package/dist/crawlers/statistics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/crawlers/statistics.ts"],"names":[],"mappings":";;;;AAAA,yCAAyC;AACzC,oDAAoB;AACpB,gCAA2C;AAC3C,iEAA4D;AAE5D,oDAAiD;AAEjD;;GAEG;AACH,MAAM,GAAG;IAAT;QACI;;;;mBAAmC,IAAI;WAAC;QACxC;;;;mBAAe,CAAC;WAAC;QACjB;;;;;WAAgC;IAepC,CAAC;IAbG,GAAG;QACC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC;IACvB,CAAC;IAED,MAAM;QACF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAU,CAAC;QACnD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,UAAU;QACN,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;CACJ;AAED;;;;;;;;;GASG;AACH,MAAa,UAAU;IA6BnB;;OAEG;IACH,YAAY,UAA6B,EAAE;QA7B3C;;WAEG;QACH;;;;mBAAc,UAAU,CAAC,EAAE,EAAE;WAAC,CAAC,sEAAsE;QAErG;;WAEG;QACH;;;;;WAA8B;QAE9B;;WAEG;QACH;;;;mBAA2C,EAAE;WAAC;QAE9C;;;;mBAA8C,IAAI;WAAC;QACnD;;;;mBAA0B,0BAA0B,IAAI,CAAC,EAAE,EAAE;WAAC;QAC9D;;;;;WAAkC;QAClC;;;;;WAA2B;QAC3B;;;;;WAAsC;QACtC;;;;mBAA6B,IAAI,GAAG,EAAwB;WAAC;QAC7D;;;;mBAAuB,SAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;WAAC;QAClE;;;;;WAA+B;QAC/B;;;;;WAA6B;QAC7B;;;;;WAA6B;QAMzB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC9B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SAC7B,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,eAAe,GAAG,EAAE,EACpB,UAAU,GAAG,YAAY,EACzB,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE,GAC3C,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QAEvC,4BAA4B;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK;QACD,IAAI,CAAC,KAAK,GAAG;YACT,gBAAgB,EAAE,CAAC;YACnB,cAAc,EAAE,CAAC;YACjB,eAAe,EAAE,CAAC;YAClB,uBAAuB,EAAE,CAAC;YAC1B,yBAAyB,EAAE,CAAC;YAC5B,wBAAwB,EAAE,QAAQ;YAClC,wBAAwB,EAAE,CAAC;YAC3B,gCAAgC,EAAE,CAAC;YACnC,kCAAkC,EAAE,CAAC;YACrC,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;YACvB,gBAAgB,EAAE,IAAI;YACtB,oBAAoB,EAAE,CAAC;SAC1B,CAAC;QAEF,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEhC,IAAI,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC;IAED;;;OAGG;IACH,QAAQ,CAAC,EAAmB;QACxB,IAAI,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,EAAE,CAAC;QACV,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,EAAmB;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,iBAAiB,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,kCAAkC,IAAI,iBAAiB,CAAC;QACnE,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YAAE,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QACrH,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YAAE,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QACrH,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;;OAGG;IACH,OAAO,CAAC,EAAmB;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,CAAC,KAAK,CAAC,gCAAgC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC5B,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,SAAS;QACL,MAAM,EACF,cAAc,EACd,gBAAgB,EAChB,gCAAgC,EAChC,kCAAkC,GACrC,GAAG,IAAI,CAAC,KAAK,CAAC;QACf,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QACpD,MAAM,YAAY,GAAG,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAE7C,OAAO;YACH,8BAA8B,EAAE,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,cAAc,CAAC,IAAI,QAAQ;YACzG,gCAAgC,EAAE,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,gBAAgB,CAAC,IAAI,QAAQ;YAC/G,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC;YAC3E,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC;YACvE,0BAA0B,EAAE,kCAAkC,GAAG,gCAAgC;YACjG,aAAa,EAAE,cAAc,GAAG,gBAAgB;YAChD,oBAAoB,EAAE,WAAW;SACpC,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAChB,IAAI,CAAC,aAAa,GAAG,MAAM,+BAAa,CAAC,IAAI,EAAE,CAAC;QAEhD,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAElC,IAAI,IAAI,CAAC,KAAK,CAAC,gBAAgB,KAAK,IAAI,EAAE;YACtC,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,IAAI,IAAI,EAAE,CAAC;SAC5C;QAED,IAAI,CAAC,MAAM,CAAC,EAAE,qCAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEvD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC3B,GAAG,IAAI,CAAC,SAAS,EAAE;gBACnB,cAAc,EAAE,IAAI,CAAC,qBAAqB;aAC7C,CAAC,CAAC;QACP,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC;QAE1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAES,qBAAqB,CAAC,GAAQ;QACpC,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;QACpC,IAAI,UAAU,GAAG,CAAC;YAAE,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC;YAC3E,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,GAAG,CAAC;YAC5C,CAAC,CAAC,CAAC,CAAC;IACZ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACd,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACrB,OAAO;SACV;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAE9E,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,oBAAoB;QAChC,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACrB,OAAO;SACV;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAA0B,IAAI,CAAC,eAAe,CAAC,CAAC;QAEpG,IAAI,CAAC,UAAU;YAAE,OAAO;QAExB,4EAA4E;QAC5E,wEAAwE;QACxE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,EAAE;YAClD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,8CAA8C,EAAE;gBAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,KAAK,EAAE,UAAU;aACpB,CAAC,CAAC;SACN;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAEjG,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACrE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC;QAC1D,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,UAAU,CAAC,cAAc,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,UAAU,CAAC,eAAe,CAAC;QAExD,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,UAAU,CAAC,gCAAgC,CAAC;QAC1F,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,UAAU,CAAC,kCAAkC,CAAC;QAC9F,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,wCAAwC;QACxC,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5G,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,oBAAoB,GAAG,UAAU,CAAC,oBAAoB,CAAC;QAClE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAiB,GAAG,UAAU,CAAC,yBAAyB,CAAC,CAAC;QAEzG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAChD,CAAC;IAES,SAAS;QACf,uFAAuF;QACvF,IAAI,CAAC,MAAM,CAAC,GAAG,qCAA0B,IAAI,CAAC,QAAQ,CAAC,CAAC;QAExD,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,aAAa,CAAC,IAAI,CAAC,WAAqB,CAAC,CAAC;YAC1C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;SAC3B;IACL,CAAC;IAED;;;OAGG;IACH,MAAM;QACF,4EAA4E;QAC5E,oEAAoE;QACpE,8BAA8B;QAC9B,OAAO;YACH,GAAG,IAAI,CAAC,KAAK;YACb,yBAAyB,EAAE,IAAI,CAAC,aAAa;YAC7C,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAC7G,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAC1G,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;YACjD,OAAO,EAAE,IAAI,CAAC,EAAE;YAChB,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC1C,GAAG,IAAI,CAAC,SAAS,EAAE;SACtB,CAAC;IACN,CAAC;;AAhRL,gCAiRC;AAhRG;;;;WAAoB,CAAC;GAAC"}

package/dist/enqueue_links/enqueue_links.d.ts

@@ -0,0 +1,138 @@
+import { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform } from './shared';
+import { RequestQueue } from '../storages/request_queue';
+import { BatchAddRequestsResult } from '../storages/storage';
+export interface EnqueueLinksOptions {
+    /** Limit the count of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
+    limit?: number;
+    /** An array of URLs to enqueue. */
+    urls: string[];
+    /** A request queue to which the URLs will be enqueued. */
+    requestQueue: RequestQueue;
+    /** A CSS selector matching links to be enqueued. */
+    selector?: string;
+    /**
+     * A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
+     * since the relative URL resolution is done inside the browser automatically.
+     */
+    baseUrl?: string;
+    /**
+     * An array of glob pattern strings or plain objects
+     * containing glob pattern strings matching the URLs to be enqueued.
+     *
+     * The plain objects must include at least the `glob` property, which holds the glob pattern string.
+     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
+     *
+     * The matching is always case-insensitive.
+     * If you need case-sensitive matching, use `regexps` property directly.
+     *
+     * If `globs` is an empty array or `undefined`, then the function
+     * enqueues the links with the same subdomain.
+     */
+    globs?: GlobInput[];
+    /**
+     * An array of regular expressions or plain objects
+     * containing regular expressions matching the URLs to be enqueued.
+     *
+     * The plain objects must include at least the `regexp` property, which holds the regular expression.
+     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
+     *
+     * If `regexps` is an empty array or `undefined`, then the function
+     * enqueues the links with the same subdomain.
+     */
+    regexps?: RegExpInput[];
+    /**
+     * *NOTE:* In future versions of SDK the options will be removed.
+     * Please use `globs` or `regexps` instead.
+     *
+     * An array of {@link PseudoUrl} strings or plain objects
+     * containing {@link PseudoUrl} strings matching the URLs to be enqueued.
+     *
+     * The plain objects must include at least the `purl` property, which holds the pseudo-URL string.
+     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
+     *
+     * With a pseudo-URL string, the matching is always case-insensitive.
+     * If you need case-sensitive matching, use `regexps` property directly.
+     *
+     * If `pseudoUrls` is an empty array or `undefined`, then the function
+     * enqueues the links with the same subdomain.
+     *
+     * @deprecated prefer using `globs` or `regexps` instead
+     */
+    pseudoUrls?: PseudoUrlInput[];
+    /**
+     * Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used
+     * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
+     * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
+     * or to dynamically update or create `userData`.
+     *
+     * For example: by adding `keepUrlFragment: true` to the `request` object, URL fragments will not be removed
+     * when `uniqueKey` is computed.
+     *
+     * **Example:**
+     * ```javascript
+     * {
+     *     transformRequestFunction: (request) => {
+     *         request.userData.foo = 'bar';
+     *         request.keepUrlFragment = true;
+     *         return request;
+     *     }
+     * }
+     * ```
+     *
+     * Note that `transformRequestFunction` has a priority over request options
+     * specified in `globs`, `regexps`, or `pseudoUrls` objects,
+     * and thus some options could be over-written by `transformRequestFunction`.
+     */
+    transformRequestFunction?: RequestTransform;
+    /**
+     * The strategy to use when enqueueing the urls.
+     * @default EnqueueStrategy.SameSubdomain
+     */
+    strategy?: EnqueueStrategy | 'all' | 'same-subdomain' | 'same-hostname';
+}
+export declare enum EnqueueStrategy {
+    /**
+     * Matches any URLs found
+     */
+    All = "all",
+    /**
+     * Matches any URLs that have the same subdomain as the base URL
+     */
+    SameSubdomain = "same-subdomain",
+    /**
+     * Matches any URLs that have the same hostname.
+     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
+     * `https://example.com`.
+     */
+    SameHostname = "same-hostname"
+}
+/**
+ * The function finds elements matching a specific CSS selector (HTML anchor (`<a>`) by default)
+ * either in a Puppeteer/Playwright page, or in a Cheerio object (parsed HTML),
+ * and enqueues the URLs in their `href` attributes to the provided {@link RequestQueue}.
+ * If you're looking to find URLs in JavaScript heavy pages where links are not available
+ * in `href` elements, but rather navigations are triggered in click handlers
+ * see {@link puppeteerUtils.enqueueLinksByClickingElements}.
+ *
+ * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
+ * and override settings of the enqueued {@link Request} objects.
+ *
+ * **Example usage**
+ *
+ * ```javascript
+ * await enqueueLinks({
+ *   page,
+ *   requestQueue,
+ *   selector: 'a.product-detail',
+ *   pseudoUrls: [
+ *       'https://www.example.com/handbags/[.*]',
+ *       'https://www.example.com/purses/[.*]'
+ *   ],
+ * });
+ * ```
+ *
+ * @param options All `enqueueLinks()` parameters are passed via an options object.
+ * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
+ */
+export declare function enqueueLinks(options: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
+//# sourceMappingURL=enqueue_links.d.ts.map

package/dist/enqueue_links/enqueue_links.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"enqueue_links.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/enqueue_links.ts"],"names":[],"mappings":"AAGA,OAAO,EAMH,SAAS,EACT,cAAc,EACd,WAAW,EACX,gBAAgB,EAEnB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE7D,MAAM,WAAW,mBAAmB;IAChC,qHAAqH;IACrH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,IAAI,EAAE,MAAM,EAAE,CAAC;IAEf,0DAA0D;IAC1D,YAAY,EAAE,YAAY,CAAC;IAE3B,oDAAoD;IACpD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;IAEpB;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IAExB;;;;;;;;;;;;;;;;;OAiBG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAE9B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,wBAAwB,CAAC,EAAE,gBAAgB,CAAC;IAE5C;;;OAGG;IACH,QAAQ,CAAC,EAAE,eAAe,GAAG,KAAK,GAAG,gBAAgB,GAAG,eAAe,CAAC;CAC3E;AAED,oBAAY,eAAe;IACvB;;OAEG;IACH,GAAG,QAAQ;IACX;;OAEG;IACH,aAAa,mBAAmB;IAChC;;;;OAIG;IACH,YAAY,kBAAkB;CACjC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAmGhG"}