apify 2.3.1-beta.4 → 3.0.0-alpha.0

package/build/crawlers/statistics.js

@@ -1,331 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const tslib_1 = require("tslib");
-/* eslint-disable max-classes-per-file */
-const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
-const key_value_store_1 = require("../storages/key_value_store");
-const constants_1 = require("../constants");
-const utils_log_1 = (0, tslib_1.__importDefault)(require("../utils_log"));
-const events_1 = (0, tslib_1.__importDefault)(require("../events"));
-/**
- * @ignore
- */
-class Job {
-    constructor() {
-        this.lastRunAt = null;
-        this.runs = 0;
-    }
-    run() {
-        this.lastRunAt = Date.now();
-        return ++this.runs;
-    }
-    finish() {
-        this.durationMillis = Date.now() - this.lastRunAt;
-        return this.durationMillis;
-    }
-    retryCount() {
-        return Math.max(0, this.runs - 1);
-    }
-}
-/**
- * The statistics class provides an interface to collecting and logging run
- * statistics for requests.
- *
- * All statistic information is saved on key value store
- * under the key SDK_CRAWLER_STATISTICS_*, persists between
- * migrations and abort/resurrect
- *
- * @property {StatisticState} state
- *   Current statistic state used for doing calculations on {@link Statistics#calculate} calls
- * @property {number} id
- *   Statistic instance id
- * @property {number[]} requestRetryHistogram
- *   Contains the current retries histogram.
- *   Index 0 means 0 retries, index 2, 2 retries,
- *   and so on
- */
-class Statistics {
-    /**
-     * @param {StatisticsOptions} [options]
-     * @hideconstructor
-     */
-    constructor(options = {}) {
-        (0, ow_1.default)(options, ow_1.default.object.exactShape({
-            logIntervalSecs: ow_1.default.optional.number,
-            logMessage: ow_1.default.optional.string,
-        }));
-        const { logIntervalSecs = 60, logMessage = 'Statistics', } = options;
-        this.log = utils_log_1.default.child({ prefix: 'Statistics' });
-        this.logIntervalMillis = logIntervalSecs * 1000;
-        this.logMessage = logMessage;
-        this.keyValueStore = null;
-        // assign an id while incrementing so it can be saved/restored from KV
-        this.id = Statistics.id++;
-        this.persistStateKey = `SDK_CRAWLER_STATISTICS_${this.id}`;
-        this.listener = this.persistState.bind(this);
-        this.requestRetryHistogram = [];
-        /**
-         * @private
-         * @type {Object<string|number, Job>}
-         */
-        this.requestsInProgress = new Map();
-        // initialize by "resetting"
-        this.reset();
-    }
-    /**
-     * Set the current statistic instance to pristine values
-     */
-    reset() {
-        this.state = {
-            requestsFinished: 0,
-            requestsFailed: 0,
-            requestsRetries: 0,
-            requestsFailedPerMinute: 0,
-            requestsFinishedPerMinute: 0,
-            requestMinDurationMillis: Infinity,
-            requestMaxDurationMillis: 0,
-            requestTotalFailedDurationMillis: 0,
-            requestTotalFinishedDurationMillis: 0,
-            crawlerStartedAt: null,
-            crawlerFinishedAt: null,
-            statsPersistedAt: null,
-            crawlerRuntimeMillis: 0,
-        };
-        this.requestRetryHistogram.length = 0;
-        this.requestsInProgress.clear();
-        this.instanceStart = Date.now();
-        this._teardown();
-    }
-    /**
-     * Starts a job
-     *
-     * @param {number|string} id
-     * @ignore
-     */
-    startJob(id) {
-        let job = this.requestsInProgress.get(id);
-        if (!job)
-            job = new Job();
-        job.run();
-        this.requestsInProgress.set(id, job);
-    }
-    /**
-     * Mark job as finished and sets the state
-     *
-     * @param {number|string} id
-     * @ignore
-     */
-    finishJob(id) {
-        const job = this.requestsInProgress.get(id);
-        if (!job)
-            return;
-        const jobDurationMillis = job.finish();
-        this.state.requestsFinished++;
-        this.state.requestTotalFinishedDurationMillis += jobDurationMillis;
-        this._saveRetryCountForJob(job);
-        if (jobDurationMillis < this.state.requestMinDurationMillis)
-            this.state.requestMinDurationMillis = jobDurationMillis;
-        if (jobDurationMillis > this.state.requestMaxDurationMillis)
-            this.state.requestMaxDurationMillis = jobDurationMillis;
-        this.requestsInProgress.delete(id);
-    }
-    /**
-     * Mark job as failed and sets the state
-     *
-     * @param {number|string} id
-     * @ignore
-     */
-    failJob(id) {
-        const job = this.requestsInProgress.get(id);
-        if (!job)
-            return;
-        this.state.requestTotalFailedDurationMillis += job.finish();
-        this.state.requestsFailed++;
-        this._saveRetryCountForJob(job);
-        this.requestsInProgress.delete(id);
-    }
-    /**
-     * Calculate the current statistics
-     */
-    calculate() {
-        const { requestsFailed, requestsFinished, requestTotalFailedDurationMillis, requestTotalFinishedDurationMillis, } = this.state;
-        const totalMillis = Date.now() - this.instanceStart;
-        const totalMinutes = totalMillis / 1000 / 60;
-        return {
-            requestAvgFailedDurationMillis: Math.round(requestTotalFailedDurationMillis / requestsFailed) || Infinity,
-            requestAvgFinishedDurationMillis: Math.round(requestTotalFinishedDurationMillis / requestsFinished) || Infinity,
-            requestsFinishedPerMinute: Math.round(requestsFinished / totalMinutes) || 0,
-            requestsFailedPerMinute: Math.floor(requestsFailed / totalMinutes) || 0,
-            requestTotalDurationMillis: requestTotalFinishedDurationMillis + requestTotalFailedDurationMillis,
-            requestsTotal: requestsFailed + requestsFinished,
-            crawlerRuntimeMillis: totalMillis,
-        };
-    }
-    /**
-     * Initializes the key value store for persisting the statistics,
-     * displaying the current state in predefined intervals
-     */
-    async startCapturing() {
-        this.keyValueStore = await (0, key_value_store_1.openKeyValueStore)();
-        await this._maybeLoadStatistics();
-        if (this.state.crawlerStartedAt === null) {
-            this.state.crawlerStartedAt = new Date();
-        }
-        events_1.default.on(constants_1.ACTOR_EVENT_NAMES_EX.PERSIST_STATE, this.listener);
-        this.logInterval = setInterval(() => {
-            this.log.info(this.logMessage, {
-                ...this.calculate(),
-                retryHistogram: this.requestRetryHistogram,
-            });
-        }, this.logIntervalMillis);
-    }
-    /**
-     * Stops logging and remove event listeners, then persist
-     */
-    async stopCapturing() {
-        this._teardown();
-        this.state.crawlerFinishedAt = new Date();
-        await this.persistState();
-    }
-    /**
-     * @param {Job} job
-     * @ignore
-     * @protected
-     * @internal
-     */
-    _saveRetryCountForJob(job) {
-        const retryCount = job.retryCount();
-        if (retryCount > 0)
-            this.state.requestsRetries++;
-        this.requestRetryHistogram[retryCount] = this.requestRetryHistogram[retryCount]
-            ? this.requestRetryHistogram[retryCount] + 1
-            : 1;
-    }
-    /**
-     * Persist internal state to the key value store
-     */
-    async persistState() {
-        // this might be called before startCapturing was called without using await, should not crash
-        if (!this.keyValueStore) {
-            return;
-        }
-        this.log.debug('Persisting state', { persistStateKey: this.persistStateKey });
-        await this.keyValueStore.setValue(this.persistStateKey, this.toJSON());
-    }
-    /**
-     * Loads the current statistic from the key value store if any
-     * @ignore
-     * @protected
-     * @internal
-     */
-    async _maybeLoadStatistics() {
-        // this might be called before startCapturing was called without using await, should not crash
-        if (!this.keyValueStore) {
-            return;
-        }
-        const savedState = await this.keyValueStore.getValue(this.persistStateKey);
-        if (!savedState)
-            return;
-        // We saw a run where the requestRetryHistogram was not iterable and crashed
-        // the actor. Adding some logging to monitor this problem in the future.
-        if (!Array.isArray(savedState.requestRetryHistogram)) {
-            this.log.warning('Received invalid state from Key-value store.', {
-                persistStateKey: this.persistStateKey,
-                state: savedState,
-            });
-        }
-        this.log.debug('Recreating state from KeyValueStore', { persistStateKey: this.persistStateKey });
-        this.requestRetryHistogram.push(...savedState.requestRetryHistogram);
-        this.state.requestsFinished = savedState.requestsFinished;
-        this.state.requestsFailed = savedState.requestsFailed;
-        this.state.requestsRetries = savedState.requestsRetries;
-        this.state.requestTotalFailedDurationMillis = savedState.requestTotalFailedDurationMillis;
-        this.state.requestTotalFinishedDurationMillis = savedState.requestTotalFinishedDurationMillis;
-        this.state.requestMinDurationMillis = savedState.requestMinDurationMillis;
-        this.state.requestMaxDurationMillis = savedState.requestMaxDurationMillis;
-        // persisted state uses ISO date strings
-        this.state.crawlerFinishedAt = savedState.crawlerFinishedAt ? new Date(savedState.crawlerFinishedAt) : null;
-        this.state.crawlerStartedAt = savedState.crawlerStartedAt ? new Date(savedState.crawlerStartedAt) : null;
-        this.state.statsPersistedAt = savedState.statsPersistedAt ? new Date(savedState.statsPersistedAt) : null;
-        this.state.crawlerRuntimeMillis = savedState.crawlerRuntimeMillis;
-        this.instanceStart = Date.now() - (this.state.statsPersistedAt - savedState.crawlerLastStartTimestamp);
-        this.log.debug('Loaded from KeyValueStore');
-    }
-    /**
-     * @ignore
-     * @protected
-     * @internal
-     */
-    _teardown() {
-        // this can be called before a call to startCapturing happens (or in a 'finally' block)
-        events_1.default.removeListener(constants_1.ACTOR_EVENT_NAMES_EX.PERSIST_STATE, this.listener);
-        if (this.logInterval) {
-            clearInterval(this.logInterval);
-            this.logInterval = null;
-        }
-    }
-    /**
-     * Make this class serializable when called with `JSON.stringify(statsInstance)` directly
-     * or through `keyValueStore.setValue('KEY', statsInstance)`
-     *
-     * @returns {StatisticPersistedState & StatisticState}
-     */
-    toJSON() {
-        // merge all the current state information that can be used from the outside
-        // without the need to reconstruct for the sake of stats.calculate()
-        // omit duplicated information
-        return {
-            ...this.state,
-            crawlerLastStartTimestamp: this.instanceStart,
-            crawlerFinishedAt: this.state.crawlerFinishedAt ? new Date(this.state.crawlerFinishedAt).toISOString() : null,
-            crawlerStartedAt: this.state.crawlerStartedAt ? new Date(this.state.crawlerStartedAt).toISOString() : null,
-            requestRetryHistogram: this.requestRetryHistogram,
-            statsId: this.id,
-            statsPersistedAt: new Date().toISOString(),
-            ...this.calculate(),
-        };
-    }
-}
-Statistics.id = 0;
-exports.default = Statistics;
-/**
- * @ignore
- * @typedef StatisticsOptions
- * @property {number} [logIntervalSecs]
- * @property {string} [logMessage]
- */
-/**
- * Format of the persisted stats
- *
- * @typedef StatisticPersistedState
- * @property {number[]} requestRetryHistogram
- * @property {number} statsId
- * @property {number} requestAvgFailedDurationMillis
- * @property {number} requestAvgFinishedDurationMillis
- * @property {number} requestsFinishedPerMinute
- * @property {number} requestsFailedPerMinute
- * @property {number} requestTotalDurationMillis
- * @property {number} requestsTotal
- * @property {number} crawlerRuntimeMillis
- * @property {number} crawlerLastStartTimestamp
- * @property {string} statsPersistedAt
- */
-/**
- * Contains the statistics state
- *
- * @typedef StatisticState
- * @property {number} requestsFinished
- * @property {number} requestsFailed
- * @property {number} requestsRetries
- * @property {number} requestsFailedPerMinute
- * @property {number} requestsFinishedPerMinute
- * @property {number} requestMinDurationMillis
- * @property {number} requestMaxDurationMillis
- * @property {number} requestTotalFailedDurationMillis
- * @property {number} requestTotalFinishedDurationMillis
- * @property {Date|string|null} crawlerStartedAt
- * @property {Date|string|null} crawlerFinishedAt
- * @property {number} crawlerRuntimeMillis
- * @property {Date|string|null} statsPersistedAt
- */
-//# sourceMappingURL=statistics.js.map

package/build/crawlers/statistics.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"statistics.js","sourceRoot":"","sources":["../../src/crawlers/statistics.js"],"names":[],"mappings":";;;AAAA,yCAAyC;AACzC,yDAAoB;AACpB,iEAAgE;AAChE,4CAAoD;AACpD,0EAAsC;AACtC,oEAA+B;AAE/B;;GAEG;AACH,MAAM,GAAG;IACL;QACI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;IAClB,CAAC;IAED,GAAG;QACC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC;IACvB,CAAC;IAED,MAAM;QACF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;QAClD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC/B,CAAC;IAED,UAAU;QACN,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IACtC,CAAC;CACJ;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU;IACZ;;;OAGG;IACH,YAAY,OAAO,GAAG,EAAE;QACpB,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YACnC,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;SACjC,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,eAAe,GAAG,EAAE,EACpB,UAAU,GAAG,YAAY,GAC5B,GAAG,OAAO,CAAC;QAEZ,IAAI,CAAC,GAAG,GAAG,mBAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,iBAAiB,GAAG,eAAe,GAAG,IAAI,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,sEAAsE;QACtE,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,EAAE,EAAE,CAAC;QAC1B,IAAI,CAAC,eAAe,GAAG,0BAA0B,IAAI,CAAC,EAAE,EAAE,CAAC;QAC3D,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,IAAI,CAAC,qBAAqB,GAAG,EAAE,CAAC;QAEhC;;;WAGG;QACH,IAAI,CAAC,kBAAkB,GAAG,IAAI,GAAG,EAAE,CAAC;QAEpC,4BAA4B;QAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED;;OAEG;IACH,KAAK;QACD,IAAI,CAAC,KAAK,GAAG;YACT,gBAAgB,EAAE,CAAC;YACnB,cAAc,EAAE,CAAC;YACjB,eAAe,EAAE,CAAC;YAClB,uBAAuB,EAAE,CAAC;YAC1B,yBAAyB,EAAE,CAAC;YAC5B,wBAAwB,EAAE,QAAQ;YAClC,wBAAwB,EAAE,CAAC;YAC3B,gCAAgC,EAAE,CAAC;YACnC,kCAAkC,EAAE,CAAC;YACrC,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;YACvB,gBAAgB,EAAE,IAAI;YACtB,oBAAoB,EAAE,CAAC;SAC1B,CAAC;QAEF,IAAI,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEhC,IAAI,CAAC,SAAS,EAAE,CAAC;IACrB,CAAC;IAED;;;;;OAKG;IACH,QAAQ,CAAC,EAAE;QACP,IAAI,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAC1B,GAAG,CAAC,GAAG,EAAE,CAAC;QACV,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IAED;;;;;OAKG;IACH,SAAS,CAAC,EAAE;QACR,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,iBAAiB,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,KAAK,CAAC,kCAAkC,IAAI,iBAAiB,CAAC;QACnE,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YAAE,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QACrH,IAAI,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB;YAAE,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,iBAAiB,CAAC;QACrH,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;;;;OAKG;IACH,OAAO,CAAC,EAAE;QACN,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,IAAI,CAAC,KAAK,CAAC,gCAAgC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC5D,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC5B,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,SAAS;QACL,MAAM,EACF,cAAc,EACd,gBAAgB,EAChB,gCAAgC,EAChC,kCAAkC,GACrC,GAAG,IAAI,CAAC,KAAK,CAAC;QACf,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;QACpD,MAAM,YAAY,GAAG,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;QAE7C,OAAO;YACH,8BAA8B,EAAE,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,cAAc,CAAC,IAAI,QAAQ;YACzG,gCAAgC,EAAE,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,gBAAgB,CAAC,IAAI,QAAQ;YAC/G,yBAAyB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,YAAY,CAAC,IAAI,CAAC;YAC3E,uBAAuB,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC;YACvE,0BAA0B,EAAE,kCAAkC,GAAG,gCAAgC;YACjG,aAAa,EAAE,cAAc,GAAG,gBAAgB;YAChD,oBAAoB,EAAE,WAAW;SACpC,CAAC;IACN,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAChB,IAAI,CAAC,aAAa,GAAG,MAAM,IAAA,mCAAiB,GAAE,CAAC;QAE/C,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAElC,IAAI,IAAI,CAAC,KAAK,CAAC,gBAAgB,KAAK,IAAI,EAAE;YACtC,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,IAAI,IAAI,EAAE,CAAC;SAC5C;QAED,gBAAM,CAAC,EAAE,CAAC,gCAAoB,CAAC,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE7D,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC3B,GAAG,IAAI,CAAC,SAAS,EAAE;gBACnB,cAAc,EAAE,IAAI,CAAC,qBAAqB;aAC7C,CAAC,CAAC;QACP,CAAC,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,IAAI,CAAC,SAAS,EAAE,CAAC;QAEjB,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC;QAE1C,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,qBAAqB,CAAC,GAAG;QACrB,MAAM,UAAU,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;QACpC,IAAI,UAAU,GAAG,CAAC;YAAE,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC;QACjD,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC;YAC3E,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,UAAU,CAAC,GAAG,CAAC;YAC5C,CAAC,CAAC,CAAC,CAAC;IACZ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QACd,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACrB,OAAO;SACV;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAE9E,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,oBAAoB;QACtB,8FAA8F;QAC9F,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACrB,OAAO;SACV;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAE3E,IAAI,CAAC,UAAU;YAAE,OAAO;QAExB,4EAA4E;QAC5E,wEAAwE;QACxE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,qBAAqB,CAAC,EAAE;YAClD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,8CAA8C,EAAE;gBAC7D,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,KAAK,EAAE,UAAU;aACpB,CAAC,CAAC;SACN;QAED,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,qCAAqC,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAEjG,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,qBAAqB,CAAC,CAAC;QACrE,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC;QAC1D,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,UAAU,CAAC,cAAc,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,UAAU,CAAC,eAAe,CAAC;QAExD,IAAI,CAAC,KAAK,CAAC,gCAAgC,GAAG,UAAU,CAAC,gCAAgC,CAAC;QAC1F,IAAI,CAAC,KAAK,CAAC,kCAAkC,GAAG,UAAU,CAAC,kCAAkC,CAAC;QAC9F,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,IAAI,CAAC,KAAK,CAAC,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC;QAC1E,wCAAwC;QACxC,IAAI,CAAC,KAAK,CAAC,iBAAiB,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5G,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACzG,IAAI,CAAC,KAAK,CAAC,oBAAoB,GAAG,UAAU,CAAC,oBAAoB,CAAC;QAClE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC,yBAAyB,CAAC,CAAC;QAEvG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAChD,CAAC;IAED;;;;OAIG;IACH,SAAS;QACL,uFAAuF;QACvF,gBAAM,CAAC,cAAc,CAAC,gCAAoB,CAAC,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEzE,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAChC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;SAC3B;IACL,CAAC;IAED;;;;;OAKG;IACH,MAAM;QACF,4EAA4E;QAC5E,oEAAoE;QACpE,8BAA8B;QAC9B,OAAO;YACH,GAAG,IAAI,CAAC,KAAK;YACb,yBAAyB,EAAE,IAAI,CAAC,aAAa;YAC7C,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAC7G,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI;YAC1G,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;YACjD,OAAO,EAAE,IAAI,CAAC,EAAE;YAChB,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC1C,GAAG,IAAI,CAAC,SAAS,EAAE;SACtB,CAAC;IACN,CAAC;CACJ;AAED,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC;AAElB,kBAAe,UAAU,CAAC;AAE1B;;;;;GAKG;AAEH;;;;;;;;;;;;;;;GAeG;AAEH;;;;;;;;;;;;;;;;;GAiBG"}

package/build/enqueue_links/click_elements.d.ts

@@ -1,179 +0,0 @@
-/**
- * The function finds elements matching a specific CSS selector in a Puppeteer page,
- * clicks all those elements using a mouse move and a left mouse button click and intercepts
- * all the navigation requests that are subsequently produced by the page. The intercepted
- * requests, including their methods, headers and payloads are then enqueued to a provided
- * {@link RequestQueue}. This is useful to crawl JavaScript heavy pages where links are not available
- * in `href` elements, but rather navigations are triggered in click handlers.
- * If you're looking to find URLs in `href` attributes of the page, see {@link utils#enqueueLinks}.
- *
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
- * and override settings of the enqueued {@link Request} objects.
- *
- * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
- * such as changing the Z-index of elements being clicked and their visibility. Therefore,
- * it is recommended to only use this function as the last operation in the page.
- *
- * **USING HEADFUL BROWSER**: When using a headful browser, this function will only be able to click elements
- * in the focused tab, effectively limiting concurrency to 1. In headless mode, full concurrency can be achieved.
- *
- * **PERFORMANCE**: Clicking elements with a mouse and intercepting requests is not a low level operation
- * that takes nanoseconds. It's not very CPU intensive, but it takes time. We strongly recommend limiting
- * the scope of the clicking as much as possible by using a specific selector that targets only the elements
- * that you assume or know will produce a navigation. You can certainly click everything by using
- * the `*` selector, but be prepared to wait minutes to get results on a large and complex page.
- *
- * **Example usage**
- *
- * ```javascript
- * await Apify.utils.puppeteer.enqueueLinksByClickingElements({
- *   page,
- *   requestQueue,
- *   selector: 'a.product-detail',
- *   pseudoUrls: [
- *       'https://www.example.com/handbags/[.*]'
- *       'https://www.example.com/purses/[.*]'
- *   ],
- * });
- * ```
- * @param {object} options
- *   All `enqueueLinksByClickingElements()` parameters are passed
- *   via an options object with the following keys:
- * @param {Page} options.page
- *   Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
- * @param {RequestQueue} options.requestQueue
- *   A request queue to which the URLs will be enqueued.
- * @param {string} options.selector
- *   A CSS selector matching elements to be clicked on. Unlike in {@link utils#enqueueLinks}, there is no default
- *   value. This is to prevent suboptimal use of this function by using it too broadly.
- * @param {Array<(string|RegExp|Object<string, *>)>} [options.pseudoUrls]
- *   An array of {@link PseudoUrl}s matching the URLs to be enqueued,
- *   or an array of strings or RegExps or plain Objects from which the {@link PseudoUrl}s can be constructed.
- *
- *   The plain objects must include at least the `purl` property, which holds the pseudo-URL string or RegExp.
- *   All remaining keys will be used as the `requestTemplate` argument of the {@link PseudoUrl} constructor,
- *   which lets you specify special properties for the enqueued {@link Request} objects.
- *
- *   If `pseudoUrls` is an empty array, `null` or `undefined`, then the function
- *   enqueues all links found on the page.
- * @param {object} [options.clickOptions]
- *   click options for use in Puppeteer's click handler
- * @param {number} [options.clickOptions.clickCount]
- *   Number of clicks to be executed. Defaults to 1
- * @param {number} [options.clickOptions.delay]
- *   Time to wait between mousedown and mouseup in milliseconds. Defaults to 0
- * @param {RequestTransform} [options.transformRequestFunction]
- *   Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used
- *   to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
- *   when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
- *   or to dynamically update or create `userData`.
- *
- *   For example: by adding `useExtendedUniqueKey: true` to the `request` object, `uniqueKey` will be computed from
- *   a combination of `url`, `method` and `payload` which enables crawling of websites that navigate using form submits
- *   (POST requests).
- *
- *   **Example:**
- *   ```javascript
- *   {
- *       transformRequestFunction: (request) => {
- *           request.userData.foo = 'bar';
- *           request.useExtendedUniqueKey = true;
- *           return request;
- *       }
- *   }
- *   ```
- * @param {number} [options.waitForPageIdleSecs=1]
- *   Clicking in the page triggers various asynchronous operations that lead to new URLs being shown
- *   by the browser. It could be a simple JavaScript redirect or opening of a new tab in the browser.
- *   These events often happen only some time after the actual click. Requests typically take milliseconds
- *   while new tabs open in hundreds of milliseconds.
- *
- *   To be able to capture all those events, the `enqueueLinksByClickingElements()` function repeatedly waits
- *   for the `waitForPageIdleSecs`. By repeatedly we mean that whenever a relevant event is triggered, the timer
- *   is restarted. As long as new events keep coming, the function will not return, unless
- *   the below `maxWaitForPageIdleSecs` timeout is reached.
- *
- *   You may want to reduce this for example when you're sure that your clicks do not open new tabs,
- *   or increase when you're not getting all the expected URLs.
- * @param {number} [options.maxWaitForPageIdleSecs=5]
- *   This is the maximum period for which the function will keep tracking events, even if more events keep coming.
- *   Its purpose is to prevent a deadlock in the page by periodic events, often unrelated to the clicking itself.
- *   See `waitForPageIdleSecs` above for an explanation.
- * @return {Promise<Array<QueueOperationInfo>>}
- *   Promise that resolves to an array of {@link QueueOperationInfo} objects.
- * @memberOf puppeteer
- * @name enqueueLinksByClickingElements
- * @function
- */
-export function enqueueLinksByClickingElements(options: {
-    page: Page;
-    requestQueue: RequestQueue;
-    selector: string;
-    pseudoUrls?: (string | RegExp | {
-        [x: string]: any;
-    })[] | undefined;
-    clickOptions?: {
-        clickCount?: number | undefined;
-        delay?: number | undefined;
-    } | undefined;
-    transformRequestFunction?: RequestTransform | undefined;
-    waitForPageIdleSecs?: number | undefined;
-    maxWaitForPageIdleSecs?: number | undefined;
-}): Promise<Array<QueueOperationInfo>>;
-/**
- * Clicks all elements of given page matching given selector.
- * Catches and intercepts all initiated navigation requests and opened pages.
- * Returns a list of all target URLs.
- *
- * @param {object} options
- * @param {Page} options.page
- * @param {string} options.selector
- * @param {number} [options.waitForPageIdleMillis]
- * @param {number} [options.maxWaitForPageIdleMillis]
- * @param {object} [clickOptions]
- * @param {number} [clickOptions.clickCount]
- * @param {number} [clickOptions.delay]
- * @return {Promise<Array<*>>}
- * @ignore
- */
-export function clickElementsAndInterceptNavigationRequests(options: {
-    page: Page;
-    selector: string;
-    waitForPageIdleMillis?: number | undefined;
-    maxWaitForPageIdleMillis?: number | undefined;
-}): Promise<Array<any>>;
-/**
- * We're only interested in pages created by the page we're currently clicking in.
- * There will generally be a lot of other targets being created in the browser.
- * @param {Page} page
- * @param {Target} target
- * @return {boolean}
- */
-export function isTargetRelevant(page: Page, target: Target): boolean;
-/**
- * Click all elements matching the given selector. To be able to do this using
- * Puppeteer's `.click()` we need to make sure the elements are reachable by mouse,
- * so we first move them to the top of the page's stacking context and then click.
- * We do all in series to prevent elements from hiding one another. Therefore,
- * for large element sets, this will take considerable amount of time.
- *
- * @param {Page} page
- * @param {string} selector
- * @param {object} [clickOptions]
- * @param {number} [clickOptions.clickCount]
- * @param {number} [clickOptions.delay]
- * @return {Promise<void>}
- * @ignore
- */
-export function clickElements(page: Page, selector: string, clickOptions?: {
-    clickCount?: number | undefined;
-    delay?: number | undefined;
-} | undefined): Promise<void>;
-// @ts-ignore optional peer dependency
-import { Page } from "puppeteer";
-import { RequestQueue } from "../storages/request_queue";
-import { RequestTransform } from "./shared";
-import { QueueOperationInfo } from "../storages/request_queue";
-// @ts-ignore optional peer dependency
-import { Target } from "puppeteer";
-//# sourceMappingURL=click_elements.d.ts.map

package/build/enqueue_links/click_elements.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"click_elements.d.ts","sourceRoot":"","sources":["../../src/enqueue_links/click_elements.js"],"names":[],"mappings":"AAkBA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0GG;AACH;IAlEyB,IAAI,EAAlB,IAAI;IAEkB,YAAY,EAAlC,YAAY;IAEI,QAAQ,EAAxB,MAAM;IAG6C,UAAU;;;IAU5C,YAAY;;;;IAMF,wBAAwB;IAoBlC,mBAAmB;IAanB,sBAAsB;IAItC,QAAQ,MAAM,kBAAkB,CAAC,CAAC,CA8C7C;AAED;;;;;;;;;;;;;;;GAeG;AACH;IAVyB,IAAI,EAAlB,IAAI;IACY,QAAQ,EAAxB,MAAM;IACW,qBAAqB;IACrB,wBAAwB;IAIxC,QAAQ,UAAQ,CAAC,CAoC5B;AA6DD;;;;;;GAMG;AACH,uCAJW,IAAI,UACJ,MAAM,GACL,OAAO,CAKlB;AA2CD;;;;;;;;;;;;;;GAcG;AACH,oCARW,IAAI,YACJ,MAAM;;;gBAIL,QAAQ,IAAI,CAAC,CAyBxB"}