@crawlee/basic 3.0.0-alpha.2 → 3.0.0-alpha.20

package/internals/basic-crawler.js

@@ -0,0 +1,690 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BasicCrawler = void 0;
+const tslib_1 = require("tslib");
+const log_1 = tslib_1.__importDefault(require("@apify/log"));
+const timeout_1 = require("@apify/timeout");
+const utilities_1 = require("@apify/utilities");
+const core_1 = require("@crawlee/core");
+const ow_1 = tslib_1.__importStar(require("ow"));
+/**
+ * Since there's no set number of seconds before the container is terminated after
+ * a migration event, we need some reasonable number to use for RequestList persistence.
+ * Once a migration event is received, the Crawler will be paused and it will wait for
+ * this long before persisting the RequestList state. This should allow most healthy
+ * requests to finish and be marked as handled, thus lowering the amount of duplicate
+ * results after migration.
+ * @ignore
+ */
+const SAFE_MIGRATION_WAIT_MILLIS = 20000;
+/**
+ * Provides a simple framework for parallel crawling of web pages.
+ * The URLs to crawl are fed either from a static list of URLs
+ * or from a dynamic queue of URLs enabling recursive crawling of websites.
+ *
+ * `BasicCrawler` is a low-level tool that requires the user to implement the page
+ * download and data extraction functionality themselves.
+ * If you want a crawler that already facilitates this functionality,
+ * please consider using {@link CheerioCrawler}, {@link PuppeteerCrawler} or {@link PlaywrightCrawler}.
+ *
+ * `BasicCrawler` invokes the user-provided {@link BasicCrawlerOptions.requestHandler}
+ * for each {@link Request} object, which represents a single URL to crawl.
+ * The {@link Request} objects are fed from the {@link RequestList} or the {@link RequestQueue}
+ * instances provided by the {@link BasicCrawlerOptions.requestList} or {@link BasicCrawlerOptions.requestQueue}
+ * constructor options, respectively.
+ *
+ * If both {@link BasicCrawlerOptions.requestList} and {@link BasicCrawlerOptions.requestQueue} options are used,
+ * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
+ * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ *
+ * The crawler finishes if there are no more {@link Request} objects to crawl.
+ *
+ * New requests are only dispatched when there is enough free CPU and memory available,
+ * using the functionality provided by the {@link AutoscaledPool} class.
+ * All {@link AutoscaledPool} configuration options can be passed to the `autoscaledPoolOptions`
+ * parameter of the `BasicCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
+ * {@link AutoscaledPool} options are available directly in the `BasicCrawler` constructor.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const { gotScraping } = require('got-scraping');
+ *
+ * // Prepare a list of URLs to crawl
+ * const requestList = new RequestList({
+ *   sources: [
+ *       { url: 'http://www.example.com/page-1' },
+ *       { url: 'http://www.example.com/page-2' },
+ *   ],
+ * });
+ * await requestList.initialize();
+ *
+ * // Crawl the URLs
+ * const crawler = new BasicCrawler({
+ *     requestList,
+ *     handleRequestFunction: async ({ request }) => {
+ *         // 'request' contains an instance of the Request class
+ *         // Here we simply fetch the HTML of the page and store it to a dataset
+ *         const { body } = await gotScraping({
+ *             url: request.url,
+ *             method: request.method,
+ *             body: request.payload,
+ *             headers: request.headers,
+ *         });
+ *
+ *         await Actor.pushData({
+ *             url: request.url,
+ *             html: body,
+ *         })
+ *     },
+ * });
+ *
+ * await crawler.run();
+ * ```
+ * @category Crawlers
+ */
+class BasicCrawler {
+    /**
+     * All `BasicCrawler` parameters are passed via an options object.
+     */
+    constructor(options, config = core_1.Configuration.getGlobalConfig()) {
+        Object.defineProperty(this, "config", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: config
+        });
+        /**
+         * Static list of URLs to be processed.
+         */
+        Object.defineProperty(this, "stats", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request}s.
+         * Either `requestList` or `requestQueue` option must be provided (or both).
+         * Only available if used by the crawler.
+         */
+        Object.defineProperty(this, "requestList", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
+         * A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request}s.
+         * Either `requestList` or `requestQueue` option must be provided (or both).
+         * Only available if used by the crawler.
+         */
+        Object.defineProperty(this, "requestQueue", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session}s.
+         * Only available if used by the crawler.
+         */
+        Object.defineProperty(this, "sessionPool", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
+         * Note that this property is only initialized after calling the {@link BasicCrawler.run} function.
+         * You can use it to change the concurrency settings on the fly,
+         * to pause the crawler by calling {@link AutoscaledPool.pause}
+         * or to abort it by calling {@link AutoscaledPool.abort}.
+         */
+        Object.defineProperty(this, "autoscaledPool", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "log", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "requestHandler", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "failedRequestHandler", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "requestHandlerTimeoutMillis", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "internalTimeoutMillis", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxRequestRetries", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "handledRequestsCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "sessionPoolOptions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "useSessionPool", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "crawlingContexts", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+        Object.defineProperty(this, "autoscaledPoolOptions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "events", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        (0, ow_1.default)(options, 'BasicCrawlerOptions', ow_1.default.object.exactShape(BasicCrawler.optionsShape));
+        const { requestList, requestQueue, maxRequestRetries = 3, maxRequestsPerCrawl, autoscaledPoolOptions = {}, sessionPoolOptions = {}, useSessionPool = true, 
+        // AutoscaledPool shorthands
+        minConcurrency, maxConcurrency, 
+        // internal
+        log = log_1.default.child({ prefix: this.constructor.name }), 
+        // Old and new request handler methods
+        handleRequestFunction, requestHandler, handleRequestTimeoutSecs, requestHandlerTimeoutSecs, handleFailedRequestFunction, failedRequestHandler, } = options;
+        this.requestList = requestList;
+        this.requestQueue = requestQueue;
+        this.log = log;
+        this.events = config.getEventManager();
+        this._handlePropertyNameChange({
+            newName: 'requestHandler',
+            oldName: 'handleRequestFunction',
+            propertyKey: 'requestHandler',
+            newProperty: requestHandler,
+            oldProperty: handleRequestFunction,
+        });
+        this._handlePropertyNameChange({
+            newName: 'failedRequestHandler',
+            oldName: 'handleFailedRequestFunction',
+            propertyKey: 'failedRequestHandler',
+            newProperty: failedRequestHandler,
+            oldProperty: handleFailedRequestFunction,
+            allowUndefined: true,
+        });
+        let newRequestHandlerTimeout;
+        if (!handleRequestTimeoutSecs) {
+            if (!requestHandlerTimeoutSecs) {
+                newRequestHandlerTimeout = 60000;
+            }
+            else {
+                newRequestHandlerTimeout = requestHandlerTimeoutSecs * 1000;
+            }
+        }
+        else if (requestHandlerTimeoutSecs) {
+            newRequestHandlerTimeout = requestHandlerTimeoutSecs * 1000;
+        }
+        this._handlePropertyNameChange({
+            newName: 'requestHandlerTimeoutSecs',
+            oldName: 'handleRequestTimeoutSecs',
+            propertyKey: 'requestHandlerTimeoutMillis',
+            newProperty: newRequestHandlerTimeout,
+            oldProperty: handleRequestTimeoutSecs ? handleRequestTimeoutSecs * 1000 : undefined,
+        });
+        const tryEnv = (val) => (val == null ? null : +val);
+        // allow at least 5min for internal timeouts
+        this.internalTimeoutMillis = tryEnv(process.env.CRAWLEE_INTERNAL_TIMEOUT) ?? Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
+        // override the default internal timeout of request queue to respect `requestHandlerTimeoutMillis`
+        if (this.requestQueue) {
+            this.requestQueue.internalTimeoutMillis = this.internalTimeoutMillis;
+        }
+        this.maxRequestRetries = maxRequestRetries;
+        this.handledRequestsCount = 0;
+        this.stats = new core_1.Statistics({ logMessage: `${log.getOptions().prefix} request statistics:`, config });
+        this.sessionPoolOptions = {
+            ...sessionPoolOptions,
+            log,
+        };
+        this.useSessionPool = useSessionPool;
+        this.crawlingContexts = new Map();
+        const maxSignedInteger = 2 ** 31 - 1;
+        if (this.requestHandlerTimeoutMillis > maxSignedInteger) {
+            log.warning(`requestHandlerTimeoutMillis ${this.requestHandlerTimeoutMillis}`
+                + `does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`);
+            this.requestHandlerTimeoutMillis = maxSignedInteger;
+        }
+        let shouldLogMaxPagesExceeded = true;
+        const isMaxPagesExceeded = () => maxRequestsPerCrawl && maxRequestsPerCrawl <= this.handledRequestsCount;
+        const { isFinishedFunction } = autoscaledPoolOptions;
+        const basicCrawlerAutoscaledPoolConfiguration = {
+            minConcurrency,
+            maxConcurrency,
+            runTaskFunction: this._runTaskFunction.bind(this),
+            isTaskReadyFunction: async () => {
+                if (isMaxPagesExceeded()) {
+                    if (shouldLogMaxPagesExceeded) {
+                        log.info('Crawler reached the maxRequestsPerCrawl limit of '
+                            + `${maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
+                        shouldLogMaxPagesExceeded = false;
+                    }
+                    return false;
+                }
+                return this._isTaskReadyFunction();
+            },
+            isFinishedFunction: async () => {
+                if (isMaxPagesExceeded()) {
+                    log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${maxRequestsPerCrawl} requests `
+                        + 'and all requests that were in progress at that time have now finished. '
+                        + `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`);
+                    return true;
+                }
+                const isFinished = isFinishedFunction
+                    ? await isFinishedFunction()
+                    : await this._defaultIsFinishedFunction();
+                if (isFinished) {
+                    const reason = isFinishedFunction
+                        ? 'Crawler\'s custom isFinishedFunction() returned true, the crawler will shut down.'
+                        : 'All the requests from request list and/or request queue have been processed, the crawler will shut down.';
+                    log.info(reason);
+                }
+                return isFinished;
+            },
+            log,
+        };
+        this.autoscaledPoolOptions = { ...autoscaledPoolOptions, ...basicCrawlerAutoscaledPoolConfiguration };
+        // Attach a listener to handle migration and aborting events gracefully.
+        this.events.on("migrating" /* MIGRATING */, this._pauseOnMigration.bind(this));
+        this.events.on("aborting" /* ABORTING */, this._pauseOnMigration.bind(this));
+    }
+    /**
+     * Runs the crawler. Returns a promise that gets resolved once all the requests are processed.
+     */
+    async run() {
+        await this._init();
+        await this.stats.startCapturing();
+        try {
+            await this.autoscaledPool.run();
+        }
+        finally {
+            await this.teardown();
+            await this.stats.stopCapturing();
+        }
+        const finalStats = this.stats.calculate();
+        const stats = {
+            requestsFinished: this.stats.state.requestsFinished,
+            requestsFailed: this.stats.state.requestsFailed,
+            retryHistogram: this.stats.requestRetryHistogram,
+            ...finalStats,
+        };
+        this.log.info('Final request statistics:', stats);
+        return stats;
+    }
+    async getRequestQueue() {
+        this.requestQueue ?? (this.requestQueue = await core_1.RequestQueue.open());
+        return this.requestQueue;
+    }
+    /**
+     * Adds requests to be processed by the crawler
+     * @param requests The requests to add
+     * @param options Options for the request queue
+     */
+    async addRequests(requests, options = {}) {
+        (0, ow_1.default)(requests, ow_1.default.array.ofType(ow_1.default.any(ow_1.default.string, ow_1.default.object.partialShape({
+            url: ow_1.default.string,
+            id: ow_1.default.undefined,
+        }))));
+        (0, ow_1.default)(options, ow_1.default.object.exactShape({
+            forefront: ow_1.default.optional.boolean,
+        }));
+        const requestQueue = await this.getRequestQueue();
+        const builtRequests = (0, core_1.createRequests)(requests);
+        return requestQueue.addRequests(builtRequests);
+    }
+    async _init() {
+        // Initialize AutoscaledPool before awaiting _loadHandledRequestCount(),
+        // so that the caller can get a reference to it before awaiting the promise returned from run()
+        // (otherwise there would be no way)
+        this.autoscaledPool = new core_1.AutoscaledPool(this.autoscaledPoolOptions, this.config);
+        if (this.useSessionPool) {
+            this.sessionPool = await core_1.SessionPool.open(this.sessionPoolOptions);
+            // Assuming there are not more than 20 browsers running at once;
+            this.sessionPool.setMaxListeners(20);
+        }
+        await this._loadHandledRequestCount();
+    }
+    async _runRequestHandler(crawlingContext) {
+        await this.requestHandler(crawlingContext);
+    }
+    async _pauseOnMigration() {
+        if (this.autoscaledPool) {
+            // if run wasn't called, this is going to crash
+            await this.autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS)
+                .catch((err) => {
+                if (err.message.includes('running tasks did not finish')) {
+                    this.log.error('The crawler was paused due to migration to another host, '
+                        + 'but some requests did not finish in time. Those requests\' results may be duplicated.');
+                }
+                else {
+                    throw err;
+                }
+            });
+        }
+        const requestListPersistPromise = (async () => {
+            if (this.requestList) {
+                if (await this.requestList.isFinished())
+                    return;
+                await this.requestList.persistState()
+                    .catch((err) => {
+                    if (err.message.includes('Cannot persist state.')) {
+                        this.log.error('The crawler attempted to persist its request list\'s state and failed due to missing or '
+                            + 'invalid config. Make sure to use either RequestList.open() or the "stateKeyPrefix" option of RequestList '
+                            + 'constructor to ensure your crawling state is persisted through host migrations and restarts.');
+                    }
+                    else {
+                        this.log.exception(err, 'An unexpected error occured when the crawler '
+                            + 'attempted to persist its request list\'s state.');
+                    }
+                });
+            }
+        })();
+        await Promise.all([
+            requestListPersistPromise,
+            this.stats.persistState(),
+        ]);
+    }
+    /**
+     * Fetches request from either RequestList or RequestQueue. If request comes from a RequestList
+     * and RequestQueue is present then enqueues it to the queue first.
+     */
+    async _fetchNextRequest() {
+        if (!this.requestList)
+            return this.requestQueue.fetchNextRequest();
+        const request = await this.requestList.fetchNextRequest();
+        if (!this.requestQueue)
+            return request;
+        if (!request)
+            return this.requestQueue.fetchNextRequest();
+        try {
+            await this.requestQueue.addRequest(request, { forefront: true });
+        }
+        catch (err) {
+            // If requestQueue.addRequest() fails here then we must reclaim it back to
+            // the RequestList because probably it's not yet in the queue!
+            this.log.error('Adding of request from the RequestList to the RequestQueue failed, reclaiming request back to the list.', { request });
+            await this.requestList.reclaimRequest(request);
+            return null;
+        }
+        await this.requestList.markRequestHandled(request);
+        return this.requestQueue.fetchNextRequest();
+    }
+    /**
+     * Wrapper around requestHandler that fetches requests from RequestList/RequestQueue
+     * then retries them in a case of an error, etc.
+     */
+    async _runTaskFunction() {
+        const source = this.requestQueue || this.requestList || await this.getRequestQueue();
+        let request;
+        let session;
+        await this._timeoutAndRetry(async () => {
+            request = await this._fetchNextRequest();
+        }, this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+        (0, timeout_1.tryCancel)();
+        if (this.useSessionPool) {
+            await this._timeoutAndRetry(async () => {
+                session = await this.sessionPool.getSession();
+            }, this.internalTimeoutMillis, `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+        }
+        (0, timeout_1.tryCancel)();
+        if (!request)
+            return;
+        // Reset loadedUrl so an old one is not carried over to retries.
+        request.loadedUrl = undefined;
+        const statisticsId = request.id || request.uniqueKey;
+        this.stats.startJob(statisticsId);
+        // Shared crawling context
+        // @ts-expect-error It is assignable, but TS says otherwise...
+        const crawlingContext = {
+            id: (0, utilities_1.cryptoRandomObjectId)(10),
+            crawler: this,
+            log: this.log,
+            request,
+            session,
+            enqueueLinks: async (enqueueOptions) => {
+                return (0, core_1.enqueueLinks)({
+                    ...enqueueOptions,
+                    requestQueue: await this.getRequestQueue(),
+                });
+            },
+        };
+        this.crawlingContexts.set(crawlingContext.id, crawlingContext);
+        try {
+            await (0, timeout_1.addTimeoutToPromise)(() => this._runRequestHandler(crawlingContext), this.requestHandlerTimeoutMillis, `handleRequestFunction timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds (${request.id}).`);
+            await this._timeoutAndRetry(() => source.markRequestHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+            this.stats.finishJob(statisticsId);
+            this.handledRequestsCount++;
+            // reclaim session if request finishes successfully
+            session?.markGood();
+        }
+        catch (err) {
+            try {
+                await this._timeoutAndRetry(() => this._requestFunctionErrorHandler(err, crawlingContext, source), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+            }
+            catch (secondaryError) {
+                this.log.exception(secondaryError, 'runTaskFunction error handler threw an exception. '
+                    + 'This places the crawler and its underlying storages into an unknown state and crawling will be terminated. '
+                    + 'This may have happened due to an internal error of Apify\'s API or due to a misconfigured crawler. '
+                    + 'If you are sure that there is no error in your code, selecting "Restart on error" in the actor\'s settings'
+                    + 'will make sure that the run continues where it left off, if programmed to handle restarts correctly.');
+                throw secondaryError;
+            }
+        }
+        finally {
+            this.crawlingContexts.delete(crawlingContext.id);
+        }
+    }
+    /**
+     * Run async callback with given timeout and retry.
+     * @ignore
+     */
+    async _timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
+        try {
+            await (0, timeout_1.addTimeoutToPromise)(handler, timeout, error);
+        }
+        catch (e) {
+            if (retried <= maxRetries) { // we retry on any error, not just timeout
+                this.log.warning(`${e.message} (retrying ${retried}/${maxRetries})`);
+                return this._timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
+            }
+            throw e;
+        }
+    }
+    /**
+     * Returns true if either RequestList or RequestQueue have a request ready for processing.
+     */
+    async _isTaskReadyFunction() {
+        // First check RequestList, since it's only in memory.
+        const isRequestListEmpty = this.requestList ? (await this.requestList.isEmpty()) : true;
+        // If RequestList is not empty, task is ready, no reason to check RequestQueue.
+        if (!isRequestListEmpty)
+            return true;
+        // If RequestQueue is not empty, task is ready, return true, otherwise false.
+        return this.requestQueue ? !(await this.requestQueue.isEmpty()) : false;
+    }
+    /**
+     * Returns true if both RequestList and RequestQueue have all requests finished.
+     */
+    async _defaultIsFinishedFunction() {
+        const [isRequestListFinished, isRequestQueueFinished,] = await Promise.all([
+            this.requestList ? this.requestList.isFinished() : true,
+            this.requestQueue ? this.requestQueue.isFinished() : true,
+        ]);
+        // If both are finished, return true, otherwise return false.
+        return isRequestListFinished && isRequestQueueFinished;
+    }
+    /**
+     * Handles errors thrown by user provided handleRequestFunction()
+     */
+    async _requestFunctionErrorHandler(error, crawlingContext, source) {
+        const { request } = crawlingContext;
+        request.pushErrorMessage(error);
+        const shouldRetryRequest = !request.noRetry && request.retryCount < this.maxRequestRetries;
+        if (shouldRetryRequest) {
+            request.retryCount++;
+            const { url, retryCount, id } = request;
+            this.log.exception(error, 'handleRequestFunction failed, reclaiming failed request back to the list or queue', { url, retryCount, id });
+            await source.reclaimRequest(request);
+        }
+        else {
+            // If we get here, the request is either not retryable
+            // or failed more than retryCount times and will not be retried anymore.
+            // Mark the request as failed and do not retry.
+            this.handledRequestsCount++;
+            await source.markRequestHandled(request);
+            this.stats.failJob(request.id || request.url);
+            // @ts-expect-error It is assignable, but TS says otherwise...
+            const castedErrorContext = crawlingContext;
+            castedErrorContext.error = error;
+            await this._handleFailedRequestHandler(castedErrorContext); // This function prints an error message.
+        }
+    }
+    async _handleFailedRequestHandler(crawlingContext) {
+        if (this.failedRequestHandler) {
+            await this.failedRequestHandler(crawlingContext);
+        }
+        else {
+            const { id, url, method, uniqueKey } = crawlingContext.request;
+            this.log.exception(crawlingContext.error, 'Request failed and reached maximum retries', { id, url, method, uniqueKey });
+        }
+    }
+    /**
+     * Updates handledRequestsCount from possibly stored counts,
+     * usually after worker migration. Since one of the stores
+     * needs to have priority when both are present,
+     * it is the request queue, because generally, the request
+     * list will first be dumped into the queue and then left
+     * empty.
+     */
+    async _loadHandledRequestCount() {
+        if (this.requestQueue) {
+            this.handledRequestsCount = await this.requestQueue.handledCount();
+        }
+        else if (this.requestList) {
+            this.handledRequestsCount = this.requestList.handledCount();
+        }
+    }
+    async _executeHooks(hooks, ...args) {
+        if (Array.isArray(hooks) && hooks.length) {
+            for (const hook of hooks) {
+                await hook(...args);
+            }
+        }
+    }
+    /**
+     * Function for cleaning up after all request are processed.
+     * @ignore
+     */
+    async teardown() {
+        if (this.useSessionPool) {
+            await this.sessionPool.teardown();
+        }
+    }
+    _handlePropertyNameChange({ newProperty, newName, oldProperty, oldName, propertyKey, allowUndefined = false, }) {
+        if (newProperty && oldProperty) {
+            this.log.warning([
+                `Both "${newName}" and "${oldName}" were provided in the crawler options.`,
+                `"${oldName}" has been renamed to "${newName}", and will be removed in a future version.`,
+                `As such, "${newName}" will be used instead.`,
+            ].join('\n'));
+            // @ts-expect-error Assigning to possibly readonly properties
+            this[propertyKey] = newProperty;
+        }
+        else if (oldProperty) {
+            this.log.warning([
+                `"${oldName}" has been renamed to "${newName}", and will be removed in a future version.`,
+                `The provided value will be used, but you should rename "${oldName}" to "${newName}" in your crawler options.`,
+            ].join('\n'));
+            // @ts-expect-error Assigning to possibly readonly properties
+            this[propertyKey] = oldProperty;
+        }
+        else if (newProperty) {
+            // @ts-expect-error Assigning to possibly readonly properties
+            this[propertyKey] = newProperty;
+        }
+        else if (!allowUndefined) {
+            throw new ow_1.ArgumentError(`"${newName}" must be provided in the crawler options`, this.constructor);
+        }
+    }
+    _getCookieHeaderFromRequest(request) {
+        return request.headers?.Cookie ?? request.headers?.cookie ?? '';
+    }
+}
+exports.BasicCrawler = BasicCrawler;
+Object.defineProperty(BasicCrawler, "optionsShape", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: {
+        requestList: ow_1.default.optional.object.validate(core_1.validators.requestList),
+        requestQueue: ow_1.default.optional.object.validate(core_1.validators.requestQueue),
+        // Subclasses override this function instead of passing it
+        // in constructor, so this validation needs to apply only
+        // if the user creates an instance of BasicCrawler directly.
+        // TODO: remove .optional from requestHandler once migration period is over
+        requestHandler: ow_1.default.optional.function,
+        // TODO: remove in a future release
+        handleRequestFunction: ow_1.default.optional.function,
+        requestHandlerTimeoutSecs: ow_1.default.optional.number,
+        // TODO: remove in a future release
+        handleRequestTimeoutSecs: ow_1.default.optional.number,
+        failedRequestHandler: ow_1.default.optional.function,
+        // TODO: remove in a future release
+        handleFailedRequestFunction: ow_1.default.optional.function,
+        maxRequestRetries: ow_1.default.optional.number,
+        maxRequestsPerCrawl: ow_1.default.optional.number,
+        autoscaledPoolOptions: ow_1.default.optional.object,
+        sessionPoolOptions: ow_1.default.optional.object,
+        useSessionPool: ow_1.default.optional.boolean,
+        // AutoscaledPool shorthands
+        minConcurrency: ow_1.default.optional.number,
+        maxConcurrency: ow_1.default.optional.number,
+        // internal
+        log: ow_1.default.optional.object,
+    }
+});
+//# sourceMappingURL=basic-crawler.js.map