@crawlee/basic 3.13.5 → 4.0.0-beta.0

package/internals/basic-crawler.js

@@ -1,20 +1,16 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.BasicCrawler = void 0;
-exports.createBasicRouter = createBasicRouter;
-const tslib_1 = require("tslib");
-const node_path_1 = require("node:path");
-const core_1 = require("@crawlee/core");
-const utils_1 = require("@crawlee/utils");
-const sync_1 = require("csv-stringify/sync");
-const fs_extra_1 = require("fs-extra");
-const ow_1 = tslib_1.__importStar(require("ow"));
-const tldts_1 = require("tldts");
-const datastructures_1 = require("@apify/datastructures");
-const log_1 = tslib_1.__importStar(require("@apify/log"));
-const timeout_1 = require("@apify/timeout");
-const utilities_1 = require("@apify/utilities");
-const send_request_1 = require("./send-request");
+import { writeFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { AutoscaledPool, Configuration, CriticalError, Dataset, enqueueLinks, EnqueueStrategy, GotScrapingHttpClient, KeyValueStore, mergeCookies, NonRetryableError, purgeDefaultStorages, RequestProvider, RequestQueue, RequestQueueV1, RequestState, RetryRequestError, Router, SessionError, SessionPool, Statistics, validators, } from '@crawlee/core';
+import { RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';
+import { stringify } from 'csv-stringify/sync';
+import { ensureDir, writeJSON } from 'fs-extra/esm';
+import ow from 'ow';
+import { getDomain } from 'tldts';
+import { LruCache } from '@apify/datastructures';
+import defaultLog, { LogLevel } from '@apify/log';
+import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
+import { cryptoRandomObjectId } from '@apify/utilities';
+import { createSendRequest } from './send-request.js';
 /**
  * Since there's no set number of seconds before the container is terminated after
  * a migration event, we need some reasonable number to use for RequestList persistence.
@@ -89,308 +85,137 @@ const SAFE_MIGRATION_WAIT_MILLIS = 20000;
  * ```
  * @category Crawlers
  */
-class BasicCrawler {
+export class BasicCrawler {
+    config;
+    static CRAWLEE_STATE_KEY = 'CRAWLEE_STATE';
+    /**
+     * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
+     */
+    stats;
+    /**
+     * A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request|requests}.
+     * Only available if used by the crawler.
+     */
+    requestList;
+    /**
+     * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
+     * A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request|requests}.
+     * Only available if used by the crawler.
+     */
+    requestQueue;
+    /**
+     * A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session|sessions}.
+     * Only available if used by the crawler.
+     */
+    sessionPool;
+    /**
+     * A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
+     * > *NOTE:* This property is only initialized after calling the {@link BasicCrawler.run|`crawler.run()`} function.
+     * We can use it to change the concurrency settings on the fly,
+     * to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
+     * or to abort it by calling {@link AutoscaledPool.abort|`autoscaledPool.abort()`}.
+     */
+    autoscaledPool;
+    /**
+     * Default {@link Router} instance that will be used if we don't specify any {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
+     * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
+     */
+    router = Router.create();
+    running = false;
+    hasFinishedBefore = false;
+    log;
+    requestHandler;
+    errorHandler;
+    failedRequestHandler;
+    requestHandlerTimeoutMillis;
+    internalTimeoutMillis;
+    maxRequestRetries;
+    sameDomainDelayMillis;
+    domainAccessedTime;
+    maxSessionRotations;
+    handledRequestsCount;
+    statusMessageLoggingInterval;
+    statusMessageCallback;
+    sessionPoolOptions;
+    useSessionPool;
+    crawlingContexts = new Map();
+    autoscaledPoolOptions;
+    events;
+    httpClient;
+    retryOnBlocked;
+    respectRobotsTxtFile;
+    onSkippedRequest;
+    _closeEvents;
+    experiments;
+    robotsTxtFileCache;
+    _experimentWarnings = {};
+    static optionsShape = {
+        requestList: ow.optional.object.validate(validators.requestList),
+        requestQueue: ow.optional.object.validate(validators.requestQueue),
+        // Subclasses override this function instead of passing it
+        // in constructor, so this validation needs to apply only
+        // if the user creates an instance of BasicCrawler directly.
+        requestHandler: ow.optional.function,
+        requestHandlerTimeoutSecs: ow.optional.number,
+        errorHandler: ow.optional.function,
+        failedRequestHandler: ow.optional.function,
+        maxRequestRetries: ow.optional.number,
+        sameDomainDelaySecs: ow.optional.number,
+        maxSessionRotations: ow.optional.number,
+        maxRequestsPerCrawl: ow.optional.number,
+        autoscaledPoolOptions: ow.optional.object,
+        sessionPoolOptions: ow.optional.object,
+        useSessionPool: ow.optional.boolean,
+        statusMessageLoggingInterval: ow.optional.number,
+        statusMessageCallback: ow.optional.function,
+        retryOnBlocked: ow.optional.boolean,
+        respectRobotsTxtFile: ow.optional.boolean,
+        onSkippedRequest: ow.optional.function,
+        httpClient: ow.optional.object,
+        // AutoscaledPool shorthands
+        minConcurrency: ow.optional.number,
+        maxConcurrency: ow.optional.number,
+        maxRequestsPerMinute: ow.optional.number.integerOrInfinite.positive.greaterThanOrEqual(1),
+        keepAlive: ow.optional.boolean,
+        // internal
+        log: ow.optional.object,
+        experiments: ow.optional.object,
+        statisticsOptions: ow.optional.object,
+    };
     /**
      * All `BasicCrawler` parameters are passed via an options object.
      */
-    constructor(options = {}, config = core_1.Configuration.getGlobalConfig()) {
-        Object.defineProperty(this, "config", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: config
-        });
-        /**
-         * A reference to the underlying {@link Statistics} class that collects and logs run statistics for requests.
-         */
-        Object.defineProperty(this, "stats", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        /**
-         * A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request|requests}.
-         * Only available if used by the crawler.
-         */
-        Object.defineProperty(this, "requestList", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        /**
-         * Dynamic queue of URLs to be processed. This is useful for recursive crawling of websites.
-         * A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request|requests}.
-         * Only available if used by the crawler.
-         */
-        Object.defineProperty(this, "requestQueue", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        /**
-         * A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session|sessions}.
-         * Only available if used by the crawler.
-         */
-        Object.defineProperty(this, "sessionPool", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        /**
-         * A reference to the underlying {@link AutoscaledPool} class that manages the concurrency of the crawler.
-         * > *NOTE:* This property is only initialized after calling the {@link BasicCrawler.run|`crawler.run()`} function.
-         * We can use it to change the concurrency settings on the fly,
-         * to pause the crawler by calling {@link AutoscaledPool.pause|`autoscaledPool.pause()`}
-         * or to abort it by calling {@link AutoscaledPool.abort|`autoscaledPool.abort()`}.
-         */
-        Object.defineProperty(this, "autoscaledPool", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        /**
-         * Default {@link Router} instance that will be used if we don't specify any {@link BasicCrawlerOptions.requestHandler|`requestHandler`}.
-         * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
-         */
-        Object.defineProperty(this, "router", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: core_1.Router.create()
-        });
-        Object.defineProperty(this, "running", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: false
-        });
-        Object.defineProperty(this, "hasFinishedBefore", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: false
-        });
-        Object.defineProperty(this, "log", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "requestHandler", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "errorHandler", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "failedRequestHandler", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "requestHandlerTimeoutMillis", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "internalTimeoutMillis", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "maxRequestRetries", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "sameDomainDelayMillis", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "domainAccessedTime", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "maxSessionRotations", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "handledRequestsCount", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "statusMessageLoggingInterval", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "statusMessageCallback", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "sessionPoolOptions", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "useSessionPool", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "crawlingContexts", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: new Map()
-        });
-        Object.defineProperty(this, "autoscaledPoolOptions", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "events", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "httpClient", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "retryOnBlocked", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "respectRobotsTxtFile", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "onSkippedRequest", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "_closeEvents", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "experiments", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "robotsTxtFileCache", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "_experimentWarnings", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: {}
-        });
-        (0, ow_1.default)(options, 'BasicCrawlerOptions', ow_1.default.object.exactShape(BasicCrawler.optionsShape));
+    constructor(options = {}, config = Configuration.getGlobalConfig()) {
+        this.config = config;
+        ow(options, 'BasicCrawlerOptions', ow.object.exactShape(BasicCrawler.optionsShape));
         const { requestList, requestQueue, maxRequestRetries = 3, sameDomainDelaySecs = 0, maxSessionRotations = 10, maxRequestsPerCrawl, autoscaledPoolOptions = {}, keepAlive, sessionPoolOptions = {}, useSessionPool = true, 
         // AutoscaledPool shorthands
-        minConcurrency, maxConcurrency, maxRequestsPerMinute, retryOnBlocked = false, respectRobotsTxtFile = false, onSkippedRequest, 
+        minConcurrency, maxConcurrency, maxRequestsPerMinute, retryOnBlocked = false, respectRobotsTxtFile = false, onSkippedRequest, requestHandler, requestHandlerTimeoutSecs, errorHandler, failedRequestHandler, statusMessageLoggingInterval = 10, statusMessageCallback, statisticsOptions, httpClient, 
         // internal
-        log = log_1.default.child({ prefix: this.constructor.name }), experiments = {}, 
-        // Old and new request handler methods
-        handleRequestFunction, requestHandler, handleRequestTimeoutSecs, requestHandlerTimeoutSecs, errorHandler, handleFailedRequestFunction, failedRequestHandler, statusMessageLoggingInterval = 10, statusMessageCallback, statisticsOptions, httpClient, } = options;
+        log = defaultLog.child({ prefix: this.constructor.name }), experiments = {}, } = options;
         this.requestList = requestList;
         this.requestQueue = requestQueue;
-        this.httpClient = httpClient ?? new core_1.GotScrapingHttpClient();
+        this.httpClient = httpClient ?? new GotScrapingHttpClient();
         this.log = log;
         this.statusMessageLoggingInterval = statusMessageLoggingInterval;
         this.statusMessageCallback = statusMessageCallback;
         this.events = config.getEventManager();
         this.domainAccessedTime = new Map();
         this.experiments = experiments;
-        this.robotsTxtFileCache = new datastructures_1.LruCache({ maxLength: 1000 });
-        this._handlePropertyNameChange({
-            newName: 'requestHandler',
-            oldName: 'handleRequestFunction',
-            propertyKey: 'requestHandler',
-            newProperty: requestHandler,
-            oldProperty: handleRequestFunction,
-            allowUndefined: true, // fallback to the default router
-        });
-        if (!this.requestHandler) {
-            this.requestHandler = this.router;
-        }
+        this.robotsTxtFileCache = new LruCache({ maxLength: 1000 });
+        // FIXME any
+        this.requestHandler = requestHandler ?? this.router;
+        this.failedRequestHandler = failedRequestHandler;
         this.errorHandler = errorHandler;
-        this._handlePropertyNameChange({
-            newName: 'failedRequestHandler',
-            oldName: 'handleFailedRequestFunction',
-            propertyKey: 'failedRequestHandler',
-            newProperty: failedRequestHandler,
-            oldProperty: handleFailedRequestFunction,
-            allowUndefined: true,
-        });
-        let newRequestHandlerTimeout;
-        if (!handleRequestTimeoutSecs) {
-            if (!requestHandlerTimeoutSecs) {
-                newRequestHandlerTimeout = 60000;
-            }
-            else {
-                newRequestHandlerTimeout = requestHandlerTimeoutSecs * 1000;
-            }
+        if (requestHandlerTimeoutSecs) {
+            this.requestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
         }
-        else if (requestHandlerTimeoutSecs) {
-            newRequestHandlerTimeout = requestHandlerTimeoutSecs * 1000;
+        else {
+            this.requestHandlerTimeoutMillis = 60_000;
         }
         this.retryOnBlocked = retryOnBlocked;
         this.respectRobotsTxtFile = respectRobotsTxtFile;
         this.onSkippedRequest = onSkippedRequest;
-        this._handlePropertyNameChange({
-            newName: 'requestHandlerTimeoutSecs',
-            oldName: 'handleRequestTimeoutSecs',
-            propertyKey: 'requestHandlerTimeoutMillis',
-            newProperty: newRequestHandlerTimeout,
-            oldProperty: handleRequestTimeoutSecs ? handleRequestTimeoutSecs * 1000 : undefined,
-        });
         const tryEnv = (val) => (val == null ? null : +val);
         // allow at least 5min for internal timeouts
         this.internalTimeoutMillis =
@@ -406,7 +231,7 @@ class BasicCrawler {
         this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
         this.maxSessionRotations = maxSessionRotations;
         this.handledRequestsCount = 0;
-        this.stats = new core_1.Statistics({
+        this.stats = new Statistics({
             logMessage: `${log.getOptions().prefix} request statistics:`,
             log,
             config,
@@ -484,7 +309,7 @@ class BasicCrawler {
      * @param error The error to check.
      */
     isProxyError(error) {
-        return utils_1.ROTATE_PROXY_ERRORS.some((x) => this._getMessageFromError(error)?.includes(x));
+        return ROTATE_PROXY_ERRORS.some((x) => this._getMessageFromError(error)?.includes(x));
     }
     /**
      * Checks whether the given crawling context is getting blocked by anti-bot protection using several heuristics.
@@ -499,13 +324,13 @@ class BasicCrawler {
      */
     async setStatusMessage(message, options = {}) {
         const data = options.isStatusMessageTerminal != null ? { terminal: options.isStatusMessageTerminal } : undefined;
-        this.log.internal(log_1.LogLevel[options.level ?? 'DEBUG'], message, data);
+        this.log.internal(LogLevel[options.level ?? 'DEBUG'], message, data);
         const client = this.config.getStorageClient();
         if (!client.setStatusMessage) {
             return;
         }
         // just to be sure, this should be fast
-        await (0, timeout_1.addTimeoutToPromise)(async () => client.setStatusMessage(message, options), 1000, 'Setting status message timed out after 1s').catch((e) => this.log.debug(e.message));
+        await addTimeoutToPromise(async () => client.setStatusMessage(message, options), 1000, 'Setting status message timed out after 1s').catch((e) => this.log.debug(e.message));
     }
     getPeriodicLogger() {
         let previousState = { ...this.stats.state };
@@ -571,7 +396,7 @@ class BasicCrawler {
             await this.sessionPool?.resetStore();
         }
         this.running = true;
-        await (0, core_1.purgeDefaultStorages)({ onlyPurgeOnce: true });
+        await purgeDefaultStorages({ onlyPurgeOnce: true });
         if (requests) {
             await this.addRequests(requests, addRequestsOptions);
         }
@@ -653,11 +478,11 @@ class BasicCrawler {
         if (!this.requestQueue && this.requestList) {
             this.log.warningOnce('When using RequestList and RequestQueue at the same time, you should instantiate both explicitly and provide them in the crawler options, to ensure correctly handled restarts of the crawler.');
         }
-        this.requestQueue ?? (this.requestQueue = await this._getRequestQueue());
+        this.requestQueue ??= await this._getRequestQueue();
         return this.requestQueue;
     }
     async useState(defaultValue = {}) {
-        const kvs = await core_1.KeyValueStore.open(null, { config: this.config });
+        const kvs = await KeyValueStore.open(null, { config: this.config });
         return kvs.getAutoSavedValue(BasicCrawler.CRAWLEE_STATE_KEY, defaultValue);
     }
     /**
@@ -711,7 +536,7 @@ class BasicCrawler {
      * Retrieves the specified {@link Dataset}, or the default crawler {@link Dataset}.
      */
     async getDataset(idOrName) {
-        return core_1.Dataset.open(idOrName, { config: this.config });
+        return Dataset.open(idOrName, { config: this.config });
     }
     /**
      * Retrieves data from the default crawler {@link Dataset} by calling {@link Dataset.getData}.
@@ -738,14 +563,14 @@ class BasicCrawler {
         const dataset = await this.getDataset();
         const items = await dataset.export(options);
         if (format === 'csv') {
-            const value = (0, sync_1.stringify)([Object.keys(items[0]), ...items.map((item) => Object.values(item))]);
-            await (0, fs_extra_1.ensureDir)((0, node_path_1.dirname)(path));
-            await (0, fs_extra_1.writeFile)(path, value);
+            const value = stringify([Object.keys(items[0]), ...items.map((item) => Object.values(item))]);
+            await ensureDir(dirname(path));
+            await writeFile(path, value);
             this.log.info(`Export to ${path} finished!`);
         }
         if (format === 'json') {
-            await (0, fs_extra_1.ensureDir)((0, node_path_1.dirname)(path));
-            await (0, fs_extra_1.writeJSON)(path, items, { spaces: 4 });
+            await ensureDir(dirname(path));
+            await writeJSON(path, items, { spaces: 4 });
             this.log.info(`Export to ${path} finished!`);
         }
         return items;
@@ -758,9 +583,9 @@ class BasicCrawler {
         // Initialize AutoscaledPool before awaiting _loadHandledRequestCount(),
         // so that the caller can get a reference to it before awaiting the promise returned from run()
         // (otherwise there would be no way)
-        this.autoscaledPool = new core_1.AutoscaledPool(this.autoscaledPoolOptions, this.config);
+        this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions, this.config);
         if (this.useSessionPool) {
-            this.sessionPool = await core_1.SessionPool.open(this.sessionPoolOptions, this.config);
+            this.sessionPool = await SessionPool.open(this.sessionPoolOptions, this.config);
             // Assuming there are not more than 20 browsers running at once;
             this.sessionPool.setMaxListeners(20);
         }
@@ -795,7 +620,7 @@ class BasicCrawler {
             if (cachedRobotsTxtFile) {
                 return cachedRobotsTxtFile;
             }
-            const robotsTxtFile = await utils_1.RobotsTxtFile.find(url);
+            const robotsTxtFile = await RobotsTxtFile.find(url);
             this.robotsTxtFileCache.add(origin, robotsTxtFile);
             return robotsTxtFile;
         }
@@ -873,7 +698,7 @@ class BasicCrawler {
      * should be ignored and will be reclaimed to the queue once ready.
      */
     delayRequest(request, source) {
-        const domain = (0, tldts_1.getDomain)(request.url);
+        const domain = getDomain(request.url);
         if (!domain || !request) {
             return false;
         }
@@ -883,7 +708,7 @@ class BasicCrawler {
             this.domainAccessedTime.set(domain, now);
             return false;
         }
-        if (source instanceof core_1.RequestQueueV1) {
+        if (source instanceof RequestQueueV1) {
             // eslint-disable-next-line dot-notation
             source['inProgress']?.delete(request.id);
         }
@@ -891,7 +716,7 @@ class BasicCrawler {
         this.log.debug(`Request ${request.url} (${request.id}) will be reclaimed after ${delay} milliseconds due to same domain delay`);
         setTimeout(async () => {
             this.log.debug(`Adding request ${request.url} (${request.id}) back to the queue`);
-            if (source instanceof core_1.RequestQueueV1) {
+            if (source instanceof RequestQueueV1) {
                 // eslint-disable-next-line dot-notation
                 source['inProgress'].add(request.id);
             }
@@ -910,19 +735,19 @@ class BasicCrawler {
         await this._timeoutAndRetry(async () => {
             request = await this._fetchNextRequest();
         }, this.internalTimeoutMillis, `Fetching next request timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-        (0, timeout_1.tryCancel)();
+        tryCancel();
         if (this.useSessionPool) {
             await this._timeoutAndRetry(async () => {
                 session = await this.sessionPool.getSession();
             }, this.internalTimeoutMillis, `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
         }
-        (0, timeout_1.tryCancel)();
+        tryCancel();
         if (!request || this.delayRequest(request, source)) {
             return;
         }
         if (!(await this.isAllowedBasedOnRobotsTxtFile(request.url))) {
             this.log.warning(`Skipping request ${request.url} (${request.id}) because it is disallowed based on robots.txt`);
-            request.state = core_1.RequestState.SKIPPED;
+            request.state = RequestState.SKIPPED;
             request.noRetry = true;
             await source.markRequestHandled(request);
             await this.onSkippedRequest?.({
@@ -940,13 +765,13 @@ class BasicCrawler {
         // All missing properties (that extend CrawlingContext) are set dynamically,
         // but TS does not know that, so otherwise it would throw when compiling.
         const crawlingContext = {
-            id: (0, utilities_1.cryptoRandomObjectId)(10),
+            id: cryptoRandomObjectId(10),
             crawler: this,
             log: this.log,
             request,
             session,
             enqueueLinks: async (options) => {
-                return (0, core_1.enqueueLinks)({
+                return enqueueLinks({
                     // specify the RQ first to allow overriding it
                     requestQueue: await this.getRequestQueue(),
                     robotsTxtFile: await this.getRobotsTxtFileForUrl(request.url),
@@ -957,42 +782,42 @@ class BasicCrawler {
             addRequests: this.addRequests.bind(this),
             pushData: this.pushData.bind(this),
             useState: this.useState.bind(this),
-            sendRequest: (0, send_request_1.createSendRequest)(this.httpClient, request, session, () => crawlingContext.proxyInfo?.url),
-            getKeyValueStore: async (idOrName) => core_1.KeyValueStore.open(idOrName, { config: this.config }),
+            sendRequest: createSendRequest(this.httpClient, request, session, () => crawlingContext.proxyInfo?.url),
+            getKeyValueStore: async (idOrName) => KeyValueStore.open(idOrName, { config: this.config }),
         };
         this.crawlingContexts.set(crawlingContext.id, crawlingContext);
         let isRequestLocked = true;
         try {
-            request.state = core_1.RequestState.REQUEST_HANDLER;
-            await (0, timeout_1.addTimeoutToPromise)(async () => this._runRequestHandler(crawlingContext), this.requestHandlerTimeoutMillis, `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds (${request.id}).`);
+            request.state = RequestState.REQUEST_HANDLER;
+            await addTimeoutToPromise(async () => this._runRequestHandler(crawlingContext), this.requestHandlerTimeoutMillis, `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds (${request.id}).`);
             await this._timeoutAndRetry(async () => source.markRequestHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
             isRequestLocked = false; // markRequestHandled succeeded and unlocked the request
             this.stats.finishJob(statisticsId, request.retryCount);
             this.handledRequestsCount++;
             // reclaim session if request finishes successfully
-            request.state = core_1.RequestState.DONE;
+            request.state = RequestState.DONE;
             crawlingContext.session?.markGood();
         }
         catch (err) {
             try {
-                request.state = core_1.RequestState.ERROR_HANDLER;
-                await (0, timeout_1.addTimeoutToPromise)(async () => this._requestFunctionErrorHandler(err, crawlingContext, source), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
-                if (!(err instanceof core_1.CriticalError)) {
+                request.state = RequestState.ERROR_HANDLER;
+                await addTimeoutToPromise(async () => this._requestFunctionErrorHandler(err, crawlingContext, source), this.internalTimeoutMillis, `Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
+                if (!(err instanceof CriticalError)) {
                     isRequestLocked = false; // _requestFunctionErrorHandler calls either markRequestHandled or reclaimRequest
                 }
-                request.state = core_1.RequestState.DONE;
+                request.state = RequestState.DONE;
             }
             catch (secondaryError) {
                 if (!secondaryError.triggeredFromUserHandler &&
                     // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
-                    !(secondaryError instanceof core_1.CriticalError)) {
+                    !(secondaryError instanceof CriticalError)) {
                     const apifySpecific = process.env.APIFY_IS_AT_HOME
                         ? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`
                         : '';
                     this.log.exception(secondaryError, 'An exception occurred during handling of failed request. ' +
                         `This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`);
                 }
-                request.state = core_1.RequestState.ERROR;
+                request.state = RequestState.ERROR;
                 throw secondaryError;
             }
             // decrease the session score if the request fails (but the error handler did not throw)
@@ -1002,7 +827,7 @@ class BasicCrawler {
             await this._cleanupContext(crawlingContext);
             this.crawlingContexts.delete(crawlingContext.id);
             // Safety net - release the lock if nobody managed to do it before
-            if (isRequestLocked && source instanceof core_1.RequestProvider) {
+            if (isRequestLocked && source instanceof RequestProvider) {
                 try {
                     await source.client.deleteRequestLock(request.id);
                 }
@@ -1018,7 +843,7 @@ class BasicCrawler {
      */
     async _timeoutAndRetry(handler, timeout, error, maxRetries = 3, retried = 1) {
         try {
-            await (0, timeout_1.addTimeoutToPromise)(handler, timeout, error);
+            await addTimeoutToPromise(handler, timeout, error);
         }
         catch (e) {
             if (retried <= maxRetries) {
@@ -1055,7 +880,7 @@ class BasicCrawler {
     }
     async _rotateSession(crawlingContext) {
         const { request } = crawlingContext;
-        request.sessionRotationCount ?? (request.sessionRotationCount = 0);
+        request.sessionRotationCount ??= 0;
         request.sessionRotationCount++;
         crawlingContext.session?.retire();
     }
@@ -1065,14 +890,14 @@ class BasicCrawler {
     async _requestFunctionErrorHandler(error, crawlingContext, source) {
         const { request } = crawlingContext;
         request.pushErrorMessage(error);
-        if (error instanceof core_1.CriticalError) {
+        if (error instanceof CriticalError) {
             throw error;
         }
         const shouldRetryRequest = this._canRequestBeRetried(request, error);
         if (shouldRetryRequest) {
             await this.stats.errorTrackerRetry.addAsync(error, crawlingContext);
-            await this._tagUserHandlerError(() => this.errorHandler?.(this._augmentContextWithDeprecatedError(crawlingContext, error), error));
-            if (error instanceof core_1.SessionError) {
+            await this.errorHandler?.(crawlingContext, error);
+            if (error instanceof SessionError) {
                 await this._rotateSession(crawlingContext);
             }
             if (!request.noRetry) {
@@ -1123,7 +948,7 @@ class BasicCrawler {
         const message = this._getMessageFromError(error, true);
         this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
         if (this.failedRequestHandler) {
-            await this._tagUserHandlerError(() => this.failedRequestHandler?.(this._augmentContextWithDeprecatedError(crawlingContext, error), error));
+            await this.failedRequestHandler?.(crawlingContext, error);
         }
     }
     /**
@@ -1138,7 +963,7 @@ class BasicCrawler {
         const stackLines = error?.stack ? error.stack.split('\n') : new Error().stack.split('\n').slice(2);
         const baseDir = process.cwd();
         const userLine = stackLines.find((line) => line.includes(baseDir) && !line.includes('node_modules'));
-        if (error instanceof timeout_1.TimeoutError) {
+        if (error instanceof TimeoutError) {
             return process.env.CRAWLEE_VERBOSE_LOG ? error.stack : error.message || error; // stack in timeout errors does not really help
         }
         return process.env.CRAWLEE_VERBOSE_LOG || forceStack
@@ -1148,28 +973,18 @@ class BasicCrawler {
     _canRequestBeRetried(request, error) {
         // Request should never be retried, or the error encountered makes it not able to be retried, or the session rotation limit has been reached
         if (request.noRetry ||
-            error instanceof core_1.NonRetryableError ||
-            (error instanceof core_1.SessionError && this.maxSessionRotations <= (request.sessionRotationCount ?? 0))) {
+            error instanceof NonRetryableError ||
+            (error instanceof SessionError && this.maxSessionRotations <= (request.sessionRotationCount ?? 0))) {
             return false;
         }
         // User requested retry (we ignore retry count here as its explicitly told by the user to retry)
-        if (error instanceof core_1.RetryRequestError) {
+        if (error instanceof RetryRequestError) {
             return true;
         }
         // Ensure there are more retries available for the request
         const maxRequestRetries = request.maxRetries ?? this.maxRequestRetries;
         return request.retryCount < maxRequestRetries;
     }
-    _augmentContextWithDeprecatedError(context, error) {
-        Object.defineProperty(context, 'error', {
-            get: () => {
-                this.log.deprecated("The 'error' property of the crawling context is deprecated, and it is now passed as the second parameter in 'errorHandler' and 'failedRequestHandler'. Please update your code, as this property will be removed in a future version.");
-                return error;
-            },
-            configurable: true,
-        });
-        return context;
-    }
     /**
      * Updates handledRequestsCount from possibly stored counts,
      * usually after worker migration. Since one of the stores
@@ -1207,36 +1022,10 @@ class BasicCrawler {
         }
         await this.autoscaledPool?.abort();
     }
-    _handlePropertyNameChange({ newProperty, newName, oldProperty, oldName, propertyKey, allowUndefined = false, }) {
-        if (newProperty && oldProperty) {
-            this.log.warning([
-                `Both "${newName}" and "${oldName}" were provided in the crawler options.`,
-                `"${oldName}" has been renamed to "${newName}", and will be removed in a future version.`,
-                `As such, "${newName}" will be used instead.`,
-            ].join('\n'));
-            // @ts-expect-error Assigning to possibly readonly properties
-            this[propertyKey] = newProperty;
-        }
-        else if (oldProperty) {
-            this.log.warning([
-                `"${oldName}" has been renamed to "${newName}", and will be removed in a future version.`,
-                `The provided value will be used, but you should rename "${oldName}" to "${newName}" in your crawler options.`,
-            ].join('\n'));
-            // @ts-expect-error Assigning to possibly readonly properties
-            this[propertyKey] = oldProperty;
-        }
-        else if (newProperty) {
-            // @ts-expect-error Assigning to possibly readonly properties
-            this[propertyKey] = newProperty;
-        }
-        else if (!allowUndefined) {
-            throw new ow_1.ArgumentError(`"${newName}" must be provided in the crawler options`, this.constructor);
-        }
-    }
     _getCookieHeaderFromRequest(request) {
         if (request.headers?.Cookie && request.headers?.cookie) {
             this.log.warning(`Encountered mixed casing for the cookie headers for request ${request.url} (${request.id}). Their values will be merged.`);
-            return (0, core_1.mergeCookies)(request.url, [request.headers.cookie, request.headers.Cookie]);
+            return mergeCookies(request.url, [request.headers.cookie, request.headers.Cookie]);
         }
         return request.headers?.Cookie || request.headers?.cookie || '';
     }
@@ -1247,9 +1036,9 @@ class BasicCrawler {
                 this.log.info('Using the old RequestQueue implementation without request locking.');
                 this._experimentWarnings.requestLocking = true;
             }
-            return core_1.RequestQueueV1.open(null, { config: this.config });
+            return RequestQueueV1.open(null, { config: this.config });
         }
-        return core_1.RequestQueue.open(null, { config: this.config });
+        return RequestQueue.open(null, { config: this.config });
     }
     requestMatchesEnqueueStrategy(request) {
         const { url, loadedUrl } = request;
@@ -1266,80 +1055,29 @@ class BasicCrawler {
         const baseUrl = new URL(url);
         const loadedBaseUrl = new URL(loadedUrl);
         switch (strategy) {
-            case core_1.EnqueueStrategy.SameHostname: {
+            case EnqueueStrategy.SameHostname: {
                 return baseUrl.hostname === loadedBaseUrl.hostname;
             }
-            case core_1.EnqueueStrategy.SameDomain: {
-                const baseUrlHostname = (0, tldts_1.getDomain)(baseUrl.hostname, { mixedInputs: false });
+            case EnqueueStrategy.SameDomain: {
+                const baseUrlHostname = getDomain(baseUrl.hostname, { mixedInputs: false });
                 if (baseUrlHostname) {
-                    const loadedBaseUrlHostname = (0, tldts_1.getDomain)(loadedBaseUrl.hostname, { mixedInputs: false });
+                    const loadedBaseUrlHostname = getDomain(loadedBaseUrl.hostname, { mixedInputs: false });
                     return baseUrlHostname === loadedBaseUrlHostname;
                 }
                 // Can happen for IPs, we just check like same origin
                 return baseUrl.origin === loadedBaseUrl.origin;
             }
-            case core_1.EnqueueStrategy.SameOrigin: {
+            case EnqueueStrategy.SameOrigin: {
                 // Same as hostname, but also checks protocol
                 return baseUrl.origin === loadedBaseUrl.origin;
             }
-            case core_1.EnqueueStrategy.All:
+            case EnqueueStrategy.All:
             default: {
                 return baseUrl.protocol === 'http:' || baseUrl.protocol === 'https:';
             }
         }
     }
 }
-exports.BasicCrawler = BasicCrawler;
-Object.defineProperty(BasicCrawler, "CRAWLEE_STATE_KEY", {
-    enumerable: true,
-    configurable: true,
-    writable: true,
-    value: 'CRAWLEE_STATE'
-});
-Object.defineProperty(BasicCrawler, "optionsShape", {
-    enumerable: true,
-    configurable: true,
-    writable: true,
-    value: {
-        requestList: ow_1.default.optional.object.validate(core_1.validators.requestList),
-        requestQueue: ow_1.default.optional.object.validate(core_1.validators.requestQueue),
-        // Subclasses override this function instead of passing it
-        // in constructor, so this validation needs to apply only
-        // if the user creates an instance of BasicCrawler directly.
-        requestHandler: ow_1.default.optional.function,
-        // TODO: remove in a future release
-        handleRequestFunction: ow_1.default.optional.function,
-        requestHandlerTimeoutSecs: ow_1.default.optional.number,
-        // TODO: remove in a future release
-        handleRequestTimeoutSecs: ow_1.default.optional.number,
-        errorHandler: ow_1.default.optional.function,
-        failedRequestHandler: ow_1.default.optional.function,
-        // TODO: remove in a future release
-        handleFailedRequestFunction: ow_1.default.optional.function,
-        maxRequestRetries: ow_1.default.optional.number,
-        sameDomainDelaySecs: ow_1.default.optional.number,
-        maxSessionRotations: ow_1.default.optional.number,
-        maxRequestsPerCrawl: ow_1.default.optional.number,
-        autoscaledPoolOptions: ow_1.default.optional.object,
-        sessionPoolOptions: ow_1.default.optional.object,
-        useSessionPool: ow_1.default.optional.boolean,
-        statusMessageLoggingInterval: ow_1.default.optional.number,
-        statusMessageCallback: ow_1.default.optional.function,
-        retryOnBlocked: ow_1.default.optional.boolean,
-        respectRobotsTxtFile: ow_1.default.optional.boolean,
-        onSkippedRequest: ow_1.default.optional.function,
-        httpClient: ow_1.default.optional.object,
-        // AutoscaledPool shorthands
-        minConcurrency: ow_1.default.optional.number,
-        maxConcurrency: ow_1.default.optional.number,
-        maxRequestsPerMinute: ow_1.default.optional.number.integerOrInfinite.positive.greaterThanOrEqual(1),
-        keepAlive: ow_1.default.optional.boolean,
-        // internal
-        log: ow_1.default.optional.object,
-        experiments: ow_1.default.optional.object,
-        statisticsOptions: ow_1.default.optional.object,
-    }
-});
 /**
  * Creates new {@link Router} instance that works based on request labels.
  * This instance can then serve as a {@link BasicCrawlerOptions.requestHandler|`requestHandler`} of our {@link BasicCrawler}.
@@ -1364,7 +1102,7 @@ Object.defineProperty(BasicCrawler, "optionsShape", {
  * await crawler.run();
  * ```
  */
-function createBasicRouter(routes) {
-    return core_1.Router.create(routes);
+export function createBasicRouter(routes) {
+    return Router.create(routes);
 }
 //# sourceMappingURL=basic-crawler.js.map