@crawlee/basic 4.0.0-beta.155 → 4.0.0-beta.157

package/internals/basic-crawler.js

@@ -7,7 +7,7 @@ import { RobotsTxtFile } from '@crawlee/utils';
 import { getDomain } from 'tldts';
 import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
-import { addTimeoutToPromise, extendTimeout, TimeoutError, tryCancel } from '@apify/timeout';
+import { addTimeoutToPromise, extendTimeout, storage as timeoutStorage, TimeoutError, tryCancel } from '@apify/timeout';
 import { cryptoRandomObjectId } from '@apify/utilities';
 import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, timeoutExpiredKey, } from './request-timeout.js';
 import { createSendRequest } from './send-request.js';
@@ -797,115 +797,119 @@ export class BasicCrawler {
      * @param [options] Options for the request queue.
      */
     async run(requests, options) {
-        if (this.running) {
-            throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
-        }
-        const { purgeRequestQueue, ...addRequestsOptions } = options ?? {};
-        if (this.hasFinishedBefore) {
-            // When executing the run method for the second time explicitly,
-            // we need to purge the RQ to allow processing the same requests again — this is important so users can
-            // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
-            // ignored — as a failed request is still handled.
-            // `purgeRequestQueue` unset purges only storage the crawler opened itself (see `#purgeableExtent`);
-            // `true` also purges a caller-supplied manager, `false` purges nothing.
-            if (purgeRequestQueue === undefined && this.#purgeableExtent === 'ambiguous') {
-                throw new Error('Cannot decide what to purge before running again: `sameDomainDelaySecs` paces the request ' +
-                    'manager you supplied, so the per-domain queues that have to be emptied are the ' +
-                    "crawler's while the manager underneath them is yours. Say which you want: " +
-                    '`run(requests, { purgeRequestQueue: true })` empties both, `false` empties neither.');
+        // A crawl is the top level of its own transaction and timeout scope, not a participant in the caller's.
+        return withDirectStorageAccess(async () => timeoutStorage.exit(async () => {
+            if (this.running) {
+                throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
             }
-            if (purgeRequestQueue !== false && (this.#purgeableExtent === 'all' || purgeRequestQueue === true)) {
-                // One call from the outside in reaches everything the manager wraps, a pacer's per-domain queues
-                // included.
-                await this.requestManager?.purge?.();
+            const { purgeRequestQueue, ...addRequestsOptions } = options ?? {};
+            if (this.hasFinishedBefore) {
+                // When executing the run method for the second time explicitly,
+                // we need to purge the RQ to allow processing the same requests again — this is important so users can
+                // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
+                // ignored — as a failed request is still handled.
+                // `purgeRequestQueue` unset purges only storage the crawler opened itself (see `#purgeableExtent`);
+                // `true` also purges a caller-supplied manager, `false` purges nothing.
+                if (purgeRequestQueue === undefined && this.#purgeableExtent === 'ambiguous') {
+                    throw new Error('Cannot decide what to purge before running again: `sameDomainDelaySecs` paces the request ' +
+                        'manager you supplied, so the per-domain queues that have to be emptied are the ' +
+                        "crawler's while the manager underneath them is yours. Say which you want: " +
+                        '`run(requests, { purgeRequestQueue: true })` empties both, `false` empties neither.');
+                }
+                if (purgeRequestQueue !== false &&
+                    (this.#purgeableExtent === 'all' || purgeRequestQueue === true)) {
+                    // One call from the outside in reaches everything the manager wraps, a pacer's per-domain queues
+                    // included.
+                    await this.requestManager?.purge?.();
+                }
+                // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
+                await this.#statisticsDep.ifOwned(async (stats) => {
+                    stats.reset();
+                    await stats.resetStore();
+                });
+                await this.#sessionPoolDep.ifOwned((pool) => pool.resetStore());
             }
-            // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
-            await this.#statisticsDep.ifOwned(async (stats) => {
-                stats.reset();
-                await stats.resetStore();
-            });
-            await this.#sessionPoolDep.ifOwned((pool) => pool.resetStore());
-        }
-        this.#unexpectedStop = false;
-        this.running = true;
-        this.#loggedPerRun.clear();
-        await purgeDefaultStorages({
-            onlyPurgeOnce: true,
-            storageBackend: serviceLocator.getStorageBackend(),
-            configuration: serviceLocator.getConfiguration(),
-        });
-        if (requests) {
-            await this.addRequests(requests, addRequestsOptions);
-        }
-        try {
-            await this.init();
-            await this.statistics.startCapturing();
-        }
-        catch (error) {
-            // Clean up here before propagating, otherwise a failed startup would leave the process hanging.
-            await this.teardown().catch((teardownError) => {
-                this.log.exception(teardownError, 'Cleaning up after a failed crawler startup failed.');
+            this.#unexpectedStop = false;
+            this.running = true;
+            this.#loggedPerRun.clear();
+            await purgeDefaultStorages({
+                onlyPurgeOnce: true,
+                storageBackend: serviceLocator.getStorageBackend(),
+                configuration: serviceLocator.getConfiguration(),
             });
-            // The run never began, so let the instance be run again instead of leaving it wedged as `running`.
-            this.running = false;
-            throw error;
-        }
-        const periodicLogger = this.getPeriodicLogger();
-        this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
-        const sigintHandler = async () => {
-            this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
-            await this.pauseOnMigration();
-            await this.#autoscaledPool.abort();
-        };
-        // Attach a listener to handle migration and aborting events gracefully.
-        const boundPauseOnMigration = this.pauseOnMigration.bind(this);
-        process.once('SIGINT', sigintHandler);
-        const eventManager = serviceLocator.getEventManager();
-        eventManager.on(EventType.MIGRATING, boundPauseOnMigration);
-        eventManager.on(EventType.ABORTING, boundPauseOnMigration);
-        let stats = {};
-        try {
-            await this.#autoscaledPool.run();
-        }
-        finally {
-            await this.statistics.stopCapturing();
-            await this.teardown();
-            process.off('SIGINT', sigintHandler);
-            eventManager.off(EventType.MIGRATING, boundPauseOnMigration);
-            eventManager.off(EventType.ABORTING, boundPauseOnMigration);
-            const finalStats = this.statistics.calculate();
-            stats = {
-                requestsFinished: this.statistics.state.requestsFinished,
-                requestsFailed: this.statistics.state.requestsFailed,
-                retryHistogram: this.statistics.requestRetryHistogram,
-                ...finalStats,
-            };
-            this.log.info('Final request statistics:', stats);
-            if (this.statistics.errorTracker.total !== 0) {
-                const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
-                this.log.info(`Error analysis:`, {
-                    totalErrors: this.statistics.errorTracker.total,
-                    uniqueErrors: this.statistics.errorTracker.getUniqueErrorCount(),
-                    mostCommonErrors: this.statistics.errorTracker.getMostPopularErrors(3).map(prettify),
+            if (requests) {
+                await this.addRequests(requests, addRequestsOptions);
+            }
+            try {
+                await this.init();
+                await this.statistics.startCapturing();
+            }
+            catch (error) {
+                // Clean up here before propagating, otherwise a failed startup would leave the process hanging.
+                await this.teardown().catch((teardownError) => {
+                    this.log.exception(teardownError, 'Cleaning up after a failed crawler startup failed.');
                 });
+                // The run never began, so let the instance be run again instead of leaving it wedged as `running`.
+                this.running = false;
+                throw error;
             }
-            const client = serviceLocator.getStorageBackend();
-            if (client.teardown) {
-                let finished = false;
-                setTimeout(() => {
-                    if (!finished) {
-                        this.log.info('Waiting for the storage to write its state to file system.');
-                    }
-                }, 1000);
-                await client.teardown();
-                finished = true;
+            const periodicLogger = this.getPeriodicLogger();
+            this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
+            const sigintHandler = async () => {
+                this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
+                await this.pauseOnMigration();
+                await this.#autoscaledPool.abort();
+            };
+            // Attach a listener to handle migration and aborting events gracefully.
+            const boundPauseOnMigration = this.pauseOnMigration.bind(this);
+            process.once('SIGINT', sigintHandler);
+            const eventManager = serviceLocator.getEventManager();
+            eventManager.on(EventType.MIGRATING, boundPauseOnMigration);
+            eventManager.on(EventType.ABORTING, boundPauseOnMigration);
+            let stats = {};
+            try {
+                await this.#autoscaledPool.run();
             }
-            periodicLogger.stop();
-            this.setStatusMessage(`Finished! Total ${this.statistics.state.requestsFinished + this.statistics.state.requestsFailed} requests: ${this.statistics.state.requestsFinished} succeeded, ${this.statistics.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
-            this.running = false;
-            this.hasFinishedBefore = true;
-        }
-        return stats;
+            finally {
+                await this.statistics.stopCapturing();
+                await this.teardown();
+                process.off('SIGINT', sigintHandler);
+                eventManager.off(EventType.MIGRATING, boundPauseOnMigration);
+                eventManager.off(EventType.ABORTING, boundPauseOnMigration);
+                const finalStats = this.statistics.calculate();
+                stats = {
+                    requestsFinished: this.statistics.state.requestsFinished,
+                    requestsFailed: this.statistics.state.requestsFailed,
+                    retryHistogram: this.statistics.requestRetryHistogram,
+                    ...finalStats,
+                };
+                this.log.info('Final request statistics:', stats);
+                if (this.statistics.errorTracker.total !== 0) {
+                    const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
+                    this.log.info(`Error analysis:`, {
+                        totalErrors: this.statistics.errorTracker.total,
+                        uniqueErrors: this.statistics.errorTracker.getUniqueErrorCount(),
+                        mostCommonErrors: this.statistics.errorTracker.getMostPopularErrors(3).map(prettify),
+                    });
+                }
+                const client = serviceLocator.getStorageBackend();
+                if (client.teardown) {
+                    let finished = false;
+                    setTimeout(() => {
+                        if (!finished) {
+                            this.log.info('Waiting for the storage to write its state to file system.');
+                        }
+                    }, 1000);
+                    await client.teardown();
+                    finished = true;
+                }
+                periodicLogger.stop();
+                this.setStatusMessage(`Finished! Total ${this.statistics.state.requestsFinished + this.statistics.state.requestsFailed} requests: ${this.statistics.state.requestsFinished} succeeded, ${this.statistics.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
+                this.running = false;
+                this.hasFinishedBefore = true;
+            }
+            return stats;
+        }));
     }
     /**
      * Gracefully stops the current run of the crawler.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.155",
+    "version": "4.0.0-beta.157",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.155",
-        "@crawlee/http-client": "4.0.0-beta.155",
-        "@crawlee/types": "4.0.0-beta.155",
-        "@crawlee/utils": "4.0.0-beta.155",
+        "@crawlee/core": "4.0.0-beta.157",
+        "@crawlee/http-client": "4.0.0-beta.157",
+        "@crawlee/types": "4.0.0-beta.157",
+        "@crawlee/utils": "4.0.0-beta.157",
         "csv-stringify": "^6.5.2",
         "tldts": "^7.0.6",
         "tslib": "^2.8.1",
@@ -53,7 +53,7 @@
         "zod": "^4.4.3"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.155"
+        "@crawlee/impit-client": "^4.0.0-beta.157"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "e7c221dfb88342e0738aa7b0e160c05f976fcf6c"
+    "gitHead": "a1b3d4f72a2759a7179af6648083f57d022125a9"
 }