@crawlee/basic 4.0.0-beta.165 → 4.0.0-beta.167

package/internals/basic-crawler.d.ts

@@ -398,13 +398,15 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
      */
     get sessionPool(): ISessionPool;
+    /** Whether a {@link BasicCrawler.run|`run()`} is in progress on this instance. */
+    get running(): boolean;
     /**
-     * The concurrency governor this run is booking its requests against — either the
+     * The concurrency governor the run in progress is booking its requests against — either the
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
      * crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
      *
-     * > *NOTE:* `undefined` until {@link BasicCrawler.run|`crawler.run()`} has resolved it. A crawler-owned default
-     * is also rebuilt for every run, so the instance is not stable across runs.
+     * > *NOTE:* `undefined` outside a run, and a crawler-owned default is rebuilt for every run — so read it
+     * during a run rather than caching it across runs.
      *
      * {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
      * owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
@@ -433,7 +435,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
         request: Request;
     }, CrawlingContext>;
     get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
-    running: boolean;
     /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
     get hasFinishedBefore(): boolean;
     get log(): CrawleeLogger;
@@ -616,7 +617,6 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      */
     private validateRequestUserData;
     useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
-    private logOncePerRun;
     /**
      * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
      * adding the rest in background. You can configure the batch size via `batchSize` option and the sleep time in between
@@ -751,13 +751,22 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     private errorAbsolvesSession;
     private canRequestBeRetried;
     /**
-     * Stops the crawler immediately.
+     * Ends the run in progress immediately, without waiting for the requests already in flight.
      *
-     * This method doesn't wait for currently active requests to finish.
+     * This runs at the end of every {@link BasicCrawler.run|`run()`}, so what it releases is per-run only, and
+     * a repeated `run()` has to find everything it needs either intact or re-establishable. What outlives a run is
+     * released by {@link BasicCrawler.destroy|`destroy()`} instead.
      *
      * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
      */
     teardown(): Promise<void>;
+    /**
+     * Releases what the crawler owns beyond a single run — in the browser crawlers, the browser pool. A finished
+     * `run()` has already released everything a run owns, so this is only needed before dropping a crawler you
+     * could otherwise have run again.
+     */
+    destroy(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
     protected getCookieHeaderFromRequest(request: Request): string;
     private requestMatchesEnqueueStrategy;
 }

package/internals/basic-crawler.js

@@ -1,6 +1,6 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { applyRequestTransform, AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, buildEnqueueStrategyPatterns, ConcurrencySystem, Configuration, constructUrlPatternObjects, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createRequestOptions, createSkippedRequestArgs, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, filterRequestOptionsByPatterns, getObjectType, KeyValueStore, log, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, PersistentRateLimitError, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, ThrottlingRequestManager, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
+import { applyRequestTransform, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, buildEnqueueStrategyPatterns, ConcurrencySystem, Configuration, constructUrlPatternObjects, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, createRequestOptions, createSkippedRequestArgs, createStorageTransaction, Request, CriticalError, currentStorageTransaction, Dataset, EnqueueStrategy, EventManager, EventType, filterRequestOptionsByPatterns, getObjectType, KeyValueStore, log, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, PersistentRateLimitError, purgeDefaultStorages, RequestHandlerError, parseRetryAfterHeader, RequestThrottledError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, ThrottlingRequestManager, validateUserData, validators, withDirectStorageAccess, } from '@crawlee/core';
 import { BaseHttpClient, FetchHttpClient } from '@crawlee/http-client';
 import { isAsyncIterable, isIterable, parseArgument, ROTATE_PROXY_ERRORS, schemas } from '@crawlee/utils/internal';
 import { RobotsTxtFile } from '@crawlee/utils';
@@ -9,6 +9,7 @@ import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
 import { addTimeoutToPromise, extendTimeout, storage as timeoutStorage, TimeoutError, tryCancel } from '@apify/timeout';
 import { cryptoRandomObjectId } from '@apify/utilities';
+import { CrawlerRun } from './crawler-run.js';
 import { extendTimeoutKey, navigationDeadlineKey, raceWithTimeout, timeoutExpiredKey, } from './request-timeout.js';
 import { createSendRequest } from './send-request.js';
 class LazyDefaultHttpClient extends BaseHttpClient {
@@ -120,35 +121,36 @@ export class BasicCrawler {
      * the concurrency shortcuts folded in (owned, so the crawler starts and stops it).
      */
     #resolveConcurrencySystem;
-    /** As resolved by `init()`. Absent until the first run, so a `teardown()` before it is a no-op. */
-    #concurrencySystemDep;
+    /** The last run started on this instance. It outlives the run, so anything that drives one goes through `#liveRun`. */
+    #run;
+    #concurrencySystem;
+    /** The run in progress, if any — a finished run is history and cannot be driven. */
+    get #liveRun() {
+        return this.#run?.isLive === true ? this.#run : undefined;
+    }
+    /** The run whose task loop is dispatching, if any — only that one can be paused and resumed. */
+    get #dispatchingRun() {
+        return this.#liveRun?.isDispatching === true ? this.#liveRun : undefined;
+    }
+    /** Whether a {@link BasicCrawler.run|`run()`} is in progress on this instance. */
+    get running() {
+        return this.#liveRun !== undefined;
+    }
     /**
-     * The concurrency governor this run is booking its requests against — either the
+     * The concurrency governor the run in progress is booking its requests against — either the
      * {@link BasicCrawlerOptions.concurrencySystem|`concurrencySystem`} that was injected, or the default the
      * crawler built for itself. Read it for telemetry: `desiredConcurrency`, `currentConcurrency`, `isRunning`.
      *
-     * > *NOTE:* `undefined` until {@link BasicCrawler.run|`crawler.run()`} has resolved it. A crawler-owned default
-     * is also rebuilt for every run, so the instance is not stable across runs.
+     * > *NOTE:* `undefined` outside a run, and a crawler-owned default is rebuilt for every run — so read it
+     * during a run rather than caching it across runs.
      *
      * {@link IConcurrencySystem} is deliberately read-only. Tuning concurrency *while a crawl is running* means
      * owning the instance: build a {@link ConcurrencySystem} yourself and inject it, then set
      * `minConcurrency`/`maxConcurrency`/`desiredConcurrency` on your own reference.
      */
     get concurrencySystem() {
-        return this.#concurrencySystemDep?.maybeValue;
+        return this.#concurrencySystem?.value;
     }
-    /**
-     * The task loop that dispatches this run's requests. Private on purpose — it is a bare parallel task runner with
-     * no configuration left of its own (see {@link ConcurrencySystem}), and everything a caller legitimately did
-     * with it now has a crawler-level counterpart: {@link BasicCrawler.pause|`pause()`},
-     * {@link BasicCrawler.resume|`resume()`}, {@link BasicCrawler.teardown|`teardown()`} and
-     * {@link BasicCrawler.concurrencySystem|`concurrencySystem`}.
-     */
-    #autoscaledPool;
-    /** A pending nudge of the task loop, armed when the request manager announces when it will have work again. */
-    #taskLoopWakeTimer;
-    /** When the pending wake-up is due, so an earlier one can replace a later one. */
-    #taskLoopWakeAt = 0;
     /**
      * A reference to the underlying {@link IProxyConfiguration} instance that manages the crawler's proxies.
      * Only available if used by the crawler.
@@ -181,9 +183,7 @@ export class BasicCrawler {
         }
         return this.#contextPipeline;
     }
-    running = false;
     #hasFinishedBefore = false;
-    #unexpectedStop = false;
     /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
     get hasFinishedBefore() {
         return this.#hasFinishedBefore;
@@ -223,8 +223,6 @@ export class BasicCrawler {
     /** The resolved per-storage-type write policy overrides forwarded to each request's transaction. */
     #storageWritePolicy;
     #onSkippedRequest;
-    #closeEvents;
-    #loggedPerRun = new Set();
     #robotsTxtFileCache;
     #identity;
     #contextPipelineOptions;
@@ -497,12 +495,12 @@ export class BasicCrawler {
                 },
                 isTaskReadyFunction: async () => {
                     if (isMaxPagesExceeded()) {
-                        this.logOncePerRun('shuttingDown', 'Crawler reached the maxRequestsPerCrawl limit of ' +
+                        this.#liveRun?.logOnce('shuttingDown', 'Crawler reached the maxRequestsPerCrawl limit of ' +
                             `${this.#maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
                         return false;
                     }
-                    if (this.#unexpectedStop) {
-                        this.logOncePerRun('shuttingDown', 'No new requests are allowed because the `stop()` method has been called. ' +
+                    if (this.#run?.stopRequested) {
+                        this.#liveRun?.logOnce('shuttingDown', 'No new requests are allowed because the `stop()` method has been called. ' +
                             'Ongoing requests will be allowed to complete.');
                         return false;
                     }
@@ -515,7 +513,7 @@ export class BasicCrawler {
                             `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`);
                         return true;
                     }
-                    if (this.#unexpectedStop) {
+                    if (this.#run?.stopRequested) {
                         this.log.info('The crawler has finished all the remaining ongoing requests and will shut down now.');
                         return true;
                     }
@@ -619,6 +617,15 @@ export class BasicCrawler {
             registerDeferredCleanup: (cleanup) => {
                 deferredCleanup.push(cleanup);
             },
+            afterStorageCommit: (callback) => {
+                const transaction = currentStorageTransaction();
+                if (transaction === undefined) {
+                    throw new Error('afterStorageCommit() needs an active storage transaction, and there is none. ' +
+                        "With `transactionalStorage: false` the request's writes are applied as they are " +
+                        'made, so they throw at the call site instead of at commit time.');
+                }
+                transaction.afterCommit(callback);
+            },
             extendTimeout: (secs) => {
                 const extraMillis = secs * 1000;
                 // the current `addTimeoutToPromise` window (the request handler, or a navigation hook)...
@@ -804,18 +811,29 @@ export class BasicCrawler {
                     await pool.resetStore();
                 });
             }
-            this.#unexpectedStop = false;
-            this.running = true;
-            this.#loggedPerRun.clear();
-            await purgeDefaultStorages({
-                onlyPurgeOnce: true,
-                storageBackend: serviceLocator.getStorageBackend(),
-                configuration: serviceLocator.getConfiguration(),
+            // An owned governor is rebuilt for every run, so it always starts from a clean slate — stale
+            // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this
+            // run's scaling. An injected one is long-lived and its lifecycle belongs to the caller.
+            this.#concurrencySystem = this.#resolveConcurrencySystem();
+            const run = new CrawlerRun({
+                log: this.log,
+                concurrencySystem: this.#concurrencySystem.value,
+                taskLoopOptions: this.#taskLoopOptions,
+                consumer: this.#identity,
             });
-            if (requests) {
-                await this.addRequests(requests, options);
-            }
+            this.#run = run;
             try {
+                await purgeDefaultStorages({
+                    onlyPurgeOnce: true,
+                    storageBackend: serviceLocator.getStorageBackend(),
+                    configuration: serviceLocator.getConfiguration(),
+                });
+                if (requests) {
+                    await this.addRequests(requests, options);
+                }
+                // An injected governor is the caller's to run; an owned one is started here and stopped again
+                // below if the startup fails. Either way the task loop rejects one that is not running.
+                await this.#concurrencySystem.ifOwned((system) => system.start());
                 await this.init();
                 await this.statistics.startCapturing();
             }
@@ -825,7 +843,8 @@ export class BasicCrawler {
                     this.log.exception(teardownError, 'Cleaning up after a failed crawler startup failed.');
                 });
                 // The run never began, so let the instance be run again instead of leaving it wedged as `running`.
-                this.running = false;
+                await run.finish();
+                await this.#concurrencySystem.ifOwned((system) => system.stop());
                 throw error;
             }
             const periodicLogger = this.getPeriodicLogger();
@@ -833,7 +852,7 @@ export class BasicCrawler {
             const sigintHandler = async () => {
                 this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
                 await this.pauseOnMigration();
-                await this.#autoscaledPool.abort();
+                await run.abort();
             };
             // Attach a listener to handle migration and aborting events gracefully.
             const boundPauseOnMigration = this.pauseOnMigration.bind(this);
@@ -843,7 +862,7 @@ export class BasicCrawler {
             eventManager.on(EventType.ABORTING, boundPauseOnMigration);
             let stats = {};
             try {
-                await this.#autoscaledPool.run();
+                await run.dispatchRequests();
             }
             finally {
                 await this.statistics.stopCapturing();
@@ -896,7 +915,8 @@ export class BasicCrawler {
                 }
                 periodicLogger.stop();
                 this.setStatusMessage(`Finished! Total ${this.statistics.state.requestsFinished + this.statistics.state.requestsFailed} requests: ${this.statistics.state.requestsFinished} succeeded, ${this.statistics.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
-                this.running = false;
+                await run.finish();
+                await this.#concurrencySystem.ifOwned((system) => system.stop());
                 this.#hasFinishedBefore = true;
             }
             return stats;
@@ -910,11 +930,12 @@ export class BasicCrawler {
      * To stop the crawler immediately, use {@link BasicCrawler.teardown|`crawler.teardown()`} instead.
      */
     stop(reason = 'The crawler has been gracefully stopped.') {
-        if (this.#unexpectedStop) {
+        const run = this.#liveRun;
+        if (run === undefined) {
+            this.log.warning('Cannot stop a crawler that is not running.');
             return;
         }
-        this.log.info(reason);
-        this.#unexpectedStop = true;
+        run.stop(reason);
     }
     /**
      * Stops dispatching new requests, letting the in-progress ones finish. Resolves once they have settled, or rejects
@@ -925,22 +946,24 @@ export class BasicCrawler {
      * throughout, since a shared one may still be serving other crawlers.
      */
     async pause(timeoutSecs) {
-        if (!this.#autoscaledPool) {
+        const run = this.#dispatchingRun;
+        if (run === undefined) {
             this.log.warning('Cannot pause a crawler that is not running.');
             return;
         }
-        await this.#autoscaledPool.pause(timeoutSecs);
+        await run.pause(timeoutSecs);
     }
     /**
      * Resumes a run suspended with {@link BasicCrawler.pause|`pause()`}, letting the crawler dispatch requests
      * again. A no-op on a crawler that is not paused.
      */
     resume() {
-        if (!this.#autoscaledPool) {
+        const run = this.#dispatchingRun;
+        if (run === undefined) {
             this.log.warning('Cannot resume a crawler that is not running.');
             return;
         }
-        this.#autoscaledPool.resume();
+        run.resume();
     }
     /**
      * Returns the crawler's {@link IRequestManager|request manager}, opening the default {@link RequestQueue}
@@ -1048,22 +1071,18 @@ export class BasicCrawler {
         // A skipped request is a *successful* outcome, but the interrupt still unwinds through the
         // transaction scope, which rolls back - so the skip bookkeeping must write directly.
         await withDirectStorageAccess(async () => {
+            // Enqueueing happens outside a run too - `addRequests()` before the first `run()` - so these dedupe
+            // against the logger for the crawler's lifetime rather than per run.
             if (options.reason === 'limit') {
-                this.logOncePerRun('maxRequestsPerCrawl', 'The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
-                    `${this.#maxRequestsPerCrawl} requests and no further requests will be added.`);
+                this.log.info('The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
+                    `${this.#maxRequestsPerCrawl} requests and no further requests will be added.`, undefined, { once: true });
             }
             if (options.reason === 'depth') {
-                this.logOncePerRun('maxCrawlDepth', `The crawler reached the maxCrawlDepth limit of ${this.#maxCrawlDepth} and no further requests will be enqueued.`);
+                this.log.info(`The crawler reached the maxCrawlDepth limit of ${this.#maxCrawlDepth} and no further requests will be enqueued.`, undefined, { once: true });
             }
             await this.#onSkippedRequest?.(options);
         });
     }
-    logOncePerRun(key, message, level = 'info') {
-        if (!this.#loggedPerRun.has(key)) {
-            this.log[level](message);
-            this.#loggedPerRun.add(key);
-        }
-    }
     /**
      * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
      * adding the rest in background. You can configure the batch size via `batchSize` option and the sleep time in between
@@ -1268,10 +1287,14 @@ export class BasicCrawler {
      * Initializes the crawler.
      */
     async init() {
+        const run = this.#liveRun;
+        if (run === undefined) {
+            throw new Error('There is no crawl in progress - `run()` is the only thing that starts one.');
+        }
         const eventManager = serviceLocator.getEventManager();
         if (!eventManager.isInitialized()) {
             await eventManager.init();
-            this.#closeEvents = true;
+            run.ownsEventManager = true;
         }
         // Warn once at startup if the internal timeout is shorter than the phases it is meant to outlast. It is
         // floored per request so it will not actually cut them short, but the configured value is then effectively
@@ -1283,16 +1306,6 @@ export class BasicCrawler {
                 `and request handler timeouts combined (${phasesMillis / 1000}s); it will be raised per request ` +
                 `so it does not cut them short.`);
         }
-        // An owned governor is rebuilt (and started) for every run, so it always starts from a clean slate — stale
-        // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this run's
-        // scaling. An injected one is long-lived and its lifecycle belongs to the caller.
-        this.#concurrencySystemDep = this.#resolveConcurrencySystem();
-        await this.#concurrencySystemDep.ifOwned((system) => system.start());
-        this.#autoscaledPool = new AutoscaledPool({
-            ...this.#taskLoopOptions,
-            concurrencySystem: this.#concurrencySystemDep.value,
-            consumer: this.#identity,
-        });
         await this.getRequestManager();
     }
     /**
@@ -1415,10 +1428,10 @@ export class BasicCrawler {
             return true;
         }
         const domain = hostnameOrUrl(url);
-        this.logOncePerRun(`rateLimitNotThrottled:${domain}`, `"${domain}" responded with HTTP 429 (Too Many Requests), but the crawler's request manager does not ` +
+        this.log.warningOnce(`"${domain}" responded with HTTP 429 (Too Many Requests), but the crawler's request manager does not ` +
             'pace that domain, so the response is handled like any other, with no per-domain delay. Set ' +
             `\`sameDomainDelaySecs\`, or pass a \`ThrottlingRequestManager\` covering "${domain}" as ` +
-            '`requestManager`, to honour `Retry-After` and apply exponential backoff instead.', 'warning');
+            '`requestManager`, to honour `Retry-After` and apply exponential backoff instead.');
         return false;
     }
     /** Hands a robots.txt `Crawl-delay` to the request manager, warning if it will not be honoured. */
@@ -1434,10 +1447,10 @@ export class BasicCrawler {
             return;
         }
         const domain = hostnameOrUrl(url);
-        this.logOncePerRun(`crawlDelayIgnored:${domain}`, `robots.txt for "${domain}" defines a crawl-delay of ${delaySeconds}s, but the crawler's request ` +
+        this.log.warningOnce(`robots.txt for "${domain}" defines a crawl-delay of ${delaySeconds}s, but the crawler's request ` +
             'manager does not pace that domain, so its requests will not be paced. Set ' +
             `\`sameDomainDelaySecs\`, or pass a \`ThrottlingRequestManager\` covering "${domain}" as ` +
-            '`requestManager`.', 'warning');
+            '`requestManager`.');
     }
     async getRobotsTxtFileForUrl(url) {
         if (!this.#respectRobotsTxtFile) {
@@ -1459,9 +1472,9 @@ export class BasicCrawler {
         }
     }
     async pauseOnMigration() {
-        if (this.#autoscaledPool) {
-            // if run wasn't called, this is going to crash
-            await this.#autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
+        const run = this.#dispatchingRun;
+        if (run) {
+            await run.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
                 if (err.message.includes('running tasks did not finish')) {
                     this.log.error('The crawler was paused due to migration to another host, ' +
                         "but some requests did not finish in time. Those requests' results may be duplicated.");
@@ -1615,36 +1628,10 @@ export class BasicCrawler {
         }
         const state = await this.requestManager.checkReadiness();
         if (state.status === 'waiting' && state.readyAt !== undefined) {
-            this.#scheduleTaskLoopWake(state.readyAt);
+            this.#run?.scheduleWake(state.readyAt);
         }
         return state.status === 'ready';
     }
-    /**
-     * Nudges the task loop at `readyAt`, on a single timer that only an earlier one replaces. The pool polls anyway
-     * every `maybeRunIntervalSecs` (0.5s by default), so this only shortens the wait - hence one timer rather than
-     * one per probe, and `unref`'d so it never keeps the process alive.
-     */
-    #scheduleTaskLoopWake(readyAt) {
-        if (this.#taskLoopWakeTimer !== undefined) {
-            if (this.#taskLoopWakeAt <= readyAt) {
-                return;
-            }
-            clearTimeout(this.#taskLoopWakeTimer);
-        }
-        this.#taskLoopWakeAt = readyAt;
-        this.#taskLoopWakeTimer = setTimeout(() => {
-            this.#taskLoopWakeTimer = undefined;
-            void this.#autoscaledPool?.notify();
-        }, Math.max(0, readyAt - Date.now()));
-        this.#taskLoopWakeTimer.unref();
-    }
-    /** Drops a pending task-loop wake-up, so a finished run leaves no timer behind. */
-    #clearTaskLoopWake() {
-        if (this.#taskLoopWakeTimer !== undefined) {
-            clearTimeout(this.#taskLoopWakeTimer);
-            this.#taskLoopWakeTimer = undefined;
-        }
-    }
     /**
      * Unwraps errors thrown by the context pipeline to get the actual user error.
      * RequestHandlerError and ContextPipelineInitializationError wrap the actual error.
@@ -1770,28 +1757,42 @@ export class BasicCrawler {
         return request.retryCount < maxRequestRetries;
     }
     /**
-     * Stops the crawler immediately.
+     * Ends the run in progress immediately, without waiting for the requests already in flight.
      *
-     * This method doesn't wait for currently active requests to finish.
+     * This runs at the end of every {@link BasicCrawler.run|`run()`}, so what it releases is per-run only, and
+     * a repeated `run()` has to find everything it needs either intact or re-establishable. What outlives a run is
+     * released by {@link BasicCrawler.destroy|`destroy()`} instead.
      *
      * To stop the crawler gracefully (waiting for all running requests to finish), use {@link BasicCrawler.stop|`crawler.stop()`} instead.
      */
     async teardown() {
+        const eventManager = serviceLocator.getEventManager();
+        const ownsEventManager = this.#run?.ownsEventManager === true;
         // First, so that the pool writes its final state once - its teardown stops it listening before the
         // persistence event below goes out.
         await this.#sessionPoolDep.ifOwned(async (pool) => pool.teardown());
         // When this crawler initialized the event manager, its close() call emits
         // the final persistence event after the crawler-specific state has been
         // saved. External event managers still need an explicit event here.
-        if (!this.#closeEvents) {
-            serviceLocator.getEventManager().emit(EventType.PERSIST_STATE, { isMigrating: false });
+        if (!ownsEventManager) {
+            eventManager.emit(EventType.PERSIST_STATE, { isMigrating: false });
         }
-        if (this.#closeEvents) {
-            await serviceLocator.getEventManager().close();
+        if (ownsEventManager) {
+            await eventManager.close();
         }
-        this.#clearTaskLoopWake();
-        await this.#autoscaledPool?.abort();
-        await this.#concurrencySystemDep?.ifOwned((system) => system.stop());
+        await this.#run?.end();
+    }
+    /**
+     * Releases what the crawler owns beyond a single run — in the browser crawlers, the browser pool. A finished
+     * `run()` has already released everything a run owns, so this is only needed before dropping a crawler you
+     * could otherwise have run again.
+     */
+    async destroy() {
+        // An abandoned run - one whose `run()` never returned - would otherwise keep dispatching requests.
+        await this.#run?.end();
+    }
+    async [Symbol.asyncDispose]() {
+        await this.destroy();
     }
     getCookieHeaderFromRequest(request) {
         if (request.headers?.Cookie && request.headers?.cookie) {

package/internals/crawler-run.d.ts

@@ -0,0 +1,74 @@
+import type { AutoscaledPoolOptions, CrawleeLogger, IConcurrencySystem } from '@crawlee/core';
+/** Everything a run needs to exist. Whatever a crawler resolves per run is resolved before the run is built. */
+export interface CrawlerRunSetup {
+    log: CrawleeLogger;
+    /** The governor this run books its requests against, started and stopped by whoever owns it. */
+    concurrencySystem: IConcurrencySystem;
+    taskLoopOptions: Omit<AutoscaledPoolOptions, 'concurrencySystem' | 'consumer'>;
+    consumer: AutoscaledPoolOptions['consumer'];
+}
+/**
+ * Everything that belongs to a single {@link BasicCrawler.run|`crawler.run()`} — the task loop dispatching its
+ * requests, the concurrency governor those requests are booked against, and the bookkeeping that must not survive
+ * into a later run.
+ *
+ * The crawler builds a fresh one for every run, which is what keeps per-run state from leaking across runs: there
+ * is no reset checklist to keep in sync. A finished run stays readable as the history of the last crawl, so
+ * liveness is asked about explicitly (`isLive`) rather than inferred from presence.
+ *
+ * The lifecycle is `dispatchRequests()` and `finish()`. Everything a run owns exists from construction, so there
+ * is no half-built state to guard against in between.
+ *
+ * @internal
+ */
+export declare class CrawlerRun {
+    #private;
+    /** Whether this run initialized the ambient event manager, and is therefore the one that closes it. */
+    ownsEventManager: boolean;
+    constructor(setup: CrawlerRunSetup);
+    /**
+     * Runs the task loop until the crawl is over or {@link CrawlerRun.abort|aborted}. The governor has to be
+     * running by now — the task loop refuses to book anything against one that is not.
+     */
+    dispatchRequests(): Promise<void>;
+    /** Whether the `run()` call this belongs to is still executing. */
+    get isLive(): boolean;
+    /**
+     * Whether the task loop is dispatching requests. Pausing one that has not started would not suspend a crawl,
+     * it would keep it from ever starting.
+     */
+    get isDispatching(): boolean;
+    /** Whether a graceful shutdown of this run has been requested. */
+    get stopRequested(): boolean;
+    /** Asks the run to stop taking new requests. Only the first call logs `reason`. */
+    stop(reason: string): void;
+    /** Stops dispatching new requests, resolving once the ones in flight have settled. */
+    pause(timeoutSecs?: number): Promise<void>;
+    /** Resumes dispatching after a {@link CrawlerRun.pause|`pause()`}. */
+    resume(): void;
+    /** Ends the crawl without waiting for the requests in flight, so a pending `crawl()` resolves. */
+    abort(): Promise<void>;
+    /**
+     * Logs `message` the first time this run asks for it under `key`. Keyed rather than deduped by text, so that
+     * two messages describing the same shutdown say it once between them.
+     */
+    logOnce(key: string, message: string): void;
+    /**
+     * Nudges the task loop at `readyAt`, when the request manager says it will have work again. The loop polls on
+     * its own anyway, so this only shortens the wait — hence a single timer that only an earlier wake-up replaces,
+     * `unref`'d so it never keeps the process alive. A run that has ended arms nothing.
+     */
+    scheduleWake(readyAt: number): void;
+    /**
+     * Ends the run: drops a pending wake-up and aborts the task loop. Idempotent, so aborting a run through
+     * `teardown()` and then finishing it does the work once. The governor is left alone — it belongs to whoever
+     * started it.
+     */
+    end(): Promise<void>;
+    /**
+     * Ends the run and marks it as history: the crawler stops reporting itself as running, and what is left is
+     * readable but no longer drivable. Ending here too means a subclass `teardown()` that forgets `super` cannot
+     * leave the task loop dispatching.
+     */
+    finish(): Promise<void>;
+}

package/internals/crawler-run.js

@@ -0,0 +1,142 @@
+import { AutoscaledPool } from '@crawlee/core';
+/**
+ * Everything that belongs to a single {@link BasicCrawler.run|`crawler.run()`} — the task loop dispatching its
+ * requests, the concurrency governor those requests are booked against, and the bookkeeping that must not survive
+ * into a later run.
+ *
+ * The crawler builds a fresh one for every run, which is what keeps per-run state from leaking across runs: there
+ * is no reset checklist to keep in sync. A finished run stays readable as the history of the last crawl, so
+ * liveness is asked about explicitly (`isLive`) rather than inferred from presence.
+ *
+ * The lifecycle is `dispatchRequests()` and `finish()`. Everything a run owns exists from construction, so there
+ * is no half-built state to guard against in between.
+ *
+ * @internal
+ */
+export class CrawlerRun {
+    #log;
+    #autoscaledPool;
+    #loggedOnce = new Set();
+    #stopRequested = false;
+    #dispatching = false;
+    #ended = false;
+    #finished = false;
+    #wakeTimer;
+    #wakeAt = 0;
+    /** Whether this run initialized the ambient event manager, and is therefore the one that closes it. */
+    ownsEventManager = false;
+    constructor(setup) {
+        this.#log = setup.log;
+        this.#autoscaledPool = new AutoscaledPool({
+            ...setup.taskLoopOptions,
+            concurrencySystem: setup.concurrencySystem,
+            consumer: setup.consumer,
+        });
+    }
+    /**
+     * Runs the task loop until the crawl is over or {@link CrawlerRun.abort|aborted}. The governor has to be
+     * running by now — the task loop refuses to book anything against one that is not.
+     */
+    async dispatchRequests() {
+        this.#dispatching = true;
+        try {
+            await this.#autoscaledPool.run();
+        }
+        finally {
+            this.#dispatching = false;
+        }
+    }
+    /** Whether the `run()` call this belongs to is still executing. */
+    get isLive() {
+        return !this.#finished;
+    }
+    /**
+     * Whether the task loop is dispatching requests. Pausing one that has not started would not suspend a crawl,
+     * it would keep it from ever starting.
+     */
+    get isDispatching() {
+        return this.#dispatching;
+    }
+    /** Whether a graceful shutdown of this run has been requested. */
+    get stopRequested() {
+        return this.#stopRequested;
+    }
+    /** Asks the run to stop taking new requests. Only the first call logs `reason`. */
+    stop(reason) {
+        if (this.#stopRequested) {
+            return;
+        }
+        this.#log.info(reason);
+        this.#stopRequested = true;
+    }
+    /** Stops dispatching new requests, resolving once the ones in flight have settled. */
+    async pause(timeoutSecs) {
+        await this.#autoscaledPool.pause(timeoutSecs);
+    }
+    /** Resumes dispatching after a {@link CrawlerRun.pause|`pause()`}. */
+    resume() {
+        this.#autoscaledPool.resume();
+    }
+    /** Ends the crawl without waiting for the requests in flight, so a pending `crawl()` resolves. */
+    async abort() {
+        await this.#autoscaledPool.abort();
+    }
+    /**
+     * Logs `message` the first time this run asks for it under `key`. Keyed rather than deduped by text, so that
+     * two messages describing the same shutdown say it once between them.
+     */
+    logOnce(key, message) {
+        if (this.#loggedOnce.has(key)) {
+            return;
+        }
+        this.#log.info(message);
+        this.#loggedOnce.add(key);
+    }
+    /**
+     * Nudges the task loop at `readyAt`, when the request manager says it will have work again. The loop polls on
+     * its own anyway, so this only shortens the wait — hence a single timer that only an earlier wake-up replaces,
+     * `unref`'d so it never keeps the process alive. A run that has ended arms nothing.
+     */
+    scheduleWake(readyAt) {
+        if (this.#ended) {
+            return;
+        }
+        if (this.#wakeTimer !== undefined) {
+            if (this.#wakeAt <= readyAt) {
+                return;
+            }
+            clearTimeout(this.#wakeTimer);
+        }
+        this.#wakeAt = readyAt;
+        this.#wakeTimer = setTimeout(() => {
+            this.#wakeTimer = undefined;
+            void this.#autoscaledPool.notify();
+        }, Math.max(0, readyAt - Date.now()));
+        this.#wakeTimer.unref();
+    }
+    /**
+     * Ends the run: drops a pending wake-up and aborts the task loop. Idempotent, so aborting a run through
+     * `teardown()` and then finishing it does the work once. The governor is left alone — it belongs to whoever
+     * started it.
+     */
+    async end() {
+        if (this.#ended) {
+            return;
+        }
+        this.#ended = true;
+        if (this.#wakeTimer !== undefined) {
+            clearTimeout(this.#wakeTimer);
+            this.#wakeTimer = undefined;
+        }
+        await this.#autoscaledPool.abort();
+    }
+    /**
+     * Ends the run and marks it as history: the crawler stops reporting itself as running, and what is left is
+     * readable but no longer drivable. Ending here too means a subclass `teardown()` that forgets `super` cannot
+     * leave the task loop dispatching.
+     */
+    async finish() {
+        await this.end();
+        this.#finished = true;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.165",
+    "version": "4.0.0-beta.167",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/core": "4.0.0-beta.165",
-        "@crawlee/http-client": "4.0.0-beta.165",
-        "@crawlee/types": "4.0.0-beta.165",
-        "@crawlee/utils": "4.0.0-beta.165",
+        "@crawlee/core": "4.0.0-beta.167",
+        "@crawlee/http-client": "4.0.0-beta.167",
+        "@crawlee/types": "4.0.0-beta.167",
+        "@crawlee/utils": "4.0.0-beta.167",
         "csv-stringify": "^6.5.2",
         "tldts": "^7.0.6",
         "tslib": "^2.8.1",
@@ -53,7 +53,7 @@
         "zod": "^4.5.4"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "4.0.0-beta.165"
+        "@crawlee/impit-client": "4.0.0-beta.167"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "e41d837b93d16b5e5c726d4dc00aa6f3f1282b51"
+    "gitHead": "8b26080bf07dc602aa0970e12fb66e7e9cba509b"
 }