@crawlee/basic 4.0.0-beta.157 → 4.0.0-beta.159

package/internals/basic-crawler.d.ts

@@ -434,7 +434,8 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     }, CrawlingContext>;
     get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
     running: boolean;
-    hasFinishedBefore: boolean;
+    /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
+    get hasFinishedBefore(): boolean;
     get log(): CrawleeLogger;
     protected readonly requestHandler: RequestHandler<ExtendedContext>;
     protected readonly internalTimeoutMillis: number;
@@ -553,8 +554,12 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
      * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
      *
+     * Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous
+     * run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one
+     * if that is what you want.
+     *
      * @param [requests] The requests to add.
-     * @param [options] Options for the request queue.
+     * @param [options] Options for adding the initial requests.
      */
     run(requests?: TypedRequestsLike<Routes>, options?: CrawlerRunOptions): Promise<FinalStatistics>;
     /**
@@ -766,17 +771,6 @@ export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions, En
 export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {
 }
 export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
-    /**
-     * Controls whether the request queue is purged between repeated `run()` calls on the same crawler instance.
-     * Purging clears all requests and resets internal counters, allowing the same URLs to be processed again.
-     *
-     * - **`undefined`** (default) — only the crawler's own (auto-created) queue is purged.
-     *   A user-supplied `requestQueue` is left untouched.
-     * - **`true`** — the queue is always purged, even if it was supplied by the user.
-     * - **`false`** — nothing is purged. Only genuinely new requests will be processed;
-     *   note that even a failed request is considered handled.
-     */
-    purgeRequestQueue?: boolean;
 }
 /**
  * Creates new {@link Router} instance that works based on request labels.

package/internals/basic-crawler.js

@@ -182,8 +182,12 @@ export class BasicCrawler {
         return this.#contextPipeline;
     }
     running = false;
-    hasFinishedBefore = false;
+    #hasFinishedBefore = false;
     #unexpectedStop = false;
+    /** Whether a `run()` on this instance has already finished - a repeated one continues where it left off. */
+    get hasFinishedBefore() {
+        return this.#hasFinishedBefore;
+    }
     #log;
     get log() {
         return this.#log;
@@ -195,16 +199,6 @@ export class BasicCrawler {
     internalTimeoutMillis;
     #maxRequestRetries;
     #maxCrawlDepth;
-    /**
-     * How much of {@link BasicCrawler.requestManager} the crawler may empty between repeated `run()` calls.
-     *
-     * - `all` — nothing under it came from the caller, so one `purge()` on the outside covers everything.
-     * - `none` — the caller supplied it and the crawler put nothing of its own inside.
-     * - `ambiguous` — the caller supplied it, but `sameDomainDelaySecs` put the crawler's own per-domain queues
-     *   underneath: purging empties the caller's storage too, skipping leaves ours stale. A repeated `run()` asks
-     *   rather than guessing.
-     */
-    #purgeableExtent;
     #maxRequestsPerCrawl;
     get handledRequestsCount() {
         return this.statistics.state.requestsFinished + this.statistics.state.requestsFailed;
@@ -357,14 +351,6 @@ export class BasicCrawler {
                 }) ??
                     false);
             const pacerNeeded = sameDomainDelaySecs > 0 && !floorTaken;
-            // Our per-domain queues under a manager the caller owns is the case with no right answer; a floor
-            // it took leaves nothing of ours behind. See `#purgeableExtent`.
-            if (suppliedManager === undefined) {
-                this.#purgeableExtent = 'all';
-            }
-            else {
-                this.#purgeableExtent = pacerNeeded ? 'ambiguous' : 'none';
-            }
             // Built here rather than at first use so it can sit *inside* the tandem below, which is where a
             // loader's transferred requests pass through it.
             const writableManager = pacerNeeded
@@ -793,8 +779,12 @@ export class BasicCrawler {
      * We can use the `requests` parameter to enqueue the initial requests — it is a shortcut for
      * running {@link BasicCrawler.addRequests|`crawler.addRequests()`} before {@link BasicCrawler.run|`crawler.run()`}.
      *
+     * Calling `run()` again on the same instance keeps crawling the same request manager - requests the previous
+     * run handled (a failed one counts as handled) are not processed again. Purge the queue or open a fresh one
+     * if that is what you want.
+     *
      * @param [requests] The requests to add.
-     * @param [options] Options for the request queue.
+     * @param [options] Options for adding the initial requests.
      */
     async run(requests, options) {
         // A crawl is the top level of its own transaction and timeout scope, not a participant in the caller's.
@@ -802,26 +792,7 @@ export class BasicCrawler {
             if (this.running) {
                 throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
             }
-            const { purgeRequestQueue, ...addRequestsOptions } = options ?? {};
-            if (this.hasFinishedBefore) {
-                // When executing the run method for the second time explicitly,
-                // we need to purge the RQ to allow processing the same requests again — this is important so users can
-                // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
-                // ignored — as a failed request is still handled.
-                // `purgeRequestQueue` unset purges only storage the crawler opened itself (see `#purgeableExtent`);
-                // `true` also purges a caller-supplied manager, `false` purges nothing.
-                if (purgeRequestQueue === undefined && this.#purgeableExtent === 'ambiguous') {
-                    throw new Error('Cannot decide what to purge before running again: `sameDomainDelaySecs` paces the request ' +
-                        'manager you supplied, so the per-domain queues that have to be emptied are the ' +
-                        "crawler's while the manager underneath them is yours. Say which you want: " +
-                        '`run(requests, { purgeRequestQueue: true })` empties both, `false` empties neither.');
-                }
-                if (purgeRequestQueue !== false &&
-                    (this.#purgeableExtent === 'all' || purgeRequestQueue === true)) {
-                    // One call from the outside in reaches everything the manager wraps, a pacer's per-domain queues
-                    // included.
-                    await this.requestManager?.purge?.();
-                }
+            if (this.#hasFinishedBefore) {
                 // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
                 await this.#statisticsDep.ifOwned(async (stats) => {
                     stats.reset();
@@ -838,7 +809,7 @@ export class BasicCrawler {
                 configuration: serviceLocator.getConfiguration(),
             });
             if (requests) {
-                await this.addRequests(requests, addRequestsOptions);
+                await this.addRequests(requests, options);
             }
             try {
                 await this.init();
@@ -884,6 +855,22 @@ export class BasicCrawler {
                     ...finalStats,
                 };
                 this.log.info('Final request statistics:', stats);
+                // A crawl that did nothing while the manager holds only handled requests is a mistake whoever
+                // handled them - this run, another crawler on the same queue, or a previous process. Starting
+                // against handled requests is not: that is what resuming a crawl looks like.
+                if (stats.requestsFinished + stats.requestsFailed === 0) {
+                    // Never let the diagnostic itself break the run.
+                    const alreadyHandled = (await this.requestManager?.getHandledCount().catch(() => 0)) ?? 0;
+                    if (alreadyHandled > 0) {
+                        this.log.warningOnce('This crawl processed no requests - the request manager holds ' +
+                            `${alreadyHandled} request${alreadyHandled === 1 ? '' : 's'}, all of them ` +
+                            'already handled, and a failed request counts as handled too. Nothing ' +
+                            'empties a queue between runs, so to crawl them again, purge it ' +
+                            '(`await queue.purge()`) or use a fresh one (e.g. ' +
+                            '`RequestQueue.open({ alias: "second-run" })`) with a freshly created ' +
+                            'crawler instance.');
+                    }
+                }
                 if (this.statistics.errorTracker.total !== 0) {
                     const prettify = ([count, info]) => `${count}x: ${info.at(-1).trim()} (${info[0]})`;
                     this.log.info(`Error analysis:`, {
@@ -906,7 +893,7 @@ export class BasicCrawler {
                 periodicLogger.stop();
                 this.setStatusMessage(`Finished! Total ${this.statistics.state.requestsFinished + this.statistics.state.requestsFailed} requests: ${this.statistics.state.requestsFinished} succeeded, ${this.statistics.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
                 this.running = false;
-                this.hasFinishedBefore = true;
+                this.#hasFinishedBefore = true;
             }
             return stats;
         }));

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.157",
+    "version": "4.0.0-beta.159",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.157",
-        "@crawlee/http-client": "4.0.0-beta.157",
-        "@crawlee/types": "4.0.0-beta.157",
-        "@crawlee/utils": "4.0.0-beta.157",
+        "@crawlee/core": "4.0.0-beta.159",
+        "@crawlee/http-client": "4.0.0-beta.159",
+        "@crawlee/types": "4.0.0-beta.159",
+        "@crawlee/utils": "4.0.0-beta.159",
         "csv-stringify": "^6.5.2",
         "tldts": "^7.0.6",
         "tslib": "^2.8.1",
@@ -53,7 +53,7 @@
         "zod": "^4.4.3"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.157"
+        "@crawlee/impit-client": "4.0.0-beta.159"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "a1b3d4f72a2759a7179af6648083f57d022125a9"
+    "gitHead": "9f91c03967915be4e9b077a717e0e3494ad0a101"
 }