@crawlee/basic 4.0.0-beta.236 → 4.0.0-beta.238

package/internals/basic-crawler.js

@@ -795,18 +795,6 @@ export class BasicCrawler {
             if (this.running) {
                 throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
             }
-            if (this.#hasFinishedBefore) {
-                // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
-                await this.#statisticsDep.ifOwned(async (stats) => {
-                    stats.reset();
-                    await stats.resetStore();
-                });
-                // A session pool the crawler built follows the same blank-slate rule as its statistics.
-                await this.#sessionPoolDep.ifOwned(async (pool) => {
-                    pool.reset();
-                    await pool.resetStore();
-                });
-            }
             // An owned governor is rebuilt for every run, so it always starts from a clean slate — stale
             // resource snapshots or a previous run's scaled desired concurrency would otherwise distort this
             // run's scaling. An injected one is long-lived and its lifecycle belongs to the caller.
@@ -819,6 +807,18 @@ export class BasicCrawler {
             });
             this.#run = run;
             try {
+                if (this.#hasFinishedBefore) {
+                    // A supplied statistics instance keeps whatever state it was handed - only wipe a default we built.
+                    await this.#statisticsDep.ifOwned(async (stats) => {
+                        stats.reset();
+                        await stats.resetStore();
+                    });
+                    // A session pool the crawler built follows the same blank-slate rule as its statistics.
+                    await this.#sessionPoolDep.ifOwned(async (pool) => {
+                        pool.reset();
+                        await pool.resetStore();
+                    });
+                }
                 await purgeDefaultStorages({
                     onlyPurgeOnce: true,
                     storageBackend: serviceLocator.getStorageBackend(),
@@ -1455,7 +1455,15 @@ export class BasicCrawler {
             if (cachedRobotsTxtFile) {
                 return cachedRobotsTxtFile;
             }
-            const robotsTxtFile = await RobotsTxtFile.find(url, { logger: this.log });
+            // Fetch robots.txt the way pages are fetched: through the crawler's HTTP client, with a session's browser
+            // fingerprint and proxy. Sites with bot protection answer a bare request with 4xx or 5xx, and that status
+            // alone would then decide whether everything is allowed or disallowed.
+            const robotsTxtFile = await RobotsTxtFile.find(url, {
+                httpClient: this.httpClient,
+                session: await this.sessionPool.getSession(),
+                timeoutMillis: 30_000,
+                logger: this.log,
+            });
             this.#robotsTxtFileCache.add(origin, robotsTxtFile);
             return robotsTxtFile;
         }

package/internals/crawler-run.js

@@ -38,6 +38,8 @@ export class CrawlerRun {
      * running by now — the task loop refuses to book anything against one that is not.
      */
     async dispatchRequests() {
+        if (this.#ended)
+            return;
         this.#dispatching = true;
         try {
             await this.#autoscaledPool.run();

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.236",
+    "version": "4.0.0-beta.238",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.13.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/core": "4.0.0-beta.236",
-        "@crawlee/http-client": "4.0.0-beta.236",
-        "@crawlee/types": "4.0.0-beta.236",
-        "@crawlee/utils": "4.0.0-beta.236",
+        "@crawlee/core": "4.0.0-beta.238",
+        "@crawlee/http-client": "4.0.0-beta.238",
+        "@crawlee/types": "4.0.0-beta.238",
+        "@crawlee/utils": "4.0.0-beta.238",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "csv-stringify": "^6.5.2",
@@ -57,7 +57,7 @@
         "zod": "^4.5.4"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "4.0.0-beta.236"
+        "@crawlee/impit-client": "4.0.0-beta.238"
     },
     "lerna": {
         "command": {
@@ -66,5 +66,5 @@
             }
         }
     },
-    "gitHead": "2fd655a2eda0a16125ebeb455ec835d9386d8846"
+    "gitHead": "8c6b2f3b5a811ca68672b6c68b6587a925baf2aa"
 }