@crawlee/basic 4.0.0-beta.169 → 4.0.0-beta.170

package/internals/basic-crawler.js

@@ -201,7 +201,7 @@ export class BasicCrawler {
     #maxCrawlDepth;
     #maxRequestsPerCrawl;
     get handledRequestsCount() {
-        return this.statistics.state.requestsFinished + this.statistics.state.requestsFailed;
+        return this.statistics.state.requestsSucceeded + this.statistics.state.requestsFailed;
     }
     #statusMessageLoggingInterval;
     #statusMessageCallback;
@@ -447,8 +447,9 @@ export class BasicCrawler {
                     }
                     // Started here, rather than in `handleRequest`, so that a failure during context pipeline
                     // initialization (e.g. a browser page timing out before the request handler ever runs) is
-                    // still accounted for by `failJob` below - which is a no-op without a matching `startJob`.
-                    this.statistics.startJob(request.id || request.uniqueKey);
+                    // still accounted for by `recordRequestFailure` below - which is a no-op without a
+                    // matching `recordRequestStart`.
+                    this.statistics.recordRequestStart(request.id || request.uniqueKey);
                     const crawlingContext = { request };
                     try {
                         // The transaction spans the whole pipeline call, covering the navigation hooks
@@ -465,7 +466,7 @@ export class BasicCrawler {
                         // ContextPipelineInterruptedError means the request was intentionally skipped
                         // (e.g., doesn't match enqueue strategy after redirect). Just return gracefully.
                         if (error instanceof ContextPipelineInterruptedError) {
-                            this.statistics.discardJob(request.id || request.uniqueKey);
+                            this.statistics.discardRequestRecord(request.id || request.uniqueKey);
                             await this.timeoutAndRetry(async () => this.requestManager?.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${crawlingContext.request.url} (${crawlingContext.request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
                             return;
                         }
@@ -762,7 +763,7 @@ export class BasicCrawler {
             }
             else {
                 const total = await this.requestManager?.getTotalCount();
-                message = `Crawled ${this.statistics.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.statistics.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
+                message = `Crawled ${this.statistics.state.requestsSucceeded}${total ? `/${total}` : ''} pages, ${this.statistics.state.requestsFailed} failed requests, desired concurrency ${this.concurrencySystem?.desiredConcurrency ?? 0}.`;
             }
             if (this.#statusMessageCallback) {
                 await this.#statusMessageCallback({
@@ -872,7 +873,7 @@ export class BasicCrawler {
                 eventManager.off(EventType.ABORTING, boundPauseOnMigration);
                 const finalStats = this.statistics.calculate();
                 stats = {
-                    requestsFinished: this.statistics.state.requestsFinished,
+                    requestsSucceeded: this.statistics.state.requestsSucceeded,
                     requestsFailed: this.statistics.state.requestsFailed,
                     retryHistogram: this.statistics.requestRetryHistogram,
                     ...finalStats,
@@ -881,7 +882,7 @@ export class BasicCrawler {
                 // A crawl that did nothing while the manager holds only handled requests is a mistake whoever
                 // handled them - this run, another crawler on the same queue, or a previous process. Starting
                 // against handled requests is not: that is what resuming a crawl looks like.
-                if (stats.requestsFinished + stats.requestsFailed === 0) {
+                if (stats.requestsSucceeded + stats.requestsFailed === 0) {
                     // Never let the diagnostic itself break the run.
                     const alreadyHandled = (await this.requestManager?.getHandledCount().catch(() => 0)) ?? 0;
                     if (alreadyHandled > 0) {
@@ -914,7 +915,7 @@ export class BasicCrawler {
                     finished = true;
                 }
                 periodicLogger.stop();
-                this.setStatusMessage(`Finished! Total ${this.statistics.state.requestsFinished + this.statistics.state.requestsFailed} requests: ${this.statistics.state.requestsFinished} succeeded, ${this.statistics.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
+                this.setStatusMessage(`Finished! Total ${this.statistics.state.requestsSucceeded + this.statistics.state.requestsFailed} requests: ${this.statistics.state.requestsSucceeded} succeeded, ${this.statistics.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
                 await run.finish();
                 await this.#concurrencySystem.ifOwned((system) => system.stop());
                 this.#hasFinishedBefore = true;
@@ -1534,7 +1535,7 @@ export class BasicCrawler {
             await transaction?.commit();
             await this.timeoutAndRetry(async () => requestSource.markRequestAsHandled(request), this.internalTimeoutMillis, `Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`);
             isRequestLocked = false; // markRequestAsHandled succeeded and unlocked the request
-            this.statistics.finishJob(statisticsId, request.retryCount);
+            this.statistics.recordRequestSuccess(statisticsId, request.retryCount);
             // reclaim session if request finishes successfully
             request.state = RequestState.DONE;
             crawlingContext.session.markGood();
@@ -1704,7 +1705,7 @@ export class BasicCrawler {
         // or failed more than retryCount times and will not be retried anymore.
         // Mark the request as failed and do not retry.
         await source.markRequestAsHandled(request);
-        this.statistics.failJob(request.id || request.uniqueKey, request.retryCount);
+        this.statistics.recordRequestFailure(request.id || request.uniqueKey, request.retryCount);
         await this.handleFailedRequestHandler(crawlingContext, error); // This function prints an error message.
     }
     async handleFailedRequestHandler(crawlingContext, error) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.169",
+    "version": "4.0.0-beta.170",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/core": "4.0.0-beta.169",
-        "@crawlee/http-client": "4.0.0-beta.169",
-        "@crawlee/types": "4.0.0-beta.169",
-        "@crawlee/utils": "4.0.0-beta.169",
+        "@crawlee/core": "4.0.0-beta.170",
+        "@crawlee/http-client": "4.0.0-beta.170",
+        "@crawlee/types": "4.0.0-beta.170",
+        "@crawlee/utils": "4.0.0-beta.170",
         "csv-stringify": "^6.5.2",
         "tldts": "^7.0.6",
         "tslib": "^2.8.1",
@@ -53,7 +53,7 @@
         "zod": "^4.5.4"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "4.0.0-beta.169"
+        "@crawlee/impit-client": "4.0.0-beta.170"
     },
     "lerna": {
         "command": {
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "a42dc93fa0ef68180d83ab12af362082690f73e2"
+    "gitHead": "86e1c891da983125e9e18f1a2327fbcf5e0f5fc2"
 }