@crawlee/core 4.0.0-beta.169 → 4.0.0-beta.170

package/autoscaling/system_status.d.ts

@@ -80,12 +80,12 @@ export interface LoadSignalInfo {
     actualRatio: number;
 }
 export interface FinalStatistics {
-    requestsFinished: number;
+    requestsSucceeded: number;
     requestsFailed: number;
     retryHistogram: number[];
     requestAvgFailedDurationMillis: number;
-    requestAvgFinishedDurationMillis: number;
-    requestsFinishedPerMinute: number;
+    requestAvgSucceededDurationMillis: number;
+    requestsSucceededPerMinute: number;
     requestsFailedPerMinute: number;
     requestTotalDurationMillis: number;
     requestsTotal: number;

package/crawlers/statistics.d.ts

@@ -33,16 +33,16 @@ export interface IStatistics<StateExtension extends object = {}> {
     readonly errorTrackerRetry: ErrorTracker;
     /** The live statistics state the crawler reads for status messages and the final summary. */
     readonly state: StatisticState & StateExtension;
-    /** Retries histogram - index `i` holds the number of requests that finished after `i` retries. */
+    /** Retries histogram - index `i` holds the number of requests that settled after `i` retries. */
     readonly requestRetryHistogram: number[];
-    /** Marks a request as started, so its duration can be measured on finish/fail. */
-    startJob(id: number | string): void;
-    /** Marks a started request as finished, updating the finished counters and durations. */
-    finishJob(id: number | string, retryCount: number): void;
+    /** Marks a request as started, so its duration can be measured on success/failure. */
+    recordRequestStart(id: number | string): void;
+    /** Marks a started request as succeeded, updating the succeeded counters and durations. */
+    recordRequestSuccess(id: number | string, retryCount: number): void;
     /** Marks a started request as failed, updating the failed counters and durations. */
-    failJob(id: number | string, retryCount: number): void;
-    /** Drops a started request without counting it as finished or failed (e.g. skipped by robots.txt). */
-    discardJob(id: number | string): void;
+    recordRequestFailure(id: number | string, retryCount: number): void;
+    /** Drops a started request without counting it as succeeded or failed (e.g. skipped by robots.txt). */
+    discardRequestRecord(id: number | string): void;
     /** Increments the counter for the given HTTP status code. */
     registerStatusCode(code: number): void;
     /** Computes the derived aggregates (averages, per-minute rates, totals) from the current state. */
@@ -61,15 +61,15 @@ export interface IStatistics<StateExtension extends object = {}> {
 export interface CalculatedStatistics {
     /** Mean duration of a failed request, in milliseconds; `Infinity` when nothing has failed. */
     requestAvgFailedDurationMillis: number;
-    /** Mean duration of a finished request, in milliseconds; `Infinity` when nothing has finished. */
-    requestAvgFinishedDurationMillis: number;
-    /** Requests finished per minute over the run so far. */
-    requestsFinishedPerMinute: number;
+    /** Mean duration of a succeeded request, in milliseconds; `Infinity` when nothing has succeeded. */
+    requestAvgSucceededDurationMillis: number;
+    /** Requests succeeded per minute over the run so far. */
+    requestsSucceededPerMinute: number;
     /** Requests failed per minute over the run so far. */
     requestsFailedPerMinute: number;
-    /** Combined duration of all finished and failed requests, in milliseconds. */
+    /** Combined duration of all succeeded and failed requests, in milliseconds. */
     requestTotalDurationMillis: number;
-    /** Total number of settled requests (finished plus failed). */
+    /** Total number of settled requests (succeeded plus failed). */
     requestsTotal: number;
     /** Wall-clock runtime since capturing started, in milliseconds. */
     crawlerRuntimeMillis: number;
@@ -131,27 +131,19 @@ export declare class Statistics<StateExtension extends object = {}, PersistedSta
      * Increments the status code counter.
      */
     registerStatusCode(code: number): void;
-    /**
-     * Starts a job
-     * @ignore
-     */
-    startJob(id: number | string): void;
-    /**
-     * Mark job as finished and sets the state
-     * @ignore
-     */
-    finishJob(id: number | string, retryCount: number): void;
-    /**
-     * Mark job as failed and sets the state
-     * @ignore
-     */
-    failJob(id: number | string, retryCount: number): void;
-    /**
-     * Discards a started job without affecting the finished/failed counters, e.g. when a request
-     * turns out to be skipped (robots.txt, enqueue strategy) after `startJob` was already called for it.
+    /** @ignore */
+    recordRequestStart(id: number | string): void;
+    /** @ignore */
+    recordRequestSuccess(id: number | string, retryCount: number): void;
+    /** @ignore */
+    recordRequestFailure(id: number | string, retryCount: number): void;
+    /**
+     * Discards a started request without affecting the succeeded/failed counters, e.g. when a request
+     * turns out to be skipped (robots.txt, enqueue strategy) after `recordRequestStart` was already
+     * called for it.
      * @ignore
      */
-    discardJob(id: number | string): void;
+    discardRequestRecord(id: number | string): void;
     /**
      * Calculate the current statistics
      */
@@ -165,7 +157,7 @@ export declare class Statistics<StateExtension extends object = {}, PersistedSta
      * Stops logging and remove event listeners, then persist
      */
     stopCapturing(): Promise<void>;
-    private saveRetryCountForJob;
+    private saveRetryCountForRequest;
     /**
      * Persist internal state to the key value store.
      *
@@ -277,12 +269,12 @@ export interface StatisticStateExtensionOptions<StateExtension extends object, P
  * Format of the persisted stats.
  *
  * The `null`s are `Infinity` on the way out - JSON has no infinity, so a record written before anything
- * finished or failed carries a `null` in its place.
+ * succeeded or failed carries a `null` in its place.
  */
-export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt' | 'crawlerStartedAt' | 'crawlerFinishedAt' | 'requestMinDurationMillis' | 'requestsFailedPerMinute' | 'requestsFinishedPerMinute' | 'requestRetryHistogram' | 'instanceStart'> {
+export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt' | 'crawlerStartedAt' | 'crawlerFinishedAt' | 'requestMinDurationMillis' | 'requestsFailedPerMinute' | 'requestsSucceededPerMinute' | 'requestRetryHistogram' | 'instanceStart'> {
     statsId: string;
     requestsFailedPerMinute: number | null;
-    requestsFinishedPerMinute: number | null;
+    requestsSucceededPerMinute: number | null;
     /** ISO strings - the live state keeps these as `Date`s. */
     crawlerStartedAt: string | null;
     crawlerFinishedAt: string | null;
@@ -291,7 +283,7 @@ export interface StatisticPersistedState extends Omit<StatisticState, 'statsPers
     /** A retry count that no request ever reached leaves a `null` here. */
     requestRetryHistogram: (number | null)[];
     requestAvgFailedDurationMillis: number | null;
-    requestAvgFinishedDurationMillis: number | null;
+    requestAvgSucceededDurationMillis: number | null;
     requestTotalDurationMillis: number;
     requestsTotal: number;
     /** {@link StatisticState.instanceStart} of the run that wrote the record. */
@@ -301,15 +293,15 @@ export interface StatisticPersistedState extends Omit<StatisticState, 'statsPers
  * Contains the statistics state
  */
 export interface StatisticState {
-    requestsFinished: number;
+    requestsSucceeded: number;
     requestsFailed: number;
     requestsRetries: number;
     requestsFailedPerMinute: number;
-    requestsFinishedPerMinute: number;
+    requestsSucceededPerMinute: number;
     requestMinDurationMillis: number;
     requestMaxDurationMillis: number;
     requestTotalFailedDurationMillis: number;
-    requestTotalFinishedDurationMillis: number;
+    requestTotalSucceededDurationMillis: number;
     crawlerStartedAt: Date | string | null;
     crawlerFinishedAt: Date | string | null;
     crawlerRuntimeMillis: number;
@@ -317,7 +309,7 @@ export interface StatisticState {
     errors: Record<string, unknown>;
     retryErrors: Record<string, unknown>;
     requestsWithStatusCode: Record<string, number>;
-    /** Retries histogram - index `i` holds the number of requests that finished after `i` retries. */
+    /** Retries histogram - index `i` holds the number of requests that settled after `i` retries. */
     requestRetryHistogram: number[];
     /**
      * When the current capture window started, as a `Date.now()` timestamp. Rebased on load so that the runtime

package/crawlers/statistics.js

@@ -7,15 +7,13 @@ import { ErrorTracker } from './error_tracker.js';
 /**
  * @ignore
  */
-class Job {
+class RequestProcessingRecord {
     #lastRunAt = null;
-    #durationMillis;
     run() {
         this.#lastRunAt = Date.now();
     }
     finish() {
-        this.#durationMillis = Date.now() - this.#lastRunAt;
-        return this.#durationMillis;
+        return Date.now() - this.#lastRunAt;
     }
 }
 const statisticsOptionsSchema = z.strictObject({
@@ -53,15 +51,15 @@ const errorTrackerConfig = {
  * validated by their own conversion on the way back, so the keys it does not know about are simply dropped here.
  */
 const persistedStatisticState = z.object({
-    requestsFinished: z.number(),
+    requestsSucceeded: z.number(),
     requestsFailed: z.number(),
     requestsRetries: z.number(),
     requestsFailedPerMinute: z.number().nullable(),
-    requestsFinishedPerMinute: z.number().nullable(),
+    requestsSucceededPerMinute: z.number().nullable(),
     requestMinDurationMillis: z.number().nullable(),
     requestMaxDurationMillis: z.number(),
     requestTotalFailedDurationMillis: z.number(),
-    requestTotalFinishedDurationMillis: z.number(),
+    requestTotalSucceededDurationMillis: z.number(),
     crawlerStartedAt: z.string().nullable(),
     crawlerFinishedAt: z.string().nullable(),
     statsPersistedAt: z.string(),
@@ -72,7 +70,7 @@ const persistedStatisticState = z.object({
     requestRetryHistogram: z.array(z.number().nullable()),
     statsId: z.string(),
     requestAvgFailedDurationMillis: z.number().nullable(),
-    requestAvgFinishedDurationMillis: z.number().nullable(),
+    requestAvgSucceededDurationMillis: z.number().nullable(),
     requestTotalDurationMillis: z.number(),
     requestsTotal: z.number(),
     requestsWithStatusCode: z.record(z.string(), z.number()),
@@ -97,11 +95,11 @@ function buildStatisticStateCodec(statistics) {
     return z.codec(persistedStatisticState, z.custom(), {
         decode: (record) => ({
             ...statistics.defaultState(),
-            requestsFinished: record.requestsFinished,
+            requestsSucceeded: record.requestsSucceeded,
             requestsFailed: record.requestsFailed,
             requestsRetries: record.requestsRetries,
             requestTotalFailedDurationMillis: record.requestTotalFailedDurationMillis,
-            requestTotalFinishedDurationMillis: record.requestTotalFinishedDurationMillis,
+            requestTotalSucceededDurationMillis: record.requestTotalSucceededDurationMillis,
             // Restoring the `null` as-is would make every later `duration < min` comparison fail, leaving the
             // minimum `null` for the rest of the run.
             requestMinDurationMillis: record.requestMinDurationMillis ?? Infinity,
@@ -117,10 +115,12 @@ function buildStatisticStateCodec(statistics) {
             instanceStart: Date.now() - (new Date(record.statsPersistedAt).getTime() - record.crawlerLastStartTimestamp),
         }),
         encode: (state) => {
+            // The per-minute rates fall through into `counters` and are overwritten below by `calculate()`'s,
+            // which are the ones the record carries.
             const { requestsWithStatusCode, errors, retryErrors, requestRetryHistogram, instanceStart, ...counters } = state;
             // Every rate and average `calculate()` derives is `Infinity` until the run is long enough, or until
-            // something has finished or failed, to divide by.
-            const { requestAvgFailedDurationMillis, requestAvgFinishedDurationMillis, requestsFailedPerMinute, requestsFinishedPerMinute, ...aggregates } = statistics.calculate();
+            // something has succeeded or failed, to divide by.
+            const { requestAvgFailedDurationMillis, requestAvgSucceededDurationMillis, requestsFailedPerMinute, requestsSucceededPerMinute, ...aggregates } = statistics.calculate();
             return {
                 ...counters,
                 requestMinDurationMillis: finiteOrNull(state.requestMinDurationMillis),
@@ -134,9 +134,9 @@ function buildStatisticStateCodec(statistics) {
                 statsId: statistics.statsId,
                 ...aggregates,
                 requestAvgFailedDurationMillis: finiteOrNull(requestAvgFailedDurationMillis),
-                requestAvgFinishedDurationMillis: finiteOrNull(requestAvgFinishedDurationMillis),
+                requestAvgSucceededDurationMillis: finiteOrNull(requestAvgSucceededDurationMillis),
                 requestsFailedPerMinute: finiteOrNull(requestsFailedPerMinute),
-                requestsFinishedPerMinute: finiteOrNull(requestsFinishedPerMinute),
+                requestsSucceededPerMinute: finiteOrNull(requestsSucceededPerMinute),
                 requestsWithStatusCode,
                 errors,
                 retryErrors,
@@ -284,15 +284,15 @@ export class Statistics {
     /** The built-in half of {@link Statistics.defaultState}, before any custom fields are merged over it. */
     #builtInDefaultState() {
         return {
-            requestsFinished: 0,
+            requestsSucceeded: 0,
             requestsFailed: 0,
             requestsRetries: 0,
             requestsFailedPerMinute: 0,
-            requestsFinishedPerMinute: 0,
+            requestsSucceededPerMinute: 0,
             requestMinDurationMillis: Infinity,
             requestMaxDurationMillis: 0,
             requestTotalFailedDurationMillis: 0,
-            requestTotalFinishedDurationMillis: 0,
+            requestTotalSucceededDurationMillis: 0,
             crawlerStartedAt: null,
             crawlerFinishedAt: null,
             statsPersistedAt: null,
@@ -323,70 +323,62 @@ export class Statistics {
         }
         this.state.requestsWithStatusCode[s]++;
     }
-    /**
-     * Starts a job
-     * @ignore
-     */
-    startJob(id) {
-        let job = this.#requestsInProgress.get(id);
-        if (!job)
-            job = new Job();
-        job.run();
-        this.#requestsInProgress.set(id, job);
+    /** @ignore */
+    recordRequestStart(id) {
+        let record = this.#requestsInProgress.get(id);
+        if (!record)
+            record = new RequestProcessingRecord();
+        record.run();
+        this.#requestsInProgress.set(id, record);
     }
-    /**
-     * Mark job as finished and sets the state
-     * @ignore
-     */
-    finishJob(id, retryCount) {
-        const job = this.#requestsInProgress.get(id);
-        if (!job)
+    /** @ignore */
+    recordRequestSuccess(id, retryCount) {
+        const record = this.#requestsInProgress.get(id);
+        if (!record)
             return;
-        const jobDurationMillis = job.finish();
-        this.state.requestsFinished++;
-        this.state.requestTotalFinishedDurationMillis += jobDurationMillis;
-        this.saveRetryCountForJob(retryCount);
-        if (jobDurationMillis < this.state.requestMinDurationMillis)
-            this.state.requestMinDurationMillis = jobDurationMillis;
-        if (jobDurationMillis > this.state.requestMaxDurationMillis)
-            this.state.requestMaxDurationMillis = jobDurationMillis;
+        const durationMillis = record.finish();
+        this.state.requestsSucceeded++;
+        this.state.requestTotalSucceededDurationMillis += durationMillis;
+        this.saveRetryCountForRequest(retryCount);
+        if (durationMillis < this.state.requestMinDurationMillis)
+            this.state.requestMinDurationMillis = durationMillis;
+        if (durationMillis > this.state.requestMaxDurationMillis)
+            this.state.requestMaxDurationMillis = durationMillis;
         this.#requestsInProgress.delete(id);
     }
-    /**
-     * Mark job as failed and sets the state
-     * @ignore
-     */
-    failJob(id, retryCount) {
-        const job = this.#requestsInProgress.get(id);
-        if (!job)
+    /** @ignore */
+    recordRequestFailure(id, retryCount) {
+        const record = this.#requestsInProgress.get(id);
+        if (!record)
             return;
-        this.state.requestTotalFailedDurationMillis += job.finish();
+        this.state.requestTotalFailedDurationMillis += record.finish();
         this.state.requestsFailed++;
-        this.saveRetryCountForJob(retryCount);
+        this.saveRetryCountForRequest(retryCount);
         this.#requestsInProgress.delete(id);
     }
     /**
-     * Discards a started job without affecting the finished/failed counters, e.g. when a request
-     * turns out to be skipped (robots.txt, enqueue strategy) after `startJob` was already called for it.
+     * Discards a started request without affecting the succeeded/failed counters, e.g. when a request
+     * turns out to be skipped (robots.txt, enqueue strategy) after `recordRequestStart` was already
+     * called for it.
      * @ignore
      */
-    discardJob(id) {
+    discardRequestRecord(id) {
         this.#requestsInProgress.delete(id);
     }
     /**
      * Calculate the current statistics
      */
     calculate() {
-        const { requestsFailed, requestsFinished, requestTotalFailedDurationMillis, requestTotalFinishedDurationMillis, } = this.state;
+        const { requestsFailed, requestsSucceeded, requestTotalFailedDurationMillis, requestTotalSucceededDurationMillis, } = this.state;
         const totalMillis = Date.now() - this.state.instanceStart;
         const totalMinutes = totalMillis / 1000 / 60;
         return {
             requestAvgFailedDurationMillis: Math.round(requestTotalFailedDurationMillis / requestsFailed) || Infinity,
-            requestAvgFinishedDurationMillis: Math.round(requestTotalFinishedDurationMillis / requestsFinished) || Infinity,
-            requestsFinishedPerMinute: Math.round(requestsFinished / totalMinutes) || 0,
+            requestAvgSucceededDurationMillis: Math.round(requestTotalSucceededDurationMillis / requestsSucceeded) || Infinity,
+            requestsSucceededPerMinute: Math.round(requestsSucceeded / totalMinutes) || 0,
             requestsFailedPerMinute: Math.floor(requestsFailed / totalMinutes) || 0,
-            requestTotalDurationMillis: requestTotalFinishedDurationMillis + requestTotalFailedDurationMillis,
-            requestsTotal: requestsFailed + requestsFinished,
+            requestTotalDurationMillis: requestTotalSucceededDurationMillis + requestTotalFailedDurationMillis,
+            requestsTotal: requestsFailed + requestsSucceeded,
             crawlerRuntimeMillis: totalMillis,
         };
     }
@@ -420,7 +412,7 @@ export class Statistics {
         this.state.crawlerFinishedAt = new Date();
         await this.#recoverableState.teardown();
     }
-    saveRetryCountForJob(retryCount) {
+    saveRetryCountForRequest(retryCount) {
         if (retryCount > 0)
             this.state.requestsRetries++;
         this.requestRetryHistogram[retryCount] ??= 0;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.169",
+    "version": "4.0.0-beta.170",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,10 +52,10 @@
         "@apify/log": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/fs-storage": "4.0.0-beta.169",
-        "@crawlee/http-client": "4.0.0-beta.169",
-        "@crawlee/types": "4.0.0-beta.169",
-        "@crawlee/utils": "4.0.0-beta.169",
+        "@crawlee/fs-storage": "4.0.0-beta.170",
+        "@crawlee/http-client": "4.0.0-beta.170",
+        "@crawlee/types": "4.0.0-beta.170",
+        "@crawlee/utils": "4.0.0-beta.170",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -78,5 +78,5 @@
             }
         }
     },
-    "gitHead": "a42dc93fa0ef68180d83ab12af362082690f73e2"
+    "gitHead": "86e1c891da983125e9e18f1a2327fbcf5e0f5fc2"
 }