@crawlee/core 4.0.0-beta.101 → 4.0.0-beta.103

package/crawlers/error_snapshotter.d.ts

@@ -25,9 +25,7 @@ interface ErrorSnapshot {
  * ```ts
  * const crawler = new BasicCrawler({
  *   // ...
- *   statisticsOptions: {
- *     saveErrorSnapshots: true,
- *   },
+ *   statistics: new Statistics({ saveErrorSnapshots: true }),
  * });
  * ```
  */

package/crawlers/error_snapshotter.js

@@ -7,9 +7,7 @@ import crypto from 'node:crypto';
  * ```ts
  * const crawler = new BasicCrawler({
  *   // ...
- *   statisticsOptions: {
- *     saveErrorSnapshots: true,
- *   },
+ *   statistics: new Statistics({ saveErrorSnapshots: true }),
  * });
  * ```
  */

package/crawlers/statistics.d.ts

@@ -11,6 +11,65 @@ export interface PersistenceOptions {
      */
     enable?: boolean;
 }
+/**
+ * The statistics surface a crawler depends on: recording per-request outcomes, tracking errors, and driving the
+ * capture lifecycle for a run. Injected via the crawler's `statistics` option, so a custom implementation (or a
+ * {@link Statistics} subclass tracking extra fields) can be plugged in without subclassing the crawler.
+ *
+ * The owned-only mutators the crawler uses to *own* a default it built - `reset()`/`resetStore()` - are deliberately
+ * absent: an injected instance is borrowed, and the crawler never wipes it. Those live on the concrete
+ * {@link Statistics} only.
+ *
+ * @category Crawlers
+ */
+export interface IStatistics {
+    /** Tracker for errors on the final retry of a request. */
+    readonly errorTracker: ErrorTracker;
+    /** Tracker for errors on retries prior to the final one. */
+    readonly errorTrackerRetry: ErrorTracker;
+    /** The live statistics state the crawler reads for status messages and the final summary. */
+    readonly state: StatisticState;
+    /** Retries histogram - index `i` holds the number of requests that finished after `i` retries. */
+    readonly requestRetryHistogram: number[];
+    /** Marks a request as started, so its duration can be measured on finish/fail. */
+    startJob(id: number | string): void;
+    /** Marks a started request as finished, updating the finished counters and durations. */
+    finishJob(id: number | string, retryCount: number): void;
+    /** Marks a started request as failed, updating the failed counters and durations. */
+    failJob(id: number | string, retryCount: number): void;
+    /** Drops a started request without counting it as finished or failed (e.g. skipped by robots.txt). */
+    discardJob(id: number | string): void;
+    /** Increments the counter for the given HTTP status code. */
+    registerStatusCode(code: number): void;
+    /** Computes the derived aggregates (averages, per-minute rates, totals) from the current state. */
+    calculate(): CalculatedStatistics;
+    /** Begins a capture window: loads any persisted state, subscribes to persistence events, starts periodic logging. */
+    startCapturing(): Promise<void>;
+    /** Ends the capture window: stops logging, unsubscribes, and persists the final state. */
+    stopCapturing(): Promise<void>;
+    /**
+     * Persists the current state to the key-value store. Optional - the crawler calls it on migration, but a backend
+     * with no persistence of its own can omit it.
+     */
+    persistState?(options?: PersistenceOptions): Promise<void>;
+}
+/** The derived aggregates computed by {@link IStatistics.calculate} from the current {@link StatisticState}. */
+export interface CalculatedStatistics {
+    /** Mean duration of a failed request, in milliseconds; `Infinity` when nothing has failed. */
+    requestAvgFailedDurationMillis: number;
+    /** Mean duration of a finished request, in milliseconds; `Infinity` when nothing has finished. */
+    requestAvgFinishedDurationMillis: number;
+    /** Requests finished per minute over the run so far. */
+    requestsFinishedPerMinute: number;
+    /** Requests failed per minute over the run so far. */
+    requestsFailedPerMinute: number;
+    /** Combined duration of all finished and failed requests, in milliseconds. */
+    requestTotalDurationMillis: number;
+    /** Total number of settled requests (finished plus failed). */
+    requestsTotal: number;
+    /** Wall-clock runtime since capturing started, in milliseconds. */
+    crawlerRuntimeMillis: number;
+}
 /**
  * The statistics class provides an interface to collecting and logging run
  * statistics for requests.
@@ -21,7 +80,7 @@ export interface PersistenceOptions {
  *
  * @category Crawlers
  */
-export declare class Statistics {
+export declare class Statistics implements IStatistics {
     private static id;
     /**
      * An error tracker for final retry errors.
@@ -56,7 +115,8 @@ export declare class Statistics {
     private persistenceOptions;
     private get events();
     /**
-     * @internal
+     * Construct a statistics instance to pass to a crawler via its `statistics` option, e.g. to preconfigure
+     * persistence or error snapshots, share it across sequential runs, or subclass it to track extra fields.
      */
     constructor(options?: StatisticsOptions);
     /**
@@ -95,15 +155,7 @@ export declare class Statistics {
     /**
      * Calculate the current statistics
      */
-    calculate(): {
-        requestAvgFailedDurationMillis: number;
-        requestAvgFinishedDurationMillis: number;
-        requestsFinishedPerMinute: number;
-        requestsFailedPerMinute: number;
-        requestTotalDurationMillis: number;
-        requestsTotal: number;
-        crawlerRuntimeMillis: number;
-    };
+    calculate(): CalculatedStatistics;
     /**
      * Initializes the key value store for persisting the statistics,
      * displaying the current state in predefined intervals

package/crawlers/statistics.js

@@ -74,7 +74,8 @@ export class Statistics {
         return this._events;
     }
     /**
-     * @internal
+     * Construct a statistics instance to pass to a crawler via its `statistics` option, e.g. to preconfigure
+     * persistence or error snapshots, share it across sequential runs, or subclass it to track extra fields.
      */
     constructor(options = {}) {
         ow(options, ow.object.exactShape({
@@ -225,6 +226,11 @@ export class Statistics {
      * displaying the current state in predefined intervals
      */
     async startCapturing() {
+        // A single instance drives one logging interval and one PERSIST_STATE listener, so a second concurrent
+        // capture (e.g. sharing one instance across crawlers running at once) would orphan the first. Fail loudly.
+        if (this.logInterval) {
+            throw new Error('Statistics.startCapturing() was already called - this instance is already capturing.');
+        }
         this.keyValueStore ??= await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
         if (this.state.crawlerStartedAt === null) {
             this.state.crawlerStartedAt = new Date();

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/core",
-    "version": "4.0.0-beta.101",
+    "version": "4.0.0-beta.103",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -52,9 +52,9 @@
         "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/fs-storage": "4.0.0-beta.101",
-        "@crawlee/types": "4.0.0-beta.101",
-        "@crawlee/utils": "4.0.0-beta.101",
+        "@crawlee/fs-storage": "4.0.0-beta.103",
+        "@crawlee/types": "4.0.0-beta.103",
+        "@crawlee/utils": "4.0.0-beta.103",
         "@sapphire/async-queue": "^1.5.5",
         "@sapphire/shapeshift": "^4.0.0",
         "@vladfrangu/async_event_emitter": "^2.4.6",
@@ -78,5 +78,5 @@
             }
         }
     },
-    "gitHead": "1fd886ee601c053d6fa8cada24b751f9cbe0a539"
+    "gitHead": "fe5d0ae11067683e27a2d17b4edd226ccf112cf5"
 }