@crawlee/core 4.0.0-beta.12 → 4.0.0-beta.121

package/crawlers/statistics.d.ts

@@ -1,6 +1,5 @@
-import type { Log } from '@apify/log';
-import { Configuration } from '../configuration.js';
-import { KeyValueStore } from '../storages/key_value_store.js';
+import type { CrawleeLogger } from '../log.js';
+import type { KeyValueStore } from '../storages/key_value_store.js';
 import { ErrorTracker } from './error_tracker.js';
 /**
  * Persistence-related options to control how and when crawler's data gets persisted.
@@ -12,65 +11,123 @@ export interface PersistenceOptions {
      */
     enable?: boolean;
 }
+/**
+ * The statistics surface a crawler depends on: recording per-request outcomes, tracking errors, and driving the
+ * capture lifecycle for a run. Injected via the crawler's `statistics` option, so a custom implementation (or a
+ * {@link Statistics} subclass tracking extra fields) can be plugged in without subclassing the crawler.
+ *
+ * The owned-only mutators the crawler uses to *own* a default it built - `reset()`/`resetStore()` - are deliberately
+ * absent: an injected instance is borrowed, and the crawler never wipes it. Those live on the concrete
+ * {@link Statistics} only.
+ *
+ * @category Crawlers
+ */
+export interface IStatistics {
+    /** Tracker for errors on the final retry of a request. */
+    readonly errorTracker: ErrorTracker;
+    /** Tracker for errors on retries prior to the final one. */
+    readonly errorTrackerRetry: ErrorTracker;
+    /** The live statistics state the crawler reads for status messages and the final summary. */
+    readonly state: StatisticState;
+    /** Retries histogram - index `i` holds the number of requests that finished after `i` retries. */
+    readonly requestRetryHistogram: number[];
+    /** Marks a request as started, so its duration can be measured on finish/fail. */
+    startJob(id: number | string): void;
+    /** Marks a started request as finished, updating the finished counters and durations. */
+    finishJob(id: number | string, retryCount: number): void;
+    /** Marks a started request as failed, updating the failed counters and durations. */
+    failJob(id: number | string, retryCount: number): void;
+    /** Drops a started request without counting it as finished or failed (e.g. skipped by robots.txt). */
+    discardJob(id: number | string): void;
+    /** Increments the counter for the given HTTP status code. */
+    registerStatusCode(code: number): void;
+    /** Computes the derived aggregates (averages, per-minute rates, totals) from the current state. */
+    calculate(): CalculatedStatistics;
+    /** Begins a capture window: loads any persisted state, subscribes to persistence events, starts periodic logging. */
+    startCapturing(): Promise<void>;
+    /** Ends the capture window: stops logging, unsubscribes, and persists the final state. */
+    stopCapturing(): Promise<void>;
+    /**
+     * Persists the current state to the key-value store. Optional - the crawler calls it on migration, but a backend
+     * with no persistence of its own can omit it.
+     */
+    persistState?(): Promise<void>;
+}
+/** The derived aggregates computed by {@link IStatistics.calculate} from the current {@link StatisticState}. */
+export interface CalculatedStatistics {
+    /** Mean duration of a failed request, in milliseconds; `Infinity` when nothing has failed. */
+    requestAvgFailedDurationMillis: number;
+    /** Mean duration of a finished request, in milliseconds; `Infinity` when nothing has finished. */
+    requestAvgFinishedDurationMillis: number;
+    /** Requests finished per minute over the run so far. */
+    requestsFinishedPerMinute: number;
+    /** Requests failed per minute over the run so far. */
+    requestsFailedPerMinute: number;
+    /** Combined duration of all finished and failed requests, in milliseconds. */
+    requestTotalDurationMillis: number;
+    /** Total number of settled requests (finished plus failed). */
+    requestsTotal: number;
+    /** Wall-clock runtime since capturing started, in milliseconds. */
+    crawlerRuntimeMillis: number;
+}
 /**
  * The statistics class provides an interface to collecting and logging run
  * statistics for requests.
  *
  * All statistic information is saved on key value store
- * under the key `SDK_CRAWLER_STATISTICS_*`, persists between
+ * under the key `CRAWLEE_CRAWLER_STATISTICS_*`, persists between
  * migrations and abort/resurrect
  *
  * @category Crawlers
  */
-export declare class Statistics {
+export declare class Statistics implements IStatistics {
+    #private;
     private static id;
     /**
      * An error tracker for final retry errors.
      */
-    errorTracker: ErrorTracker;
+    readonly errorTracker: ErrorTracker;
     /**
      * An error tracker for retry errors prior to the final retry.
      */
-    errorTrackerRetry: ErrorTracker;
+    readonly errorTrackerRetry: ErrorTracker;
     /**
      * Statistic instance id.
      */
-    readonly id: number;
+    readonly id: string;
+    protected readonly persistStateKey: string;
+    private readonly log;
     /**
      * Current statistic state used for doing calculations on {@link Statistics.calculate} calls
      */
-    state: StatisticState;
+    get state(): StatisticState;
     /**
      * Contains the current retries histogram. Index 0 means 0 retries, index 2, 2 retries, and so on
      */
-    readonly requestRetryHistogram: number[];
-    /**
-     * Contains the associated Configuration instance
-     */
-    private readonly config;
-    protected keyValueStore?: KeyValueStore;
-    protected persistStateKey: string;
-    private logIntervalMillis;
-    private logMessage;
-    private listener;
-    private requestsInProgress;
-    private readonly log;
-    private instanceStart;
-    private logInterval;
-    private events;
-    private persistenceOptions;
+    get requestRetryHistogram(): number[];
     /**
-     * @internal
+     * Construct a statistics instance to pass to a crawler via its `statistics` option, e.g. to preconfigure
+     * persistence or error snapshots, share it across sequential runs, or subclass it to track extra fields.
      */
     constructor(options?: StatisticsOptions);
     /**
-     * Set the current statistic instance to pristine values
+     * Set the current statistic instance to pristine values.
+     *
+     * The persisted record is left alone - use {@link Statistics.resetStore} to clear that as well.
      */
     reset(): void;
     /**
-     * @param options - Override the persistence options provided in the constructor
+     * The pristine state a new instance starts with and {@link Statistics.reset} restores.
+     *
+     * A subclass tracking extra fields declares their initial values here.
+     */
+    protected defaultState(): StatisticState;
+    /**
+     * Clear the persisted statistics record, leaving the in-memory state alone.
+     *
+     * Throws while capturing - the next PERSIST_STATE event would write the record straight back.
      */
-    resetStore(options?: PersistenceOptions): Promise<void>;
+    resetStore(): Promise<void>;
     /**
      * Increments the status code counter.
      */
@@ -90,18 +147,16 @@ export declare class Statistics {
      * @ignore
      */
     failJob(id: number | string, retryCount: number): void;
+    /**
+     * Discards a started job without affecting the finished/failed counters, e.g. when a request
+     * turns out to be skipped (robots.txt, enqueue strategy) after `startJob` was already called for it.
+     * @ignore
+     */
+    discardJob(id: number | string): void;
     /**
      * Calculate the current statistics
      */
-    calculate(): {
-        requestAvgFailedDurationMillis: number;
-        requestAvgFinishedDurationMillis: number;
-        requestsFinishedPerMinute: number;
-        requestsFailedPerMinute: number;
-        requestTotalDurationMillis: number;
-        requestsTotal: number;
-        crawlerRuntimeMillis: number;
-    };
+    calculate(): CalculatedStatistics;
     /**
      * Initializes the key value store for persisting the statistics,
      * displaying the current state in predefined intervals
@@ -111,17 +166,25 @@ export declare class Statistics {
      * Stops logging and remove event listeners, then persist
      */
     stopCapturing(): Promise<void>;
-    protected _saveRetryCountForJob(retryCount: number): void;
+    private saveRetryCountForJob;
     /**
-     * Persist internal state to the key value store
-     * @param options - Override the persistence options provided in the constructor
+     * Persist internal state to the key value store.
+     *
+     * Statistics are bookkeeping - a store that refuses the write is worth a warning, not a failed crawl. The
+     * crawler calls this from its migration handler, where a rejection would go unhandled.
      */
-    persistState(options?: PersistenceOptions): Promise<void>;
+    persistState(): Promise<void>;
     /**
-     * Loads the current statistic from the key value store if any
+     * Rebuilds the state from a persisted record.
+     *
+     * A subclass tracking extra fields restores them here, on top of the result of `super.deserializeState()`.
      */
-    protected _maybeLoadStatistics(): Promise<void>;
-    protected _teardown(): void;
+    protected deserializeState(persistedState: StatisticPersistedState): StatisticState;
+    /**
+     * Builds the record written to the key value store, merging in the derived aggregates so that a consumer
+     * reading the record does not have to reconstruct them.
+     */
+    protected serializeState(state: StatisticState): StatisticPersistedState;
     /**
      * Make this class serializable when called with `JSON.stringify(statsInstance)` directly
      * or through `keyValueStore.setValue('KEY', statsInstance)`
@@ -146,17 +209,12 @@ export interface StatisticsOptions {
      * Parent logger instance, the statistics will create a child logger from this.
      * @default crawler.log
      */
-    log?: Log;
+    log?: CrawleeLogger;
     /**
      * Key value store instance to persist the statistics.
      * If not provided, the default one will be used when capturing starts
      */
     keyValueStore?: KeyValueStore;
-    /**
-     * Configuration instance to use
-     * @default Configuration.getGlobalConfig()
-     */
-    config?: Configuration;
     /**
      * Control how and when to persist the statistics.
      */
@@ -166,19 +224,39 @@ export interface StatisticsOptions {
      * @default false
      */
     saveErrorSnapshots?: boolean;
+    /**
+     * A unique identifier for this statistics instance. This ID is used for persistence
+     * to the key value store, ensuring the same statistics can be loaded after script restarts.
+     *
+     * If not provided, an auto-incremented ID will be used for backward compatibility.
+     * This means statistics may not persist correctly across script restarts
+     * if crawler creation order changes.
+     */
+    id?: string;
 }
 /**
- * Format of the persisted stats
+ * Format of the persisted stats.
+ *
+ * The `null`s are `Infinity` on the way out - JSON has no infinity, so a record written before anything
+ * finished or failed carries a `null` in its place.
  */
-export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt'> {
-    requestRetryHistogram: number[];
-    statsId: number;
-    requestAvgFailedDurationMillis: number;
-    requestAvgFinishedDurationMillis: number;
+export interface StatisticPersistedState extends Omit<StatisticState, 'statsPersistedAt' | 'crawlerStartedAt' | 'crawlerFinishedAt' | 'requestMinDurationMillis' | 'requestsFailedPerMinute' | 'requestsFinishedPerMinute' | 'requestRetryHistogram' | 'instanceStart'> {
+    statsId: string;
+    requestsFailedPerMinute: number | null;
+    requestsFinishedPerMinute: number | null;
+    /** ISO strings - the live state keeps these as `Date`s. */
+    crawlerStartedAt: string | null;
+    crawlerFinishedAt: string | null;
+    statsPersistedAt: string;
+    requestMinDurationMillis: number | null;
+    /** A retry count that no request ever reached leaves a `null` here. */
+    requestRetryHistogram: (number | null)[];
+    requestAvgFailedDurationMillis: number | null;
+    requestAvgFinishedDurationMillis: number | null;
     requestTotalDurationMillis: number;
     requestsTotal: number;
+    /** {@link StatisticState.instanceStart} of the run that wrote the record. */
     crawlerLastStartTimestamp: number;
-    statsPersistedAt: string;
 }
 /**
  * Contains the statistics state
@@ -200,5 +278,11 @@ export interface StatisticState {
     errors: Record<string, unknown>;
     retryErrors: Record<string, unknown>;
     requestsWithStatusCode: Record<string, number>;
+    /** Retries histogram - index `i` holds the number of requests that finished after `i` retries. */
+    requestRetryHistogram: number[];
+    /**
+     * When the current capture window started, as a `Date.now()` timestamp. Rebased on load so that the runtime
+     * reported by {@link Statistics.calculate} spans a migration rather than restarting from zero.
+     */
+    instanceStart: number;
 }
-//# sourceMappingURL=statistics.d.ts.map