@crawlee/core 4.0.0-beta.104 → 4.0.0-beta.106

package/autoscaling/memory_load_signal.js

@@ -15,92 +15,91 @@ const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10_000;
 export class MemoryLoadSignal {
     name = 'memInfo';
     overloadedRatio;
-    store = new SnapshotStore();
-    maxUsedRatio;
+    #store = new SnapshotStore();
+    #maxUsedRatio;
     /** All resolved in `start()`, before anything that reads them can fire. */
-    config;
-    log;
-    maxMemoryBytes;
-    events;
-    maxMemoryRatio;
-    lastLoggedCriticalMemoryOverloadAt = null;
+    #config;
+    #log;
+    #maxMemoryBytes;
+    #events;
+    #maxMemoryRatio;
+    #lastLoggedCriticalMemoryOverloadAt = null;
     constructor(options = {}) {
-        this.maxUsedRatio = options.maxUsedRatio ?? 0.9;
+        this.#maxUsedRatio = options.maxUsedRatio ?? 0.9;
         this.overloadedRatio = options.overloadedRatio ?? 0.2;
         this.handle = this.handle.bind(this);
     }
     async start(context) {
-        this.store.useSampleWindow(context.maxSampleWindowMillis);
+        this.#store.useSampleWindow(context.maxSampleWindowMillis);
         // A new session starts from a clean slate, so it is not judged on measurements from before the downtime.
-        this.store.clear();
+        this.#store.clear();
         // Resolved here rather than in the constructor: an instance built ahead of time (to be wrapped, or shared
         // between systems) must not capture whichever services happened to be registered at that moment.
-        this.config = serviceLocator.getConfiguration();
-        this.events = serviceLocator.getEventManager();
-        this.log = serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
-        const memoryMbytes = this.config.memoryMbytes ?? 0;
+        this.#config = serviceLocator.getConfiguration();
+        this.#events = serviceLocator.getEventManager();
+        this.#log = serviceLocator.getLogger().child({ prefix: 'MemoryLoadSignal' });
+        const memoryMbytes = this.#config.memoryMbytes ?? 0;
         if (memoryMbytes > 0) {
-            this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
+            this.#maxMemoryBytes = memoryMbytes * 1024 * 1024;
         }
         else {
-            this.maxMemoryRatio = this.config.availableMemoryRatio;
-            if (!this.maxMemoryRatio) {
+            this.#maxMemoryRatio = this.#config.availableMemoryRatio;
+            if (!this.#maxMemoryRatio) {
                 throw new Error('availableMemoryRatio is not set in configuration.');
             }
             else {
-                this.log.debug(`Setting max memory of this run to ${this.maxMemoryRatio * 100} % of available memory. ` +
+                this.#log.debug(`Setting max memory of this run to ${this.#maxMemoryRatio * 100} % of available memory. ` +
                     'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
             }
             // Fallback memory measurement in case memTotalBytes is missing from SystemInfo.
-            this.maxMemoryBytes = await this._getTotalMemoryBytes();
+            this.#maxMemoryBytes = await this.getTotalMemoryBytes();
         }
-        this.events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
+        this.#events.on("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
     }
     async stop() {
-        this.events?.off("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
-        this.events = undefined;
+        this.#events?.off("systemInfo" /* EventType.SYSTEM_INFO */, this.handle);
+        this.#events = undefined;
     }
     getSample(sampleDurationMillis) {
-        return this.store.getSample(sampleDurationMillis);
+        return this.#store.getSample(sampleDurationMillis);
     }
     /** @internal Records a snapshot from a `SYSTEM_INFO` payload. Exposed for tests. */
     handle(systemInfo) {
         const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
         const { memCurrentBytes, memTotalBytes } = systemInfo;
-        let maxMemoryBytes = this.maxMemoryBytes;
-        if (this.maxMemoryRatio !== undefined && this.maxMemoryRatio > 0) {
-            maxMemoryBytes = this.maxMemoryRatio * (memTotalBytes ?? this.maxMemoryBytes);
+        let maxMemoryBytes = this.#maxMemoryBytes;
+        if (this.#maxMemoryRatio !== undefined && this.#maxMemoryRatio > 0) {
+            maxMemoryBytes = this.#maxMemoryRatio * (memTotalBytes ?? this.#maxMemoryBytes);
         }
         const snapshot = {
             createdAt,
-            isOverloaded: memCurrentBytes / maxMemoryBytes > this.maxUsedRatio,
+            isOverloaded: memCurrentBytes / maxMemoryBytes > this.#maxUsedRatio,
             usedBytes: memCurrentBytes,
         };
-        this.store.push(snapshot, createdAt);
-        this._memoryOverloadWarning(systemInfo, maxMemoryBytes);
+        this.#store.push(snapshot, createdAt);
+        this.memoryOverloadWarning(systemInfo, maxMemoryBytes);
     }
-    /** @internal */
-    _memoryOverloadWarning(systemInfo, maxMemoryBytes) {
-        const effectiveMax = maxMemoryBytes ?? this.maxMemoryBytes;
+    memoryOverloadWarning(systemInfo, maxMemoryBytes) {
+        const effectiveMax = maxMemoryBytes ?? this.#maxMemoryBytes;
         const { memCurrentBytes } = systemInfo;
         const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
-        if (this.lastLoggedCriticalMemoryOverloadAt &&
-            +createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
+        if (this.#lastLoggedCriticalMemoryOverloadAt &&
+            +createdAt < +this.#lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS)
             return;
-        const maxDesiredMemoryBytes = this.maxUsedRatio * effectiveMax;
-        const reserveMemory = effectiveMax * (1 - this.maxUsedRatio) * RESERVE_MEMORY_RATIO;
+        const maxDesiredMemoryBytes = this.#maxUsedRatio * effectiveMax;
+        const reserveMemory = effectiveMax * (1 - this.#maxUsedRatio) * RESERVE_MEMORY_RATIO;
         const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
         const isCriticalOverload = memCurrentBytes > criticalOverloadBytes;
         if (isCriticalOverload) {
             const usedPercentage = Math.round((memCurrentBytes / effectiveMax) * 100);
             const toMb = (bytes) => Math.round(bytes / 1024 ** 2);
-            this.log.warning('Memory is critically overloaded. ' +
+            this.#log.warning('Memory is critically overloaded. ' +
                 `Using ${toMb(memCurrentBytes)} MB of ${toMb(effectiveMax)} MB (${usedPercentage}%). Consider increasing available memory.`);
-            this.lastLoggedCriticalMemoryOverloadAt = createdAt;
+            this.#lastLoggedCriticalMemoryOverloadAt = createdAt;
         }
     }
-    async _getTotalMemoryBytes() {
-        const containerized = this.config.containerized ?? (await isContainerized());
+    async getTotalMemoryBytes() {
+        const containerized = this.#config.containerized ?? (await isContainerized());
         return (await getMemoryInfo({ containerized, logger: serviceLocator.getLogger() })).totalBytes;
     }
 }

package/autoscaling/snapshotter.d.ts

@@ -61,10 +61,7 @@ export type SnapshotterOptions = Omit<LoadSignalsOptions, 'custom'>;
  * @internal
  */
 export declare class Snapshotter {
-    private readonly memorySignal?;
-    private readonly eventLoopSignal?;
-    private readonly cpuSignal?;
-    private readonly clientSignal?;
+    #private;
     /**
      * Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
      * instances. Signals switched off through the options are simply absent — the system status reports them as

package/autoscaling/snapshotter.js

@@ -13,10 +13,10 @@ import { MemoryLoadSignal } from './memory_load_signal.js';
  */
 export class Snapshotter {
     // Absent when switched off through the corresponding option (e.g. `client: false`).
-    memorySignal;
-    eventLoopSignal;
-    cpuSignal;
-    clientSignal;
+    #memorySignal;
+    #eventLoopSignal;
+    #cpuSignal;
+    #clientSignal;
     /**
      * Returns the enabled built-in signals, so `SystemStatus` can iterate them alongside any custom `LoadSignal`
      * instances. Signals switched off through the options are simply absent — the system status reports them as
@@ -24,10 +24,10 @@ export class Snapshotter {
      */
     getLoadSignals() {
         const builtin = [
-            this.memorySignal,
-            this.eventLoopSignal,
-            this.cpuSignal,
-            this.clientSignal,
+            this.#memorySignal,
+            this.#eventLoopSignal,
+            this.#cpuSignal,
+            this.#clientSignal,
         ];
         return builtin.filter((signal) => signal !== undefined);
     }
@@ -39,13 +39,13 @@ export class Snapshotter {
         // Each signal resolves its own ambient dependencies when started, and is told the window it will be sampled
         // over then too - so there is nothing to thread in here beyond the caller's tuning.
         if (memory !== false)
-            this.memorySignal = new MemoryLoadSignal(memory);
+            this.#memorySignal = new MemoryLoadSignal(memory);
         if (eventLoop !== false)
-            this.eventLoopSignal = new EventLoopLoadSignal(eventLoop);
+            this.#eventLoopSignal = new EventLoopLoadSignal(eventLoop);
         if (cpu !== false)
-            this.cpuSignal = new CpuLoadSignal(cpu);
+            this.#cpuSignal = new CpuLoadSignal(cpu);
         if (client !== false)
-            this.clientSignal = new ClientLoadSignal(client);
+            this.#clientSignal = new ClientLoadSignal(client);
     }
     /**
      * Starts capturing snapshots at configured intervals. The `context` carries the sample window the signals will

package/autoscaling/system_status.d.ts

@@ -105,9 +105,7 @@ export interface FinalStatistics {
  * @internal
  */
 export declare class SystemStatus {
-    private readonly currentHistoryMillis;
-    private readonly historyMillis;
-    private readonly signals;
+    #private;
     constructor(options: SystemStatusOptions);
     /**
      * The widest window any signal will be queried with, and therefore exactly how much history the signals are asked

package/autoscaling/system_status.js

@@ -32,14 +32,14 @@ const BUILTIN_SIGNAL_NAMES = new Set(Object.keys(BUILTIN_SIGNAL_OPTION_KEYS));
  * @internal
  */
 export class SystemStatus {
-    currentHistoryMillis;
-    historyMillis;
-    signals;
+    #currentHistoryMillis;
+    #historyMillis;
+    #signals;
     constructor(options) {
         const { currentHistorySecs = DEFAULT_CURRENT_HISTORY_SECS, historySecs = DEFAULT_SNAPSHOT_HISTORY_SECS, snapshotter, loadSignals = [], } = options;
-        this.currentHistoryMillis = currentHistorySecs * 1000;
-        this.historyMillis = historySecs * 1000;
-        this.signals = [...snapshotter.getLoadSignals(), ...loadSignals];
+        this.#currentHistoryMillis = currentHistorySecs * 1000;
+        this.#historyMillis = historySecs * 1000;
+        this.#signals = [...snapshotter.getLoadSignals(), ...loadSignals];
         this.assertUniqueSignalNames();
     }
     /**
@@ -48,7 +48,7 @@ export class SystemStatus {
      * defaults.
      */
     get maxSampleWindowMillis() {
-        return Math.max(this.currentHistoryMillis, this.historyMillis);
+        return Math.max(this.#currentHistoryMillis, this.#historyMillis);
     }
     /**
      * Signal names are the keys of the reported {@link SystemInfo}, so a duplicate would leave a status object that
@@ -57,7 +57,7 @@ export class SystemStatus {
      */
     assertUniqueSignalNames() {
         const seen = new Set();
-        for (const { name } of this.signals) {
+        for (const { name } of this.#signals) {
             if (!seen.has(name)) {
                 seen.add(name);
                 continue;
@@ -85,7 +85,7 @@ export class SystemStatus {
      * and `true` otherwise.
      */
     getCurrentStatus() {
-        return this.isSystemIdle(this.currentHistoryMillis);
+        return this.isSystemIdle(this.#currentHistoryMillis);
     }
     /**
      * Returns an {@link SystemInfo} object with the following structure:
@@ -103,7 +103,7 @@ export class SystemStatus {
      * `historySecs` seconds and `true` otherwise.
      */
     getHistoricalStatus() {
-        return this.isSystemIdle(this.historyMillis);
+        return this.isSystemIdle(this.#historyMillis);
     }
     /**
      * Returns a system status object.
@@ -117,7 +117,7 @@ export class SystemStatus {
             clientInfo: { isOverloaded: false, limitRatio: 0, actualRatio: 0 },
         };
         let loadSignalInfo;
-        for (const signal of this.signals) {
+        for (const signal of this.#signals) {
             const sample = signal.getSample(sampleDurationMillis);
             const info = evaluateLoadSignalSample(sample, signal.overloadedRatio);
             if (info.isOverloaded) {

package/configuration.d.ts

@@ -123,12 +123,12 @@ export interface Configuration extends ResolvedConfigValues {
  * `containerized` | `CRAWLEE_CONTAINERIZED` | -
  */
 export declare class Configuration {
+    #private;
     /**
      * Field definitions for this configuration class.
      * Subclasses override this to register additional fields.
      */
     protected static fields: Record<string, ConfigField>;
-    private resolvedValues;
     /**
      * Creates new `Configuration` instance with provided options.
      * Constructor options take precedence over environment variables, which take precedence

package/configuration.js

@@ -147,7 +147,7 @@ export class Configuration {
      * Subclasses override this to register additional fields.
      */
     static fields = crawleeConfigFields;
-    resolvedValues;
+    #resolvedValues;
     /**
      * Creates new `Configuration` instance with provided options.
      * Constructor options take precedence over environment variables, which take precedence
@@ -156,7 +156,7 @@ export class Configuration {
     constructor(options = {}) {
         const fields = this.constructor.fields;
         const fileOptions = Configuration.loadFileOptions();
-        this.resolvedValues = Configuration.resolveAll(fields, options, fileOptions);
+        this.#resolvedValues = Configuration.resolveAll(fields, options, fileOptions);
         this.registerAccessors();
         // Set the log level
         const logLevel = this.logLevel;
@@ -209,7 +209,7 @@ export class Configuration {
         const descriptors = {};
         for (const key of Object.keys(fields)) {
             descriptors[key] = {
-                get: () => this.resolvedValues[key],
+                get: () => this.#resolvedValues[key],
                 set() {
                     throw new TypeError('Configuration is immutable. Pass options via the constructor instead.');
                 },

package/crawlers/context_pipeline.js

@@ -26,12 +26,12 @@ export class ContextPipeline {
  * properties from the `ContextPipeline` interface, making type checking more reliable.
  */
 class ContextPipelineImpl extends ContextPipeline {
-    middleware;
-    parent;
+    #middleware;
+    #parent;
     constructor(middleware, parent) {
         super();
-        this.middleware = middleware;
-        this.parent = parent;
+        this.#middleware = middleware;
+        this.#parent = parent;
     }
     /**
      * @inheritdoc
@@ -50,8 +50,8 @@ class ContextPipelineImpl extends ContextPipeline {
     *middlewareChain() {
         let step = this;
         while (step !== undefined) {
-            yield step.middleware;
-            step = step.parent;
+            yield step.#middleware;
+            step = step.#parent;
         }
     }
     /**

package/crawlers/crawler_commons.d.ts

@@ -1,12 +1,11 @@
 import type { Dictionary, HttpRequestOptions, ISession, ProxyInfo, SendRequestOptions } from '@crawlee/types';
 import type { ReadonlyDeep, SetRequired } from 'type-fest';
-import type { Configuration } from '../configuration.js';
 import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links.js';
 import type { CrawleeLogger } from '../log.js';
 import type { Request, RequestOptions, Source } from '../request.js';
 import type { StorageIdentifier } from '../storages/storage_instance_manager.js';
 import type { Dataset } from '../storages/dataset.js';
-import { KeyValueStore, type RecordOptions } from '../storages/key_value_store.js';
+import type { KeyValueStore } from '../storages/key_value_store.js';
 import type { RequestQueueOperationOptions } from '../storages/request_queue.js';
 /** @internal */
 export type IsAny<T> = 0 extends 1 & T ? true : false;
@@ -176,6 +175,11 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
     sendRequest: (requestOverrides?: Partial<HttpRequestOptions>, optionsOverrides?: SendRequestOptions) => Promise<Response>;
     /**
      * Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.
+     *
+     * The callback runs *outside* the request's storage transaction, so storage writes made here are
+     * applied immediately and are **not** rolled back when the request fails. In
+     * {@link AdaptivePlaywrightCrawler} it also runs once per request handler attempt, so a write
+     * here can land more than once for a single request. Push results from the request handler itself.
      */
     registerDeferredCleanup(cleanup: () => Promise<unknown>): void;
     /**
@@ -198,58 +202,4 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
      */
     extendTimeout(secs: number): void;
 }
-/**
- * A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
- *
- * @experimental
- */
-export declare class RequestHandlerResult {
-    private configuration;
-    private crawleeStateKey;
-    private _keyValueStoreChanges;
-    private pushDataCalls;
-    private addRequestsCalls;
-    constructor(configuration: Configuration, crawleeStateKey: string);
-    /**
-     * A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
-     */
-    get calls(): ReadonlyDeep<{
-        pushData: Parameters<RestrictedCrawlingContext['pushData']>[];
-        addRequests: Parameters<RestrictedCrawlingContext['addRequests']>[];
-    }>;
-    /**
-     * A record of changes made to key-value stores by a request handler.
-     */
-    get keyValueStoreChanges(): ReadonlyDeep<Record<string, Record<string, {
-        changedValue: unknown;
-        options?: RecordOptions;
-    }>>>;
-    /**
-     * Items added to datasets by a request handler.
-     */
-    get datasetItems(): ReadonlyDeep<{
-        item: Dictionary;
-        datasetIdentifier?: string | StorageIdentifier;
-    }[]>;
-    /**
-     * URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
-     */
-    get enqueuedUrls(): ReadonlyDeep<{
-        url: string;
-        label?: string;
-    }[]>;
-    /**
-     * URL lists enqueued to the request queue by a request handler via {@link RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
-     */
-    get enqueuedUrlLists(): ReadonlyDeep<{
-        listUrl: string;
-        label?: string;
-    }[]>;
-    pushData: RestrictedCrawlingContext['pushData'];
-    addRequests: RestrictedCrawlingContext['addRequests'];
-    useState: RestrictedCrawlingContext['useState'];
-    getKeyValueStore: RestrictedCrawlingContext['getKeyValueStore'];
-    private getKeyValueStoreChangedValue;
-    private setKeyValueStoreChangedValue;
-}
 export {};

package/crawlers/crawler_commons.js

@@ -1,107 +1 @@
-import { KeyValueStore } from '../storages/key_value_store.js';
-/**
- * A partial implementation of {@link RestrictedCrawlingContext} that stores parameters of calls to context methods for later inspection.
- *
- * @experimental
- */
-export class RequestHandlerResult {
-    configuration;
-    crawleeStateKey;
-    _keyValueStoreChanges = {};
-    pushDataCalls = [];
-    addRequestsCalls = [];
-    constructor(configuration, crawleeStateKey) {
-        this.configuration = configuration;
-        this.crawleeStateKey = crawleeStateKey;
-    }
-    /**
-     * A record of calls to {@link RestrictedCrawlingContext.pushData}, {@link RestrictedCrawlingContext.addRequests}, {@link RestrictedCrawlingContext.enqueueLinks} made by a request handler.
-     */
-    get calls() {
-        return {
-            pushData: this.pushDataCalls,
-            addRequests: this.addRequestsCalls,
-        };
-    }
-    /**
-     * A record of changes made to key-value stores by a request handler.
-     */
-    get keyValueStoreChanges() {
-        return this._keyValueStoreChanges;
-    }
-    /**
-     * Items added to datasets by a request handler.
-     */
-    get datasetItems() {
-        return this.pushDataCalls.flatMap(([data, datasetIdentifier]) => (Array.isArray(data) ? data : [data]).map((item) => ({ item, datasetIdentifier })));
-    }
-    /**
-     * URLs enqueued to the request queue by a request handler, either via {@link RestrictedCrawlingContext.addRequests} or {@link RestrictedCrawlingContext.enqueueLinks}
-     */
-    get enqueuedUrls() {
-        const result = [];
-        for (const [requests] of this.addRequestsCalls) {
-            for (const request of requests) {
-                if (typeof request === 'object' &&
-                    (!('requestsFromUrl' in request) || request.requestsFromUrl !== undefined) &&
-                    request.url !== undefined) {
-                    result.push({ url: request.url, label: request.label });
-                }
-                else if (typeof request === 'string') {
-                    result.push({ url: request });
-                }
-            }
-        }
-        return result;
-    }
-    /**
-     * URL lists enqueued to the request queue by a request handler via {@link RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
-     */
-    get enqueuedUrlLists() {
-        const result = [];
-        for (const [requests] of this.addRequestsCalls) {
-            for (const request of requests) {
-                if (typeof request === 'object' &&
-                    'requestsFromUrl' in request &&
-                    request.requestsFromUrl !== undefined) {
-                    result.push({ listUrl: request.requestsFromUrl, label: request.label });
-                }
-            }
-        }
-        return result;
-    }
-    pushData = async (data, datasetIdOrName) => {
-        this.pushDataCalls.push([data, datasetIdOrName]);
-    };
-    addRequests = async (requests, options = {}) => {
-        this.addRequestsCalls.push([requests, options]);
-    };
-    useState = async (defaultValue) => {
-        const store = await this.getKeyValueStore(undefined);
-        return await store.getAutoSavedValue(this.crawleeStateKey, defaultValue);
-    };
-    getKeyValueStore = async (identifier) => {
-        const store = await KeyValueStore.open(identifier, { configuration: this.configuration });
-        const storeId = store.id;
-        return {
-            id: storeId ?? this.configuration.defaultKeyValueStoreId,
-            name: store.name,
-            getValue: async (key) => this.getKeyValueStoreChangedValue(storeId, key) ?? (await store.getValue(key)),
-            setValue: async (key, value, options) => {
-                this.setKeyValueStoreChangedValue(storeId, key, value, options);
-            },
-            getAutoSavedValue: store.getAutoSavedValue.bind(store),
-            getPublicUrl: store.getPublicUrl.bind(store),
-        };
-    };
-    getKeyValueStoreChangedValue = (storeKey, key) => {
-        const id = storeKey ?? this.configuration.defaultKeyValueStoreId;
-        this._keyValueStoreChanges[id] ??= {};
-        return this.keyValueStoreChanges[id][key]?.changedValue ?? null;
-    };
-    setKeyValueStoreChangedValue = (storeKey, key, changedValue, options) => {
-        const id = storeKey ?? this.configuration.defaultKeyValueStoreId;
-        this._keyValueStoreChanges[id] ??= {};
-        this._keyValueStoreChanges[id][key] = { changedValue, options };
-    };
-}
+export {};

package/crawlers/index.d.ts

@@ -1,5 +1,5 @@
 export * from './context_pipeline.js';
-export * from './crawler_commons.js';
+export type * from './crawler_commons.js';
 export * from './statistics.js';
 export * from './error_tracker.js';
 export * from './error_snapshotter.js';

package/crawlers/index.js

@@ -1,5 +1,4 @@
 export * from './context_pipeline.js';
-export * from './crawler_commons.js';
 export * from './statistics.js';
 export * from './error_tracker.js';
 export * from './error_snapshotter.js';

package/crawlers/statistics.d.ts

@@ -81,6 +81,7 @@ export interface CalculatedStatistics {
  * @category Crawlers
  */
 export declare class Statistics implements IStatistics {
+    #private;
     private static id;
     /**
      * An error tracker for final retry errors.
@@ -104,15 +105,7 @@ export declare class Statistics implements IStatistics {
     readonly requestRetryHistogram: number[];
     protected keyValueStore?: KeyValueStore;
     protected readonly persistStateKey: string;
-    private logIntervalMillis;
-    private logMessage;
-    private listener;
-    private requestsInProgress;
     private readonly log;
-    private instanceStart;
-    private logInterval;
-    private _events?;
-    private persistenceOptions;
     private get events();
     /**
      * Construct a statistics instance to pass to a crawler via its `statistics` option, e.g. to preconfigure