@crawlee/core 4.0.0-beta.104 → 4.0.0-beta.106

package/crawlers/statistics.js

@@ -6,14 +6,14 @@ import { ErrorTracker } from './error_tracker.js';
  * @ignore
  */
 class Job {
-    lastRunAt = null;
-    durationMillis;
+    #lastRunAt = null;
+    #durationMillis;
     run() {
-        this.lastRunAt = Date.now();
+        this.#lastRunAt = Date.now();
     }
     finish() {
-        this.durationMillis = Date.now() - this.lastRunAt;
-        return this.durationMillis;
+        this.#durationMillis = Date.now() - this.#lastRunAt;
+        return this.#durationMillis;
     }
 }
 const errorTrackerConfig = {
@@ -35,6 +35,7 @@ const errorTrackerConfig = {
  * @category Crawlers
  */
 export class Statistics {
+    // kept as TS-private: statistics tests read the static counter directly
     static id = 0;
     /**
      * An error tracker for final retry errors.
@@ -58,20 +59,20 @@ export class Statistics {
     requestRetryHistogram = [];
     keyValueStore = undefined;
     persistStateKey;
-    logIntervalMillis;
-    logMessage;
-    listener;
-    requestsInProgress = new Map();
+    #logIntervalMillis;
+    #logMessage;
+    #listener;
+    #requestsInProgress = new Map();
     log;
-    instanceStart;
-    logInterval;
-    _events;
-    persistenceOptions;
+    #instanceStart;
+    #logInterval;
+    #events;
+    #persistenceOptions;
     get events() {
-        if (!this._events) {
-            this._events = serviceLocator.getEventManager();
+        if (!this.#events) {
+            this.#events = serviceLocator.getEventManager();
         }
-        return this._events;
+        return this.#events;
     }
     /**
      * Construct a statistics instance to pass to a crawler via its `statistics` option, e.g. to preconfigure
@@ -95,11 +96,11 @@ export class Statistics {
         this.log = (options.log ?? serviceLocator.getLogger()).child({ prefix: 'Statistics' });
         this.errorTracker = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
         this.errorTrackerRetry = new ErrorTracker({ ...errorTrackerConfig, saveErrorSnapshots });
-        this.logIntervalMillis = logIntervalSecs * 1000;
-        this.logMessage = logMessage;
+        this.#logIntervalMillis = logIntervalSecs * 1000;
+        this.#logMessage = logMessage;
         this.keyValueStore = keyValueStore;
-        this.listener = this.persistState.bind(this);
-        this.persistenceOptions = persistenceOptions;
+        this.#listener = this.persistState.bind(this);
+        this.#persistenceOptions = persistenceOptions;
         // initialize by "resetting"
         this.reset();
     }
@@ -128,15 +129,15 @@ export class Statistics {
             retryErrors: this.errorTrackerRetry.result,
         };
         this.requestRetryHistogram.length = 0;
-        this.requestsInProgress.clear();
-        this.instanceStart = Date.now();
+        this.#requestsInProgress.clear();
+        this.#instanceStart = Date.now();
         this.teardown();
     }
     /**
      * @param options - Override the persistence options provided in the constructor
      */
     async resetStore(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
+        if (!this.#persistenceOptions.enable && !options?.enable) {
             return;
         }
         if (!this.keyValueStore) {
@@ -159,18 +160,18 @@ export class Statistics {
      * @ignore
      */
     startJob(id) {
-        let job = this.requestsInProgress.get(id);
+        let job = this.#requestsInProgress.get(id);
         if (!job)
             job = new Job();
         job.run();
-        this.requestsInProgress.set(id, job);
+        this.#requestsInProgress.set(id, job);
     }
     /**
      * Mark job as finished and sets the state
      * @ignore
      */
     finishJob(id, retryCount) {
-        const job = this.requestsInProgress.get(id);
+        const job = this.#requestsInProgress.get(id);
         if (!job)
             return;
         const jobDurationMillis = job.finish();
@@ -181,20 +182,20 @@ export class Statistics {
             this.state.requestMinDurationMillis = jobDurationMillis;
         if (jobDurationMillis > this.state.requestMaxDurationMillis)
             this.state.requestMaxDurationMillis = jobDurationMillis;
-        this.requestsInProgress.delete(id);
+        this.#requestsInProgress.delete(id);
     }
     /**
      * Mark job as failed and sets the state
      * @ignore
      */
     failJob(id, retryCount) {
-        const job = this.requestsInProgress.get(id);
+        const job = this.#requestsInProgress.get(id);
         if (!job)
             return;
         this.state.requestTotalFailedDurationMillis += job.finish();
         this.state.requestsFailed++;
         this.saveRetryCountForJob(retryCount);
-        this.requestsInProgress.delete(id);
+        this.#requestsInProgress.delete(id);
     }
     /**
      * Discards a started job without affecting the finished/failed counters, e.g. when a request
@@ -202,14 +203,14 @@ export class Statistics {
      * @ignore
      */
     discardJob(id) {
-        this.requestsInProgress.delete(id);
+        this.#requestsInProgress.delete(id);
     }
     /**
      * Calculate the current statistics
      */
     calculate() {
         const { requestsFailed, requestsFinished, requestTotalFailedDurationMillis, requestTotalFinishedDurationMillis, } = this.state;
-        const totalMillis = Date.now() - this.instanceStart;
+        const totalMillis = Date.now() - this.#instanceStart;
         const totalMinutes = totalMillis / 1000 / 60;
         return {
             requestAvgFailedDurationMillis: Math.round(requestTotalFailedDurationMillis / requestsFailed) || Infinity,
@@ -228,23 +229,23 @@ export class Statistics {
     async startCapturing() {
         // A single instance drives one logging interval and one PERSIST_STATE listener, so a second concurrent
         // capture (e.g. sharing one instance across crawlers running at once) would orphan the first. Fail loudly.
-        if (this.logInterval) {
+        if (this.#logInterval) {
             throw new Error('Statistics.startCapturing() was already called - this instance is already capturing.');
         }
         this.keyValueStore ??= await KeyValueStore.open(null, { configuration: serviceLocator.getConfiguration() });
         if (this.state.crawlerStartedAt === null) {
             this.state.crawlerStartedAt = new Date();
         }
-        if (this.persistenceOptions.enable) {
+        if (this.#persistenceOptions.enable) {
             await this.maybeLoadStatistics();
-            this.events.on("persistState" /* EventType.PERSIST_STATE */, this.listener);
+            this.events.on("persistState" /* EventType.PERSIST_STATE */, this.#listener);
         }
-        this.logInterval = setInterval(() => {
-            this.log.info(this.logMessage, {
+        this.#logInterval = setInterval(() => {
+            this.log.info(this.#logMessage, {
                 ...this.calculate(),
                 retryHistogram: this.requestRetryHistogram,
             });
-        }, this.logIntervalMillis);
+        }, this.#logIntervalMillis);
     }
     /**
      * Stops logging and remove event listeners, then persist
@@ -265,7 +266,7 @@ export class Statistics {
      * @param options - Override the persistence options provided in the constructor
      */
     async persistState(options) {
-        if (!this.persistenceOptions.enable && !options?.enable) {
+        if (!this.#persistenceOptions.enable && !options?.enable) {
             return;
         }
         // this might be called before startCapturing was called without using await, should not crash
@@ -312,17 +313,17 @@ export class Statistics {
         this.state.crawlerStartedAt = savedState.crawlerStartedAt ? new Date(savedState.crawlerStartedAt) : null;
         this.state.statsPersistedAt = savedState.statsPersistedAt ? new Date(savedState.statsPersistedAt) : null;
         this.state.crawlerRuntimeMillis = savedState.crawlerRuntimeMillis;
-        this.instanceStart = Date.now() - (+this.state.statsPersistedAt - savedState.crawlerLastStartTimestamp);
+        this.#instanceStart = Date.now() - (+this.state.statsPersistedAt - savedState.crawlerLastStartTimestamp);
         this.log.debug('Loaded from KeyValueStore');
     }
     teardown() {
         // this can be called before a call to startCapturing happens (or in a 'finally' block)
         // Only unsubscribe if event manager was already resolved — avoid eagerly resolving it
         // (e.g. during the constructor's reset() call, which would capture the wrong context)
-        this._events?.off("persistState" /* EventType.PERSIST_STATE */, this.listener);
-        if (this.logInterval) {
-            clearInterval(this.logInterval);
-            this.logInterval = null;
+        this.#events?.off("persistState" /* EventType.PERSIST_STATE */, this.#listener);
+        if (this.#logInterval) {
+            clearInterval(this.#logInterval);
+            this.#logInterval = null;
         }
     }
     /**
@@ -335,7 +336,7 @@ export class Statistics {
         // omit duplicated information
         const result = {
             ...this.state,
-            crawlerLastStartTimestamp: this.instanceStart,
+            crawlerLastStartTimestamp: this.#instanceStart,
             crawlerFinishedAt: this.state.crawlerFinishedAt
                 ? new Date(this.state.crawlerFinishedAt).toISOString()
                 : null,

package/events/event_manager.d.ts

@@ -41,12 +41,12 @@ interface Intervals {
     systemInfo?: BetterIntervalID;
 }
 export declare abstract class EventManager {
+    #private;
     protected events: AsyncEventEmitter<{}>;
     protected initialized: boolean;
     protected intervals: Intervals;
 // @ts-ignore optional peer dependency or compatibility with es2022
     protected log: import("@crawlee/types").CrawleeLogger;
-    private persistStateIntervalMillis;
     constructor(options: EventManagerOptions);
     /**
      * Initializes the event manager by starting the `persistState` event interval.

package/events/event_manager.js

@@ -15,9 +15,9 @@ export class EventManager {
     initialized = false;
     intervals = {};
     log = serviceLocator.getLogger().child({ prefix: 'Events' });
-    persistStateIntervalMillis;
+    #persistStateIntervalMillis;
     constructor(options) {
-        this.persistStateIntervalMillis = options.persistStateIntervalMillis;
+        this.#persistStateIntervalMillis = options.persistStateIntervalMillis;
         this.events.setMaxListeners(50);
     }
     /**
@@ -31,7 +31,7 @@ export class EventManager {
         this.intervals.persistState = betterSetInterval((intervalCallback) => {
             this.emit("persistState" /* EventType.PERSIST_STATE */, { isMigrating: false });
             intervalCallback();
-        }, this.persistStateIntervalMillis);
+        }, this.#persistStateIntervalMillis);
         this.initialized = true;
     }
     /**

package/events/local_event_manager.d.ts

@@ -5,7 +5,7 @@ export interface LocalEventManagerOptions extends EventManagerOptions {
     systemInfoIntervalMillis: number;
 }
 export declare class LocalEventManager extends EventManager {
-    private systemInfoIntervalMillis;
+    #private;
     constructor(options: LocalEventManagerOptions);
     /**
      * Creates a new `LocalEventManager` based on the provided `Configuration`.

package/events/local_event_manager.js

@@ -2,10 +2,10 @@ import { betterClearInterval, betterSetInterval } from '@apify/utilities';
 import { serviceLocator } from '../service_locator.js';
 import { EventManager } from './event_manager.js';
 export class LocalEventManager extends EventManager {
-    systemInfoIntervalMillis;
+    #systemInfoIntervalMillis;
     constructor(options) {
         super(options);
-        this.systemInfoIntervalMillis = options.systemInfoIntervalMillis;
+        this.#systemInfoIntervalMillis = options.systemInfoIntervalMillis;
     }
     /**
      * Creates a new `LocalEventManager` based on the provided `Configuration`.
@@ -28,7 +28,7 @@ export class LocalEventManager extends EventManager {
         }
         await super.init();
         this.emitSystemInfoEvent = this.emitSystemInfoEvent.bind(this);
-        this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), this.systemInfoIntervalMillis);
+        this.intervals.systemInfo = betterSetInterval(this.emitSystemInfoEvent.bind(this), this.#systemInfoIntervalMillis);
     }
     /**
      * @inheritDoc

package/log.js

@@ -30,6 +30,8 @@ import log, { Log, Logger, LoggerJson, LoggerText, LogLevel } from '@apify/log';
  * ```
  */
 export class BaseCrawleeLogger {
+    // kept as TS-private: the adaptive crawler's log proxy calls non-intercepted methods with `this === proxy`,
+    // where `#`-field access would throw at runtime
     options;
     warningsLogged = new Set();
     constructor(options = {}) {
@@ -87,7 +89,9 @@ export class BaseCrawleeLogger {
  */
 export class ApifyLogAdapter extends BaseCrawleeLogger {
     apifyLog;
-    constructor(apifyLog, options) {
+    constructor(
+    // kept as a TS-private parameter property: reached through the adaptive crawler's log proxy, see above
+    apifyLog, options) {
         super(options ?? {});
         this.apifyLog = apifyLog;
     }

package/memory-storage/memory-storage.d.ts

@@ -10,12 +10,8 @@ export interface MemoryStorageOptions {
     logger?: CrawleeLogger;
 }
 export declare class MemoryStorageBackend implements storage.StorageBackend {
+    #private;
     readonly logger?: CrawleeLogger;
-    /**
-     * Unique per-instance cache partition key. Mirrors the way `FileSystemStorageBackend` partitions its
-     * cache by storage directory: two distinct `MemoryStorageBackend` instances must not share cached backends.
-     */
-    private readonly instanceCacheKey;
     readonly keyValueStoreBackendCache: KeyValueStoreBackend[];
     readonly datasetBackendCache: DatasetBackend[];
     readonly requestQueueBackendCache: RequestQueueBackend[];

package/memory-storage/memory-storage.js

@@ -8,7 +8,7 @@ export class MemoryStorageBackend {
      * Unique per-instance cache partition key. Mirrors the way `FileSystemStorageBackend` partitions its
      * cache by storage directory: two distinct `MemoryStorageBackend` instances must not share cached backends.
      */
-    instanceCacheKey = `MemoryStorageBackend:${randomUUID()}`;
+    #instanceCacheKey = `MemoryStorageBackend:${randomUUID()}`;
     keyValueStoreBackendCache = [];
     datasetBackendCache = [];
     requestQueueBackendCache = [];
@@ -20,7 +20,7 @@ export class MemoryStorageBackend {
      * cache partitions in the storage backend cache.
      */
     getStorageBackendCacheKey() {
-        return this.instanceCacheKey;
+        return this.#instanceCacheKey;
     }
     static resolveStorageKey(options) {
         const isAlias = 'alias' in options && !!options.alias;

package/memory-storage/resource-clients/dataset.d.ts

@@ -14,6 +14,7 @@ export interface DatasetBackendOptions {
     storageBackend: MemoryStorageBackend;
 }
 export declare class DatasetBackend<Data extends Dictionary = Dictionary> extends BaseClient implements storage.DatasetBackend<Data> {
+    #private;
     name?: string;
     /**
      * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
@@ -24,7 +25,6 @@ export declare class DatasetBackend<Data extends Dictionary = Dictionary> extend
     accessedAt: Date;
     modifiedAt: Date;
     itemCount: number;
-    private readonly datasetEntries;
     private readonly storageBackend;
     constructor(options: DatasetBackendOptions);
     getMetadata(): Promise<storage.DatasetInfo>;

package/memory-storage/resource-clients/dataset.js

@@ -22,7 +22,8 @@ export class DatasetBackend extends BaseClient {
     accessedAt = new Date();
     modifiedAt = new Date();
     itemCount = 0;
-    datasetEntries = new Map();
+    #datasetEntries = new Map();
+    // kept as TS-private: storage-backend tests read this field at runtime
     storageBackend;
     constructor(options) {
         super(options.id ?? randomUUID());
@@ -39,12 +40,12 @@ export class DatasetBackend extends BaseClient {
         if (storeIndex !== -1) {
             const [oldBackend] = this.storageBackend.datasetBackendCache.splice(storeIndex, 1);
             oldBackend.itemCount = 0;
-            oldBackend.datasetEntries.clear();
+            oldBackend.#datasetEntries.clear();
         }
     }
     async purge() {
         this.itemCount = 0;
-        this.datasetEntries.clear();
+        this.#datasetEntries.clear();
         this.updateTimestamps(true);
     }
     getData(options = {}) {
@@ -67,7 +68,7 @@ export class DatasetBackend extends BaseClient {
         const items = [];
         for (let idx = start; idx < end; idx++) {
             const entryNumber = this.generateLocalEntryName(idx);
-            items.push(this.datasetEntries.get(entryNumber));
+            items.push(this.#datasetEntries.get(entryNumber));
         }
         this.updateTimestamps(false);
         return {
@@ -82,7 +83,7 @@ export class DatasetBackend extends BaseClient {
     async pushData(items) {
         for (const entry of items) {
             const idx = this.generateLocalEntryName(++this.itemCount);
-            this.datasetEntries.set(idx, JSON.parse(JSON.stringify(entry)));
+            this.#datasetEntries.set(idx, JSON.parse(JSON.stringify(entry)));
         }
         this.updateTimestamps(true);
     }

package/memory-storage/resource-clients/key-value-store.d.ts

@@ -19,6 +19,7 @@ export interface InternalKeyRecord {
     extension: string;
 }
 export declare class KeyValueStoreBackend extends BaseClient implements storage.KeyValueStoreBackend {
+    #private;
     name?: string;
     /**
      * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
@@ -28,7 +29,6 @@ export declare class KeyValueStoreBackend extends BaseClient implements storage.
     createdAt: Date;
     accessedAt: Date;
     modifiedAt: Date;
-    private readonly keyValueEntries;
     private readonly storageBackend;
     constructor(options: KeyValueStoreBackendOptions);
     getMetadata(): Promise<storage.KeyValueStoreInfo>;

package/memory-storage/resource-clients/key-value-store.js

@@ -19,7 +19,8 @@ export class KeyValueStoreBackend extends BaseClient {
     createdAt = new Date();
     accessedAt = new Date();
     modifiedAt = new Date();
-    keyValueEntries = new Map();
+    #keyValueEntries = new Map();
+    // kept as TS-private: storage-backend tests read this field at runtime
     storageBackend;
     constructor(options) {
         super(options.id ?? randomUUID());
@@ -35,11 +36,11 @@ export class KeyValueStoreBackend extends BaseClient {
         const storeIndex = this.storageBackend.keyValueStoreBackendCache.findIndex((store) => store.id === this.id);
         if (storeIndex !== -1) {
             const [oldBackend] = this.storageBackend.keyValueStoreBackendCache.splice(storeIndex, 1);
-            oldBackend.keyValueEntries.clear();
+            oldBackend.#keyValueEntries.clear();
         }
     }
     async purge() {
-        this.keyValueEntries.clear();
+        this.#keyValueEntries.clear();
         this.updateTimestamps(true);
     }
     /**
@@ -49,9 +50,9 @@ export class KeyValueStoreBackend extends BaseClient {
      * preserve the bare `INPUT` key only.
      */
     async purgeExceptInput() {
-        for (const key of this.keyValueEntries.keys()) {
+        for (const key of this.#keyValueEntries.keys()) {
             if (key !== KEY_VALUE_STORE_INPUT_KEY) {
-                this.keyValueEntries.delete(key);
+                this.#keyValueEntries.delete(key);
             }
         }
         this.updateTimestamps(true);
@@ -65,7 +66,7 @@ export class KeyValueStoreBackend extends BaseClient {
         })
             .parse(options);
         const items = [];
-        for (const record of this.keyValueEntries.values()) {
+        for (const record of this.#keyValueEntries.values()) {
             const size = Buffer.byteLength(record.value);
             items.push({
                 key: record.key,
@@ -114,11 +115,11 @@ export class KeyValueStoreBackend extends BaseClient {
      */
     async recordExists(key) {
         s.string().parse(key);
-        return this.keyValueEntries.has(key);
+        return this.#keyValueEntries.has(key);
     }
     async getValue(key) {
         s.string().parse(key);
-        const entry = this.keyValueEntries.get(key);
+        const entry = this.#keyValueEntries.get(key);
         if (!entry) {
             return undefined;
         }
@@ -169,19 +170,19 @@ export class KeyValueStoreBackend extends BaseClient {
         const normalizedValue = typeof value === 'string'
             ? Buffer.from(value, 'utf-8')
             : toBuffer(value);
-        const _record = {
+        const normalizedRecord = {
             extension,
             key,
             value: normalizedValue,
             contentType,
         };
-        this.keyValueEntries.set(key, _record);
+        this.#keyValueEntries.set(key, normalizedRecord);
         this.updateTimestamps(true);
     }
     async deleteValue(key) {
         s.string().parse(key);
-        if (this.keyValueEntries.has(key)) {
-            this.keyValueEntries.delete(key);
+        if (this.#keyValueEntries.has(key)) {
+            this.#keyValueEntries.delete(key);
             this.updateTimestamps(true);
         }
     }

package/memory-storage/resource-clients/request-queue.d.ts

@@ -22,6 +22,7 @@ export interface InternalRequest {
     json: string;
 }
 export declare class RequestQueueBackend extends BaseClient implements storage.RequestQueueBackend {
+    #private;
     name?: string;
     /**
      * The key used for cache lookup. For named storages, this equals the name. For alias (unnamed)
@@ -33,26 +34,6 @@ export declare class RequestQueueBackend extends BaseClient implements storage.R
     modifiedAt: Date;
     handledRequestCount: number;
     pendingRequestCount: number;
-    /**
-     * Serializes every operation that reads-then-writes this backend's shared queue state — the
-     * `requests` map, the `forefrontRequestIds` array, the `inProgressRequestIds` set and the request
-     * counts. Those mutations span `await` points, so without this mutex a concurrent operation could
-     * interleave and corrupt them (e.g. a head scan pruning `forefrontRequestIds` while
-     * `addBatchOfRequests` pushes to it). Held by every mutating method as well as by `isEmpty`/
-     * `isFinished`, whose head scan also prunes `forefrontRequestIds`.
-     */
-    private readonly queueStateMutex;
-    private forefrontRequestIds;
-    /**
-     * IDs of requests currently fetched but not yet handled or reclaimed. A request in this set is
-     * "in progress" and will not be handed out again by {@link fetchNextRequest}.
-     *
-     * Unlike the file-system / platform clients, the in-memory queue lives entirely within a single
-     * process and is never shared with another consumer, so there is no need for an expiring,
-     * cross-process-visible lock — tracking in-progress requests in this set is enough.
-     */
-    private readonly inProgressRequestIds;
-    private readonly requests;
     private readonly storageBackend;
     constructor(options: RequestQueueBackendOptions);
     getMetadata(): Promise<storage.RequestQueueInfo>;
@@ -90,7 +71,7 @@ export declare class RequestQueueBackend extends BaseClient implements storage.R
     listItems(): Promise<storage.UpdateRequestSchema[]>;
     toRequestQueueInfo(): storage.RequestQueueInfo;
     private updateTimestamps;
-    private _jsonToRequest;
-    private _createInternalRequest;
-    private _calculateOrderNo;
+    private jsonToRequest;
+    private createInternalRequest;
+    private calculateOrderNo;
 }