@crawlee/core 4.0.0-beta.105 → 4.0.0-beta.107

package/storages/request_queue.js

@@ -56,26 +56,28 @@ export class RequestQueue {
     id;
     name;
     backend;
-    proxyConfiguration;
+    #proxyConfiguration;
     log;
+    // kept as TS-private: request_queue tests read this cache directly
     requestCache;
     /**
      * Remembers the `requestId` of every request already submitted to the client — including background
      * batches that `requestCache` skips — so overlapping URL sets aren't re-submitted.
      * See {@link RequestDeduplicationCache} for why this is a separate, cheaper cache.
      */
-    requestSeenCache;
-    queuePausedForMigration = false;
+    #requestSeenCache;
+    #queuePausedForMigration = false;
+    // kept as TS-private: packages/core/test request-queue tests write this counter directly
     inProgressRequestBatchCount = 0;
     /**
      * The largest expected request-processing time (in seconds) seen so far via
      * {@link setExpectedRequestProcessingTimeSecs}. Used to ensure that value is only ever raised, never
      * lowered, before being forwarded to the storage backend.
      */
-    expectedRequestProcessingSecs = 0;
-    httpClient;
-    events;
-    statsTracker = new StorageStatsTracker({
+    #expectedRequestProcessingSecs = 0;
+    #httpClient;
+    #events;
+    #statsTracker = new StorageStatsTracker({
         writeCount: 0,
         headItemReadCount: 0,
     });
@@ -84,7 +86,7 @@ export class RequestQueue {
      * queue-head reads issued to the underlying storage backend). Counted per backend call.
      */
     get stats() {
-        return this.statsTracker.current;
+        return this.#statsTracker.current;
     }
     /**
      * @internal
@@ -92,14 +94,14 @@ export class RequestQueue {
     constructor(options) {
         this.id = options.metadata.id;
         this.name = options.metadata.name;
-        this.events = serviceLocator.getEventManager();
+        this.#events = serviceLocator.getEventManager();
         this.backend = options.backend;
-        this.proxyConfiguration = options.proxyConfiguration;
+        this.#proxyConfiguration = options.proxyConfiguration;
         this.requestCache = new LruCache({ maxLength: MAX_CACHED_REQUESTS });
-        this.requestSeenCache = new RequestDeduplicationCache();
+        this.#requestSeenCache = new RequestDeduplicationCache();
         this.log = serviceLocator.getLogger().child({ prefix: `RequestQueue(${this.id}, ${this.name ?? 'no-name'})` });
-        this.events.on("migrating" /* EventType.MIGRATING */, async () => {
-            this.queuePausedForMigration = true;
+        this.#events.on("migrating" /* EventType.MIGRATING */, async () => {
+            this.#queuePausedForMigration = true;
         });
     }
     /**
@@ -169,7 +171,7 @@ export class RequestQueue {
                 forefront,
             };
         }
-        this.statsTracker.add('writeCount');
+        this.#statsTracker.add('writeCount');
         const { processedRequests } = await this.backend.addBatchOfRequests([request], { forefront });
         this.recordRequestJournalEntry(transaction, [request], forefront, true);
         const queueOperationInfo = {
@@ -178,7 +180,7 @@ export class RequestQueue {
             forefront,
         };
         this.cacheRequest(cacheKey, queueOperationInfo);
-        this.requestSeenCache.add(cacheKey, request.id);
+        this.#requestSeenCache.add(cacheKey, request.id);
         return queueOperationInfo;
     }
     /**
@@ -240,7 +242,7 @@ export class RequestQueue {
         // reading saves a probe. Same lookup as the write-through path.
         const cacheKey = getRequestId(request.uniqueKey);
         const cachedInfo = this.requestCache.get(cacheKey);
-        const knownRequestId = cachedInfo?.id ?? this.requestSeenCache.get(cacheKey);
+        const knownRequestId = cachedInfo?.id ?? this.#requestSeenCache.get(cacheKey);
         if (knownRequestId) {
             this.recordRequestJournalEntry(transaction, [request], forefront, false);
             return {
@@ -306,7 +308,7 @@ export class RequestQueue {
                 : []);
             if (requests.length === 0)
                 continue;
-            this.statsTracker.add('writeCount');
+            this.#statsTracker.add('writeCount');
             const { processedRequests, unprocessedRequests } = await this.backend.addBatchOfRequests(requests, {
                 forefront,
             });
@@ -314,7 +316,7 @@ export class RequestQueue {
             for (const processed of processedRequests) {
                 const cacheKey = getRequestId(processed.uniqueKey);
                 this.cacheRequest(cacheKey, { ...processed, forefront });
-                this.requestSeenCache.add(cacheKey, processed.requestId);
+                this.#requestSeenCache.add(cacheKey, processed.requestId);
             }
             if (unprocessedRequests.length > 0) {
                 // Warn and skip, rather than retry or fail. `unprocessedRequests` is what remains after
@@ -389,7 +391,7 @@ export class RequestQueue {
             const cacheKey = getCachedRequestId(request.uniqueKey);
             // Prefer the full `requestCache` record; fall back to the dedup cache for background batches it skips.
             const cachedInfo = this.requestCache.get(cacheKey);
-            const knownRequestId = cachedInfo?.id ?? this.requestSeenCache.get(cacheKey);
+            const knownRequestId = cachedInfo?.id ?? this.#requestSeenCache.get(cacheKey);
             if (knownRequestId) {
                 request.id = knownRequestId;
                 results.processedRequests.push({
@@ -408,7 +410,7 @@ export class RequestQueue {
         if (!requestsToAdd.size) {
             return results;
         }
-        this.statsTracker.add('writeCount');
+        this.#statsTracker.add('writeCount');
         const apiResults = await this.backend.addBatchOfRequests([...requestsToAdd.values()], { forefront });
         // Report unprocessed requests
         results.unprocessedRequests = apiResults.unprocessedRequests;
@@ -421,7 +423,7 @@ export class RequestQueue {
                 this.cacheRequest(cacheKey, { ...newRequest, forefront });
             }
             // Unlike `requestCache`, populate this on every batch (including background ones).
-            this.requestSeenCache.add(cacheKey, newRequest.requestId);
+            this.#requestSeenCache.add(cacheKey, newRequest.requestId);
         }
         return results;
     }
@@ -609,10 +611,10 @@ export class RequestQueue {
      */
     async fetchNextRequest() {
         rejectOperationInTransaction('RequestQueue.fetchNextRequest()', 'it is part of the crawler request-processing bookkeeping, which a transaction must not affect.');
-        if (this.queuePausedForMigration) {
+        if (this.#queuePausedForMigration) {
             return null;
         }
-        this.statsTracker.add('headItemReadCount');
+        this.#statsTracker.add('headItemReadCount');
         const requestOptions = await this.backend.fetchNextRequest();
         if (!requestOptions)
             return null;
@@ -633,7 +635,7 @@ export class RequestQueue {
         }));
         const forefront = this.requestCache.get(getRequestId(request.uniqueKey))?.forefront ?? false;
         const handledAt = request.handledAt ?? new Date().toISOString();
-        this.statsTracker.add('writeCount');
+        this.#statsTracker.add('writeCount');
         const processedRequest = await this.backend.markRequestAsHandled({
             ...request,
             handledAt,
@@ -667,7 +669,7 @@ export class RequestQueue {
             forefront: ow.optional.boolean,
         }));
         const { forefront = false } = options;
-        this.statsTracker.add('writeCount');
+        this.#statsTracker.add('writeCount');
         const processedRequest = await this.backend.reclaimRequest(request, { forefront });
         // The request was not in progress — nothing to reclaim.
         if (!processedRequest) {
@@ -730,10 +732,10 @@ export class RequestQueue {
      * short the reservation of a long-lived one and have its in-flight request stolen.
      */
     async setExpectedRequestProcessingTimeSecs(secs) {
-        if (secs <= this.expectedRequestProcessingSecs) {
+        if (secs <= this.#expectedRequestProcessingSecs) {
             return;
         }
-        this.expectedRequestProcessingSecs = secs;
+        this.#expectedRequestProcessingSecs = secs;
         await this.backend.setExpectedRequestProcessingTimeSecs?.(secs);
     }
     /**
@@ -769,12 +771,12 @@ export class RequestQueue {
         await this.backend.purge();
         // Reset in-memory bookkeeping so the queue behaves as if freshly opened.
         this.requestCache.clear();
-        this.requestSeenCache.clear();
+        this.#requestSeenCache.clear();
         this.inProgressRequestBatchCount = 0;
         // Reset the expected-processing-time high-water mark too, otherwise the monotonic-raise guard
         // in `setExpectedRequestProcessingTimeSecs` would let a value raised in an earlier run leak into a
         // later one and silently swallow a lower hint (the queue is meant to be reusable across runs).
-        this.expectedRequestProcessingSecs = 0;
+        this.#expectedRequestProcessingSecs = 0;
     }
     /**
      * @inheritdoc
@@ -842,10 +844,10 @@ export class RequestQueue {
         // Download remote resource and parse URLs.
         let urlsArr;
         try {
-            urlsArr = await this._downloadListOfUrls({
+            urlsArr = await this.downloadListOfUrls({
                 url: requestsFromUrl,
                 urlRegExp: regex,
-                proxyUrl: (await this.proxyConfiguration?.newProxyInfo())?.url,
+                proxyUrl: (await this.#proxyConfiguration?.newProxyInfo())?.url,
             });
         }
         catch (err) {
@@ -877,10 +879,10 @@ export class RequestQueue {
     /**
      * @internal wraps public utility for mocking purposes
      */
-    async _downloadListOfUrls(options) {
+    async downloadListOfUrls(options) {
         return downloadListOfUrls({
             ...options,
-            httpClient: this.httpClient,
+            httpClient: this.#httpClient,
         });
     }
     /**
@@ -919,8 +921,8 @@ export class RequestQueue {
             backendOpener: () => storageBackend.createRequestQueueBackend(resolved),
             backendCacheKey: storageBackend.getStorageBackendCacheKey?.() ?? storageBackend.constructor.name,
         });
-        queue.proxyConfiguration = options.proxyConfiguration;
-        queue.httpClient = options.httpClient;
+        queue.#proxyConfiguration = options.proxyConfiguration;
+        queue.#httpClient = options.httpClient;
         return queue;
     }
 }

package/storages/sitemap_request_loader.d.ts

@@ -80,55 +80,12 @@ export interface SitemapRequestLoaderOptions extends UrlConstraints {
  * The loading of the sitemap is performed in the background so that crawling can start before the sitemap is fully loaded.
  */
 export declare class SitemapRequestLoader implements IRequestLoader {
+    #private;
     /**
      * Set of URLs that were returned by `fetchNextRequest()` and not marked as handled yet.
      * @internal
      */
     inProgress: Set<string>;
-    /**
-     * Map of returned Request objects that have not been marked as handled yet.
-     *
-     * We use this to persist custom user fields on the in-progress requests.
-     */
-    private requestData;
-    /**
-     * Object for keeping track of the sitemap parsing progress.
-     */
-    private sitemapParsingProgress;
-    /**
-     * Object stream of URLs parsed from the sitemaps.
-     * Using `highWaterMark`, this can manage the speed of the sitemap loading.
-     *
-     * Fetch the next URL to be processed using `fetchNextRequest()`.
-     */
-    private urlQueueStream;
-    /**
-     * Indicates whether the request list sitemap loading was aborted.
-     *
-     * If the loading was aborted before the sitemaps were fully loaded, the request list might be missing some URLs.
-     * The `isSitemapFullyLoaded` method can be used to check if the sitemaps were fully loaded.
-     *
-     * If the loading is aborted and all the requests are handled, `isFinished()` will return `true`.
-     */
-    private abortLoading;
-    /** Number of URLs that were marked as handled */
-    private handledUrlCount;
-    private persistStateKey?;
-    private store?;
-    private closed;
-    /**
-     * Proxy URL to be used for sitemap loading.
-     */
-    private proxyUrl?;
-    /**
-     * Logger instance.
-     */
-    private log;
-    private urlExcludePatternObjects;
-    private urlPatternObjects;
-    /** EventManager used to handle persistence */
-    private events;
-    private persistenceOptions;
     /** @internal */
     private constructor();
     /**

package/storages/sitemap_request_loader.js

@@ -25,11 +25,11 @@ export class SitemapRequestLoader {
      *
      * We use this to persist custom user fields on the in-progress requests.
      */
-    requestData = new Map();
+    #requestData = new Map();
     /**
      * Object for keeping track of the sitemap parsing progress.
      */
-    sitemapParsingProgress = {
+    #sitemapParsingProgress = {
         /**
          * URL of the sitemap that is currently being parsed. `null` if no sitemap is being parsed.
          */
@@ -49,7 +49,7 @@ export class SitemapRequestLoader {
      *
      * Fetch the next URL to be processed using `fetchNextRequest()`.
      */
-    urlQueueStream;
+    #urlQueueStream;
     /**
      * Indicates whether the request list sitemap loading was aborted.
      *
@@ -58,25 +58,25 @@ export class SitemapRequestLoader {
      *
      * If the loading is aborted and all the requests are handled, `isFinished()` will return `true`.
      */
-    abortLoading = false;
+    #abortLoading = false;
     /** Number of URLs that were marked as handled */
-    handledUrlCount = 0;
-    persistStateKey;
-    store;
-    closed = false;
+    #handledUrlCount = 0;
+    #persistStateKey;
+    #store;
+    #closed = false;
     /**
      * Proxy URL to be used for sitemap loading.
      */
-    proxyUrl;
+    #proxyUrl;
     /**
      * Logger instance.
      */
-    log;
-    urlExcludePatternObjects = [];
-    urlPatternObjects = [];
+    #log;
+    #urlExcludePatternObjects = [];
+    #urlPatternObjects = [];
     /** EventManager used to handle persistence */
-    events;
-    persistenceOptions;
+    #events;
+    #persistenceOptions;
     /** @internal */
     constructor(options) {
         const urlPatternValidator = ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'));
@@ -94,19 +94,19 @@ export class SitemapRequestLoader {
             httpClient: ow.optional.object,
         }));
         const { include, exclude } = options;
-        this.log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestLoader' });
+        this.#log = serviceLocator.getLogger().child({ prefix: 'SitemapRequestLoader' });
         if (exclude?.length) {
-            this.urlExcludePatternObjects.push(...constructUrlPatternObjects(exclude));
+            this.#urlExcludePatternObjects.push(...constructUrlPatternObjects(exclude));
         }
         if (include?.length) {
-            this.urlPatternObjects.push(...constructUrlPatternObjects(include));
+            this.#urlPatternObjects.push(...constructUrlPatternObjects(include));
         }
-        this.persistStateKey = options.persistStateKey;
-        this.persistenceOptions = { enable: true, ...options.persistenceOptions };
-        this.proxyUrl = options.proxyUrl;
-        this.urlQueueStream = this.createNewStream(options.maxBufferSize ?? 200);
-        this.sitemapParsingProgress.pendingSitemapUrls = new Set(options.sitemapUrls);
-        this.events = serviceLocator.getEventManager();
+        this.#persistStateKey = options.persistStateKey;
+        this.#persistenceOptions = { enable: true, ...options.persistenceOptions };
+        this.#proxyUrl = options.proxyUrl;
+        this.#urlQueueStream = this.createNewStream(options.maxBufferSize ?? 200);
+        this.#sitemapParsingProgress.pendingSitemapUrls = new Set(options.sitemapUrls);
+        this.#events = serviceLocator.getEventManager();
         this.persistState = this.persistState.bind(this);
     }
     /**
@@ -139,8 +139,8 @@ export class SitemapRequestLoader {
      * @returns `true` if the URL matches the patterns, `false` otherwise.
      */
     isUrlMatchingPatterns(url) {
-        return (!this.urlExcludePatternObjects.some(this.matchesUrl(url)) &&
-            (this.urlPatternObjects.length === 0 || this.urlPatternObjects.some(this.matchesUrl(url))));
+        return (!this.#urlExcludePatternObjects.some(this.matchesUrl(url)) &&
+            (this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(this.matchesUrl(url))));
     }
     /**
      * Adds a URL to the queue of parsed URLs.
@@ -149,13 +149,13 @@ export class SitemapRequestLoader {
      */
     async pushNextUrl(url) {
         return new Promise((resolve) => {
-            if (this.closed || (url && !this.isUrlMatchingPatterns(url))) {
+            if (this.#closed || (url && !this.isUrlMatchingPatterns(url))) {
                 resolve();
                 return;
             }
-            if (!this.urlQueueStream.push(url)) {
+            if (!this.#urlQueueStream.push(url)) {
                 // This doesn't work with the 'drain' event (it's not emitted for some reason).
-                this.urlQueueStream.once('readdata', () => {
+                this.#urlQueueStream.once('readdata', () => {
                     resolve();
                 });
             }
@@ -172,21 +172,21 @@ export class SitemapRequestLoader {
      */
     async readNextUrl() {
         return new Promise((resolve) => {
-            if (this.closed) {
+            if (this.#closed) {
                 resolve(null);
                 return;
             }
-            const result = this.urlQueueStream.read();
+            const result = this.#urlQueueStream.read();
             if (!result && !this.isSitemapFullyLoaded()) {
-                this.urlQueueStream.once('readable', () => {
-                    const nextUrl = this.urlQueueStream.read();
+                this.#urlQueueStream.once('readable', () => {
+                    const nextUrl = this.#urlQueueStream.read();
                     resolve(nextUrl);
                 });
             }
             else {
                 resolve(result);
             }
-            this.urlQueueStream.emit('readdata');
+            this.#urlQueueStream.emit('readdata');
         });
     }
     /**
@@ -195,8 +195,8 @@ export class SitemapRequestLoader {
      * If this is `false`, the background processing is either still in progress or was aborted.
      */
     isSitemapFullyLoaded() {
-        return (this.sitemapParsingProgress.inProgressSitemapUrl === null &&
-            this.sitemapParsingProgress.pendingSitemapUrls.size === 0);
+        return (this.#sitemapParsingProgress.inProgressSitemapUrl === null &&
+            this.#sitemapParsingProgress.pendingSitemapUrls.size === 0);
     }
     /**
      * Start processing the sitemaps and loading the URLs.
@@ -204,34 +204,34 @@ export class SitemapRequestLoader {
      * Resolves once all the sitemaps URLs have been fully loaded (sets `isSitemapFullyLoaded` to `true`).
      */
     async load({ parseSitemapOptions, }) {
-        while (!this.isSitemapFullyLoaded() && !this.abortLoading) {
-            const sitemapUrl = this.sitemapParsingProgress.inProgressSitemapUrl ??
-                this.sitemapParsingProgress.pendingSitemapUrls.values().next().value;
+        while (!this.isSitemapFullyLoaded() && !this.#abortLoading) {
+            const sitemapUrl = this.#sitemapParsingProgress.inProgressSitemapUrl ??
+                this.#sitemapParsingProgress.pendingSitemapUrls.values().next().value;
             try {
-                for await (const item of parseSitemap([{ type: 'url', url: sitemapUrl }], this.proxyUrl, {
+                for await (const item of parseSitemap([{ type: 'url', url: sitemapUrl }], this.#proxyUrl, {
                     ...parseSitemapOptions,
                     maxDepth: 0,
                     emitNestedSitemaps: true,
                 })) {
                     if (!item.originSitemapUrl) {
                         // This is a nested sitemap
-                        this.sitemapParsingProgress.pendingSitemapUrls.add(item.loc);
+                        this.#sitemapParsingProgress.pendingSitemapUrls.add(item.loc);
                         continue;
                     }
-                    if (!this.sitemapParsingProgress.inProgressEntries.has(item.loc)) {
+                    if (!this.#sitemapParsingProgress.inProgressEntries.has(item.loc)) {
                         await this.pushNextUrl(item.loc);
-                        this.sitemapParsingProgress.inProgressEntries.add(item.loc);
+                        this.#sitemapParsingProgress.inProgressEntries.add(item.loc);
                     }
                 }
             }
             catch (e) {
-                this.log.error('Error loading sitemap contents:', e);
+                this.#log.error('Error loading sitemap contents:', e);
             }
-            this.sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);
-            this.sitemapParsingProgress.inProgressEntries.clear();
-            this.sitemapParsingProgress.inProgressSitemapUrl = null;
+            this.#sitemapParsingProgress.pendingSitemapUrls.delete(sitemapUrl);
+            this.#sitemapParsingProgress.inProgressEntries.clear();
+            this.#sitemapParsingProgress.inProgressSitemapUrl = null;
         }
-        this.urlQueueStream.end();
+        this.#urlQueueStream.end();
     }
     /**
      * Open a sitemap and start processing it.
@@ -250,15 +250,15 @@ export class SitemapRequestLoader {
         void requestList.load({
             parseSitemapOptions: { logger: serviceLocator.getLogger(), ...options.parseSitemapOptions, httpClient },
         });
-        if (requestList.persistenceOptions.enable) {
-            requestList.events.on("persistState" /* EventType.PERSIST_STATE */, requestList.persistState);
+        if (requestList.#persistenceOptions.enable) {
+            requestList.#events.on("persistState" /* EventType.PERSIST_STATE */, requestList.persistState);
         }
         options?.signal?.addEventListener('abort', () => {
-            requestList.abortLoading = true;
+            requestList.#abortLoading = true;
         });
         if (options.timeoutMillis) {
             setTimeout(() => {
-                requestList.abortLoading = true;
+                requestList.#abortLoading = true;
             }, options.timeoutMillis);
         }
         return requestList;
@@ -269,14 +269,14 @@ export class SitemapRequestLoader {
     async getTotalCount() {
         // Total known so far = not-yet-fetched (still buffered in the stream) + in-progress (fetched but not
         // yet handled) + already handled.
-        return this.urlQueueStream.readableLength + this.inProgress.size + this.handledUrlCount;
+        return this.#urlQueueStream.readableLength + this.inProgress.size + this.#handledUrlCount;
     }
     /**
      * @inheritDoc
      */
     async getPendingCount() {
         // Pending = everything not yet handled = not-yet-fetched + in-progress.
-        return this.urlQueueStream.readableLength + this.inProgress.size;
+        return this.#urlQueueStream.readableLength + this.inProgress.size;
     }
     /**
      * Combines this list with a request manager (a {@link RequestQueue} by default) into a
@@ -293,31 +293,31 @@ export class SitemapRequestLoader {
      * @inheritDoc
      */
     async isFinished() {
-        return ((await this.isEmpty()) && this.inProgress.size === 0 && (this.isSitemapFullyLoaded() || this.abortLoading));
+        return ((await this.isEmpty()) && this.inProgress.size === 0 && (this.isSitemapFullyLoaded() || this.#abortLoading));
     }
     /**
      * @inheritDoc
      */
     async isEmpty() {
-        return this.urlQueueStream.readableLength === 0;
+        return this.#urlQueueStream.readableLength === 0;
     }
     /**
      * @inheritDoc
      */
     async getHandledCount() {
-        return this.handledUrlCount;
+        return this.#handledUrlCount;
     }
     /**
      * @inheritDoc
      */
     async persistState() {
-        if (this.persistStateKey === undefined) {
+        if (this.#persistStateKey === undefined) {
             return;
         }
-        this.store ??= await KeyValueStore.open();
+        this.#store ??= await KeyValueStore.open();
         const urlQueue = [];
-        while (this.urlQueueStream.readableLength > 0) {
-            const url = this.urlQueueStream.read();
+        while (this.#urlQueueStream.readableLength > 0) {
+            const url = this.#urlQueueStream.read();
             if (url === null) {
                 break;
             }
@@ -325,48 +325,48 @@ export class SitemapRequestLoader {
         }
         // Create a new stream, as we have read all the URLs from the current one.
         // Pushing the urls back to the original stream might not be possible if it has been ended.
-        const newStream = this.createNewStream(this.urlQueueStream.readableHighWaterMark);
+        const newStream = this.createNewStream(this.#urlQueueStream.readableHighWaterMark);
         for (const url of urlQueue) {
             newStream.push(url);
         }
-        if (this.urlQueueStream.writableEnded) {
+        if (this.#urlQueueStream.writableEnded) {
             newStream.end();
         }
-        this.urlQueueStream = newStream;
-        await this.store.setValue(this.persistStateKey, {
+        this.#urlQueueStream = newStream;
+        await this.#store.setValue(this.#persistStateKey, {
             sitemapParsingProgress: {
-                pendingSitemapUrls: Array.from(this.sitemapParsingProgress.pendingSitemapUrls),
-                inProgressSitemapUrl: this.sitemapParsingProgress.inProgressSitemapUrl,
-                inProgressEntries: Array.from(this.sitemapParsingProgress.inProgressEntries),
+                pendingSitemapUrls: Array.from(this.#sitemapParsingProgress.pendingSitemapUrls),
+                inProgressSitemapUrl: this.#sitemapParsingProgress.inProgressSitemapUrl,
+                inProgressEntries: Array.from(this.#sitemapParsingProgress.inProgressEntries),
             },
             // Re-queue in-progress requests to the front so they are retried if the state is restored.
             urlQueue: [...this.inProgress, ...urlQueue],
-            requestData: Array.from(this.requestData.entries()),
-            abortLoading: this.abortLoading,
-            closed: this.closed,
+            requestData: Array.from(this.#requestData.entries()),
+            abortLoading: this.#abortLoading,
+            closed: this.#closed,
         });
     }
     async restoreState() {
         await purgeDefaultStorages({ onlyPurgeOnce: true });
-        if (this.persistStateKey === undefined) {
+        if (this.#persistStateKey === undefined) {
             return;
         }
-        this.store ??= await KeyValueStore.open();
-        const state = await this.store.getValue(this.persistStateKey);
+        this.#store ??= await KeyValueStore.open();
+        const state = await this.#store.getValue(this.#persistStateKey);
         if (state === null) {
             return;
         }
-        this.sitemapParsingProgress = {
+        this.#sitemapParsingProgress = {
             pendingSitemapUrls: new Set(state.sitemapParsingProgress.pendingSitemapUrls),
             inProgressSitemapUrl: state.sitemapParsingProgress.inProgressSitemapUrl,
             inProgressEntries: new Set(state.sitemapParsingProgress.inProgressEntries),
         };
-        this.requestData = new Map(state.requestData ?? []);
+        this.#requestData = new Map(state.requestData ?? []);
         for (const url of state.urlQueue) {
-            this.urlQueueStream.push(url);
+            this.#urlQueueStream.push(url);
         }
-        this.abortLoading = state.abortLoading;
-        this.closed = state.closed;
+        this.#abortLoading = state.abortLoading;
+        this.#closed = state.closed;
     }
     /**
      * @inheritDoc
@@ -377,11 +377,11 @@ export class SitemapRequestLoader {
             return null;
         }
         // A restored in-progress request already has its Request data; don't overwrite it.
-        if (!this.requestData.has(nextUrl)) {
-            this.requestData.set(nextUrl, new Request({ url: nextUrl }));
+        if (!this.#requestData.has(nextUrl)) {
+            this.#requestData.set(nextUrl, new Request({ url: nextUrl }));
         }
         this.inProgress.add(nextUrl);
-        return this.requestData.get(nextUrl);
+        return this.#requestData.get(nextUrl);
     }
     /**
      * @inheritDoc
@@ -400,20 +400,20 @@ export class SitemapRequestLoader {
      * Calling `fetchNextRequest()` after this method will always return `null`.
      */
     async teardown() {
-        this.closed = true;
-        this.abortLoading = true;
-        this.events.off("persistState" /* EventType.PERSIST_STATE */, this.persistState);
+        this.#closed = true;
+        this.#abortLoading = true;
+        this.#events.off("persistState" /* EventType.PERSIST_STATE */, this.persistState);
         await this.persistState();
-        this.urlQueueStream.emit('readdata'); // unblocks the potentially waiting `pushNextUrl` call
+        this.#urlQueueStream.emit('readdata'); // unblocks the potentially waiting `pushNextUrl` call
     }
     /**
      * @inheritDoc
      */
     async markRequestAsHandled(request) {
-        this.handledUrlCount += 1;
+        this.#handledUrlCount += 1;
         this.ensureInProgress(request.url);
         this.inProgress.delete(request.url);
-        this.requestData.delete(request.url);
+        this.#requestData.delete(request.url);
     }
     ensureInProgress(url) {
         if (!this.inProgress.has(url)) {