@crawlee/basic 4.0.0-beta.211 → 4.0.0-beta.212

package/internals/sitemap_request_loader.d.ts

@@ -95,49 +95,12 @@ export declare class SitemapRequestLoader implements IRequestLoader {
     inProgress: Set<string>;
     /** @internal */
     private constructor();
-    /**
-     * Creates a new object stream with the specified highWaterMark.
-     * @param highWaterMark High water mark for the stream (the maximum number of objects the stream will buffer).
-     * @returns A new object stream.
-     */
-    private createNewStream;
-    /**
-     * Returns a function that checks whether the provided pattern matches the closure URL.
-     * @param url URL to be checked.
-     * @returns A matcher function that checks whether the pattern matches the closure URL.
-     */
-    private matchesUrl;
-    /**
-     * Checks whether the URL matches the `include` / `exclude` patterns provided in the `options`.
-     * @param url URL to be checked.
-     * @returns `true` if the URL matches the patterns, `false` otherwise.
-     */
-    private isUrlMatchingPatterns;
-    /**
-     * Adds a URL to the queue of parsed URLs.
-     *
-     * Blocks if the stream is full until it is drained.
-     */
-    private pushNextUrl;
-    /**
-     * Reads the next URL from the queue of parsed URLs.
-     *
-     * If the stream is empty, blocks until a new URL is pushed.
-     * @returns The next URL from the queue or `null` if we have read all URLs.
-     */
-    private readNextUrl;
     /**
      * Indicates whether the background processing of sitemap contents has successfully finished.
      *
      * If this is `false`, the background processing is either still in progress or was aborted.
      */
     isSitemapFullyLoaded(): boolean;
-    /**
-     * Start processing the sitemaps and loading the URLs.
-     *
-     * Resolves once all the sitemaps URLs have been fully loaded (sets `isSitemapFullyLoaded` to `true`).
-     */
-    private load;
     /**
      * Open a sitemap and start processing it.
      *
@@ -172,7 +135,6 @@ export declare class SitemapRequestLoader implements IRequestLoader {
      * @inheritDoc
      */
     persistState(): Promise<void>;
-    private restoreState;
     /**
      * @inheritDoc
      */
@@ -192,6 +154,5 @@ export declare class SitemapRequestLoader implements IRequestLoader {
      * @inheritDoc
      */
     markRequestAsHandled(request: Request): Promise<void>;
-    private ensureInProgress;
 }
 export {};

package/internals/sitemap_request_loader.js

@@ -107,7 +107,7 @@ export class SitemapRequestLoader {
         this.#persistenceOptions = { enable: true, ...persistenceOptions };
         this.#proxyUrl = proxyUrl;
         this.#enqueueStrategy = enqueueStrategy;
-        this.#urlQueueStream = this.createNewStream(maxBufferSize);
+        this.#urlQueueStream = this.#createNewStream(maxBufferSize);
         this.#sitemapParsingProgress.pendingSitemapUrls = new Set(sitemapUrls);
         this.#events = serviceLocator.getEventManager();
         this.persistState = this.persistState.bind(this);
@@ -117,7 +117,7 @@ export class SitemapRequestLoader {
      * @param highWaterMark High water mark for the stream (the maximum number of objects the stream will buffer).
      * @returns A new object stream.
      */
-    createNewStream(highWaterMark) {
+    #createNewStream(highWaterMark) {
         return new Transform({
             objectMode: true,
             highWaterMark,
@@ -128,7 +128,7 @@ export class SitemapRequestLoader {
      * @param url URL to be checked.
      * @returns A matcher function that checks whether the pattern matches the closure URL.
      */
-    matchesUrl(url) {
+    #matchesUrl(url) {
         return (patternObject) => {
             const { regexp, glob } = patternObject;
             const matchesRegex = (regexp && url.match(regexp)) || false;
@@ -141,18 +141,18 @@ export class SitemapRequestLoader {
      * @param url URL to be checked.
      * @returns `true` if the URL matches the patterns, `false` otherwise.
      */
-    isUrlMatchingPatterns(url) {
-        return (!this.#urlExcludePatternObjects.some(this.matchesUrl(url)) &&
-            (this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(this.matchesUrl(url))));
+    #isUrlMatchingPatterns(url) {
+        return (!this.#urlExcludePatternObjects.some(this.#matchesUrl(url)) &&
+            (this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(this.#matchesUrl(url))));
     }
     /**
      * Adds a URL to the queue of parsed URLs.
      *
      * Blocks if the stream is full until it is drained.
      */
-    async pushNextUrl(url) {
+    async #pushNextUrl(url) {
         return new Promise((resolve) => {
-            if (this.#closed || (url && !this.isUrlMatchingPatterns(url))) {
+            if (this.#closed || (url && !this.#isUrlMatchingPatterns(url))) {
                 resolve();
                 return;
             }
@@ -173,7 +173,7 @@ export class SitemapRequestLoader {
      * If the stream is empty, blocks until a new URL is pushed.
      * @returns The next URL from the queue or `null` if we have read all URLs.
      */
-    async readNextUrl() {
+    async #readNextUrl() {
         return new Promise((resolve) => {
             if (this.#closed) {
                 resolve(null);
@@ -206,7 +206,7 @@ export class SitemapRequestLoader {
      *
      * Resolves once all the sitemaps URLs have been fully loaded (sets `isSitemapFullyLoaded` to `true`).
      */
-    async load({ parseSitemapOptions, }) {
+    async #load({ parseSitemapOptions, }) {
         while (!this.isSitemapFullyLoaded() && !this.#abortLoading) {
             const sitemapUrl = this.#sitemapParsingProgress.inProgressSitemapUrl ??
                 this.#sitemapParsingProgress.pendingSitemapUrls.values().next().value;
@@ -223,7 +223,7 @@ export class SitemapRequestLoader {
                         continue;
                     }
                     if (!this.#sitemapParsingProgress.inProgressEntries.has(item.loc)) {
-                        await this.pushNextUrl(item.loc);
+                        await this.#pushNextUrl(item.loc);
                         this.#sitemapParsingProgress.inProgressEntries.add(item.loc);
                     }
                 }
@@ -250,8 +250,8 @@ export class SitemapRequestLoader {
             ...restOptions,
             persistStateKey: options.persistStateKey ?? STATE_PERSISTENCE_KEY,
         });
-        await requestList.restoreState();
-        void requestList.load({
+        await requestList.#restoreState();
+        void requestList.#load({
             parseSitemapOptions: { logger: serviceLocator.getLogger(), ...options.parseSitemapOptions, httpClient },
         });
         if (requestList.#persistenceOptions.enable) {
@@ -328,7 +328,7 @@ export class SitemapRequestLoader {
         // Create a new stream, as we have read all the URLs from the current one.
         // Pushing the urls back to the original stream might not be possible if it has been ended.
         const previousStream = this.#urlQueueStream;
-        const newStream = this.createNewStream(previousStream.readableHighWaterMark);
+        const newStream = this.#createNewStream(previousStream.readableHighWaterMark);
         for (const url of urlQueue) {
             newStream.push(url);
         }
@@ -355,7 +355,7 @@ export class SitemapRequestLoader {
             closed: this.#closed,
         });
     }
-    async restoreState() {
+    async #restoreState() {
         await purgeDefaultStorages({ onlyPurgeOnce: true });
         if (this.#persistStateKey === undefined) {
             return;
@@ -381,7 +381,7 @@ export class SitemapRequestLoader {
      * @inheritDoc
      */
     async fetchNextRequest() {
-        const nextUrl = await this.readNextUrl();
+        const nextUrl = await this.#readNextUrl();
         if (!nextUrl) {
             return null;
         }
@@ -420,11 +420,11 @@ export class SitemapRequestLoader {
      */
     async markRequestAsHandled(request) {
         this.#handledUrlCount += 1;
-        this.ensureInProgress(request.url);
+        this.#ensureInProgress(request.url);
         this.inProgress.delete(request.url);
         this.#requestData.delete(request.url);
     }
-    ensureInProgress(url) {
+    #ensureInProgress(url) {
         if (!this.inProgress.has(url)) {
             throw new Error(`The request is not being processed (url: ${url})`);
         }

package/internals/throttling_request_manager.d.ts

@@ -153,7 +153,6 @@ export interface ThrottlingRequestManagerOptions<T extends IRequestManager = IRe
  */
 export declare class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager {
     #private;
-    private readonly config;
     constructor(options: ThrottlingRequestManagerOptions<T>, config?: Configuration);
     /**
      * The wrapped manager, holding every request whose domain is not throttled. `undefined` until an `inner`

package/internals/throttling_request_manager.js

@@ -91,7 +91,6 @@ const MAX_INNER_MIGRATIONS = 1000;
  * @category Sources
  */
 export class ThrottlingRequestManager {
-    config;
     #innerFactory;
     #innerPromise;
     #resolvedInner;
@@ -104,6 +103,7 @@ export class ThrottlingRequestManager {
     #throttleBy;
     #maxThrottledDomains;
     #persistStateKey;
+    #config;
     #subManagers = new Map();
     #domainStates = new Map();
     #log;
@@ -142,12 +142,12 @@ export class ThrottlingRequestManager {
         return this.#listedDomains.size > 0 || this.#throttlesEveryDomain;
     }
     constructor(options, config = serviceLocator.getConfiguration()) {
-        this.config = config;
+        this.#config = config;
         parseArgument(options, throttlingRequestManagerOptionsSchema, 'ThrottlingRequestManagerOptions');
         if (options.inner === undefined) {
             // Deferred like any other factory, so a manager nobody fetches from opens no storage. The opener is
             // assigned below and read only when this runs.
-            this.#innerFactory = () => this.#requestManagerOpener(null, { configuration: this.config });
+            this.#innerFactory = () => this.#requestManagerOpener(null, { configuration: this.#config });
         }
         else if (typeof options.inner === 'function') {
             this.#innerFactory = options.inner;
@@ -301,7 +301,7 @@ export class ThrottlingRequestManager {
             subManager = this.#requestManagerOpener(
             // Backends use the alias as a directory name, and an IPv6 literal is full of characters
             // Windows will not accept. Ordinary hostnames survive this untouched.
-            { alias: `throttled-${encodeURIComponent(domain)}` }, { configuration: this.config });
+            { alias: `throttled-${encodeURIComponent(domain)}` }, { configuration: this.#config });
             this.#subManagers.set(domain, subManager);
             this.#ensureDomainState(domain);
         }
@@ -333,7 +333,7 @@ export class ThrottlingRequestManager {
     async #ensureSubManagers() {
         this.#subManagersReady ??= (async () => {
             if (this.#throttlesEveryDomain) {
-                this.#domainListStore = await KeyValueStore.open(null, { configuration: this.config });
+                this.#domainListStore = await KeyValueStore.open(null, { configuration: this.#config });
                 for (const domain of (await this.#domainListStore.getValue(this.#persistStateKey)) ?? []) {
                     this.#discoveredDomains.add(domain);
                 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.211",
+    "version": "4.0.0-beta.212",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/core": "4.0.0-beta.211",
-        "@crawlee/http-client": "4.0.0-beta.211",
-        "@crawlee/types": "4.0.0-beta.211",
-        "@crawlee/utils": "4.0.0-beta.211",
+        "@crawlee/core": "4.0.0-beta.212",
+        "@crawlee/http-client": "4.0.0-beta.212",
+        "@crawlee/types": "4.0.0-beta.212",
+        "@crawlee/utils": "4.0.0-beta.212",
         "@sapphire/async-queue": "^1.5.5",
         "@standard-schema/spec": "^1.0.0",
         "csv-stringify": "^6.5.2",
@@ -57,7 +57,7 @@
         "zod": "^4.5.4"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "4.0.0-beta.211"
+        "@crawlee/impit-client": "4.0.0-beta.212"
     },
     "lerna": {
         "command": {
@@ -66,5 +66,5 @@
             }
         }
     },
-    "gitHead": "17bab8ab5b98554ca2db1d09bc5f61c59194a760"
+    "gitHead": "171a63a5d7f2a6d74cbce58479204ac20d30b87e"
 }