@crawlee/utils 4.0.0-beta.211 → 4.0.0-beta.212

package/internals/robots.d.ts

@@ -53,7 +53,6 @@ export declare class RobotsTxtFile {
      * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
      */
     static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
-    private static load;
     /**
      * Get crawl delay for a given user agent.
      * @param [userAgent] relevant user agent, default to `*`

package/internals/robots.js

@@ -43,7 +43,7 @@ export class RobotsTxtFile {
         const robotsTxtFileUrl = new URL(url);
         robotsTxtFileUrl.pathname = '/robots.txt';
         robotsTxtFileUrl.search = '';
-        return RobotsTxtFile.load(robotsTxtFileUrl.toString(), options);
+        return RobotsTxtFile.#load(robotsTxtFileUrl.toString(), options);
     }
     /**
      * Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
@@ -55,7 +55,7 @@ export class RobotsTxtFile {
         // @ts-ignore
         return new RobotsTxtFile(url, robotsParser(url, content), proxyUrl);
     }
-    static async load(url, options) {
+    static async #load(url, options) {
         const { proxyUrl, logger, httpClient = new FetchHttpClient() } = options || {};
         const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
             proxyUrl,

package/internals/sitemap.d.ts

@@ -82,6 +82,7 @@ export declare function parseSitemap<T extends ParseSitemapOptions>(initialSourc
  * ```
  */
 export declare class Sitemap {
+    #private;
     readonly urls: string[];
     constructor(urls: string[]);
     /**
@@ -103,7 +104,6 @@ export declare class Sitemap {
      * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
      */
     static fromXmlString(content: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
-    private static parse;
 }
 /**
  * Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,

package/internals/sitemap.js

@@ -14,16 +14,16 @@ class SitemapTxtParser extends Transform {
         super({
             readableObjectMode: true,
             transform: (chunk, _encoding, callback) => {
-                this.processBuffer(this.#decoder.write(chunk), false);
+                this.#processBuffer(this.#decoder.write(chunk), false);
                 callback();
             },
             flush: (callback) => {
-                this.processBuffer(this.#decoder.end(), true);
+                this.#processBuffer(this.#decoder.end(), true);
                 callback();
             },
         });
     }
-    processBuffer(input, finalize) {
+    #processBuffer(input, finalize) {
         this.#buffer += input;
         if (finalize || this.#buffer.includes('\n')) {
             const parts = this.#buffer
@@ -52,7 +52,7 @@ class SitemapXmlParser extends Transform {
     #currentTag = undefined;
     #url = {};
     static async create() {
-        const { SAXParser } = await import('sax');
+        const { SAXParser } = (await import('sax')).default;
         return new SitemapXmlParser(new SAXParser(true));
     }
     constructor(parser) {
@@ -72,13 +72,13 @@ class SitemapXmlParser extends Transform {
             },
         });
         this.#parser = parser;
-        this.#parser.onopentag = this.onOpenTag.bind(this);
-        this.#parser.onclosetag = this.onCloseTag.bind(this);
-        this.#parser.ontext = this.onText.bind(this);
-        this.#parser.oncdata = this.onText.bind(this);
+        this.#parser.onopentag = this.#onOpenTag.bind(this);
+        this.#parser.onclosetag = this.#onCloseTag.bind(this);
+        this.#parser.ontext = this.#onText.bind(this);
+        this.#parser.oncdata = this.#onText.bind(this);
         this.#parser.onerror = this.destroy.bind(this);
     }
-    onOpenTag(node) {
+    #onOpenTag(node) {
         if (this.#rootTagName !== undefined) {
             if (node.name === 'loc' ||
                 node.name === 'lastmod' ||
@@ -94,7 +94,7 @@ class SitemapXmlParser extends Transform {
             this.#rootTagName = 'sitemapindex';
         }
     }
-    onCloseTag(name) {
+    #onCloseTag(name) {
         if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
             this.#currentTag = undefined;
         }
@@ -105,7 +105,7 @@ class SitemapXmlParser extends Transform {
             this.#url = {};
         }
     }
-    onText(text) {
+    #onText(text) {
         if (this.#currentTag === 'loc') {
             if (this.#rootTagName === 'sitemapindex') {
                 this.push({ type: 'sitemapUrl', url: text.trim() });
@@ -323,7 +323,7 @@ export class Sitemap {
      * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
      */
     static async load(urls, proxyUrl, parseSitemapOptions) {
-        return await this.parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
+        return await this.#parse((Array.isArray(urls) ? urls : [urls]).map((url) => ({ type: 'url', url })), proxyUrl, parseSitemapOptions);
     }
     /**
      * Parse XML sitemap content from a string and return URLs of referenced pages. If the sitemap references other sitemaps, they will be loaded via HTTP.
@@ -331,9 +331,9 @@ export class Sitemap {
      * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
      */
     static async fromXmlString(content, proxyUrl, parseSitemapOptions) {
-        return await this.parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
+        return await this.#parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
     }
-    static async parse(sources, proxyUrl, parseSitemapOptions) {
+    static async #parse(sources, proxyUrl, parseSitemapOptions) {
         const urls = [];
         try {
             for await (const item of parseSitemap(sources, proxyUrl, parseSitemapOptions)) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/utils",
-    "version": "4.0.0-beta.211",
+    "version": "4.0.0-beta.212",
     "description": "A set of shared utilities that can be used by crawlers",
     "engines": {
         "node": ">=22.0.0"
@@ -44,8 +44,8 @@
     "dependencies": {
         "@apify/ps-tree": "^1.2.0",
         "@apify/validations": "^1.1.0",
-        "@crawlee/http-client": "4.0.0-beta.211",
-        "@crawlee/types": "4.0.0-beta.211",
+        "@crawlee/http-client": "4.0.0-beta.212",
+        "@crawlee/types": "4.0.0-beta.212",
         "@types/sax": "^1.2.7",
         "cheerio": "^1.0.0",
         "domhandler": "^5.0.3",
@@ -64,5 +64,5 @@
             }
         }
     },
-    "gitHead": "17bab8ab5b98554ca2db1d09bc5f61c59194a760"
+    "gitHead": "171a63a5d7f2a6d74cbce58479204ac20d30b87e"
 }