@crawlee/utils 4.0.0-beta.112 → 4.0.0-beta.114

package/internals/sitemap.js

@@ -3,7 +3,6 @@ import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
 import { StringDecoder } from 'node:string_decoder';
 import { createGunzip } from 'node:zlib';
 import { FetchHttpClient } from '@crawlee/http-client';
-import sax from 'sax';
 import MIMEType from 'whatwg-mimetype';
 import { mergeAsyncIterables } from './iterables.js';
 import { RobotsTxtFile } from './robots.js';
@@ -47,11 +46,15 @@ class SitemapTxtParser extends Transform {
 }
 class SitemapXmlParser extends Transform {
     #decoder = new StringDecoder('utf8');
-    #parser = new sax.SAXParser(true);
+    #parser;
     #rootTagName;
     #currentTag = undefined;
     #url = {};
-    constructor() {
+    static async create() {
+        const { SAXParser } = await import('sax');
+        return new SitemapXmlParser(new SAXParser(true));
+    }
+    constructor(parser) {
         super({
             readableObjectMode: true,
             transform: (chunk, _encoding, callback) => {
@@ -67,6 +70,7 @@ class SitemapXmlParser extends Transform {
                 callback();
             },
         });
+        this.#parser = parser;
         this.#parser.onopentag = this.onOpenTag.bind(this);
         this.#parser.onclosetag = this.onCloseTag.bind(this);
         this.#parser.ontext = this.onText.bind(this);
@@ -131,7 +135,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
     const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, logger, } = options ?? {};
     const sources = [...initialSources];
     const visitedSitemapUrls = new Set();
-    const createParser = (contentType = '', url) => {
+    const createParser = async (contentType = '', url) => {
         let mimeType;
         try {
             mimeType = new MIMEType(contentType);
@@ -140,7 +144,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
             mimeType = null;
         }
         if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {
-            return new SitemapXmlParser();
+            return SitemapXmlParser.create();
         }
         if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
             return new SitemapTxtParser();
@@ -194,7 +198,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
                                 sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
                             }
                         }
-                        items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType ?? undefined, sitemapUrl), (e) => {
+                        items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), await createParser(contentType ?? undefined, sitemapUrl), (e) => {
                             if (e !== undefined && e !== null) {
                                 error = { type: 'parser', error: e };
                             }
@@ -222,7 +226,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
             }
         }
         else if (source.type === 'raw') {
-            items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {
+            items = pipeline(Readable.from([source.content]), await createParser('text/xml'), (error) => {
                 if (error !== undefined) {
                     logger?.warning(`Malformed sitemap content: ${error}`);
                 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/utils",
-    "version": "4.0.0-beta.112",
+    "version": "4.0.0-beta.114",
     "description": "A set of shared utilities that can be used by crawlers",
     "engines": {
         "node": ">=22.0.0"
@@ -43,8 +43,8 @@
     },
     "dependencies": {
         "@apify/ps-tree": "^1.2.0",
-        "@crawlee/http-client": "4.0.0-beta.112",
-        "@crawlee/types": "4.0.0-beta.112",
+        "@crawlee/http-client": "4.0.0-beta.114",
+        "@crawlee/types": "4.0.0-beta.114",
         "@types/sax": "^1.2.7",
         "cheerio": "^1.0.0",
         "domhandler": "^5.0.3",
@@ -62,5 +62,5 @@
             }
         }
     },
-    "gitHead": "5d1e107d4dc4ffd21f1c6cc5b1a1c10b59ca2b47"
+    "gitHead": "e00fd1671498a90451e62730d58cae1f5d89f2be"
 }