@crawlee/utils 4.0.0-beta.12 → 4.0.0-beta.121

package/internals/sitemap.js

@@ -2,29 +2,30 @@ import { createHash } from 'node:crypto';
 import { PassThrough, pipeline, Readable, Transform } from 'node:stream';
 import { StringDecoder } from 'node:string_decoder';
 import { createGunzip } from 'node:zlib';
-import sax from 'sax';
+import { FetchHttpClient } from '@crawlee/http-client';
 import MIMEType from 'whatwg-mimetype';
-import log from '@apify/log';
+import { mergeAsyncIterables } from './iterables.js';
+import { RobotsTxtFile } from './robots.js';
 class SitemapTxtParser extends Transform {
-    decoder = new StringDecoder('utf8');
-    buffer = '';
+    #decoder = new StringDecoder('utf8');
+    #buffer = '';
     constructor() {
         super({
             readableObjectMode: true,
             transform: (chunk, _encoding, callback) => {
-                this.processBuffer(this.decoder.write(chunk), false);
+                this.processBuffer(this.#decoder.write(chunk), false);
                 callback();
             },
             flush: (callback) => {
-                this.processBuffer(this.decoder.end(), true);
+                this.processBuffer(this.#decoder.end(), true);
                 callback();
             },
         });
     }
     processBuffer(input, finalize) {
-        this.buffer += input;
-        if (finalize || this.buffer.includes('\n')) {
-            const parts = this.buffer
+        this.#buffer += input;
+        if (finalize || this.#buffer.includes('\n')) {
+            const parts = this.#buffer
                 .split('\n')
                 .map((part) => part.trim())
                 .filter((part) => part.length > 0);
@@ -32,101 +33,109 @@ class SitemapTxtParser extends Transform {
                 for (const url of parts) {
                     this.push({ type: 'url', loc: url });
                 }
-                this.buffer = '';
+                this.#buffer = '';
             }
             else if (parts.length > 0) {
                 for (const url of parts.slice(0, -1)) {
                     this.push({ type: 'url', loc: url });
                 }
-                this.buffer = parts.at(-1);
+                this.#buffer = parts.at(-1);
             }
         }
     }
 }
 class SitemapXmlParser extends Transform {
-    decoder = new StringDecoder('utf8');
-    parser = new sax.SAXParser(true);
-    rootTagName;
-    currentTag = undefined;
-    url = {};
-    constructor() {
+    #decoder = new StringDecoder('utf8');
+    #parser;
+    #rootTagName;
+    #currentTag = undefined;
+    #url = {};
+    static async create() {
+        const { SAXParser } = await import('sax');
+        return new SitemapXmlParser(new SAXParser(true));
+    }
+    constructor(parser) {
         super({
             readableObjectMode: true,
             transform: (chunk, _encoding, callback) => {
-                this.parser.write(this.decoder.write(chunk));
+                this.#parser.write(this.#decoder.write(chunk));
                 callback();
             },
             flush: (callback) => {
-                const rest = this.decoder.end();
+                const rest = this.#decoder.end();
                 if (rest.length > 0) {
-                    this.parser.write(rest);
+                    this.#parser.write(rest);
                 }
-                this.parser.end();
+                this.#parser.end();
                 callback();
             },
         });
-        this.parser.onopentag = this.onOpenTag.bind(this);
-        this.parser.onclosetag = this.onCloseTag.bind(this);
-        this.parser.ontext = this.onText.bind(this);
-        this.parser.oncdata = this.onText.bind(this);
-        this.parser.onerror = this.destroy.bind(this);
+        this.#parser = parser;
+        this.#parser.onopentag = this.onOpenTag.bind(this);
+        this.#parser.onclosetag = this.onCloseTag.bind(this);
+        this.#parser.ontext = this.onText.bind(this);
+        this.#parser.oncdata = this.onText.bind(this);
+        this.#parser.onerror = this.destroy.bind(this);
     }
     onOpenTag(node) {
-        if (this.rootTagName !== undefined) {
+        if (this.#rootTagName !== undefined) {
             if (node.name === 'loc' ||
                 node.name === 'lastmod' ||
                 node.name === 'priority' ||
                 node.name === 'changefreq') {
-                this.currentTag = node.name;
+                this.#currentTag = node.name;
             }
         }
         if (node.name === 'urlset') {
-            this.rootTagName = 'urlset';
+            this.#rootTagName = 'urlset';
         }
         if (node.name === 'sitemapindex') {
-            this.rootTagName = 'sitemapindex';
+            this.#rootTagName = 'sitemapindex';
         }
     }
     onCloseTag(name) {
         if (name === 'loc' || name === 'lastmod' || name === 'priority' || name === 'changefreq') {
-            this.currentTag = undefined;
+            this.#currentTag = undefined;
         }
-        if (name === 'url' && this.url.loc !== undefined) {
-            this.push({ type: 'url', ...this.url, loc: this.url.loc });
-            this.url = {};
+        if (name === 'url') {
+            if (this.#url.loc !== undefined) {
+                this.push({ type: 'url', ...this.#url, loc: this.#url.loc });
+            }
+            this.#url = {};
         }
     }
     onText(text) {
-        if (this.currentTag === 'loc') {
-            if (this.rootTagName === 'sitemapindex') {
+        if (this.#currentTag === 'loc') {
+            if (this.#rootTagName === 'sitemapindex') {
                 this.push({ type: 'sitemapUrl', url: text.trim() });
             }
-            if (this.rootTagName === 'urlset') {
-                this.url ??= {};
-                this.url.loc = text.trim();
+            if (this.#rootTagName === 'urlset') {
+                this.#url ??= {};
+                this.#url.loc = text.trim();
             }
         }
         text = text.trim();
-        if (this.currentTag === 'lastmod') {
-            this.url.lastmod = new Date(text);
+        if (this.#currentTag === 'lastmod') {
+            const lastmod = new Date(text);
+            if (!Number.isNaN(lastmod.getTime())) {
+                this.#url.lastmod = lastmod;
+            }
         }
-        if (this.currentTag === 'priority') {
-            this.url.priority = Number(text);
+        if (this.#currentTag === 'priority') {
+            this.#url.priority = Number(text);
         }
-        if (this.currentTag === 'changefreq') {
+        if (this.#currentTag === 'changefreq') {
             if (['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'].includes(text)) {
-                this.url.changefreq = text;
+                this.#url.changefreq = text;
             }
         }
     }
 }
 export async function* parseSitemap(initialSources, proxyUrl, options) {
-    const { gotScraping } = await import('got-scraping');
-    const { fileTypeStream } = await import('file-type');
-    const { emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, networkTimeouts } = options ?? {};
+    const { httpClient = new FetchHttpClient(), emitNestedSitemaps = false, maxDepth = Infinity, sitemapRetries = 3, timeoutMillis: timeout = 30000, reportNetworkErrors = true, nestedSitemapFilter, logger, } = options ?? {};
     const sources = [...initialSources];
     const visitedSitemapUrls = new Set();
-    const createParser = (contentType = '', url) => {
+    const createParser = async (contentType = '', url) => {
         let mimeType;
         try {
             mimeType = new MIMEType(contentType);
@@ -135,7 +144,7 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
             mimeType = null;
         }
         if (mimeType?.isXML() || url?.pathname.endsWith('.xml')) {
-            return new SitemapXmlParser();
+            return SitemapXmlParser.create();
         }
         if (mimeType?.essence === 'text/plain' || url?.pathname.endsWith('.txt')) {
             return new SitemapTxtParser();
@@ -145,7 +154,6 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
     while (sources.length > 0) {
         const source = sources.shift();
         if ((source?.depth ?? 0) > maxDepth) {
-            log.debug(`Skipping sitemap ${source.type === 'url' ? source.url : ''} because it reached max depth ${maxDepth}.`);
             continue;
         }
         let items = null;
@@ -155,23 +163,29 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
             let retriesLeft = sitemapRetries + 1;
             while (retriesLeft-- > 0) {
                 try {
-                    const sitemapStream = await new Promise((resolve, reject) => {
-                        const request = gotScraping.stream({
-                            url: sitemapUrl,
-                            proxyUrl,
+                    let sitemapResponse;
+                    try {
+                        sitemapResponse = await httpClient.sendRequest(new Request(sitemapUrl, {
                             method: 'GET',
-                            timeout: networkTimeouts,
                             headers: {
-                                accept: 'text/plain, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8',
+                                accept: '*/*',
                             },
+                        }), {
+                            proxyUrl,
+                            timeoutMillis: timeout,
                         });
-                        request.on('response', () => resolve(request));
-                        request.on('error', reject);
-                    });
+                    }
+                    catch (error) {
+                        sitemapResponse = null;
+                    }
                     let error = null;
-                    if (sitemapStream.response.statusCode >= 200 && sitemapStream.response.statusCode < 300) {
-                        let contentType = sitemapStream.response.headers['content-type'];
-                        const streamWithType = await fileTypeStream(sitemapStream);
+                    if (sitemapResponse && sitemapResponse.status >= 200 && sitemapResponse.status < 300) {
+                        let contentType = sitemapResponse.headers.get('content-type');
+                        if (sitemapResponse.body === null) {
+                            break;
+                        }
+                        const { fileTypeStream } = await import('file-type');
+                        const streamWithType = await fileTypeStream(Readable.fromWeb(sitemapResponse.body));
                         if (streamWithType.fileType !== undefined) {
                             contentType = streamWithType.fileType.mime;
                         }
@@ -184,29 +198,37 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
                                 sitemapUrl.pathname = sitemapUrl.pathname.substring(0, sitemapUrl.pathname.length - 3);
                             }
                         }
-                        items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), createParser(contentType, sitemapUrl), (e) => {
-                            if (e !== undefined) {
-                                error = e;
+                        items = pipeline(streamWithType, isGzipped ? createGunzip() : new PassThrough(), await createParser(contentType ?? undefined, sitemapUrl), (e) => {
+                            if (e !== undefined && e !== null) {
+                                error = { type: 'parser', error: e };
                             }
                         });
                     }
                     else {
-                        error = new Error(`Failed to fetch sitemap: ${sitemapUrl}, status code: ${sitemapStream.response.statusCode}`);
+                        error = {
+                            type: 'fetch',
+                            error: new Error(`Failed to fetch sitemap: ${sitemapUrl}, status code: ${sitemapResponse?.status}`),
+                        };
                     }
                     if (error !== null) {
-                        throw error;
+                        const shouldIgnoreError = error.type === 'fetch' && !reportNetworkErrors;
+                        if (!shouldIgnoreError) {
+                            throw error.error;
+                        }
+                    }
+                    else {
+                        break;
                     }
-                    break;
                 }
                 catch (e) {
-                    log.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
+                    logger?.warning(`Malformed sitemap content: ${sitemapUrl}, ${retriesLeft === 0 ? 'no retries left.' : 'retrying...'} (${e})`);
                 }
             }
         }
         else if (source.type === 'raw') {
-            items = pipeline(Readable.from([source.content]), createParser('text/xml'), (error) => {
+            items = pipeline(Readable.from([source.content]), await createParser('text/xml'), (error) => {
                 if (error !== undefined) {
-                    log.warning(`Malformed sitemap content: ${error}`);
+                    logger?.warning(`Malformed sitemap content: ${error}`);
                 }
             });
         }
@@ -215,6 +237,10 @@ export async function* parseSitemap(initialSources, proxyUrl, options) {
         }
         for await (const item of items) {
             if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
+                if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
+                    logger?.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
+                    continue;
+                }
                 sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
                 if (emitNestedSitemaps) {
                     yield { loc: item.url, originSitemapUrl: null };
@@ -254,7 +280,7 @@ export class Sitemap {
      * @param url The domain URL to fetch the sitemap for.
      * @param proxyUrl A proxy to be used for fetching the sitemap file.
      */
-    static async tryCommonNames(url, proxyUrl) {
+    static async tryCommonNames(url, proxyUrl, parseSitemapOptions) {
         const sitemapUrls = [];
         const sitemapUrl = new URL(url);
         sitemapUrl.search = '';
@@ -262,7 +288,7 @@ export class Sitemap {
         sitemapUrls.push(sitemapUrl.toString());
         sitemapUrl.pathname = '/sitemap.txt';
         sitemapUrls.push(sitemapUrl.toString());
-        return Sitemap.load(sitemapUrls, proxyUrl);
+        return Sitemap.load(sitemapUrls, proxyUrl, { reportNetworkErrors: false, ...parseSitemapOptions });
     }
     /**
      * Fetch sitemap content from given URL or URLs and return URLs of referenced pages.
@@ -277,8 +303,8 @@ export class Sitemap {
      * @param content XML sitemap content
      * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
      */
-    static async fromXmlString(content, proxyUrl) {
-        return await this.parse([{ type: 'raw', content }], proxyUrl);
+    static async fromXmlString(content, proxyUrl, parseSitemapOptions) {
+        return await this.parse([{ type: 'raw', content }], proxyUrl, parseSitemapOptions);
     }
     static async parse(sources, proxyUrl, parseSitemapOptions) {
         const urls = [];
@@ -287,10 +313,128 @@ export class Sitemap {
                 urls.push(item.loc);
             }
         }
-        catch {
+        catch (e) {
+            parseSitemapOptions?.logger?.warning(`Sitemap.load: Failed to load sitemap, returning empty result. (${e})`);
             return new Sitemap([]);
         }
         return new Sitemap(urls);
     }
 }
-//# sourceMappingURL=sitemap.js.map
+/**
+ * Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,
+ * the default `sitemap.xml` & `sitemap.txt` files and the URLs themselves.
+ * @param `urls` The list of URLs to discover sitemaps for.
+ * @param `options` Options for sitemap discovery
+ * @returns An async iterable with the discovered sitemap URLs.
+ */
+export async function* discoverValidSitemaps(urls, options = {}) {
+    const { proxyUrl, timeoutMillis = 60_000, signal: externalSignal, requestTimeoutMillis = 20_000, httpClient = new FetchHttpClient(), logger, } = options;
+    const controller = new AbortController();
+    const timeoutHandle = setTimeout(() => controller.abort(), timeoutMillis);
+    const onExternalAbort = () => controller.abort();
+    if (externalSignal) {
+        if (externalSignal.aborted) {
+            controller.abort();
+        }
+        else {
+            externalSignal.addEventListener('abort', onExternalAbort, { once: true });
+        }
+    }
+    const signal = controller.signal;
+    const sitemapUrls = new Set();
+    const addSitemapUrl = (url) => {
+        const sizeBefore = sitemapUrls.size;
+        sitemapUrls.add(url);
+        if (sitemapUrls.size > sizeBefore) {
+            return url;
+        }
+        return undefined;
+    };
+    const urlExists = async (url) => {
+        if (!httpClient) {
+            return false;
+        }
+        try {
+            const response = await httpClient.sendRequest(new Request(url, { method: 'HEAD' }), {
+                proxyUrl,
+                timeoutMillis: requestTimeoutMillis,
+                signal,
+            });
+            return response.status >= 200 && response.status < 400;
+        }
+        catch {
+            return false;
+        }
+    };
+    const discoverSitemapsForDomainUrls = async function* (hostname, domainUrls) {
+        if (!hostname) {
+            return;
+        }
+        try {
+            const robotsFile = await RobotsTxtFile.find(domainUrls[0], {
+                proxyUrl,
+                timeoutMillis: requestTimeoutMillis,
+                signal,
+                httpClient,
+                logger,
+            });
+            for (const sitemapUrl of robotsFile.getSitemaps()) {
+                if (addSitemapUrl(sitemapUrl)) {
+                    yield sitemapUrl;
+                }
+            }
+        }
+        catch (err) {
+            logger?.warning(`Failed to fetch robots.txt file for ${hostname}`, { error: err });
+        }
+        const sitemapUrl = domainUrls.find((url) => /sitemap\.(?:xml|txt)(?:\.gz)?$/i.test(url));
+        if (sitemapUrl !== undefined) {
+            if (addSitemapUrl(sitemapUrl)) {
+                yield sitemapUrl;
+            }
+        }
+        else {
+            const firstUrl = new URL(domainUrls[0]);
+            const possibleSitemapPathnames = ['/sitemap.xml', '/sitemap.txt', '/sitemap_index.xml'];
+            const candidateSitemapUrls = possibleSitemapPathnames.map((pathname) => {
+                firstUrl.pathname = pathname;
+                return firstUrl.toString();
+            });
+            const candidateResults = await Promise.allSettled(candidateSitemapUrls.map(urlExists));
+            for (const [index, result] of candidateResults.entries()) {
+                const candidateSitemapUrl = candidateSitemapUrls[index];
+                if (result.status === 'fulfilled') {
+                    if (result.value && addSitemapUrl(candidateSitemapUrl)) {
+                        yield candidateSitemapUrl;
+                    }
+                }
+                else {
+                    logger?.debug(`Failed to check sitemap candidate ${candidateSitemapUrl} for ${hostname}`, {
+                        error: result.reason,
+                    });
+                }
+            }
+        }
+    };
+    const groupedUrls = urls.reduce((acc, url) => {
+        const hostname = new URL(url)?.hostname ?? '';
+        acc[hostname] ??= [];
+        acc[hostname].push(url);
+        return acc;
+    }, {});
+    const iterables = Object.entries(groupedUrls).map(([hostname, domainUrls]) => discoverSitemapsForDomainUrls(hostname, domainUrls));
+    const discoveredUrls = new Set();
+    try {
+        for await (const url of mergeAsyncIterables(...iterables)) {
+            if (discoveredUrls.has(url)) {
+                continue;
+            }
+            discoveredUrls.add(url);
+            yield url;
+        }
+    }
+    finally {
+        clearTimeout(timeoutHandle);
+        externalSignal?.removeEventListener('abort', onExternalAbort);
+    }
+}

package/internals/social.d.ts

@@ -488,5 +488,4 @@ export declare const DISCORD_REGEX_GLOBAL: RegExp;
  *   so that the caller doesn't need to parse the HTML document again, if needed.
  * @return An object with the social handles.
  */
-export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): SocialHandles;
-//# sourceMappingURL=social.d.ts.map
+export declare function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): Promise<SocialHandles>;

package/internals/social.js

@@ -1,7 +1,9 @@
-import * as cheerio from 'cheerio';
 import { htmlToText } from './cheerio.js';
 // Regex inspired by https://zapier.com/blog/extract-links-email-phone-regex/
-const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
+// The dot-atom local part and domain labels use RFC 5321 length bounds ({1,64}, {0,62})
+// instead of unbounded quantifiers to avoid quadratic backtracking (ReDoS) on long
+// dotted or hyphenated inputs, e.g. text scraped by parseHandlesFromHtml().
+const EMAIL_REGEX_STRING = '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]{1,64}){0,32}|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\\])';
 /**
  * Regular expression to exactly match a single email address.
  * It has the following form: `/^...$/i`.
@@ -587,7 +589,8 @@ export const DISCORD_REGEX_GLOBAL = new RegExp(DISCORD_REGEX_STRING, 'ig');
  *   so that the caller doesn't need to parse the HTML document again, if needed.
  * @return An object with the social handles.
  */
-export function parseHandlesFromHtml(html, data = null) {
+export async function parseHandlesFromHtml(html, data = null) {
+    const cheerio = await import('cheerio');
     const result = {
         emails: [],
         phones: [],
@@ -606,7 +609,7 @@ export function parseHandlesFromHtml(html, data = null) {
     const $ = cheerio.load(html, { xml: { decodeEntities: true } });
     if (data)
         data.$ = $;
-    const text = htmlToText($);
+    const text = await htmlToText($);
     if (data)
         data.text = text;
     // NOTE: we need to parse each text separately, orherwise we might concatenate unrelated texts
@@ -644,4 +647,3 @@ export function parseHandlesFromHtml(html, data = null) {
     }
     return result;
 }
-//# sourceMappingURL=social.js.map

package/internals/url.d.ts

@@ -1,4 +1,4 @@
-export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;
+import type { SearchParams } from '@crawlee/types';
 /**
  * Appends search (query string) parameters to a URL, replacing the original value (if any).
  *
@@ -7,4 +7,3 @@ export type SearchParams = string | URLSearchParams | Record<string, string | nu
  * @internal
  */
 export declare function applySearchParams(url: URL, searchParams: SearchParams | undefined): void;
-//# sourceMappingURL=url.d.ts.map

package/internals/url.js

@@ -19,7 +19,7 @@ export function applySearchParams(url, searchParams) {
     }
     else {
         newSearchParams = new URLSearchParams();
-        for (const [key, value] of Object.entries(newSearchParams)) {
+        for (const [key, value] of Object.entries(searchParams)) {
             if (value === undefined) {
                 newSearchParams.delete(key);
             }
@@ -33,4 +33,3 @@ export function applySearchParams(url, searchParams) {
     }
     url.search = newSearchParams.toString();
 }
-//# sourceMappingURL=url.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/utils",
-    "version": "4.0.0-beta.12",
+    "version": "4.0.0-beta.121",
     "description": "A set of shared utilities that can be used by crawlers",
     "engines": {
         "node": ">=22.0.0"
@@ -8,6 +8,7 @@
     "type": "module",
     "exports": {
         ".": "./index.js",
+        "./internal": "./internal.js",
         "./package.json": "./package.json"
     },
     "keywords": [
@@ -35,19 +36,19 @@
     },
     "homepage": "https://crawlee.dev",
     "scripts": {
-        "build": "yarn clean && yarn compile && yarn copy",
+        "build": "pnpm clean && pnpm compile && pnpm copy",
         "clean": "rimraf ./dist",
         "compile": "tsc -p tsconfig.build.json",
         "copy": "tsx ../../scripts/copy.ts"
     },
     "dependencies": {
-        "@apify/log": "^2.5.18",
         "@apify/ps-tree": "^1.2.0",
-        "@crawlee/types": "4.0.0-beta.12",
+        "@crawlee/http-client": "4.0.0-beta.121",
+        "@crawlee/types": "4.0.0-beta.121",
         "@types/sax": "^1.2.7",
         "cheerio": "^1.0.0",
+        "domhandler": "^5.0.3",
         "file-type": "^21.0.0",
-        "got-scraping": "^4.1.1",
         "ow": "^2.0.0",
         "robots-parser": "^3.0.1",
         "sax": "^1.4.1",
@@ -61,5 +62,5 @@
             }
         }
     },
-    "gitHead": "06431733e5cfa6cc3a4c105427592f87424037ca"
+    "gitHead": "5027317de626f5ba6de5047ae9341a898258cc5a"
 }

package/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,OAAO,KAAK,MAAM,MAAM,uBAAuB,CAAC;AAChD,cAAc,yBAAyB,CAAC;AACxC,cAAc,kCAAkC,CAAC;AACjD,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AAEnC,OAAO,EAAE,oBAAoB,EAAE,SAAS,EAAE,MAAM,qCAAqC,CAAC;AACtF,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,wCAAwC,CAAC;AAEnF,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC"}

package/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,OAAO,KAAK,MAAM,MAAM,uBAAuB,CAAC;AAChD,cAAc,yBAAyB,CAAC;AACxC,cAAc,kCAAkC,CAAC;AACjD,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AAEnC,OAAO,EAAE,oBAAoB,EAAa,MAAM,qCAAqC,CAAC;AACtF,OAAO,EAAE,aAAa,EAAc,MAAM,wCAAwC,CAAC"}

package/internals/blocked.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"blocked.d.ts","sourceRoot":"","sources":["../../src/internals/blocked.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,8BAA8B,UAA0E,CAAC;AAEtH;;GAEG;AACH,eAAO,MAAM,mBAAmB,UAI/B,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,mBAAmB,UAM/B,CAAC"}

package/internals/blocked.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"blocked.js","sourceRoot":"","sources":["../../src/internals/blocked.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC,qEAAqE,CAAC,CAAC;AAEtH;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IAC/B,GAAG,8BAA8B;IACjC,0DAA0D;IAC1D,oCAAoC;CACvC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IAC/B,YAAY;IACZ,cAAc;IACd,6BAA6B;IAC7B,8BAA8B;IAC9B,sBAAsB;CACzB,CAAC"}

package/internals/cheerio.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"cheerio.d.ts","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAK1C,MAAM,MAAM,WAAW,GAAG,UAAU,CAAC;AAOrC;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,UAAU,CAAC,oBAAoB,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,CAwC7E;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,SAAM,EAAE,OAAO,SAAK,GAAG,MAAM,EAAE,CAwB5F"}

package/internals/cheerio.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAInD,gIAAgI;AAChI,MAAM,eAAe,GAAG,uCAAuC,CAAC;AAChE,MAAM,gBAAgB,GAClB,sGAAsG,CAAC;AAE3G;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,UAAU,CAAC,oBAA0C;IACjE,IAAI,CAAC,oBAAoB;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,CAAC,GAAG,OAAO,oBAAoB,KAAK,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACjH,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,MAAM,OAAO,GAAG,CAAC,KAAiB,EAAE,EAAE;QAClC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACvB,qDAAqD;gBACrD,IAAI,KAAK,CAAC;gBACV,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK;oBAAE,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC;;oBAC/D,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC5C,+EAA+E;gBAC/E,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC9E,IAAI,IAAI,KAAK,CAAC;YAClB,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvE,qCAAqC;YACzC,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC/B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACJ,2EAA2E;gBAC3E,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvD,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;gBACtD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;YACzD,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,kFAAkF;IAClF,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;IACxB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAE7C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CAAC,CAAa,EAAE,QAAQ,GAAG,GAAG,EAAE,OAAO,GAAG,EAAE;IAC9E,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,eAAe,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAE9D,IAAI,eAAe,EAAE,CAAC;QAClB,OAAO,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,CAAC,QAAQ,CAAC;SACb,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACnC,GAAG,EAAE;SACL,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACV,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACX,qBAAqB,IAAI,uCAAuC;gBAC5D,2DAA2D,CAClE,CAAC;QACN,CAAC;QACD,OAAO,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1D,CAAC,CAAC;SACD,MAAM,CAAC,OAAO,CAAa,CAAC;AACrC,CAAC"}

package/internals/chunk.d.ts

@@ -1,2 +0,0 @@
-export declare function chunk<T>(array: readonly T[], chunkSize: number): T[][];
-//# sourceMappingURL=chunk.d.ts.map

package/internals/chunk.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../../src/internals/chunk.ts"],"names":[],"mappings":"AA2BA,wBAAgB,KAAK,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,EAAE,EAAE,CAQtE"}