@crawlee/utils 4.0.0-beta.10 → 4.0.0-beta.101

package/README.md

@@ -1,19 +1,23 @@
 <h1 align="center">
     <a href="https://crawlee.dev">
         <picture>
-          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
-          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
         </picture>
     </a>
-    <br>
+    <br />
     <small>A web scraping and browser automation library</small>
 </h1>
 
-<p align=center>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
-    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+<p align="center">
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
+</p>
+
+<p align="center">
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
 
-> Crawlee for Python is open for early adopters. 🐍  [👉 Checkout the source code 👈](https://github.com/apify/crawlee-python).
+> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
 
 ## Installation
 
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
 We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
 
 ```bash
-npm install crawlee@3.12.3-beta.13
+npm install crawlee@next
 ```
 
 If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
 {
     "overrides": {
        "apify": {
-           "@crawlee/core": "3.12.3-beta.13",
-           "@crawlee/types": "3.12.3-beta.13",
-           "@crawlee/utils": "3.12.3-beta.13"
+           "@crawlee/core": "$crawlee",
+           "@crawlee/types": "$crawlee",
+           "@crawlee/utils": "$crawlee"
        }
     }
 }

package/index.d.ts

@@ -1,16 +1,10 @@
 export * from './internals/blocked.js';
 export * from './internals/cheerio.js';
-export * from './internals/chunk.js';
 export * from './internals/extract-urls.js';
 export * from './internals/general.js';
-export * from './internals/debug.js';
 export * as social from './internals/social.js';
-export * from './internals/typedefs.js';
 export * from './internals/open_graph_parser.js';
 export * from './internals/robots.js';
 export * from './internals/sitemap.js';
+export * from './internals/iterables.js';
 export * from './internals/url.js';
-export { getCurrentCpuTicksV2, CpuSample } from './internals/system-info/cpu-info.js';
-export { getMemoryInfo, MemoryInfo } from './internals/system-info/memory-info.js';
-export { Dictionary, Awaitable, Constructor } from '@crawlee/types';
-//# sourceMappingURL=index.d.ts.map

package/index.js

@@ -1,15 +1,10 @@
 export * from './internals/blocked.js';
 export * from './internals/cheerio.js';
-export * from './internals/chunk.js';
 export * from './internals/extract-urls.js';
 export * from './internals/general.js';
-export * from './internals/debug.js';
 export * as social from './internals/social.js';
-export * from './internals/typedefs.js';
 export * from './internals/open_graph_parser.js';
 export * from './internals/robots.js';
 export * from './internals/sitemap.js';
+export * from './internals/iterables.js';
 export * from './internals/url.js';
-export { getCurrentCpuTicksV2 } from './internals/system-info/cpu-info.js';
-export { getMemoryInfo } from './internals/system-info/memory-info.js';
-//# sourceMappingURL=index.js.map

package/internals/blocked.d.ts

@@ -7,4 +7,3 @@ export declare const RETRY_CSS_SELECTORS: string[];
  * Content of proxy errors that should trigger a retry, as the proxy is likely getting blocked / is malfunctioning.
  */
 export declare const ROTATE_PROXY_ERRORS: string[];
-//# sourceMappingURL=blocked.d.ts.map

package/internals/blocked.js

@@ -17,4 +17,3 @@ export const ROTATE_PROXY_ERRORS = [
     'ERR_TUNNEL_CONNECTION_FAILED',
     'Proxy responded with',
 ];
-//# sourceMappingURL=blocked.js.map

package/internals/cheerio.d.ts

@@ -1,5 +1,7 @@
 import type { CheerioAPI } from 'cheerio';
 export type CheerioRoot = CheerioAPI;
+export type { CheerioAPI, Cheerio } from 'cheerio';
+export type { Element } from 'domhandler';
 /**
  * The function converts a HTML document to a plain text.
  *
@@ -27,7 +29,7 @@ export type CheerioRoot = CheerioAPI;
  * @param htmlOrCheerioElement HTML text or parsed HTML represented using a [cheerio](https://www.npmjs.com/package/cheerio) function.
  * @return Plain text
  */
-export declare function htmlToText(htmlOrCheerioElement: string | CheerioRoot): string;
+export declare function htmlToText(htmlOrCheerioElement: string | CheerioRoot): Promise<string>;
 /**
  * Extracts URLs from a given Cheerio object.
  *
@@ -38,4 +40,3 @@ export declare function htmlToText(htmlOrCheerioElement: string | CheerioRoot):
  * @return An array of absolute URLs
  */
 export declare function extractUrlsFromCheerio($: CheerioAPI, selector?: string, baseUrl?: string): string[];
-//# sourceMappingURL=cheerio.d.ts.map

package/internals/cheerio.js

@@ -1,4 +1,3 @@
-import * as cheerio from 'cheerio';
 import { tryAbsoluteURL } from './extract-urls.js';
 // NOTE: We are skipping 'noscript' since it's content is evaluated as text, instead of HTML elements. That damages the results.
 const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript)$/i;
@@ -30,10 +29,11 @@ const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|
  * @param htmlOrCheerioElement HTML text or parsed HTML represented using a [cheerio](https://www.npmjs.com/package/cheerio) function.
  * @return Plain text
  */
-export function htmlToText(htmlOrCheerioElement) {
+export async function htmlToText(htmlOrCheerioElement) {
+    const { load } = await import('cheerio');
     if (!htmlOrCheerioElement)
         return '';
-    const $ = typeof htmlOrCheerioElement === 'function' ? htmlOrCheerioElement : cheerio.load(htmlOrCheerioElement);
+    const $ = typeof htmlOrCheerioElement === 'function' ? htmlOrCheerioElement : load(htmlOrCheerioElement);
     let text = '';
     const process = (elems) => {
         const len = elems ? elems.length : 0;
@@ -42,7 +42,7 @@ export function htmlToText(htmlOrCheerioElement) {
             if (elem.type === 'text') {
                 // Compress spaces, unless we're inside <pre> element
                 let compr;
-                if (elem.parent && elem.parent.tagName === 'pre')
+                if (elem.parent?.tagName === 'pre')
                     compr = elem.data;
                 else
                     compr = elem.data.replace(/\s+/g, ' ');
@@ -107,4 +107,3 @@ export function extractUrlsFromCheerio($, selector = 'a', baseUrl = '') {
     })
         .filter(Boolean);
 }
-//# sourceMappingURL=cheerio.js.map

package/internals/extract-urls.d.ts

@@ -1,3 +1,4 @@
+import type { BaseHttpClient } from '@crawlee/types';
 export interface DownloadListOfUrlsOptions {
     /**
      * URL to the file
@@ -16,6 +17,10 @@ export interface DownloadListOfUrlsOptions {
     urlRegExp?: RegExp;
     /** Allows to use a proxy for the download request. */
     proxyUrl?: string;
+    /**
+     * Custom HTTP client to use for downloading the file.
+     */
+    httpClient?: BaseHttpClient;
 }
 /**
  * Returns a promise that resolves to an array of urls parsed from the resource available at the provided url.
@@ -41,4 +46,3 @@ export declare function extractUrls(options: ExtractUrlsOptions): string[];
  * Helper function used to validate URLs used when extracting URLs from a page
  */
 export declare function tryAbsoluteURL(href: string, baseUrl: string): string | undefined;
-//# sourceMappingURL=extract-urls.d.ts.map

package/internals/extract-urls.js

@@ -1,4 +1,4 @@
-import { gotScraping } from 'got-scraping';
+import { FetchHttpClient } from '@crawlee/http-client';
 import ow from 'ow';
 import { URL_NO_COMMAS_REGEX } from './general.js';
 /**
@@ -11,15 +11,19 @@ export async function downloadListOfUrls(options) {
         encoding: ow.optional.string,
         urlRegExp: ow.optional.regExp,
         proxyUrl: ow.optional.string,
+        httpClient: ow.optional.object,
     }));
-    const { url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX, proxyUrl } = options;
+    const { url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX, proxyUrl, httpClient = new FetchHttpClient(), } = options;
     // Try to detect wrong urls and fix them. Currently, detects only sharing url instead of csv download one.
-    const match = url.match(/^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/?/);
+    const match = /^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/?/.exec(url);
     let fixedUrl = url;
     if (match) {
         fixedUrl = `${match[1]}/gviz/tq?tqx=out:csv`;
     }
-    const { body: string } = await gotScraping({ url: fixedUrl, encoding, proxyUrl });
+    const response = await httpClient.sendRequest(new Request(fixedUrl, { method: 'GET' }), {
+        proxyUrl,
+    });
+    const string = new TextDecoder(encoding).decode(new Uint8Array(await response.arrayBuffer()));
     return extractUrls({ string, urlRegExp });
 }
 /**
@@ -49,4 +53,3 @@ export function tryAbsoluteURL(href, baseUrl) {
         return undefined;
     }
 }
-//# sourceMappingURL=extract-urls.js.map

package/internals/general.d.ts

@@ -8,25 +8,6 @@ export declare const URL_NO_COMMAS_REGEX: RegExp;
  * Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
  */
 export declare const URL_WITH_COMMAS_REGEX: RegExp;
-/**
- * Returns a `Promise` that resolves to true if the code is running in a Docker container.
- */
-export declare function isDocker(forceReset?: boolean): Promise<boolean>;
-/**
- * Detects if crawlee is running in a containerized environment.
- */
-export declare function isContainerized(): Promise<boolean>;
-export declare function isLambda(): boolean;
-/**
- * gets the cgroup version by checking for a file at /sys/fs/cgroup/memory
- * @returns "V1" or "V2" for the version of cgroup or null if cgroup is not found.
- */
-export declare function getCgroupsVersion(forceReset?: boolean): Promise<"V1" | "V2" | null>;
-/**
- * Computes a weighted average of an array of numbers, complemented by an array of weights.
- * @ignore
- */
-export declare function weightedAvg(arrValues: number[], arrWeights: number[]): number;
 /**
  * Returns a `Promise` that resolves after a specific period of time. This is useful to implement waiting
  * in your code, e.g. to prevent overloading of target website or to avoid bot detection.
@@ -44,14 +25,8 @@ export declare function weightedAvg(arrValues: number[], arrWeights: number[]):
  * @param millis Period of time to sleep, in milliseconds. If not a positive number, the returned promise resolves immediately.
  */
 export declare function sleep(millis?: number): Promise<void>;
-/**
- * Converts SNAKE_CASE to camelCase.
- * @ignore
- */
-export declare function snakeCaseToCamelCase(snakeCaseStr: string): string;
 /**
  * Traverses DOM and expands shadow-root elements (created by custom components).
  * @ignore
  */
 export declare function expandShadowRoots(document: Document): string;
-//# sourceMappingURL=general.d.ts.map

package/internals/general.js

@@ -1,108 +1,14 @@
-import fs from 'node:fs/promises';
 import { setTimeout } from 'node:timers/promises';
 /**
  * Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters
  * and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
  */
-export const URL_NO_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()]*)?/giu;
+export const URL_NO_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()'*]*)?/giu;
 /**
  * Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query.
  * Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
  */
-export const URL_WITH_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()]*)?/giu;
-let isDockerPromiseCache;
-async function createIsDockerPromise() {
-    const promise1 = fs
-        .stat('/.dockerenv')
-        .then(() => true)
-        .catch(() => false);
-    const promise2 = fs
-        .readFile('/proc/self/cgroup', 'utf8')
-        .then((content) => content.includes('docker'))
-        .catch(() => false);
-    const [result1, result2] = await Promise.all([promise1, promise2]);
-    return result1 || result2;
-}
-/**
- * Returns a `Promise` that resolves to true if the code is running in a Docker container.
- */
-export async function isDocker(forceReset) {
-    // Parameter forceReset is just internal for unit tests.
-    if (!isDockerPromiseCache || forceReset)
-        isDockerPromiseCache = createIsDockerPromise();
-    return isDockerPromiseCache;
-}
-let isContainerizedResult;
-/**
- * Detects if crawlee is running in a containerized environment.
- */
-export async function isContainerized() {
-    // Value is very unlikley to change. Cache the result after the first execution.
-    if (isContainerizedResult !== undefined) {
-        return isContainerizedResult;
-    }
-    // return false if running in aws lambda
-    if (isLambda()) {
-        isContainerizedResult = false;
-        return isContainerizedResult;
-    }
-    const dockerenvCheck = fs
-        .stat('/.dockerenv')
-        .then(() => true)
-        .catch(() => false);
-    const cgroupCheck = fs
-        .readFile('/proc/self/cgroup', 'utf8')
-        .then((content) => content.includes('docker'))
-        .catch(() => false);
-    const [dockerenvResult, cgroupResult] = await Promise.all([dockerenvCheck, cgroupCheck]);
-    isContainerizedResult = dockerenvResult || cgroupResult || !!process.env.KUBERNETES_SERVICE_HOST;
-    return isContainerizedResult;
-}
-export function isLambda() {
-    return !!process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;
-}
-let _cgroupsVersion;
-/**
- * gets the cgroup version by checking for a file at /sys/fs/cgroup/memory
- * @returns "V1" or "V2" for the version of cgroup or null if cgroup is not found.
- */
-export async function getCgroupsVersion(forceReset) {
-    // Parameter forceReset is just internal for unit tests.
-    if (_cgroupsVersion !== undefined && !forceReset) {
-        return _cgroupsVersion;
-    }
-    try {
-        // If this directory does not exists, cgroups are not available
-        await fs.access('/sys/fs/cgroup/');
-    }
-    catch (e) {
-        _cgroupsVersion = null;
-        return null;
-    }
-    _cgroupsVersion = 'V1';
-    try {
-        // If this directory does not exists, assume the container is using cgroups V2
-        await fs.access('/sys/fs/cgroup/memory/');
-    }
-    catch (e) {
-        _cgroupsVersion = 'V2';
-    }
-    return _cgroupsVersion;
-}
-/**
- * Computes a weighted average of an array of numbers, complemented by an array of weights.
- * @ignore
- */
-export function weightedAvg(arrValues, arrWeights) {
-    const result = arrValues
-        .map((value, i) => {
-        const weight = arrWeights[i];
-        const sum = value * weight;
-        return [sum, weight];
-    })
-        .reduce((p, c) => [p[0] + c[0], p[1] + c[1]], [0, 0]);
-    return result[0] / result[1];
-}
+export const URL_WITH_COMMAS_REGEX = /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()'*]*)?/giu;
 /**
  * Returns a `Promise` that resolves after a specific period of time. This is useful to implement waiting
  * in your code, e.g. to prevent overloading of target website or to avoid bot detection.
@@ -122,19 +28,6 @@ export function weightedAvg(arrValues, arrWeights) {
 export async function sleep(millis) {
     return setTimeout(millis ?? undefined);
 }
-/**
- * Converts SNAKE_CASE to camelCase.
- * @ignore
- */
-export function snakeCaseToCamelCase(snakeCaseStr) {
-    return snakeCaseStr
-        .toLowerCase()
-        .split('_')
-        .map((part, index) => {
-        return index > 0 ? part.charAt(0).toUpperCase() + part.slice(1) : part;
-    })
-        .join('');
-}
 /**
  * Traverses DOM and expands shadow-root elements (created by custom components).
  * @ignore
@@ -164,4 +57,3 @@ export function expandShadowRoots(document) {
     replaceShadowDomsWithHtml(document.body);
     return document.documentElement.outerHTML;
 }
-//# sourceMappingURL=general.js.map

package/internals/iterables.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Type guard that checks if a value is iterable (has Symbol.iterator).
+ * @internal
+ *
+ * **Example usage:**
+ * ```ts
+ * if (isIterable(someValue)) {
+ *   for (const item of someValue) {
+ *     console.log(item);
+ *   }
+ * }
+ * ```
+ */
+export declare function isIterable<T>(value: unknown): value is Iterable<T>;
+/**
+ * Type guard that checks if a value is async iterable (has Symbol.asyncIterator).
+ * @internal
+ *
+ * **Example usage:**
+ * ```ts
+ * if (isAsyncIterable(someValue)) {
+ *   for await (const item of someValue) {
+ *     console.log(item);
+ *   }
+ * }
+ * ```
+ */
+export declare function isAsyncIterable<T>(value: unknown): value is AsyncIterable<T>;
+/**
+ * Merges multiple async iterables into a single async iterable, yielding values concurrently.
+ *
+ * **Example usage:**
+ * ```ts
+ * const asyncIterable1 = async function* () {
+ *   yield 1; yield 3; yield 5;
+ * };
+ *
+ * const asyncIterable2 = async function* () {
+ *   yield 2; yield 4; yield 6;
+ * };
+ *
+ * for await (const value of mergeAsyncIterables(asyncIterable1(), asyncIterable2())) {
+ *   console.log(value);
+ * }
+ * ```
+ */
+export declare function mergeAsyncIterables<T>(...iterables: AsyncIterable<T>[]): AsyncIterable<T>;

package/internals/iterables.js

@@ -0,0 +1,96 @@
+/**
+ * Type guard that checks if a value is iterable (has Symbol.iterator).
+ * @internal
+ *
+ * **Example usage:**
+ * ```ts
+ * if (isIterable(someValue)) {
+ *   for (const item of someValue) {
+ *     console.log(item);
+ *   }
+ * }
+ * ```
+ */
+export function isIterable(value) {
+    if (value == null || typeof value === 'string' || ArrayBuffer.isView(value)) {
+        return false;
+    }
+    if (Array.isArray(value)) {
+        return true;
+    }
+    return typeof Object(value)[Symbol.iterator] === 'function';
+}
+/**
+ * Type guard that checks if a value is async iterable (has Symbol.asyncIterator).
+ * @internal
+ *
+ * **Example usage:**
+ * ```ts
+ * if (isAsyncIterable(someValue)) {
+ *   for await (const item of someValue) {
+ *     console.log(item);
+ *   }
+ * }
+ * ```
+ */
+export function isAsyncIterable(value) {
+    if (value == null || typeof value === 'string' || ArrayBuffer.isView(value)) {
+        return false;
+    }
+    return typeof Object(value)[Symbol.asyncIterator] === 'function';
+}
+// Source - https://stackoverflow.com/a/71288323
+/**
+ * Merges multiple async iterables into a single async iterable, yielding values concurrently.
+ *
+ * **Example usage:**
+ * ```ts
+ * const asyncIterable1 = async function* () {
+ *   yield 1; yield 3; yield 5;
+ * };
+ *
+ * const asyncIterable2 = async function* () {
+ *   yield 2; yield 4; yield 6;
+ * };
+ *
+ * for await (const value of mergeAsyncIterables(asyncIterable1(), asyncIterable2())) {
+ *   console.log(value);
+ * }
+ * ```
+ */
+export async function* mergeAsyncIterables(...iterables) {
+    const asyncIterators = iterables.map((iterable) => iterable[Symbol.asyncIterator]());
+    const results = [];
+    let count = asyncIterators.length;
+    const never = new Promise(() => { });
+    async function getNext(asyncIterator, index) {
+        const result = await asyncIterator.next();
+        return {
+            index,
+            result,
+        };
+    }
+    const nextPromises = asyncIterators.map(getNext);
+    try {
+        while (count) {
+            const { index, result } = await Promise.race(nextPromises);
+            if (result.done) {
+                nextPromises[index] = never;
+                results[index] = result.value;
+                count--;
+            }
+            else {
+                nextPromises[index] = getNext(asyncIterators[index], index);
+                yield result.value;
+            }
+        }
+    }
+    finally {
+        for (const [index, iterator] of asyncIterators.entries()) {
+            // no await here - see https://github.com/tc39/proposal-async-iteration/issues/126
+            if (nextPromises[index] !== never && iterator.return != null)
+                void iterator.return();
+        }
+    }
+    return results;
+}

package/internals/open_graph_parser.d.ts

@@ -14,7 +14,6 @@ type OpenGraphResult = string | string[] | Dictionary<string | Dictionary>;
  * Currently existing properties are kept up to date.
  * @returns Scraped OpenGraph properties as an object.
  */
-export declare function parseOpenGraph(raw: string, additionalProperties?: OpenGraphProperty[]): Dictionary<OpenGraphResult>;
-export declare function parseOpenGraph($: CheerioAPI, additionalProperties?: OpenGraphProperty[]): Dictionary<OpenGraphResult>;
+export declare function parseOpenGraph(raw: string, additionalProperties?: OpenGraphProperty[]): Promise<Dictionary<OpenGraphResult>>;
+export declare function parseOpenGraph($: CheerioAPI, additionalProperties?: OpenGraphProperty[]): Promise<Dictionary<OpenGraphResult>>;
 export {};
-//# sourceMappingURL=open_graph_parser.d.ts.map

package/internals/open_graph_parser.js

@@ -1,4 +1,3 @@
-import { load } from 'cheerio';
 /**
  * To be used with the spread operator. Ensures that the item is defined, and is not empty.
  *
@@ -255,32 +254,32 @@ const OPEN_GRAPH_PROPERTIES = [
         outputName: 'articleInfo',
         children: [
             {
-                name: 'music:published_time',
+                name: 'article:published_time',
                 outputName: 'publishedTime',
                 children: [],
             },
             {
-                name: 'music:modified_time',
+                name: 'article:modified_time',
                 outputName: 'modifiedTime',
                 children: [],
             },
             {
-                name: 'music:expiration_time',
+                name: 'article:expiration_time',
                 outputName: 'expirationTime',
                 children: [],
             },
             {
-                name: 'music:author',
+                name: 'article:author',
                 outputName: 'author',
                 children: [],
             },
             {
-                name: 'music:section',
+                name: 'article:section',
                 outputName: 'section',
                 children: [],
             },
             {
-                name: 'music:tag',
+                name: 'article:tag',
                 outputName: 'tag',
                 children: [],
             },
@@ -366,7 +365,8 @@ const parseOpenGraphProperty = (property, $) => {
         }, {}),
     };
 };
-export function parseOpenGraph(item, additionalProperties) {
+export async function parseOpenGraph(item, additionalProperties) {
+    const { load } = await import('cheerio');
     const $ = typeof item === 'string' ? load(item) : item;
     return [...(additionalProperties || []), ...OPEN_GRAPH_PROPERTIES].reduce((acc, curr) => {
         return {
@@ -375,4 +375,3 @@ export function parseOpenGraph(item, additionalProperties) {
         };
     }, {});
 }
-//# sourceMappingURL=open_graph_parser.js.map

package/internals/robots.d.ts

@@ -1,3 +1,4 @@
+import type { BaseHttpClient, CrawleeLogger } from '@crawlee/types';
 import { Sitemap } from './sitemap.js';
 /**
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
@@ -20,13 +21,23 @@ import { Sitemap } from './sitemap.js';
 export declare class RobotsTxtFile {
     private robots;
     private proxyUrl?;
+    private logger?;
     private constructor();
     /**
      * Determine the location of a robots.txt file for a URL and fetch it.
      * @param url the URL to fetch robots.txt for
-     * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
+     * @param [options] additional options
+     * @param [options.signal] an AbortSignal to cancel the request
+     * @param [options.timeoutMillis] timeout in milliseconds for the request
+     * @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
      */
-    static find(url: string, proxyUrl?: string): Promise<RobotsTxtFile>;
+    static find(url: string, options?: {
+        signal?: AbortSignal;
+        timeoutMillis?: number;
+        proxyUrl?: string;
+        httpClient?: BaseHttpClient;
+        logger?: CrawleeLogger;
+    }): Promise<RobotsTxtFile>;
     /**
      * Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
      * @param url the URL for robots.txt file
@@ -34,7 +45,7 @@ export declare class RobotsTxtFile {
      * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
      */
     static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
-    protected static load(url: string, proxyUrl?: string): Promise<RobotsTxtFile>;
+    private static load;
     /**
      * Check if a URL should be crawled by robots.
      * @param url the URL to check against the rules in robots.txt
@@ -54,5 +65,3 @@ export declare class RobotsTxtFile {
      */
     parseUrlsFromSitemaps(): Promise<string[]>;
 }
-export { RobotsTxtFile as RobotsFile };
-//# sourceMappingURL=robots.d.ts.map