@crawlee/utils 4.0.0-beta.7 → 4.0.0-beta.71

package/internals/iterables.js

@@ -0,0 +1,222 @@
+import { inspect } from 'node:util';
+/**
+ * Type guard that checks if a value is iterable (has Symbol.iterator).
+ * @internal
+ *
+ * **Example usage:**
+ * ```ts
+ * if (isIterable(someValue)) {
+ *   for (const item of someValue) {
+ *     console.log(item);
+ *   }
+ * }
+ * ```
+ */
+export function isIterable(value) {
+    if (value == null || typeof value === 'string' || ArrayBuffer.isView(value)) {
+        return false;
+    }
+    if (Array.isArray(value)) {
+        return true;
+    }
+    return typeof Object(value)[Symbol.iterator] === 'function';
+}
+/**
+ * Type guard that checks if a value is async iterable (has Symbol.asyncIterator).
+ * @internal
+ *
+ * **Example usage:**
+ * ```ts
+ * if (isAsyncIterable(someValue)) {
+ *   for await (const item of someValue) {
+ *     console.log(item);
+ *   }
+ * }
+ * ```
+ */
+export function isAsyncIterable(value) {
+    if (value == null || typeof value === 'string' || ArrayBuffer.isView(value)) {
+        return false;
+    }
+    return typeof Object(value)[Symbol.asyncIterator] === 'function';
+}
+/**
+ * Converts any iterable or async iterable to an async iterable.
+ * @internal
+ *
+ * @yields Each item from the input iterable
+ *
+ * **Example usage:**
+ * ```ts
+ * const syncArray = [1, 2, 3];
+ * for await (const item of asyncifyIterable(syncArray)) {
+ *   console.log(item); // 1, 2, 3
+ * }
+ * ```
+ */
+export async function* asyncifyIterable(iterable) {
+    yield* iterable;
+}
+/**
+ * Lazily splits the input async iterable into chunks of specified size.
+ * The last chunk may contain fewer items if the total number of items
+ * is not evenly divisible by the chunk size.
+ * @internal
+ *
+ * @yields Arrays of items, each containing up to chunkSize items
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   for (let i = 1; i <= 10; i++) yield i;
+ * };
+ *
+ * for await (const chunk of chunkedAsyncIterable(numbers(), 3)) {
+ *   console.log(chunk); // [1, 2, 3], [4, 5, 6], [7, 8, 9], [10]
+ * }
+ * ```
+ */
+export async function* chunkedAsyncIterable(iterable, chunkSize) {
+    if (typeof chunkSize !== 'number' || chunkSize < 1) {
+        throw new Error(`Chunk size must be a positive number (${inspect(chunkSize)}) received`);
+    }
+    let chunk = [];
+    for await (const item of iterable) {
+        chunk.push(item);
+        if (chunk.length >= chunkSize) {
+            yield chunk;
+            chunk = [];
+        }
+    }
+    if (chunk.length) {
+        yield chunk;
+    }
+}
+/**
+ * Wraps an async iterable to provide peek functionality, allowing you to look at
+ * the next value without consuming it from the iterator.
+ * @internal
+ *
+ * @param iterable - The async iterable to make peekable
+ *
+ * **Example usage:**
+ * ```ts
+ * const numbers = async function* () {
+ *   yield 1; yield 2; yield 3;
+ * };
+ *
+ * const peekable = peekableAsyncIterable(numbers());
+ * const iterator = peekable[Symbol.asyncIterator]();
+ *
+ * console.log(await iterator.peek()); // 1 (doesn't consume)
+ * console.log(await iterator.peek()); // 1 (still doesn't consume)
+ * console.log(await iterator.next()); // { value: 1, done: false } (now consumed)
+ * console.log(await iterator.peek()); // 2 (next value)
+ * ```
+ */
+export function peekableAsyncIterable(iterable) {
+    const iterator = asyncifyIterable(iterable)[Symbol.asyncIterator]();
+    let peekedValue;
+    let isExhausted = false;
+    const peekableIterator = {
+        async next() {
+            // If we have peeked a value, return it and clear the peek
+            if (peekedValue !== undefined) {
+                const result = peekedValue;
+                peekedValue = undefined;
+                if (result.done) {
+                    isExhausted = true;
+                    return { done: true, value: undefined };
+                }
+                return { done: false, value: result.value };
+            }
+            if (isExhausted) {
+                return { done: true, value: undefined };
+            }
+            const result = await iterator.next();
+            if (result.done) {
+                isExhausted = true;
+            }
+            return result;
+        },
+        async peek() {
+            if (peekedValue !== undefined) {
+                return peekedValue.done ? undefined : peekedValue.value;
+            }
+            if (isExhausted) {
+                return undefined;
+            }
+            const result = await iterator.next();
+            peekedValue = { done: result.done ?? false, value: result.value };
+            if (result.done) {
+                isExhausted = true;
+                return undefined;
+            }
+            return result.value;
+        },
+        [Symbol.asyncIterator]() {
+            return this;
+        },
+    };
+    return {
+        [Symbol.asyncIterator]() {
+            return peekableIterator;
+        },
+    };
+}
+// Source - https://stackoverflow.com/a/71288323
+/**
+ * Merges multiple async iterables into a single async iterable, yielding values concurrently.
+ *
+ * **Example usage:**
+ * ```ts
+ * const asyncIterable1 = async function* () {
+ *   yield 1; yield 3; yield 5;
+ * };
+ *
+ * const asyncIterable2 = async function* () {
+ *   yield 2; yield 4; yield 6;
+ * };
+ *
+ * for await (const value of mergeAsyncIterables(asyncIterable1(), asyncIterable2())) {
+ *   console.log(value);
+ * }
+ * ```
+ */
+export async function* mergeAsyncIterables(...iterables) {
+    const asyncIterators = iterables.map((iterable) => iterable[Symbol.asyncIterator]());
+    const results = [];
+    let count = asyncIterators.length;
+    const never = new Promise(() => { });
+    async function getNext(asyncIterator, index) {
+        const result = await asyncIterator.next();
+        return {
+            index,
+            result,
+        };
+    }
+    const nextPromises = asyncIterators.map(getNext);
+    try {
+        while (count) {
+            const { index, result } = await Promise.race(nextPromises);
+            if (result.done) {
+                nextPromises[index] = never;
+                results[index] = result.value;
+                count--;
+            }
+            else {
+                nextPromises[index] = getNext(asyncIterators[index], index);
+                yield result.value;
+            }
+        }
+    }
+    finally {
+        for (const [index, iterator] of asyncIterators.entries()) {
+            // no await here - see https://github.com/tc39/proposal-async-iteration/issues/126
+            if (nextPromises[index] !== never && iterator.return != null)
+                void iterator.return();
+        }
+    }
+    return results;
+}
+//# sourceMappingURL=iterables.js.map

package/internals/iterables.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"iterables.js","sourceRoot":"","sources":["../../src/internals/iterables.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,UAAU,CAAI,KAAc;IACxC,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC;AAChE,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,eAAe,CAAI,KAAc;IAC7C,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1E,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,OAAO,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,UAAU,CAAC;AACrE,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAAI,QAAwC;IAC/E,KAAK,CAAC,CAAC,QAAQ,CAAC;AACpB,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,oBAAoB,CACvC,QAAwC,EACxC,SAAiB;IAEjB,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IAC7F,CAAC;IAED,IAAI,KAAK,GAAQ,EAAE,CAAC;IAEpB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEjB,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,KAAK,CAAC;YACZ,KAAK,GAAG,EAAE,CAAC;QACf,CAAC;IACL,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACf,MAAM,KAAK,CAAC;IAChB,CAAC;AACL,CAAC;AAyBD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,qBAAqB,CAAI,QAAwC;IAC7E,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;IACpE,IAAI,WAAoD,CAAC;IACzD,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB,MAAM,gBAAgB,GAA6B;QAC/C,KAAK,CAAC,IAAI;YACN,0DAA0D;YAC1D,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC5B,MAAM,MAAM,GAAG,WAAW,CAAC;gBAC3B,WAAW,GAAG,SAAS,CAAC;gBAExB,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;oBACd,WAAW,GAAG,IAAI,CAAC;oBACnB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;gBAC5C,CAAC;gBAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YAChD,CAAC;YAED,IAAI,WAAW,EAAE,CAAC;gBACd,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;YAC5C,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAErC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACd,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC;YAED,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,KAAK,CAAC,IAAI;YACN,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;gBAC5B,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC;YAC5D,CAAC;YAED,IAAI,WAAW,EAAE,CAAC;gBACd,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACrC,WAAW,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;YAElE,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACd,WAAW,GAAG,IAAI,CAAC;gBACnB,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,OAAO,MAAM,CAAC,KAAK,CAAC;QACxB,CAAC;QAED,CAAC,MAAM,CAAC,aAAa,CAAC;YAClB,OAAO,IAAI,CAAC;QAChB,CAAC;KACJ,CAAC;IAEF,OAAO;QACH,CAAC,MAAM,CAAC,aAAa,CAAC;YAClB,OAAO,gBAAgB,CAAC;QAC5B,CAAC;KACJ,CAAC;AACN,CAAC;AAED,gDAAgD;AAChD;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,mBAAmB,CAAI,GAAG,SAA6B;IAC1E,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IACrF,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,IAAI,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC;IAClC,MAAM,KAAK,GAAmB,IAAI,OAAO,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACpD,KAAK,UAAU,OAAO,CAAC,aAA+B,EAAE,KAAa;QACjE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAC1C,OAAO;YACH,KAAK;YACL,MAAM;SACT,CAAC;IACN,CAAC;IACD,MAAM,YAAY,GAAG,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACjD,IAAI,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACX,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC3D,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACd,YAAY,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;gBAC5B,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;gBAC9B,KAAK,EAAE,CAAC;YACZ,CAAC;iBAAM,CAAC;gBACJ,YAAY,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;gBAC5D,MAAM,MAAM,CAAC,KAAK,CAAC;YACvB,CAAC;QACL,CAAC;IACL,CAAC;YAAS,CAAC;QACP,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,cAAc,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,kFAAkF;YAClF,IAAI,YAAY,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,QAAQ,CAAC,MAAM,IAAI,IAAI;gBAAE,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;QACzF,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC"}

package/internals/open_graph_parser.d.ts

@@ -14,7 +14,7 @@ type OpenGraphResult = string | string[] | Dictionary<string | Dictionary>;
  * Currently existing properties are kept up to date.
  * @returns Scraped OpenGraph properties as an object.
  */
-export declare function parseOpenGraph(raw: string, additionalProperties?: OpenGraphProperty[]): Dictionary<OpenGraphResult>;
-export declare function parseOpenGraph($: CheerioAPI, additionalProperties?: OpenGraphProperty[]): Dictionary<OpenGraphResult>;
+export declare function parseOpenGraph(raw: string, additionalProperties?: OpenGraphProperty[]): Promise<Dictionary<OpenGraphResult>>;
+export declare function parseOpenGraph($: CheerioAPI, additionalProperties?: OpenGraphProperty[]): Promise<Dictionary<OpenGraphResult>>;
 export {};
 //# sourceMappingURL=open_graph_parser.d.ts.map

package/internals/open_graph_parser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"open_graph_parser.d.ts","sourceRoot":"","sources":["../../src/internals/open_graph_parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAG1C,MAAM,WAAW,iBAAiB;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CACjC;AAED,KAAK,eAAe,GAAG,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC;AA2X3E;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,oBAAoB,CAAC,EAAE,iBAAiB,EAAE,GAAG,UAAU,CAAC,eAAe,CAAC,CAAC;AACrH,wBAAgB,cAAc,CAAC,CAAC,EAAE,UAAU,EAAE,oBAAoB,CAAC,EAAE,iBAAiB,EAAE,GAAG,UAAU,CAAC,eAAe,CAAC,CAAC"}
+{"version":3,"file":"open_graph_parser.d.ts","sourceRoot":"","sources":["../../src/internals/open_graph_parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE1C,MAAM,WAAW,iBAAiB;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CACjC;AAED,KAAK,eAAe,GAAG,MAAM,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC;AA2X3E;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAChC,GAAG,EAAE,MAAM,EACX,oBAAoB,CAAC,EAAE,iBAAiB,EAAE,GAC3C,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC;AACxC,wBAAsB,cAAc,CAChC,CAAC,EAAE,UAAU,EACb,oBAAoB,CAAC,EAAE,iBAAiB,EAAE,GAC3C,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,CAAC"}

package/internals/open_graph_parser.js

@@ -1,4 +1,3 @@
-import { load } from 'cheerio';
 /**
  * To be used with the spread operator. Ensures that the item is defined, and is not empty.
  *
@@ -366,7 +365,8 @@ const parseOpenGraphProperty = (property, $) => {
         }, {}),
     };
 };
-export function parseOpenGraph(item, additionalProperties) {
+export async function parseOpenGraph(item, additionalProperties) {
+    const { load } = await import('cheerio');
     const $ = typeof item === 'string' ? load(item) : item;
     return [...(additionalProperties || []), ...OPEN_GRAPH_PROPERTIES].reduce((acc, curr) => {
         return {

package/internals/open_graph_parser.js.map

@@ -1 +1 @@
-{"version":3,"file":"open_graph_parser.js","sourceRoot":"","sources":["../../src/internals/open_graph_parser.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAU/B;;;;;;GAMG;AACH,MAAM,cAAc,GAAG,CAAC,GAAW,EAAE,IAAS,EAAE,EAAE,CAC9C,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAE/E,MAAM,qBAAqB,GAAwB;IAC/C;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,MAAM;QAClB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,iBAAiB;gBACvB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,KAAK;QACjB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,gBAAgB;QACtB,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,eAAe;QACrB,UAAU,EAAE,YAAY;QACxB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,WAAW;QACjB,UAAU,EAAE,QAAQ;QACpB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,UAAU;QACtB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,iBAAiB;gBACvB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD,kDAAkD;IAClD,iHAAiH;IACjH,mDAAmD;IACnD;QACI,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,WAAW;QACvB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,WAAW;QACvB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;oBACD;wBACI,IAAI,EAAE,mBAAmB;wBACzB,UAAU,EAAE,OAAO;wBACnB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,YAAY;gBAClB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,iBAAiB;wBACvB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;oBACD;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,OAAO;wBACnB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,sBAAsB;gBAC5B,UAAU,EAAE,eAAe;gBAC3B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,cAAc;gBAC1B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,uBAAuB;gBAC7B,UAAU,EAAE,gBAAgB;gBAC5B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,MAAM;QACZ,UAAU,EAAE,UAAU;QACtB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,mBAAmB;gBACzB,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,UAAU;gBAChB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,mBAAmB;gBACzB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,kBAAkB;gBACxB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;CACJ,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,kBAAkB,IAAI,IAAI,CAAC;AAE3E,MAAM,sBAAsB,GAAG,CAAC,QAA2B,EAAE,CAAa,EAAuC,EAAE;IAC/G,6GAA6G;IAC7G,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,qBAAqB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAEnG,4FAA4F;IAC5F,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAE,MAAM,CAAC,CAAC,CAAY,CAAC,CAAC,CAAE,MAAmB,CAAC;IAElF,sEAAsE;IACtE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC;IAE9C,wGAAwG;IACxG,OAAO;QACH,qGAAqG;QACrG,+EAA+E;QAC/E,wGAAwG;QACxG,uDAAuD;QACvD,GAAG,cAAc,CAAC,GAAG,QAAQ,CAAC,UAAU,OAAO,EAAE,OAAO,CAAC;QACzD,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CACvB,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;YACV,MAAM,MAAM,GAAG,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC/C,IAAI,MAAM,KAAK,SAAS;gBAAE,OAAO,GAAG,CAAC;YAErC,OAAO;gBACH,GAAG,GAAG;gBACN,GAAG,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;aACtE,CAAC;QACN,CAAC,EACD,EAAqC,CACxC;KACJ,CAAC;AACN,CAAC,CAAC;AAYF,MAAM,UAAU,cAAc,CAAC,IAAyB,EAAE,oBAA0C;IAChG,MAAM,CAAC,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvD,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,EAAE,CAAC,EAAE,GAAG,qBAAqB,CAAC,CAAC,MAAM,CACrE,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QACV,OAAO;YACH,GAAG,GAAG;YACN,GAAG,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;SACtE,CAAC;IACN,CAAC,EACD,EAAiC,CACpC,CAAC;AACN,CAAC"}
+{"version":3,"file":"open_graph_parser.js","sourceRoot":"","sources":["../../src/internals/open_graph_parser.ts"],"names":[],"mappings":"AAWA;;;;;;GAMG;AACH,MAAM,cAAc,GAAG,CAAC,GAAW,EAAE,IAAS,EAAE,EAAE,CAC9C,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAE/E,MAAM,qBAAqB,GAAwB;IAC/C;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,MAAM;QAClB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,iBAAiB;gBACvB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE,KAAK;QACjB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,gBAAgB;QACtB,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,eAAe;QACrB,UAAU,EAAE,YAAY;QACxB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,WAAW;QACjB,UAAU,EAAE,QAAQ;QACpB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,cAAc;QACpB,UAAU,EAAE,UAAU;QACtB,QAAQ,EAAE,EAAE;KACf;IACD;QACI,IAAI,EAAE,UAAU;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,iBAAiB;gBACvB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD,kDAAkD;IAClD,iHAAiH;IACjH,mDAAmD;IACnD;QACI,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,WAAW;QACvB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,WAAW;QACvB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,OAAO;gBACnB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;oBACD;wBACI,IAAI,EAAE,mBAAmB;wBACzB,UAAU,EAAE,OAAO;wBACnB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,YAAY;gBAClB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE;oBACN;wBACI,IAAI,EAAE,iBAAiB;wBACvB,UAAU,EAAE,MAAM;wBAClB,QAAQ,EAAE,EAAE;qBACf;oBACD;wBACI,IAAI,EAAE,kBAAkB;wBACxB,UAAU,EAAE,OAAO;wBACnB,QAAQ,EAAE,EAAE;qBACf;iBACJ;aACJ;YACD;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,sBAAsB;gBAC5B,UAAU,EAAE,eAAe;gBAC3B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,cAAc;gBAC1B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,uBAAuB;gBAC7B,UAAU,EAAE,gBAAgB;gBAC5B,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,cAAc;gBACpB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,MAAM;QACZ,UAAU,EAAE,UAAU;QACtB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,aAAa;gBACnB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,WAAW;gBACjB,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,mBAAmB;gBACzB,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,UAAU;gBAChB,UAAU,EAAE,KAAK;gBACjB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;IACD;QACI,IAAI,EAAE,SAAS;QACf,UAAU,EAAE,aAAa;QACzB,QAAQ,EAAE;YACN;gBACI,IAAI,EAAE,oBAAoB;gBAC1B,UAAU,EAAE,WAAW;gBACvB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,mBAAmB;gBACzB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,kBAAkB;gBACxB,UAAU,EAAE,UAAU;gBACtB,QAAQ,EAAE,EAAE;aACf;YACD;gBACI,IAAI,EAAE,gBAAgB;gBACtB,UAAU,EAAE,QAAQ;gBACpB,QAAQ,EAAE,EAAE;aACf;SACJ;KACJ;CACJ,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,kBAAkB,IAAI,IAAI,CAAC;AAE3E,MAAM,sBAAsB,GAAG,CAAC,QAA2B,EAAE,CAAa,EAAuC,EAAE;IAC/G,6GAA6G;IAC7G,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,qBAAqB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAEnG,4FAA4F;IAC5F,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAE,MAAM,CAAC,CAAC,CAAY,CAAC,CAAC,CAAE,MAAmB,CAAC;IAElF,sEAAsE;IACtE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC;IAE9C,wGAAwG;IACxG,OAAO;QACH,qGAAqG;QACrG,+EAA+E;QAC/E,wGAAwG;QACxG,uDAAuD;QACvD,GAAG,cAAc,CAAC,GAAG,QAAQ,CAAC,UAAU,OAAO,EAAE,OAAO,CAAC;QACzD,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CACvB,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;YACV,MAAM,MAAM,GAAG,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC/C,IAAI,MAAM,KAAK,SAAS;gBAAE,OAAO,GAAG,CAAC;YAErC,OAAO;gBACH,GAAG,GAAG;gBACN,GAAG,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;aACtE,CAAC;QACN,CAAC,EACD,EAAqC,CACxC;KACJ,CAAC;AACN,CAAC,CAAC;AAkBF,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAyB,EAAE,oBAA0C;IACtG,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvD,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,EAAE,CAAC,EAAE,GAAG,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;QACpF,OAAO;YACH,GAAG,GAAG;YACN,GAAG,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,sBAAsB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;SACtE,CAAC;IACN,CAAC,EAAE,EAAiC,CAAC,CAAC;AAC1C,CAAC"}

package/internals/robots.d.ts

@@ -1,3 +1,4 @@
+import type { BaseHttpClient, CrawleeLogger } from '@crawlee/types';
 import { Sitemap } from './sitemap.js';
 /**
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
@@ -20,13 +21,23 @@ import { Sitemap } from './sitemap.js';
 export declare class RobotsTxtFile {
     private robots;
     private proxyUrl?;
+    private logger?;
     private constructor();
     /**
      * Determine the location of a robots.txt file for a URL and fetch it.
      * @param url the URL to fetch robots.txt for
-     * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
+     * @param [options] additional options
+     * @param [options.signal] an AbortSignal to cancel the request
+     * @param [options.timeoutMillis] timeout in milliseconds for the request
+     * @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
      */
-    static find(url: string, proxyUrl?: string): Promise<RobotsTxtFile>;
+    static find(url: string, options?: {
+        signal?: AbortSignal;
+        timeoutMillis?: number;
+        proxyUrl?: string;
+        httpClient?: BaseHttpClient;
+        logger?: CrawleeLogger;
+    }): Promise<RobotsTxtFile>;
     /**
      * Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
      * @param url the URL for robots.txt file
@@ -34,7 +45,13 @@ export declare class RobotsTxtFile {
      * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
      */
     static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
-    protected static load(url: string, proxyUrl?: string): Promise<RobotsTxtFile>;
+    protected static load(url: string, options?: {
+        signal?: AbortSignal;
+        timeoutMillis?: number;
+        proxyUrl?: string;
+        httpClient?: BaseHttpClient;
+        logger?: CrawleeLogger;
+    }): Promise<RobotsTxtFile>;
     /**
      * Check if a URL should be crawled by robots.
      * @param url the URL to check against the rules in robots.txt

package/internals/robots.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../../src/internals/robots.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAIvC;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,aAAa;IAElB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,QAAQ,CAAC;IAFrB,OAAO;IAKP;;;;OAIG;WACU,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IAQzE;;;;;OAKG;IACH,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,aAAa;qBAKpD,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IAiCnF;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,SAAM,GAAG,OAAO;IAIhD;;OAEG;IACH,WAAW,IAAI,MAAM,EAAE;IAIvB;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,OAAO,CAAC;IAIvC;;OAEG;IACG,qBAAqB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;CAGnD;AAGD,OAAO,EAAE,aAAa,IAAI,UAAU,EAAE,CAAC"}
+{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../../src/internals/robots.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAIpE,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,aAAa;IAElB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,QAAQ,CAAC;IACjB,OAAO,CAAC,MAAM,CAAC;IAHnB,OAAO;IAMP;;;;;;;OAOG;WACU,IAAI,CACb,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,UAAU,CAAC,EAAE,cAAc,CAAC;QAC5B,MAAM,CAAC,EAAE,aAAa,CAAC;KAC1B,GACF,OAAO,CAAC,aAAa,CAAC;IAQzB;;;;;OAKG;IACH,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,aAAa;qBAKpD,IAAI,CACvB,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QACN,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,UAAU,CAAC,EAAE,cAAc,CAAC;QAC5B,MAAM,CAAC,EAAE,aAAa,CAAC;KAC1B,GACF,OAAO,CAAC,aAAa,CAAC;IAgCzB;;;;OAIG;IACH,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,SAAM,GAAG,OAAO;IAIhD;;OAEG;IACH,WAAW,IAAI,MAAM,EAAE;IAIvB;;OAEG;IACG,aAAa,IAAI,OAAO,CAAC,OAAO,CAAC;IAIvC;;OAEG;IACG,qBAAqB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;CAGnD;AAGD,OAAO,EAAE,aAAa,IAAI,UAAU,EAAE,CAAC"}

package/internals/robots.js

@@ -1,7 +1,6 @@
-import { gotScraping } from 'got-scraping';
+import { FetchHttpClient } from '@crawlee/http-client';
 import robotsParser from 'robots-parser';
 import { Sitemap } from './sitemap.js';
-let HTTPError;
 /**
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
  *
@@ -23,20 +22,25 @@ let HTTPError;
 export class RobotsTxtFile {
     robots;
     proxyUrl;
-    constructor(robots, proxyUrl) {
+    logger;
+    constructor(robots, proxyUrl, logger) {
         this.robots = robots;
         this.proxyUrl = proxyUrl;
+        this.logger = logger;
     }
     /**
      * Determine the location of a robots.txt file for a URL and fetch it.
      * @param url the URL to fetch robots.txt for
-     * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
+     * @param [options] additional options
+     * @param [options.signal] an AbortSignal to cancel the request
+     * @param [options.timeoutMillis] timeout in milliseconds for the request
+     * @param [options.proxyUrl] a proxy to be used for fetching the robots.txt file
      */
-    static async find(url, proxyUrl) {
+    static async find(url, options) {
         const robotsTxtFileUrl = new URL(url);
         robotsTxtFileUrl.pathname = '/robots.txt';
         robotsTxtFileUrl.search = '';
-        return RobotsTxtFile.load(robotsTxtFileUrl.toString(), proxyUrl);
+        return RobotsTxtFile.load(robotsTxtFileUrl.toString(), options);
     }
     /**
      * Allows providing the URL and robots.txt content explicitly instead of loading it from the target site.
@@ -48,33 +52,28 @@ export class RobotsTxtFile {
         // @ts-ignore
         return new RobotsTxtFile(robotsParser(url, content), proxyUrl);
     }
-    static async load(url, proxyUrl) {
-        if (!HTTPError) {
-            HTTPError = (await import('got-scraping')).HTTPError;
-        }
-        try {
-            const response = await gotScraping({
-                url,
-                proxyUrl,
-                method: 'GET',
-                responseType: 'text',
-            });
-            // @ts-ignore
-            return new RobotsTxtFile(robotsParser(url.toString(), response.body), proxyUrl);
+    static async load(url, options) {
+        const { proxyUrl, logger, httpClient = new FetchHttpClient() } = options || {};
+        const response = await httpClient.sendRequest(new Request(url, { method: 'GET' }), {
+            proxyUrl,
+            timeoutMillis: options?.timeoutMillis,
+            signal: options?.signal,
+        });
+        if (response.status < 200 || response.status >= 300) {
+            throw new Error(`Failed to load robots.txt from ${url}: HTTP ${response.status}`);
         }
-        catch (e) {
-            if (e instanceof HTTPError && e.response.statusCode === 404) {
-                return new RobotsTxtFile({
-                    isAllowed() {
-                        return true;
-                    },
-                    getSitemaps() {
-                        return [];
-                    },
-                }, proxyUrl);
-            }
-            throw e;
+        if (response.status === 404) {
+            return new RobotsTxtFile({
+                isAllowed() {
+                    return true;
+                },
+                getSitemaps() {
+                    return [];
+                },
+            }, proxyUrl, logger);
         }
+        // @ts-ignore
+        return new RobotsTxtFile(robotsParser(url.toString(), await response.text()), proxyUrl, logger);
     }
     /**
      * Check if a URL should be crawled by robots.
@@ -94,7 +93,7 @@ export class RobotsTxtFile {
      * Parse all the sitemaps referenced in the robots file.
      */
     async parseSitemaps() {
-        return Sitemap.load(this.robots.getSitemaps(), this.proxyUrl);
+        return Sitemap.load(this.robots.getSitemaps(), this.proxyUrl, { logger: this.logger });
     }
     /**
      * Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.

package/internals/robots.js.map

@@ -1 +1 @@
-{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/internals/robots.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE3C,OAAO,YAAY,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,IAAI,SAAgC,CAAC;AAErC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,OAAO,aAAa;IAEV;IACA;IAFZ,YACY,MAAgD,EAChD,QAAiB;QADjB,WAAM,GAAN,MAAM,CAA0C;QAChD,aAAQ,GAAR,QAAQ,CAAS;IAC1B,CAAC;IAEJ;;;;OAIG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,QAAiB;QAC5C,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,gBAAgB,CAAC,QAAQ,GAAG,aAAa,CAAC;QAC1C,gBAAgB,CAAC,MAAM,GAAG,EAAE,CAAC;QAE7B,OAAO,aAAa,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,CAAC;IACrE,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,IAAI,CAAC,GAAW,EAAE,OAAe,EAAE,QAAiB;QACvD,aAAa;QACb,OAAO,IAAI,aAAa,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;IACnE,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,QAAiB;QACtD,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,SAAS,GAAG,CAAC,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC;QACzD,CAAC;QAED,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC;gBAC/B,GAAG;gBACH,QAAQ;gBACR,MAAM,EAAE,KAAK;gBACb,YAAY,EAAE,MAAM;aACvB,CAAC,CAAC;YAEH,aAAa;YACb,OAAO,IAAI,aAAa,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAC;QACpF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAI,CAAC,YAAY,SAAS,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gBAC1D,OAAO,IAAI,aAAa,CACpB;oBACI,SAAS;wBACL,OAAO,IAAI,CAAC;oBAChB,CAAC;oBACD,WAAW;wBACP,OAAO,EAAE,CAAC;oBACd,CAAC;iBACJ,EACD,QAAQ,CACX,CAAC;YACN,CAAC;YACD,MAAM,CAAC,CAAC;QACZ,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,GAAW,EAAE,SAAS,GAAG,GAAG;QAClC,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,CAAC,+FAA+F;IACzJ,CAAC;IAED;;OAEG;IACH,WAAW;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACvB,OAAO,CAAC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC;IAC7C,CAAC;CACJ;AAED,+BAA+B;AAC/B,OAAO,EAAE,aAAa,IAAI,UAAU,EAAE,CAAC"}
+{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/internals/robots.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAGvD,OAAO,YAAY,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,OAAO,aAAa;IAEV;IACA;IACA;IAHZ,YACY,MAAgD,EAChD,QAAiB,EACjB,MAAsB;QAFtB,WAAM,GAAN,MAAM,CAA0C;QAChD,aAAQ,GAAR,QAAQ,CAAS;QACjB,WAAM,GAAN,MAAM,CAAgB;IAC/B,CAAC;IAEJ;;;;;;;OAOG;IACH,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,GAAW,EACX,OAMC;QAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,gBAAgB,CAAC,QAAQ,GAAG,aAAa,CAAC;QAC1C,gBAAgB,CAAC,MAAM,GAAG,EAAE,CAAC;QAE7B,OAAO,aAAa,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,IAAI,CAAC,GAAW,EAAE,OAAe,EAAE,QAAiB;QACvD,aAAa;QACb,OAAO,IAAI,aAAa,CAAC,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;IACnE,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,IAAI,CACvB,GAAW,EACX,OAMC;QAED,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,GAAG,IAAI,eAAe,EAAE,EAAE,GAAG,OAAO,IAAI,EAAE,CAAC;QAE/E,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE;YAC/E,QAAQ;YACR,aAAa,EAAE,OAAO,EAAE,aAAa;YACrC,MAAM,EAAE,OAAO,EAAE,MAAM;SAC1B,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CAAC,kCAAkC,GAAG,UAAU,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACtF,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC1B,OAAO,IAAI,aAAa,CACpB;gBACI,SAAS;oBACL,OAAO,IAAI,CAAC;gBAChB,CAAC;gBACD,WAAW;oBACP,OAAO,EAAE,CAAC;gBACd,CAAC;aACJ,EACD,QAAQ,EACR,MAAM,CACT,CAAC;QACN,CAAC;QAED,aAAa;QACb,OAAO,IAAI,aAAa,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IACpG,CAAC;IAED;;;;OAIG;IACH,SAAS,CAAC,GAAW,EAAE,SAAS,GAAG,GAAG;QAClC,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,CAAC,+FAA+F;IACzJ,CAAC;IAED;;OAEG;IACH,WAAW;QACP,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa;QACf,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3F,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB;QACvB,OAAO,CAAC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC;IAC7C,CAAC;CACJ;AAED,+BAA+B;AAC/B,OAAO,EAAE,aAAa,IAAI,UAAU,EAAE,CAAC"}

package/internals/sitemap.d.ts

@@ -1,5 +1,4 @@
-// @ts-ignore optional peer dependency or compatibility with es2022
-import type { Delays } from 'got-scraping';
+import type { BaseHttpClient, CrawleeLogger } from '@crawlee/types';
 interface SitemapUrlData {
     loc: string;
     lastmod?: Date;
@@ -36,9 +35,22 @@ export interface ParseSitemapOptions {
      */
     sitemapRetries?: number;
     /**
-     * Network timeouts for sitemap fetching. See [Got documentation](https://github.com/sindresorhus/got/blob/main/documentation/6-timeout.md) for more details.
+     * Timeout settings for network requests when fetching sitemaps. By default this is `30000` milliseconds (30 seconds).
      */
-    networkTimeouts?: Delays;
+    timeoutMillis?: number;
+    /**
+     * If true, the parser will log a warning if it fails to fetch a sitemap due to a network error
+     * @default true
+     */
+    reportNetworkErrors?: boolean;
+    /**
+     * Custom HTTP client to be used for fetching sitemaps.
+     */
+    httpClient?: BaseHttpClient;
+    /**
+     * Optional logger for reporting warnings during sitemap parsing.
+     */
+    logger?: CrawleeLogger;
 }
 export declare function parseSitemap<T extends ParseSitemapOptions>(initialSources: SitemapSource[], proxyUrl?: string, options?: T): AsyncIterable<T['emitNestedSitemaps'] extends true ? SitemapUrl | NestedSitemap : SitemapUrl>;
 /**
@@ -62,7 +74,7 @@ export declare class Sitemap {
      * @param url The domain URL to fetch the sitemap for.
      * @param proxyUrl A proxy to be used for fetching the sitemap file.
      */
-    static tryCommonNames(url: string, proxyUrl?: string): Promise<Sitemap>;
+    static tryCommonNames(url: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
     /**
      * Fetch sitemap content from given URL or URLs and return URLs of referenced pages.
      * @param urls sitemap URL(s)
@@ -74,8 +86,47 @@ export declare class Sitemap {
      * @param content XML sitemap content
      * @param proxyUrl URL of a proxy to be used for fetching sitemap contents
      */
-    static fromXmlString(content: string, proxyUrl?: string): Promise<Sitemap>;
+    static fromXmlString(content: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
     protected static parse(sources: SitemapSource[], proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
 }
+/**
+ * Given a list of URLs, discover related sitemap files for these domains by checking the `robots.txt` file,
+ * the default `sitemap.xml` & `sitemap.txt` files and the URLs themselves.
+ * @param `urls` The list of URLs to discover sitemaps for.
+ * @param `options` Options for sitemap discovery
+ * @returns An async iterable with the discovered sitemap URLs.
+ */
+export declare function discoverValidSitemaps(urls: string[], options?: {
+    /**
+     * Proxy URL to be used for network requests.
+     */
+    proxyUrl?: string;
+    /**
+     * Timeout in milliseconds for the entire `discoverValidSitemaps` call.
+     * An `AbortController` is created internally and its signal is passed to every HTTP request,
+     * so the whole discovery operation is cancelled once the timeout elapses.
+     * Defaults to `60_000` ms (60 seconds) to prevent indefinite hangs.
+     */
+    timeoutMillis?: number;
+    /**
+     * An external `AbortSignal` to cancel the entire discovery operation.
+     * If both `signal` and `timeout` are provided, the operation is cancelled
+     * when either the signal is aborted or the timeout elapses (whichever comes first).
+     */
+    signal?: AbortSignal;
+    /**
+     * Timeout in milliseconds for each individual HTTP request during discovery.
+     * Defaults to `20000` ms (20 seconds).
+     */
+    requestTimeoutMillis?: number;
+    /**
+     * HTTP client to be used for network requests.
+     */
+    httpClient?: BaseHttpClient;
+    /**
+     * Optional logger for reporting warnings during sitemap discovery.
+     */
+    logger?: CrawleeLogger;
+}): AsyncIterable<string>;
 export {};
 //# sourceMappingURL=sitemap.d.ts.map

package/internals/sitemap.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAM3C,UAAU,cAAc;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,IAAI,CAAC;IACf,UAAU,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;IACvF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG;IACtC,gBAAgB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,UAAU,aAAa;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,gBAAgB,EAAE,IAAI,CAAC;CAC1B;AAED,KAAK,aAAa,GAAG,CAAC;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA8I5G,MAAM,WAAW,mBAAmB;IAChC;;OAEG;IACH,kBAAkB,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC;IAClC;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,wBAAuB,YAAY,CAAC,CAAC,SAAS,mBAAmB,EAC7D,cAAc,EAAE,aAAa,EAAE,EAC/B,QAAQ,CAAC,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,CAAC,GACZ,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,SAAS,IAAI,GAAG,UAAU,GAAG,aAAa,GAAG,UAAU,CAAC,CAgJ/F;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,OAAO;IACJ,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE;gBAAd,IAAI,EAAE,MAAM,EAAE;IAEnC;;;;;OAKG;WACU,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAe7E;;;;OAIG;WACU,IAAI,CACb,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,EACvB,QAAQ,CAAC,EAAE,MAAM,EACjB,mBAAmB,CAAC,EAAE,mBAAmB,GAC1C,OAAO,CAAC,OAAO,CAAC;IAQnB;;;;OAIG;WACU,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;qBAIzD,KAAK,CACxB,OAAO,EAAE,aAAa,EAAE,EACxB,QAAQ,CAAC,EAAE,MAAM,EACjB,mBAAmB,CAAC,EAAE,mBAAmB,GAC1C,OAAO,CAAC,OAAO,CAAC;CAatB"}
+{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/internals/sitemap.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAQpE,UAAU,cAAc;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,IAAI,CAAC;IACf,UAAU,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,CAAC;IACvF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG;IACtC,gBAAgB,EAAE,MAAM,CAAC;CAC5B,CAAC;AAEF,UAAU,aAAa;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,gBAAgB,EAAE,IAAI,CAAC;CAC1B;AAED,KAAK,aAAa,GAAG,CAAC;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AA8I5G,MAAM,WAAW,mBAAmB;IAChC;;OAEG;IACH,kBAAkB,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC;IAClC;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,wBAAuB,YAAY,CAAC,CAAC,SAAS,mBAAmB,EAC7D,cAAc,EAAE,aAAa,EAAE,EAC/B,QAAQ,CAAC,EAAE,MAAM,EACjB,OAAO,CAAC,EAAE,CAAC,GACZ,aAAa,CAAC,CAAC,CAAC,oBAAoB,CAAC,SAAS,IAAI,GAAG,UAAU,GAAG,aAAa,GAAG,UAAU,CAAC,CAgK/F;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,OAAO;IACJ,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE;gBAAd,IAAI,EAAE,MAAM,EAAE;IAEnC;;;;;OAKG;WACU,cAAc,CACvB,GAAG,EAAE,MAAM,EACX,QAAQ,CAAC,EAAE,MAAM,EACjB,mBAAmB,CAAC,EAAE,mBAAmB,GAC1C,OAAO,CAAC,OAAO,CAAC;IAenB;;;;OAIG;WACU,IAAI,CACb,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,EACvB,QAAQ,CAAC,EAAE,MAAM,EACjB,mBAAmB,CAAC,EAAE,mBAAmB,GAC1C,OAAO,CAAC,OAAO,CAAC;IAQnB;;;;OAIG;WACU,aAAa,CACtB,OAAO,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,EACjB,mBAAmB,CAAC,EAAE,mBAAmB,GAC1C,OAAO,CAAC,OAAO,CAAC;qBAII,KAAK,CACxB,OAAO,EAAE,aAAa,EAAE,EACxB,QAAQ,CAAC,EAAE,MAAM,EACjB,mBAAmB,CAAC,EAAE,mBAAmB,GAC1C,OAAO,CAAC,OAAO,CAAC;CAgBtB;AAED;;;;;;GAMG;AACH,wBAAuB,qBAAqB,CACxC,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE;IACL;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;OAEG;IACH,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B;;OAEG;IACH,MAAM,CAAC,EAAE,aAAa,CAAC;CACrB,GACP,aAAa,CAAC,MAAM,CAAC,CAkIvB"}