@crawlee/utils 3.0.0-alpha.2 → 3.0.0-alpha.22

package/index.d.ts

@@ -0,0 +1,7 @@
+export * from './internals/cheerio';
+export * from './internals/extract-urls';
+export * from './internals/general';
+export * from './internals/memory-info';
+export * as social from './internals/social';
+export * from './internals/typedefs';
+//# sourceMappingURL=index.d.ts.map

package/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qBAAqB,CAAC;AACpC,cAAc,yBAAyB,CAAC;AACxC,OAAO,KAAK,MAAM,MAAM,oBAAoB,CAAC;AAC7C,cAAc,sBAAsB,CAAC"}

package/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;AAAA,8DAAoC;AACpC,mEAAyC;AACzC,8DAAoC;AACpC,kEAAwC;AACxC,qEAA6C;AAC7C,+DAAqC"}

package/index.mjs

@@ -0,0 +1,18 @@
+import mod from "./index.js";
+
+export default mod;
+export const URL_NO_COMMAS_REGEX = mod.URL_NO_COMMAS_REGEX;
+export const URL_WITH_COMMAS_REGEX = mod.URL_WITH_COMMAS_REGEX;
+export const downloadListOfUrls = mod.downloadListOfUrls;
+export const entries = mod.entries;
+export const extractUrls = mod.extractUrls;
+export const getMemoryInfo = mod.getMemoryInfo;
+export const htmlToText = mod.htmlToText;
+export const isDocker = mod.isDocker;
+export const keys = mod.keys;
+export const parseContentTypeFromResponse = mod.parseContentTypeFromResponse;
+export const purgeLocalStorage = mod.purgeLocalStorage;
+export const sleep = mod.sleep;
+export const snakeCaseToCamelCase = mod.snakeCaseToCamelCase;
+export const social = mod.social;
+export const weightedAvg = mod.weightedAvg;

package/internals/cheerio.d.ts

@@ -0,0 +1,32 @@
+import { load } from 'cheerio';
+export declare type CheerioRoot = ReturnType<typeof load>;
+/**
+ * The function converts a HTML document to a plain text.
+ *
+ * The plain text generated by the function is similar to a text captured
+ * by pressing Ctrl+A and Ctrl+C on a page when loaded in a web browser.
+ * The function doesn't aspire to preserve the formatting or to be perfectly correct with respect to HTML specifications.
+ * However, it attempts to generate newlines and whitespaces in and around HTML elements
+ * to avoid merging distinct parts of text and thus enable extraction of data from the text (e.g. phone numbers).
+ *
+ * **Example usage**
+ * ```javascript
+ * const text = htmlToText('<html><body>Some text</body></html>');
+ * console.log(text);
+ * ```
+ *
+ * Note that the function uses [cheerio](https://www.npmjs.com/package/cheerio) to parse the HTML.
+ * Optionally, to avoid duplicate parsing of HTML and thus improve performance, you can pass
+ * an existing Cheerio object to the function instead of the HTML text. The HTML should be parsed
+ * with the `decodeEntities` option set to `true`. For example:
+ *
+ * ```javascript
+ * const cheerio = require('cheerio');
+ * const html = '<html><body>Some text</body></html>';
+ * const text = htmlToText(cheerio.load(html, { decodeEntities: true }));
+ * ```
+ * @param htmlOrCheerioElement HTML text or parsed HTML represented using a [cheerio](https://www.npmjs.com/package/cheerio) function.
+ * @return Plain text
+ */
+export declare function htmlToText(htmlOrCheerioElement: string | CheerioRoot): string;
+//# sourceMappingURL=cheerio.d.ts.map

package/internals/cheerio.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cheerio.d.ts","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":"AAAA,OAAgB,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAExC,oBAAY,WAAW,GAAG,UAAU,CAAC,OAAO,IAAI,CAAC,CAAC;AAMlD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,UAAU,CAAC,oBAAoB,EAAE,MAAM,GAAG,WAAW,GAAG,MAAM,CAiD7E"}

package/internals/cheerio.js

@@ -0,0 +1,94 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.htmlToText = void 0;
+const tslib_1 = require("tslib");
+const cheerio_1 = tslib_1.__importDefault(require("cheerio"));
+// NOTE: We are skipping 'noscript' since it's content is evaluated as text, instead of HTML elements. That damages the results.
+const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript)$/i;
+const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|dl|div|fieldset|form|table|tr|select|option)$/i;
+/**
+ * The function converts a HTML document to a plain text.
+ *
+ * The plain text generated by the function is similar to a text captured
+ * by pressing Ctrl+A and Ctrl+C on a page when loaded in a web browser.
+ * The function doesn't aspire to preserve the formatting or to be perfectly correct with respect to HTML specifications.
+ * However, it attempts to generate newlines and whitespaces in and around HTML elements
+ * to avoid merging distinct parts of text and thus enable extraction of data from the text (e.g. phone numbers).
+ *
+ * **Example usage**
+ * ```javascript
+ * const text = htmlToText('<html><body>Some text</body></html>');
+ * console.log(text);
+ * ```
+ *
+ * Note that the function uses [cheerio](https://www.npmjs.com/package/cheerio) to parse the HTML.
+ * Optionally, to avoid duplicate parsing of HTML and thus improve performance, you can pass
+ * an existing Cheerio object to the function instead of the HTML text. The HTML should be parsed
+ * with the `decodeEntities` option set to `true`. For example:
+ *
+ * ```javascript
+ * const cheerio = require('cheerio');
+ * const html = '<html><body>Some text</body></html>';
+ * const text = htmlToText(cheerio.load(html, { decodeEntities: true }));
+ * ```
+ * @param htmlOrCheerioElement HTML text or parsed HTML represented using a [cheerio](https://www.npmjs.com/package/cheerio) function.
+ * @return Plain text
+ */
+function htmlToText(htmlOrCheerioElement) {
+    if (!htmlOrCheerioElement)
+        return '';
+    // TODO: Add support for "html" being a Cheerio element, otherwise the only way
+    //  to use it is e.g. htmlToText($('p').html())) which is inefficient
+    //  Also, it seems this doesn't work well in CheerioScraper, e.g. htmlToText($)
+    //  produces really text with a lot of HTML elements in it. Let's just deprecate this sort of usage,
+    //  and make the parameter "htmlOrCheerioElement"
+    const $ = typeof htmlOrCheerioElement === 'function' ? htmlOrCheerioElement : cheerio_1.default.load(htmlOrCheerioElement, { decodeEntities: true });
+    let text = '';
+    // TODO: the type for elems is very annoying to work with.
+    //  The correct type is Node[] from cheerio but it needs a lot more casting in each branch, or alternatively,
+    //  use the is* methods from domhandler (isText, isTag, isComment, etc.)
+    // @ts-expect-error
+    const process = (elems) => {
+        const len = elems ? elems.length : 0;
+        for (let i = 0; i < len; i++) {
+            const elem = elems[i];
+            if (elem.type === 'text') {
+                // Compress spaces, unless we're inside <pre> element
+                let compr;
+                if (elem.parent && elem.parent.tagName === 'pre')
+                    compr = elem.data;
+                else
+                    compr = elem.data.replace(/\s+/g, ' ');
+                // If text is empty or ends with a whitespace, don't add the leading whitepsace
+                if (compr.startsWith(' ') && /(^|\s)$/.test(text))
+                    compr = compr.substr(1);
+                text += compr;
+            }
+            else if (elem.type === 'comment' || SKIP_TAGS_REGEX.test(elem.tagName)) {
+                // Skip comments and special elements
+            }
+            else if (elem.tagName === 'br') {
+                text += '\n';
+            }
+            else if (elem.tagName === 'td') {
+                process(elem.children);
+                text += '\t';
+            }
+            else {
+                // Block elements must be surrounded by newlines (unless beginning of text)
+                const isBlockTag = BLOCK_TAGS_REGEX.test(elem.tagName);
+                if (isBlockTag && !/(^|\n)$/.test(text))
+                    text += '\n';
+                process(elem.children);
+                if (isBlockTag && !text.endsWith('\n'))
+                    text += '\n';
+            }
+        }
+    };
+    // If HTML document has body, only convert that, otherwise convert the entire HTML
+    const $body = $('body');
+    process($body.length > 0 ? $body : $.root());
+    return text.trim();
+}
+exports.htmlToText = htmlToText;
+//# sourceMappingURL=cheerio.js.map

package/internals/cheerio.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../src/internals/cheerio.ts"],"names":[],"mappings":";;;;AAAA,8DAAwC;AAIxC,gIAAgI;AAChI,MAAM,eAAe,GAAG,uCAAuC,CAAC;AAChE,MAAM,gBAAgB,GAAG,sGAAsG,CAAC;AAEhI;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,SAAgB,UAAU,CAAC,oBAA0C;IACjE,IAAI,CAAC,oBAAoB;QAAE,OAAO,EAAE,CAAC;IAErC,+EAA+E;IAC/E,qEAAqE;IACrE,+EAA+E;IAC/E,oGAAoG;IACpG,iDAAiD;IACjD,MAAM,CAAC,GAAG,OAAO,oBAAoB,KAAK,UAAU,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,iBAAO,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3I,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,0DAA0D;IAC1D,6GAA6G;IAC7G,wEAAwE;IACxE,mBAAmB;IACnB,MAAM,OAAO,GAAG,CAAC,KAAK,EAAE,EAAE;QACtB,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE;YAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;gBACtB,qDAAqD;gBACrD,IAAI,KAAK,CAAC;gBACV,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK;oBAAE,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC;;oBAC/D,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC5C,+EAA+E;gBAC/E,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAC3E,IAAI,IAAI,KAAK,CAAC;aACjB;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;gBACtE,qCAAqC;aACxC;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE;gBAC9B,IAAI,IAAI,IAAI,CAAC;aAChB;iBAAM,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE;gBAC9B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,IAAI,IAAI,CAAC;aAChB;iBAAM;gBACH,2EAA2E;gBAC3E,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvD,IAAI,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;gBACtD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,IAAI,UAAU,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAAE,IAAI,IAAI,IAAI,CAAC;aACxD;SACJ;IACL,CAAC,CAAC;IAEF,kFAAkF;IAClF,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;IACxB,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAE7C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AAjDD,gCAiDC"}

package/internals/extract-urls.d.ts

@@ -0,0 +1,39 @@
+/// <reference types="node" />
+export interface DownloadListOfUrlsOptions {
+    /**
+     * URL to the file
+     */
+    url: string;
+    /**
+     * The encoding of the file.
+     * @default 'utf8'
+     */
+    encoding?: BufferEncoding;
+    /**
+     * Custom regular expression to identify the URLs in the file to extract.
+     * The regular expression should be case-insensitive and have global flag set (i.e. `/something/gi`).
+     * @default Actor.utils.URL_NO_COMMAS_REGEX
+     */
+    urlRegExp?: RegExp;
+}
+/**
+ * Returns a promise that resolves to an array of urls parsed from the resource available at the provided url.
+ * Optionally, custom regular expression and encoding may be provided.
+ */
+export declare function downloadListOfUrls(options: DownloadListOfUrlsOptions): Promise<string[]>;
+export interface ExtractUrlsOptions {
+    /**
+     * The string to extract URLs from.
+     */
+    string: string;
+    /**
+     * Custom regular expression
+     * @default URL_NO_COMMAS_REGEX
+     */
+    urlRegExp?: RegExp;
+}
+/**
+ * Collects all URLs in an arbitrary string to an array, optionally using a custom regular expression.
+ */
+export declare function extractUrls(options: ExtractUrlsOptions): string[];
+//# sourceMappingURL=extract-urls.d.ts.map

package/internals/extract-urls.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-urls.d.ts","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":";AAIA,MAAM,WAAW,yBAAyB;IACtC;;OAEG;IACH,GAAG,EAAE,MAAM,CAAC;IAEZ;;;OAGG;IACH,QAAQ,CAAC,EAAE,cAAc,CAAC;IAE1B;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAqB9F;AAED,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,kBAAkB,GAAG,MAAM,EAAE,CAOjE"}

package/internals/extract-urls.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractUrls = exports.downloadListOfUrls = void 0;
+const tslib_1 = require("tslib");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const got_scraping_1 = require("got-scraping");
+const general_1 = require("./general");
+/**
+ * Returns a promise that resolves to an array of urls parsed from the resource available at the provided url.
+ * Optionally, custom regular expression and encoding may be provided.
+ */
+async function downloadListOfUrls(options) {
+    (0, ow_1.default)(options, ow_1.default.object.exactShape({
+        url: ow_1.default.string.url,
+        encoding: ow_1.default.optional.string,
+        urlRegExp: ow_1.default.optional.regExp,
+        proxyUrl: ow_1.default.optional.string,
+    }));
+    const { url, encoding = 'utf8', urlRegExp = general_1.URL_NO_COMMAS_REGEX, proxyUrl } = options;
+    // Try to detect wrong urls and fix them. Currently, detects only sharing url instead of csv download one.
+    const match = url.match(/^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/?/);
+    let fixedUrl = url;
+    if (match) {
+        fixedUrl = `${match[1]}/gviz/tq?tqx=out:csv`;
+    }
+    // @ts-expect-error https://github.com/apify/got-scraping/issues/66
+    const { body: string } = await (0, got_scraping_1.gotScraping)({ url: fixedUrl, encoding, proxyUrl });
+    return extractUrls({ string, urlRegExp });
+}
+exports.downloadListOfUrls = downloadListOfUrls;
+/**
+ * Collects all URLs in an arbitrary string to an array, optionally using a custom regular expression.
+ */
+function extractUrls(options) {
+    (0, ow_1.default)(options, ow_1.default.object.exactShape({
+        string: ow_1.default.string,
+        urlRegExp: ow_1.default.optional.regExp,
+    }));
+    const { string, urlRegExp = general_1.URL_NO_COMMAS_REGEX } = options;
+    return string.match(urlRegExp) || [];
+}
+exports.extractUrls = extractUrls;
+//# sourceMappingURL=extract-urls.js.map

package/internals/extract-urls.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-urls.js","sourceRoot":"","sources":["../../src/internals/extract-urls.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,+CAA2C;AAC3C,uCAAgD;AAsBhD;;;GAGG;AACI,KAAK,UAAU,kBAAkB,CAAC,OAAkC;IACvE,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,GAAG;QAClB,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC5B,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC7B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAC/B,CAAC,CAAC,CAAC;IACJ,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,MAAM,EAAE,SAAS,GAAG,6BAAmB,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;IAEtF,0GAA0G;IAC1G,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;IACzF,IAAI,QAAQ,GAAG,GAAG,CAAC;IAEnB,IAAI,KAAK,EAAE;QACP,QAAQ,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,sBAAsB,CAAC;KAChD;IAED,mEAAmE;IACnE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,0BAAW,EAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;IAElF,OAAO,WAAW,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AAC9C,CAAC;AArBD,gDAqBC;AAeD;;GAEG;AACH,SAAgB,WAAW,CAAC,OAA2B;IACnD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;QAC7B,MAAM,EAAE,YAAE,CAAC,MAAM;QACjB,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAChC,CAAC,CAAC,CAAC;IACJ,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,6BAAmB,EAAE,GAAG,OAAO,CAAC;IAC5D,OAAO,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;AACzC,CAAC;AAPD,kCAOC"}

package/internals/general.d.ts

@@ -0,0 +1,62 @@
+/// <reference types="node" />
+import type { IncomingMessage } from 'node:http';
+/**
+ * Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters
+ * and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
+ */
+export declare const URL_NO_COMMAS_REGEX: RegExp;
+/**
+ * Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query.
+ * Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
+ */
+export declare const URL_WITH_COMMAS_REGEX: RegExp;
+/**
+ * Returns a `Promise` that resolves to true if the code is running in a Docker container.
+ */
+export declare function isDocker(forceReset?: boolean): Promise<boolean>;
+/**
+ * Computes a weighted average of an array of numbers, complemented by an array of weights.
+ * @ignore
+ */
+export declare function weightedAvg(arrValues: number[], arrWeights: number[]): number;
+/**
+ * Returns a `Promise` that resolves after a specific period of time. This is useful to implement waiting
+ * in your code, e.g. to prevent overloading of target website or to avoid bot detection.
+ *
+ * **Example usage:**
+ *
+ * ```
+ * const { sleep } = require('@crawlee/utils');
+ *
+ * ...
+ *
+ * // Sleep 1.5 seconds
+ * await sleep(1500);
+ * ```
+ * @param millis Period of time to sleep, in milliseconds. If not a positive number, the returned promise resolves immediately.
+ */
+export declare function sleep(millis?: number): Promise<void>;
+/**
+ * Converts SNAKE_CASE to camelCase.
+ * @ignore
+ */
+export declare function snakeCaseToCamelCase(snakeCaseStr: string): string;
+/**
+ * Cleans up the local storage folder created when testing locally.
+ * This is useful in the event you are debugging your code locally.
+ *
+ * Be careful as this will remove the folder you provide and everything in it!
+ *
+ * @param [folder] The folder to clean up
+ */
+export declare function purgeLocalStorage(folder?: string): Promise<void>;
+/**
+ * Gets parsed content type from response object
+ * @param response HTTP response object
+ * @ignore
+ */
+export declare function parseContentTypeFromResponse(response: IncomingMessage): {
+    type: string;
+    charset: BufferEncoding;
+};
+//# sourceMappingURL=general.d.ts.map

package/internals/general.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"general.d.ts","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":";AAIA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAMjD;;;GAGG;AACH,eAAO,MAAM,mBAAmB,QAA0J,CAAC;AAE3L;;;GAGG;AACH,eAAO,MAAM,qBAAqB,QAA2J,CAAC;AAkB9L;;GAEG;AACH,wBAAgB,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAK/D;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,MAAM,CAS7E;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEpD;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAUjE;AAED;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAQtE;AAED;;;;GAIG;AACH,wBAAgB,4BAA4B,CAAC,QAAQ,EAAE,eAAe,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,cAAc,CAAA;CAAE,CA6BjH"}

package/internals/general.js

@@ -0,0 +1,143 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseContentTypeFromResponse = exports.purgeLocalStorage = exports.snakeCaseToCamelCase = exports.sleep = exports.weightedAvg = exports.isDocker = exports.URL_WITH_COMMAS_REGEX = exports.URL_NO_COMMAS_REGEX = void 0;
+const tslib_1 = require("tslib");
+const consts_1 = require("@apify/consts");
+const content_type_1 = tslib_1.__importDefault(require("content-type"));
+const mime_types_1 = tslib_1.__importDefault(require("mime-types"));
+const promises_1 = tslib_1.__importDefault(require("node:fs/promises"));
+const node_path_1 = tslib_1.__importDefault(require("node:path"));
+const promises_2 = require("node:timers/promises");
+const node_url_1 = require("node:url");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+/**
+ * Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters
+ * and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
+ */
+exports.URL_NO_COMMAS_REGEX = RegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+.~#?&//=\\(\\)]*)?', 'giu'); // eslint-disable-line
+/**
+ * Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query.
+ * Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
+ */
+exports.URL_WITH_COMMAS_REGEX = RegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+,.~#?&//=\\(\\)]*)?', 'giu'); // eslint-disable-line
+let isDockerPromiseCache;
+async function createIsDockerPromise() {
+    const promise1 = promises_1.default.stat('/.dockerenv')
+        .then(() => true)
+        .catch(() => false);
+    const promise2 = promises_1.default.readFile('/proc/self/cgroup', 'utf8')
+        .then((content) => content.includes('docker'))
+        .catch(() => false);
+    const [result1, result2] = await Promise.all([promise1, promise2]);
+    return result1 || result2;
+}
+/**
+ * Returns a `Promise` that resolves to true if the code is running in a Docker container.
+ */
+function isDocker(forceReset) {
+    // Parameter forceReset is just internal for unit tests.
+    if (!isDockerPromiseCache || forceReset)
+        isDockerPromiseCache = createIsDockerPromise();
+    return isDockerPromiseCache;
+}
+exports.isDocker = isDocker;
+/**
+ * Computes a weighted average of an array of numbers, complemented by an array of weights.
+ * @ignore
+ */
+function weightedAvg(arrValues, arrWeights) {
+    const result = arrValues.map((value, i) => {
+        const weight = arrWeights[i];
+        const sum = value * weight;
+        return [sum, weight];
+    }).reduce((p, c) => [p[0] + c[0], p[1] + c[1]], [0, 0]);
+    return result[0] / result[1];
+}
+exports.weightedAvg = weightedAvg;
+/**
+ * Returns a `Promise` that resolves after a specific period of time. This is useful to implement waiting
+ * in your code, e.g. to prevent overloading of target website or to avoid bot detection.
+ *
+ * **Example usage:**
+ *
+ * ```
+ * const { sleep } = require('@crawlee/utils');
+ *
+ * ...
+ *
+ * // Sleep 1.5 seconds
+ * await sleep(1500);
+ * ```
+ * @param millis Period of time to sleep, in milliseconds. If not a positive number, the returned promise resolves immediately.
+ */
+function sleep(millis) {
+    return (0, promises_2.setTimeout)(millis);
+}
+exports.sleep = sleep;
+/**
+ * Converts SNAKE_CASE to camelCase.
+ * @ignore
+ */
+function snakeCaseToCamelCase(snakeCaseStr) {
+    return snakeCaseStr
+        .toLowerCase()
+        .split('_')
+        .map((part, index) => {
+        return index > 0
+            ? part.charAt(0).toUpperCase() + part.slice(1)
+            : part;
+    })
+        .join('');
+}
+exports.snakeCaseToCamelCase = snakeCaseToCamelCase;
+/**
+ * Cleans up the local storage folder created when testing locally.
+ * This is useful in the event you are debugging your code locally.
+ *
+ * Be careful as this will remove the folder you provide and everything in it!
+ *
+ * @param [folder] The folder to clean up
+ */
+async function purgeLocalStorage(folder) {
+    // If the user did not provide a folder, try to get it from the env variables, or the default one
+    if (!folder) {
+        folder = process.env[consts_1.ENV_VARS.LOCAL_STORAGE_DIR] || 'apify_storage';
+    }
+    // Clear the folder
+    await promises_1.default.rm(folder, { recursive: true, force: true });
+}
+exports.purgeLocalStorage = purgeLocalStorage;
+/**
+ * Gets parsed content type from response object
+ * @param response HTTP response object
+ * @ignore
+ */
+function parseContentTypeFromResponse(response) {
+    (0, ow_1.default)(response, ow_1.default.object.partialShape({
+        url: ow_1.default.string.url,
+        headers: ow_1.default.object,
+    }));
+    const { url, headers } = response;
+    let parsedContentType;
+    if (headers['content-type']) {
+        try {
+            parsedContentType = content_type_1.default.parse(headers['content-type']);
+        }
+        catch {
+            // Can not parse content type from Content-Type header. Try to parse it from file extension.
+        }
+    }
+    // Parse content type from file extension as fallback
+    if (!parsedContentType) {
+        const parsedUrl = new node_url_1.URL(url);
+        const contentTypeFromExtname = mime_types_1.default.contentType(node_path_1.default.extname(parsedUrl.pathname))
+            || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5
+        parsedContentType = content_type_1.default.parse(contentTypeFromExtname);
+    }
+    return {
+        type: parsedContentType.type,
+        charset: parsedContentType.parameters.charset,
+    };
+}
+exports.parseContentTypeFromResponse = parseContentTypeFromResponse;
+//# sourceMappingURL=general.js.map

package/internals/general.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"general.js","sourceRoot":"","sources":["../../src/internals/general.ts"],"names":[],"mappings":";;;;AAAA,0CAAyC;AACzC,wEAA6C;AAC7C,oEAA8B;AAC9B,wEAAkC;AAElC,kEAA6B;AAC7B,mDAAkD;AAClD,uCAA+B;AAC/B,oDAAoB;AAEpB;;;GAGG;AACU,QAAA,mBAAmB,GAAG,MAAM,CAAC,wIAAwI,EAAE,KAAK,CAAC,CAAC,CAAC,sBAAsB;AAElN;;;GAGG;AACU,QAAA,qBAAqB,GAAG,MAAM,CAAC,yIAAyI,EAAE,KAAK,CAAC,CAAC,CAAC,sBAAsB;AAErN,IAAI,oBAAsC,CAAC;AAE3C,KAAK,UAAU,qBAAqB;IAChC,MAAM,QAAQ,GAAG,kBAAE,CAAC,IAAI,CAAC,aAAa,CAAC;SAClC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;SAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,QAAQ,GAAG,kBAAE,CAAC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACpD,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SAC7C,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;IAExB,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAEnE,OAAO,OAAO,IAAI,OAAO,CAAC;AAC9B,CAAC;AAED;;GAEG;AACH,SAAgB,QAAQ,CAAC,UAAoB;IACzC,wDAAwD;IACxD,IAAI,CAAC,oBAAoB,IAAI,UAAU;QAAE,oBAAoB,GAAG,qBAAqB,EAAE,CAAC;IAExF,OAAO,oBAAoB,CAAC;AAChC,CAAC;AALD,4BAKC;AAED;;;GAGG;AACH,SAAgB,WAAW,CAAC,SAAmB,EAAE,UAAoB;IACjE,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;QACtC,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC;QAE3B,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAExD,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;AACjC,CAAC;AATD,kCASC;AAED;;;;;;;;;;;;;;;GAeG;AACH,SAAgB,KAAK,CAAC,MAAe;IACjC,OAAO,IAAA,qBAAU,EAAC,MAAM,CAAC,CAAC;AAC9B,CAAC;AAFD,sBAEC;AAED;;;GAGG;AACH,SAAgB,oBAAoB,CAAC,YAAoB;IACrD,OAAO,YAAY;SACd,WAAW,EAAE;SACb,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACjB,OAAO,KAAK,GAAG,CAAC;YACZ,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;YAC9C,CAAC,CAAC,IAAI,CAAC;IACf,CAAC,CAAC;SACD,IAAI,CAAC,EAAE,CAAC,CAAC;AAClB,CAAC;AAVD,oDAUC;AAED;;;;;;;GAOG;AACI,KAAK,UAAU,iBAAiB,CAAC,MAAe;IACnD,iGAAiG;IACjG,IAAI,CAAC,MAAM,EAAE;QACT,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAQ,CAAC,iBAAiB,CAAC,IAAI,eAAe,CAAC;KACvE;IAED,mBAAmB;IACnB,MAAM,kBAAE,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAC1D,CAAC;AARD,8CAQC;AAED;;;;GAIG;AACH,SAAgB,4BAA4B,CAAC,QAAyB;IAClE,IAAA,YAAE,EAAC,QAAQ,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QAChC,GAAG,EAAE,YAAE,CAAC,MAAM,CAAC,GAAG;QAClB,OAAO,EAAE,YAAE,CAAC,MAAM;KACrB,CAAC,CAAC,CAAC;IAEJ,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC;IAClC,IAAI,iBAAiB,CAAC;IAEtB,IAAI,OAAO,CAAC,cAAc,CAAC,EAAE;QACzB,IAAI;YACA,iBAAiB,GAAG,sBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;SACxE;QAAC,MAAM;YACJ,4FAA4F;SAC/F;KACJ;IAED,qDAAqD;IACrD,IAAI,CAAC,iBAAiB,EAAE;QACpB,MAAM,SAAS,GAAG,IAAI,cAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,sBAAsB,GAAG,oBAAI,CAAC,WAAW,CAAC,mBAAI,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;eAC1E,yCAAyC,CAAC,CAAC,0FAA0F;QAC5I,iBAAiB,GAAG,sBAAiB,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;KACvE;IAED,OAAO;QACH,IAAI,EAAE,iBAAiB,CAAC,IAAI;QAC5B,OAAO,EAAE,iBAAiB,CAAC,UAAU,CAAC,OAAyB;KAClE,CAAC;AACN,CAAC;AA7BD,oEA6BC"}

package/internals/memory-info.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Describes memory usage of the process.
+ */
+export interface MemoryInfo {
+    /** Total memory available in the system or container */
+    totalBytes: number;
+    /** Amount of free memory in the system or container */
+    freeBytes: number;
+    /** Amount of memory used (= totalBytes - freeBytes) */
+    usedBytes: number;
+    /** Amount of memory used the current Node.js process */
+    mainProcessBytes: number;
+    /** Amount of memory used by child processes of the current Node.js process */
+    childProcessesBytes: number;
+}
+/**
+ * Returns memory statistics of the process and the system, see {@link MemoryInfo}.
+ *
+ * If the process runs inside of Docker, the `getMemoryInfo` gets container memory limits,
+ * otherwise it gets system memory limits.
+ *
+ * Beware that the function is quite inefficient because it spawns a new process.
+ * Therefore you shouldn't call it too often, like more than once per second.
+ */
+export declare function getMemoryInfo(): Promise<MemoryInfo>;
+//# sourceMappingURL=memory-info.d.ts.map

package/internals/memory-info.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-info.d.ts","sourceRoot":"","sources":["../../src/internals/memory-info.ts"],"names":[],"mappings":"AAqBA;;GAEG;AACH,MAAM,WAAW,UAAU;IACvB,wDAAwD;IACxD,UAAU,EAAE,MAAM,CAAC;IAEnB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAElB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAC;IAElB,wDAAwD;IACxD,gBAAgB,EAAE,MAAM,CAAC;IAEzB,8EAA8E;IAC9E,mBAAmB,EAAE,MAAM,CAAC;CAC/B;AAED;;;;;;;;GAQG;AACH,wBAAsB,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC,CAmHzD"}

package/internals/memory-info.js

@@ -0,0 +1,142 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getMemoryInfo = void 0;
+const tslib_1 = require("tslib");
+const log_1 = tslib_1.__importDefault(require("@apify/log"));
+// @ts-expect-error We need to add typings for @apify/ps-tree
+const ps_tree_1 = tslib_1.__importDefault(require("@apify/ps-tree"));
+const node_child_process_1 = require("node:child_process");
+const promises_1 = tslib_1.__importDefault(require("node:fs/promises"));
+const node_os_1 = tslib_1.__importDefault(require("node:os"));
+const node_util_1 = tslib_1.__importDefault(require("node:util"));
+const general_1 = require("./general");
+const MEMORY_FILE_PATHS = {
+    TOTAL: {
+        V1: '/sys/fs/cgroup/memory/memory.limit_in_bytes',
+        V2: '/sys/fs/cgroup/memory.max',
+    },
+    USED: {
+        V1: '/sys/fs/cgroup/memory/memory.usage_in_bytes',
+        V2: '/sys/fs/cgroup/memory.current',
+    },
+};
+/**
+ * Returns memory statistics of the process and the system, see {@link MemoryInfo}.
+ *
+ * If the process runs inside of Docker, the `getMemoryInfo` gets container memory limits,
+ * otherwise it gets system memory limits.
+ *
+ * Beware that the function is quite inefficient because it spawns a new process.
+ * Therefore you shouldn't call it too often, like more than once per second.
+ */
+async function getMemoryInfo() {
+    const psTreePromised = node_util_1.default.promisify(ps_tree_1.default);
+    // lambda does *not* have `ps` and other command line tools
+    // required to extract memory usage.
+    const isLambdaEnvironment = process.platform === 'linux'
+        && !!process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;
+    const isDockerVar = !isLambdaEnvironment && await (0, general_1.isDocker)();
+    let mainProcessBytes = -1;
+    let childProcessesBytes = 0;
+    if (isLambdaEnvironment) {
+        // reported in bytes
+        mainProcessBytes = process.memoryUsage().rss;
+        // TODO this is quite ugly, let's introduce some local variables
+        // https://stackoverflow.com/a/55914335/129415
+        childProcessesBytes = +(0, node_child_process_1.execSync)('cat /proc/meminfo')
+            .toString()
+            .split(/[\n: ]/)
+            .filter((val) => val.trim())[19]
+            * 1000 // meminfo reports in kb, not bytes
+            // the total used memory is reported by meminfo
+            // subtract memory used by the main node process
+            // in order to infer memory used by any child processes
+            - mainProcessBytes;
+    }
+    else {
+        // Query both root and child processes
+        const processes = await psTreePromised(process.pid, true);
+        processes.forEach((rec) => {
+            // Skip the 'ps' or 'wmic' commands used by ps-tree to query the processes
+            if (rec.COMMAND === 'ps' || rec.COMMAND === 'WMIC.exe') {
+                return;
+            }
+            const bytes = parseInt(rec.RSS, 10);
+            // Obtain main process' memory separately
+            if (rec.PID === `${process.pid}`) {
+                mainProcessBytes = bytes;
+                return;
+            }
+            childProcessesBytes += bytes;
+        });
+    }
+    let totalBytes;
+    let usedBytes;
+    let freeBytes;
+    if (isLambdaEnvironment) {
+        // memory size is defined in megabytes
+        totalBytes = parseInt(process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE, 10) * 1000000;
+        usedBytes = mainProcessBytes + childProcessesBytes;
+        freeBytes = totalBytes - usedBytes;
+        log_1.default.debug(`lambda size of ${totalBytes} with ${freeBytes} free bytes`);
+    }
+    else if (isDockerVar) {
+        // When running inside Docker container, use container memory limits
+        // Check whether cgroups V1 or V2 is used
+        let cgroupsVersion = 'V1';
+        try {
+            // If this directory does not exists, assume docker is using cgroups V2
+            await promises_1.default.access('/sys/fs/cgroup/memory/');
+        }
+        catch {
+            cgroupsVersion = 'V2';
+        }
+        try {
+            let [totalBytesStr, usedBytesStr] = await Promise.all([
+                promises_1.default.readFile(MEMORY_FILE_PATHS.TOTAL[cgroupsVersion], 'utf8'),
+                promises_1.default.readFile(MEMORY_FILE_PATHS.USED[cgroupsVersion], 'utf8'),
+            ]);
+            // Cgroups V2 files contains newline character. Getting rid of it for better handling in later part of the code.
+            totalBytesStr = totalBytesStr.replace(/[^a-zA-Z0-9 ]/g, '');
+            usedBytesStr = usedBytesStr.replace(/[^a-zA-Z0-9 ]/g, '');
+            // Cgroups V2 contains 'max' string if memory is not limited
+            // See https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git/tree/Documentation/admin-guide/cgroup-v2.rst (see "memory.max")
+            if (totalBytesStr === 'max') {
+                totalBytes = node_os_1.default.totalmem();
+                // Cgroups V1 is set to number related to platform and page size if memory is not limited
+                // See https://unix.stackexchange.com/q/420906
+            }
+            else {
+                totalBytes = parseInt(totalBytesStr, 10);
+                const containerRunsWithUnlimitedMemory = totalBytes > Number.MAX_SAFE_INTEGER;
+                if (containerRunsWithUnlimitedMemory)
+                    totalBytes = node_os_1.default.totalmem();
+            }
+            usedBytes = parseInt(usedBytesStr, 10);
+            freeBytes = totalBytes - usedBytes;
+        }
+        catch (err) {
+            // log.deprecated logs a warning only once
+            log_1.default.deprecated('Your environment is Docker, but your system does not support memory cgroups. '
+                + 'If you\'re running containers with limited memory, memory auto-scaling will not work properly.\n\n'
+                + `Cause: ${err.message}`);
+            totalBytes = node_os_1.default.totalmem();
+            freeBytes = node_os_1.default.freemem();
+            usedBytes = totalBytes - freeBytes;
+        }
+    }
+    else {
+        totalBytes = node_os_1.default.totalmem();
+        freeBytes = node_os_1.default.freemem();
+        usedBytes = totalBytes - freeBytes;
+    }
+    return {
+        totalBytes,
+        freeBytes,
+        usedBytes,
+        mainProcessBytes,
+        childProcessesBytes,
+    };
+}
+exports.getMemoryInfo = getMemoryInfo;
+//# sourceMappingURL=memory-info.js.map