apify 2.3.1-beta.3 → 3.0.0-alpha.1

package/build/utils.js

@@ -1,754 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.publicUtils = exports.purgeLocalStorage = exports.waitForRunToFinish = exports.parseContentTypeFromResponse = exports.printOutdatedSdkWarning = exports.snakeCaseToCamelCase = exports.sleep = exports.isAtHome = exports.getTypicalChromeExecutablePath = exports.getFirstKey = exports.getMemoryInfo = exports.weightedAvg = exports.isDocker = exports.addCharsetToContentType = exports.apifyClient = exports.logSystemInfo = exports.newClient = void 0;
-const tslib_1 = require("tslib");
-const ps_tree_1 = (0, tslib_1.__importDefault)(require("@apify/ps-tree"));
-const child_process_1 = require("child_process");
-const apify_client_1 = require("apify-client");
-const package_json_1 = require("apify-client/package.json");
-const consts_1 = require("@apify/consts");
-// eslint-disable-next-line import/no-duplicates
-const cheerio_1 = (0, tslib_1.__importDefault)(require("cheerio"));
-const content_type_1 = (0, tslib_1.__importDefault)(require("content-type"));
-const fs_1 = (0, tslib_1.__importDefault)(require("fs"));
-const mime_types_1 = (0, tslib_1.__importDefault)(require("mime-types"));
-const os_1 = (0, tslib_1.__importDefault)(require("os"));
-const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
-const path_1 = (0, tslib_1.__importDefault)(require("path"));
-const semver_1 = (0, tslib_1.__importDefault)(require("semver"));
-const underscore_1 = (0, tslib_1.__importDefault)(require("underscore"));
-const url_1 = require("url");
-const util_1 = (0, tslib_1.__importDefault)(require("util"));
-const rimraf_1 = (0, tslib_1.__importDefault)(require("rimraf"));
-const package_json_2 = require("../package.json");
-const utils_log_1 = (0, tslib_1.__importDefault)(require("./utils_log"));
-const utils_request_1 = require("./utils_request");
-const configuration_1 = require("./configuration");
-/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
-const rimrafp = util_1.default.promisify(rimraf_1.default);
-/**
- * Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters
- * and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
- * @memberOf utils
- */
-const URL_NO_COMMAS_REGEX = RegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+.~#?&//=\\(\\)]*)?', 'giu'); // eslint-disable-line
-/**
- * Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query.
- * Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
- * @memberOf utils
- */
-const URL_WITH_COMMAS_REGEX = RegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+,.~#?&//=\\(\\)]*)?', 'giu'); // eslint-disable-line
-const MEMORY_FILE_PATHS = {
-    TOTAL: {
-        V1: '/sys/fs/cgroup/memory/memory.limit_in_bytes',
-        V2: '/sys/fs/cgroup/memory.max',
-    },
-    USED: {
-        V1: '/sys/fs/cgroup/memory/memory.usage_in_bytes',
-        V2: '/sys/fs/cgroup/memory.current',
-    },
-};
-// Set encoding to utf-8 so fs.readFile returns string instead of buffer
-const MEMORY_FILE_ENCODING = 'utf-8';
-const psTreePromised = util_1.default.promisify(ps_tree_1.default);
-/**
- * Returns a new instance of the Apify API client. The `ApifyClient` class is provided
- * by the <a href="https://www.npmjs.com/package/apify-client" target="_blank">apify-client</a>
- * NPM package, and it is automatically configured using the `APIFY_API_BASE_URL`, and `APIFY_TOKEN`
- * environment variables. You can override the token via the available options. That's useful
- * if you want to use the client as a different Apify user than the SDK internals are using.
- *
- * @param {object} [options]
- * @param {string} [options.token]
- * @param {string} [options.maxRetries]
- * @param {string} [options.minDelayBetweenRetriesMillis]
- * @memberof module:Apify
- * @function
- * @name newClient
- * @return {ApifyClient}
- */
-const newClient = (options = {}) => {
-    (0, ow_1.default)(options, ow_1.default.object.exactShape({
-        baseUrl: ow_1.default.optional.string.url,
-        token: ow_1.default.optional.string,
-        maxRetries: ow_1.default.optional.number,
-        minDelayBetweenRetriesMillis: ow_1.default.optional.number,
-    }));
-    const { baseUrl = process.env[consts_1.ENV_VARS.API_BASE_URL], token = process.env[consts_1.ENV_VARS.TOKEN], } = options;
-    return new apify_client_1.ApifyClient({
-        ...options,
-        baseUrl,
-        token,
-    });
-};
-exports.newClient = newClient;
-/**
- * Logs info about system, node version and apify package version.
- */
-const logSystemInfo = () => {
-    utils_log_1.default.info('System info', {
-        apifyVersion: package_json_2.version,
-        apifyClientVersion: package_json_1.version,
-        osType: os_1.default.type(),
-        nodeVersion: process.version,
-    });
-};
-exports.logSystemInfo = logSystemInfo;
-/**
- * The default instance of `ApifyClient` used internally
- * by the SDK.
- *
- * @type {*}
- * @ignore
- */
-exports.apifyClient = new Proxy(
-/**
- * Getting the client directly from the global Configuration invokes storage
- * creation right away during the module import. Using the proxy object delays this
- * until the first time the client is accessed.
- * This allows the user to set the envVars and other options before the client is
- * actually used.
- */
-{}, {
-    get(target, prop) {
-        return target[prop] ?? Reflect.get(configuration_1.Configuration.getGlobalConfig().getClient(), prop);
-    },
-    // for mocking purpose in tests
-    set(target, prop, value) {
-        target[prop] = value;
-        return true;
-    },
-    getOwnPropertyDescriptor(target, prop) {
-        return Reflect.getOwnPropertyDescriptor(configuration_1.Configuration.getGlobalConfig().getClient(), prop);
-    },
-    getPrototypeOf() {
-        return Object.getPrototypeOf(configuration_1.Configuration.getGlobalConfig().getClient());
-    },
-});
-/**
- * Adds charset=utf-8 to given content type if this parameter is missing.
- *
- * @param {string} contentType
- * @returns {string}
- *
- * @ignore
- */
-const addCharsetToContentType = (contentType) => {
-    if (!contentType)
-        return contentType;
-    const parsed = content_type_1.default.parse(contentType);
-    if (parsed.parameters.charset)
-        return contentType;
-    parsed.parameters.charset = 'utf-8';
-    return content_type_1.default.format(parsed);
-};
-exports.addCharsetToContentType = addCharsetToContentType;
-let isDockerPromiseCache;
-const createIsDockerPromise = () => {
-    const promise1 = util_1.default
-        .promisify(fs_1.default.stat)('/.dockerenv')
-        .then(() => true)
-        .catch(() => false);
-    const promise2 = util_1.default
-        .promisify(fs_1.default.readFile)('/proc/self/cgroup', 'utf8')
-        .then((content) => content.indexOf('docker') !== -1)
-        .catch(() => false);
-    return Promise
-        .all([promise1, promise2])
-        .then(([result1, result2]) => result1 || result2);
-};
-/**
- * Returns a `Promise` that resolves to true if the code is running in a Docker container.
- *
- * @param {boolean} forceReset
- * @return {Promise<boolean>}
- *
- * @memberof utils
- * @name isDocker
- * @function
- */
-const isDocker = (forceReset) => {
-    // Parameter forceReset is just internal for unit tests.
-    if (!isDockerPromiseCache || forceReset)
-        isDockerPromiseCache = createIsDockerPromise();
-    return isDockerPromiseCache;
-};
-exports.isDocker = isDocker;
-/**
- * Computes a weighted average of an array of numbers, complemented by an array of weights.
- *
- * @param {number[]} arrValues
- * @param {number[]} arrWeights
- * @return {number}
- *
- * @ignore
- */
-const weightedAvg = (arrValues, arrWeights) => {
-    const result = arrValues.map((value, i) => {
-        const weight = arrWeights[i];
-        const sum = value * weight; // eslint-disable-line no-shadow
-        return [sum, weight];
-    }).reduce((p, c) => [p[0] + c[0], p[1] + c[1]], [0, 0]);
-    return result[0] / result[1];
-};
-exports.weightedAvg = weightedAvg;
-/**
- * Describes memory usage of an Actor.
- *
- * @typedef MemoryInfo
- * @property {number} totalBytes Total memory available in the system or container
- * @property {number} freeBytes Amount of free memory in the system or container
- * @property {number} usedBytes Amount of memory used (= totalBytes - freeBytes)
- * @property {number} mainProcessBytes Amount of memory used the current Node.js process
- * @property {number} childProcessesBytes Amount of memory used by child processes of the current Node.js process
- */
-/**
- * Returns memory statistics of the process and the system, see {@link MemoryInfo}.
- *
- * If the process runs inside of Docker, the `getMemoryInfo` gets container memory limits,
- * otherwise it gets system memory limits.
- *
- * Beware that the function is quite inefficient because it spawns a new process.
- * Therefore you shouldn't call it too often, like more than once per second.
- *
- * @returns {Promise<MemoryInfo>}
- *
- * @memberof module:Apify
- * @name getMemoryInfo
- * @function
- */
-const getMemoryInfo = async () => {
-    // lambda does *not* have `ps` and other command line tools
-    // required to extract memory usage.
-    const isLambdaEnvironment = process.platform === 'linux'
-        && !!process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE;
-    // module.exports must be here so that we can mock it.
-    const isDockerVar = !isLambdaEnvironment && (await module.exports.isDocker());
-    let mainProcessBytes = -1;
-    let childProcessesBytes = 0;
-    if (isLambdaEnvironment) {
-        // reported in bytes
-        mainProcessBytes = process.memoryUsage().rss;
-        // https://stackoverflow.com/a/55914335/129415
-        childProcessesBytes = (0, child_process_1.execSync)('cat /proc/meminfo')
-            .toString()
-            .split(/[\n: ]/)
-            .filter((val) => val.trim())[19]
-            // meminfo reports in kb, not bytes
-            * 1000
-            // the total used memory is reported by meminfo
-            // subtract memory used by the main node proces
-            // in order to infer memory used by any child processes
-            - mainProcessBytes;
-    }
-    else {
-        // Query both root and child processes
-        const processes = await psTreePromised(process.pid, true);
-        processes.forEach((rec) => {
-            // Skip the 'ps' or 'wmic' commands used by ps-tree to query the processes
-            if (rec.COMMAND === 'ps' || rec.COMMAND === 'WMIC.exe') {
-                return;
-            }
-            const bytes = parseInt(rec.RSS, 10);
-            // Obtain main process' memory separately
-            if (rec.PID === `${process.pid}`) {
-                mainProcessBytes = bytes;
-                return;
-            }
-            childProcessesBytes += bytes;
-        });
-    }
-    let totalBytes;
-    let usedBytes;
-    let freeBytes;
-    if (isLambdaEnvironment) {
-        // memory size is defined in megabytes
-        totalBytes = parseInt(process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE, 10) * 1000000;
-        usedBytes = mainProcessBytes + childProcessesBytes;
-        freeBytes = totalBytes - usedBytes;
-        utils_log_1.default.debug(`lambda size of ${totalBytes} with ${freeBytes} free bytes`);
-    }
-    else if (isDockerVar) {
-        // When running inside Docker container, use container memory limits
-        // This must be promisified here so that we can mock it.
-        const readPromised = util_1.default.promisify(fs_1.default.readFile);
-        const accessPromised = util_1.default.promisify(fs_1.default.access);
-        // Check wheter cgroups V1 or V2 is used
-        let cgroupsVersion = 'V1';
-        try {
-            // If this directory does not exists, assume docker is using cgroups V2
-            await accessPromised('/sys/fs/cgroup/memory/', fs_1.default.constants.R_OK);
-        }
-        catch (err) {
-            cgroupsVersion = 'V2';
-        }
-        try {
-            let [totalBytesStr, usedBytesStr] = await Promise.all([
-                readPromised(MEMORY_FILE_PATHS.TOTAL[cgroupsVersion], MEMORY_FILE_ENCODING),
-                readPromised(MEMORY_FILE_PATHS.USED[cgroupsVersion], MEMORY_FILE_ENCODING),
-            ]);
-            // Cgroups V2 files contains newline character. Getting rid of it for better handling in later part of the code.
-            totalBytesStr = totalBytesStr.replace(/[^a-zA-Z0-9 ]/g, '');
-            usedBytesStr = usedBytesStr.replace(/[^a-zA-Z0-9 ]/g, '');
-            // Cgroups V2 contains 'max' string if memory is not limited
-            // See https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git/tree/Documentation/admin-guide/cgroup-v2.rst (see "memory.max")
-            if (totalBytesStr === 'max') {
-                totalBytes = os_1.default.totalmem();
-                // Cgroups V1 is set to number related to platform and page size if memory is not limited
-                // See https://unix.stackexchange.com/q/420906
-            }
-            else {
-                totalBytes = parseInt(totalBytesStr, 10);
-                const containerRunsWithUnlimitedMemory = totalBytes > Number.MAX_SAFE_INTEGER;
-                if (containerRunsWithUnlimitedMemory)
-                    totalBytes = os_1.default.totalmem();
-            }
-            usedBytes = parseInt(usedBytesStr, 10);
-            freeBytes = totalBytes - usedBytes;
-        }
-        catch (err) {
-            // log.deprecated logs a warning only once
-            utils_log_1.default.deprecated('Your environment is Docker, but your system does not support memory cgroups. '
-                + 'If you\'re running containers with limited memory, memory auto-scaling will not work properly.\n\n'
-                + `Cause: ${err.message}`);
-            totalBytes = os_1.default.totalmem();
-            freeBytes = os_1.default.freemem();
-            usedBytes = totalBytes - freeBytes;
-        }
-    }
-    else {
-        totalBytes = os_1.default.totalmem();
-        freeBytes = os_1.default.freemem();
-        usedBytes = totalBytes - freeBytes;
-    }
-    return {
-        totalBytes,
-        freeBytes,
-        usedBytes,
-        mainProcessBytes,
-        childProcessesBytes,
-    };
-};
-exports.getMemoryInfo = getMemoryInfo;
-/**
- * Helper function that returns the first key from plain object.
- *
- * @ignore
- */
-const getFirstKey = (dict) => {
-    for (const key in dict) { // eslint-disable-line
-        return key;
-    }
-};
-exports.getFirstKey = getFirstKey;
-/**
- * Gets a typical path to Chrome executable, depending on the current operating system.
- *
- * @return {string}
- * @ignore
- */
-const getTypicalChromeExecutablePath = () => {
-    /**
-     * Return path of Chrome executable by its OS environment variable to deal with non-english language OS.
-     * Taking also in account the old [chrome 380177 issue](https://bugs.chromium.org/p/chromium/issues/detail?id=380177).
-     *
-     * @returns {string}
-     * @ignore
-     */
-    const getWin32Path = () => {
-        let chromeExecutablePath = 'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe';
-        const path00 = `${process.env.ProgramFiles}\\Google\\Chrome\\Application\\chrome.exe`;
-        const path86 = `${process.env['ProgramFiles(x86)']}\\Google\\Chrome\\Application\\chrome.exe`;
-        if (fs_1.default.existsSync(path00)) {
-            chromeExecutablePath = path00;
-        }
-        else if (fs_1.default.existsSync(path86)) {
-            chromeExecutablePath = path86;
-        }
-        return chromeExecutablePath;
-    };
-    switch (os_1.default.platform()) {
-        case 'darwin': return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
-        case 'win32': return getWin32Path();
-        default: return '/usr/bin/google-chrome';
-    }
-};
-exports.getTypicalChromeExecutablePath = getTypicalChromeExecutablePath;
-/**
- * Returns `true` when code is running on Apify platform and `false` otherwise (for example locally).
- *
- * @returns {boolean}
- *
- * @memberof module:Apify
- * @name isAtHome
- * @function
- */
-const isAtHome = () => !!process.env[consts_1.ENV_VARS.IS_AT_HOME];
-exports.isAtHome = isAtHome;
-/**
- * Returns a `Promise` that resolves after a specific period of time. This is useful to implement waiting
- * in your code, e.g. to prevent overloading of target website or to avoid bot detection.
- *
- * **Example usage:**
- *
- * ```
- * const Apify = require('apify');
- *
- * ...
- *
- * // Sleep 1.5 seconds
- * await Apify.utils.sleep(1500);
- * ```
- * @param {number} millis Period of time to sleep, in milliseconds. If not a positive number, the returned promise resolves immediately.
- * @memberof utils
- * @name sleep
- * @function
- * @return {Promise<void>}
- */
-const sleep = (millis) => {
-    return new Promise((res) => setTimeout(res, millis));
-};
-exports.sleep = sleep;
-/**
- * Returns a promise that resolves to an array of urls parsed from the resource available at the provided url.
- * Optionally, custom regular expression and encoding may be provided.
- *
- * @param {object} options
- * @param {string} options.url URL to the file
- * @param {string} [options.encoding='utf8'] The encoding of the file.
- * @param {string} [options.proxyUrl] The proxy url to be used for the request.
- * @param {RegExp} [options.urlRegExp=URL_NO_COMMAS_REGEX]
- *   Custom regular expression to identify the URLs in the file to extract.
- *   The regular expression should be case-insensitive and have global flag set (i.e. `/something/gi`).
- * @returns {Promise<Array<string>>}
- * @memberOf utils
- */
-const downloadListOfUrls = async (options) => {
-    (0, ow_1.default)(options, ow_1.default.object.exactShape({
-        url: ow_1.default.string.url,
-        encoding: ow_1.default.optional.string,
-        urlRegExp: ow_1.default.optional.regExp,
-        proxyUrl: ow_1.default.optional.string,
-    }));
-    const { url, encoding = 'utf8', urlRegExp = URL_NO_COMMAS_REGEX, proxyUrl } = options;
-    // Try to detect wrong urls and fix them. Currently, detects only sharing url instead of csv download one.
-    const match = url.match(/^(https:\/\/docs\.google\.com\/spreadsheets\/d\/(?:\w|-)+)\/?/);
-    let fixedUrl = url;
-    if (match) {
-        fixedUrl = `${match[1]}/gviz/tq?tqx=out:csv`;
-    }
-    const { body: string } = await (0, utils_request_1.requestAsBrowser)({ url: fixedUrl, encoding, proxyUrl });
-    return extractUrls({ string, urlRegExp });
-};
-/**
- * Collects all URLs in an arbitrary string to an array, optionally using a custom regular expression.
- * @param {object} options
- * @param {string} options.string
- * @param {RegExp} [options.urlRegExp=Apify.utils.URL_NO_COMMAS_REGEX]
- * @returns {string[]}
- * @memberOf utils
- */
-const extractUrls = (options) => {
-    (0, ow_1.default)(options, ow_1.default.object.exactShape({
-        string: ow_1.default.string,
-        urlRegExp: ow_1.default.optional.regExp,
-    }));
-    const { string, urlRegExp = URL_NO_COMMAS_REGEX } = options;
-    return string.match(urlRegExp) || [];
-};
-// NOTE: We skipping 'noscript' since it's content is evaluated as text, instead of HTML elements. That damages the results.
-const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript)$/i;
-const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|dl|div|fieldset|form|table|tr|select|option)$/i;
-/**
- * The function converts a HTML document to a plain text.
- *
- * The plain text generated by the function is similar to a text captured
- * by pressing Ctrl+A and Ctrl+C on a page when loaded in a web browser.
- * The function doesn't aspire to preserve the formatting or to be perfectly correct with respect to HTML specifications.
- * However, it attempts to generate newlines and whitespaces in and around HTML elements
- * to avoid merging distinct parts of text and thus enable extraction of data from the text (e.g. phone numbers).
- *
- * **Example usage**
- * ```javascript
- * const text = htmlToText('<html><body>Some text</body></html>');
- * console.log(text);
- * ```
- *
- * Note that the function uses [cheerio](https://www.npmjs.com/package/cheerio) to parse the HTML.
- * Optionally, to avoid duplicate parsing of HTML and thus improve performance, you can pass
- * an existing Cheerio object to the function instead of the HTML text. The HTML should be parsed
- * with the `decodeEntities` option set to `true`. For example:
- *
- * ```javascript
- * const cheerio = require('cheerio');
- * const html = '<html><body>Some text</body></html>';
- * const text = htmlToText(cheerio.load(html, { decodeEntities: true }));
- * ```
- * @param {(string|CheerioAPI)} html HTML text or parsed HTML represented using a
- * [cheerio](https://www.npmjs.com/package/cheerio) function.
- * @return {string} Plain text
- * @memberOf utils
- * @function
- */
-const htmlToText = (html) => {
-    if (!html)
-        return '';
-    // TODO: Add support for "html" being a Cheerio element, otherwise the only way
-    //  to use it is e.g. htmlToText($('p').html())) which is inefficient
-    //  Also, it seems this doesn't work well in CheerioScraper, e.g. htmlToText($)
-    //  produces really text with a lot of HTML elements in it. Let's just deprecate this sort of usage,
-    //  and make the parameter "htmlOrCheerioElement"
-    /**
-     * @type {CheerioAPI}
-     * @ignore
-     */
-    const $ = typeof html === 'function' ? html : cheerio_1.default.load(html, { decodeEntities: true });
-    let text = '';
-    const process = (elems) => {
-        const len = elems ? elems.length : 0;
-        for (let i = 0; i < len; i++) {
-            const elem = elems[i];
-            if (elem.type === 'text') {
-                // Compress spaces, unless we're inside <pre> element
-                let compr;
-                if (elem.parent && elem.parent.tagName === 'pre')
-                    compr = elem.data;
-                else
-                    compr = elem.data.replace(/\s+/g, ' ');
-                // If text is empty or ends with a whitespace, don't add the leading whitepsace
-                if (compr.startsWith(' ') && /(^|\s)$/.test(text))
-                    compr = compr.substr(1);
-                text += compr;
-            }
-            else if (elem.type === 'comment' || SKIP_TAGS_REGEX.test(elem.tagName)) {
-                // Skip comments and special elements
-            }
-            else if (elem.tagName === 'br') {
-                text += '\n';
-            }
-            else if (elem.tagName === 'td') {
-                process(elem.children);
-                text += '\t';
-            }
-            else {
-                // Block elements must be surrounded by newlines (unless beginning of text)
-                const isBlockTag = BLOCK_TAGS_REGEX.test(elem.tagName);
-                if (isBlockTag && !/(^|\n)$/.test(text))
-                    text += '\n';
-                process(elem.children);
-                if (isBlockTag && !text.endsWith('\n'))
-                    text += '\n';
-            }
-        }
-    };
-    // If HTML document has body, only convert that, otherwise convert the entire HTML
-    const $body = $('body');
-    process($body.length > 0 ? $body : $.root());
-    return text.trim();
-};
-/**
- * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
- *
- * @param {(Request|RequestOptions)} request [Apify.Request](https://sdk.apify.com/docs/api/request) object.
- * @param {(*|IncomingMessage|PuppeteerResponse)} [response]
- *   Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
- *   or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
- * @param {Object<string, *>} [additionalFields] Object containing additional fields to be added.
-
- * @return {Object<string, *>}
- */
-const createRequestDebugInfo = (request, response = {}, additionalFields = {}) => {
-    (0, ow_1.default)(request, ow_1.default.object);
-    (0, ow_1.default)(response, ow_1.default.object);
-    (0, ow_1.default)(additionalFields, ow_1.default.object);
-    return {
-        requestId: request.id,
-        url: request.url,
-        loadedUrl: request.loadedUrl,
-        method: request.method,
-        retryCount: request.retryCount,
-        errorMessages: request.errorMessages,
-        // Puppeteer response has .status() funtion and NodeJS response ,statusCode property.
-        statusCode: underscore_1.default.isFunction(response.status) ? response.status() : response.statusCode,
-        ...additionalFields,
-    };
-};
-/**
- * Converts SNAKE_CASE to camelCase.
- *
- * @param {string} snakeCaseStr
- * @return {string}
- * @ignore
- */
-const snakeCaseToCamelCase = (snakeCaseStr) => {
-    return snakeCaseStr
-        .toLowerCase()
-        .split('_')
-        .map((part, index) => {
-        return index > 0
-            ? part.charAt(0).toUpperCase() + part.slice(1)
-            : part;
-    })
-        .join('');
-};
-exports.snakeCaseToCamelCase = snakeCaseToCamelCase;
-/**
- * Prints a warning if this version of Apify SDK is outdated.
- *
- * @ignore
- */
-const printOutdatedSdkWarning = () => {
-    if (process.env[consts_1.ENV_VARS.DISABLE_OUTDATED_WARNING])
-        return;
-    const latestApifyVersion = process.env[consts_1.ENV_VARS.SDK_LATEST_VERSION];
-    if (!latestApifyVersion || !semver_1.default.lt(package_json_2.version, latestApifyVersion))
-        return;
-    // eslint-disable-next-line
-    utils_log_1.default.warning(`You are using an outdated version (${package_json_2.version}) of Apify SDK. We recommend you to update to the latest version (${latestApifyVersion}).
-         Read more about Apify SDK versioning at: https://help.apify.com/en/articles/3184510-updates-and-versioning-of-apify-sdk`);
-};
-exports.printOutdatedSdkWarning = printOutdatedSdkWarning;
-/**
- * Gets parsed content type from response object
- * @param {IncomingMessage} response - HTTP response object
- * @return {{ type: string, charset: string }}
- * @ignore
- */
-const parseContentTypeFromResponse = (response) => {
-    (0, ow_1.default)(response, ow_1.default.object.partialShape({
-        url: ow_1.default.string.url,
-        headers: ow_1.default.object,
-    }));
-    const { url, headers } = response;
-    let parsedContentType;
-    if (headers['content-type']) {
-        try {
-            parsedContentType = content_type_1.default.parse(headers['content-type']);
-        }
-        catch (err) {
-            // Can not parse content type from Content-Type header. Try to parse it from file extension.
-        }
-    }
-    // Parse content type from file extension as fallback
-    if (!parsedContentType) {
-        const parsedUrl = new url_1.URL(url);
-        const contentTypeFromExtname = mime_types_1.default.contentType(path_1.default.extname(parsedUrl.pathname))
-            || 'application/octet-stream; charset=utf-8'; // Fallback content type, specified in https://tools.ietf.org/html/rfc7231#section-3.1.1.5
-        parsedContentType = content_type_1.default.parse(contentTypeFromExtname);
-    }
-    return {
-        type: parsedContentType.type,
-        charset: parsedContentType.parameters.charset,
-    };
-};
-exports.parseContentTypeFromResponse = parseContentTypeFromResponse;
-/**
- * Returns a promise that resolves with the finished Run object when the provided actor run finishes
- * or with the unfinished Run object when the `waitSecs` timeout lapses. The promise is NOT rejected
- * based on run status. You can inspect the `status` property of the Run object to find out its status.
- *
- * This is useful when you need to chain actor executions. Similar effect can be achieved
- * by using webhooks, so be sure to review which technique fits your use-case better.
- *
- * @param {object} options
- * @param {string} options.actorId
- *  ID of the actor that started the run.
- * @param {string} options.runId
- *  ID of the run itself.
- * @param {string} [options.waitSecs]
- *  Maximum time to wait for the run to finish, in seconds.
- *  If the limit is reached, the returned promise is resolved to a run object that will have
- *  status `READY` or `RUNNING`. If `waitSecs` omitted, the function waits indefinitely.
- * @param {string} [options.token]
- *  You can supply an Apify token to override the default one
- *  that's used by the default ApifyClient instance.
- *  E.g. you can track other users' runs.
- * @returns {Promise<ActorRun>}
- * @memberOf utils
- * @name waitForRunToFinish
- * @function
- * @deprecated
- *  Please use the 'waitForFinish' functions of 'apify-client'.
- * @ignore
- */
-const waitForRunToFinish = async (options) => {
-    (0, ow_1.default)(options, ow_1.default.object.exactShape({
-        actorId: ow_1.default.string,
-        runId: ow_1.default.string,
-        waitSecs: ow_1.default.optional.number,
-    }));
-    const { actorId, runId, waitSecs, } = options;
-    let run;
-    const startedAt = Date.now();
-    const shouldRepeat = () => {
-        if (waitSecs && (Date.now() - startedAt) / 1000 >= waitSecs)
-            return false;
-        if (run && consts_1.ACT_JOB_TERMINAL_STATUSES.includes(run.status))
-            return false;
-        return true;
-    };
-    while (shouldRepeat()) {
-        const waitForFinish = waitSecs
-            ? Math.round(waitSecs - (Date.now() - startedAt) / 1000)
-            : 999999;
-        run = await exports.apifyClient.run(runId, actorId).waitForFinish({ waitSecs: waitForFinish });
-        // It might take some time for database replicas to get up-to-date,
-        // so getRun() might return null. Wait a little bit and try it again.
-        if (!run)
-            await (0, exports.sleep)(250);
-    }
-    if (!run) {
-        throw new Error('Waiting for run to finish failed. Cannot fetch actor run details from the server.');
-    }
-    return run;
-};
-exports.waitForRunToFinish = waitForRunToFinish;
-/**
- * Cleans up the local storage folder created when testing locally.
- * This is useful in the event you are debugging your code locally.
- *
- * Be careful as this will remove the folder you provide and everything in it!
- *
- * @param {string} [folder] The folder to clean up
- * @returns {Promise<void>}
- * @memberOf utils
- * @name purgeLocalStorage
- * @function
- */
-const purgeLocalStorage = async (folder) => {
-    // If the user did not provide a folder, try to get it from the env variables, or the default one
-    if (!folder) {
-        folder = process.env[consts_1.ENV_VARS.LOCAL_STORAGE_DIR] || 'apify_storage';
-    }
-    // Clear the folder
-    await rimrafp(folder);
-};
-exports.purgeLocalStorage = purgeLocalStorage;
-/**
- * A namespace that contains various utilities.
- *
- * **Example usage:**
- *
- * ```javascript
- * const Apify = require('apify');
- *
- * ...
- *
- * // Sleep 1.5 seconds
- * await Apify.utils.sleep(1500);
- * ```
- * @namespace utils
- */
-exports.publicUtils = {
-    isDocker: exports.isDocker,
-    sleep: exports.sleep,
-    downloadListOfUrls,
-    extractUrls,
-    htmlToText,
-    URL_NO_COMMAS_REGEX,
-    URL_WITH_COMMAS_REGEX,
-    createRequestDebugInfo,
-    waitForRunToFinish: exports.waitForRunToFinish,
-    purgeLocalStorage: exports.purgeLocalStorage,
-};
-//# sourceMappingURL=utils.js.map