apify 2.3.1-beta.4 → 3.0.0-alpha.0

package/build/utils_request.js

@@ -1,385 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.requestAsBrowser = void 0;
-const tslib_1 = require("tslib");
-const got_scraping_1 = require("got-scraping");
-const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
-const utils_log_1 = (0, tslib_1.__importDefault)(require("./utils_log"));
-/* eslint-enable no-unused-vars,import/named,import/order */
-/**
- * @typedef {(IncomingMessage & { body: string })} RequestAsBrowserResult
- */
-/**
- * @typedef RequestAsBrowserOptions
- * @property {string} url
- *  URL of the target endpoint. Supports both HTTP and HTTPS schemes.
- * @property {string} [method="GET"]
- *  HTTP method.
- * @property {Object<string, string>} [headers]
- *  Additional HTTP headers to add. It's only recommended to use this option,
- *  with headers that are typically added by websites, such as cookies. Overriding
- *  default browser headers will remove the masking this function provides.
- * @property {string} [proxyUrl]
- *  An HTTP proxy to be passed down to the HTTP request. Supports proxy authentication with Basic Auth.
- * @property {object} [headerGeneratorOptions]
- *  Configuration to be used for generating correct browser headers.
- *  See the [`header-generator`](https://github.com/apify/header-generator) library.
- * @property {string} [languageCode=en]
- *  Two-letter ISO 639 language code.
- * @property {string} [countryCode=US]
- *  Two-letter ISO 3166 country code.
- * @property {boolean} [useMobileVersion]
- *  If `true`, the function uses User-Agent of a mobile browser.
- * @property {boolean} [ignoreSslErrors=true]
- *  If set to true, SSL/TLS certificate errors will be ignored.
- * @property {boolean} [useInsecureHttpParser=true]
- *  Node.js' HTTP parser is stricter than parsers used by web browsers, which prevents scraping of websites
- *  whose servers do not comply with HTTP specs, either by accident or due to some anti-scraping protections,
- *  causing e.g. the `invalid header value char` error. The `useInsecureHttpParser` option forces
- *  the HTTP parser to ignore certain errors which lets you scrape such websites.
- *  However, it will also open your application to some security vulnerabilities,
- *  although the risk should be negligible as these vulnerabilities mainly relate to server applications, not clients.
- *  Learn more in this [blog post](https://snyk.io/blog/node-js-release-fixes-a-critical-http-security-vulnerability/).
- * @property {AbortFunction} [abortFunction]
- *  Function accepts `response` object as a single parameter and should return `true` or `false`.
- *  If function returns true, request gets aborted.
- * @property {boolean} [useHttp2=true]
- *  If set to false, it will prevent use of HTTP2 requests. This is strongly discouraged. Websites
- *  expect HTTP2 connections, because browsers use HTTP2 by default. It will automatically downgrade
- *  to HTTP/1.1 for websites that do not support HTTP2.
- * @property {object} [sessionToken]
- *  A unique object used to generate browser headers. By default, new headers are generated on every call.
- *  Set this option to make these headers persistent.
- */
-/**
- * @callback AbortFunction
- * @param {IncomingMessage} response
- * @returns {boolean}
- */
-/**
- * **IMPORTANT:** This function uses an insecure version of HTTP parser by default
- * and also ignores SSL/TLS errors. This is very useful in scraping, because it allows bypassing
- * certain anti-scraping walls, but it also exposes some vulnerability. For other than scraping
- * scenarios, please set `useInsecureHttpParser: false` and `ignoreSslErrors: false`.
- *
- * Sends a HTTP request that looks like a request sent by a web browser,
- * fully emulating browser's HTTP headers. It uses HTTP2 by default for Node 12+.
- *
- * This function is useful for web scraping of websites that send the full HTML in the first response.
- * Thanks to this function, the target web server has no simple way to find out the request
- * hasn't been sent by a human's web browser. Using a headless browser for such requests
- * is an order of magnitude more resource-intensive than this function.
- *
- * The function emulates the Chrome and Firefox web browsers. If you want more control
- * over the browsers and their versions, use the `headerGeneratorOptions` property.
- * You can find more info in the readme of the [`header-generator`](https://github.com/apify/header-generator) library.
- *
- * Internally, the function uses the [`got-scraping`](https://github.com/apify/got-scraping) library to perform the request.
- * All `options` not recognized by this function are passed to it so see it for more details.
- *
- * **Example usage:**
- * ```js
- * const Apify = require('apify');
- *
- * const { utils: { requestAsBrowser } } = Apify;
- *
- * ...
- *
- * const response = await requestAsBrowser({ url: 'https://www.example.com/' });
- *
- * const html = response.body;
- * const status = response.statusCode;
- * const contentType = response.headers['content-type'];
- * ```
- *
- * @param {RequestAsBrowserOptions} options All `requestAsBrowser` configuration options.
- *
- * @return {Promise<RequestAsBrowserResult>} The result can be various objects, but it will always be like a
- * [Node.js HTTP response stream](https://nodejs.org/api/http.html#http_class_http_incomingmessage)
- * with a 'body' property for the parsed response body, unless the 'stream' option is used.
- * @memberOf utils
- * @name requestAsBrowser
- * @function
- */
-const requestAsBrowser = async (options = {}) => {
-    logDeprecatedOptions(options);
-    (0, ow_1.default)(options, 'RequestAsBrowserOptions', ow_1.default.object.partialShape({
-        payload: ow_1.default.optional.any(ow_1.default.string, ow_1.default.buffer),
-        proxyUrl: ow_1.default.optional.string.url,
-        languageCode: ow_1.default.optional.string.length(2),
-        countryCode: ow_1.default.optional.string.length(2),
-        useMobileVersion: ow_1.default.optional.boolean,
-        abortFunction: ow_1.default.optional.function,
-        ignoreSslErrors: ow_1.default.optional.boolean,
-        useInsecureHttpParser: ow_1.default.optional.boolean,
-        useHttp2: ow_1.default.optional.boolean,
-        timeoutSecs: ow_1.default.optional.number,
-        throwOnHttpErrors: ow_1.default.optional.boolean,
-        headerGeneratorOptions: ow_1.default.optional.object,
-        stream: ow_1.default.optional.boolean,
-        decodeBody: ow_1.default.optional.boolean,
-        sessionToken: ow_1.default.optional.object,
-    }));
-    (0, ow_1.default)(options, 'RequestAsBrowserOptions', ow_1.default.object.validate((opts) => ({
-        validator: areBodyOptionsCompatible(opts),
-        message: (label) => `The 'payload', 'body', 'json' and 'form' options of ${label} are mutually exclusive.`,
-    })));
-    // We created the `got-scraping` package which replaced underlying @apify/http-request.
-    // At the same time, we want users to be able to use requestAsBrowser without breaking changes.
-    // So we do a lot of property mapping here, to make sure that everything works as expected.
-    // TODO Update this with SDK v3 and use `got-scraping` API directly.
-    const { payload, // alias for body to allow direct passing of our Request objects
-    json, headerGeneratorOptions, languageCode = 'en', countryCode = 'US', useMobileVersion = false, abortFunction = () => false, ignoreSslErrors = true, useInsecureHttpParser = true, useHttp2 = true, timeoutSecs = 30, throwOnHttpErrors = false, stream = false, decodeBody = true, forceUrlEncoding, // TODO remove in v3. It's not used, but we keep it here to prevent validation errors in got.
-    ...gotParams } = options;
-    const gotScrapingOptions = {
-        insecureHTTPParser: useInsecureHttpParser,
-        http2: useHttp2,
-        timeout: { request: timeoutSecs * 1000 },
-        throwHttpErrors: throwOnHttpErrors,
-        isStream: stream,
-        decompress: decodeBody,
-        // We overwrite the above arguments because we want to give the official
-        // got interface a priority over our requestAsBrowser one.
-        // E.g. { isStream: false, stream: true } should produce { isStream: false }.
-        ...gotParams,
-        https: {
-            ...gotParams.https,
-            rejectUnauthorized: !ignoreSslErrors,
-        },
-    };
-    // Order is important
-    normalizePayloadOption(payload, gotScrapingOptions);
-    normalizeJsonOption(json, gotScrapingOptions);
-    ensureCorrectHttp2Headers(gotScrapingOptions);
-    maybeAddAbortHook(abortFunction, gotScrapingOptions);
-    if (!headerGeneratorOptions) {
-        // Values that respect old requestAsBrowser user-agents and settings
-        gotScrapingOptions.headerGeneratorOptions = {
-            devices: useMobileVersion ? ['mobile'] : ['desktop'],
-            locales: [`${languageCode}-${countryCode}`],
-        };
-    }
-    else {
-        gotScrapingOptions.headerGeneratorOptions = headerGeneratorOptions;
-    }
-    // Return the promise directly
-    if (!gotScrapingOptions.isStream) {
-        return (0, got_scraping_1.gotScraping)(gotScrapingOptions);
-    }
-    // abortFunction must be handled separately for streams :(
-    const duplexStream = (0, got_scraping_1.gotScraping)(gotScrapingOptions);
-    ensureRequestIsDispatched(duplexStream, gotScrapingOptions);
-    return new Promise((resolve, reject) => {
-        duplexStream
-            .on('error', reject)
-            .on('response', (res) => {
-            try {
-                const shouldAbort = abortFunction(res);
-                if (shouldAbort) {
-                    const err = new Error(`Request for ${gotScrapingOptions.url} aborted due to abortFunction.`);
-                    duplexStream.destroy(err);
-                    return reject(err);
-                }
-            }
-            catch (e) {
-                duplexStream.destroy(e);
-                return reject(e);
-            }
-            addResponsePropertiesToStream(duplexStream, res);
-            return resolve(duplexStream);
-        });
-    });
-};
-exports.requestAsBrowser = requestAsBrowser;
-/**
- * `got` has a `body` option and 2 helpers, `json` and `form`, to provide specific bodies.
- * Those options are mutually exclusive. `requestAsBrowser` also supports `payload` as
- * an alias of `body`. It must be exclusive as well.
- * @param {RequestAsBrowserOptions} requestAsBrowserOptions
- * @return {boolean}
- * @private
- * @ignore
- */
-function areBodyOptionsCompatible(requestAsBrowserOptions) {
-    const { payload, json, body, form } = requestAsBrowserOptions;
-    // A boolean is old requestAsBrowser interface and not a real "body"
-    // See the normalizeJsonOption function.
-    const jsonBody = typeof json === 'boolean' ? undefined : json;
-    const possibleOpts = [payload, jsonBody, body, form];
-    const usedOpts = possibleOpts.filter((opt) => opt !== undefined);
-    // Only a single option out of the 4 can be used.
-    return usedOpts.length <= 1;
-}
-/**
- * got-scraping uses 'body', but we also support 'payload' from {@link Request}.
- * @param {string|Buffer} payload
- * @param {GotScrapingOptions} gotScrapingOptions
- * @ignore
- * @private
- */
-function normalizePayloadOption(payload, gotScrapingOptions) {
-    if (payload !== undefined)
-        gotScrapingOptions.body = payload;
-}
-/**
- * `json` is a boolean flag in `requestAsBrowser`, but a `body` alias that
- * adds a 'content-type: application/json' header in got. To stay backwards
- * compatible we need to figure out which option the user provided.
- * @param {*} json
- * @param {GotScrapingOptions} gotScrapingOptions
- * @ignore
- * @private
- */
-function normalizeJsonOption(json, gotScrapingOptions) {
-    // If it's a boolean, then it's the old requestAsBrowser API.
-    // If it's true, it means the user expects a JSON response.
-    const deprecationMessage = `"options.json" of type: Boolean is deprecated.`
-        + 'If you expect a JSON response, use "options.responseType = \'json\'"'
-        + 'Use "options.json" with a plain object to provide a JSON body.';
-    if (json === true) {
-        utils_log_1.default.deprecated(deprecationMessage);
-        gotScrapingOptions.responseType = 'json';
-        gotScrapingOptions.https.ciphers = undefined;
-    }
-    else if (json === false) {
-        utils_log_1.default.deprecated(deprecationMessage);
-        // Do nothing, it means the user expects something else than JSON.
-    }
-    else {
-        // If it's something else, we let `got` handle it as a request body.
-        gotScrapingOptions.json = json;
-    }
-}
-/**
- * 'connection' and 'host' headers are forbidden when using HTTP2. We delete
- * them from user-provided headers because we switched the default from HTTP1 to 2.
- * @param {GotScrapingOptions} gotScrapingOptions
- * @ignore
- * @private
- */
-function ensureCorrectHttp2Headers(gotScrapingOptions) {
-    if (gotScrapingOptions.http2 && gotScrapingOptions.headers) {
-        gotScrapingOptions.headers = { ...gotScrapingOptions.headers };
-        // eslint-disable-next-line no-restricted-syntax, guard-for-in
-        for (const key in gotScrapingOptions.headers) {
-            const lkey = key.toLowerCase();
-            if (lkey === 'connection' || lkey === 'host') {
-                delete gotScrapingOptions.headers[key];
-            }
-        }
-    }
-}
-/**
- * `abortFunction` is an old `requestAsBrowser` interface for aborting requests before
- * the response body is read to save bandwidth.
- * @param {function} abortFunction
- * @param {GotScrapingOptions} gotScrapingOptions
- * @ignore
- * @private
- */
-function maybeAddAbortHook(abortFunction, gotScrapingOptions) {
-    // Stream aborting must be handled on the response object because `got`
-    // does not execute `afterResponse` hooks for streams :(
-    if (gotScrapingOptions.isStream)
-        return;
-    const abortHook = (response) => {
-        const shouldAbort = abortFunction(response);
-        if (shouldAbort) {
-            throw new Error(`Request for ${gotScrapingOptions.url} aborted due to abortFunction.`);
-        }
-        return response;
-    };
-    const { hooks } = gotScrapingOptions;
-    const fixedHooks = {
-        ...hooks,
-        afterResponse: [
-            ...((hooks && hooks.afterResponse) || []),
-            abortHook,
-        ],
-    };
-    gotScrapingOptions.hooks = fixedHooks;
-}
-/**
- * 'got' will not dispatch non-GET request stream until a body is provided.
- * @param {stream.Duplex} duplexStream
- * @param {GotScrapingOptions} gotScrapingOptions
- */
-function ensureRequestIsDispatched(duplexStream, gotScrapingOptions) {
-    const { method } = gotScrapingOptions;
-    const bodyIsEmpty = gotScrapingOptions.body === undefined
-        && gotScrapingOptions.json === undefined
-        && gotScrapingOptions.form === undefined;
-    if (method && method.toLowerCase() !== 'get' && bodyIsEmpty) {
-        duplexStream.end();
-    }
-}
-/**
- * @param {RequestAsBrowserOptions} options
- * @ignore
- * @private
- */
-function logDeprecatedOptions(options) {
-    const deprecatedOptions = [
-        // 'json' is handled in the JSON handler, because it has a conflict of types
-        ['languageCode', 'headerGeneratorOptions.locales'],
-        ['countryCode', 'headerGeneratorOptions.locales'],
-        ['useMobileVersion', 'headerGeneratorOptions.devices'],
-        ['payload', 'body'],
-        ['useHttp2', 'http2'],
-        ['stream', 'isStream'],
-        ['decodeBody', 'decompress'],
-        ['throwOnHttpErrors', 'throwHttpErrors'],
-        ['timeoutSecs', 'timeout.request'],
-        ['ignoreSslErrors', 'https.rejectUnauthorized'],
-        ['abortFunction'], // custom message below
-    ];
-    for (const [deprecatedOption, newOption] of deprecatedOptions) {
-        if (options[deprecatedOption] !== undefined) {
-            // This will log only for the first property thanks to log.deprecated logging only once.
-            const initialMessage = 'requestAsBrowser internal implementation has been replaced with the got-scraping module. '
-                + 'To make the switch without breaking changes, we mapped all existing options to the got-scraping options. '
-                + 'This mapping will be removed in SDK v3 and we advise you to update your code using the hints below: ';
-            utils_log_1.default.deprecated(initialMessage);
-            if (deprecatedOption === 'abortFunction') {
-                utils_log_1.default.deprecated(`"options.${deprecatedOption}" is deprecated.`
-                    + 'Use a request cancellation process appropriate for your request type.'
-                    + 'Either a Stream or a Promise. See Got documentation for more info: https://github.com/sindresorhus/got');
-            }
-            else {
-                utils_log_1.default.deprecated(`"options.${deprecatedOption}" is deprecated. Use "options.${newOption}" instead.`);
-            }
-        }
-    }
-}
-/**
- * The stream object returned from got does not have the below properties.
- * At the same time, you can't read data directly from the response stream,
- * because they won't get emitted unless you also read from the primary
- * got stream. To be able to work with only one stream, we move the expected props
- * from the response stream to the got stream.
- * @param {GotStream} stream
- * @param {http.IncomingMessage} response
- * @return {GotStream}
- * @ignore
- * @private
- */
-function addResponsePropertiesToStream(stream, response) {
-    const properties = [
-        'statusCode', 'statusMessage', 'headers',
-        'complete', 'httpVersion', 'rawHeaders',
-        'rawTrailers', 'trailers', 'url',
-        'request',
-    ];
-    response.on('end', () => {
-        Object.assign(stream.rawTrailers, response.rawTrailers);
-        Object.assign(stream.trailers, response.trailers);
-        stream.complete = response.complete;
-    });
-    for (const prop of properties) {
-        if (!(prop in stream)) {
-            stream[prop] = response[prop];
-        }
-    }
-    return stream;
-}
-//# sourceMappingURL=utils_request.js.map

package/build/utils_request.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"utils_request.js","sourceRoot":"","sources":["../src/utils_request.js"],"names":[],"mappings":";;;;AAAA,+CAA2C;AAC3C,yDAAoB;AACpB,yEAA8B;AAI9B,4DAA4D;AAE5D;;GAEG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAEH;;;;GAIG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AACI,MAAM,gBAAgB,GAAG,KAAK,EAAE,OAAO,GAAG,EAAE,EAAE,EAAE;IACnD,oBAAoB,CAAC,OAAO,CAAC,CAAC;IAC9B,IAAA,YAAE,EAAC,OAAO,EAAE,yBAAyB,EAAE,YAAE,CAAC,MAAM,CAAC,YAAY,CAAC;QAC1D,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC;QAC9C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;QAChC,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAC1C,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACzC,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACrC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QACnC,eAAe,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACpC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC1C,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC7B,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACtC,sBAAsB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC1C,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC3B,UAAU,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC/B,YAAY,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KACnC,CAAC,CAAC,CAAC;IAEJ,IAAA,YAAE,EAAC,OAAO,EAAE,yBAAyB,EAAE,YAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACjE,SAAS,EAAE,wBAAwB,CAAC,IAAI,CAAC;QACzC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,uDAAuD,KAAK,0BAA0B;KAC7G,CAAC,CAAC,CAAC,CAAC;IAEL,uFAAuF;IACvF,+FAA+F;IAC/F,2FAA2F;IAC3F,oEAAoE;IACpE,MAAM,EACF,OAAO,EAAE,gEAAgE;IACzE,IAAI,EACJ,sBAAsB,EACtB,YAAY,GAAG,IAAI,EACnB,WAAW,GAAG,IAAI,EAClB,gBAAgB,GAAG,KAAK,EACxB,aAAa,GAAG,GAAG,EAAE,CAAC,KAAK,EAC3B,eAAe,GAAG,IAAI,EACtB,qBAAqB,GAAG,IAAI,EAC5B,QAAQ,GAAG,IAAI,EACf,WAAW,GAAG,EAAE,EAChB,iBAAiB,GAAG,KAAK,EACzB,MAAM,GAAG,KAAK,EACd,UAAU,GAAG,IAAI,EACjB,gBAAgB,EAAE,6FAA6F;IAC/G,GAAG,SAAS,EACf,GAAG,OAAO,CAAC;IAEZ,MAAM,kBAAkB,GAAG;QACvB,kBAAkB,EAAE,qBAAqB;QACzC,KAAK,EAAE,QAAQ;QACf,OAAO,EAAE,EAAE,OAAO,EAAE,WAAW,GAAG,IAAI,EAAE;QACxC,eAAe,EAAE,iBAAiB;QAClC,QAAQ,EAAE,MAAM;QAChB,UAAU,EAAE,UAAU;QACtB,wEAAwE;QACxE,0DAA0D;QAC1D,6EAA6E;QAC7E,GAAG,SAAS;QACZ,KAAK,EAAE;YACH,GAAG,SAAS,CAAC,KAAK;YAClB,kBAAkB,EAAE,CAAC,eAAe;SACvC;KACJ,CAAC;IAEF,qBAAqB;IACrB,sBAAsB,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;IACpD,mBAAmB,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;IAC9C,yBAAyB,CAAC,kBAAkB,CAAC,CAAC;IAC9C,iBAAiB,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAAC;IACrD,IAAI,CAAC,sBAAsB,EAAE;QACzB,oEAAoE;QACpE,kBAAkB,CAAC,sBAAsB,GAAG;YACxC,OAAO,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACpD,OAAO,EAAE,CAAC,GAAG,YAAY,IAAI,WAAW,EAAE,CAAC;SAC9C,CAAC;KACL;SAAM;QACH,kBAAkB,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;KACtE;IAED,8BAA8B;IAC9B,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE;QAC9B,OAAO,IAAA,0BAAW,EAAC,kBAAkB,CAAC,CAAC;KAC1C;IAED,0DAA0D;IAC1D,MAAM,YAAY,GAAG,IAAA,0BAAW,EAAC,kBAAkB,CAAC,CAAC;IAErD,yBAAyB,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC;IAE5D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnC,YAAY;aACP,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC;aACnB,EAAE,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE;YACpB,IAAI;gBACA,MAAM,WAAW,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;gBACvC,IAAI,WAAW,EAAE;oBACb,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,eAAe,kBAAkB,CAAC,GAAG,gCAAgC,CAAC,CAAC;oBAC7F,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;oBAC1B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;iBACtB;aACJ;YAAC,OAAO,CAAC,EAAE;gBACR,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACxB,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;aACpB;YAED,6BAA6B,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;YAEjD,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACX,CAAC,CAAC,CAAC;AACP,CAAC,CAAC;AA/GW,QAAA,gBAAgB,oBA+G3B;AAEF;;;;;;;;GAQG;AACH,SAAS,wBAAwB,CAAC,uBAAuB;IACrD,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,uBAAuB,CAAC;IAC9D,oEAAoE;IACpE,wCAAwC;IACxC,MAAM,QAAQ,GAAG,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;IAE9D,MAAM,YAAY,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IACrD,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC;IAEjE,iDAAiD;IACjD,OAAO,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;AAChC,CAAC;AAED;;;;;;GAMG;AACH,SAAS,sBAAsB,CAAC,OAAO,EAAE,kBAAkB;IACvD,IAAI,OAAO,KAAK,SAAS;QAAE,kBAAkB,CAAC,IAAI,GAAG,OAAO,CAAC;AACjE,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,mBAAmB,CAAC,IAAI,EAAE,kBAAkB;IACjD,6DAA6D;IAC7D,2DAA2D;IAC3D,MAAM,kBAAkB,GAAG,gDAAgD;UACrE,sEAAsE;UACtE,gEAAgE,CAAC;IACvE,IAAI,IAAI,KAAK,IAAI,EAAE;QACf,mBAAG,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;QACnC,kBAAkB,CAAC,YAAY,GAAG,MAAM,CAAC;QACzC,kBAAkB,CAAC,KAAK,CAAC,OAAO,GAAG,SAAS,CAAC;KAChD;SAAM,IAAI,IAAI,KAAK,KAAK,EAAE;QACvB,mBAAG,CAAC,UAAU,CAAC,kBAAkB,CAAC,CAAC;QACnC,kEAAkE;KACrE;SAAM;QACH,oEAAoE;QACpE,kBAAkB,CAAC,IAAI,GAAG,IAAI,CAAC;KAClC;AACL,CAAC;AAED;;;;;;GAMG;AACH,SAAS,yBAAyB,CAAC,kBAAkB;IACjD,IAAI,kBAAkB,CAAC,KAAK,IAAI,kBAAkB,CAAC,OAAO,EAAE;QACxD,kBAAkB,CAAC,OAAO,GAAG,EAAE,GAAG,kBAAkB,CAAC,OAAO,EAAE,CAAC;QAE/D,8DAA8D;QAC9D,KAAK,MAAM,GAAG,IAAI,kBAAkB,CAAC,OAAO,EAAE;YAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;YAE/B,IAAI,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,MAAM,EAAE;gBAC1C,OAAO,kBAAkB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;aAC1C;SACJ;KACJ;AACL,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,iBAAiB,CAAC,aAAa,EAAE,kBAAkB;IACxD,uEAAuE;IACvE,wDAAwD;IACxD,IAAI,kBAAkB,CAAC,QAAQ;QAAE,OAAO;IAExC,MAAM,SAAS,GAAG,CAAC,QAAQ,EAAE,EAAE;QAC3B,MAAM,WAAW,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,WAAW,EAAE;YACb,MAAM,IAAI,KAAK,CAAC,eAAe,kBAAkB,CAAC,GAAG,gCAAgC,CAAC,CAAC;SAC1F;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC,CAAC;IAEF,MAAM,EAAE,KAAK,EAAE,GAAG,kBAAkB,CAAC;IACrC,MAAM,UAAU,GAAG;QACf,GAAG,KAAK;QACR,aAAa,EAAE;YACX,GAAG,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YACzC,SAAS;SACZ;KACJ,CAAC;IAEF,kBAAkB,CAAC,KAAK,GAAG,UAAU,CAAC;AAC1C,CAAC;AAED;;;;GAIG;AACH,SAAS,yBAAyB,CAAC,YAAY,EAAE,kBAAkB;IAC/D,MAAM,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC;IACtC,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,KAAK,SAAS;WAClD,kBAAkB,CAAC,IAAI,KAAK,SAAS;WACrC,kBAAkB,CAAC,IAAI,KAAK,SAAS,CAAC;IAE7C,IAAI,MAAM,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,WAAW,EAAE;QACzD,YAAY,CAAC,GAAG,EAAE,CAAC;KACtB;AACL,CAAC;AAED;;;;GAIG;AACH,SAAS,oBAAoB,CAAC,OAAO;IACjC,MAAM,iBAAiB,GAAG;QACtB,4EAA4E;QAC5E,CAAC,cAAc,EAAE,gCAAgC,CAAC;QAClD,CAAC,aAAa,EAAE,gCAAgC,CAAC;QACjD,CAAC,kBAAkB,EAAE,gCAAgC,CAAC;QACtD,CAAC,SAAS,EAAE,MAAM,CAAC;QACnB,CAAC,UAAU,EAAE,OAAO,CAAC;QACrB,CAAC,QAAQ,EAAE,UAAU,CAAC;QACtB,CAAC,YAAY,EAAE,YAAY,CAAC;QAC5B,CAAC,mBAAmB,EAAE,iBAAiB,CAAC;QACxC,CAAC,aAAa,EAAE,iBAAiB,CAAC;QAClC,CAAC,iBAAiB,EAAE,0BAA0B,CAAC;QAC/C,CAAC,eAAe,CAAC,EAAE,uBAAuB;KAC7C,CAAC;IAEF,KAAK,MAAM,CAAC,gBAAgB,EAAE,SAAS,CAAC,IAAI,iBAAiB,EAAE;QAC3D,IAAI,OAAO,CAAC,gBAAgB,CAAC,KAAK,SAAS,EAAE;YACzC,wFAAwF;YACxF,MAAM,cAAc,GAAG,2FAA2F;kBAC5G,2GAA2G;kBAC3G,sGAAsG,CAAC;YAC7G,mBAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;YAE/B,IAAI,gBAAgB,KAAK,eAAe,EAAE;gBACtC,mBAAG,CAAC,UAAU,CAAC,YAAY,gBAAgB,kBAAkB;sBACvD,uEAAuE;sBACvE,wGAAwG,CAAC,CAAC;aACnH;iBAAM;gBACH,mBAAG,CAAC,UAAU,CAAC,YAAY,gBAAgB,iCAAiC,SAAS,YAAY,CAAC,CAAC;aACtG;SACJ;KACJ;AACL,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,6BAA6B,CAAC,MAAM,EAAE,QAAQ;IACnD,MAAM,UAAU,GAAG;QACf,YAAY,EAAE,eAAe,EAAE,SAAS;QACxC,UAAU,EAAE,aAAa,EAAE,YAAY;QACvC,aAAa,EAAE,UAAU,EAAE,KAAK;QAChC,SAAS;KACZ,CAAC;IAEF,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACpB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAElD,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE;QAC3B,IAAI,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,EAAE;YACnB,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;SACjC;KACJ;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}

package/build/utils_social.d.ts

@@ -1,210 +0,0 @@
-export namespace socialUtils {
-    export { emailsFromText };
-    export { emailsFromUrls };
-    export { phonesFromText };
-    export { phonesFromUrls };
-    export { parseHandlesFromHtml };
-    export { EMAIL_REGEX };
-    export { EMAIL_REGEX_GLOBAL };
-    export { LINKEDIN_REGEX };
-    export { LINKEDIN_REGEX_GLOBAL };
-    export { INSTAGRAM_REGEX };
-    export { INSTAGRAM_REGEX_GLOBAL };
-    export { TWITTER_REGEX };
-    export { TWITTER_REGEX_GLOBAL };
-    export { FACEBOOK_REGEX };
-    export { FACEBOOK_REGEX_GLOBAL };
-    export { YOUTUBE_REGEX };
-    export { YOUTUBE_REGEX_GLOBAL };
-    export { TIKTOK_REGEX };
-    export { TIKTOK_REGEX_GLOBAL };
-    export { PINTEREST_REGEX };
-    export { PINTEREST_REGEX_GLOBAL };
-    export { DISCORD_REGEX };
-    export { DISCORD_REGEX_GLOBAL };
-}
-/**
- * Representation of social handles parsed from a HTML page.
- *
- * The object has the following structure:
- *
- * ```
- * {
- *   emails: String[],
- *   phones: String[],
- *   phonesUncertain: String[],
- *   linkedIns: String[],
- *   twitters: String[],
- *   instagrams: String[],
- *   facebooks: String[],
- *   youtubes: String[],
- *   tiktoks: String[],
- *   pinterests: String[],
- *   discords: String[],
- * }
- * ```
- */
-export type SocialHandles = {
-    emails: string[];
-    phones: string[];
-    phonesUncertain: string[];
-    linkedIns: string[];
-    twitters: string[];
-    instagrams: string[];
-    facebooks: string[];
-    youtubes: string[];
-    tiktoks: string[];
-    pinterests: string[];
-    discords: string[];
-};
-/**
- * The function extracts email addresses from a plain text.
- * Note that the function preserves the order of emails and keep duplicates.
- * @param {string} text Text to search in.
- * @return {string[]} Array of emails addresses found.
- * If no emails are found, the function returns an empty array.
- * @memberOf social
- */
-declare function emailsFromText(text: string): string[];
-/**
- * The function extracts email addresses from a list of URLs.
- * Basically it looks for all `mailto:` URLs and returns valid email addresses from them.
- * Note that the function preserves the order of emails and keep duplicates.
- * @param {string[]} urls Array of URLs.
- * @return {string[]} Array of emails addresses found.
- * If no emails are found, the function returns an empty array.
- * @memberOf social
- */
-declare function emailsFromUrls(urls: string[]): string[];
-/**
- * The function attempts to extract phone numbers from a text. Please note that
- * the results might not be accurate, since phone numbers appear in a large variety of formats and conventions.
- * If you encounter some problems, please [file an issue](https://github.com/apify/apify-js/issues).
- * @param {string} text Text to search the phone numbers in.
- * @return {string[]} Array of phone numbers found.
- * If no phone numbers are found, the function returns an empty array.
- * @memberOf social
- */
-declare function phonesFromText(text: string): string[];
-/**
- * Finds phone number links in an array of URLs and extracts the phone numbers from them.
- * Note that the phone number links look like `tel://123456789`, `tel:/123456789` or `tel:123456789`.
- * @param {string[]} urls Array of URLs.
- * @return {string[]} Array of phone numbers found.
- * If no phone numbers are found, the function returns an empty array.
- * @memberOf social
- */
-declare function phonesFromUrls(urls: string[]): string[];
-/**
- * Representation of social handles parsed from a HTML page.
- *
- * The object has the following structure:
- *
- * ```
- * {
- *   emails: String[],
- *   phones: String[],
- *   phonesUncertain: String[],
- *   linkedIns: String[],
- *   twitters: String[],
- *   instagrams: String[],
- *   facebooks: String[],
- *   youtubes: String[],
- *   tiktoks: String[],
- *   pinterests: String[],
- *   discords: String[],
- * }
- * ```
- * @typedef SocialHandles
- * @property {string[]} emails
- * @property {string[]} phones
- * @property {string[]} phonesUncertain
- * @property {string[]} linkedIns
- * @property {string[]} twitters
- * @property {string[]} instagrams
- * @property {string[]} facebooks
- * @property {string[]} youtubes
- * @property {string[]} tiktoks
- * @property {string[]} pinterests
- * @property {string[]} discords
- */
-/**
- * The function attempts to extract emails, phone numbers and social profile URLs from a HTML document,
- * specifically LinkedIn, Twitter, Instagram and Facebook profile URLs.
- * The function removes duplicates from the resulting arrays and sorts the items alphabetically.
- *
- * Note that the `phones` field contains phone numbers extracted from the special phone links
- * such as `[call us](tel:+1234556789)` (see {@link social#phonesFromUrls})
- * and potentially other sources with high certainty, while `phonesUncertain` contains phone numbers
- * extracted from the plain text, which might be very inaccurate.
- *
- * **Example usage:**
- * ```javascript
- * const Apify = require('apify');
- *
- * const browser = await Apify.launchPuppeteer();
- * const page = await browser.newPage();
- * await page.goto('http://www.example.com');
- * const html = await page.content();
- *
- * const result = Apify.utils.social.parseHandlesFromHtml(html);
- * console.log('Social handles:');
- * console.dir(result);
- * ```
- *
- * @param {string} html HTML text
- * @param {*|null} [data] Optional object which will receive the `text` and `$` properties
- *   that contain text content of the HTML and `cheerio` object, respectively. This is an optimization
- *   so that the caller doesn't need to parse the HTML document again, if needed.
- * @return {SocialHandles} An object with the social handles.
- *
- * @memberOf social
- */
-declare function parseHandlesFromHtml(html: string, data?: any | null): SocialHandles;
-/**
- * Regular expression to exactly match a single email address.
- * It has the following form: `/^...$/i`.
- * @type {RegExp}
- * @memberOf social
- */
-declare const EMAIL_REGEX: RegExp;
-/**
- * Regular expression to find multiple email addresses in a text.
- * It has the following form: `/.../ig`.
- * @type {RegExp}
- * @memberOf social
- */
-declare const EMAIL_REGEX_GLOBAL: RegExp;
-/** @type RegExp */
-declare let LINKEDIN_REGEX: RegExp;
-/** @type RegExp */
-declare let LINKEDIN_REGEX_GLOBAL: RegExp;
-/** @type RegExp */
-declare let INSTAGRAM_REGEX: RegExp;
-/** @type RegExp */
-declare let INSTAGRAM_REGEX_GLOBAL: RegExp;
-/** @type RegExp */
-declare let TWITTER_REGEX: RegExp;
-/** @type RegExp */
-declare let TWITTER_REGEX_GLOBAL: RegExp;
-/** @type RegExp */
-declare let FACEBOOK_REGEX: RegExp;
-/** @type RegExp */
-declare let FACEBOOK_REGEX_GLOBAL: RegExp;
-/** @type RegExp */
-declare let YOUTUBE_REGEX: RegExp;
-/** @type RegExp */
-declare let YOUTUBE_REGEX_GLOBAL: RegExp;
-/** @type RegExp */
-declare let TIKTOK_REGEX: RegExp;
-/** @type RegExp */
-declare let TIKTOK_REGEX_GLOBAL: RegExp;
-/** @type RegExp */
-declare let PINTEREST_REGEX: RegExp;
-/** @type RegExp */
-declare let PINTEREST_REGEX_GLOBAL: RegExp;
-declare let DISCORD_REGEX: any;
-/** @type RegExp */
-declare let DISCORD_REGEX_GLOBAL: RegExp;
-export {};
-//# sourceMappingURL=utils_social.d.ts.map

package/build/utils_social.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"utils_social.d.ts","sourceRoot":"","sources":["../src/utils_social.js"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YAgrBc,MAAM,EAAE;YACR,MAAM,EAAE;qBACR,MAAM,EAAE;eACR,MAAM,EAAE;cACR,MAAM,EAAE;gBACR,MAAM,EAAE;eACR,MAAM,EAAE;cACR,MAAM,EAAE;aACR,MAAM,EAAE;gBACR,MAAM,EAAE;cACR,MAAM,EAAE;;AA9pBtB;;;;;;;GAOG;AACH,sCALW,MAAM,GACL,MAAM,EAAE,CAOnB;AAED;;;;;;;;GAQG;AACH,sCALW,MAAM,EAAE,GACP,MAAM,EAAE,CAgBnB;AAgED;;;;;;;;GAQG;AACH,sCALW,MAAM,GACL,MAAM,EAAE,CAqBnB;AAED;;;;;;;GAOG;AACH,sCALW,MAAM,EAAE,GACP,MAAM,EAAE,CAgBnB;AA6eD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,4CARW,MAAM,SACN,MAAE,IAAI,GAGL,aAAa,CA4DxB;AA3wBD;;;;;GAKG;AACH,2BAHU,MAAM,CAG+C;AAE/D;;;;;GAKG;AACH,kCAHU,MAAM,CAGgD;AAwLhE,mBAAmB;AACnB,4BADU,MAAM,CACG;AACnB,mBAAmB;AACnB,mCADU,MAAM,CACU;AAC1B,mBAAmB;AACnB,6BADU,MAAM,CACI;AACpB,mBAAmB;AACnB,oCADU,MAAM,CACW;AAC3B,mBAAmB;AACnB,2BADU,MAAM,CACE;AAClB,mBAAmB;AACnB,kCADU,MAAM,CACS;AACzB,mBAAmB;AACnB,4BADU,MAAM,CACG;AACnB,mBAAmB;AACnB,mCADU,MAAM,CACU;AAC1B,mBAAmB;AACnB,2BADU,MAAM,CACE;AAClB,mBAAmB;AACnB,kCADU,MAAM,CACS;AACzB,mBAAmB;AACnB,0BADU,MAAM,CACC;AACjB,mBAAmB;AACnB,iCADU,MAAM,CACQ;AACxB,mBAAmB;AACnB,6BADU,MAAM,CACI;AACpB,mBAAmB;AACnB,oCADU,MAAM,CACW;AAC3B,+BAAkB;AAClB,mBAAmB;AACnB,kCADU,MAAM,CACS"}