apify 2.3.1-beta.4 → 3.0.0-alpha.0

package/build/proxy_configuration.js

@@ -1,517 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createProxyConfiguration = exports.ProxyConfiguration = void 0;
-const tslib_1 = require("tslib");
-const consts_1 = require("@apify/consts");
-const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
-const constants_1 = require("./constants");
-const utils_1 = require("./utils");
-const utils_request_1 = require("./utils_request");
-const utils_log_1 = (0, tslib_1.__importDefault)(require("./utils_log"));
-const configuration_1 = require("./configuration");
-// CONSTANTS
-const PROTOCOL = 'http';
-// https://docs.apify.com/proxy/datacenter-proxy#username-parameters
-const MAX_SESSION_ID_LENGTH = 50;
-const CHECK_ACCESS_REQUEST_TIMEOUT_MILLIS = 4000;
-const CHECK_ACCESS_MAX_ATTEMPTS = 2;
-/**
- * @callback ProxyConfigurationFunction
- * @param {string|number} sessionId
- * @returns {string}
- */
-/**
- * @typedef ProxyConfigurationOptions
- * @property {string} [password]
- *   User's password for the proxy. By default, it is taken from the `APIFY_PROXY_PASSWORD`
- *   environment variable, which is automatically set by the system when running the actors.
- * @property {string[]} [groups]
- *   An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy).
- *   If not provided, the proxy will select the groups automatically.
- * @property {string} [countryCode]
- *   If set and relevant proxies are available in your Apify account, all proxied requests will
- *   use IP addresses that are geolocated to the specified country. For example `GB` for IPs
- *   from Great Britain. Note that online services often have their own rules for handling
- *   geolocation and thus the country selection is a best attempt at geolocation, rather than
- *   a guaranteed hit. This parameter is optional, by default, each proxied request is assigned
- *   an IP address from a random country. The country code needs to be a two letter ISO country code. See the
- *   [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).
- *   This parameter is optional, by default, the proxy uses all available proxy servers from all countries.
- *   on the Apify cloud, or when using the [Apify CLI](https://github.com/apify/apify-cli).
- * @property {string[]} [apifyProxyGroups]
- *   Same option as `groups` which can be used to
- *   configurate the proxy by UI input schema. You should use the `groups` option in your crawler code.
- * @property {string} [apifyProxyCountry]
- *   Same option as `countryCode` which can be used to
- *   configurate the proxy by UI input schema. You should use the `countryCode` option in your crawler code.
- * @property {string[]} [proxyUrls]
- *   An array of custom proxy URLs to be rotated.
- *   Custom proxies are not compatible with Apify Proxy and an attempt to use both
- *   configuration options will cause an error to be thrown on initialize.
- * @property {ProxyConfigurationFunction} [newUrlFunction]
- *   Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter
- *   and should always return stringified proxy URL.
- *   This function is used to generate the URL when {@link ProxyConfiguration.newUrl} or {@link ProxyConfiguration.newProxyInfo} is called.
- */
-/**
- * The main purpose of the ProxyInfo object is to provide information
- * about the current proxy connection used by the crawler for the request.
- * Outside of crawlers, you can get this object by calling {@link ProxyConfiguration.newProxyInfo}.
- *
- * **Example usage:**
- *
- * ```javascript
- *
- * const proxyConfiguration = await Apify.createProxyConfiguration({
- *   groups: ['GROUP1', 'GROUP2'] // List of Apify Proxy groups
- *   countryCode: 'US',
- * });
- *
- * // Getting proxyInfo object by calling class method directly
- * const proxyInfo = proxyConfiguration.newProxyInfo();
- *
- * // In crawler
- * const crawler = new Apify.CheerioCrawler({
- *   // ...
- *   proxyConfiguration,
- *   handlePageFunction: ({ proxyInfo }) => {
- *      // Getting used proxy URL
- *       const proxyUrl = proxyInfo.url;
- *
- *      // Getting ID of used Session
- *       const sessionIdentifier = proxyInfo.sessionId;
- *   }
- * })
- *
- * ```
- * @typedef ProxyInfo
- * @property {string} [sessionId]
- *   The identifier of used {@link Session}, if used.
- * @property {string} url
- *   The URL of the proxy.
- * @property {string[]} groups
- *   An array of proxy groups to be used by the [Apify Proxy](https://docs.apify.com/proxy).
- *   If not provided, the proxy will select the groups automatically.
- * @property {string} [countryCode]
- *   If set and relevant proxies are available in your Apify account, all proxied requests will
- *   use IP addresses that are geolocated to the specified country. For example `GB` for IPs
- *   from Great Britain. Note that online services often have their own rules for handling
- *   geolocation and thus the country selection is a best attempt at geolocation, rather than
- *   a guaranteed hit. This parameter is optional, by default, each proxied request is assigned
- *   an IP address from a random country. The country code needs to be a two letter ISO country code. See the
- *   [full list of available country codes](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements).
- *   This parameter is optional, by default, the proxy uses all available proxy servers from all countries.
- * @property {string} password
- *   User's password for the proxy. By default, it is taken from the `APIFY_PROXY_PASSWORD`
- *   environment variable, which is automatically set by the system when running the actors
- *   on the Apify cloud, or when using the [Apify CLI](https://github.com/apify/apify-cli).
- * @property {string} hostname
- *   Hostname of your proxy.
- * @property {string} port
- *   Proxy port.
- */
-/**
- * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
- * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
- * them to use the selected proxies for all connections. You can get information about the currently used proxy by inspecting
- * the {@link ProxyInfo} property in your crawler's page function. There, you can inspect the proxy's URL and other attributes.
- *
- * The proxy servers are managed by [Apify Proxy](https://docs.apify.com/proxy). To be able to use Apify Proxy,
- * you need an Apify account and access to the selected proxies. If you provide no configuration option,
- * the proxies will be managed automatically using a smart algorithm.
- *
- * If you want to use your own proxies, use the {@link ProxyConfigurationOptions.proxyUrls} option. Your list of proxy URLs will
- * be rotated by the configuration if this option is provided.
- *
- * **Example usage:**
- *
- * ```javascript
- *
- * const proxyConfiguration = await Apify.createProxyConfiguration({
- *   groups: ['GROUP1', 'GROUP2'] // List of Apify Proxy groups
- *   countryCode: 'US',
- * });
- *
- * const crawler = new Apify.CheerioCrawler({
- *   // ...
- *   proxyConfiguration,
- *   handlePageFunction: ({ proxyInfo }) => {
- *      const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
- *   }
- * })
- *
- * ```
- * @hideconstructor
- */
-class ProxyConfiguration {
-    /**
-     * Configuration of proxy.
-     *
-     * @param {ProxyConfigurationOptions} [options] All `ProxyConfiguration` options.
-     * @param {Configuration} [config]
-     */
-    constructor(options = {}, config = configuration_1.Configuration.getGlobalConfig()) {
-        (0, ow_1.default)(options, ow_1.default.object.exactShape({
-            groups: ow_1.default.optional.array.ofType(ow_1.default.string.matches(consts_1.APIFY_PROXY_VALUE_REGEX)),
-            apifyProxyGroups: ow_1.default.optional.array.ofType(ow_1.default.string.matches(consts_1.APIFY_PROXY_VALUE_REGEX)),
-            countryCode: ow_1.default.optional.string.matches(constants_1.COUNTRY_CODE_REGEX),
-            apifyProxyCountry: ow_1.default.optional.string.matches(constants_1.COUNTRY_CODE_REGEX),
-            proxyUrls: ow_1.default.optional.array.nonEmpty.ofType(ow_1.default.string.url),
-            password: ow_1.default.optional.string,
-            newUrlFunction: ow_1.default.optional.function,
-        }));
-        const { groups = [], apifyProxyGroups = [], countryCode, apifyProxyCountry, proxyUrls, password = config.get('proxyPassword'), newUrlFunction, } = options;
-        const groupsToUse = groups.length ? groups : apifyProxyGroups;
-        const countryCodeToUse = countryCode || apifyProxyCountry;
-        const hostname = config.get('proxyHostname');
-        const port = config.get('proxyPort');
-        // Validation
-        if (((proxyUrls || newUrlFunction) && ((groupsToUse.length) || countryCodeToUse))) {
-            this._throwCannotCombineCustomWithApify();
-        }
-        if (proxyUrls && newUrlFunction)
-            this._throwCannotCombineCustomMethods();
-        this.groups = groupsToUse;
-        this.countryCode = countryCodeToUse;
-        this.password = password;
-        this.hostname = hostname;
-        this.port = port;
-        this.nextCustomUrlIndex = 0;
-        this.proxyUrls = proxyUrls;
-        this.usedProxyUrls = new Map();
-        this.newUrlFunction = newUrlFunction;
-        this.usesApifyProxy = !this.proxyUrls && !this.newUrlFunction;
-        this.log = utils_log_1.default.child({ prefix: 'ProxyConfiguration' });
-        this.config = config;
-        this.isManInTheMiddle = false;
-        if (proxyUrls && proxyUrls.some((url) => url.includes('apify.com'))) {
-            this.log.warning('Some Apify proxy features may work incorrectly. Please consider setting up Apify properties instead of `proxyUrls`.\n'
-                + 'See https://sdk.apify.com/docs/guides/proxy-management#apify-proxy-configuration');
-        }
-    }
-    /**
-     * Loads proxy password if token is provided and checks access to Apify Proxy and provided proxy groups
-     * if Apify Proxy configuration is used.
-     * Also checks if country has access to Apify Proxy groups if the country code is provided.
-     *
-     * You should use the {@link Apify.createProxyConfiguration} function to create a pre-initialized
-     * `ProxyConfiguration` instance instead of calling this manually.
-     *
-     * @returns {Promise<void>}
-     */
-    async initialize() {
-        if (this.usesApifyProxy) {
-            await this._setPasswordIfToken();
-            await this._checkAccess();
-        }
-    }
-    /**
-     * This function creates a new {@link ProxyInfo} info object.
-     * It is used by CheerioCrawler and PuppeteerCrawler to generate proxy URLs and also to allow the user to inspect
-     * the currently used proxy via the handlePageFunction parameter: proxyInfo.
-     * Use it if you want to work with a rich representation of a proxy URL.
-     * If you need the URL string only, use {@link ProxyConfiguration.newUrl}.
-     * @param {string|number} [sessionId]
-     *  Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
-     *  you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
-     *  When the provided sessionId is a number, it's converted to a string. Property sessionId of
-     *  {@link ProxyInfo} is always returned as a type string.
-     *
-     *  All the HTTP requests going through the proxy with the same session identifier
-     *  will use the same target proxy server (i.e. the same IP address).
-     *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
-     * @return {ProxyInfo} represents information about used proxy and its configuration.
-     */
-    newProxyInfo(sessionId) {
-        if (typeof sessionId === 'number')
-            sessionId = `${sessionId}`;
-        (0, ow_1.default)(sessionId, ow_1.default.optional.string.maxLength(MAX_SESSION_ID_LENGTH).matches(consts_1.APIFY_PROXY_VALUE_REGEX));
-        const url = this.newUrl(sessionId);
-        const { groups, countryCode, password, port, hostname } = this.usesApifyProxy ? this : new URL(url);
-        return {
-            sessionId,
-            url,
-            groups,
-            countryCode,
-            password,
-            hostname,
-            port,
-        };
-    }
-    /**
-     * Returns a new proxy URL based on provided configuration options and the `sessionId` parameter.
-     * @param {string|number} [sessionId]
-     *  Represents the identifier of user {@link Session} that can be managed by the {@link SessionPool} or
-     *  you can use the Apify Proxy [Session](https://docs.apify.com/proxy#sessions) identifier.
-     *  When the provided sessionId is a number, it's converted to a string.
-     *
-     *  All the HTTP requests going through the proxy with the same session identifier
-     *  will use the same target proxy server (i.e. the same IP address).
-     *  The identifier must not be longer than 50 characters and include only the following: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
-     * @return {string} A string with a proxy URL, including authentication credentials and port number.
-     *  For example, `http://bob:password123@proxy.example.com:8000`
-     */
-    newUrl(sessionId) {
-        if (typeof sessionId === 'number')
-            sessionId = `${sessionId}`;
-        (0, ow_1.default)(sessionId, ow_1.default.optional.string.maxLength(MAX_SESSION_ID_LENGTH).matches(consts_1.APIFY_PROXY_VALUE_REGEX));
-        if (this.newUrlFunction) {
-            return this._callNewUrlFunction(sessionId);
-        }
-        if (this.proxyUrls) {
-            return this._handleCustomUrl(sessionId);
-        }
-        const username = this._getUsername(sessionId);
-        const { password, hostname, port } = this;
-        return `${PROTOCOL}://${username}:${password}@${hostname}:${port}`;
-    }
-    /**
-     * Returns proxy username.
-     * @param {string} [sessionId]
-     * @return {string} the proxy username
-     * @ignore
-     * @protected
-     * @internal
-     */
-    _getUsername(sessionId) {
-        let username;
-        const { groups, countryCode } = this;
-        const parts = [];
-        if (groups && groups.length) {
-            parts.push(`groups-${groups.join('+')}`);
-        }
-        if (sessionId) {
-            parts.push(`session-${sessionId}`);
-        }
-        if (countryCode) {
-            parts.push(`country-${countryCode}`);
-        }
-        username = parts.join(',');
-        if (parts.length === 0)
-            username = 'auto';
-        return username;
-    }
-    /**
-     * Checks if Apify Token is provided in env
-     * and gets the password via API and sets it to env
-     * @returns {Promise<void>}
-     * @ignore
-     * @protected
-     * @internal
-     */
-    async _setPasswordIfToken() {
-        const token = this.config.get('token');
-        if (token) {
-            const { proxy: { password } } = await utils_1.apifyClient.user().get();
-            if (this.password) {
-                if (this.password !== password) {
-                    this.log.warning('The Apify Proxy password you provided belongs to'
-                        + ' a different user than the Apify token you are using. Are you sure this is correct?');
-                }
-            }
-            else {
-                this.password = password;
-            }
-        }
-        if (!this.password) {
-            throw new Error(`Apify Proxy password must be provided using options.password or the "${consts_1.ENV_VARS.PROXY_PASSWORD}" environment variable.`
-                + `If you add the "${consts_1.ENV_VARS.TOKEN}" environment variable, the password will be automatically inferred.`);
-        }
-    }
-    /**
-     * Checks whether the user has access to the proxies specified in the provided ProxyConfigurationOptions.
-     * If the check can not be made, it only prints a warning and allows the program to continue. This is to
-     * prevent program crashes caused by short downtimes of Proxy.
-     *
-     * @returns {Promise<void>}
-     * @protected
-     * @ignore
-     * @internal
-     */
-    async _checkAccess() {
-        const status = await this._fetchStatus();
-        if (status) {
-            const { connected, connectionError, isManInTheMiddle } = status;
-            this.isManInTheMiddle = isManInTheMiddle;
-            if (!connected)
-                this._throwApifyProxyConnectionError(connectionError);
-        }
-        else {
-            this.log.warning('Apify Proxy access check timed out. Watch out for errors with status code 407. '
-                + 'If you see some, it most likely means you don\'t have access to either all or some of the proxies you\'re trying to use.');
-        }
-    }
-    /**
-     * Apify Proxy can be down for a second or a minute, but this should not crash processes.
-     *
-     * @return {Promise<{ connected: boolean, connectionError: string } | undefined>}
-     * @protected
-     * @ignore
-     * @internal
-     */
-    async _fetchStatus() {
-        const requestOpts = {
-            url: `${this.config.get('proxyStatusUrl')}/?format=json`,
-            proxyUrl: this.newUrl(),
-            timeout: { request: CHECK_ACCESS_REQUEST_TIMEOUT_MILLIS },
-            responseType: 'json',
-        };
-        for (let attempt = 1; attempt <= CHECK_ACCESS_MAX_ATTEMPTS; attempt++) {
-            try {
-                const response = await (0, utils_request_1.requestAsBrowser)(requestOpts);
-                return response.body;
-            }
-            catch (err) {
-                // retry connection errors
-            }
-        }
-    }
-    /**
-     * Handles custom url rotation with session
-     * @param {string} [sessionId]
-     * @returns {string} url
-     * @protected
-     * @ignore
-     * @internal
-     */
-    _handleCustomUrl(sessionId) {
-        let customUrlToUse;
-        if (sessionId) {
-            if (this.usedProxyUrls.has(sessionId)) {
-                customUrlToUse = this.usedProxyUrls.get(sessionId);
-            }
-            else {
-                customUrlToUse = this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
-                this.usedProxyUrls.set(sessionId, customUrlToUse);
-            }
-        }
-        else {
-            customUrlToUse = this.proxyUrls[this.nextCustomUrlIndex++ % this.proxyUrls.length];
-        }
-        return customUrlToUse;
-    }
-    /**
-     * Calls the custom newUrlFunction and checks format of its return value
-     * @param {string} [sessionId]
-     * @protected
-     * @ignore
-     * @internal
-     */
-    _callNewUrlFunction(sessionId) {
-        let proxyUrl;
-        try {
-            proxyUrl = this.newUrlFunction(sessionId);
-            new URL(proxyUrl); // eslint-disable-line no-new
-        }
-        catch (err) {
-            this._throwNewUrlFunctionInvalid(err);
-        }
-        return proxyUrl;
-    }
-    /**
-     * Throws invalid custom newUrlFunction return
-     * @param {Error} err
-     * @protected
-     * @ignore
-     * @internal
-     */
-    _throwNewUrlFunctionInvalid(err) {
-        throw new Error(`The provided newUrlFunction did not return a valid URL.\nCause: ${err.message}`);
-    }
-    /**
-     * Throws invalid proxy value error
-     * @param {string} param
-     * @param {string} value
-     * @protected
-     * @ignore
-     * @internal
-     */
-    _throwInvalidProxyValueError(param, value) {
-        throw new Error(`The provided proxy ${param} name "${value}" can only contain the following characters: 0-9, a-z, A-Z, ".", "_" and "~"`);
-    }
-    /**
-     * Throws Apify Proxy is not connected
-     * @protected
-     * @ignore
-     * @internal
-     */
-    _throwApifyProxyConnectionError(errorMessage) {
-        throw new Error(errorMessage);
-    }
-    /**
-     * Throws cannot combine custom proxies with Apify Proxy
-     * @protected
-     * @ignore
-     * @internal
-     */
-    _throwCannotCombineCustomWithApify() {
-        throw new Error('Cannot combine custom proxies with Apify Proxy!'
-            + 'It is not allowed to set "options.proxyUrls" or "options.newUrlFunction" combined with '
-            + '"options.groups" or "options.apifyProxyGroups" and "options.countryCode" or "options.apifyProxyCountry".');
-    }
-    /**
-     * Throws cannot combine custom 2 custom methods
-     * @protected
-     * @ignore
-     * @internal
-     */
-    _throwCannotCombineCustomMethods() {
-        throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".');
-    }
-}
-exports.ProxyConfiguration = ProxyConfiguration;
-/**
- * Creates a proxy configuration and returns a promise resolving to an instance
- * of the {@link ProxyConfiguration} class that is already initialized.
- *
- * Configures connection to a proxy server with the provided options. Proxy servers are used to prevent target websites from blocking
- * your crawlers based on IP address rate limits or blacklists. Setting proxy configuration in your crawlers automatically configures
- * them to use the selected proxies for all connections.
- *
- * For more details and code examples, see the {@link ProxyConfiguration} class.
- *
- * ```javascript
- *
- * // Returns initialized proxy configuration class
- * const proxyConfiguration = await Apify.createProxyConfiguration({
- *     groups: ['GROUP1', 'GROUP2'] // List of Apify proxy groups
- *     countryCode: 'US'
- * });
- *
- * const crawler = new Apify.CheerioCrawler({
- *   // ...
- *   proxyConfiguration,
- *   handlePageFunction: ({ proxyInfo }) => {
- *       const usedProxyUrl = proxyInfo.url; // Getting the proxy URL
- *   }
- * })
- *
- * ```
- *
- * For compatibility with existing Actor Input UI (Input Schema), this function
- * returns `undefined` when the following object is passed as `proxyConfigurationOptions`.
- *
- * ```
- * { useApifyProxy: false }
- * ```
- *
- * @param {ProxyConfigurationOptions} [proxyConfigurationOptions]
- * @returns {Promise<ProxyConfiguration|undefined>}
- * @memberof module:Apify
- * @name createProxyConfiguration
- * @function
- */
-const createProxyConfiguration = async (proxyConfigurationOptions = {}) => {
-    // Compatibility fix for Input UI where proxy: None returns { useApifyProxy: false }
-    // Without this, it would cause proxy to use the zero config / auto mode.
-    const { useApifyProxy, ...options } = proxyConfigurationOptions;
-    const dontUseApifyProxy = useApifyProxy === false;
-    const dontUseCustomProxies = !proxyConfigurationOptions.proxyUrls;
-    if (dontUseApifyProxy && dontUseCustomProxies)
-        return undefined;
-    const proxyConfiguration = new ProxyConfiguration(options);
-    await proxyConfiguration.initialize();
-    return proxyConfiguration;
-};
-exports.createProxyConfiguration = createProxyConfiguration;
-//# sourceMappingURL=proxy_configuration.js.map

package/build/proxy_configuration.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"proxy_configuration.js","sourceRoot":"","sources":["../src/proxy_configuration.js"],"names":[],"mappings":";;;;AAAA,0CAAkE;AAClE,yDAAoB;AACpB,2CAAiD;AACjD,mCAAsC;AACtC,mDAAmD;AACnD,yEAAqC;AACrC,mDAAgD;AAEhD,YAAY;AACZ,MAAM,QAAQ,GAAG,MAAM,CAAC;AACxB,oEAAoE;AACpE,MAAM,qBAAqB,GAAG,EAAE,CAAC;AACjC,MAAM,mCAAmC,GAAG,IAAK,CAAC;AAClD,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAEpC;;;;GAIG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwDG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAa,kBAAkB;IAC3B;;;;;OAKG;IACH,YAAY,OAAO,GAAG,EAAE,EAAE,MAAM,GAAG,6BAAa,CAAC,eAAe,EAAE;QAC9D,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7B,MAAM,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,gCAAuB,CAAC,CAAC;YAC5E,gBAAgB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,gCAAuB,CAAC,CAAC;YACtF,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,8BAAkB,CAAC;YAC3D,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,8BAAkB,CAAC;YACjE,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAE,CAAC,MAAM,CAAC,GAAG,CAAC;YAC3D,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;YAC5B,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;SACvC,CAAC,CAAC,CAAC;QAEJ,MAAM,EACF,MAAM,GAAG,EAAE,EACX,gBAAgB,GAAG,EAAE,EACrB,WAAW,EACX,iBAAiB,EACjB,SAAS,EACT,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,EACtC,cAAc,GACjB,GAAG,OAAO,CAAC;QAEZ,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,gBAAgB,CAAC;QAC9D,MAAM,gBAAgB,GAAG,WAAW,IAAI,iBAAiB,CAAC;QAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAErC,aAAa;QACb,IAAI,CAAC,CAAC,SAAS,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,gBAAgB,CAAC,CAAC,EAAE;YAC/E,IAAI,CAAC,kCAAkC,EAAE,CAAC;SAC7C;QACD,IAAI,SAAS,IAAI,cAAc;YAAE,IAAI,CAAC,gCAAgC,EAAE,CAAC;QAEzE,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,gBAAgB,CAAC;QACpC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC;QAC5B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,aAAa,GAAG,IAAI,GAAG,EAAE,CAAC;QAC/B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,cAAc,GAAG,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC;QAC9D,IAAI,CAAC,GAAG,GAAG,mBAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;QAC9D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAE9B,IAAI,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,EAAE;YACjE,IAAI,CAAC,GAAG,CAAC,OAAO,CACZ,uHAAuH;kBACrH,kFAAkF,CACvF,CAAC;SACL;IACL,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAEjC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;SAC7B;IACL,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,YAAY,CAAC,SAAS;QAClB,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAC9D,IAAA,YAAE,EAAC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC,OAAO,CAAC,gCAAuB,CAAC,CAAC,CAAC;QACpG,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAEnC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAEpG,OAAO;YACH,SAAS;YACT,GAAG;YACH,MAAM;YACN,WAAW;YACX,QAAQ;YACR,QAAQ;YACR,IAAI;SACP,CAAC;IACN,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,MAAM,CAAC,SAAS;QACZ,IAAI,OAAO,SAAS,KAAK,QAAQ;YAAE,SAAS,GAAG,GAAG,SAAS,EAAE,CAAC;QAC9D,IAAA,YAAE,EAAC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC,OAAO,CAAC,gCAAuB,CAAC,CAAC,CAAC;QACpG,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,OAAO,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;SAC9C;QACD,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;SAC3C;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAC9C,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QAE1C,OAAO,GAAG,QAAQ,MAAM,QAAQ,IAAI,QAAQ,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;IACvE,CAAC;IAED;;;;;;;OAOG;IACH,YAAY,CAAC,SAAS;QAClB,IAAI,QAAQ,CAAC;QACb,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC;QACrC,MAAM,KAAK,GAAG,EAAE,CAAC;QAEjB,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE;YACzB,KAAK,CAAC,IAAI,CAAC,UAAU,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;SAC5C;QACD,IAAI,SAAS,EAAE;YACX,KAAK,CAAC,IAAI,CAAC,WAAW,SAAS,EAAE,CAAC,CAAC;SACtC;QACD,IAAI,WAAW,EAAE;YACb,KAAK,CAAC,IAAI,CAAC,WAAW,WAAW,EAAE,CAAC,CAAC;SACxC;QAED,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE3B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,QAAQ,GAAG,MAAM,CAAC;QAE1C,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,mBAAmB;QACrB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,KAAK,EAAE;YACP,MAAM,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,EAAE,GAAG,MAAM,mBAAW,CAAC,IAAI,EAAE,CAAC,GAAG,EAAE,CAAC;YAC/D,IAAI,IAAI,CAAC,QAAQ,EAAE;gBACf,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE;oBAC5B,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,kDAAkD;0BACjE,qFAAqF,CAAC,CAAC;iBAC5F;aACJ;iBAAM;gBACH,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;aAC5B;SACJ;QACD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE;YAChB,MAAM,IAAI,KAAK,CAAC,wEAAwE,iBAAQ,CAAC,cAAc,yBAAyB;kBAClI,mBAAmB,iBAAQ,CAAC,KAAK,sEAAsE,CAAC,CAAC;SAClH;IACL,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,YAAY;QACd,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QACzC,IAAI,MAAM,EAAE;YACR,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,gBAAgB,EAAE,GAAG,MAAM,CAAC;YAChE,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;YAEzC,IAAI,CAAC,SAAS;gBAAE,IAAI,CAAC,+BAA+B,CAAC,eAAe,CAAC,CAAC;SACzE;aAAM;YACH,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,iFAAiF;kBAC5F,0HAA0H,CAAC,CAAC;SACrI;IACL,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,YAAY;QACd,MAAM,WAAW,GAAG;YAChB,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,gBAAgB,CAAC,eAAe;YACxD,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE;YACvB,OAAO,EAAE,EAAE,OAAO,EAAE,mCAAmC,EAAE;YACzD,YAAY,EAAE,MAAM;SACvB,CAAC;QACF,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,yBAAyB,EAAE,OAAO,EAAE,EAAE;YACnE,IAAI;gBACA,MAAM,QAAQ,GAAG,MAAM,IAAA,gCAAgB,EAAC,WAAW,CAAC,CAAC;gBACrD,OAAO,QAAQ,CAAC,IAAI,CAAC;aACxB;YAAC,OAAO,GAAG,EAAE;gBACV,0BAA0B;aAC7B;SACJ;IACL,CAAC;IAED;;;;;;;OAOG;IACH,gBAAgB,CAAC,SAAS;QACtB,IAAI,cAAc,CAAC;QACnB,IAAI,SAAS,EAAE;YACX,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;gBACnC,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;aACtD;iBAAM;gBACH,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACnF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;aACrD;SACJ;aAAM;YACH,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,kBAAkB,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;SACtF;QACD,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED;;;;;;OAMG;IACH,mBAAmB,CAAC,SAAS;QACzB,IAAI,QAAQ,CAAC;QACb,IAAI;YACA,QAAQ,GAAG,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAC1C,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,6BAA6B;SACnD;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,2BAA2B,CAAC,GAAG,CAAC,CAAC;SACzC;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;;;;;OAMG;IACH,2BAA2B,CAAC,GAAG;QAC3B,MAAM,IAAI,KAAK,CAAC,mEAAmE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACtG,CAAC;IAED;;;;;;;OAOG;IACH,4BAA4B,CAAC,KAAK,EAAE,KAAK;QACrC,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,UAAU,KAAK,8EAA8E,CAAC,CAAC;IAC9I,CAAC;IAED;;;;;OAKG;IACH,+BAA+B,CAAC,YAAY;QACxC,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC;IAClC,CAAC;IAED;;;;;OAKG;IACH,kCAAkC;QAC9B,MAAM,IAAI,KAAK,CAAC,iDAAiD;cAC3D,yFAAyF;cACzF,0GAA0G,CAAC,CAAC;IACtH,CAAC;IAED;;;;;OAKG;IACH,gCAAgC;QAC5B,MAAM,IAAI,KAAK,CAAC,6GAA6G,CAAC,CAAC;IACnI,CAAC;CACJ;AAtVD,gDAsVC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACI,MAAM,wBAAwB,GAAG,KAAK,EAAE,yBAAyB,GAAG,EAAE,EAAE,EAAE;IAC7E,oFAAoF;IACpF,yEAAyE;IACzE,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO,EAAE,GAAG,yBAAyB,CAAC;IAChE,MAAM,iBAAiB,GAAG,aAAa,KAAK,KAAK,CAAC;IAClD,MAAM,oBAAoB,GAAG,CAAC,yBAAyB,CAAC,SAAS,CAAC;IAClE,IAAI,iBAAiB,IAAI,oBAAoB;QAAE,OAAO,SAAS,CAAC;IAEhE,MAAM,kBAAkB,GAAG,IAAI,kBAAkB,CAAC,OAAO,CAAC,CAAC;IAC3D,MAAM,kBAAkB,CAAC,UAAU,EAAE,CAAC;IAEtC,OAAO,kBAAkB,CAAC;AAC9B,CAAC,CAAC;AAZW,QAAA,wBAAwB,4BAYnC"}

package/build/pseudo_url.d.ts

@@ -1,86 +0,0 @@
-export default PseudoUrl;
-/**
- * Represents a pseudo-URL (PURL) - an URL pattern used by web crawlers
- * to specify which URLs should the crawler visit.
- * This class is used by the {@link utils#enqueueLinks} function.
- *
- * A PURL is simply a URL with special directives enclosed in `[]` brackets.
- * Currently, the only supported directive is `[RegExp]`,
- * which defines a JavaScript-style regular expression to match against the URL.
- *
- * The `PseudoUrl` class can be constructed either using a pseudo-URL string
- * or a regular expression (an instance of the `RegExp` object).
- * With a pseudo-URL string, the matching is always case-insensitive.
- * If you need case-sensitive matching, use an appropriate `RegExp` object.
- *
- * For example, a PURL `http://www.example.com/pages/[(\w|-)*]` will match all of the following URLs:
- *
- * - `http://www.example.com/pages/`
- * - `http://www.example.com/pages/my-awesome-page`
- * - `http://www.example.com/pages/something`
- *
- * Be careful to correctly escape special characters in the pseudo-URL string.
- * If either `[` or `]` is part of the normal query string, it must be encoded as `[\x5B]` or `[\x5D]`,
- * respectively. For example, the following PURL:
- * ```http
- * http://www.example.com/search?do[\x5B]load[\x5D]=1
- * ```
- * will match the URL:
- * ```http
- * http://www.example.com/search?do[load]=1
- * ```
- *
- * If the regular expression in the pseudo-URL contains a backslash character (\),
- * you need to escape it with another back backslash, as shown in the example below.
- *
- * **Example usage:**
- *
- * ```javascript
- * // Using a pseudo-URL string
- * const purl = new Apify.PseudoUrl('http://www.example.com/pages/[(\\w|-)+]', {
- *   userData: { foo: 'bar' },
- * });
- *
- * // Using a regular expression
- * const purl2 = new Apify.PseudoUrl(/http:\/\/www\.example\.com\/pages\/(\w|-)+/);
- *
- * if (purl.matches('http://www.example.com/pages/my-awesome-page')) console.log('Match!');
- * ```
- */
-declare class PseudoUrl {
-    /**
-     * @param {(string|RegExp)} purl
-     *   A pseudo-URL string or a regular expression object.
-     *   Using a `RegExp` instance enables more granular control,
-     *   such as making the matching case sensitive.
-     * @param {RequestOptions} requestTemplate
-     *   Options for the new {@link Request} instances created for matching URLs
-     *   by the {@link utils#enqueueLinks} function.
-     */
-    constructor(purl: (string | RegExp), requestTemplate?: RequestOptions);
-    regex: RegExp;
-    requestTemplate: RequestOptions;
-    /**
-     * Determines whether a URL matches this pseudo-URL pattern.
-     *
-     * @param {string} url URL to be matched.
-     * @return {boolean} Returns `true` if given URL matches pseudo-URL.
-     */
-    matches(url: string): boolean;
-    /**
-     * Creates a Request object from a provided `requestTemplate` and a given URL
-     * or an object that specifies ${@link Request} properties. In case of a collision
-     * the properties will override the template, except for `userData`, which will
-     * be merged together, with the `userData` property having preference over the template.
-     * This enables dynamic overriding of the template.
-     *
-     * @param {(string|Object<string, *>)} urlOrProps
-     * @return {Request}
-     */
-    createRequest(urlOrProps: (string | {
-        [x: string]: any;
-    })): Request;
-}
-import { RequestOptions } from "./request";
-import Request from "./request";
-//# sourceMappingURL=pseudo_url.d.ts.map

package/build/pseudo_url.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"pseudo_url.d.ts","sourceRoot":"","sources":["../src/pseudo_url.js"],"names":[],"mappings":";AAmDA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH;IACI;;;;;;;;OAQG;IACH,kBARW,CAAC,MAAM,GAAC,MAAM,CAAC,oBAIf,cAAc,EAiBxB;IARO,cAAiB;IAOrB,gCAAsC;IAG1C;;;;;OAKG;IACH,aAHW,MAAM,GACL,OAAO,CAIlB;IAED;;;;;;;;;OASG;IACH,0BAHW,CAAC,MAAM;YAAQ,MAAM;KAAI,CAAC,GACzB,OAAO,CAOlB;CACJ"}