apify 2.3.1-beta.4 → 3.0.0-alpha.0

package/build/pseudo_url.js

@@ -1,153 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const tslib_1 = require("tslib");
-const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
-const underscore_1 = (0, tslib_1.__importDefault)(require("underscore"));
-const utils_log_1 = (0, tslib_1.__importDefault)(require("./utils_log"));
-const request_1 = (0, tslib_1.__importDefault)(require("./request")); // eslint-disable-line import/named,no-unused-vars
-/**
- * Parses PURL into Regex string.
- * @ignore
- */
-const parsePurl = (purl) => {
-    const trimmedPurl = purl.trim();
-    if (trimmedPurl.length === 0)
-        throw new Error(`Cannot parse PURL '${trimmedPurl}': it must be an non-empty string`);
-    let regex = '^';
-    try {
-        let openBrackets = 0;
-        for (let i = 0; i < trimmedPurl.length; i++) {
-            const ch = trimmedPurl.charAt(i);
-            if (ch === '[' && ++openBrackets === 1) {
-                // Beginning of '[regex]' section
-                // Enclose regex in () brackets to enforce operator priority
-                regex += '(';
-            }
-            else if (ch === ']' && openBrackets > 0 && --openBrackets === 0) {
-                // End of '[regex]' section
-                regex += ')';
-            }
-            else if (openBrackets > 0) {
-                // Inside '[regex]' section
-                regex += ch;
-            }
-            else {
-                // Outside '[regex]' section, parsing the URL part
-                const code = ch.charCodeAt(0);
-                if ((code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122)) {
-                    // Alphanumeric character => copy it.
-                    regex += ch;
-                }
-                else {
-                    // Special character => escape it
-                    const hex = code < 16 ? `0${code.toString(16)}` : code.toString(16);
-                    regex += `\\x${hex}`;
-                }
-            }
-        }
-        regex += '$';
-    }
-    catch (err) {
-        throw new Error(`Cannot parse PURL '${purl}': ${err}`);
-    }
-    return regex;
-};
-/**
- * Represents a pseudo-URL (PURL) - an URL pattern used by web crawlers
- * to specify which URLs should the crawler visit.
- * This class is used by the {@link utils#enqueueLinks} function.
- *
- * A PURL is simply a URL with special directives enclosed in `[]` brackets.
- * Currently, the only supported directive is `[RegExp]`,
- * which defines a JavaScript-style regular expression to match against the URL.
- *
- * The `PseudoUrl` class can be constructed either using a pseudo-URL string
- * or a regular expression (an instance of the `RegExp` object).
- * With a pseudo-URL string, the matching is always case-insensitive.
- * If you need case-sensitive matching, use an appropriate `RegExp` object.
- *
- * For example, a PURL `http://www.example.com/pages/[(\w|-)*]` will match all of the following URLs:
- *
- * - `http://www.example.com/pages/`
- * - `http://www.example.com/pages/my-awesome-page`
- * - `http://www.example.com/pages/something`
- *
- * Be careful to correctly escape special characters in the pseudo-URL string.
- * If either `[` or `]` is part of the normal query string, it must be encoded as `[\x5B]` or `[\x5D]`,
- * respectively. For example, the following PURL:
- * ```http
- * http://www.example.com/search?do[\x5B]load[\x5D]=1
- * ```
- * will match the URL:
- * ```http
- * http://www.example.com/search?do[load]=1
- * ```
- *
- * If the regular expression in the pseudo-URL contains a backslash character (\),
- * you need to escape it with another back backslash, as shown in the example below.
- *
- * **Example usage:**
- *
- * ```javascript
- * // Using a pseudo-URL string
- * const purl = new Apify.PseudoUrl('http://www.example.com/pages/[(\\w|-)+]', {
- *   userData: { foo: 'bar' },
- * });
- *
- * // Using a regular expression
- * const purl2 = new Apify.PseudoUrl(/http:\/\/www\.example\.com\/pages\/(\w|-)+/);
- *
- * if (purl.matches('http://www.example.com/pages/my-awesome-page')) console.log('Match!');
- * ```
- */
-class PseudoUrl {
-    /**
-     * @param {(string|RegExp)} purl
-     *   A pseudo-URL string or a regular expression object.
-     *   Using a `RegExp` instance enables more granular control,
-     *   such as making the matching case sensitive.
-     * @param {RequestOptions} requestTemplate
-     *   Options for the new {@link Request} instances created for matching URLs
-     *   by the {@link utils#enqueueLinks} function.
-     */
-    constructor(purl, requestTemplate = {}) {
-        (0, ow_1.default)(purl, ow_1.default.any(ow_1.default.string, ow_1.default.regExp));
-        (0, ow_1.default)(requestTemplate, ow_1.default.object);
-        if (purl instanceof RegExp) {
-            this.regex = purl;
-        }
-        else {
-            const regex = parsePurl(purl);
-            utils_log_1.default.debug('PURL parsed', { purl, regex });
-            this.regex = new RegExp(regex, 'i');
-        }
-        this.requestTemplate = requestTemplate;
-    }
-    /**
-     * Determines whether a URL matches this pseudo-URL pattern.
-     *
-     * @param {string} url URL to be matched.
-     * @return {boolean} Returns `true` if given URL matches pseudo-URL.
-     */
-    matches(url) {
-        return underscore_1.default.isString(url) && url.match(this.regex) !== null;
-    }
-    /**
-     * Creates a Request object from a provided `requestTemplate` and a given URL
-     * or an object that specifies ${@link Request} properties. In case of a collision
-     * the properties will override the template, except for `userData`, which will
-     * be merged together, with the `userData` property having preference over the template.
-     * This enables dynamic overriding of the template.
-     *
-     * @param {(string|Object<string, *>)} urlOrProps
-     * @return {Request}
-     */
-    createRequest(urlOrProps) {
-        const props = typeof urlOrProps === 'string' ? { url: urlOrProps } : urlOrProps;
-        props.userData = { ...this.requestTemplate.userData, ...props.userData };
-        const options = { ...this.requestTemplate, ...props }; // props.userData will override template with merged data.
-        return new request_1.default(options);
-    }
-}
-exports.default = PseudoUrl;
-//# sourceMappingURL=pseudo_url.js.map

package/build/pseudo_url.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"pseudo_url.js","sourceRoot":"","sources":["../src/pseudo_url.js"],"names":[],"mappings":";;;AAAA,yDAAoB;AACpB,yEAA2B;AAC3B,yEAA8B;AAC9B,qEAAoD,CAAC,kDAAkD;AAEvG;;;GAGG;AACH,MAAM,SAAS,GAAG,CAAC,IAAI,EAAE,EAAE;IACvB,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,WAAW,mCAAmC,CAAC,CAAC;IAEpH,IAAI,KAAK,GAAG,GAAG,CAAC;IAEhB,IAAI;QACA,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACzC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAEjC,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,YAAY,KAAK,CAAC,EAAE;gBACpC,iCAAiC;gBACjC,4DAA4D;gBAC5D,KAAK,IAAI,GAAG,CAAC;aAChB;iBAAM,IAAI,EAAE,KAAK,GAAG,IAAI,YAAY,GAAG,CAAC,IAAI,EAAE,YAAY,KAAK,CAAC,EAAE;gBAC/D,2BAA2B;gBAC3B,KAAK,IAAI,GAAG,CAAC;aAChB;iBAAM,IAAI,YAAY,GAAG,CAAC,EAAE;gBACzB,2BAA2B;gBAC3B,KAAK,IAAI,EAAE,CAAC;aACf;iBAAM;gBACH,kDAAkD;gBAClD,MAAM,IAAI,GAAG,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBAC9B,IAAI,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,GAAG,CAAC,EAAE;oBACzF,qCAAqC;oBACrC,KAAK,IAAI,EAAE,CAAC;iBACf;qBAAM;oBACH,iCAAiC;oBACjC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;oBACpE,KAAK,IAAI,MAAM,GAAG,EAAE,CAAC;iBACxB;aACJ;SACJ;QACD,KAAK,IAAI,GAAG,CAAC;KAChB;IAAC,OAAO,GAAG,EAAE;QACV,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,MAAM,GAAG,EAAE,CAAC,CAAC;KAC1D;IAED,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,MAAM,SAAS;IACX;;;;;;;;OAQG;IACH,YAAY,IAAI,EAAE,eAAe,GAAG,EAAE;QAClC,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,GAAG,CAAC,YAAE,CAAC,MAAM,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC,CAAC;QACvC,IAAA,YAAE,EAAC,eAAe,EAAE,YAAE,CAAC,MAAM,CAAC,CAAC;QAE/B,IAAI,IAAI,YAAY,MAAM,EAAE;YACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;SACrB;aAAM;YACH,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;YAC9B,mBAAG,CAAC,KAAK,CAAC,aAAa,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;SACvC;QAED,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3C,CAAC;IAED;;;;;OAKG;IACH,OAAO,CAAC,GAAG;QACP,OAAO,oBAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC;IAC7D,CAAC;IAED;;;;;;;;;OASG;IACH,aAAa,CAAC,UAAU;QACpB,MAAM,KAAK,GAAG,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;QAChF,KAAK,CAAC,QAAQ,GAAG,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;QACzE,MAAM,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,eAAe,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC,0DAA0D;QACjH,OAAO,IAAI,iBAAO,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;CACJ;AAED,kBAAe,SAAS,CAAC"}

package/build/puppeteer_request_interception.d.ts

@@ -1,8 +0,0 @@
-export function addInterceptRequestHandler(page: Page, handler: InterceptHandler): Promise<void>;
-export function removeInterceptRequestHandler(page: Page, handler: InterceptHandler): Promise<void>;
-export type InterceptHandler = (request: PuppeteerRequest) => any;
-// @ts-ignore optional peer dependency
-import { Page } from "puppeteer";
-// @ts-ignore optional peer dependency
-import { HTTPRequest as PuppeteerRequest } from "puppeteer";
-//# sourceMappingURL=puppeteer_request_interception.d.ts.map

package/build/puppeteer_request_interception.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"puppeteer_request_interception.d.ts","sourceRoot":"","sources":["../src/puppeteer_request_interception.js"],"names":[],"mappings":"AA2KO,iDAPI,IAAI,WAEJ,gBAAgB,GACf,QAAQ,IAAI,CAAC,CAsCxB;AAYM,oDAPI,IAAI,WAEJ,gBAAgB,GACf,QAAQ,IAAI,CAAC,CAmCxB;yCA/MU,gBAAgB"}

package/build/puppeteer_request_interception.js

@@ -1,235 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.removeInterceptRequestHandler = exports.addInterceptRequestHandler = void 0;
-const tslib_1 = require("tslib");
-const events_1 = require("events");
-const ow_1 = (0, tslib_1.__importDefault)(require("ow"));
-const underscore_1 = (0, tslib_1.__importDefault)(require("underscore"));
-const utils_log_1 = (0, tslib_1.__importDefault)(require("./utils_log"));
-// We use weak maps here so that the content gets discarted after page gets closed.
-const pageInterceptRequestHandlersMap = new WeakMap(); // Maps page to an array of request interception handlers.
-const pageInterceptRequestMasterHandlerMap = new WeakMap(); // Maps page to master request interception handler.
-const pageInterceptedRequestsMap = new WeakMap(); // Maps page to a set of its pending intercepted requests.
-/**
- * Enables observation of changes of internal state
- * to be able to queue other actions based on it.
- * @ignore
- */
-class ObservableSet extends events_1.EventEmitter {
-    constructor() {
-        super();
-        this.set = new Set();
-    }
-    add(value) {
-        this.set.add(value);
-        this.emit('add', value);
-        return this.set;
-    }
-    delete(value) {
-        const success = this.set.delete(value);
-        this.emit('delete', value);
-        return success;
-    }
-    get size() {
-        return this.set.size;
-    }
-}
-/**
- * @callback InterceptHandler
- * @param {PuppeteerRequest} request
- */
-/**
- * Makes all request headers capitalized to more look like in browser
- * @param {Object<string, string>} headers
- * @returns {Object<string, string>}
- */
-const browserifyHeaders = (headers) => {
-    const finalHeaders = {};
-    // eslint-disable-next-line prefer-const
-    for (let [key, value] of Object.entries(headers)) {
-        key = key.toLowerCase()
-            .split('-')
-            .map((str) => str.charAt(0).toUpperCase() + str.slice(1))
-            .join('-');
-        finalHeaders[key] = value;
-    }
-    return finalHeaders;
-};
-/**
- * Executes an array for given intercept request handlers for a given request object.
- *
- * @param {PuppeteerRequest} request Puppeteer's Request object.
- * @param {Array<InterceptHandler>} interceptRequestHandlers An array of intercept request handlers.
- * @ignore
- */
-const handleRequest = async (request, interceptRequestHandlers) => {
-    // If there are no intercept handlers, it means that request interception is not enabled (anymore)
-    // and therefore .abort() .respond() and .continue() would throw and crash the process.
-    if (!interceptRequestHandlers.length)
-        return;
-    let wasAborted = false;
-    let wasResponded = false;
-    let wasContinued = false;
-    const accumulatedOverrides = {
-        headers: browserifyHeaders(request.headers()),
-    };
-    const originalContinue = request.continue.bind(request);
-    request.continue = (overrides = {}) => {
-        wasContinued = true;
-        const headers = browserifyHeaders({ ...accumulatedOverrides.headers, ...overrides.headers });
-        Object.assign(accumulatedOverrides, overrides, { headers });
-    };
-    request.abort = underscore_1.default.wrap(request.abort.bind(request), (abort, ...args) => {
-        wasAborted = true;
-        return abort(...args);
-    });
-    request.respond = underscore_1.default.wrap(request.respond.bind(request), (respond, ...args) => {
-        wasResponded = true;
-        return respond(...args);
-    });
-    for (const handler of interceptRequestHandlers) {
-        wasContinued = false;
-        await handler(request);
-        // Check that one of the functions was called.
-        if (!wasAborted && !wasResponded && !wasContinued) {
-            throw new Error('Intercept request handler must call one of request.continue|respond|abort() methods!');
-        }
-        // If request was aborted or responded then we can finish immediately.
-        if (wasAborted || wasResponded)
-            return;
-    }
-    return originalContinue(accumulatedOverrides);
-};
-/**
- * Adds request interception handler in similar to `page.on('request', handler);` but in addition to that
- * supports multiple parallel handlers.
- *
- * All the handlers are executed sequentially in the order as they were added.
- * Each of the handlers must call one of `request.continue()`, `request.abort()` and `request.respond()`.
- * In addition to that any of the handlers may modify the request object (method, postData, headers)
- * by passing its overrides to `request.continue()`.
- * If multiple handlers modify same property then the last one wins. Headers are merged separately so you can
- * override only a value of specific header.
- *
- * If one the handlers calls `request.abort()` or `request.respond()` then request is not propagated further
- * to any of the remaining handlers.
- *
- *
- * **Example usage:**
- *
- * ```javascript
- * // Replace images with placeholder.
- * await addInterceptRequestHandler(page, (request) => {
- *     if (request.resourceType() === 'image') {
- *         return request.respond({
- *             statusCode: 200,
- *             contentType: 'image/jpeg',
- *             body: placeholderImageBuffer,
- *         });
- *     }
- *     return request.continue();
- * });
- *
- * // Abort all the scripts.
- * await addInterceptRequestHandler(page, (request) => {
- *     if (request.resourceType() === 'script') return request.abort();
- *     return request.continue();
- * });
- *
- * // Change requests to post.
- * await addInterceptRequestHandler(page, (request) => {
- *     return request.continue({
- *          method: 'POST',
- *     });
- * });
- *
- * await page.goto('http://example.com');
- * ```
- *
- * @param {Page} page
- *   Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
- * @param {InterceptHandler} handler Request interception handler.
- * @return {Promise<void>}
- * @memberOf puppeteer
- * @name addInterceptRequestHandler
- */
-const addInterceptRequestHandler = async (page, handler) => {
-    (0, ow_1.default)(page, ow_1.default.object.hasKeys('goto', 'evaluate'));
-    (0, ow_1.default)(handler, ow_1.default.function);
-    if (!pageInterceptRequestHandlersMap.has(page)) {
-        pageInterceptRequestHandlersMap.set(page, []);
-    }
-    if (!pageInterceptedRequestsMap.has(page)) {
-        pageInterceptedRequestsMap.set(page, new ObservableSet());
-    }
-    const handlersArray = pageInterceptRequestHandlersMap.get(page);
-    handlersArray.push(handler);
-    // First handler was just added at this point so we need to set up request interception.
-    if (handlersArray.length === 1) {
-        await page.setRequestInterception(true);
-        // This is a handler that gets set in page.on('request', ...) and that executes all the user
-        // added custom handlers.
-        const masterHandler = async (request) => {
-            const interceptedRequests = pageInterceptedRequestsMap.get(page);
-            interceptedRequests.add(request);
-            const interceptHandlers = pageInterceptRequestHandlersMap.get(page);
-            try {
-                await handleRequest(request, interceptHandlers);
-            }
-            finally {
-                interceptedRequests.delete(request);
-            }
-        };
-        pageInterceptRequestMasterHandlerMap.set(page, masterHandler);
-        page.on('request', masterHandler);
-    }
-};
-exports.addInterceptRequestHandler = addInterceptRequestHandler;
-/**
- * Removes request interception handler for given page.
- *
- * @param {Page} page
- *   Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
- * @param {InterceptHandler} handler Request interception handler.
- * @return {Promise<void>}
- * @memberOf puppeteer
- * @name removeInterceptRequestHandler
- */
-const removeInterceptRequestHandler = async (page, handler) => {
-    (0, ow_1.default)(page, ow_1.default.object.hasKeys('goto', 'evaluate'));
-    (0, ow_1.default)(handler, ow_1.default.function);
-    const handlersArray = pageInterceptRequestHandlersMap
-        .get(page)
-        .filter((item) => item !== handler);
-    pageInterceptRequestHandlersMap.set(page, handlersArray);
-    if (handlersArray.length === 0) {
-        const interceptedRequestsInProgress = pageInterceptedRequestsMap.get(page);
-        // Since handlers can be async, we can't simply turn off request interception
-        // when there are no handlers, because some handlers could still
-        // be in progress and request.abort|respond|continue() would throw.
-        if (interceptedRequestsInProgress.size === 0) {
-            await disableRequestInterception(page);
-        }
-        else {
-            const onDelete = async () => {
-                if (interceptedRequestsInProgress.size === 0) {
-                    try {
-                        await disableRequestInterception(page);
-                        interceptedRequestsInProgress.removeListener('delete', onDelete);
-                    }
-                    catch (error) {
-                        utils_log_1.default.debug('Error while disabling request interception', { error });
-                    }
-                }
-            };
-            interceptedRequestsInProgress.on('delete', onDelete);
-        }
-    }
-};
-exports.removeInterceptRequestHandler = removeInterceptRequestHandler;
-async function disableRequestInterception(page) {
-    await page.setRequestInterception(false);
-    const requestHandler = pageInterceptRequestMasterHandlerMap.get(page);
-    page.removeListener('request', requestHandler);
-}
-//# sourceMappingURL=puppeteer_request_interception.js.map

package/build/puppeteer_request_interception.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"puppeteer_request_interception.js","sourceRoot":"","sources":["../src/puppeteer_request_interception.js"],"names":[],"mappings":";;;;AAAA,mCAAsC;AACtC,yDAAoB;AACpB,yEAA2B;AAE3B,yEAA8B;AAE9B,mFAAmF;AACnF,MAAM,+BAA+B,GAAG,IAAI,OAAO,EAAE,CAAC,CAAC,0DAA0D;AACjH,MAAM,oCAAoC,GAAG,IAAI,OAAO,EAAE,CAAC,CAAC,oDAAoD;AAChH,MAAM,0BAA0B,GAAG,IAAI,OAAO,EAAE,CAAC,CAAC,0DAA0D;AAE5G;;;;GAIG;AACH,MAAM,aAAc,SAAQ,qBAAY;IACpC;QACI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;IACzB,CAAC;IAED,GAAG,CAAC,KAAK;QACL,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC,GAAG,CAAC;IACpB,CAAC;IAED,MAAM,CAAC,KAAK;QACR,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC3B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,IAAI,IAAI;QACJ,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;IACzB,CAAC;CACJ;AAED;;;GAGG;AAEH;;;;GAIG;AACH,MAAM,iBAAiB,GAAG,CAAC,OAAO,EAAE,EAAE;IAClC,MAAM,YAAY,GAAG,EAAE,CAAC;IACxB,wCAAwC;IACxC,KAAK,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;QAC9C,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE;aAClB,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aACxD,IAAI,CAAC,GAAG,CAAC,CAAC;QAEf,YAAY,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;KAC7B;IAED,OAAO,YAAY,CAAC;AACxB,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,aAAa,GAAG,KAAK,EAAE,OAAO,EAAE,wBAAwB,EAAE,EAAE;IAC9D,kGAAkG;IAClG,uFAAuF;IACvF,IAAI,CAAC,wBAAwB,CAAC,MAAM;QAAE,OAAO;IAE7C,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,MAAM,oBAAoB,GAAG;QACzB,OAAO,EAAE,iBAAiB,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;KAChD,CAAC;IAEF,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxD,OAAO,CAAC,QAAQ,GAAG,CAAC,SAAS,GAAG,EAAE,EAAE,EAAE;QAClC,YAAY,GAAG,IAAI,CAAC;QACpB,MAAM,OAAO,GAAG,iBAAiB,CAAC,EAAE,GAAG,oBAAoB,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,oBAAoB,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAChE,CAAC,CAAC;IAEF,OAAO,CAAC,KAAK,GAAG,oBAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,IAAI,EAAE,EAAE;QACnE,UAAU,GAAG,IAAI,CAAC;QAElB,OAAO,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,OAAO,GAAG,oBAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,EAAE,EAAE;QACzE,YAAY,GAAG,IAAI,CAAC;QAEpB,OAAO,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC;IAC5B,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,OAAO,IAAI,wBAAwB,EAAE;QAC5C,YAAY,GAAG,KAAK,CAAC;QAErB,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACvB,8CAA8C;QAC9C,IAAI,CAAC,UAAU,IAAI,CAAC,YAAY,IAAI,CAAC,YAAY,EAAE;YAC/C,MAAM,IAAI,KAAK,CAAC,sFAAsF,CAAC,CAAC;SAC3G;QAED,sEAAsE;QACtE,IAAI,UAAU,IAAI,YAAY;YAAE,OAAO;KAC1C;IAED,OAAO,gBAAgB,CAAC,oBAAoB,CAAC,CAAC;AAClD,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoDG;AACI,MAAM,0BAA0B,GAAG,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;IAC9D,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;IAChD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;IAEzB,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QAC5C,+BAA+B,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;KACjD;IACD,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACvC,0BAA0B,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,aAAa,EAAE,CAAC,CAAC;KAC7D;IAED,MAAM,aAAa,GAAG,+BAA+B,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAChE,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAE5B,wFAAwF;IACxF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE;QAC5B,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAExC,4FAA4F;QAC5F,yBAAyB;QACzB,MAAM,aAAa,GAAG,KAAK,EAAE,OAAO,EAAE,EAAE;YACpC,MAAM,mBAAmB,GAAG,0BAA0B,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACjE,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACjC,MAAM,iBAAiB,GAAG,+BAA+B,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACpE,IAAI;gBACA,MAAM,aAAa,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;aACnD;oBAAS;gBACN,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;aACvC;QACL,CAAC,CAAC;QAEF,oCAAoC,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAC9D,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;KACrC;AACL,CAAC,CAAC;AAlCW,QAAA,0BAA0B,8BAkCrC;AAEF;;;;;;;;;GASG;AACI,MAAM,6BAA6B,GAAG,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;IACjE,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;IAChD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;IAEzB,MAAM,aAAa,GAAG,+BAA+B;SAChD,GAAG,CAAC,IAAI,CAAC;SACT,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;IAExC,+BAA+B,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IAEzD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE;QAC5B,MAAM,6BAA6B,GAAG,0BAA0B,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC3E,6EAA6E;QAC7E,gEAAgE;QAChE,mEAAmE;QACnE,IAAI,6BAA6B,CAAC,IAAI,KAAK,CAAC,EAAE;YAC1C,MAAM,0BAA0B,CAAC,IAAI,CAAC,CAAC;SAC1C;aAAM;YACH,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;gBACxB,IAAI,6BAA6B,CAAC,IAAI,KAAK,CAAC,EAAE;oBAC1C,IAAI;wBACA,MAAM,0BAA0B,CAAC,IAAI,CAAC,CAAC;wBACvC,6BAA6B,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;qBACpE;oBAAC,OAAO,KAAK,EAAE;wBACZ,mBAAG,CAAC,KAAK,CAAC,4CAA4C,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;qBACtE;iBACJ;YACL,CAAC,CAAC;YACF,6BAA6B,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;SACxD;KACJ;AACL,CAAC,CAAC;AA/BW,QAAA,6BAA6B,iCA+BxC;AAEF,KAAK,UAAU,0BAA0B,CAAC,IAAI;IAC1C,MAAM,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,cAAc,GAAG,oCAAoC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACtE,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;AACnD,CAAC"}

package/build/puppeteer_utils.d.ts

@@ -1,250 +0,0 @@
-export function gotoExtended(page: Page, request: Request, gotoOptions?: any): Promise<(Response | null)>;
-export function infiniteScroll(page: Page, options?: {
-    timeoutSecs?: number | undefined;
-    waitForSecs?: number | undefined;
-    scrollDownAndUp?: boolean | undefined;
-    buttonSelector?: string | undefined;
-    stopScrollCallback?: Function | undefined;
-} | undefined): Promise<void>;
-export namespace puppeteerUtils {
-    export { injectFile };
-    export { injectJQuery };
-    export { injectUnderscore };
-    export { enqueueLinksByClickingElements };
-    export { blockRequests };
-    export { blockResources };
-    export { cacheResponses };
-    export { compileScript };
-    export { gotoExtended };
-    export { addInterceptRequestHandler };
-    export { removeInterceptRequestHandler };
-    export { infiniteScroll };
-    export { saveSnapshot };
-}
-export type CompiledScriptParams = {
-    page: Page;
-    request: Request;
-};
-export type CompiledScriptFunction = (params: CompiledScriptParams) => Promise<any>;
-// @ts-ignore optional peer dependency
-import { Page } from "puppeteer";
-/**
- * Injects a JavaScript file into a Puppeteer page.
- * Unlike Puppeteer's `addScriptTag` function, this function works on pages
- * with arbitrary Cross-Origin Resource Sharing (CORS) policies.
- *
- * File contents are cached for up to 10 files to limit file system access.
- *
- * @param {Page} page
- *   Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
- * @param {string} filePath File path
- * @param {object} [options]
- * @param {boolean} [options.surviveNavigations]
- *   Enables the injected script to survive page navigations and reloads without need to be re-injected manually.
- *   This does not mean, however, that internal state will be preserved. Just that it will be automatically
- *   re-injected on each navigation before any other scripts get the chance to execute.
- * @return {Promise<*>}
- * @memberOf puppeteer
- */
-declare function injectFile(page: Page, filePath: string, options?: {
-    surviveNavigations?: boolean | undefined;
-} | undefined): Promise<any>;
-/**
- * Injects the [jQuery](https://jquery.com/) library into a Puppeteer page.
- * jQuery is often useful for various web scraping and crawling tasks.
- * For example, it can help extract text from HTML elements using CSS selectors.
- *
- * Beware that the injected jQuery object will be set to the `window.$` variable and thus it might cause conflicts with
- * other libraries included by the page that use the same variable name (e.g. another version of jQuery).
- * This can affect functionality of page's scripts.
- *
- * The injected jQuery will survive page navigations and reloads.
- *
- * **Example usage:**
- * ```javascript
- * await Apify.utils.puppeteer.injectJQuery(page);
- * const title = await page.evaluate(() => {
- *   return $('head title').text();
- * });
- * ```
- *
- * Note that `injectJQuery()` does not affect the Puppeteer's
- * [`page.$()`](https://pptr.dev/#?product=Puppeteer&show=api-pageselector)
- * function in any way.
- *
- * @param {Page} page
- *   Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
- * @return {Promise<*>}
- * @memberOf puppeteer
- */
-declare function injectJQuery(page: Page): Promise<any>;
-/**
- * Injects the [Underscore](https://underscorejs.org/) library into a Puppeteer page.
- *
- * Beware that the injected Underscore object will be set to the `window._` variable and thus it might cause conflicts with
- * libraries included by the page that use the same variable name.
- * This can affect functionality of page's scripts.
- *
- * The injected Underscore will survive page navigations and reloads.
- *
- * **Example usage:**
- * ```javascript
- * await Apify.utils.puppeteer.injectUnderscore(page);
- * const escapedHtml = await page.evaluate(() => {
- *   return _.escape('<h1>Hello</h1>');
- * });
- * ```
- *
- * @param {Page} page Puppeteer [Page](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#class-page) object.
- * @return {Promise<*>}
- * @memberOf puppeteer
- */
-declare function injectUnderscore(page: Page): Promise<any>;
-import { enqueueLinksByClickingElements } from "./enqueue_links/click_elements";
-/**
- * Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.
- * This is useful to speed up crawling of websites, since it reduces the amount
- * of data that needs to be downloaded from the web, but it may break some websites
- * or unexpectedly prevent loading of resources.
- *
- * By default, the function will block all URLs including the following patterns:
- *
- * ```json
- * [".css", ".jpg", ".jpeg", ".png", ".svg", ".gif", ".woff", ".pdf", ".zip"]
- * ```
- *
- * If you want to extend this list further, use the `extraUrlPatterns` option,
- * which will keep blocking the default patterns, as well as add your custom ones.
- * If you would like to block only specific patterns, use the `urlPatterns` option,
- * which will override the defaults and block only URLs with your custom patterns.
- *
- * This function does not use Puppeteer's request interception and therefore does not interfere
- * with browser cache. It's also faster than blocking requests using interception,
- * because the blocking happens directly in the browser without the round-trip to Node.js,
- * but it does not provide the extra benefits of request interception.
- *
- * The function will never block main document loads and their respective redirects.
- *
- * **Example usage**
- * ```javascript
- * const Apify = require('apify');
- *
- * const browser = await Apify.launchPuppeteer();
- * const page = await browser.newPage();
- *
- * // Block all requests to URLs that include `adsbygoogle.js` and also all defaults.
- * await Apify.utils.puppeteer.blockRequests(page, {
- *     extraUrlPatterns: ['adsbygoogle.js'],
- * });
- *
- * await page.goto('https://cnn.com');
- * ```
- *
- * @param {Page} page
- *   Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
- * @param {object} [options]
- * @param {string[]} [options.urlPatterns]
- *   The patterns of URLs to block from being loaded by the browser.
- *   Only `*` can be used as a wildcard. It is also automatically added to the beginning
- *   and end of the pattern. This limitation is enforced by the DevTools protocol.
- *   `.png` is the same as `*.png*`.
- * @param {string[]} [options.extraUrlPatterns]
- *   If you just want to append to the default blocked patterns, use this property.
- * @return {Promise<void>}
- * @memberOf puppeteer
- */
-declare function blockRequests(page: Page, options?: {
-    urlPatterns?: string[] | undefined;
-    extraUrlPatterns?: string[] | undefined;
-} | undefined): Promise<void>;
-/**
- * `blockResources()` has a high impact on performance in recent versions of Puppeteer.
- * 'Until this resolves, please use `Apify.utils.puppeteer.blockRequests()`.
- * @deprecated
- */
-declare function blockResources(page: any, resourceTypes?: string[]): Promise<void>;
-/**
- * *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal
- * performance. Until this resolves, we suggest just relying on the in-browser cache unless absolutely necessary.
- *
- * Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.
- * *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
- *   This issue should be resolved or atleast mitigated in future iterations of this feature.
- * @param {Page} page
- *   Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
- * @param {Object<string, *>} cache
- *   Object in which responses are stored
- * @param {Array<(string|RegExp)>} responseUrlRules
- *   List of rules that are used to check if the response should be cached.
- *   String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).
- * @return {Promise<void>}
- * @memberOf puppeteer
- * @deprecated
- */
-declare function cacheResponses(page: Page, cache: {
-    [x: string]: any;
-}, responseUrlRules: Array<(string | RegExp)>): Promise<void>;
-/**
- * Compiles a Puppeteer script into an async function that may be executed at any time
- * by providing it with the following object:
- * ```
- * {
- *    page: Page,
- *    request: Request,
- * }
- * ```
- * Where `page` is a Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page)
- * and `request` is a {@link Request}.
- *
- * The function is compiled by using the `scriptString` parameter as the function's body,
- * so any limitations to function bodies apply. Return value of the compiled function
- * is the return value of the function body = the `scriptString` parameter.
- *
- * As a security measure, no globals such as `process` or `require` are accessible
- * from within the function body. Note that the function does not provide a safe
- * sandbox and even though globals are not easily accessible, malicious code may
- * still execute in the main process via prototype manipulation. Therefore you
- * should only use this function to execute sanitized or safe code.
- *
- * Custom context may also be provided using the `context` parameter. To improve security,
- * make sure to only pass the really necessary objects to the context. Preferably making
- * secured copies beforehand.
- *
- * @param {string} scriptString
- * @param {Object<string, *>} context
- * @return {CompiledScriptFunction}
- * @memberOf puppeteer
- */
-declare function compileScript(scriptString: string, context?: {
-    [x: string]: any;
-}): CompiledScriptFunction;
-import { addInterceptRequestHandler } from "./puppeteer_request_interception";
-import { removeInterceptRequestHandler } from "./puppeteer_request_interception";
-/**
- * Saves a full screenshot and HTML of the current page into a Key-Value store.
- * @param {Page} page
- *   Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
- * @param {object} [options]
- * @param {string} [options.key=SNAPSHOT]
- *   Key under which the screenshot and HTML will be saved. `.jpg` will be appended for screenshot and `.html` for HTML.
- * @param {number} [options.screenshotQuality=50]
- *   The quality of the image, between 0-100. Higher quality images have bigger size and require more storage.
- * @param {boolean} [options.saveScreenshot=true]
- *   If true, it will save a full screenshot of the current page as a record with `key` appended by `.jpg`.
- * @param {boolean} [options.saveHtml=true]
- *   If true, it will save a full HTML of the current page as a record with `key` appended by `.html`.
- * @param {string|null} [options.keyValueStoreName=null]
- *   Name or id of the Key-Value store where snapshot is saved. By default it is saved to default Key-Value store.
- * @returns {Promise<void>}
- * @memberOf puppeteer
- * @name saveSnapshot
- */
-declare function saveSnapshot(page: Page, options?: {
-    key?: string | undefined;
-    screenshotQuality?: number | undefined;
-    saveScreenshot?: boolean | undefined;
-    saveHtml?: boolean | undefined;
-    keyValueStoreName?: string | null | undefined;
-} | undefined): Promise<void>;
-export {};
-//# sourceMappingURL=puppeteer_utils.d.ts.map

package/build/puppeteer_utils.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"puppeteer_utils.d.ts","sourceRoot":"","sources":["../src/puppeteer_utils.js"],"names":[],"mappings":"AAsVO,mCATI,IAAI,WAEJ,OAAO,sBAEN,QAAQ,CAAC,WAAS,IAAI,CAAC,CAAC,CA2CnC;AAsBM,qCAjBI,IAAI;;;;;;gBAaF,QAAQ,IAAI,CAAC,CAyGzB;;;;;;;;;;;;;;;;;UAhea,IAAI;aACJ,OAAO;;8CAIV,oBAAoB,KAClB,YAAU;;AAOvB;;;;;;;;;;;;;;;;;GAiBG;AACH,kCAXW,IAAI,YAEJ,MAAM;;gBAML,YAAU,CAmBrB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,oCALW,IAAI,GAEH,YAAU,CAMrB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wCAJW,IAAI,GACH,YAAU,CAMrB;;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,qCAbW,IAAI;;;gBAUH,QAAQ,IAAI,CAAC,CAkBxB;AAED;;;;GAIG;AACH,oFAQC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,sCAXW,IAAI;QAEG,MAAM;qBAEb,MAAM,CAAC,MAAM,GAAC,MAAM,CAAC,CAAC,GAGrB,QAAQ,IAAI,CAAC,CA+CxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,6CALW,MAAM;QACC,MAAM;IACZ,sBAAsB,CAiBjC;;;AAsLD;;;;;;;;;;;;;;;;;;GAkBG;AACH,oCAjBW,IAAI;;;;;;gBAaF,QAAQ,IAAI,CAAC,CAsCzB"}