@crawlee/puppeteer 3.0.0-alpha.2 → 3.0.0-alpha.22

package/internals/utils/puppeteer_request_interception.js

@@ -0,0 +1,225 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removeInterceptRequestHandler = exports.addInterceptRequestHandler = void 0;
+const tslib_1 = require("tslib");
+const events_1 = require("events");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const log_1 = tslib_1.__importDefault(require("@apify/log"));
+// We use weak maps here so that the content gets discarded after page gets closed.
+const pageInterceptRequestHandlersMap = new WeakMap(); // Maps page to an array of request interception handlers.
+const pageInterceptRequestMasterHandlerMap = new WeakMap(); // Maps page to master request interception handler.
+const pageInterceptedRequestsMap = new WeakMap(); // Maps page to a set of its pending intercepted requests.
+/**
+ * Enables observation of changes of internal state to be able to queue other actions based on it.
+ * @ignore
+ */
+class ObservableSet extends events_1.EventEmitter {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "set", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Set()
+        });
+    }
+    add(value) {
+        this.set.add(value);
+        this.emit('add', value);
+        return this.set;
+    }
+    delete(value) {
+        const success = this.set.delete(value);
+        this.emit('delete', value);
+        return success;
+    }
+    get size() {
+        return this.set.size;
+    }
+}
+/**
+ * Makes all request headers capitalized to more look like in browser
+ */
+function browserifyHeaders(headers) {
+    const finalHeaders = {};
+    // eslint-disable-next-line prefer-const
+    for (let [key, value] of Object.entries(headers)) {
+        key = key.toLowerCase()
+            .split('-')
+            .map((str) => str.charAt(0).toUpperCase() + str.slice(1))
+            .join('-');
+        finalHeaders[key] = value;
+    }
+    return finalHeaders;
+}
+/**
+ * Executes an array for given intercept request handlers for a given request object.
+ *
+ * @param request Puppeteer's Request object.
+ * @param interceptRequestHandlers An array of intercept request handlers.
+ * @ignore
+ */
+async function handleRequest(request, interceptRequestHandlers) {
+    // If there are no intercept handlers, it means that request interception is not enabled (anymore)
+    // and therefore .abort() .respond() and .continue() would throw and crash the process.
+    if (!interceptRequestHandlers?.length)
+        return;
+    let wasAborted = false;
+    let wasResponded = false;
+    let wasContinued = false;
+    const accumulatedOverrides = {
+        headers: browserifyHeaders(request.headers()),
+    };
+    const originalContinue = request.continue.bind(request);
+    request.continue = async (overrides = {}) => {
+        wasContinued = true;
+        const headers = browserifyHeaders({ ...accumulatedOverrides.headers, ...overrides.headers });
+        Object.assign(accumulatedOverrides, overrides, { headers });
+    };
+    const { abort, respond } = request;
+    request.abort = async (...args) => {
+        wasAborted = true;
+        return abort.call(request, ...args);
+    };
+    request.respond = async (...args) => {
+        wasResponded = true;
+        return respond.call(request, ...args);
+    };
+    for (const handler of interceptRequestHandlers) {
+        wasContinued = false;
+        await handler(request);
+        // Check that one of the functions was called.
+        if (!wasAborted && !wasResponded && !wasContinued) {
+            throw new Error('Intercept request handler must call one of request.continue|respond|abort() methods!');
+        }
+        // If request was aborted or responded then we can finish immediately.
+        if (wasAborted || wasResponded)
+            return;
+    }
+    return originalContinue(accumulatedOverrides);
+}
+/**
+ * Adds request interception handler in similar to `page.on('request', handler);` but in addition to that
+ * supports multiple parallel handlers.
+ *
+ * All the handlers are executed sequentially in the order as they were added.
+ * Each of the handlers must call one of `request.continue()`, `request.abort()` and `request.respond()`.
+ * In addition to that any of the handlers may modify the request object (method, postData, headers)
+ * by passing its overrides to `request.continue()`.
+ * If multiple handlers modify same property then the last one wins. Headers are merged separately so you can
+ * override only a value of specific header.
+ *
+ * If one the handlers calls `request.abort()` or `request.respond()` then request is not propagated further
+ * to any of the remaining handlers.
+ *
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * // Replace images with placeholder.
+ * await addInterceptRequestHandler(page, (request) => {
+ *     if (request.resourceType() === 'image') {
+ *         return request.respond({
+ *             statusCode: 200,
+ *             contentType: 'image/jpeg',
+ *             body: placeholderImageBuffer,
+ *         });
+ *     }
+ *     return request.continue();
+ * });
+ *
+ * // Abort all the scripts.
+ * await addInterceptRequestHandler(page, (request) => {
+ *     if (request.resourceType() === 'script') return request.abort();
+ *     return request.continue();
+ * });
+ *
+ * // Change requests to post.
+ * await addInterceptRequestHandler(page, (request) => {
+ *     return request.continue({
+ *          method: 'POST',
+ *     });
+ * });
+ *
+ * await page.goto('http://example.com');
+ * ```
+ *
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param handler Request interception handler.
+ */
+async function addInterceptRequestHandler(page, handler) {
+    (0, ow_1.default)(page, ow_1.default.object.hasKeys('goto', 'evaluate'));
+    (0, ow_1.default)(handler, ow_1.default.function);
+    if (!pageInterceptRequestHandlersMap.has(page)) {
+        pageInterceptRequestHandlersMap.set(page, []);
+    }
+    if (!pageInterceptedRequestsMap.has(page)) {
+        pageInterceptedRequestsMap.set(page, new ObservableSet());
+    }
+    const handlersArray = pageInterceptRequestHandlersMap.get(page);
+    handlersArray.push(handler);
+    // First handler was just added at this point so we need to set up request interception.
+    if (handlersArray.length === 1) {
+        await page.setRequestInterception(true);
+        // This is a handler that gets set in page.on('request', ...) and that executes all the user
+        // added custom handlers.
+        const masterHandler = async (request) => {
+            const interceptedRequests = pageInterceptedRequestsMap.get(page);
+            interceptedRequests.add(request);
+            const interceptHandlers = pageInterceptRequestHandlersMap.get(page);
+            try {
+                await handleRequest(request, interceptHandlers);
+            }
+            finally {
+                interceptedRequests.delete(request);
+            }
+        };
+        pageInterceptRequestMasterHandlerMap.set(page, masterHandler);
+        page.on('request', masterHandler);
+    }
+}
+exports.addInterceptRequestHandler = addInterceptRequestHandler;
+/**
+ * Removes request interception handler for given page.
+ *
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param handler Request interception handler.
+ */
+async function removeInterceptRequestHandler(page, handler) {
+    (0, ow_1.default)(page, ow_1.default.object.hasKeys('goto', 'evaluate'));
+    (0, ow_1.default)(handler, ow_1.default.function);
+    const handlersArray = pageInterceptRequestHandlersMap
+        .get(page)
+        .filter((item) => item !== handler);
+    pageInterceptRequestHandlersMap.set(page, handlersArray);
+    if (handlersArray.length === 0) {
+        const interceptedRequestsInProgress = pageInterceptedRequestsMap.get(page);
+        // Since handlers can be async, we can't simply turn off request interception
+        // when there are no handlers, because some handlers could still
+        // be in progress and request.abort|respond|continue() would throw.
+        if (interceptedRequestsInProgress.size === 0) {
+            await disableRequestInterception(page);
+        }
+        else {
+            const onDelete = async () => {
+                if (interceptedRequestsInProgress.size === 0) {
+                    try {
+                        await disableRequestInterception(page);
+                        interceptedRequestsInProgress.removeListener('delete', onDelete);
+                    }
+                    catch (error) {
+                        log_1.default.debug('Error while disabling request interception', { error });
+                    }
+                }
+            };
+            interceptedRequestsInProgress.on('delete', onDelete);
+        }
+    }
+}
+exports.removeInterceptRequestHandler = removeInterceptRequestHandler;
+async function disableRequestInterception(page) {
+    await page.setRequestInterception(false);
+    const requestHandler = pageInterceptRequestMasterHandlerMap.get(page);
+    page.off('request', requestHandler);
+}
+//# sourceMappingURL=puppeteer_request_interception.js.map

package/internals/utils/puppeteer_request_interception.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"puppeteer_request_interception.js","sourceRoot":"","sources":["../../../src/internals/utils/puppeteer_request_interception.ts"],"names":[],"mappings":";;;;AAAA,mCAAsC;AACtC,oDAAoB;AAGpB,6DAA6B;AAE7B,mFAAmF;AACnF,MAAM,+BAA+B,GAAsC,IAAI,OAAO,EAAE,CAAC,CAAC,0DAA0D;AACpJ,MAAM,oCAAoC,GAAG,IAAI,OAAO,EAAE,CAAC,CAAC,oDAAoD;AAChH,MAAM,0BAA0B,GAAG,IAAI,OAAO,EAAE,CAAC,CAAC,0DAA0D;AAE5G;;;GAGG;AACH,MAAM,aAAiB,SAAQ,qBAAY;IAA3C;;QACI;;;;mBAAM,IAAI,GAAG,EAAK;WAAC;IAiBvB,CAAC;IAfG,GAAG,CAAC,KAAQ;QACR,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC,GAAG,CAAC;IACpB,CAAC;IAED,MAAM,CAAC,KAAQ;QACX,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC3B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,IAAI,IAAI;QACJ,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;IACzB,CAAC;CACJ;AAID;;GAEG;AACH,SAAS,iBAAiB,CAAC,OAA+B;IACtD,MAAM,YAAY,GAAuB,EAAE,CAAC;IAC5C,wCAAwC;IACxC,KAAK,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;QAC9C,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE;aAClB,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aACxD,IAAI,CAAC,GAAG,CAAC,CAAC;QAEf,YAAY,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;KAC7B;IAED,OAAO,YAAY,CAAC;AACxB,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,aAAa,CAAC,OAAyB,EAAE,wBAA6C;IACjG,kGAAkG;IAClG,uFAAuF;IACvF,IAAI,CAAC,wBAAwB,EAAE,MAAM;QAAE,OAAO;IAE9C,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,MAAM,oBAAoB,GAAG;QACzB,OAAO,EAAE,iBAAiB,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;KAChD,CAAC;IAEF,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxD,OAAO,CAAC,QAAQ,GAAG,KAAK,EAAE,SAAS,GAAG,EAAE,EAAE,EAAE;QACxC,YAAY,GAAG,IAAI,CAAC;QACpB,MAAM,OAAO,GAAG,iBAAiB,CAAC,EAAE,GAAG,oBAAoB,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,oBAAoB,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAChE,CAAC,CAAC;IAEF,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IACnC,OAAO,CAAC,KAAK,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,EAAE;QAC9B,UAAU,GAAG,IAAI,CAAC;QAClB,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IACxC,CAAC,CAAC;IACF,OAAO,CAAC,OAAO,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,EAAE;QAChC,YAAY,GAAG,IAAI,CAAC;QACpB,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IAC1C,CAAC,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,wBAAwB,EAAE;QAC5C,YAAY,GAAG,KAAK,CAAC;QAErB,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACvB,8CAA8C;QAC9C,IAAI,CAAC,UAAU,IAAI,CAAC,YAAY,IAAI,CAAC,YAAY,EAAE;YAC/C,MAAM,IAAI,KAAK,CAAC,sFAAsF,CAAC,CAAC;SAC3G;QAED,sEAAsE;QACtE,IAAI,UAAU,IAAI,YAAY;YAAE,OAAO;KAC1C;IAED,OAAO,gBAAgB,CAAC,oBAAoB,CAAC,CAAC;AAClD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AACI,KAAK,UAAU,0BAA0B,CAAC,IAAU,EAAE,OAAyB;IAClF,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;IAChD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;IAEzB,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QAC5C,+BAA+B,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;KACjD;IAED,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACvC,0BAA0B,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,aAAa,EAAE,CAAC,CAAC;KAC7D;IAED,MAAM,aAAa,GAAG,+BAA+B,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;IACjE,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAE5B,wFAAwF;IACxF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE;QAC5B,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAExC,4FAA4F;QAC5F,yBAAyB;QACzB,MAAM,aAAa,GAAG,KAAK,EAAE,OAAoB,EAAE,EAAE;YACjD,MAAM,mBAAmB,GAAG,0BAA0B,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACjE,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACjC,MAAM,iBAAiB,GAAG,+BAA+B,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACpE,IAAI;gBACA,MAAM,aAAa,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;aACnD;oBAAS;gBACN,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;aACvC;QACL,CAAC,CAAC;QAEF,oCAAoC,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAC9D,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;KACrC;AACL,CAAC;AAnCD,gEAmCC;AAED;;;;;GAKG;AACI,KAAK,UAAU,6BAA6B,CAAC,IAAU,EAAE,OAAyB;IACrF,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;IAChD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;IAEzB,MAAM,aAAa,GAAG,+BAA+B;SAChD,GAAG,CAAC,IAAI,CAAE;SACV,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;IAExC,+BAA+B,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IAEzD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE;QAC5B,MAAM,6BAA6B,GAAG,0BAA0B,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC3E,6EAA6E;QAC7E,gEAAgE;QAChE,mEAAmE;QACnE,IAAI,6BAA6B,CAAC,IAAI,KAAK,CAAC,EAAE;YAC1C,MAAM,0BAA0B,CAAC,IAAI,CAAC,CAAC;SAC1C;aAAM;YACH,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;gBACxB,IAAI,6BAA6B,CAAC,IAAI,KAAK,CAAC,EAAE;oBAC1C,IAAI;wBACA,MAAM,0BAA0B,CAAC,IAAI,CAAC,CAAC;wBACvC,6BAA6B,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;qBACpE;oBAAC,OAAO,KAAK,EAAE;wBACZ,aAAG,CAAC,KAAK,CAAC,4CAA4C,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;qBACtE;iBACJ;YACL,CAAC,CAAC;YACF,6BAA6B,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;SACxD;KACJ;AACL,CAAC;AA/BD,sEA+BC;AAED,KAAK,UAAU,0BAA0B,CAAC,IAAU;IAChD,MAAM,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,cAAc,GAAG,oCAAoC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACtE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;AACxC,CAAC"}

package/internals/utils/puppeteer_utils.d.ts

@@ -0,0 +1,314 @@
+/**
+ * A namespace that contains various utilities for
+ * [Puppeteer](https://github.com/puppeteer/puppeteer) - the headless Chrome Node API.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const Apify = require('apify');
+ * const { puppeteer } = Actor.utils;
+ *
+ * // Open https://www.example.com in Puppeteer
+ * const browser = await Actor.launchPuppeteer();
+ * const page = await browser.newPage();
+ * await page.goto('https://www.example.com');
+ *
+ * // Inject jQuery into a page
+ * await puppeteer.injectJQuery(page);
+ * ```
+ * @module puppeteerUtils
+ */
+import { Page, HTTPResponse, ResponseForRequest } from 'puppeteer';
+import { Request, storage } from '@crawlee/core';
+import { Dictionary } from '@crawlee/utils';
+import { enqueueLinksByClickingElements, EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements';
+import { addInterceptRequestHandler, InterceptHandler, removeInterceptRequestHandler } from './puppeteer_request_interception';
+import { PuppeteerCrawlingContext } from '../puppeteer-crawler';
+export interface DirectNavigationOptions {
+    /**
+     * Maximum operation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout. The
+     * default value can be changed by using the browserContext.setDefaultNavigationTimeout(timeout),
+     * browserContext.setDefaultTimeout(timeout), page.setDefaultNavigationTimeout(timeout) or
+     * page.setDefaultTimeout(timeout) methods.
+     */
+    timeout?: number;
+    /**
+     * When to consider operation succeeded, defaults to `load`. Events can be either:
+     * - `'domcontentloaded'` - consider operation to be finished when the `DOMContentLoaded` event is fired.
+     * - `'load'` - consider operation to be finished when the `load` event is fired.
+     * - `'networkidle'` - consider operation to be finished when there are no network connections for at least `500` ms.
+     */
+    waitUntil?: 'domcontentloaded' | 'load' | 'networkidle';
+    /**
+     * Referer header value. If provided it will take preference over the referer header value set by page.setExtraHTTPHeaders(headers).
+     */
+    referer?: string;
+}
+export interface InjectFileOptions {
+    /**
+     * Enables the injected script to survive page navigations and reloads without need to be re-injected manually.
+     * This does not mean, however, that internal state will be preserved. Just that it will be automatically
+     * re-injected on each navigation before any other scripts get the chance to execute.
+     */
+    surviveNavigations?: boolean;
+}
+export interface BlockRequestsOptions {
+    /**
+     * The patterns of URLs to block from being loaded by the browser.
+     * Only `*` can be used as a wildcard. It is also automatically added to the beginning
+     * and end of the pattern. This limitation is enforced by the DevTools protocol.
+     * `.png` is the same as `*.png*`.
+     */
+    urlPatterns?: string[];
+    /**
+     * If you just want to append to the default blocked patterns, use this property.
+     */
+    extraUrlPatterns?: string[];
+}
+export interface CompiledScriptParams {
+    page: Page;
+    request: Request;
+}
+export declare type CompiledScriptFunction = (params: CompiledScriptParams) => Promise<unknown>;
+/**
+ * Injects a JavaScript file into a Puppeteer page.
+ * Unlike Puppeteer's `addScriptTag` function, this function works on pages
+ * with arbitrary Cross-Origin Resource Sharing (CORS) policies.
+ *
+ * File contents are cached for up to 10 files to limit file system access.
+ *
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param filePath File path
+ * @param [options]
+ */
+export declare function injectFile(page: Page, filePath: string, options?: InjectFileOptions): Promise<unknown>;
+/**
+ * Injects the [jQuery](https://jquery.com/) library into a Puppeteer page.
+ * jQuery is often useful for various web scraping and crawling tasks.
+ * For example, it can help extract text from HTML elements using CSS selectors.
+ *
+ * Beware that the injected jQuery object will be set to the `window.$` variable and thus it might cause conflicts with
+ * other libraries included by the page that use the same variable name (e.g. another version of jQuery).
+ * This can affect functionality of page's scripts.
+ *
+ * The injected jQuery will survive page navigations and reloads.
+ *
+ * **Example usage:**
+ * ```javascript
+ * await Actor.utils.puppeteer.injectJQuery(page);
+ * const title = await page.evaluate(() => {
+ *   return $('head title').text();
+ * });
+ * ```
+ *
+ * Note that `injectJQuery()` does not affect the Puppeteer's
+ * [`page.$()`](https://pptr.dev/#?product=Puppeteer&show=api-pageselector)
+ * function in any way.
+ *
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ */
+export declare function injectJQuery(page: Page): Promise<unknown>;
+/**
+ * Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.
+ * This is useful to speed up crawling of websites, since it reduces the amount
+ * of data that needs to be downloaded from the web, but it may break some websites
+ * or unexpectedly prevent loading of resources.
+ *
+ * By default, the function will block all URLs including the following patterns:
+ *
+ * ```json
+ * [".css", ".jpg", ".jpeg", ".png", ".svg", ".gif", ".woff", ".pdf", ".zip"]
+ * ```
+ *
+ * If you want to extend this list further, use the `extraUrlPatterns` option,
+ * which will keep blocking the default patterns, as well as add your custom ones.
+ * If you would like to block only specific patterns, use the `urlPatterns` option,
+ * which will override the defaults and block only URLs with your custom patterns.
+ *
+ * This function does not use Puppeteer's request interception and therefore does not interfere
+ * with browser cache. It's also faster than blocking requests using interception,
+ * because the blocking happens directly in the browser without the round-trip to Node.js,
+ * but it does not provide the extra benefits of request interception.
+ *
+ * The function will never block main document loads and their respective redirects.
+ *
+ * **Example usage**
+ * ```javascript
+ * const Apify = require('apify');
+ *
+ * const browser = await Actor.launchPuppeteer();
+ * const page = await browser.newPage();
+ *
+ * // Block all requests to URLs that include `adsbygoogle.js` and also all defaults.
+ * await Actor.utils.puppeteer.blockRequests(page, {
+ *     extraUrlPatterns: ['adsbygoogle.js'],
+ * });
+ *
+ * await page.goto('https://cnn.com');
+ * ```
+ *
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param [options]
+ */
+export declare function blockRequests(page: Page, options?: BlockRequestsOptions): Promise<void>;
+/**
+ * `blockResources()` has a high impact on performance in recent versions of Puppeteer.
+ * Until this resolves, please use `Actor.utils.puppeteer.blockRequests()`.
+ * @deprecated
+ */
+export declare const blockResources: (page: Page, resourceTypes?: string[]) => Promise<void>;
+/**
+ * *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal
+ * performance. Until this resolves, we suggest just relying on the in-browser cache unless absolutely necessary.
+ *
+ * Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.
+ * *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.
+ *   This issue should be resolved or atleast mitigated in future iterations of this feature.
+ * @param page
+ *   Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param cache
+ *   Object in which responses are stored
+ * @param responseUrlRules
+ *   List of rules that are used to check if the response should be cached.
+ *   String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).
+ * @deprecated
+ */
+export declare function cacheResponses(page: Page, cache: Dictionary<Partial<ResponseForRequest>>, responseUrlRules: (string | RegExp)[]): Promise<void>;
+/**
+ * Compiles a Puppeteer script into an async function that may be executed at any time
+ * by providing it with the following object:
+ * ```
+ * {
+ *    page: Page,
+ *    request: Request,
+ * }
+ * ```
+ * Where `page` is a Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page)
+ * and `request` is a {@link Request}.
+ *
+ * The function is compiled by using the `scriptString` parameter as the function's body,
+ * so any limitations to function bodies apply. Return value of the compiled function
+ * is the return value of the function body = the `scriptString` parameter.
+ *
+ * As a security measure, no globals such as `process` or `require` are accessible
+ * from within the function body. Note that the function does not provide a safe
+ * sandbox and even though globals are not easily accessible, malicious code may
+ * still execute in the main process via prototype manipulation. Therefore you
+ * should only use this function to execute sanitized or safe code.
+ *
+ * Custom context may also be provided using the `context` parameter. To improve security,
+ * make sure to only pass the really necessary objects to the context. Preferably making
+ * secured copies beforehand.
+ *
+ */
+export declare function compileScript(scriptString: string, context?: Dictionary): CompiledScriptFunction;
+/**
+ * Extended version of Puppeteer's `page.goto()` allowing to perform requests with HTTP method other than GET,
+ * with custom headers and POST payload. URL, method, headers and payload are taken from
+ * request parameter that must be an instance of Request class.
+ *
+ * *NOTE:* In recent versions of Puppeteer using requests other than GET, overriding headers and adding payloads disables
+ * browser cache which degrades performance.
+ *
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param request
+ * @param [gotoOptions] Custom options for `page.goto()`.
+ */
+export declare function gotoExtended(page: Page, request: Request, gotoOptions?: DirectNavigationOptions): Promise<HTTPResponse | null>;
+export interface InfiniteScrollOptions {
+    /**
+     * How many seconds to scroll for. If 0, will scroll until bottom of page.
+     * @default 1
+     */
+    timeoutSecs?: number;
+    /**
+     * How many seconds to wait for no new content to load before exit.
+     * @default 4
+     */
+    waitForSecs?: number;
+    /**
+     * If true, it will scroll up a bit after each scroll down. This is required on some websites for the scroll to work.
+     * @default false
+     */
+    scrollDownAndUp?: boolean;
+    /**
+     * Optionally checks and clicks a button if it appears while scrolling. This is required on some websites for the scroll to work.
+     */
+    buttonSelector?: string;
+    /**
+     * This function is called after every scroll and stops the scrolling process if it returns `true`. The function can be `async`.
+     */
+    stopScrollCallback?: () => unknown | Promise<unknown>;
+}
+/**
+ * Scrolls to the bottom of a page, or until it times out.
+ * Loads dynamic content when it hits the bottom of a page, and then continues scrolling.
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param [options]
+ */
+export declare function infiniteScroll(page: Page, options?: InfiniteScrollOptions): Promise<void>;
+export interface SaveSnapshotOptions {
+    /**
+     * Key under which the screenshot and HTML will be saved. `.jpg` will be appended for screenshot and `.html` for HTML.
+     * @default 'SNAPSHOT'
+     */
+    key?: string;
+    /**
+     * The quality of the image, between 0-100. Higher quality images have bigger size and require more storage.
+     * @default 50
+     */
+    screenshotQuality?: number;
+    /**
+     * If true, it will save a full screenshot of the current page as a record with `key` appended by `.jpg`.
+     * @default true
+     */
+    saveScreenshot?: boolean;
+    /**
+     * If true, it will save a full HTML of the current page as a record with `key` appended by `.html`.
+     * @default true
+     */
+    saveHtml?: boolean;
+    /**
+     * Name or id of the Key-Value store where snapshot is saved. By default it is saved to default Key-Value store.
+     * @default null
+     */
+    keyValueStoreName?: string | null;
+}
+/**
+ * Saves a full screenshot and HTML of the current page into a Key-Value store.
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param [options]
+ */
+export declare function saveSnapshot(page: Page, options?: SaveSnapshotOptions): Promise<void>;
+export interface PuppeteerContextUtils {
+    injectFile(filePath: string, options?: InjectFileOptions): Promise<unknown>;
+    injectJQuery(): Promise<unknown>;
+    enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>): Promise<storage.BatchAddRequestsResult>;
+    blockRequests(options?: BlockRequestsOptions): Promise<void>;
+    blockResources(resourceTypes?: string[]): Promise<void>;
+    cacheResponses(cache: Dictionary<Partial<ResponseForRequest>>, responseUrlRules: (string | RegExp)[]): Promise<void>;
+    compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
+    addInterceptRequestHandler(handler: InterceptHandler): Promise<void>;
+    removeInterceptRequestHandler(handler: InterceptHandler): Promise<void>;
+    infiniteScroll(options?: InfiniteScrollOptions): Promise<void>;
+    saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
+}
+/** @internal */
+export declare function registerUtilsToContext(context: PuppeteerCrawlingContext): void;
+export { enqueueLinksByClickingElements, addInterceptRequestHandler, removeInterceptRequestHandler, };
+/** @internal */
+export declare const puppeteerUtils: {
+    injectFile: typeof injectFile;
+    injectJQuery: typeof injectJQuery;
+    enqueueLinksByClickingElements: typeof enqueueLinksByClickingElements;
+    blockRequests: typeof blockRequests;
+    blockResources: (page: Page, resourceTypes?: string[]) => Promise<void>;
+    cacheResponses: typeof cacheResponses;
+    compileScript: typeof compileScript;
+    gotoExtended: typeof gotoExtended;
+    addInterceptRequestHandler: typeof addInterceptRequestHandler;
+    removeInterceptRequestHandler: typeof removeInterceptRequestHandler;
+    infiniteScroll: typeof infiniteScroll;
+    saveSnapshot: typeof saveSnapshot;
+};
+//# sourceMappingURL=puppeteer_utils.d.ts.map

package/internals/utils/puppeteer_utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"puppeteer_utils.d.ts","sourceRoot":"","sources":["../../../src/internals/utils/puppeteer_utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAOH,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,kBAAkB,EAAmC,MAAM,WAAW,CAAC;AAEpG,OAAO,EAAiB,OAAO,EAAE,OAAO,EAAc,MAAM,eAAe,CAAC;AAC5E,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,8BAA8B,EAAE,qCAAqC,EAAE,MAAM,iCAAiC,CAAC;AACxH,OAAO,EAAE,0BAA0B,EAAE,gBAAgB,EAAE,6BAA6B,EAAE,MAAM,kCAAkC,CAAC;AAC/H,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAUhE,MAAM,WAAW,uBAAuB;IACpC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,kBAAkB,GAAG,MAAM,GAAG,aAAa,CAAC;IAExD;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAC9B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,MAAM,WAAW,oBAAoB;IACjC;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,oBAAoB;IACjC,IAAI,EAAE,IAAI,CAAC;IACX,OAAO,EAAE,OAAO,CAAC;CACpB;AAED,oBAAY,sBAAsB,GAAG,CAAC,MAAM,EAAE,oBAAoB,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAOxF;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,iBAAsB,GAAG,OAAO,CAAC,OAAO,CAAC,CAgBhH;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,CAGzD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,oBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC,CAgBjG;AAED;;;;GAIG;AACH,eAAO,MAAM,cAAc,SAAgB,IAAI,4CAQ9C,CAAC;AAEF;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA4CrJ;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE,UAAgC,GAAG,sBAAsB,CAcrH;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,GAAE,uBAA4B,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAuCxI;AAED,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAE1B;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,kBAAkB,CAAC,EAAE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACzD;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAqGnG;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,GAAE,mBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC,CAmC/F;AAED,MAAM,WAAW,qBAAqB;IAClC,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5E,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IACjC,8BAA8B,CAAC,OAAO,EAAE,IAAI,CAAC,qCAAqC,EAAE,MAAM,GAAG,cAAc,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;IACvJ,aAAa,CAAC,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7D,cAAc,CAAC,aAAa,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD,cAAc,CAAC,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,EAAE,gBAAgB,EAAE,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACrH,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,UAAU,GAAG,sBAAsB,CAAC;IAC9E,0BAA0B,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACrE,6BAA6B,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxE,cAAc,CAAC,OAAO,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/D,YAAY,CAAC,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9D;AAED,gBAAgB;AAChB,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,wBAAwB,GAAG,IAAI,CAkB9E;AAED,OAAO,EACH,8BAA8B,EAC9B,0BAA0B,EAC1B,6BAA6B,GAChC,CAAC;AAEF,gBAAgB;AAChB,eAAO,MAAM,cAAc;;;;;2BAvagB,IAAI;;;;;;;;CAob9C,CAAC"}