@crawlee/puppeteer 3.0.0-alpha.2 → 3.0.0-alpha.3

package/dist/internals/stealth/index.d.ts

@@ -0,0 +1,40 @@
+import { Page, Browser } from 'puppeteer';
+/**
+ * Configuration of stealth tricks for a proper hiding effect all of them should be set to true.
+ * These tricks are applied only when the `stealth` option is set to `true`.
+ */
+export interface StealthOptions {
+    /** If plugins should be added to the navigator. */
+    addPlugins?: boolean;
+    /** Emulates window Iframe. */
+    emulateWindowFrame?: boolean;
+    /** Emulates graphic card. */
+    emulateWebGL?: boolean;
+    /** Emulates console.debug to return null. */
+    emulateConsoleDebug?: boolean;
+    /** Adds languages to the navigator. */
+    addLanguage?: boolean;
+    /** Hides the webdriver by changing the navigator proto. */
+    hideWebDriver?: boolean;
+    /** Fakes interaction with permissions. */
+    hackPermissions?: boolean;
+    /** Adds the chrome runtime properties. */
+    mockChrome?: boolean;
+    /** Adds the chrome runtime properties inside the every newly created iframe. */
+    mockChromeInIframe?: boolean;
+    /** Sets device memory to other value than 0. */
+    mockDeviceMemory?: boolean;
+}
+/**
+ *  The main purpose of this function is to override newPage function and attach selected tricks.
+ * @param browser puppeteer browser instance
+ * @param options
+ * @returns Instance of Browser from puppeteer package
+ */
+export declare function applyStealthToBrowser(browser: Browser, options: StealthOptions): Browser;
+/**
+ * Applies stealth tricks to the puppeteer page
+ * @internal
+ */
+export declare function applyStealthTricks(page: Page, evaluationDebugMessage: string, options: StealthOptions): Promise<void>;
+//# sourceMappingURL=index.d.ts.map

package/dist/internals/stealth/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/internals/stealth/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQ1C;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC3B,mDAAmD;IACnD,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,8BAA8B;IAC9B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,6BAA6B;IAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,6CAA6C;IAC7C,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,uCAAuC;IACvC,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,2DAA2D;IAC3D,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,0CAA0C;IAC1C,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,0CAA0C;IAC1C,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,gFAAgF;IAChF,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAkBD;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAwBxF;AAoCD;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,IAAI,EAAE,sBAAsB,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAwBrH"}

package/dist/internals/stealth/index.js

@@ -0,0 +1,108 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.applyStealthTricks = exports.applyStealthToBrowser = void 0;
+const tslib_1 = require("tslib");
+const utilities_1 = require("@apify/utilities");
+const log_1 = tslib_1.__importDefault(require("@apify/log"));
+const utils_1 = require("@crawlee/utils");
+const hidingTricks = tslib_1.__importStar(require("./hiding_tricks"));
+const log = log_1.default.child({ prefix: 'Stealth' });
+const DEFAULT_STEALTH_OPTIONS = {
+    addPlugins: true,
+    emulateWindowFrame: true,
+    emulateWebGL: true,
+    emulateConsoleDebug: true,
+    addLanguage: true,
+    hackPermissions: true,
+    mockChrome: true,
+    mockChromeInIframe: true,
+    mockDeviceMemory: true,
+};
+const STEALTH_ERROR_MESSAGE_PREFIX = 'StealthError';
+const MAX_IFRAMES = 10;
+const alreadyWrapped = Symbol('alreadyWrapped');
+/**
+ *  The main purpose of this function is to override newPage function and attach selected tricks.
+ * @param browser puppeteer browser instance
+ * @param options
+ * @returns Instance of Browser from puppeteer package
+ */
+function applyStealthToBrowser(browser, options) {
+    const modifiedBrowser = browser;
+    const opts = { ...DEFAULT_STEALTH_OPTIONS, ...options };
+    const defaultContext = browser.defaultBrowserContext();
+    const contextPrototype = Object.getPrototypeOf(defaultContext);
+    const prevNewPage = contextPrototype.newPage;
+    if (!contextPrototype.newPage[alreadyWrapped]) {
+        contextPrototype.newPage = async function (...args) {
+            const page = await prevNewPage.bind(this)(...args);
+            const evaluationDebugMessage = generateEvaluationDebugMessage();
+            addStealthDebugToPage(page, evaluationDebugMessage);
+            await applyStealthTricks(page, evaluationDebugMessage, opts);
+            return page;
+        };
+        contextPrototype.newPage[alreadyWrapped] = true;
+    }
+    return modifiedBrowser;
+}
+exports.applyStealthToBrowser = applyStealthToBrowser;
+function generateEvaluationDebugMessage() {
+    const minLength = 6;
+    const maxLength = 10;
+    const randomLength = Math.random() * (maxLength - minLength) + minLength;
+    return (0, utilities_1.cryptoRandomObjectId)(randomLength);
+}
+/**
+ * Logs the stealth errors in browser to the node stdout.
+ * @param page puppeteer page instance
+ * @param evaluationDebugMessage debug message
+ */
+function addStealthDebugToPage(page, evaluationDebugMessage) {
+    let warningLogged = false;
+    let counter = 1;
+    page.on('console', (msg) => {
+        const text = msg.text();
+        if (text.includes(STEALTH_ERROR_MESSAGE_PREFIX)) {
+            log.error(text);
+        }
+        else if (text.includes(evaluationDebugMessage)) {
+            if (counter > MAX_IFRAMES && !warningLogged) {
+                log.warning(`Evaluating hiding tricks in too many iframes (limit: ${MAX_IFRAMES}).`
+                    + 'You might experience some performance issues. Try setting \'stealth\' false');
+                warningLogged = true;
+            }
+            counter++;
+            log.debug('Tricks evaluated', { counter });
+        }
+    });
+}
+/**
+ * Applies stealth tricks to the puppeteer page
+ * @internal
+ */
+function applyStealthTricks(page, evaluationDebugMessage, options) {
+    const functions = (0, utils_1.keys)(options)
+        .filter((key) => {
+        return options[key];
+    })
+        .map((key) => hidingTricks[key].toString());
+    /* istanbul ignore next */
+    const addFunctions = (functionsArr, errorMessagePrefix, debugMessage) => {
+        // eslint-disable-next-line no-console
+        console.log(debugMessage);
+        // add functions
+        for (const func of functionsArr) {
+            try {
+                eval(func)(); // eslint-disable-line
+            }
+            catch (e) {
+                const err = e;
+                // eslint-disable-next-line no-console
+                console.error(`${errorMessagePrefix}: Failed to apply stealth trick reason: ${err.message}`);
+            }
+        }
+    };
+    return page.evaluateOnNewDocument(addFunctions, functions, STEALTH_ERROR_MESSAGE_PREFIX, evaluationDebugMessage);
+}
+exports.applyStealthTricks = applyStealthTricks;
+//# sourceMappingURL=index.js.map

package/dist/internals/stealth/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/internals/stealth/index.ts"],"names":[],"mappings":";;;;AACA,gDAAwD;AACxD,6DAA8B;AAC9B,0CAAsC;AACtC,sEAAgD;AAEhD,MAAM,GAAG,GAAG,aAAI,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;AA6B9C,MAAM,uBAAuB,GAAG;IAC5B,UAAU,EAAE,IAAI;IAChB,kBAAkB,EAAE,IAAI;IACxB,YAAY,EAAE,IAAI;IAClB,mBAAmB,EAAE,IAAI;IACzB,WAAW,EAAE,IAAI;IACjB,eAAe,EAAE,IAAI;IACrB,UAAU,EAAE,IAAI;IAChB,kBAAkB,EAAE,IAAI;IACxB,gBAAgB,EAAE,IAAI;CACzB,CAAC;AAEF,MAAM,4BAA4B,GAAG,cAAc,CAAC;AACpD,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,MAAM,cAAc,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;AAEhD;;;;;GAKG;AACH,SAAgB,qBAAqB,CAAC,OAAgB,EAAE,OAAuB;IAC3E,MAAM,eAAe,GAAG,OAAO,CAAC;IAChC,MAAM,IAAI,GAAG,EAAE,GAAG,uBAAuB,EAAE,GAAG,OAAO,EAAE,CAAC;IACxD,MAAM,cAAc,GAAG,OAAO,CAAC,qBAAqB,EAAE,CAAC;IACvD,MAAM,gBAAgB,GAAG,MAAM,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;IAE/D,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC;IAE7C,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE;QAC3C,gBAAgB,CAAC,OAAO,GAAG,KAAK,WAAW,GAAG,IAAe;YACzD,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;YAEnD,MAAM,sBAAsB,GAAG,8BAA8B,EAAE,CAAC;YAEhE,qBAAqB,CAAC,IAAI,EAAE,sBAAsB,CAAC,CAAC;YACpD,MAAM,kBAAkB,CAAC,IAAI,EAAE,sBAAsB,EAAE,IAAI,CAAC,CAAC;YAE7D,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC;QAEF,gBAAgB,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC;KACnD;IAED,OAAO,eAAe,CAAC;AAC3B,CAAC;AAxBD,sDAwBC;AAED,SAAS,8BAA8B;IACnC,MAAM,SAAS,GAAG,CAAC,CAAC;IACpB,MAAM,SAAS,GAAG,EAAE,CAAC;IACrB,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;IAEzE,OAAO,IAAA,gCAAoB,EAAC,YAAY,CAAC,CAAC;AAC9C,CAAC;AACD;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,IAAU,EAAE,sBAA8B;IACrE,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;QACvB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,QAAQ,CAAC,4BAA4B,CAAC,EAAE;YAC7C,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;SACnB;aAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE;YAC9C,IAAI,OAAO,GAAG,WAAW,IAAI,CAAC,aAAa,EAAE;gBACzC,GAAG,CAAC,OAAO,CACP,wDAAwD,WAAW,IAAI;sBACrE,6EAA6E,CAClF,CAAC;gBAEF,aAAa,GAAG,IAAI,CAAC;aACxB;YACD,OAAO,EAAE,CAAC;YACV,GAAG,CAAC,KAAK,CAAC,kBAAkB,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;SAC9C;IACL,CAAC,CAAC,CAAC;AACP,CAAC;AAED;;;GAGG;AACH,SAAgB,kBAAkB,CAAC,IAAU,EAAE,sBAA8B,EAAE,OAAuB;IAClG,MAAM,SAAS,GAAG,IAAA,YAAI,EAAC,OAAO,CAAC;SAC1B,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;QACZ,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,GAAgC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IAE7E,0BAA0B;IAC1B,MAAM,YAAY,GAAG,CAAC,YAAsB,EAAE,kBAA0B,EAAE,YAAoB,EAAE,EAAE;QAC9F,sCAAsC;QACtC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAC1B,gBAAgB;QAChB,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE;YAC7B,IAAI;gBACA,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,sBAAsB;aACvC;YAAC,OAAO,CAAC,EAAE;gBACR,MAAM,GAAG,GAAG,CAAU,CAAC;gBACvB,sCAAsC;gBACtC,OAAO,CAAC,KAAK,CAAC,GAAG,kBAAkB,2CAA2C,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;aAChG;SACJ;IACL,CAAC,CAAC;IAEF,OAAO,IAAI,CAAC,qBAAqB,CAAC,YAAY,EAAE,SAAS,EAAE,4BAA4B,EAAE,sBAAsB,CAAC,CAAC;AACrH,CAAC;AAxBD,gDAwBC"}

package/dist/internals/utils/puppeteer_request_interception.d.ts

@@ -0,0 +1,60 @@
+import { HTTPRequest as PuppeteerRequest, Page } from 'puppeteer';
+export declare type InterceptHandler = (request: PuppeteerRequest) => unknown;
+/**
+ * Adds request interception handler in similar to `page.on('request', handler);` but in addition to that
+ * supports multiple parallel handlers.
+ *
+ * All the handlers are executed sequentially in the order as they were added.
+ * Each of the handlers must call one of `request.continue()`, `request.abort()` and `request.respond()`.
+ * In addition to that any of the handlers may modify the request object (method, postData, headers)
+ * by passing its overrides to `request.continue()`.
+ * If multiple handlers modify same property then the last one wins. Headers are merged separately so you can
+ * override only a value of specific header.
+ *
+ * If one the handlers calls `request.abort()` or `request.respond()` then request is not propagated further
+ * to any of the remaining handlers.
+ *
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * // Replace images with placeholder.
+ * await addInterceptRequestHandler(page, (request) => {
+ *     if (request.resourceType() === 'image') {
+ *         return request.respond({
+ *             statusCode: 200,
+ *             contentType: 'image/jpeg',
+ *             body: placeholderImageBuffer,
+ *         });
+ *     }
+ *     return request.continue();
+ * });
+ *
+ * // Abort all the scripts.
+ * await addInterceptRequestHandler(page, (request) => {
+ *     if (request.resourceType() === 'script') return request.abort();
+ *     return request.continue();
+ * });
+ *
+ * // Change requests to post.
+ * await addInterceptRequestHandler(page, (request) => {
+ *     return request.continue({
+ *          method: 'POST',
+ *     });
+ * });
+ *
+ * await page.goto('http://example.com');
+ * ```
+ *
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param handler Request interception handler.
+ */
+export declare function addInterceptRequestHandler(page: Page, handler: InterceptHandler): Promise<void>;
+/**
+ * Removes request interception handler for given page.
+ *
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param handler Request interception handler.
+ */
+export declare function removeInterceptRequestHandler(page: Page, handler: InterceptHandler): Promise<void>;
+//# sourceMappingURL=puppeteer_request_interception.d.ts.map

package/dist/internals/utils/puppeteer_request_interception.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"puppeteer_request_interception.d.ts","sourceRoot":"","sources":["../../../src/internals/utils/puppeteer_request_interception.ts"],"names":[],"mappings":"AAEA,OAAO,EAAe,WAAW,IAAI,gBAAgB,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAiC/E,oBAAY,gBAAgB,GAAG,CAAC,OAAO,EAAE,gBAAgB,KAAK,OAAO,CAAC;AAwEtE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AACH,wBAAsB,0BAA0B,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAmCrG;AAED;;;;;GAKG;AACH,wBAAsB,6BAA6B,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CA+BxG"}

package/dist/internals/utils/puppeteer_request_interception.js

@@ -0,0 +1,225 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removeInterceptRequestHandler = exports.addInterceptRequestHandler = void 0;
+const tslib_1 = require("tslib");
+const events_1 = require("events");
+const ow_1 = tslib_1.__importDefault(require("ow"));
+const log_1 = tslib_1.__importDefault(require("@apify/log"));
+// We use weak maps here so that the content gets discarded after page gets closed.
+const pageInterceptRequestHandlersMap = new WeakMap(); // Maps page to an array of request interception handlers.
+const pageInterceptRequestMasterHandlerMap = new WeakMap(); // Maps page to master request interception handler.
+const pageInterceptedRequestsMap = new WeakMap(); // Maps page to a set of its pending intercepted requests.
+/**
+ * Enables observation of changes of internal state to be able to queue other actions based on it.
+ * @ignore
+ */
+class ObservableSet extends events_1.EventEmitter {
+    constructor() {
+        super(...arguments);
+        Object.defineProperty(this, "set", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Set()
+        });
+    }
+    add(value) {
+        this.set.add(value);
+        this.emit('add', value);
+        return this.set;
+    }
+    delete(value) {
+        const success = this.set.delete(value);
+        this.emit('delete', value);
+        return success;
+    }
+    get size() {
+        return this.set.size;
+    }
+}
+/**
+ * Makes all request headers capitalized to more look like in browser
+ */
+function browserifyHeaders(headers) {
+    const finalHeaders = {};
+    // eslint-disable-next-line prefer-const
+    for (let [key, value] of Object.entries(headers)) {
+        key = key.toLowerCase()
+            .split('-')
+            .map((str) => str.charAt(0).toUpperCase() + str.slice(1))
+            .join('-');
+        finalHeaders[key] = value;
+    }
+    return finalHeaders;
+}
+/**
+ * Executes an array for given intercept request handlers for a given request object.
+ *
+ * @param request Puppeteer's Request object.
+ * @param interceptRequestHandlers An array of intercept request handlers.
+ * @ignore
+ */
+async function handleRequest(request, interceptRequestHandlers) {
+    // If there are no intercept handlers, it means that request interception is not enabled (anymore)
+    // and therefore .abort() .respond() and .continue() would throw and crash the process.
+    if (!interceptRequestHandlers?.length)
+        return;
+    let wasAborted = false;
+    let wasResponded = false;
+    let wasContinued = false;
+    const accumulatedOverrides = {
+        headers: browserifyHeaders(request.headers()),
+    };
+    const originalContinue = request.continue.bind(request);
+    request.continue = async (overrides = {}) => {
+        wasContinued = true;
+        const headers = browserifyHeaders({ ...accumulatedOverrides.headers, ...overrides.headers });
+        Object.assign(accumulatedOverrides, overrides, { headers });
+    };
+    const { abort, respond } = request;
+    request.abort = async (...args) => {
+        wasAborted = true;
+        return abort.call(request, ...args);
+    };
+    request.respond = async (...args) => {
+        wasResponded = true;
+        return respond.call(request, ...args);
+    };
+    for (const handler of interceptRequestHandlers) {
+        wasContinued = false;
+        await handler(request);
+        // Check that one of the functions was called.
+        if (!wasAborted && !wasResponded && !wasContinued) {
+            throw new Error('Intercept request handler must call one of request.continue|respond|abort() methods!');
+        }
+        // If request was aborted or responded then we can finish immediately.
+        if (wasAborted || wasResponded)
+            return;
+    }
+    return originalContinue(accumulatedOverrides);
+}
+/**
+ * Adds request interception handler in similar to `page.on('request', handler);` but in addition to that
+ * supports multiple parallel handlers.
+ *
+ * All the handlers are executed sequentially in the order as they were added.
+ * Each of the handlers must call one of `request.continue()`, `request.abort()` and `request.respond()`.
+ * In addition to that any of the handlers may modify the request object (method, postData, headers)
+ * by passing its overrides to `request.continue()`.
+ * If multiple handlers modify same property then the last one wins. Headers are merged separately so you can
+ * override only a value of specific header.
+ *
+ * If one the handlers calls `request.abort()` or `request.respond()` then request is not propagated further
+ * to any of the remaining handlers.
+ *
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * // Replace images with placeholder.
+ * await addInterceptRequestHandler(page, (request) => {
+ *     if (request.resourceType() === 'image') {
+ *         return request.respond({
+ *             statusCode: 200,
+ *             contentType: 'image/jpeg',
+ *             body: placeholderImageBuffer,
+ *         });
+ *     }
+ *     return request.continue();
+ * });
+ *
+ * // Abort all the scripts.
+ * await addInterceptRequestHandler(page, (request) => {
+ *     if (request.resourceType() === 'script') return request.abort();
+ *     return request.continue();
+ * });
+ *
+ * // Change requests to post.
+ * await addInterceptRequestHandler(page, (request) => {
+ *     return request.continue({
+ *          method: 'POST',
+ *     });
+ * });
+ *
+ * await page.goto('http://example.com');
+ * ```
+ *
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param handler Request interception handler.
+ */
+async function addInterceptRequestHandler(page, handler) {
+    (0, ow_1.default)(page, ow_1.default.object.hasKeys('goto', 'evaluate'));
+    (0, ow_1.default)(handler, ow_1.default.function);
+    if (!pageInterceptRequestHandlersMap.has(page)) {
+        pageInterceptRequestHandlersMap.set(page, []);
+    }
+    if (!pageInterceptedRequestsMap.has(page)) {
+        pageInterceptedRequestsMap.set(page, new ObservableSet());
+    }
+    const handlersArray = pageInterceptRequestHandlersMap.get(page);
+    handlersArray.push(handler);
+    // First handler was just added at this point so we need to set up request interception.
+    if (handlersArray.length === 1) {
+        await page.setRequestInterception(true);
+        // This is a handler that gets set in page.on('request', ...) and that executes all the user
+        // added custom handlers.
+        const masterHandler = async (request) => {
+            const interceptedRequests = pageInterceptedRequestsMap.get(page);
+            interceptedRequests.add(request);
+            const interceptHandlers = pageInterceptRequestHandlersMap.get(page);
+            try {
+                await handleRequest(request, interceptHandlers);
+            }
+            finally {
+                interceptedRequests.delete(request);
+            }
+        };
+        pageInterceptRequestMasterHandlerMap.set(page, masterHandler);
+        page.on('request', masterHandler);
+    }
+}
+exports.addInterceptRequestHandler = addInterceptRequestHandler;
+/**
+ * Removes request interception handler for given page.
+ *
+ * @param page Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
+ * @param handler Request interception handler.
+ */
+async function removeInterceptRequestHandler(page, handler) {
+    (0, ow_1.default)(page, ow_1.default.object.hasKeys('goto', 'evaluate'));
+    (0, ow_1.default)(handler, ow_1.default.function);
+    const handlersArray = pageInterceptRequestHandlersMap
+        .get(page)
+        .filter((item) => item !== handler);
+    pageInterceptRequestHandlersMap.set(page, handlersArray);
+    if (handlersArray.length === 0) {
+        const interceptedRequestsInProgress = pageInterceptedRequestsMap.get(page);
+        // Since handlers can be async, we can't simply turn off request interception
+        // when there are no handlers, because some handlers could still
+        // be in progress and request.abort|respond|continue() would throw.
+        if (interceptedRequestsInProgress.size === 0) {
+            await disableRequestInterception(page);
+        }
+        else {
+            const onDelete = async () => {
+                if (interceptedRequestsInProgress.size === 0) {
+                    try {
+                        await disableRequestInterception(page);
+                        interceptedRequestsInProgress.removeListener('delete', onDelete);
+                    }
+                    catch (error) {
+                        log_1.default.debug('Error while disabling request interception', { error });
+                    }
+                }
+            };
+            interceptedRequestsInProgress.on('delete', onDelete);
+        }
+    }
+}
+exports.removeInterceptRequestHandler = removeInterceptRequestHandler;
+async function disableRequestInterception(page) {
+    await page.setRequestInterception(false);
+    const requestHandler = pageInterceptRequestMasterHandlerMap.get(page);
+    page.off('request', requestHandler);
+}
+//# sourceMappingURL=puppeteer_request_interception.js.map

package/dist/internals/utils/puppeteer_request_interception.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"puppeteer_request_interception.js","sourceRoot":"","sources":["../../../src/internals/utils/puppeteer_request_interception.ts"],"names":[],"mappings":";;;;AAAA,mCAAsC;AACtC,oDAAoB;AAGpB,6DAA6B;AAE7B,mFAAmF;AACnF,MAAM,+BAA+B,GAAsC,IAAI,OAAO,EAAE,CAAC,CAAC,0DAA0D;AACpJ,MAAM,oCAAoC,GAAG,IAAI,OAAO,EAAE,CAAC,CAAC,oDAAoD;AAChH,MAAM,0BAA0B,GAAG,IAAI,OAAO,EAAE,CAAC,CAAC,0DAA0D;AAE5G;;;GAGG;AACH,MAAM,aAAiB,SAAQ,qBAAY;IAA3C;;QACI;;;;mBAAM,IAAI,GAAG,EAAK;WAAC;IAiBvB,CAAC;IAfG,GAAG,CAAC,KAAQ;QACR,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACxB,OAAO,IAAI,CAAC,GAAG,CAAC;IACpB,CAAC;IAED,MAAM,CAAC,KAAQ;QACX,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC3B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,IAAI,IAAI;QACJ,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;IACzB,CAAC;CACJ;AAID;;GAEG;AACH,SAAS,iBAAiB,CAAC,OAA+B;IACtD,MAAM,YAAY,GAAuB,EAAE,CAAC;IAC5C,wCAAwC;IACxC,KAAK,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;QAC9C,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE;aAClB,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aACxD,IAAI,CAAC,GAAG,CAAC,CAAC;QAEf,YAAY,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;KAC7B;IAED,OAAO,YAAY,CAAC;AACxB,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,aAAa,CAAC,OAAyB,EAAE,wBAA6C;IACjG,kGAAkG;IAClG,uFAAuF;IACvF,IAAI,CAAC,wBAAwB,EAAE,MAAM;QAAE,OAAO;IAE9C,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,MAAM,oBAAoB,GAAG;QACzB,OAAO,EAAE,iBAAiB,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;KAChD,CAAC;IAEF,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxD,OAAO,CAAC,QAAQ,GAAG,KAAK,EAAE,SAAS,GAAG,EAAE,EAAE,EAAE;QACxC,YAAY,GAAG,IAAI,CAAC;QACpB,MAAM,OAAO,GAAG,iBAAiB,CAAC,EAAE,GAAG,oBAAoB,CAAC,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,oBAAoB,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAChE,CAAC,CAAC;IAEF,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;IACnC,OAAO,CAAC,KAAK,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,EAAE;QAC9B,UAAU,GAAG,IAAI,CAAC;QAClB,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IACxC,CAAC,CAAC;IACF,OAAO,CAAC,OAAO,GAAG,KAAK,EAAE,GAAG,IAAI,EAAE,EAAE;QAChC,YAAY,GAAG,IAAI,CAAC;QACpB,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IAC1C,CAAC,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,wBAAwB,EAAE;QAC5C,YAAY,GAAG,KAAK,CAAC;QAErB,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACvB,8CAA8C;QAC9C,IAAI,CAAC,UAAU,IAAI,CAAC,YAAY,IAAI,CAAC,YAAY,EAAE;YAC/C,MAAM,IAAI,KAAK,CAAC,sFAAsF,CAAC,CAAC;SAC3G;QAED,sEAAsE;QACtE,IAAI,UAAU,IAAI,YAAY;YAAE,OAAO;KAC1C;IAED,OAAO,gBAAgB,CAAC,oBAAoB,CAAC,CAAC;AAClD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AACI,KAAK,UAAU,0BAA0B,CAAC,IAAU,EAAE,OAAyB;IAClF,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;IAChD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;IAEzB,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QAC5C,+BAA+B,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;KACjD;IAED,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QACvC,0BAA0B,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,aAAa,EAAE,CAAC,CAAC;KAC7D;IAED,MAAM,aAAa,GAAG,+BAA+B,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;IACjE,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAE5B,wFAAwF;IACxF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE;QAC5B,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAExC,4FAA4F;QAC5F,yBAAyB;QACzB,MAAM,aAAa,GAAG,KAAK,EAAE,OAAoB,EAAE,EAAE;YACjD,MAAM,mBAAmB,GAAG,0BAA0B,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACjE,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACjC,MAAM,iBAAiB,GAAG,+BAA+B,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACpE,IAAI;gBACA,MAAM,aAAa,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;aACnD;oBAAS;gBACN,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;aACvC;QACL,CAAC,CAAC;QAEF,oCAAoC,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAC9D,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;KACrC;AACL,CAAC;AAnCD,gEAmCC;AAED;;;;;GAKG;AACI,KAAK,UAAU,6BAA6B,CAAC,IAAU,EAAE,OAAyB;IACrF,IAAA,YAAE,EAAC,IAAI,EAAE,YAAE,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;IAChD,IAAA,YAAE,EAAC,OAAO,EAAE,YAAE,CAAC,QAAQ,CAAC,CAAC;IAEzB,MAAM,aAAa,GAAG,+BAA+B;SAChD,GAAG,CAAC,IAAI,CAAE;SACV,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;IAExC,+BAA+B,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IAEzD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE;QAC5B,MAAM,6BAA6B,GAAG,0BAA0B,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC3E,6EAA6E;QAC7E,gEAAgE;QAChE,mEAAmE;QACnE,IAAI,6BAA6B,CAAC,IAAI,KAAK,CAAC,EAAE;YAC1C,MAAM,0BAA0B,CAAC,IAAI,CAAC,CAAC;SAC1C;aAAM;YACH,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;gBACxB,IAAI,6BAA6B,CAAC,IAAI,KAAK,CAAC,EAAE;oBAC1C,IAAI;wBACA,MAAM,0BAA0B,CAAC,IAAI,CAAC,CAAC;wBACvC,6BAA6B,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;qBACpE;oBAAC,OAAO,KAAK,EAAE;wBACZ,aAAG,CAAC,KAAK,CAAC,4CAA4C,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;qBACtE;iBACJ;YACL,CAAC,CAAC;YACF,6BAA6B,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;SACxD;KACJ;AACL,CAAC;AA/BD,sEA+BC;AAED,KAAK,UAAU,0BAA0B,CAAC,IAAU;IAChD,MAAM,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,cAAc,GAAG,oCAAoC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACtE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;AACxC,CAAC"}