@crawlee/puppeteer 4.0.0-beta.8 → 4.0.0-beta.80

package/README.md

@@ -1,19 +1,23 @@
 <h1 align="center">
     <a href="https://crawlee.dev">
         <picture>
-          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
-          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
         </picture>
     </a>
-    <br>
+    <br />
     <small>A web scraping and browser automation library</small>
 </h1>
 
-<p align=center>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
-    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
-    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
-    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+<p align="center">
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
+</p>
+
+<p align="center">
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
 </p>
 
 Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
@@ -24,7 +28,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 > 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
 
-> Crawlee for Python is open for early adopters. 🐍  [👉 Checkout the source code 👈](https://github.com/apify/crawlee-python).
+> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
 
 ## Installation
 
@@ -85,7 +89,7 @@ By default, Crawlee stores data to `./storage` in the current working directory.
 We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
 
 ```bash
-npm install crawlee@3.12.3-beta.13
+npm install crawlee@next
 ```
 
 If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
@@ -94,9 +98,9 @@ If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need
 {
     "overrides": {
        "apify": {
-           "@crawlee/core": "3.12.3-beta.13",
-           "@crawlee/types": "3.12.3-beta.13",
-           "@crawlee/utils": "3.12.3-beta.13"
+           "@crawlee/core": "$crawlee",
+           "@crawlee/types": "$crawlee",
+           "@crawlee/utils": "$crawlee"
        }
     }
 }

package/index.d.ts

@@ -7,4 +7,3 @@ export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
 export type { BlockRequestsOptions, CompiledScriptFunction, CompiledScriptParams, DirectNavigationOptions as PuppeteerDirectNavigationOptions, InfiniteScrollOptions, InjectFileOptions, SaveSnapshotOptions, } from './internals/utils/puppeteer_utils.js';
 export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
 export type { EnqueueLinksByClickingElementsOptions } from './internals/enqueue-links/click-elements.js';
-//# sourceMappingURL=index.d.ts.map

package/index.js

@@ -4,4 +4,3 @@ export * from './internals/puppeteer-launcher.js';
 export * as puppeteerRequestInterception from './internals/utils/puppeteer_request_interception.js';
 export * as puppeteerUtils from './internals/utils/puppeteer_utils.js';
 export * as puppeteerClickElements from './internals/enqueue-links/click-elements.js';
-//# sourceMappingURL=index.js.map

package/internals/enqueue-links/click-elements.d.ts

@@ -1,4 +1,4 @@
-import type { GlobInput, PseudoUrlInput, RegExpInput, RequestProvider, RequestTransform } from '@crawlee/browser';
+import type { GlobInput, IRequestManager, PseudoUrlInput, RegExpInput, RequestTransform, SkippedRequestCallback } from '@crawlee/browser';
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { ClickOptions, Page, Target } from 'puppeteer';
@@ -8,9 +8,9 @@ export interface EnqueueLinksByClickingElementsOptions {
      */
     page: Page;
     /**
-     * A request queue to which the URLs will be enqueued.
+     * * A request manager to which the URLs will be enqueued.
      */
-    requestQueue: RequestProvider;
+    requestManager: IRequestManager;
     /**
      * A CSS selector matching elements to be clicked on. Unlike in {@link enqueueLinks}, there is no default
      * value. This is to prevent suboptimal use of this function by using it too broadly.
@@ -39,6 +39,16 @@ export interface EnqueueLinksByClickingElementsOptions {
      * after clicking on elements matching the provided CSS selector.
      */
     globs?: GlobInput[];
+    /**
+     * An array of glob pattern strings, regexp patterns or plain objects
+     * containing patterns matching URLs that will **never** be enqueued.
+     *
+     * The plain objects must include either the `glob` property or the `regexp` property.
+     *
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, provide a regexp.
+     */
+    exclude?: readonly (GlobInput | RegExpInput)[];
     /**
      * An array of regular expressions or plain objects
      * containing regular expressions matching the URLs to be enqueued.
@@ -72,25 +82,28 @@ export interface EnqueueLinksByClickingElementsOptions {
      */
     pseudoUrls?: PseudoUrlInput[];
     /**
-     * Just before a new {@link Request} is constructed and enqueued to the {@link RequestQueue}, this function can be used
-     * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
-     * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
-     * or to dynamically update or create `userData`.
-     *
-     * For example: by adding `useExtendedUniqueKey: true` to the `request` object, `uniqueKey` will be computed from
-     * a combination of `url`, `method` and `payload` which enables crawling of websites that navigate using form submits
-     * (POST requests).
+     * After {@link Request} objects are constructed and filtered by URL patterns (`globs`, `regexps`, `pseudoUrls`),
+     * this function can be used to remove them or modify their contents such as `userData`, `payload` or, most importantly
+     * `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share the same URL,
+     * but differ in methods or payloads, or to dynamically update or create `userData`.
      *
      * **Example:**
      * ```javascript
      * {
      *     transformRequestFunction: (request) => {
      *         request.userData.foo = 'bar';
-     *         request.useExtendedUniqueKey = true;
      *         return request;
      *     }
      * }
      * ```
+     *
+     * Note that `transformRequestFunction` has the highest priority and can overwrite request options
+     * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
+     *
+     * The function receives a {@link RequestOptions} object and can return either:
+     * - The modified {@link RequestOptions} object
+     * - `'unchanged'` to keep the original options as-is
+     * - A falsy value or `'skip'` to exclude the request from the queue
      */
     transformRequestFunction?: RequestTransform;
     /**
@@ -130,6 +143,12 @@ export interface EnqueueLinksByClickingElementsOptions {
      * @default false
      */
     skipNavigation?: boolean;
+    /**
+     * When a request is skipped for some reason, you can use this callback to act on it.
+     * This is fired for requests skipped because they don't match enqueueLinks filters
+     * or because they were removed by `transformRequestFunction`.
+     */
+    onSkippedRequest?: SkippedRequestCallback;
 }
 /**
  * The function finds elements matching a specific CSS selector in a Puppeteer page,
@@ -161,7 +180,7 @@ export interface EnqueueLinksByClickingElementsOptions {
  * ```javascript
  * await utils.puppeteer.enqueueLinksByClickingElements({
  *   page,
- *   requestQueue,
+ *   requestManager,
  *   selector: 'a.product-detail',
  *   pseudoUrls: [
  *       'https://www.example.com/handbags/[.*]'
@@ -202,6 +221,7 @@ export declare function isTargetRelevant(page: Page, target: Target): boolean;
  * for large element sets, this will take considerable amount of time.
  * @ignore
  */
-export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions): Promise<void>;
+export declare function clickElements(page: Page, selector: string, clickOptions?: ClickOptions & {
+    clickCount?: number;
+}): Promise<void>;
 export {};
-//# sourceMappingURL=click-elements.d.ts.map

package/internals/enqueue-links/click-elements.js

@@ -1,10 +1,9 @@
 import { URL } from 'node:url';
-import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, createRequests, } from '@crawlee/browser';
+import { applyRequestTransform, constructGlobObjectsFromGlobs, constructRegExpObjectsFromPseudoUrls, constructRegExpObjectsFromRegExps, createRequestOptions, filterRequestOptionsByPatterns, Request, serviceLocator, } from '@crawlee/browser';
 import ow from 'ow';
-import log_ from '@apify/log';
 import { addInterceptRequestHandler, removeInterceptRequestHandler } from '../utils/puppeteer_request_interception.js';
 const STARTING_Z_INDEX = 2147400000;
-const log = log_.child({ prefix: 'Puppeteer Click Elements' });
+const getLog = () => serviceLocator.getChildLog('Puppeteer Click Elements');
 /**
  * The function finds elements matching a specific CSS selector in a Puppeteer page,
  * clicks all those elements using a mouse move and a left mouse button click and intercepts
@@ -35,7 +34,7 @@ const log = log_.child({ prefix: 'Puppeteer Click Elements' });
  * ```javascript
  * await utils.puppeteer.enqueueLinksByClickingElements({
  *   page,
- *   requestQueue,
+ *   requestManager,
  *   selector: 'a.product-detail',
  *   pseudoUrls: [
  *       'https://www.example.com/handbags/[.*]'
@@ -49,26 +48,41 @@ const log = log_.child({ prefix: 'Puppeteer Click Elements' });
 export async function enqueueLinksByClickingElements(options) {
     ow(options, ow.object.exactShape({
         page: ow.object.hasKeys('goto', 'evaluate'),
-        requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
+        requestManager: ow.object.hasKeys('fetchNextRequest', 'addRequestsBatched'),
         selector: ow.string,
         userData: ow.optional.object,
-        clickOptions: ow.optional.object.hasKeys('clickCount', 'delay'),
+        clickOptions: ow.optional.object,
         pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
         globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
         regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
+        exclude: ow.optional.array.ofType(ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp'))),
         transformRequestFunction: ow.optional.function,
         waitForPageIdleSecs: ow.optional.number,
         maxWaitForPageIdleSecs: ow.optional.number,
         label: ow.optional.string,
         forefront: ow.optional.boolean,
         skipNavigation: ow.optional.boolean,
+        onSkippedRequest: ow.optional.function,
     }));
-    const { page, requestQueue, selector, clickOptions, pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, } = options;
+    const { page, requestManager, selector, clickOptions, 
+    // oxlint-disable-next-line typescript/no-deprecated -- still accepted for backwards compat
+    pseudoUrls, globs, regexps, transformRequestFunction, waitForPageIdleSecs = 1, maxWaitForPageIdleSecs = 5, forefront, exclude, onSkippedRequest, } = options;
     const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
     const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
+    const urlExcludePatternObjects = [];
     const urlPatternObjects = [];
+    if (exclude?.length) {
+        for (const excl of exclude) {
+            if (typeof excl === 'string' || 'glob' in excl) {
+                urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
+            }
+            else if (excl instanceof RegExp || 'regexp' in excl) {
+                urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
+            }
+        }
+    }
     if (pseudoUrls?.length) {
-        log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
+        getLog().deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
         urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
     }
     if (globs?.length) {
@@ -84,12 +98,21 @@ export async function enqueueLinksByClickingElements(options) {
         maxWaitForPageIdleMillis,
         clickOptions,
     });
-    let requestOptions = createRequestOptions(interceptedRequests, options);
+    const requestOptions = createRequestOptions(interceptedRequests, options);
+    const skippedByFilters = [];
+    let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, (url) => skippedByFilters.push(url));
+    if (onSkippedRequest && skippedByFilters.length > 0) {
+        await Promise.all(skippedByFilters.map(async (url) => onSkippedRequest({ url, reason: 'filters' })));
+    }
     if (transformRequestFunction) {
-        requestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r);
+        const skippedByTransform = [];
+        filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
+        if (onSkippedRequest && skippedByTransform.length > 0) {
+            await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest({ url: r.url, reason: 'transform' })));
+        }
     }
-    const requests = createRequests(requestOptions, urlPatternObjects);
-    const { addedRequests } = await requestQueue.addRequestsBatched(requests, { forefront });
+    const requests = filteredOptions.map((opts) => new Request(opts));
+    const { addedRequests } = await requestManager.addRequestsBatched(requests, { forefront });
     return { processedRequests: addedRequests, unprocessedRequests: [] };
 }
 /**
@@ -130,6 +153,7 @@ function createInterceptRequestHandler(page, requests) {
             url,
             headers: req.headers(),
             method: req.method(),
+            // oxlint-disable-next-line typescript/no-deprecated -- fetchPostData() is async and adds a CDP roundtrip per request; keep the sync page-cached read
             payload: req.postData(),
         }));
         if (req.redirectChain().length) {
@@ -163,7 +187,9 @@ function createTargetCreatedHandler(page, requests) {
             await createdPage.close();
         }
         catch (err) {
-            log.debug('enqueueLinksByClickingElements: Could not close spawned page.', { error: err.stack });
+            getLog().debug('enqueueLinksByClickingElements: Could not close spawned page.', {
+                error: err.stack,
+            });
         }
     };
 }
@@ -172,6 +198,7 @@ function createTargetCreatedHandler(page, requests) {
  * There will generally be a lot of other targets being created in the browser.
  */
 export function isTargetRelevant(page, target) {
+    // oxlint-disable-next-line typescript/no-deprecated -- the non-deprecated replacement (opener.page()) is async and would force every call site to await, including EventEmitter callbacks
     return target.type() === 'page' && page.target() === target.opener();
 }
 /**
@@ -218,8 +245,12 @@ async function preventHistoryNavigation(page) {
  * @ignore
  */
 export async function clickElements(page, selector, clickOptions) {
+    // Puppeteer 25 removed the deprecated `clickCount` option in favor of `count`, older versions ignore `count`, so we pass both.
+    if (clickOptions?.clickCount !== undefined && clickOptions.count === undefined) {
+        clickOptions = { ...clickOptions, count: clickOptions.clickCount };
+    }
     const elementHandles = await page.$$(selector);
-    log.debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
+    getLog().debug(`enqueueLinksByClickingElements: There are ${elementHandles.length} elements to click.`);
     let clickedElementsCount = 0;
     let zIndex = STARTING_Z_INDEX;
     let shouldLogWarning = true;
@@ -232,15 +263,15 @@ export async function clickElements(page, selector, clickOptions) {
         catch (err) {
             const e = err;
             if (shouldLogWarning && e.stack.includes('is detached from document')) {
-                log.warning(`An element with selector ${selector} that you're trying to click has been removed from the page. ` +
+                getLog().warning(`An element with selector ${selector} that you're trying to click has been removed from the page. ` +
                     'This was probably caused by an earlier click which triggered some JavaScript on the page that caused it to change. ' +
                     'If you\'re trying to enqueue pagination links, we suggest using the "next" button, if available and going one by one.');
                 shouldLogWarning = false;
             }
-            log.debug('enqueueLinksByClickingElements: Click failed.', { stack: e.stack });
+            getLog().debug('enqueueLinksByClickingElements: Click failed.', { stack: e.stack });
         }
     }
-    log.debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
+    getLog().debug(`enqueueLinksByClickingElements: Successfully clicked ${clickedElementsCount} elements out of ${elementHandles.length}`);
 }
 /* istanbul ignore next */
 /**
@@ -277,7 +308,6 @@ function updateElementCssToEnableMouseClick(el, zIndex) {
 async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdleMillis, }) {
     return new Promise((resolve) => {
         let timeout;
-        let maxTimeout;
         const context = page.browserContext();
         function newTabTracker(target) {
             if (isTargetRelevant(page, target))
@@ -291,7 +321,7 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
             }, waitForPageIdleMillis);
         }
         function maxTimeoutHandler() {
-            log.debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. ` +
+            getLog().debug(`enqueueLinksByClickingElements: Page still showed activity after ${maxWaitForPageIdleMillis}ms. ` +
                 'This is probably due to the website itself dispatching requests, but some links may also have been missed.');
             finish();
         }
@@ -301,7 +331,7 @@ async function waitForPageIdle({ page, waitForPageIdleMillis, maxWaitForPageIdle
             context.off('targetcreated', newTabTracker);
             resolve();
         }
-        maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
+        const maxTimeout = setTimeout(maxTimeoutHandler, maxWaitForPageIdleMillis);
         activityHandler(); // We call this once manually in case there would be no requests at all.
         page.on('request', activityHandler);
         page.on('framenavigated', activityHandler);
@@ -325,8 +355,7 @@ async function restoreHistoryNavigationAndSaveCapturedUrls(page, requests) {
             requests.add(JSON.stringify({ url }));
         }
         catch (err) {
-            log.debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
+            getLog().debug('enqueueLinksByClickingElements: Failed to ', { error: err.stack });
         }
     });
 }
-//# sourceMappingURL=click-elements.js.map

package/internals/puppeteer-crawler.d.ts

@@ -1,20 +1,20 @@
-import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, BrowserRequestHandler, GetUserDataFromRequest, RouterRoutes } from '@crawlee/browser';
-import { BrowserCrawler, Configuration } from '@crawlee/browser';
-import type { PuppeteerController, PuppeteerPlugin } from '@crawlee/browser-pool';
+import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
+import { BrowserCrawler } from '@crawlee/browser';
+import type { PuppeteerPlugin } from '@crawlee/browser-pool';
 import type { Dictionary } from '@crawlee/types';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { HTTPResponse, LaunchOptions, Page } from 'puppeteer';
+import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/click-elements.js';
 import type { PuppeteerLaunchContext } from './puppeteer-launcher.js';
-import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/puppeteer_utils.js';
-export interface PuppeteerCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<PuppeteerCrawler, Page, HTTPResponse, PuppeteerController, UserData>, PuppeteerContextUtils {
+import type { InterceptHandler } from './utils/puppeteer_request_interception.js';
+import type { BlockRequestsOptions, DirectNavigationOptions, InfiniteScrollOptions, InjectFileOptions, PuppeteerContextUtils, SaveSnapshotOptions } from './utils/puppeteer_utils.js';
+export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;
+export interface PuppeteerCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
 }
 // @ts-ignore optional peer dependency or compatibility with es2022
-export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext, PuppeteerGoToOptions> {
+export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext> {
 }
-export interface PuppeteerRequestHandler extends BrowserRequestHandler<PuppeteerCrawlingContext> {
-}
-export type PuppeteerGoToOptions = Parameters<Page['goto']>[1];
-export interface PuppeteerCrawlerOptions extends BrowserCrawlerOptions<PuppeteerCrawlingContext, {
+export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, {
     browserPlugins: [PuppeteerPlugin];
 }> {
     /**
@@ -23,25 +23,25 @@ export interface PuppeteerCrawlerOptions extends BrowserCrawlerOptions<Puppeteer
     launchContext?: PuppeteerLaunchContext;
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
-     * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
-     * which are passed to the `page.goto()` function the crawler calls to navigate.
+     * or browser properties before navigation. The function receives the `crawlingContext`; the options object
+     * forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
+     * A hook may optionally return a partial object whose properties are merged into the crawling context
+     * (e.g. to override context members for subsequent hooks and pipeline stages).
      * Example:
      * ```
      * preNavigationHooks: [
-     *     async (crawlingContext, gotoOptions) => {
-     *         const { page } = crawlingContext;
+     *     async ({ page, gotoOptions }) => {
      *         await page.evaluate((attr) => { window.foo = attr; }, 'bar');
+     *         gotoOptions.timeout = 60_000;
      *     },
      * ]
      * ```
-     *
-     * Modyfing `pageOptions` is supported only in Playwright incognito.
-     * See {@link PrePageCreateHook}
      */
     preNavigationHooks?: PuppeteerHook[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
-     * The function accepts `crawlingContext` as the only parameter.
+     * The function accepts `crawlingContext` as the only parameter. A hook may optionally return a partial object
+     * whose properties are merged into the crawling context (e.g. to override `response` after solving a challenge).
      * Example:
      * ```
      * postNavigationHooks: [
@@ -67,13 +67,15 @@ export interface PuppeteerCrawlerOptions extends BrowserCrawlerOptions<Puppeteer
  * If the target website doesn't need JavaScript, consider using {@link CheerioCrawler},
  * which downloads the pages using raw HTTP requests and is about 10x faster.
  *
- * The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link PuppeteerCrawlerOptions.requestList}
- * or {@link PuppeteerCrawlerOptions.requestQueue} constructor options, respectively.
+ * The source URLs are represented using {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link PuppeteerCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). To read from a read-only source such
+ * as a {@link RequestList} while still being able to enqueue new requests, combine it with a queue into a
+ * {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the
+ * result as `requestManager`.
  *
- * If both {@link PuppeteerCrawlerOptions.requestList} and {@link PuppeteerCrawlerOptions.requestQueue} are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * > The {@link PuppeteerCrawlerOptions.requestList|`requestList`} and {@link PuppeteerCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
@@ -119,11 +121,9 @@ export interface PuppeteerCrawlerOptions extends BrowserCrawlerOptions<Puppeteer
  * ```
  * @category Crawlers
  */
-export declare class PuppeteerCrawler extends BrowserCrawler<{
+export declare class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension> extends BrowserCrawler<Page, HTTPResponse, {
     browserPlugins: [PuppeteerPlugin];
-}, LaunchOptions, PuppeteerCrawlingContext> {
-    private readonly options;
-    readonly config: Configuration;
+}, LaunchOptions, PuppeteerCrawlingContext, ContextExtension, ExtendedContext> {
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
         browserPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
@@ -138,17 +138,17 @@ export declare class PuppeteerCrawler extends BrowserCrawler<{
 // @ts-ignore optional peer dependency or compatibility with es2022
         headless: import("ow").AnyPredicate<string | boolean>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        browserPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        remoteBrowser: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        useSessionPool: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreShadowRoots: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreIframes: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -165,24 +165,40 @@ export declare class PuppeteerCrawler extends BrowserCrawler<{
         maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         autoscaledPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        blockedStatusCodes: import("ow").ArrayPredicate<number>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        respectRobotsTxtFile: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -191,18 +207,34 @@ export declare class PuppeteerCrawler extends BrowserCrawler<{
         maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
     };
     /**
      * All `PuppeteerCrawler` parameters are passed via an options object.
      */
-    constructor(options?: PuppeteerCrawlerOptions, config?: Configuration);
-    protected _runRequestHandler(context: PuppeteerCrawlingContext): Promise<void>;
+    constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext>);
+// @ts-ignore optional peer dependency or compatibility with es2022
+    protected buildContextPipeline(): import("@crawlee/browser").ContextPipeline<import("@crawlee/browser").CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, HTTPResponse, Dictionary, Dictionary> & {
+        injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;
+        injectJQuery: () => Promise<void>;
+        waitForSelector: (selector: string, timeoutMs?: number) => Promise<void>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<import("@crawlee/browser").CheerioAPI>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        enqueueLinksByClickingElements: (options: Omit<EnqueueLinksByClickingElementsOptions, "page" | "requestManager">) => Promise<import("@crawlee/types").BatchAddRequestsResult>;
+        blockRequests: (options?: BlockRequestsOptions) => Promise<void>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        compileScript: (scriptString: string, ctx?: Dictionary) => import("./utils/puppeteer_utils.js").CompiledScriptFunction;
+        addInterceptRequestHandler: (handler: InterceptHandler) => Promise<void>;
+        removeInterceptRequestHandler: (handler: InterceptHandler) => Promise<void>;
+        infiniteScroll: (options?: InfiniteScrollOptions) => Promise<void>;
+        saveSnapshot: (options?: SaveSnapshotOptions) => Promise<void>;
+        closeCookieModals: () => Promise<void>;
+    }>;
+    private enhanceContext;
     protected _navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<HTTPResponse | null>;
 }
 /**
@@ -229,6 +261,6 @@ export declare class PuppeteerCrawler extends BrowserCrawler<{
  * await crawler.run();
  * ```
  */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/browser").RouterHandler<Context>;
-//# sourceMappingURL=puppeteer-crawler.d.ts.map
+export declare function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export declare function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;