@crawlee/playwright 4.0.0-rc.0 → 4.0.0-rc.1

package/internals/playwright-crawler.d.ts

@@ -1,12 +1,11 @@
-import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
+import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
 import { BrowserCrawler } from '@crawlee/browser';
 import type { Dictionary } from '@crawlee/types';
 // @ts-ignore optional peer dependency or compatibility with es2022
-import type { Download, LaunchOptions, Page, Response } from 'playwright';
+import type { Page, Response } from 'playwright';
 import { z } from 'zod';
-import type { EnqueueLinksByClickingElementsOptions } from './enqueue-links/click-elements.js';
 import type { PlaywrightLaunchContext } from './playwright-launcher.js';
-import type { BlockRequestsOptions, DirectNavigationOptions, HandleCloudflareChallengeOptions, InfiniteScrollOptions, InjectFileOptions, PlaywrightContextUtils, SaveSnapshotOptions } from './utils/playwright-utils.js';
+import type { DirectNavigationOptions, HandleCloudflareChallengeOptions, PlaywrightContextUtils } from './utils/playwright-utils.js';
 export type PlaywrightGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
 export interface PlaywrightCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, Response, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {
 }
@@ -21,29 +20,6 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
      * Can be also set via {@link Configuration}.
      */
     headless?: boolean;
-    /**
-     * Function that is called to process each request.
-     *
-     * The function receives the {@link PlaywrightCrawlingContext} as an argument, where:
-     * - `request` is an instance of the {@link Request} object with details about the URL to open, HTTP method etc.
-     * - `page` is an instance of the `Playwright`
-     * [`Page`](https://playwright.dev/docs/api/class-page)
-     * - `response` is an instance of the `Playwright`
-     * [`Response`](https://playwright.dev/docs/api/class-response),
-     * which is the main resource response as returned by `page.goto(request.url)`.
-     *
-     * The function must return a promise, which is then awaited by the crawler.
-     *
-     * If the function throws an exception, the crawler will try to re-crawl the
-     * request later, up to `option.maxRequestRetries` times.
-     * If all the retries fail, the crawler calls the function
-     * provided to the `failedRequestHandler` parameter.
-     * To make this work, you should **always**
-     * let your function throw exceptions rather than catch them.
-     * The exceptions are logged to the request using the
-     * {@link Request.pushErrorMessage} function.
-     */
-    requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
      * or browser properties before navigation. The function receives the `crawlingContext`; the options object
@@ -144,21 +120,12 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
  * ```
  * @category Crawlers
  */
-export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, Response, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+    #private;
+    /**
+     * @internal
+     */
     protected static optionsShape: {
-        headless: z.ZodOptional<z.ZodBoolean>;
-        launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
-        navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
-        preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
-        postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
-        launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
-        browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
-        remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
-        saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
-        proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
-        ignoreIframes: z.ZodDefault<z.ZodBoolean>;
-        ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
         contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
         requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -190,27 +157,23 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
         }, z.core.$strict>]>>;
         onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
+        httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").Configuration, import("@crawlee/browser").Configuration>>;
+        configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
         storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").EventManager, import("@crawlee/browser").EventManager>>;
+        eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
         logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
         minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
         keepAlive: z.ZodOptional<z.ZodBoolean>;
         statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         id: z.ZodOptional<z.ZodString>;
-    };
-    protected static optionsSchema: z.ZodObject<{
-        headless: z.ZodOptional<z.ZodBoolean>;
-        launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
         preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
         postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
-        launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
         browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
         browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
         remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
@@ -218,6 +181,11 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
         proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
         ignoreIframes: z.ZodDefault<z.ZodBoolean>;
         ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
+        launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
+        headless: z.ZodOptional<z.ZodBoolean>;
+    };
+    /** @internal */
+    protected static optionsSchema: z.ZodObject<{
         contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
         requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -249,44 +217,38 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
         }, z.core.$strict>]>>;
         onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
+        httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").Configuration, import("@crawlee/browser").Configuration>>;
+        configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
         storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").EventManager, import("@crawlee/browser").EventManager>>;
+        eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
         logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
         minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
+        initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
         maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
         keepAlive: z.ZodOptional<z.ZodBoolean>;
         statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         id: z.ZodOptional<z.ZodString>;
+        navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
+        preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+        postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
+        browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
+        remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
+        proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        ignoreIframes: z.ZodDefault<z.ZodBoolean>;
+        ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
+        launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
+        headless: z.ZodOptional<z.ZodBoolean>;
     }, z.core.$strict>;
     /**
      * All `PlaywrightCrawler` parameters are passed via an options object.
      */
     constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
-// @ts-ignore optional peer dependency or compatibility with es2022
-    protected buildContextPipeline(): import("@crawlee/browser").ContextPipeline<import("@crawlee/browser").CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, Response, Dictionary, Dictionary> & {
-        injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;
-        injectJQuery: () => Promise<void>;
-        blockRequests: (options?: BlockRequestsOptions) => Promise<void>;
-        waitForSelector: (selector: string, timeoutMs?: number) => Promise<void>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<import("@crawlee/browser").CheerioAPI>;
-        infiniteScroll: (options?: InfiniteScrollOptions) => Promise<void>;
-        listDownloads: () => Promise<Download[]>;
-        saveSnapshot: (options?: SaveSnapshotOptions) => Promise<void>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        enqueueLinksByClickingElements: (options: Omit<EnqueueLinksByClickingElementsOptions, "page" | "requestManager">) => Promise<import("@crawlee/types").BatchAddRequestsResult>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        compileScript: (scriptString: string, ctx?: Dictionary) => import("./utils/playwright-utils.js").CompiledScriptFunction;
-        closeCookieModals: () => Promise<void>;
-        handleCloudflareChallenge: (options?: HandleCloudflareChallengeOptions) => Promise<Response | undefined>;
-    }>;
     protected navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<Response | null>;
-    private enhanceContext;
 }
 /**
  * Returns a `postNavigationHooks`-ready hook that runs {@link PlaywrightContextUtils.handleCloudflareChallenge}

package/internals/playwright-crawler.js

@@ -1,7 +1,8 @@
-import { assertBrowserPoolNotConfigured, BrowserCrawler, parseArgument, RequestState, Router, schemas, serviceLocator, } from '@crawlee/browser';
+import { BrowserCrawler, RequestState, Router, serviceLocator } from '@crawlee/browser';
+import { assertBrowserPoolNotConfigured, parseArgument, schemas } from '@crawlee/utils/internal';
 import { z } from 'zod';
 import { playwrightBrowserPool, remotePlaywrightBrowserPool } from './playwright-browser-pool.js';
-import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
+import { blockRequests, compileScript, enqueueLinksByClickingElements, gotoExtended, handleCloudflareChallenge, infiniteScroll, injectFile, injectJQuery, parseWithCheerio, saveSnapshot, } from './utils/playwright-utils.js';
 /**
  * Provides a simple framework for parallel crawling of web pages
  * using headless Chromium, Firefox and Webkit browsers with [Playwright](https://github.com/microsoft/playwright).
@@ -68,18 +69,22 @@ import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
  * @category Crawlers
  */
 export class PlaywrightCrawler extends BrowserCrawler {
+    /**
+     * @internal
+     */
     static optionsShape = {
         ...BrowserCrawler.optionsShape,
+        launchContext: schemas.anyObject.default(() => ({})),
         headless: z.boolean().optional(),
-        launcher: schemas.anyObject.optional(),
     };
+    /** @internal */
     static optionsSchema = z.strictObject(PlaywrightCrawler.optionsShape);
     /**
      * All `PlaywrightCrawler` parameters are passed via an options object.
      */
     constructor(options = {}) {
         const parsedOptions = parseArgument(options, PlaywrightCrawler.optionsSchema, 'PlaywrightCrawlerOptions');
-        const { launchContext, headless, configuration, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
+        const { launchContext, headless, configuration, ...browserCrawlerOptions } = parsedOptions;
         if (launchContext.proxyUrl) {
             throw new Error('PlaywrightCrawlerOptions.launchContext.proxyUrl is not allowed in PlaywrightCrawler.' +
                 'Use PlaywrightCrawlerOptions.proxyConfiguration');
@@ -93,21 +98,20 @@ export class PlaywrightCrawler extends BrowserCrawler {
         }
         super({
             ...browserCrawlerOptions,
-            launchContext,
             configuration,
             browserPoolBuilder: (remoteBrowser) => remoteBrowser
                 ? remotePlaywrightBrowserPool({ ...remoteBrowser, launchContext, headless, configuration })
                 : playwrightBrowserPool({ launchContext, headless, configuration }),
-            contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
+            contextPipelineBuilder: () => this.#buildContextPipeline(),
         });
     }
-    buildContextPipeline() {
-        return super.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) });
+    #buildContextPipeline() {
+        return this.buildContextPipeline().compose(this.#enhanceContext.bind(this));
     }
     async navigationHandler(crawlingContext, gotoOptions) {
         return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
     }
-    async enhanceContext(context) {
+    async #enhanceContext(context) {
         const waitForSelector = async (selector, timeoutMs = 5_000) => {
             const locator = context.page.locator(selector).first();
             await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
@@ -115,38 +119,37 @@ export class PlaywrightCrawler extends BrowserCrawler {
         const downloads = [];
         context.page.on('download', (download) => downloads.push(download));
         return {
-            injectFile: async (filePath, options) => playwrightUtils.injectFile(context.page, filePath, options),
+            injectFile: async (filePath, options) => injectFile(context.page, filePath, options),
             injectJQuery: async () => {
                 if (context.request.state === RequestState.BEFORE_NAV) {
                     context.log.warning('Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.');
-                    await playwrightUtils.injectJQuery(context.page);
+                    await injectJQuery(context.page);
                     return;
                 }
-                await playwrightUtils.injectJQuery(context.page, { surviveNavigations: false });
+                await injectJQuery(context.page, { surviveNavigations: false });
             },
-            blockRequests: async (options) => playwrightUtils.blockRequests(context.page, options),
+            blockRequests: async (options) => blockRequests(context.page, options),
             waitForSelector,
             parseWithCheerio: async (selector, timeoutMs = 5_000) => {
                 if (selector) {
                     await waitForSelector(selector, timeoutMs);
                 }
-                return playwrightUtils.parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
+                return parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
             },
-            infiniteScroll: async (options) => playwrightUtils.infiniteScroll(context.page, options),
+            infiniteScroll: async (options) => infiniteScroll(context.page, options),
             listDownloads: async () => downloads,
-            saveSnapshot: async (options) => playwrightUtils.saveSnapshot(context.page, {
+            saveSnapshot: async (options) => saveSnapshot(context.page, {
                 ...options,
                 configuration: serviceLocator.getConfiguration(),
             }),
-            enqueueLinksByClickingElements: async (options) => playwrightUtils.enqueueLinksByClickingElements({
+            enqueueLinksByClickingElements: async (options) => enqueueLinksByClickingElements({
                 ...options,
                 page: context.page,
                 requestManager: this.requestManager,
             }),
-            compileScript: (scriptString, ctx) => playwrightUtils.compileScript(scriptString, ctx),
-            closeCookieModals: async () => playwrightUtils.closeCookieModals(context.page),
+            compileScript: (scriptString, ctx) => compileScript(scriptString, ctx),
             handleCloudflareChallenge: async (options) => {
-                return playwrightUtils.handleCloudflareChallenge(context.page, context.request.url, options);
+                return handleCloudflareChallenge(context.page, context.request.url, options);
             },
         };
     }

package/internals/playwright-launcher.d.ts

@@ -72,11 +72,10 @@ export interface PlaywrightLaunchContext extends BrowserLaunchContext<LaunchOpti
  */
 export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin> {
     readonly configuration: Configuration;
+    /**
+     * @internal
+     */
     protected static optionsShape: {
-// @ts-ignore optional peer dependency or compatibility with es2022
-        launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        launchContextOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
         proxyUrl: z.ZodOptional<z.ZodURL>;
         useChrome: z.ZodOptional<z.ZodBoolean>;
         useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
@@ -86,12 +85,11 @@ export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin
 // @ts-ignore optional peer dependency or compatibility with es2022
         launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
         userAgent: z.ZodOptional<z.ZodString>;
-    };
-    protected static optionsSchema: z.ZodObject<{
 // @ts-ignore optional peer dependency or compatibility with es2022
         launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        launchContextOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+    };
+    /** @internal */
+    protected static optionsSchema: z.ZodObject<{
         proxyUrl: z.ZodOptional<z.ZodURL>;
         useChrome: z.ZodOptional<z.ZodBoolean>;
         useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
@@ -101,6 +99,8 @@ export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin
 // @ts-ignore optional peer dependency or compatibility with es2022
         launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
         userAgent: z.ZodOptional<z.ZodString>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
     }, z.core.$strict>;
     /**
      * All `PlaywrightLauncher` parameters are passed via this launchContext object.

package/internals/playwright-launcher.js

@@ -1,5 +1,6 @@
-import { BrowserLauncher, Configuration, parseArgument, schemas } from '@crawlee/browser';
+import { BrowserLauncher, Configuration } from '@crawlee/browser';
 import { PlaywrightPlugin } from '@crawlee/browser-pool';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
 import { z } from 'zod';
 /**
  * `PlaywrightLauncher` is based on the `BrowserLauncher`. It launches `playwright` browser instance.
@@ -7,12 +8,15 @@ import { z } from 'zod';
  */
 export class PlaywrightLauncher extends BrowserLauncher {
     configuration;
+    /**
+     * @internal
+     */
     static optionsShape = {
         ...BrowserLauncher.optionsShape,
-        // Passthrough schemas — the launcher module object must keep its prototype through parsing.
+        // Passthrough schema — the launcher module object must keep its prototype through parsing.
         launcher: schemas.anyObject.optional(),
-        launchContextOptions: schemas.anyObject.optional(),
     };
+    /** @internal */
     static optionsSchema = z.strictObject(PlaywrightLauncher.optionsShape);
     /**
      * All `PlaywrightLauncher` parameters are passed via this launchContext object.

package/internals/utils/playwright-utils.d.ts

@@ -19,12 +19,11 @@
  */
 import { Configuration, type Request } from '@crawlee/browser';
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type CheerioRoot } from '@crawlee/utils/internal';
+import type { CheerioAPI } from 'cheerio';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Download, Page, Response } from 'playwright';
 import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
-import { RenderingTypePredictor } from './rendering-type-prediction.js';
 export interface InjectFileOptions {
     /**
      * Enables the injected script to survive page navigations and reloads without need to be re-injected manually.
@@ -287,8 +286,7 @@ export declare function saveSnapshot(page: Page, options?: SaveSnapshotOptions):
  * @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object.
  * @param ignoreShadowRoots
  */
-export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;
-export declare function closeCookieModals(page: Page): Promise<void>;
+export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioAPI>;
 export interface HandleCloudflareChallengeOptions {
     /** Logging defaults to the `debug` level, use this flag to log to `info` level instead. */
     verbose?: boolean;
@@ -335,7 +333,7 @@ export interface HandleCloudflareChallengeOptions {
  * @param url current URL for request identification, only used for logging
  * @param [options]
  */
-declare function handleCloudflareChallenge(page: Page, url: string, options?: HandleCloudflareChallengeOptions): Promise<Response | undefined>;
+export declare function handleCloudflareChallenge(page: Page, url: string, options?: HandleCloudflareChallengeOptions): Promise<Response | undefined>;
 export interface PlaywrightContextUtils {
     /**
      * Injects a JavaScript file into current `page`.
@@ -434,7 +432,7 @@ export interface PlaywrightContextUtils {
      * });
      * ```
      */
-    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
     /**
      * Scrolls to the bottom of a page, or until it times out.
      * Loads dynamic content when it hits the bottom of a page, and then continues scrolling.
@@ -513,19 +511,6 @@ export interface PlaywrightContextUtils {
      * secured copies beforehand.
      */
     compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
-    /**
-     * Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
-     *
-     * Note that this method requires the idcac-playwright package to be installed.
-     * Crawlee does not include it by default due to licensing issues.
-     *
-     * To use this method, please install the package manually by running:
-     *
-     * ```bash
-     * npm install idcac-playwright
-     * ```
-     */
-    closeCookieModals(): Promise<void>;
     /**
      * This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
      * It will try to detect the Cloudflare page, click on the checkbox, and wait for 10 seconds (configurable
@@ -572,18 +557,3 @@ export interface PlaywrightContextUtils {
     listDownloads(): Promise<Download[]>;
 }
 export { enqueueLinksByClickingElements };
-/** @internal */
-export declare const playwrightUtils: {
-    injectFile: typeof injectFile;
-    injectJQuery: typeof injectJQuery;
-    gotoExtended: typeof gotoExtended;
-    blockRequests: typeof blockRequests;
-    enqueueLinksByClickingElements: typeof enqueueLinksByClickingElements;
-    parseWithCheerio: typeof parseWithCheerio;
-    infiniteScroll: typeof infiniteScroll;
-    saveSnapshot: typeof saveSnapshot;
-    compileScript: typeof compileScript;
-    closeCookieModals: typeof closeCookieModals;
-    RenderingTypePredictor: typeof RenderingTypePredictor;
-    handleCloudflareChallenge: typeof handleCloudflareChallenge;
-};

package/internals/utils/playwright-utils.js

@@ -20,12 +20,12 @@
 import { readFile } from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import vm from 'node:vm';
-import { Configuration, KeyValueStore, parseArgument, schemas, serviceLocator, SessionError, validators, } from '@crawlee/browser';
-import { expandShadowRoots, sleep } from '@crawlee/utils';
+import { Configuration, KeyValueStore, serviceLocator, SessionError, validators } from '@crawlee/browser';
+import { sleep } from '@crawlee/utils';
+import { expandShadowRoots, parseArgument, schemas } from '@crawlee/utils/internal';
 import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
-import { RenderingTypePredictor } from './rendering-type-prediction.js';
 const getLog = () => serviceLocator.getChildLog('Playwright Utils');
 const require = createRequire(import.meta.url);
 const jqueryPath = require.resolve('jquery');
@@ -404,6 +404,7 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
         ? null
         : (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
     const pageContent = html || (await page.content());
+    // Full cheerio (parse5) import for browser compliant parser
     const { load } = await import('cheerio');
     const $ = load(pageContent);
     if (page.frames().length > 1 && !ignoreIframes) {
@@ -437,35 +438,6 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
     }
     return $;
 }
-let idcacPlaywright = null;
-async function getIdcacPlaywright() {
-    if (idcacPlaywright)
-        return idcacPlaywright;
-    try {
-        idcacPlaywright = await import('idcac-playwright');
-    }
-    catch (error) {
-        getLog().warning(`Failed to import 'idcac-playwright'.
-
-We recently made idcac-playwright an optional dependency due to licensing issues.
-To use this feature, please install it manually by running
-
-npm install idcac-playwright
-
-Original error message follows:
-
-${error.message}
-`);
-    }
-    return idcacPlaywright;
-}
-export async function closeCookieModals(page) {
-    parseArgument(page, validators.browserPage);
-    const idcac = await getIdcacPlaywright();
-    if (idcac?.getInjectableScript()) {
-        await page.evaluate(idcac.getInjectableScript());
-    }
-}
 /**
  * This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
  * It will try to detect the Cloudflare page, click on the checkbox, and wait for 10 seconds (configurable
@@ -490,7 +462,7 @@ export async function closeCookieModals(page) {
  * @param url current URL for request identification, only used for logging
  * @param [options]
  */
-async function handleCloudflareChallenge(page, url, options = {}) {
+export async function handleCloudflareChallenge(page, url, options = {}) {
     options.isBlockedCallback ??= async () => {
         const isBlocked = await page.evaluate(() => {
             return document.querySelector('h1')?.textContent?.trim().includes('Sorry, you have been blocked');
@@ -512,7 +484,11 @@ async function handleCloudflareChallenge(page, url, options = {}) {
     };
     // check if we ended up on the CF challenge page
     const isChallenge = async () => {
-        return options.isChallengeCallback(page).catch(() => false);
+        return options.isChallengeCallback(page).catch(async () => {
+            // the challenge page navigates itself, which destroys the context of an in-flight check
+            await page.waitForLoadState('domcontentloaded').catch(() => { });
+            return options.isChallengeCallback(page).catch(() => false);
+        });
     };
     if (!(await isChallenge())) {
         await retryBlocked();
@@ -576,18 +552,3 @@ async function handleCloudflareChallenge(page, url, options = {}) {
     return (await page.reload()) ?? undefined;
 }
 export { enqueueLinksByClickingElements };
-/** @internal */
-export const playwrightUtils = {
-    injectFile,
-    injectJQuery,
-    gotoExtended,
-    blockRequests,
-    enqueueLinksByClickingElements,
-    parseWithCheerio,
-    infiniteScroll,
-    saveSnapshot,
-    compileScript,
-    closeCookieModals,
-    RenderingTypePredictor,
-    handleCloudflareChallenge,
-};

package/internals/utils/rendering-type-prediction.d.ts

@@ -1,4 +1,5 @@
 import type { RecoverableStatePersistenceOptions, Request } from '@crawlee/core';
+import type { Awaitable } from '@crawlee/types';
 export type RenderingType = 'clientOnly' | 'static';
 export interface RenderingTypePredictorOptions {
     /** A number between 0 and 1 that determines the desired ratio of rendering type detections */
@@ -12,13 +13,24 @@ export interface RenderingTypePredictorOptions {
  * @experimental
  */
 export interface IRenderingTypePredictor {
-    /** Predict the rendering type for a request, and how likely the crawler should be to verify it. */
-    predict(request: Request): {
+    /**
+     * Predict the rendering type for a request, and how likely the crawler should be to verify it.
+     *
+     * Called once per request, before navigation - the crawler awaits it, so prefer loading whatever the
+     * prediction needs up front over per-request I/O.
+     */
+    predict(request: Request): Awaitable<{
         renderingType: RenderingType;
         detectionProbabilityRecommendation: number;
-    };
-    /** Report a detected rendering type, so that future predictions can take it into account. */
-    storeResult(requests: Request | Request[], renderingType: RenderingType): void;
+    }>;
+    /**
+     * Report a detected rendering type, so that future predictions can take it into account.
+     *
+     * The crawler does not await this per detection - it collects the returned promises and drains them in
+     * `teardown()`, so an implementation that persists results asynchronously can keep batching its writes
+     * and still be sure they land before the crawl ends.
+     */
+    storeResult(requests: Request | Request[], renderingType: RenderingType): Awaitable<void>;
 }
 /**
  * Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.
@@ -27,12 +39,16 @@ export interface IRenderingTypePredictor {
  */
 export declare class RenderingTypePredictor implements IRenderingTypePredictor {
     #private;
-    private state;
     constructor({ detectionRatio, persistenceOptions }: RenderingTypePredictorOptions);
     /**
      * Initialize the predictor by restoring persisted state.
      */
     initialize(): Promise<void>;
+    /**
+     * Stop persisting the model, writing it out one last time. `initialize()` reopens the persistence window.
+     */
+    teardown(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
     /**
      * Predict the rendering type for a given URL and request label.
      */
@@ -44,7 +60,4 @@ export declare class RenderingTypePredictor implements IRenderingTypePredictor {
      * Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
      */
     storeResult(requests: Request | Request[], renderingType: RenderingType): void;
-    private resultCount;
-    private calculateFeatureVector;
-    private retrain;
 }