@crawlee/playwright 4.0.0-beta.11 → 4.0.0-beta.111

package/internals/utils/playwright-utils.d.ts

@@ -17,11 +17,11 @@
  * ```
  * @module playwrightUtils
  */
-import { Configuration, type Request, type Session } from '@crawlee/browser';
-import type { BatchAddRequestsResult } from '@crawlee/types';
-import { type CheerioRoot, type Dictionary } from '@crawlee/utils';
+import { Configuration, type Request } from '@crawlee/browser';
+import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
+import { type CheerioRoot } from '@crawlee/utils/internal';
 // @ts-ignore optional peer dependency or compatibility with es2022
-import type { Page, Response } from 'playwright';
+import type { Download, Page, Response } from 'playwright';
 import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
 import { RenderingTypePredictor } from './rendering-type-prediction.js';
@@ -265,9 +265,9 @@ export interface SaveSnapshotOptions {
     keyValueStoreName?: string | null;
     /**
      * Configuration of the crawler that will be used to save the snapshot.
-     * @default Configuration.getGlobalConfig()
+     * @default Configuration.getGlobalConfiguration()
      */
-    config?: Configuration;
+    configuration?: Configuration;
 }
 /**
  * Saves a full screenshot and HTML of the current page into a Key-Value store.
@@ -303,6 +303,13 @@ export interface HandleCloudflareChallengeOptions {
     isChallengeCallback?: (page: Page) => Promise<boolean>;
     /** Allows overriding the detection of Cloudflare "blocked page". */
     isBlockedCallback?: (page: Page) => Promise<boolean>;
+    /** Allows overriding how the checkbox click position is calculated. */
+    clickPositionCallback?: (page: Page) => Promise<{
+        x: number;
+        y: number;
+    } | null>;
+    /** Optional delay (in seconds) before the first click attempt on the challenge checkbox. Defaults to 1s. */
+    preChallengeSleepSecs?: number;
 }
 /**
  * This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
@@ -311,24 +318,24 @@ export interface HandleCloudflareChallengeOptions {
  * result in a SessionError which will be automatically retried, so only successful requests will get
  * into the `requestHandler`.
  *
+ * On a successfully solved challenge the page is reloaded and the new {@link Response} is returned, so
+ * it can be propagated back to the crawling context via a hook return value (see
+ * {@link handleCloudflareChallengeHook}).
+ *
  * Works best with camoufox.
  *
  * **Example usage**
  * ```ts
  * postNavigationHooks: [
- *     async ({ handleCloudflareChallenge }) => {
- *         await handleCloudflareChallenge();
- *     },
+ *     async (context) => ({ response: await context.handleCloudflareChallenge() }),
  * ],
  * ```
  *
  * @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object
  * @param url current URL for request identification, only used for logging
- * @param [session] current session object
  * @param [options]
  */
-declare function handleCloudflareChallenge(page: Page, url: string, session?: Session, options?: HandleCloudflareChallengeOptions): Promise<void>;
-/** @internal */
+declare function handleCloudflareChallenge(page: Page, url: string, options?: HandleCloudflareChallengeOptions): Promise<Response | undefined>;
 export interface PlaywrightContextUtils {
     /**
      * Injects a JavaScript file into current `page`.
@@ -447,8 +454,7 @@ export interface PlaywrightContextUtils {
      * in `href` elements, but rather navigations are triggered in click handlers.
      * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
      *
-     * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
-     * and override settings of the enqueued {@link Request} objects.
+     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
      *
      * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
      * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -469,9 +475,9 @@ export interface PlaywrightContextUtils {
      * async requestHandler({ enqueueLinksByClickingElements }) {
      *     await enqueueLinksByClickingElements({
      *         selector: 'a.product-detail',
-     *         globs: [
-     *             'https://www.example.com/handbags/**'
-     *             'https://www.example.com/purses/**'
+     *         include: [
+     *             'https://www.example.com/handbags/**',
+     *             'https://www.example.com/purses/**',
      *         ],
      *     });
      * });
@@ -479,7 +485,7 @@ export interface PlaywrightContextUtils {
      *
      * @returns Promise that resolves to {@link BatchAddRequestsResult} object.
      */
-    enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>): Promise<BatchAddRequestsResult>;
+    enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestManager'>): Promise<BatchAddRequestsResult>;
     /**
      * Compiles a Playwright script into an async function that may be executed at any time
      * by providing it with the following object:
@@ -509,6 +515,15 @@ export interface PlaywrightContextUtils {
     compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
     /**
      * Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
+     *
+     * Note that this method requires the idcac-playwright package to be installed.
+     * Crawlee does not include it by default due to licensing issues.
+     *
+     * To use this method, please install the package manually by running:
+     *
+     * ```bash
+     * npm install idcac-playwright
+     * ```
      */
     closeCookieModals(): Promise<void>;
     /**
@@ -518,20 +533,43 @@ export interface PlaywrightContextUtils {
      * result in a SessionError which will be automatically retried, so only successful requests will get
      * into the `requestHandler`.
      *
-     * Works best with camoufox.
+     * On a successfully solved challenge the page is reloaded and the new {@link Response} is returned,
+     * which can be returned from the hook to update the crawling context's `response`. For the common case,
+     * prefer the pre-wrapped {@link handleCloudflareChallengeHook} hook.
      *
      * **Example usage**
      * ```ts
      * postNavigationHooks: [
-     *     async ({ handleCloudflareChallenge }) => {
-     *         await handleCloudflareChallenge();
-     *     },
+     *     async (context) => ({ response: await context.handleCloudflareChallenge() }),
      * ],
      * ```
      *
      * @param [options]
      */
-    handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise<void>;
+    handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise<Response | undefined>;
+    /**
+     * Returns the list of {@link https://playwright.dev/docs/api/class-download | Download} objects
+     * collected during the current page navigation and request handler.
+     *
+     * Useful for accessing files that the page downloads automatically.
+     * For most use cases, prefer re-enqueueing the URL to {@link FileDownload}.
+     * Use this only when direct access to the Playwright `Download` object is required.
+     *
+     * **Example usage**
+     * ```ts
+     * requestHandler: async ({ listDownloads }) => {
+     *     for (const download of await listDownloads()) {
+     *         try {
+     *             const stream = await download.createReadStream();
+     *             // stream to storage...
+     *         } catch {
+     *             // download failed or was cancelled
+     *         }
+     *     }
+     * },
+     * ```
+     */
+    listDownloads(): Promise<Download[]>;
 }
 export { enqueueLinksByClickingElements };
 /** @internal */
@@ -549,4 +587,3 @@ export declare const playwrightUtils: {
     RenderingTypePredictor: typeof RenderingTypePredictor;
     handleCloudflareChallenge: typeof handleCloudflareChallenge;
 };
-//# sourceMappingURL=playwright-utils.d.ts.map

package/internals/utils/playwright-utils.js

@@ -20,16 +20,13 @@
 import { readFile } from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import vm from 'node:vm';
-import { Configuration, KeyValueStore, SessionError, validators } from '@crawlee/browser';
+import { Configuration, KeyValueStore, serviceLocator, SessionError, validators } from '@crawlee/browser';
 import { expandShadowRoots, sleep } from '@crawlee/utils';
-import * as cheerio from 'cheerio';
-import { getInjectableScript as getCookieClosingScript } from 'idcac-playwright';
 import ow from 'ow';
 import { LruCache } from '@apify/datastructures';
-import log_ from '@apify/log';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
 import { RenderingTypePredictor } from './rendering-type-prediction.js';
-const log = log_.child({ prefix: 'Playwright Utils' });
+const getLog = () => serviceLocator.getChildLog('Playwright Utils');
 const require = createRequire(import.meta.url);
 const jqueryPath = require.resolve('jquery');
 const MAX_INJECT_FILE_CACHE_SIZE = 10;
@@ -64,7 +61,7 @@ export async function injectFile(page, filePath, options = {}) {
     if (options.surviveNavigations) {
         page.on('framenavigated', async () => page
             .evaluate(contents)
-            .catch((error) => log.warning('An error occurred during the script injection!', { error })));
+            .catch((error) => getLog().warning('An error occurred during the script injection!', { error })));
     }
     return evalP;
 }
@@ -121,9 +118,9 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
     ow(gotoOptions, ow.object);
     const { url, method, headers, payload } = request;
     const isEmpty = (o) => !o || Object.keys(o).length === 0;
-    if (method !== 'GET' || payload || !isEmpty(headers)) {
+    if (method !== 'GET' || payload) {
         // This is not deprecated, we use it to log only once.
-        log.deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
+        getLog().deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
             'in recent versions of Playwright. Use only when necessary.');
         let wasCalled = false;
         const interceptRequestHandler = async (route) => {
@@ -144,12 +141,15 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
                 await route.continue(overrides);
             }
             catch (error) {
-                log.debug('Error inside request interceptor', { error });
+                getLog().debug('Error inside request interceptor', { error });
             }
             return undefined;
         };
         await page.route('**/*', interceptRequestHandler);
     }
+    else if (!isEmpty(headers)) {
+        await page.setExtraHTTPHeaders(headers);
+    }
     return page.goto(url, gotoOptions);
 }
 /**
@@ -213,7 +213,7 @@ export async function blockRequests(page, options = {}) {
         await client.send('Network.setBlockedURLs', { urls: patternsToBlock });
     }
     catch {
-        log.warning('blockRequests() helper is incompatible with non-Chromium browsers.');
+        getLog().warning('blockRequests() helper is incompatible with non-Chromium browsers.');
     }
 }
 /**
@@ -249,7 +249,7 @@ export function compileScript(scriptString, context = Object.create(null)) {
         func = vm.runInNewContext(funcString, context); // "Secure" the context by removing prototypes, unless custom context is provided.
     }
     catch (err) {
-        log.exception(err, 'Cannot compile script!');
+        getLog().exception(err, 'Cannot compile script!');
         throw err;
     }
     if (typeof func !== 'function')
@@ -356,12 +356,12 @@ export async function saveSnapshot(page, options = {}) {
         saveScreenshot: ow.optional.boolean,
         saveHtml: ow.optional.boolean,
         keyValueStoreName: ow.optional.string,
-        config: ow.optional.object,
+        configuration: ow.optional.object,
     }));
-    const { key = 'SNAPSHOT', screenshotQuality = 50, saveScreenshot = true, saveHtml = true, keyValueStoreName, config, } = options;
+    const { key = 'SNAPSHOT', screenshotQuality = 50, saveScreenshot = true, saveHtml = true, keyValueStoreName, configuration, } = options;
     try {
-        const store = await KeyValueStore.open(keyValueStoreName, {
-            config: config ?? Configuration.getGlobalConfig(),
+        const store = await KeyValueStore.open(keyValueStoreName ? { name: keyValueStoreName } : null, {
+            configuration: configuration ?? Configuration.getGlobalConfiguration(),
         });
         if (saveScreenshot) {
             const screenshotName = `${key}.jpg`;
@@ -397,35 +397,71 @@ export async function saveSnapshot(page, options = {}) {
  */
 export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIframes = false) {
     ow(page, ow.object.validate(validators.browserPage));
+    const html = ignoreShadowRoots
+        ? null
+        : (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
+    const pageContent = html || (await page.content());
+    const { load } = await import('cheerio');
+    const $ = load(pageContent);
     if (page.frames().length > 1 && !ignoreIframes) {
         const frames = await page.$$('iframe');
-        await Promise.all(frames.map(async (frame) => {
+        const cheerioIframes = $('iframe').toArray();
+        if (frames.length !== cheerioIframes.length) {
+            serviceLocator
+                .getLogger()
+                .warning(`parseWithCheerio: iframe count mismatch between live DOM (${frames.length}) and page snapshot (${cheerioIframes.length}). Some iframes may not be expanded.`);
+        }
+        await Promise.all(frames.map(async (frame, index) => {
             try {
                 const iframe = await frame.contentFrame();
-                if (iframe) {
-                    const contents = await iframe.content();
-                    await frame.evaluate((f, c) => {
-                        const replacementNode = document.createElement('div');
-                        replacementNode.innerHTML = c;
-                        replacementNode.className = 'crawlee-iframe-replacement';
-                        f.replaceWith(replacementNode);
-                    }, contents);
+                if (iframe && cheerioIframes[index]) {
+                    const getIframeHTML = async () => {
+                        try {
+                            return iframe.locator('body').first().innerHTML();
+                        }
+                        catch {
+                            return iframe.content();
+                        }
+                    };
+                    const contents = await getIframeHTML();
+                    $(cheerioIframes[index]).replaceWith(`<div class="crawlee-iframe-replacement">${contents}</div>`);
                 }
             }
             catch (error) {
-                log.warning(`Failed to extract iframe content: ${error}`);
+                getLog().warning(`Failed to extract iframe content: ${error}`);
             }
         }));
     }
-    const html = ignoreShadowRoots
-        ? null
-        : (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
-    const pageContent = html || (await page.content());
-    return cheerio.load(pageContent);
+    return $;
+}
+let idcacPlaywright = null;
+async function getIdcacPlaywright() {
+    if (idcacPlaywright)
+        return idcacPlaywright;
+    try {
+        idcacPlaywright = await import('idcac-playwright');
+    }
+    catch (error) {
+        getLog().warning(`Failed to import 'idcac-playwright'.
+
+We recently made idcac-playwright an optional dependency due to licensing issues.
+To use this feature, please install it manually by running
+
+npm install idcac-playwright
+
+Original error message follows:
+
+${error.message}
+`);
+    }
+    return idcacPlaywright;
 }
 export async function closeCookieModals(page) {
     ow(page, ow.object.validate(validators.browserPage));
-    await page.evaluate(getCookieClosingScript());
+    const idcac = await getIdcacPlaywright();
+    if (idcac?.getInjectableScript()) {
+        await page.evaluate(idcac.getInjectableScript());
+    }
 }
 /**
  * This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
@@ -434,30 +470,24 @@ export async function closeCookieModals(page) {
  * result in a SessionError which will be automatically retried, so only successful requests will get
  * into the `requestHandler`.
  *
+ * On a successfully solved challenge the page is reloaded and the new {@link Response} is returned, so
+ * it can be propagated back to the crawling context via a hook return value (see
+ * {@link handleCloudflareChallengeHook}).
+ *
  * Works best with camoufox.
  *
  * **Example usage**
  * ```ts
  * postNavigationHooks: [
- *     async ({ handleCloudflareChallenge }) => {
- *         await handleCloudflareChallenge();
- *     },
+ *     async (context) => ({ response: await context.handleCloudflareChallenge() }),
  * ],
  * ```
  *
  * @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object
  * @param url current URL for request identification, only used for logging
- * @param [session] current session object
  * @param [options]
  */
-async function handleCloudflareChallenge(page, url, session, options = {}) {
-    // eslint-disable-next-line dot-notation
-    const blockedStatusCodes = session?.['sessionPool']['blockedStatusCodes'];
-    // Cloudflare pages are 403, which are blocked by default
-    if (blockedStatusCodes?.includes(403)) {
-        const idx = blockedStatusCodes.indexOf(403);
-        blockedStatusCodes.splice(idx, 1);
-    }
+async function handleCloudflareChallenge(page, url, options = {}) {
     options.isBlockedCallback ??= async () => {
         const isBlocked = await page.evaluate(() => {
             return document.querySelector('h1')?.textContent?.trim().includes('Sorry, you have been blocked');
@@ -466,7 +496,9 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
     };
     options.isChallengeCallback ??= async () => {
         return await page.evaluate(async () => {
-            return !!document.querySelector('.footer > .footer-inner > .diagnostic-wrapper > .ray-id');
+            // Cloudflare nests the ray ID under varying wrapper elements, so we match by descendants
+            // instead of a direct-child chain (e.g. a `.footer-wrapper` was inserted in between).
+            return !!document.querySelector('.footer .footer-inner .diagnostic-wrapper .ray-id');
         });
     };
     const retryBlocked = async () => {
@@ -481,31 +513,41 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
     };
     if (!(await isChallenge())) {
         await retryBlocked();
-        return;
+        return undefined;
     }
     const logLevel = options.verbose ? 'info' : 'debug';
-    log[logLevel](`Detected Cloudflare challenge at ${url}, trying to solve it. This can take up to ${10 + (options.sleepSecs ?? 10)} seconds.`);
+    getLog()[logLevel](`Detected Cloudflare challenge at ${url}, trying to solve it. This can take up to ${10 + (options.sleepSecs ?? 10)} seconds.`);
     const bb = await page
         .evaluate(() => {
-        const div = document.querySelector('.main-content div');
+        // Prefer the actual challenge widget (the box holding the Turnstile checkbox input);
+        // fall back to the first content div for older challenge layouts.
+        const div = document.querySelector('.main-content div:has(input[id^="cf-chl-widget-"])') ??
+            document.querySelector('.main-content div');
         return div?.getBoundingClientRect();
     })
         .catch(() => undefined);
     if (!bb) {
-        return;
+        return undefined;
     }
     const randomOffset = (range) => {
         return Math.round(100 * range * Math.random()) / 100;
     };
-    const x = bb.x + 30;
-    const y = bb.y + 25;
+    let x = bb.x + 30;
+    let y = bb.y + 25;
     // try to click the checkbox every second
     for (let i = 0; i < 10; i++) {
-        await sleep(1000);
+        await sleep((options.preChallengeSleepSecs ?? 1) * 1000);
         // break early if we are no longer on the CF challenge page
         if (!(await isChallenge())) {
             break;
         }
+        if (options.clickPositionCallback) {
+            const pos = await options.clickPositionCallback(page);
+            if (pos) {
+                x = pos.x;
+                y = pos.y;
+            }
+        }
         if (options.clickCallback) {
             await options.clickCallback(page, { x, y });
             continue;
@@ -513,7 +555,10 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
         // we can click on the text too, so X can be a bit larger
         const xRandomized = x + randomOffset(10);
         const yRandomized = y + randomOffset(10);
-        log[logLevel](`Trying to click on the Cloudflare checkbox at ${url}`, { x: xRandomized, y: yRandomized });
+        getLog()[logLevel](`Trying to click on the Cloudflare checkbox at ${url}`, {
+            x: xRandomized,
+            y: yRandomized,
+        });
         await page.mouse.click(xRandomized, yRandomized);
         // sometimes the checkbox is lower (could be caused by a lag when rendering the logo)
         await page.mouse.click(xRandomized, yRandomized + 35);
@@ -523,6 +568,9 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
         throw new SessionError(`Blocked by Cloudflare when processing ${url}`);
     }
     await retryBlocked();
+    // Reload to obtain a fresh Response without the challenge interstitial, which the caller can
+    // propagate back into the crawling context so downstream status-code checks see the new value.
+    return (await page.reload()) ?? undefined;
 }
 export { enqueueLinksByClickingElements };
 /** @internal */
@@ -540,4 +588,3 @@ export const playwrightUtils = {
     RenderingTypePredictor,
     handleCloudflareChallenge,
 };
-//# sourceMappingURL=playwright-utils.js.map

package/internals/utils/rendering-type-prediction.d.ts

@@ -1,21 +1,38 @@
-import type { Request } from '@crawlee/core';
+import type { RecoverableStatePersistenceOptions, Request } from '@crawlee/core';
 export type RenderingType = 'clientOnly' | 'static';
-type URLComponents = string[];
-type FeatureVector = [staticResultsSimilarity: number, clientOnlyResultsSimilarity: number];
 export interface RenderingTypePredictorOptions {
     /** A number between 0 and 1 that determines the desired ratio of rendering type detections */
     detectionRatio: number;
+    persistenceOptions?: Partial<RecoverableStatePersistenceOptions>;
+}
+/**
+ * Minimal contract that any object passed to {@link AdaptivePlaywrightCrawler} as its
+ * `renderingTypePredictor` option must satisfy.
+ *
+ * @experimental
+ */
+export interface IRenderingTypePredictor {
+    /** Predict the rendering type for a request, and how likely the crawler should be to verify it. */
+    predict(request: Request): {
+        renderingType: RenderingType;
+        detectionProbabilityRecommendation: number;
+    };
+    /** Report a detected rendering type, so that future predictions can take it into account. */
+    storeResult(requests: Request | Request[], renderingType: RenderingType): void;
 }
 /**
  * Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.
  *
  * @experimental
  */
-export declare class RenderingTypePredictor {
-    private renderingTypeDetectionResults;
-    private detectionRatio;
-    private logreg;
-    constructor({ detectionRatio }: RenderingTypePredictorOptions);
+export declare class RenderingTypePredictor implements IRenderingTypePredictor {
+    #private;
+    private state;
+    constructor({ detectionRatio, persistenceOptions }: RenderingTypePredictorOptions);
+    /**
+     * Initialize the predictor by restoring persisted state.
+     */
+    initialize(): Promise<void>;
     /**
      * Predict the rendering type for a given URL and request label.
      */
@@ -26,10 +43,8 @@ export declare class RenderingTypePredictor {
     /**
      * Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
      */
-    storeResult({ url, loadedUrl, label }: Request, renderingType: RenderingType): void;
+    storeResult(requests: Request | Request[], renderingType: RenderingType): void;
     private resultCount;
-    protected calculateFeatureVector(url: URLComponents, label: string | undefined): FeatureVector;
-    protected retrain(): void;
+    private calculateFeatureVector;
+    private retrain;
 }
-export {};
-//# sourceMappingURL=rendering-type-prediction.d.ts.map