@crawlee/playwright 4.0.0-beta.2 → 4.0.0-beta.200

package/internals/utils/playwright-utils.js

@@ -20,20 +20,47 @@
 import { readFile } from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import vm from 'node:vm';
-import { Configuration, KeyValueStore, RequestState, SessionError, validators, } from '@crawlee/browser';
-import { expandShadowRoots, sleep } from '@crawlee/utils';
-import * as cheerio from 'cheerio';
-import { getInjectableScript as getCookieClosingScript } from 'idcac-playwright';
-import ow from 'ow';
+import { Configuration, KeyValueStore, serviceLocator, SessionError, validators } from '@crawlee/browser';
+import { sleep } from '@crawlee/utils';
+import { expandShadowRoots, parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
-import log_ from '@apify/log';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
-import { RenderingTypePredictor } from './rendering-type-prediction.js';
-const log = log_.child({ prefix: 'Playwright Utils' });
+const getLog = () => serviceLocator.getChildLog('Playwright Utils');
 const require = createRequire(import.meta.url);
 const jqueryPath = require.resolve('jquery');
 const MAX_INJECT_FILE_CACHE_SIZE = 10;
 const DEFAULT_BLOCK_REQUEST_URL_PATTERNS = ['.css', '.jpg', '.jpeg', '.png', '.svg', '.gif', '.woff', '.pdf', '.zip'];
+const filePathSchema = z.string();
+const injectFileOptionsSchema = z.strictObject({
+    surviveNavigations: z.boolean().optional(),
+});
+const gotoExtendedRequestSchema = z.looseObject({
+    url: z.url(),
+    method: z.string().optional(),
+    headers: schemas.anyObject.optional(),
+    payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
+});
+const blockRequestsOptionsSchema = z.strictObject({
+    urlPatterns: schemas.arrayOf(z.string(), 'strings').default(DEFAULT_BLOCK_REQUEST_URL_PATTERNS),
+    extraUrlPatterns: schemas.arrayOf(z.string(), 'strings').default(() => []),
+});
+const infiniteScrollOptionsSchema = z.strictObject({
+    timeoutSecs: schemas.anyNumber.default(0),
+    maxScrollHeight: schemas.anyNumber.default(0),
+    waitForSecs: schemas.anyNumber.default(4),
+    scrollDownAndUp: z.boolean().default(false),
+    buttonSelector: z.string().optional(),
+    stopScrollCallback: schemas.anyFunction.optional(),
+});
+const saveSnapshotOptionsSchema = z.strictObject({
+    key: z.string().min(1).default('SNAPSHOT'),
+    screenshotQuality: schemas.anyNumber.default(50),
+    saveScreenshot: z.boolean().default(true),
+    saveHtml: z.boolean().default(true),
+    keyValueStoreName: z.string().optional(),
+    configuration: schemas.anyObject.optional(),
+});
 /**
  * Cache contents of previously injected files to limit file system access.
  */
@@ -50,21 +77,19 @@ const injectedFilesCache = new LruCache({ maxLength: MAX_INJECT_FILE_CACHE_SIZE
  * @param [options]
  */
 export async function injectFile(page, filePath, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(filePath, ow.string);
-    ow(options, ow.object.exactShape({
-        surviveNavigations: ow.optional.boolean,
-    }));
+    parseArgument(page, validators.browserPage);
+    parseArgument(filePath, filePathSchema);
+    const { surviveNavigations } = parseArgument(options, injectFileOptionsSchema);
     let contents = injectedFilesCache.get(filePath);
     if (!contents) {
         contents = await readFile(filePath, 'utf8');
         injectedFilesCache.add(filePath, contents);
     }
     const evalP = page.evaluate(contents);
-    if (options.surviveNavigations) {
+    if (surviveNavigations) {
         page.on('framenavigated', async () => page
             .evaluate(contents)
-            .catch((error) => log.warning('An error occurred during the script injection!', { error })));
+            .catch((error) => getLog().warning('An error occurred during the script injection!', { error })));
     }
     return evalP;
 }
@@ -95,7 +120,7 @@ export async function injectFile(page, filePath, options = {}) {
  * @param [options.surviveNavigations] Opt-out option to disable the JQuery reinjection after navigation.
  */
 export async function injectJQuery(page, options) {
-    ow(page, ow.object.validate(validators.browserPage));
+    parseArgument(page, validators.browserPage);
     return injectFile(page, jqueryPath, { surviveNavigations: options?.surviveNavigations ?? true });
 }
 /**
@@ -111,19 +136,14 @@ export async function injectJQuery(page, options) {
  * @param [gotoOptions] Custom options for `page.goto()`.
  */
 export async function gotoExtended(page, request, gotoOptions = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(request, ow.object.partialShape({
-        url: ow.string.url,
-        method: ow.optional.string,
-        headers: ow.optional.object,
-        payload: ow.optional.any(ow.string, ow.uint8Array),
-    }));
-    ow(gotoOptions, ow.object);
+    parseArgument(page, validators.browserPage);
+    parseArgument(request, gotoExtendedRequestSchema);
+    parseArgument(gotoOptions, schemas.anyObject);
     const { url, method, headers, payload } = request;
     const isEmpty = (o) => !o || Object.keys(o).length === 0;
-    if (method !== 'GET' || payload || !isEmpty(headers)) {
+    if (method !== 'GET' || payload) {
         // This is not deprecated, we use it to log only once.
-        log.deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
+        getLog().deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
             'in recent versions of Playwright. Use only when necessary.');
         let wasCalled = false;
         const interceptRequestHandler = async (route) => {
@@ -144,12 +164,15 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
                 await route.continue(overrides);
             }
             catch (error) {
-                log.debug('Error inside request interceptor', { error });
+                getLog().debug('Error inside request interceptor', { error });
             }
             return undefined;
         };
         await page.route('**/*', interceptRequestHandler);
     }
+    else if (!isEmpty(headers)) {
+        await page.setExtraHTTPHeaders(headers);
+    }
     return page.goto(url, gotoOptions);
 }
 /**
@@ -200,12 +223,8 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
  * @param [options]
  */
 export async function blockRequests(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        urlPatterns: ow.optional.array.ofType(ow.string),
-        extraUrlPatterns: ow.optional.array.ofType(ow.string),
-    }));
-    const { urlPatterns = DEFAULT_BLOCK_REQUEST_URL_PATTERNS, extraUrlPatterns = [] } = options;
+    parseArgument(page, validators.browserPage);
+    const { urlPatterns, extraUrlPatterns } = parseArgument(options, blockRequestsOptionsSchema);
     const patternsToBlock = [...urlPatterns, ...extraUrlPatterns];
     try {
         const client = await page.context().newCDPSession(page);
@@ -213,7 +232,7 @@ export async function blockRequests(page, options = {}) {
         await client.send('Network.setBlockedURLs', { urls: patternsToBlock });
     }
     catch {
-        log.warning('blockRequests() helper is incompatible with non-Chromium browsers.');
+        getLog().warning('blockRequests() helper is incompatible with non-Chromium browsers.');
     }
 }
 /**
@@ -249,7 +268,7 @@ export function compileScript(scriptString, context = Object.create(null)) {
         func = vm.runInNewContext(funcString, context); // "Secure" the context by removing prototypes, unless custom context is provided.
     }
     catch (err) {
-        log.exception(err, 'Cannot compile script!');
+        getLog().exception(err, 'Cannot compile script!');
         throw err;
     }
     if (typeof func !== 'function')
@@ -263,16 +282,8 @@ export function compileScript(scriptString, context = Object.create(null)) {
  * @param [options]
  */
 export async function infiniteScroll(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        timeoutSecs: ow.optional.number,
-        maxScrollHeight: ow.optional.number,
-        waitForSecs: ow.optional.number,
-        scrollDownAndUp: ow.optional.boolean,
-        buttonSelector: ow.optional.string,
-        stopScrollCallback: ow.optional.function,
-    }));
-    const { timeoutSecs = 0, maxScrollHeight = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback, } = options;
+    parseArgument(page, validators.browserPage);
+    const { timeoutSecs, maxScrollHeight, waitForSecs, scrollDownAndUp, buttonSelector, stopScrollCallback } = parseArgument(options, infiniteScrollOptionsSchema);
     let finished;
     const startTime = Date.now();
     const CHECK_INTERVAL_MILLIS = 1000;
@@ -349,19 +360,11 @@ export async function infiniteScroll(page, options = {}) {
  * @param [options]
  */
 export async function saveSnapshot(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        key: ow.optional.string.nonEmpty,
-        screenshotQuality: ow.optional.number,
-        saveScreenshot: ow.optional.boolean,
-        saveHtml: ow.optional.boolean,
-        keyValueStoreName: ow.optional.string,
-        config: ow.optional.object,
-    }));
-    const { key = 'SNAPSHOT', screenshotQuality = 50, saveScreenshot = true, saveHtml = true, keyValueStoreName, config, } = options;
+    parseArgument(page, validators.browserPage);
+    const { key, screenshotQuality, saveScreenshot, saveHtml, keyValueStoreName, configuration } = parseArgument(options, saveSnapshotOptionsSchema);
     try {
-        const store = await KeyValueStore.open(keyValueStoreName, {
-            config: config ?? Configuration.getGlobalConfig(),
+        const store = await KeyValueStore.open(keyValueStoreName ? { name: keyValueStoreName } : null, {
+            configuration: configuration ?? Configuration.getGlobalConfiguration(),
         });
         if (saveScreenshot) {
             const screenshotName = `${key}.jpg`;
@@ -396,36 +399,44 @@ export async function saveSnapshot(page, options = {}) {
  * @param ignoreShadowRoots
  */
 export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIframes = false) {
-    ow(page, ow.object.validate(validators.browserPage));
+    parseArgument(page, validators.browserPage);
+    const html = ignoreShadowRoots
+        ? null
+        : (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
+    const pageContent = html || (await page.content());
+    // Full cheerio (parse5) import for browser compliant parser
+    const { load } = await import('cheerio');
+    const $ = load(pageContent);
     if (page.frames().length > 1 && !ignoreIframes) {
         const frames = await page.$$('iframe');
-        await Promise.all(frames.map(async (frame) => {
+        const cheerioIframes = $('iframe').toArray();
+        if (frames.length !== cheerioIframes.length) {
+            serviceLocator
+                .getLogger()
+                .warning(`parseWithCheerio: iframe count mismatch between live DOM (${frames.length}) and page snapshot (${cheerioIframes.length}). Some iframes may not be expanded.`);
+        }
+        await Promise.all(frames.map(async (frame, index) => {
             try {
                 const iframe = await frame.contentFrame();
-                if (iframe) {
-                    const contents = await iframe.content();
-                    await frame.evaluate((f, c) => {
-                        const replacementNode = document.createElement('div');
-                        replacementNode.innerHTML = c;
-                        replacementNode.className = 'crawlee-iframe-replacement';
-                        f.replaceWith(replacementNode);
-                    }, contents);
+                if (iframe && cheerioIframes[index]) {
+                    const getIframeHTML = async () => {
+                        try {
+                            return iframe.locator('body').first().innerHTML();
+                        }
+                        catch {
+                            return iframe.content();
+                        }
+                    };
+                    const contents = await getIframeHTML();
+                    $(cheerioIframes[index]).replaceWith(`<div class="crawlee-iframe-replacement">${contents}</div>`);
                 }
             }
             catch (error) {
-                log.warning(`Failed to extract iframe content: ${error}`);
+                getLog().warning(`Failed to extract iframe content: ${error}`);
             }
         }));
     }
-    const html = ignoreShadowRoots
-        ? null
-        : (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
-    const pageContent = html || (await page.content());
-    return cheerio.load(pageContent);
-}
-export async function closeCookieModals(page) {
-    ow(page, ow.object.validate(validators.browserPage));
-    await page.evaluate(getCookieClosingScript());
+    return $;
 }
 /**
  * This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
@@ -434,30 +445,24 @@ export async function closeCookieModals(page) {
  * result in a SessionError which will be automatically retried, so only successful requests will get
  * into the `requestHandler`.
  *
+ * On a successfully solved challenge the page is reloaded and the new {@link Response} is returned, so
+ * it can be propagated back to the crawling context via a hook return value (see
+ * {@link handleCloudflareChallengeHook}).
+ *
  * Works best with camoufox.
  *
  * **Example usage**
  * ```ts
  * postNavigationHooks: [
- *     async ({ handleCloudflareChallenge }) => {
- *         await handleCloudflareChallenge();
- *     },
+ *     async (context) => ({ response: await context.handleCloudflareChallenge() }),
  * ],
  * ```
  *
  * @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object
  * @param url current URL for request identification, only used for logging
- * @param [session] current session object
  * @param [options]
  */
-async function handleCloudflareChallenge(page, url, session, options = {}) {
-    // eslint-disable-next-line dot-notation
-    const blockedStatusCodes = session?.['sessionPool']['blockedStatusCodes'];
-    // Cloudflare pages are 403, which are blocked by default
-    if (blockedStatusCodes?.includes(403)) {
-        const idx = blockedStatusCodes.indexOf(403);
-        blockedStatusCodes.splice(idx, 1);
-    }
+export async function handleCloudflareChallenge(page, url, options = {}) {
     options.isBlockedCallback ??= async () => {
         const isBlocked = await page.evaluate(() => {
             return document.querySelector('h1')?.textContent?.trim().includes('Sorry, you have been blocked');
@@ -466,7 +471,9 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
     };
     options.isChallengeCallback ??= async () => {
         return await page.evaluate(async () => {
-            return !!document.querySelector('.footer > .footer-inner > .diagnostic-wrapper > .ray-id');
+            // Cloudflare keeps reshuffling the wrapper elements between `.footer-inner` and `.ray-id`,
+            // so only the stable outer classes are matched.
+            return !!document.querySelector('.footer .footer-inner .ray-id');
         });
     };
     const retryBlocked = async () => {
@@ -481,31 +488,41 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
     };
     if (!(await isChallenge())) {
         await retryBlocked();
-        return;
+        return undefined;
     }
     const logLevel = options.verbose ? 'info' : 'debug';
-    log[logLevel](`Detected Cloudflare challenge at ${url}, trying to solve it. This can take up to ${10 + (options.sleepSecs ?? 10)} seconds.`);
+    getLog()[logLevel](`Detected Cloudflare challenge at ${url}, trying to solve it. This can take up to ${10 + (options.sleepSecs ?? 10)} seconds.`);
     const bb = await page
         .evaluate(() => {
-        const div = document.querySelector('.main-content div');
+        // Prefer the actual challenge widget (the box holding the Turnstile checkbox input);
+        // fall back to the first content div for older challenge layouts.
+        const div = document.querySelector('.main-content div:has(input[id^="cf-chl-widget-"])') ??
+            document.querySelector('.main-content div');
         return div?.getBoundingClientRect();
     })
         .catch(() => undefined);
     if (!bb) {
-        return;
+        return undefined;
     }
     const randomOffset = (range) => {
         return Math.round(100 * range * Math.random()) / 100;
     };
-    const x = bb.x + 30;
-    const y = bb.y + 25;
+    let x = bb.x + 30;
+    let y = bb.y + 25;
     // try to click the checkbox every second
     for (let i = 0; i < 10; i++) {
-        await sleep(1000);
+        await sleep((options.preChallengeSleepSecs ?? 1) * 1000);
         // break early if we are no longer on the CF challenge page
         if (!(await isChallenge())) {
             break;
         }
+        if (options.clickPositionCallback) {
+            const pos = await options.clickPositionCallback(page);
+            if (pos) {
+                x = pos.x;
+                y = pos.y;
+            }
+        }
         if (options.clickCallback) {
             await options.clickCallback(page, { x, y });
             continue;
@@ -513,7 +530,10 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
         // we can click on the text too, so X can be a bit larger
         const xRandomized = x + randomOffset(10);
         const yRandomized = y + randomOffset(10);
-        log[logLevel](`Trying to click on the Cloudflare checkbox at ${url}`, { x: xRandomized, y: yRandomized });
+        getLog()[logLevel](`Trying to click on the Cloudflare checkbox at ${url}`, {
+            x: xRandomized,
+            y: yRandomized,
+        });
         await page.mouse.click(xRandomized, yRandomized);
         // sometimes the checkbox is lower (could be caused by a lag when rendering the logo)
         await page.mouse.click(xRandomized, yRandomized + 35);
@@ -523,55 +543,8 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
         throw new SessionError(`Blocked by Cloudflare when processing ${url}`);
     }
     await retryBlocked();
-}
-export function registerUtilsToContext(context, crawlerOptions) {
-    context.injectFile = async (filePath, options) => injectFile(context.page, filePath, options);
-    context.injectJQuery = async () => {
-        if (context.request.state === RequestState.BEFORE_NAV) {
-            log.warning('Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.');
-            await injectJQuery(context.page);
-            return;
-        }
-        await injectJQuery(context.page, { surviveNavigations: false });
-    };
-    context.blockRequests = async (options) => blockRequests(context.page, options);
-    context.waitForSelector = async (selector, timeoutMs = 5_000) => {
-        const locator = context.page.locator(selector).first();
-        await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-    };
-    context.parseWithCheerio = async (selector, timeoutMs = 5_000) => {
-        if (selector) {
-            await context.waitForSelector(selector, timeoutMs);
-        }
-        return parseWithCheerio(context.page, crawlerOptions.ignoreShadowRoots, crawlerOptions.ignoreIframes);
-    };
-    context.infiniteScroll = async (options) => infiniteScroll(context.page, options);
-    context.saveSnapshot = async (options) => saveSnapshot(context.page, { ...options, config: context.crawler.config });
-    context.enqueueLinksByClickingElements = async (options) => enqueueLinksByClickingElements({
-        ...options,
-        page: context.page,
-        requestQueue: context.crawler.requestQueue,
-    });
-    context.compileScript = (scriptString, ctx) => compileScript(scriptString, ctx);
-    context.closeCookieModals = async () => closeCookieModals(context.page);
-    context.handleCloudflareChallenge = async (options) => {
-        return handleCloudflareChallenge(context.page, context.request.url, context.session, options);
-    };
+    // Reload to obtain a fresh Response without the challenge interstitial, which the caller can
+    // propagate back into the crawling context so downstream status-code checks see the new value.
+    return (await page.reload()) ?? undefined;
 }
 export { enqueueLinksByClickingElements };
-/** @internal */
-export const playwrightUtils = {
-    injectFile,
-    injectJQuery,
-    gotoExtended,
-    blockRequests,
-    enqueueLinksByClickingElements,
-    parseWithCheerio,
-    infiniteScroll,
-    saveSnapshot,
-    compileScript,
-    closeCookieModals,
-    RenderingTypePredictor,
-    handleCloudflareChallenge,
-};
-//# sourceMappingURL=playwright-utils.js.map

package/internals/utils/rendering-type-prediction.d.ts

@@ -1,21 +1,54 @@
-import type { Request } from '@crawlee/core';
+import type { RecoverableStatePersistenceOptions, Request } from '@crawlee/core';
+import type { Awaitable } from '@crawlee/types';
 export type RenderingType = 'clientOnly' | 'static';
-type URLComponents = string[];
-type FeatureVector = [staticResultsSimilarity: number, clientOnlyResultsSimilarity: number];
 export interface RenderingTypePredictorOptions {
     /** A number between 0 and 1 that determines the desired ratio of rendering type detections */
     detectionRatio: number;
+    persistenceOptions?: Partial<RecoverableStatePersistenceOptions>;
+}
+/**
+ * Minimal contract that any object passed to {@link AdaptivePlaywrightCrawler} as its
+ * `renderingTypePredictor` option must satisfy.
+ *
+ * @experimental
+ */
+export interface IRenderingTypePredictor {
+    /**
+     * Predict the rendering type for a request, and how likely the crawler should be to verify it.
+     *
+     * Called once per request, before navigation - the crawler awaits it, so prefer loading whatever the
+     * prediction needs up front over per-request I/O.
+     */
+    predict(request: Request): Awaitable<{
+        renderingType: RenderingType;
+        detectionProbabilityRecommendation: number;
+    }>;
+    /**
+     * Report a detected rendering type, so that future predictions can take it into account.
+     *
+     * The crawler does not await this per detection - it collects the returned promises and drains them in
+     * `teardown()`, so an implementation that persists results asynchronously can keep batching its writes
+     * and still be sure they land before the crawl ends.
+     */
+    storeResult(requests: Request | Request[], renderingType: RenderingType): Awaitable<void>;
 }
 /**
  * Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.
  *
  * @experimental
  */
-export declare class RenderingTypePredictor {
-    private renderingTypeDetectionResults;
-    private detectionRatio;
-    private logreg;
-    constructor({ detectionRatio }: RenderingTypePredictorOptions);
+export declare class RenderingTypePredictor implements IRenderingTypePredictor {
+    #private;
+    constructor({ detectionRatio, persistenceOptions }: RenderingTypePredictorOptions);
+    /**
+     * Initialize the predictor by restoring persisted state.
+     */
+    initialize(): Promise<void>;
+    /**
+     * Stop persisting the model, writing it out one last time. `initialize()` reopens the persistence window.
+     */
+    teardown(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
     /**
      * Predict the rendering type for a given URL and request label.
      */
@@ -26,10 +59,8 @@ export declare class RenderingTypePredictor {
     /**
      * Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
      */
-    storeResult({ url, loadedUrl, label }: Request, renderingType: RenderingType): void;
+    storeResult(requests: Request | Request[], renderingType: RenderingType): void;
     private resultCount;
-    protected calculateFeatureVector(url: URLComponents, label: string | undefined): FeatureVector;
-    protected retrain(): void;
+    private calculateFeatureVector;
+    private retrain;
 }
-export {};
-//# sourceMappingURL=rendering-type-prediction.d.ts.map