@crawlee/playwright 4.0.0-beta.9 → 4.0.0-beta.91

package/internals/utils/playwright-utils.js

@@ -20,16 +20,14 @@
 import { readFile } from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import vm from 'node:vm';
-import { Configuration, KeyValueStore, RequestState, SessionError, validators, } from '@crawlee/browser';
+import { Configuration, KeyValueStore, serviceLocator, SessionError, validators } from '@crawlee/browser';
 import { expandShadowRoots, sleep } from '@crawlee/utils';
 import * as cheerio from 'cheerio';
-import { getInjectableScript as getCookieClosingScript } from 'idcac-playwright';
 import ow from 'ow';
 import { LruCache } from '@apify/datastructures';
-import log_ from '@apify/log';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
 import { RenderingTypePredictor } from './rendering-type-prediction.js';
-const log = log_.child({ prefix: 'Playwright Utils' });
+const getLog = () => serviceLocator.getChildLog('Playwright Utils');
 const require = createRequire(import.meta.url);
 const jqueryPath = require.resolve('jquery');
 const MAX_INJECT_FILE_CACHE_SIZE = 10;
@@ -64,7 +62,7 @@ export async function injectFile(page, filePath, options = {}) {
     if (options.surviveNavigations) {
         page.on('framenavigated', async () => page
             .evaluate(contents)
-            .catch((error) => log.warning('An error occurred during the script injection!', { error })));
+            .catch((error) => getLog().warning('An error occurred during the script injection!', { error })));
     }
     return evalP;
 }
@@ -121,9 +119,9 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
     ow(gotoOptions, ow.object);
     const { url, method, headers, payload } = request;
     const isEmpty = (o) => !o || Object.keys(o).length === 0;
-    if (method !== 'GET' || payload || !isEmpty(headers)) {
+    if (method !== 'GET' || payload) {
         // This is not deprecated, we use it to log only once.
-        log.deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
+        getLog().deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
             'in recent versions of Playwright. Use only when necessary.');
         let wasCalled = false;
         const interceptRequestHandler = async (route) => {
@@ -144,12 +142,15 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
                 await route.continue(overrides);
             }
             catch (error) {
-                log.debug('Error inside request interceptor', { error });
+                getLog().debug('Error inside request interceptor', { error });
             }
             return undefined;
         };
         await page.route('**/*', interceptRequestHandler);
     }
+    else if (!isEmpty(headers)) {
+        await page.setExtraHTTPHeaders(headers);
+    }
     return page.goto(url, gotoOptions);
 }
 /**
@@ -213,7 +214,7 @@ export async function blockRequests(page, options = {}) {
         await client.send('Network.setBlockedURLs', { urls: patternsToBlock });
     }
     catch {
-        log.warning('blockRequests() helper is incompatible with non-Chromium browsers.');
+        getLog().warning('blockRequests() helper is incompatible with non-Chromium browsers.');
     }
 }
 /**
@@ -249,7 +250,7 @@ export function compileScript(scriptString, context = Object.create(null)) {
         func = vm.runInNewContext(funcString, context); // "Secure" the context by removing prototypes, unless custom context is provided.
     }
     catch (err) {
-        log.exception(err, 'Cannot compile script!');
+        getLog().exception(err, 'Cannot compile script!');
         throw err;
     }
     if (typeof func !== 'function')
@@ -356,12 +357,12 @@ export async function saveSnapshot(page, options = {}) {
         saveScreenshot: ow.optional.boolean,
         saveHtml: ow.optional.boolean,
         keyValueStoreName: ow.optional.string,
-        config: ow.optional.object,
+        configuration: ow.optional.object,
     }));
-    const { key = 'SNAPSHOT', screenshotQuality = 50, saveScreenshot = true, saveHtml = true, keyValueStoreName, config, } = options;
+    const { key = 'SNAPSHOT', screenshotQuality = 50, saveScreenshot = true, saveHtml = true, keyValueStoreName, configuration, } = options;
     try {
-        const store = await KeyValueStore.open(keyValueStoreName, {
-            config: config ?? Configuration.getGlobalConfig(),
+        const store = await KeyValueStore.open(keyValueStoreName ? { name: keyValueStoreName } : null, {
+            configuration: configuration ?? Configuration.getGlobalConfiguration(),
         });
         if (saveScreenshot) {
             const screenshotName = `${key}.jpg`;
@@ -397,35 +398,70 @@ export async function saveSnapshot(page, options = {}) {
  */
 export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIframes = false) {
     ow(page, ow.object.validate(validators.browserPage));
+    const html = ignoreShadowRoots
+        ? null
+        : (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
+    const pageContent = html || (await page.content());
+    const $ = cheerio.load(pageContent);
     if (page.frames().length > 1 && !ignoreIframes) {
         const frames = await page.$$('iframe');
-        await Promise.all(frames.map(async (frame) => {
+        const cheerioIframes = $('iframe').toArray();
+        if (frames.length !== cheerioIframes.length) {
+            serviceLocator
+                .getLogger()
+                .warning(`parseWithCheerio: iframe count mismatch between live DOM (${frames.length}) and page snapshot (${cheerioIframes.length}). Some iframes may not be expanded.`);
+        }
+        await Promise.all(frames.map(async (frame, index) => {
             try {
                 const iframe = await frame.contentFrame();
-                if (iframe) {
-                    const contents = await iframe.content();
-                    await frame.evaluate((f, c) => {
-                        const replacementNode = document.createElement('div');
-                        replacementNode.innerHTML = c;
-                        replacementNode.className = 'crawlee-iframe-replacement';
-                        f.replaceWith(replacementNode);
-                    }, contents);
+                if (iframe && cheerioIframes[index]) {
+                    const getIframeHTML = async () => {
+                        try {
+                            return iframe.locator('body').first().innerHTML();
+                        }
+                        catch {
+                            return iframe.content();
+                        }
+                    };
+                    const contents = await getIframeHTML();
+                    $(cheerioIframes[index]).replaceWith(`<div class="crawlee-iframe-replacement">${contents}</div>`);
                 }
             }
             catch (error) {
-                log.warning(`Failed to extract iframe content: ${error}`);
+                getLog().warning(`Failed to extract iframe content: ${error}`);
             }
         }));
     }
-    const html = ignoreShadowRoots
-        ? null
-        : (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
-    const pageContent = html || (await page.content());
-    return cheerio.load(pageContent);
+    return $;
+}
+let idcacPlaywright = null;
+async function getIdcacPlaywright() {
+    if (idcacPlaywright)
+        return idcacPlaywright;
+    try {
+        idcacPlaywright = await import('idcac-playwright');
+    }
+    catch (error) {
+        getLog().warning(`Failed to import 'idcac-playwright'.
+
+We recently made idcac-playwright an optional dependency due to licensing issues.
+To use this feature, please install it manually by running
+
+npm install idcac-playwright
+
+Original error message follows:
+
+${error.message}
+`);
+    }
+    return idcacPlaywright;
 }
 export async function closeCookieModals(page) {
     ow(page, ow.object.validate(validators.browserPage));
-    await page.evaluate(getCookieClosingScript());
+    const idcac = await getIdcacPlaywright();
+    if (idcac?.getInjectableScript()) {
+        await page.evaluate(idcac.getInjectableScript());
+    }
 }
 /**
  * This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
@@ -434,30 +470,24 @@ export async function closeCookieModals(page) {
  * result in a SessionError which will be automatically retried, so only successful requests will get
  * into the `requestHandler`.
  *
+ * On a successfully solved challenge the page is reloaded and the new {@link Response} is returned, so
+ * it can be propagated back to the crawling context via a hook return value (see
+ * {@link handleCloudflareChallengeHook}).
+ *
  * Works best with camoufox.
  *
  * **Example usage**
  * ```ts
  * postNavigationHooks: [
- *     async ({ handleCloudflareChallenge }) => {
- *         await handleCloudflareChallenge();
- *     },
+ *     async (context) => ({ response: await context.handleCloudflareChallenge() }),
  * ],
  * ```
  *
  * @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object
  * @param url current URL for request identification, only used for logging
- * @param [session] current session object
  * @param [options]
  */
-async function handleCloudflareChallenge(page, url, session, options = {}) {
-    // eslint-disable-next-line dot-notation
-    const blockedStatusCodes = session?.['sessionPool']['blockedStatusCodes'];
-    // Cloudflare pages are 403, which are blocked by default
-    if (blockedStatusCodes?.includes(403)) {
-        const idx = blockedStatusCodes.indexOf(403);
-        blockedStatusCodes.splice(idx, 1);
-    }
+async function handleCloudflareChallenge(page, url, options = {}) {
     options.isBlockedCallback ??= async () => {
         const isBlocked = await page.evaluate(() => {
             return document.querySelector('h1')?.textContent?.trim().includes('Sorry, you have been blocked');
@@ -466,7 +496,9 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
     };
     options.isChallengeCallback ??= async () => {
         return await page.evaluate(async () => {
-            return !!document.querySelector('.footer > .footer-inner > .diagnostic-wrapper > .ray-id');
+            // Cloudflare nests the ray ID under varying wrapper elements, so we match by descendants
+            // instead of a direct-child chain (e.g. a `.footer-wrapper` was inserted in between).
+            return !!document.querySelector('.footer .footer-inner .diagnostic-wrapper .ray-id');
         });
     };
     const retryBlocked = async () => {
@@ -481,31 +513,41 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
     };
     if (!(await isChallenge())) {
         await retryBlocked();
-        return;
+        return undefined;
     }
     const logLevel = options.verbose ? 'info' : 'debug';
-    log[logLevel](`Detected Cloudflare challenge at ${url}, trying to solve it. This can take up to ${10 + (options.sleepSecs ?? 10)} seconds.`);
+    getLog()[logLevel](`Detected Cloudflare challenge at ${url}, trying to solve it. This can take up to ${10 + (options.sleepSecs ?? 10)} seconds.`);
     const bb = await page
         .evaluate(() => {
-        const div = document.querySelector('.main-content div');
+        // Prefer the actual challenge widget (the box holding the Turnstile checkbox input);
+        // fall back to the first content div for older challenge layouts.
+        const div = document.querySelector('.main-content div:has(input[id^="cf-chl-widget-"])') ??
+            document.querySelector('.main-content div');
         return div?.getBoundingClientRect();
     })
         .catch(() => undefined);
     if (!bb) {
-        return;
+        return undefined;
     }
     const randomOffset = (range) => {
         return Math.round(100 * range * Math.random()) / 100;
     };
-    const x = bb.x + 30;
-    const y = bb.y + 25;
+    let x = bb.x + 30;
+    let y = bb.y + 25;
     // try to click the checkbox every second
     for (let i = 0; i < 10; i++) {
-        await sleep(1000);
+        await sleep((options.preChallengeSleepSecs ?? 1) * 1000);
         // break early if we are no longer on the CF challenge page
         if (!(await isChallenge())) {
             break;
         }
+        if (options.clickPositionCallback) {
+            const pos = await options.clickPositionCallback(page);
+            if (pos) {
+                x = pos.x;
+                y = pos.y;
+            }
+        }
         if (options.clickCallback) {
             await options.clickCallback(page, { x, y });
             continue;
@@ -513,7 +555,10 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
         // we can click on the text too, so X can be a bit larger
         const xRandomized = x + randomOffset(10);
         const yRandomized = y + randomOffset(10);
-        log[logLevel](`Trying to click on the Cloudflare checkbox at ${url}`, { x: xRandomized, y: yRandomized });
+        getLog()[logLevel](`Trying to click on the Cloudflare checkbox at ${url}`, {
+            x: xRandomized,
+            y: yRandomized,
+        });
         await page.mouse.click(xRandomized, yRandomized);
         // sometimes the checkbox is lower (could be caused by a lag when rendering the logo)
         await page.mouse.click(xRandomized, yRandomized + 35);
@@ -523,40 +568,9 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
         throw new SessionError(`Blocked by Cloudflare when processing ${url}`);
     }
     await retryBlocked();
-}
-export function registerUtilsToContext(context, crawlerOptions) {
-    context.injectFile = async (filePath, options) => injectFile(context.page, filePath, options);
-    context.injectJQuery = async () => {
-        if (context.request.state === RequestState.BEFORE_NAV) {
-            log.warning('Using injectJQuery() in preNavigationHooks leads to unstable results. Use it in a postNavigationHook or a requestHandler instead.');
-            await injectJQuery(context.page);
-            return;
-        }
-        await injectJQuery(context.page, { surviveNavigations: false });
-    };
-    context.blockRequests = async (options) => blockRequests(context.page, options);
-    context.waitForSelector = async (selector, timeoutMs = 5_000) => {
-        const locator = context.page.locator(selector).first();
-        await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-    };
-    context.parseWithCheerio = async (selector, timeoutMs = 5_000) => {
-        if (selector) {
-            await context.waitForSelector(selector, timeoutMs);
-        }
-        return parseWithCheerio(context.page, crawlerOptions.ignoreShadowRoots, crawlerOptions.ignoreIframes);
-    };
-    context.infiniteScroll = async (options) => infiniteScroll(context.page, options);
-    context.saveSnapshot = async (options) => saveSnapshot(context.page, { ...options, config: context.crawler.config });
-    context.enqueueLinksByClickingElements = async (options) => enqueueLinksByClickingElements({
-        ...options,
-        page: context.page,
-        requestQueue: context.crawler.requestQueue,
-    });
-    context.compileScript = (scriptString, ctx) => compileScript(scriptString, ctx);
-    context.closeCookieModals = async () => closeCookieModals(context.page);
-    context.handleCloudflareChallenge = async (options) => {
-        return handleCloudflareChallenge(context.page, context.request.url, context.session, options);
-    };
+    // Reload to obtain a fresh Response without the challenge interstitial, which the caller can
+    // propagate back into the crawling context so downstream status-code checks see the new value.
+    return (await page.reload()) ?? undefined;
 }
 export { enqueueLinksByClickingElements };
 /** @internal */
@@ -574,4 +588,3 @@ export const playwrightUtils = {
     RenderingTypePredictor,
     handleCloudflareChallenge,
 };
-//# sourceMappingURL=playwright-utils.js.map

package/internals/utils/rendering-type-prediction.d.ts

@@ -1,21 +1,38 @@
-import type { Request } from '@crawlee/core';
+import type { RecoverableStatePersistenceOptions, Request } from '@crawlee/core';
 export type RenderingType = 'clientOnly' | 'static';
-type URLComponents = string[];
-type FeatureVector = [staticResultsSimilarity: number, clientOnlyResultsSimilarity: number];
 export interface RenderingTypePredictorOptions {
     /** A number between 0 and 1 that determines the desired ratio of rendering type detections */
     detectionRatio: number;
+    persistenceOptions?: Partial<RecoverableStatePersistenceOptions>;
+}
+/**
+ * Minimal contract that any object passed to {@link AdaptivePlaywrightCrawler} as its
+ * `renderingTypePredictor` option must satisfy.
+ *
+ * @experimental
+ */
+export interface IRenderingTypePredictor {
+    /** Predict the rendering type for a request, and how likely the crawler should be to verify it. */
+    predict(request: Request): {
+        renderingType: RenderingType;
+        detectionProbabilityRecommendation: number;
+    };
+    /** Report a detected rendering type, so that future predictions can take it into account. */
+    storeResult(requests: Request | Request[], renderingType: RenderingType): void;
 }
 /**
  * Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.
  *
  * @experimental
  */
-export declare class RenderingTypePredictor {
-    private renderingTypeDetectionResults;
+export declare class RenderingTypePredictor implements IRenderingTypePredictor {
     private detectionRatio;
-    private logreg;
-    constructor({ detectionRatio }: RenderingTypePredictorOptions);
+    private state;
+    constructor({ detectionRatio, persistenceOptions }: RenderingTypePredictorOptions);
+    /**
+     * Initialize the predictor by restoring persisted state.
+     */
+    initialize(): Promise<void>;
     /**
      * Predict the rendering type for a given URL and request label.
      */
@@ -26,10 +43,8 @@ export declare class RenderingTypePredictor {
     /**
      * Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
      */
-    storeResult({ url, loadedUrl, label }: Request, renderingType: RenderingType): void;
+    storeResult(requests: Request | Request[], renderingType: RenderingType): void;
     private resultCount;
-    protected calculateFeatureVector(url: URLComponents, label: string | undefined): FeatureVector;
-    protected retrain(): void;
+    private calculateFeatureVector;
+    private retrain;
 }
-export {};
-//# sourceMappingURL=rendering-type-prediction.d.ts.map

package/internals/utils/rendering-type-prediction.js

@@ -1,3 +1,4 @@
+import { RecoverableState } from '@crawlee/core';
 import LogisticRegression from 'ml-logistic-regression';
 import { Matrix } from 'ml-matrix';
 import stringComparison from 'string-comparison';
@@ -9,10 +10,17 @@ const calculateUrlSimilarity = (a, b) => {
     if (a[0] !== b[0]) {
         return 0;
     }
-    for (let i = 1; i < Math.max(a.length, b.length); i++) {
+    const maxLength = Math.max(a.length, b.length);
+    // Only the hostname is present (no path components to compare) - the hosts already match.
+    if (maxLength <= 1) {
+        return 1;
+    }
+    for (let i = 1; i < maxLength; i++) {
         values.push(stringComparison.jaroWinkler.similarity(a[i] ?? '', b[i] ?? '') > 0.8 ? 1 : 0);
     }
-    return sum(values) / Math.max(a.length, b.length);
+    // The first component (index 0, the hostname) is excluded from the comparison above,
+    // so it must also be excluded from the denominator of the weighted average.
+    return sum(values) / (maxLength - 1);
 };
 const sum = (values) => values.reduce((acc, value) => acc + value);
 const mean = (values) => (values.length > 0 ? sum(values) / values.length : undefined);
@@ -22,27 +30,58 @@ const mean = (values) => (values.length > 0 ? sum(values) / values.length : unde
  * @experimental
  */
 export class RenderingTypePredictor {
-    renderingTypeDetectionResults = new Map();
     detectionRatio;
-    logreg;
-    constructor({ detectionRatio }) {
+    state;
+    constructor({ detectionRatio, persistenceOptions }) {
         this.detectionRatio = detectionRatio;
-        this.logreg = new LogisticRegression({ numSteps: 1000, learningRate: 0.05 });
+        this.state = new RecoverableState({
+            defaultState: {
+                logreg: new LogisticRegression({ numSteps: 1000, learningRate: 0.05 }),
+                detectionResults: new Map(),
+            },
+            serialize: (state) => JSON.stringify({
+                logreg: state.logreg.toJSON(),
+                detectionResults: Array.from(state.detectionResults.entries()).map(([renderingType, urlPartsByLabel]) => ({
+                    renderingType,
+                    urlPartsByLabel: Array.from(urlPartsByLabel.entries()).map(([label, urlParts]) => ({
+                        label,
+                        urlParts,
+                    })),
+                })),
+            }),
+            deserialize: (serializedState) => {
+                const { logreg, detectionResults = [] } = JSON.parse(serializedState);
+                return {
+                    logreg: LogisticRegression.load(logreg),
+                    detectionResults: new Map(detectionResults.map((serializedItem) => [
+                        serializedItem.renderingType,
+                        new Map(serializedItem.urlPartsByLabel.map((item) => [item.label, item.urlParts])),
+                    ])),
+                };
+            },
+            persistStateKey: 'rendering-type-predictor-state',
+            persistenceEnabled: true,
+            ...persistenceOptions,
+        });
+    }
+    /**
+     * Initialize the predictor by restoring persisted state.
+     */
+    async initialize() {
+        await this.state.initialize();
     }
     /**
      * Predict the rendering type for a given URL and request label.
      */
     predict({ url, loadedUrl, label }) {
-        if (this.logreg.classifiers.length === 0) {
+        const { logreg } = this.state.currentValue;
+        if (logreg.classifiers.length === 0) {
             return { renderingType: 'clientOnly', detectionProbabilityRecommendation: 1 };
         }
         const predictionUrl = new URL(loadedUrl ?? url);
         const urlFeature = new Matrix([this.calculateFeatureVector(urlComponents(predictionUrl), label)]);
-        const [prediction] = this.logreg.predict(urlFeature);
-        const scores = [
-            this.logreg.classifiers[0].testScores(urlFeature),
-            this.logreg.classifiers[1].testScores(urlFeature),
-        ];
+        const [prediction] = logreg.predict(urlFeature);
+        const scores = [logreg.classifiers[0].testScores(urlFeature), logreg.classifiers[1].testScores(urlFeature)];
         return {
             renderingType: prediction === 1 ? 'static' : 'clientOnly',
             detectionProbabilityRecommendation: Math.abs(scores[0] - scores[1]) < 0.1
@@ -53,26 +92,29 @@ export class RenderingTypePredictor {
     /**
      * Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
      */
-    storeResult({ url, loadedUrl, label }, renderingType) {
-        const resultUrl = new URL(loadedUrl ?? url);
-        if (!this.renderingTypeDetectionResults.has(renderingType)) {
-            this.renderingTypeDetectionResults.set(renderingType, new Map());
-        }
-        if (!this.renderingTypeDetectionResults.get(renderingType).has(label)) {
-            this.renderingTypeDetectionResults.get(renderingType).set(label, []);
+    storeResult(requests, renderingType) {
+        const state = this.state.currentValue;
+        for (const { url, loadedUrl, label } of Array.isArray(requests) ? requests : [requests]) {
+            const resultUrl = new URL(loadedUrl ?? url);
+            if (!state.detectionResults.has(renderingType)) {
+                state.detectionResults.set(renderingType, new Map());
+            }
+            if (!state.detectionResults.get(renderingType).has(label)) {
+                state.detectionResults.get(renderingType).set(label, []);
+            }
+            state.detectionResults.get(renderingType).get(label).push(urlComponents(resultUrl));
         }
-        this.renderingTypeDetectionResults.get(renderingType).get(label).push(urlComponents(resultUrl));
         this.retrain();
     }
     resultCount(label) {
-        return Array.from(this.renderingTypeDetectionResults.values())
+        return Array.from(this.state.currentValue.detectionResults.values())
             .map((results) => results.get(label)?.length ?? 0)
             .reduce((acc, value) => acc + value, 0);
     }
     calculateFeatureVector(url, label) {
         return [
-            mean((this.renderingTypeDetectionResults.get('static')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
-            mean((this.renderingTypeDetectionResults.get('clientOnly')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
+            mean((this.state.currentValue.detectionResults.get('static')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
+            mean((this.state.currentValue.detectionResults.get('clientOnly')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
         ];
     }
     retrain() {
@@ -81,7 +123,7 @@ export class RenderingTypePredictor {
             [1, 0],
         ];
         const Y = [0, 1];
-        for (const [renderingType, urlsByLabel] of this.renderingTypeDetectionResults.entries()) {
+        for (const [renderingType, urlsByLabel] of this.state.currentValue.detectionResults.entries()) {
             for (const [label, urls] of urlsByLabel) {
                 for (const url of urls) {
                     X.push(this.calculateFeatureVector(url, label));
@@ -89,7 +131,6 @@ export class RenderingTypePredictor {
                 }
             }
         }
-        this.logreg.train(new Matrix(X), Matrix.columnVector(Y));
+        this.state.currentValue.logreg.train(new Matrix(X), Matrix.columnVector(Y));
     }
 }
-//# sourceMappingURL=rendering-type-prediction.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.9",
+    "version": "4.0.0-beta.91",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -38,7 +38,7 @@
     },
     "homepage": "https://crawlee.dev",
     "scripts": {
-        "build": "yarn clean && yarn compile && yarn copy",
+        "build": "pnpm clean && pnpm compile && pnpm copy",
         "clean": "rimraf ./dist",
         "compile": "tsc -p tsconfig.build.json",
         "copy": "tsx ../../scripts/copy.ts"
@@ -48,27 +48,32 @@
     },
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
-        "@apify/log": "^2.5.18",
         "@apify/timeout": "^0.3.2",
-        "@crawlee/browser": "4.0.0-beta.9",
-        "@crawlee/browser-pool": "4.0.0-beta.9",
-        "@crawlee/core": "4.0.0-beta.9",
-        "@crawlee/types": "4.0.0-beta.9",
-        "@crawlee/utils": "4.0.0-beta.9",
+        "@crawlee/basic": "4.0.0-beta.91",
+        "@crawlee/browser": "4.0.0-beta.91",
+        "@crawlee/browser-pool": "4.0.0-beta.91",
+        "@crawlee/cheerio": "4.0.0-beta.91",
+        "@crawlee/core": "4.0.0-beta.91",
+        "@crawlee/types": "4.0.0-beta.91",
+        "@crawlee/utils": "4.0.0-beta.91",
         "cheerio": "^1.0.0",
         "idcac-playwright": "^0.1.3",
         "jquery": "^3.7.1",
-        "lodash.isequal": "^4.5.0",
         "ml-logistic-regression": "^2.0.0",
         "ml-matrix": "^6.12.1",
         "ow": "^2.0.0",
         "string-comparison": "^1.3.0",
-        "tslib": "^2.8.1"
+        "tslib": "^2.8.1",
+        "type-fest": "^4.0.0"
     },
     "peerDependencies": {
+        "idcac-playwright": "^0.2.0",
         "playwright": "*"
     },
     "peerDependenciesMeta": {
+        "idcac-playwright": {
+            "optional": true
+        },
         "playwright": {
             "optional": true
         }
@@ -80,5 +85,5 @@
             }
         }
     },
-    "gitHead": "d2489f2800cd7d9f15815fc6b5e195dc243703d2"
+    "gitHead": "1e3e1ca10f24be53e1d527b3ea25f456baebecbf"
 }

package/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,mCAAmC,CAAC;AAClD,cAAc,oCAAoC,CAAC;AACnD,cAAc,4CAA4C,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,gDAAgD,CAAC;AAExF,OAAO,KAAK,eAAe,MAAM,uCAAuC,CAAC;AACzE,OAAO,KAAK,uBAAuB,MAAM,6CAA6C,CAAC;AACvF,YAAY,EAAE,uBAAuB,IAAI,iCAAiC,EAAE,MAAM,uCAAuC,CAAC;AAC1H,YAAY,EAAE,aAAa,EAAE,MAAM,gDAAgD,CAAC"}

package/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,mCAAmC,CAAC;AAClD,cAAc,oCAAoC,CAAC;AACnD,cAAc,4CAA4C,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,gDAAgD,CAAC;AAExF,OAAO,KAAK,eAAe,MAAM,uCAAuC,CAAC;AACzE,OAAO,KAAK,uBAAuB,MAAM,6CAA6C,CAAC"}

package/internals/adaptive-playwright-crawler.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"adaptive-playwright-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/adaptive-playwright-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,aAAa,EAA0B,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAE1G,OAAO,KAAK,EACR,oBAAoB,EACpB,sBAAsB,EACtB,yBAAyB,EACzB,YAAY,EAEZ,iBAAiB,EACjB,cAAc,EACjB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAU,UAAU,EAA4B,MAAM,eAAe,CAAC;AAClH,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5D,OAAO,EAAE,KAAK,WAAW,EAA0B,MAAM,gBAAgB,CAAC;AAC1E,OAAO,EAAE,KAAK,OAAO,EAAQ,MAAM,SAAS,CAAC;AAE7C,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AAGtC,OAAO,KAAK,EACR,wBAAwB,EACxB,yBAAyB,EACzB,qBAAqB,EACxB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAsB,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAElG,KAAK,MAAM,CAAC,OAAO,IACb;IAAE,MAAM,EAAE,OAAO,CAAC;IAAC,EAAE,EAAE,IAAI,CAAC;IAAC,IAAI,CAAC,EAAE,YAAY,EAAE,CAAA;CAAE,GACpD;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,EAAE,EAAE,KAAK,CAAC;IAAC,IAAI,CAAC,EAAE,YAAY,EAAE,CAAA;CAAE,CAAC;AAE3D,UAAU,uCAAwC,SAAQ,cAAc;IACpE,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,2BAA2B,CAAC,EAAE,MAAM,CAAC;CACxC;AAQD,cAAM,mCAAoC,SAAQ,UAAU;IAC/C,KAAK,EAAE,uCAAuC,CAAe;gBAE1D,OAAO,GAAE,iBAAsB;IAKlC,KAAK,IAAI,IAAI;cAOG,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAe9D,8BAA8B,IAAI,IAAI;IAKtC,6BAA6B,IAAI,IAAI;IAKrC,+BAA+B,IAAI,IAAI;CAI1C;AAED,MAAM,WAAW,gCAAgC,CAAC,QAAQ,SAAS,UAAU,GAAG,UAAU,CACtF,SAAQ,yBAAyB,CAAC,QAAQ,CAAC;IAC3C;;OAEG;IACH,QAAQ,EAAE,oBAAoB,CAAC;IAE/B;;OAEG;IACH,IAAI,EAAE,IAAI,CAAC;IAEX;;;OAGG;IACH,aAAa,CAAC,CAAC,GAAG,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAElF;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErE;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACjF;AAED,UAAU,YACN,SAAQ,WAAW,CACf,IAAI,CAAC,gCAAgC,EAAE,IAAI,GAAG,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,KAAK,CAAC,GAAG;IAAE,IAAI,CAAC,EAAE,IAAI,CAAA;CAAE,EAC5G,qBAAqB,CACxB;CAAG;AAER,MAAM,WAAW,gCACb,SAAQ,IAAI,CAAC,wBAAwB,EAAE,gBAAgB,GAAG,oBAAoB,GAAG,qBAAqB,CAAC;IACvG;;;;;;;;;;OAUG;IACH,cAAc,CAAC,EAAE,CAAC,eAAe,EAAE,aAAa,CAAC,gCAAgC,CAAC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;IAEvG;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,YAAY,EAAE,CAAC;IAEpC;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,YAAY,EAAE,CAAC;IAErC;;;OAGG;IACH,2BAA2B,CAAC,EAAE,MAAM,CAAC;IAErC;;;;OAIG;IACH,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,oBAAoB,KAAK,OAAO,CAAC;IAE1D;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,oBAAoB,KAAK,OAAO,CAAC;IAE7F;;OAEG;IACH,sBAAsB,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,SAAS,GAAG,aAAa,CAAC,CAAC;IAEjF;;;OAGG;IACH,0BAA0B,CAAC,EAAE,OAAO,CAAC;CACxC;AAED,QAAA,MAAM,eAAe,mGASX,CAAC;AAEX,KAAK,YAAY,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;AAE7F;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,yBAA0B,SAAQ,iBAAiB;aAkBtC,MAAM;IAjB5B,OAAO,CAAC,sBAAsB,CAA0D;IACxF,OAAO,CAAC,sBAAsB,CAA0E;IACxG,OAAO,CAAC,aAAa,CAAiE;IACtF,OAAO,CAAC,gBAAgB,CAAoE;IAC5F,OAAO,CAAC,0BAA0B,CAAU;IAC5C,SAAiB,KAAK,EAAE,mCAAmC,CAAC;IAE5D;;;OAGG;IAEH,SAAkB,MAAM,EAAE,aAAa,CAAC,gCAAgC,CAAC,CACnB;gBAGlD,OAAO,GAAE,gCAAqC,EAC5B,MAAM,gBAAkC;cA4CrC,kBAAkB,CAAC,eAAe,EAAE,yBAAyB,GAAG,OAAO,CAAC,IAAI,CAAC;cAuEtF,YAAY,CACxB,eAAe,EAAE,yBAAyB,EAC1C,EAAE,KAAK,EAAE,oBAAoB,EAAE,EAAE,oBAAoB,GACtD,OAAO,CAAC,IAAI,CAAC;IAgBhB,SAAS,CAAC,kBAAkB,CAAC,CAAC,EAAE,KAAK,SAAS,GAAG,EAAE,EAC/C,IAAI,EAAE,CAAC,GAAG,IAAI,EAAE,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,GACrC,CAAC,GAAG,IAAI,EAAE,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC;cAQjB,0BAA0B,CACtC,eAAe,EAAE,yBAAyB,GAC3C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC;cA8FhF,8BAA8B,CAC1C,eAAe,EAAE,yBAAyB,EAC1C,YAAY,CAAC,EAAE,UAAU,GAC1B,OAAO,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAqGxC,OAAO,CAAC,cAAc;CAYzB;AAED,wBAAgB,8BAA8B,CAC1C,OAAO,SAAS,gCAAgC,GAAG,gCAAgC,EACnF,QAAQ,SAAS,UAAU,GAAG,sBAAsB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAC1E,MAAM,CAAC,EAAE,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,0BAEzC"}