@crawlee/playwright 4.0.0-beta.98 → 4.0.0-rc.0

package/internals/playwright-crawler.js

@@ -1,6 +1,6 @@
-import { BrowserCrawler, RequestState, Router, serviceLocator } from '@crawlee/browser';
-import ow from 'ow';
-import { PlaywrightLauncher } from './playwright-launcher.js';
+import { assertBrowserPoolNotConfigured, BrowserCrawler, parseArgument, RequestState, Router, schemas, serviceLocator, } from '@crawlee/browser';
+import { z } from 'zod';
+import { playwrightBrowserPool, remotePlaywrightBrowserPool } from './playwright-browser-pool.js';
 import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
 /**
  * Provides a simple framework for parallel crawling of web pages
@@ -70,46 +70,41 @@ import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
 export class PlaywrightCrawler extends BrowserCrawler {
     static optionsShape = {
         ...BrowserCrawler.optionsShape,
-        browserPoolOptions: ow.optional.object,
-        launcher: ow.optional.object,
-        ignoreIframes: ow.optional.boolean,
-        ignoreShadowRoots: ow.optional.boolean,
+        headless: z.boolean().optional(),
+        launcher: schemas.anyObject.optional(),
     };
+    static optionsSchema = z.strictObject(PlaywrightCrawler.optionsShape);
     /**
      * All `PlaywrightCrawler` parameters are passed via an options object.
      */
     constructor(options = {}) {
-        ow(options, 'PlaywrightCrawlerOptions', ow.object.exactShape(PlaywrightCrawler.optionsShape));
-        const { launchContext = {}, headless, contextPipelineBuilder, ...browserCrawlerOptions } = options;
-        const browserPoolOptions = {
-            ...options.browserPoolOptions,
-        };
+        const parsedOptions = parseArgument(options, PlaywrightCrawler.optionsSchema, 'PlaywrightCrawlerOptions');
+        const { launchContext, headless, configuration, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
         if (launchContext.proxyUrl) {
             throw new Error('PlaywrightCrawlerOptions.launchContext.proxyUrl is not allowed in PlaywrightCrawler.' +
                 'Use PlaywrightCrawlerOptions.proxyConfiguration');
         }
-        // `browserPlugins` is working when it's not overridden by `launchContext`,
-        // which for crawlers it is always overridden. Hence the error to use the other option.
-        if (browserPoolOptions.browserPlugins) {
-            throw new Error('browserPoolOptions.browserPlugins is disallowed. Use launchContext.launcher instead.');
-        }
-        if (headless != null) {
-            launchContext.launchOptions ??= {};
-            launchContext.launchOptions.headless = headless;
+        if (options.browserPool) {
+            // The raw options, not the parsed ones: `launchContext` has a default, so by now it is always set.
+            assertBrowserPoolNotConfigured(new.target.name, {
+                launchContext: options.launchContext,
+                headless: options.headless,
+            });
         }
-        const playwrightLauncher = new PlaywrightLauncher(launchContext, options.configuration);
-        browserPoolOptions.browserPlugins = [playwrightLauncher.createBrowserPlugin()];
         super({
             ...browserCrawlerOptions,
             launchContext,
-            browserPoolOptions,
+            configuration,
+            browserPoolBuilder: (remoteBrowser) => remoteBrowser
+                ? remotePlaywrightBrowserPool({ ...remoteBrowser, launchContext, headless, configuration })
+                : playwrightBrowserPool({ launchContext, headless, configuration }),
             contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
         });
     }
     buildContextPipeline() {
         return super.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) });
     }
-    async _navigationHandler(crawlingContext, gotoOptions) {
+    async navigationHandler(crawlingContext, gotoOptions) {
         return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
     }
     async enhanceContext(context) {

package/internals/playwright-launcher.d.ts

@@ -3,6 +3,7 @@ import { BrowserLauncher, Configuration } from '@crawlee/browser';
 import { PlaywrightPlugin } from '@crawlee/browser-pool';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Browser, BrowserType, LaunchOptions } from 'playwright';
+import { z } from 'zod';
 /**
  * Apify extends the launch options of Playwright.
  * You can use any of the Playwright compatible
@@ -73,26 +74,34 @@ export declare class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin
     readonly configuration: Configuration;
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
-        launcher: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        launchContextOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        launchContextOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        proxyUrl: z.ZodOptional<z.ZodURL>;
+        useChrome: z.ZodOptional<z.ZodBoolean>;
+        useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
+        browserPerProxy: z.ZodOptional<z.ZodBoolean>;
+        ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
+        userDataDir: z.ZodOptional<z.ZodString>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        proxyUrl: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        useChrome: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreProxyCertificate: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+        launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        userAgent: z.ZodOptional<z.ZodString>;
+    };
+    protected static optionsSchema: z.ZodObject<{
 // @ts-ignore optional peer dependency or compatibility with es2022
-        userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        launchOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        launchContextOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        proxyUrl: z.ZodOptional<z.ZodURL>;
+        useChrome: z.ZodOptional<z.ZodBoolean>;
+        useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
+        browserPerProxy: z.ZodOptional<z.ZodBoolean>;
+        ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
+        userDataDir: z.ZodOptional<z.ZodString>;
 // @ts-ignore optional peer dependency or compatibility with es2022
-        userAgent: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-    };
+        launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types").Dictionary, import("@crawlee/types").Dictionary>>;
+        userAgent: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
     /**
      * All `PlaywrightLauncher` parameters are passed via this launchContext object.
      */

package/internals/playwright-launcher.js

@@ -1,6 +1,6 @@
-import { BrowserLauncher, Configuration } from '@crawlee/browser';
+import { BrowserLauncher, Configuration, parseArgument, schemas } from '@crawlee/browser';
 import { PlaywrightPlugin } from '@crawlee/browser-pool';
-import ow from 'ow';
+import { z } from 'zod';
 /**
  * `PlaywrightLauncher` is based on the `BrowserLauncher`. It launches `playwright` browser instance.
  * @ignore
@@ -9,21 +9,23 @@ export class PlaywrightLauncher extends BrowserLauncher {
     configuration;
     static optionsShape = {
         ...BrowserLauncher.optionsShape,
-        launcher: ow.optional.object,
-        launchContextOptions: ow.optional.object,
+        // Passthrough schemas — the launcher module object must keep its prototype through parsing.
+        launcher: schemas.anyObject.optional(),
+        launchContextOptions: schemas.anyObject.optional(),
     };
+    static optionsSchema = z.strictObject(PlaywrightLauncher.optionsShape);
     /**
      * All `PlaywrightLauncher` parameters are passed via this launchContext object.
      */
     constructor(launchContext = {}, configuration = Configuration.getGlobalConfiguration()) {
-        ow(launchContext, 'PlaywrightLauncherOptions', ow.object.exactShape(PlaywrightLauncher.optionsShape));
-        const { launcher = BrowserLauncher.requireLauncherOrThrow('playwright', 'apify/actor-node-playwright-*').chromium, } = launchContext;
-        const { launchOptions = {}, ...rest } = launchContext;
+        const parsedContext = parseArgument(launchContext, PlaywrightLauncher.optionsSchema, 'PlaywrightLaunchContext');
+        const { launcher = BrowserLauncher.requireLauncherOrThrow('playwright', 'apify/actor-node-playwright-*').chromium, } = parsedContext;
+        const { launchOptions = {}, ...rest } = parsedContext;
         super({
             ...rest,
             launchOptions: {
                 ...launchOptions,
-                executablePath: getDefaultExecutablePath(launchContext, configuration),
+                executablePath: getDefaultExecutablePath(parsedContext, configuration),
             },
             launcher,
         }, configuration);

package/internals/utils/playwright-utils.d.ts

@@ -19,7 +19,7 @@
  */
 import { Configuration, type Request } from '@crawlee/browser';
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type CheerioRoot } from '@crawlee/utils';
+import { type CheerioRoot } from '@crawlee/utils/internal';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Download, Page, Response } from 'playwright';
 import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
@@ -336,7 +336,6 @@ export interface HandleCloudflareChallengeOptions {
  * @param [options]
  */
 declare function handleCloudflareChallenge(page: Page, url: string, options?: HandleCloudflareChallengeOptions): Promise<Response | undefined>;
-/** @internal */
 export interface PlaywrightContextUtils {
     /**
      * Injects a JavaScript file into current `page`.
@@ -455,8 +454,7 @@ export interface PlaywrightContextUtils {
      * in `href` elements, but rather navigations are triggered in click handlers.
      * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
      *
-     * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
-     * and override settings of the enqueued {@link Request} objects.
+     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
      *
      * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
      * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -477,9 +475,9 @@ export interface PlaywrightContextUtils {
      * async requestHandler({ enqueueLinksByClickingElements }) {
      *     await enqueueLinksByClickingElements({
      *         selector: 'a.product-detail',
-     *         globs: [
-     *             'https://www.example.com/handbags/**'
-     *             'https://www.example.com/purses/**'
+     *         include: [
+     *             'https://www.example.com/handbags/**',
+     *             'https://www.example.com/purses/**',
      *         ],
      *     });
      * });

package/internals/utils/playwright-utils.js

@@ -20,10 +20,9 @@
 import { readFile } from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import vm from 'node:vm';
-import { Configuration, KeyValueStore, serviceLocator, SessionError, validators } from '@crawlee/browser';
+import { Configuration, KeyValueStore, parseArgument, schemas, serviceLocator, SessionError, validators, } from '@crawlee/browser';
 import { expandShadowRoots, sleep } from '@crawlee/utils';
-import * as cheerio from 'cheerio';
-import ow from 'ow';
+import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
 import { RenderingTypePredictor } from './rendering-type-prediction.js';
@@ -32,6 +31,36 @@ const require = createRequire(import.meta.url);
 const jqueryPath = require.resolve('jquery');
 const MAX_INJECT_FILE_CACHE_SIZE = 10;
 const DEFAULT_BLOCK_REQUEST_URL_PATTERNS = ['.css', '.jpg', '.jpeg', '.png', '.svg', '.gif', '.woff', '.pdf', '.zip'];
+const filePathSchema = z.string();
+const injectFileOptionsSchema = z.strictObject({
+    surviveNavigations: z.boolean().optional(),
+});
+const gotoExtendedRequestSchema = z.looseObject({
+    url: z.url(),
+    method: z.string().optional(),
+    headers: schemas.anyObject.optional(),
+    payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
+});
+const blockRequestsOptionsSchema = z.strictObject({
+    urlPatterns: schemas.arrayOf(z.string(), 'strings').default(DEFAULT_BLOCK_REQUEST_URL_PATTERNS),
+    extraUrlPatterns: schemas.arrayOf(z.string(), 'strings').default(() => []),
+});
+const infiniteScrollOptionsSchema = z.strictObject({
+    timeoutSecs: schemas.anyNumber.default(0),
+    maxScrollHeight: schemas.anyNumber.default(0),
+    waitForSecs: schemas.anyNumber.default(4),
+    scrollDownAndUp: z.boolean().default(false),
+    buttonSelector: z.string().optional(),
+    stopScrollCallback: schemas.anyFunction.optional(),
+});
+const saveSnapshotOptionsSchema = z.strictObject({
+    key: z.string().min(1).default('SNAPSHOT'),
+    screenshotQuality: schemas.anyNumber.default(50),
+    saveScreenshot: z.boolean().default(true),
+    saveHtml: z.boolean().default(true),
+    keyValueStoreName: z.string().optional(),
+    configuration: schemas.anyObject.optional(),
+});
 /**
  * Cache contents of previously injected files to limit file system access.
  */
@@ -48,18 +77,16 @@ const injectedFilesCache = new LruCache({ maxLength: MAX_INJECT_FILE_CACHE_SIZE
  * @param [options]
  */
 export async function injectFile(page, filePath, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(filePath, ow.string);
-    ow(options, ow.object.exactShape({
-        surviveNavigations: ow.optional.boolean,
-    }));
+    parseArgument(page, validators.browserPage);
+    parseArgument(filePath, filePathSchema);
+    const { surviveNavigations } = parseArgument(options, injectFileOptionsSchema);
     let contents = injectedFilesCache.get(filePath);
     if (!contents) {
         contents = await readFile(filePath, 'utf8');
         injectedFilesCache.add(filePath, contents);
     }
     const evalP = page.evaluate(contents);
-    if (options.surviveNavigations) {
+    if (surviveNavigations) {
         page.on('framenavigated', async () => page
             .evaluate(contents)
             .catch((error) => getLog().warning('An error occurred during the script injection!', { error })));
@@ -93,7 +120,7 @@ export async function injectFile(page, filePath, options = {}) {
  * @param [options.surviveNavigations] Opt-out option to disable the JQuery reinjection after navigation.
  */
 export async function injectJQuery(page, options) {
-    ow(page, ow.object.validate(validators.browserPage));
+    parseArgument(page, validators.browserPage);
     return injectFile(page, jqueryPath, { surviveNavigations: options?.surviveNavigations ?? true });
 }
 /**
@@ -109,14 +136,9 @@ export async function injectJQuery(page, options) {
  * @param [gotoOptions] Custom options for `page.goto()`.
  */
 export async function gotoExtended(page, request, gotoOptions = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(request, ow.object.partialShape({
-        url: ow.string.url,
-        method: ow.optional.string,
-        headers: ow.optional.object,
-        payload: ow.optional.any(ow.string, ow.uint8Array),
-    }));
-    ow(gotoOptions, ow.object);
+    parseArgument(page, validators.browserPage);
+    parseArgument(request, gotoExtendedRequestSchema);
+    parseArgument(gotoOptions, schemas.anyObject);
     const { url, method, headers, payload } = request;
     const isEmpty = (o) => !o || Object.keys(o).length === 0;
     if (method !== 'GET' || payload) {
@@ -201,12 +223,8 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
  * @param [options]
  */
 export async function blockRequests(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        urlPatterns: ow.optional.array.ofType(ow.string),
-        extraUrlPatterns: ow.optional.array.ofType(ow.string),
-    }));
-    const { urlPatterns = DEFAULT_BLOCK_REQUEST_URL_PATTERNS, extraUrlPatterns = [] } = options;
+    parseArgument(page, validators.browserPage);
+    const { urlPatterns, extraUrlPatterns } = parseArgument(options, blockRequestsOptionsSchema);
     const patternsToBlock = [...urlPatterns, ...extraUrlPatterns];
     try {
         const client = await page.context().newCDPSession(page);
@@ -264,16 +282,8 @@ export function compileScript(scriptString, context = Object.create(null)) {
  * @param [options]
  */
 export async function infiniteScroll(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        timeoutSecs: ow.optional.number,
-        maxScrollHeight: ow.optional.number,
-        waitForSecs: ow.optional.number,
-        scrollDownAndUp: ow.optional.boolean,
-        buttonSelector: ow.optional.string,
-        stopScrollCallback: ow.optional.function,
-    }));
-    const { timeoutSecs = 0, maxScrollHeight = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback, } = options;
+    parseArgument(page, validators.browserPage);
+    const { timeoutSecs, maxScrollHeight, waitForSecs, scrollDownAndUp, buttonSelector, stopScrollCallback } = parseArgument(options, infiniteScrollOptionsSchema);
     let finished;
     const startTime = Date.now();
     const CHECK_INTERVAL_MILLIS = 1000;
@@ -350,16 +360,8 @@ export async function infiniteScroll(page, options = {}) {
  * @param [options]
  */
 export async function saveSnapshot(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        key: ow.optional.string.nonEmpty,
-        screenshotQuality: ow.optional.number,
-        saveScreenshot: ow.optional.boolean,
-        saveHtml: ow.optional.boolean,
-        keyValueStoreName: ow.optional.string,
-        configuration: ow.optional.object,
-    }));
-    const { key = 'SNAPSHOT', screenshotQuality = 50, saveScreenshot = true, saveHtml = true, keyValueStoreName, configuration, } = options;
+    parseArgument(page, validators.browserPage);
+    const { key, screenshotQuality, saveScreenshot, saveHtml, keyValueStoreName, configuration } = parseArgument(options, saveSnapshotOptionsSchema);
     try {
         const store = await KeyValueStore.open(keyValueStoreName ? { name: keyValueStoreName } : null, {
             configuration: configuration ?? Configuration.getGlobalConfiguration(),
@@ -397,12 +399,13 @@ export async function saveSnapshot(page, options = {}) {
  * @param ignoreShadowRoots
  */
 export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIframes = false) {
-    ow(page, ow.object.validate(validators.browserPage));
+    parseArgument(page, validators.browserPage);
     const html = ignoreShadowRoots
         ? null
         : (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
     const pageContent = html || (await page.content());
-    const $ = cheerio.load(pageContent);
+    const { load } = await import('cheerio');
+    const $ = load(pageContent);
     if (page.frames().length > 1 && !ignoreIframes) {
         const frames = await page.$$('iframe');
         const cheerioIframes = $('iframe').toArray();
@@ -457,7 +460,7 @@ ${error.message}
     return idcacPlaywright;
 }
 export async function closeCookieModals(page) {
-    ow(page, ow.object.validate(validators.browserPage));
+    parseArgument(page, validators.browserPage);
     const idcac = await getIdcacPlaywright();
     if (idcac?.getInjectableScript()) {
         await page.evaluate(idcac.getInjectableScript());
@@ -496,9 +499,9 @@ async function handleCloudflareChallenge(page, url, options = {}) {
     };
     options.isChallengeCallback ??= async () => {
         return await page.evaluate(async () => {
-            // Cloudflare nests the ray ID under varying wrapper elements, so we match by descendants
-            // instead of a direct-child chain (e.g. a `.footer-wrapper` was inserted in between).
-            return !!document.querySelector('.footer .footer-inner .diagnostic-wrapper .ray-id');
+            // Cloudflare keeps reshuffling the wrapper elements between `.footer-inner` and `.ray-id`,
+            // so only the stable outer classes are matched.
+            return !!document.querySelector('.footer .footer-inner .ray-id');
         });
     };
     const retryBlocked = async () => {

package/internals/utils/rendering-type-prediction.d.ts

@@ -26,7 +26,7 @@ export interface IRenderingTypePredictor {
  * @experimental
  */
 export declare class RenderingTypePredictor implements IRenderingTypePredictor {
-    private detectionRatio;
+    #private;
     private state;
     constructor({ detectionRatio, persistenceOptions }: RenderingTypePredictorOptions);
     /**

package/internals/utils/rendering-type-prediction.js

@@ -2,6 +2,7 @@ import { RecoverableState } from '@crawlee/core';
 import LogisticRegression from 'ml-logistic-regression';
 import { Matrix } from 'ml-matrix';
 import stringComparison from 'string-comparison';
+import { z } from 'zod';
 const urlComponents = (url) => {
     return [url.hostname, ...url.pathname.split('/')];
 };
@@ -24,41 +25,57 @@ const calculateUrlSimilarity = (a, b) => {
 };
 const sum = (values) => values.reduce((acc, value) => acc + value);
 const mean = (values) => (values.length > 0 ? sum(values) / values.length : undefined);
+const renderingType = z.enum(['clientOnly', 'static']);
+const predictorState = z.object({
+    logreg: z.instanceof(LogisticRegression),
+    detectionResults: z.map(renderingType, z.map(z.string().optional(), z.array(z.array(z.string())))),
+});
+const persistedState = z.object({
+    logreg: z
+        .record(z.string(), z.unknown())
+        .prefault(() => new LogisticRegression({ numSteps: 1000, learningRate: 0.05 }).toJSON()),
+    detectionResults: z
+        .array(z.object({
+        renderingType,
+        urlPartsByLabel: z.array(z.object({
+            label: z.string().optional(),
+            urlParts: z.array(z.array(z.string())),
+        })),
+    }))
+        .prefault([]),
+});
+const stateCodec = z.codec(persistedState, predictorState, {
+    decode: ({ logreg, detectionResults }) => ({
+        logreg: LogisticRegression.load(logreg),
+        detectionResults: new Map(detectionResults.map(({ renderingType, urlPartsByLabel }) => [
+            renderingType,
+            new Map(urlPartsByLabel.map(({ label, urlParts }) => [label, urlParts])),
+        ])),
+    }),
+    encode: ({ logreg, detectionResults }) => ({
+        logreg: logreg.toJSON(),
+        detectionResults: Array.from(detectionResults.entries()).map(([renderingType, urlPartsByLabel]) => ({
+            renderingType,
+            urlPartsByLabel: Array.from(urlPartsByLabel.entries()).map(([label, urlParts]) => ({ label, urlParts })),
+        })),
+    }),
+});
 /**
  * Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.
  *
  * @experimental
  */
 export class RenderingTypePredictor {
-    detectionRatio;
+    #detectionRatio;
+    // kept as TS-private: tests reach for it at runtime
     state;
     constructor({ detectionRatio, persistenceOptions }) {
-        this.detectionRatio = detectionRatio;
+        this.#detectionRatio = detectionRatio;
         this.state = new RecoverableState({
-            defaultState: {
-                logreg: new LogisticRegression({ numSteps: 1000, learningRate: 0.05 }),
-                detectionResults: new Map(),
-            },
-            serialize: (state) => JSON.stringify({
-                logreg: state.logreg.toJSON(),
-                detectionResults: Array.from(state.detectionResults.entries()).map(([renderingType, urlPartsByLabel]) => ({
-                    renderingType,
-                    urlPartsByLabel: Array.from(urlPartsByLabel.entries()).map(([label, urlParts]) => ({
-                        label,
-                        urlParts,
-                    })),
-                })),
-            }),
-            deserialize: (serializedState) => {
-                const { logreg, detectionResults = [] } = JSON.parse(serializedState);
-                return {
-                    logreg: LogisticRegression.load(logreg),
-                    detectionResults: new Map(detectionResults.map((serializedItem) => [
-                        serializedItem.renderingType,
-                        new Map(serializedItem.urlPartsByLabel.map((item) => [item.label, item.urlParts])),
-                    ])),
-                };
-            },
+            defaultState: () => stateCodec.decode({}),
+            // The codec validates in the decode direction, so it is a Standard Schema as-is; encoding needs a call.
+            deserialize: stateCodec,
+            serialize: (state) => stateCodec.encode(state),
             persistStateKey: 'rendering-type-predictor-state',
             persistenceEnabled: true,
             ...persistenceOptions,
@@ -86,7 +103,7 @@ export class RenderingTypePredictor {
             renderingType: prediction === 1 ? 'static' : 'clientOnly',
             detectionProbabilityRecommendation: Math.abs(scores[0] - scores[1]) < 0.1
                 ? 1
-                : this.detectionRatio * Math.max(1, 5 - this.resultCount(label)),
+                : this.#detectionRatio * Math.max(1, 5 - this.resultCount(label)),
         };
     }
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.98",
+    "version": "4.0.0-rc.0",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,22 +49,21 @@
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
         "@apify/timeout": "^0.4.4",
-        "@crawlee/basic": "4.0.0-beta.98",
-        "@crawlee/browser": "4.0.0-beta.98",
-        "@crawlee/browser-pool": "4.0.0-beta.98",
-        "@crawlee/cheerio": "4.0.0-beta.98",
-        "@crawlee/core": "4.0.0-beta.98",
-        "@crawlee/types": "4.0.0-beta.98",
-        "@crawlee/utils": "4.0.0-beta.98",
+        "@crawlee/basic": "4.0.0-rc.0",
+        "@crawlee/browser": "4.0.0-rc.0",
+        "@crawlee/browser-pool": "4.0.0-rc.0",
+        "@crawlee/cheerio": "4.0.0-rc.0",
+        "@crawlee/core": "4.0.0-rc.0",
+        "@crawlee/types": "4.0.0-rc.0",
+        "@crawlee/utils": "4.0.0-rc.0",
         "cheerio": "^1.0.0",
-        "idcac-playwright": "^0.1.3",
         "jquery": "^3.7.1",
         "ml-logistic-regression": "^2.0.0",
         "ml-matrix": "^6.12.1",
-        "ow": "^2.0.0",
         "string-comparison": "^1.3.0",
         "tslib": "^2.8.1",
-        "type-fest": "^4.0.0"
+        "type-fest": "^4.0.0",
+        "zod": "^4.4.3"
     },
     "peerDependencies": {
         "idcac-playwright": "^0.2.0",
@@ -85,5 +84,5 @@
             }
         }
     },
-    "gitHead": "3b8cd86b13e253ab5fc71e631e12a68f7465cee5"
+    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
 }