@crawlee/playwright 4.0.0-beta.19 → 4.0.0-beta.191

package/internals/utils/playwright-utils.js

@@ -20,19 +20,48 @@
 import { readFile } from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import vm from 'node:vm';
-import { Configuration, KeyValueStore, SessionError, validators } from '@crawlee/browser';
+import { Configuration, KeyValueStore, serviceLocator, SessionError, validators } from '@crawlee/browser';
 import { expandShadowRoots, sleep } from '@crawlee/utils';
-import * as cheerio from 'cheerio';
-import ow from 'ow';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
-import log_ from '@apify/log';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
 import { RenderingTypePredictor } from './rendering-type-prediction.js';
-const log = log_.child({ prefix: 'Playwright Utils' });
+const getLog = () => serviceLocator.getChildLog('Playwright Utils');
 const require = createRequire(import.meta.url);
 const jqueryPath = require.resolve('jquery');
 const MAX_INJECT_FILE_CACHE_SIZE = 10;
 const DEFAULT_BLOCK_REQUEST_URL_PATTERNS = ['.css', '.jpg', '.jpeg', '.png', '.svg', '.gif', '.woff', '.pdf', '.zip'];
+const filePathSchema = z.string();
+const injectFileOptionsSchema = z.strictObject({
+    surviveNavigations: z.boolean().optional(),
+});
+const gotoExtendedRequestSchema = z.looseObject({
+    url: z.url(),
+    method: z.string().optional(),
+    headers: schemas.anyObject.optional(),
+    payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
+});
+const blockRequestsOptionsSchema = z.strictObject({
+    urlPatterns: schemas.arrayOf(z.string(), 'strings').default(DEFAULT_BLOCK_REQUEST_URL_PATTERNS),
+    extraUrlPatterns: schemas.arrayOf(z.string(), 'strings').default(() => []),
+});
+const infiniteScrollOptionsSchema = z.strictObject({
+    timeoutSecs: schemas.anyNumber.default(0),
+    maxScrollHeight: schemas.anyNumber.default(0),
+    waitForSecs: schemas.anyNumber.default(4),
+    scrollDownAndUp: z.boolean().default(false),
+    buttonSelector: z.string().optional(),
+    stopScrollCallback: schemas.anyFunction.optional(),
+});
+const saveSnapshotOptionsSchema = z.strictObject({
+    key: z.string().min(1).default('SNAPSHOT'),
+    screenshotQuality: schemas.anyNumber.default(50),
+    saveScreenshot: z.boolean().default(true),
+    saveHtml: z.boolean().default(true),
+    keyValueStoreName: z.string().optional(),
+    configuration: schemas.anyObject.optional(),
+});
 /**
  * Cache contents of previously injected files to limit file system access.
  */
@@ -49,21 +78,19 @@ const injectedFilesCache = new LruCache({ maxLength: MAX_INJECT_FILE_CACHE_SIZE
  * @param [options]
  */
 export async function injectFile(page, filePath, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(filePath, ow.string);
-    ow(options, ow.object.exactShape({
-        surviveNavigations: ow.optional.boolean,
-    }));
+    parseArgument(page, validators.browserPage);
+    parseArgument(filePath, filePathSchema);
+    const { surviveNavigations } = parseArgument(options, injectFileOptionsSchema);
     let contents = injectedFilesCache.get(filePath);
     if (!contents) {
         contents = await readFile(filePath, 'utf8');
         injectedFilesCache.add(filePath, contents);
     }
     const evalP = page.evaluate(contents);
-    if (options.surviveNavigations) {
+    if (surviveNavigations) {
         page.on('framenavigated', async () => page
             .evaluate(contents)
-            .catch((error) => log.warning('An error occurred during the script injection!', { error })));
+            .catch((error) => getLog().warning('An error occurred during the script injection!', { error })));
     }
     return evalP;
 }
@@ -94,7 +121,7 @@ export async function injectFile(page, filePath, options = {}) {
  * @param [options.surviveNavigations] Opt-out option to disable the JQuery reinjection after navigation.
  */
 export async function injectJQuery(page, options) {
-    ow(page, ow.object.validate(validators.browserPage));
+    parseArgument(page, validators.browserPage);
     return injectFile(page, jqueryPath, { surviveNavigations: options?.surviveNavigations ?? true });
 }
 /**
@@ -110,19 +137,14 @@ export async function injectJQuery(page, options) {
  * @param [gotoOptions] Custom options for `page.goto()`.
  */
 export async function gotoExtended(page, request, gotoOptions = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(request, ow.object.partialShape({
-        url: ow.string.url,
-        method: ow.optional.string,
-        headers: ow.optional.object,
-        payload: ow.optional.any(ow.string, ow.uint8Array),
-    }));
-    ow(gotoOptions, ow.object);
+    parseArgument(page, validators.browserPage);
+    parseArgument(request, gotoExtendedRequestSchema);
+    parseArgument(gotoOptions, schemas.anyObject);
     const { url, method, headers, payload } = request;
     const isEmpty = (o) => !o || Object.keys(o).length === 0;
-    if (method !== 'GET' || payload || !isEmpty(headers)) {
+    if (method !== 'GET' || payload) {
         // This is not deprecated, we use it to log only once.
-        log.deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
+        getLog().deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance ' +
             'in recent versions of Playwright. Use only when necessary.');
         let wasCalled = false;
         const interceptRequestHandler = async (route) => {
@@ -143,12 +165,15 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
                 await route.continue(overrides);
             }
             catch (error) {
-                log.debug('Error inside request interceptor', { error });
+                getLog().debug('Error inside request interceptor', { error });
             }
             return undefined;
         };
         await page.route('**/*', interceptRequestHandler);
     }
+    else if (!isEmpty(headers)) {
+        await page.setExtraHTTPHeaders(headers);
+    }
     return page.goto(url, gotoOptions);
 }
 /**
@@ -199,12 +224,8 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
  * @param [options]
  */
 export async function blockRequests(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        urlPatterns: ow.optional.array.ofType(ow.string),
-        extraUrlPatterns: ow.optional.array.ofType(ow.string),
-    }));
-    const { urlPatterns = DEFAULT_BLOCK_REQUEST_URL_PATTERNS, extraUrlPatterns = [] } = options;
+    parseArgument(page, validators.browserPage);
+    const { urlPatterns, extraUrlPatterns } = parseArgument(options, blockRequestsOptionsSchema);
     const patternsToBlock = [...urlPatterns, ...extraUrlPatterns];
     try {
         const client = await page.context().newCDPSession(page);
@@ -212,7 +233,7 @@ export async function blockRequests(page, options = {}) {
         await client.send('Network.setBlockedURLs', { urls: patternsToBlock });
     }
     catch {
-        log.warning('blockRequests() helper is incompatible with non-Chromium browsers.');
+        getLog().warning('blockRequests() helper is incompatible with non-Chromium browsers.');
     }
 }
 /**
@@ -248,7 +269,7 @@ export function compileScript(scriptString, context = Object.create(null)) {
         func = vm.runInNewContext(funcString, context); // "Secure" the context by removing prototypes, unless custom context is provided.
     }
     catch (err) {
-        log.exception(err, 'Cannot compile script!');
+        getLog().exception(err, 'Cannot compile script!');
         throw err;
     }
     if (typeof func !== 'function')
@@ -262,16 +283,8 @@ export function compileScript(scriptString, context = Object.create(null)) {
  * @param [options]
  */
 export async function infiniteScroll(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        timeoutSecs: ow.optional.number,
-        maxScrollHeight: ow.optional.number,
-        waitForSecs: ow.optional.number,
-        scrollDownAndUp: ow.optional.boolean,
-        buttonSelector: ow.optional.string,
-        stopScrollCallback: ow.optional.function,
-    }));
-    const { timeoutSecs = 0, maxScrollHeight = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback, } = options;
+    parseArgument(page, validators.browserPage);
+    const { timeoutSecs, maxScrollHeight, waitForSecs, scrollDownAndUp, buttonSelector, stopScrollCallback } = parseArgument(options, infiniteScrollOptionsSchema);
     let finished;
     const startTime = Date.now();
     const CHECK_INTERVAL_MILLIS = 1000;
@@ -348,19 +361,11 @@ export async function infiniteScroll(page, options = {}) {
  * @param [options]
  */
 export async function saveSnapshot(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        key: ow.optional.string.nonEmpty,
-        screenshotQuality: ow.optional.number,
-        saveScreenshot: ow.optional.boolean,
-        saveHtml: ow.optional.boolean,
-        keyValueStoreName: ow.optional.string,
-        config: ow.optional.object,
-    }));
-    const { key = 'SNAPSHOT', screenshotQuality = 50, saveScreenshot = true, saveHtml = true, keyValueStoreName, config, } = options;
+    parseArgument(page, validators.browserPage);
+    const { key, screenshotQuality, saveScreenshot, saveHtml, keyValueStoreName, configuration } = parseArgument(options, saveSnapshotOptionsSchema);
     try {
-        const store = await KeyValueStore.open(keyValueStoreName, {
-            config: config ?? Configuration.getGlobalConfig(),
+        const store = await KeyValueStore.open(keyValueStoreName ? { name: keyValueStoreName } : null, {
+            configuration: configuration ?? Configuration.getGlobalConfiguration(),
         });
         if (saveScreenshot) {
             const screenshotName = `${key}.jpg`;
@@ -395,13 +400,26 @@ export async function saveSnapshot(page, options = {}) {
  * @param ignoreShadowRoots
  */
 export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIframes = false) {
-    ow(page, ow.object.validate(validators.browserPage));
+    parseArgument(page, validators.browserPage);
+    const html = ignoreShadowRoots
+        ? null
+        : (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
+    const pageContent = html || (await page.content());
+    // Full cheerio (parse5) import for browser compliant parser
+    const { load } = await import('cheerio');
+    const $ = load(pageContent);
     if (page.frames().length > 1 && !ignoreIframes) {
         const frames = await page.$$('iframe');
-        await Promise.all(frames.map(async (frame) => {
+        const cheerioIframes = $('iframe').toArray();
+        if (frames.length !== cheerioIframes.length) {
+            serviceLocator
+                .getLogger()
+                .warning(`parseWithCheerio: iframe count mismatch between live DOM (${frames.length}) and page snapshot (${cheerioIframes.length}). Some iframes may not be expanded.`);
+        }
+        await Promise.all(frames.map(async (frame, index) => {
             try {
                 const iframe = await frame.contentFrame();
-                if (iframe) {
+                if (iframe && cheerioIframes[index]) {
                     const getIframeHTML = async () => {
                         try {
                             return iframe.locator('body').first().innerHTML();
@@ -411,53 +429,15 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
                         }
                     };
                     const contents = await getIframeHTML();
-                    await frame.evaluate((f, c) => {
-                        const replacementNode = document.createElement('div');
-                        replacementNode.innerHTML = c;
-                        replacementNode.className = 'crawlee-iframe-replacement';
-                        f.replaceWith(replacementNode);
-                    }, contents);
+                    $(cheerioIframes[index]).replaceWith(`<div class="crawlee-iframe-replacement">${contents}</div>`);
                 }
             }
             catch (error) {
-                log.warning(`Failed to extract iframe content: ${error}`);
+                getLog().warning(`Failed to extract iframe content: ${error}`);
             }
         }));
     }
-    const html = ignoreShadowRoots
-        ? null
-        : (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
-    const pageContent = html || (await page.content());
-    return cheerio.load(pageContent);
-}
-let idcacPlaywright = null;
-async function getIdcacPlaywright() {
-    if (idcacPlaywright)
-        return idcacPlaywright;
-    try {
-        idcacPlaywright = await import('idcac-playwright');
-    }
-    catch (error) {
-        log.warning(`Failed to import 'idcac-playwright'.
-
-We recently made idcac-playwright an optional dependency due to licensing issues.
-To use this feature, please install it manually by running
-
-npm install idcac-playwright
-
-Original error message follows:
-
-${error.message}
-`);
-    }
-    return idcacPlaywright;
-}
-export async function closeCookieModals(page) {
-    ow(page, ow.object.validate(validators.browserPage));
-    const idcac = await getIdcacPlaywright();
-    if (idcac?.getInjectableScript()) {
-        await page.evaluate(idcac.getInjectableScript());
-    }
+    return $;
 }
 /**
  * This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
@@ -466,30 +446,24 @@ export async function closeCookieModals(page) {
  * result in a SessionError which will be automatically retried, so only successful requests will get
  * into the `requestHandler`.
  *
+ * On a successfully solved challenge the page is reloaded and the new {@link Response} is returned, so
+ * it can be propagated back to the crawling context via a hook return value (see
+ * {@link handleCloudflareChallengeHook}).
+ *
  * Works best with camoufox.
  *
  * **Example usage**
  * ```ts
  * postNavigationHooks: [
- *     async ({ handleCloudflareChallenge }) => {
- *         await handleCloudflareChallenge();
- *     },
+ *     async (context) => ({ response: await context.handleCloudflareChallenge() }),
  * ],
  * ```
  *
  * @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object
  * @param url current URL for request identification, only used for logging
- * @param [session] current session object
  * @param [options]
  */
-async function handleCloudflareChallenge(page, url, session, options = {}) {
-    // eslint-disable-next-line dot-notation
-    const blockedStatusCodes = session?.['sessionPool']['blockedStatusCodes'];
-    // Cloudflare pages are 403, which are blocked by default
-    if (blockedStatusCodes?.includes(403)) {
-        const idx = blockedStatusCodes.indexOf(403);
-        blockedStatusCodes.splice(idx, 1);
-    }
+async function handleCloudflareChallenge(page, url, options = {}) {
     options.isBlockedCallback ??= async () => {
         const isBlocked = await page.evaluate(() => {
             return document.querySelector('h1')?.textContent?.trim().includes('Sorry, you have been blocked');
@@ -498,7 +472,9 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
     };
     options.isChallengeCallback ??= async () => {
         return await page.evaluate(async () => {
-            return !!document.querySelector('.footer > .footer-inner > .diagnostic-wrapper > .ray-id');
+            // Cloudflare keeps reshuffling the wrapper elements between `.footer-inner` and `.ray-id`,
+            // so only the stable outer classes are matched.
+            return !!document.querySelector('.footer .footer-inner .ray-id');
         });
     };
     const retryBlocked = async () => {
@@ -513,31 +489,41 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
     };
     if (!(await isChallenge())) {
         await retryBlocked();
-        return;
+        return undefined;
     }
     const logLevel = options.verbose ? 'info' : 'debug';
-    log[logLevel](`Detected Cloudflare challenge at ${url}, trying to solve it. This can take up to ${10 + (options.sleepSecs ?? 10)} seconds.`);
+    getLog()[logLevel](`Detected Cloudflare challenge at ${url}, trying to solve it. This can take up to ${10 + (options.sleepSecs ?? 10)} seconds.`);
     const bb = await page
         .evaluate(() => {
-        const div = document.querySelector('.main-content div');
+        // Prefer the actual challenge widget (the box holding the Turnstile checkbox input);
+        // fall back to the first content div for older challenge layouts.
+        const div = document.querySelector('.main-content div:has(input[id^="cf-chl-widget-"])') ??
+            document.querySelector('.main-content div');
         return div?.getBoundingClientRect();
     })
         .catch(() => undefined);
     if (!bb) {
-        return;
+        return undefined;
     }
     const randomOffset = (range) => {
         return Math.round(100 * range * Math.random()) / 100;
     };
-    const x = bb.x + 30;
-    const y = bb.y + 25;
+    let x = bb.x + 30;
+    let y = bb.y + 25;
     // try to click the checkbox every second
     for (let i = 0; i < 10; i++) {
-        await sleep(1000);
+        await sleep((options.preChallengeSleepSecs ?? 1) * 1000);
         // break early if we are no longer on the CF challenge page
         if (!(await isChallenge())) {
             break;
         }
+        if (options.clickPositionCallback) {
+            const pos = await options.clickPositionCallback(page);
+            if (pos) {
+                x = pos.x;
+                y = pos.y;
+            }
+        }
         if (options.clickCallback) {
             await options.clickCallback(page, { x, y });
             continue;
@@ -545,7 +531,10 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
         // we can click on the text too, so X can be a bit larger
         const xRandomized = x + randomOffset(10);
         const yRandomized = y + randomOffset(10);
-        log[logLevel](`Trying to click on the Cloudflare checkbox at ${url}`, { x: xRandomized, y: yRandomized });
+        getLog()[logLevel](`Trying to click on the Cloudflare checkbox at ${url}`, {
+            x: xRandomized,
+            y: yRandomized,
+        });
         await page.mouse.click(xRandomized, yRandomized);
         // sometimes the checkbox is lower (could be caused by a lag when rendering the logo)
         await page.mouse.click(xRandomized, yRandomized + 35);
@@ -555,6 +544,9 @@ async function handleCloudflareChallenge(page, url, session, options = {}) {
         throw new SessionError(`Blocked by Cloudflare when processing ${url}`);
     }
     await retryBlocked();
+    // Reload to obtain a fresh Response without the challenge interstitial, which the caller can
+    // propagate back into the crawling context so downstream status-code checks see the new value.
+    return (await page.reload()) ?? undefined;
 }
 export { enqueueLinksByClickingElements };
 /** @internal */
@@ -568,8 +560,6 @@ export const playwrightUtils = {
     infiniteScroll,
     saveSnapshot,
     compileScript,
-    closeCookieModals,
     RenderingTypePredictor,
     handleCloudflareChallenge,
 };
-//# sourceMappingURL=playwright-utils.js.map

package/internals/utils/rendering-type-prediction.d.ts

@@ -1,26 +1,54 @@
 import type { RecoverableStatePersistenceOptions, Request } from '@crawlee/core';
+import type { Awaitable } from '@crawlee/types';
 export type RenderingType = 'clientOnly' | 'static';
-type URLComponents = string[];
-type FeatureVector = [staticResultsSimilarity: number, clientOnlyResultsSimilarity: number];
 export interface RenderingTypePredictorOptions {
     /** A number between 0 and 1 that determines the desired ratio of rendering type detections */
     detectionRatio: number;
     persistenceOptions?: Partial<RecoverableStatePersistenceOptions>;
 }
+/**
+ * Minimal contract that any object passed to {@link AdaptivePlaywrightCrawler} as its
+ * `renderingTypePredictor` option must satisfy.
+ *
+ * @experimental
+ */
+export interface IRenderingTypePredictor {
+    /**
+     * Predict the rendering type for a request, and how likely the crawler should be to verify it.
+     *
+     * Called once per request, before navigation - the crawler awaits it, so prefer loading whatever the
+     * prediction needs up front over per-request I/O.
+     */
+    predict(request: Request): Awaitable<{
+        renderingType: RenderingType;
+        detectionProbabilityRecommendation: number;
+    }>;
+    /**
+     * Report a detected rendering type, so that future predictions can take it into account.
+     *
+     * The crawler does not await this per detection - it collects the returned promises and drains them in
+     * `teardown()`, so an implementation that persists results asynchronously can keep batching its writes
+     * and still be sure they land before the crawl ends.
+     */
+    storeResult(requests: Request | Request[], renderingType: RenderingType): Awaitable<void>;
+}
 /**
  * Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.
  *
  * @experimental
  */
-export declare class RenderingTypePredictor {
-    private renderingTypeDetectionResults;
-    private detectionRatio;
-    private state;
+export declare class RenderingTypePredictor implements IRenderingTypePredictor {
+    #private;
     constructor({ detectionRatio, persistenceOptions }: RenderingTypePredictorOptions);
     /**
      * Initialize the predictor by restoring persisted state.
      */
     initialize(): Promise<void>;
+    /**
+     * Stop persisting the model, writing it out one last time. `initialize()` reopens the persistence window.
+     */
+    teardown(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
     /**
      * Predict the rendering type for a given URL and request label.
      */
@@ -31,10 +59,8 @@ export declare class RenderingTypePredictor {
     /**
      * Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
      */
-    storeResult({ url, loadedUrl, label }: Request, renderingType: RenderingType): void;
+    storeResult(requests: Request | Request[], renderingType: RenderingType): void;
     private resultCount;
-    protected calculateFeatureVector(url: URLComponents, label: string | undefined): FeatureVector;
-    protected retrain(): void;
+    private calculateFeatureVector;
+    private retrain;
 }
-export {};
-//# sourceMappingURL=rendering-type-prediction.d.ts.map

package/internals/utils/rendering-type-prediction.js

@@ -2,6 +2,7 @@ import { RecoverableState } from '@crawlee/core';
 import LogisticRegression from 'ml-logistic-regression';
 import { Matrix } from 'ml-matrix';
 import stringComparison from 'string-comparison';
+import { z } from 'zod';
 const urlComponents = (url) => {
     return [url.hostname, ...url.pathname.split('/')];
 };
@@ -10,28 +11,70 @@ const calculateUrlSimilarity = (a, b) => {
     if (a[0] !== b[0]) {
         return 0;
     }
-    for (let i = 1; i < Math.max(a.length, b.length); i++) {
+    const maxLength = Math.max(a.length, b.length);
+    // Only the hostname is present (no path components to compare) - the hosts already match.
+    if (maxLength <= 1) {
+        return 1;
+    }
+    for (let i = 1; i < maxLength; i++) {
         values.push(stringComparison.jaroWinkler.similarity(a[i] ?? '', b[i] ?? '') > 0.8 ? 1 : 0);
     }
-    return sum(values) / Math.max(a.length, b.length);
+    // The first component (index 0, the hostname) is excluded from the comparison above,
+    // so it must also be excluded from the denominator of the weighted average.
+    return sum(values) / (maxLength - 1);
 };
 const sum = (values) => values.reduce((acc, value) => acc + value);
 const mean = (values) => (values.length > 0 ? sum(values) / values.length : undefined);
+const renderingType = z.enum(['clientOnly', 'static']);
+const predictorState = z.object({
+    logreg: z.instanceof(LogisticRegression),
+    detectionResults: z.map(renderingType, z.map(z.string().optional(), z.array(z.array(z.string())))),
+});
+const persistedState = z.object({
+    logreg: z
+        .record(z.string(), z.unknown())
+        .prefault(() => new LogisticRegression({ numSteps: 1000, learningRate: 0.05 }).toJSON()),
+    detectionResults: z
+        .array(z.object({
+        renderingType,
+        urlPartsByLabel: z.array(z.object({
+            label: z.string().optional(),
+            urlParts: z.array(z.array(z.string())),
+        })),
+    }))
+        .prefault([]),
+});
+const stateCodec = z.codec(persistedState, predictorState, {
+    decode: ({ logreg, detectionResults }) => ({
+        logreg: LogisticRegression.load(logreg),
+        detectionResults: new Map(detectionResults.map(({ renderingType, urlPartsByLabel }) => [
+            renderingType,
+            new Map(urlPartsByLabel.map(({ label, urlParts }) => [label, urlParts])),
+        ])),
+    }),
+    encode: ({ logreg, detectionResults }) => ({
+        logreg: logreg.toJSON(),
+        detectionResults: Array.from(detectionResults.entries()).map(([renderingType, urlPartsByLabel]) => ({
+            renderingType,
+            urlPartsByLabel: Array.from(urlPartsByLabel.entries()).map(([label, urlParts]) => ({ label, urlParts })),
+        })),
+    }),
+});
 /**
  * Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.
  *
  * @experimental
  */
 export class RenderingTypePredictor {
-    renderingTypeDetectionResults = new Map();
-    detectionRatio;
-    state;
+    #detectionRatio;
+    #state;
     constructor({ detectionRatio, persistenceOptions }) {
-        this.detectionRatio = detectionRatio;
-        this.state = new RecoverableState({
-            defaultState: { logreg: new LogisticRegression({ numSteps: 1000, learningRate: 0.05 }) },
-            serialize: (state) => JSON.stringify({ logreg: state.logreg.toJSON() }),
-            deserialize: (serializedState) => ({ logreg: LogisticRegression.load(JSON.parse(serializedState).logreg) }),
+        this.#detectionRatio = detectionRatio;
+        this.#state = new RecoverableState({
+            defaultState: () => stateCodec.decode({}),
+            // The codec validates in the decode direction, so it is a Standard Schema as-is; encoding needs a call.
+            deserialize: stateCodec,
+            serialize: (state) => stateCodec.encode(state),
             persistStateKey: 'rendering-type-predictor-state',
             persistenceEnabled: true,
             ...persistenceOptions,
@@ -41,13 +84,22 @@ export class RenderingTypePredictor {
      * Initialize the predictor by restoring persisted state.
      */
     async initialize() {
-        await this.state.initialize();
+        await this.#state.initialize();
+    }
+    /**
+     * Stop persisting the model, writing it out one last time. `initialize()` reopens the persistence window.
+     */
+    async teardown() {
+        await this.#state.teardown();
+    }
+    async [Symbol.asyncDispose]() {
+        await this.teardown();
     }
     /**
      * Predict the rendering type for a given URL and request label.
      */
     predict({ url, loadedUrl, label }) {
-        const { logreg } = this.state.currentValue;
+        const { logreg } = this.#state.currentValue;
         if (logreg.classifiers.length === 0) {
             return { renderingType: 'clientOnly', detectionProbabilityRecommendation: 1 };
         }
@@ -59,32 +111,35 @@ export class RenderingTypePredictor {
             renderingType: prediction === 1 ? 'static' : 'clientOnly',
             detectionProbabilityRecommendation: Math.abs(scores[0] - scores[1]) < 0.1
                 ? 1
-                : this.detectionRatio * Math.max(1, 5 - this.resultCount(label)),
+                : this.#detectionRatio * Math.max(1, 5 - this.resultCount(label)),
         };
     }
     /**
      * Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
      */
-    storeResult({ url, loadedUrl, label }, renderingType) {
-        const resultUrl = new URL(loadedUrl ?? url);
-        if (!this.renderingTypeDetectionResults.has(renderingType)) {
-            this.renderingTypeDetectionResults.set(renderingType, new Map());
-        }
-        if (!this.renderingTypeDetectionResults.get(renderingType).has(label)) {
-            this.renderingTypeDetectionResults.get(renderingType).set(label, []);
+    storeResult(requests, renderingType) {
+        const state = this.#state.currentValue;
+        for (const { url, loadedUrl, label } of Array.isArray(requests) ? requests : [requests]) {
+            const resultUrl = new URL(loadedUrl ?? url);
+            if (!state.detectionResults.has(renderingType)) {
+                state.detectionResults.set(renderingType, new Map());
+            }
+            if (!state.detectionResults.get(renderingType).has(label)) {
+                state.detectionResults.get(renderingType).set(label, []);
+            }
+            state.detectionResults.get(renderingType).get(label).push(urlComponents(resultUrl));
         }
-        this.renderingTypeDetectionResults.get(renderingType).get(label).push(urlComponents(resultUrl));
         this.retrain();
     }
     resultCount(label) {
-        return Array.from(this.renderingTypeDetectionResults.values())
+        return Array.from(this.#state.currentValue.detectionResults.values())
             .map((results) => results.get(label)?.length ?? 0)
             .reduce((acc, value) => acc + value, 0);
     }
     calculateFeatureVector(url, label) {
         return [
-            mean((this.renderingTypeDetectionResults.get('static')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
-            mean((this.renderingTypeDetectionResults.get('clientOnly')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
+            mean((this.#state.currentValue.detectionResults.get('static')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
+            mean((this.#state.currentValue.detectionResults.get('clientOnly')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
         ];
     }
     retrain() {
@@ -93,7 +148,7 @@ export class RenderingTypePredictor {
             [1, 0],
         ];
         const Y = [0, 1];
-        for (const [renderingType, urlsByLabel] of this.renderingTypeDetectionResults.entries()) {
+        for (const [renderingType, urlsByLabel] of this.#state.currentValue.detectionResults.entries()) {
             for (const [label, urls] of urlsByLabel) {
                 for (const url of urls) {
                     X.push(this.calculateFeatureVector(url, label));
@@ -101,7 +156,6 @@ export class RenderingTypePredictor {
                 }
             }
         }
-        this.state.currentValue.logreg.train(new Matrix(X), Matrix.columnVector(Y));
+        this.#state.currentValue.logreg.train(new Matrix(X), Matrix.columnVector(Y));
     }
 }
-//# sourceMappingURL=rendering-type-prediction.js.map