@crawlee/playwright 4.0.0-beta.99 → 4.0.0-rc.1

package/internals/utils/playwright-utils.d.ts

@@ -19,12 +19,11 @@
  */
 import { Configuration, type Request } from '@crawlee/browser';
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type CheerioRoot } from '@crawlee/utils';
+import type { CheerioAPI } from 'cheerio';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Download, Page, Response } from 'playwright';
 import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
-import { RenderingTypePredictor } from './rendering-type-prediction.js';
 export interface InjectFileOptions {
     /**
      * Enables the injected script to survive page navigations and reloads without need to be re-injected manually.
@@ -287,8 +286,7 @@ export declare function saveSnapshot(page: Page, options?: SaveSnapshotOptions):
  * @param page Playwright [`Page`](https://playwright.dev/docs/api/class-page) object.
  * @param ignoreShadowRoots
  */
-export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;
-export declare function closeCookieModals(page: Page): Promise<void>;
+export declare function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioAPI>;
 export interface HandleCloudflareChallengeOptions {
     /** Logging defaults to the `debug` level, use this flag to log to `info` level instead. */
     verbose?: boolean;
@@ -335,7 +333,7 @@ export interface HandleCloudflareChallengeOptions {
  * @param url current URL for request identification, only used for logging
  * @param [options]
  */
-declare function handleCloudflareChallenge(page: Page, url: string, options?: HandleCloudflareChallengeOptions): Promise<Response | undefined>;
+export declare function handleCloudflareChallenge(page: Page, url: string, options?: HandleCloudflareChallengeOptions): Promise<Response | undefined>;
 export interface PlaywrightContextUtils {
     /**
      * Injects a JavaScript file into current `page`.
@@ -434,7 +432,7 @@ export interface PlaywrightContextUtils {
      * });
      * ```
      */
-    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
     /**
      * Scrolls to the bottom of a page, or until it times out.
      * Loads dynamic content when it hits the bottom of a page, and then continues scrolling.
@@ -454,8 +452,7 @@ export interface PlaywrightContextUtils {
      * in `href` elements, but rather navigations are triggered in click handlers.
      * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
      *
-     * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
-     * and override settings of the enqueued {@link Request} objects.
+     * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
      *
      * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
      * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -476,9 +473,9 @@ export interface PlaywrightContextUtils {
      * async requestHandler({ enqueueLinksByClickingElements }) {
      *     await enqueueLinksByClickingElements({
      *         selector: 'a.product-detail',
-     *         globs: [
-     *             'https://www.example.com/handbags/**'
-     *             'https://www.example.com/purses/**'
+     *         include: [
+     *             'https://www.example.com/handbags/**',
+     *             'https://www.example.com/purses/**',
      *         ],
      *     });
      * });
@@ -514,19 +511,6 @@ export interface PlaywrightContextUtils {
      * secured copies beforehand.
      */
     compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
-    /**
-     * Tries to close cookie consent modals on the page. Based on the I Don't Care About Cookies browser extension.
-     *
-     * Note that this method requires the idcac-playwright package to be installed.
-     * Crawlee does not include it by default due to licensing issues.
-     *
-     * To use this method, please install the package manually by running:
-     *
-     * ```bash
-     * npm install idcac-playwright
-     * ```
-     */
-    closeCookieModals(): Promise<void>;
     /**
      * This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
      * It will try to detect the Cloudflare page, click on the checkbox, and wait for 10 seconds (configurable
@@ -573,18 +557,3 @@ export interface PlaywrightContextUtils {
     listDownloads(): Promise<Download[]>;
 }
 export { enqueueLinksByClickingElements };
-/** @internal */
-export declare const playwrightUtils: {
-    injectFile: typeof injectFile;
-    injectJQuery: typeof injectJQuery;
-    gotoExtended: typeof gotoExtended;
-    blockRequests: typeof blockRequests;
-    enqueueLinksByClickingElements: typeof enqueueLinksByClickingElements;
-    parseWithCheerio: typeof parseWithCheerio;
-    infiniteScroll: typeof infiniteScroll;
-    saveSnapshot: typeof saveSnapshot;
-    compileScript: typeof compileScript;
-    closeCookieModals: typeof closeCookieModals;
-    RenderingTypePredictor: typeof RenderingTypePredictor;
-    handleCloudflareChallenge: typeof handleCloudflareChallenge;
-};

package/internals/utils/playwright-utils.js

@@ -21,17 +21,46 @@ import { readFile } from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import vm from 'node:vm';
 import { Configuration, KeyValueStore, serviceLocator, SessionError, validators } from '@crawlee/browser';
-import { expandShadowRoots, sleep } from '@crawlee/utils';
-import * as cheerio from 'cheerio';
-import ow from 'ow';
+import { sleep } from '@crawlee/utils';
+import { expandShadowRoots, parseArgument, schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { LruCache } from '@apify/datastructures';
 import { enqueueLinksByClickingElements } from '../enqueue-links/click-elements.js';
-import { RenderingTypePredictor } from './rendering-type-prediction.js';
 const getLog = () => serviceLocator.getChildLog('Playwright Utils');
 const require = createRequire(import.meta.url);
 const jqueryPath = require.resolve('jquery');
 const MAX_INJECT_FILE_CACHE_SIZE = 10;
 const DEFAULT_BLOCK_REQUEST_URL_PATTERNS = ['.css', '.jpg', '.jpeg', '.png', '.svg', '.gif', '.woff', '.pdf', '.zip'];
+const filePathSchema = z.string();
+const injectFileOptionsSchema = z.strictObject({
+    surviveNavigations: z.boolean().optional(),
+});
+const gotoExtendedRequestSchema = z.looseObject({
+    url: z.url(),
+    method: z.string().optional(),
+    headers: schemas.anyObject.optional(),
+    payload: z.union([z.string(), z.instanceof(Uint8Array)]).optional(),
+});
+const blockRequestsOptionsSchema = z.strictObject({
+    urlPatterns: schemas.arrayOf(z.string(), 'strings').default(DEFAULT_BLOCK_REQUEST_URL_PATTERNS),
+    extraUrlPatterns: schemas.arrayOf(z.string(), 'strings').default(() => []),
+});
+const infiniteScrollOptionsSchema = z.strictObject({
+    timeoutSecs: schemas.anyNumber.default(0),
+    maxScrollHeight: schemas.anyNumber.default(0),
+    waitForSecs: schemas.anyNumber.default(4),
+    scrollDownAndUp: z.boolean().default(false),
+    buttonSelector: z.string().optional(),
+    stopScrollCallback: schemas.anyFunction.optional(),
+});
+const saveSnapshotOptionsSchema = z.strictObject({
+    key: z.string().min(1).default('SNAPSHOT'),
+    screenshotQuality: schemas.anyNumber.default(50),
+    saveScreenshot: z.boolean().default(true),
+    saveHtml: z.boolean().default(true),
+    keyValueStoreName: z.string().optional(),
+    configuration: schemas.anyObject.optional(),
+});
 /**
  * Cache contents of previously injected files to limit file system access.
  */
@@ -48,18 +77,16 @@ const injectedFilesCache = new LruCache({ maxLength: MAX_INJECT_FILE_CACHE_SIZE
  * @param [options]
  */
 export async function injectFile(page, filePath, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(filePath, ow.string);
-    ow(options, ow.object.exactShape({
-        surviveNavigations: ow.optional.boolean,
-    }));
+    parseArgument(page, validators.browserPage);
+    parseArgument(filePath, filePathSchema);
+    const { surviveNavigations } = parseArgument(options, injectFileOptionsSchema);
     let contents = injectedFilesCache.get(filePath);
     if (!contents) {
         contents = await readFile(filePath, 'utf8');
         injectedFilesCache.add(filePath, contents);
     }
     const evalP = page.evaluate(contents);
-    if (options.surviveNavigations) {
+    if (surviveNavigations) {
         page.on('framenavigated', async () => page
             .evaluate(contents)
             .catch((error) => getLog().warning('An error occurred during the script injection!', { error })));
@@ -93,7 +120,7 @@ export async function injectFile(page, filePath, options = {}) {
  * @param [options.surviveNavigations] Opt-out option to disable the JQuery reinjection after navigation.
  */
 export async function injectJQuery(page, options) {
-    ow(page, ow.object.validate(validators.browserPage));
+    parseArgument(page, validators.browserPage);
     return injectFile(page, jqueryPath, { surviveNavigations: options?.surviveNavigations ?? true });
 }
 /**
@@ -109,14 +136,9 @@ export async function injectJQuery(page, options) {
  * @param [gotoOptions] Custom options for `page.goto()`.
  */
 export async function gotoExtended(page, request, gotoOptions = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(request, ow.object.partialShape({
-        url: ow.string.url,
-        method: ow.optional.string,
-        headers: ow.optional.object,
-        payload: ow.optional.any(ow.string, ow.uint8Array),
-    }));
-    ow(gotoOptions, ow.object);
+    parseArgument(page, validators.browserPage);
+    parseArgument(request, gotoExtendedRequestSchema);
+    parseArgument(gotoOptions, schemas.anyObject);
     const { url, method, headers, payload } = request;
     const isEmpty = (o) => !o || Object.keys(o).length === 0;
     if (method !== 'GET' || payload) {
@@ -201,12 +223,8 @@ export async function gotoExtended(page, request, gotoOptions = {}) {
  * @param [options]
  */
 export async function blockRequests(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        urlPatterns: ow.optional.array.ofType(ow.string),
-        extraUrlPatterns: ow.optional.array.ofType(ow.string),
-    }));
-    const { urlPatterns = DEFAULT_BLOCK_REQUEST_URL_PATTERNS, extraUrlPatterns = [] } = options;
+    parseArgument(page, validators.browserPage);
+    const { urlPatterns, extraUrlPatterns } = parseArgument(options, blockRequestsOptionsSchema);
     const patternsToBlock = [...urlPatterns, ...extraUrlPatterns];
     try {
         const client = await page.context().newCDPSession(page);
@@ -264,16 +282,8 @@ export function compileScript(scriptString, context = Object.create(null)) {
  * @param [options]
  */
 export async function infiniteScroll(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        timeoutSecs: ow.optional.number,
-        maxScrollHeight: ow.optional.number,
-        waitForSecs: ow.optional.number,
-        scrollDownAndUp: ow.optional.boolean,
-        buttonSelector: ow.optional.string,
-        stopScrollCallback: ow.optional.function,
-    }));
-    const { timeoutSecs = 0, maxScrollHeight = 0, waitForSecs = 4, scrollDownAndUp = false, buttonSelector, stopScrollCallback, } = options;
+    parseArgument(page, validators.browserPage);
+    const { timeoutSecs, maxScrollHeight, waitForSecs, scrollDownAndUp, buttonSelector, stopScrollCallback } = parseArgument(options, infiniteScrollOptionsSchema);
     let finished;
     const startTime = Date.now();
     const CHECK_INTERVAL_MILLIS = 1000;
@@ -350,16 +360,8 @@ export async function infiniteScroll(page, options = {}) {
  * @param [options]
  */
 export async function saveSnapshot(page, options = {}) {
-    ow(page, ow.object.validate(validators.browserPage));
-    ow(options, ow.object.exactShape({
-        key: ow.optional.string.nonEmpty,
-        screenshotQuality: ow.optional.number,
-        saveScreenshot: ow.optional.boolean,
-        saveHtml: ow.optional.boolean,
-        keyValueStoreName: ow.optional.string,
-        configuration: ow.optional.object,
-    }));
-    const { key = 'SNAPSHOT', screenshotQuality = 50, saveScreenshot = true, saveHtml = true, keyValueStoreName, configuration, } = options;
+    parseArgument(page, validators.browserPage);
+    const { key, screenshotQuality, saveScreenshot, saveHtml, keyValueStoreName, configuration } = parseArgument(options, saveSnapshotOptionsSchema);
     try {
         const store = await KeyValueStore.open(keyValueStoreName ? { name: keyValueStoreName } : null, {
             configuration: configuration ?? Configuration.getGlobalConfiguration(),
@@ -397,12 +399,14 @@ export async function saveSnapshot(page, options = {}) {
  * @param ignoreShadowRoots
  */
 export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIframes = false) {
-    ow(page, ow.object.validate(validators.browserPage));
+    parseArgument(page, validators.browserPage);
     const html = ignoreShadowRoots
         ? null
         : (await page.evaluate(`(${expandShadowRoots.toString()})(document)`));
     const pageContent = html || (await page.content());
-    const $ = cheerio.load(pageContent);
+    // Full cheerio (parse5) import for browser compliant parser
+    const { load } = await import('cheerio');
+    const $ = load(pageContent);
     if (page.frames().length > 1 && !ignoreIframes) {
         const frames = await page.$$('iframe');
         const cheerioIframes = $('iframe').toArray();
@@ -434,35 +438,6 @@ export async function parseWithCheerio(page, ignoreShadowRoots = false, ignoreIf
     }
     return $;
 }
-let idcacPlaywright = null;
-async function getIdcacPlaywright() {
-    if (idcacPlaywright)
-        return idcacPlaywright;
-    try {
-        idcacPlaywright = await import('idcac-playwright');
-    }
-    catch (error) {
-        getLog().warning(`Failed to import 'idcac-playwright'.
-
-We recently made idcac-playwright an optional dependency due to licensing issues.
-To use this feature, please install it manually by running
-
-npm install idcac-playwright
-
-Original error message follows:
-
-${error.message}
-`);
-    }
-    return idcacPlaywright;
-}
-export async function closeCookieModals(page) {
-    ow(page, ow.object.validate(validators.browserPage));
-    const idcac = await getIdcacPlaywright();
-    if (idcac?.getInjectableScript()) {
-        await page.evaluate(idcac.getInjectableScript());
-    }
-}
 /**
  * This helper tries to solve the Cloudflare challenge automatically by clicking on the checkbox.
  * It will try to detect the Cloudflare page, click on the checkbox, and wait for 10 seconds (configurable
@@ -487,7 +462,7 @@ export async function closeCookieModals(page) {
  * @param url current URL for request identification, only used for logging
  * @param [options]
  */
-async function handleCloudflareChallenge(page, url, options = {}) {
+export async function handleCloudflareChallenge(page, url, options = {}) {
     options.isBlockedCallback ??= async () => {
         const isBlocked = await page.evaluate(() => {
             return document.querySelector('h1')?.textContent?.trim().includes('Sorry, you have been blocked');
@@ -496,9 +471,9 @@ async function handleCloudflareChallenge(page, url, options = {}) {
     };
     options.isChallengeCallback ??= async () => {
         return await page.evaluate(async () => {
-            // Cloudflare nests the ray ID under varying wrapper elements, so we match by descendants
-            // instead of a direct-child chain (e.g. a `.footer-wrapper` was inserted in between).
-            return !!document.querySelector('.footer .footer-inner .diagnostic-wrapper .ray-id');
+            // Cloudflare keeps reshuffling the wrapper elements between `.footer-inner` and `.ray-id`,
+            // so only the stable outer classes are matched.
+            return !!document.querySelector('.footer .footer-inner .ray-id');
         });
     };
     const retryBlocked = async () => {
@@ -509,7 +484,11 @@ async function handleCloudflareChallenge(page, url, options = {}) {
     };
     // check if we ended up on the CF challenge page
     const isChallenge = async () => {
-        return options.isChallengeCallback(page).catch(() => false);
+        return options.isChallengeCallback(page).catch(async () => {
+            // the challenge page navigates itself, which destroys the context of an in-flight check
+            await page.waitForLoadState('domcontentloaded').catch(() => { });
+            return options.isChallengeCallback(page).catch(() => false);
+        });
     };
     if (!(await isChallenge())) {
         await retryBlocked();
@@ -573,18 +552,3 @@ async function handleCloudflareChallenge(page, url, options = {}) {
     return (await page.reload()) ?? undefined;
 }
 export { enqueueLinksByClickingElements };
-/** @internal */
-export const playwrightUtils = {
-    injectFile,
-    injectJQuery,
-    gotoExtended,
-    blockRequests,
-    enqueueLinksByClickingElements,
-    parseWithCheerio,
-    infiniteScroll,
-    saveSnapshot,
-    compileScript,
-    closeCookieModals,
-    RenderingTypePredictor,
-    handleCloudflareChallenge,
-};

package/internals/utils/rendering-type-prediction.d.ts

@@ -1,4 +1,5 @@
 import type { RecoverableStatePersistenceOptions, Request } from '@crawlee/core';
+import type { Awaitable } from '@crawlee/types';
 export type RenderingType = 'clientOnly' | 'static';
 export interface RenderingTypePredictorOptions {
     /** A number between 0 and 1 that determines the desired ratio of rendering type detections */
@@ -12,13 +13,24 @@ export interface RenderingTypePredictorOptions {
  * @experimental
  */
 export interface IRenderingTypePredictor {
-    /** Predict the rendering type for a request, and how likely the crawler should be to verify it. */
-    predict(request: Request): {
+    /**
+     * Predict the rendering type for a request, and how likely the crawler should be to verify it.
+     *
+     * Called once per request, before navigation - the crawler awaits it, so prefer loading whatever the
+     * prediction needs up front over per-request I/O.
+     */
+    predict(request: Request): Awaitable<{
         renderingType: RenderingType;
         detectionProbabilityRecommendation: number;
-    };
-    /** Report a detected rendering type, so that future predictions can take it into account. */
-    storeResult(requests: Request | Request[], renderingType: RenderingType): void;
+    }>;
+    /**
+     * Report a detected rendering type, so that future predictions can take it into account.
+     *
+     * The crawler does not await this per detection - it collects the returned promises and drains them in
+     * `teardown()`, so an implementation that persists results asynchronously can keep batching its writes
+     * and still be sure they land before the crawl ends.
+     */
+    storeResult(requests: Request | Request[], renderingType: RenderingType): Awaitable<void>;
 }
 /**
  * Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.
@@ -26,13 +38,17 @@ export interface IRenderingTypePredictor {
  * @experimental
  */
 export declare class RenderingTypePredictor implements IRenderingTypePredictor {
-    private detectionRatio;
-    private state;
+    #private;
     constructor({ detectionRatio, persistenceOptions }: RenderingTypePredictorOptions);
     /**
      * Initialize the predictor by restoring persisted state.
      */
     initialize(): Promise<void>;
+    /**
+     * Stop persisting the model, writing it out one last time. `initialize()` reopens the persistence window.
+     */
+    teardown(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
     /**
      * Predict the rendering type for a given URL and request label.
      */
@@ -44,7 +60,4 @@ export declare class RenderingTypePredictor implements IRenderingTypePredictor {
      * Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
      */
     storeResult(requests: Request | Request[], renderingType: RenderingType): void;
-    private resultCount;
-    private calculateFeatureVector;
-    private retrain;
 }

package/internals/utils/rendering-type-prediction.js

@@ -2,6 +2,7 @@ import { RecoverableState } from '@crawlee/core';
 import LogisticRegression from 'ml-logistic-regression';
 import { Matrix } from 'ml-matrix';
 import stringComparison from 'string-comparison';
+import { z } from 'zod';
 const urlComponents = (url) => {
     return [url.hostname, ...url.pathname.split('/')];
 };
@@ -24,41 +25,56 @@ const calculateUrlSimilarity = (a, b) => {
 };
 const sum = (values) => values.reduce((acc, value) => acc + value);
 const mean = (values) => (values.length > 0 ? sum(values) / values.length : undefined);
+const renderingType = z.enum(['clientOnly', 'static']);
+const predictorState = z.object({
+    logreg: z.instanceof(LogisticRegression),
+    detectionResults: z.map(renderingType, z.map(z.string().optional(), z.array(z.array(z.string())))),
+});
+const persistedState = z.object({
+    logreg: z
+        .record(z.string(), z.unknown())
+        .prefault(() => new LogisticRegression({ numSteps: 1000, learningRate: 0.05 }).toJSON()),
+    detectionResults: z
+        .array(z.object({
+        renderingType,
+        urlPartsByLabel: z.array(z.object({
+            label: z.string().optional(),
+            urlParts: z.array(z.array(z.string())),
+        })),
+    }))
+        .prefault([]),
+});
+const stateCodec = z.codec(persistedState, predictorState, {
+    decode: ({ logreg, detectionResults }) => ({
+        logreg: LogisticRegression.load(logreg),
+        detectionResults: new Map(detectionResults.map(({ renderingType, urlPartsByLabel }) => [
+            renderingType,
+            new Map(urlPartsByLabel.map(({ label, urlParts }) => [label, urlParts])),
+        ])),
+    }),
+    encode: ({ logreg, detectionResults }) => ({
+        logreg: logreg.toJSON(),
+        detectionResults: Array.from(detectionResults.entries()).map(([renderingType, urlPartsByLabel]) => ({
+            renderingType,
+            urlPartsByLabel: Array.from(urlPartsByLabel.entries()).map(([label, urlParts]) => ({ label, urlParts })),
+        })),
+    }),
+});
 /**
  * Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.
  *
  * @experimental
  */
 export class RenderingTypePredictor {
-    detectionRatio;
-    state;
+    #detectionRatio;
+    #state;
     constructor({ detectionRatio, persistenceOptions }) {
-        this.detectionRatio = detectionRatio;
-        this.state = new RecoverableState({
-            defaultState: {
-                logreg: new LogisticRegression({ numSteps: 1000, learningRate: 0.05 }),
-                detectionResults: new Map(),
-            },
-            serialize: (state) => JSON.stringify({
-                logreg: state.logreg.toJSON(),
-                detectionResults: Array.from(state.detectionResults.entries()).map(([renderingType, urlPartsByLabel]) => ({
-                    renderingType,
-                    urlPartsByLabel: Array.from(urlPartsByLabel.entries()).map(([label, urlParts]) => ({
-                        label,
-                        urlParts,
-                    })),
-                })),
-            }),
-            deserialize: (serializedState) => {
-                const { logreg, detectionResults = [] } = JSON.parse(serializedState);
-                return {
-                    logreg: LogisticRegression.load(logreg),
-                    detectionResults: new Map(detectionResults.map((serializedItem) => [
-                        serializedItem.renderingType,
-                        new Map(serializedItem.urlPartsByLabel.map((item) => [item.label, item.urlParts])),
-                    ])),
-                };
-            },
+        this.#detectionRatio = detectionRatio;
+        this.#state = new RecoverableState({
+            defaultState: () => stateCodec.decode({}),
+            // The codec validates in the decode direction, so it is a Standard Schema as-is; encoding needs a call.
+            deserialize: stateCodec,
+            serialize: (state) => stateCodec.encode(state),
             persistStateKey: 'rendering-type-predictor-state',
             persistenceEnabled: true,
             ...persistenceOptions,
@@ -68,32 +84,41 @@ export class RenderingTypePredictor {
      * Initialize the predictor by restoring persisted state.
      */
     async initialize() {
-        await this.state.initialize();
+        await this.#state.initialize();
+    }
+    /**
+     * Stop persisting the model, writing it out one last time. `initialize()` reopens the persistence window.
+     */
+    async teardown() {
+        await this.#state.teardown();
+    }
+    async [Symbol.asyncDispose]() {
+        await this.teardown();
     }
     /**
      * Predict the rendering type for a given URL and request label.
      */
     predict({ url, loadedUrl, label }) {
-        const { logreg } = this.state.currentValue;
+        const { logreg } = this.#state.currentValue;
         if (logreg.classifiers.length === 0) {
             return { renderingType: 'clientOnly', detectionProbabilityRecommendation: 1 };
         }
         const predictionUrl = new URL(loadedUrl ?? url);
-        const urlFeature = new Matrix([this.calculateFeatureVector(urlComponents(predictionUrl), label)]);
+        const urlFeature = new Matrix([this.#calculateFeatureVector(urlComponents(predictionUrl), label)]);
         const [prediction] = logreg.predict(urlFeature);
         const scores = [logreg.classifiers[0].testScores(urlFeature), logreg.classifiers[1].testScores(urlFeature)];
         return {
             renderingType: prediction === 1 ? 'static' : 'clientOnly',
             detectionProbabilityRecommendation: Math.abs(scores[0] - scores[1]) < 0.1
                 ? 1
-                : this.detectionRatio * Math.max(1, 5 - this.resultCount(label)),
+                : this.#detectionRatio * Math.max(1, 5 - this.#resultCount(label)),
         };
     }
     /**
      * Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
      */
     storeResult(requests, renderingType) {
-        const state = this.state.currentValue;
+        const state = this.#state.currentValue;
         for (const { url, loadedUrl, label } of Array.isArray(requests) ? requests : [requests]) {
             const resultUrl = new URL(loadedUrl ?? url);
             if (!state.detectionResults.has(renderingType)) {
@@ -104,33 +129,33 @@ export class RenderingTypePredictor {
             }
             state.detectionResults.get(renderingType).get(label).push(urlComponents(resultUrl));
         }
-        this.retrain();
+        this.#retrain();
     }
-    resultCount(label) {
-        return Array.from(this.state.currentValue.detectionResults.values())
+    #resultCount(label) {
+        return Array.from(this.#state.currentValue.detectionResults.values())
             .map((results) => results.get(label)?.length ?? 0)
             .reduce((acc, value) => acc + value, 0);
     }
-    calculateFeatureVector(url, label) {
+    #calculateFeatureVector(url, label) {
         return [
-            mean((this.state.currentValue.detectionResults.get('static')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
-            mean((this.state.currentValue.detectionResults.get('clientOnly')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
+            mean((this.#state.currentValue.detectionResults.get('static')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
+            mean((this.#state.currentValue.detectionResults.get('clientOnly')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
         ];
     }
-    retrain() {
+    #retrain() {
         const X = [
             [0, 1],
             [1, 0],
         ];
         const Y = [0, 1];
-        for (const [renderingType, urlsByLabel] of this.state.currentValue.detectionResults.entries()) {
+        for (const [renderingType, urlsByLabel] of this.#state.currentValue.detectionResults.entries()) {
             for (const [label, urls] of urlsByLabel) {
                 for (const url of urls) {
-                    X.push(this.calculateFeatureVector(url, label));
+                    X.push(this.#calculateFeatureVector(url, label));
                     Y.push(renderingType === 'static' ? 1 : 0);
                 }
             }
         }
-        this.state.currentValue.logreg.train(new Matrix(X), Matrix.columnVector(Y));
+        this.#state.currentValue.logreg.train(new Matrix(X), Matrix.columnVector(Y));
     }
 }