@crawlee/playwright 4.0.0-rc.0 → 4.0.0-rc.1

package/internals/utils/rendering-type-prediction.js

@@ -67,11 +67,10 @@ const stateCodec = z.codec(persistedState, predictorState, {
  */
 export class RenderingTypePredictor {
     #detectionRatio;
-    // kept as TS-private: tests reach for it at runtime
-    state;
+    #state;
     constructor({ detectionRatio, persistenceOptions }) {
         this.#detectionRatio = detectionRatio;
-        this.state = new RecoverableState({
+        this.#state = new RecoverableState({
             defaultState: () => stateCodec.decode({}),
             // The codec validates in the decode direction, so it is a Standard Schema as-is; encoding needs a call.
             deserialize: stateCodec,
@@ -85,32 +84,41 @@ export class RenderingTypePredictor {
      * Initialize the predictor by restoring persisted state.
      */
     async initialize() {
-        await this.state.initialize();
+        await this.#state.initialize();
+    }
+    /**
+     * Stop persisting the model, writing it out one last time. `initialize()` reopens the persistence window.
+     */
+    async teardown() {
+        await this.#state.teardown();
+    }
+    async [Symbol.asyncDispose]() {
+        await this.teardown();
     }
     /**
      * Predict the rendering type for a given URL and request label.
      */
     predict({ url, loadedUrl, label }) {
-        const { logreg } = this.state.currentValue;
+        const { logreg } = this.#state.currentValue;
         if (logreg.classifiers.length === 0) {
             return { renderingType: 'clientOnly', detectionProbabilityRecommendation: 1 };
         }
         const predictionUrl = new URL(loadedUrl ?? url);
-        const urlFeature = new Matrix([this.calculateFeatureVector(urlComponents(predictionUrl), label)]);
+        const urlFeature = new Matrix([this.#calculateFeatureVector(urlComponents(predictionUrl), label)]);
         const [prediction] = logreg.predict(urlFeature);
         const scores = [logreg.classifiers[0].testScores(urlFeature), logreg.classifiers[1].testScores(urlFeature)];
         return {
             renderingType: prediction === 1 ? 'static' : 'clientOnly',
             detectionProbabilityRecommendation: Math.abs(scores[0] - scores[1]) < 0.1
                 ? 1
-                : this.#detectionRatio * Math.max(1, 5 - this.resultCount(label)),
+                : this.#detectionRatio * Math.max(1, 5 - this.#resultCount(label)),
         };
     }
     /**
      * Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
      */
     storeResult(requests, renderingType) {
-        const state = this.state.currentValue;
+        const state = this.#state.currentValue;
         for (const { url, loadedUrl, label } of Array.isArray(requests) ? requests : [requests]) {
             const resultUrl = new URL(loadedUrl ?? url);
             if (!state.detectionResults.has(renderingType)) {
@@ -121,33 +129,33 @@ export class RenderingTypePredictor {
             }
             state.detectionResults.get(renderingType).get(label).push(urlComponents(resultUrl));
         }
-        this.retrain();
+        this.#retrain();
     }
-    resultCount(label) {
-        return Array.from(this.state.currentValue.detectionResults.values())
+    #resultCount(label) {
+        return Array.from(this.#state.currentValue.detectionResults.values())
             .map((results) => results.get(label)?.length ?? 0)
             .reduce((acc, value) => acc + value, 0);
     }
-    calculateFeatureVector(url, label) {
+    #calculateFeatureVector(url, label) {
         return [
-            mean((this.state.currentValue.detectionResults.get('static')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
-            mean((this.state.currentValue.detectionResults.get('clientOnly')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
+            mean((this.#state.currentValue.detectionResults.get('static')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
+            mean((this.#state.currentValue.detectionResults.get('clientOnly')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
         ];
     }
-    retrain() {
+    #retrain() {
         const X = [
             [0, 1],
             [1, 0],
         ];
         const Y = [0, 1];
-        for (const [renderingType, urlsByLabel] of this.state.currentValue.detectionResults.entries()) {
+        for (const [renderingType, urlsByLabel] of this.#state.currentValue.detectionResults.entries()) {
             for (const [label, urls] of urlsByLabel) {
                 for (const url of urls) {
-                    X.push(this.calculateFeatureVector(url, label));
+                    X.push(this.#calculateFeatureVector(url, label));
                     Y.push(renderingType === 'static' ? 1 : 0);
                 }
             }
         }
-        this.state.currentValue.logreg.train(new Matrix(X), Matrix.columnVector(Y));
+        this.#state.currentValue.logreg.train(new Matrix(X), Matrix.columnVector(Y));
     }
 }

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-rc.0",
+    "version": "4.0.0-rc.1",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
@@ -47,32 +47,27 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/datastructures": "^2.0.3",
-        "@apify/timeout": "^0.4.4",
-        "@crawlee/basic": "4.0.0-rc.0",
-        "@crawlee/browser": "4.0.0-rc.0",
-        "@crawlee/browser-pool": "4.0.0-rc.0",
-        "@crawlee/cheerio": "4.0.0-rc.0",
-        "@crawlee/core": "4.0.0-rc.0",
-        "@crawlee/types": "4.0.0-rc.0",
-        "@crawlee/utils": "4.0.0-rc.0",
+        "@apify/datastructures": "^3.0.1",
+        "@apify/timeout": "^1.0.1",
+        "@crawlee/basic": "4.0.0-rc.1",
+        "@crawlee/browser": "4.0.0-rc.1",
+        "@crawlee/browser-pool": "4.0.0-rc.1",
+        "@crawlee/cheerio": "4.0.0-rc.1",
+        "@crawlee/core": "4.0.0-rc.1",
+        "@crawlee/types": "4.0.0-rc.1",
+        "@crawlee/utils": "4.0.0-rc.1",
         "cheerio": "^1.0.0",
         "jquery": "^3.7.1",
         "ml-logistic-regression": "^2.0.0",
         "ml-matrix": "^6.12.1",
         "string-comparison": "^1.3.0",
         "tslib": "^2.8.1",
-        "type-fest": "^4.0.0",
-        "zod": "^4.4.3"
+        "zod": "^4.5.4"
     },
     "peerDependencies": {
-        "idcac-playwright": "^0.2.0",
         "playwright": "*"
     },
     "peerDependenciesMeta": {
-        "idcac-playwright": {
-            "optional": true
-        },
         "playwright": {
             "optional": true
         }
@@ -84,5 +79,5 @@
             }
         }
     },
-    "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }