@crawlee/playwright 4.0.0-beta.118 → 4.0.0-beta.119

package/internals/utils/rendering-type-prediction.js

@@ -2,6 +2,7 @@ import { RecoverableState } from '@crawlee/core';
 import LogisticRegression from 'ml-logistic-regression';
 import { Matrix } from 'ml-matrix';
 import stringComparison from 'string-comparison';
+import { z } from 'zod';
 const urlComponents = (url) => {
     return [url.hostname, ...url.pathname.split('/')];
 };
@@ -24,6 +25,41 @@ const calculateUrlSimilarity = (a, b) => {
 };
 const sum = (values) => values.reduce((acc, value) => acc + value);
 const mean = (values) => (values.length > 0 ? sum(values) / values.length : undefined);
+const renderingType = z.enum(['clientOnly', 'static']);
+const predictorState = z.object({
+    logreg: z.instanceof(LogisticRegression),
+    detectionResults: z.map(renderingType, z.map(z.string().optional(), z.array(z.array(z.string())))),
+});
+const persistedState = z.object({
+    logreg: z
+        .record(z.string(), z.unknown())
+        .prefault(() => new LogisticRegression({ numSteps: 1000, learningRate: 0.05 }).toJSON()),
+    detectionResults: z
+        .array(z.object({
+        renderingType,
+        urlPartsByLabel: z.array(z.object({
+            label: z.string().optional(),
+            urlParts: z.array(z.array(z.string())),
+        })),
+    }))
+        .prefault([]),
+});
+const stateCodec = z.codec(persistedState, predictorState, {
+    decode: ({ logreg, detectionResults }) => ({
+        logreg: LogisticRegression.load(logreg),
+        detectionResults: new Map(detectionResults.map(({ renderingType, urlPartsByLabel }) => [
+            renderingType,
+            new Map(urlPartsByLabel.map(({ label, urlParts }) => [label, urlParts])),
+        ])),
+    }),
+    encode: ({ logreg, detectionResults }) => ({
+        logreg: logreg.toJSON(),
+        detectionResults: Array.from(detectionResults.entries()).map(([renderingType, urlPartsByLabel]) => ({
+            renderingType,
+            urlPartsByLabel: Array.from(urlPartsByLabel.entries()).map(([label, urlParts]) => ({ label, urlParts })),
+        })),
+    }),
+});
 /**
  * Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.
  *
@@ -36,30 +72,10 @@ export class RenderingTypePredictor {
     constructor({ detectionRatio, persistenceOptions }) {
         this.#detectionRatio = detectionRatio;
         this.state = new RecoverableState({
-            defaultState: {
-                logreg: new LogisticRegression({ numSteps: 1000, learningRate: 0.05 }),
-                detectionResults: new Map(),
-            },
-            serialize: (state) => JSON.stringify({
-                logreg: state.logreg.toJSON(),
-                detectionResults: Array.from(state.detectionResults.entries()).map(([renderingType, urlPartsByLabel]) => ({
-                    renderingType,
-                    urlPartsByLabel: Array.from(urlPartsByLabel.entries()).map(([label, urlParts]) => ({
-                        label,
-                        urlParts,
-                    })),
-                })),
-            }),
-            deserialize: (serializedState) => {
-                const { logreg, detectionResults = [] } = JSON.parse(serializedState);
-                return {
-                    logreg: LogisticRegression.load(logreg),
-                    detectionResults: new Map(detectionResults.map((serializedItem) => [
-                        serializedItem.renderingType,
-                        new Map(serializedItem.urlPartsByLabel.map((item) => [item.label, item.urlParts])),
-                    ])),
-                };
-            },
+            defaultState: () => stateCodec.decode({}),
+            // The codec validates in the decode direction, so it is a Standard Schema as-is; encoding needs a call.
+            deserialize: stateCodec,
+            serialize: (state) => stateCodec.encode(state),
             persistStateKey: 'rendering-type-predictor-state',
             persistenceEnabled: true,
             ...persistenceOptions,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.118",
+    "version": "4.0.0-beta.119",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,13 +49,13 @@
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
         "@apify/timeout": "^0.4.4",
-        "@crawlee/basic": "4.0.0-beta.118",
-        "@crawlee/browser": "4.0.0-beta.118",
-        "@crawlee/browser-pool": "4.0.0-beta.118",
-        "@crawlee/cheerio": "4.0.0-beta.118",
-        "@crawlee/core": "4.0.0-beta.118",
-        "@crawlee/types": "4.0.0-beta.118",
-        "@crawlee/utils": "4.0.0-beta.118",
+        "@crawlee/basic": "4.0.0-beta.119",
+        "@crawlee/browser": "4.0.0-beta.119",
+        "@crawlee/browser-pool": "4.0.0-beta.119",
+        "@crawlee/cheerio": "4.0.0-beta.119",
+        "@crawlee/core": "4.0.0-beta.119",
+        "@crawlee/types": "4.0.0-beta.119",
+        "@crawlee/utils": "4.0.0-beta.119",
         "cheerio": "^1.0.0",
         "jquery": "^3.7.1",
         "ml-logistic-regression": "^2.0.0",
@@ -63,7 +63,8 @@
         "ow": "^2.0.0",
         "string-comparison": "^1.3.0",
         "tslib": "^2.8.1",
-        "type-fest": "^4.0.0"
+        "type-fest": "^4.0.0",
+        "zod": "^4.1.0"
     },
     "peerDependencies": {
         "idcac-playwright": "^0.2.0",
@@ -84,5 +85,5 @@
             }
         }
     },
-    "gitHead": "7306bd05a126662898a4f2fe7779f6b53425cadf"
+    "gitHead": "552fe2371a3c6e9e9f3514010536ea6abdc27eb4"
 }