@crawlee/playwright 3.7.4-beta.8 → 3.8.0

package/internals/utils/rendering-type-prediction.js

@@ -0,0 +1,108 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RenderingTypePredictor = void 0;
+const tslib_1 = require("tslib");
+const ml_logistic_regression_1 = tslib_1.__importDefault(require("ml-logistic-regression"));
+const ml_matrix_1 = require("ml-matrix");
+const string_comparison_1 = tslib_1.__importDefault(require("string-comparison"));
+const urlComponents = (url) => {
+    return [url.hostname, ...url.pathname.split('/')];
+};
+const calculateUrlSimilarity = (a, b) => {
+    const values = [];
+    if (a[0] !== b[0]) {
+        return 0;
+    }
+    for (let i = 1; i < Math.max(a.length, b.length); i++) {
+        values.push(string_comparison_1.default.jaroWinkler.similarity(a[i] ?? '', b[i] ?? '') > 0.8 ? 1 : 0);
+    }
+    return sum(values) / Math.max(a.length, b.length);
+};
+const sum = (values) => values.reduce((acc, value) => acc + value);
+const mean = (values) => (values.length > 0 ? sum(values) / values.length : undefined);
+/**
+ * Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.
+ *
+ * @experimental
+ */
+class RenderingTypePredictor {
+    constructor({ detectionRatio }) {
+        Object.defineProperty(this, "renderingTypeDetectionResults", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+        Object.defineProperty(this, "detectionRatio", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "logreg", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.detectionRatio = detectionRatio;
+        this.logreg = new ml_logistic_regression_1.default({ numSteps: 1000, learningRate: 0.05 });
+    }
+    /**
+     * Predict the rendering type for a given URL and request label.
+     */
+    predict(url, label) {
+        if (this.logreg.classifiers.length === 0) {
+            return { renderingType: 'clientOnly', detectionProbabilityRecommendation: 1 };
+        }
+        const urlFeature = new ml_matrix_1.Matrix([this.calculateFeatureVector(urlComponents(url), label)]);
+        const [prediction] = this.logreg.predict(urlFeature);
+        const scores = [this.logreg.classifiers[0].testScores(urlFeature), this.logreg.classifiers[1].testScores(urlFeature)];
+        return {
+            renderingType: prediction === 1 ? 'static' : 'clientOnly',
+            detectionProbabilityRecommendation: Math.abs(scores[0] - scores[1]) < 0.1 ? 1 : this.detectionRatio * Math.max(1, 5 - this.resultCount(label)),
+        };
+    }
+    /**
+     * Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
+     */
+    storeResult(url, label, renderingType) {
+        if (!this.renderingTypeDetectionResults.has(renderingType)) {
+            this.renderingTypeDetectionResults.set(renderingType, new Map());
+        }
+        if (!this.renderingTypeDetectionResults.get(renderingType).has(label)) {
+            this.renderingTypeDetectionResults.get(renderingType).set(label, []);
+        }
+        this.renderingTypeDetectionResults.get(renderingType).get(label).push(urlComponents(url));
+        this.retrain();
+    }
+    resultCount(label) {
+        return Array.from(this.renderingTypeDetectionResults.values())
+            .map((results) => results.get(label)?.length ?? 0)
+            .reduce((acc, value) => acc + value, 0);
+    }
+    calculateFeatureVector(url, label) {
+        return [
+            mean((this.renderingTypeDetectionResults.get('static')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
+            mean((this.renderingTypeDetectionResults.get('clientOnly')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
+        ];
+    }
+    retrain() {
+        const X = [
+            [0, 1],
+            [1, 0],
+        ];
+        const Y = [0, 1];
+        for (const [renderingType, urlsByLabel] of this.renderingTypeDetectionResults.entries()) {
+            for (const [label, urls] of urlsByLabel) {
+                for (const url of urls) {
+                    X.push(this.calculateFeatureVector(url, label));
+                    Y.push(renderingType === 'static' ? 1 : 0);
+                }
+            }
+        }
+        this.logreg.train(new ml_matrix_1.Matrix(X), ml_matrix_1.Matrix.columnVector(Y));
+    }
+}
+exports.RenderingTypePredictor = RenderingTypePredictor;
+//# sourceMappingURL=rendering-type-prediction.js.map

package/internals/utils/rendering-type-prediction.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rendering-type-prediction.js","sourceRoot":"","sources":["../../../src/internals/utils/rendering-type-prediction.ts"],"names":[],"mappings":";;;;AAAA,4FAAwD;AACxD,yCAAmC;AACnC,kFAAiD;AAMjD,MAAM,aAAa,GAAG,CAAC,GAAQ,EAAiB,EAAE;IAC9C,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;AACtD,CAAC,CAAC;AAEF,MAAM,sBAAsB,GAAG,CAAC,CAAgB,EAAE,CAAgB,EAAsB,EAAE;IACtF,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAChB,OAAO,CAAC,CAAC;IACb,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACpD,MAAM,CAAC,IAAI,CAAC,2BAAgB,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/F,CAAC;IAED,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;AACtD,CAAC,CAAC;AAEF,MAAM,GAAG,GAAG,CAAC,MAAgB,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;AAC7E,MAAM,IAAI,GAAG,CAAC,MAAgB,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;AASjG;;;;GAIG;AACH,MAAa,sBAAsB;IAK/B,YAAY,EAAE,cAAc,EAAiC;QAJrD;;;;mBAAgC,IAAI,GAAG,EAA2D;WAAC;QACnG;;;;;WAAuB;QACvB;;;;;WAA2B;QAG/B,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,MAAM,GAAG,IAAI,gCAAkB,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;IACjF,CAAC;IAED;;OAEG;IACI,OAAO,CAAC,GAAQ,EAAE,KAAyB;QAC9C,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvC,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,kCAAkC,EAAE,CAAC,EAAE,CAAC;QAClF,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,kBAAM,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;QACxF,MAAM,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC;QAEtH,OAAO;YACH,aAAa,EAAE,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY;YACzD,kCAAkC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;SACjJ,CAAC;IACN,CAAC;IAED;;OAEG;IACI,WAAW,CAAC,GAAQ,EAAE,KAAyB,EAAE,aAA4B;QAChF,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;YACzD,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,aAAa,CAAE,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACrE,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,aAAa,CAAE,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC1E,CAAC;QAED,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,aAAa,CAAE,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5F,IAAI,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;IAEO,WAAW,CAAC,KAAyB;QACzC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,6BAA6B,CAAC,MAAM,EAAE,CAAC;aACzD,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;aACjD,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;IAChD,CAAC;IAES,sBAAsB,CAAC,GAAkB,EAAE,KAAyB;QAC1E,OAAO;YACH,IAAI,CAAC,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAC7I,IAAI,CAAC,CAAC,IAAI,CAAC,6BAA6B,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,sBAAsB,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;SACpJ,CAAC;IACN,CAAC;IAES,OAAO;QACb,MAAM,CAAC,GAAoB;YACvB,CAAC,CAAC,EAAE,CAAC,CAAC;YACN,CAAC,CAAC,EAAE,CAAC,CAAC;SACT,CAAC;QACF,MAAM,CAAC,GAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAE3B,KAAK,MAAM,CAAC,aAAa,EAAE,WAAW,CAAC,IAAI,IAAI,CAAC,6BAA6B,CAAC,OAAO,EAAE,EAAE,CAAC;YACtF,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,WAAW,EAAE,CAAC;gBACtC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;oBACrB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;oBAChD,CAAC,CAAC,IAAI,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/C,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,kBAAM,CAAC,CAAC,CAAC,EAAE,kBAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC;CACJ;AA3ED,wDA2EC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "3.7.4-beta.8",
+    "version": "3.8.0",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
@@ -55,14 +55,19 @@
     "dependencies": {
         "@apify/datastructures": "^2.0.0",
         "@apify/log": "^2.4.0",
-        "@crawlee/browser": "3.7.4-beta.8",
-        "@crawlee/browser-pool": "3.7.4-beta.8",
-        "@crawlee/types": "3.7.4-beta.8",
-        "@crawlee/utils": "3.7.4-beta.8",
+        "@apify/timeout": "^0.3.1",
+        "@crawlee/browser": "3.8.0",
+        "@crawlee/browser-pool": "3.8.0",
+        "@crawlee/core": "3.8.0",
+        "@crawlee/types": "3.8.0",
+        "@crawlee/utils": "3.8.0",
         "cheerio": "^1.0.0-rc.12",
         "idcac-playwright": "^0.1.2",
         "jquery": "^3.6.0",
+        "lodash.isequal": "^4.5.0",
+        "ml-logistic-regression": "^2.0.0",
         "ow": "^0.28.1",
+        "string-comparison": "^1.3.0",
         "tslib": "^2.4.0"
     },
     "peerDependencies": {
@@ -80,5 +85,5 @@
             }
         }
     },
-    "gitHead": "83abc8f1527c11bac121512097b50c6124ebf4fa"
+    "gitHead": "bd430d3de22c0a9f064cb00654a6cad3bc6cd601"
 }