@crawlee/playwright 4.0.0-beta.210 → 4.0.0-beta.212

package/internals/adaptive-playwright-crawler.d.ts

@@ -185,18 +185,7 @@ export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<nev
     #private;
     constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
     protected init(): Promise<void>;
-    private adaptCheerioContext;
-    private adaptPlaywrightContext;
-    /**
-     * Runs one request handler attempt inside its own {@link StorageTransaction}, wrapping the inner
-     * (static or browser) context pipeline. The transaction is pushed to `transactions` *at creation
-     * time, before the `try`* - the `ok: false` branch of the returned {@link Result} carries no
-     * result, and failed attempts are routine here. The caller owns the outcome and disposal.
-     */
-    private crawlOne;
     protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
-    private enqueueLinks;
-    private createLogProxy;
     /**
      * Number of rendering type detections that have not settled yet, including results the predictor is
      * still persisting.

package/internals/adaptive-playwright-crawler.js

@@ -185,8 +185,8 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         });
         this.#staticCrawler = staticCrawler;
         this.#browserCrawler = browserCrawler;
-        this.#staticContextPipeline = staticCrawler.contextPipeline.compose(this.adaptCheerioContext.bind(this));
-        this.#browserContextPipeline = browserCrawler.contextPipeline.compose(this.adaptPlaywrightContext.bind(this));
+        this.#staticContextPipeline = staticCrawler.contextPipeline.compose(this.#adaptCheerioContext.bind(this));
+        this.#browserContextPipeline = browserCrawler.contextPipeline.compose(this.#adaptPlaywrightContext.bind(this));
     }
     async init() {
         // A crawler can be run again after a teardown.
@@ -225,7 +225,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             },
         }));
     }
-    async adaptCheerioContext(cheerioContext) {
+    async #adaptCheerioContext(cheerioContext) {
         return {
             get page() {
                 throw new Error('Page object was used in HTTP-only request handler');
@@ -238,12 +238,12 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             },
             enqueueLinks: async (options = {}) => {
                 const urls = extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
-                return (await this.enqueueLinks(urls, options, cheerioContext.request));
+                return (await this.#enqueueLinks(urls, options, cheerioContext.request));
             },
             response: cheerioContext.response,
         };
     }
-    async adaptPlaywrightContext(playwrightContext) {
+    async #adaptPlaywrightContext(playwrightContext) {
         // Capture the original response to avoid infinite recursion when the getter is copied to the context
         const originalResponse = playwrightContext.response;
         return {
@@ -270,7 +270,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 const locator = playwrightContext.page.locator(selector).first();
                 await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
                 const urls = await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl);
-                return (await this.enqueueLinks(urls, options, playwrightContext.request));
+                return (await this.#enqueueLinks(urls, options, playwrightContext.request));
             },
         };
     }
@@ -280,7 +280,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
      * time, before the `try`* - the `ok: false` branch of the returned {@link Result} carries no
      * result, and failed attempts are routine here. The caller owns the outcome and disposal.
      */
-    async crawlOne(renderingType, context, useStateFunction, transactions) {
+    async #crawlOne(renderingType, context, useStateFunction, transactions) {
         const transaction = createStorageTransaction({
             policy: this.#attemptWritePolicy,
             commitTimeoutMillis: this.internalTimeoutMillis,
@@ -290,7 +290,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         const deferredCleanup = [];
         const attemptBoundContextHelpers = {
             useState: useStateFunction,
-            log: this.createLogProxy(context.log, logs),
+            log: this.#createLogProxy(context.log, logs),
             registerDeferredCleanup: (cleanup) => deferredCleanup.push(cleanup),
         };
         const subCrawlerContext = Object.defineProperties({}, Object.getOwnPropertyDescriptors(context));
@@ -340,7 +340,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
                 crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
                 this.statistics.state.httpOnlyRequestHandlerRuns++;
-                const plainHTTPRun = await this.crawlOne('static', crawlingContext, crawlingContext.useState, transactions);
+                const plainHTTPRun = await this.#crawlOne('static', crawlingContext, crawlingContext.useState, transactions);
                 if (plainHTTPRun.ok && this.#resultChecker(plainHTTPRun.result)) {
                     crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
                     plainHTTPRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
@@ -385,7 +385,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                     return this.stateCopy;
                 },
             };
-            const browserRun = await this.crawlOne('clientOnly', crawlingContext, stateTracker.getLiveState.bind(stateTracker), transactions);
+            const browserRun = await this.#crawlOne('clientOnly', crawlingContext, stateTracker.getLiveState.bind(stateTracker), transactions);
             if (!browserRun.ok) {
                 throw browserRun.error;
             }
@@ -396,7 +396,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                     crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
                     // The detection attempt's transaction is never committed - its writes exist only for the
                     // result comparison.
-                    const plainHTTPRun = await this.crawlOne('static', crawlingContext, stateTracker.getStateCopy.bind(stateTracker), transactions);
+                    const plainHTTPRun = await this.#crawlOne('static', crawlingContext, stateTracker.getStateCopy.bind(stateTracker), transactions);
                     const detectionResult = (() => {
                         if (!plainHTTPRun.ok) {
                             return 'clientOnly';
@@ -432,7 +432,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             }
         }
     }
-    async enqueueLinks(urls, options, request) {
+    async #enqueueLinks(urls, options, request) {
         const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
             enqueueStrategy: options?.strategy,
             finalRequestUrl: request.loadedUrl,
@@ -448,7 +448,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             strategy: options.strategy ?? EnqueueStrategy.SameHostname,
         });
     }
-    createLogProxy(log, logs) {
+    #createLogProxy(log, logs) {
         return new Proxy(log, {
             get(target, propertyName) {
                 if (proxyLogMethods.includes(propertyName)) {

package/internals/playwright-crawler.d.ts

@@ -249,7 +249,6 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
      */
     constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
     protected navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<Response | null>;
-    private enhanceContext;
 }
 /**
  * Returns a `postNavigationHooks`-ready hook that runs {@link PlaywrightContextUtils.handleCloudflareChallenge}

package/internals/playwright-crawler.js

@@ -106,12 +106,12 @@ export class PlaywrightCrawler extends BrowserCrawler {
         });
     }
     #buildContextPipeline() {
-        return this.buildContextPipeline().compose(this.enhanceContext.bind(this));
+        return this.buildContextPipeline().compose(this.#enhanceContext.bind(this));
     }
     async navigationHandler(crawlingContext, gotoOptions) {
         return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
     }
-    async enhanceContext(context) {
+    async #enhanceContext(context) {
         const waitForSelector = async (selector, timeoutMs = 5_000) => {
             const locator = context.page.locator(selector).first();
             await locator.waitFor({ timeout: timeoutMs, state: 'attached' });

package/internals/utils/rendering-type-prediction.d.ts

@@ -60,7 +60,4 @@ export declare class RenderingTypePredictor implements IRenderingTypePredictor {
      * Store the rendering type for a given URL and request label. This updates the underlying prediction model, which may be costly.
      */
     storeResult(requests: Request | Request[], renderingType: RenderingType): void;
-    private resultCount;
-    private calculateFeatureVector;
-    private retrain;
 }

package/internals/utils/rendering-type-prediction.js

@@ -104,14 +104,14 @@ export class RenderingTypePredictor {
             return { renderingType: 'clientOnly', detectionProbabilityRecommendation: 1 };
         }
         const predictionUrl = new URL(loadedUrl ?? url);
-        const urlFeature = new Matrix([this.calculateFeatureVector(urlComponents(predictionUrl), label)]);
+        const urlFeature = new Matrix([this.#calculateFeatureVector(urlComponents(predictionUrl), label)]);
         const [prediction] = logreg.predict(urlFeature);
         const scores = [logreg.classifiers[0].testScores(urlFeature), logreg.classifiers[1].testScores(urlFeature)];
         return {
             renderingType: prediction === 1 ? 'static' : 'clientOnly',
             detectionProbabilityRecommendation: Math.abs(scores[0] - scores[1]) < 0.1
                 ? 1
-                : this.#detectionRatio * Math.max(1, 5 - this.resultCount(label)),
+                : this.#detectionRatio * Math.max(1, 5 - this.#resultCount(label)),
         };
     }
     /**
@@ -129,20 +129,20 @@ export class RenderingTypePredictor {
             }
             state.detectionResults.get(renderingType).get(label).push(urlComponents(resultUrl));
         }
-        this.retrain();
+        this.#retrain();
     }
-    resultCount(label) {
+    #resultCount(label) {
         return Array.from(this.#state.currentValue.detectionResults.values())
             .map((results) => results.get(label)?.length ?? 0)
             .reduce((acc, value) => acc + value, 0);
     }
-    calculateFeatureVector(url, label) {
+    #calculateFeatureVector(url, label) {
         return [
             mean((this.#state.currentValue.detectionResults.get('static')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
             mean((this.#state.currentValue.detectionResults.get('clientOnly')?.get(label) ?? []).map((otherUrl) => calculateUrlSimilarity(url, otherUrl) ?? 0)) ?? 0,
         ];
     }
-    retrain() {
+    #retrain() {
         const X = [
             [0, 1],
             [1, 0],
@@ -151,7 +151,7 @@ export class RenderingTypePredictor {
         for (const [renderingType, urlsByLabel] of this.#state.currentValue.detectionResults.entries()) {
             for (const [label, urls] of urlsByLabel) {
                 for (const url of urls) {
-                    X.push(this.calculateFeatureVector(url, label));
+                    X.push(this.#calculateFeatureVector(url, label));
                     Y.push(renderingType === 'static' ? 1 : 0);
                 }
             }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.210",
+    "version": "4.0.0-beta.212",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,13 +49,13 @@
     "dependencies": {
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
-        "@crawlee/basic": "4.0.0-beta.210",
-        "@crawlee/browser": "4.0.0-beta.210",
-        "@crawlee/browser-pool": "4.0.0-beta.210",
-        "@crawlee/cheerio": "4.0.0-beta.210",
-        "@crawlee/core": "4.0.0-beta.210",
-        "@crawlee/types": "4.0.0-beta.210",
-        "@crawlee/utils": "4.0.0-beta.210",
+        "@crawlee/basic": "4.0.0-beta.212",
+        "@crawlee/browser": "4.0.0-beta.212",
+        "@crawlee/browser-pool": "4.0.0-beta.212",
+        "@crawlee/cheerio": "4.0.0-beta.212",
+        "@crawlee/core": "4.0.0-beta.212",
+        "@crawlee/types": "4.0.0-beta.212",
+        "@crawlee/utils": "4.0.0-beta.212",
         "cheerio": "^1.0.0",
         "jquery": "^3.7.1",
         "ml-logistic-regression": "^2.0.0",
@@ -80,5 +80,5 @@
             }
         }
     },
-    "gitHead": "037dff5d651d88e96749cc79d17bc6d1e154f4e1"
+    "gitHead": "171a63a5d7f2a6d74cbce58479204ac20d30b87e"
 }