@crawlee/playwright 4.0.0-beta.163 → 4.0.0-beta.164

package/internals/adaptive-playwright-crawler.d.ts

@@ -198,6 +198,24 @@ export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<nev
     protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
     private enqueueLinks;
     private createLogProxy;
+    /**
+     * Number of rendering type detections that have not settled yet, including results the predictor is
+     * still persisting.
+     */
+    get inFlightRenderingTypeDetectionCount(): number;
+    /**
+     * Waits for in-flight rendering type detections to settle, bounded by `timeoutMillis` (defaults to the
+     * internal timeout).
+     */
+    drainRenderingDetections({ timeoutMillis }?: {
+        timeoutMillis?: number;
+    }): Promise<void>;
+    /**
+     * Stops the crawler immediately, but not before rendering type detections already under way (and results
+     * the predictor is still persisting) have settled - see
+     * {@link AdaptivePlaywrightCrawler.drainRenderingDetections|`drainRenderingDetections()`}. Requests
+     * that are still running are not waited for, unlike {@link BasicCrawler.stop|`stop()`}.
+     */
     teardown(): Promise<void>;
 }
 export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;

package/internals/adaptive-playwright-crawler.js

@@ -1,3 +1,4 @@
+import { setTimeout as delay } from 'node:timers/promises';
 import { isDeepStrictEqual } from 'node:util';
 import { BasicCrawler } from '@crawlee/basic';
 import { extractUrlsFromPage } from '@crawlee/browser';
@@ -76,6 +77,14 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
      * a discarded attempt's enqueues must never reach the queue.
      */
     #attemptWritePolicy;
+    /**
+     * In-flight rendering type detections, plus the pending results of an asynchronous `storeResult`.
+     */
+    #activeDetections = new Set();
+    /**
+     * Set once `teardown()` starts, so that requests still in the pool stop opening new detections.
+     */
+    #shutDown = false;
     #teardownHooks = [];
     constructor(options = {}) {
         const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statistics, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks = [], postNavigationHooks = [], extendContext, contextPipelineBuilder, transactionalStorage, launchContext, headless, browserPool, remoteBrowser, ...rest } = options;
@@ -180,6 +189,8 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         });
     }
     async init() {
+        // A crawler can be run again after a teardown.
+        this.#shutDown = false;
         // Only the predictor we built ourselves is ours to initialize - an injected one is borrowed, so its
         // lifecycle (including restoring persisted state) stays with whoever created it.
         await this.#renderingTypePredictor.ifOwned((predictor) => predictor.initialize());
@@ -314,8 +325,8 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         }
     }
     async runRequestHandler(crawlingContext) {
-        const renderingTypePrediction = this.#renderingTypePredictor.value.predict(crawlingContext.request);
-        const shouldDetectRenderingType = Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;
+        const renderingTypePrediction = await this.#renderingTypePredictor.value.predict(crawlingContext.request);
+        const shouldDetectRenderingType = !this.#shutDown && Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;
         if (!shouldDetectRenderingType) {
             crawlingContext.log.debug(`Predicted rendering type ${renderingTypePrediction.renderingType} for ${crawlingContext.request.url}`);
         }
@@ -378,27 +389,36 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             browserRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
             await browserRun.result.commit();
             if (shouldDetectRenderingType) {
-                crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
-                // The detection attempt's transaction is never committed - its writes exist only for the
-                // result comparison.
-                const plainHTTPRun = await this.crawlOne('static', crawlingContext, stateTracker.getStateCopy.bind(stateTracker), transactions);
-                const detectionResult = (() => {
-                    if (!plainHTTPRun.ok) {
-                        return 'clientOnly';
-                    }
-                    const comparisonResult = this.#resultComparator(plainHTTPRun.result, browserRun.result);
-                    if (comparisonResult === true || comparisonResult === 'equal') {
-                        return 'static';
-                    }
-                    if (comparisonResult === false || comparisonResult === 'different') {
-                        return 'clientOnly';
+                const detectionPromise = (async () => {
+                    crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
+                    // The detection attempt's transaction is never committed - its writes exist only for the
+                    // result comparison.
+                    const plainHTTPRun = await this.crawlOne('static', crawlingContext, stateTracker.getStateCopy.bind(stateTracker), transactions);
+                    const detectionResult = (() => {
+                        if (!plainHTTPRun.ok) {
+                            return 'clientOnly';
+                        }
+                        const comparisonResult = this.#resultComparator(plainHTTPRun.result, browserRun.result);
+                        if (comparisonResult === true || comparisonResult === 'equal') {
+                            return 'static';
+                        }
+                        if (comparisonResult === false || comparisonResult === 'different') {
+                            return 'clientOnly';
+                        }
+                        return undefined;
+                    })();
+                    crawlingContext.log.debug(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
+                    if (detectionResult !== undefined) {
+                        // Deliberately not awaited: a predictor that persists asynchronously gets to keep
+                        // batching its writes, and the drain below catches whatever is still pending.
+                        const stored = this.#renderingTypePredictor.value.storeResult(crawlingContext.request, detectionResult);
+                        if (stored !== undefined) {
+                            // Nothing downstream awaits this, so a failed write would otherwise be silent.
+                            void this.#trackDetection(Promise.resolve(stored).catch((error) => this.log.exception(error, `Failed to store the rendering type detection result for ${crawlingContext.request.url}`)));
+                        }
                     }
-                    return undefined;
                 })();
-                crawlingContext.log.debug(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
-                if (detectionResult !== undefined) {
-                    this.#renderingTypePredictor.value.storeResult(crawlingContext.request, detectionResult);
-                }
+                await this.#trackDetection(detectionPromise);
             }
         }
         finally {
@@ -443,7 +463,66 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             },
         });
     }
+    #trackDetection(promise) {
+        this.#activeDetections.add(promise);
+        // Not `finally()`: the promise it derives would reject on its own and go unhandled. A rejection here
+        // belongs to whoever awaits the original, or to `allSettled` in the drain.
+        void promise.catch(() => { }).then(() => this.#activeDetections.delete(promise));
+        return promise;
+    }
+    /**
+     * Number of rendering type detections that have not settled yet, including results the predictor is
+     * still persisting.
+     */
+    get inFlightRenderingTypeDetectionCount() {
+        return this.#activeDetections.size;
+    }
+    /**
+     * Waits for in-flight rendering type detections to settle, bounded by `timeoutMillis` (defaults to the
+     * internal timeout).
+     */
+    async drainRenderingDetections({ timeoutMillis } = {}) {
+        if (this.#activeDetections.size === 0) {
+            return;
+        }
+        const drained = (async () => {
+            while (this.#activeDetections.size > 0) {
+                await Promise.allSettled(Array.from(this.#activeDetections));
+            }
+        })();
+        const millis = timeoutMillis ?? this.internalTimeoutMillis;
+        // A caller opting out of the bound would otherwise get an immediate spurious timeout - `setTimeout`
+        // clamps a non-finite delay to 1ms.
+        if (!Number.isFinite(millis)) {
+            await drained;
+            return;
+        }
+        const abortTimer = new AbortController();
+        try {
+            const outcome = await Promise.race([
+                drained.then(() => 'drained'),
+                delay(millis, 'timedOut', { signal: abortTimer.signal }).catch(() => 'aborted'),
+            ]);
+            if (outcome === 'timedOut') {
+                this.log.warning(`Timed out after ${millis / 1e3} seconds waiting for ${this.#activeDetections.size} rendering type detection(s) to settle - their results may be lost.`);
+            }
+        }
+        finally {
+            abortTimer.abort();
+        }
+    }
+    /**
+     * Stops the crawler immediately, but not before rendering type detections already under way (and results
+     * the predictor is still persisting) have settled - see
+     * {@link AdaptivePlaywrightCrawler.drainRenderingDetections|`drainRenderingDetections()`}. Requests
+     * that are still running are not waited for, unlike {@link BasicCrawler.stop|`stop()`}.
+     */
     async teardown() {
+        // Called from outside `run()` - under `keepAlive`, say - the pool keeps dispatching until
+        // `super.teardown()` aborts it, and a request starting during the drain would open a detection the
+        // drain has already passed. Closing that first makes the drain a fence.
+        this.#shutDown = true;
+        await this.drainRenderingDetections();
         await super.teardown();
         // Mirrors the owned-only `initialize()` in `init()` - without this, the predictor we built keeps its
         // PERSIST_STATE listener registered after the crawl and never gets a final write.

package/internals/utils/rendering-type-prediction.d.ts

@@ -1,4 +1,5 @@
 import type { RecoverableStatePersistenceOptions, Request } from '@crawlee/core';
+import type { Awaitable } from '@crawlee/types';
 export type RenderingType = 'clientOnly' | 'static';
 export interface RenderingTypePredictorOptions {
     /** A number between 0 and 1 that determines the desired ratio of rendering type detections */
@@ -12,13 +13,24 @@ export interface RenderingTypePredictorOptions {
  * @experimental
  */
 export interface IRenderingTypePredictor {
-    /** Predict the rendering type for a request, and how likely the crawler should be to verify it. */
-    predict(request: Request): {
+    /**
+     * Predict the rendering type for a request, and how likely the crawler should be to verify it.
+     *
+     * Called once per request, before navigation - the crawler awaits it, so prefer loading whatever the
+     * prediction needs up front over per-request I/O.
+     */
+    predict(request: Request): Awaitable<{
         renderingType: RenderingType;
         detectionProbabilityRecommendation: number;
-    };
-    /** Report a detected rendering type, so that future predictions can take it into account. */
-    storeResult(requests: Request | Request[], renderingType: RenderingType): void;
+    }>;
+    /**
+     * Report a detected rendering type, so that future predictions can take it into account.
+     *
+     * The crawler does not await this per detection - it collects the returned promises and drains them in
+     * `teardown()`, so an implementation that persists results asynchronously can keep batching its writes
+     * and still be sure they land before the crawl ends.
+     */
+    storeResult(requests: Request | Request[], renderingType: RenderingType): Awaitable<void>;
 }
 /**
  * Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.163",
+    "version": "4.0.0-beta.164",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,13 +49,13 @@
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
         "@apify/timeout": "^0.4.4",
-        "@crawlee/basic": "4.0.0-beta.163",
-        "@crawlee/browser": "4.0.0-beta.163",
-        "@crawlee/browser-pool": "4.0.0-beta.163",
-        "@crawlee/cheerio": "4.0.0-beta.163",
-        "@crawlee/core": "4.0.0-beta.163",
-        "@crawlee/types": "4.0.0-beta.163",
-        "@crawlee/utils": "4.0.0-beta.163",
+        "@crawlee/basic": "4.0.0-beta.164",
+        "@crawlee/browser": "4.0.0-beta.164",
+        "@crawlee/browser-pool": "4.0.0-beta.164",
+        "@crawlee/cheerio": "4.0.0-beta.164",
+        "@crawlee/core": "4.0.0-beta.164",
+        "@crawlee/types": "4.0.0-beta.164",
+        "@crawlee/utils": "4.0.0-beta.164",
         "cheerio": "^1.0.0",
         "jquery": "^3.7.1",
         "ml-logistic-regression": "^2.0.0",
@@ -80,5 +80,5 @@
             }
         }
     },
-    "gitHead": "3bcb1a803a1ba3082734c4b1aee72f7b3b51f427"
+    "gitHead": "a2e4e2a3aa3a364c8a44fd2e49ca57f500d2c01f"
 }