@crawlee/playwright 4.0.0-beta.89 → 4.0.0-beta.90

package/index.d.ts

@@ -6,4 +6,4 @@ export { RenderingTypePredictor } from './internals/utils/rendering-type-predict
 export * as playwrightUtils from './internals/utils/playwright-utils.js';
 export * as playwrightClickElements from './internals/enqueue-links/click-elements.js';
 export type { DirectNavigationOptions as PlaywrightDirectNavigationOptions } from './internals/utils/playwright-utils.js';
-export type { RenderingType } from './internals/utils/rendering-type-prediction.js';
+export type { IRenderingTypePredictor, RenderingType } from './internals/utils/rendering-type-prediction.js';

package/internals/adaptive-playwright-crawler.d.ts

@@ -10,7 +10,7 @@ import type { AnyNode } from 'domhandler';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Page } from 'playwright';
 import type { PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';
-import { RenderingTypePredictor } from './utils/rendering-type-prediction.js';
+import { type IRenderingTypePredictor } from './utils/rendering-type-prediction.js';
 interface AdaptivePlaywrightCrawlerStatisticState extends StatisticState {
     httpOnlyRequestHandlerRuns?: number;
     browserRequestHandlerRuns?: number;
@@ -137,9 +137,11 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
      */
     resultComparator?: (resultA: RequestHandlerResult, resultB: RequestHandlerResult) => boolean | 'equal' | 'different' | 'inconclusive';
     /**
-     * A custom rendering type predictor
+     * A custom rendering type predictor. A predictor passed here is borrowed - the crawler never drives its
+     * lifecycle, so set it up yourself (the built-in {@link RenderingTypePredictor} needs `initialize()`).
+     * Omit the option and the crawler builds its own from `renderingTypeDetectionRatio` - and initializes it.
      */
-    renderingTypePredictor?: Pick<RenderingTypePredictor, 'predict' | 'storeResult' | 'initialize'>;
+    renderingTypePredictor?: IRenderingTypePredictor;
     /**
      * Prevent direct access to storage in request handlers (only allow using context helpers).
      * Defaults to `true`

package/internals/adaptive-playwright-crawler.js

@@ -2,11 +2,11 @@ import { isDeepStrictEqual } from 'node:util';
 import { BasicCrawler } from '@crawlee/basic';
 import { extractUrlsFromPage } from '@crawlee/browser';
 import { CheerioCrawler } from '@crawlee/cheerio';
-import { RequestHandlerError, RequestHandlerResult, resolveBaseUrlForEnqueueLinksFiltering, Router, serviceLocator, Statistics, withCheckedStorageAccess, } from '@crawlee/core';
+import { OwnedOrInjected, RequestHandlerError, RequestHandlerResult, resolveBaseUrlForEnqueueLinksFiltering, Router, serviceLocator, Statistics, withCheckedStorageAccess, } from '@crawlee/core';
 import { extractUrlsFromCheerio } from '@crawlee/utils';
 import { addTimeoutToPromise } from '@apify/timeout';
 import { PlaywrightCrawler } from './playwright-crawler.js';
-import { RenderingTypePredictor } from './utils/rendering-type-prediction.js';
+import { RenderingTypePredictor, } from './utils/rendering-type-prediction.js';
 class AdaptivePlaywrightCrawlerStatistics extends Statistics {
     state = null; // this needs to be assigned for a valid override, but the initialization is done by a reset() call from the parent constructor
     constructor(options = {}) {
@@ -103,8 +103,9 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
         });
         this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
-        this.renderingTypePredictor =
-            renderingTypePredictor ?? new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio });
+        // `renderingTypeDetectionRatio` only configures the default predictor - an injected one brings its own
+        // detection ratio (and its own state), so the option is ignored in that case.
+        this.renderingTypePredictor = OwnedOrInjected.resolve(renderingTypePredictor, () => new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio }));
         this.resultChecker = resultChecker ?? (() => true);
         this.shouldPropagateError = shouldPropagateError ?? (() => false);
         if (resultComparator !== undefined) {
@@ -162,7 +163,9 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         this.preventDirectStorageAccess = preventDirectStorageAccess;
     }
     async _init() {
-        await this.renderingTypePredictor.initialize();
+        // Only the predictor we built ourselves is ours to initialize - an injected one is borrowed, so its
+        // lifecycle (including restoring persisted state) stays with whoever created it.
+        await this.renderingTypePredictor.ifOwned((predictor) => predictor.initialize());
         return await super._init();
     }
     buildContextPipeline() {
@@ -302,7 +305,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         }
     }
     async runRequestHandler(crawlingContext) {
-        const renderingTypePrediction = this.renderingTypePredictor.predict(crawlingContext.request);
+        const renderingTypePrediction = this.renderingTypePredictor.value.predict(crawlingContext.request);
         const shouldDetectRenderingType = Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;
         if (!shouldDetectRenderingType) {
             crawlingContext.log.debug(`Predicted rendering type ${renderingTypePrediction.renderingType} for ${crawlingContext.request.url}`);
@@ -382,7 +385,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 })();
                 crawlingContext.log.debug(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
                 if (detectionResult !== undefined) {
-                    this.renderingTypePredictor.storeResult(crawlingContext.request, detectionResult);
+                    this.renderingTypePredictor.value.storeResult(crawlingContext.request, detectionResult);
                 }
             }
         }

package/internals/utils/rendering-type-prediction.d.ts

@@ -5,12 +5,27 @@ export interface RenderingTypePredictorOptions {
     detectionRatio: number;
     persistenceOptions?: Partial<RecoverableStatePersistenceOptions>;
 }
+/**
+ * Minimal contract that any object passed to {@link AdaptivePlaywrightCrawler} as its
+ * `renderingTypePredictor` option must satisfy.
+ *
+ * @experimental
+ */
+export interface IRenderingTypePredictor {
+    /** Predict the rendering type for a request, and how likely the crawler should be to verify it. */
+    predict(request: Request): {
+        renderingType: RenderingType;
+        detectionProbabilityRecommendation: number;
+    };
+    /** Report a detected rendering type, so that future predictions can take it into account. */
+    storeResult(requests: Request | Request[], renderingType: RenderingType): void;
+}
 /**
  * Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.
  *
  * @experimental
  */
-export declare class RenderingTypePredictor {
+export declare class RenderingTypePredictor implements IRenderingTypePredictor {
     private detectionRatio;
     private state;
     constructor({ detectionRatio, persistenceOptions }: RenderingTypePredictorOptions);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.89",
+    "version": "4.0.0-beta.90",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,13 +49,13 @@
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
         "@apify/timeout": "^0.3.2",
-        "@crawlee/basic": "4.0.0-beta.89",
-        "@crawlee/browser": "4.0.0-beta.89",
-        "@crawlee/browser-pool": "4.0.0-beta.89",
-        "@crawlee/cheerio": "4.0.0-beta.89",
-        "@crawlee/core": "4.0.0-beta.89",
-        "@crawlee/types": "4.0.0-beta.89",
-        "@crawlee/utils": "4.0.0-beta.89",
+        "@crawlee/basic": "4.0.0-beta.90",
+        "@crawlee/browser": "4.0.0-beta.90",
+        "@crawlee/browser-pool": "4.0.0-beta.90",
+        "@crawlee/cheerio": "4.0.0-beta.90",
+        "@crawlee/core": "4.0.0-beta.90",
+        "@crawlee/types": "4.0.0-beta.90",
+        "@crawlee/utils": "4.0.0-beta.90",
         "cheerio": "^1.0.0",
         "idcac-playwright": "^0.1.3",
         "jquery": "^3.7.1",
@@ -85,5 +85,5 @@
             }
         }
     },
-    "gitHead": "9279941162eb4be0a9113f768b5bd79d27e66eba"
+    "gitHead": "7afdd44c5b6850b4775cfc08ffbbdd1791ae6402"
 }