@crawlee/playwright 4.0.0-beta.105 → 4.0.0-beta.107

package/internals/adaptive-playwright-crawler.d.ts

@@ -174,23 +174,10 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
  * @experimental
  */
 export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes> {
-    private renderingTypePredictor;
-    private resultChecker;
-    private shouldPropagateError;
-    private resultComparator;
-    private staticContextPipeline;
-    private browserContextPipeline;
-    private individualRequestHandlerTimeoutMillis;
+    #private;
     get stats(): AdaptivePlaywrightCrawlerStatistics;
-    private inFlightRenderingTypeDetections;
-    /**
-     * The write policy of the per-attempt transactions. Defaults the request queue to `deferred`:
-     * a discarded attempt's enqueues must never reach the queue.
-     */
-    private readonly attemptWritePolicy;
-    private teardownHooks;
     constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
-    protected _init(): Promise<void>;
+    protected init(): Promise<void>;
     protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, CrawlingContext<Dictionary> & {
         readonly request: LoadedRequest<Request<Dictionary>>;
         readonly response: Response;

package/internals/adaptive-playwright-crawler.js

@@ -83,24 +83,23 @@ const proxyLogMethods = [
  * @experimental
  */
 export class AdaptivePlaywrightCrawler extends BasicCrawler {
-    renderingTypePredictor;
-    resultChecker;
-    shouldPropagateError;
-    resultComparator;
-    staticContextPipeline;
-    browserContextPipeline;
-    individualRequestHandlerTimeoutMillis;
+    #renderingTypePredictor;
+    #resultChecker;
+    #shouldPropagateError;
+    #resultComparator;
+    #staticContextPipeline;
+    #browserContextPipeline;
+    #individualRequestHandlerTimeoutMillis;
     // The constructor always injects an `AdaptivePlaywrightCrawlerStatistics`, so narrowing the cast is sound.
     get stats() {
         return super.stats;
     }
-    inFlightRenderingTypeDetections = 0;
     /**
      * The write policy of the per-attempt transactions. Defaults the request queue to `deferred`:
      * a discarded attempt's enqueues must never reach the queue.
      */
-    attemptWritePolicy;
-    teardownHooks = [];
+    #attemptWritePolicy;
+    #teardownHooks = [];
     constructor(options = {}) {
         const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statistics, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks = [], postNavigationHooks = [], extendContext, contextPipelineBuilder, transactionalStorage, ...rest } = options;
         // The user's value is replaced by `false` in the `super` call below — validate it separately.
@@ -134,24 +133,24 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             // user-facing option (validated above) to those.
             transactionalStorage: false,
         });
-        this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
+        this.#individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
         // `renderingTypeDetectionRatio` only configures the default predictor - an injected one brings its own
         // detection ratio (and its own state), so the option is ignored in that case.
-        this.renderingTypePredictor = OwnedOrInjected.resolve(renderingTypePredictor, () => new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio }));
-        this.attemptWritePolicy = {
+        this.#renderingTypePredictor = OwnedOrInjected.resolve(renderingTypePredictor, () => new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio }));
+        this.#attemptWritePolicy = {
             requestQueue: 'deferred',
             ...(typeof transactionalStorage === 'object' ? transactionalStorage : {}),
         };
-        this.resultChecker = resultChecker ?? (() => true);
-        this.shouldPropagateError = shouldPropagateError ?? (() => false);
+        this.#resultChecker = resultChecker ?? (() => true);
+        this.#shouldPropagateError = shouldPropagateError ?? (() => false);
         if (resultComparator !== undefined) {
-            this.resultComparator = resultComparator;
+            this.#resultComparator = resultComparator;
         }
         else if (resultChecker !== undefined) {
-            this.resultComparator = (resultA, resultB) => this.resultChecker(resultA) && this.resultChecker(resultB);
+            this.#resultComparator = (resultA, resultB) => this.#resultChecker(resultA) && this.#resultChecker(resultB);
         }
         else {
-            this.resultComparator = (resultA, resultB) => {
+            this.#resultComparator = (resultA, resultB) => {
                 return (resultA.datasetItems.length === resultB.datasetItems.length &&
                     resultA.datasetItems.every((itemA, i) => {
                         const itemB = resultB.datasetItems[i];
@@ -181,19 +180,19 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             postNavigationHooks: postNavigationHooks,
             extendContext,
         });
-        this.teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));
-        this.staticContextPipeline = staticCrawler.contextPipeline.compose({
+        this.#teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));
+        this.#staticContextPipeline = staticCrawler.contextPipeline.compose({
             action: this.adaptCheerioContext.bind(this),
         });
-        this.browserContextPipeline = browserCrawler.contextPipeline.compose({
+        this.#browserContextPipeline = browserCrawler.contextPipeline.compose({
             action: this.adaptPlaywrightContext.bind(this),
         });
     }
-    async _init() {
+    async init() {
         // Only the predictor we built ourselves is ours to initialize - an injected one is borrowed, so its
         // lifecycle (including restoring persisted state) stays with whoever created it.
-        await this.renderingTypePredictor.ifOwned((predictor) => predictor.initialize());
-        return await super._init();
+        await this.#renderingTypePredictor.ifOwned((predictor) => predictor.initialize());
+        return await super.init();
     }
     buildContextPipeline() {
         const errorMessage = (prop) => `The \`${prop}\` property is not available on the outer context pipeline of AdaptivePlaywrightCrawler - it is provided by the inner (static/browser) pipelines`;
@@ -289,7 +288,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
      */
     async crawlOne(renderingType, context, useStateFunction, transactions) {
         const transaction = createStorageTransaction({
-            policy: this.attemptWritePolicy,
+            policy: this.#attemptWritePolicy,
             commitTimeoutMillis: this.internalTimeoutMillis,
         });
         transactions.push(transaction);
@@ -309,16 +308,16 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         try {
             const callAdaptiveRequestHandler = async () => {
                 if (renderingType === 'static') {
-                    await this.staticContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));
+                    await this.#staticContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));
                 }
                 else if (renderingType === 'clientOnly') {
-                    await this.browserContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));
+                    await this.#browserContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));
                 }
             };
             // this crawler overrides `runRequestHandler` and times each rendering-type run itself, so it has
             // to resolve any per-route override too - otherwise routes would be silently ignored here
             const routeTimeoutSecs = this.requestHandler.getTimeoutSecs?.(context.request.label);
-            const timeoutMillis = routeTimeoutSecs === undefined ? this.individualRequestHandlerTimeoutMillis : routeTimeoutSecs * 1000;
+            const timeoutMillis = routeTimeoutSecs === undefined ? this.#individualRequestHandlerTimeoutMillis : routeTimeoutSecs * 1000;
             await addTimeoutToPromise(async () => transaction.run(callAdaptiveRequestHandler), timeoutMillis, 'Request handler timed out');
             return { result: transaction, ok: true, logs };
         }
@@ -330,14 +329,11 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         }
     }
     async runRequestHandler(crawlingContext) {
-        const renderingTypePrediction = this.renderingTypePredictor.value.predict(crawlingContext.request);
+        const renderingTypePrediction = this.#renderingTypePredictor.value.predict(crawlingContext.request);
         const shouldDetectRenderingType = Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;
         if (!shouldDetectRenderingType) {
             crawlingContext.log.debug(`Predicted rendering type ${renderingTypePrediction.renderingType} for ${crawlingContext.request.url}`);
         }
-        if (shouldDetectRenderingType) {
-            this.inFlightRenderingTypeDetections += 1;
-        }
         // Every transaction created for this request - up to two, since the static-then-browser
         // fall-through and the browser-then-detection pair are mutually exclusive. Disposed in the
         // `finally` below, not earlier: the comparators read the journals after `crawlOne` returns.
@@ -347,7 +343,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
                 this.stats.trackHttpOnlyRequestHandlerRun();
                 const plainHTTPRun = await this.crawlOne('static', crawlingContext, crawlingContext.useState, transactions);
-                if (plainHTTPRun.ok && this.resultChecker(plainHTTPRun.result)) {
+                if (plainHTTPRun.ok && this.#resultChecker(plainHTTPRun.result)) {
                     crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
                     plainHTTPRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
                     await plainHTTPRun.result.commit();
@@ -358,7 +354,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                     const actualError = plainHTTPRun.error instanceof RequestHandlerError
                         ? plainHTTPRun.error.cause
                         : plainHTTPRun.error;
-                    if (await this.shouldPropagateError(actualError, crawlingContext)) {
+                    if (await this.#shouldPropagateError(actualError, crawlingContext)) {
                         throw actualError;
                     }
                     crawlingContext.log.exception(actualError, `HTTP-only request handler failed for ${crawlingContext.request.url}`);
@@ -405,7 +401,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                     if (!plainHTTPRun.ok) {
                         return 'clientOnly';
                     }
-                    const comparisonResult = this.resultComparator(plainHTTPRun.result, browserRun.result);
+                    const comparisonResult = this.#resultComparator(plainHTTPRun.result, browserRun.result);
                     if (comparisonResult === true || comparisonResult === 'equal') {
                         return 'static';
                     }
@@ -416,14 +412,11 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 })();
                 crawlingContext.log.debug(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
                 if (detectionResult !== undefined) {
-                    this.renderingTypePredictor.value.storeResult(crawlingContext.request, detectionResult);
+                    this.#renderingTypePredictor.value.storeResult(crawlingContext.request, detectionResult);
                 }
             }
         }
         finally {
-            if (shouldDetectRenderingType) {
-                this.inFlightRenderingTypeDetections -= 1;
-            }
             // A still-open transaction here belongs to a discarded attempt - roll it back, then release.
             for (const transaction of transactions) {
                 transaction.rollback();
@@ -456,7 +449,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
     }
     async teardown() {
         await super.teardown();
-        for (const hook of this.teardownHooks) {
+        for (const hook of this.#teardownHooks) {
             await hook();
         }
     }

package/internals/playwright-crawler.d.ts

@@ -264,7 +264,7 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
         closeCookieModals: () => Promise<void>;
         handleCloudflareChallenge: (options?: HandleCloudflareChallengeOptions) => Promise<Response | undefined>;
     }>;
-    protected _navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<Response | null>;
+    protected navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: DirectNavigationOptions): Promise<Response | null>;
     private enhanceContext;
 }
 /**

package/internals/playwright-crawler.js

@@ -109,7 +109,7 @@ export class PlaywrightCrawler extends BrowserCrawler {
     buildContextPipeline() {
         return super.buildContextPipeline().compose({ action: this.enhanceContext.bind(this) });
     }
-    async _navigationHandler(crawlingContext, gotoOptions) {
+    async navigationHandler(crawlingContext, gotoOptions) {
         return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
     }
     async enhanceContext(context) {

package/internals/utils/rendering-type-prediction.d.ts

@@ -26,7 +26,7 @@ export interface IRenderingTypePredictor {
  * @experimental
  */
 export declare class RenderingTypePredictor implements IRenderingTypePredictor {
-    private detectionRatio;
+    #private;
     private state;
     constructor({ detectionRatio, persistenceOptions }: RenderingTypePredictorOptions);
     /**

package/internals/utils/rendering-type-prediction.js

@@ -30,10 +30,11 @@ const mean = (values) => (values.length > 0 ? sum(values) / values.length : unde
  * @experimental
  */
 export class RenderingTypePredictor {
-    detectionRatio;
+    #detectionRatio;
+    // kept as TS-private: tests reach for it at runtime
     state;
     constructor({ detectionRatio, persistenceOptions }) {
-        this.detectionRatio = detectionRatio;
+        this.#detectionRatio = detectionRatio;
         this.state = new RecoverableState({
             defaultState: {
                 logreg: new LogisticRegression({ numSteps: 1000, learningRate: 0.05 }),
@@ -86,7 +87,7 @@ export class RenderingTypePredictor {
             renderingType: prediction === 1 ? 'static' : 'clientOnly',
             detectionProbabilityRecommendation: Math.abs(scores[0] - scores[1]) < 0.1
                 ? 1
-                : this.detectionRatio * Math.max(1, 5 - this.resultCount(label)),
+                : this.#detectionRatio * Math.max(1, 5 - this.resultCount(label)),
         };
     }
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.105",
+    "version": "4.0.0-beta.107",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,15 +49,14 @@
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
         "@apify/timeout": "^0.4.4",
-        "@crawlee/basic": "4.0.0-beta.105",
-        "@crawlee/browser": "4.0.0-beta.105",
-        "@crawlee/browser-pool": "4.0.0-beta.105",
-        "@crawlee/cheerio": "4.0.0-beta.105",
-        "@crawlee/core": "4.0.0-beta.105",
-        "@crawlee/types": "4.0.0-beta.105",
-        "@crawlee/utils": "4.0.0-beta.105",
+        "@crawlee/basic": "4.0.0-beta.107",
+        "@crawlee/browser": "4.0.0-beta.107",
+        "@crawlee/browser-pool": "4.0.0-beta.107",
+        "@crawlee/cheerio": "4.0.0-beta.107",
+        "@crawlee/core": "4.0.0-beta.107",
+        "@crawlee/types": "4.0.0-beta.107",
+        "@crawlee/utils": "4.0.0-beta.107",
         "cheerio": "^1.0.0",
-        "idcac-playwright": "^0.1.3",
         "jquery": "^3.7.1",
         "ml-logistic-regression": "^2.0.0",
         "ml-matrix": "^6.12.1",
@@ -85,5 +84,5 @@
             }
         }
     },
-    "gitHead": "26073a822c7699ac487931383a39192bc3daae7a"
+    "gitHead": "5a3cb6242d0ae261df93c5e03cfebbb9b736e6b7"
 }