@crawlee/playwright 4.0.0-beta.238 → 4.0.0-beta.239

package/internals/adaptive-playwright-crawler.d.ts

@@ -83,7 +83,7 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =
     parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
     enqueueLinks(options?: EnqueueLinksOptions): Promise<unknown>;
 }
-interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'session' | 'proxyInfo' | 'log'> {
+interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'session' | 'proxyInfo' | 'log' | 'extendTimeout'> {
     page?: Page;
     request: CrawlingRequest;
     gotoOptions?: PlaywrightGotoOptions;
@@ -98,6 +98,12 @@ export interface AdaptiveResultContext {
     request: CrawlingRequest;
 }
 export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'preNavigationHooks' | 'postNavigationHooks' | 'contextPipelineBuilder'>, Pick<PlaywrightCrawlerOptions, 'launchContext' | 'headless' | 'browserPool' | 'remoteBrowser'> {
+    /**
+     * Timeout for the navigation phase of each attempt, in seconds, shared by the `preNavigationHooks`, the
+     * navigation and the `postNavigationHooks`. Applies to both the HTTP-only and the browser attempt. When not
+     * set, each attempt keeps its own default: 30 seconds for HTTP-only and 60 seconds for the browser.
+     */
+    navigationTimeoutSecs?: number;
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.
      * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,

package/internals/adaptive-playwright-crawler.js

@@ -1,6 +1,6 @@
 import { setTimeout as delay } from 'node:timers/promises';
 import { isDeepStrictEqual } from 'node:util';
-import { BasicCrawler, ContextPipeline, ContextPipelineInitializationError, ContextPipelineInterruptedError, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/basic';
+import { BasicCrawler, ContextPipeline, ContextPipelineInitializationError, ContextPipelineInterruptedError, navigationDeadlineKey, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/basic';
 import { extractUrlsFromPage } from '@crawlee/browser';
 import { CheerioCrawler } from '@crawlee/cheerio';
 import { ResponseWithUrl } from '@crawlee/http-client';
@@ -89,7 +89,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
      */
     #shutDown = false;
     constructor(options = {}) {
-        const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statistics, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks = [], postNavigationHooks = [], extendContext, transactionalStorage, launchContext, headless, browserPool, remoteBrowser, ...rest } = options;
+        const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statistics, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks = [], postNavigationHooks = [], extendContext, transactionalStorage, navigationTimeoutSecs, launchContext, headless, browserPool, remoteBrowser, ...rest } = options;
         // The user's value is replaced by `false` in the `super` call below — validate it separately,
         // wrapped in an object so the error still names the field.
         parseArgument({ transactionalStorage }, z.object({ transactionalStorage: BasicCrawler.optionsShape.transactionalStorage }), 'AdaptivePlaywrightCrawlerOptions');
@@ -170,6 +170,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             preNavigationHooks,
             postNavigationHooks,
             extendContext,
+            navigationTimeoutSecs,
         });
         const browserCrawler = new PlaywrightCrawler({
             ...rest,
@@ -177,6 +178,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             preNavigationHooks: preNavigationHooks,
             postNavigationHooks: postNavigationHooks,
             extendContext,
+            navigationTimeoutSecs,
             launchContext,
             headless,
             browserPool,
@@ -276,12 +278,20 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         transactions.push(transaction);
         const logs = [];
         const deferredCleanup = [];
+        const subCrawlerContext = Object.defineProperties({}, Object.getOwnPropertyDescriptors(context));
         const attemptBoundContextHelpers = {
             useState: useStateFunction,
             log: this.#createLogProxy(context.log, logs),
             registerDeferredCleanup: (cleanup) => deferredCleanup.push(cleanup),
+            // The sub-crawler pipeline keeps this attempt's navigation deadline on `subCrawlerContext`, but the
+            // outer `extendTimeout` only pushes the deadline it finds on `context`, so push this one as well.
+            extendTimeout: (secs) => {
+                context.extendTimeout(secs);
+                if (subCrawlerContext[navigationDeadlineKey] !== undefined) {
+                    subCrawlerContext[navigationDeadlineKey] += secs * 1000;
+                }
+            },
         };
-        const subCrawlerContext = Object.defineProperties({}, Object.getOwnPropertyDescriptors(context));
         // Mark attempt-bound helpers as non-configurable so they survive the sub-crawler context pipeline
         // (which would otherwise override them with the sub-crawler's own versions, losing the binding).
         for (const [key, descriptor] of Object.entries(Object.getOwnPropertyDescriptors(attemptBoundContextHelpers))) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.238",
+    "version": "4.0.0-beta.239",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.13.0"
@@ -49,14 +49,14 @@
     "dependencies": {
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
-        "@crawlee/basic": "4.0.0-beta.238",
-        "@crawlee/browser": "4.0.0-beta.238",
-        "@crawlee/browser-pool": "4.0.0-beta.238",
-        "@crawlee/cheerio": "4.0.0-beta.238",
-        "@crawlee/core": "4.0.0-beta.238",
-        "@crawlee/http-client": "4.0.0-beta.238",
-        "@crawlee/types": "4.0.0-beta.238",
-        "@crawlee/utils": "4.0.0-beta.238",
+        "@crawlee/basic": "4.0.0-beta.239",
+        "@crawlee/browser": "4.0.0-beta.239",
+        "@crawlee/browser-pool": "4.0.0-beta.239",
+        "@crawlee/cheerio": "4.0.0-beta.239",
+        "@crawlee/core": "4.0.0-beta.239",
+        "@crawlee/http-client": "4.0.0-beta.239",
+        "@crawlee/types": "4.0.0-beta.239",
+        "@crawlee/utils": "4.0.0-beta.239",
         "cheerio": "^1.0.0",
         "domhandler": "^6.0.0",
         "jquery": "^3.7.1",
@@ -81,5 +81,5 @@
             }
         }
     },
-    "gitHead": "8c6b2f3b5a811ca68672b6c68b6587a925baf2aa"
+    "gitHead": "5fadaae0607c497eeffd0029fb17955f69d2ed46"
 }