@crawlee/http 4.0.0-beta.96 → 4.0.0-beta.97

package/internals/http-crawler.d.ts

@@ -21,7 +21,11 @@ JSONData extends JsonValue = any, // with default to Dictionary we cant use a ty
 ContextExtension = Dictionary<never>> = ErrorHandler<CrawlingContext, HttpCrawlingContext<UserData, JSONData> & ContextExtension>;
 export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext, Routes> {
     /**
-     * Timeout in which the HTTP request to the resource needs to finish, given in seconds.
+     * Timeout for the whole navigation phase, given in seconds. A single window shared by the
+     * `preNavigationHooks`, the navigation (the HTTP request to the resource), and the `postNavigationHooks` -
+     * so a slow hook eats into the same budget the navigation uses. Separate from the
+     * {@link BasicCrawlerOptions.requestHandlerTimeoutSecs|`requestHandlerTimeoutSecs`}, which times only the
+     * request handler.
      */
     navigationTimeoutSecs?: number;
     /**
@@ -344,6 +348,7 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
      * All `HttpCrawlerOptions` parameters are passed via an options object.
      */
     constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
+    protected getNavigationTimeoutMillis(): number;
     /**
      * Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
      * concurrency shortcuts on top. Not called for a supplied

package/internals/http-crawler.js

@@ -1,13 +1,13 @@
 import { Readable } from 'node:stream';
 import util from 'node:util';
-import { BasicCrawler, ContextPipeline, NavigationSkippedError, RequestState, Router, SessionError, } from '@crawlee/basic';
+import { BasicCrawler, ContextPipeline, NavigationSkippedError, remainingNavigationWindowMillis, RequestState, Router, SessionError, } from '@crawlee/basic';
 import { getCookiesFromResponse } from '@crawlee/core';
 import { ResponseWithUrl } from '@crawlee/http-client';
 import { RETRY_CSS_SELECTORS } from '@crawlee/utils';
 import contentTypeParser from 'content-type';
 import iconv from 'iconv-lite';
 import ow from 'ow';
-import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
+import { addTimeoutToPromise, storage, TimeoutError, tryCancel } from '@apify/timeout';
 import { extractCharsetFromHtmlBytes, parseContentTypeFromResponse, processHttpRequestOptions } from './utils.js';
 /**
  * Default mime types, which HttpScraper supports.
@@ -161,6 +161,9 @@ export class HttpCrawler extends BasicCrawler {
         ];
         this.saveResponseCookies = saveResponseCookies;
     }
+    getNavigationTimeoutMillis() {
+        return this.navigationTimeoutMillis;
+    }
     /**
      * Folds {@link HTTP_OPTIMIZED_CONCURRENCY_SYSTEM_OPTIONS} into the default system, keeping the user's
      * concurrency shortcuts on top. Not called for a supplied
@@ -179,15 +182,26 @@ export class HttpCrawler extends BasicCrawler {
         const skipGuard = (action) => ({
             action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
         });
+        // A single navigation window covers the pre-navigation hooks, the navigation, and the post-navigation
+        // hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow hook eats into the same
+        // window the navigation uses instead of each step being timed on its own.
+        const navigationTimedOut = `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`;
+        const windowGuard = (step) => skipGuard(async (ctx) => {
+            const remaining = remainingNavigationWindowMillis(ctx, this.navigationTimeoutMillis);
+            if (remaining <= 0) {
+                throw new TimeoutError(navigationTimedOut);
+            }
+            return addTimeoutToPromise(async () => step(ctx), remaining, navigationTimedOut);
+        });
         let pipeline = ContextPipeline.create().compose({
             action: this.prepareHttpRequest.bind(this),
         });
         for (const hook of this.preNavigationHooks) {
-            pipeline = pipeline.compose(skipGuard(hook));
+            pipeline = pipeline.compose(windowGuard(hook));
         }
         let pipelineWithNavigation = pipeline.compose(skipGuard(this.makeHttpRequest.bind(this)));
         for (const hook of this.postNavigationHooks) {
-            pipelineWithNavigation = pipelineWithNavigation.compose(skipGuard(hook));
+            pipelineWithNavigation = pipelineWithNavigation.compose(windowGuard(hook));
         }
         return pipelineWithNavigation
             .compose({ action: this.processHttpResponse.bind(this) })
@@ -217,7 +231,10 @@ export class HttpCrawler extends BasicCrawler {
         tryCancel();
         const { request, session } = crawlingContext;
         const proxyUrl = crawlingContext.proxyInfo?.url;
-        const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        // Bound the request by whatever is left of the shared navigation window (the pre-navigation hooks may
+        // have already spent part of it), so it produces a clean navigation-timeout error rather than the raw
+        // client abort.
+        const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }), Math.max(1, remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis)), `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
         tryCancel();
         request.loadedUrl = httpResponse?.url;
         request.state = RequestState.AFTER_NAV;
@@ -244,7 +261,14 @@ export class HttpCrawler extends BasicCrawler {
             };
         }
         tryCancel();
-        const parsed = await this.parseResponse(crawlingContext.request, crawlingContext.response);
+        // Reading the body is still part of the navigation, so it draws from the same shared window: on a server
+        // that streams the body slowly the request completes (headers arrive) but the body read would otherwise
+        // run unbounded. `extendTimeout` from a post-navigation hook has already pushed this deadline out if asked.
+        const remaining = remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis);
+        if (remaining <= 0) {
+            throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        }
+        const parsed = await addTimeoutToPromise(async () => this.parseResponse(crawlingContext.request, crawlingContext.response), remaining, `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
         tryCancel();
         const response = parsed.response;
         const contentType = parsed.contentType;
@@ -474,7 +498,7 @@ export class HttpCrawler extends BasicCrawler {
      */
     handleRequestTimeout(session) {
         session.markBad();
-        throw new Error(`request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        throw new Error(`Request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
     }
     _abortDownloadOfBody(request, response) {
         const { status } = response;
@@ -498,6 +522,12 @@ export class HttpCrawler extends BasicCrawler {
         // session jar. Reads still go through the session (so session.setCookie() in pre-nav
         // hooks keeps working) but a per-request clone is passed in so writes are discarded.
         const cookieJar = this.saveResponseCookies ? session.cookieJar : await session.cookieJar.clone();
+        // Bind the request to the shared navigation window instead of a fixed per-request timeout, so
+        // `extendTimeout()` can push the deadline and a fixed `AbortSignal.timeout` won't fire on its own and
+        // kill a lazily-read body mid-extension. This aborts the socket only during the header phase; the body
+        // read is bounded separately at the promise level (see `processHttpResponse`), so a slow-streaming body
+        // still fails cleanly with a navigation timeout, though the socket is left to close on its own.
+        const cancelSignal = storage.getStore()?.cancelTask.signal;
         const response = await this.httpClient.sendRequest(new Request(opts.url, {
             body: opts.body ? Readable.toWeb(opts.body) : undefined,
             headers: new Headers(opts.headers),
@@ -507,7 +537,8 @@ export class HttpCrawler extends BasicCrawler {
         }), {
             session,
             cookieJar,
-            timeoutMillis: opts.timeout,
+            signal: cancelSignal,
+            timeoutMillis: cancelSignal ? undefined : opts.timeout,
         });
         return response;
     };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http",
-    "version": "4.0.0-beta.96",
+    "version": "4.0.0-beta.97",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -47,13 +47,13 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/timeout": "^0.3.2",
+        "@apify/timeout": "^0.4.4",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/basic": "4.0.0-beta.96",
-        "@crawlee/core": "4.0.0-beta.96",
-        "@crawlee/http-client": "4.0.0-beta.96",
-        "@crawlee/types": "4.0.0-beta.96",
-        "@crawlee/utils": "4.0.0-beta.96",
+        "@crawlee/basic": "4.0.0-beta.97",
+        "@crawlee/core": "4.0.0-beta.97",
+        "@crawlee/http-client": "4.0.0-beta.97",
+        "@crawlee/types": "4.0.0-beta.97",
+        "@crawlee/utils": "4.0.0-beta.97",
         "@types/content-type": "^1.1.8",
         "cheerio": "^1.0.0",
         "content-type": "^1.0.5",
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "b5c20fe6cecad57a6e3dd26b4137776d4b62588a"
+    "gitHead": "43008dd43f4832d083353ba1b731c0b4607c9cfa"
 }