@crawlee/http 4.0.0-beta.80 → 4.0.0-beta.82

package/internals/http-crawler.d.ts

@@ -1,9 +1,8 @@
 import type { BasicCrawlerOptions, CrawlingContext, ErrorHandler, GetUserDataFromRequest, Request as CrawleeRequest, RequestHandler, RequireContextPipeline, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/basic';
 import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
 import { type LoadedRequest } from '@crawlee/core';
-import type { Awaitable, Dictionary, ISession } from '@crawlee/types';
+import type { Awaitable, Dictionary } from '@crawlee/types';
 import { type CheerioRoot } from '@crawlee/utils';
-import type { RequestLike, ResponseLike } from 'content-type';
 import type { JsonValue } from 'type-fest';
 export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawler
@@ -24,6 +23,11 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
      *
      * A hook may optionally return a partial object whose properties are merged into the crawling context,
      * allowing the hook to override context members for subsequent hooks and pipeline stages.
+     *
+     * The context is built up in the following order: base context (`request`, `session`, helpers, ...) ->
+     * `extendContext` -> `preNavigationHooks` -> navigation -> `postNavigationHooks` -> `requestHandler`.
+     * This means the members added by `extendContext` are already available here, but navigation-dependent
+     * members (e.g. `response`, `body`, `$`) are not.
      * Example:
      * ```
      * preNavigationHooks: [
@@ -33,7 +37,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
      * ]
      * ```
      */
-    preNavigationHooks?: InternalHttpHook<CrawlingContext>[];
+    preNavigationHooks?: InternalHttpHook<CrawlingContext, ContextExtension>[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
      * The function accepts `crawlingContext` as the only parameter.
@@ -51,7 +55,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
      * ]
      * ```
      */
-    postNavigationHooks?: ((crawlingContext: CrawlingContextWithResponse) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
+    postNavigationHooks?: ((crawlingContext: CrawlingContextWithResponse & ContextExtension) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
     /**
      * An array of [MIME types](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types)
      * you want the crawler to load and process. By default, only `text/html`, `application/xhtml+xml`, `text/xml`, `application/xml`,
@@ -91,7 +95,7 @@ export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext
 /**
  * @internal
  */
-export type InternalHttpHook<Context> = (crawlingContext: Context) => Awaitable<void | Partial<Context>>;
+export type InternalHttpHook<Context, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
 export type HttpHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
 JSONData extends JsonValue = any> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;
 interface CrawlingContextWithResponse<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
@@ -228,14 +232,14 @@ JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData,
  * @category Crawlers
  */
 export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
-    protected preNavigationHooks: InternalHttpHook<CrawlingContext>[];
-    protected postNavigationHooks: ((crawlingContext: CrawlingContextWithResponse) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
-    protected saveResponseCookies: boolean;
-    protected navigationTimeoutMillis: number;
-    protected ignoreSslErrors: boolean;
-    protected suggestResponseEncoding?: string;
-    protected forceResponseEncoding?: string;
-    protected readonly supportedMimeTypes: Set<string>;
+    private preNavigationHooks;
+    private postNavigationHooks;
+    private saveResponseCookies;
+    private navigationTimeoutMillis;
+    private ignoreSslErrors;
+    private suggestResponseEncoding?;
+    private forceResponseEncoding?;
+    private readonly supportedMimeTypes;
     protected static optionsShape: {
 // @ts-ignore optional peer dependency or compatibility with es2022
         navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
@@ -337,64 +341,30 @@ export declare class HttpCrawler<Context extends InternalHttpCrawlingContext<any
      * on the request such as only downloading the request body if the
      * received content type matches text/html, application/xml, application/xhtml+xml.
      */
-    protected _requestFunction({ request, session, proxyUrl }: RequestFunctionOptions): Promise<Response>;
+    private requestFunction;
     /**
      * Encodes and parses response according to the provided content type
      */
-    protected _parseResponse(request: CrawleeRequest, response: Response): Promise<{
-        response: Response;
-        contentType: {
-            type: string;
-            encoding: BufferEncoding;
-        };
-        body: string;
-    } | {
-        body: Buffer<ArrayBuffer>;
-        response: Response;
-        contentType: {
-            type: string;
-            encoding: BufferEncoding;
-        };
-    }>;
+    private parseResponse;
     /**
      * Combines the provided `requestOptions` with mandatory (non-overridable) values.
      */
-    protected _getRequestOptions(request: CrawleeRequest, session: ISession, proxyUrl?: string): {
-        url: string;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        method: import("@crawlee/types").AllowedHttpMethods;
-        proxyUrl: string | undefined;
-        timeout: number;
-        sessionToken: ISession;
-        headers: Record<string, string> | undefined;
-        https: {
-            rejectUnauthorized: boolean;
-        };
-        body: string | undefined;
-    };
-    protected _encodeResponse(request: CrawleeRequest, response: Response, encoding: BufferEncoding): {
-        encoding: BufferEncoding;
-        response: Response;
-    };
+    private getRequestOptions;
+    private encodeResponse;
     /**
      * Checks and extends supported mime types
      */
-    protected _extendSupportedMimeTypes(additionalMimeTypes: (string | RequestLike | ResponseLike)[]): void;
+    private extendSupportedMimeTypes;
     /**
      * Handles timeout request
      */
-    protected _handleRequestTimeout(session: ISession): void;
+    private handleRequestTimeout;
     private _abortDownloadOfBody;
     /**
      * @internal wraps public utility for mocking purposes
      */
     private _requestAsBrowser;
 }
-interface RequestFunctionOptions {
-    request: CrawleeRequest;
-    session: ISession;
-    proxyUrl?: string;
-}
 /**
  * Creates new {@link Router} instance that works based on request labels.
  * This instance can then serve as a `requestHandler` of your {@link HttpCrawler}.

package/internals/http-crawler.js

@@ -94,6 +94,10 @@ const HTTP_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
  * @category Crawlers
  */
 export class HttpCrawler extends BasicCrawler {
+    // Internal storage uses the base (non-extended) context types. The public option types are
+    // extension-aware for consumer DX, but internally the pipeline composes hooks against the
+    // concrete crawling context, which does not statically carry `ContextExtension`. The members
+    // added by `extendContext` are present at runtime regardless.
     preNavigationHooks;
     postNavigationHooks;
     saveResponseCookies;
@@ -129,7 +133,7 @@ export class HttpCrawler extends BasicCrawler {
         });
         this.supportedMimeTypes = new Set([...HTML_AND_XML_MIME_TYPES, APPLICATION_JSON_MIME_TYPE]);
         if (additionalMimeTypes.length)
-            this._extendSupportedMimeTypes(additionalMimeTypes);
+            this.extendSupportedMimeTypes(additionalMimeTypes);
         if (suggestResponseEncoding && forceResponseEncoding) {
             this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
         }
@@ -137,6 +141,9 @@ export class HttpCrawler extends BasicCrawler {
         this.ignoreSslErrors = ignoreSslErrors;
         this.suggestResponseEncoding = suggestResponseEncoding;
         this.forceResponseEncoding = forceResponseEncoding;
+        // Cast away the extension-aware option types to the base internal storage types (see the field
+        // declarations above). This is sound - the hooks only ever receive the base context plus the
+        // members `extendContext` added at runtime.
         this.preNavigationHooks = preNavigationHooks;
         this.postNavigationHooks = [
             ({ request, response }) => this._abortDownloadOfBody(request, response),
@@ -188,7 +195,7 @@ export class HttpCrawler extends BasicCrawler {
         tryCancel();
         const { request, session } = crawlingContext;
         const proxyUrl = crawlingContext.proxyInfo?.url;
-        const httpResponse = await addTimeoutToPromise(async () => this._requestFunction({ request, session, proxyUrl }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+        const httpResponse = await addTimeoutToPromise(async () => this.requestFunction({ request, session, proxyUrl }), this.navigationTimeoutMillis, `request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
         tryCancel();
         request.loadedUrl = httpResponse?.url;
         request.state = RequestState.AFTER_NAV;
@@ -215,7 +222,7 @@ export class HttpCrawler extends BasicCrawler {
             };
         }
         tryCancel();
-        const parsed = await this._parseResponse(crawlingContext.request, crawlingContext.response);
+        const parsed = await this.parseResponse(crawlingContext.request, crawlingContext.response);
         tryCancel();
         const response = parsed.response;
         const contentType = parsed.contentType;
@@ -291,15 +298,15 @@ export class HttpCrawler extends BasicCrawler {
      * on the request such as only downloading the request body if the
      * received content type matches text/html, application/xml, application/xhtml+xml.
      */
-    async _requestFunction({ request, session, proxyUrl }) {
-        const opts = this._getRequestOptions(request, session, proxyUrl);
+    async requestFunction({ request, session, proxyUrl }) {
+        const opts = this.getRequestOptions(request, session, proxyUrl);
         try {
             return await this._requestAsBrowser(opts, session);
         }
         catch (e) {
             if (e instanceof Error && e.constructor.name === 'TimeoutError') {
-                this._handleRequestTimeout(session);
-                return new Response(); // this will never happen, as _handleRequestTimeout always throws
+                this.handleRequestTimeout(session);
+                return new Response(); // this will never happen, as handleRequestTimeout always throws
             }
             if (this.isProxyError(e)) {
                 throw new SessionError(this._getMessageFromError(e));
@@ -312,10 +319,10 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Encodes and parses response according to the provided content type
      */
-    async _parseResponse(request, response) {
+    async parseResponse(request, response) {
         const { status } = response;
         const { type, charset } = parseContentTypeFromResponse(response);
-        const { response: reencodedResponse, encoding } = this._encodeResponse(request, response, charset);
+        const { response: reencodedResponse, encoding } = this.encodeResponse(request, response, charset);
         const contentType = { type, encoding };
         if (status >= 400 && status <= 599) {
             this.stats.registerStatusCode(status);
@@ -361,7 +368,7 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Combines the provided `requestOptions` with mandatory (non-overridable) values.
      */
-    _getRequestOptions(request, session, proxyUrl) {
+    getRequestOptions(request, session, proxyUrl) {
         const requestOptions = {
             url: request.url,
             method: request.method,
@@ -389,7 +396,7 @@ export class HttpCrawler extends BasicCrawler {
             requestOptions.body = request.payload ?? '';
         return requestOptions;
     }
-    _encodeResponse(request, response, encoding) {
+    encodeResponse(request, response, encoding) {
         if (this.forceResponseEncoding) {
             encoding = this.forceResponseEncoding;
         }
@@ -425,7 +432,7 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Checks and extends supported mime types
      */
-    _extendSupportedMimeTypes(additionalMimeTypes) {
+    extendSupportedMimeTypes(additionalMimeTypes) {
         for (const mimeType of additionalMimeTypes) {
             if (mimeType === '*/*') {
                 this.supportedMimeTypes.add(mimeType);
@@ -443,7 +450,7 @@ export class HttpCrawler extends BasicCrawler {
     /**
      * Handles timeout request
      */
-    _handleRequestTimeout(session) {
+    handleRequestTimeout(session) {
         session.markBad();
         throw new Error(`request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http",
-    "version": "4.0.0-beta.80",
+    "version": "4.0.0-beta.82",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,11 +49,11 @@
     "dependencies": {
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/basic": "4.0.0-beta.80",
-        "@crawlee/core": "4.0.0-beta.80",
-        "@crawlee/http-client": "4.0.0-beta.80",
-        "@crawlee/types": "4.0.0-beta.80",
-        "@crawlee/utils": "4.0.0-beta.80",
+        "@crawlee/basic": "4.0.0-beta.82",
+        "@crawlee/core": "4.0.0-beta.82",
+        "@crawlee/http-client": "4.0.0-beta.82",
+        "@crawlee/types": "4.0.0-beta.82",
+        "@crawlee/utils": "4.0.0-beta.82",
         "@types/content-type": "^1.1.8",
         "cheerio": "^1.0.0",
         "content-type": "^1.0.5",
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "96c57b4a0c999e4b2bd198792490af28db7aa42d"
+    "gitHead": "eb1096f7c7743d124375ef011fbbadb19476822e"
 }