@crawlee/http 4.0.0-beta.184 → 4.0.0-beta.186

package/internals/dom-crawler.js

@@ -23,22 +23,17 @@ export class DOMCrawler extends HttpCrawler {
     buildContextPipeline() {
         return super
             .buildContextPipeline()
-            .compose({
-            action: async (context) => this.#parseContent(context),
-            cleanup: async (context) => {
-                // The `skipNavigation` placeholders below throw on access, so there is nothing to clean up.
-                if (!context.request.skipNavigation) {
-                    await this.#parser.cleanup?.(context);
-                }
-            },
-        })
-            .compose({ action: async (context) => this.#addHelpers(context) });
+            .compose(this.#parseContent.bind(this))
+            .compose(async (context) => this.#addHelpers(context));
     }
-    async #parseContent(context) {
+    async #parseContent(context, onCleanup) {
         try {
-            return await this.#parser.parse(context);
+            const parsed = await this.#parser.parse(context);
+            onCleanup(() => this.#parser.cleanup?.(parsed));
+            return parsed;
         }
         catch (err) {
+            // The `skipNavigation` placeholders below throw on access, so there is nothing to clean up.
             if (err instanceof NavigationSkippedError) {
                 return Object.defineProperties({}, Object.fromEntries(Object.keys(this.#parser.placeholderMembers).map((member) => [
                     member,

package/internals/file-download.js

@@ -3,7 +3,6 @@ import { BasicCrawler } from '@crawlee/basic';
 import { ResponseWithUrl } from '@crawlee/http-client';
 import { Router } from '../index.js';
 import { parseContentTypeFromResponse } from './utils.js';
-const kBodyDrained = Symbol('bodyDrained');
 /**
  * Creates a transform stream that throws an error if the source data speed is below the specified minimum speed.
  * This `Transform` checks the amount of data every `checkProgressInterval` milliseconds.
@@ -114,32 +113,28 @@ export class FileDownload extends BasicCrawler {
         });
     }
     buildContextPipeline() {
-        return super.buildContextPipeline().compose({
-            action: async (context) => this.initiateDownload(context),
-            cleanup: async (context) => {
-                if (!context.response.bodyUsed) {
-                    // Nobody consumed the body — cancel it so the
-                    // underlying connection can be released.
-                    await context.response.body?.cancel();
-                }
-                await context[kBodyDrained];
-            },
-        });
+        return super.buildContextPipeline().compose(this.initiateDownload.bind(this));
     }
-    async initiateDownload(context) {
+    async initiateDownload(context, onCleanup) {
         const response = await this.httpClient.sendRequest(context.request.intoFetchAPIRequest(), {
             session: context.session,
         });
         const { type, charset: encoding } = parseContentTypeFromResponse(response);
         context.request.url = response.url;
         const { response: trackedResponse, bodyDrained } = trackBodyConsumption(response);
-        const contextExtension = {
+        onCleanup(async () => {
+            if (!trackedResponse.bodyUsed) {
+                // Nobody consumed the body — cancel it so the
+                // underlying connection can be released.
+                await trackedResponse.body?.cancel();
+            }
+            await bodyDrained;
+        });
+        return {
             request: context.request,
             response: trackedResponse,
             contentType: { type, encoding },
-            [kBodyDrained]: bodyDrained,
         };
-        return contextExtension;
     }
 }
 /**

package/internals/http-crawler.js

@@ -182,9 +182,7 @@ export class HttpCrawler extends BasicCrawler {
     buildContextPipeline() {
         // When navigation is skipped, `prepareHttpRequest` has already installed throwing getters for
         // the response-derived members, so the guarded action is bypassed and the context left untouched.
-        const skipGuard = (action) => ({
-            action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
-        });
+        const skipGuard = (action) => async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {}));
         // A single navigation window covers the pre-navigation hooks, the navigation, and the post-navigation
         // hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow hook eats into the same
         // window the navigation uses instead of each step being timed on its own.
@@ -196,9 +194,7 @@ export class HttpCrawler extends BasicCrawler {
             }
             return addTimeoutToPromise(async () => step(ctx), remaining, navigationTimedOut);
         });
-        let pipeline = ContextPipeline.create().compose({
-            action: this.prepareHttpRequest.bind(this),
-        });
+        let pipeline = ContextPipeline.create().compose(this.prepareHttpRequest.bind(this));
         for (const hook of this.#preNavigationHooks) {
             pipeline = pipeline.compose(windowGuard(hook));
         }
@@ -207,8 +203,8 @@ export class HttpCrawler extends BasicCrawler {
             pipelineWithNavigation = pipelineWithNavigation.compose(windowGuard(hook));
         }
         return pipelineWithNavigation
-            .compose({ action: this.processHttpResponse.bind(this) })
-            .compose({ action: this.handleBlockedRequestByContent.bind(this) });
+            .compose(this.processHttpResponse.bind(this))
+            .compose(this.handleBlockedRequestByContent.bind(this));
     }
     async prepareHttpRequest(crawlingContext) {
         const { request } = crawlingContext;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/http",
-    "version": "4.0.0-beta.184",
+    "version": "4.0.0-beta.186",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,11 +49,11 @@
     "dependencies": {
         "@apify/timeout": "^1.0.1",
         "@apify/utilities": "^3.0.1",
-        "@crawlee/basic": "4.0.0-beta.184",
-        "@crawlee/core": "4.0.0-beta.184",
-        "@crawlee/http-client": "4.0.0-beta.184",
-        "@crawlee/types": "4.0.0-beta.184",
-        "@crawlee/utils": "4.0.0-beta.184",
+        "@crawlee/basic": "4.0.0-beta.186",
+        "@crawlee/core": "4.0.0-beta.186",
+        "@crawlee/http-client": "4.0.0-beta.186",
+        "@crawlee/types": "4.0.0-beta.186",
+        "@crawlee/utils": "4.0.0-beta.186",
         "@types/content-type": "^1.1.8",
         "cheerio": "^1.0.0",
         "content-type": "^1.0.5",
@@ -70,5 +70,5 @@
             }
         }
     },
-    "gitHead": "19ebeeb9bb90e38496cbe90565c8ebb02e102193"
+    "gitHead": "d15c77259254640d55d0492a196d1bc6a89ecd0b"
 }