@crawlee/basic 4.0.0-beta.81 → 4.0.0-beta.82

package/internals/basic-crawler.d.ts

@@ -46,7 +46,13 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      */
     requestHandler?: RequestHandler<ExtendedContext>;
     /**
-     * Allows the user to extend the crawling context passed to the request handler with custom functionality.
+     * Allows the user to extend the crawling context with custom functionality (helpers, references, etc.).
+     *
+     * `extendContext` runs *before* navigation, so the returned members are visible to the
+     * `preNavigationHooks`, `postNavigationHooks`, and the `requestHandler` alike. As a consequence,
+     * the `context` passed to `extendContext` is the pre-navigation {@link CrawlingContext} and does
+     * **not** include navigation-dependent members (e.g. `page`, `response`, `$`, `body`). If you need
+     * those, use a `postNavigationHook` or the `requestHandler` instead.
      *
      * **Example usage:**
      *
@@ -66,7 +72,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * });
      * ```
      */
-    extendContext?: (context: Context) => Awaitable<ContextExtension>;
+    extendContext?: (context: CrawlingContext) => Awaitable<ContextExtension>;
     /**
      * *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter.
      *

package/internals/basic-crawler.js

@@ -507,13 +507,27 @@ export class BasicCrawler {
         return { enqueueLinks: enqueueLinksWrapper, addRequests, sendRequest };
     }
     buildFinalContextPipeline() {
-        let contextPipeline = (this.contextPipelineOptions.contextPipelineBuilder?.() ??
+        const subclassPipeline = (this.contextPipelineOptions.contextPipelineBuilder?.() ??
             this.buildContextPipeline());
+        // `extendContext` runs *before* the subclass navigation pipeline (which includes the
+        // pre/post-navigation hooks). This makes the extension visible to those hooks and to the
+        // request handler alike. The trade-off is that `extendContext` cannot access
+        // navigation-dependent context members (e.g. `page`, `response`, `$`, `body`), as those
+        // don't exist yet at this point in the pipeline.
+        // The `extendContext` output (`ContextExtension`) is carried through the subclass pipeline at
+        // runtime (the pipeline copies each middleware's returned members onto the shared context), but
+        // TypeScript cannot express that `Context` transitively includes `ContextExtension` here. The
+        // casts below are sound because `buildFinalContextPipeline` is declared to return the fully
+        // resolved `ExtendedContext` (= `Context & ContextExtension`).
         const { extendContext } = this.contextPipelineOptions;
+        let contextPipeline;
         if (extendContext !== undefined) {
-            contextPipeline = contextPipeline.compose({
-                action: async (context) => await extendContext(context),
-            });
+            contextPipeline = ContextPipeline.create()
+                .compose({ action: async (context) => await extendContext(context) })
+                .chain(subclassPipeline);
+        }
+        else {
+            contextPipeline = subclassPipeline;
         }
         contextPipeline = contextPipeline.compose({
             action: async (context) => {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.81",
+    "version": "4.0.0-beta.82",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.81",
-        "@crawlee/http-client": "4.0.0-beta.81",
-        "@crawlee/types": "4.0.0-beta.81",
-        "@crawlee/utils": "4.0.0-beta.81",
+        "@crawlee/core": "4.0.0-beta.82",
+        "@crawlee/http-client": "4.0.0-beta.82",
+        "@crawlee/types": "4.0.0-beta.82",
+        "@crawlee/utils": "4.0.0-beta.82",
         "csv-stringify": "^6.5.2",
         "fs-extra": "^11.3.0",
         "ow": "^2.0.0",
@@ -54,7 +54,7 @@
         "type-fest": "^4.41.0"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.81"
+        "@crawlee/impit-client": "^4.0.0-beta.82"
     },
     "lerna": {
         "command": {
@@ -63,5 +63,5 @@
             }
         }
     },
-    "gitHead": "80dc6b4fc82237e63a51a71153809ec8dfd0cc50"
+    "gitHead": "eb1096f7c7743d124375ef011fbbadb19476822e"
 }