@crawlee/basic 4.0.0-beta.81 → 4.0.0-beta.83

package/internals/basic-crawler.d.ts

@@ -46,7 +46,13 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      */
     requestHandler?: RequestHandler<ExtendedContext>;
     /**
-     * Allows the user to extend the crawling context passed to the request handler with custom functionality.
+     * Allows the user to extend the crawling context with custom functionality (helpers, references, etc.).
+     *
+     * `extendContext` runs *before* navigation, so the returned members are visible to the
+     * `preNavigationHooks`, `postNavigationHooks`, and the `requestHandler` alike. As a consequence,
+     * the `context` passed to `extendContext` is the pre-navigation {@link CrawlingContext} and does
+     * **not** include navigation-dependent members (e.g. `page`, `response`, `$`, `body`). If you need
+     * those, use a `postNavigationHook` or the `requestHandler` instead.
      *
      * **Example usage:**
      *
@@ -66,7 +72,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingC
      * });
      * ```
      */
-    extendContext?: (context: Context) => Awaitable<ContextExtension>;
+    extendContext?: (context: CrawlingContext) => Awaitable<ContextExtension>;
     /**
      * *Intended for BasicCrawler subclasses*. Prepares a context pipeline that transforms the initial crawling context into the shape given by the `Context` type parameter.
      *
@@ -405,24 +411,19 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
      * via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
      */
     protected requestManager?: IRequestManager;
+    /** Backs the {@link BasicCrawler.sessionPool|`sessionPool`} getter. */
+    private sessionPoolDep;
     /**
      * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
      * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
      */
-    readonly sessionPool: ISessionPool;
-    /**
-     * Set when the crawler constructed its own {@link SessionPool} (no `sessionPool` option was provided).
-     * Holds the same instance as `sessionPool`, but typed as the concrete class so the crawler can call
-     * lifecycle methods (`resetStore`, `teardown`) that aren't part of {@link ISessionPool}. A user-supplied
-     * pool is never owned and never torn down by the crawler.
-     */
-    private ownedSessionPool?;
+    get sessionPool(): ISessionPool;
     /**
-     * Set when the crawler constructed its own request manager (no `requestManager`, `requestQueue`, or `requestList`
-     * option was provided). The owned manager is purged (not dropped) between repeated `run()` calls.
-     * A user-supplied manager is never purged by the crawler.
+     * Tracks **only** the queue the crawler opens for itself — not the {@link RequestManagerTandem} that may wrap it
+     * around a user-supplied `requestList` — so the owned-only purge between repeated `run()` calls never reaches
+     * through to a borrowed loader. Filled lazily in {@link BasicCrawler.openOwnedRequestQueue|`openOwnedRequestQueue()`}.
      */
-    private ownedRequestManager?;
+    private ownedRequestQueue;
     /**
      * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
      * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
@@ -637,7 +638,7 @@ export declare class BasicCrawler<Context extends CrawlingContext = CrawlingCont
     getRequestQueue(): Promise<IRequestManager>;
     /**
      * Opens the default {@link RequestQueue}, applies the crawler's timeouts to it and records it as the
-     * crawler-owned manager (so it gets purged between repeated `run()` calls).
+     * crawler-owned queue (so it gets purged between repeated `run()` calls).
      * @private
      */
     private openOwnedRequestQueue;

package/internals/basic-crawler.js

@@ -1,6 +1,6 @@
 import { writeFile } from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, CriticalError, Dataset, enqueueLinks, EnqueueStrategy, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, purgeDefaultStorages, RequestHandlerError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, validateUserData, validators, } from '@crawlee/core';
+import { AutoscaledPool, bindMethodsToServiceLocator, BLOCKED_STATUS_CODES, ContextPipeline, ContextPipelineCleanupError, ContextPipelineInitializationError, ContextPipelineInterruptedError, CriticalError, Dataset, enqueueLinks, EnqueueStrategy, KeyValueStore, log, LogLevel, mergeCookies, MissingSessionError, NavigationSkippedError, NonRetryableError, OwnedOrInjected, purgeDefaultStorages, RequestHandlerError, RequestManagerTandem, RequestQueue, RequestState, RetryRequestError, Router, ServiceLocator, serviceLocator, Session, SessionError, SessionPool, Statistics, validateUserData, validators, } from '@crawlee/core';
 import { FetchHttpClient } from '@crawlee/http-client';
 import { getObjectType, isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';
 import { stringify } from 'csv-stringify/sync';
@@ -60,24 +60,21 @@ export class BasicCrawler {
      * via {@link BasicCrawler.getRequestManager|`getRequestManager()`}.
      */
     requestManager;
+    /** Backs the {@link BasicCrawler.sessionPool|`sessionPool`} getter. */
+    sessionPoolDep;
     /**
      * A reference to the underlying session pool that manages the crawler's {@link Session|sessions}. Typed as
      * {@link ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
      */
-    sessionPool;
-    /**
-     * Set when the crawler constructed its own {@link SessionPool} (no `sessionPool` option was provided).
-     * Holds the same instance as `sessionPool`, but typed as the concrete class so the crawler can call
-     * lifecycle methods (`resetStore`, `teardown`) that aren't part of {@link ISessionPool}. A user-supplied
-     * pool is never owned and never torn down by the crawler.
-     */
-    ownedSessionPool;
+    get sessionPool() {
+        return this.sessionPoolDep.value;
+    }
     /**
-     * Set when the crawler constructed its own request manager (no `requestManager`, `requestQueue`, or `requestList`
-     * option was provided). The owned manager is purged (not dropped) between repeated `run()` calls.
-     * A user-supplied manager is never purged by the crawler.
+     * Tracks **only** the queue the crawler opens for itself — not the {@link RequestManagerTandem} that may wrap it
+     * around a user-supplied `requestList` — so the owned-only purge between repeated `run()` calls never reaches
+     * through to a borrowed loader. Filled lazily in {@link BasicCrawler.openOwnedRequestQueue|`openOwnedRequestQueue()`}.
      */
-    ownedRequestManager;
+    ownedRequestQueue = OwnedOrInjected.resolve();
     /**
      * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
      * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
@@ -297,18 +294,12 @@ export class BasicCrawler {
                     '`proxyInfo` they were created with. Configure proxies on the pool instead, ' +
                     'e.g. via `addSession({ proxyInfo })` or a custom `createSessionFunction`.');
             }
-            if (sessionPool) {
-                this.sessionPool = sessionPool;
-            }
-            else {
-                this.ownedSessionPool = new SessionPool({
-                    createSessionFunction: async (opts) => new Session({
-                        ...opts?.sessionOptions,
-                        proxyInfo: opts?.sessionOptions?.proxyInfo ?? (await this.proxyConfiguration?.newProxyInfo()),
-                    }),
-                });
-                this.sessionPool = this.ownedSessionPool;
-            }
+            this.sessionPoolDep = OwnedOrInjected.resolve(sessionPool, () => new SessionPool({
+                createSessionFunction: async (opts) => new Session({
+                    ...opts?.sessionOptions,
+                    proxyInfo: opts?.sessionOptions?.proxyInfo ?? (await this.proxyConfiguration?.newProxyInfo()),
+                }),
+            }));
             this.blockedStatusCodes = new Set(blockedStatusCodesInput ?? BLOCKED_STATUS_CODES);
             const maxSignedInteger = 2 ** 31 - 1;
             if (this.requestHandlerTimeoutMillis > maxSignedInteger) {
@@ -507,13 +498,27 @@ export class BasicCrawler {
         return { enqueueLinks: enqueueLinksWrapper, addRequests, sendRequest };
     }
     buildFinalContextPipeline() {
-        let contextPipeline = (this.contextPipelineOptions.contextPipelineBuilder?.() ??
+        const subclassPipeline = (this.contextPipelineOptions.contextPipelineBuilder?.() ??
             this.buildContextPipeline());
+        // `extendContext` runs *before* the subclass navigation pipeline (which includes the
+        // pre/post-navigation hooks). This makes the extension visible to those hooks and to the
+        // request handler alike. The trade-off is that `extendContext` cannot access
+        // navigation-dependent context members (e.g. `page`, `response`, `$`, `body`), as those
+        // don't exist yet at this point in the pipeline.
+        // The `extendContext` output (`ContextExtension`) is carried through the subclass pipeline at
+        // runtime (the pipeline copies each middleware's returned members onto the shared context), but
+        // TypeScript cannot express that `Context` transitively includes `ContextExtension` here. The
+        // casts below are sound because `buildFinalContextPipeline` is declared to return the fully
+        // resolved `ExtendedContext` (= `Context & ContextExtension`).
         const { extendContext } = this.contextPipelineOptions;
+        let contextPipeline;
         if (extendContext !== undefined) {
-            contextPipeline = contextPipeline.compose({
-                action: async (context) => await extendContext(context),
-            });
+            contextPipeline = ContextPipeline.create()
+                .compose({ action: async (context) => await extendContext(context) })
+                .chain(subclassPipeline);
+        }
+        else {
+            contextPipeline = subclassPipeline;
         }
         contextPipeline = contextPipeline.compose({
             action: async (context) => {
@@ -620,17 +625,17 @@ export class BasicCrawler {
             // we need to purge the RQ to allow processing the same requests again — this is important so users can
             // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
             // ignored — as a failed request is still handled.
-            // By default (purgeRequestQueue unset or true), only the manager we created ourselves (ownedRequestManager) is purged.
+            // By default (`purgeRequestQueue` unset), only the queue we opened ourselves is purged.
             // When `purgeRequestQueue` is explicitly `true`, we also purge a user-supplied manager.
             // When `purgeRequestQueue` is explicitly `false`, nothing is purged.
             const shouldPurge = purgeRequestQueue !== false;
-            const managerToPurge = this.ownedRequestManager ?? (purgeRequestQueue === true ? this.requestManager : undefined);
+            const managerToPurge = this.ownedRequestQueue.maybeValue ?? (purgeRequestQueue === true ? this.requestManager : undefined);
             if (managerToPurge?.purge && shouldPurge) {
                 await managerToPurge.purge();
             }
             this.stats.reset();
             await this.stats.resetStore();
-            await this.ownedSessionPool?.resetStore();
+            await this.sessionPoolDep.ifOwned((pool) => pool.resetStore());
         }
         this.unexpectedStop = false;
         this.running = true;
@@ -742,7 +747,7 @@ export class BasicCrawler {
     }
     /**
      * Opens the default {@link RequestQueue}, applies the crawler's timeouts to it and records it as the
-     * crawler-owned manager (so it gets purged between repeated `run()` calls).
+     * crawler-owned queue (so it gets purged between repeated `run()` calls).
      * @private
      */
     async openOwnedRequestQueue() {
@@ -750,8 +755,7 @@ export class BasicCrawler {
         // subsequent instances get their own queue via a unique alias so they don't collide.
         const identifier = this.identity.instanceIndex === 0 ? null : { alias: `__default_${this.identity.id}__` };
         const requestQueue = await RequestQueue.open(identifier, { config: serviceLocator.getConfiguration() });
-        this.ownedRequestManager = requestQueue;
-        return requestQueue;
+        return this.ownedRequestQueue.set(requestQueue);
     }
     /**
      * Tells a request manager how long we expect to hold a fetched request, so that one backed by a
@@ -1350,7 +1354,7 @@ export class BasicCrawler {
         if (this._closeEvents) {
             await serviceLocator.getEventManager().close();
         }
-        await this.ownedSessionPool?.teardown();
+        await this.sessionPoolDep.ifOwned((pool) => pool.teardown());
         await this.autoscaledPool?.abort();
     }
     _getCookieHeaderFromRequest(request) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/basic",
-    "version": "4.0.0-beta.81",
+    "version": "4.0.0-beta.83",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -42,10 +42,10 @@
         "@apify/datastructures": "^2.0.0",
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
-        "@crawlee/core": "4.0.0-beta.81",
-        "@crawlee/http-client": "4.0.0-beta.81",
-        "@crawlee/types": "4.0.0-beta.81",
-        "@crawlee/utils": "4.0.0-beta.81",
+        "@crawlee/core": "4.0.0-beta.83",
+        "@crawlee/http-client": "4.0.0-beta.83",
+        "@crawlee/types": "4.0.0-beta.83",
+        "@crawlee/utils": "4.0.0-beta.83",
         "csv-stringify": "^6.5.2",
         "fs-extra": "^11.3.0",
         "ow": "^2.0.0",
@@ -54,7 +54,7 @@
         "type-fest": "^4.41.0"
     },
     "optionalDependencies": {
-        "@crawlee/impit-client": "^4.0.0-beta.81"
+        "@crawlee/impit-client": "^4.0.0-beta.83"
     },
     "lerna": {
         "command": {
@@ -63,5 +63,5 @@
             }
         }
     },
-    "gitHead": "80dc6b4fc82237e63a51a71153809ec8dfd0cc50"
+    "gitHead": "5f25b90c4914d7e097732cf761e440aeba799137"
 }