@crawlee/browser 4.0.0-beta.81 → 4.0.0-beta.83

package/internals/browser-crawler.d.ts

@@ -31,8 +31,8 @@ export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Re
      */
     enqueueLinks: (options?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
 }
-export type BrowserHook<Context = BrowserCrawlingContext> = (crawlingContext: Context) => Awaitable<void | Partial<Context>>;
-export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<Page, Response, Dictionary>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions<Context, ExtendedContext>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
+export type BrowserHook<Context = BrowserCrawlingContext, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
+export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<Page, Response, Dictionary>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
     launchContext?: BrowserLaunchContext<any, any>;
     /**
      * An existing browser pool instance to use. When provided, the crawler will use this pool directly instead of
@@ -124,8 +124,13 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
      *
      * A hook may optionally return a partial object whose properties are merged into the crawling context,
      * allowing the hook to override context members for subsequent hooks and pipeline stages.
+     *
+     * The context is built up in the following order: base context (`request`, `session`, helpers, ...) ->
+     * `extendContext` -> `preNavigationHooks` -> navigation -> `postNavigationHooks` -> `requestHandler`.
+     * This means the members added by `extendContext` are already available here, but navigation-dependent
+     * members (e.g. `page`, `response`) are not.
      */
-    preNavigationHooks?: BrowserHook<Context>[];
+    preNavigationHooks?: BrowserHook<Context, ContextExtension>[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
      * The function accepts `crawlingContext` as the only parameter.
@@ -151,7 +156,7 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
      * ]
      * ```
      */
-    postNavigationHooks?: BrowserHook<Context>[];
+    postNavigationHooks?: BrowserHook<Context, ContextExtension>[];
     /**
      * Timeout in which page navigation needs to finish, in seconds.
      */
@@ -219,17 +224,13 @@ export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Res
  * @category Crawlers
  */
 export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, LaunchOptions extends Dictionary | undefined = Dictionary, Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<Page, Response, Dictionary>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, GoToOptions extends Dictionary = Dictionary> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
+    /** Backs the {@link BrowserCrawler.browserPool|`browserPool`} getter. */
+    private browserPoolDep;
     /**
      * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
      * {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
      */
-    browserPool: IBrowserPool<Page>;
-    /**
-     * Set when the crawler constructed its own pool (a {@link BrowserPool}, or a {@link RemoteBrowserPool}
-     * built from the `remoteBrowser` option). Holds the same instance as `browserPool` but is the only reference
-     * the crawler tears down — a user-supplied `browserPool` is never owned and never destroyed by the crawler.
-     */
-    private ownedBrowserPool?;
+    get browserPool(): IBrowserPool<Page>;
     launchContext: BrowserLaunchContext<LaunchOptions, unknown>;
     protected readonly ignoreShadowRoots: boolean;
     protected readonly ignoreIframes: boolean;

package/internals/browser-crawler.js

@@ -1,4 +1,4 @@
-import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, enqueueLinks, handleRequestTimeout, NavigationSkippedError, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
+import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, enqueueLinks, handleRequestTimeout, NavigationSkippedError, OwnedOrInjected, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
 import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
 import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';
 import ow from 'ow';
@@ -48,17 +48,15 @@ const readContextField = (ctx, key) => ctx[key];
  * @category Crawlers
  */
 export class BrowserCrawler extends BasicCrawler {
+    /** Backs the {@link BrowserCrawler.browserPool|`browserPool`} getter. */
+    browserPoolDep;
     /**
      * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
      * {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
      */
-    browserPool;
-    /**
-     * Set when the crawler constructed its own pool (a {@link BrowserPool}, or a {@link RemoteBrowserPool}
-     * built from the `remoteBrowser` option). Holds the same instance as `browserPool` but is the only reference
-     * the crawler tears down — a user-supplied `browserPool` is never owned and never destroyed by the crawler.
-     */
-    ownedBrowserPool;
+    get browserPool() {
+        return this.browserPoolDep.value;
+    }
     launchContext;
     ignoreShadowRoots;
     ignoreIframes;
@@ -104,10 +102,13 @@ export class BrowserCrawler extends BasicCrawler {
                     .compose({ action: this.handleBlockedRequestByContent.bind(this) })
                     .compose({ action: this.restoreRequestState.bind(this) });
             },
-            extendContext: extendContext,
+            extendContext,
         });
         this.launchContext = launchContext;
         this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
+        // The public option hooks are extension-aware; internal storage uses the base context type
+        // (the pipeline composes hooks against the concrete context, which does not statically carry
+        // `ContextExtension`). The extension members are present at runtime regardless.
         this.preNavigationHooks = preNavigationHooks;
         this.postNavigationHooks = postNavigationHooks;
         this.ignoreIframes = ignoreIframes;
@@ -117,35 +118,34 @@ export class BrowserCrawler extends BasicCrawler {
             this.launchContext.launchOptions.headless = headless;
         }
         this.saveResponseCookies = saveResponseCookies;
-        // `browserPool` wins over `remoteBrowser` — a passed-in pool is used as-is, the sugar is ignored.
-        if (browserPool) {
-            this.browserPool = browserPool;
-            return;
-        }
-        const resolvedBrowserPoolOptions = browserPoolOptions ?? {};
-        if (launchContext?.userAgent) {
-            if (resolvedBrowserPoolOptions.useFingerprints)
-                this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');
-            resolvedBrowserPoolOptions.useFingerprints = false;
-        }
-        if (remoteBrowser) {
-            // The crawler already built the right plugin for its browser — hand it to a RemoteBrowserPool so the
-            // remote connection is always for the matching browser (no plugin to construct, no way to mismatch).
-            const { browserPlugins, ...remoteBrowserPoolOptions } = resolvedBrowserPoolOptions;
-            const remotePool = new RemoteBrowserPool({
-                browserPlugins: browserPlugins,
-                ...remoteBrowser,
-                browserPoolOptions: remoteBrowserPoolOptions,
+        // `browserPool` wins over `remoteBrowser` — a passed-in pool is used as-is (borrowed), the sugar is ignored.
+        // The default is only built when no pool was injected, so all the option/launchContext fiddling below stays
+        // inside the factory.
+        this.browserPoolDep = OwnedOrInjected.resolve(browserPool, () => {
+            const resolvedBrowserPoolOptions = browserPoolOptions ?? {};
+            if (launchContext?.userAgent) {
+                if (resolvedBrowserPoolOptions.useFingerprints)
+                    this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');
+                resolvedBrowserPoolOptions.useFingerprints = false;
+            }
+            if (remoteBrowser) {
+                // The crawler already built the right plugin for its browser — hand it to a RemoteBrowserPool so the
+                // remote connection is always for the matching browser (no plugin to construct, no way to mismatch).
+                const { browserPlugins, ...remoteBrowserPoolOptions } = resolvedBrowserPoolOptions;
+                return new RemoteBrowserPool({
+                    browserPlugins: browserPlugins,
+                    ...remoteBrowser,
+                    browserPoolOptions: remoteBrowserPoolOptions,
+                });
+            }
+            // Double cast: `BrowserPool` implements `IBrowserPool<PageReturn>`, where `PageReturn` is derived from the
+            // plugin/controller generics and doesn't overlap with the crawler's free `Page` type param, so TS won't
+            // narrow it directly. The concrete pool does satisfy the `Page`/`destroy` contract at runtime — this is the
+            // long-standing `Page` variance gap, not a `destroy`-related hole.
+            return new BrowserPool({
+                ...resolvedBrowserPoolOptions,
             });
-            this.ownedBrowserPool = remotePool;
-            this.browserPool = remotePool;
-            return;
-        }
-        const ownedBrowserPool = new BrowserPool({
-            ...resolvedBrowserPoolOptions,
         });
-        this.ownedBrowserPool = ownedBrowserPool;
-        this.browserPool = ownedBrowserPool;
     }
     buildContextPipeline() {
         return ContextPipeline.create().compose({
@@ -360,7 +360,7 @@ export class BrowserCrawler extends BasicCrawler {
      * @ignore
      */
     async teardown() {
-        await this.ownedBrowserPool?.destroy();
+        await this.browserPoolDep.ifOwned((pool) => pool.destroy());
         await super.teardown();
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/browser",
-    "version": "4.0.0-beta.81",
+    "version": "4.0.0-beta.83",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -48,10 +48,10 @@
     },
     "dependencies": {
         "@apify/timeout": "^0.3.2",
-        "@crawlee/basic": "4.0.0-beta.81",
-        "@crawlee/browser-pool": "4.0.0-beta.81",
-        "@crawlee/types": "4.0.0-beta.81",
-        "@crawlee/utils": "4.0.0-beta.81",
+        "@crawlee/basic": "4.0.0-beta.83",
+        "@crawlee/browser-pool": "4.0.0-beta.83",
+        "@crawlee/types": "4.0.0-beta.83",
+        "@crawlee/utils": "4.0.0-beta.83",
         "ow": "^2.0.0",
         "tslib": "^2.8.1",
         "type-fest": "^4.41.0"
@@ -75,5 +75,5 @@
             }
         }
     },
-    "gitHead": "80dc6b4fc82237e63a51a71153809ec8dfd0cc50"
+    "gitHead": "5f25b90c4914d7e097732cf761e440aeba799137"
 }