@crawlee/browser 4.0.0-beta.122 → 4.0.0-beta.124

package/internals/browser-crawler.d.ts

@@ -1,9 +1,7 @@
-import type { BasicCrawlerOptions, BasicCrawlingContext, CrawlingContext, EnqueueLinksOptions, ErrorHandler, GetUserDataFromRequest, IRequestManager, LoadedRequest, Request, RequestHandler, RouterHandler, SkippedRequestCallback } from '@crawlee/basic';
+import type { AddRequestsBatchedResult, BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, ErrorHandler, ExtractLinksOptions, GetUserDataFromRequest, LoadedRequest, Request, RequestHandler, RouterHandler } from '@crawlee/basic';
 import { BasicCrawler, ContextPipeline } from '@crawlee/basic';
 import type { BrowserController, BrowserPlugin, BrowserPoolHooks, BrowserPoolOptions, CommonPage, CrawlerRemoteBrowserOptions, InferBrowserPluginArray, LaunchContext } from '@crawlee/browser-pool';
-import type { Awaitable, BatchAddRequestsResult, Dictionary, IBrowserPool } from '@crawlee/types';
-import type { RobotsTxtFile } from '@crawlee/utils';
-import type { ReadonlyDeep } from 'type-fest';
+import type { Awaitable, Dictionary, IBrowserPool } from '@crawlee/types';
 import { z } from 'zod';
 import type { BrowserLaunchContext } from './browser-launcher.js';
 interface BaseResponse {
@@ -30,10 +28,14 @@ GoToOptions extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
      * object (or return `{ gotoOptions: ... }`) to influence the navigation.
      */
     gotoOptions: GoToOptions;
+    /**
+     * Extracts URLs from the current page, without adding them to the request queue.
+     */
+    extractLinks: (options?: ExtractLinksOptions) => Promise<string[]>;
     /**
      * Helper function for extracting URLs from the current page and adding them to the request queue.
      */
-    enqueueLinks: (options?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
+    enqueueLinks: (options?: EnqueueLinksOptions) => Promise<AddRequestsBatchedResult>;
 }
 export type BrowserHook<Context = BrowserCrawlingContext, ContextExtension = {}> = (crawlingContext: Context & ContextExtension) => Awaitable<void | Partial<Context>>;
 export interface BrowserCrawlerOptions<Page extends CommonPage = CommonPage, Response extends BaseResponse = BaseResponse, Context extends BrowserCrawlingContext<Page, Response> = BrowserCrawlingContext<Page, Response>, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions<Context, ContextExtension, ExtendedContext>, 'requestHandler' | 'failedRequestHandler' | 'errorHandler'> {
@@ -397,26 +399,6 @@ export declare abstract class BrowserCrawler<Page extends CommonPage = CommonPag
      */
     teardown(): Promise<void>;
 }
-/** @internal */
-interface EnqueueLinksInternalOptions {
-    options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestManager'>> & Pick<EnqueueLinksOptions, 'requestManager'>;
-    page: CommonPage;
-    requestManager: IRequestManager;
-    robotsTxtFile?: RobotsTxtFile;
-    onSkippedRequest?: SkippedRequestCallback;
-    originalRequestUrl: string;
-    finalRequestUrl?: string;
-}
-/** @internal */
-interface BoundEnqueueLinksInternalOptions {
-    enqueueLinks: BasicCrawlingContext['enqueueLinks'];
-    options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestManager'>> & Pick<EnqueueLinksOptions, 'requestManager'>;
-    originalRequestUrl: string;
-    finalRequestUrl?: string;
-    page: CommonPage;
-}
-/** @internal */
-export declare function browserCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
 /**
  * Extracts URLs from a given page.
  * @ignore

package/internals/browser-crawler.js

@@ -1,4 +1,4 @@
-import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, enqueueLinks, NavigationSkippedError, OwnedOrInjected, parseArgument, remainingNavigationWindowMillis, RequestState, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, schemas, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
+import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, OwnedOrInjected, parseArgument, remainingNavigationWindowMillis, RequestState, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, schemas, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
 import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
 import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS } from '@crawlee/utils/internal';
 import { sleep } from '@crawlee/utils';
@@ -222,7 +222,10 @@ export class BrowserCrawler extends BasicCrawler {
             session: crawlingContext.session,
         });
         tryCancel();
-        const contextEnqueueLinks = crawlingContext.enqueueLinks;
+        const addRequests = crawlingContext.addRequests;
+        const extractLinks = async (options) => {
+            return extractUrlsFromPage(page, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
+        };
         return {
             page,
             get response() {
@@ -231,20 +234,20 @@ export class BrowserCrawler extends BasicCrawler {
             get gotoOptions() {
                 throw new Error('The `gotoOptions` property is not available until `prepareNavigation` runs.');
             },
-            enqueueLinks: async (enqueueOptions = {}) => {
-                return (await browserCrawlerEnqueueLinks({
-                    options: {
-                        ...enqueueOptions,
-                        limit: await this.calculateEnqueuedRequestLimit(enqueueOptions?.limit),
-                    },
-                    page,
-                    requestManager: await this.getRequestManager(),
-                    robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
-                    onSkippedRequest: this.handleSkippedRequest,
-                    originalRequestUrl: crawlingContext.request.url,
+            extractLinks,
+            enqueueLinks: async (options = {}) => {
+                const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
+                    enqueueStrategy: options.strategy,
                     finalRequestUrl: crawlingContext.request.loadedUrl,
-                    enqueueLinks: contextEnqueueLinks,
-                })); // TODO make this type safe
+                    originalRequestUrl: crawlingContext.request.url,
+                    userProvidedBaseUrl: options.baseUrl,
+                });
+                const urls = await extractLinks(options);
+                return addRequests(urls, {
+                    ...options,
+                    baseUrl,
+                    strategy: options.strategy ?? EnqueueStrategy.SameHostname,
+                });
             },
         };
     }
@@ -448,36 +451,6 @@ export class BrowserCrawler extends BasicCrawler {
         await super.teardown();
     }
 }
-/** @internal */
-function containsEnqueueLinks(options) {
-    return !!options.enqueueLinks;
-}
-/** @internal */
-export async function browserCrawlerEnqueueLinks(options) {
-    const { options: enqueueLinksOptions, finalRequestUrl, originalRequestUrl, page } = options;
-    const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
-        enqueueStrategy: enqueueLinksOptions?.strategy,
-        finalRequestUrl,
-        originalRequestUrl,
-        userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
-    });
-    const urls = await extractUrlsFromPage(page, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
-    if (containsEnqueueLinks(options)) {
-        return options.enqueueLinks({
-            urls,
-            baseUrl,
-            ...enqueueLinksOptions,
-        });
-    }
-    return enqueueLinks({
-        requestManager: options.requestManager,
-        robotsTxtFile: options.robotsTxtFile,
-        onSkippedRequest: options.onSkippedRequest,
-        urls,
-        baseUrl,
-        ...enqueueLinksOptions,
-    });
-}
 /**
  * Extracts URLs from a given page.
  * @ignore

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/browser",
-    "version": "4.0.0-beta.122",
+    "version": "4.0.0-beta.124",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -48,10 +48,10 @@
     },
     "dependencies": {
         "@apify/timeout": "^0.4.4",
-        "@crawlee/basic": "4.0.0-beta.122",
-        "@crawlee/browser-pool": "4.0.0-beta.122",
-        "@crawlee/types": "4.0.0-beta.122",
-        "@crawlee/utils": "4.0.0-beta.122",
+        "@crawlee/basic": "4.0.0-beta.124",
+        "@crawlee/browser-pool": "4.0.0-beta.124",
+        "@crawlee/types": "4.0.0-beta.124",
+        "@crawlee/utils": "4.0.0-beta.124",
         "tslib": "^2.8.1",
         "type-fest": "^4.41.0",
         "zod": "^4.4.3"
@@ -75,5 +75,5 @@
             }
         }
     },
-    "gitHead": "2c3e87fefdb9e1fca4c144f03167d8524cfdc2e5"
+    "gitHead": "0694ee1b94c755b98141671baa93cc363f2bf8e3"
 }