@crawlee/browser 3.0.0-alpha.3 → 3.0.0-alpha.30

package/{dist/index.mjs → index.mjs}

@@ -40,7 +40,6 @@ export const Snapshotter = mod.Snapshotter;
 export const Statistics = mod.Statistics;
 export const StorageManager = mod.StorageManager;
 export const SystemStatus = mod.SystemStatus;
-export const basicCrawlerEnqueueLinks = mod.basicCrawlerEnqueueLinks;
 export const browserCrawlerEnqueueLinks = mod.browserCrawlerEnqueueLinks;
 export const checkAndSerialize = mod.checkAndSerialize;
 export const chunkBySize = mod.chunkBySize;
@@ -59,11 +58,9 @@ export const handleRequestTimeout = mod.handleRequestTimeout;
 export const log = mod.log;
 export const maybeStringify = mod.maybeStringify;
 export const mergeCookies = mod.mergeCookies;
-export const openDataset = mod.openDataset;
-export const pushData = mod.pushData;
+export const resolveBaseUrl = mod.resolveBaseUrl;
 export const serializeArray = mod.serializeArray;
 export const storage = mod.storage;
-export const throwOnBlockedRequest = mod.throwOnBlockedRequest;
 export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
 export const validateGlobPattern = mod.validateGlobPattern;
 export const validators = mod.validators;

package/{dist/internals → internals}/browser-crawler.d.ts

@@ -3,7 +3,7 @@ import { BasicCrawler, BasicCrawlerOptions } from '@crawlee/basic';
 import { Awaitable, Dictionary } from '@crawlee/utils';
 import { BrowserController, BrowserPlugin, BrowserPool, BrowserPoolHooks, BrowserPoolOptions, CommonPage, InferBrowserPluginArray, LaunchContext } from '@crawlee/browser-pool';
 import { BrowserLaunchContext } from './browser-launcher';
-export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response = Dictionary<any>, ProvidedController = BrowserController> extends CrawlingContext {
+export interface BrowserCrawlingContext<Page extends CommonPage = CommonPage, Response = Dictionary, ProvidedController = BrowserController> extends CrawlingContext {
     browserController: ProvidedController;
     page: Page;
     response?: Response;
@@ -17,8 +17,8 @@ export declare type BrowserCrawlerHandleFailedRequest = (inputs: BrowserCrawlerH
 export declare type BrowserCrawlerHandleRequest<Context extends BrowserCrawlingContext = BrowserCrawlingContext> = (inputs: Context) => Awaitable<void>;
 export declare type BrowserCrawlerEnqueueLinksOptions = Omit<EnqueueLinksOptions, 'requestQueue' | 'urls'>;
 export declare type BrowserHook<Context = BrowserCrawlingContext, GoToOptions extends Record<PropertyKey, any> | undefined = Dictionary> = (crawlingContext: Context, gotoOptions: GoToOptions) => Awaitable<void>;
-export declare type GotoFunction<Context = BrowserCrawlingContext, GoToOptions extends Record<PropertyKey, any> | undefined = Dictionary> = (context: Context, gotoOptions?: GoToOptions) => Awaitable<any>;
-export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext = BrowserCrawlingContext, GoToOptions extends Record<PropertyKey, any> | undefined = Dictionary, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions, 'requestHandler' | 'handleRequestFunction' | 'failedRequestHandler' | 'handleFailedRequestFunction'> {
+export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext = BrowserCrawlingContext, InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions, __BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<InternalBrowserPoolOptions['browserPlugins']>, __BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>, __LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>> extends Omit<BasicCrawlerOptions, 'requestHandler' | 'handleRequestFunction' | 'failedRequestHandler' | 'handleFailedRequestFunction'> {
+    launchContext?: BrowserLaunchContext<any, any>;
     /**
      * Function that is called to process each request.
      * It is passed an object with the following fields:
@@ -101,10 +101,6 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
      * @deprecated `handlePageFunction` has been renamed to `requestHandler` and will be removed in a future version.
      */
     handlePageFunction?: BrowserCrawlerHandleRequest<Context>;
-    /**
-     * Navigation function for corresponding library. `page.goto(url)` is supported by both `playwright` and `puppeteer`.
-     */
-    gotoFunction?: GotoFunction<Context, GoToOptions>;
     /**
      * A function to handle requests that failed more than `option.maxRequestRetries` times.
      *
@@ -194,10 +190,6 @@ export interface BrowserCrawlerOptions<Context extends BrowserCrawlingContext =
      * Timeout in which page navigation needs to finish, in seconds.
      */
     navigationTimeoutSecs?: number;
-    /**
-     * @deprecated Use `navigationTimeoutSecs` instead
-     */
-    gotoTimeoutSecs?: number;
     /**
      * If cookies should be persisted between sessions.
      * This can only be used when `useSessionPool` is set to `true`.
@@ -256,18 +248,15 @@ export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends
     launchContext?: BrowserLaunchContext<LaunchOptions, unknown>;
     protected userProvidedRequestHandler: BrowserCrawlerHandleRequest<Context>;
     protected navigationTimeoutMillis: number;
-    protected gotoFunction?: GotoFunction<Context, GoToOptions>;
-    protected defaultGotoOptions: GoToOptions;
     protected preNavigationHooks: BrowserHook<Context>[];
     protected postNavigationHooks: BrowserHook<Context>[];
     protected persistCookiesPerSession: boolean;
     protected static optionsShape: {
         handlePageFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-        gotoFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
-        gotoTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
         navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
         preNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
         postNavigationHooks: import("ow").ArrayPredicate<unknown> & import("ow").BasePredicate<unknown[] | undefined>;
+        launchContext: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
         browserPoolOptions: import("ow").ObjectPredicate<object>;
         sessionPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
         persistCookiesPerSession: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
@@ -291,18 +280,19 @@ export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends
     /**
      * All `BrowserCrawler` parameters are passed via an options object.
      */
-    protected constructor(options: BrowserCrawlerOptions<Context, GoToOptions>);
+    protected constructor(options: BrowserCrawlerOptions<Context>);
     /**
      * Wrapper around handlePageFunction that opens and closes pages etc.
      */
     protected _runRequestHandler(crawlingContext: Context): Promise<void>;
     protected _enhanceCrawlingContextWithPageInfo(crawlingContext: Context, page: CommonPage, useIncognitoPages?: boolean): void;
     protected _handleNavigation(crawlingContext: Context): Promise<void>;
+    protected _applyCookies({ session, request, page, browserController }: Context, preHooksCookies: string, postHooksCookies: string): Promise<void>;
     /**
      * Marks session bad in case of navigation timeout.
      */
     protected _handleNavigationTimeout(crawlingContext: Context, error: Error): void;
-    protected _navigationHandler(crawlingContext: Context, gotoOptions: GoToOptions): Promise<any>;
+    protected abstract _navigationHandler(crawlingContext: Context, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
     /**
      * Should be overridden in case of different automation library that does not support this response API.
      * @todo: This can be also done as a postNavigation hook except the loadedUrl marking.
@@ -320,10 +310,11 @@ export declare abstract class BrowserCrawler<InternalBrowserPoolOptions extends
 interface EnqueueLinksInternalOptions {
     options?: BrowserCrawlerEnqueueLinksOptions;
     page: CommonPage;
-    requestQueue?: RequestQueue;
-    defaultBaseUrl?: string;
+    requestQueue: RequestQueue;
+    originalRequestUrl: string;
+    finalRequestUrl?: string;
 }
 /** @internal */
-export declare function browserCrawlerEnqueueLinks({ options, page, requestQueue, defaultBaseUrl }: EnqueueLinksInternalOptions): Promise<storage.BatchAddRequestsResult>;
+export declare function browserCrawlerEnqueueLinks({ options, page, requestQueue, originalRequestUrl, finalRequestUrl, }: EnqueueLinksInternalOptions): Promise<storage.BatchAddRequestsResult>;
 export {};
 //# sourceMappingURL=browser-crawler.d.ts.map

package/internals/browser-crawler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-crawler.d.ts","sourceRoot":"","sources":["../../src/internals/browser-crawler.ts"],"names":[],"mappings":"AACA,OAAO,EACH,mBAAmB,EACnB,+BAA+B,EAC/B,eAAe,EAIf,kBAAkB,EAElB,YAAY,EAGZ,OAAO,EAEV,MAAM,eAAe,CAAC;AACvB,OAAO,EAEH,YAAY,EACZ,mBAAmB,EACtB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,EAEH,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EAEV,uBAAuB,EACvB,aAAa,EAChB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,MAAM,WAAW,sBAAsB,CACnC,IAAI,SAAS,UAAU,GAAG,UAAU,EACpC,QAAQ,GAAG,UAAU,EACrB,kBAAkB,GAAG,iBAAiB,CACxC,SAAQ,eAAe;IACrB,iBAAiB,EAAE,kBAAkB,CAAC;IACtC,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,OAAO,EAAE,cAAc,CAAC;IACxB,YAAY,EAAE,CAAC,OAAO,CAAC,EAAE,iCAAiC,KAAK,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;CAC1G;AAED,MAAM,WAAW,sCAAuC,SAAQ,+BAA+B;IAC3F,OAAO,EAAE,cAAc,CAAC;CAC3B;AAED,oBAAY,iCAAiC,GAAG,CAAC,MAAM,EAAE,sCAAsC,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAEpH,oBAAY,2BAA2B,CAAC,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAEhJ,oBAAY,iCAAiC,GAAG,IAAI,CAAC,mBAAmB,EAAE,cAAc,GAAG,MAAM,CAAC,CAAA;AAElG,oBAAY,WAAW,CACnB,OAAO,GAAG,sBAAsB,EAChC,WAAW,SAAS,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,GAAG,SAAS,GAAG,UAAU,IACrE,CAAC,eAAe,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,KAAK,SAAS,CAAC,IAAI,CAAC,CAAC;AAE5E,MAAM,WAAW,qBAAqB,CAClC,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,EAC/D,0BAA0B,SAAS,kBAAkB,GAAG,kBAAkB,EAC1E,gBAAgB,SAAS,aAAa,EAAE,GAAG,uBAAuB,CAAC,0BAA0B,CAAC,gBAAgB,CAAC,CAAC,EAChH,yBAAyB,SAAS,iBAAiB,GAAG,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC,EAC9G,qBAAqB,SAAS,aAAa,GAAG,UAAU,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAC3G,SAAQ,IAAI,CACV,mBAAmB,EAEjB,gBAAgB,GAChB,uBAAuB,GAEvB,sBAAsB,GACtB,6BAA6B,CAClC;IACG,aAAa,CAAC,EAAE,oBAAoB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC/C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAsCG;IACH,cAAc,EAAE,2BAA2B,CAAC,OAAO,CAAC,CAAC;IAErD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAwCG;IACH,kBAAkB,CAAC,EAAE,2BAA2B,CAAC,OAAO,CAAC,CAAC;IAE1D;;;;;;;;;;;;;;;;;;;OAmBG;IACH,oBAAoB,CAAC,EAAE,iCAAiC,CAAC;IAEzD;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,2BAA2B,CAAC,EAAE,iCAAiC,CAAC;IAEhE;;;OAGG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,yBAAyB,EAAE,qBAAqB,CAAC,CAAC,CAAC;IAE/H;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;;;;;;;;;;OAaG;IACH,kBAAkB,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IAE5C;;;;;;;;;;;;;;OAcG;IACH,mBAAmB,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IAE7C;;OAEG;IACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B;;;OAGG;IACH,wBAAwB,CAAC,EAAE,OAAO,CAAC;CACtC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,8BAAsB,cAAc,CAChC,0BAA0B,SAAS,kBAAkB,GAAG,kBAAkB,EAC1E,aAAa,GAAG,UAAU,EAC1B,OAAO,SAAS,sBAAsB,GAAG,sBAAsB,EAC/D,WAAW,SAAS,MAAM,CAAC,WAAW,EAAE,GAAG,CAAC,GAAG,UAAU,CAC3D,SAAQ,YAAY,CAAC,OAAO,CAAC;IAC3B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IAExC;;;;OAIG;IACH,WAAW,EAAE,WAAW,CAAC,0BAA0B,CAAC,CAAC;IAErD,aAAa,CAAC,EAAE,oBAAoB,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAE7D,SAAS,CAAC,0BAA0B,EAAG,2BAA2B,CAAC,OAAO,CAAC,CAAC;IAC5E,SAAS,CAAC,uBAAuB,EAAE,MAAM,CAAC;IAC1C,SAAS,CAAC,kBAAkB,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IACrD,SAAS,CAAC,mBAAmB,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IACtD,SAAS,CAAC,wBAAwB,EAAE,OAAO,CAAC;IAE5C,iBAA0B,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;MAcpC;IAEF;;OAEG;IACH,SAAS,aAAa,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC;IAkF7D;;OAEG;cACsB,kBAAkB,CAAC,eAAe,EAAE,OAAO;IAkEpE,SAAS,CAAC,mCAAmC,CAAC,eAAe,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,iBAAiB,CAAC,EAAE,OAAO,GAAG,IAAI;cA6B5G,iBAAiB,CAAC,eAAe,EAAE,OAAO;cAwB1C,aAAa,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM;IAevI;;OAEG;IACH,SAAS,CAAC,wBAAwB,CAAC,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI;IAQhF,SAAS,CAAC,QAAQ,CAAC,kBAAkB,CAAC,eAAe,EAAE,OAAO,EAAE,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;IAE1I;;;OAGG;cACa,gBAAgB,CAAC,eAAe,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;cAczD,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBlG,SAAS,CAAC,+BAA+B,CAAC,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,IAAI;IAkBjH;;;OAGG;IACY,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAI3C;AAED,gBAAgB;AAChB,UAAU,2BAA2B;IACjC,OAAO,CAAC,EAAE,iCAAiC,CAAC;IAC5C,IAAI,EAAE,UAAU,CAAC;IACjB,YAAY,EAAE,YAAY,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,gBAAgB;AAChB,wBAAsB,0BAA0B,CAAC,EAC7C,OAAO,EACP,IAAI,EACJ,YAAY,EACZ,kBAAkB,EAClB,eAAe,GAClB,EAAE,2BAA2B,2CAgB7B"}

package/{dist/internals → internals}/browser-crawler.js

@@ -7,6 +7,7 @@ const core_1 = require("@crawlee/core");
 const basic_1 = require("@crawlee/basic");
 const browser_pool_1 = require("@crawlee/browser-pool");
 const ow_1 = tslib_1.__importDefault(require("ow"));
+const tough_cookie_1 = require("tough-cookie");
 /**
  * Provides a simple framework for parallel crawling of web pages
  * using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
@@ -50,9 +51,7 @@ class BrowserCrawler extends basic_1.BasicCrawler {
      */
     constructor(options) {
         (0, ow_1.default)(options, 'BrowserCrawlerOptions', ow_1.default.object.exactShape(BrowserCrawler.optionsShape));
-        const { navigationTimeoutSecs = 60, requestHandlerTimeoutSecs = 60, gotoFunction, // deprecated
-        gotoTimeoutSecs, // deprecated
-        persistCookiesPerSession, proxyConfiguration, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], 
+        const { navigationTimeoutSecs = 60, requestHandlerTimeoutSecs = 60, persistCookiesPerSession, proxyConfiguration, launchContext, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], 
         // Ignored
         handleRequestFunction, requestHandler: userProvidedRequestHandler, handlePageFunction, failedRequestHandler, handleFailedRequestFunction, ...basicCrawlerOptions } = options;
         super({
@@ -99,18 +98,6 @@ class BrowserCrawler extends basic_1.BasicCrawler {
             writable: true,
             value: void 0
         });
-        Object.defineProperty(this, "gotoFunction", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "defaultGotoOptions", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
         Object.defineProperty(this, "preNavigationHooks", {
             enumerable: true,
             configurable: true,
@@ -148,14 +135,8 @@ class BrowserCrawler extends basic_1.BasicCrawler {
         if (!this.useSessionPool && persistCookiesPerSession) {
             throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
         }
-        if (gotoTimeoutSecs) {
-            this.log.deprecated('Option "gotoTimeoutSecs" is deprecated. Use "navigationTimeoutSecs" instead.');
-        }
-        this.navigationTimeoutMillis = (gotoTimeoutSecs || navigationTimeoutSecs) * 1000;
-        this.gotoFunction = gotoFunction;
-        this.defaultGotoOptions = {
-            timeout: this.navigationTimeoutMillis,
-        };
+        this.launchContext = launchContext;
+        this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
         this.proxyConfiguration = proxyConfiguration;
         this.preNavigationHooks = preNavigationHooks;
         this.postNavigationHooks = postNavigationHooks;
@@ -165,6 +146,10 @@ class BrowserCrawler extends basic_1.BasicCrawler {
         else {
             this.persistCookiesPerSession = false;
         }
+        if (launchContext?.userAgent) {
+            browserPoolOptions.useFingerprints = false;
+            this.log.info('Disabling automatic fingerprint injection because custom user agent has been provided.');
+        }
         const { preLaunchHooks = [], postLaunchHooks = [], ...rest } = browserPoolOptions;
         this.browserPool = new browser_pool_1.BrowserPool({
             ...rest,
@@ -188,7 +173,7 @@ class BrowserCrawler extends basic_1.BasicCrawler {
         const useIncognitoPages = this.launchContext?.useIncognitoPages;
         if (this.proxyConfiguration && useIncognitoPages) {
             const { session } = crawlingContext;
-            const proxyInfo = this.proxyConfiguration.newProxyInfo(session?.id);
+            const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id);
             crawlingContext.proxyInfo = proxyInfo;
             newPageOptions.proxyUrl = proxyInfo.url;
             if (this.proxyConfiguration.isManInTheMiddle) {
@@ -209,24 +194,19 @@ class BrowserCrawler extends basic_1.BasicCrawler {
         // For example, `sessionPoolOptions.sessionOptions.maxUsageCount` can be `1`.
         // So we must not save the session prior to making sure it was used only once, otherwise we would use it twice.
         const { request, session } = crawlingContext;
-        if (this.useSessionPool) {
-            const sessionCookies = session.getPuppeteerCookies(request.url);
-            if (sessionCookies.length) {
-                await crawlingContext.browserController.setCookies(page, sessionCookies);
-                (0, timeout_1.tryCancel)();
-            }
-        }
         try {
-            await this._handleNavigation(crawlingContext);
-            (0, timeout_1.tryCancel)();
-            await this._responseHandler(crawlingContext);
-            (0, timeout_1.tryCancel)();
-            // save cookies
-            // TODO: Should we save the cookies also after/only the handle page?
-            if (this.persistCookiesPerSession) {
-                const cookies = await crawlingContext.browserController.getCookies(page);
+            if (!request.skipNavigation) {
+                await this._handleNavigation(crawlingContext);
                 (0, timeout_1.tryCancel)();
-                session?.setPuppeteerCookies(cookies, request.loadedUrl);
+                await this._responseHandler(crawlingContext);
+                (0, timeout_1.tryCancel)();
+                // save cookies
+                // TODO: Should we save the cookies also after/only the handle page?
+                if (this.persistCookiesPerSession) {
+                    const cookies = await crawlingContext.browserController.getCookies(page);
+                    (0, timeout_1.tryCancel)();
+                    session?.setPuppeteerCookies(cookies, request.loadedUrl);
+                }
             }
             await (0, timeout_1.addTimeoutToPromise)(() => Promise.resolve(this.userProvidedRequestHandler(crawlingContext)), this.requestHandlerTimeoutMillis, `requestHandler timed out after ${this.requestHandlerTimeoutMillis / 1000} seconds.`);
             (0, timeout_1.tryCancel)();
@@ -256,16 +236,20 @@ class BrowserCrawler extends basic_1.BasicCrawler {
                 options: enqueueOptions,
                 page,
                 requestQueue: await this.getRequestQueue(),
-                defaultBaseUrl: new URL(crawlingContext.request.loadedUrl ?? crawlingContext.request.url).origin,
+                originalRequestUrl: new URL(crawlingContext.request.url).origin,
+                finalRequestUrl: new URL(crawlingContext.request.loadedUrl ?? crawlingContext.request.url).origin,
             });
         };
     }
     async _handleNavigation(crawlingContext) {
-        const gotoOptions = { ...this.defaultGotoOptions };
+        const gotoOptions = { timeout: this.navigationTimeoutMillis };
+        const preNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
         await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
         (0, timeout_1.tryCancel)();
+        const postNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
+        await this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
         try {
-            crawlingContext.response = await this._navigationHandler(crawlingContext, gotoOptions);
+            crawlingContext.response = await this._navigationHandler(crawlingContext, gotoOptions) ?? undefined;
         }
         catch (error) {
             this._handleNavigationTimeout(crawlingContext, error);
@@ -274,6 +258,16 @@ class BrowserCrawler extends basic_1.BasicCrawler {
         (0, timeout_1.tryCancel)();
         await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
     }
+    async _applyCookies({ session, request, page, browserController }, preHooksCookies, postHooksCookies) {
+        const sessionCookie = session?.getPuppeteerCookies(request.url) ?? [];
+        const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => tough_cookie_1.Cookie.parse(c)?.toJSON());
+        const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => tough_cookie_1.Cookie.parse(c)?.toJSON());
+        await browserController.setCookies(page, [
+            ...sessionCookie,
+            ...parsedPreHooksCookies,
+            ...parsedPostHooksCookies,
+        ].filter((c) => typeof c !== 'undefined'));
+    }
     /**
      * Marks session bad in case of navigation timeout.
      */
@@ -283,14 +277,6 @@ class BrowserCrawler extends basic_1.BasicCrawler {
             (0, core_1.handleRequestTimeout)({ session, errorMessage: error.message });
         }
     }
-    async _navigationHandler(crawlingContext, gotoOptions) {
-        if (!this.gotoFunction) {
-            // @TODO: although it is optional in the validation,
-            //   because when you make automation library specific you can override this handler.
-            throw new Error('BrowserCrawler: You must specify a gotoFunction!');
-        }
-        return this.gotoFunction(crawlingContext, gotoOptions);
-    }
     /**
      * Should be overridden in case of different automation library that does not support this response API.
      * @todo: This can be also done as a postNavigation hook except the loadedUrl marking.
@@ -299,7 +285,7 @@ class BrowserCrawler extends basic_1.BasicCrawler {
         const { response, session, request, page } = crawlingContext;
         if (this.sessionPool && response && session) {
             if (typeof response === 'object' && typeof response.status === 'function') {
-                (0, core_1.throwOnBlockedRequest)(session, response.status());
+                this._throwOnBlockedRequest(session, response.status());
             }
             else {
                 this.log.debug('Got a malformed Browser response.', { request, response });
@@ -313,7 +299,7 @@ class BrowserCrawler extends basic_1.BasicCrawler {
             launchContextExtends.session = await this.sessionPool.getSession();
         }
         if (this.proxyConfiguration) {
-            const proxyInfo = this.proxyConfiguration.newProxyInfo(launchContextExtends.session && launchContextExtends.session.id);
+            const proxyInfo = await this.proxyConfiguration.newProxyInfo(launchContextExtends.session?.id);
             launchContext.proxyUrl = proxyInfo.url;
             launchContextExtends.proxyInfo = proxyInfo;
             // Disable SSL verification for MITM proxies
@@ -358,11 +344,10 @@ Object.defineProperty(BrowserCrawler, "optionsShape", {
     value: {
         ...basic_1.BasicCrawler.optionsShape,
         handlePageFunction: ow_1.default.optional.function,
-        gotoFunction: ow_1.default.optional.function,
-        gotoTimeoutSecs: ow_1.default.optional.number.greaterThan(0),
         navigationTimeoutSecs: ow_1.default.optional.number.greaterThan(0),
         preNavigationHooks: ow_1.default.optional.array,
         postNavigationHooks: ow_1.default.optional.array,
+        launchContext: ow_1.default.optional.object,
         browserPoolOptions: ow_1.default.object,
         sessionPoolOptions: ow_1.default.optional.object,
         persistCookiesPerSession: ow_1.default.optional.boolean,
@@ -371,12 +356,16 @@ Object.defineProperty(BrowserCrawler, "optionsShape", {
     }
 });
 /** @internal */
-async function browserCrawlerEnqueueLinks({ options, page, requestQueue, defaultBaseUrl }) {
-    const baseUrl = options?.baseUrl ?? defaultBaseUrl;
+async function browserCrawlerEnqueueLinks({ options, page, requestQueue, originalRequestUrl, finalRequestUrl, }) {
+    const baseUrl = (0, core_1.resolveBaseUrl)({
+        enqueueStrategy: options?.strategy,
+        finalRequestUrl,
+        originalRequestUrl,
+        userProvidedBaseUrl: options?.baseUrl,
+    });
     const urls = await extractUrlsFromPage(page, options?.selector ?? 'a', baseUrl);
     return (0, core_1.enqueueLinks)({
-        // FIXME this does not make much sense, as the instance would be ignored by any crawler - the argument needs to be required
-        requestQueue: requestQueue ?? await core_1.RequestQueue.open(),
+        requestQueue,
         urls,
         baseUrl,
         ...options,

package/internals/browser-crawler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-crawler.js","sourceRoot":"","sources":["../../src/internals/browser-crawler.ts"],"names":[],"mappings":";;;;AAAA,4CAAgE;AAChE,wCAcuB;AACvB,0CAIwB;AAExB,wDAW+B;AAC/B,oDAAoB;AACpB,+CAAsC;AAyOtC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AACH,MAAsB,cAKpB,SAAQ,oBAAqB;IAsC3B;;OAEG;IACH,YAAsB,OAAuC;QACzD,IAAA,YAAE,EAAC,OAAO,EAAE,uBAAuB,EAAE,YAAE,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC;QACxF,MAAM,EACF,qBAAqB,GAAG,EAAE,EAC1B,yBAAyB,GAAG,EAAE,EAC9B,wBAAwB,EACxB,kBAAkB,EAClB,aAAa,EACb,kBAAkB,EAClB,kBAAkB,GAAG,EAAE,EACvB,mBAAmB,GAAG,EAAE;QACxB,UAAU;QACV,qBAAqB,EAErB,cAAc,EAAE,0BAA0B,EAC1C,kBAAkB,EAElB,oBAAoB,EACpB,2BAA2B,EAC3B,GAAG,mBAAmB,EACzB,GAAG,OAAO,CAAC;QAEZ,KAAK,CAAC;YACF,GAAG,mBAAmB;YACtB,cAAc,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,IAAI,CAAC;YAC7D,yBAAyB,EAAE,qBAAqB,GAAG,yBAAyB,GAAG,yCAAiC;SACnH,CAAC,CAAC;QAlEP;;;WAGG;QACH;;;;;WAAwC;QAExC;;;;WAIG;QACH;;;;;WAAqD;QAErD;;;;;WAA6D;QAE7D;;;;;WAA4E;QAC5E;;;;;WAA0C;QAC1C;;;;;WAAqD;QACrD;;;;;WAAsD;QACtD;;;;;WAA4C;QAiDxC,IAAI,CAAC,yBAAyB,CAAC;YAC3B,OAAO,EAAE,gBAAgB;YACzB,OAAO,EAAE,oBAAoB;YAC7B,WAAW,EAAE,4BAA4B;YACzC,WAAW,EAAE,0BAA0B;YACvC,WAAW,EAAE,kBAAkB;SAClC,CAAC,CAAC;QAEH,IAAI,CAAC,yBAAyB,CAAC;YAC3B,OAAO,EAAE,sBAAsB;YAC/B,OAAO,EAAE,6BAA6B;YACtC,WAAW,EAAE,sBAAsB;YACnC,WAAW,EAAE,oBAAoB;YACjC,WAAW,EAAE,2BAA2B;YACxC,cAAc,EAAE,IAAI;SACvB,CAAC,CAAC;QAEH,uEAAuE;QACvE,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,wBAAwB,EAAE;YAClD,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;SACtG;QAED,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,uBAAuB,GAAG,qBAAqB,GAAG,IAAI,CAAC;QAC5D,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAE/C,IAAI,IAAI,CAAC,cAAc,EAAE;YACrB,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,KAAK,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC;SAC5G;aAAM;YACH,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;SACzC;QAED,IAAI,aAAa,EAAE,SAAS,EAAE;YAC1B,kBAAkB,CAAC,eAAe,GAAG,KAAK,CAAC;YAC3C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,wFAAwF,CAAC,CAAC;SAC3G;QAED,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,eAAe,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,kBAAkB,CAAC;QAElF,IAAI,CAAC,WAAW,GAAG,IAAI,0BAAW,CAA6B;YAC3D,GAAG,IAAW;YACd,cAAc,EAAE;gBACZ,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;gBACpC,GAAG,cAAc;aACpB;YACD,eAAe,EAAE;gBACb,IAAI,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC/C,GAAG,eAAe;aACrB;SACJ,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACgB,KAAK,CAAC,kBAAkB,CAAC,eAAwB;QAChE,MAAM,cAAc,GAAe;YAC/B,EAAE,EAAE,eAAe,CAAC,EAAE;SACzB,CAAC;QAEF,MAAM,iBAAiB,GAAG,IAAI,CAAC,aAAa,EAAE,iBAAiB,CAAC;QAEhE,IAAI,IAAI,CAAC,kBAAkB,IAAI,iBAAiB,EAAE;YAC9C,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;YAEpC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAC1E,eAAe,CAAC,SAAS,GAAG,SAAS,CAAC;YAEtC,cAAc,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC;YAExC,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;gBAC1C;;;mBAGG;gBACH,cAAc,CAAC,WAAW,GAAG;oBACzB,iBAAiB,EAAE,IAAI;iBAC1B,CAAC;aACL;SACJ;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,cAAc,CAAe,CAAC;QAC1E,IAAA,mBAAS,GAAE,CAAC;QACZ,IAAI,CAAC,mCAAmC,CAAC,eAAe,EAAE,IAAI,EAAE,iBAAiB,CAAC,CAAC;QAEnF,+BAA+B;QAC/B,iEAAiE;QACjE,6EAA6E;QAC7E,+GAA+G;QAC/G,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAE7C,IAAI;YACA,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE;gBACzB,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC;gBAC9C,IAAA,mBAAS,GAAE,CAAC;gBAEZ,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;gBAC7C,IAAA,mBAAS,GAAE,CAAC;gBAEZ,eAAe;gBACf,oEAAoE;gBACpE,IAAI,IAAI,CAAC,wBAAwB,EAAE;oBAC/B,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;oBACzE,IAAA,mBAAS,GAAE,CAAC;oBACZ,OAAO,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,SAAU,CAAC,CAAC;iBAC7D;aACJ;YAED,MAAM,IAAA,6BAAmB,EACrB,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,0BAA0B,CAAC,eAAe,CAAC,CAAC,EACvE,IAAI,CAAC,2BAA2B,EAChC,kCAAkC,IAAI,CAAC,2BAA2B,GAAG,IAAI,WAAW,CACvF,CAAC;YACF,IAAA,mBAAS,GAAE,CAAC;YAEZ,IAAI,OAAO;gBAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;SACnC;gBAAS;YACN,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;SAC/F;IACL,CAAC;IAES,mCAAmC,CAAC,eAAwB,EAAE,IAAgB,EAAE,iBAA2B;QACjH,eAAe,CAAC,IAAI,GAAG,IAAI,CAAC;QAE5B,gFAAgF;QAChF,6HAA6H;QAC7H,kGAAkG;QAClG,2DAA2D;QAC3D,MAAM,yBAAyB,GAAG,IAAI,CAAC,WAAW,CAAC,0BAA0B,CAAC,IAAW,CAAiC,CAAC;QAC3H,eAAe,CAAC,iBAAiB,GAAG,yBAAyB,CAAC;QAE9D,IAAI,CAAC,iBAAiB,EAAE;YACpB,eAAe,CAAC,OAAO,GAAG,yBAAyB,CAAC,aAAa,CAAC,OAAkB,CAAC;SACxF;QAED,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE;YAC5B,eAAe,CAAC,SAAS,GAAG,yBAAyB,CAAC,aAAa,CAAC,SAAsB,CAAC;SAC9F;QAED,eAAe,CAAC,YAAY,GAAG,KAAK,EAAE,cAAc,EAAE,EAAE;YACpD,OAAO,0BAA0B,CAAC;gBAC9B,OAAO,EAAE,cAAc;gBACvB,IAAI;gBACJ,YAAY,EAAE,MAAM,IAAI,CAAC,eAAe,EAAE;gBAC1C,kBAAkB,EAAE,IAAI,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM;gBAC/D,eAAe,EAAE,IAAI,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS,IAAI,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM;aACpG,CAAC,CAAC;QACP,CAAC,CAAC;IACN,CAAC;IAES,KAAK,CAAC,iBAAiB,CAAC,eAAwB;QACtD,MAAM,WAAW,GAAG,EAAE,OAAO,EAAE,IAAI,CAAC,uBAAuB,EAA4B,CAAC;QAExF,MAAM,yBAAyB,GAAG,IAAI,CAAC,2BAA2B,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAE5F,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,kBAAkB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;QAChF,IAAA,mBAAS,GAAE,CAAC;QAEZ,MAAM,0BAA0B,GAAG,IAAI,CAAC,2BAA2B,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAE7F,MAAM,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,yBAAyB,EAAE,0BAA0B,CAAC,CAAC;QAEjG,IAAI;YACA,eAAe,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,WAAW,CAAC,IAAI,SAAS,CAAC;SACvG;QAAC,OAAO,KAAK,EAAE;YACZ,IAAI,CAAC,wBAAwB,CAAC,eAAe,EAAE,KAAc,CAAC,CAAC;YAE/D,MAAM,KAAK,CAAC;SACf;QAED,IAAA,mBAAS,GAAE,CAAC;QACZ,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,mBAAmB,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;IACrF,CAAC;IAES,KAAK,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAW,EAAE,eAAuB,EAAE,gBAAwB;QACnI,MAAM,aAAa,GAAG,OAAO,EAAE,mBAAmB,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtE,MAAM,qBAAqB,GAAG,eAAe,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,qBAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QACnG,MAAM,sBAAsB,GAAG,gBAAgB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,qBAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QAErG,MAAM,iBAAiB,CAAC,UAAU,CAC9B,IAAI,EACJ;YACI,GAAG,aAAa;YAChB,GAAG,qBAAqB;YACxB,GAAG,sBAAsB;SAC5B,CAAC,MAAM,CAAC,CAAC,CAAC,EAA0B,EAAE,CAAC,OAAO,CAAC,KAAK,WAAW,CAAC,CACpE,CAAC;IACN,CAAC;IAED;;OAEG;IACO,wBAAwB,CAAC,eAAwB,EAAE,KAAY;QACrE,MAAM,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC;QAEpC,IAAI,KAAK,IAAI,KAAK,CAAC,WAAW,CAAC,IAAI,KAAK,cAAc,EAAE;YACpD,IAAA,2BAAoB,EAAC,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SAClE;IACL,CAAC;IAID;;;OAGG;IACO,KAAK,CAAC,gBAAgB,CAAC,eAAwB;QACrD,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC;QAE7D,IAAI,IAAI,CAAC,WAAW,IAAI,QAAQ,IAAI,OAAO,EAAE;YACzC,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE;gBACvE,IAAI,CAAC,sBAAsB,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;aAC3D;iBAAM;gBACH,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,mCAAmC,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;aAC9E;SACJ;QAED,OAAO,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC;IACzC,CAAC;IAES,KAAK,CAAC,oBAAoB,CAAC,OAAe,EAAE,aAA4B;QAC9E,MAAM,oBAAoB,GAAiD,EAAE,CAAC;QAE9E,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,oBAAoB,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;SACtE;QAED,IAAI,IAAI,CAAC,kBAAkB,EAAE;YACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,oBAAoB,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAC/F,aAAa,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC;YACvC,oBAAoB,CAAC,SAAS,GAAG,SAAS,CAAC;YAE3C,4CAA4C;YAC5C,IAAI,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,EAAE;gBAC1C;;;mBAGG;gBACF,aAAa,CAAC,aAA4B,CAAC,iBAAiB,GAAG,IAAI,CAAC;aACxE;SACJ;QAED,aAAa,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAES,+BAA+B,CAAC,OAAe,EAAE,iBAA+C;QACtG,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB,MAAM,QAAQ,GAAG,CAAC,OAAgB,EAAE,EAAE;gBAClC,MAAM,EAAE,aAAa,EAAE,GAAG,iBAAiB,CAAC;gBAC5C,IAAI,OAAO,CAAC,EAAE,KAAM,aAAa,CAAC,OAAmB,CAAC,EAAE,EAAE;oBACtD,IAAI,CAAC,WAAW,CAAC,uBAAuB,CACpC,iBAAsG,CACzG,CAAC;iBACL;YACL,CAAC,CAAC;YAEF,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,4BAAqB,EAAE,QAAQ,CAAC,CAAC;YACrD,iBAAiB,CAAC,EAAE,uCAA2C,GAAG,EAAE;gBAChE,OAAO,IAAI,CAAC,WAAY,CAAC,cAAc,CAAC,4BAAqB,EAAE,QAAQ,CAAC,CAAC;YAC7E,CAAC,CAAC,CAAC;SACN;IACL,CAAC;IAED;;;OAGG;IACM,KAAK,CAAC,QAAQ;QACnB,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC3B,CAAC;;AA1VL,wCA2VC;AAhUG;;;;WAAyC;QACrC,GAAG,oBAAY,CAAC,YAAY;QAC5B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,QAAQ;QAExC,qBAAqB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;QACxD,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QACrC,mBAAmB,EAAE,YAAE,CAAC,QAAQ,CAAC,KAAK;QAEtC,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACjC,kBAAkB,EAAE,YAAE,CAAC,MAAM;QAC7B,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACtC,wBAAwB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC7C,cAAc,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACnC,kBAAkB,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAU,CAAC,kBAAkB,CAAC;KACjF;GAAC;AA6TN,gBAAgB;AACT,KAAK,UAAU,0BAA0B,CAAC,EAC7C,OAAO,EACP,IAAI,EACJ,YAAY,EACZ,kBAAkB,EAClB,eAAe,GACW;IAC1B,MAAM,OAAO,GAAG,IAAA,qBAAc,EAAC;QAC3B,eAAe,EAAE,OAAO,EAAE,QAAQ;QAClC,eAAe;QACf,kBAAkB;QAClB,mBAAmB,EAAE,OAAO,EAAE,OAAO;KACxC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,IAAW,EAAE,OAAO,EAAE,QAAQ,IAAI,GAAG,EAAE,OAAO,CAAC,CAAC;IAEvF,OAAO,IAAA,mBAAY,EAAC;QAChB,YAAY;QACZ,IAAI;QACJ,OAAO;QACP,GAAG,OAAO;KACb,CAAC,CAAC;AACP,CAAC;AAtBD,gEAsBC;AAED;;;GAGG;AACH,wDAAwD;AACxD,KAAK,UAAU,mBAAmB,CAAC,IAA0B,EAAE,QAAgB,EAAE,OAAgB;IAC7F,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,OAA0B,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEpJ,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,IAAY,EAAE,EAAE;QAC7B,yHAAyH;QACzH,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,6CAA6C;QACtG,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,EAAE;YAC7B,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,+CAA+C;kBAC9E,+EAA+E,CAAC,CAAC;SAC9F;QACD,OAAO,OAAO;YACV,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI;YAC/B,CAAC,CAAC,IAAI,CAAC;IACf,CAAC,CAAC,CAAC;AACP,CAAC"}

package/{dist/internals → internals}/browser-launcher.d.ts

@@ -1,6 +1,13 @@
 import { Dictionary, Constructor } from '@crawlee/utils';
 import { BrowserPlugin, BrowserPluginOptions } from '@crawlee/browser-pool';
 export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginOptions<TOptions> {
+    /**
+     * URL to a HTTP proxy server. It must define the port number,
+     * and it may also contain proxy username and password.
+     *
+     * Example: `http://bob:pass123@proxy.example.com:1234`.
+     */
+    proxyUrl?: string;
     /**
      * If `true` and `executablePath` is not set,
      * the launcher will launch full Google Chrome browser available on the machine
@@ -11,6 +18,24 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginO
      * @default false
      */
     useChrome?: boolean;
+    /**
+    * With this option selected, all pages will be opened in a new incognito browser context.
+    * This means they will not share cookies nor cache and their resources will not be throttled by one another.
+    * @default false
+    */
+    useIncognitoPages?: boolean;
+    /**
+    * Sets the [User Data Directory](https://chromium.googlesource.com/chromium/src/+/master/docs/user_data_dir.md) path.
+    * The user data directory contains profile data such as history, bookmarks, and cookies, as well as other per-installation local state.
+    * If not specified, a temporary directory is used instead.
+    */
+    userDataDir?: string;
+    /**
+     * The `User-Agent` HTTP header used by the browser.
+     * If not provided, the function sets `User-Agent` to a reasonable default
+     * to reduce the chance of detection of the crawler.
+     */
+    userAgent?: string;
     launcher?: Launcher;
 }
 /**
@@ -31,6 +56,7 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
         useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
         userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
         launchOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
+        userAgent: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
     };
     static requireLauncherOrThrow<T>(launcher: string, apifyImageName: string): T;
     /**
@@ -46,9 +72,7 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
      * @returns Browser instance.
      */
     launch(): LaunchResult;
-    createLaunchOptions(): {
-        args?: string[];
-    } & Dictionary;
+    createLaunchOptions(): Dictionary;
     protected _getDefaultHeadlessOption(): boolean;
     protected _getChromeExecutablePath(): string;
     /**

package/internals/browser-launcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-launcher.d.ts","sourceRoot":"","sources":["../../src/internals/browser-launcher.ts"],"names":[],"mappings":"AAIA,OAAO,EACH,UAAU,EACV,WAAW,EACd,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,aAAa,EACb,oBAAoB,EACvB,MAAM,uBAAuB,CAAC;AAO/B,MAAM,WAAW,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAE,SAAQ,oBAAoB,CAAC,QAAQ,CAAC;IAC5F;;;;;OAKG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;;;;;;OAQG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;MAIE;IACF,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B;;;;MAIE;IACF,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACvB;AAED;;;GAGG;AACH,8BAAsB,eAAe,CACjC,MAAM,SAAS,aAAa,EAC5B,QAAQ,GAAG,MAAM,CAAC,SAAS,CAAC,EAC5B,CAAC,SAAS,WAAW,CAAC,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,EACnD,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACxD,YAAY,SAAS,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAEhF,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,aAAa,EAAE,UAAU,CAAC;IAC1B,uBAAuB,EAAE,UAAU,CAAC;IAEpC,MAAM,EAAG,CAAC,CAAC;IACX,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,SAAS,CAAC,MAAM,CAAC,YAAY;;;;;;;MAO3B;IAEF,MAAM,CAAC,sBAAsB,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,CAAC;IAkB7E;;OAEG;gBACS,aAAa,EAAE,oBAAoB,CAAC,aAAa,EAAE,QAAQ,CAAC;IAqBxE;;OAEG;IACH,mBAAmB,IAAI,MAAM;IAQ7B;;;OAGG;IACH,MAAM,IAAI,YAAY;IAOtB,mBAAmB,IAAI,UAAU;IA2BjC,SAAS,CAAC,yBAAyB,IAAI,OAAO;IAI9C,SAAS,CAAC,wBAAwB,IAAI,MAAM;IAI5C;;OAEG;IACH,SAAS,CAAC,+BAA+B,IAAI,MAAM;IA8BnD,SAAS,CAAC,yBAAyB,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI;CAe/D"}

package/{dist/internals → internals}/browser-launcher.js

@@ -6,6 +6,10 @@ const os_1 = tslib_1.__importDefault(require("os"));
 const fs_1 = tslib_1.__importDefault(require("fs"));
 const ow_1 = tslib_1.__importDefault(require("ow"));
 const consts_1 = require("@apify/consts");
+const DEFAULT_VIEWPORT = {
+    width: 1366,
+    height: 768,
+};
 /**
  * Abstract class for creating browser launchers, such as `PlaywrightLauncher` and `PuppeteerLauncher`.
  * @ignore
@@ -58,12 +62,13 @@ class BrowserLauncher {
             writable: true,
             value: void 0
         });
-        const { launcher, proxyUrl, useChrome, launchOptions = {}, ...otherLaunchContextProps } = launchContext;
+        const { launcher, proxyUrl, useChrome, userAgent, launchOptions = {}, ...otherLaunchContextProps } = launchContext;
         this._validateProxyUrlProtocol(proxyUrl);
         // those need to be reassigned otherwise they are {} in types
         this.launcher = launcher;
         this.proxyUrl = proxyUrl;
         this.useChrome = useChrome;
+        this.userAgent = userAgent;
         this.launchOptions = launchOptions;
         this.otherLaunchContextProps = otherLaunchContextProps;
     }
@@ -76,16 +81,10 @@ class BrowserLauncher {
             if (e.code === 'MODULE_NOT_FOUND') {
                 const msg = `Cannot find module '${launcher}'. Did you you install the '${launcher}' package?\n`
                     + `Make sure you have '${launcher}' in your package.json dependencies and in your package-lock.json, if you use it.`;
-                try {
-                    // eslint-disable-next-line global-require, @typescript-eslint/no-var-requires
-                    const apify = require('apify');
-                    // TODO: this might be doable better to know if its used on apify or not...
-                    if (apify.isAtHome()) {
-                        e.message = `${msg}\nOn the Apify platform, '${launcher}' can only be used with the ${apifyImageName} Docker image.`;
-                    }
-                    // eslint-disable-next-line no-empty
+                // TODO we should not depend on apify env vars here ideally
+                if (process.env.APIFY_IS_AT_HOME) {
+                    e.message = `${msg}\nOn the Apify platform, '${launcher}' can only be used with the ${apifyImageName} Docker image.`;
                 }
-                catch { }
             }
             throw err;
         }
@@ -111,8 +110,17 @@ class BrowserLauncher {
     }
     createLaunchOptions() {
         const launchOptions = {
+            args: [],
+            defaultViewport: DEFAULT_VIEWPORT,
             ...this.launchOptions,
         };
+        // TODO is this needed? should be controlled via public options preferably
+        if (process.env.APIFY_IS_AT_HOME) {
+            launchOptions.args.push('--no-sandbox');
+        }
+        if (this.userAgent) {
+            launchOptions.args.push(`--user-agent=${this.userAgent}`);
+        }
         if (launchOptions.headless == null) {
             launchOptions.headless = this._getDefaultHeadlessOption();
         }
@@ -184,6 +192,7 @@ Object.defineProperty(BrowserLauncher, "optionsShape", {
         useIncognitoPages: ow_1.default.optional.boolean,
         userDataDir: ow_1.default.optional.string,
         launchOptions: ow_1.default.optional.object,
+        userAgent: ow_1.default.optional.string,
     }
 });
 //# sourceMappingURL=browser-launcher.js.map

package/internals/browser-launcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-launcher.js","sourceRoot":"","sources":["../../src/internals/browser-launcher.ts"],"names":[],"mappings":";;;;AAAA,oDAAoB;AACpB,oDAAoB;AACpB,oDAAoB;AACpB,0CAAyC;AAUzC,MAAM,gBAAgB,GAAG;IACrB,KAAK,EAAE,IAAI;IACX,MAAM,EAAE,GAAG;CACd,CAAC;AA8CF;;;GAGG;AACH,MAAsB,eAAe;IA2CjC;;OAEG;IACH,YAAY,aAA4D;QAvCxE;;;;;WAAmB;QACnB;;;;;WAAkB;QAClB;;;;;WAAoB;QACpB;;;;;WAA0B;QAC1B;;;;;WAAoC;QACpC,mCAAmC;QACnC;;;;;WAAW;QACX;;;;;WAAmB;QAiCf,MAAM,EACF,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,SAAS,EACT,aAAa,GAAG,EAAE,EAClB,GAAG,uBAAuB,EAC7B,GAAG,aAAa,CAAC;QAElB,IAAI,CAAC,yBAAyB,CAAC,QAAQ,CAAC,CAAC;QAEzC,6DAA6D;QAC7D,IAAI,CAAC,QAAQ,GAAG,QAAS,CAAC;QAC1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,uBAAuB,GAAG,uBAAqC,CAAC;IACzE,CAAC;IAxCD,MAAM,CAAC,sBAAsB,CAAI,QAAgB,EAAE,cAAsB;QACrE,IAAI;YACA,OAAO,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,sBAAsB;SACnD;QAAC,OAAO,GAAG,EAAE;YACV,MAAM,CAAC,GAAG,GAA+B,CAAC;YAC1C,IAAI,CAAC,CAAC,IAAI,KAAK,kBAAkB,EAAE;gBAC/B,MAAM,GAAG,GAAG,uBAAuB,QAAQ,+BAA+B,QAAQ,cAAc;sBAC1F,uBAAuB,QAAQ,mFAAmF,CAAC;gBACzH,2DAA2D;gBAC3D,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE;oBAC9B,CAAC,CAAC,OAAO,GAAG,GAAG,GAAG,6BAA6B,QAAQ,+BAA+B,cAAc,gBAAgB,CAAC;iBACxH;aACJ;YAED,MAAM,GAAG,CAAC;SACb;IACL,CAAC;IA0BD;;OAEG;IACH,mBAAmB;QACf,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE;YAClC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,aAAa,EAAE,IAAI,CAAC,mBAAmB,EAAE;YACzC,GAAG,IAAI,CAAC,uBAAuB;SAClC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,MAAM;QACF,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC1C,MAAM,OAAO,GAAG,MAAM,CAAC,mBAAmB,EAAE,CAAC;QAE7C,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAiB,CAAC;IAClD,CAAC;IAED,mBAAmB;QACf,MAAM,aAAa,GAAoC;YACnD,IAAI,EAAE,EAAE;YACR,eAAe,EAAE,gBAAgB;YACjC,GAAG,IAAI,CAAC,aAAa;SACxB,CAAC;QAEF,0EAA0E;QAC1E,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE;YAC9B,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;SAC3C;QAED,IAAI,IAAI,CAAC,SAAS,EAAE;YAChB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;SAC7D;QAED,IAAI,aAAa,CAAC,QAAQ,IAAI,IAAI,EAAE;YAChC,aAAa,CAAC,QAAQ,GAAG,IAAI,CAAC,yBAAyB,EAAE,CAAC;SAC7D;QAED,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,aAAa,CAAC,cAAc,EAAE;YACjD,aAAa,CAAC,cAAc,GAAG,IAAI,CAAC,wBAAwB,EAAE,CAAC;SAClE;QAED,OAAO,aAAa,CAAC;IACzB,CAAC;IAES,yBAAyB;QAC/B,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAQ,CAAC,QAAQ,CAAC,KAAK,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAQ,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC;IACxF,CAAC;IAES,wBAAwB;QAC9B,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAQ,CAAC,sBAAsB,CAAC,IAAI,IAAI,CAAC,+BAA+B,EAAE,CAAC;IAClG,CAAC;IAED;;OAEG;IACO,+BAA+B;QACrC;;;;;;WAMG;QACH,MAAM,YAAY,GAAG,GAAG,EAAE;YACtB,IAAI,oBAAoB,GAAG,4DAA4D,CAAC;YACxF,MAAM,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,2CAA2C,CAAC;YACtF,MAAM,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,2CAA2C,CAAC;YAE9F,IAAI,YAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE;gBACvB,oBAAoB,GAAG,MAAM,CAAC;aACjC;iBAAM,IAAI,YAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE;gBAC9B,oBAAoB,GAAG,MAAM,CAAC;aACjC;YACD,OAAO,oBAAoB,CAAC;QAChC,CAAC,CAAC;QACF,QAAQ,YAAE,CAAC,QAAQ,EAAE,EAAE;YACnB,KAAK,QAAQ;gBACT,OAAO,8DAA8D,CAAC;YAC1E,KAAK,OAAO;gBACR,OAAO,YAAY,EAAE,CAAC;YAC1B;gBACI,OAAO,wBAAwB,CAAC;SACvC;IACL,CAAC;IAES,yBAAyB,CAAC,QAAiB;QACjD,IAAI,CAAC,QAAQ;YAAE,OAAO;QAEtB,IAAI,CAAC,8BAA8B,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;YAChD,MAAM,IAAI,KAAK,CAAC,6CAA6C,QAAQ,GAAG,CAAC,CAAC;SAC7E;QAED,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE9B,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,EAAE;YAC9B,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE;gBACvD,MAAM,IAAI,KAAK,CAAC,kFAAkF,CAAC,CAAC;aACvG;SACJ;IACL,CAAC;;AA3KL,0CA4KC;AA5JG;;;;WAAgC;QAC5B,QAAQ,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG;QAChC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QAC9B,iBAAiB,EAAE,YAAE,CAAC,QAAQ,CAAC,OAAO;QACtC,WAAW,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QAC/B,aAAa,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;QACjC,SAAS,EAAE,YAAE,CAAC,QAAQ,CAAC,MAAM;KAChC;GAAC"}

package/package.json

@@ -1,15 +1,15 @@
 {
     "name": "@crawlee/browser",
-    "version": "3.0.0-alpha.3",
+    "version": "3.0.0-alpha.30",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=16.0.0"
     },
-    "types": "dist/index.d.ts",
+    "types": "index.d.ts",
     "exports": {
         ".": {
-            "import": "./dist/index.mjs",
-            "require": "./dist/index.js"
+            "import": "./index.mjs",
+            "require": "./index.js"
         }
     },
     "keywords": [
@@ -39,20 +39,17 @@
         "url": "https://github.com/apify/apify-js/issues"
     },
     "homepage": "https://sdk.apify.com/",
-    "files": [
-        "dist"
-    ],
     "scripts": {
         "build": "npm run clean && npm run compile && npm run copy",
         "clean": "rimraf ./dist",
-        "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs",
+        "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./index.js ./index.mjs",
         "copy": "ts-node -T ../../scripts/copy.ts"
     },
     "publishConfig": {
         "access": "public"
     },
     "dependencies": {
-        "@crawlee/basic": "^3.0.0-alpha.3"
-    },
-    "gitHead": "548b65585f29cd3c662acddbb30abfb79d0579e6"
+        "@crawlee/basic": "^3.0.0-alpha.30",
+        "tough-cookie": "^4.0.0"
+    }
 }