@crawlee/browser-pool 4.0.0-beta.99 → 4.0.0-rc.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -36,7 +36,7 @@ export interface BrowserSpecification {
36
36
  */
37
37
  httpVersion?: HttpVersion;
38
38
  }
39
- export declare const enum OperatingSystemsName {
39
+ export declare enum OperatingSystemsName {
40
40
  linux = "linux",
41
41
  macos = "macos",
42
42
  windows = "windows",
@@ -49,7 +49,7 @@ export declare const enum OperatingSystemsName {
49
49
  */
50
50
  ios = "ios"
51
51
  }
52
- export declare const enum DeviceCategory {
52
+ export declare enum DeviceCategory {
53
53
  /**
54
54
  * Describes mobile devices (mobile phones, tablets...). These devices usually have smaller, vertical screens and load lighter versions of websites.
55
55
  * > Note: Generating `android` and `ios` devices will not work without setting the device to `mobile` first.
@@ -1,10 +1,10 @@
1
1
  import { PlaywrightPlugin } from '../playwright/playwright-plugin.js';
2
2
  import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js';
3
- import { BrowserName } from './types.js';
3
+ import { BrowserName, DeviceCategory, OperatingSystemsName } from './types.js';
4
4
  export const getGeneratorDefaultOptions = (launchContext) => {
5
5
  const { browserPlugin, launchOptions } = launchContext;
6
6
  const options = {
7
- devices: ["desktop" /* DeviceCategory.desktop */],
7
+ devices: [DeviceCategory.desktop],
8
8
  locales: ['en-US'],
9
9
  browsers: [getBrowserName(browserPlugin, launchOptions)],
10
10
  operatingSystems: [getOperatingSystem()],
@@ -34,11 +34,11 @@ const getOperatingSystem = () => {
34
34
  switch (platform) {
35
35
  case 'win32':
36
36
  // platform is win32 even for 64-bit
37
- return "windows" /* OperatingSystemsName.windows */;
37
+ return OperatingSystemsName.windows;
38
38
  case 'darwin':
39
- return "macos" /* OperatingSystemsName.macos */;
39
+ return OperatingSystemsName.macos;
40
40
  default:
41
41
  // consider everything else a linux
42
- return "linux" /* OperatingSystemsName.linux */;
42
+ return OperatingSystemsName.linux;
43
43
  }
44
44
  };
@@ -56,6 +56,7 @@ export interface LaunchContextOptions<Library extends CommonLibrary = CommonLibr
56
56
  isRemote?: boolean;
57
57
  }
58
58
  export declare class LaunchContext<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
59
+ #private;
59
60
  id?: string;
60
61
  browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
61
62
  launchOptions: LibraryOptions;
@@ -64,15 +65,13 @@ export declare class LaunchContext<Library extends CommonLibrary = CommonLibrary
64
65
  userDataDir: string;
65
66
  readonly isRemote: boolean;
66
67
  ignoreProxyCertificate?: boolean;
67
- private _proxyUrl?;
68
- private readonly _reservedFieldNames;
69
68
  fingerprint?: BrowserFingerprintWithHeaders;
70
69
  /**
71
70
  * Token identifying the remote browser session this context connected to, set by the plugin and read by
72
71
  * the {@link RemoteBrowserPool} to release the session on close. Only present for remote connections.
73
72
  * @internal
74
73
  */
75
- _remoteToken?: number;
74
+ remoteToken?: number;
76
75
  [K: PropertyKey]: unknown;
77
76
  constructor(options: LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
78
77
  /**
package/launch-context.js CHANGED
@@ -7,15 +7,15 @@ export class LaunchContext {
7
7
  userDataDir;
8
8
  isRemote;
9
9
  ignoreProxyCertificate;
10
- _proxyUrl;
11
- _reservedFieldNames = [...Reflect.ownKeys(this), 'extend'];
10
+ #proxyUrl;
11
+ #reservedFieldNames;
12
12
  fingerprint;
13
13
  /**
14
14
  * Token identifying the remote browser session this context connected to, set by the plugin and read by
15
15
  * the {@link RemoteBrowserPool} to release the session on close. Only present for remote connections.
16
16
  * @internal
17
17
  */
18
- _remoteToken;
18
+ remoteToken;
19
19
  constructor(options) {
20
20
  const { id, browserPlugin, launchOptions, proxyUrl, useIncognitoPages, browserPerProxy, userDataDir = '', ignoreProxyCertificate, isRemote, } = options;
21
21
  this.id = id;
@@ -26,7 +26,10 @@ export class LaunchContext {
26
26
  this.userDataDir = userDataDir;
27
27
  this.ignoreProxyCertificate = ignoreProxyCertificate ?? false;
28
28
  this.isRemote = isRemote ?? false;
29
- this._proxyUrl = proxyUrl;
29
+ this.#proxyUrl = proxyUrl;
30
+ // Computed here (not in a field initializer) so that all fields already exist; the accessors live on
31
+ // the prototype, so they are never own keys and have to be listed explicitly.
32
+ this.#reservedFieldNames = [...Reflect.ownKeys(this), 'proxyUrl', 'remoteToken', 'extend'];
30
33
  }
31
34
  /**
32
35
  * Extend the launch context with any extra fields.
@@ -37,7 +40,7 @@ export class LaunchContext {
37
40
  */
38
41
  extend(fields) {
39
42
  Object.entries(fields).forEach(([key, value]) => {
40
- if (this._reservedFieldNames.includes(key)) {
43
+ if (this.#reservedFieldNames.includes(key)) {
41
44
  throw new Error(`Cannot extend LaunchContext with key: ${key}, because it's reserved.`);
42
45
  }
43
46
  else {
@@ -51,7 +54,7 @@ export class LaunchContext {
51
54
  */
52
55
  set proxyUrl(url) {
53
56
  if (!url) {
54
- this._proxyUrl = undefined;
57
+ this.#proxyUrl = undefined;
55
58
  return;
56
59
  }
57
60
  const urlInstance = new URL(url);
@@ -59,12 +62,12 @@ export class LaunchContext {
59
62
  urlInstance.search = '';
60
63
  urlInstance.hash = '';
61
64
  // https://www.chromium.org/developers/design-documents/network-settings/#command-line-options-for-proxy-settings
62
- this._proxyUrl = urlInstance.href.slice(0, -1);
65
+ this.#proxyUrl = urlInstance.href.slice(0, -1);
63
66
  }
64
67
  /**
65
68
  * Returns the proxy URL of the browser.
66
69
  */
67
70
  get proxyUrl() {
68
- return this._proxyUrl;
71
+ return this.#proxyUrl;
69
72
  }
70
73
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crawlee/browser-pool",
3
- "version": "4.0.0-beta.99",
3
+ "version": "4.0.0-rc.0",
4
4
  "description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
5
5
  "engines": {
6
6
  "node": ">=22.0.0"
@@ -31,18 +31,18 @@
31
31
  },
32
32
  "dependencies": {
33
33
  "@apify/timeout": "^0.4.4",
34
- "@crawlee/core": "4.0.0-beta.99",
35
- "@crawlee/types": "4.0.0-beta.99",
34
+ "@crawlee/core": "4.0.0-rc.0",
35
+ "@crawlee/types": "4.0.0-rc.0",
36
36
  "fingerprint-generator": "^2.1.68",
37
37
  "fingerprint-injector": "^2.1.68",
38
38
  "lodash.merge": "^4.6.2",
39
39
  "nanoid": "^5.1.5",
40
- "ow": "^2.0.0",
41
40
  "p-limit": "^6.2.0",
42
41
  "proxy-chain": "^2.5.8",
43
42
  "quick-lru": "^7.0.1",
44
43
  "tiny-typed-emitter": "^2.1.0",
45
- "tslib": "^2.8.1"
44
+ "tslib": "^2.8.1",
45
+ "zod": "^4.4.3"
46
46
  },
47
47
  "peerDependencies": {
48
48
  "playwright": "*",
@@ -63,5 +63,5 @@
63
63
  }
64
64
  }
65
65
  },
66
- "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
66
+ "gitHead": "79ab33dacdacb83e0197e6516d145f3aceef80c7"
67
67
  }
@@ -9,10 +9,7 @@ export interface BrowserOptions {
9
9
  * Browser wrapper created to have consistent API with persistent and non-persistent contexts.
10
10
  */
11
11
  export declare class PlaywrightBrowser extends EventEmitter {
12
- private _browserContext;
13
- private _version;
14
- private _isConnected;
15
- private _browserType?;
12
+ #private;
16
13
  constructor(options: BrowserOptions);
17
14
  [Symbol.asyncDispose](): Promise<void>;
18
15
  close(): Promise<void>;
@@ -20,7 +17,7 @@ export declare class PlaywrightBrowser extends EventEmitter {
20
17
  isConnected(): boolean;
21
18
  version(): string;
22
19
  /** @internal */
23
- _setBrowserType(browserType: BrowserType): void;
20
+ setBrowserType(browserType: BrowserType): void;
24
21
  browserType(): BrowserType;
25
22
  newPage(...args: Parameters<BrowserContext['newPage']>): ReturnType<BrowserContext['newPage']>;
26
23
  newContext(): Promise<never>;
@@ -3,17 +3,17 @@ import { EventEmitter } from 'node:events';
3
3
  * Browser wrapper created to have consistent API with persistent and non-persistent contexts.
4
4
  */
5
5
  export class PlaywrightBrowser extends EventEmitter {
6
- _browserContext;
7
- _version;
8
- _isConnected = true;
9
- _browserType;
6
+ #browserContext;
7
+ #version;
8
+ #isConnected = true;
9
+ #browserType;
10
10
  constructor(options) {
11
11
  super();
12
12
  const { browserContext, version } = options;
13
- this._browserContext = browserContext;
14
- this._version = version;
15
- this._browserContext.once('close', () => {
16
- this._isConnected = false;
13
+ this.#browserContext = browserContext;
14
+ this.#version = version;
15
+ this.#browserContext.once('close', () => {
16
+ this.#isConnected = false;
17
17
  this.emit('disconnected');
18
18
  });
19
19
  }
@@ -21,26 +21,26 @@ export class PlaywrightBrowser extends EventEmitter {
21
21
  await this.close();
22
22
  }
23
23
  async close() {
24
- await this._browserContext.close();
24
+ await this.#browserContext.close();
25
25
  }
26
26
  contexts() {
27
- return [this._browserContext];
27
+ return [this.#browserContext];
28
28
  }
29
29
  isConnected() {
30
- return this._isConnected;
30
+ return this.#isConnected;
31
31
  }
32
32
  version() {
33
- return this._version;
33
+ return this.#version;
34
34
  }
35
35
  /** @internal */
36
- _setBrowserType(browserType) {
37
- this._browserType = browserType;
36
+ setBrowserType(browserType) {
37
+ this.#browserType = browserType;
38
38
  }
39
39
  browserType() {
40
- return this._browserType;
40
+ return this.#browserType;
41
41
  }
42
42
  async newPage(...args) {
43
- return this._browserContext.newPage(...args);
43
+ return this.#browserContext.newPage(...args);
44
44
  }
45
45
  async newContext() {
46
46
  throw new Error('Function `newContext()` is not available in incognito mode');
@@ -6,15 +6,15 @@ import type { RemoteConnection, RemoteConnectionParameters } from '../remote-bro
6
6
  import type { SafeParameters } from '../utils.js';
7
7
  import { PlaywrightController } from './playwright-controller.js';
8
8
  export declare class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<BrowserType['launch']>[0], PlaywrightBrowser> {
9
- private _browserVersion?;
9
+ #private;
10
10
  /**
11
11
  * Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
12
12
  * accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
13
13
  */
14
14
  useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
15
15
  protected _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser>;
16
- private _throwOnFailedLaunch;
16
+ private throwOnFailedLaunch;
17
17
  createController(): PlaywrightController;
18
- protected _addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;
19
- protected _isChromiumBasedBrowser(): boolean;
18
+ protected addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;
19
+ protected isChromiumBasedBrowser(): boolean;
20
20
  }
@@ -5,7 +5,7 @@ import { getLocalProxyAddress } from '../proxy-server.js';
5
5
  import { PlaywrightBrowser as PlaywrightBrowserWithPersistentContext } from './playwright-browser.js';
6
6
  import { PlaywrightController } from './playwright-controller.js';
7
7
  export class PlaywrightPlugin extends BrowserPlugin {
8
- _browserVersion;
8
+ #browserVersion;
9
9
  /**
10
10
  * Playwright remote connections only support incognito pages — `connect()` / `connectOverCDP()` don't
11
11
  * accept persistent contexts. Force it on (and inform the user) when wired for a remote connection.
@@ -20,7 +20,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
20
20
  }
21
21
  async _launch(launchContext) {
22
22
  if (this.remoteConnection) {
23
- return this._connectToRemoteBrowser(launchContext, async (url) => {
23
+ return this.connectToRemoteBrowser(launchContext, async (url) => {
24
24
  const connectOptions = (this.remoteConnectionParameters?.connectOptions ?? {});
25
25
  if (this.remoteConnectionParameters?.protocol === 'playwright') {
26
26
  this.log.info('Connecting to remote browser via connect (Playwright WebSocket).');
@@ -53,7 +53,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
53
53
  try {
54
54
  if (useIncognitoPages) {
55
55
  browser = await this.library.launch(launchOptions).catch((error) => {
56
- return this._throwOnFailedLaunch(launchContext, error);
56
+ return this.throwOnFailedLaunch(launchContext, error);
57
57
  });
58
58
  if (anonymizedProxyUrl) {
59
59
  browser.on('disconnected', async () => {
@@ -65,7 +65,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
65
65
  const browserContext = await this.library
66
66
  .launchPersistentContext(userDataDir, launchOptions)
67
67
  .catch((error) => {
68
- return this._throwOnFailedLaunch(launchContext, error);
68
+ return this.throwOnFailedLaunch(launchContext, error);
69
69
  });
70
70
  browserContext.once('close', () => {
71
71
  if (userDataDir.includes('apify-playwright-firefox-taac-')) {
@@ -80,18 +80,20 @@ export class PlaywrightPlugin extends BrowserPlugin {
80
80
  await close();
81
81
  });
82
82
  }
83
- if (!this._browserVersion) {
83
+ if (!this.#browserVersion) {
84
84
  // Launches unused browser just to get the browser version.
85
85
  const inactiveBrowser = await this.library.launch(launchOptions);
86
- this._browserVersion = inactiveBrowser.version();
86
+ this.#browserVersion = inactiveBrowser.version();
87
87
  inactiveBrowser.close().catch((error) => {
88
88
  this.log.exception(error, 'Failed to close browser.');
89
89
  });
90
90
  }
91
- browser = new PlaywrightBrowserWithPersistentContext({
91
+ const persistentBrowser = new PlaywrightBrowserWithPersistentContext({
92
92
  browserContext,
93
- version: this._browserVersion,
93
+ version: this.#browserVersion,
94
94
  });
95
+ persistentBrowser.setBrowserType(this.library);
96
+ browser = persistentBrowser;
95
97
  }
96
98
  }
97
99
  catch (error) {
@@ -100,13 +102,13 @@ export class PlaywrightPlugin extends BrowserPlugin {
100
102
  }
101
103
  return browser;
102
104
  }
103
- _throwOnFailedLaunch(launchContext, cause) {
104
- this._throwAugmentedLaunchError(cause, launchContext.launchOptions?.executablePath, '`apify/actor-node-playwright-*` (with a correct browser name)', 'Try installing the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).');
105
+ throwOnFailedLaunch(launchContext, cause) {
106
+ this.throwAugmentedLaunchError(cause, launchContext.launchOptions?.executablePath, '`apify/actor-node-playwright-*` (with a correct browser name)', 'Try installing the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).');
105
107
  }
106
108
  createController() {
107
109
  return new PlaywrightController(this);
108
110
  }
109
- async _addProxyToLaunchOptions(launchContext) {
111
+ async addProxyToLaunchOptions(launchContext) {
110
112
  launchContext.launchOptions ??= {};
111
113
  const { launchOptions, proxyUrl } = launchContext;
112
114
  if (proxyUrl) {
@@ -118,7 +120,7 @@ export class PlaywrightPlugin extends BrowserPlugin {
118
120
  };
119
121
  }
120
122
  }
121
- _isChromiumBasedBrowser() {
123
+ isChromiumBasedBrowser() {
122
124
  const name = this.library.name();
123
125
  return name === 'chromium';
124
126
  }
@@ -12,6 +12,6 @@ export declare class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer, Pup
12
12
  useRemoteConnection(connection: RemoteConnection, parameters?: RemoteConnectionParameters): void;
13
13
  protected _launch(launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<PuppeteerTypes.Browser>;
14
14
  createController(): PuppeteerController;
15
- protected _addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;
16
- protected _isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;
15
+ protected addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;
16
+ protected isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;
17
17
  }
@@ -27,7 +27,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
27
27
  const { useIncognitoPages, proxyUrl, ignoreProxyCertificate } = launchContext;
28
28
  let browser;
29
29
  if (this.remoteConnection) {
30
- browser = await this._connectToRemoteBrowser(launchContext, async (url) => {
30
+ browser = await this.connectToRemoteBrowser(launchContext, async (url) => {
31
31
  const connectOptions = this.remoteConnectionParameters?.connectOptions ?? {};
32
32
  this.log.info('Connecting to remote browser via connect (CDP).');
33
33
  return this.library.connect({ ...connectOptions, browserWSEndpoint: url });
@@ -73,7 +73,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
73
73
  }
74
74
  catch (error) {
75
75
  await close();
76
- this._throwAugmentedLaunchError(error, launchContext.launchOptions?.executablePath, '`apify/actor-node-puppeteer-chrome`', "Try installing a browser, if it's missing, by running `npx @puppeteer/browsers install chromium --path [path]` and pointing `executablePath` to the downloaded executable (https://pptr.dev/browsers-api)");
76
+ this.throwAugmentedLaunchError(error, launchContext.launchOptions?.executablePath, '`apify/actor-node-puppeteer-chrome`', "Try installing a browser, if it's missing, by running `npx @puppeteer/browsers install chromium --path [path]` and pointing `executablePath` to the downloaded executable (https://pptr.dev/browsers-api)");
77
77
  }
78
78
  }
79
79
  }
@@ -177,7 +177,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
177
177
  createController() {
178
178
  return new PuppeteerController(this);
179
179
  }
180
- async _addProxyToLaunchOptions(_launchContext) {
180
+ async addProxyToLaunchOptions(_launchContext) {
181
181
  /*
182
182
  // DO NOT USE YET! DOING SO DISABLES CACHE WHICH IS 50% PERFORMANCE HIT!
183
183
  launchContext.launchOptions ??= {};
@@ -204,7 +204,7 @@ export class PuppeteerPlugin extends BrowserPlugin {
204
204
  }
205
205
  */
206
206
  }
207
- _isChromiumBasedBrowser(_launchContext) {
207
+ isChromiumBasedBrowser(_launchContext) {
208
208
  return true;
209
209
  }
210
210
  }
@@ -95,10 +95,10 @@ export interface RemoteBrowserPoolOptions {
95
95
  slotPollIntervalMillis?: number;
96
96
  }
97
97
  /**
98
- * The remote-connection configuration a browser crawler accepts on its `remoteBrowser` option. It is the
99
- * {@link RemoteBrowserPoolOptions} a user supplies *minus* the parts the crawler provides itself the
100
- * `browserPlugins` (the crawler builds the correct one for its browser) and `browserPoolOptions` (taken from
101
- * the crawler's own `browserPoolOptions`). This is what makes the crawler path both terse and mismatch-proof.
98
+ * The remote-connection configuration a browser crawler accepts on its `remoteBrowser` option: the
99
+ * {@link RemoteBrowserPoolOptions} minus the `browserPlugins` (the crawler builds the correct one for its
100
+ * browser, which is what makes this path mismatch-proof) and minus `browserPoolOptions` tuning the wrapping
101
+ * pool means building the pool yourself, through the `remote*BrowserPool()` factory for your crawler.
102
102
  */
103
103
  export type CrawlerRemoteBrowserOptions = Omit<RemoteBrowserPoolOptions, 'browserPlugins' | 'browserPoolOptions'>;
104
104
  /**
@@ -134,15 +134,9 @@ export type CrawlerRemoteBrowserOptions = Omit<RemoteBrowserPoolOptions, 'browse
134
134
  * @category Browser management
135
135
  */
136
136
  export declare class RemoteBrowserPool<Page = unknown> implements IBrowserPool<Page> {
137
+ #private;
137
138
  /** The wrapped pool that performs the remote connections and serves pages. */
138
139
  readonly browserPool: BrowserPool;
139
- /** The wrapped pool viewed through the {@link IBrowserPool} contract (the bare type widens pages to `never`). */
140
- private readonly pool;
141
- private readonly registry;
142
- private readonly slotPollIntervalMillis;
143
- private readonly log;
144
- /** Shared by all `newPage` callers waiting for a free slot, so they don't each register their own listeners. */
145
- private _capacityChange?;
146
140
  constructor(options: RemoteBrowserPoolOptions);
147
141
  /** Maximum number of remote browsers that may be open at the same time. */
148
142
  get maxOpenBrowsers(): number;
@@ -160,11 +154,11 @@ export declare class RemoteBrowserPool<Page = unknown> implements IBrowserPool<P
160
154
  /** Closes all browsers, releases any still-open remote sessions, and tears down the wrapped pool. */
161
155
  destroy(): Promise<void>;
162
156
  /** Resolves once the wrapped pool can serve another page without exceeding `maxOpenBrowsers`. */
163
- private _waitForFreeSlot;
157
+ private waitForFreeSlot;
164
158
  /**
165
159
  * Resolves on the next browser-retired / page-closed event, or after `slotPollIntervalMillis`. All
166
160
  * concurrently-waiting `newPage` calls share a single promise (and a single pair of event listeners)
167
161
  * per tick, so a fleet of saturated callers doesn't fan out into N listener pairs on the pool.
168
162
  */
169
- private _nextCapacityChange;
163
+ private nextCapacityChange;
170
164
  }
@@ -1,5 +1,6 @@
1
1
  import { serviceLocator } from '@crawlee/core';
2
2
  import { BrowserPool } from './browser-pool.js';
3
+ import { BROWSER_CONTROLLER_EVENTS, BROWSER_POOL_EVENTS } from './events.js';
3
4
  import { RemoteBrowserProvider } from './remote-browser-provider.js';
4
5
  /**
5
6
  * Owns the lifecycle of remote browser sessions for a single {@link RemoteBrowserPool}: endpoint
@@ -7,18 +8,18 @@ import { RemoteBrowserProvider } from './remote-browser-provider.js';
7
8
  * {@link RemoteConnection} so it can be injected into a plugin.
8
9
  */
9
10
  class RemoteSessionRegistry {
10
- endpoint;
11
- onRelease;
12
- log;
13
- sessions = new Map();
14
- nextToken = 0;
11
+ #sessions = new Map();
12
+ #nextToken = 0;
13
+ #endpoint;
14
+ #onRelease;
15
+ #log;
15
16
  constructor(endpoint, onRelease, log) {
16
- this.endpoint = endpoint;
17
- this.onRelease = onRelease;
18
- this.log = log;
17
+ this.#endpoint = endpoint;
18
+ this.#onRelease = onRelease;
19
+ this.#log = log;
19
20
  }
20
21
  async resolve(options) {
21
- const resolved = typeof this.endpoint === 'function' ? await this.endpoint(options) : this.endpoint;
22
+ const resolved = typeof this.#endpoint === 'function' ? await this.#endpoint(options) : this.#endpoint;
22
23
  let result;
23
24
  if (typeof resolved === 'string') {
24
25
  if (!resolved)
@@ -31,30 +32,30 @@ class RemoteSessionRegistry {
31
32
  else {
32
33
  result = resolved;
33
34
  }
34
- const token = this.nextToken++;
35
- this.sessions.set(token, { url: result.url, context: result.context, released: false });
35
+ const token = this.#nextToken++;
36
+ this.#sessions.set(token, { url: result.url, context: result.context, released: false });
36
37
  return { url: result.url, token };
37
38
  }
38
39
  async release(token) {
39
- const session = this.sessions.get(token);
40
+ const session = this.#sessions.get(token);
40
41
  // Release at most once per session — guards a close()/teardown race (the `released` flag is set
41
42
  // synchronously before the awaited onRelease, so releaseAll() can't double-fire an in-flight release).
42
43
  if (!session || session.released)
43
44
  return;
44
45
  session.released = true;
45
46
  try {
46
- await this.onRelease?.({ endpoint: session.url, context: session.context });
47
+ await this.#onRelease?.({ endpoint: session.url, context: session.context });
47
48
  }
48
49
  catch (err) {
49
- this.log.warning('Remote browser release() failed.', { error: err?.message });
50
+ this.#log.warning('Remote browser release() failed.', { error: err?.message });
50
51
  }
51
52
  finally {
52
- this.sessions.delete(token);
53
+ this.#sessions.delete(token);
53
54
  }
54
55
  }
55
56
  /** Releases every session that is still open. Called on pool teardown so no remote session leaks. */
56
57
  async releaseAll() {
57
- await Promise.all([...this.sessions.keys()].map(async (token) => this.release(token)));
58
+ await Promise.all([...this.#sessions.keys()].map(async (token) => this.release(token)));
58
59
  }
59
60
  }
60
61
  /**
@@ -92,16 +93,16 @@ export class RemoteBrowserPool {
92
93
  /** The wrapped pool that performs the remote connections and serves pages. */
93
94
  browserPool;
94
95
  /** The wrapped pool viewed through the {@link IBrowserPool} contract (the bare type widens pages to `never`). */
95
- pool;
96
- registry;
97
- slotPollIntervalMillis;
98
- log;
96
+ #pool;
97
+ #registry;
98
+ #slotPollIntervalMillis;
99
+ #log;
99
100
  /** Shared by all `newPage` callers waiting for a free slot, so they don't each register their own listeners. */
100
- _capacityChange;
101
+ #capacityChange;
101
102
  constructor(options) {
102
103
  const { browserPlugins, endpoint, release, maxOpenBrowsers, connection = {}, browserPoolOptions = {}, slotPollIntervalMillis = 500, } = options;
103
- this.log = serviceLocator.getLogger().child({ prefix: 'RemoteBrowserPool' });
104
- this.slotPollIntervalMillis = slotPollIntervalMillis;
104
+ this.#log = serviceLocator.getLogger().child({ prefix: 'RemoteBrowserPool' });
105
+ this.#slotPollIntervalMillis = slotPollIntervalMillis;
105
106
  // A RemoteBrowserProvider carries its own endpoint, release, and maxOpenBrowsers.
106
107
  const provider = endpoint instanceof RemoteBrowserProvider ? endpoint : undefined;
107
108
  const resolvedEndpoint = provider
@@ -111,20 +112,20 @@ export class RemoteBrowserPool {
111
112
  ? ({ context }) => provider.release(context)
112
113
  : release;
113
114
  const resolvedMax = maxOpenBrowsers ?? provider?.maxOpenBrowsers;
114
- this.registry = new RemoteSessionRegistry(resolvedEndpoint, resolvedRelease, this.log);
115
+ this.#registry = new RemoteSessionRegistry(resolvedEndpoint, resolvedRelease, this.#log);
115
116
  // Wire every plugin for remote connection.
116
117
  for (const plugin of browserPlugins) {
117
- plugin.useRemoteConnection(this.registry, connection);
118
+ plugin.useRemoteConnection(this.#registry, connection);
118
119
  }
119
120
  this.browserPool = new BrowserPool({ ...browserPoolOptions, browserPlugins });
120
- this.pool = this.browserPool;
121
+ this.#pool = this.browserPool;
121
122
  // Release a browser's remote session once it closes. The registry dedupes (close() schedules a delayed
122
123
  // kill(), so BROWSER_CLOSED can fire twice), and destroy()'s releaseAll() backstops any that never close.
123
- this.browserPool.on("browserLaunched" /* BROWSER_POOL_EVENTS.BROWSER_LAUNCHED */, (controller) => {
124
- controller.once("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, () => {
125
- const token = controller.launchContext._remoteToken;
124
+ this.browserPool.on(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, (controller) => {
125
+ controller.once(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, () => {
126
+ const token = controller.launchContext.remoteToken;
126
127
  if (token !== undefined)
127
- void this.registry.release(token);
128
+ void this.#registry.release(token);
128
129
  });
129
130
  });
130
131
  if (resolvedMax !== undefined) {
@@ -143,28 +144,28 @@ export class RemoteBrowserPool {
143
144
  * allows it (either a new browser slot is free, or an active browser still has page capacity).
144
145
  */
145
146
  async newPage(options) {
146
- await this._waitForFreeSlot();
147
- return this.pool.newPage(options);
147
+ await this.waitForFreeSlot();
148
+ return this.#pool.newPage(options);
148
149
  }
149
150
  async closePage(page, options) {
150
- return this.pool.closePage(page, options);
151
+ return this.#pool.closePage(page, options);
151
152
  }
152
153
  async extractPageState(page) {
153
- return this.pool.extractPageState(page);
154
+ return this.#pool.extractPageState(page);
154
155
  }
155
156
  async injectPageState(page, state) {
156
- return this.pool.injectPageState(page, state);
157
+ return this.#pool.injectPageState(page, state);
157
158
  }
158
159
  /** Closes all browsers, releases any still-open remote sessions, and tears down the wrapped pool. */
159
160
  async destroy() {
160
161
  await this.browserPool.destroy();
161
162
  // Backstop: release any sessions whose browser never emitted a close (e.g. dropped on teardown).
162
- await this.registry.releaseAll();
163
+ await this.#registry.releaseAll();
163
164
  }
164
165
  /** Resolves once the wrapped pool can serve another page without exceeding `maxOpenBrowsers`. */
165
- async _waitForFreeSlot() {
166
+ async waitForFreeSlot() {
166
167
  while (!this.browserPool.hasFreeBrowserSlot() && !this.browserPool.hasActiveBrowserWithFreeCapacity()) {
167
- await this._nextCapacityChange();
168
+ await this.nextCapacityChange();
168
169
  }
169
170
  }
170
171
  /**
@@ -172,20 +173,20 @@ export class RemoteBrowserPool {
172
173
  * concurrently-waiting `newPage` calls share a single promise (and a single pair of event listeners)
173
174
  * per tick, so a fleet of saturated callers doesn't fan out into N listener pairs on the pool.
174
175
  */
175
- _nextCapacityChange() {
176
- this._capacityChange ??= new Promise((resolve) => {
176
+ nextCapacityChange() {
177
+ this.#capacityChange ??= new Promise((resolve) => {
177
178
  const done = () => {
178
179
  clearTimeout(timer);
179
- this.browserPool.off("browserRetired" /* BROWSER_POOL_EVENTS.BROWSER_RETIRED */, done);
180
- this.browserPool.off("pageClosed" /* BROWSER_POOL_EVENTS.PAGE_CLOSED */, done);
181
- this._capacityChange = undefined;
180
+ this.browserPool.off(BROWSER_POOL_EVENTS.BROWSER_RETIRED, done);
181
+ this.browserPool.off(BROWSER_POOL_EVENTS.PAGE_CLOSED, done);
182
+ this.#capacityChange = undefined;
182
183
  resolve();
183
184
  };
184
- const timer = setTimeout(done, this.slotPollIntervalMillis);
185
+ const timer = setTimeout(done, this.#slotPollIntervalMillis);
185
186
  timer.unref?.();
186
- this.browserPool.once("browserRetired" /* BROWSER_POOL_EVENTS.BROWSER_RETIRED */, done);
187
- this.browserPool.once("pageClosed" /* BROWSER_POOL_EVENTS.PAGE_CLOSED */, done);
187
+ this.browserPool.once(BROWSER_POOL_EVENTS.BROWSER_RETIRED, done);
188
+ this.browserPool.once(BROWSER_POOL_EVENTS.PAGE_CLOSED, done);
188
189
  });
189
- return this._capacityChange;
190
+ return this.#capacityChange;
190
191
  }
191
192
  }