@crawlee/browser-pool 4.0.0-beta.115 → 4.0.0-beta.117
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -2,6 +2,7 @@ import { serviceLocator } from '@crawlee/core';
|
|
|
2
2
|
import { nanoid } from 'nanoid';
|
|
3
3
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
4
4
|
import { tryCancel } from '@apify/timeout';
|
|
5
|
+
import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
|
|
5
6
|
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
|
|
6
7
|
/**
|
|
7
8
|
* The `BrowserController` serves two purposes. First, it is the base class that
|
|
@@ -89,7 +90,7 @@ export class BrowserController extends TypedEmitter {
|
|
|
89
90
|
catch (error) {
|
|
90
91
|
this.log.debug(`Could not close browser.\nCause: ${error.message}`, { id: this.id });
|
|
91
92
|
}
|
|
92
|
-
this.emit(
|
|
93
|
+
this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
|
|
93
94
|
const killTimer = setTimeout(() => {
|
|
94
95
|
this._kill().catch((err) => {
|
|
95
96
|
this.log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
|
|
@@ -105,7 +106,7 @@ export class BrowserController extends TypedEmitter {
|
|
|
105
106
|
async kill() {
|
|
106
107
|
await this.#hasBrowserPromise;
|
|
107
108
|
await this._kill();
|
|
108
|
-
this.emit(
|
|
109
|
+
this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
|
|
109
110
|
}
|
|
110
111
|
/**
|
|
111
112
|
* Opens new browser page.
|
package/browser-pool.js
CHANGED
|
@@ -8,6 +8,7 @@ import pLimit from 'p-limit';
|
|
|
8
8
|
import QuickLRU from 'quick-lru';
|
|
9
9
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
10
10
|
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
|
|
11
|
+
import { BROWSER_POOL_EVENTS } from './events.js';
|
|
11
12
|
import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePageCreateHook, } from './fingerprinting/hooks.js';
|
|
12
13
|
const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
|
|
13
14
|
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
|
|
@@ -309,7 +310,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
309
310
|
}
|
|
310
311
|
await this.executeHooks(this.postPageCreateHooks, page, browserController);
|
|
311
312
|
tryCancel();
|
|
312
|
-
this.emit(
|
|
313
|
+
this.emit(BROWSER_POOL_EVENTS.PAGE_CREATED, page);
|
|
313
314
|
return page;
|
|
314
315
|
}
|
|
315
316
|
/**
|
|
@@ -324,7 +325,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
324
325
|
if (hasBeenRetiredOrKilled)
|
|
325
326
|
return;
|
|
326
327
|
this.retiredBrowserControllers.add(browserController);
|
|
327
|
-
this.emit(
|
|
328
|
+
this.emit(BROWSER_POOL_EVENTS.BROWSER_RETIRED, browserController);
|
|
328
329
|
this.startingBrowserControllers.delete(browserController);
|
|
329
330
|
this.activeBrowserControllers.delete(browserController);
|
|
330
331
|
}
|
|
@@ -485,7 +486,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
485
486
|
browserController.activate();
|
|
486
487
|
this.startingBrowserControllers.delete(browserController);
|
|
487
488
|
this.activeBrowserControllers.add(browserController);
|
|
488
|
-
this.emit(
|
|
489
|
+
this.emit(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, browserController);
|
|
489
490
|
return browserController;
|
|
490
491
|
}
|
|
491
492
|
/**
|
|
@@ -542,7 +543,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
542
543
|
await this.executeHooks(this.postPageCloseHooks, pageId, browserController);
|
|
543
544
|
this.pages.delete(pageId);
|
|
544
545
|
this.closeRetiredBrowserWithNoPages(browserController);
|
|
545
|
-
this.emit(
|
|
546
|
+
this.emit(BROWSER_POOL_EVENTS.PAGE_CLOSED, page);
|
|
546
547
|
};
|
|
547
548
|
}
|
|
548
549
|
async executeHooks(hooks, ...args) {
|
package/events.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
export declare
|
|
1
|
+
export declare enum BROWSER_POOL_EVENTS {
|
|
2
2
|
BROWSER_LAUNCHED = "browserLaunched",
|
|
3
3
|
BROWSER_RETIRED = "browserRetired",
|
|
4
4
|
BROWSER_CLOSED = "browserClosed",
|
|
5
5
|
PAGE_CREATED = "pageCreated",
|
|
6
6
|
PAGE_CLOSED = "pageClosed"
|
|
7
7
|
}
|
|
8
|
-
export declare
|
|
8
|
+
export declare enum BROWSER_CONTROLLER_EVENTS {
|
|
9
9
|
BROWSER_CLOSED = "browserClosed"
|
|
10
10
|
}
|
|
@@ -36,7 +36,7 @@ export interface BrowserSpecification {
|
|
|
36
36
|
*/
|
|
37
37
|
httpVersion?: HttpVersion;
|
|
38
38
|
}
|
|
39
|
-
export declare
|
|
39
|
+
export declare enum OperatingSystemsName {
|
|
40
40
|
linux = "linux",
|
|
41
41
|
macos = "macos",
|
|
42
42
|
windows = "windows",
|
|
@@ -49,7 +49,7 @@ export declare const enum OperatingSystemsName {
|
|
|
49
49
|
*/
|
|
50
50
|
ios = "ios"
|
|
51
51
|
}
|
|
52
|
-
export declare
|
|
52
|
+
export declare enum DeviceCategory {
|
|
53
53
|
/**
|
|
54
54
|
* Describes mobile devices (mobile phones, tablets...). These devices usually have smaller, vertical screens and load lighter versions of websites.
|
|
55
55
|
* > Note: Generating `android` and `ios` devices will not work without setting the device to `mobile` first.
|
package/fingerprinting/utils.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { PlaywrightPlugin } from '../playwright/playwright-plugin.js';
|
|
2
2
|
import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js';
|
|
3
|
-
import { BrowserName } from './types.js';
|
|
3
|
+
import { BrowserName, DeviceCategory, OperatingSystemsName } from './types.js';
|
|
4
4
|
export const getGeneratorDefaultOptions = (launchContext) => {
|
|
5
5
|
const { browserPlugin, launchOptions } = launchContext;
|
|
6
6
|
const options = {
|
|
7
|
-
devices: [
|
|
7
|
+
devices: [DeviceCategory.desktop],
|
|
8
8
|
locales: ['en-US'],
|
|
9
9
|
browsers: [getBrowserName(browserPlugin, launchOptions)],
|
|
10
10
|
operatingSystems: [getOperatingSystem()],
|
|
@@ -34,11 +34,11 @@ const getOperatingSystem = () => {
|
|
|
34
34
|
switch (platform) {
|
|
35
35
|
case 'win32':
|
|
36
36
|
// platform is win32 even for 64-bit
|
|
37
|
-
return
|
|
37
|
+
return OperatingSystemsName.windows;
|
|
38
38
|
case 'darwin':
|
|
39
|
-
return
|
|
39
|
+
return OperatingSystemsName.macos;
|
|
40
40
|
default:
|
|
41
41
|
// consider everything else a linux
|
|
42
|
-
return
|
|
42
|
+
return OperatingSystemsName.linux;
|
|
43
43
|
}
|
|
44
44
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser-pool",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.117",
|
|
4
4
|
"description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -31,8 +31,8 @@
|
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
33
|
"@apify/timeout": "^0.4.4",
|
|
34
|
-
"@crawlee/core": "4.0.0-beta.
|
|
35
|
-
"@crawlee/types": "4.0.0-beta.
|
|
34
|
+
"@crawlee/core": "4.0.0-beta.117",
|
|
35
|
+
"@crawlee/types": "4.0.0-beta.117",
|
|
36
36
|
"fingerprint-generator": "^2.1.68",
|
|
37
37
|
"fingerprint-injector": "^2.1.68",
|
|
38
38
|
"lodash.merge": "^4.6.2",
|
|
@@ -63,5 +63,5 @@
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
},
|
|
66
|
-
"gitHead": "
|
|
66
|
+
"gitHead": "7c9c8c83da02da0444130cb95368045d30881c3b"
|
|
67
67
|
}
|
package/remote-browser-pool.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { serviceLocator } from '@crawlee/core';
|
|
2
2
|
import { BrowserPool } from './browser-pool.js';
|
|
3
|
+
import { BROWSER_CONTROLLER_EVENTS, BROWSER_POOL_EVENTS } from './events.js';
|
|
3
4
|
import { RemoteBrowserProvider } from './remote-browser-provider.js';
|
|
4
5
|
/**
|
|
5
6
|
* Owns the lifecycle of remote browser sessions for a single {@link RemoteBrowserPool}: endpoint
|
|
@@ -120,8 +121,8 @@ export class RemoteBrowserPool {
|
|
|
120
121
|
this.#pool = this.browserPool;
|
|
121
122
|
// Release a browser's remote session once it closes. The registry dedupes (close() schedules a delayed
|
|
122
123
|
// kill(), so BROWSER_CLOSED can fire twice), and destroy()'s releaseAll() backstops any that never close.
|
|
123
|
-
this.browserPool.on(
|
|
124
|
-
controller.once(
|
|
124
|
+
this.browserPool.on(BROWSER_POOL_EVENTS.BROWSER_LAUNCHED, (controller) => {
|
|
125
|
+
controller.once(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, () => {
|
|
125
126
|
const token = controller.launchContext.remoteToken;
|
|
126
127
|
if (token !== undefined)
|
|
127
128
|
void this.#registry.release(token);
|
|
@@ -176,15 +177,15 @@ export class RemoteBrowserPool {
|
|
|
176
177
|
this.#capacityChange ??= new Promise((resolve) => {
|
|
177
178
|
const done = () => {
|
|
178
179
|
clearTimeout(timer);
|
|
179
|
-
this.browserPool.off(
|
|
180
|
-
this.browserPool.off(
|
|
180
|
+
this.browserPool.off(BROWSER_POOL_EVENTS.BROWSER_RETIRED, done);
|
|
181
|
+
this.browserPool.off(BROWSER_POOL_EVENTS.PAGE_CLOSED, done);
|
|
181
182
|
this.#capacityChange = undefined;
|
|
182
183
|
resolve();
|
|
183
184
|
};
|
|
184
185
|
const timer = setTimeout(done, this.#slotPollIntervalMillis);
|
|
185
186
|
timer.unref?.();
|
|
186
|
-
this.browserPool.once(
|
|
187
|
-
this.browserPool.once(
|
|
187
|
+
this.browserPool.once(BROWSER_POOL_EVENTS.BROWSER_RETIRED, done);
|
|
188
|
+
this.browserPool.once(BROWSER_POOL_EVENTS.PAGE_CLOSED, done);
|
|
188
189
|
});
|
|
189
190
|
return this.#capacityChange;
|
|
190
191
|
}
|