@crawlee/browser-pool 4.0.0-beta.121 → 4.0.0-beta.122
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/browser-pool.js +19 -19
- package/package.json +6 -6
package/browser-pool.js
CHANGED
|
@@ -1,18 +1,34 @@
|
|
|
1
1
|
import { AsyncResource } from 'node:async_hooks';
|
|
2
|
-
import { SessionError, serviceLocator } from '@crawlee/core';
|
|
2
|
+
import { parseArgument, schemas, SessionError, serviceLocator } from '@crawlee/core';
|
|
3
3
|
import { FingerprintGenerator } from 'fingerprint-generator';
|
|
4
4
|
import { FingerprintInjector } from 'fingerprint-injector';
|
|
5
5
|
import { nanoid } from 'nanoid';
|
|
6
|
-
import ow from 'ow';
|
|
7
6
|
import pLimit from 'p-limit';
|
|
8
7
|
import QuickLRU from 'quick-lru';
|
|
9
8
|
import { TypedEmitter } from 'tiny-typed-emitter';
|
|
9
|
+
import { z } from 'zod';
|
|
10
10
|
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
|
|
11
11
|
import { BROWSER_POOL_EVENTS } from './events.js';
|
|
12
12
|
import { createFingerprintPreLaunchHook, createPostPageCreateHook, createPrePageCreateHook, } from './fingerprinting/hooks.js';
|
|
13
13
|
const PAGE_CLOSE_TIMEOUT_MILLIS = 5000;
|
|
14
14
|
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
|
|
15
15
|
const BROWSER_KILLER_INTERVAL_MILLIS = 10 * 1000;
|
|
16
|
+
const browserPoolOptionsSchema = z.strictObject({
|
|
17
|
+
browserPlugins: schemas.anyArray.refine((value) => value.length >= 1, 'Expected a non-empty array'),
|
|
18
|
+
maxOpenPagesPerBrowser: schemas.anyNumber.default(20),
|
|
19
|
+
retireBrowserAfterPageCount: schemas.anyNumber.default(100),
|
|
20
|
+
operationTimeoutSecs: schemas.anyNumber.default(15),
|
|
21
|
+
closeInactiveBrowserAfterSecs: schemas.anyNumber.default(300),
|
|
22
|
+
retireInactiveBrowserAfterSecs: schemas.anyNumber.default(10),
|
|
23
|
+
preLaunchHooks: schemas.anyArray.default(() => []),
|
|
24
|
+
postLaunchHooks: schemas.anyArray.default(() => []),
|
|
25
|
+
prePageCreateHooks: schemas.anyArray.default(() => []),
|
|
26
|
+
postPageCreateHooks: schemas.anyArray.default(() => []),
|
|
27
|
+
prePageCloseHooks: schemas.anyArray.default(() => []),
|
|
28
|
+
postPageCloseHooks: schemas.anyArray.default(() => []),
|
|
29
|
+
useFingerprints: z.boolean().default(true),
|
|
30
|
+
fingerprintOptions: schemas.anyObject.default(() => ({})),
|
|
31
|
+
});
|
|
16
32
|
/**
|
|
17
33
|
* The `BrowserPool` class is the most important class of the `browser-pool` module.
|
|
18
34
|
* It manages opening and closing of browsers and their pages and its constructor
|
|
@@ -94,23 +110,7 @@ export class BrowserPool extends TypedEmitter {
|
|
|
94
110
|
super();
|
|
95
111
|
this.#log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
|
|
96
112
|
this.browserKillerInterval.unref();
|
|
97
|
-
|
|
98
|
-
browserPlugins: ow.array.minLength(1),
|
|
99
|
-
maxOpenPagesPerBrowser: ow.optional.number,
|
|
100
|
-
retireBrowserAfterPageCount: ow.optional.number,
|
|
101
|
-
operationTimeoutSecs: ow.optional.number,
|
|
102
|
-
closeInactiveBrowserAfterSecs: ow.optional.number,
|
|
103
|
-
retireInactiveBrowserAfterSecs: ow.optional.number,
|
|
104
|
-
preLaunchHooks: ow.optional.array,
|
|
105
|
-
postLaunchHooks: ow.optional.array,
|
|
106
|
-
prePageCreateHooks: ow.optional.array,
|
|
107
|
-
postPageCreateHooks: ow.optional.array,
|
|
108
|
-
prePageCloseHooks: ow.optional.array,
|
|
109
|
-
postPageCloseHooks: ow.optional.array,
|
|
110
|
-
useFingerprints: ow.optional.boolean,
|
|
111
|
-
fingerprintOptions: ow.optional.object,
|
|
112
|
-
}));
|
|
113
|
-
const { browserPlugins, maxOpenPagesPerBrowser = 20, retireBrowserAfterPageCount = 100, operationTimeoutSecs = 15, closeInactiveBrowserAfterSecs = 300, retireInactiveBrowserAfterSecs = 10, preLaunchHooks = [], postLaunchHooks = [], prePageCreateHooks = [], postPageCreateHooks = [], prePageCloseHooks = [], postPageCloseHooks = [], useFingerprints = true, fingerprintOptions = {}, } = options;
|
|
113
|
+
const { browserPlugins, maxOpenPagesPerBrowser, retireBrowserAfterPageCount, operationTimeoutSecs, closeInactiveBrowserAfterSecs, retireInactiveBrowserAfterSecs, preLaunchHooks, postLaunchHooks, prePageCreateHooks, postPageCreateHooks, prePageCloseHooks, postPageCloseHooks, useFingerprints, fingerprintOptions, } = parseArgument(options, browserPoolOptionsSchema);
|
|
114
114
|
const firstPluginConstructor = browserPlugins[0].constructor;
|
|
115
115
|
for (let i = 1; i < browserPlugins.length; i++) {
|
|
116
116
|
const providedPlugin = browserPlugins[i];
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/browser-pool",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.122",
|
|
4
4
|
"description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -31,18 +31,18 @@
|
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
33
|
"@apify/timeout": "^0.4.4",
|
|
34
|
-
"@crawlee/core": "4.0.0-beta.
|
|
35
|
-
"@crawlee/types": "4.0.0-beta.
|
|
34
|
+
"@crawlee/core": "4.0.0-beta.122",
|
|
35
|
+
"@crawlee/types": "4.0.0-beta.122",
|
|
36
36
|
"fingerprint-generator": "^2.1.68",
|
|
37
37
|
"fingerprint-injector": "^2.1.68",
|
|
38
38
|
"lodash.merge": "^4.6.2",
|
|
39
39
|
"nanoid": "^5.1.5",
|
|
40
|
-
"ow": "^2.0.0",
|
|
41
40
|
"p-limit": "^6.2.0",
|
|
42
41
|
"proxy-chain": "^2.5.8",
|
|
43
42
|
"quick-lru": "^7.0.1",
|
|
44
43
|
"tiny-typed-emitter": "^2.1.0",
|
|
45
|
-
"tslib": "^2.8.1"
|
|
44
|
+
"tslib": "^2.8.1",
|
|
45
|
+
"zod": "^4.4.3"
|
|
46
46
|
},
|
|
47
47
|
"peerDependencies": {
|
|
48
48
|
"playwright": "*",
|
|
@@ -63,5 +63,5 @@
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
},
|
|
66
|
-
"gitHead": "
|
|
66
|
+
"gitHead": "2c3e87fefdb9e1fca4c144f03167d8524cfdc2e5"
|
|
67
67
|
}
|