@browserless.io/browserless 2.8.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -1
- package/README.md +41 -3
- package/assets/debugger.png +0 -0
- package/bin/scaffold/src/hello-world.http.ts +3 -2
- package/build/browserless.d.ts +5 -4
- package/build/browserless.js +15 -11
- package/build/browsers/chrome.cdp.d.ts +2 -2
- package/build/browsers/chrome.cdp.js +2 -2
- package/build/browsers/chrome.playwright.d.ts +2 -2
- package/build/browsers/chrome.playwright.js +2 -2
- package/build/browsers/chromium.cdp.d.ts +4 -4
- package/build/browsers/chromium.cdp.js +49 -32
- package/build/browsers/chromium.playwright.d.ts +4 -4
- package/build/browsers/chromium.playwright.js +14 -13
- package/build/browsers/firefox.playwright.d.ts +4 -4
- package/build/browsers/firefox.playwright.js +14 -13
- package/build/browsers/index.d.ts +24 -8
- package/build/browsers/index.js +20 -15
- package/build/browsers/webkit.playwright.d.ts +4 -4
- package/build/browsers/webkit.playwright.js +14 -13
- package/build/config.d.ts +3 -0
- package/build/config.js +3 -0
- package/build/file-system.d.ts +2 -3
- package/build/file-system.js +2 -2
- package/build/index.js +7 -7
- package/build/limiter.d.ts +2 -3
- package/build/limiter.js +11 -11
- package/build/logger.d.ts +16 -9
- package/build/logger.js +32 -16
- package/build/monitoring.d.ts +2 -3
- package/build/monitoring.js +4 -4
- package/build/router.d.ts +1 -3
- package/build/router.js +21 -22
- package/build/routes/chrome/http/content.post.body.json +8 -8
- package/build/routes/chrome/http/pdf.post.body.json +8 -8
- package/build/routes/chrome/http/scrape.post.body.json +8 -8
- package/build/routes/chrome/http/screenshot.post.body.json +8 -8
- package/build/routes/chromium/http/content.post.body.json +8 -8
- package/build/routes/chromium/http/pdf.post.body.json +8 -8
- package/build/routes/chromium/http/scrape.post.body.json +8 -8
- package/build/routes/chromium/http/screenshot.post.body.json +8 -8
- package/build/routes/management/http/static.get.js +3 -3
- package/build/routes/management/tests/management.spec.js +9 -0
- package/build/server.d.ts +1 -3
- package/build/server.js +33 -30
- package/build/shared/content.http.d.ts +1 -1
- package/build/shared/content.http.js +4 -1
- package/build/shared/download.http.js +9 -9
- package/build/shared/function.http.js +2 -2
- package/build/shared/json-protocol.http.d.ts +3 -3
- package/build/shared/json-protocol.http.js +2 -2
- package/build/shared/json-version.http.d.ts +3 -3
- package/build/shared/json-version.http.js +2 -2
- package/build/shared/pdf.http.d.ts +1 -1
- package/build/shared/pdf.http.js +6 -4
- package/build/shared/performance.http.js +1 -0
- package/build/shared/scrape.http.d.ts +1 -1
- package/build/shared/scrape.http.js +4 -1
- package/build/shared/screenshot.http.d.ts +1 -1
- package/build/shared/screenshot.http.js +4 -1
- package/build/shared/utils/function/handler.js +7 -7
- package/build/shared/utils/performance/child.js +4 -4
- package/build/shared/utils/performance/main.d.ts +1 -1
- package/build/shared/utils/performance/main.js +5 -7
- package/build/shared/utils/performance/types.d.ts +2 -1
- package/build/utils.d.ts +1 -1
- package/build/utils.js +6 -2
- package/package.json +18 -15
- package/scripts/install-debugger.js +55 -15
- package/src/browserless.ts +21 -12
- package/src/browsers/chrome.cdp.ts +2 -5
- package/src/browsers/chrome.playwright.ts +2 -5
- package/src/browsers/chromium.cdp.ts +84 -35
- package/src/browsers/chromium.playwright.ts +26 -13
- package/src/browsers/firefox.playwright.ts +28 -13
- package/src/browsers/index.ts +24 -16
- package/src/browsers/webkit.playwright.ts +28 -13
- package/src/config.ts +4 -0
- package/src/file-system.ts +2 -7
- package/src/index.ts +7 -7
- package/src/limiter.ts +13 -11
- package/src/logger.ts +39 -18
- package/src/monitoring.ts +6 -8
- package/src/router.ts +20 -20
- package/src/routes/management/http/static.get.ts +3 -5
- package/src/routes/management/tests/management.spec.ts +15 -0
- package/src/server.ts +43 -30
- package/src/shared/content.http.ts +5 -1
- package/src/shared/download.http.ts +9 -9
- package/src/shared/function.http.ts +2 -2
- package/src/shared/json-protocol.http.ts +8 -3
- package/src/shared/json-version.http.ts +8 -4
- package/src/shared/pdf.http.ts +6 -4
- package/src/shared/performance.http.ts +1 -0
- package/src/shared/scrape.http.ts +5 -1
- package/src/shared/screenshot.http.ts +4 -1
- package/src/shared/utils/function/handler.ts +7 -7
- package/src/shared/utils/performance/child.ts +4 -4
- package/src/shared/utils/performance/main.ts +5 -6
- package/src/shared/utils/performance/types.ts +2 -1
- package/src/utils.ts +7 -2
- package/static/docs/swagger.json +10 -10
- package/static/docs/swagger.min.json +9 -9
- package/static/function/client.js +1646 -2917
- package/static/function/index.html +1646 -2917
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ServerError,
|
|
1
|
+
import { ServerError, } from '@browserless.io/browserless';
|
|
2
2
|
import playwright from 'playwright-core';
|
|
3
3
|
import { EventEmitter } from 'events';
|
|
4
4
|
import httpProxy from 'http-proxy';
|
|
@@ -7,15 +7,16 @@ export class FirefoxPlaywright extends EventEmitter {
|
|
|
7
7
|
config;
|
|
8
8
|
userDataDir;
|
|
9
9
|
running = false;
|
|
10
|
+
logger;
|
|
10
11
|
proxy = httpProxy.createProxyServer();
|
|
11
12
|
browser = null;
|
|
12
13
|
browserWSEndpoint = null;
|
|
13
|
-
|
|
14
|
-
constructor({ config, userDataDir, }) {
|
|
14
|
+
constructor({ config, userDataDir, logger, }) {
|
|
15
15
|
super();
|
|
16
16
|
this.userDataDir = userDataDir;
|
|
17
17
|
this.config = config;
|
|
18
|
-
this.
|
|
18
|
+
this.logger = logger;
|
|
19
|
+
this.logger.info(`Starting new ${this.constructor.name} instance`);
|
|
19
20
|
}
|
|
20
21
|
cleanListeners() {
|
|
21
22
|
this.removeAllListeners();
|
|
@@ -23,7 +24,7 @@ export class FirefoxPlaywright extends EventEmitter {
|
|
|
23
24
|
isRunning = () => this.running;
|
|
24
25
|
close = async () => {
|
|
25
26
|
if (this.browser) {
|
|
26
|
-
this.
|
|
27
|
+
this.logger.trace(`Closing ${this.constructor.name} process and all listeners`);
|
|
27
28
|
this.emit('close');
|
|
28
29
|
this.cleanListeners();
|
|
29
30
|
this.browser.close();
|
|
@@ -34,16 +35,16 @@ export class FirefoxPlaywright extends EventEmitter {
|
|
|
34
35
|
};
|
|
35
36
|
pages = async () => [];
|
|
36
37
|
getPageId = () => {
|
|
37
|
-
throw new ServerError(`#getPageId is not yet supported with this
|
|
38
|
+
throw new ServerError(`#getPageId is not yet supported with ${this.constructor.name}.`);
|
|
38
39
|
};
|
|
39
40
|
makeLiveURL = () => {
|
|
40
|
-
throw new ServerError(`Live URLs are not yet supported with this
|
|
41
|
+
throw new ServerError(`Live URLs are not yet supported with ${this.constructor.name}.`);
|
|
41
42
|
};
|
|
42
43
|
newPage = async () => {
|
|
43
|
-
throw new ServerError(`Can't create new page with this
|
|
44
|
+
throw new ServerError(`Can't create new page with ${this.constructor.name}`);
|
|
44
45
|
};
|
|
45
46
|
launch = async (options = {}) => {
|
|
46
|
-
this.
|
|
47
|
+
this.logger.info(`Launching ${this.constructor.name} Handler`);
|
|
47
48
|
this.browser = await playwright.firefox.launchServer({
|
|
48
49
|
...options,
|
|
49
50
|
args: [
|
|
@@ -53,7 +54,7 @@ export class FirefoxPlaywright extends EventEmitter {
|
|
|
53
54
|
executablePath: playwright.firefox.executablePath(),
|
|
54
55
|
});
|
|
55
56
|
const browserWSEndpoint = this.browser.wsEndpoint();
|
|
56
|
-
this.
|
|
57
|
+
this.logger.info(`${this.constructor.name} is running on ${browserWSEndpoint}`);
|
|
57
58
|
this.browserWSEndpoint = browserWSEndpoint;
|
|
58
59
|
this.running = true;
|
|
59
60
|
return this.browser;
|
|
@@ -72,14 +73,14 @@ export class FirefoxPlaywright extends EventEmitter {
|
|
|
72
73
|
return externalURL.href;
|
|
73
74
|
};
|
|
74
75
|
proxyPageWebSocket = async () => {
|
|
75
|
-
|
|
76
|
+
this.logger.warn(`Not yet implemented in ${this.constructor.name}`);
|
|
76
77
|
};
|
|
77
78
|
proxyWebSocket = async (req, socket, head) => new Promise((resolve, reject) => {
|
|
78
79
|
if (!this.browserWSEndpoint) {
|
|
79
80
|
throw new ServerError(`No browserWSEndpoint found, did you launch first?`);
|
|
80
81
|
}
|
|
81
82
|
socket.once('close', resolve);
|
|
82
|
-
this.
|
|
83
|
+
this.logger.info(`Proxying ${req.parsed.href} to ${this.constructor.name} ${this.browserWSEndpoint}`);
|
|
83
84
|
// Delete headers known to cause issues
|
|
84
85
|
delete req.headers.origin;
|
|
85
86
|
req.url = '';
|
|
@@ -87,7 +88,7 @@ export class FirefoxPlaywright extends EventEmitter {
|
|
|
87
88
|
changeOrigin: true,
|
|
88
89
|
target: this.browserWSEndpoint,
|
|
89
90
|
}, (error) => {
|
|
90
|
-
this.
|
|
91
|
+
this.logger.error(`Error proxying session to ${this.constructor.name}: ${error}`);
|
|
91
92
|
this.close();
|
|
92
93
|
return reject(error);
|
|
93
94
|
});
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
import { BrowserHTTPRoute, BrowserInstance, BrowserWebsocketRoute, BrowserlessSession, BrowserlessSessionJSON, CDPJSONPayload, ChromiumCDP, Config, Hooks, Request } from '@browserless.io/browserless';
|
|
1
|
+
import { BrowserHTTPRoute, BrowserInstance, BrowserServerOptions, BrowserWebsocketRoute, BrowserlessSession, BrowserlessSessionJSON, CDPJSONPayload, CDPLaunchOptions, ChromiumCDP, Config, Hooks, Logger, Request } from '@browserless.io/browserless';
|
|
3
2
|
import { Page } from 'puppeteer-core';
|
|
4
3
|
export declare class BrowserManager {
|
|
5
4
|
protected config: Config;
|
|
@@ -7,11 +6,11 @@ export declare class BrowserManager {
|
|
|
7
6
|
protected browsers: Map<BrowserInstance, BrowserlessSession>;
|
|
8
7
|
protected launching: Map<string, Promise<unknown>>;
|
|
9
8
|
protected timers: Map<string, number>;
|
|
10
|
-
protected
|
|
9
|
+
protected log: Logger;
|
|
11
10
|
protected chromeBrowsers: (typeof ChromiumCDP)[];
|
|
12
11
|
protected playwrightBrowserNames: string[];
|
|
13
12
|
constructor(config: Config, hooks: Hooks);
|
|
14
|
-
|
|
13
|
+
protected browserIsChrome: (b: BrowserInstance) => boolean;
|
|
15
14
|
protected removeUserDataDir: (userDataDir: string | null) => Promise<void>;
|
|
16
15
|
protected onNewPage: (req: Request, page: Page) => Promise<void>;
|
|
17
16
|
/**
|
|
@@ -19,24 +18,41 @@ export declare class BrowserManager {
|
|
|
19
18
|
* and modifies URLs to set them to the appropriate addresses configured.
|
|
20
19
|
* When both Chrome and Chromium are installed, defaults to Chromium.
|
|
21
20
|
*/
|
|
22
|
-
getProtocolJSON: () => Promise<object>;
|
|
21
|
+
getProtocolJSON: (logger: Logger) => Promise<object>;
|
|
23
22
|
/**
|
|
24
23
|
* Returns the /json/version API from Chromium or Chrome, whichever is installed,
|
|
25
24
|
* and modifies URLs to set them to the appropriate addresses configured.
|
|
26
25
|
* When both Chrome and Chromium are installed, defaults to Chromium.
|
|
27
26
|
*/
|
|
28
|
-
getVersionJSON: () => Promise<CDPJSONPayload>;
|
|
27
|
+
getVersionJSON: (logger: Logger) => Promise<CDPJSONPayload>;
|
|
29
28
|
/**
|
|
30
29
|
* Returns a list of all Chrome-like browsers (both Chromium and Chrome) with
|
|
31
30
|
* their respective /json/list contents. URLs are modified so that subsequent
|
|
32
31
|
* calls can be forwarded to the appropriate destination
|
|
33
32
|
*/
|
|
34
33
|
getJSONList: () => Promise<Array<CDPJSONPayload>>;
|
|
35
|
-
|
|
34
|
+
protected generateSessionJson: (browser: BrowserInstance, session: BrowserlessSession) => Promise<{
|
|
35
|
+
browser: string;
|
|
36
|
+
browserId: string;
|
|
37
|
+
initialConnectURL: string;
|
|
38
|
+
killURL: string | null;
|
|
39
|
+
running: boolean;
|
|
40
|
+
timeAliveMs: number;
|
|
41
|
+
type: string;
|
|
42
|
+
id: string | null;
|
|
43
|
+
isTempDataDir: boolean;
|
|
44
|
+
launchOptions: CDPLaunchOptions | BrowserServerOptions;
|
|
45
|
+
numbConnected: number;
|
|
46
|
+
resolver: (val: unknown) => void;
|
|
47
|
+
routePath: string | string[];
|
|
48
|
+
startedOn: number;
|
|
49
|
+
ttl: number;
|
|
50
|
+
userDataDir: string | null;
|
|
51
|
+
}[]>;
|
|
36
52
|
close: (browser: BrowserInstance, session: BrowserlessSession) => Promise<void>;
|
|
37
53
|
getAllSessions: () => Promise<BrowserlessSessionJSON[]>;
|
|
38
54
|
complete: (browser: BrowserInstance) => Promise<void>;
|
|
39
|
-
getBrowserForRequest: (req: Request, router: BrowserHTTPRoute | BrowserWebsocketRoute) => Promise<BrowserInstance>;
|
|
55
|
+
getBrowserForRequest: (req: Request, router: BrowserHTTPRoute | BrowserWebsocketRoute, logger: Logger) => Promise<BrowserInstance>;
|
|
40
56
|
shutdown: () => Promise<void>;
|
|
41
57
|
/**
|
|
42
58
|
* Left blank for downstream SDK modules to optionally implement.
|
package/build/browsers/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { BLESS_PAGE_IDENTIFIER, BadRequest, ChromeCDP, ChromePlaywright, ChromiumCDP, ChromiumPlaywright, FirefoxPlaywright, HTTPManagementRoutes, NotFound, ServerError, WebkitPlaywright, availableBrowsers, convertIfBase64,
|
|
1
|
+
import { BLESS_PAGE_IDENTIFIER, BadRequest, ChromeCDP, ChromePlaywright, ChromiumCDP, ChromiumPlaywright, FirefoxPlaywright, HTTPManagementRoutes, Logger, NotFound, ServerError, WebkitPlaywright, availableBrowsers, convertIfBase64, exists, generateDataDir, makeExternalURL, noop, parseBooleanParam, } from '@browserless.io/browserless';
|
|
2
2
|
import { deleteAsync } from 'del';
|
|
3
3
|
import path from 'path';
|
|
4
4
|
export class BrowserManager {
|
|
@@ -7,7 +7,7 @@ export class BrowserManager {
|
|
|
7
7
|
browsers = new Map();
|
|
8
8
|
launching = new Map();
|
|
9
9
|
timers = new Map();
|
|
10
|
-
|
|
10
|
+
log = new Logger('browser-manager');
|
|
11
11
|
chromeBrowsers = [ChromiumCDP, ChromeCDP];
|
|
12
12
|
playwrightBrowserNames = [
|
|
13
13
|
ChromiumPlaywright.name,
|
|
@@ -22,9 +22,9 @@ export class BrowserManager {
|
|
|
22
22
|
browserIsChrome = (b) => this.chromeBrowsers.some((chromeBrowser) => b instanceof chromeBrowser);
|
|
23
23
|
removeUserDataDir = async (userDataDir) => {
|
|
24
24
|
if (userDataDir && (await exists(userDataDir))) {
|
|
25
|
-
this.
|
|
25
|
+
this.log.info(`Deleting data directory "${userDataDir}"`);
|
|
26
26
|
await deleteAsync(userDataDir, { force: true }).catch((err) => {
|
|
27
|
-
this.
|
|
27
|
+
this.log.error(`Error cleaning up user-data-dir "${err}" at ${userDataDir}`);
|
|
28
28
|
});
|
|
29
29
|
}
|
|
30
30
|
};
|
|
@@ -36,7 +36,7 @@ export class BrowserManager {
|
|
|
36
36
|
* and modifies URLs to set them to the appropriate addresses configured.
|
|
37
37
|
* When both Chrome and Chromium are installed, defaults to Chromium.
|
|
38
38
|
*/
|
|
39
|
-
getProtocolJSON = async () => {
|
|
39
|
+
getProtocolJSON = async (logger) => {
|
|
40
40
|
const Browser = (await availableBrowsers).find((InstalledBrowser) => this.chromeBrowsers.some((ChromeBrowser) => InstalledBrowser === ChromeBrowser));
|
|
41
41
|
if (!Browser) {
|
|
42
42
|
throw new Error(`No Chrome or Chromium browsers are installed!`);
|
|
@@ -44,6 +44,7 @@ export class BrowserManager {
|
|
|
44
44
|
const browser = new Browser({
|
|
45
45
|
blockAds: false,
|
|
46
46
|
config: this.config,
|
|
47
|
+
logger,
|
|
47
48
|
userDataDir: null,
|
|
48
49
|
});
|
|
49
50
|
await browser.launch();
|
|
@@ -62,8 +63,8 @@ export class BrowserManager {
|
|
|
62
63
|
* and modifies URLs to set them to the appropriate addresses configured.
|
|
63
64
|
* When both Chrome and Chromium are installed, defaults to Chromium.
|
|
64
65
|
*/
|
|
65
|
-
getVersionJSON = async () => {
|
|
66
|
-
this.
|
|
66
|
+
getVersionJSON = async (logger) => {
|
|
67
|
+
this.log.info(`Launching Chromium to generate /json/version results`);
|
|
67
68
|
const Browser = (await availableBrowsers).find((InstalledBrowser) => this.chromeBrowsers.some((ChromeBrowser) => InstalledBrowser === ChromeBrowser));
|
|
68
69
|
if (!Browser) {
|
|
69
70
|
throw new ServerError(`No Chrome or Chromium browsers are installed!`);
|
|
@@ -71,6 +72,7 @@ export class BrowserManager {
|
|
|
71
72
|
const browser = new Browser({
|
|
72
73
|
blockAds: false,
|
|
73
74
|
config: this.config,
|
|
75
|
+
logger,
|
|
74
76
|
userDataDir: null,
|
|
75
77
|
});
|
|
76
78
|
await browser.launch();
|
|
@@ -178,15 +180,15 @@ export class BrowserManager {
|
|
|
178
180
|
};
|
|
179
181
|
close = async (browser, session) => {
|
|
180
182
|
const cleanupACtions = [];
|
|
181
|
-
this.
|
|
183
|
+
this.log.info(`${session.numbConnected} Client(s) are currently connected`);
|
|
182
184
|
// Don't close if there's clients still connected
|
|
183
185
|
if (session.numbConnected > 0) {
|
|
184
186
|
return;
|
|
185
187
|
}
|
|
186
|
-
this.
|
|
188
|
+
this.log.info(`Closing browser session`);
|
|
187
189
|
cleanupACtions.push(() => browser.close());
|
|
188
190
|
if (session.isTempDataDir) {
|
|
189
|
-
this.
|
|
191
|
+
this.log.info(`Deleting "${session.userDataDir}" user-data-dir and session from memory`);
|
|
190
192
|
this.browsers.delete(browser);
|
|
191
193
|
cleanupACtions.push(() => this.removeUserDataDir(session.userDataDir));
|
|
192
194
|
}
|
|
@@ -204,7 +206,7 @@ export class BrowserManager {
|
|
|
204
206
|
complete = async (browser) => {
|
|
205
207
|
const session = this.browsers.get(browser);
|
|
206
208
|
if (!session) {
|
|
207
|
-
this.
|
|
209
|
+
this.log.info(`Couldn't locate session for browser, proceeding with close`);
|
|
208
210
|
return browser.close();
|
|
209
211
|
}
|
|
210
212
|
const { id, resolver } = session;
|
|
@@ -215,7 +217,7 @@ export class BrowserManager {
|
|
|
215
217
|
--session.numbConnected;
|
|
216
218
|
this.close(browser, session);
|
|
217
219
|
};
|
|
218
|
-
getBrowserForRequest = async (req, router) => {
|
|
220
|
+
getBrowserForRequest = async (req, router, logger) => {
|
|
219
221
|
const { browser: Browser } = router;
|
|
220
222
|
const blockAds = parseBooleanParam(req.parsed.searchParams, 'blockAds', false);
|
|
221
223
|
const decodedLaunchOptions = convertIfBase64(req.parsed.searchParams.get('launch') || '{}');
|
|
@@ -228,7 +230,7 @@ export class BrowserManager {
|
|
|
228
230
|
if (found) {
|
|
229
231
|
const [browser, session] = found;
|
|
230
232
|
++session.numbConnected;
|
|
231
|
-
this.debug(`Located browser with ID ${id}`);
|
|
233
|
+
this.log.debug(`Located browser with ID ${id}`);
|
|
232
234
|
return browser;
|
|
233
235
|
}
|
|
234
236
|
throw new NotFound(`Couldn't locate browser "${id}" for request "${req.parsed.pathname}"`);
|
|
@@ -259,6 +261,8 @@ export class BrowserManager {
|
|
|
259
261
|
}));
|
|
260
262
|
const found = allPages.flat().find((b) => b.id === id);
|
|
261
263
|
if (found) {
|
|
264
|
+
const session = this.browsers.get(found.browser);
|
|
265
|
+
++session.numbConnected;
|
|
262
266
|
return found.browser;
|
|
263
267
|
}
|
|
264
268
|
throw new NotFound(`Couldn't locate browser "${id}" for request "${req.parsed.pathname}"`);
|
|
@@ -299,6 +303,7 @@ export class BrowserManager {
|
|
|
299
303
|
const browser = new Browser({
|
|
300
304
|
blockAds,
|
|
301
305
|
config: this.config,
|
|
306
|
+
logger,
|
|
302
307
|
userDataDir,
|
|
303
308
|
});
|
|
304
309
|
const connectionMeta = {
|
|
@@ -323,7 +328,7 @@ export class BrowserManager {
|
|
|
323
328
|
return browser;
|
|
324
329
|
};
|
|
325
330
|
shutdown = async () => {
|
|
326
|
-
this.
|
|
331
|
+
this.log.info(`Closing down browser instances`);
|
|
327
332
|
const sessions = Array.from(this.browsers);
|
|
328
333
|
await Promise.all(sessions.map(([b]) => b.close()));
|
|
329
334
|
const timers = Array.from(this.timers);
|
|
@@ -332,7 +337,7 @@ export class BrowserManager {
|
|
|
332
337
|
this.browsers = new Map();
|
|
333
338
|
this.timers = new Map();
|
|
334
339
|
await this.stop();
|
|
335
|
-
this.
|
|
340
|
+
this.log.info(`Shutdown complete`);
|
|
336
341
|
};
|
|
337
342
|
/**
|
|
338
343
|
* Left blank for downstream SDK modules to optionally implement.
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
/// <reference types="debug" />
|
|
3
2
|
/// <reference types="node" />
|
|
4
3
|
/// <reference types="node" />
|
|
5
4
|
/// <reference types="node" />
|
|
6
|
-
import { BrowserServerOptions, Config, Request } from '@browserless.io/browserless';
|
|
5
|
+
import { BrowserServerOptions, Config, Logger, Request } from '@browserless.io/browserless';
|
|
7
6
|
import playwright, { Page } from 'playwright-core';
|
|
8
7
|
import { Duplex } from 'stream';
|
|
9
8
|
import { EventEmitter } from 'events';
|
|
@@ -15,9 +14,10 @@ export declare class WebkitPlaywright extends EventEmitter {
|
|
|
15
14
|
protected proxy: httpProxy<import("http").IncomingMessage, import("http").ServerResponse<import("http").IncomingMessage>>;
|
|
16
15
|
protected browser: playwright.BrowserServer | null;
|
|
17
16
|
protected browserWSEndpoint: string | null;
|
|
18
|
-
protected
|
|
19
|
-
constructor({ config, userDataDir, }: {
|
|
17
|
+
protected logger: Logger;
|
|
18
|
+
constructor({ config, userDataDir, logger, }: {
|
|
20
19
|
config: Config;
|
|
20
|
+
logger: Logger;
|
|
21
21
|
userDataDir: WebkitPlaywright['userDataDir'];
|
|
22
22
|
});
|
|
23
23
|
protected cleanListeners(): void;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ServerError,
|
|
1
|
+
import { ServerError, } from '@browserless.io/browserless';
|
|
2
2
|
import playwright from 'playwright-core';
|
|
3
3
|
import { EventEmitter } from 'events';
|
|
4
4
|
import httpProxy from 'http-proxy';
|
|
@@ -10,12 +10,13 @@ export class WebkitPlaywright extends EventEmitter {
|
|
|
10
10
|
proxy = httpProxy.createProxyServer();
|
|
11
11
|
browser = null;
|
|
12
12
|
browserWSEndpoint = null;
|
|
13
|
-
|
|
14
|
-
constructor({ config, userDataDir, }) {
|
|
13
|
+
logger;
|
|
14
|
+
constructor({ config, userDataDir, logger, }) {
|
|
15
15
|
super();
|
|
16
16
|
this.userDataDir = userDataDir;
|
|
17
17
|
this.config = config;
|
|
18
|
-
this.
|
|
18
|
+
this.logger = logger;
|
|
19
|
+
this.logger.info(`Starting new ${this.constructor.name} instance`);
|
|
19
20
|
}
|
|
20
21
|
cleanListeners() {
|
|
21
22
|
this.removeAllListeners();
|
|
@@ -23,7 +24,7 @@ export class WebkitPlaywright extends EventEmitter {
|
|
|
23
24
|
isRunning = () => this.running;
|
|
24
25
|
close = async () => {
|
|
25
26
|
if (this.browser) {
|
|
26
|
-
this.
|
|
27
|
+
this.logger.info(`Closing ${this.constructor.name} process and all listeners`);
|
|
27
28
|
this.emit('close');
|
|
28
29
|
this.cleanListeners();
|
|
29
30
|
this.browser.close();
|
|
@@ -34,16 +35,16 @@ export class WebkitPlaywright extends EventEmitter {
|
|
|
34
35
|
};
|
|
35
36
|
pages = async () => [];
|
|
36
37
|
getPageId = () => {
|
|
37
|
-
throw new ServerError(`#getPageId is not yet supported with this
|
|
38
|
+
throw new ServerError(`#getPageId is not yet supported with ${this.constructor.name}.`);
|
|
38
39
|
};
|
|
39
40
|
makeLiveURL = () => {
|
|
40
|
-
throw new ServerError(`Live URLs are not yet supported with this
|
|
41
|
+
throw new ServerError(`Live URLs are not yet supported with ${this.constructor.name}.`);
|
|
41
42
|
};
|
|
42
43
|
newPage = async () => {
|
|
43
|
-
throw new ServerError(`Can't create new page with this
|
|
44
|
+
throw new ServerError(`Can't create new page with ${this.constructor.name}`);
|
|
44
45
|
};
|
|
45
46
|
launch = async (options = {}) => {
|
|
46
|
-
this.
|
|
47
|
+
this.logger.info(`Launching ${this.constructor.name} Handler`);
|
|
47
48
|
this.browser = await playwright.webkit.launchServer({
|
|
48
49
|
...options,
|
|
49
50
|
args: [
|
|
@@ -53,7 +54,7 @@ export class WebkitPlaywright extends EventEmitter {
|
|
|
53
54
|
executablePath: playwright.webkit.executablePath(),
|
|
54
55
|
});
|
|
55
56
|
const browserWSEndpoint = this.browser.wsEndpoint();
|
|
56
|
-
this.
|
|
57
|
+
this.logger.info(`${this.constructor.name} is running on ${browserWSEndpoint}`);
|
|
57
58
|
this.browserWSEndpoint = browserWSEndpoint;
|
|
58
59
|
this.running = true;
|
|
59
60
|
return this.browser;
|
|
@@ -72,14 +73,14 @@ export class WebkitPlaywright extends EventEmitter {
|
|
|
72
73
|
return externalURL.href;
|
|
73
74
|
};
|
|
74
75
|
proxyPageWebSocket = async () => {
|
|
75
|
-
|
|
76
|
+
this.logger.warn(`Not yet implemented`);
|
|
76
77
|
};
|
|
77
78
|
proxyWebSocket = async (req, socket, head) => new Promise((resolve, reject) => {
|
|
78
79
|
if (!this.browserWSEndpoint) {
|
|
79
80
|
throw new ServerError(`No browserWSEndpoint found, did you launch first?`);
|
|
80
81
|
}
|
|
81
82
|
socket.once('close', resolve);
|
|
82
|
-
this.
|
|
83
|
+
this.logger.info(`Proxying ${req.parsed.href} to ${this.constructor.name} ${this.browserWSEndpoint}`);
|
|
83
84
|
// Delete headers known to cause issues
|
|
84
85
|
delete req.headers.origin;
|
|
85
86
|
req.url = '';
|
|
@@ -87,7 +88,7 @@ export class WebkitPlaywright extends EventEmitter {
|
|
|
87
88
|
changeOrigin: true,
|
|
88
89
|
target: this.browserWSEndpoint,
|
|
89
90
|
}, (error) => {
|
|
90
|
-
this.
|
|
91
|
+
this.logger.error(`Error proxying session to ${this.constructor.name}: ${error}`);
|
|
91
92
|
this.close();
|
|
92
93
|
return reject(error);
|
|
93
94
|
});
|
package/build/config.d.ts
CHANGED
|
@@ -18,6 +18,7 @@ export declare class Config extends EventEmitter {
|
|
|
18
18
|
protected queued: number;
|
|
19
19
|
protected timeout: number;
|
|
20
20
|
protected static: string;
|
|
21
|
+
protected debuggerDir: string;
|
|
21
22
|
protected retries: number;
|
|
22
23
|
protected allowFileProtocol: boolean;
|
|
23
24
|
protected allowGet: boolean;
|
|
@@ -53,6 +54,7 @@ export declare class Config extends EventEmitter {
|
|
|
53
54
|
getQueued: () => number;
|
|
54
55
|
getTimeout: () => number;
|
|
55
56
|
getStatic: () => string;
|
|
57
|
+
getDebuggerDir: () => string;
|
|
56
58
|
getRetries: () => number;
|
|
57
59
|
getAllowFileProtocol: () => boolean;
|
|
58
60
|
getCPULimit: () => number;
|
|
@@ -63,6 +65,7 @@ export declare class Config extends EventEmitter {
|
|
|
63
65
|
getRejectAlertURL: () => string | null;
|
|
64
66
|
getTimeoutAlertURL: () => string | null;
|
|
65
67
|
getErrorAlertURL: () => string | null;
|
|
68
|
+
hasDebugger: () => Promise<boolean>;
|
|
66
69
|
/**
|
|
67
70
|
* If true, allows GET style calls on our browser-based APIs, using
|
|
68
71
|
* ?body=JSON format.
|
package/build/config.js
CHANGED
|
@@ -124,6 +124,7 @@ export class Config extends EventEmitter {
|
|
|
124
124
|
process.env.CONNECTION_TIMEOUT ??
|
|
125
125
|
'30000');
|
|
126
126
|
static = process.env.STATIC ?? path.join(__dirname, '..', 'static');
|
|
127
|
+
debuggerDir = path.join(this.static, 'debugger');
|
|
127
128
|
retries = +(process.env.RETRIES ?? '5');
|
|
128
129
|
allowFileProtocol = !!parseEnvVars(false, 'ALLOW_FILE_PROTOCOL');
|
|
129
130
|
allowGet = !!parseEnvVars(false, 'ALLOW_GET', 'ENABLE_API_GET');
|
|
@@ -159,6 +160,7 @@ export class Config extends EventEmitter {
|
|
|
159
160
|
getQueued = () => this.queued;
|
|
160
161
|
getTimeout = () => this.timeout;
|
|
161
162
|
getStatic = () => this.static;
|
|
163
|
+
getDebuggerDir = () => this.debuggerDir;
|
|
162
164
|
getRetries = () => this.retries;
|
|
163
165
|
getAllowFileProtocol = () => this.allowFileProtocol;
|
|
164
166
|
getCPULimit = () => this.maxCpu;
|
|
@@ -169,6 +171,7 @@ export class Config extends EventEmitter {
|
|
|
169
171
|
getRejectAlertURL = () => this.rejectAlertURL;
|
|
170
172
|
getTimeoutAlertURL = () => this.timeoutAlertURL;
|
|
171
173
|
getErrorAlertURL = () => this.errorAlertURL;
|
|
174
|
+
hasDebugger = () => exists(this.debuggerDir);
|
|
172
175
|
/**
|
|
173
176
|
* If true, allows GET style calls on our browser-based APIs, using
|
|
174
177
|
* ?body=JSON format.
|
package/build/file-system.d.ts
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
/// <reference types="debug" />
|
|
3
2
|
/// <reference types="node" />
|
|
4
|
-
import { Config } from '@browserless.io/browserless';
|
|
3
|
+
import { Config, Logger } from '@browserless.io/browserless';
|
|
5
4
|
import { EventEmitter } from 'events';
|
|
6
5
|
export declare class FileSystem extends EventEmitter {
|
|
7
6
|
protected config: Config;
|
|
8
7
|
protected fsMap: Map<string, string[]>;
|
|
9
8
|
protected currentAESKey: Buffer;
|
|
10
|
-
protected
|
|
9
|
+
protected logger: Logger;
|
|
11
10
|
constructor(config: Config);
|
|
12
11
|
/**
|
|
13
12
|
* Appends contents to a file-path for persistance. File contents are
|
package/build/file-system.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Logger, decrypt, encrypt } from '@browserless.io/browserless';
|
|
2
2
|
import { readFile, writeFile } from 'fs/promises';
|
|
3
3
|
import { EventEmitter } from 'events';
|
|
4
4
|
export class FileSystem extends EventEmitter {
|
|
5
5
|
config;
|
|
6
6
|
fsMap = new Map();
|
|
7
7
|
currentAESKey;
|
|
8
|
-
|
|
8
|
+
logger = new Logger('file-system');
|
|
9
9
|
constructor(config) {
|
|
10
10
|
super();
|
|
11
11
|
this.config = config;
|
package/build/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { Browserless,
|
|
1
|
+
import { Browserless, Logger } from '@browserless.io/browserless';
|
|
2
2
|
(async () => {
|
|
3
3
|
const browserless = new Browserless();
|
|
4
|
-
const
|
|
4
|
+
const logger = new Logger('index.js');
|
|
5
5
|
browserless.start();
|
|
6
6
|
process
|
|
7
7
|
.on('unhandledRejection', async (reason, promise) => {
|
|
@@ -13,27 +13,27 @@ import { Browserless, createLogger } from '@browserless.io/browserless';
|
|
|
13
13
|
process.exit(1);
|
|
14
14
|
})
|
|
15
15
|
.once('SIGTERM', async () => {
|
|
16
|
-
|
|
16
|
+
logger.info(`SIGTERM received, saving and closing down`);
|
|
17
17
|
await browserless.stop();
|
|
18
18
|
process.exit(0);
|
|
19
19
|
})
|
|
20
20
|
.once('SIGINT', async () => {
|
|
21
|
-
|
|
21
|
+
logger.info(`SIGINT received, saving and closing down`);
|
|
22
22
|
await browserless.stop();
|
|
23
23
|
process.exit(0);
|
|
24
24
|
})
|
|
25
25
|
.once('SIGHUP', async () => {
|
|
26
|
-
|
|
26
|
+
logger.info(`SIGHUP received, saving and closing down`);
|
|
27
27
|
await browserless.stop();
|
|
28
28
|
process.exit(0);
|
|
29
29
|
})
|
|
30
30
|
.once('SIGUSR2', async () => {
|
|
31
|
-
|
|
31
|
+
logger.info(`SIGUSR2 received, saving and closing down`);
|
|
32
32
|
await browserless.stop();
|
|
33
33
|
process.exit(0);
|
|
34
34
|
})
|
|
35
35
|
.once('exit', () => {
|
|
36
|
-
|
|
36
|
+
logger.info(`Process is finished, exiting`);
|
|
37
37
|
process.exit(0);
|
|
38
38
|
});
|
|
39
39
|
})();
|
package/build/limiter.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
import { AfterResponse, Config, Hooks, Metrics, Monitoring, WebHooks } from '@browserless.io/browserless';
|
|
1
|
+
import { AfterResponse, Config, Hooks, Logger, Metrics, Monitoring, WebHooks } from '@browserless.io/browserless';
|
|
3
2
|
import q from 'queue';
|
|
4
3
|
export type LimitFn<TArgs extends unknown[], TResult> = (...args: TArgs) => Promise<TResult>;
|
|
5
4
|
export type ErrorFn<TArgs extends unknown[]> = (...args: TArgs) => void;
|
|
@@ -17,7 +16,7 @@ export declare class Limiter extends q {
|
|
|
17
16
|
protected webhooks: WebHooks;
|
|
18
17
|
protected hooks: Hooks;
|
|
19
18
|
protected queued: number;
|
|
20
|
-
protected
|
|
19
|
+
protected logger: Logger;
|
|
21
20
|
constructor(config: Config, metrics: Metrics, monitor: Monitoring, webhooks: WebHooks, hooks: Hooks);
|
|
22
21
|
protected handleEnd(): void;
|
|
23
22
|
protected jobEnd(jobInfo: AfterResponse): void;
|
package/build/limiter.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Logger, TooManyRequests, } from '@browserless.io/browserless';
|
|
2
2
|
import q from 'queue';
|
|
3
3
|
export class Limiter extends q {
|
|
4
4
|
config;
|
|
@@ -7,7 +7,7 @@ export class Limiter extends q {
|
|
|
7
7
|
webhooks;
|
|
8
8
|
hooks;
|
|
9
9
|
queued;
|
|
10
|
-
|
|
10
|
+
logger = new Logger('limiter');
|
|
11
11
|
constructor(config, metrics, monitor, webhooks, hooks) {
|
|
12
12
|
super({
|
|
13
13
|
autostart: true,
|
|
@@ -20,17 +20,17 @@ export class Limiter extends q {
|
|
|
20
20
|
this.webhooks = webhooks;
|
|
21
21
|
this.hooks = hooks;
|
|
22
22
|
this.queued = config.getQueued();
|
|
23
|
-
this.
|
|
23
|
+
this.logger.info(`Concurrency: ${this.concurrency} queue: ${this.queued} timeout: ${this.timeout}ms`);
|
|
24
24
|
config.on('concurrent', (concurrency) => {
|
|
25
|
-
this.
|
|
25
|
+
this.logger.info(`Concurrency updated to ${concurrency}`);
|
|
26
26
|
this.concurrency = concurrency;
|
|
27
27
|
});
|
|
28
28
|
config.on('queued', (queued) => {
|
|
29
|
-
this.
|
|
29
|
+
this.logger.info(`Queue updated to ${queued}`);
|
|
30
30
|
this.queued = queued;
|
|
31
31
|
});
|
|
32
32
|
config.on('timeout', (timeout) => {
|
|
33
|
-
this.
|
|
33
|
+
this.logger.info(`Timeout updated to ${timeout}ms`);
|
|
34
34
|
this.timeout = timeout <= 0 ? 0 : timeout;
|
|
35
35
|
});
|
|
36
36
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -49,7 +49,7 @@ export class Limiter extends q {
|
|
|
49
49
|
}
|
|
50
50
|
handleSuccess({ detail: { job } }) {
|
|
51
51
|
const timeUsed = Date.now() - job.start;
|
|
52
|
-
this.
|
|
52
|
+
this.logger.info(`Job has succeeded after ${timeUsed.toLocaleString()}ms of activity.`);
|
|
53
53
|
this.metrics.addSuccessful(Date.now() - job.start);
|
|
54
54
|
// @TODO Figure out a better argument handling for jobs
|
|
55
55
|
this.jobEnd({
|
|
@@ -60,10 +60,10 @@ export class Limiter extends q {
|
|
|
60
60
|
}
|
|
61
61
|
handleJobTimeout({ detail: { next, job }, }) {
|
|
62
62
|
const timeUsed = Date.now() - job.start;
|
|
63
|
-
this.
|
|
63
|
+
this.logger.warn(`Job has hit timeout after ${timeUsed.toLocaleString()}ms of activity.`);
|
|
64
64
|
this.metrics.addTimedout(Date.now() - job.start);
|
|
65
65
|
this.webhooks.callTimeoutAlertURL();
|
|
66
|
-
this.
|
|
66
|
+
this.logger.info(`Calling timeout handler`);
|
|
67
67
|
job?.onTimeoutFn(job);
|
|
68
68
|
this.jobEnd({
|
|
69
69
|
req: job.args[0],
|
|
@@ -73,7 +73,7 @@ export class Limiter extends q {
|
|
|
73
73
|
next();
|
|
74
74
|
}
|
|
75
75
|
handleFail({ detail: { error, job }, }) {
|
|
76
|
-
this.
|
|
76
|
+
this.logger.info(`Recording failed stat, cleaning up: "${error?.toString()}"`);
|
|
77
77
|
this.metrics.addError(Date.now() - job.start);
|
|
78
78
|
this.webhooks.callErrorAlertURL(error?.toString() ?? 'Unknown Error');
|
|
79
79
|
this.jobEnd({
|
|
@@ -83,7 +83,7 @@ export class Limiter extends q {
|
|
|
83
83
|
});
|
|
84
84
|
}
|
|
85
85
|
logQueue(message) {
|
|
86
|
-
this.
|
|
86
|
+
this.logger.info(`(Running: ${this.executing}, Pending: ${this.waiting}) ${message} `);
|
|
87
87
|
}
|
|
88
88
|
get executing() {
|
|
89
89
|
return this.length > this.concurrency ? this.concurrency : this.length;
|
package/build/logger.d.ts
CHANGED
|
@@ -1,12 +1,19 @@
|
|
|
1
|
-
import { Request
|
|
1
|
+
import { Request } from '@browserless.io/browserless';
|
|
2
2
|
export declare class Logger {
|
|
3
3
|
protected prefix: string;
|
|
4
|
-
protected request
|
|
5
|
-
protected
|
|
6
|
-
protected
|
|
7
|
-
protected
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
4
|
+
protected request?: Request | undefined;
|
|
5
|
+
protected _trace: (...args: unknown[]) => void;
|
|
6
|
+
protected _debug: (...args: unknown[]) => void;
|
|
7
|
+
protected _info: (...args: unknown[]) => void;
|
|
8
|
+
protected _warn: (...args: unknown[]) => void;
|
|
9
|
+
protected _error: (...args: unknown[]) => void;
|
|
10
|
+
protected _fatal: (...args: unknown[]) => void;
|
|
11
|
+
constructor(prefix: string, request?: Request | undefined);
|
|
12
|
+
protected get reqInfo(): string;
|
|
13
|
+
trace: (...messages: unknown[]) => void;
|
|
14
|
+
debug: (...messages: unknown[]) => void;
|
|
15
|
+
info: (...messages: unknown[]) => void;
|
|
16
|
+
warn: (...messages: unknown[]) => void;
|
|
17
|
+
error: (...messages: unknown[]) => void;
|
|
18
|
+
fatal: (...messages: unknown[]) => void;
|
|
12
19
|
}
|