@browserless.io/browserless 2.7.1 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -1
- package/README.md +41 -3
- package/assets/debugger.png +0 -0
- package/bin/browserless.js +8 -4
- package/bin/scaffold/README.md +6 -4
- package/bin/scaffold/src/hello-world.http.ts +7 -1
- package/build/browserless.d.ts +8 -5
- package/build/browserless.js +23 -22
- package/build/browsers/chrome.cdp.d.ts +2 -2
- package/build/browsers/chrome.cdp.js +2 -2
- package/build/browsers/chrome.playwright.d.ts +2 -2
- package/build/browsers/chrome.playwright.js +2 -2
- package/build/browsers/chromium.cdp.d.ts +4 -4
- package/build/browsers/chromium.cdp.js +49 -32
- package/build/browsers/chromium.playwright.d.ts +4 -4
- package/build/browsers/chromium.playwright.js +14 -13
- package/build/browsers/firefox.playwright.d.ts +4 -4
- package/build/browsers/firefox.playwright.js +14 -13
- package/build/browsers/index.d.ts +24 -8
- package/build/browsers/index.js +20 -15
- package/build/browsers/webkit.playwright.d.ts +4 -4
- package/build/browsers/webkit.playwright.js +14 -13
- package/build/config.d.ts +3 -0
- package/build/config.js +3 -0
- package/build/exports.d.ts +1 -0
- package/build/exports.js +1 -0
- package/build/file-system.d.ts +2 -3
- package/build/file-system.js +2 -2
- package/build/index.js +7 -7
- package/build/limiter.d.ts +2 -3
- package/build/limiter.js +11 -11
- package/build/logger.d.ts +19 -0
- package/build/logger.js +43 -0
- package/build/monitoring.d.ts +2 -3
- package/build/monitoring.js +4 -4
- package/build/router.d.ts +4 -5
- package/build/router.js +31 -28
- package/build/routes/chrome/http/content.post.body.json +8 -8
- package/build/routes/chrome/http/pdf.post.body.json +9 -9
- package/build/routes/chrome/http/scrape.post.body.json +8 -8
- package/build/routes/chrome/http/screenshot.post.body.json +8 -8
- package/build/routes/chromium/http/content.post.body.json +8 -8
- package/build/routes/chromium/http/pdf.post.body.json +9 -9
- package/build/routes/chromium/http/scrape.post.body.json +8 -8
- package/build/routes/chromium/http/screenshot.post.body.json +8 -8
- package/build/routes/firefox/ws/playwright.d.ts +2 -2
- package/build/routes/firefox/ws/playwright.js +1 -1
- package/build/routes/management/http/static.get.d.ts +2 -2
- package/build/routes/management/http/static.get.js +8 -10
- package/build/routes/management/tests/management.spec.js +9 -0
- package/build/routes/webkit/ws/playwright.d.ts +2 -2
- package/build/routes/webkit/ws/playwright.js +1 -1
- package/build/sdk-utils.js +23 -10
- package/build/server.d.ts +4 -5
- package/build/server.js +38 -33
- package/build/shared/browser.ws.d.ts +2 -2
- package/build/shared/browser.ws.js +1 -1
- package/build/shared/chromium.playwright.ws.d.ts +2 -2
- package/build/shared/chromium.playwright.ws.js +1 -1
- package/build/shared/chromium.ws.d.ts +2 -2
- package/build/shared/chromium.ws.js +1 -1
- package/build/shared/content.http.d.ts +2 -2
- package/build/shared/content.http.js +4 -1
- package/build/shared/download.http.d.ts +2 -2
- package/build/shared/download.http.js +11 -12
- package/build/shared/function.http.d.ts +2 -2
- package/build/shared/function.http.js +4 -5
- package/build/shared/json-protocol.http.d.ts +3 -3
- package/build/shared/json-protocol.http.js +2 -2
- package/build/shared/json-version.http.d.ts +3 -3
- package/build/shared/json-version.http.js +2 -2
- package/build/shared/page.ws.d.ts +2 -2
- package/build/shared/page.ws.js +1 -1
- package/build/shared/pdf.http.d.ts +2 -2
- package/build/shared/pdf.http.js +6 -4
- package/build/shared/performance.http.d.ts +2 -2
- package/build/shared/performance.http.js +2 -1
- package/build/shared/scrape.http.d.ts +2 -2
- package/build/shared/scrape.http.js +4 -1
- package/build/shared/screenshot.http.d.ts +2 -2
- package/build/shared/screenshot.http.js +4 -1
- package/build/shared/utils/function/handler.d.ts +2 -3
- package/build/shared/utils/function/handler.js +8 -8
- package/build/shared/utils/performance/child.js +4 -4
- package/build/shared/utils/performance/main.d.ts +1 -1
- package/build/shared/utils/performance/main.js +5 -7
- package/build/shared/utils/performance/types.d.ts +2 -1
- package/build/types.d.ts +6 -15
- package/build/types.js +1 -10
- package/build/utils.d.ts +1 -1
- package/build/utils.js +6 -2
- package/package.json +18 -15
- package/scripts/install-debugger.js +55 -15
- package/src/browserless.ts +29 -21
- package/src/browsers/chrome.cdp.ts +2 -5
- package/src/browsers/chrome.playwright.ts +2 -5
- package/src/browsers/chromium.cdp.ts +84 -35
- package/src/browsers/chromium.playwright.ts +26 -13
- package/src/browsers/firefox.playwright.ts +28 -13
- package/src/browsers/index.ts +24 -16
- package/src/browsers/webkit.playwright.ts +28 -13
- package/src/config.ts +4 -0
- package/src/exports.ts +1 -0
- package/src/file-system.ts +2 -7
- package/src/index.ts +7 -7
- package/src/limiter.ts +13 -11
- package/src/logger.ts +52 -0
- package/src/monitoring.ts +6 -8
- package/src/router.ts +29 -27
- package/src/routes/firefox/ws/playwright.ts +2 -0
- package/src/routes/management/http/static.get.ts +13 -10
- package/src/routes/management/tests/management.spec.ts +15 -0
- package/src/routes/webkit/ws/playwright.ts +2 -0
- package/src/sdk-utils.ts +20 -2
- package/src/server.ts +47 -32
- package/src/shared/browser.ws.ts +2 -0
- package/src/shared/chromium.playwright.ws.ts +2 -0
- package/src/shared/chromium.ws.ts +2 -0
- package/src/shared/content.http.ts +6 -0
- package/src/shared/download.http.ts +14 -11
- package/src/shared/function.http.ts +5 -4
- package/src/shared/json-protocol.http.ts +8 -3
- package/src/shared/json-version.http.ts +8 -4
- package/src/shared/page.ws.ts +2 -0
- package/src/shared/pdf.http.ts +7 -3
- package/src/shared/performance.http.ts +3 -0
- package/src/shared/scrape.http.ts +6 -0
- package/src/shared/screenshot.http.ts +5 -0
- package/src/shared/utils/function/handler.ts +9 -13
- package/src/shared/utils/performance/child.ts +4 -4
- package/src/shared/utils/performance/main.ts +5 -6
- package/src/shared/utils/performance/types.ts +2 -1
- package/src/types.ts +5 -9
- package/src/utils.ts +7 -2
- package/static/docs/swagger.json +11 -11
- package/static/docs/swagger.min.json +10 -10
- package/static/function/client.js +1656 -2916
- package/static/function/index.html +1656 -2916
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ServerError,
|
|
1
|
+
import { ServerError, } from '@browserless.io/browserless';
|
|
2
2
|
import playwright from 'playwright-core';
|
|
3
3
|
import { EventEmitter } from 'events';
|
|
4
4
|
import httpProxy from 'http-proxy';
|
|
@@ -10,12 +10,13 @@ export class WebkitPlaywright extends EventEmitter {
|
|
|
10
10
|
proxy = httpProxy.createProxyServer();
|
|
11
11
|
browser = null;
|
|
12
12
|
browserWSEndpoint = null;
|
|
13
|
-
|
|
14
|
-
constructor({ config, userDataDir, }) {
|
|
13
|
+
logger;
|
|
14
|
+
constructor({ config, userDataDir, logger, }) {
|
|
15
15
|
super();
|
|
16
16
|
this.userDataDir = userDataDir;
|
|
17
17
|
this.config = config;
|
|
18
|
-
this.
|
|
18
|
+
this.logger = logger;
|
|
19
|
+
this.logger.info(`Starting new ${this.constructor.name} instance`);
|
|
19
20
|
}
|
|
20
21
|
cleanListeners() {
|
|
21
22
|
this.removeAllListeners();
|
|
@@ -23,7 +24,7 @@ export class WebkitPlaywright extends EventEmitter {
|
|
|
23
24
|
isRunning = () => this.running;
|
|
24
25
|
close = async () => {
|
|
25
26
|
if (this.browser) {
|
|
26
|
-
this.
|
|
27
|
+
this.logger.info(`Closing ${this.constructor.name} process and all listeners`);
|
|
27
28
|
this.emit('close');
|
|
28
29
|
this.cleanListeners();
|
|
29
30
|
this.browser.close();
|
|
@@ -34,16 +35,16 @@ export class WebkitPlaywright extends EventEmitter {
|
|
|
34
35
|
};
|
|
35
36
|
pages = async () => [];
|
|
36
37
|
getPageId = () => {
|
|
37
|
-
throw new ServerError(`#getPageId is not yet supported with this
|
|
38
|
+
throw new ServerError(`#getPageId is not yet supported with ${this.constructor.name}.`);
|
|
38
39
|
};
|
|
39
40
|
makeLiveURL = () => {
|
|
40
|
-
throw new ServerError(`Live URLs are not yet supported with this
|
|
41
|
+
throw new ServerError(`Live URLs are not yet supported with ${this.constructor.name}.`);
|
|
41
42
|
};
|
|
42
43
|
newPage = async () => {
|
|
43
|
-
throw new ServerError(`Can't create new page with this
|
|
44
|
+
throw new ServerError(`Can't create new page with ${this.constructor.name}`);
|
|
44
45
|
};
|
|
45
46
|
launch = async (options = {}) => {
|
|
46
|
-
this.
|
|
47
|
+
this.logger.info(`Launching ${this.constructor.name} Handler`);
|
|
47
48
|
this.browser = await playwright.webkit.launchServer({
|
|
48
49
|
...options,
|
|
49
50
|
args: [
|
|
@@ -53,7 +54,7 @@ export class WebkitPlaywright extends EventEmitter {
|
|
|
53
54
|
executablePath: playwright.webkit.executablePath(),
|
|
54
55
|
});
|
|
55
56
|
const browserWSEndpoint = this.browser.wsEndpoint();
|
|
56
|
-
this.
|
|
57
|
+
this.logger.info(`${this.constructor.name} is running on ${browserWSEndpoint}`);
|
|
57
58
|
this.browserWSEndpoint = browserWSEndpoint;
|
|
58
59
|
this.running = true;
|
|
59
60
|
return this.browser;
|
|
@@ -72,14 +73,14 @@ export class WebkitPlaywright extends EventEmitter {
|
|
|
72
73
|
return externalURL.href;
|
|
73
74
|
};
|
|
74
75
|
proxyPageWebSocket = async () => {
|
|
75
|
-
|
|
76
|
+
this.logger.warn(`Not yet implemented`);
|
|
76
77
|
};
|
|
77
78
|
proxyWebSocket = async (req, socket, head) => new Promise((resolve, reject) => {
|
|
78
79
|
if (!this.browserWSEndpoint) {
|
|
79
80
|
throw new ServerError(`No browserWSEndpoint found, did you launch first?`);
|
|
80
81
|
}
|
|
81
82
|
socket.once('close', resolve);
|
|
82
|
-
this.
|
|
83
|
+
this.logger.info(`Proxying ${req.parsed.href} to ${this.constructor.name} ${this.browserWSEndpoint}`);
|
|
83
84
|
// Delete headers known to cause issues
|
|
84
85
|
delete req.headers.origin;
|
|
85
86
|
req.url = '';
|
|
@@ -87,7 +88,7 @@ export class WebkitPlaywright extends EventEmitter {
|
|
|
87
88
|
changeOrigin: true,
|
|
88
89
|
target: this.browserWSEndpoint,
|
|
89
90
|
}, (error) => {
|
|
90
|
-
this.
|
|
91
|
+
this.logger.error(`Error proxying session to ${this.constructor.name}: ${error}`);
|
|
91
92
|
this.close();
|
|
92
93
|
return reject(error);
|
|
93
94
|
});
|
package/build/config.d.ts
CHANGED
|
@@ -18,6 +18,7 @@ export declare class Config extends EventEmitter {
|
|
|
18
18
|
protected queued: number;
|
|
19
19
|
protected timeout: number;
|
|
20
20
|
protected static: string;
|
|
21
|
+
protected debuggerDir: string;
|
|
21
22
|
protected retries: number;
|
|
22
23
|
protected allowFileProtocol: boolean;
|
|
23
24
|
protected allowGet: boolean;
|
|
@@ -53,6 +54,7 @@ export declare class Config extends EventEmitter {
|
|
|
53
54
|
getQueued: () => number;
|
|
54
55
|
getTimeout: () => number;
|
|
55
56
|
getStatic: () => string;
|
|
57
|
+
getDebuggerDir: () => string;
|
|
56
58
|
getRetries: () => number;
|
|
57
59
|
getAllowFileProtocol: () => boolean;
|
|
58
60
|
getCPULimit: () => number;
|
|
@@ -63,6 +65,7 @@ export declare class Config extends EventEmitter {
|
|
|
63
65
|
getRejectAlertURL: () => string | null;
|
|
64
66
|
getTimeoutAlertURL: () => string | null;
|
|
65
67
|
getErrorAlertURL: () => string | null;
|
|
68
|
+
hasDebugger: () => Promise<boolean>;
|
|
66
69
|
/**
|
|
67
70
|
* If true, allows GET style calls on our browser-based APIs, using
|
|
68
71
|
* ?body=JSON format.
|
package/build/config.js
CHANGED
|
@@ -124,6 +124,7 @@ export class Config extends EventEmitter {
|
|
|
124
124
|
process.env.CONNECTION_TIMEOUT ??
|
|
125
125
|
'30000');
|
|
126
126
|
static = process.env.STATIC ?? path.join(__dirname, '..', 'static');
|
|
127
|
+
debuggerDir = path.join(this.static, 'debugger');
|
|
127
128
|
retries = +(process.env.RETRIES ?? '5');
|
|
128
129
|
allowFileProtocol = !!parseEnvVars(false, 'ALLOW_FILE_PROTOCOL');
|
|
129
130
|
allowGet = !!parseEnvVars(false, 'ALLOW_GET', 'ENABLE_API_GET');
|
|
@@ -159,6 +160,7 @@ export class Config extends EventEmitter {
|
|
|
159
160
|
getQueued = () => this.queued;
|
|
160
161
|
getTimeout = () => this.timeout;
|
|
161
162
|
getStatic = () => this.static;
|
|
163
|
+
getDebuggerDir = () => this.debuggerDir;
|
|
162
164
|
getRetries = () => this.retries;
|
|
163
165
|
getAllowFileProtocol = () => this.allowFileProtocol;
|
|
164
166
|
getCPULimit = () => this.maxCpu;
|
|
@@ -169,6 +171,7 @@ export class Config extends EventEmitter {
|
|
|
169
171
|
getRejectAlertURL = () => this.rejectAlertURL;
|
|
170
172
|
getTimeoutAlertURL = () => this.timeoutAlertURL;
|
|
171
173
|
getErrorAlertURL = () => this.errorAlertURL;
|
|
174
|
+
hasDebugger = () => exists(this.debuggerDir);
|
|
172
175
|
/**
|
|
173
176
|
* If true, allows GET style calls on our browser-based APIs, using
|
|
174
177
|
* ?body=JSON format.
|
package/build/exports.d.ts
CHANGED
package/build/exports.js
CHANGED
package/build/file-system.d.ts
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
/// <reference types="debug" />
|
|
3
2
|
/// <reference types="node" />
|
|
4
|
-
import { Config } from '@browserless.io/browserless';
|
|
3
|
+
import { Config, Logger } from '@browserless.io/browserless';
|
|
5
4
|
import { EventEmitter } from 'events';
|
|
6
5
|
export declare class FileSystem extends EventEmitter {
|
|
7
6
|
protected config: Config;
|
|
8
7
|
protected fsMap: Map<string, string[]>;
|
|
9
8
|
protected currentAESKey: Buffer;
|
|
10
|
-
protected
|
|
9
|
+
protected logger: Logger;
|
|
11
10
|
constructor(config: Config);
|
|
12
11
|
/**
|
|
13
12
|
* Appends contents to a file-path for persistance. File contents are
|
package/build/file-system.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Logger, decrypt, encrypt } from '@browserless.io/browserless';
|
|
2
2
|
import { readFile, writeFile } from 'fs/promises';
|
|
3
3
|
import { EventEmitter } from 'events';
|
|
4
4
|
export class FileSystem extends EventEmitter {
|
|
5
5
|
config;
|
|
6
6
|
fsMap = new Map();
|
|
7
7
|
currentAESKey;
|
|
8
|
-
|
|
8
|
+
logger = new Logger('file-system');
|
|
9
9
|
constructor(config) {
|
|
10
10
|
super();
|
|
11
11
|
this.config = config;
|
package/build/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { Browserless,
|
|
1
|
+
import { Browserless, Logger } from '@browserless.io/browserless';
|
|
2
2
|
(async () => {
|
|
3
3
|
const browserless = new Browserless();
|
|
4
|
-
const
|
|
4
|
+
const logger = new Logger('index.js');
|
|
5
5
|
browserless.start();
|
|
6
6
|
process
|
|
7
7
|
.on('unhandledRejection', async (reason, promise) => {
|
|
@@ -13,27 +13,27 @@ import { Browserless, createLogger } from '@browserless.io/browserless';
|
|
|
13
13
|
process.exit(1);
|
|
14
14
|
})
|
|
15
15
|
.once('SIGTERM', async () => {
|
|
16
|
-
|
|
16
|
+
logger.info(`SIGTERM received, saving and closing down`);
|
|
17
17
|
await browserless.stop();
|
|
18
18
|
process.exit(0);
|
|
19
19
|
})
|
|
20
20
|
.once('SIGINT', async () => {
|
|
21
|
-
|
|
21
|
+
logger.info(`SIGINT received, saving and closing down`);
|
|
22
22
|
await browserless.stop();
|
|
23
23
|
process.exit(0);
|
|
24
24
|
})
|
|
25
25
|
.once('SIGHUP', async () => {
|
|
26
|
-
|
|
26
|
+
logger.info(`SIGHUP received, saving and closing down`);
|
|
27
27
|
await browserless.stop();
|
|
28
28
|
process.exit(0);
|
|
29
29
|
})
|
|
30
30
|
.once('SIGUSR2', async () => {
|
|
31
|
-
|
|
31
|
+
logger.info(`SIGUSR2 received, saving and closing down`);
|
|
32
32
|
await browserless.stop();
|
|
33
33
|
process.exit(0);
|
|
34
34
|
})
|
|
35
35
|
.once('exit', () => {
|
|
36
|
-
|
|
36
|
+
logger.info(`Process is finished, exiting`);
|
|
37
37
|
process.exit(0);
|
|
38
38
|
});
|
|
39
39
|
})();
|
package/build/limiter.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
import { AfterResponse, Config, Hooks, Metrics, Monitoring, WebHooks } from '@browserless.io/browserless';
|
|
1
|
+
import { AfterResponse, Config, Hooks, Logger, Metrics, Monitoring, WebHooks } from '@browserless.io/browserless';
|
|
3
2
|
import q from 'queue';
|
|
4
3
|
export type LimitFn<TArgs extends unknown[], TResult> = (...args: TArgs) => Promise<TResult>;
|
|
5
4
|
export type ErrorFn<TArgs extends unknown[]> = (...args: TArgs) => void;
|
|
@@ -17,7 +16,7 @@ export declare class Limiter extends q {
|
|
|
17
16
|
protected webhooks: WebHooks;
|
|
18
17
|
protected hooks: Hooks;
|
|
19
18
|
protected queued: number;
|
|
20
|
-
protected
|
|
19
|
+
protected logger: Logger;
|
|
21
20
|
constructor(config: Config, metrics: Metrics, monitor: Monitoring, webhooks: WebHooks, hooks: Hooks);
|
|
22
21
|
protected handleEnd(): void;
|
|
23
22
|
protected jobEnd(jobInfo: AfterResponse): void;
|
package/build/limiter.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Logger, TooManyRequests, } from '@browserless.io/browserless';
|
|
2
2
|
import q from 'queue';
|
|
3
3
|
export class Limiter extends q {
|
|
4
4
|
config;
|
|
@@ -7,7 +7,7 @@ export class Limiter extends q {
|
|
|
7
7
|
webhooks;
|
|
8
8
|
hooks;
|
|
9
9
|
queued;
|
|
10
|
-
|
|
10
|
+
logger = new Logger('limiter');
|
|
11
11
|
constructor(config, metrics, monitor, webhooks, hooks) {
|
|
12
12
|
super({
|
|
13
13
|
autostart: true,
|
|
@@ -20,17 +20,17 @@ export class Limiter extends q {
|
|
|
20
20
|
this.webhooks = webhooks;
|
|
21
21
|
this.hooks = hooks;
|
|
22
22
|
this.queued = config.getQueued();
|
|
23
|
-
this.
|
|
23
|
+
this.logger.info(`Concurrency: ${this.concurrency} queue: ${this.queued} timeout: ${this.timeout}ms`);
|
|
24
24
|
config.on('concurrent', (concurrency) => {
|
|
25
|
-
this.
|
|
25
|
+
this.logger.info(`Concurrency updated to ${concurrency}`);
|
|
26
26
|
this.concurrency = concurrency;
|
|
27
27
|
});
|
|
28
28
|
config.on('queued', (queued) => {
|
|
29
|
-
this.
|
|
29
|
+
this.logger.info(`Queue updated to ${queued}`);
|
|
30
30
|
this.queued = queued;
|
|
31
31
|
});
|
|
32
32
|
config.on('timeout', (timeout) => {
|
|
33
|
-
this.
|
|
33
|
+
this.logger.info(`Timeout updated to ${timeout}ms`);
|
|
34
34
|
this.timeout = timeout <= 0 ? 0 : timeout;
|
|
35
35
|
});
|
|
36
36
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -49,7 +49,7 @@ export class Limiter extends q {
|
|
|
49
49
|
}
|
|
50
50
|
handleSuccess({ detail: { job } }) {
|
|
51
51
|
const timeUsed = Date.now() - job.start;
|
|
52
|
-
this.
|
|
52
|
+
this.logger.info(`Job has succeeded after ${timeUsed.toLocaleString()}ms of activity.`);
|
|
53
53
|
this.metrics.addSuccessful(Date.now() - job.start);
|
|
54
54
|
// @TODO Figure out a better argument handling for jobs
|
|
55
55
|
this.jobEnd({
|
|
@@ -60,10 +60,10 @@ export class Limiter extends q {
|
|
|
60
60
|
}
|
|
61
61
|
handleJobTimeout({ detail: { next, job }, }) {
|
|
62
62
|
const timeUsed = Date.now() - job.start;
|
|
63
|
-
this.
|
|
63
|
+
this.logger.warn(`Job has hit timeout after ${timeUsed.toLocaleString()}ms of activity.`);
|
|
64
64
|
this.metrics.addTimedout(Date.now() - job.start);
|
|
65
65
|
this.webhooks.callTimeoutAlertURL();
|
|
66
|
-
this.
|
|
66
|
+
this.logger.info(`Calling timeout handler`);
|
|
67
67
|
job?.onTimeoutFn(job);
|
|
68
68
|
this.jobEnd({
|
|
69
69
|
req: job.args[0],
|
|
@@ -73,7 +73,7 @@ export class Limiter extends q {
|
|
|
73
73
|
next();
|
|
74
74
|
}
|
|
75
75
|
handleFail({ detail: { error, job }, }) {
|
|
76
|
-
this.
|
|
76
|
+
this.logger.info(`Recording failed stat, cleaning up: "${error?.toString()}"`);
|
|
77
77
|
this.metrics.addError(Date.now() - job.start);
|
|
78
78
|
this.webhooks.callErrorAlertURL(error?.toString() ?? 'Unknown Error');
|
|
79
79
|
this.jobEnd({
|
|
@@ -83,7 +83,7 @@ export class Limiter extends q {
|
|
|
83
83
|
});
|
|
84
84
|
}
|
|
85
85
|
logQueue(message) {
|
|
86
|
-
this.
|
|
86
|
+
this.logger.info(`(Running: ${this.executing}, Pending: ${this.waiting}) ${message} `);
|
|
87
87
|
}
|
|
88
88
|
get executing() {
|
|
89
89
|
return this.length > this.concurrency ? this.concurrency : this.length;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { Request } from '@browserless.io/browserless';
|
|
2
|
+
export declare class Logger {
|
|
3
|
+
protected prefix: string;
|
|
4
|
+
protected request?: Request | undefined;
|
|
5
|
+
protected _trace: (...args: unknown[]) => void;
|
|
6
|
+
protected _debug: (...args: unknown[]) => void;
|
|
7
|
+
protected _info: (...args: unknown[]) => void;
|
|
8
|
+
protected _warn: (...args: unknown[]) => void;
|
|
9
|
+
protected _error: (...args: unknown[]) => void;
|
|
10
|
+
protected _fatal: (...args: unknown[]) => void;
|
|
11
|
+
constructor(prefix: string, request?: Request | undefined);
|
|
12
|
+
protected get reqInfo(): string;
|
|
13
|
+
trace: (...messages: unknown[]) => void;
|
|
14
|
+
debug: (...messages: unknown[]) => void;
|
|
15
|
+
info: (...messages: unknown[]) => void;
|
|
16
|
+
warn: (...messages: unknown[]) => void;
|
|
17
|
+
error: (...messages: unknown[]) => void;
|
|
18
|
+
fatal: (...messages: unknown[]) => void;
|
|
19
|
+
}
|
package/build/logger.js
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { createLogger } from '@browserless.io/browserless';
|
|
2
|
+
export class Logger {
|
|
3
|
+
prefix;
|
|
4
|
+
request;
|
|
5
|
+
_trace;
|
|
6
|
+
_debug;
|
|
7
|
+
_info;
|
|
8
|
+
_warn;
|
|
9
|
+
_error;
|
|
10
|
+
_fatal;
|
|
11
|
+
constructor(prefix, request) {
|
|
12
|
+
this.prefix = prefix;
|
|
13
|
+
this.request = request;
|
|
14
|
+
const logger = createLogger(prefix);
|
|
15
|
+
this._trace = logger.extend('trace');
|
|
16
|
+
this._debug = logger.extend('debug');
|
|
17
|
+
this._info = logger.extend('info');
|
|
18
|
+
this._warn = logger.extend('warn');
|
|
19
|
+
this._error = logger.extend('error');
|
|
20
|
+
this._fatal = logger.extend('fatal');
|
|
21
|
+
}
|
|
22
|
+
get reqInfo() {
|
|
23
|
+
return this.request ? this.request.socket.remoteAddress ?? 'Unknown' : '';
|
|
24
|
+
}
|
|
25
|
+
trace = (...messages) => {
|
|
26
|
+
this._trace(this.reqInfo, ...messages);
|
|
27
|
+
};
|
|
28
|
+
debug = (...messages) => {
|
|
29
|
+
this._debug(this.reqInfo, ...messages);
|
|
30
|
+
};
|
|
31
|
+
info = (...messages) => {
|
|
32
|
+
this._info(this.reqInfo, ...messages);
|
|
33
|
+
};
|
|
34
|
+
warn = (...messages) => {
|
|
35
|
+
this._warn(this.reqInfo, ...messages);
|
|
36
|
+
};
|
|
37
|
+
error = (...messages) => {
|
|
38
|
+
this._error(this.reqInfo, ...messages);
|
|
39
|
+
};
|
|
40
|
+
fatal = (...messages) => {
|
|
41
|
+
this._fatal(this.reqInfo, ...messages);
|
|
42
|
+
};
|
|
43
|
+
}
|
package/build/monitoring.d.ts
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
/// <reference types="debug" />
|
|
2
1
|
/// <reference types="node" />
|
|
3
|
-
import { Config, IResourceLoad } from '@browserless.io/browserless';
|
|
2
|
+
import { Config, IResourceLoad, Logger } from '@browserless.io/browserless';
|
|
4
3
|
import { EventEmitter } from 'events';
|
|
5
4
|
export declare class Monitoring extends EventEmitter {
|
|
6
5
|
protected config: Config;
|
|
7
|
-
protected log:
|
|
6
|
+
protected log: Logger;
|
|
8
7
|
constructor(config: Config);
|
|
9
8
|
getMachineStats: () => Promise<IResourceLoad>;
|
|
10
9
|
overloaded: () => Promise<{
|
package/build/monitoring.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Logger } from '@browserless.io/browserless';
|
|
2
2
|
import { EventEmitter } from 'events';
|
|
3
3
|
import si from 'systeminformation';
|
|
4
4
|
export class Monitoring extends EventEmitter {
|
|
5
5
|
config;
|
|
6
|
-
log =
|
|
6
|
+
log = new Logger('hardware');
|
|
7
7
|
constructor(config) {
|
|
8
8
|
super();
|
|
9
9
|
this.config = config;
|
|
@@ -13,7 +13,7 @@ export class Monitoring extends EventEmitter {
|
|
|
13
13
|
si.currentLoad(),
|
|
14
14
|
si.mem(),
|
|
15
15
|
]).catch((err) => {
|
|
16
|
-
this.log(`Error checking machine stats`, err);
|
|
16
|
+
this.log.error(`Error checking machine stats`, err);
|
|
17
17
|
return [null, null];
|
|
18
18
|
});
|
|
19
19
|
const cpu = cpuLoad ? cpuLoad.currentLoadUser / 100 : null;
|
|
@@ -27,7 +27,7 @@ export class Monitoring extends EventEmitter {
|
|
|
27
27
|
const { cpu, memory } = await this.getMachineStats();
|
|
28
28
|
const cpuInt = cpu && Math.ceil(cpu * 100);
|
|
29
29
|
const memoryInt = memory && Math.ceil(memory * 100);
|
|
30
|
-
this.log(`Checking overload status: CPU ${cpuInt}% Memory ${memoryInt}%`);
|
|
30
|
+
this.log.info(`Checking overload status: CPU ${cpuInt}% Memory ${memoryInt}%`);
|
|
31
31
|
const cpuOverloaded = !!(cpuInt && cpuInt >= this.config.getCPULimit());
|
|
32
32
|
const memoryOverloaded = !!(memoryInt && memoryInt >= this.config.getMemoryLimit());
|
|
33
33
|
return {
|
package/build/router.d.ts
CHANGED
|
@@ -1,19 +1,18 @@
|
|
|
1
|
-
/// <reference types="debug" />
|
|
2
1
|
/// <reference types="node" />
|
|
3
2
|
/// <reference types="node" />
|
|
4
3
|
/// <reference types="node" />
|
|
5
|
-
import { BrowserHTTPRoute, BrowserManager, BrowserWebsocketRoute, Config, HTTPRoute, Limiter, Request, Response, WebSocketRoute } from '@browserless.io/browserless';
|
|
4
|
+
import { BrowserHTTPRoute, BrowserManager, BrowserWebsocketRoute, Config, HTTPRoute, Limiter, Logger, Request, Response, WebSocketRoute } from '@browserless.io/browserless';
|
|
6
5
|
import { EventEmitter } from 'events';
|
|
7
6
|
import stream from 'stream';
|
|
8
7
|
export declare class Router extends EventEmitter {
|
|
9
8
|
protected config: Config;
|
|
10
9
|
protected browserManager: BrowserManager;
|
|
11
10
|
protected limiter: Limiter;
|
|
12
|
-
protected
|
|
13
|
-
protected
|
|
11
|
+
protected logger: typeof Logger;
|
|
12
|
+
protected log: Logger;
|
|
14
13
|
protected httpRoutes: Array<HTTPRoute | BrowserHTTPRoute>;
|
|
15
14
|
protected webSocketRoutes: Array<WebSocketRoute | BrowserWebsocketRoute>;
|
|
16
|
-
constructor(config: Config, browserManager: BrowserManager, limiter: Limiter);
|
|
15
|
+
constructor(config: Config, browserManager: BrowserManager, limiter: Limiter, logger: typeof Logger);
|
|
17
16
|
protected getTimeout(req: Request): number | undefined;
|
|
18
17
|
protected onQueueFullHTTP: (_req: Request, res: Response) => void;
|
|
19
18
|
protected onQueueFullWebSocket: (_req: Request, socket: stream.Duplex) => void;
|
package/build/router.js
CHANGED
|
@@ -1,49 +1,51 @@
|
|
|
1
|
-
import { HTTPManagementRoutes,
|
|
1
|
+
import { HTTPManagementRoutes, Logger, contentTypes, isConnected, writeResponse, } from '@browserless.io/browserless';
|
|
2
2
|
import { EventEmitter } from 'events';
|
|
3
3
|
import micromatch from 'micromatch';
|
|
4
4
|
export class Router extends EventEmitter {
|
|
5
5
|
config;
|
|
6
6
|
browserManager;
|
|
7
7
|
limiter;
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
logger;
|
|
9
|
+
log = new Logger('router');
|
|
10
10
|
httpRoutes = [];
|
|
11
11
|
webSocketRoutes = [];
|
|
12
|
-
constructor(config, browserManager, limiter) {
|
|
12
|
+
constructor(config, browserManager, limiter, logger) {
|
|
13
13
|
super();
|
|
14
14
|
this.config = config;
|
|
15
15
|
this.browserManager = browserManager;
|
|
16
16
|
this.limiter = limiter;
|
|
17
|
+
this.logger = logger;
|
|
17
18
|
}
|
|
18
19
|
getTimeout(req) {
|
|
19
20
|
const timer = req.parsed.searchParams.get('timeout');
|
|
20
21
|
return timer ? +timer : undefined;
|
|
21
22
|
}
|
|
22
23
|
onQueueFullHTTP = (_req, res) => {
|
|
23
|
-
this.log(`Queue is full, sending 429 response`);
|
|
24
|
+
this.log.warn(`Queue is full, sending 429 response`);
|
|
24
25
|
return writeResponse(res, 429, 'Too many requests');
|
|
25
26
|
};
|
|
26
27
|
onQueueFullWebSocket = (_req, socket) => {
|
|
27
|
-
this.log(`Queue is full, sending 429 response`);
|
|
28
|
+
this.log.warn(`Queue is full, sending 429 response`);
|
|
28
29
|
return writeResponse(socket, 429, 'Too many requests');
|
|
29
30
|
};
|
|
30
31
|
onHTTPTimeout = (_req, res) => {
|
|
31
|
-
this.log(`HTTP job has timedout, sending 429 response`);
|
|
32
|
+
this.log.error(`HTTP job has timedout, sending 429 response`);
|
|
32
33
|
return writeResponse(res, 408, 'Request has timed out');
|
|
33
34
|
};
|
|
34
35
|
onWebsocketTimeout = (_req, socket) => {
|
|
35
|
-
this.log(`Websocket job has timedout, sending 429 response`);
|
|
36
|
+
this.log.error(`Websocket job has timedout, sending 429 response`);
|
|
36
37
|
return writeResponse(socket, 408, 'Request has timed out');
|
|
37
38
|
};
|
|
38
39
|
wrapHTTPHandler = (route, handler) => async (req, res) => {
|
|
39
40
|
if (!isConnected(res)) {
|
|
40
|
-
this.log(`HTTP Request has closed prior to running`);
|
|
41
|
+
this.log.warn(`HTTP Request has closed prior to running`);
|
|
41
42
|
return Promise.resolve();
|
|
42
43
|
}
|
|
44
|
+
const logger = new this.logger(route.name, req);
|
|
43
45
|
if ('browser' in route && route.browser) {
|
|
44
|
-
const browser = await this.browserManager.getBrowserForRequest(req, route);
|
|
46
|
+
const browser = await this.browserManager.getBrowserForRequest(req, route, logger);
|
|
45
47
|
if (!isConnected(res)) {
|
|
46
|
-
this.log(`HTTP Request has closed prior to running`);
|
|
48
|
+
this.log.warn(`HTTP Request has closed prior to running`);
|
|
47
49
|
this.browserManager.complete(browser);
|
|
48
50
|
return Promise.resolve();
|
|
49
51
|
}
|
|
@@ -51,36 +53,37 @@ export class Router extends EventEmitter {
|
|
|
51
53
|
return writeResponse(res, 500, `Error loading the browser`);
|
|
52
54
|
}
|
|
53
55
|
try {
|
|
54
|
-
this.
|
|
56
|
+
this.log.trace(`Running found HTTP handler.`);
|
|
55
57
|
return await Promise.race([
|
|
56
|
-
handler(req, res, browser),
|
|
58
|
+
handler(req, res, logger, browser),
|
|
57
59
|
new Promise((resolve, reject) => {
|
|
58
60
|
res.once('close', () => {
|
|
59
61
|
if (!res.writableEnded) {
|
|
60
62
|
reject(new Error(`Request closed prior to writing results`));
|
|
61
63
|
}
|
|
62
|
-
this.
|
|
64
|
+
this.log.trace(`Response has been written, resolving`);
|
|
63
65
|
resolve(null);
|
|
64
66
|
});
|
|
65
67
|
}),
|
|
66
68
|
]);
|
|
67
69
|
}
|
|
68
70
|
finally {
|
|
69
|
-
this.
|
|
71
|
+
this.log.trace(`HTTP Request handler has finished.`);
|
|
70
72
|
this.browserManager.complete(browser);
|
|
71
73
|
}
|
|
72
74
|
}
|
|
73
|
-
return handler(req, res);
|
|
75
|
+
return handler(req, res, logger);
|
|
74
76
|
};
|
|
75
77
|
wrapWebSocketHandler = (route, handler) => async (req, socket, head) => {
|
|
76
78
|
if (!isConnected(socket)) {
|
|
77
|
-
this.log(`WebSocket Request has closed prior to running`);
|
|
79
|
+
this.log.warn(`WebSocket Request has closed prior to running`);
|
|
78
80
|
return Promise.resolve();
|
|
79
81
|
}
|
|
82
|
+
const logger = new this.logger(route.name, req);
|
|
80
83
|
if ('browser' in route && route.browser) {
|
|
81
|
-
const browser = await this.browserManager.getBrowserForRequest(req, route);
|
|
84
|
+
const browser = await this.browserManager.getBrowserForRequest(req, route, logger);
|
|
82
85
|
if (!isConnected(socket)) {
|
|
83
|
-
this.log(`WebSocket Request has closed prior to running`);
|
|
86
|
+
this.log.warn(`WebSocket Request has closed prior to running`);
|
|
84
87
|
this.browserManager.complete(browser);
|
|
85
88
|
return Promise.resolve();
|
|
86
89
|
}
|
|
@@ -88,19 +91,19 @@ export class Router extends EventEmitter {
|
|
|
88
91
|
return writeResponse(socket, 500, `Error loading the browser.`);
|
|
89
92
|
}
|
|
90
93
|
try {
|
|
91
|
-
this.
|
|
92
|
-
await handler(req, socket, head, browser);
|
|
94
|
+
this.log.trace(`Running found WebSocket handler.`);
|
|
95
|
+
await handler(req, socket, head, logger, browser);
|
|
93
96
|
}
|
|
94
97
|
finally {
|
|
95
|
-
this.
|
|
98
|
+
this.log.trace(`WebSocket Request handler has finished.`);
|
|
96
99
|
this.browserManager.complete(browser);
|
|
97
100
|
}
|
|
98
101
|
return;
|
|
99
102
|
}
|
|
100
|
-
return handler(req, socket, head);
|
|
103
|
+
return handler(req, socket, head, logger);
|
|
101
104
|
};
|
|
102
105
|
registerHTTPRoute(route) {
|
|
103
|
-
this.
|
|
106
|
+
this.log.trace(`Registering HTTP ${route.method.toUpperCase()} ${route.path}`);
|
|
104
107
|
const bound = route.handler.bind(route);
|
|
105
108
|
const wrapped = this.wrapHTTPHandler(route, bound);
|
|
106
109
|
route.handler = route.concurrency
|
|
@@ -110,13 +113,13 @@ export class Router extends EventEmitter {
|
|
|
110
113
|
const registeredPaths = this.httpRoutes.map((r) => r.path).flat();
|
|
111
114
|
const duplicatePaths = registeredPaths.filter((path) => route.path.includes(path));
|
|
112
115
|
if (duplicatePaths.length) {
|
|
113
|
-
this.log(`Found duplicate routes: ${duplicatePaths.join(', ')}`);
|
|
116
|
+
this.log.warn(`Found duplicate routes: ${duplicatePaths.join(', ')}`);
|
|
114
117
|
}
|
|
115
118
|
this.httpRoutes.push(route);
|
|
116
119
|
return route;
|
|
117
120
|
}
|
|
118
121
|
registerWebSocketRoute(route) {
|
|
119
|
-
this.
|
|
122
|
+
this.log.trace(`Registering WebSocket "${route.path}"`);
|
|
120
123
|
const bound = route.handler.bind(route);
|
|
121
124
|
const wrapped = this.wrapWebSocketHandler(route, bound);
|
|
122
125
|
route.handler = route.concurrency
|
|
@@ -126,7 +129,7 @@ export class Router extends EventEmitter {
|
|
|
126
129
|
const registeredPaths = this.webSocketRoutes.map((r) => r.path).flat();
|
|
127
130
|
const duplicatePaths = registeredPaths.filter((path) => route.path.includes(path));
|
|
128
131
|
if (duplicatePaths.length) {
|
|
129
|
-
this.log(`Found duplicate routes: ${duplicatePaths.join(', ')}`);
|
|
132
|
+
this.log.warn(`Found duplicate routes: ${duplicatePaths.join(', ')}`);
|
|
130
133
|
}
|
|
131
134
|
this.webSocketRoutes.push(route);
|
|
132
135
|
return route;
|
|
@@ -141,7 +144,7 @@ export class Router extends EventEmitter {
|
|
|
141
144
|
// Once registered, paths are always an array here.
|
|
142
145
|
r.path.some((p) => micromatch.isMatch(req.parsed.pathname, p)) &&
|
|
143
146
|
r.method === req.method?.toLocaleLowerCase() &&
|
|
144
|
-
(accepts.some((a) => a.
|
|
147
|
+
(accepts.some((a) => a.includes('*/*')) ||
|
|
145
148
|
r.contentTypes.some((contentType) => accepts.includes(contentType))) &&
|
|
146
149
|
((!contentType && r.accepts.includes(contentTypes.any)) ||
|
|
147
150
|
r.accepts.includes(contentType))) ||
|