@browserless.io/browserless 2.8.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -1
- package/README.md +41 -3
- package/assets/debugger.png +0 -0
- package/bin/scaffold/src/hello-world.http.ts +3 -2
- package/build/browserless.d.ts +5 -4
- package/build/browserless.js +15 -11
- package/build/browsers/chrome.cdp.d.ts +2 -2
- package/build/browsers/chrome.cdp.js +2 -2
- package/build/browsers/chrome.playwright.d.ts +2 -2
- package/build/browsers/chrome.playwright.js +2 -2
- package/build/browsers/chromium.cdp.d.ts +4 -4
- package/build/browsers/chromium.cdp.js +49 -32
- package/build/browsers/chromium.playwright.d.ts +4 -4
- package/build/browsers/chromium.playwright.js +14 -13
- package/build/browsers/firefox.playwright.d.ts +4 -4
- package/build/browsers/firefox.playwright.js +14 -13
- package/build/browsers/index.d.ts +24 -8
- package/build/browsers/index.js +20 -15
- package/build/browsers/webkit.playwright.d.ts +4 -4
- package/build/browsers/webkit.playwright.js +14 -13
- package/build/config.d.ts +3 -0
- package/build/config.js +3 -0
- package/build/file-system.d.ts +2 -3
- package/build/file-system.js +2 -2
- package/build/index.js +7 -7
- package/build/limiter.d.ts +2 -3
- package/build/limiter.js +11 -11
- package/build/logger.d.ts +16 -9
- package/build/logger.js +32 -16
- package/build/monitoring.d.ts +2 -3
- package/build/monitoring.js +4 -4
- package/build/router.d.ts +1 -3
- package/build/router.js +21 -22
- package/build/routes/chrome/http/content.post.body.json +8 -8
- package/build/routes/chrome/http/pdf.post.body.json +8 -8
- package/build/routes/chrome/http/scrape.post.body.json +8 -8
- package/build/routes/chrome/http/screenshot.post.body.json +8 -8
- package/build/routes/chromium/http/content.post.body.json +8 -8
- package/build/routes/chromium/http/pdf.post.body.json +8 -8
- package/build/routes/chromium/http/scrape.post.body.json +8 -8
- package/build/routes/chromium/http/screenshot.post.body.json +8 -8
- package/build/routes/management/http/static.get.js +3 -3
- package/build/routes/management/tests/management.spec.js +9 -0
- package/build/server.d.ts +1 -3
- package/build/server.js +33 -30
- package/build/shared/content.http.d.ts +1 -1
- package/build/shared/content.http.js +4 -1
- package/build/shared/download.http.js +9 -9
- package/build/shared/function.http.js +2 -2
- package/build/shared/json-protocol.http.d.ts +3 -3
- package/build/shared/json-protocol.http.js +2 -2
- package/build/shared/json-version.http.d.ts +3 -3
- package/build/shared/json-version.http.js +2 -2
- package/build/shared/pdf.http.d.ts +1 -1
- package/build/shared/pdf.http.js +6 -4
- package/build/shared/performance.http.js +1 -0
- package/build/shared/scrape.http.d.ts +1 -1
- package/build/shared/scrape.http.js +4 -1
- package/build/shared/screenshot.http.d.ts +1 -1
- package/build/shared/screenshot.http.js +4 -1
- package/build/shared/utils/function/handler.js +7 -7
- package/build/shared/utils/performance/child.js +4 -4
- package/build/shared/utils/performance/main.d.ts +1 -1
- package/build/shared/utils/performance/main.js +5 -7
- package/build/shared/utils/performance/types.d.ts +2 -1
- package/build/utils.d.ts +1 -1
- package/build/utils.js +6 -2
- package/package.json +18 -15
- package/scripts/install-debugger.js +55 -15
- package/src/browserless.ts +21 -12
- package/src/browsers/chrome.cdp.ts +2 -5
- package/src/browsers/chrome.playwright.ts +2 -5
- package/src/browsers/chromium.cdp.ts +84 -35
- package/src/browsers/chromium.playwright.ts +26 -13
- package/src/browsers/firefox.playwright.ts +28 -13
- package/src/browsers/index.ts +24 -16
- package/src/browsers/webkit.playwright.ts +28 -13
- package/src/config.ts +4 -0
- package/src/file-system.ts +2 -7
- package/src/index.ts +7 -7
- package/src/limiter.ts +13 -11
- package/src/logger.ts +39 -18
- package/src/monitoring.ts +6 -8
- package/src/router.ts +20 -20
- package/src/routes/management/http/static.get.ts +3 -5
- package/src/routes/management/tests/management.spec.ts +15 -0
- package/src/server.ts +43 -30
- package/src/shared/content.http.ts +5 -1
- package/src/shared/download.http.ts +9 -9
- package/src/shared/function.http.ts +2 -2
- package/src/shared/json-protocol.http.ts +8 -3
- package/src/shared/json-version.http.ts +8 -4
- package/src/shared/pdf.http.ts +6 -4
- package/src/shared/performance.http.ts +1 -0
- package/src/shared/scrape.http.ts +5 -1
- package/src/shared/screenshot.http.ts +4 -1
- package/src/shared/utils/function/handler.ts +7 -7
- package/src/shared/utils/performance/child.ts +4 -4
- package/src/shared/utils/performance/main.ts +5 -6
- package/src/shared/utils/performance/types.ts +2 -1
- package/src/utils.ts +7 -2
- package/static/docs/swagger.json +10 -10
- package/static/docs/swagger.min.json +9 -9
- package/static/function/client.js +1646 -2917
- package/static/function/index.html +1646 -2917
|
@@ -484,14 +484,14 @@
|
|
|
484
484
|
"length": {
|
|
485
485
|
"type": "number"
|
|
486
486
|
},
|
|
487
|
-
"__@toStringTag@
|
|
487
|
+
"__@toStringTag@252176": {
|
|
488
488
|
"type": "string",
|
|
489
489
|
"const": "Uint8Array"
|
|
490
490
|
}
|
|
491
491
|
},
|
|
492
492
|
"required": [
|
|
493
493
|
"BYTES_PER_ELEMENT",
|
|
494
|
-
"__@toStringTag@
|
|
494
|
+
"__@toStringTag@252176",
|
|
495
495
|
"buffer",
|
|
496
496
|
"byteLength",
|
|
497
497
|
"byteOffset",
|
|
@@ -526,13 +526,13 @@
|
|
|
526
526
|
"byteLength": {
|
|
527
527
|
"type": "number"
|
|
528
528
|
},
|
|
529
|
-
"__@toStringTag@
|
|
529
|
+
"__@toStringTag@252176": {
|
|
530
530
|
"type": "string"
|
|
531
531
|
}
|
|
532
532
|
},
|
|
533
533
|
"additionalProperties": false,
|
|
534
534
|
"required": [
|
|
535
|
-
"__@toStringTag@
|
|
535
|
+
"__@toStringTag@252176",
|
|
536
536
|
"byteLength"
|
|
537
537
|
]
|
|
538
538
|
},
|
|
@@ -542,18 +542,18 @@
|
|
|
542
542
|
"byteLength": {
|
|
543
543
|
"type": "number"
|
|
544
544
|
},
|
|
545
|
-
"__@species@
|
|
545
|
+
"__@species@252277": {
|
|
546
546
|
"$ref": "#/definitions/SharedArrayBuffer"
|
|
547
547
|
},
|
|
548
|
-
"__@toStringTag@
|
|
548
|
+
"__@toStringTag@252176": {
|
|
549
549
|
"type": "string",
|
|
550
550
|
"const": "SharedArrayBuffer"
|
|
551
551
|
}
|
|
552
552
|
},
|
|
553
553
|
"additionalProperties": false,
|
|
554
554
|
"required": [
|
|
555
|
-
"__@species@
|
|
556
|
-
"__@toStringTag@
|
|
555
|
+
"__@species@252277",
|
|
556
|
+
"__@toStringTag@252176",
|
|
557
557
|
"byteLength"
|
|
558
558
|
]
|
|
559
559
|
},
|
|
@@ -4,7 +4,7 @@ import path from 'path';
|
|
|
4
4
|
const pathMap = new Map();
|
|
5
5
|
const streamFile = (logger, res, file, contentType) => new Promise((resolve, reject) => {
|
|
6
6
|
if (contentType) {
|
|
7
|
-
logger.
|
|
7
|
+
logger.debug(`Setting content-type ${contentType}`);
|
|
8
8
|
res.setHeader('Content-Type', contentType);
|
|
9
9
|
}
|
|
10
10
|
return createReadStream(file)
|
|
@@ -49,10 +49,10 @@ export default class StaticGetRoute extends HTTPRoute {
|
|
|
49
49
|
throw new NotFound(`No route or file found for resource ${req.method}: ${pathname}`);
|
|
50
50
|
}
|
|
51
51
|
if (foundFilePaths.length > 1) {
|
|
52
|
-
logger.
|
|
52
|
+
logger.warn(`Multiple files found for request to "${pathname}". Only the first file is served, so please name your files uniquely.`);
|
|
53
53
|
}
|
|
54
54
|
const [foundFilePath] = foundFilePaths;
|
|
55
|
-
logger.
|
|
55
|
+
logger.info(`Found new file "${foundFilePath}", caching path and serving`);
|
|
56
56
|
const contentType = mimeTypes.get(path.extname(foundFilePath));
|
|
57
57
|
if (contentType) {
|
|
58
58
|
res.setHeader('Content-Type', contentType);
|
|
@@ -45,4 +45,13 @@ describe('Management APIs', function () {
|
|
|
45
45
|
expect(res.status).to.equal(204);
|
|
46
46
|
});
|
|
47
47
|
});
|
|
48
|
+
it('allows HEAD requests to /active', async () => {
|
|
49
|
+
await start();
|
|
50
|
+
await fetch('http://localhost:3000/active?token=6R0W53R135510', {
|
|
51
|
+
method: 'HEAD',
|
|
52
|
+
}).then(async (res) => {
|
|
53
|
+
expect(res.headers.get('content-type')).to.equal('text/plain; charset=UTF-8');
|
|
54
|
+
expect(res.status).to.equal(204);
|
|
55
|
+
});
|
|
56
|
+
});
|
|
48
57
|
});
|
package/build/server.d.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
/// <reference types="debug" />
|
|
3
2
|
/// <reference types="node" />
|
|
4
3
|
/// <reference types="node" />
|
|
5
4
|
/// <reference types="node" />
|
|
@@ -24,8 +23,7 @@ export declare class HTTPServer extends EventEmitter {
|
|
|
24
23
|
protected server: http.Server;
|
|
25
24
|
protected port: number;
|
|
26
25
|
protected host?: string;
|
|
27
|
-
protected
|
|
28
|
-
protected verbose: import("debug").Debugger;
|
|
26
|
+
protected logger: BlessLogger;
|
|
29
27
|
constructor(config: Config, metrics: Metrics, token: Token, router: Router, hooks: Hooks, Logger: typeof BlessLogger);
|
|
30
28
|
protected onHTTPUnauthorized: (_req: Request, res: Response) => void;
|
|
31
29
|
protected onWebsocketUnauthorized: (_req: Request, socket: stream.Duplex) => void;
|
package/build/server.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as http from 'http';
|
|
2
|
-
import { BadRequest, NotFound, Timeout, TooManyRequests, Unauthorized, contentTypes, convertPathToURL,
|
|
2
|
+
import { BadRequest, Logger as BlessLogger, NotFound, Timeout, TooManyRequests, Unauthorized, contentTypes, convertPathToURL, queryParamsToObject, readBody, shimLegacyRequests, writeResponse, } from '@browserless.io/browserless';
|
|
3
3
|
import { EventEmitter } from 'events';
|
|
4
4
|
// @ts-ignore
|
|
5
5
|
import Enjoi from 'enjoi';
|
|
@@ -13,8 +13,7 @@ export class HTTPServer extends EventEmitter {
|
|
|
13
13
|
server = http.createServer();
|
|
14
14
|
port;
|
|
15
15
|
host;
|
|
16
|
-
|
|
17
|
-
verbose = createLogger('server:verbose');
|
|
16
|
+
logger = new BlessLogger('server');
|
|
18
17
|
constructor(config, metrics, token, router, hooks, Logger) {
|
|
19
18
|
super();
|
|
20
19
|
this.config = config;
|
|
@@ -25,20 +24,20 @@ export class HTTPServer extends EventEmitter {
|
|
|
25
24
|
this.Logger = Logger;
|
|
26
25
|
this.host = config.getHost();
|
|
27
26
|
this.port = config.getPort();
|
|
28
|
-
this.
|
|
27
|
+
this.logger.info(`Server instantiated with host "${this.host}" on port "${this.port}" using token "${this.config.getToken()}"`);
|
|
29
28
|
}
|
|
30
29
|
onHTTPUnauthorized = (_req, res) => {
|
|
31
|
-
this.
|
|
30
|
+
this.logger.error(`HTTP request is not properly authorized, responding with 401`);
|
|
32
31
|
this.metrics.addUnauthorized();
|
|
33
32
|
return writeResponse(res, 401, 'Bad or missing authentication.');
|
|
34
33
|
};
|
|
35
34
|
onWebsocketUnauthorized = (_req, socket) => {
|
|
36
|
-
this.
|
|
35
|
+
this.logger.error(`Websocket request is not properly authorized, responding with 401`);
|
|
37
36
|
this.metrics.addUnauthorized();
|
|
38
37
|
return writeResponse(socket, 401, 'Bad or missing authentication.');
|
|
39
38
|
};
|
|
40
39
|
async start() {
|
|
41
|
-
this.
|
|
40
|
+
this.logger.info(`HTTP Server is starting`);
|
|
42
41
|
this.server.on('request', this.handleRequest);
|
|
43
42
|
this.server.on('upgrade', this.handleWebSocket);
|
|
44
43
|
const listenMessage = [
|
|
@@ -50,13 +49,13 @@ export class HTTPServer extends EventEmitter {
|
|
|
50
49
|
host: this.host,
|
|
51
50
|
port: this.port,
|
|
52
51
|
}, undefined, () => {
|
|
53
|
-
this.
|
|
52
|
+
this.logger.info(listenMessage);
|
|
54
53
|
r(undefined);
|
|
55
54
|
});
|
|
56
55
|
});
|
|
57
56
|
}
|
|
58
57
|
handleRequest = async (request, res) => {
|
|
59
|
-
this.
|
|
58
|
+
this.logger.trace(`Handling inbound HTTP request on "${request.method}: ${request.url}"`);
|
|
60
59
|
const req = request;
|
|
61
60
|
const proceed = await this.hooks.before({ req, res });
|
|
62
61
|
req.parsed = convertPathToURL(request.url || '', this.config);
|
|
@@ -70,6 +69,10 @@ export class HTTPServer extends EventEmitter {
|
|
|
70
69
|
return res.end();
|
|
71
70
|
}
|
|
72
71
|
}
|
|
72
|
+
if (req.method?.toLowerCase() === 'head') {
|
|
73
|
+
this.logger.debug(`Inbound HEAD request, setting to GET`);
|
|
74
|
+
req.method = 'GET';
|
|
75
|
+
}
|
|
73
76
|
if (this.config.getAllowGetCalls() &&
|
|
74
77
|
req.method === 'GET' &&
|
|
75
78
|
req.parsed.searchParams.has('body')) {
|
|
@@ -80,13 +83,13 @@ export class HTTPServer extends EventEmitter {
|
|
|
80
83
|
}
|
|
81
84
|
const route = await this.router.getRouteForHTTPRequest(req);
|
|
82
85
|
if (!route) {
|
|
83
|
-
this.
|
|
86
|
+
this.logger.error(`No matching HTTP route handler for "${req.method}: ${req.parsed.href}"`);
|
|
84
87
|
writeResponse(res, 404, 'Not Found');
|
|
85
88
|
return Promise.resolve();
|
|
86
89
|
}
|
|
87
|
-
this.
|
|
90
|
+
this.logger.trace(`Found matching HTTP route handler "${route.path}"`);
|
|
88
91
|
if (route?.auth) {
|
|
89
|
-
this.
|
|
92
|
+
this.logger.trace(`Authorizing HTTP request to "${request.url}"`);
|
|
90
93
|
const isPermitted = await this.token.isAuthorized(req, route);
|
|
91
94
|
if (!isPermitted) {
|
|
92
95
|
return this.onHTTPUnauthorized(req, res);
|
|
@@ -104,7 +107,7 @@ export class HTTPServer extends EventEmitter {
|
|
|
104
107
|
return Promise.resolve();
|
|
105
108
|
}
|
|
106
109
|
if (route.querySchema) {
|
|
107
|
-
this.
|
|
110
|
+
this.logger.trace(`Validating route query-params with QUERY schema`);
|
|
108
111
|
try {
|
|
109
112
|
const schema = Enjoi.schema(route.querySchema);
|
|
110
113
|
const valid = schema.validate(req.queryParams, {
|
|
@@ -114,19 +117,19 @@ export class HTTPServer extends EventEmitter {
|
|
|
114
117
|
const errorDetails = valid.error.details
|
|
115
118
|
.map(({ message, context, }) => context?.message || message)
|
|
116
119
|
.join('\n');
|
|
117
|
-
this.
|
|
120
|
+
this.logger.error(`HTTP query-params contain errors sending 400:${errorDetails}`);
|
|
118
121
|
writeResponse(res, 400, `Query-parameter validation failed: ${errorDetails}`, contentTypes.text);
|
|
119
122
|
return Promise.resolve();
|
|
120
123
|
}
|
|
121
124
|
}
|
|
122
125
|
catch (e) {
|
|
123
|
-
this.
|
|
126
|
+
this.logger.error(`Error parsing body schema`, e);
|
|
124
127
|
writeResponse(res, 500, 'There was an error handling your request', contentTypes.text);
|
|
125
128
|
return Promise.resolve();
|
|
126
129
|
}
|
|
127
130
|
}
|
|
128
131
|
if (route.bodySchema) {
|
|
129
|
-
this.
|
|
132
|
+
this.logger.trace(`Validating route payload with BODY schema`);
|
|
130
133
|
try {
|
|
131
134
|
const schema = Enjoi.schema(route.bodySchema);
|
|
132
135
|
const valid = schema.validate(body, { abortEarly: false });
|
|
@@ -134,13 +137,13 @@ export class HTTPServer extends EventEmitter {
|
|
|
134
137
|
const errorDetails = valid.error.details
|
|
135
138
|
.map(({ message, context, }) => context?.message || message)
|
|
136
139
|
.join('\n');
|
|
137
|
-
this.
|
|
140
|
+
this.logger.error(`HTTP body contain errors sending 400:${errorDetails}`);
|
|
138
141
|
writeResponse(res, 400, `POST Body validation failed: ${errorDetails}`, contentTypes.text);
|
|
139
142
|
return Promise.resolve();
|
|
140
143
|
}
|
|
141
144
|
}
|
|
142
145
|
catch (e) {
|
|
143
|
-
this.
|
|
146
|
+
this.logger.error(`Error parsing body schema`, e);
|
|
144
147
|
writeResponse(res, 500, 'There was an error handling your request', contentTypes.text);
|
|
145
148
|
return Promise.resolve();
|
|
146
149
|
}
|
|
@@ -148,7 +151,7 @@ export class HTTPServer extends EventEmitter {
|
|
|
148
151
|
return route
|
|
149
152
|
.handler(req, res, new this.Logger(route.name, req))
|
|
150
153
|
.then(() => {
|
|
151
|
-
this.
|
|
154
|
+
this.logger.trace('HTTP connection complete');
|
|
152
155
|
})
|
|
153
156
|
.catch((e) => {
|
|
154
157
|
if (e instanceof BadRequest) {
|
|
@@ -170,7 +173,7 @@ export class HTTPServer extends EventEmitter {
|
|
|
170
173
|
});
|
|
171
174
|
};
|
|
172
175
|
handleWebSocket = async (request, socket, head) => {
|
|
173
|
-
this.
|
|
176
|
+
this.logger.trace(`Handling inbound WebSocket request on "${request.url}"`);
|
|
174
177
|
const req = request;
|
|
175
178
|
const proceed = await this.hooks.before({ head, req, socket });
|
|
176
179
|
req.parsed = convertPathToURL(request.url || '', this.config);
|
|
@@ -180,16 +183,16 @@ export class HTTPServer extends EventEmitter {
|
|
|
180
183
|
req.queryParams = queryParamsToObject(req.parsed.searchParams);
|
|
181
184
|
const route = await this.router.getRouteForWebSocketRequest(req);
|
|
182
185
|
if (route) {
|
|
183
|
-
this.
|
|
186
|
+
this.logger.trace(`Found matching WebSocket route handler "${route.path}"`);
|
|
184
187
|
if (route?.auth) {
|
|
185
|
-
this.
|
|
188
|
+
this.logger.trace(`Authorizing WebSocket request to "${req.parsed.href}"`);
|
|
186
189
|
const isPermitted = await this.token.isAuthorized(req, route);
|
|
187
190
|
if (!isPermitted) {
|
|
188
191
|
return this.onWebsocketUnauthorized(req, socket);
|
|
189
192
|
}
|
|
190
193
|
}
|
|
191
194
|
if (route.querySchema) {
|
|
192
|
-
this.
|
|
195
|
+
this.logger.trace(`Validating route query-params with QUERY schema`);
|
|
193
196
|
try {
|
|
194
197
|
const schema = Enjoi.schema(route.querySchema);
|
|
195
198
|
const valid = schema.validate(req.queryParams, {
|
|
@@ -199,13 +202,13 @@ export class HTTPServer extends EventEmitter {
|
|
|
199
202
|
const errorDetails = valid.error.details
|
|
200
203
|
.map(({ message, context, }) => context?.message || message)
|
|
201
204
|
.join('\n');
|
|
202
|
-
this.
|
|
205
|
+
this.logger.error(`WebSocket query-params contain errors sending 400:${errorDetails}`);
|
|
203
206
|
writeResponse(socket, 400, `Query-parameter validation failed: ${errorDetails}`, contentTypes.text);
|
|
204
207
|
return Promise.resolve();
|
|
205
208
|
}
|
|
206
209
|
}
|
|
207
210
|
catch (e) {
|
|
208
|
-
this.
|
|
211
|
+
this.logger.error(`Error parsing query-params schema`, e);
|
|
209
212
|
writeResponse(socket, 500, 'There was an error handling your request', contentTypes.text);
|
|
210
213
|
return Promise.resolve();
|
|
211
214
|
}
|
|
@@ -213,7 +216,7 @@ export class HTTPServer extends EventEmitter {
|
|
|
213
216
|
return route
|
|
214
217
|
.handler(req, socket, head, new this.Logger(route.name, req))
|
|
215
218
|
.then(() => {
|
|
216
|
-
this.
|
|
219
|
+
this.logger.trace('Websocket connection complete');
|
|
217
220
|
})
|
|
218
221
|
.catch((e) => {
|
|
219
222
|
if (e instanceof BadRequest) {
|
|
@@ -228,20 +231,20 @@ export class HTTPServer extends EventEmitter {
|
|
|
228
231
|
if (e instanceof TooManyRequests) {
|
|
229
232
|
return writeResponse(socket, 429, e.message);
|
|
230
233
|
}
|
|
231
|
-
this.
|
|
234
|
+
this.logger.error(`Error handling request at "${route.path}": ${e}\n${e.stack}`);
|
|
232
235
|
return writeResponse(socket, 500, e.message);
|
|
233
236
|
});
|
|
234
237
|
}
|
|
235
|
-
this.
|
|
238
|
+
this.logger.error(`No matching WebSocket route handler for "${req.parsed.href}"`);
|
|
236
239
|
return writeResponse(socket, 404, 'Not Found');
|
|
237
240
|
};
|
|
238
241
|
async shutdown() {
|
|
239
|
-
this.
|
|
242
|
+
this.logger.info(`HTTP Server is shutting down`);
|
|
240
243
|
await new Promise((r) => this.server.close(r));
|
|
241
244
|
this.server && this.server.removeAllListeners();
|
|
242
245
|
// @ts-ignore garbage collect this reference
|
|
243
246
|
this.server = null;
|
|
244
|
-
this.
|
|
247
|
+
this.logger.info(`HTTP Server shutdown complete`);
|
|
245
248
|
}
|
|
246
249
|
/**
|
|
247
250
|
* Left blank for downstream SDK modules to optionally implement.
|
|
@@ -43,5 +43,5 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
|
|
|
43
43
|
method: Methods;
|
|
44
44
|
path: HTTPRoutes[];
|
|
45
45
|
tags: APITags[];
|
|
46
|
-
handler: (req: Request, res: ServerResponse,
|
|
46
|
+
handler: (req: Request, res: ServerResponse, logger: Logger, browser: BrowserInstance) => Promise<void>;
|
|
47
47
|
}
|
|
@@ -10,7 +10,8 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
|
|
|
10
10
|
method = Methods.post;
|
|
11
11
|
path = [HTTPRoutes.content, HTTPRoutes.chromiumContent];
|
|
12
12
|
tags = [APITags.browserAPI];
|
|
13
|
-
handler = async (req, res,
|
|
13
|
+
handler = async (req, res, logger, browser) => {
|
|
14
|
+
logger.info('Content API invoked with body:', req.body);
|
|
14
15
|
const contentType = !req.headers.accept || req.headers.accept?.includes('*')
|
|
15
16
|
? contentTypes.html
|
|
16
17
|
: req.headers.accept;
|
|
@@ -53,6 +54,7 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
|
|
|
53
54
|
page.on('request', (req) => {
|
|
54
55
|
if (!!rejectRequestPattern.find((pattern) => req.url().match(pattern)) ||
|
|
55
56
|
rejectResourceTypes.includes(req.resourceType())) {
|
|
57
|
+
logger.debug(`Aborting request ${req.method()}: ${req.url()}`);
|
|
56
58
|
return req.abort();
|
|
57
59
|
}
|
|
58
60
|
const interceptor = requestInterceptors.find((r) => req.url().match(r.pattern));
|
|
@@ -102,6 +104,7 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
|
|
|
102
104
|
}
|
|
103
105
|
const markup = await page.content();
|
|
104
106
|
page.close().catch(noop);
|
|
107
|
+
logger.info('Content API request completed');
|
|
105
108
|
return writeResponse(res, 200, markup, contentTypes.html);
|
|
106
109
|
};
|
|
107
110
|
}
|
|
@@ -24,11 +24,11 @@ export default class ChromiumDownloadPostRoute extends BrowserHTTPRoute {
|
|
|
24
24
|
handler = async (req, res, logger, browser) => new Promise(async (resolve, reject) => {
|
|
25
25
|
const config = this.config();
|
|
26
26
|
const downloadPath = path.join(await config.getDownloadsDir(), `.browserless.download.${id()}`);
|
|
27
|
-
logger.
|
|
27
|
+
logger.info(`Generating a download directory at "${downloadPath}"`);
|
|
28
28
|
await mkdir(downloadPath);
|
|
29
29
|
const handler = functionHandler(config, logger, { downloadPath });
|
|
30
30
|
const response = await handler(req, browser).catch((e) => {
|
|
31
|
-
logger.
|
|
31
|
+
logger.error(`Error running download code handler: "${e}"`);
|
|
32
32
|
reject(e);
|
|
33
33
|
return null;
|
|
34
34
|
});
|
|
@@ -36,10 +36,10 @@ export default class ChromiumDownloadPostRoute extends BrowserHTTPRoute {
|
|
|
36
36
|
return;
|
|
37
37
|
}
|
|
38
38
|
const { page } = response;
|
|
39
|
-
logger.
|
|
39
|
+
logger.info(`Download function has returned, finding downloads...`);
|
|
40
40
|
async function checkIfDownloadComplete() {
|
|
41
41
|
if (res.headersSent) {
|
|
42
|
-
logger.
|
|
42
|
+
logger.trace(`Request headers have been sent, terminating download watch.`);
|
|
43
43
|
return null;
|
|
44
44
|
}
|
|
45
45
|
const [fileName] = await readdir(downloadPath);
|
|
@@ -47,20 +47,20 @@ export default class ChromiumDownloadPostRoute extends BrowserHTTPRoute {
|
|
|
47
47
|
await sleep(500);
|
|
48
48
|
return checkIfDownloadComplete();
|
|
49
49
|
}
|
|
50
|
-
logger.
|
|
50
|
+
logger.info(`All files have finished downloading`);
|
|
51
51
|
return path.join(downloadPath, fileName);
|
|
52
52
|
}
|
|
53
53
|
const filePath = await checkIfDownloadComplete();
|
|
54
|
-
logger.
|
|
54
|
+
logger.info(`Closing pages.`);
|
|
55
55
|
page.close();
|
|
56
56
|
page.removeAllListeners();
|
|
57
57
|
const rmDownload = once(() => filePath &&
|
|
58
58
|
deleteAsync(filePath, { force: true })
|
|
59
59
|
.then(() => {
|
|
60
|
-
logger.
|
|
60
|
+
logger.info(`Successfully deleted downloads from disk at "${filePath}"`);
|
|
61
61
|
})
|
|
62
62
|
.catch((err) => {
|
|
63
|
-
logger.
|
|
63
|
+
logger.error(`Error cleaning up downloaded files: "${err}" at "${filePath}"`);
|
|
64
64
|
}));
|
|
65
65
|
if (res.headersSent || !filePath) {
|
|
66
66
|
rmDownload();
|
|
@@ -78,7 +78,7 @@ export default class ChromiumDownloadPostRoute extends BrowserHTTPRoute {
|
|
|
78
78
|
}
|
|
79
79
|
})
|
|
80
80
|
.on('end', () => {
|
|
81
|
-
logger.
|
|
81
|
+
logger.info(`Downloads successfully sent`);
|
|
82
82
|
rmDownload();
|
|
83
83
|
return resolve();
|
|
84
84
|
})
|
|
@@ -22,7 +22,7 @@ export default class ChromiumFunctionPostRoute extends BrowserHTTPRoute {
|
|
|
22
22
|
const config = this.config();
|
|
23
23
|
const handler = functionHandler(config, logger);
|
|
24
24
|
const { contentType, payload, page } = await handler(req, browser);
|
|
25
|
-
logger.
|
|
25
|
+
logger.info(`Got function response of "${contentType}"`);
|
|
26
26
|
page.close();
|
|
27
27
|
page.removeAllListeners();
|
|
28
28
|
if (contentType === 'uint8array') {
|
|
@@ -33,7 +33,7 @@ export default class ChromiumFunctionPostRoute extends BrowserHTTPRoute {
|
|
|
33
33
|
throw new BadRequest(`Couldn't determine function's response type.`);
|
|
34
34
|
}
|
|
35
35
|
else {
|
|
36
|
-
logger.
|
|
36
|
+
logger.info(`Sending file-type response of "${type}"`);
|
|
37
37
|
const readStream = new Stream.PassThrough();
|
|
38
38
|
readStream.end(response);
|
|
39
39
|
res.setHeader('Content-Type', type);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { APITags, HTTPRoute, HTTPRoutes, Methods, Request, Response, contentTypes } from '@browserless.io/browserless';
|
|
1
|
+
import { APITags, HTTPRoute, HTTPRoutes, Logger, Methods, Request, Response, contentTypes } from '@browserless.io/browserless';
|
|
2
2
|
export type ResponseSchema = object;
|
|
3
3
|
export default class ChromiumJSONProtocolGetRoute extends HTTPRoute {
|
|
4
|
-
|
|
4
|
+
protected cachedProtocol: object | undefined;
|
|
5
5
|
name: string;
|
|
6
6
|
accepts: contentTypes[];
|
|
7
7
|
auth: boolean;
|
|
@@ -12,5 +12,5 @@ export default class ChromiumJSONProtocolGetRoute extends HTTPRoute {
|
|
|
12
12
|
method: Methods;
|
|
13
13
|
path: HTTPRoutes;
|
|
14
14
|
tags: APITags[];
|
|
15
|
-
handler: (_req: Request, res: Response) => Promise<void>;
|
|
15
|
+
handler: (_req: Request, res: Response, logger: Logger) => Promise<void>;
|
|
16
16
|
}
|
|
@@ -11,10 +11,10 @@ export default class ChromiumJSONProtocolGetRoute extends HTTPRoute {
|
|
|
11
11
|
method = Methods.get;
|
|
12
12
|
path = HTTPRoutes.jsonProtocol;
|
|
13
13
|
tags = [APITags.browserAPI];
|
|
14
|
-
handler = async (_req, res) => {
|
|
14
|
+
handler = async (_req, res, logger) => {
|
|
15
15
|
const browserManager = this.browserManager();
|
|
16
16
|
if (!this.cachedProtocol) {
|
|
17
|
-
this.cachedProtocol = await browserManager.getProtocolJSON();
|
|
17
|
+
this.cachedProtocol = await browserManager.getProtocolJSON(logger);
|
|
18
18
|
}
|
|
19
19
|
return jsonResponse(res, 200, this.cachedProtocol);
|
|
20
20
|
};
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { APITags, BrowserManager, HTTPRoute, HTTPRoutes, Methods, Request, Response, UnwrapPromise, contentTypes } from '@browserless.io/browserless';
|
|
1
|
+
import { APITags, BrowserManager, HTTPRoute, HTTPRoutes, Logger, Methods, Request, Response, UnwrapPromise, contentTypes } from '@browserless.io/browserless';
|
|
2
2
|
export type ResponseSchema = UnwrapPromise<ReturnType<BrowserManager['getVersionJSON']>>;
|
|
3
3
|
export default class ChromiumJSONVersionGetRoute extends HTTPRoute {
|
|
4
|
-
|
|
4
|
+
protected cachedJSON: ResponseSchema | undefined;
|
|
5
5
|
name: string;
|
|
6
6
|
accepts: contentTypes[];
|
|
7
7
|
auth: boolean;
|
|
@@ -12,5 +12,5 @@ export default class ChromiumJSONVersionGetRoute extends HTTPRoute {
|
|
|
12
12
|
method: Methods;
|
|
13
13
|
path: HTTPRoutes;
|
|
14
14
|
tags: APITags[];
|
|
15
|
-
handler: (req: Request, res: Response) => Promise<void>;
|
|
15
|
+
handler: (req: Request, res: Response, logger: Logger) => Promise<void>;
|
|
16
16
|
}
|
|
@@ -11,14 +11,14 @@ export default class ChromiumJSONVersionGetRoute extends HTTPRoute {
|
|
|
11
11
|
method = Methods.get;
|
|
12
12
|
path = HTTPRoutes.jsonVersion;
|
|
13
13
|
tags = [APITags.browserAPI];
|
|
14
|
-
handler = async (req, res) => {
|
|
14
|
+
handler = async (req, res, logger) => {
|
|
15
15
|
const baseUrl = req.parsed.host;
|
|
16
16
|
const protocol = req.parsed.protocol.includes('s') ? 'wss' : 'ws';
|
|
17
17
|
try {
|
|
18
18
|
if (!this.cachedJSON) {
|
|
19
19
|
const browserManager = this.browserManager();
|
|
20
20
|
this.cachedJSON = {
|
|
21
|
-
...(await browserManager.getVersionJSON()),
|
|
21
|
+
...(await browserManager.getVersionJSON(logger)),
|
|
22
22
|
webSocketDebuggerUrl: `${protocol}://${baseUrl}`,
|
|
23
23
|
};
|
|
24
24
|
}
|
|
@@ -43,5 +43,5 @@ export default class ChromiumPDFPostRoute extends BrowserHTTPRoute {
|
|
|
43
43
|
method: Methods;
|
|
44
44
|
path: HTTPRoutes[];
|
|
45
45
|
tags: APITags[];
|
|
46
|
-
handler: (req: Request, res: ServerResponse,
|
|
46
|
+
handler: (req: Request, res: ServerResponse, logger: Logger, browser: BrowserInstance) => Promise<void>;
|
|
47
47
|
}
|
package/build/shared/pdf.http.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { APITags, BadRequest, BrowserHTTPRoute, BrowserlessRoutes, ChromiumCDP, HTTPRoutes, Methods, bestAttemptCatch, contentTypes, dedent, noop, sleep, waitForEvent as waitForEvt, waitForFunction as waitForFn,
|
|
1
|
+
import { APITags, BadRequest, BrowserHTTPRoute, BrowserlessRoutes, ChromiumCDP, HTTPRoutes, Methods, bestAttemptCatch, contentTypes, dedent, noop, sleep, waitForEvent as waitForEvt, waitForFunction as waitForFn, } from '@browserless.io/browserless';
|
|
2
2
|
import { Stream } from 'stream';
|
|
3
3
|
export default class ChromiumPDFPostRoute extends BrowserHTTPRoute {
|
|
4
4
|
name = BrowserlessRoutes.ChromiumPDFPostRoute;
|
|
@@ -16,13 +16,13 @@ export default class ChromiumPDFPostRoute extends BrowserHTTPRoute {
|
|
|
16
16
|
method = Methods.post;
|
|
17
17
|
path = [HTTPRoutes.pdf, HTTPRoutes.chromiumPdf];
|
|
18
18
|
tags = [APITags.browserAPI];
|
|
19
|
-
handler = async (req, res,
|
|
19
|
+
handler = async (req, res, logger, browser) => {
|
|
20
|
+
logger.info('PDF API invoked with body:', req.body);
|
|
20
21
|
const contentType = !req.headers.accept || req.headers.accept?.includes('*')
|
|
21
22
|
? 'application/pdf'
|
|
22
23
|
: req.headers.accept;
|
|
23
24
|
if (!req.body) {
|
|
24
|
-
|
|
25
|
-
return;
|
|
25
|
+
throw new BadRequest(`Couldn't parse JSON body`);
|
|
26
26
|
}
|
|
27
27
|
res.setHeader('Content-Type', contentType);
|
|
28
28
|
const { url, gotoOptions, authenticate, html, addScriptTag = [], addStyleTag = [], cookies = [], emulateMediaType, rejectRequestPattern = [], requestInterceptors = [], rejectResourceTypes = [], options, setExtraHTTPHeaders, setJavaScriptEnabled, userAgent, viewport, waitForEvent, waitForFunction, waitForSelector, waitForTimeout, bestAttempt = false, } = req.body;
|
|
@@ -60,6 +60,7 @@ export default class ChromiumPDFPostRoute extends BrowserHTTPRoute {
|
|
|
60
60
|
page.on('request', (req) => {
|
|
61
61
|
if (!!rejectRequestPattern.find((pattern) => req.url().match(pattern)) ||
|
|
62
62
|
rejectResourceTypes.includes(req.resourceType())) {
|
|
63
|
+
logger.debug(`Aborting request ${req.method()}: ${req.url()}`);
|
|
63
64
|
return req.abort();
|
|
64
65
|
}
|
|
65
66
|
const interceptor = requestInterceptors.find((r) => req.url().match(r.pattern));
|
|
@@ -112,5 +113,6 @@ export default class ChromiumPDFPostRoute extends BrowserHTTPRoute {
|
|
|
112
113
|
readStream.end(pdfBuffer);
|
|
113
114
|
await new Promise((r) => readStream.pipe(res).once('close', r));
|
|
114
115
|
page.close().catch(noop);
|
|
116
|
+
logger.info('PDF API request completed');
|
|
115
117
|
};
|
|
116
118
|
}
|
|
@@ -115,5 +115,5 @@ export default class ChromiumScrapePostRoute extends BrowserHTTPRoute {
|
|
|
115
115
|
method: Methods;
|
|
116
116
|
path: HTTPRoutes[];
|
|
117
117
|
tags: APITags[];
|
|
118
|
-
handler: (req: Request, res: ServerResponse,
|
|
118
|
+
handler: (req: Request, res: ServerResponse, logger: Logger, browser: BrowserInstance) => Promise<void>;
|
|
119
119
|
}
|
|
@@ -54,7 +54,8 @@ export default class ChromiumScrapePostRoute extends BrowserHTTPRoute {
|
|
|
54
54
|
method = Methods.post;
|
|
55
55
|
path = [HTTPRoutes.scrape, HTTPRoutes.chromiumScrape];
|
|
56
56
|
tags = [APITags.browserAPI];
|
|
57
|
-
handler = async (req, res,
|
|
57
|
+
handler = async (req, res, logger, browser) => {
|
|
58
|
+
logger.info('Scrape API invoked with body:', req.body);
|
|
58
59
|
const contentType = !req.headers.accept || req.headers.accept?.includes('*')
|
|
59
60
|
? contentTypes.html
|
|
60
61
|
: req.headers.accept;
|
|
@@ -121,6 +122,7 @@ export default class ChromiumScrapePostRoute extends BrowserHTTPRoute {
|
|
|
121
122
|
page.on('request', (req) => {
|
|
122
123
|
if (!!rejectRequestPattern.find((pattern) => req.url().match(pattern)) ||
|
|
123
124
|
rejectResourceTypes.includes(req.resourceType())) {
|
|
125
|
+
logger.debug(`Aborting request ${req.method()}: ${req.url()}`);
|
|
124
126
|
return req.abort();
|
|
125
127
|
}
|
|
126
128
|
const interceptor = requestInterceptors.find((r) => req.url().match(r.pattern));
|
|
@@ -198,6 +200,7 @@ export default class ChromiumScrapePostRoute extends BrowserHTTPRoute {
|
|
|
198
200
|
debug: debugData,
|
|
199
201
|
};
|
|
200
202
|
page.close().catch(noop);
|
|
203
|
+
logger.info('Scrape API request completed');
|
|
201
204
|
return jsonResponse(res, 200, response, false);
|
|
202
205
|
};
|
|
203
206
|
}
|
|
@@ -46,5 +46,5 @@ export default class ScreenshotPost extends BrowserHTTPRoute {
|
|
|
46
46
|
method: Methods;
|
|
47
47
|
path: HTTPRoutes[];
|
|
48
48
|
tags: APITags[];
|
|
49
|
-
handler: (req: Request, res: ServerResponse,
|
|
49
|
+
handler: (req: Request, res: ServerResponse, logger: Logger, browser: BrowserInstance) => Promise<void>;
|
|
50
50
|
}
|
|
@@ -15,7 +15,8 @@ export default class ScreenshotPost extends BrowserHTTPRoute {
|
|
|
15
15
|
method = Methods.post;
|
|
16
16
|
path = [HTTPRoutes.screenshot, HTTPRoutes.chromiumScreenshot];
|
|
17
17
|
tags = [APITags.browserAPI];
|
|
18
|
-
handler = async (req, res,
|
|
18
|
+
handler = async (req, res, logger, browser) => {
|
|
19
|
+
logger.info('Screenshot API invoked with body:', req.body);
|
|
19
20
|
const contentType = !req.headers.accept || req.headers.accept?.includes('*')
|
|
20
21
|
? 'image/png'
|
|
21
22
|
: req.headers.accept;
|
|
@@ -61,6 +62,7 @@ export default class ScreenshotPost extends BrowserHTTPRoute {
|
|
|
61
62
|
page.on('request', (req) => {
|
|
62
63
|
if (!!rejectRequestPattern.find((pattern) => req.url().match(pattern)) ||
|
|
63
64
|
rejectResourceTypes.includes(req.resourceType())) {
|
|
65
|
+
logger.debug(`Aborting request ${req.method()}: ${req.url()}`);
|
|
64
66
|
return req.abort();
|
|
65
67
|
}
|
|
66
68
|
const interceptor = requestInterceptors.find((r) => req.url().match(r.pattern));
|
|
@@ -122,5 +124,6 @@ export default class ScreenshotPost extends BrowserHTTPRoute {
|
|
|
122
124
|
readStream.end(buffer);
|
|
123
125
|
await new Promise((r) => readStream.pipe(res).once('close', r));
|
|
124
126
|
page.close().catch(noop);
|
|
127
|
+
logger.info('Screenshot API request completed');
|
|
125
128
|
};
|
|
126
129
|
}
|