@browserless.io/browserless 2.7.1 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -1
- package/README.md +41 -3
- package/assets/debugger.png +0 -0
- package/bin/browserless.js +8 -4
- package/bin/scaffold/README.md +6 -4
- package/bin/scaffold/src/hello-world.http.ts +7 -1
- package/build/browserless.d.ts +8 -5
- package/build/browserless.js +23 -22
- package/build/browsers/chrome.cdp.d.ts +2 -2
- package/build/browsers/chrome.cdp.js +2 -2
- package/build/browsers/chrome.playwright.d.ts +2 -2
- package/build/browsers/chrome.playwright.js +2 -2
- package/build/browsers/chromium.cdp.d.ts +4 -4
- package/build/browsers/chromium.cdp.js +49 -32
- package/build/browsers/chromium.playwright.d.ts +4 -4
- package/build/browsers/chromium.playwright.js +14 -13
- package/build/browsers/firefox.playwright.d.ts +4 -4
- package/build/browsers/firefox.playwright.js +14 -13
- package/build/browsers/index.d.ts +24 -8
- package/build/browsers/index.js +20 -15
- package/build/browsers/webkit.playwright.d.ts +4 -4
- package/build/browsers/webkit.playwright.js +14 -13
- package/build/config.d.ts +3 -0
- package/build/config.js +3 -0
- package/build/exports.d.ts +1 -0
- package/build/exports.js +1 -0
- package/build/file-system.d.ts +2 -3
- package/build/file-system.js +2 -2
- package/build/index.js +7 -7
- package/build/limiter.d.ts +2 -3
- package/build/limiter.js +11 -11
- package/build/logger.d.ts +19 -0
- package/build/logger.js +43 -0
- package/build/monitoring.d.ts +2 -3
- package/build/monitoring.js +4 -4
- package/build/router.d.ts +4 -5
- package/build/router.js +31 -28
- package/build/routes/chrome/http/content.post.body.json +8 -8
- package/build/routes/chrome/http/pdf.post.body.json +9 -9
- package/build/routes/chrome/http/scrape.post.body.json +8 -8
- package/build/routes/chrome/http/screenshot.post.body.json +8 -8
- package/build/routes/chromium/http/content.post.body.json +8 -8
- package/build/routes/chromium/http/pdf.post.body.json +9 -9
- package/build/routes/chromium/http/scrape.post.body.json +8 -8
- package/build/routes/chromium/http/screenshot.post.body.json +8 -8
- package/build/routes/firefox/ws/playwright.d.ts +2 -2
- package/build/routes/firefox/ws/playwright.js +1 -1
- package/build/routes/management/http/static.get.d.ts +2 -2
- package/build/routes/management/http/static.get.js +8 -10
- package/build/routes/management/tests/management.spec.js +9 -0
- package/build/routes/webkit/ws/playwright.d.ts +2 -2
- package/build/routes/webkit/ws/playwright.js +1 -1
- package/build/sdk-utils.js +23 -10
- package/build/server.d.ts +4 -5
- package/build/server.js +38 -33
- package/build/shared/browser.ws.d.ts +2 -2
- package/build/shared/browser.ws.js +1 -1
- package/build/shared/chromium.playwright.ws.d.ts +2 -2
- package/build/shared/chromium.playwright.ws.js +1 -1
- package/build/shared/chromium.ws.d.ts +2 -2
- package/build/shared/chromium.ws.js +1 -1
- package/build/shared/content.http.d.ts +2 -2
- package/build/shared/content.http.js +4 -1
- package/build/shared/download.http.d.ts +2 -2
- package/build/shared/download.http.js +11 -12
- package/build/shared/function.http.d.ts +2 -2
- package/build/shared/function.http.js +4 -5
- package/build/shared/json-protocol.http.d.ts +3 -3
- package/build/shared/json-protocol.http.js +2 -2
- package/build/shared/json-version.http.d.ts +3 -3
- package/build/shared/json-version.http.js +2 -2
- package/build/shared/page.ws.d.ts +2 -2
- package/build/shared/page.ws.js +1 -1
- package/build/shared/pdf.http.d.ts +2 -2
- package/build/shared/pdf.http.js +6 -4
- package/build/shared/performance.http.d.ts +2 -2
- package/build/shared/performance.http.js +2 -1
- package/build/shared/scrape.http.d.ts +2 -2
- package/build/shared/scrape.http.js +4 -1
- package/build/shared/screenshot.http.d.ts +2 -2
- package/build/shared/screenshot.http.js +4 -1
- package/build/shared/utils/function/handler.d.ts +2 -3
- package/build/shared/utils/function/handler.js +8 -8
- package/build/shared/utils/performance/child.js +4 -4
- package/build/shared/utils/performance/main.d.ts +1 -1
- package/build/shared/utils/performance/main.js +5 -7
- package/build/shared/utils/performance/types.d.ts +2 -1
- package/build/types.d.ts +6 -15
- package/build/types.js +1 -10
- package/build/utils.d.ts +1 -1
- package/build/utils.js +6 -2
- package/package.json +18 -15
- package/scripts/install-debugger.js +55 -15
- package/src/browserless.ts +29 -21
- package/src/browsers/chrome.cdp.ts +2 -5
- package/src/browsers/chrome.playwright.ts +2 -5
- package/src/browsers/chromium.cdp.ts +84 -35
- package/src/browsers/chromium.playwright.ts +26 -13
- package/src/browsers/firefox.playwright.ts +28 -13
- package/src/browsers/index.ts +24 -16
- package/src/browsers/webkit.playwright.ts +28 -13
- package/src/config.ts +4 -0
- package/src/exports.ts +1 -0
- package/src/file-system.ts +2 -7
- package/src/index.ts +7 -7
- package/src/limiter.ts +13 -11
- package/src/logger.ts +52 -0
- package/src/monitoring.ts +6 -8
- package/src/router.ts +29 -27
- package/src/routes/firefox/ws/playwright.ts +2 -0
- package/src/routes/management/http/static.get.ts +13 -10
- package/src/routes/management/tests/management.spec.ts +15 -0
- package/src/routes/webkit/ws/playwright.ts +2 -0
- package/src/sdk-utils.ts +20 -2
- package/src/server.ts +47 -32
- package/src/shared/browser.ws.ts +2 -0
- package/src/shared/chromium.playwright.ws.ts +2 -0
- package/src/shared/chromium.ws.ts +2 -0
- package/src/shared/content.http.ts +6 -0
- package/src/shared/download.http.ts +14 -11
- package/src/shared/function.http.ts +5 -4
- package/src/shared/json-protocol.http.ts +8 -3
- package/src/shared/json-version.http.ts +8 -4
- package/src/shared/page.ws.ts +2 -0
- package/src/shared/pdf.http.ts +7 -3
- package/src/shared/performance.http.ts +3 -0
- package/src/shared/scrape.http.ts +6 -0
- package/src/shared/screenshot.http.ts +5 -0
- package/src/shared/utils/function/handler.ts +9 -13
- package/src/shared/utils/performance/child.ts +4 -4
- package/src/shared/utils/performance/main.ts +5 -6
- package/src/shared/utils/performance/types.ts +2 -1
- package/src/types.ts +5 -9
- package/src/utils.ts +7 -2
- package/static/docs/swagger.json +11 -11
- package/static/docs/swagger.min.json +10 -10
- package/static/function/client.js +1656 -2916
- package/static/function/index.html +1656 -2916
package/build/server.d.ts
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
/// <reference types="debug" />
|
|
3
2
|
/// <reference types="node" />
|
|
4
3
|
/// <reference types="node" />
|
|
5
4
|
/// <reference types="node" />
|
|
6
5
|
import * as http from 'http';
|
|
7
6
|
import * as stream from 'stream';
|
|
8
|
-
import { Config, Hooks, Metrics, Request, Response, Router, Token } from '@browserless.io/browserless';
|
|
7
|
+
import { Logger as BlessLogger, Config, Hooks, Metrics, Request, Response, Router, Token } from '@browserless.io/browserless';
|
|
9
8
|
import { EventEmitter } from 'events';
|
|
10
9
|
export interface HTTPServerOptions {
|
|
11
10
|
concurrent: number;
|
|
@@ -20,12 +19,12 @@ export declare class HTTPServer extends EventEmitter {
|
|
|
20
19
|
protected token: Token;
|
|
21
20
|
protected router: Router;
|
|
22
21
|
protected hooks: Hooks;
|
|
22
|
+
protected Logger: typeof BlessLogger;
|
|
23
23
|
protected server: http.Server;
|
|
24
24
|
protected port: number;
|
|
25
25
|
protected host?: string;
|
|
26
|
-
protected
|
|
27
|
-
|
|
28
|
-
constructor(config: Config, metrics: Metrics, token: Token, router: Router, hooks: Hooks);
|
|
26
|
+
protected logger: BlessLogger;
|
|
27
|
+
constructor(config: Config, metrics: Metrics, token: Token, router: Router, hooks: Hooks, Logger: typeof BlessLogger);
|
|
29
28
|
protected onHTTPUnauthorized: (_req: Request, res: Response) => void;
|
|
30
29
|
protected onWebsocketUnauthorized: (_req: Request, socket: stream.Duplex) => void;
|
|
31
30
|
start(): Promise<void>;
|
package/build/server.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as http from 'http';
|
|
2
|
-
import { BadRequest, NotFound, Timeout, TooManyRequests, Unauthorized, contentTypes, convertPathToURL,
|
|
2
|
+
import { BadRequest, Logger as BlessLogger, NotFound, Timeout, TooManyRequests, Unauthorized, contentTypes, convertPathToURL, queryParamsToObject, readBody, shimLegacyRequests, writeResponse, } from '@browserless.io/browserless';
|
|
3
3
|
import { EventEmitter } from 'events';
|
|
4
4
|
// @ts-ignore
|
|
5
5
|
import Enjoi from 'enjoi';
|
|
@@ -9,34 +9,35 @@ export class HTTPServer extends EventEmitter {
|
|
|
9
9
|
token;
|
|
10
10
|
router;
|
|
11
11
|
hooks;
|
|
12
|
+
Logger;
|
|
12
13
|
server = http.createServer();
|
|
13
14
|
port;
|
|
14
15
|
host;
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
constructor(config, metrics, token, router, hooks) {
|
|
16
|
+
logger = new BlessLogger('server');
|
|
17
|
+
constructor(config, metrics, token, router, hooks, Logger) {
|
|
18
18
|
super();
|
|
19
19
|
this.config = config;
|
|
20
20
|
this.metrics = metrics;
|
|
21
21
|
this.token = token;
|
|
22
22
|
this.router = router;
|
|
23
23
|
this.hooks = hooks;
|
|
24
|
+
this.Logger = Logger;
|
|
24
25
|
this.host = config.getHost();
|
|
25
26
|
this.port = config.getPort();
|
|
26
|
-
this.
|
|
27
|
+
this.logger.info(`Server instantiated with host "${this.host}" on port "${this.port}" using token "${this.config.getToken()}"`);
|
|
27
28
|
}
|
|
28
29
|
onHTTPUnauthorized = (_req, res) => {
|
|
29
|
-
this.
|
|
30
|
+
this.logger.error(`HTTP request is not properly authorized, responding with 401`);
|
|
30
31
|
this.metrics.addUnauthorized();
|
|
31
32
|
return writeResponse(res, 401, 'Bad or missing authentication.');
|
|
32
33
|
};
|
|
33
34
|
onWebsocketUnauthorized = (_req, socket) => {
|
|
34
|
-
this.
|
|
35
|
+
this.logger.error(`Websocket request is not properly authorized, responding with 401`);
|
|
35
36
|
this.metrics.addUnauthorized();
|
|
36
37
|
return writeResponse(socket, 401, 'Bad or missing authentication.');
|
|
37
38
|
};
|
|
38
39
|
async start() {
|
|
39
|
-
this.
|
|
40
|
+
this.logger.info(`HTTP Server is starting`);
|
|
40
41
|
this.server.on('request', this.handleRequest);
|
|
41
42
|
this.server.on('upgrade', this.handleWebSocket);
|
|
42
43
|
const listenMessage = [
|
|
@@ -48,13 +49,13 @@ export class HTTPServer extends EventEmitter {
|
|
|
48
49
|
host: this.host,
|
|
49
50
|
port: this.port,
|
|
50
51
|
}, undefined, () => {
|
|
51
|
-
this.
|
|
52
|
+
this.logger.info(listenMessage);
|
|
52
53
|
r(undefined);
|
|
53
54
|
});
|
|
54
55
|
});
|
|
55
56
|
}
|
|
56
57
|
handleRequest = async (request, res) => {
|
|
57
|
-
this.
|
|
58
|
+
this.logger.trace(`Handling inbound HTTP request on "${request.method}: ${request.url}"`);
|
|
58
59
|
const req = request;
|
|
59
60
|
const proceed = await this.hooks.before({ req, res });
|
|
60
61
|
req.parsed = convertPathToURL(request.url || '', this.config);
|
|
@@ -68,6 +69,10 @@ export class HTTPServer extends EventEmitter {
|
|
|
68
69
|
return res.end();
|
|
69
70
|
}
|
|
70
71
|
}
|
|
72
|
+
if (req.method?.toLowerCase() === 'head') {
|
|
73
|
+
this.logger.debug(`Inbound HEAD request, setting to GET`);
|
|
74
|
+
req.method = 'GET';
|
|
75
|
+
}
|
|
71
76
|
if (this.config.getAllowGetCalls() &&
|
|
72
77
|
req.method === 'GET' &&
|
|
73
78
|
req.parsed.searchParams.has('body')) {
|
|
@@ -78,13 +83,13 @@ export class HTTPServer extends EventEmitter {
|
|
|
78
83
|
}
|
|
79
84
|
const route = await this.router.getRouteForHTTPRequest(req);
|
|
80
85
|
if (!route) {
|
|
81
|
-
this.
|
|
86
|
+
this.logger.error(`No matching HTTP route handler for "${req.method}: ${req.parsed.href}"`);
|
|
82
87
|
writeResponse(res, 404, 'Not Found');
|
|
83
88
|
return Promise.resolve();
|
|
84
89
|
}
|
|
85
|
-
this.
|
|
90
|
+
this.logger.trace(`Found matching HTTP route handler "${route.path}"`);
|
|
86
91
|
if (route?.auth) {
|
|
87
|
-
this.
|
|
92
|
+
this.logger.trace(`Authorizing HTTP request to "${request.url}"`);
|
|
88
93
|
const isPermitted = await this.token.isAuthorized(req, route);
|
|
89
94
|
if (!isPermitted) {
|
|
90
95
|
return this.onHTTPUnauthorized(req, res);
|
|
@@ -102,7 +107,7 @@ export class HTTPServer extends EventEmitter {
|
|
|
102
107
|
return Promise.resolve();
|
|
103
108
|
}
|
|
104
109
|
if (route.querySchema) {
|
|
105
|
-
this.
|
|
110
|
+
this.logger.trace(`Validating route query-params with QUERY schema`);
|
|
106
111
|
try {
|
|
107
112
|
const schema = Enjoi.schema(route.querySchema);
|
|
108
113
|
const valid = schema.validate(req.queryParams, {
|
|
@@ -112,19 +117,19 @@ export class HTTPServer extends EventEmitter {
|
|
|
112
117
|
const errorDetails = valid.error.details
|
|
113
118
|
.map(({ message, context, }) => context?.message || message)
|
|
114
119
|
.join('\n');
|
|
115
|
-
this.
|
|
120
|
+
this.logger.error(`HTTP query-params contain errors sending 400:${errorDetails}`);
|
|
116
121
|
writeResponse(res, 400, `Query-parameter validation failed: ${errorDetails}`, contentTypes.text);
|
|
117
122
|
return Promise.resolve();
|
|
118
123
|
}
|
|
119
124
|
}
|
|
120
125
|
catch (e) {
|
|
121
|
-
this.
|
|
126
|
+
this.logger.error(`Error parsing body schema`, e);
|
|
122
127
|
writeResponse(res, 500, 'There was an error handling your request', contentTypes.text);
|
|
123
128
|
return Promise.resolve();
|
|
124
129
|
}
|
|
125
130
|
}
|
|
126
131
|
if (route.bodySchema) {
|
|
127
|
-
this.
|
|
132
|
+
this.logger.trace(`Validating route payload with BODY schema`);
|
|
128
133
|
try {
|
|
129
134
|
const schema = Enjoi.schema(route.bodySchema);
|
|
130
135
|
const valid = schema.validate(body, { abortEarly: false });
|
|
@@ -132,21 +137,21 @@ export class HTTPServer extends EventEmitter {
|
|
|
132
137
|
const errorDetails = valid.error.details
|
|
133
138
|
.map(({ message, context, }) => context?.message || message)
|
|
134
139
|
.join('\n');
|
|
135
|
-
this.
|
|
140
|
+
this.logger.error(`HTTP body contain errors sending 400:${errorDetails}`);
|
|
136
141
|
writeResponse(res, 400, `POST Body validation failed: ${errorDetails}`, contentTypes.text);
|
|
137
142
|
return Promise.resolve();
|
|
138
143
|
}
|
|
139
144
|
}
|
|
140
145
|
catch (e) {
|
|
141
|
-
this.
|
|
146
|
+
this.logger.error(`Error parsing body schema`, e);
|
|
142
147
|
writeResponse(res, 500, 'There was an error handling your request', contentTypes.text);
|
|
143
148
|
return Promise.resolve();
|
|
144
149
|
}
|
|
145
150
|
}
|
|
146
151
|
return route
|
|
147
|
-
.handler(req, res)
|
|
152
|
+
.handler(req, res, new this.Logger(route.name, req))
|
|
148
153
|
.then(() => {
|
|
149
|
-
this.
|
|
154
|
+
this.logger.trace('HTTP connection complete');
|
|
150
155
|
})
|
|
151
156
|
.catch((e) => {
|
|
152
157
|
if (e instanceof BadRequest) {
|
|
@@ -168,7 +173,7 @@ export class HTTPServer extends EventEmitter {
|
|
|
168
173
|
});
|
|
169
174
|
};
|
|
170
175
|
handleWebSocket = async (request, socket, head) => {
|
|
171
|
-
this.
|
|
176
|
+
this.logger.trace(`Handling inbound WebSocket request on "${request.url}"`);
|
|
172
177
|
const req = request;
|
|
173
178
|
const proceed = await this.hooks.before({ head, req, socket });
|
|
174
179
|
req.parsed = convertPathToURL(request.url || '', this.config);
|
|
@@ -178,16 +183,16 @@ export class HTTPServer extends EventEmitter {
|
|
|
178
183
|
req.queryParams = queryParamsToObject(req.parsed.searchParams);
|
|
179
184
|
const route = await this.router.getRouteForWebSocketRequest(req);
|
|
180
185
|
if (route) {
|
|
181
|
-
this.
|
|
186
|
+
this.logger.trace(`Found matching WebSocket route handler "${route.path}"`);
|
|
182
187
|
if (route?.auth) {
|
|
183
|
-
this.
|
|
188
|
+
this.logger.trace(`Authorizing WebSocket request to "${req.parsed.href}"`);
|
|
184
189
|
const isPermitted = await this.token.isAuthorized(req, route);
|
|
185
190
|
if (!isPermitted) {
|
|
186
191
|
return this.onWebsocketUnauthorized(req, socket);
|
|
187
192
|
}
|
|
188
193
|
}
|
|
189
194
|
if (route.querySchema) {
|
|
190
|
-
this.
|
|
195
|
+
this.logger.trace(`Validating route query-params with QUERY schema`);
|
|
191
196
|
try {
|
|
192
197
|
const schema = Enjoi.schema(route.querySchema);
|
|
193
198
|
const valid = schema.validate(req.queryParams, {
|
|
@@ -197,21 +202,21 @@ export class HTTPServer extends EventEmitter {
|
|
|
197
202
|
const errorDetails = valid.error.details
|
|
198
203
|
.map(({ message, context, }) => context?.message || message)
|
|
199
204
|
.join('\n');
|
|
200
|
-
this.
|
|
205
|
+
this.logger.error(`WebSocket query-params contain errors sending 400:${errorDetails}`);
|
|
201
206
|
writeResponse(socket, 400, `Query-parameter validation failed: ${errorDetails}`, contentTypes.text);
|
|
202
207
|
return Promise.resolve();
|
|
203
208
|
}
|
|
204
209
|
}
|
|
205
210
|
catch (e) {
|
|
206
|
-
this.
|
|
211
|
+
this.logger.error(`Error parsing query-params schema`, e);
|
|
207
212
|
writeResponse(socket, 500, 'There was an error handling your request', contentTypes.text);
|
|
208
213
|
return Promise.resolve();
|
|
209
214
|
}
|
|
210
215
|
}
|
|
211
216
|
return route
|
|
212
|
-
.handler(req, socket, head)
|
|
217
|
+
.handler(req, socket, head, new this.Logger(route.name, req))
|
|
213
218
|
.then(() => {
|
|
214
|
-
this.
|
|
219
|
+
this.logger.trace('Websocket connection complete');
|
|
215
220
|
})
|
|
216
221
|
.catch((e) => {
|
|
217
222
|
if (e instanceof BadRequest) {
|
|
@@ -226,20 +231,20 @@ export class HTTPServer extends EventEmitter {
|
|
|
226
231
|
if (e instanceof TooManyRequests) {
|
|
227
232
|
return writeResponse(socket, 429, e.message);
|
|
228
233
|
}
|
|
229
|
-
this.
|
|
234
|
+
this.logger.error(`Error handling request at "${route.path}": ${e}\n${e.stack}`);
|
|
230
235
|
return writeResponse(socket, 500, e.message);
|
|
231
236
|
});
|
|
232
237
|
}
|
|
233
|
-
this.
|
|
238
|
+
this.logger.error(`No matching WebSocket route handler for "${req.parsed.href}"`);
|
|
234
239
|
return writeResponse(socket, 404, 'Not Found');
|
|
235
240
|
};
|
|
236
241
|
async shutdown() {
|
|
237
|
-
this.
|
|
242
|
+
this.logger.info(`HTTP Server is shutting down`);
|
|
238
243
|
await new Promise((r) => this.server.close(r));
|
|
239
244
|
this.server && this.server.removeAllListeners();
|
|
240
245
|
// @ts-ignore garbage collect this reference
|
|
241
246
|
this.server = null;
|
|
242
|
-
this.
|
|
247
|
+
this.logger.info(`HTTP Server shutdown complete`);
|
|
243
248
|
}
|
|
244
249
|
/**
|
|
245
250
|
* Left blank for downstream SDK modules to optionally implement.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
/// <reference types="node" />
|
|
3
|
-
import { APITags, BrowserWebsocketRoute, CDPLaunchOptions, ChromiumCDP, Request, SystemQueryParameters, WebsocketRoutes } from '@browserless.io/browserless';
|
|
3
|
+
import { APITags, BrowserWebsocketRoute, CDPLaunchOptions, ChromiumCDP, Logger, Request, SystemQueryParameters, WebsocketRoutes } from '@browserless.io/browserless';
|
|
4
4
|
import { Duplex } from 'stream';
|
|
5
5
|
export interface QuerySchema extends SystemQueryParameters {
|
|
6
6
|
launch?: CDPLaunchOptions | string;
|
|
@@ -13,5 +13,5 @@ export default class ChromiumBrowserWebSocketRoute extends BrowserWebsocketRoute
|
|
|
13
13
|
description: string;
|
|
14
14
|
path: WebsocketRoutes;
|
|
15
15
|
tags: APITags[];
|
|
16
|
-
handler: (req: Request, socket: Duplex, head: Buffer, browser: ChromiumCDP) => Promise<void>;
|
|
16
|
+
handler: (req: Request, socket: Duplex, head: Buffer, _logger: Logger, browser: ChromiumCDP) => Promise<void>;
|
|
17
17
|
}
|
|
@@ -9,5 +9,5 @@ export default class ChromiumBrowserWebSocketRoute extends BrowserWebsocketRoute
|
|
|
9
9
|
must already be launched in order to not return a 404.`);
|
|
10
10
|
path = WebsocketRoutes.browser;
|
|
11
11
|
tags = [APITags.browserWS];
|
|
12
|
-
handler = async (req, socket, head, browser) => browser.proxyWebSocket(req, socket, head);
|
|
12
|
+
handler = async (req, socket, head, _logger, browser) => browser.proxyWebSocket(req, socket, head);
|
|
13
13
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
/// <reference types="node" />
|
|
3
|
-
import { APITags, BrowserServerOptions, BrowserWebsocketRoute, ChromiumPlaywright, Request, SystemQueryParameters, WebsocketRoutes } from '@browserless.io/browserless';
|
|
3
|
+
import { APITags, BrowserServerOptions, BrowserWebsocketRoute, ChromiumPlaywright, Logger, Request, SystemQueryParameters, WebsocketRoutes } from '@browserless.io/browserless';
|
|
4
4
|
import { Duplex } from 'stream';
|
|
5
5
|
export interface QuerySchema extends SystemQueryParameters {
|
|
6
6
|
launch?: BrowserServerOptions | string;
|
|
@@ -13,5 +13,5 @@ export default class ChromiumPlaywrightWebSocketRoute extends BrowserWebsocketRo
|
|
|
13
13
|
description: string;
|
|
14
14
|
path: WebsocketRoutes[];
|
|
15
15
|
tags: APITags[];
|
|
16
|
-
handler: (req: Request, socket: Duplex, head: Buffer, browser: ChromiumPlaywright) => Promise<void>;
|
|
16
|
+
handler: (req: Request, socket: Duplex, head: Buffer, _logger: Logger, browser: ChromiumPlaywright) => Promise<void>;
|
|
17
17
|
}
|
|
@@ -10,7 +10,7 @@ export default class ChromiumPlaywrightWebSocketRoute extends BrowserWebsocketRo
|
|
|
10
10
|
WebsocketRoutes.chromiumPlaywright,
|
|
11
11
|
];
|
|
12
12
|
tags = [APITags.browserWS];
|
|
13
|
-
handler = async (req, socket, head, browser) => {
|
|
13
|
+
handler = async (req, socket, head, _logger, browser) => {
|
|
14
14
|
const isPlaywright = req.headers['user-agent']
|
|
15
15
|
?.toLowerCase()
|
|
16
16
|
.includes('playwright');
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
/// <reference types="node" />
|
|
3
|
-
import { APITags, BrowserWebsocketRoute, CDPLaunchOptions, ChromiumCDP, Request, SystemQueryParameters, WebsocketRoutes } from '@browserless.io/browserless';
|
|
3
|
+
import { APITags, BrowserWebsocketRoute, CDPLaunchOptions, ChromiumCDP, Logger, Request, SystemQueryParameters, WebsocketRoutes } from '@browserless.io/browserless';
|
|
4
4
|
import { Duplex } from 'stream';
|
|
5
5
|
export interface QuerySchema extends SystemQueryParameters {
|
|
6
6
|
launch?: CDPLaunchOptions | string;
|
|
@@ -13,5 +13,5 @@ export default class ChromiumCDPWebSocketRoute extends BrowserWebsocketRoute {
|
|
|
13
13
|
description: string;
|
|
14
14
|
path: WebsocketRoutes[];
|
|
15
15
|
tags: APITags[];
|
|
16
|
-
handler: (req: Request, socket: Duplex, head: Buffer, browser: ChromiumCDP) => Promise<void>;
|
|
16
|
+
handler: (req: Request, socket: Duplex, head: Buffer, _logger: Logger, browser: ChromiumCDP) => Promise<void>;
|
|
17
17
|
}
|
|
@@ -7,5 +7,5 @@ export default class ChromiumCDPWebSocketRoute extends BrowserWebsocketRoute {
|
|
|
7
7
|
description = `Launch and connect to Chromium with a library like puppeteer or others that work over chrome-devtools-protocol.`;
|
|
8
8
|
path = [WebsocketRoutes['/'], WebsocketRoutes.chromium];
|
|
9
9
|
tags = [APITags.browserWS];
|
|
10
|
-
handler = async (req, socket, head, browser) => browser.proxyWebSocket(req, socket, head);
|
|
10
|
+
handler = async (req, socket, head, _logger, browser) => browser.proxyWebSocket(req, socket, head);
|
|
11
11
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import { APITags, BrowserHTTPRoute, BrowserInstance, CDPLaunchOptions, ChromiumCDP, HTTPRoutes, Methods, Request, SystemQueryParameters, WaitForEventOptions, WaitForFunctionOptions, WaitForSelectorOptions, bestAttempt, contentTypes, rejectRequestPattern, rejectResourceTypes, requestInterceptors, setJavaScriptEnabled } from '@browserless.io/browserless';
|
|
2
|
+
import { APITags, BrowserHTTPRoute, BrowserInstance, CDPLaunchOptions, ChromiumCDP, HTTPRoutes, Logger, Methods, Request, SystemQueryParameters, WaitForEventOptions, WaitForFunctionOptions, WaitForSelectorOptions, bestAttempt, contentTypes, rejectRequestPattern, rejectResourceTypes, requestInterceptors, setJavaScriptEnabled } from '@browserless.io/browserless';
|
|
3
3
|
import { Page } from 'puppeteer-core';
|
|
4
4
|
import { ServerResponse } from 'http';
|
|
5
5
|
export interface BodySchema {
|
|
@@ -43,5 +43,5 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
|
|
|
43
43
|
method: Methods;
|
|
44
44
|
path: HTTPRoutes[];
|
|
45
45
|
tags: APITags[];
|
|
46
|
-
handler: (req: Request, res: ServerResponse, browser: BrowserInstance) => Promise<void>;
|
|
46
|
+
handler: (req: Request, res: ServerResponse, logger: Logger, browser: BrowserInstance) => Promise<void>;
|
|
47
47
|
}
|
|
@@ -10,7 +10,8 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
|
|
|
10
10
|
method = Methods.post;
|
|
11
11
|
path = [HTTPRoutes.content, HTTPRoutes.chromiumContent];
|
|
12
12
|
tags = [APITags.browserAPI];
|
|
13
|
-
handler = async (req, res, browser) => {
|
|
13
|
+
handler = async (req, res, logger, browser) => {
|
|
14
|
+
logger.info('Content API invoked with body:', req.body);
|
|
14
15
|
const contentType = !req.headers.accept || req.headers.accept?.includes('*')
|
|
15
16
|
? contentTypes.html
|
|
16
17
|
: req.headers.accept;
|
|
@@ -53,6 +54,7 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
|
|
|
53
54
|
page.on('request', (req) => {
|
|
54
55
|
if (!!rejectRequestPattern.find((pattern) => req.url().match(pattern)) ||
|
|
55
56
|
rejectResourceTypes.includes(req.resourceType())) {
|
|
57
|
+
logger.debug(`Aborting request ${req.method()}: ${req.url()}`);
|
|
56
58
|
return req.abort();
|
|
57
59
|
}
|
|
58
60
|
const interceptor = requestInterceptors.find((r) => req.url().match(r.pattern));
|
|
@@ -102,6 +104,7 @@ export default class ChromiumContentPostRoute extends BrowserHTTPRoute {
|
|
|
102
104
|
}
|
|
103
105
|
const markup = await page.content();
|
|
104
106
|
page.close().catch(noop);
|
|
107
|
+
logger.info('Content API request completed');
|
|
105
108
|
return writeResponse(res, 200, markup, contentTypes.html);
|
|
106
109
|
};
|
|
107
110
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import { APITags, BrowserHTTPRoute, BrowserInstance, CDPLaunchOptions, ChromiumCDP, HTTPRoutes, Methods, Request, SystemQueryParameters, contentTypes } from '@browserless.io/browserless';
|
|
2
|
+
import { APITags, BrowserHTTPRoute, BrowserInstance, CDPLaunchOptions, ChromiumCDP, HTTPRoutes, Logger, Methods, Request, SystemQueryParameters, contentTypes } from '@browserless.io/browserless';
|
|
3
3
|
import { ServerResponse } from 'http';
|
|
4
4
|
interface JSONSchema {
|
|
5
5
|
code: string;
|
|
@@ -25,6 +25,6 @@ export default class ChromiumDownloadPostRoute extends BrowserHTTPRoute {
|
|
|
25
25
|
method: Methods;
|
|
26
26
|
path: HTTPRoutes[];
|
|
27
27
|
tags: APITags[];
|
|
28
|
-
handler: (req: Request, res: ServerResponse, browser: BrowserInstance) => Promise<void>;
|
|
28
|
+
handler: (req: Request, res: ServerResponse, logger: Logger, browser: BrowserInstance) => Promise<void>;
|
|
29
29
|
}
|
|
30
30
|
export {};
|
|
@@ -21,15 +21,14 @@ export default class ChromiumDownloadPostRoute extends BrowserHTTPRoute {
|
|
|
21
21
|
method = Methods.post;
|
|
22
22
|
path = [HTTPRoutes.download, HTTPRoutes.chromiumDownload];
|
|
23
23
|
tags = [APITags.browserAPI];
|
|
24
|
-
handler = async (req, res, browser) => new Promise(async (resolve, reject) => {
|
|
25
|
-
const debug = this.debug();
|
|
24
|
+
handler = async (req, res, logger, browser) => new Promise(async (resolve, reject) => {
|
|
26
25
|
const config = this.config();
|
|
27
26
|
const downloadPath = path.join(await config.getDownloadsDir(), `.browserless.download.${id()}`);
|
|
28
|
-
|
|
27
|
+
logger.info(`Generating a download directory at "${downloadPath}"`);
|
|
29
28
|
await mkdir(downloadPath);
|
|
30
|
-
const handler = functionHandler(config,
|
|
29
|
+
const handler = functionHandler(config, logger, { downloadPath });
|
|
31
30
|
const response = await handler(req, browser).catch((e) => {
|
|
32
|
-
|
|
31
|
+
logger.error(`Error running download code handler: "${e}"`);
|
|
33
32
|
reject(e);
|
|
34
33
|
return null;
|
|
35
34
|
});
|
|
@@ -37,10 +36,10 @@ export default class ChromiumDownloadPostRoute extends BrowserHTTPRoute {
|
|
|
37
36
|
return;
|
|
38
37
|
}
|
|
39
38
|
const { page } = response;
|
|
40
|
-
|
|
39
|
+
logger.info(`Download function has returned, finding downloads...`);
|
|
41
40
|
async function checkIfDownloadComplete() {
|
|
42
41
|
if (res.headersSent) {
|
|
43
|
-
|
|
42
|
+
logger.trace(`Request headers have been sent, terminating download watch.`);
|
|
44
43
|
return null;
|
|
45
44
|
}
|
|
46
45
|
const [fileName] = await readdir(downloadPath);
|
|
@@ -48,20 +47,20 @@ export default class ChromiumDownloadPostRoute extends BrowserHTTPRoute {
|
|
|
48
47
|
await sleep(500);
|
|
49
48
|
return checkIfDownloadComplete();
|
|
50
49
|
}
|
|
51
|
-
|
|
50
|
+
logger.info(`All files have finished downloading`);
|
|
52
51
|
return path.join(downloadPath, fileName);
|
|
53
52
|
}
|
|
54
53
|
const filePath = await checkIfDownloadComplete();
|
|
55
|
-
|
|
54
|
+
logger.info(`Closing pages.`);
|
|
56
55
|
page.close();
|
|
57
56
|
page.removeAllListeners();
|
|
58
57
|
const rmDownload = once(() => filePath &&
|
|
59
58
|
deleteAsync(filePath, { force: true })
|
|
60
59
|
.then(() => {
|
|
61
|
-
|
|
60
|
+
logger.info(`Successfully deleted downloads from disk at "${filePath}"`);
|
|
62
61
|
})
|
|
63
62
|
.catch((err) => {
|
|
64
|
-
|
|
63
|
+
logger.error(`Error cleaning up downloaded files: "${err}" at "${filePath}"`);
|
|
65
64
|
}));
|
|
66
65
|
if (res.headersSent || !filePath) {
|
|
67
66
|
rmDownload();
|
|
@@ -79,7 +78,7 @@ export default class ChromiumDownloadPostRoute extends BrowserHTTPRoute {
|
|
|
79
78
|
}
|
|
80
79
|
})
|
|
81
80
|
.on('end', () => {
|
|
82
|
-
|
|
81
|
+
logger.info(`Downloads successfully sent`);
|
|
83
82
|
rmDownload();
|
|
84
83
|
return resolve();
|
|
85
84
|
})
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import { APITags, BrowserHTTPRoute, BrowserInstance, CDPLaunchOptions, ChromiumCDP, HTTPRoutes, Methods, Request, SystemQueryParameters, contentTypes } from '@browserless.io/browserless';
|
|
2
|
+
import { APITags, BrowserHTTPRoute, BrowserInstance, CDPLaunchOptions, ChromiumCDP, HTTPRoutes, Logger, Methods, Request, SystemQueryParameters, contentTypes } from '@browserless.io/browserless';
|
|
3
3
|
import { ServerResponse } from 'http';
|
|
4
4
|
interface JSONSchema {
|
|
5
5
|
code: string;
|
|
@@ -27,6 +27,6 @@ export default class ChromiumFunctionPostRoute extends BrowserHTTPRoute {
|
|
|
27
27
|
method: Methods;
|
|
28
28
|
path: HTTPRoutes[];
|
|
29
29
|
tags: APITags[];
|
|
30
|
-
handler: (req: Request, res: ServerResponse, browser: BrowserInstance) => Promise<void>;
|
|
30
|
+
handler: (req: Request, res: ServerResponse, logger: Logger, browser: BrowserInstance) => Promise<void>;
|
|
31
31
|
}
|
|
32
32
|
export {};
|
|
@@ -18,12 +18,11 @@ export default class ChromiumFunctionPostRoute extends BrowserHTTPRoute {
|
|
|
18
18
|
method = Methods.post;
|
|
19
19
|
path = [HTTPRoutes.function, HTTPRoutes.chromiumFunction];
|
|
20
20
|
tags = [APITags.browserAPI];
|
|
21
|
-
handler = async (req, res, browser) => {
|
|
22
|
-
const debug = this.debug();
|
|
21
|
+
handler = async (req, res, logger, browser) => {
|
|
23
22
|
const config = this.config();
|
|
24
|
-
const handler = functionHandler(config,
|
|
23
|
+
const handler = functionHandler(config, logger);
|
|
25
24
|
const { contentType, payload, page } = await handler(req, browser);
|
|
26
|
-
|
|
25
|
+
logger.info(`Got function response of "${contentType}"`);
|
|
27
26
|
page.close();
|
|
28
27
|
page.removeAllListeners();
|
|
29
28
|
if (contentType === 'uint8array') {
|
|
@@ -34,7 +33,7 @@ export default class ChromiumFunctionPostRoute extends BrowserHTTPRoute {
|
|
|
34
33
|
throw new BadRequest(`Couldn't determine function's response type.`);
|
|
35
34
|
}
|
|
36
35
|
else {
|
|
37
|
-
|
|
36
|
+
logger.info(`Sending file-type response of "${type}"`);
|
|
38
37
|
const readStream = new Stream.PassThrough();
|
|
39
38
|
readStream.end(response);
|
|
40
39
|
res.setHeader('Content-Type', type);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { APITags, HTTPRoute, HTTPRoutes, Methods, Request, Response, contentTypes } from '@browserless.io/browserless';
|
|
1
|
+
import { APITags, HTTPRoute, HTTPRoutes, Logger, Methods, Request, Response, contentTypes } from '@browserless.io/browserless';
|
|
2
2
|
export type ResponseSchema = object;
|
|
3
3
|
export default class ChromiumJSONProtocolGetRoute extends HTTPRoute {
|
|
4
|
-
|
|
4
|
+
protected cachedProtocol: object | undefined;
|
|
5
5
|
name: string;
|
|
6
6
|
accepts: contentTypes[];
|
|
7
7
|
auth: boolean;
|
|
@@ -12,5 +12,5 @@ export default class ChromiumJSONProtocolGetRoute extends HTTPRoute {
|
|
|
12
12
|
method: Methods;
|
|
13
13
|
path: HTTPRoutes;
|
|
14
14
|
tags: APITags[];
|
|
15
|
-
handler: (_req: Request, res: Response) => Promise<void>;
|
|
15
|
+
handler: (_req: Request, res: Response, logger: Logger) => Promise<void>;
|
|
16
16
|
}
|
|
@@ -11,10 +11,10 @@ export default class ChromiumJSONProtocolGetRoute extends HTTPRoute {
|
|
|
11
11
|
method = Methods.get;
|
|
12
12
|
path = HTTPRoutes.jsonProtocol;
|
|
13
13
|
tags = [APITags.browserAPI];
|
|
14
|
-
handler = async (_req, res) => {
|
|
14
|
+
handler = async (_req, res, logger) => {
|
|
15
15
|
const browserManager = this.browserManager();
|
|
16
16
|
if (!this.cachedProtocol) {
|
|
17
|
-
this.cachedProtocol = await browserManager.getProtocolJSON();
|
|
17
|
+
this.cachedProtocol = await browserManager.getProtocolJSON(logger);
|
|
18
18
|
}
|
|
19
19
|
return jsonResponse(res, 200, this.cachedProtocol);
|
|
20
20
|
};
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { APITags, BrowserManager, HTTPRoute, HTTPRoutes, Methods, Request, Response, UnwrapPromise, contentTypes } from '@browserless.io/browserless';
|
|
1
|
+
import { APITags, BrowserManager, HTTPRoute, HTTPRoutes, Logger, Methods, Request, Response, UnwrapPromise, contentTypes } from '@browserless.io/browserless';
|
|
2
2
|
export type ResponseSchema = UnwrapPromise<ReturnType<BrowserManager['getVersionJSON']>>;
|
|
3
3
|
export default class ChromiumJSONVersionGetRoute extends HTTPRoute {
|
|
4
|
-
|
|
4
|
+
protected cachedJSON: ResponseSchema | undefined;
|
|
5
5
|
name: string;
|
|
6
6
|
accepts: contentTypes[];
|
|
7
7
|
auth: boolean;
|
|
@@ -12,5 +12,5 @@ export default class ChromiumJSONVersionGetRoute extends HTTPRoute {
|
|
|
12
12
|
method: Methods;
|
|
13
13
|
path: HTTPRoutes;
|
|
14
14
|
tags: APITags[];
|
|
15
|
-
handler: (req: Request, res: Response) => Promise<void>;
|
|
15
|
+
handler: (req: Request, res: Response, logger: Logger) => Promise<void>;
|
|
16
16
|
}
|
|
@@ -11,14 +11,14 @@ export default class ChromiumJSONVersionGetRoute extends HTTPRoute {
|
|
|
11
11
|
method = Methods.get;
|
|
12
12
|
path = HTTPRoutes.jsonVersion;
|
|
13
13
|
tags = [APITags.browserAPI];
|
|
14
|
-
handler = async (req, res) => {
|
|
14
|
+
handler = async (req, res, logger) => {
|
|
15
15
|
const baseUrl = req.parsed.host;
|
|
16
16
|
const protocol = req.parsed.protocol.includes('s') ? 'wss' : 'ws';
|
|
17
17
|
try {
|
|
18
18
|
if (!this.cachedJSON) {
|
|
19
19
|
const browserManager = this.browserManager();
|
|
20
20
|
this.cachedJSON = {
|
|
21
|
-
...(await browserManager.getVersionJSON()),
|
|
21
|
+
...(await browserManager.getVersionJSON(logger)),
|
|
22
22
|
webSocketDebuggerUrl: `${protocol}://${baseUrl}`,
|
|
23
23
|
};
|
|
24
24
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
/// <reference types="node" />
|
|
3
|
-
import { APITags, BrowserWebsocketRoute, CDPLaunchOptions, ChromiumCDP, Request, SystemQueryParameters, WebsocketRoutes } from '@browserless.io/browserless';
|
|
3
|
+
import { APITags, BrowserWebsocketRoute, CDPLaunchOptions, ChromiumCDP, Logger, Request, SystemQueryParameters, WebsocketRoutes } from '@browserless.io/browserless';
|
|
4
4
|
import { Duplex } from 'stream';
|
|
5
5
|
export interface QuerySchema extends SystemQueryParameters {
|
|
6
6
|
launch?: CDPLaunchOptions | string;
|
|
@@ -13,5 +13,5 @@ export default class ChromiumPageWebSocketRoute extends BrowserWebsocketRoute {
|
|
|
13
13
|
description: string;
|
|
14
14
|
path: WebsocketRoutes;
|
|
15
15
|
tags: APITags[];
|
|
16
|
-
handler: (req: Request, socket: Duplex, head: Buffer, browser: ChromiumCDP) => Promise<void>;
|
|
16
|
+
handler: (req: Request, socket: Duplex, head: Buffer, _logger: Logger, browser: ChromiumCDP) => Promise<void>;
|
|
17
17
|
}
|
package/build/shared/page.ws.js
CHANGED
|
@@ -10,5 +10,5 @@ export default class ChromiumPageWebSocketRoute extends BrowserWebsocketRoute {
|
|
|
10
10
|
or by finding the page's unique ID from your library of choice.`);
|
|
11
11
|
path = WebsocketRoutes.page;
|
|
12
12
|
tags = [APITags.browserWS];
|
|
13
|
-
handler = async (req, socket, head, browser) => browser.proxyPageWebSocket(req, socket, head);
|
|
13
|
+
handler = async (req, socket, head, _logger, browser) => browser.proxyPageWebSocket(req, socket, head);
|
|
14
14
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
-
import { APITags, BrowserHTTPRoute, BrowserInstance, CDPLaunchOptions, ChromiumCDP, HTTPRoutes, Methods, Request, SystemQueryParameters, WaitForEventOptions, WaitForFunctionOptions, WaitForSelectorOptions, bestAttempt, contentTypes, rejectRequestPattern, rejectResourceTypes, requestInterceptors } from '@browserless.io/browserless';
|
|
2
|
+
import { APITags, BrowserHTTPRoute, BrowserInstance, CDPLaunchOptions, ChromiumCDP, HTTPRoutes, Logger, Methods, Request, SystemQueryParameters, WaitForEventOptions, WaitForFunctionOptions, WaitForSelectorOptions, bestAttempt, contentTypes, rejectRequestPattern, rejectResourceTypes, requestInterceptors } from '@browserless.io/browserless';
|
|
3
3
|
import { Page } from 'puppeteer-core';
|
|
4
4
|
import { ServerResponse } from 'http';
|
|
5
5
|
export interface BodySchema {
|
|
@@ -43,5 +43,5 @@ export default class ChromiumPDFPostRoute extends BrowserHTTPRoute {
|
|
|
43
43
|
method: Methods;
|
|
44
44
|
path: HTTPRoutes[];
|
|
45
45
|
tags: APITags[];
|
|
46
|
-
handler: (req: Request, res: ServerResponse, browser: BrowserInstance) => Promise<void>;
|
|
46
|
+
handler: (req: Request, res: ServerResponse, logger: Logger, browser: BrowserInstance) => Promise<void>;
|
|
47
47
|
}
|