figranium 0.12.1 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +674 -674
- package/README.md +336 -336
- package/agent.js +1 -1
- package/bin/cli.js +149 -149
- package/common-utils.js +211 -211
- package/dist/assets/{favicon-DmUMR1rm.svg → favicon-DXDXzv5K.svg} +290 -290
- package/dist/assets/index-BaVlGc48.js +18 -0
- package/dist/assets/index-T2xxnq_A.css +1 -0
- package/dist/favicon.svg +290 -290
- package/dist/figranium_icon.svg +290 -290
- package/dist/figranium_logo.svg +60 -60
- package/dist/index.html +26 -26
- package/dist/novnc.html +108 -108
- package/dist/styles.css +86 -86
- package/extraction-worker.js +211 -207
- package/headful.js +584 -569
- package/html-utils.js +24 -24
- package/package.json +82 -82
- package/proxy-rotation.js +261 -261
- package/proxy-utils.js +84 -84
- package/public/favicon.svg +290 -290
- package/public/figranium_icon.svg +290 -290
- package/public/figranium_logo.svg +60 -60
- package/public/novnc.html +108 -108
- package/public/styles.css +86 -86
- package/scrape.js +389 -389
- package/scripts/postinstall.js +21 -21
- package/server.js +626 -626
- package/src/server/cron-parser.js +325 -316
- package/src/server/routes/schedules.js +171 -171
- package/src/server/scheduler.js +379 -379
- package/url-utils.js +339 -323
- package/user-agent-settings.js +76 -76
- package/dist/assets/index-C2rVEs3q.css +0 -1
- package/dist/assets/index-CvaIUcTv.js +0 -18
package/url-utils.js
CHANGED
|
@@ -1,323 +1,339 @@
|
|
|
1
|
-
const dns = require('dns').promises;
|
|
2
|
-
const net = require('net');
|
|
3
|
-
const { ALLOW_PRIVATE_NETWORKS } = require('./src/server/constants');
|
|
4
|
-
|
|
5
|
-
/**
|
|
6
|
-
* Checks if an IP address is private.
|
|
7
|
-
*
|
|
8
|
-
* Qualifies as a private network/blocked destination:
|
|
9
|
-
*
|
|
10
|
-
* IPv4 Ranges:
|
|
11
|
-
* - 0.0.0.0/8 (Current network)
|
|
12
|
-
* - 10.0.0.0/8 (Private-Use Networks - RFC 1918)
|
|
13
|
-
* - 100.64.0.0/10 (Shared Address Space - RFC 6598)
|
|
14
|
-
* - 127.0.0.0/8 (Loopback)
|
|
15
|
-
* - 169.254.0.0/16 (Link-Local)
|
|
16
|
-
* - 172.16.0.0/12 (Private-Use Networks - RFC 1918)
|
|
17
|
-
* - 192.
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
* -
|
|
21
|
-
* -
|
|
22
|
-
* -
|
|
23
|
-
* -
|
|
24
|
-
* -
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
* -
|
|
28
|
-
* -
|
|
29
|
-
* -
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
const
|
|
99
|
-
const
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
) {
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
//
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
//
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
return
|
|
320
|
-
}
|
|
321
|
-
}
|
|
322
|
-
|
|
323
|
-
|
|
1
|
+
const dns = require('dns').promises;
|
|
2
|
+
const net = require('net');
|
|
3
|
+
const { ALLOW_PRIVATE_NETWORKS } = require('./src/server/constants');
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Checks if an IP address is private.
|
|
7
|
+
*
|
|
8
|
+
* Qualifies as a private network/blocked destination:
|
|
9
|
+
*
|
|
10
|
+
* IPv4 Ranges:
|
|
11
|
+
* - 0.0.0.0/8 (Current network)
|
|
12
|
+
* - 10.0.0.0/8 (Private-Use Networks - RFC 1918)
|
|
13
|
+
* - 100.64.0.0/10 (Shared Address Space - RFC 6598)
|
|
14
|
+
* - 127.0.0.0/8 (Loopback)
|
|
15
|
+
* - 169.254.0.0/16 (Link-Local)
|
|
16
|
+
* - 172.16.0.0/12 (Private-Use Networks - RFC 1918)
|
|
17
|
+
* - 192.0.0.0/24 (IETF Protocol Assignments)
|
|
18
|
+
* - 192.0.2.0/24 (TEST-NET-1)
|
|
19
|
+
* - 192.168.0.0/16 (Private-Use Networks - RFC 1918)
|
|
20
|
+
* - 198.18.0.0/15 (Benchmarking)
|
|
21
|
+
* - 198.51.100.0/24 (TEST-NET-2)
|
|
22
|
+
* - 203.0.113.0/24 (TEST-NET-3)
|
|
23
|
+
* - 224.0.0.0/4 (Multicast)
|
|
24
|
+
* - 240.0.0.0/4 (Reserved)
|
|
25
|
+
*
|
|
26
|
+
* IPv6 Ranges:
|
|
27
|
+
* - ::/128 (Unspecified)
|
|
28
|
+
* - ::1/128 (Loopback)
|
|
29
|
+
* - fc00::/7 (Unique Local Address)
|
|
30
|
+
* - fe80::/10 (Link-Local Unicast)
|
|
31
|
+
* - ff00::/8 (Multicast)
|
|
32
|
+
* - IPv4-mapped/compatible addresses pointing to the above IPv4 ranges
|
|
33
|
+
*
|
|
34
|
+
* Hostnames:
|
|
35
|
+
* - localhost
|
|
36
|
+
* - *.localhost
|
|
37
|
+
* - host.docker.internal
|
|
38
|
+
*
|
|
39
|
+
* @param {string} ip The IP address to check.
|
|
40
|
+
* @returns {boolean} True if the IP is private.
|
|
41
|
+
*/
|
|
42
|
+
function isPrivateIP(ip) {
|
|
43
|
+
if (net.isIPv4(ip)) {
|
|
44
|
+
const parts = ip.split('.').map(Number);
|
|
45
|
+
return (
|
|
46
|
+
parts[0] === 0 ||
|
|
47
|
+
parts[0] === 10 ||
|
|
48
|
+
(parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) ||
|
|
49
|
+
(parts[0] === 192 && parts[1] === 168) ||
|
|
50
|
+
parts[0] === 127 ||
|
|
51
|
+
(parts[0] === 169 && parts[1] === 254) ||
|
|
52
|
+
(parts[0] === 100 && parts[1] >= 64 && parts[1] <= 127) ||
|
|
53
|
+
(parts[0] === 192 && parts[1] === 0 && parts[2] === 0) || // 192.0.0.0/24
|
|
54
|
+
(parts[0] === 192 && parts[1] === 0 && parts[2] === 2) || // 192.0.2.0/24
|
|
55
|
+
(parts[0] === 198 && parts[1] >= 18 && parts[1] <= 19) || // 198.18.0.0/15
|
|
56
|
+
(parts[0] === 198 && parts[1] === 51 && parts[2] === 100) || // 198.51.100.0/24
|
|
57
|
+
(parts[0] === 203 && parts[1] === 0 && parts[2] === 113) || // 203.0.113.0/24
|
|
58
|
+
parts[0] >= 224 // 224.0.0.0/4 (Multicast) and 240.0.0.0/4 (Reserved)
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
if (net.isIPv6(ip)) {
|
|
62
|
+
const lower = ip.toLowerCase();
|
|
63
|
+
const parts = lower.split(':');
|
|
64
|
+
const last = parts[parts.length - 1];
|
|
65
|
+
|
|
66
|
+
// Handle IPv4-mapped IPv6 addresses (::ffff:1.2.3.4 or ::ffff:7f00:1)
|
|
67
|
+
const ffffIndex = parts.indexOf('ffff');
|
|
68
|
+
if (ffffIndex !== -1) {
|
|
69
|
+
const prefixAllZeros = parts.slice(0, ffffIndex).every(p => p === '' || p === '0');
|
|
70
|
+
if (prefixAllZeros) {
|
|
71
|
+
if (net.isIPv4(last)) {
|
|
72
|
+
return isPrivateIP(last);
|
|
73
|
+
}
|
|
74
|
+
const p1 = parseInt(parts[parts.length - 2], 16);
|
|
75
|
+
const p2 = parseInt(parts[parts.length - 1], 16);
|
|
76
|
+
if (!isNaN(p1) && !isNaN(p2)) {
|
|
77
|
+
return isPrivateIP(`${(p1 >> 8) & 0xff}.${p1 & 0xff}.${(p2 >> 8) & 0xff}.${p2 & 0xff}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Handle IPv4-compatible IPv6 addresses (::1.2.3.4 or ::7f00:1)
|
|
83
|
+
if (ffffIndex === -1) {
|
|
84
|
+
const prefixAllZeros = parts.slice(0, -2).every(p => p === '' || p === '0');
|
|
85
|
+
if (prefixAllZeros) {
|
|
86
|
+
if (net.isIPv4(last)) {
|
|
87
|
+
return isPrivateIP(last);
|
|
88
|
+
}
|
|
89
|
+
const p1 = parseInt(parts[parts.length - 2], 16);
|
|
90
|
+
const p2 = parseInt(parts[parts.length - 1], 16);
|
|
91
|
+
if (!isNaN(p1) && !isNaN(p2)) {
|
|
92
|
+
return isPrivateIP(`${(p1 >> 8) & 0xff}.${p1 & 0xff}.${(p2 >> 8) & 0xff}.${p2 & 0xff}`);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Native IPv6 checks (Unspecified, Loopback, Link-Local, Unique Local, Multicast)
|
|
98
|
+
const isUnspecified = parts.every(p => p === '0' || p === '0000' || p === '');
|
|
99
|
+
const isLoopback = parts.slice(0, -1).every(p => p === '0' || p === '0000' || p === '') &&
|
|
100
|
+
(parts[parts.length - 1] === '1' || parts[parts.length - 1] === '0001');
|
|
101
|
+
|
|
102
|
+
if (isUnspecified || isLoopback) return true;
|
|
103
|
+
|
|
104
|
+
const firstHex = parseInt(parts[0], 16);
|
|
105
|
+
if (!isNaN(firstHex)) {
|
|
106
|
+
if (firstHex >= 0xff00) return true; // ff00::/8 Multicast
|
|
107
|
+
if (firstHex >= 0xfe80 && firstHex <= 0xfebf) return true; // fe80::/10 Link-local
|
|
108
|
+
if (firstHex >= 0xfc00 && firstHex <= 0xfdff) return true; // fc00::/7 Unique local
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return false;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const VALID_HOSTNAME_CACHE = new Set();
|
|
115
|
+
const INVALID_HOSTNAME_CACHE = new Set();
|
|
116
|
+
const CACHE_TTL = 30000; // 30 seconds
|
|
117
|
+
let lastCacheClear = Date.now();
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Validates a URL to prevent SSRF by blocking private IP ranges.
|
|
121
|
+
* @param {string} urlStr The URL to validate.
|
|
122
|
+
* @returns {string} The validated URL string.
|
|
123
|
+
* @throws {Error} If the URL is invalid or points to a private network.
|
|
124
|
+
*/
|
|
125
|
+
async function validateUrl(urlStr) {
|
|
126
|
+
if (!urlStr) return '';
|
|
127
|
+
|
|
128
|
+
let url;
|
|
129
|
+
try {
|
|
130
|
+
url = new URL(urlStr);
|
|
131
|
+
} catch (e) {
|
|
132
|
+
throw new Error('Invalid URL');
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
136
|
+
throw new Error('Only HTTP and HTTPS protocols are allowed');
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
if (ALLOW_PRIVATE_NETWORKS) return url.href;
|
|
140
|
+
|
|
141
|
+
let hostname = url.hostname;
|
|
142
|
+
// Strip brackets from IPv6 hostnames
|
|
143
|
+
if (hostname.startsWith('[') && hostname.endsWith(']')) {
|
|
144
|
+
hostname = hostname.substring(1, hostname.length - 1);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
const lowerHost = hostname.toLowerCase();
|
|
148
|
+
|
|
149
|
+
// Cache management
|
|
150
|
+
if (Date.now() - lastCacheClear > CACHE_TTL) {
|
|
151
|
+
VALID_HOSTNAME_CACHE.clear();
|
|
152
|
+
INVALID_HOSTNAME_CACHE.clear();
|
|
153
|
+
lastCacheClear = Date.now();
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
if (VALID_HOSTNAME_CACHE.has(lowerHost)) return url.href;
|
|
157
|
+
if (INVALID_HOSTNAME_CACHE.has(lowerHost)) {
|
|
158
|
+
throw new Error('Access to private network is restricted');
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// Direct check for common private hostnames
|
|
162
|
+
if (
|
|
163
|
+
lowerHost === 'localhost' ||
|
|
164
|
+
lowerHost.endsWith('.localhost') ||
|
|
165
|
+
lowerHost === 'host.docker.internal'
|
|
166
|
+
) {
|
|
167
|
+
INVALID_HOSTNAME_CACHE.add(lowerHost);
|
|
168
|
+
throw new Error('Access to private network is restricted');
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// Resolve hostname to IP
|
|
172
|
+
try {
|
|
173
|
+
// If it's already an IP address, check it directly
|
|
174
|
+
if (net.isIP(hostname)) {
|
|
175
|
+
if (isPrivateIP(hostname)) {
|
|
176
|
+
INVALID_HOSTNAME_CACHE.add(lowerHost);
|
|
177
|
+
throw new Error('Access to private network is restricted');
|
|
178
|
+
}
|
|
179
|
+
return url.href;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// dns.lookup follows /etc/hosts and is what's typically used for connecting
|
|
183
|
+
const addresses = await dns.lookup(hostname, { all: true });
|
|
184
|
+
for (const addr of addresses) {
|
|
185
|
+
if (isPrivateIP(addr.address)) {
|
|
186
|
+
INVALID_HOSTNAME_CACHE.add(lowerHost);
|
|
187
|
+
throw new Error('Access to private network is restricted');
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
VALID_HOSTNAME_CACHE.add(lowerHost);
|
|
191
|
+
} catch (e) {
|
|
192
|
+
if (e.message === 'Access to private network is restricted') {
|
|
193
|
+
throw e;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// If we can't resolve it and it's not an IP, we allow it to proceed
|
|
197
|
+
// to the browser where it will likely fail normally.
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
return url.href;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Perform a fetch with manual redirect following and validation at each hop.
|
|
205
|
+
* Ensures sensitive headers (Authorization, Token) are stripped on cross-origin redirects.
|
|
206
|
+
* @param {string} urlStr Initial URL.
|
|
207
|
+
* @param {object} options Fetch options.
|
|
208
|
+
* @param {number} maxRedirects Maximum number of redirects to follow.
|
|
209
|
+
*/
|
|
210
|
+
async function fetchWithRedirectValidation(urlStr, options = {}, maxRedirects = 5) {
|
|
211
|
+
let currentUrl;
|
|
212
|
+
try {
|
|
213
|
+
currentUrl = new URL(urlStr);
|
|
214
|
+
} catch (e) {
|
|
215
|
+
throw new Error('Invalid URL');
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
let currentOptions = { ...options };
|
|
219
|
+
let redirectCount = 0;
|
|
220
|
+
|
|
221
|
+
while (redirectCount <= maxRedirects) {
|
|
222
|
+
// validateUrl respects ALLOW_PRIVATE_NETWORKS internally
|
|
223
|
+
const validatedHref = await validateUrl(currentUrl.href);
|
|
224
|
+
|
|
225
|
+
// Explicitly reconstruct URL from validated href to ensure taint is cleared
|
|
226
|
+
const safeUrl = new URL(validatedHref);
|
|
227
|
+
|
|
228
|
+
// CodeQL mitigation: strictly verify protocol and pass URL object to fetch
|
|
229
|
+
if (safeUrl.protocol !== 'http:' && safeUrl.protocol !== 'https:') {
|
|
230
|
+
throw new Error('Only HTTP and HTTPS protocols are allowed');
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
const response = await fetch(safeUrl, {
|
|
234
|
+
...currentOptions,
|
|
235
|
+
redirect: 'manual'
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
// Handle redirects (301, 302, 303, 307, 308)
|
|
239
|
+
if (response.status >= 300 && response.status < 400) {
|
|
240
|
+
const location = response.headers.get('location');
|
|
241
|
+
if (!location) return response;
|
|
242
|
+
|
|
243
|
+
const nextUrl = new URL(location, safeUrl.href);
|
|
244
|
+
const isCrossOrigin = nextUrl.origin !== currentUrl.origin;
|
|
245
|
+
|
|
246
|
+
// Update options for the next request (shallow copy)
|
|
247
|
+
const nextOptions = { ...currentOptions };
|
|
248
|
+
if (nextOptions.headers) {
|
|
249
|
+
nextOptions.headers = { ...nextOptions.headers };
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// Strip sensitive headers on cross-origin redirects
|
|
253
|
+
if (isCrossOrigin && nextOptions.headers) {
|
|
254
|
+
const sensitiveHeaders = ['authorization', 'x-api-key', 'token', 'cookie', 'proxy-authorization'];
|
|
255
|
+
for (const h of Object.keys(nextOptions.headers)) {
|
|
256
|
+
if (sensitiveHeaders.includes(h.toLowerCase())) {
|
|
257
|
+
delete nextOptions.headers[h];
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// Standards compliance: 301, 302, 303 redirects switch to GET and drop body
|
|
263
|
+
if ([301, 302, 303].includes(response.status)) {
|
|
264
|
+
nextOptions.method = 'GET';
|
|
265
|
+
delete nextOptions.body;
|
|
266
|
+
if (nextOptions.headers) {
|
|
267
|
+
for (const h of Object.keys(nextOptions.headers)) {
|
|
268
|
+
if (['content-type', 'content-length'].includes(h.toLowerCase())) {
|
|
269
|
+
delete nextOptions.headers[h];
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
currentUrl = nextUrl;
|
|
276
|
+
currentOptions = nextOptions;
|
|
277
|
+
redirectCount++;
|
|
278
|
+
continue;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
return response;
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
throw new Error('Too many redirects');
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* Sets up navigation protection for a Playwright context.
|
|
289
|
+
* Intercepts requests and validates destination URLs.
|
|
290
|
+
* @param {object} context Playwright context.
|
|
291
|
+
*/
|
|
292
|
+
async function setupNavigationProtection(context) {
|
|
293
|
+
if (ALLOW_PRIVATE_NETWORKS) return;
|
|
294
|
+
|
|
295
|
+
await context.route('**/*', async (route) => {
|
|
296
|
+
const request = route.request();
|
|
297
|
+
// Only validate main frame navigations for performance and to avoid breaking sub-resources
|
|
298
|
+
if (request.isNavigationRequest() && request.frame() === request.frame().page().mainFrame()) {
|
|
299
|
+
const url = request.url();
|
|
300
|
+
const currentUrl = request.frame().url();
|
|
301
|
+
|
|
302
|
+
try {
|
|
303
|
+
// If it's a same-origin navigation, skip validation for speed
|
|
304
|
+
if (currentUrl && currentUrl !== 'about:blank') {
|
|
305
|
+
const u1 = new URL(url);
|
|
306
|
+
const u2 = new URL(currentUrl);
|
|
307
|
+
if (u1.origin === u2.origin) {
|
|
308
|
+
return route.continue();
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
await validateUrl(url);
|
|
313
|
+
return route.continue();
|
|
314
|
+
} catch (err) {
|
|
315
|
+
console.error(`[SECURITY] Navigation to ${url} blocked: ${err.message}`);
|
|
316
|
+
return route.abort('blockedbyclient');
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
return route.continue();
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/**
|
|
324
|
+
* Verifies if a WebSocket origin matches the request host (CSWSH protection).
|
|
325
|
+
* @param {string} origin The Origin header value.
|
|
326
|
+
* @param {string} host The Host header value.
|
|
327
|
+
* @returns {boolean} True if the origin is valid or missing.
|
|
328
|
+
*/
|
|
329
|
+
function isValidWebSocketOrigin(origin, host) {
|
|
330
|
+
if (!origin) return true;
|
|
331
|
+
try {
|
|
332
|
+
const originHost = new URL(origin).host;
|
|
333
|
+
return !!(originHost && host && originHost === host);
|
|
334
|
+
} catch (e) {
|
|
335
|
+
return false;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
module.exports = { validateUrl, isPrivateIP, isValidWebSocketOrigin, fetchWithRedirectValidation, setupNavigationProtection };
|