@adguard/hostlist-compiler 2.1.1 → 3.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.markdownlint.json +29 -0
- package/.markdownlintignore +1 -0
- package/AGENTS.md +310 -98
- package/CHANGELOG.md +59 -8
- package/DEPLOYMENT.md +171 -0
- package/DEVELOPMENT.md +171 -72
- package/README.md +238 -143
- package/examples/china/README.md +2 -2
- package/examples/energized/README.md +4 -2
- package/examples/sdn/README.md +2 -2
- package/examples/whitelist/README.md +2 -2
- package/package.json +24 -18
- package/src/cli.js +53 -31
- package/src/compile-source.js +4 -3
- package/src/configuration.js +132 -2
- package/src/download.js +27 -0
- package/src/filter.js +2 -2
- package/src/index.d.ts +2 -0
- package/src/index.js +16 -50
- package/src/{transformations/ip-normalize.js → ip-normalize.js} +2 -2
- package/src/schemas/configuration.schema.js +137 -0
- package/src/source-types.js +17 -0
- package/src/transformations/enum.js +49 -0
- package/src/transformations/include.js +1 -1
- package/src/transformations/transform.js +40 -74
- package/src/transformations/validate-allow-ip-and-public-suffix.js +1 -1
- package/src/transformations/validate-allow-ip.js +1 -1
- package/src/utils.js +1284 -0
- package/src/validation-conflicts.js +106 -0
- package/test/cli-error-output.test.js +305 -0
- package/test/cli.test.js +47 -0
- package/test/compile-source.test.js +167 -7
- package/test/configuration.test.js +438 -0
- package/test/download.test.js +260 -0
- package/test/helpers/http-mock.js +94 -0
- package/test/index.test.js +497 -4
- package/test/{transformations/ip-normalize.test.js → ip-normalize.test.js} +1 -1
- package/test/resolve-include.test.js +8 -8
- package/test/transformations/covert-to-ascii.test.js +1 -1
- package/test/transformations/exclude.test.js +2 -2
- package/test/transformations/include.test.js +9 -3
- package/test/transformations/ip-alignment.test.js +1 -1
- package/test/transformations/transform.test.js +66 -1
- package/test/utils.test.js +1559 -0
- package/test/validation-conflicts.test.js +230 -0
- package/src/schemas/configuration.schema.json +0 -369
package/test/utils.test.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
/* eslint-disable class-methods-use-this, max-classes-per-file */
|
|
1
2
|
const utils = require('../src/utils');
|
|
3
|
+
const configurationSchema = require('../src/schemas/configuration.schema');
|
|
2
4
|
|
|
3
5
|
const { Wildcard } = utils;
|
|
4
6
|
|
|
@@ -19,6 +21,1489 @@ describe('Wildcard', () => {
|
|
|
19
21
|
});
|
|
20
22
|
});
|
|
21
23
|
|
|
24
|
+
describe('stripUrlCredentials', () => {
|
|
25
|
+
it.each([
|
|
26
|
+
[
|
|
27
|
+
'https://example.org/list.txt',
|
|
28
|
+
'https://example.org/list.txt',
|
|
29
|
+
],
|
|
30
|
+
[
|
|
31
|
+
'https://user:secret@example.org/list.txt',
|
|
32
|
+
'https://example.org/list.txt',
|
|
33
|
+
],
|
|
34
|
+
[
|
|
35
|
+
'https://user@example.org/list.txt',
|
|
36
|
+
'https://example.org/list.txt',
|
|
37
|
+
],
|
|
38
|
+
[
|
|
39
|
+
'https://user%40name:p%40ss%3Aword@example.org/list.txt',
|
|
40
|
+
'https://example.org/list.txt',
|
|
41
|
+
],
|
|
42
|
+
[
|
|
43
|
+
'//user:secret@example.org/list.txt',
|
|
44
|
+
'//example.org/list.txt',
|
|
45
|
+
],
|
|
46
|
+
[
|
|
47
|
+
'http:/user:secret@example.org/list.txt',
|
|
48
|
+
'http:/example.org/list.txt',
|
|
49
|
+
],
|
|
50
|
+
[
|
|
51
|
+
'https:user:secret@example.org/list.txt',
|
|
52
|
+
'https:example.org/list.txt',
|
|
53
|
+
],
|
|
54
|
+
[
|
|
55
|
+
'https:\\user:secret@example.org\\list.txt',
|
|
56
|
+
'https:\\example.org\\list.txt',
|
|
57
|
+
],
|
|
58
|
+
[
|
|
59
|
+
'https:\\\\user:secret@example.org\\list.txt',
|
|
60
|
+
'https:\\\\example.org\\list.txt',
|
|
61
|
+
],
|
|
62
|
+
[
|
|
63
|
+
'HTTPS://user:secret@example.org/list.txt',
|
|
64
|
+
'HTTPS://example.org/list.txt',
|
|
65
|
+
],
|
|
66
|
+
[
|
|
67
|
+
'See https://user:secret@example.org/list.txt and https://other:secret@example.net/other.txt',
|
|
68
|
+
'See https://example.org/list.txt and https://example.net/other.txt',
|
|
69
|
+
],
|
|
70
|
+
[
|
|
71
|
+
'https://proxy.example.org/fetch?u=https://user:secret@raw.example.org/list.txt',
|
|
72
|
+
'https://proxy.example.org/fetch?u=https://raw.example.org/list.txt',
|
|
73
|
+
],
|
|
74
|
+
[
|
|
75
|
+
'https://proxy.example.org/r/https://user:secret@raw.example.org/list.txt',
|
|
76
|
+
'https://proxy.example.org/r/https://raw.example.org/list.txt',
|
|
77
|
+
],
|
|
78
|
+
[
|
|
79
|
+
'https://example.org/a,https://user:secret@raw.example.org/list.txt',
|
|
80
|
+
'https://example.org/a,https://raw.example.org/list.txt',
|
|
81
|
+
],
|
|
82
|
+
[
|
|
83
|
+
'https://user:secret@example.org:8443/list.txt',
|
|
84
|
+
'https://example.org:8443/list.txt',
|
|
85
|
+
],
|
|
86
|
+
[
|
|
87
|
+
'https://user:secret@[2001:db8::1]:8443/list.txt?query=1#fragment',
|
|
88
|
+
'https://[2001:db8::1]:8443/list.txt?query=1#fragment',
|
|
89
|
+
],
|
|
90
|
+
[
|
|
91
|
+
'https://user:secret@example.org?query=1#fragment',
|
|
92
|
+
'https://example.org?query=1#fragment',
|
|
93
|
+
],
|
|
94
|
+
[
|
|
95
|
+
'https://user:secret@example.org#fragment',
|
|
96
|
+
'https://example.org#fragment',
|
|
97
|
+
],
|
|
98
|
+
[
|
|
99
|
+
'https://example.org/list@with@at.txt',
|
|
100
|
+
'https://example.org/list@with@at.txt',
|
|
101
|
+
],
|
|
102
|
+
[
|
|
103
|
+
'https://example.org/a//b@c',
|
|
104
|
+
'https://example.org/a//b@c',
|
|
105
|
+
],
|
|
106
|
+
[
|
|
107
|
+
"https://user:p'ss@example.org/list.txt",
|
|
108
|
+
'https://example.org/list.txt',
|
|
109
|
+
],
|
|
110
|
+
[
|
|
111
|
+
"https://o'brien:p@ssw0rd@example.org/list.txt",
|
|
112
|
+
'https://example.org/list.txt',
|
|
113
|
+
],
|
|
114
|
+
[
|
|
115
|
+
"https://o'brien:p@ssw0rd@example.org",
|
|
116
|
+
'https://example.org',
|
|
117
|
+
],
|
|
118
|
+
[
|
|
119
|
+
'https://John Smith:p@ss@example.org/list.txt',
|
|
120
|
+
'https://example.org/list.txt',
|
|
121
|
+
],
|
|
122
|
+
[
|
|
123
|
+
'https://John Smith:p@ss@example.org',
|
|
124
|
+
'https://example.org',
|
|
125
|
+
],
|
|
126
|
+
[
|
|
127
|
+
'https://J. Smith:secret@example.org/list.txt',
|
|
128
|
+
'https://example.org/list.txt',
|
|
129
|
+
],
|
|
130
|
+
[
|
|
131
|
+
'https://J. Smith:secret@example.org',
|
|
132
|
+
'https://example.org',
|
|
133
|
+
],
|
|
134
|
+
[
|
|
135
|
+
'https://admin user:secret@example.org',
|
|
136
|
+
'https://example.org',
|
|
137
|
+
],
|
|
138
|
+
[
|
|
139
|
+
'https://john smith:secret@example.org',
|
|
140
|
+
'https://example.org',
|
|
141
|
+
],
|
|
142
|
+
[
|
|
143
|
+
'"https://o\'brien:secret@example.org"',
|
|
144
|
+
'"https://example.org"',
|
|
145
|
+
],
|
|
146
|
+
[
|
|
147
|
+
'<https://John Smith:secret@example.org>',
|
|
148
|
+
'<https://example.org>',
|
|
149
|
+
],
|
|
150
|
+
[
|
|
151
|
+
'https://user:p"s@example.org/list.txt',
|
|
152
|
+
'https://example.org/list.txt',
|
|
153
|
+
],
|
|
154
|
+
[
|
|
155
|
+
'https://user:p<s@example.org/list.txt',
|
|
156
|
+
'https://example.org/list.txt',
|
|
157
|
+
],
|
|
158
|
+
[
|
|
159
|
+
'https://user:p>s@example.org/list.txt',
|
|
160
|
+
'https://example.org/list.txt',
|
|
161
|
+
],
|
|
162
|
+
[
|
|
163
|
+
'https://user:p s@example.org/list.txt',
|
|
164
|
+
'https://example.org/list.txt',
|
|
165
|
+
],
|
|
166
|
+
[
|
|
167
|
+
'user:secret@example.org/list.txt',
|
|
168
|
+
'user:secret@example.org/list.txt',
|
|
169
|
+
],
|
|
170
|
+
[
|
|
171
|
+
'test/dir/list.txt',
|
|
172
|
+
'test/dir/list.txt',
|
|
173
|
+
],
|
|
174
|
+
[
|
|
175
|
+
'C:\\user:secret@example.org\\list.txt',
|
|
176
|
+
'C:\\user:secret@example.org\\list.txt',
|
|
177
|
+
],
|
|
178
|
+
[
|
|
179
|
+
'C:/user@corp/list.txt',
|
|
180
|
+
'C:/user@corp/list.txt',
|
|
181
|
+
],
|
|
182
|
+
[
|
|
183
|
+
'c:/user:secret@corp/list.txt',
|
|
184
|
+
'c:/user:secret@corp/list.txt',
|
|
185
|
+
],
|
|
186
|
+
[
|
|
187
|
+
'ftp:/user:secret@example.org/list.txt',
|
|
188
|
+
'ftp:/example.org/list.txt',
|
|
189
|
+
],
|
|
190
|
+
[
|
|
191
|
+
'ftp://user:secret@example.org/list.txt',
|
|
192
|
+
'ftp://example.org/list.txt',
|
|
193
|
+
],
|
|
194
|
+
[
|
|
195
|
+
'https://user:p s s@example.org/list.txt',
|
|
196
|
+
'https://example.org/list.txt',
|
|
197
|
+
],
|
|
198
|
+
[
|
|
199
|
+
' https:///user:p s s@example.invalid/list.txt',
|
|
200
|
+
' https:///example.invalid/list.txt',
|
|
201
|
+
],
|
|
202
|
+
[
|
|
203
|
+
'https://user:p"s s@example.org/list.txt',
|
|
204
|
+
'https://example.org/list.txt',
|
|
205
|
+
],
|
|
206
|
+
[
|
|
207
|
+
'https://user:p"q"@example.org/list.txt',
|
|
208
|
+
'https://example.org/list.txt',
|
|
209
|
+
],
|
|
210
|
+
[
|
|
211
|
+
'https://example.org is maintained by user@example.com',
|
|
212
|
+
'https://example.org is maintained by user@example.com',
|
|
213
|
+
],
|
|
214
|
+
[
|
|
215
|
+
'Сайт https://фильтры.рф Контакты: пишите на admin@example.org',
|
|
216
|
+
'Сайт https://фильтры.рф Контакты: пишите на admin@example.org',
|
|
217
|
+
],
|
|
218
|
+
[
|
|
219
|
+
'List for https://intranet Contact: admin@example.org',
|
|
220
|
+
'List for https://intranet Contact: admin@example.org',
|
|
221
|
+
],
|
|
222
|
+
[
|
|
223
|
+
'List for https://intranet Contact:admin@example.org',
|
|
224
|
+
'List for https://intranet Contact:admin@example.org',
|
|
225
|
+
],
|
|
226
|
+
[
|
|
227
|
+
'List for https://my_host.org Contact: admin@example.org',
|
|
228
|
+
'List for https://my_host.org Contact: admin@example.org',
|
|
229
|
+
],
|
|
230
|
+
[
|
|
231
|
+
'List for https://x Contact: admin@example.org',
|
|
232
|
+
'List for https://x Contact: admin@example.org',
|
|
233
|
+
],
|
|
234
|
+
[
|
|
235
|
+
'List for https://x 联系: admin@example.org',
|
|
236
|
+
'List for https://x 联系: admin@example.org',
|
|
237
|
+
],
|
|
238
|
+
[
|
|
239
|
+
'https://example.org, see foo@bar.com',
|
|
240
|
+
'https://example.org, see foo@bar.com',
|
|
241
|
+
],
|
|
242
|
+
[
|
|
243
|
+
'https://user:secret@example.org and a@b',
|
|
244
|
+
'https://example.org and a@b',
|
|
245
|
+
],
|
|
246
|
+
[
|
|
247
|
+
'https://user:secret@example.org and https://other:secret@example.net/x',
|
|
248
|
+
'https://example.org and https://example.net/x',
|
|
249
|
+
],
|
|
250
|
+
[
|
|
251
|
+
'https://user:secret@example.org and https://other:secret@example.net',
|
|
252
|
+
'https://example.org and https://example.net',
|
|
253
|
+
],
|
|
254
|
+
[
|
|
255
|
+
'https://user:secret@example.org and a@b',
|
|
256
|
+
'https://example.org and a@b',
|
|
257
|
+
],
|
|
258
|
+
[
|
|
259
|
+
'https://John Smith: secret@example.org',
|
|
260
|
+
'https://example.org',
|
|
261
|
+
],
|
|
262
|
+
[
|
|
263
|
+
'See https://John Smith: secret@example.org',
|
|
264
|
+
'See https://example.org',
|
|
265
|
+
],
|
|
266
|
+
[
|
|
267
|
+
'https://user:1234, secret@example.org',
|
|
268
|
+
'https://example.org',
|
|
269
|
+
],
|
|
270
|
+
[
|
|
271
|
+
'https://user:1234, secret@example.org/list.txt',
|
|
272
|
+
'https://example.org/list.txt',
|
|
273
|
+
],
|
|
274
|
+
[
|
|
275
|
+
'See https://user:1234, secret@example.org/list.txt',
|
|
276
|
+
'See https://example.org/list.txt',
|
|
277
|
+
],
|
|
278
|
+
[
|
|
279
|
+
'https://example.org:8080 by admin@example.com',
|
|
280
|
+
'https://example.org:8080 by admin@example.com',
|
|
281
|
+
],
|
|
282
|
+
[
|
|
283
|
+
'https://example.org:8080 by admin@example.com/path',
|
|
284
|
+
'https://example.org:8080 by admin@example.com/path',
|
|
285
|
+
],
|
|
286
|
+
[
|
|
287
|
+
'https://localhost:8080, contact admin@example.com',
|
|
288
|
+
'https://localhost:8080, contact admin@example.com',
|
|
289
|
+
],
|
|
290
|
+
[
|
|
291
|
+
'https://192.168.1.1:8080, contact admin@example.com',
|
|
292
|
+
'https://192.168.1.1:8080, contact admin@example.com',
|
|
293
|
+
],
|
|
294
|
+
[
|
|
295
|
+
'Mirror at https://example.org:8080, contact admin@example.com',
|
|
296
|
+
'Mirror at https://example.org:8080, contact admin@example.com',
|
|
297
|
+
],
|
|
298
|
+
[
|
|
299
|
+
'Mirror at https://example.org:8080, contact admin@example.com/path',
|
|
300
|
+
'Mirror at https://example.org:8080, contact admin@example.com/path',
|
|
301
|
+
],
|
|
302
|
+
[
|
|
303
|
+
'Mirror at https://example.org:8080. Contact admin@example.com/path',
|
|
304
|
+
'Mirror at https://example.org:8080. Contact admin@example.com/path',
|
|
305
|
+
],
|
|
306
|
+
[
|
|
307
|
+
'Mirror at https://example.org:8080; contact admin@example.com/path',
|
|
308
|
+
'Mirror at https://example.org:8080; contact admin@example.com/path',
|
|
309
|
+
],
|
|
310
|
+
[
|
|
311
|
+
'Mirror at https://example.org:8080) contact admin@example.com/path',
|
|
312
|
+
'Mirror at https://example.org:8080) contact admin@example.com/path',
|
|
313
|
+
],
|
|
314
|
+
[
|
|
315
|
+
'Maintained at https://example.org by john@example.com. Forum: https://forum.example.org/',
|
|
316
|
+
'Maintained at https://example.org by john@example.com. Forum: https://forum.example.org/',
|
|
317
|
+
],
|
|
318
|
+
[
|
|
319
|
+
'https://John Smith:sekret@example.org by admin@example.com',
|
|
320
|
+
'https://example.org by admin@example.com',
|
|
321
|
+
],
|
|
322
|
+
[
|
|
323
|
+
"https://o'brien:sekret@example.org by admin@example.com",
|
|
324
|
+
'https://example.org by admin@example.com',
|
|
325
|
+
],
|
|
326
|
+
[
|
|
327
|
+
"https://o'brien:pass phrase@example.org",
|
|
328
|
+
'https://example.org',
|
|
329
|
+
],
|
|
330
|
+
[
|
|
331
|
+
'https://John Smith:pass phrase@example.org',
|
|
332
|
+
'https://example.org',
|
|
333
|
+
],
|
|
334
|
+
[
|
|
335
|
+
"https://o'brien:pass phrase@example.org by admin@example.com",
|
|
336
|
+
'https://example.org by admin@example.com',
|
|
337
|
+
],
|
|
338
|
+
[
|
|
339
|
+
'https://example.org for more info, contact admin@example.org/help.',
|
|
340
|
+
'https://example.org for more info, contact admin@example.org/help.',
|
|
341
|
+
],
|
|
342
|
+
[
|
|
343
|
+
'See https://example.org or mailto:admin@example.com for support',
|
|
344
|
+
'See https://example.org or mailto:admin@example.com for support',
|
|
345
|
+
],
|
|
346
|
+
[
|
|
347
|
+
'See https://example.org or e-mail:admin@example.com for support',
|
|
348
|
+
'See https://example.org or e-mail:admin@example.com for support',
|
|
349
|
+
],
|
|
350
|
+
[
|
|
351
|
+
'See https://example.org or Email:admin@example.com for support',
|
|
352
|
+
'See https://example.org or Email:admin@example.com for support',
|
|
353
|
+
],
|
|
354
|
+
[
|
|
355
|
+
'Homepage "https://example.org". Contact: admin@example.com. Thanks!',
|
|
356
|
+
'Homepage "https://example.org". Contact: admin@example.com. Thanks!',
|
|
357
|
+
],
|
|
358
|
+
[
|
|
359
|
+
'Filter list for Corp. Homepage: https://example.org. Contact: admin@example.com',
|
|
360
|
+
'Filter list for Corp. Homepage: https://example.org. Contact: admin@example.com',
|
|
361
|
+
],
|
|
362
|
+
[
|
|
363
|
+
'Mirror of https://example.org, maintainer: admin@example.com',
|
|
364
|
+
'Mirror of https://example.org, maintainer: admin@example.com',
|
|
365
|
+
],
|
|
366
|
+
[
|
|
367
|
+
'(https://example.org) Contact: admin@example.com',
|
|
368
|
+
'(https://example.org) Contact: admin@example.com',
|
|
369
|
+
],
|
|
370
|
+
[
|
|
371
|
+
'https://[2001:db8::1] by admin@example.com',
|
|
372
|
+
'https://[2001:db8::1] by admin@example.com',
|
|
373
|
+
],
|
|
374
|
+
[
|
|
375
|
+
'https://[2001:db8::1]. Contact: admin@example.com',
|
|
376
|
+
'https://[2001:db8::1]. Contact: admin@example.com',
|
|
377
|
+
],
|
|
378
|
+
[
|
|
379
|
+
'https://user:secret@[2001:db8::1] by admin@example.com',
|
|
380
|
+
'https://[2001:db8::1] by admin@example.com',
|
|
381
|
+
],
|
|
382
|
+
[
|
|
383
|
+
'https://user:secret@[2001:db8::1]. Contact: admin@example.com',
|
|
384
|
+
'https://[2001:db8::1]. Contact: admin@example.com',
|
|
385
|
+
],
|
|
386
|
+
[
|
|
387
|
+
'https://user:p@ss word@example.org',
|
|
388
|
+
'https://example.org',
|
|
389
|
+
],
|
|
390
|
+
[
|
|
391
|
+
'https://user:p@ss"word@example.org',
|
|
392
|
+
'https://example.org',
|
|
393
|
+
],
|
|
394
|
+
[
|
|
395
|
+
'https://user:p@ss<word@example.org',
|
|
396
|
+
'https://example.org',
|
|
397
|
+
],
|
|
398
|
+
[
|
|
399
|
+
'https://user:p@ss word@example.org/list.txt',
|
|
400
|
+
'https://example.org/list.txt',
|
|
401
|
+
],
|
|
402
|
+
[
|
|
403
|
+
'https://user:p@ss word@example.org by admin@example.com',
|
|
404
|
+
'https://example.org by admin@example.com',
|
|
405
|
+
],
|
|
406
|
+
[
|
|
407
|
+
'https://example.org: Contact admin@example.com',
|
|
408
|
+
'https://example.org: Contact admin@example.com',
|
|
409
|
+
],
|
|
410
|
+
[
|
|
411
|
+
'https://myhost, contact: admin@example.org',
|
|
412
|
+
'https://myhost, contact: admin@example.org',
|
|
413
|
+
],
|
|
414
|
+
[
|
|
415
|
+
'https://user, name:secret@example.org',
|
|
416
|
+
'https://example.org',
|
|
417
|
+
],
|
|
418
|
+
])('strips credentials from %s', (url, expected) => {
|
|
419
|
+
expect(utils.stripUrlCredentials(url)).toBe(expected);
|
|
420
|
+
});
|
|
421
|
+
|
|
422
|
+
it('returns a non-string value unchanged', () => {
|
|
423
|
+
expect(utils.stripUrlCredentials(undefined)).toBeUndefined();
|
|
424
|
+
expect(utils.stripUrlCredentials(null)).toBeNull();
|
|
425
|
+
});
|
|
426
|
+
|
|
427
|
+
it('does not treat a double slash inside a path as a new URL', () => {
|
|
428
|
+
const text = 'https://example.org/a//b@c';
|
|
429
|
+
|
|
430
|
+
expect(utils.stripUrlCredentials(text)).toBe(text);
|
|
431
|
+
expect(utils.stripUrlCredentials(`${text} and https://user:secret@example.net/x`))
|
|
432
|
+
.toBe(`${text} and https://example.net/x`);
|
|
433
|
+
});
|
|
434
|
+
|
|
435
|
+
it('does not change a rendered configuration that has no credentials', () => {
|
|
436
|
+
const configuration = {
|
|
437
|
+
homepage: 'https://example.org',
|
|
438
|
+
exclusions: ['@@||good.example^'],
|
|
439
|
+
sources: [{ source: 'lists/a.txt' }],
|
|
440
|
+
};
|
|
441
|
+
const dump = JSON.stringify(configuration);
|
|
442
|
+
|
|
443
|
+
expect(utils.stripUrlCredentials(dump)).toBe(dump);
|
|
444
|
+
});
|
|
445
|
+
|
|
446
|
+
it('handles long strings in linear time', () => {
|
|
447
|
+
// Jest does not fail a synchronous test that runs past its timeout,
|
|
448
|
+
// so the test needs a wall-clock bound. The bound is generous: a
|
|
449
|
+
// quadratic scan needs about 19 seconds for this input, while a linear
|
|
450
|
+
// scan takes milliseconds. A loaded runner cannot fail the bound.
|
|
451
|
+
// The credentialed URL reaches the port heuristic, which used to retry
|
|
452
|
+
// from every position inside the long password.
|
|
453
|
+
const plainText = 'a'.repeat(200_000);
|
|
454
|
+
const longPassword = 'b'.repeat(200_000);
|
|
455
|
+
const credentialedUrl = `https://reader:${longPassword}@example.org/list.txt`;
|
|
456
|
+
// Every URL start used to rescan the rest of the string for the
|
|
457
|
+
// authority end, and the strict parser scanned back to index 0 when
|
|
458
|
+
// the input had no `@`. Both scans must use the lookup tables.
|
|
459
|
+
const repeatedUrlStarts = 'https:a '.repeat(20_000);
|
|
460
|
+
// The value parser must not scan the text up to a far `@` for every
|
|
461
|
+
// URL start.
|
|
462
|
+
const proseWithEmail = `${'https:example.org '.repeat(10_000)}admin@example.com`;
|
|
463
|
+
// The strict malformed-userinfo scan must stay within the value. It
|
|
464
|
+
// used to walk back to the last `@` in the whole string for every URL
|
|
465
|
+
// start, which is quadratic on many at signs.
|
|
466
|
+
const manyAtSigns = 'https:x#a@b '.repeat(20_000);
|
|
467
|
+
const startedAt = Date.now();
|
|
468
|
+
|
|
469
|
+
expect(utils.stripUrlCredentials(plainText)).toBe(plainText);
|
|
470
|
+
expect(utils.stripUrlCredentials(credentialedUrl)).toBe('https://example.org/list.txt');
|
|
471
|
+
expect(utils.stripUrlCredentials(repeatedUrlStarts)).toBe(repeatedUrlStarts);
|
|
472
|
+
expect(utils.stripKnownUrlCredentials(repeatedUrlStarts)).toBe(repeatedUrlStarts);
|
|
473
|
+
expect(utils.stripUrlCredentials(proseWithEmail)).toBe(proseWithEmail);
|
|
474
|
+
expect(utils.stripKnownUrlCredentials(manyAtSigns)).toBe(manyAtSigns);
|
|
475
|
+
expect(Date.now() - startedAt).toBeLessThan(3000);
|
|
476
|
+
});
|
|
477
|
+
});
|
|
478
|
+
|
|
479
|
+
describe('stripUrlValueCredentials', () => {
|
|
480
|
+
it.each([
|
|
481
|
+
[
|
|
482
|
+
'https://user:secret@example.org/list.txt',
|
|
483
|
+
'https://example.org/list.txt',
|
|
484
|
+
],
|
|
485
|
+
[
|
|
486
|
+
'https://user:p s s@example.org/list.txt',
|
|
487
|
+
'https://example.org/list.txt',
|
|
488
|
+
],
|
|
489
|
+
[
|
|
490
|
+
'https://user:p"s@example.org/list.txt',
|
|
491
|
+
'https://example.org/list.txt',
|
|
492
|
+
],
|
|
493
|
+
[
|
|
494
|
+
"https://user:p'ss@example.org/list.txt",
|
|
495
|
+
'https://example.org/list.txt',
|
|
496
|
+
],
|
|
497
|
+
[
|
|
498
|
+
'//user:p s s@example.org/list.txt',
|
|
499
|
+
'//example.org/list.txt',
|
|
500
|
+
],
|
|
501
|
+
[
|
|
502
|
+
'https:user:p s s@example.org/list.txt',
|
|
503
|
+
'https:example.org/list.txt',
|
|
504
|
+
],
|
|
505
|
+
[
|
|
506
|
+
'https:\\user:p s s@example.org\\list.txt',
|
|
507
|
+
'https:\\example.org\\list.txt',
|
|
508
|
+
],
|
|
509
|
+
[
|
|
510
|
+
'https://example.org/a//b@c',
|
|
511
|
+
'https://example.org/a//b@c',
|
|
512
|
+
],
|
|
513
|
+
[
|
|
514
|
+
'C:\\user:secret@example.org\\list.txt',
|
|
515
|
+
'C:\\user:secret@example.org\\list.txt',
|
|
516
|
+
],
|
|
517
|
+
[
|
|
518
|
+
'C:/user@corp/list.txt',
|
|
519
|
+
'C:/user@corp/list.txt',
|
|
520
|
+
],
|
|
521
|
+
[
|
|
522
|
+
' https://user:secret@example.org/list.txt',
|
|
523
|
+
' https://example.org/list.txt',
|
|
524
|
+
],
|
|
525
|
+
[
|
|
526
|
+
'\thttps://user:secret@example.org/list.txt',
|
|
527
|
+
'\thttps://example.org/list.txt',
|
|
528
|
+
],
|
|
529
|
+
[
|
|
530
|
+
'<https://user:secret@example.org/>',
|
|
531
|
+
'<https://example.org/>',
|
|
532
|
+
],
|
|
533
|
+
[
|
|
534
|
+
'"https://user:secret@example.org/"',
|
|
535
|
+
'"https://example.org/"',
|
|
536
|
+
],
|
|
537
|
+
[
|
|
538
|
+
'See https://user:secret@example.org/list.txt',
|
|
539
|
+
'See https://example.org/list.txt',
|
|
540
|
+
],
|
|
541
|
+
[
|
|
542
|
+
'https://user:p"s s@example.org/list.txt',
|
|
543
|
+
'https://example.org/list.txt',
|
|
544
|
+
],
|
|
545
|
+
[
|
|
546
|
+
'https://user:p"q"@example.org/list.txt',
|
|
547
|
+
'https://example.org/list.txt',
|
|
548
|
+
],
|
|
549
|
+
[
|
|
550
|
+
'https://user:p\t s@example.org/list.txt',
|
|
551
|
+
'https://example.org/list.txt',
|
|
552
|
+
],
|
|
553
|
+
[
|
|
554
|
+
'https://example.org is maintained by user@example.com',
|
|
555
|
+
'https://example.org is maintained by user@example.com',
|
|
556
|
+
],
|
|
557
|
+
[
|
|
558
|
+
'Сайт https://фильтры.рф Контакты: пишите на admin@example.org',
|
|
559
|
+
'Сайт https://фильтры.рф Контакты: пишите на admin@example.org',
|
|
560
|
+
],
|
|
561
|
+
[
|
|
562
|
+
'List for https://intranet Contact: admin@example.org',
|
|
563
|
+
'List for https://intranet Contact: admin@example.org',
|
|
564
|
+
],
|
|
565
|
+
[
|
|
566
|
+
'List for https://intranet Contact:admin@example.org',
|
|
567
|
+
'List for https://intranet Contact:admin@example.org',
|
|
568
|
+
],
|
|
569
|
+
[
|
|
570
|
+
'List for https://my_host.org Contact: admin@example.org',
|
|
571
|
+
'List for https://my_host.org Contact: admin@example.org',
|
|
572
|
+
],
|
|
573
|
+
[
|
|
574
|
+
'List for https://x Contact: admin@example.org',
|
|
575
|
+
'List for https://x Contact: admin@example.org',
|
|
576
|
+
],
|
|
577
|
+
[
|
|
578
|
+
'List for https://x 联系: admin@example.org',
|
|
579
|
+
'List for https://x 联系: admin@example.org',
|
|
580
|
+
],
|
|
581
|
+
[
|
|
582
|
+
'https://example.org, see foo@bar.com',
|
|
583
|
+
'https://example.org, see foo@bar.com',
|
|
584
|
+
],
|
|
585
|
+
[
|
|
586
|
+
"https://o'brien:secret@example.org/list.txt",
|
|
587
|
+
'https://example.org/list.txt',
|
|
588
|
+
],
|
|
589
|
+
[
|
|
590
|
+
'https://John Smith:secret@example.org/list.txt',
|
|
591
|
+
'https://example.org/list.txt',
|
|
592
|
+
],
|
|
593
|
+
[
|
|
594
|
+
"https://o'brien:p@ssw0rd@example.org/list.txt",
|
|
595
|
+
'https://example.org/list.txt',
|
|
596
|
+
],
|
|
597
|
+
[
|
|
598
|
+
"https://o'brien:p@ssw0rd@example.org",
|
|
599
|
+
'https://example.org',
|
|
600
|
+
],
|
|
601
|
+
[
|
|
602
|
+
'https://John Smith:p@ss@example.org/list.txt',
|
|
603
|
+
'https://example.org/list.txt',
|
|
604
|
+
],
|
|
605
|
+
[
|
|
606
|
+
'https://John Smith:p@ss@example.org',
|
|
607
|
+
'https://example.org',
|
|
608
|
+
],
|
|
609
|
+
[
|
|
610
|
+
'https://J. Smith:secret@example.org/list.txt',
|
|
611
|
+
'https://example.org/list.txt',
|
|
612
|
+
],
|
|
613
|
+
[
|
|
614
|
+
'https://user: secret@example.org/list.txt',
|
|
615
|
+
'https://example.org/list.txt',
|
|
616
|
+
],
|
|
617
|
+
[
|
|
618
|
+
"https://o'brien:secret@example.org",
|
|
619
|
+
'https://example.org',
|
|
620
|
+
],
|
|
621
|
+
[
|
|
622
|
+
'https://John Smith:secret@example.org',
|
|
623
|
+
'https://example.org',
|
|
624
|
+
],
|
|
625
|
+
[
|
|
626
|
+
'https://J. Smith:secret@example.org',
|
|
627
|
+
'https://example.org',
|
|
628
|
+
],
|
|
629
|
+
[
|
|
630
|
+
'https://admin user:secret@example.org',
|
|
631
|
+
'https://example.org',
|
|
632
|
+
],
|
|
633
|
+
[
|
|
634
|
+
'https://john smith:secret@example.org',
|
|
635
|
+
'https://example.org',
|
|
636
|
+
],
|
|
637
|
+
[
|
|
638
|
+
'https://J. Smith:secret@example.org by admin@example.com',
|
|
639
|
+
'https://example.org by admin@example.com',
|
|
640
|
+
],
|
|
641
|
+
[
|
|
642
|
+
'https://user: secret@example.org',
|
|
643
|
+
'https://example.org',
|
|
644
|
+
],
|
|
645
|
+
[
|
|
646
|
+
'https://example.org team: contact@example.com',
|
|
647
|
+
'https://example.org team: contact@example.com',
|
|
648
|
+
],
|
|
649
|
+
[
|
|
650
|
+
'https://John Smith: secret@example.org',
|
|
651
|
+
'https://example.org',
|
|
652
|
+
],
|
|
653
|
+
[
|
|
654
|
+
'See https://John Smith: secret@example.org',
|
|
655
|
+
'See https://example.org',
|
|
656
|
+
],
|
|
657
|
+
[
|
|
658
|
+
'https://user:1234, secret@example.org',
|
|
659
|
+
'https://example.org',
|
|
660
|
+
],
|
|
661
|
+
[
|
|
662
|
+
'https://user:1234, secret@example.org/list.txt',
|
|
663
|
+
'https://example.org/list.txt',
|
|
664
|
+
],
|
|
665
|
+
[
|
|
666
|
+
'See https://user:1234, secret@example.org/list.txt',
|
|
667
|
+
'See https://example.org/list.txt',
|
|
668
|
+
],
|
|
669
|
+
[
|
|
670
|
+
'https://example.org:8080 by admin@example.com',
|
|
671
|
+
'https://example.org:8080 by admin@example.com',
|
|
672
|
+
],
|
|
673
|
+
[
|
|
674
|
+
'https://example.org:8080 by admin@example.com/path',
|
|
675
|
+
'https://example.org:8080 by admin@example.com/path',
|
|
676
|
+
],
|
|
677
|
+
[
|
|
678
|
+
'https://localhost:8080, contact admin@example.com',
|
|
679
|
+
'https://localhost:8080, contact admin@example.com',
|
|
680
|
+
],
|
|
681
|
+
[
|
|
682
|
+
'https://192.168.1.1:8080, contact admin@example.com',
|
|
683
|
+
'https://192.168.1.1:8080, contact admin@example.com',
|
|
684
|
+
],
|
|
685
|
+
[
|
|
686
|
+
'Mirror at https://example.org:8080, contact admin@example.com',
|
|
687
|
+
'Mirror at https://example.org:8080, contact admin@example.com',
|
|
688
|
+
],
|
|
689
|
+
[
|
|
690
|
+
'Mirror at https://example.org:8080, contact admin@example.com/path',
|
|
691
|
+
'Mirror at https://example.org:8080, contact admin@example.com/path',
|
|
692
|
+
],
|
|
693
|
+
[
|
|
694
|
+
'Mirror at https://example.org:8080. Contact admin@example.com/path',
|
|
695
|
+
'Mirror at https://example.org:8080. Contact admin@example.com/path',
|
|
696
|
+
],
|
|
697
|
+
[
|
|
698
|
+
'Mirror at https://example.org:8080; contact admin@example.com/path',
|
|
699
|
+
'Mirror at https://example.org:8080; contact admin@example.com/path',
|
|
700
|
+
],
|
|
701
|
+
[
|
|
702
|
+
'Mirror at https://example.org:8080) contact admin@example.com/path',
|
|
703
|
+
'Mirror at https://example.org:8080) contact admin@example.com/path',
|
|
704
|
+
],
|
|
705
|
+
[
|
|
706
|
+
'Maintained at https://example.org by john@example.com. Forum: https://forum.example.org/',
|
|
707
|
+
'Maintained at https://example.org by john@example.com. Forum: https://forum.example.org/',
|
|
708
|
+
],
|
|
709
|
+
[
|
|
710
|
+
'https://John Smith:sekret@example.org by admin@example.com',
|
|
711
|
+
'https://example.org by admin@example.com',
|
|
712
|
+
],
|
|
713
|
+
[
|
|
714
|
+
"https://o'brien:sekret@example.org by admin@example.com",
|
|
715
|
+
'https://example.org by admin@example.com',
|
|
716
|
+
],
|
|
717
|
+
[
|
|
718
|
+
"https://o'brien:pass phrase@example.org",
|
|
719
|
+
'https://example.org',
|
|
720
|
+
],
|
|
721
|
+
[
|
|
722
|
+
'https://John Smith:pass phrase@example.org',
|
|
723
|
+
'https://example.org',
|
|
724
|
+
],
|
|
725
|
+
[
|
|
726
|
+
"https://o'brien:pass phrase@example.org by admin@example.com",
|
|
727
|
+
'https://example.org by admin@example.com',
|
|
728
|
+
],
|
|
729
|
+
[
|
|
730
|
+
'https://example.org for more info, contact admin@example.org/help.',
|
|
731
|
+
'https://example.org for more info, contact admin@example.org/help.',
|
|
732
|
+
],
|
|
733
|
+
[
|
|
734
|
+
'See https://example.org or mailto:admin@example.com for support',
|
|
735
|
+
'See https://example.org or mailto:admin@example.com for support',
|
|
736
|
+
],
|
|
737
|
+
[
|
|
738
|
+
'See https://example.org or e-mail:admin@example.com for support',
|
|
739
|
+
'See https://example.org or e-mail:admin@example.com for support',
|
|
740
|
+
],
|
|
741
|
+
[
|
|
742
|
+
'See https://example.org or Email:admin@example.com for support',
|
|
743
|
+
'See https://example.org or Email:admin@example.com for support',
|
|
744
|
+
],
|
|
745
|
+
[
|
|
746
|
+
'Homepage "https://example.org". Contact: admin@example.com. Thanks!',
|
|
747
|
+
'Homepage "https://example.org". Contact: admin@example.com. Thanks!',
|
|
748
|
+
],
|
|
749
|
+
[
|
|
750
|
+
'Filter list for Corp. Homepage: https://example.org. Contact: admin@example.com',
|
|
751
|
+
'Filter list for Corp. Homepage: https://example.org. Contact: admin@example.com',
|
|
752
|
+
],
|
|
753
|
+
[
|
|
754
|
+
'Mirror of https://example.org, maintainer: admin@example.com',
|
|
755
|
+
'Mirror of https://example.org, maintainer: admin@example.com',
|
|
756
|
+
],
|
|
757
|
+
[
|
|
758
|
+
'(https://example.org) Contact: admin@example.com',
|
|
759
|
+
'(https://example.org) Contact: admin@example.com',
|
|
760
|
+
],
|
|
761
|
+
[
|
|
762
|
+
'https://[2001:db8::1] by admin@example.com',
|
|
763
|
+
'https://[2001:db8::1] by admin@example.com',
|
|
764
|
+
],
|
|
765
|
+
[
|
|
766
|
+
'https://[2001:db8::1]. Contact: admin@example.com',
|
|
767
|
+
'https://[2001:db8::1]. Contact: admin@example.com',
|
|
768
|
+
],
|
|
769
|
+
[
|
|
770
|
+
'https://user:secret@[2001:db8::1] by admin@example.com',
|
|
771
|
+
'https://[2001:db8::1] by admin@example.com',
|
|
772
|
+
],
|
|
773
|
+
[
|
|
774
|
+
'https://user:secret@[2001:db8::1]. Contact: admin@example.com',
|
|
775
|
+
'https://[2001:db8::1]. Contact: admin@example.com',
|
|
776
|
+
],
|
|
777
|
+
[
|
|
778
|
+
'https://user:secret@example.org and a@b',
|
|
779
|
+
'https://example.org and a@b',
|
|
780
|
+
],
|
|
781
|
+
[
|
|
782
|
+
'https://proxy.example.org/fetch?u=https://user:secret@raw.example.org/list.txt',
|
|
783
|
+
'https://proxy.example.org/fetch?u=https://raw.example.org/list.txt',
|
|
784
|
+
],
|
|
785
|
+
[
|
|
786
|
+
'https://proxy.example.org/r/https://user:secret@raw.example.org/list.txt',
|
|
787
|
+
'https://proxy.example.org/r/https://raw.example.org/list.txt',
|
|
788
|
+
],
|
|
789
|
+
[
|
|
790
|
+
'https://example.org/a,https://user:secret@raw.example.org/list.txt',
|
|
791
|
+
'https://example.org/a,https://raw.example.org/list.txt',
|
|
792
|
+
],
|
|
793
|
+
[
|
|
794
|
+
'https://user:p@ss word@example.org',
|
|
795
|
+
'https://example.org',
|
|
796
|
+
],
|
|
797
|
+
[
|
|
798
|
+
'https://user:p@ss"word@example.org',
|
|
799
|
+
'https://example.org',
|
|
800
|
+
],
|
|
801
|
+
[
|
|
802
|
+
'https://user:p@ss<word@example.org',
|
|
803
|
+
'https://example.org',
|
|
804
|
+
],
|
|
805
|
+
[
|
|
806
|
+
'https://user:p@ss word@example.org/list.txt',
|
|
807
|
+
'https://example.org/list.txt',
|
|
808
|
+
],
|
|
809
|
+
[
|
|
810
|
+
'https://user:p@ss word@example.org by admin@example.com',
|
|
811
|
+
'https://example.org by admin@example.com',
|
|
812
|
+
],
|
|
813
|
+
[
|
|
814
|
+
'https://example.org: Contact admin@example.com',
|
|
815
|
+
'https://example.org: Contact admin@example.com',
|
|
816
|
+
],
|
|
817
|
+
[
|
|
818
|
+
'https://myhost, contact: admin@example.org',
|
|
819
|
+
'https://myhost, contact: admin@example.org',
|
|
820
|
+
],
|
|
821
|
+
[
|
|
822
|
+
'https://user, name:secret@example.org',
|
|
823
|
+
'https://example.org',
|
|
824
|
+
],
|
|
825
|
+
])('strips credentials from %s', (url, expected) => {
|
|
826
|
+
expect(utils.stripUrlValueCredentials(url)).toBe(expected);
|
|
827
|
+
});
|
|
828
|
+
|
|
829
|
+
it('returns a non-string value unchanged', () => {
|
|
830
|
+
expect(utils.stripUrlValueCredentials(undefined)).toBeUndefined();
|
|
831
|
+
expect(utils.stripUrlValueCredentials(null)).toBeNull();
|
|
832
|
+
});
|
|
833
|
+
|
|
834
|
+
it('strips credentials when the path contains @', () => {
|
|
835
|
+
expect(utils.stripUrlValueCredentials('https://John Smith:secret@example.org/list@with@at.txt'))
|
|
836
|
+
.toBe('https://example.org/list@with@at.txt');
|
|
837
|
+
expect(utils.stripUrlValueCredentials("https://o'brien:secret@example.org/list@with@at.txt"))
|
|
838
|
+
.toBe('https://example.org/list@with@at.txt');
|
|
839
|
+
});
|
|
840
|
+
});
|
|
841
|
+
|
|
842
|
+
describe('credential sanitizer prose families', () => {
|
|
843
|
+
// One row per host shape and separator. A prose tail after a path-less URL
|
|
844
|
+
// must stay unchanged, while the same value with userinfo must lose only
|
|
845
|
+
// the userinfo. The cross product closes the class that single rows missed:
|
|
846
|
+
// `https://example.org: Contact ...` and
|
|
847
|
+
// `https://myhost, contact: ...` both regressed while their neighbours
|
|
848
|
+
// passed.
|
|
849
|
+
const hosts = [
|
|
850
|
+
'example.org',
|
|
851
|
+
'intranet',
|
|
852
|
+
'my_host.org',
|
|
853
|
+
'фильтры.рф',
|
|
854
|
+
'[2001:db8::1]',
|
|
855
|
+
'127.0.0.1',
|
|
856
|
+
];
|
|
857
|
+
const separators = [' ', ':', ',', ';', '.'];
|
|
858
|
+
const rows = hosts.flatMap((host) => separators.map((separator) => {
|
|
859
|
+
const prose = `https://${host}${separator} Contact: admin@example.org`;
|
|
860
|
+
const credentialed = `https://user:secret@${host}${separator} Contact: admin@example.org`;
|
|
861
|
+
|
|
862
|
+
return [host, separator, prose, credentialed];
|
|
863
|
+
}));
|
|
864
|
+
|
|
865
|
+
it.each(rows)('keeps prose after a %s host with the %j separator', (host, separator, prose) => {
|
|
866
|
+
expect(utils.stripUrlCredentials(prose)).toBe(prose);
|
|
867
|
+
expect(utils.stripUrlValueCredentials(prose)).toBe(prose);
|
|
868
|
+
});
|
|
869
|
+
|
|
870
|
+
it.each(rows)('redacts userinfo before a %s host with the %j separator', (host, separator, prose, credentialed) => {
|
|
871
|
+
expect(utils.stripUrlCredentials(credentialed)).toBe(prose);
|
|
872
|
+
expect(utils.stripUrlValueCredentials(credentialed)).toBe(prose);
|
|
873
|
+
});
|
|
874
|
+
});
|
|
875
|
+
|
|
876
|
+
describe('stripKnownUrlCredentials', () => {
|
|
877
|
+
it.each([
|
|
878
|
+
[
|
|
879
|
+
'https://user:secret@example.org/list.txt',
|
|
880
|
+
'https://example.org/list.txt',
|
|
881
|
+
],
|
|
882
|
+
[
|
|
883
|
+
'https://user:secret@example.org',
|
|
884
|
+
'https://example.org',
|
|
885
|
+
],
|
|
886
|
+
[
|
|
887
|
+
"https://o'brien:secret@example.org/list.txt",
|
|
888
|
+
'https://example.org/list.txt',
|
|
889
|
+
],
|
|
890
|
+
[
|
|
891
|
+
"https://o'brien:secret@example.org",
|
|
892
|
+
'https://example.org',
|
|
893
|
+
],
|
|
894
|
+
[
|
|
895
|
+
'https://John Smith:secret@example.org/list.txt',
|
|
896
|
+
'https://example.org/list.txt',
|
|
897
|
+
],
|
|
898
|
+
[
|
|
899
|
+
'https://John Smith:secret@example.org',
|
|
900
|
+
'https://example.org',
|
|
901
|
+
],
|
|
902
|
+
[
|
|
903
|
+
"https://o'brien:p@ssw0rd@example.org/list.txt",
|
|
904
|
+
'https://example.org/list.txt',
|
|
905
|
+
],
|
|
906
|
+
[
|
|
907
|
+
"https://o'brien:p@ssw0rd@example.org",
|
|
908
|
+
'https://example.org',
|
|
909
|
+
],
|
|
910
|
+
[
|
|
911
|
+
'https://John Smith:p@ss@example.org/list.txt',
|
|
912
|
+
'https://example.org/list.txt',
|
|
913
|
+
],
|
|
914
|
+
[
|
|
915
|
+
'https://John Smith:p@ss@example.org',
|
|
916
|
+
'https://example.org',
|
|
917
|
+
],
|
|
918
|
+
[
|
|
919
|
+
'https://J. Smith:secret@example.org/list.txt',
|
|
920
|
+
'https://example.org/list.txt',
|
|
921
|
+
],
|
|
922
|
+
[
|
|
923
|
+
'https://J. Smith:secret@example.org',
|
|
924
|
+
'https://example.org',
|
|
925
|
+
],
|
|
926
|
+
[
|
|
927
|
+
'https://admin user:secret@example.org',
|
|
928
|
+
'https://example.org',
|
|
929
|
+
],
|
|
930
|
+
[
|
|
931
|
+
'https://john smith:secret@example.org',
|
|
932
|
+
'https://example.org',
|
|
933
|
+
],
|
|
934
|
+
[
|
|
935
|
+
'https://user: secret@example.org/list.txt',
|
|
936
|
+
'https://example.org/list.txt',
|
|
937
|
+
],
|
|
938
|
+
[
|
|
939
|
+
'https://user: secret@example.org',
|
|
940
|
+
'https://example.org',
|
|
941
|
+
],
|
|
942
|
+
[
|
|
943
|
+
'https://user:p@s s@example.invalid/list.txt',
|
|
944
|
+
'https://example.invalid/list.txt',
|
|
945
|
+
],
|
|
946
|
+
[
|
|
947
|
+
' https://user:p s s@example.invalid/list.txt',
|
|
948
|
+
' https://example.invalid/list.txt',
|
|
949
|
+
],
|
|
950
|
+
[
|
|
951
|
+
'<https://user:p"s s@example.org/>',
|
|
952
|
+
'<https://example.org/>',
|
|
953
|
+
],
|
|
954
|
+
[
|
|
955
|
+
'See https://user:p"q"@example.org/list.txt',
|
|
956
|
+
'See https://example.org/list.txt',
|
|
957
|
+
],
|
|
958
|
+
[
|
|
959
|
+
'https://example.org:8080/list.txt',
|
|
960
|
+
'https://example.org:8080/list.txt',
|
|
961
|
+
],
|
|
962
|
+
[
|
|
963
|
+
'C:\\user:secret@example.org\\list.txt',
|
|
964
|
+
'C:\\user:secret@example.org\\list.txt',
|
|
965
|
+
],
|
|
966
|
+
[
|
|
967
|
+
'C:/user@corp/list.txt',
|
|
968
|
+
'C:/user@corp/list.txt',
|
|
969
|
+
],
|
|
970
|
+
[
|
|
971
|
+
'https://proxy.example.org/fetch?u=https://user:secret@raw.example.org/list.txt',
|
|
972
|
+
'https://proxy.example.org/fetch?u=https://raw.example.org/list.txt',
|
|
973
|
+
],
|
|
974
|
+
[
|
|
975
|
+
'https://proxy.example.org/r/https://user:secret@raw.example.org/list.txt',
|
|
976
|
+
'https://proxy.example.org/r/https://raw.example.org/list.txt',
|
|
977
|
+
],
|
|
978
|
+
[
|
|
979
|
+
'https://example.org/a,https://user:secret@raw.example.org/list.txt',
|
|
980
|
+
'https://example.org/a,https://raw.example.org/list.txt',
|
|
981
|
+
],
|
|
982
|
+
])('strips credentials from %s', (url, expected) => {
|
|
983
|
+
expect(utils.stripKnownUrlCredentials(url)).toBe(expected);
|
|
984
|
+
});
|
|
985
|
+
|
|
986
|
+
it.each([
|
|
987
|
+
[
|
|
988
|
+
'https://user:pa?ss@example.org/f.txt',
|
|
989
|
+
'https://example.org/f.txt',
|
|
990
|
+
],
|
|
991
|
+
[
|
|
992
|
+
'https://user:pa#ss@example.org/f.txt',
|
|
993
|
+
'https://example.org/f.txt',
|
|
994
|
+
],
|
|
995
|
+
[
|
|
996
|
+
'http://admin:#Secret1@localhost/f.txt',
|
|
997
|
+
'http://localhost/f.txt',
|
|
998
|
+
],
|
|
999
|
+
[
|
|
1000
|
+
'http://admin:?Secret1@localhost/f.txt',
|
|
1001
|
+
'http://localhost/f.txt',
|
|
1002
|
+
],
|
|
1003
|
+
[
|
|
1004
|
+
'https://admin:2024#Secret1@example.org/list.txt',
|
|
1005
|
+
'https://example.org/list.txt',
|
|
1006
|
+
],
|
|
1007
|
+
[
|
|
1008
|
+
'https://admin:2024?Secret1@example.org/list.txt',
|
|
1009
|
+
'https://example.org/list.txt',
|
|
1010
|
+
],
|
|
1011
|
+
[
|
|
1012
|
+
'https://admin:2024#Secret1@b',
|
|
1013
|
+
'https://b',
|
|
1014
|
+
],
|
|
1015
|
+
[
|
|
1016
|
+
'http://user:#Secret1@fileserver?format=adblock',
|
|
1017
|
+
'http://fileserver?format=adblock',
|
|
1018
|
+
],
|
|
1019
|
+
[
|
|
1020
|
+
'http://user:?Secret1@fileserver',
|
|
1021
|
+
'http://fileserver',
|
|
1022
|
+
],
|
|
1023
|
+
[
|
|
1024
|
+
'http://user:#Secret1@fileserver#section',
|
|
1025
|
+
'http://fileserver#section',
|
|
1026
|
+
],
|
|
1027
|
+
[
|
|
1028
|
+
'https://admin:2024#Secret1@fileserver/list.txt',
|
|
1029
|
+
'https://fileserver/list.txt',
|
|
1030
|
+
],
|
|
1031
|
+
[
|
|
1032
|
+
'https://admin:2024#Secret1@example.org/path@x',
|
|
1033
|
+
'https://example.org/path@x',
|
|
1034
|
+
],
|
|
1035
|
+
[
|
|
1036
|
+
'https://proxy.example.org/fetch?u=https://user:pa?ss@raw.example.org/list.txt',
|
|
1037
|
+
'https://proxy.example.org/fetch?u=https://raw.example.org/list.txt',
|
|
1038
|
+
],
|
|
1039
|
+
])('strips a malformed password that contains ? or #: %s', (url, expected) => {
|
|
1040
|
+
expect(utils.stripKnownUrlCredentials(url)).toBe(expected);
|
|
1041
|
+
});
|
|
1042
|
+
|
|
1043
|
+
it.each([
|
|
1044
|
+
'https://example.org:8080#contact@example.org',
|
|
1045
|
+
'https://example.org:8080?x@y',
|
|
1046
|
+
'https://example.org:?email=user@example.com',
|
|
1047
|
+
'https://localhost:8080#contact@example.org',
|
|
1048
|
+
'https://[::1]:8080#contact@example.org',
|
|
1049
|
+
'https://myhost:8080/path?email=user@example.com',
|
|
1050
|
+
'https://myhost:8080/path#contact@example.org',
|
|
1051
|
+
])('keeps a valid url with a port and an at sign in the query or fragment: %s', (url) => {
|
|
1052
|
+
expect(utils.stripKnownUrlCredentials(url)).toBe(url);
|
|
1053
|
+
});
|
|
1054
|
+
|
|
1055
|
+
it('treats a single-label port with a query at sign as userinfo', () => {
|
|
1056
|
+
// A digit-first password and a port are lexically identical. The
|
|
1057
|
+
// query form has no path to tell them apart, so the password form
|
|
1058
|
+
// wins to avoid a leak. The path form is kept by the path bound in
|
|
1059
|
+
// the malformed-userinfo scan.
|
|
1060
|
+
expect(utils.stripKnownUrlCredentials('https://myhost:8080?email=user@example.com'))
|
|
1061
|
+
.toBe('https://example.com');
|
|
1062
|
+
});
|
|
1063
|
+
|
|
1064
|
+
it('returns a non-string value unchanged', () => {
|
|
1065
|
+
expect(utils.stripKnownUrlCredentials(undefined)).toBeUndefined();
|
|
1066
|
+
expect(utils.stripKnownUrlCredentials(null)).toBeNull();
|
|
1067
|
+
});
|
|
1068
|
+
});
|
|
1069
|
+
|
|
1070
|
+
describe('stripConfigurationCredentials', () => {
|
|
1071
|
+
it('returns a sanitized copy without changing the original configuration', () => {
|
|
1072
|
+
const configuration = {
|
|
1073
|
+
name: 'Test filter',
|
|
1074
|
+
homepage: 'https://user:secret@example.org/',
|
|
1075
|
+
exclusions_sources: ['https://user:secret@example.org/exclusions.txt'],
|
|
1076
|
+
inclusions_sources: ['https://user:secret@example.org/inclusions.txt'],
|
|
1077
|
+
sources: [
|
|
1078
|
+
{
|
|
1079
|
+
name: 'source 1',
|
|
1080
|
+
source: 'https://user:secret@example.org/list.txt',
|
|
1081
|
+
exclusions_sources: ['https://user:secret@example.org/source-exclusions.txt'],
|
|
1082
|
+
inclusions_sources: ['https://user:secret@example.org/source-inclusions.txt'],
|
|
1083
|
+
},
|
|
1084
|
+
],
|
|
1085
|
+
};
|
|
1086
|
+
|
|
1087
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1088
|
+
|
|
1089
|
+
expect(sanitized).not.toBe(configuration);
|
|
1090
|
+
expect(sanitized).not.toEqual(configuration);
|
|
1091
|
+
expect(sanitized.name).toBe('Test filter');
|
|
1092
|
+
expect(sanitized.homepage).toBe('https://example.org/');
|
|
1093
|
+
expect(sanitized.exclusions_sources).toEqual(['https://example.org/exclusions.txt']);
|
|
1094
|
+
expect(sanitized.inclusions_sources).toEqual(['https://example.org/inclusions.txt']);
|
|
1095
|
+
expect(sanitized.sources[0].source).toBe('https://example.org/list.txt');
|
|
1096
|
+
expect(sanitized.sources[0].exclusions_sources).toEqual(['https://example.org/source-exclusions.txt']);
|
|
1097
|
+
expect(sanitized.sources[0].inclusions_sources).toEqual(['https://example.org/source-inclusions.txt']);
|
|
1098
|
+
|
|
1099
|
+
expect(configuration.homepage).toBe('https://user:secret@example.org/');
|
|
1100
|
+
expect(configuration.sources[0].source).toBe('https://user:secret@example.org/list.txt');
|
|
1101
|
+
});
|
|
1102
|
+
|
|
1103
|
+
it('sanitizes known URL fields with whitespace and quotes in userinfo', () => {
|
|
1104
|
+
const configuration = {
|
|
1105
|
+
homepage: 'https://user:p s s@example.org/',
|
|
1106
|
+
sources: [
|
|
1107
|
+
{
|
|
1108
|
+
source: 'https://user:p s s@example.org/list.txt',
|
|
1109
|
+
exclusions_sources: ['https://user:p s s@example.org/exclusions.txt'],
|
|
1110
|
+
inclusions_sources: ['https://user:p"s@example.org/inclusions.txt'],
|
|
1111
|
+
},
|
|
1112
|
+
],
|
|
1113
|
+
};
|
|
1114
|
+
|
|
1115
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1116
|
+
|
|
1117
|
+
expect(sanitized.homepage).toBe('https://example.org/');
|
|
1118
|
+
expect(sanitized.sources[0].source).toBe('https://example.org/list.txt');
|
|
1119
|
+
expect(sanitized.sources[0].exclusions_sources).toEqual(['https://example.org/exclusions.txt']);
|
|
1120
|
+
expect(sanitized.sources[0].inclusions_sources).toEqual(['https://example.org/inclusions.txt']);
|
|
1121
|
+
expect(configuration.homepage).toBe('https://user:p s s@example.org/');
|
|
1122
|
+
});
|
|
1123
|
+
|
|
1124
|
+
it('sanitizes known URL fields with a quote or space in the username', () => {
|
|
1125
|
+
const configuration = {
|
|
1126
|
+
homepage: "https://o'brien:secret@example.org",
|
|
1127
|
+
sources: [
|
|
1128
|
+
{
|
|
1129
|
+
source: 'https://John Smith:secret@example.org/list.txt',
|
|
1130
|
+
},
|
|
1131
|
+
],
|
|
1132
|
+
exclusions_sources: ['https://user: secret@example.org/exclusions.txt'],
|
|
1133
|
+
inclusions_sources: 'https://user:p@s s@example.org/inclusions.txt',
|
|
1134
|
+
};
|
|
1135
|
+
|
|
1136
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1137
|
+
|
|
1138
|
+
expect(sanitized.homepage).toBe('https://example.org');
|
|
1139
|
+
expect(sanitized.sources[0].source).toBe('https://example.org/list.txt');
|
|
1140
|
+
expect(sanitized.exclusions_sources).toEqual(['https://example.org/exclusions.txt']);
|
|
1141
|
+
expect(sanitized.inclusions_sources).toBe('https://example.org/inclusions.txt');
|
|
1142
|
+
expect(configuration.homepage).toBe("https://o'brien:secret@example.org");
|
|
1143
|
+
expect(configuration.sources[0].source).toBe('https://John Smith:secret@example.org/list.txt');
|
|
1144
|
+
});
|
|
1145
|
+
|
|
1146
|
+
it('sanitizes path-less known URL fields with a literal @ in the password', () => {
|
|
1147
|
+
const configuration = {
|
|
1148
|
+
homepage: 'https://user:p@s s@example.org',
|
|
1149
|
+
sources: [
|
|
1150
|
+
{
|
|
1151
|
+
source: 'https://user:p@s s@example.org',
|
|
1152
|
+
exclusions_sources: ['https://user:p@s s@example.org'],
|
|
1153
|
+
inclusions_sources: 'https://user:p@s s@example.org',
|
|
1154
|
+
},
|
|
1155
|
+
],
|
|
1156
|
+
};
|
|
1157
|
+
|
|
1158
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1159
|
+
|
|
1160
|
+
expect(sanitized.homepage).toBe('https://example.org');
|
|
1161
|
+
expect(sanitized.sources[0].source).toBe('https://example.org');
|
|
1162
|
+
expect(sanitized.sources[0].exclusions_sources).toEqual(['https://example.org']);
|
|
1163
|
+
expect(sanitized.sources[0].inclusions_sources).toBe('https://example.org');
|
|
1164
|
+
expect(configuration.homepage).toBe('https://user:p@s s@example.org');
|
|
1165
|
+
});
|
|
1166
|
+
|
|
1167
|
+
it('keeps prose that looks like a port', () => {
|
|
1168
|
+
const configuration = {
|
|
1169
|
+
description: 'https://example.org:8080 by admin@example.com',
|
|
1170
|
+
};
|
|
1171
|
+
|
|
1172
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1173
|
+
|
|
1174
|
+
expect(sanitized.description).toBe(configuration.description);
|
|
1175
|
+
});
|
|
1176
|
+
|
|
1177
|
+
it('keeps prose with a port on a host-like token and punctuation', () => {
|
|
1178
|
+
const configuration = {
|
|
1179
|
+
description: 'Mirror at https://example.org:8080, contact admin@example.com',
|
|
1180
|
+
};
|
|
1181
|
+
|
|
1182
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1183
|
+
|
|
1184
|
+
expect(sanitized.description).toBe(configuration.description);
|
|
1185
|
+
});
|
|
1186
|
+
|
|
1187
|
+
it('keeps prose that starts a label after a URL', () => {
|
|
1188
|
+
const configuration = {
|
|
1189
|
+
description: 'https://example.org team: contact@example.com',
|
|
1190
|
+
};
|
|
1191
|
+
|
|
1192
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1193
|
+
|
|
1194
|
+
expect(sanitized.description).toBe(configuration.description);
|
|
1195
|
+
});
|
|
1196
|
+
|
|
1197
|
+
it('sanitizes a description with a space in the username and after the password colon', () => {
|
|
1198
|
+
const configuration = {
|
|
1199
|
+
description: 'Mirror https://John Smith: secret@example.org',
|
|
1200
|
+
};
|
|
1201
|
+
|
|
1202
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1203
|
+
|
|
1204
|
+
expect(sanitized.description).toBe('Mirror https://example.org');
|
|
1205
|
+
});
|
|
1206
|
+
|
|
1207
|
+
it('sanitizes a description with a numeric password followed by punctuation', () => {
|
|
1208
|
+
const configuration = {
|
|
1209
|
+
description: 'Mirror https://user:1234, secret@example.org/list.txt',
|
|
1210
|
+
};
|
|
1211
|
+
|
|
1212
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1213
|
+
|
|
1214
|
+
expect(sanitized.description).toBe('Mirror https://example.org/list.txt');
|
|
1215
|
+
});
|
|
1216
|
+
|
|
1217
|
+
it('sanitizes values under unknown keys with the strict parser', () => {
|
|
1218
|
+
const configuration = {
|
|
1219
|
+
name: 'test',
|
|
1220
|
+
sources: [
|
|
1221
|
+
{
|
|
1222
|
+
url: 'https://John Smith: secret@example.org',
|
|
1223
|
+
},
|
|
1224
|
+
],
|
|
1225
|
+
unknown: 'https://user:1234, secret@example.org/list.txt',
|
|
1226
|
+
// A path-less value with a literal `@` and a space in the password
|
|
1227
|
+
// is sanitized only by the strict parser.
|
|
1228
|
+
unknownLiteralAt: 'https://user:p@s s@example.org',
|
|
1229
|
+
};
|
|
1230
|
+
|
|
1231
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1232
|
+
|
|
1233
|
+
expect(sanitized.sources[0].url).toBe('https://example.org');
|
|
1234
|
+
expect(sanitized.unknown).toBe('https://example.org/list.txt');
|
|
1235
|
+
expect(sanitized.unknownLiteralAt).toBe('https://example.org');
|
|
1236
|
+
expect(configuration.sources[0].url).toBe('https://John Smith: secret@example.org');
|
|
1237
|
+
expect(configuration.unknown).toBe('https://user:1234, secret@example.org/list.txt');
|
|
1238
|
+
expect(configuration.unknownLiteralAt).toBe('https://user:p@s s@example.org');
|
|
1239
|
+
});
|
|
1240
|
+
|
|
1241
|
+
it('sanitizes nested values under unknown keys', () => {
|
|
1242
|
+
const configuration = {
|
|
1243
|
+
unknown: {
|
|
1244
|
+
nested: ['https://John Smith: secret@example.org'],
|
|
1245
|
+
},
|
|
1246
|
+
};
|
|
1247
|
+
|
|
1248
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1249
|
+
|
|
1250
|
+
expect(sanitized.unknown.nested).toEqual(['https://example.org']);
|
|
1251
|
+
});
|
|
1252
|
+
|
|
1253
|
+
it('sanitizes URL object keys', () => {
|
|
1254
|
+
const configuration = {
|
|
1255
|
+
'https://user:secret@example.org/key.txt': 'value',
|
|
1256
|
+
'https://user:p"s@example.org/other.txt': 'value',
|
|
1257
|
+
};
|
|
1258
|
+
|
|
1259
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1260
|
+
|
|
1261
|
+
expect(sanitized['https://example.org/key.txt']).toBe('value');
|
|
1262
|
+
expect(sanitized['https://example.org/other.txt']).toBe('value');
|
|
1263
|
+
expect(configuration['https://user:secret@example.org/key.txt']).toBe('value');
|
|
1264
|
+
});
|
|
1265
|
+
|
|
1266
|
+
it('sanitizes URL object keys with whitespace in userinfo', () => {
|
|
1267
|
+
// Keys are never prose, so the strict parser must handle a whitespace
|
|
1268
|
+
// in the username and in the password.
|
|
1269
|
+
const configuration = {
|
|
1270
|
+
'https:John Smith:pa ss@example.org': 'value',
|
|
1271
|
+
};
|
|
1272
|
+
|
|
1273
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1274
|
+
|
|
1275
|
+
expect(sanitized).toEqual({ 'https:example.org': 'value' });
|
|
1276
|
+
expect(configuration['https:John Smith:pa ss@example.org']).toBe('value');
|
|
1277
|
+
});
|
|
1278
|
+
|
|
1279
|
+
it('sanitizes every string in malformed configuration shapes', () => {
|
|
1280
|
+
const configuration = {
|
|
1281
|
+
homepage: ['https://user:secret@example.org/'],
|
|
1282
|
+
exclusions_sources: 'https://user:secret@example.org/exclusions.txt',
|
|
1283
|
+
sources: [
|
|
1284
|
+
'https://user:secret@example.org/source.txt',
|
|
1285
|
+
{
|
|
1286
|
+
url: 'https://user:secret@example.org/misspelled.txt',
|
|
1287
|
+
source: ['https://user:secret@example.org/source-array.txt'],
|
|
1288
|
+
},
|
|
1289
|
+
],
|
|
1290
|
+
};
|
|
1291
|
+
|
|
1292
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1293
|
+
|
|
1294
|
+
expect(sanitized).toEqual({
|
|
1295
|
+
homepage: ['https://example.org/'],
|
|
1296
|
+
exclusions_sources: 'https://example.org/exclusions.txt',
|
|
1297
|
+
sources: [
|
|
1298
|
+
'https://example.org/source.txt',
|
|
1299
|
+
{
|
|
1300
|
+
url: 'https://example.org/misspelled.txt',
|
|
1301
|
+
source: ['https://example.org/source-array.txt'],
|
|
1302
|
+
},
|
|
1303
|
+
],
|
|
1304
|
+
});
|
|
1305
|
+
expect(configuration.homepage[0]).toBe('https://user:secret@example.org/');
|
|
1306
|
+
expect(configuration.sources[1].url).toBe('https://user:secret@example.org/misspelled.txt');
|
|
1307
|
+
});
|
|
1308
|
+
|
|
1309
|
+
it('sanitizes a malformed source string with whitespace in userinfo', () => {
|
|
1310
|
+
const configuration = {
|
|
1311
|
+
sources: ['https://user:p s s@example.org/source.txt'],
|
|
1312
|
+
};
|
|
1313
|
+
|
|
1314
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1315
|
+
|
|
1316
|
+
expect(sanitized.sources).toEqual(['https://example.org/source.txt']);
|
|
1317
|
+
expect(configuration.sources[0]).toBe('https://user:p s s@example.org/source.txt');
|
|
1318
|
+
});
|
|
1319
|
+
|
|
1320
|
+
it('does not change prose that starts with a URL', () => {
|
|
1321
|
+
const configuration = {
|
|
1322
|
+
description: 'https://example.org is maintained by user@example.com',
|
|
1323
|
+
};
|
|
1324
|
+
|
|
1325
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1326
|
+
|
|
1327
|
+
expect(sanitized.description).toBe(configuration.description);
|
|
1328
|
+
});
|
|
1329
|
+
|
|
1330
|
+
it('sanitizes strings in class instances', () => {
|
|
1331
|
+
class Source {
|
|
1332
|
+
constructor(source) {
|
|
1333
|
+
this.source = source;
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
|
|
1337
|
+
const configuration = {
|
|
1338
|
+
sources: [new Source('https://user:secret@example.org/list.txt')],
|
|
1339
|
+
};
|
|
1340
|
+
|
|
1341
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1342
|
+
|
|
1343
|
+
expect(sanitized.sources[0].source).toBe('https://example.org/list.txt');
|
|
1344
|
+
expect(configuration.sources[0].source).toBe('https://user:secret@example.org/list.txt');
|
|
1345
|
+
});
|
|
1346
|
+
|
|
1347
|
+
it('sanitizes a configuration that is itself a class instance', () => {
|
|
1348
|
+
class Configuration {
|
|
1349
|
+
constructor() {
|
|
1350
|
+
this.sources = [
|
|
1351
|
+
{ source: 'https://user:secret@example.org/list.txt' },
|
|
1352
|
+
];
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
const configuration = new Configuration();
|
|
1357
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1358
|
+
|
|
1359
|
+
expect(sanitized).toBeInstanceOf(Configuration);
|
|
1360
|
+
expect(sanitized.sources[0].source).toBe('https://example.org/list.txt');
|
|
1361
|
+
expect(configuration.sources[0].source).toBe('https://user:secret@example.org/list.txt');
|
|
1362
|
+
});
|
|
1363
|
+
});
|
|
1364
|
+
|
|
1365
|
+
describe('sanitizeConfigurationForLog', () => {
|
|
1366
|
+
it('does not change a configuration with a path-less URL and a later email', () => {
|
|
1367
|
+
// Re-scanning the rendered document treated the text between the
|
|
1368
|
+
// path-less homepage and the email as userinfo. The description key
|
|
1369
|
+
// disappeared and the homepage changed.
|
|
1370
|
+
const configuration = {
|
|
1371
|
+
homepage: 'https://example.org',
|
|
1372
|
+
description: 'admin@example.com',
|
|
1373
|
+
sources: [{ source: 'list.txt' }],
|
|
1374
|
+
};
|
|
1375
|
+
|
|
1376
|
+
const sanitized = utils.sanitizeConfigurationForLog(configuration);
|
|
1377
|
+
const dump = JSON.stringify(sanitized, 0, 4);
|
|
1378
|
+
|
|
1379
|
+
expect(dump).toBe(JSON.stringify(configuration, 0, 4));
|
|
1380
|
+
});
|
|
1381
|
+
|
|
1382
|
+
it('does not change a rendered configuration that has no credentials', () => {
|
|
1383
|
+
const configuration = {
|
|
1384
|
+
homepage: 'https://example.org',
|
|
1385
|
+
exclusions: ['@@||good.example^'],
|
|
1386
|
+
sources: [{ source: 'lists/a.txt' }],
|
|
1387
|
+
};
|
|
1388
|
+
|
|
1389
|
+
const dump = JSON.stringify(utils.sanitizeConfigurationForLog(configuration), 0, 4);
|
|
1390
|
+
|
|
1391
|
+
expect(dump).toBe(JSON.stringify(configuration, 0, 4));
|
|
1392
|
+
});
|
|
1393
|
+
|
|
1394
|
+
it('sanitizes a credentialed source in a rendered configuration with exclusions', () => {
|
|
1395
|
+
const configuration = {
|
|
1396
|
+
homepage: 'https://example.org',
|
|
1397
|
+
exclusions: ['@@||good.example^'],
|
|
1398
|
+
sources: [{ source: 'https://user:secret@example.org/list.txt' }],
|
|
1399
|
+
};
|
|
1400
|
+
|
|
1401
|
+
const dump = JSON.stringify(utils.sanitizeConfigurationForLog(configuration), 0, 4);
|
|
1402
|
+
|
|
1403
|
+
expect(dump).toContain('"homepage": "https://example.org"');
|
|
1404
|
+
expect(dump).toContain('"@@||good.example^"');
|
|
1405
|
+
expect(dump).toContain('"source": "https://example.org/list.txt"');
|
|
1406
|
+
expect(dump).not.toContain('secret');
|
|
1407
|
+
});
|
|
1408
|
+
|
|
1409
|
+
it('sanitizes a URL object, a serialized value, and URL-like keys', () => {
|
|
1410
|
+
class Serialized {
|
|
1411
|
+
toJSON() {
|
|
1412
|
+
return 'https://user:secret@example.org/serialized.txt';
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
const configuration = {
|
|
1417
|
+
homepage: new URL('https://user:secret@example.org/'),
|
|
1418
|
+
description: new Serialized(),
|
|
1419
|
+
'https://user:secret@example.org/key.txt': 'value',
|
|
1420
|
+
sources: [{ source: 'https://example.org/list.txt' }],
|
|
1421
|
+
};
|
|
1422
|
+
|
|
1423
|
+
const dump = JSON.stringify(utils.sanitizeConfigurationForLog(configuration));
|
|
1424
|
+
|
|
1425
|
+
expect(dump).toContain('"homepage":"https://example.org/"');
|
|
1426
|
+
expect(dump).toContain('"description":"https://example.org/serialized.txt"');
|
|
1427
|
+
expect(dump).toContain('"https://example.org/key.txt":"value"');
|
|
1428
|
+
expect(dump).not.toContain('secret');
|
|
1429
|
+
});
|
|
1430
|
+
|
|
1431
|
+
it('sanitizes URL-like keys introduced by toJSON', () => {
|
|
1432
|
+
class Configuration {
|
|
1433
|
+
toJSON() {
|
|
1434
|
+
return {
|
|
1435
|
+
'https://user:secret@example.org/key.txt': 'value',
|
|
1436
|
+
};
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
const dump = JSON.stringify(utils.sanitizeConfigurationForLog(new Configuration()));
|
|
1441
|
+
|
|
1442
|
+
expect(dump).toContain('"https://example.org/key.txt":"value"');
|
|
1443
|
+
expect(dump).not.toContain('secret');
|
|
1444
|
+
});
|
|
1445
|
+
|
|
1446
|
+
it('does not change the original configuration', () => {
|
|
1447
|
+
const configuration = {
|
|
1448
|
+
homepage: new URL('https://user:secret@example.org/'),
|
|
1449
|
+
};
|
|
1450
|
+
|
|
1451
|
+
utils.sanitizeConfigurationForLog(configuration);
|
|
1452
|
+
|
|
1453
|
+
expect(configuration.homepage.href).toBe('https://user:secret@example.org/');
|
|
1454
|
+
});
|
|
1455
|
+
});
|
|
1456
|
+
|
|
1457
|
+
describe('stripErrorCredentials', () => {
|
|
1458
|
+
it('sanitizes the message, stack, and cause chain in place', () => {
|
|
1459
|
+
const aggregateError = new global.AggregateError([
|
|
1460
|
+
new Error('aggregate https://user:secret@example.net/aggregate.txt'),
|
|
1461
|
+
]);
|
|
1462
|
+
const innerError = new Error('inner https:///user:secret@example.org/inner.txt');
|
|
1463
|
+
innerError.cause = aggregateError;
|
|
1464
|
+
|
|
1465
|
+
const error = new Error('outer https://user:secret@example.org/outer.txt');
|
|
1466
|
+
error.stack = 'Error: outer https://user:secret@example.org/outer.txt\n at test';
|
|
1467
|
+
error.cause = innerError;
|
|
1468
|
+
|
|
1469
|
+
const sanitized = utils.stripErrorCredentials(error);
|
|
1470
|
+
|
|
1471
|
+
expect(sanitized).toBe(error);
|
|
1472
|
+
expect(error.message).toBe('outer https://example.org/outer.txt');
|
|
1473
|
+
expect(error.stack).toBe('Error: outer https://example.org/outer.txt\n at test');
|
|
1474
|
+
expect(error.cause.message).toBe('inner https:///example.org/inner.txt');
|
|
1475
|
+
expect(error.cause.cause.errors[0].message).toBe('aggregate https://example.net/aggregate.txt');
|
|
1476
|
+
expect(JSON.stringify(error, Object.getOwnPropertyNames(error))).not.toContain('secret');
|
|
1477
|
+
});
|
|
1478
|
+
|
|
1479
|
+
it('sanitizes a string cause and a string in the errors array', () => {
|
|
1480
|
+
const error = new Error('outer');
|
|
1481
|
+
error.cause = 'cause https://user:secret@example.org/cause.txt';
|
|
1482
|
+
error.errors = ['aggregate https://user:secret@example.net/aggregate.txt'];
|
|
1483
|
+
|
|
1484
|
+
utils.stripErrorCredentials(error);
|
|
1485
|
+
|
|
1486
|
+
expect(error.cause).toBe('cause https://example.org/cause.txt');
|
|
1487
|
+
expect(error.errors).toEqual(['aggregate https://example.net/aggregate.txt']);
|
|
1488
|
+
});
|
|
1489
|
+
|
|
1490
|
+
it('applies exact replacements before the generic scan', () => {
|
|
1491
|
+
const raw = 'https:///user:p@s s@example.invalid/list.txt';
|
|
1492
|
+
const sanitized = 'https:///example.invalid/list.txt';
|
|
1493
|
+
const error = new Error(`Failed to request url '${raw}'`);
|
|
1494
|
+
|
|
1495
|
+
utils.stripErrorCredentials(error, [[raw, sanitized]]);
|
|
1496
|
+
|
|
1497
|
+
expect(error.message).toBe(`Failed to request url '${sanitized}'`);
|
|
1498
|
+
});
|
|
1499
|
+
|
|
1500
|
+
it('returns primitive values unchanged', () => {
|
|
1501
|
+
expect(utils.stripErrorCredentials('https://user:secret@example.org/')).toBe('https://user:secret@example.org/');
|
|
1502
|
+
expect(utils.stripErrorCredentials(undefined)).toBeUndefined();
|
|
1503
|
+
expect(utils.stripErrorCredentials(null)).toBeNull();
|
|
1504
|
+
});
|
|
1505
|
+
});
|
|
1506
|
+
|
|
22
1507
|
describe('substringBetween', () => {
|
|
23
1508
|
it('works', () => {
|
|
24
1509
|
let substr = utils.substringBetween('<a>test</a>', '<a>', '</a>');
|
|
@@ -34,3 +1519,77 @@ describe('substringBetween', () => {
|
|
|
34
1519
|
expect(substr).toBe(null);
|
|
35
1520
|
});
|
|
36
1521
|
});
|
|
1522
|
+
|
|
1523
|
+
describe('stripConfigurationCredentials with nested URLs', () => {
|
|
1524
|
+
it('sanitizes a URL nested in another URL with the strict parser', () => {
|
|
1525
|
+
const configuration = {
|
|
1526
|
+
homepage: 'https://proxy.example.org/fetch?u=https://user:secret@raw.example.org/list.txt',
|
|
1527
|
+
sources: [
|
|
1528
|
+
{
|
|
1529
|
+
source: 'https://proxy.example.org/r/https://user:secret@raw.example.org/list.txt',
|
|
1530
|
+
},
|
|
1531
|
+
],
|
|
1532
|
+
};
|
|
1533
|
+
|
|
1534
|
+
const sanitized = utils.stripConfigurationCredentials(configuration);
|
|
1535
|
+
|
|
1536
|
+
expect(sanitized.homepage).toBe('https://proxy.example.org/fetch?u=https://raw.example.org/list.txt');
|
|
1537
|
+
expect(sanitized.sources[0].source).toBe(
|
|
1538
|
+
'https://proxy.example.org/r/https://raw.example.org/list.txt',
|
|
1539
|
+
);
|
|
1540
|
+
expect(configuration.homepage).toBe(
|
|
1541
|
+
'https://proxy.example.org/fetch?u=https://user:secret@raw.example.org/list.txt',
|
|
1542
|
+
);
|
|
1543
|
+
});
|
|
1544
|
+
});
|
|
1545
|
+
|
|
1546
|
+
describe('credential field sets', () => {
|
|
1547
|
+
/**
|
|
1548
|
+
* Collects the fields marked with `x-credential-url` in the configuration
|
|
1549
|
+
* schema. The marker is the schema-side source of truth for the strict
|
|
1550
|
+
* credential parser: a new URL property must carry it, and this helper then
|
|
1551
|
+
* requires the matching entry in `src/utils.js`.
|
|
1552
|
+
*
|
|
1553
|
+
* @param {Object} schema - the configuration schema.
|
|
1554
|
+
* @returns {{ value: Array<String>, list: Array<String> }} sorted field
|
|
1555
|
+
* names grouped by marker kind.
|
|
1556
|
+
*/
|
|
1557
|
+
function collectCredentialUrlFields(schema) {
|
|
1558
|
+
const value = new Set();
|
|
1559
|
+
const list = new Set();
|
|
1560
|
+
const propertyGroups = [
|
|
1561
|
+
schema.properties,
|
|
1562
|
+
schema.definitions.source.properties,
|
|
1563
|
+
];
|
|
1564
|
+
|
|
1565
|
+
propertyGroups.forEach((properties) => {
|
|
1566
|
+
Object.entries(properties).forEach(([key, property]) => {
|
|
1567
|
+
if (property['x-credential-url'] === 'value') {
|
|
1568
|
+
value.add(key);
|
|
1569
|
+
}
|
|
1570
|
+
if (property['x-credential-url'] === 'list') {
|
|
1571
|
+
list.add(key);
|
|
1572
|
+
}
|
|
1573
|
+
});
|
|
1574
|
+
});
|
|
1575
|
+
|
|
1576
|
+
return {
|
|
1577
|
+
value: [...value].sort(),
|
|
1578
|
+
list: [...list].sort(),
|
|
1579
|
+
};
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
it('matches the configuration schema key sets', () => {
|
|
1583
|
+
expect([...utils.CONFIGURATION_KEYS].sort())
|
|
1584
|
+
.toEqual(Object.keys(configurationSchema.properties).sort());
|
|
1585
|
+
expect([...utils.SOURCE_KEYS].sort())
|
|
1586
|
+
.toEqual(Object.keys(configurationSchema.definitions.source.properties).sort());
|
|
1587
|
+
});
|
|
1588
|
+
|
|
1589
|
+
it('matches the URL fields marked in the configuration schema', () => {
|
|
1590
|
+
const marked = collectCredentialUrlFields(configurationSchema);
|
|
1591
|
+
|
|
1592
|
+
expect([...utils.URL_VALUE_FIELDS].sort()).toEqual(marked.value);
|
|
1593
|
+
expect([...utils.URL_LIST_FIELDS].sort()).toEqual(marked.list);
|
|
1594
|
+
});
|
|
1595
|
+
});
|