@stamhoofd/backend 2.137.3 → 2.137.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/package.json +23 -17
  2. package/src/crons/drip-emails.ts +2 -1
  3. package/src/crons.ts +4 -2
  4. package/src/email-recipient-loaders/payments.ts +3 -2
  5. package/src/endpoints/auth/CreateAdminEndpoint.ts +3 -2
  6. package/src/endpoints/auth/CreateTokenEndpoint.ts +2 -1
  7. package/src/endpoints/auth/ForgotPasswordEndpoint.ts +3 -2
  8. package/src/endpoints/auth/PatchUserEndpoint.ts +6 -3
  9. package/src/endpoints/auth/RetryEmailVerificationEndpoint.ts +2 -1
  10. package/src/endpoints/auth/SignupEndpoint.ts +4 -2
  11. package/src/endpoints/global/email/CreateEmailEndpoint.ts +2 -1
  12. package/src/endpoints/global/email/GetAdminEmailsEndpoint.test.ts +1 -1
  13. package/src/endpoints/global/email/GetAdminEmailsEndpoint.ts +2 -1
  14. package/src/endpoints/global/email/GetEmailEndpoint.ts +2 -1
  15. package/src/endpoints/global/email/GetUserEmailsEndpoint.test.ts +1 -1
  16. package/src/endpoints/global/email/GetUserEmailsEndpoint.ts +2 -1
  17. package/src/endpoints/global/email/PatchEmailEndpoint.ts +2 -1
  18. package/src/endpoints/global/files/ExportToExcelEndpoint.ts +9 -5
  19. package/src/endpoints/global/files/UploadFile.test.ts +206 -0
  20. package/src/endpoints/global/files/UploadFile.ts +31 -6
  21. package/src/endpoints/global/files/UploadImage.test.ts +177 -0
  22. package/src/endpoints/global/files/UploadImage.ts +23 -4
  23. package/src/endpoints/global/files/upload-security.test.ts +837 -0
  24. package/src/endpoints/global/members/PatchOrganizationMembersEndpoint.ts +1 -1
  25. package/src/endpoints/global/organizations/CreateOrganizationEndpoint.ts +2 -1
  26. package/src/endpoints/organization/dashboard/documents/GetDocumentTemplateXML.ts +2 -1
  27. package/src/endpoints/organization/dashboard/organization/SetOrganizationDomainEndpoint.ts +3 -2
  28. package/src/endpoints/organization/dashboard/users/PatchApiUserEndpoint.ts +5 -3
  29. package/src/endpoints/organization/dashboard/webshops/PatchWebshopOrdersEndpoint.ts +4 -3
  30. package/src/endpoints/organization/shared/GetDocumentHtml.ts +2 -1
  31. package/src/endpoints/organization/webshops/PlaceOrderEndpoint.ts +4 -3
  32. package/src/excel-loaders/balance-items.ts +1 -1
  33. package/src/excel-loaders/event-notifications.ts +6 -7
  34. package/src/excel-loaders/members.test.ts +2 -2
  35. package/src/excel-loaders/members.ts +9 -10
  36. package/src/excel-loaders/organizations.ts +14 -20
  37. package/src/excel-loaders/payments.ts +1 -1
  38. package/src/excel-loaders/platform-memberships.ts +7 -7
  39. package/src/excel-loaders/platform-sheets.test.ts +113 -0
  40. package/src/excel-loaders/receivable-balances.ts +1 -1
  41. package/src/excel-loaders/registrations.ts +9 -9
  42. package/src/helpers/AdminPermissionChecker.ts +1 -1
  43. package/src/helpers/AuthenticatedStructures.ts +12 -9
  44. package/src/helpers/MembershipCharger.ts +3 -2
  45. package/src/services/BalanceItemService.ts +2 -1
  46. package/src/services/DocumentRenderService.test.ts +229 -0
  47. package/src/services/DocumentRenderService.ts +180 -0
  48. package/src/services/EmailPreviewService.test.ts +300 -0
  49. package/src/services/EmailPreviewService.ts +239 -0
  50. package/src/services/FileSignService.test.ts +85 -0
  51. package/src/services/FileSignService.ts +23 -1
  52. package/src/services/OrderService.test.ts +308 -0
  53. package/src/services/OrderService.ts +214 -0
  54. package/src/services/OrganizationDNSService.test.ts +177 -0
  55. package/src/services/OrganizationDNSService.ts +282 -0
  56. package/src/services/OrganizationEmailService.test.ts +57 -0
  57. package/src/services/OrganizationEmailService.ts +211 -0
  58. package/src/services/PasswordForgotService.test.ts +99 -0
  59. package/src/services/PasswordForgotService.ts +38 -0
  60. package/src/services/PlatformMembershipService.test.ts +180 -0
  61. package/src/services/PlatformMembershipService.ts +281 -4
  62. package/src/services/RegistrationService.ts +3 -2
  63. package/src/services/STPackageService.test.ts +191 -0
  64. package/src/services/STPackageService.ts +114 -2
  65. package/src/services/VerificationCodeService.test.ts +71 -0
  66. package/src/services/VerificationCodeService.ts +100 -0
  67. package/tests/e2e/documents.test.ts +2 -1
  68. package/tests/e2e/private-files.test.ts +77 -14
@@ -0,0 +1,837 @@
1
+ import type { PutObjectCommand } from '@aws-sdk/client-s3';
2
+ import { S3Client } from '@aws-sdk/client-s3';
3
+ import { Request } from '@simonbackx/simple-endpoints';
4
+ import type { Organization } from '@stamhoofd/models';
5
+ import { Image, OrganizationFactory, Token, UserFactory } from '@stamhoofd/models';
6
+ import { File, PermissionLevel, Permissions } from '@stamhoofd/structures';
7
+ import { TestUtils } from '@stamhoofd/test-utils';
8
+ import type { IncomingMessage } from 'node:http';
9
+ import { Readable } from 'node:stream';
10
+
11
+ import { testServer } from '../../../../tests/helpers/TestServer.js';
12
+ import { FileSignService } from '../../../services/FileSignService.js';
13
+ import { UploadFile } from './UploadFile.js';
14
+ import { limiter, UploadImage } from './UploadImage.js';
15
+
16
+ /**
17
+ * Adversarial tests for the upload endpoints.
18
+ *
19
+ * Everything we store is served back from a domain of ours, so an upload may never end up on the file server
20
+ * with a content type a browser executes, with a key we didn't build ourselves, or with a public ACL the
21
+ * uploader wasn't allowed to ask for.
22
+ */
23
+
24
+ /**
25
+ * A real (1x1 pixel) png, so sharp can generate resolutions from it
26
+ */
27
+ const pngContent = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==', 'base64');
28
+
29
+ const svgContent = Buffer.from('<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10"><script>alert(1)</script><rect width="10" height="10" fill="red"/></svg>');
30
+
31
+ const htmlContent = Buffer.from('<!DOCTYPE html><html><body><script>alert(document.domain)</script></body></html>');
32
+
33
+ const xxeContent = Buffer.from('<?xml version="1.0"?><!DOCTYPE foo [<!ENTITY xxe SYSTEM "file:///etc/passwd">]><foo>&xxe;</foo>');
34
+
35
+ /**
36
+ * The only content types a browser may render inline: a pdf is opened in the pdf viewer, which has no access
37
+ * to the page it is served from, and a browser only ever sniffs an image as another image type.
38
+ */
39
+ const inlineSafeContentTypes = [
40
+ 'application/pdf',
41
+ 'image/jpeg',
42
+ 'image/png',
43
+ 'image/gif',
44
+ 'image/webp',
45
+ 'image/heic',
46
+ 'image/heif',
47
+ ];
48
+
49
+ /**
50
+ * Content types a browser executes on the domain it downloads them from
51
+ */
52
+ const executableContentTypes = [
53
+ 'text/html',
54
+ 'image/svg+xml',
55
+ 'image/svg',
56
+ 'text/xml',
57
+ 'application/xml',
58
+ 'application/xhtml+xml',
59
+ 'text/javascript',
60
+ 'application/javascript',
61
+ 'application/wasm',
62
+ 'application/x-httpd-php',
63
+ ];
64
+
65
+ const boundary = '--------------------------StamhoofdSecurityTest';
66
+
67
+ type RawPart = {
68
+ /**
69
+ * The raw headers of this part, without the trailing empty line
70
+ */
71
+ headers: string;
72
+ body: Buffer | string;
73
+ };
74
+
75
+ /**
76
+ * Builds the part a browser sends for <input type="file">. Pass null to leave a header out completely.
77
+ */
78
+ const filePart = (data: { filename?: string | null; contentType?: string | null; content?: Buffer | string; name?: string }): RawPart => {
79
+ let headers = `Content-Disposition: form-data; name="${data.name ?? 'file'}"`;
80
+
81
+ if (data.filename !== null && data.filename !== undefined) {
82
+ headers += `; filename="${data.filename}"`;
83
+ }
84
+
85
+ if (data.contentType !== null && data.contentType !== undefined) {
86
+ headers += `\r\nContent-Type: ${data.contentType}`;
87
+ }
88
+
89
+ return { headers, body: data.content ?? 'Hello world' };
90
+ };
91
+
92
+ const fieldPart = (name: string, value: string): RawPart => {
93
+ return { headers: `Content-Disposition: form-data; name="${name}"`, body: value };
94
+ };
95
+
96
+ describe('Upload security', () => {
97
+ const uploadFileEndpoint = new UploadFile();
98
+ const uploadImageEndpoint = new UploadImage();
99
+
100
+ /**
101
+ * All the objects that were sent to the (mocked) file server
102
+ */
103
+ let uploads: PutObjectCommand['input'][] = [];
104
+
105
+ let organization: Organization;
106
+ let adminToken: string;
107
+ let memberToken: string;
108
+ let adminUserId: string;
109
+ let memberUserId: string;
110
+
111
+ beforeAll(async () => {
112
+ organization = await new OrganizationFactory({}).create();
113
+
114
+ const admin = await new UserFactory({
115
+ organization,
116
+ permissions: Permissions.create({ level: PermissionLevel.Full }),
117
+ }).create();
118
+ adminUserId = admin.id;
119
+ adminToken = (await Token.createToken(admin)).accessToken;
120
+
121
+ // A user without any permissions: it may only upload private files
122
+ const member = await new UserFactory({ organization }).create();
123
+ memberUserId = member.id;
124
+ memberToken = (await Token.createToken(member)).accessToken;
125
+ });
126
+
127
+ beforeEach(() => {
128
+ TestUtils.setEnvironment('SPACES_BUCKET', 'test-bucket');
129
+ TestUtils.setEnvironment('SPACES_ENDPOINT', 'test.digitaloceanspaces.com');
130
+ TestUtils.setEnvironment('SPACES_KEY', 'test-key');
131
+ TestUtils.setEnvironment('SPACES_SECRET', 'test-secret');
132
+
133
+ uploads = [];
134
+ Image.s3Client = {
135
+ send: (command: PutObjectCommand) => {
136
+ uploads.push(command.input);
137
+ return Promise.resolve({});
138
+ },
139
+ } as any;
140
+
141
+ // These tests share one user, so the (per user) upload rate limit may not carry over between them
142
+ for (const window of limiter.windows) {
143
+ window.windows.clear();
144
+ }
145
+ });
146
+
147
+ afterEach(() => {
148
+ Image.s3Client = null;
149
+ });
150
+
151
+ const buildRequest = (url: string, parts: RawPart[], options: { isPrivate?: boolean; contentLength?: string; token?: string } = {}) => {
152
+ const chunks: Buffer[] = [];
153
+
154
+ for (const part of parts) {
155
+ chunks.push(Buffer.from(`--${boundary}\r\n${part.headers}\r\n\r\n`));
156
+ chunks.push(Buffer.isBuffer(part.body) ? part.body : Buffer.from(part.body));
157
+ chunks.push(Buffer.from('\r\n'));
158
+ }
159
+ chunks.push(Buffer.from(`--${boundary}--\r\n`));
160
+
161
+ const body = Buffer.concat(chunks);
162
+ const stream = Readable.from([body]) as unknown as IncomingMessage;
163
+ stream.headers = {
164
+ 'content-type': `multipart/form-data; boundary=${boundary}`,
165
+ 'content-length': options.contentLength ?? body.length.toString(),
166
+ };
167
+
168
+ const r = Request.buildJson('POST', url, organization.getApiHost());
169
+ r.headers.authorization = 'Bearer ' + (options.token ?? adminToken);
170
+ r.request = stream;
171
+ r.query = options.isPrivate ? { private: true } : {};
172
+
173
+ return r;
174
+ };
175
+
176
+ const uploadFile = async (data: { filename?: string | null; contentType?: string | null; content?: Buffer | string }, options: { isPrivate?: boolean; token?: string } = {}) => {
177
+ return await testServer.test(uploadFileEndpoint, buildRequest('/v1/upload-file', [filePart(data)], options));
178
+ };
179
+
180
+ const uploadImage = async (data: { filename?: string | null; contentType?: string | null; content?: Buffer | string; resolutions?: object[] }, options: { isPrivate?: boolean; token?: string } = {}) => {
181
+ return await testServer.test(uploadImageEndpoint, buildRequest('/v1/upload-image', [
182
+ fieldPart('resolutions', JSON.stringify(data.resolutions ?? [{ width: 50, height: 50, fit: 'inside' }])),
183
+ filePart(data),
184
+ ], options));
185
+ };
186
+
187
+ /**
188
+ * The invariants that have to hold for every object we sent to the file server, whatever was uploaded.
189
+ */
190
+ const expectSafeUploads = () => {
191
+ for (const upload of uploads) {
192
+ const key = upload.Key;
193
+ expect(key).toBeDefined();
194
+
195
+ // Only the characters we generate ourselves: a slug, uuids and slashes
196
+ expect(key, 'Unsafe key').toMatch(/^[a-z0-9\-/]+\.[a-z0-9]+$/);
197
+ expect(key!.split('/'), 'Key escapes its prefix').not.toContain('..');
198
+ expect(key, 'Key contains a relative path').not.toContain('..');
199
+
200
+ // Exactly one extension, and only in the last segment of the key
201
+ const segments = key!.split('/');
202
+ expect(segments.filter(segment => segment.includes('.')), 'More than one extension in ' + key).toHaveLength(1);
203
+ expect(segments[segments.length - 1].split('.'), 'Double extension in ' + key).toHaveLength(2);
204
+
205
+ // A browser only ever renders a file type it cannot execute
206
+ if (upload.ContentDisposition === 'inline') {
207
+ expect(inlineSafeContentTypes, 'Rendered inline: ' + upload.ContentType).toContain(upload.ContentType);
208
+ }
209
+
210
+ expect(['private', 'public-read'], 'Unexpected ACL').toContain(upload.ACL);
211
+
212
+ // The only object we store with an executable content type is the private source of an image
213
+ // upload, which is never served: it is private and stored as an attachment.
214
+ if (executableContentTypes.includes(upload.ContentType!)) {
215
+ expect(upload.ACL, 'Executable content type: ' + upload.ContentType).toBe('private');
216
+ expect(upload.ContentDisposition, 'Executable content type: ' + upload.ContentType).toBe('attachment');
217
+ }
218
+ }
219
+ };
220
+
221
+ /**
222
+ * A malformed or abusive request may be refused, but it may never result in an upload we didn't validate
223
+ */
224
+ const expectRejectedOrSafe = async (promise: Promise<unknown>) => {
225
+ await promise.catch(() => { /* refusing the request is a valid outcome */ });
226
+ expectSafeUploads();
227
+ };
228
+
229
+ describe('Key injection through the filename', () => {
230
+ const attackFilenames = [
231
+ '../../etc/passwd.pdf',
232
+ '..\\..\\evil.pdf',
233
+ '/etc/passwd.pdf',
234
+ 'folder/subfolder/evil.pdf',
235
+ 'evil.pdf/../../../root.pdf',
236
+ '....pdf',
237
+ '---.pdf',
238
+ '你好.pdf',
239
+ 'ᴬᴮᶜ.pdf',
240
+ '.....',
241
+ 'evil\u0000.pdf',
242
+ 'evil .pdf',
243
+ 'evil;rm -rf /.pdf',
244
+ 'evil\'`$().pdf',
245
+ '‮fdp.evil.pdf',
246
+ 'a'.repeat(300) + '.pdf',
247
+ '%2e%2e%2f%2e%2e%2fevil.pdf',
248
+ 'CON.pdf',
249
+ ' .pdf',
250
+ ];
251
+
252
+ test('A filename can never change the key we upload to', async () => {
253
+ for (const filename of attackFilenames) {
254
+ uploads = [];
255
+
256
+ const response = await uploadFile({ filename, contentType: 'application/pdf' });
257
+
258
+ expectSafeUploads();
259
+ expect(uploads, filename).toHaveLength(1);
260
+
261
+ const key = uploads[0].Key!;
262
+
263
+ // Inside the prefix of public uploads, and named after the id of the file
264
+ expect(key, filename).toMatch(new RegExp('^test/p/' + response.body.id + '/[a-z0-9-]+\\.pdf$'));
265
+ expect(key, filename).toBe(response.body.path);
266
+ expect(uploads[0].ContentType, filename).toBe('application/pdf');
267
+ }
268
+ });
269
+
270
+ test('A private upload always stays inside the prefix of its own user', async () => {
271
+ for (const filename of ['../../p/public.pdf', '../' + memberUserId + '/stolen.pdf', 'evil.pdf']) {
272
+ uploads = [];
273
+
274
+ await uploadFile({ filename, contentType: 'application/pdf' }, { isPrivate: true });
275
+
276
+ expectSafeUploads();
277
+ expect(uploads[0].Key, filename).toMatch(new RegExp('^test/users/' + adminUserId + '/'));
278
+ expect(uploads[0].ACL, filename).toBe('private');
279
+ }
280
+ });
281
+
282
+ test('Two uploads with the same name never write to the same key', async () => {
283
+ const first = await uploadFile({ filename: 'invoice.pdf', contentType: 'application/pdf' });
284
+ const second = await uploadFile({ filename: 'invoice.pdf', contentType: 'application/pdf' });
285
+
286
+ expect(first.body.path).not.toBe(second.body.path);
287
+ expect(uploads[0].Key).not.toBe(uploads[1].Key);
288
+ expectSafeUploads();
289
+ });
290
+
291
+ test('The name we report back always ends with the extension we stored the file with', async () => {
292
+ for (const [filename, contentType, expected] of [
293
+ ['evil.pdf.html', 'application/pdf', '.pdf'],
294
+ ['evil.html', 'application/pdf', '.pdf'],
295
+ ['evil.svg', 'image/png', '.png'],
296
+ ['photo.png', 'image/jpeg', '.jpg'],
297
+ ['notes.php', 'text/plain', '.txt'],
298
+ ] as const) {
299
+ const response = await uploadFile({ filename, contentType });
300
+
301
+ expect(response.body.name, filename).toMatch(new RegExp(expected.replace('.', '\\.') + '$'));
302
+ expect(File.resolveUploadType({ filename: response.body.name }), filename).not.toBeNull();
303
+ }
304
+ });
305
+ });
306
+
307
+ describe('Content type spoofing', () => {
308
+ test('An executable payload is never stored with a content type a browser runs', async () => {
309
+ const attacks: { filename: string; contentType: string; content: Buffer }[] = [
310
+ { filename: 'evil.html', contentType: 'text/html', content: htmlContent },
311
+ { filename: 'evil.svg', contentType: 'image/svg+xml', content: svgContent },
312
+ { filename: 'evil.svg', contentType: 'image/svg', content: svgContent },
313
+ { filename: 'evil.xml', contentType: 'text/xml', content: xxeContent },
314
+ { filename: 'evil.xml', contentType: 'application/xml', content: xxeContent },
315
+ { filename: 'evil.xhtml', contentType: 'application/xhtml+xml', content: htmlContent },
316
+ { filename: 'evil.js', contentType: 'text/javascript', content: Buffer.from('alert(1)') },
317
+ { filename: 'evil.wasm', contentType: 'application/wasm', content: Buffer.from('\0asm') },
318
+ { filename: 'evil.php', contentType: 'application/x-httpd-php', content: Buffer.from('<?php system($_GET[0]); ?>') },
319
+ { filename: 'evil.jsp', contentType: 'application/octet-stream', content: Buffer.from('<% out.print(1); %>') },
320
+ { filename: 'evil.html', contentType: 'application/octet-stream', content: htmlContent },
321
+ { filename: 'evil.html', contentType: '*/*', content: htmlContent },
322
+ { filename: 'evil.html', contentType: '', content: htmlContent },
323
+ { filename: 'evil.html', contentType: 'text/html; charset=utf-8', content: htmlContent },
324
+ { filename: 'evil.html', contentType: 'TEXT/HTML', content: htmlContent },
325
+ ];
326
+
327
+ for (const attack of attacks) {
328
+ uploads = [];
329
+ const description = attack.filename + ' as ' + attack.contentType;
330
+
331
+ await expectRejectedOrSafe(uploadFile(attack));
332
+
333
+ // Neither the content type nor the extension is on the allowlist, so nothing is stored at all
334
+ expect(uploads, description).toHaveLength(0);
335
+ }
336
+ });
337
+
338
+ test('A server config or a script is stored as the plain text file it claims to be', async () => {
339
+ // These are accepted (text/plain is on the allowlist), so the extension has to be neutralized
340
+ for (const filename of ['.htaccess', 'evil.php', 'web.config', 'evil.jsp', 'evil.js']) {
341
+ uploads = [];
342
+
343
+ const response = await uploadFile({
344
+ filename,
345
+ contentType: 'text/plain',
346
+ content: Buffer.from('AddType application/x-httpd-php .pdf'),
347
+ });
348
+
349
+ expect(uploads, filename).toHaveLength(1);
350
+ expect(uploads[0].ContentType, filename).toBe('text/plain');
351
+ expect(uploads[0].ContentDisposition, filename).toBe('attachment');
352
+ expect(uploads[0].Key, filename).toMatch(/\.txt$/);
353
+ expect(response.body.path, filename).not.toMatch(/htaccess|php|config|jsp|\.js/);
354
+ expectSafeUploads();
355
+ }
356
+ });
357
+
358
+ test('A polyglot is served as the safe type it claims to be, never as what is inside it', async () => {
359
+ // The bytes are html, but the browser is told it is a pdf and opens it in the pdf viewer
360
+ const pdf = await uploadFile({ filename: 'polyglot.pdf', contentType: 'application/pdf', content: htmlContent });
361
+ expect(uploads[0]).toMatchObject({
362
+ ContentType: 'application/pdf',
363
+ ContentDisposition: 'inline',
364
+ });
365
+ expect(pdf.body.contentType).toBe('application/pdf');
366
+
367
+ uploads = [];
368
+
369
+ // A browser never sniffs an image/png response into html
370
+ await uploadFile({ filename: 'polyglot.png', contentType: 'image/png', content: htmlContent });
371
+ expect(uploads[0]).toMatchObject({ ContentType: 'image/png' });
372
+
373
+ uploads = [];
374
+
375
+ // An svg that claims to be a png is stored, and served, as a png
376
+ await uploadFile({ filename: 'evil.svg', contentType: 'image/png', content: svgContent });
377
+ expect(uploads[0]).toMatchObject({ ContentType: 'image/png' });
378
+ expect(uploads[0].Key).toMatch(/\.png$/);
379
+
380
+ expectSafeUploads();
381
+ });
382
+
383
+ test('A content type header can never inject anything into what we store', async () => {
384
+ const attacks = [
385
+ 'application/pdf\r\nX-Injected: 1',
386
+ 'application/pdf"; ContentDisposition="inline',
387
+ 'application/pdf; charset=utf-8',
388
+ 'application/pdf, text/html',
389
+ ' application/pdf ',
390
+ ];
391
+
392
+ for (const contentType of attacks) {
393
+ uploads = [];
394
+
395
+ // The filename makes sure the upload is accepted, so we can look at what was stored
396
+ await expectRejectedOrSafe(uploadFile({ filename: 'notes.txt', contentType }));
397
+
398
+ for (const upload of uploads) {
399
+ // Only our own literals are ever stored: never a value taken from the request
400
+ expect([...inlineSafeContentTypes, 'text/plain'], contentType).toContain(upload.ContentType);
401
+ expect(upload.ContentType, contentType).not.toContain('\r');
402
+ expect(upload.ContentType, contentType).not.toContain(';');
403
+ }
404
+ }
405
+ });
406
+
407
+ test('An upload without a filename or a content type is refused instead of guessed', async () => {
408
+ await expectRejectedOrSafe(uploadFile({ filename: 'backup', contentType: 'application/octet-stream' }));
409
+ expect(uploads).toHaveLength(0);
410
+
411
+ await expectRejectedOrSafe(uploadFile({ filename: '', contentType: 'application/octet-stream' }));
412
+ expect(uploads).toHaveLength(0);
413
+
414
+ await expectRejectedOrSafe(uploadFile({ filename: 'backup' }));
415
+ expect(uploads).toHaveLength(0);
416
+ });
417
+ });
418
+
419
+ describe('Multipart abuse', () => {
420
+ test('It refuses a request with more than one file', async () => {
421
+ const request = buildRequest('/v1/upload-file', [
422
+ filePart({ filename: 'invoice.pdf', contentType: 'application/pdf' }),
423
+ filePart({ filename: 'evil.html', contentType: 'text/html', content: htmlContent }),
424
+ ]);
425
+
426
+ await expectRejectedOrSafe(testServer.test(uploadFileEndpoint, request));
427
+ expect(uploads).toHaveLength(0);
428
+ });
429
+
430
+ test('A part without a filename can never smuggle in a content type or a key', async () => {
431
+ // A part with a content type header is a file for formidable, even when it has no filename at all
432
+ const executable = buildRequest('/v1/upload-file', [
433
+ filePart({ filename: null, contentType: 'text/html', content: htmlContent }),
434
+ ]);
435
+
436
+ await expectRejectedOrSafe(testServer.test(uploadFileEndpoint, executable));
437
+ expect(uploads).toHaveLength(0);
438
+
439
+ // Without a content type header it is a field, so there is no file to upload
440
+ const field = buildRequest('/v1/upload-file', [
441
+ filePart({ filename: null, contentType: null, content: htmlContent }),
442
+ ]);
443
+
444
+ await expectRejectedOrSafe(testServer.test(uploadFileEndpoint, field));
445
+ expect(uploads).toHaveLength(0);
446
+
447
+ // An allowed content type without a filename is stored under a key we generated ourselves
448
+ const response = await testServer.test(uploadFileEndpoint, buildRequest('/v1/upload-file', [
449
+ filePart({ filename: null, contentType: 'application/pdf' }),
450
+ ]));
451
+
452
+ expectSafeUploads();
453
+ expect(response.body.name).toBeNull();
454
+ expect(uploads[0].Key).toBe('test/p/' + response.body.id + '/' + response.body.id + '.pdf');
455
+ expect(uploads[0].ContentType).toBe('application/pdf');
456
+ });
457
+
458
+ test('A part with an empty filename never ends up with an empty key', async () => {
459
+ for (const contentType of ['application/pdf', 'text/plain']) {
460
+ uploads = [];
461
+
462
+ const response = await testServer.test(uploadFileEndpoint, buildRequest('/v1/upload-file', [
463
+ filePart({ filename: '', contentType }),
464
+ ]));
465
+
466
+ expectSafeUploads();
467
+ expect(uploads[0].Key, contentType).toBe('test/p/' + response.body.id + '/' + response.body.id + '.' + (contentType === 'text/plain' ? 'txt' : 'pdf'));
468
+ }
469
+ });
470
+
471
+ test('It stores a part without a content type header safely, or not at all', async () => {
472
+ const request = buildRequest('/v1/upload-file', [
473
+ filePart({ filename: 'evil.html', contentType: null, content: htmlContent }),
474
+ ]);
475
+
476
+ await expectRejectedOrSafe(testServer.test(uploadFileEndpoint, request));
477
+ expect(uploads).toHaveLength(0);
478
+
479
+ const allowed = buildRequest('/v1/upload-file', [
480
+ filePart({ filename: 'invoice.pdf', contentType: null }),
481
+ ]);
482
+
483
+ await expectRejectedOrSafe(testServer.test(uploadFileEndpoint, allowed));
484
+ expectSafeUploads();
485
+ });
486
+
487
+ test('A quote or a second filename in the part headers never changes what we store', async () => {
488
+ const attacks = [
489
+ 'evil".pdf',
490
+ 'invoice.pdf"; filename="evil.html',
491
+ 'invoice.pdf; filename=evil.html',
492
+ 'evil.html"; filename="invoice.pdf',
493
+ ];
494
+
495
+ for (const filename of attacks) {
496
+ uploads = [];
497
+
498
+ await expectRejectedOrSafe(testServer.test(uploadFileEndpoint, buildRequest('/v1/upload-file', [
499
+ filePart({ filename, contentType: 'text/html', content: htmlContent }),
500
+ ])));
501
+
502
+ // The header parser may end up with any of the two filenames, but never with something that
503
+ // makes us store an executable file
504
+ for (const upload of uploads) {
505
+ expect(upload.ContentType, filename).not.toBe('text/html');
506
+ expect(upload.Key, filename).not.toMatch(/\.x?html?$/);
507
+ }
508
+ }
509
+ });
510
+
511
+ test('A filename with a line break never breaks the part headers open', async () => {
512
+ const request = buildRequest('/v1/upload-file', [
513
+ { headers: 'Content-Disposition: form-data; name="file"; filename="evil\r\nContent-Type: text/html\r\n.pdf"', body: htmlContent },
514
+ ]);
515
+
516
+ await expectRejectedOrSafe(testServer.test(uploadFileEndpoint, request));
517
+
518
+ for (const upload of uploads) {
519
+ expect(upload.ContentType).not.toBe('text/html');
520
+ }
521
+ });
522
+
523
+ test('A declared content length that does not match the body never bypasses validation', async () => {
524
+ const oversized = buildRequest('/v1/upload-file', [
525
+ filePart({ filename: 'evil.html', contentType: 'text/html', content: htmlContent }),
526
+ ], { contentLength: (100 * 1024 * 1024).toString() });
527
+
528
+ await expectRejectedOrSafe(testServer.test(uploadFileEndpoint, oversized));
529
+ expect(uploads).toHaveLength(0);
530
+
531
+ const undersized = buildRequest('/v1/upload-file', [
532
+ filePart({ filename: 'evil.svg', contentType: 'image/svg+xml', content: svgContent }),
533
+ ], { contentLength: '1' });
534
+
535
+ await expectRejectedOrSafe(testServer.test(uploadFileEndpoint, undersized));
536
+ expect(uploads).toHaveLength(0);
537
+ });
538
+
539
+ test('Extra fields never make us skip the validation of the file', async () => {
540
+ const withFields = buildRequest('/v1/upload-file', [
541
+ fieldPart('contentType', 'application/pdf'),
542
+ fieldPart('extension', 'pdf'),
543
+ filePart({ filename: 'evil.html', contentType: 'text/html', content: htmlContent }),
544
+ ]);
545
+
546
+ await expectRejectedOrSafe(testServer.test(uploadFileEndpoint, withFields));
547
+ expect(uploads).toHaveLength(0);
548
+ });
549
+
550
+ test('It refuses an image upload with a duplicated resolutions field', async () => {
551
+ const request = buildRequest('/v1/upload-image', [
552
+ fieldPart('resolutions', JSON.stringify([{ width: 50, height: 50, fit: 'inside' }])),
553
+ fieldPart('resolutions', JSON.stringify([{ width: 50, height: 50, fit: 'inside' }])),
554
+ filePart({ filename: 'evil.svg', contentType: 'image/svg+xml', content: svgContent }),
555
+ ]);
556
+
557
+ await expectRejectedOrSafe(testServer.test(uploadImageEndpoint, request));
558
+ expect(uploads).toHaveLength(0);
559
+ });
560
+
561
+ test('It refuses an image upload with more than one file', async () => {
562
+ const request = buildRequest('/v1/upload-image', [
563
+ fieldPart('resolutions', JSON.stringify([{ width: 50, height: 50, fit: 'inside' }])),
564
+ filePart({ filename: 'logo.png', contentType: 'image/png', content: pngContent }),
565
+ filePart({ filename: 'evil.svg', contentType: 'image/svg+xml', content: svgContent }),
566
+ ]);
567
+
568
+ await expectRejectedOrSafe(testServer.test(uploadImageEndpoint, request));
569
+ expect(uploads).toHaveLength(0);
570
+ });
571
+ });
572
+
573
+ describe('Private and public uploads', () => {
574
+ test('A user without permissions can only upload private files', async () => {
575
+ await expect(uploadFile({ filename: 'invoice.pdf', contentType: 'application/pdf' }, { token: memberToken })).rejects.toThrow();
576
+ expect(uploads).toHaveLength(0);
577
+
578
+ const response = await uploadFile({ filename: 'invoice.pdf', contentType: 'application/pdf' }, { token: memberToken, isPrivate: true });
579
+
580
+ expect(response.body.isPrivate).toBe(true);
581
+ expect(uploads[0].ACL).toBe('private');
582
+ expect(uploads[0].Key).toMatch(new RegExp('^test/users/' + memberUserId + '/'));
583
+ });
584
+
585
+ test('A private upload is never readable without a signature', async () => {
586
+ const response = await uploadFile({ filename: 'contract.pdf', contentType: 'application/pdf' }, { isPrivate: true });
587
+
588
+ expect(uploads[0].ACL).toBe('private');
589
+ expect(uploads[0].ACL).not.toBe('public-read');
590
+ expect(response.body.signature).toBeTruthy();
591
+ expect(await response.body.verify()).toBe(true);
592
+
593
+ // Changing anything about the file breaks the signature, so the path can't be swapped afterwards
594
+ response.body.path = 'test/users/' + memberUserId + '/stolen.pdf';
595
+ expect(await response.body.verify()).toBe(false);
596
+ });
597
+
598
+ test('A public upload is never signed, and its content type does not change its ACL', async () => {
599
+ for (const contentType of ['application/pdf', 'text/plain', 'image/png']) {
600
+ uploads = [];
601
+
602
+ const response = await uploadFile({ filename: 'file.' + (contentType === 'text/plain' ? 'txt' : contentType.split('/')[1]), contentType });
603
+
604
+ expect(response.body.isPrivate, contentType).toBe(false);
605
+ expect(response.body.signature, contentType).toBeNull();
606
+ expect(uploads[0].ACL, contentType).toBe('public-read');
607
+ }
608
+ });
609
+
610
+ test('A file we refuse is refused in private mode too', async () => {
611
+ for (const isPrivate of [false, true]) {
612
+ uploads = [];
613
+
614
+ await expect(uploadFile({ filename: 'evil.html', contentType: 'text/html', content: htmlContent }, { isPrivate })).rejects.toThrow(/unsupported file type/i);
615
+ expect(uploads).toHaveLength(0);
616
+ }
617
+ });
618
+ });
619
+
620
+ describe('Image uploads', () => {
621
+ /**
622
+ * Every object a browser can reach has to be a raster image generated by sharp
623
+ */
624
+ const expectNoPublicSvg = () => {
625
+ for (const upload of uploads) {
626
+ if (upload.ACL === 'public-read') {
627
+ expect(['image/png', 'image/jpeg'], 'Public image').toContain(upload.ContentType);
628
+ expect(upload.Key, 'Public image').not.toMatch(/\.svg$/);
629
+ } else {
630
+ expect(upload.ACL).toBe('private');
631
+ expect(upload.ContentDisposition).toBe('attachment');
632
+ }
633
+ }
634
+ };
635
+
636
+ test('An svg that claims to be a png is still rasterized, and never served as an svg', async () => {
637
+ await uploadImage({ filename: 'logo.png', contentType: 'image/png', content: svgContent });
638
+
639
+ expectNoPublicSvg();
640
+ expect(uploads[0]).toMatchObject({ ContentType: 'image/png', ACL: 'public-read' });
641
+
642
+ // The uploaded bytes are only kept as a private source object
643
+ expect(uploads[1]).toMatchObject({ ContentDisposition: 'attachment', ACL: 'private' });
644
+ });
645
+
646
+ test('A png that claims to be an svg is stored as the private source it is', async () => {
647
+ await uploadImage({ filename: 'logo.svg', contentType: 'image/svg+xml', content: pngContent });
648
+
649
+ expectNoPublicSvg();
650
+ expect(uploads[0]).toMatchObject({ ContentType: 'image/png', ACL: 'public-read' });
651
+ expect(uploads[1]).toMatchObject({ ContentType: 'image/svg+xml', ContentDisposition: 'attachment', ACL: 'private' });
652
+ });
653
+
654
+ test('An upload that is not an image at all never reaches the file server publicly', async () => {
655
+ await expectRejectedOrSafe(uploadImage({ filename: 'evil.png', contentType: 'image/png', content: htmlContent }));
656
+ expectNoPublicSvg();
657
+
658
+ for (const upload of uploads) {
659
+ expect(upload.ACL).not.toBe('public-read');
660
+ }
661
+ });
662
+
663
+ test('Without resolutions nothing is rasterized, so nothing may become public', async () => {
664
+ // This is the only path where the bytes we store are the bytes that were uploaded
665
+ for (const attack of [
666
+ { filename: 'evil.svg', contentType: 'image/svg+xml', content: svgContent },
667
+ { filename: 'evil.png', contentType: 'image/png', content: htmlContent },
668
+ { filename: 'evil.png', contentType: 'image/png', content: svgContent },
669
+ ]) {
670
+ uploads = [];
671
+
672
+ await expectRejectedOrSafe(uploadImage({ ...attack, resolutions: [] }));
673
+
674
+ expect(uploads.length, attack.filename).toBeLessThanOrEqual(1);
675
+ expectNoPublicSvg();
676
+ }
677
+ });
678
+
679
+ test('A resolution we cannot generate never results in a public object', async () => {
680
+ for (const resolution of [
681
+ { width: -1, height: -1, fit: 'inside' },
682
+ { width: 0, height: 0, fit: 'inside' },
683
+ { width: 100000, height: 100000, fit: 'inside' },
684
+ ]) {
685
+ uploads = [];
686
+
687
+ await expectRejectedOrSafe(uploadImage({ filename: 'logo.png', contentType: 'image/png', content: pngContent, resolutions: [resolution] }));
688
+ expectNoPublicSvg();
689
+
690
+ // A resolution we can't generate never leaves a half finished image behind
691
+ expect(uploads.length, JSON.stringify(resolution)).toBeLessThanOrEqual(2);
692
+ }
693
+ });
694
+
695
+ test('An image endpoint upload never gets an inline disposition it did not earn', async () => {
696
+ await uploadImage({ filename: 'logo.svg', contentType: 'image/svg+xml', content: svgContent });
697
+
698
+ expect(uploads).toHaveLength(2);
699
+
700
+ for (const upload of uploads) {
701
+ expect(upload.ContentDisposition).not.toBe('inline');
702
+ }
703
+ });
704
+ });
705
+
706
+ describe('Signed urls', () => {
707
+ let originalClient: S3Client;
708
+
709
+ beforeEach(() => {
710
+ originalClient = FileSignService.s3;
711
+ FileSignService.s3 = new S3Client({
712
+ forcePathStyle: false,
713
+ endpoint: 'https://test.digitaloceanspaces.com',
714
+ credentials: {
715
+ accessKeyId: 'test-key',
716
+ secretAccessKey: 'test-secret',
717
+ },
718
+ region: 'eu-west-1',
719
+ });
720
+ });
721
+
722
+ afterEach(() => {
723
+ FileSignService.s3 = originalClient;
724
+ });
725
+
726
+ const getDisposition = async (data: { path: string; contentType?: string | null }) => {
727
+ const file = new File({
728
+ id: '1c9ab9e6-1234-4c5e-9f1a-000000000000',
729
+ server: 'https://test-bucket.test.digitaloceanspaces.com',
730
+ path: data.path,
731
+ size: 100,
732
+ isPrivate: true,
733
+ contentType: data.contentType ?? null,
734
+ });
735
+
736
+ const signed = await FileSignService.withSignedUrl(file);
737
+ expect(signed?.signedUrl, data.path).toBeDefined();
738
+
739
+ return new URL(signed!.signedUrl!).searchParams.get('response-content-disposition');
740
+ };
741
+
742
+ test('A dangerous path is downloaded, whatever the content type of the struct claims', async () => {
743
+ const attacks = [
744
+ { path: 'test/users/1/a/evil.svg' },
745
+ { path: 'test/users/1/a/evil.svg', contentType: 'application/pdf' },
746
+ { path: 'test/users/1/a/evil.svg', contentType: 'image/png' },
747
+ { path: 'test/users/1/a/evil.html', contentType: 'application/pdf' },
748
+ { path: 'test/users/1/a/evil.xhtml', contentType: 'image/jpeg' },
749
+ { path: 'test/users/1/a/evil.js', contentType: 'application/pdf' },
750
+ { path: 'test/users/1/a/evil.php', contentType: 'application/pdf' },
751
+ { path: 'test/users/1/a/evil', contentType: 'application/pdf' },
752
+ { path: 'test/users/1/a/evil.', contentType: 'application/pdf' },
753
+ { path: 'test/users/1/a.pdf/evil.svg' },
754
+ { path: 'test/invoices/1c9ab9e6.xml', contentType: 'application/xml' },
755
+
756
+ // Casing and whitespace around the extension may not hide it
757
+ { path: 'test/users/1/a/EVIL.SVG', contentType: 'application/pdf' },
758
+ { path: 'test/users/1/a/evil.SvG' },
759
+ { path: 'test/users/1/a/evil.svg ', contentType: 'application/pdf' },
760
+
761
+ // A path we could never generate is refused instead of guessed
762
+ { path: '', contentType: 'application/pdf' },
763
+ { path: 'test/users/1/a/', contentType: 'application/pdf' },
764
+ ];
765
+
766
+ for (const attack of attacks) {
767
+ expect(await getDisposition(attack), JSON.stringify(attack)).toBe('attachment');
768
+ }
769
+ });
770
+
771
+ test('A dangerous content type is downloaded, whatever the path claims', async () => {
772
+ for (const contentType of executableContentTypes) {
773
+ expect(await getDisposition({ path: 'test/users/1/a/invoice.pdf', contentType }), contentType).toBe('attachment');
774
+ expect(await getDisposition({ path: 'test/users/1/a/photo.png', contentType }), contentType).toBe('attachment');
775
+ }
776
+ });
777
+
778
+ test('Only a real pdf or raster image is rendered', async () => {
779
+ expect(await getDisposition({ path: 'test/users/1/a/invoice.pdf', contentType: 'application/pdf' })).toBe('inline');
780
+ expect(await getDisposition({ path: 'test/users/1/a/photo.jpg', contentType: 'image/jpeg' })).toBe('inline');
781
+
782
+ // Files we generate ourselves don't carry a content type
783
+ expect(await getDisposition({ path: 'test/p/1/a.png' })).toBe('inline');
784
+ expect(await getDisposition({ path: 'test/p/1/A.PDF' })).toBe('inline');
785
+
786
+ // An office document is never rendered, even though we accept the upload
787
+ expect(await getDisposition({ path: 'test/users/1/a/report.docx', contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' })).toBe('attachment');
788
+ expect(await getDisposition({ path: 'test/users/1/a/notes.txt', contentType: 'text/plain' })).toBe('attachment');
789
+ });
790
+
791
+ test('The disposition is signed, so a client cannot change or strip it', async () => {
792
+ const file = new File({
793
+ id: '1c9ab9e6-1234-4c5e-9f1a-000000000000',
794
+ server: 'https://test-bucket.test.digitaloceanspaces.com',
795
+ path: 'test/users/1/a/report.docx',
796
+ size: 100,
797
+ isPrivate: true,
798
+ contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
799
+ });
800
+
801
+ // Sign the same file twice, once with the disposition we would refuse, until both urls carry the
802
+ // same timestamp: only the disposition differs between them
803
+ let attachment: URL;
804
+ let inline: URL;
805
+ let attempts = 0;
806
+
807
+ do {
808
+ attachment = new URL((await FileSignService.withSignedUrl(file))!.signedUrl!);
809
+
810
+ const spy = vi.spyOn(FileSignService, 'getContentDisposition').mockReturnValue('inline');
811
+ inline = new URL((await FileSignService.withSignedUrl(file))!.signedUrl!);
812
+ spy.mockRestore();
813
+
814
+ attempts++;
815
+ } while (attachment.searchParams.get('X-Amz-Date') !== inline.searchParams.get('X-Amz-Date') && attempts < 5);
816
+
817
+ expect(attachment.searchParams.get('response-content-disposition')).toBe('attachment');
818
+ expect(inline.searchParams.get('response-content-disposition')).toBe('inline');
819
+
820
+ // The disposition is part of what is signed, so a client can't turn a download into a render by
821
+ // editing the url: the signature no longer matches
822
+ expect(attachment.searchParams.get('X-Amz-Signature')).toBeTruthy();
823
+ expect(inline.searchParams.get('X-Amz-Signature')).not.toBe(attachment.searchParams.get('X-Amz-Signature'));
824
+ });
825
+
826
+ test('The keys we generate can never contain something the extension check would misread', async () => {
827
+ // getContentDisposition() reads the extension of the path, so our keys may never contain a
828
+ // query string, a fragment or a second extension
829
+ const response = await uploadFile({ filename: 'evil.svg?x=.pdf#.pdf', contentType: 'application/pdf' });
830
+
831
+ expect(response.body.path).not.toContain('?');
832
+ expect(response.body.path).not.toContain('#');
833
+ expect(response.body.path).toMatch(/^test\/p\/[a-z0-9-]+\/[a-z0-9-]+\.pdf$/);
834
+ expect(await getDisposition({ path: response.body.path, contentType: response.body.contentType })).toBe('inline');
835
+ });
836
+ });
837
+ });