@stamhoofd/backend 2.137.4 → 2.137.5

package/src/endpoints/global/files/UploadFile.test.ts

@@ -0,0 +1,206 @@
+import type { PutObjectCommand } from '@aws-sdk/client-s3';
+import { Request } from '@simonbackx/simple-endpoints';
+import { Image, OrganizationFactory, Token, UserFactory } from '@stamhoofd/models';
+import { PermissionLevel, Permissions } from '@stamhoofd/structures';
+import { TestUtils } from '@stamhoofd/test-utils';
+import { promises as fs } from 'fs';
+import type { IncomingMessage } from 'node:http';
+import { Readable } from 'node:stream';
+
+import { testServer } from '../../../../tests/helpers/TestServer.js';
+import { UploadFile } from './UploadFile.js';
+
+describe('Endpoint.UploadFile', () => {
+    const endpoint = new UploadFile();
+
+    /**
+     * All the objects that were sent to the (mocked) file server
+     */
+    let uploads: PutObjectCommand['input'][] = [];
+
+    /**
+     * All the paths the endpoint tried to clean up
+     */
+    let temporaryFiles: string[] = [];
+
+    beforeEach(() => {
+        TestUtils.setEnvironment('SPACES_BUCKET', 'test-bucket');
+        TestUtils.setEnvironment('SPACES_ENDPOINT', 'test.digitaloceanspaces.com');
+        TestUtils.setEnvironment('SPACES_KEY', 'test-key');
+        TestUtils.setEnvironment('SPACES_SECRET', 'test-secret');
+
+        uploads = [];
+        Image.s3Client = {
+            send: (command: PutObjectCommand) => {
+                uploads.push(command.input);
+                return Promise.resolve({});
+            },
+        } as any;
+
+        temporaryFiles = [];
+        const rm = fs.rm;
+        vi.spyOn(fs, 'rm').mockImplementation(async (path, options) => {
+            temporaryFiles.push(path as string);
+            return await rm(path, options);
+        });
+    });
+
+    afterEach(() => {
+        Image.s3Client = null;
+        vi.restoreAllMocks();
+    });
+
+    /**
+     * Builds a multipart/form-data request with a single uploaded file, like a browser would send it
+     */
+    const buildUploadRequest = async (data: { filename: string; contentType: string; content?: string; isPrivate?: boolean }) => {
+        const organization = await new OrganizationFactory({}).create();
+        const user = await new UserFactory({
+            organization,
+            permissions: Permissions.create({ level: PermissionLevel.Full }),
+        }).create();
+        const token = await Token.createToken(user);
+
+        const boundary = '--------------------------StamhoofdTest';
+        const body = Buffer.concat([
+            Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${data.filename}"\r\nContent-Type: ${data.contentType}\r\n\r\n`),
+            Buffer.from(data.content ?? 'Hello world'),
+            Buffer.from(`\r\n--${boundary}--\r\n`),
+        ]);
+
+        const stream = Readable.from([body]) as unknown as IncomingMessage;
+        stream.headers = {
+            'content-type': `multipart/form-data; boundary=${boundary}`,
+            'content-length': body.length.toString(),
+        };
+
+        const r = Request.buildJson('POST', '/v1/upload-file', organization.getApiHost());
+        r.headers.authorization = 'Bearer ' + token.accessToken;
+        r.request = stream;
+        r.query = data.isPrivate ? { private: true } : {};
+
+        return r;
+    };
+
+    test('It stores a pdf with the reported content type, and lets the browser render it', async () => {
+        const r = await buildUploadRequest({ filename: 'Mijn document.pdf', contentType: 'application/pdf' });
+        const response = await testServer.test(endpoint, r);
+
+        expect(response.body.contentType).toBe('application/pdf');
+        expect(response.body.name).toBe('Mijn document.pdf');
+        expect(response.body.path).toMatch(/\/mijn-document\.pdf$/);
+
+        expect(uploads).toHaveLength(1);
+        expect(uploads[0]).toMatchObject({
+            Key: response.body.path,
+            ContentType: 'application/pdf',
+            ContentDisposition: 'inline',
+            ACL: 'public-read',
+        });
+    });
+
+    test('It uses the extension when the uploader does not report a usable content type', async () => {
+        const r = await buildUploadRequest({ filename: 'ledenlijst.xlsx', contentType: 'application/octet-stream' });
+        const response = await testServer.test(endpoint, r);
+
+        expect(response.body.contentType).toBe('application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
+        expect(uploads[0]).toMatchObject({
+            ContentType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+            // A browser can execute an office document, so it is never rendered
+            ContentDisposition: 'attachment',
+        });
+    });
+
+    test('It ignores a content type we do not support, and uses the extension', async () => {
+        const r = await buildUploadRequest({ filename: 'document.pdf', contentType: 'text/html', content: '<script>alert(1)</script>' });
+        const response = await testServer.test(endpoint, r);
+
+        // Whatever is inside the file, a browser only ever gets it as a pdf
+        expect(response.body.contentType).toBe('application/pdf');
+        expect(uploads[0]).toMatchObject({
+            ContentType: 'application/pdf',
+        });
+    });
+
+    test('It renames a file that was uploaded with the extension of another content type', async () => {
+        const r = await buildUploadRequest({ filename: 'foto.png', contentType: 'image/jpeg' });
+        const response = await testServer.test(endpoint, r);
+
+        expect(response.body.contentType).toBe('image/jpeg');
+        expect(response.body.name).toBe('foto.jpg');
+        expect(response.body.path).toMatch(/\/foto\.jpg$/);
+        expect(uploads[0]).toMatchObject({
+            ContentType: 'image/jpeg',
+        });
+    });
+
+    test('It stores the file with the extension of the resolved content type', async () => {
+        const r = await buildUploadRequest({ filename: 'Foto.JPEG', contentType: 'image/jpg' });
+        const response = await testServer.test(endpoint, r);
+
+        expect(response.body.contentType).toBe('image/jpeg');
+        expect(response.body.name).toBe('Foto.jpeg');
+        expect(response.body.path).toMatch(/\/foto\.jpeg$/);
+        expect(uploads[0]).toMatchObject({
+            ContentType: 'image/jpeg',
+            ContentDisposition: 'inline',
+        });
+    });
+
+    test('It stores a file without a usable name', async () => {
+        const r = await buildUploadRequest({ filename: '.pdf', contentType: 'application/pdf' });
+        const response = await testServer.test(endpoint, r);
+
+        expect(response.body.name).toBeNull();
+
+        // Falls back to the id of the file, instead of storing it without a name
+        expect(response.body.path).toMatch(new RegExp('/' + response.body.id + '/' + response.body.id + '\\.pdf$'));
+    });
+
+    test('It refuses a file with a content type browsers could execute', async () => {
+        const r = await buildUploadRequest({ filename: 'evil.html', contentType: 'text/html', content: '<script>alert(1)</script>' });
+
+        await expect(testServer.test(endpoint, r)).rejects.toThrow(/unsupported file type/i);
+        expect(uploads).toHaveLength(0);
+    });
+
+    test('It refuses an svg image', async () => {
+        const r = await buildUploadRequest({ filename: 'evil.svg', contentType: 'image/svg+xml', content: '<svg xmlns="http://www.w3.org/2000/svg"><script>alert(1)</script></svg>' });
+
+        await expect(testServer.test(endpoint, r)).rejects.toThrow(/unsupported file type/i);
+        expect(uploads).toHaveLength(0);
+    });
+
+    test('It refuses a file we cannot determine a safe content type for', async () => {
+        const r = await buildUploadRequest({ filename: 'backup', contentType: 'application/octet-stream' });
+
+        await expect(testServer.test(endpoint, r)).rejects.toThrow(/unsupported file type/i);
+        expect(uploads).toHaveLength(0);
+    });
+
+    test('It stores private files as an attachment too', async () => {
+        const r = await buildUploadRequest({ filename: 'contract.pdf', contentType: 'application/pdf', isPrivate: true });
+        const response = await testServer.test(endpoint, r);
+
+        expect(response.body.isPrivate).toBe(true);
+        expect(response.body.signature).not.toBeNull();
+        expect(uploads[0]).toMatchObject({
+            ContentType: 'application/pdf',
+            ACL: 'private',
+        });
+    });
+
+    test('It cleans up the temporary file of the upload', async () => {
+        const rejected = await buildUploadRequest({ filename: 'evil.html', contentType: 'text/html' });
+        await expect(testServer.test(endpoint, rejected)).rejects.toThrow(/unsupported file type/i);
+
+        const accepted = await buildUploadRequest({ filename: 'document.pdf', contentType: 'application/pdf' });
+        await testServer.test(endpoint, accepted);
+
+        expect(temporaryFiles).toHaveLength(2);
+
+        for (const path of temporaryFiles) {
+            await expect(fs.stat(path)).rejects.toThrow(/ENOENT/);
+        }
+    });
+});

package/src/endpoints/global/files/UploadFile.ts

@@ -108,6 +108,15 @@ export class UploadFile extends Endpoint<Params, Query, Body, ResponseBody> {
             });
         });
 
+        try {
+            return await this.upload(request, file, user);
+        } finally {
+            // Formidable wrote the upload to a temporary file
+            await fs.rm(file.filepath, { force: true }).catch(() => { /* we can't do anything about this */ });
+        }
+    }
+
+    private async upload(request: DecodedRequest<Params, Query, Body>, file: FormidableFile, user: { id: string }) {
         if (!STAMHOOFD.SPACES_BUCKET || !STAMHOOFD.SPACES_ENDPOINT || !STAMHOOFD.SPACES_KEY || !STAMHOOFD.SPACES_SECRET) {
             throw new SimpleError({
                 code: 'not_available',
@@ -116,6 +125,20 @@ export class UploadFile extends Endpoint<Params, Query, Body, ResponseBody> {
             });
         }
 
+        // Never trust the content type of the uploader: it is served back to browsers when the file is
+        // downloaded, so we only allow content types we know are safe to serve.
+        const uploadType = File.resolveUploadType({ contentType: file.mimetype, filename: file.originalFilename });
+
+        if (!uploadType) {
+            throw new SimpleError({
+                code: 'invalid_file_type',
+                message: 'Unsupported file type ' + (file.mimetype ?? 'unknown') + ' for file ' + (file.originalFilename ?? 'unknown'),
+                human: $t('Dit type bestand kan je niet uploaden.'),
+                field: 'file',
+                statusCode: 400,
+            });
+        }
+
         const fileContent = await fs.readFile(file.filepath);
 
         let prefix = (STAMHOOFD.SPACES_PREFIX ?? '');
@@ -140,19 +163,19 @@ export class UploadFile extends Endpoint<Params, Query, Body, ResponseBody> {
 
         // Also include the source, in private mode
         const fileId = uuidv4();
-        const uploadExt = File.contentTypeToExtension(file.mimetype ?? '') ?? '';
 
-        const filenameWithoutExt = file.originalFilename ? File.removeExtension(file.originalFilename) : fileId;
-        const key = prefix + fileId + '/' + (Formatter.slug(filenameWithoutExt) + (uploadExt ? ('.' + uploadExt) : ''));
+        const filenameWithoutExt = file.originalFilename ? File.removeExtension(file.originalFilename) : '';
+        const key = prefix + fileId + '/' + ((Formatter.slug(filenameWithoutExt) || fileId) + '.' + uploadType.extension);
 
         const fileStruct = new File({
             id: fileId,
             server: 'https://' + STAMHOOFD.SPACES_BUCKET + '.' + STAMHOOFD.SPACES_ENDPOINT,
             path: key,
+            // Always keep the extension in sync with the content type we store the file with
+            name: filenameWithoutExt ? filenameWithoutExt + '.' + uploadType.extension : null,
             size: fileContent.length,
-            name: file.originalFilename,
             isPrivate: request.query.isPrivate,
-            contentType: file.mimetype ?? null,
+            contentType: uploadType.contentType,
         });
 
         // Generate an upload signature for this file if it is private
@@ -171,7 +194,9 @@ export class UploadFile extends Endpoint<Params, Query, Body, ResponseBody> {
             Bucket: STAMHOOFD.SPACES_BUCKET,
             Key: key,
             Body: fileContent,
-            ContentType: file.mimetype ?? 'application/octet-stream',
+            ContentType: uploadType.contentType,
+            // Only let the browser render file types it can't execute
+            ContentDisposition: uploadType.canRenderInline ? 'inline' : 'attachment',
             ACL: request.query.isPrivate ? 'private' : 'public-read',
         });
         await Image.getS3Client().send(cmd);

package/src/endpoints/global/files/UploadImage.test.ts

@@ -0,0 +1,177 @@
+import type { PutObjectCommand } from '@aws-sdk/client-s3';
+import { Request } from '@simonbackx/simple-endpoints';
+import { Image, OrganizationFactory, Token, UserFactory } from '@stamhoofd/models';
+import { PermissionLevel, Permissions } from '@stamhoofd/structures';
+import { TestUtils } from '@stamhoofd/test-utils';
+import type { IncomingMessage } from 'node:http';
+import { Readable } from 'node:stream';
+
+import { testServer } from '../../../../tests/helpers/TestServer.js';
+import { UploadImage } from './UploadImage.js';
+
+/**
+ * A real (1x1 pixel) png, so sharp can generate resolutions from it
+ */
+const pngContent = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==', 'base64');
+
+const svgContent = Buffer.from('<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10"><script>alert(1)</script><rect width="10" height="10" fill="red"/></svg>');
+
+describe('Endpoint.UploadImage', () => {
+    const endpoint = new UploadImage();
+
+    /**
+     * All the objects that were sent to the (mocked) file server
+     */
+    let uploads: PutObjectCommand['input'][] = [];
+
+    beforeEach(() => {
+        TestUtils.setEnvironment('SPACES_BUCKET', 'test-bucket');
+        TestUtils.setEnvironment('SPACES_ENDPOINT', 'test.digitaloceanspaces.com');
+        TestUtils.setEnvironment('SPACES_KEY', 'test-key');
+        TestUtils.setEnvironment('SPACES_SECRET', 'test-secret');
+
+        uploads = [];
+        Image.s3Client = {
+            send: (command: PutObjectCommand) => {
+                uploads.push(command.input);
+                return Promise.resolve({});
+            },
+        } as any;
+    });
+
+    afterEach(() => {
+        Image.s3Client = null;
+    });
+
+    /**
+     * Builds a multipart/form-data request with a single uploaded image, like a browser would send it
+     */
+    const buildUploadRequest = async (data: { filename: string; contentType: string; content: Buffer; resolutions?: object[] }) => {
+        const organization = await new OrganizationFactory({}).create();
+        const user = await new UserFactory({
+            organization,
+            permissions: Permissions.create({ level: PermissionLevel.Full }),
+        }).create();
+        const token = await Token.createToken(user);
+
+        const boundary = '--------------------------StamhoofdTest';
+        const body = Buffer.concat([
+            Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="resolutions"\r\n\r\n${JSON.stringify(data.resolutions ?? [{ width: 50, height: 50, fit: 'inside' }])}\r\n`),
+            Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${data.filename}"\r\nContent-Type: ${data.contentType}\r\n\r\n`),
+            data.content,
+            Buffer.from(`\r\n--${boundary}--\r\n`),
+        ]);
+
+        const stream = Readable.from([body]) as unknown as IncomingMessage;
+        stream.headers = {
+            'content-type': `multipart/form-data; boundary=${boundary}`,
+            'content-length': body.length.toString(),
+        };
+
+        const r = Request.buildJson('POST', '/v1/upload-image', organization.getApiHost());
+        r.headers.authorization = 'Bearer ' + token.accessToken;
+        r.request = stream;
+        r.query = {};
+
+        return r;
+    };
+
+    test('It stores a png, and never stores the original with a content type of the uploader', async () => {
+        const r = await buildUploadRequest({ filename: 'logo.png', contentType: 'image/png', content: pngContent });
+        const response = await testServer.test(endpoint, r);
+
+        expect(response.body.resolutions).toHaveLength(1);
+
+        // The generated image is public and rendered inline in the browser
+        expect(uploads[0]).toMatchObject({
+            ContentType: 'image/png',
+            ACL: 'public-read',
+        });
+
+        // Forcing a download here would break every image in our emails and pdfs
+        expect(uploads[0].ContentDisposition).toBeUndefined();
+
+        // The original is never rendered
+        expect(uploads[1]).toMatchObject({
+            ContentType: 'image/png',
+            ContentDisposition: 'attachment',
+            ACL: 'private',
+        });
+    });
+
+    /**
+     * An svg can contain scripts, so a browser may never render one: it is always stored privately and as an
+     * attachment, whatever else the upload results in.
+     */
+    const expectNoRenderableSvg = () => {
+        const svgUploads = uploads.filter(upload => upload.ContentType === 'image/svg+xml' || upload.Key?.endsWith('.svg'));
+
+        for (const upload of svgUploads) {
+            expect(upload.ContentDisposition).toBe('attachment');
+            expect(upload.ACL).toBe('private');
+        }
+
+        return svgUploads;
+    };
+
+    test('It rasterizes an svg, so the uploaded svg itself is never served', async () => {
+        const r = await buildUploadRequest({ filename: 'logo.svg', contentType: 'image/svg+xml', content: svgContent });
+        const response = await testServer.test(endpoint, r);
+
+        expect(response.body.resolutions).toHaveLength(1);
+        expect(uploads[0]).toMatchObject({
+            ContentType: 'image/png',
+            ACL: 'public-read',
+        });
+
+        expect(uploads[1]).toMatchObject({
+            ContentType: 'image/svg+xml',
+            ContentDisposition: 'attachment',
+            ACL: 'private',
+        });
+
+        // Every public file we generated from the svg is a rasterized image
+        expect(expectNoRenderableSvg()).toHaveLength(1);
+    });
+
+    test('It never publishes the svg itself, even when no resolutions are requested', async () => {
+        // Without resolutions there is nothing to rasterize, so the uploaded svg is the only stored file - and
+        // Image.getPublicPath() falls back to it
+        const r = await buildUploadRequest({ filename: 'logo.svg', contentType: 'image/svg+xml', content: svgContent, resolutions: [] });
+        const response = await testServer.test(endpoint, r);
+
+        expect(response.body.resolutions).toHaveLength(0);
+        expect(uploads).toHaveLength(1);
+        expect(expectNoRenderableSvg()).toHaveLength(1);
+    });
+
+    test('It refuses a file that is not an image, even without resolutions', async () => {
+        const r = await buildUploadRequest({
+            filename: 'evil.html',
+            contentType: 'text/html',
+            content: Buffer.from('<script>alert(1)</script>'),
+            resolutions: [],
+        });
+
+        await expect(testServer.test(endpoint, r)).rejects.toThrow(/unsupported image type/i);
+        expect(uploads).toHaveLength(0);
+    });
+
+    test('It refuses an image with a content type we cannot generate resolutions from', async () => {
+        const r = await buildUploadRequest({ filename: 'photo.tiff', contentType: 'image/tiff', content: pngContent });
+
+        await expect(testServer.test(endpoint, r)).rejects.toThrow(/unsupported image type/i);
+        expect(uploads).toHaveLength(0);
+    });
+
+    test('It uses the extension when the uploader does not report a usable content type', async () => {
+        const r = await buildUploadRequest({ filename: 'logo.png', contentType: 'application/octet-stream', content: pngContent });
+        const response = await testServer.test(endpoint, r);
+
+        expect(response.body.resolutions).toHaveLength(1);
+        expect(uploads[1]).toMatchObject({
+            ContentType: 'image/png',
+            ContentDisposition: 'attachment',
+        });
+    });
+});

package/src/endpoints/global/files/UploadImage.ts

@@ -4,7 +4,7 @@ import type { DecodedRequest, Request } from '@simonbackx/simple-endpoints';
 import { Endpoint, Response } from '@simonbackx/simple-endpoints';
 import { SimpleError } from '@simonbackx/simple-errors';
 import { Image, RateLimiter } from '@stamhoofd/models';
-import { Image as ImageStruct, ResolutionRequest } from '@stamhoofd/structures';
+import { Image as ImageStruct, ResolutionRequest, supportedImageTypes } from '@stamhoofd/structures';
 import formidable from 'formidable';
 import { promises as fs } from 'fs';
 
@@ -131,8 +131,27 @@ export class UploadImage extends Endpoint<Params, Query, Body, ResponseBody> {
             });
         });
 
-        const fileContent = await fs.readFile(file.filepath);
-        const image = await Image.create(fileContent, file.mimetype ?? undefined, resolutions, request.query.isPrivate, user);
-        return new Response(ImageStruct.create(image));
+        try {
+            // Never trust the content type of the uploader: it decides how we store the image, so only accept
+            // the image types we know we can generate our resolutions from.
+            const imageType = supportedImageTypes.resolveUpload({ contentType: file.mimetype, filename: file.originalFilename });
+
+            if (!imageType) {
+                throw new SimpleError({
+                    code: 'invalid_file_type',
+                    message: 'Unsupported image type ' + (file.mimetype ?? 'unknown') + ' for file ' + (file.originalFilename ?? 'unknown'),
+                    human: $t('Dit soort afbeelding kan je niet opladen.'),
+                    field: 'file',
+                    statusCode: 400,
+                });
+            }
+
+            const fileContent = await fs.readFile(file.filepath);
+            const image = await Image.create(fileContent, imageType.contentType, resolutions, request.query.isPrivate, user);
+            return new Response(ImageStruct.create(image));
+        } finally {
+            // Formidable wrote the upload to a temporary file
+            await fs.rm(file.filepath, { force: true }).catch(() => { /* we can't do anything about this */ });
+        }
     }
 }