@travetto/web-upload 6.0.0-rc.2

package/README.md

@@ -0,0 +1,58 @@
+<!-- This file was generated by @travetto/doc and should not be modified directly -->
+<!-- Please modify https://github.com/travetto/travetto/tree/main/module/web-upload/DOC.tsx and execute "npx trv doc" to rebuild -->
+# Web Upload Support
+
+## Provides integration between the travetto asset and web module.
+
+**Install: @travetto/web-upload**
+```bash
+npm install @travetto/web-upload
+
+# or
+
+yarn add @travetto/web-upload
+```
+
+This module provides a clean and direct mechanism for processing uploads, built upon [@fastify/busboy](https://github.com/fastify/busboy). The module also provides some best practices with respect to temporary file management.
+
+Once the files are uploaded, they are exposed via [Endpoint](https://github.com/travetto/travetto/tree/main/module/web/src/decorator/endpoint.ts#L14) parameters using the [@Upload](https://github.com/travetto/travetto/tree/main/module/web-upload/src/decorator.ts#L20) decorator.  This decorator requires the related field type to be a standard [Buffer](https://nodejs.org/api/buffer.html#class-file) object, or a [FileMap](https://github.com/travetto/travetto/tree/main/module/web-upload/src/types.ts#L4).
+
+A simple example:
+
+**Code: Web controller with upload support**
+```typescript
+import { Controller, Post, Get } from '@travetto/web';
+import { FileMap, Upload } from '@travetto/web-upload';
+
+@Controller('/simple')
+export class Simple {
+
+  @Get('/age')
+  getAge() {
+    return { age: 50 };
+  }
+
+  @Post('/age')
+  getPage() {
+    return { age: 20 };
+  }
+
+  /**
+   * @param file A file to upload
+   */
+  @Post('/file')
+  loadFile(@Upload() upload: File) {
+    return upload;
+  }
+
+  /**
+   * @param uploads A map of files that were uploaded
+   */
+  @Post('/files')
+  async loadFiles(@Upload() uploads: FileMap) {
+    for (const [, upload] of Object.entries(uploads)) {
+      return upload;
+    }
+  }
+}
+```

package/__index__.ts

@@ -0,0 +1,5 @@
+import type { } from './src/trv.d.ts';
+export * from './src/decorator.ts';
+export * from './src/config.ts';
+export * from './src/util.ts';
+export * from './src/types.ts';

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@travetto/web-upload",
+  "version": "6.0.0-rc.2",
+  "description": "Provides integration between the travetto asset and web module.",
+  "keywords": [
+    "web",
+    "travetto",
+    "typescript",
+    "upload"
+  ],
+  "homepage": "https://travetto.io",
+  "license": "MIT",
+  "author": {
+    "email": "travetto.framework@gmail.com",
+    "name": "Travetto Framework"
+  },
+  "files": [
+    "__index__.ts",
+    "src",
+    "support"
+  ],
+  "main": "__index__.ts",
+  "repository": {
+    "url": "git+https://github.com/travetto/travetto.git",
+    "directory": "module/web-upload"
+  },
+  "dependencies": {
+    "@fastify/busboy": "^3.1.1",
+    "@travetto/config": "^6.0.0-rc.2",
+    "@travetto/web": "^6.0.0-rc.2",
+    "file-type": "^20.4.1",
+    "mime": "^4.0.7"
+  },
+  "peerDependencies": {
+    "@travetto/test": "^6.0.0-rc.2"
+  },
+  "peerDependenciesMeta": {
+    "@travetto/test": {
+      "optional": true
+    }
+  },
+  "travetto": {
+    "displayName": "Web Upload Support"
+  },
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/config.ts

@@ -0,0 +1,29 @@
+import { Config } from '@travetto/config';
+import { Ignore } from '@travetto/schema';
+
+/**
+ * Config for uploading within @travetto/web
+ */
+@Config('web.upload')
+export class WebUploadConfig {
+  applies = false;
+
+  /**
+   * Max file size in bytes
+   */
+  maxSize = 10 * 1024 * 1024;
+  /**
+   * List of types to allow/exclude
+   */
+  types: string[] = [];
+  /**
+   * Cleanup temporary files after request finishes
+   */
+  cleanupFiles: boolean = true;
+
+  @Ignore()
+  uploads?: Record<string, Partial<WebUploadConfig>>;
+
+  @Ignore()
+  matcher: (contentType: string) => boolean;
+}

package/src/decorator.ts

@@ -0,0 +1,68 @@
+import { AppError, toConcrete, ClassInstance } from '@travetto/runtime';
+import { ControllerRegistry, EndpointParamConfig, Param } from '@travetto/web';
+import { SchemaRegistry } from '@travetto/schema';
+
+import { WebUploadInterceptor } from './interceptor.ts';
+import { WebUploadConfig } from './config.ts';
+import { FileMap } from './types.ts';
+import { WebUploadUtil } from './util.ts';
+
+type UploadConfig = Partial<Pick<WebUploadConfig, 'types' | 'maxSize' | 'cleanupFiles'>>;
+
+const FileMapContract = toConcrete<FileMap>();
+
+/**
+ * Allows for supporting uploads
+ *
+ * @augments `@travetto/web-upload:Upload`
+ * @augments `@travetto/web:Param`
+ */
+export function Upload(
+  param: string | Partial<EndpointParamConfig> & UploadConfig = {},
+): (inst: ClassInstance, prop: string, idx: number) => void {
+
+  if (typeof param === 'string') {
+    param = { name: param };
+  }
+
+  const finalConf = { ...param };
+
+  return (inst: ClassInstance, prop: string, idx: number): void => {
+    // Register field
+    ControllerRegistry.registerEndpointInterceptorConfig(
+      inst.constructor, inst[prop], WebUploadInterceptor,
+      {
+        applies: true,
+        maxSize: finalConf.maxSize,
+        types: finalConf.types,
+        cleanupFiles: finalConf.cleanupFiles,
+        uploads: {
+          [finalConf.name ?? prop]: {
+            maxSize: finalConf.maxSize,
+            types: finalConf.types,
+            cleanupFiles: finalConf.cleanupFiles
+          }
+        }
+      }
+    );
+
+    return Param('body', {
+      ...finalConf,
+      extract: (request, config) => {
+        const field = SchemaRegistry.getMethodSchema(inst.constructor, prop)[idx];
+
+        if (!field) {
+          throw new AppError(`Unknown field type, ensure you are using ${Blob.name}, ${File.name} or ${FileMapContract.name}`);
+        }
+
+        if (!(field.type === Blob || field.type === File || field.type === FileMapContract)) {
+          throw new AppError(`Cannot use upload decorator with ${field.type.name}, but only an ${Blob.name}, ${File.name} or ${FileMapContract.name}`);
+        }
+
+        const isMap = field.type === FileMapContract;
+        const map = WebUploadUtil.getRequestUploads(request);
+        return isMap ? map : map[config.name!];
+      }
+    })(inst, prop, idx);
+  };
+}

package/src/interceptor.ts

@@ -0,0 +1,54 @@
+import { Inject, Injectable } from '@travetto/di';
+import {
+  BodyParseInterceptor, WebInterceptor, WebInterceptorCategory, WebChainedContext,
+  WebResponse, DecompressInterceptor, WebInterceptorContext
+} from '@travetto/web';
+
+import { WebUploadConfig } from './config.ts';
+import { WebUploadUtil } from './util.ts';
+import { FileMap } from './types.ts';
+
+@Injectable()
+export class WebUploadInterceptor implements WebInterceptor<WebUploadConfig> {
+
+  category: WebInterceptorCategory = 'request';
+  runsBefore = [BodyParseInterceptor];
+  dependsOn = [DecompressInterceptor];
+
+  @Inject()
+  config: WebUploadConfig;
+
+  /**
+   * Produces final config object
+   */
+  finalizeConfig({ config: base }: WebInterceptorContext<WebUploadConfig>, inputs: Partial<WebUploadConfig>[]): WebUploadConfig {
+    base.uploads ??= {};
+    // Override the uploads object with all the data from the inputs
+    for (const [k, cfg] of inputs.flatMap(el => Object.entries(el.uploads ?? {}))) {
+      Object.assign(base.uploads[k] ??= {}, cfg);
+    }
+    return base;
+  }
+
+  applies({ config }: WebInterceptorContext<WebUploadConfig>): boolean {
+    return config.applies;
+  }
+
+  async filter({ request, config, next }: WebChainedContext<WebUploadConfig>): Promise<WebResponse> {
+    const uploads: FileMap = {};
+
+    try {
+      for await (const item of WebUploadUtil.getUploads(request, config)) {
+        uploads[item.field] = await WebUploadUtil.toFile(item, config.uploads?.[item.field] ?? config);
+      }
+
+      WebUploadUtil.setRequestUploads(request, uploads);
+
+      return await next();
+    } finally {
+      for (const [field, item] of Object.entries(uploads)) {
+        await WebUploadUtil.finishUpload(item, config.uploads?.[field] ?? config);
+      }
+    }
+  }
+}

package/src/trv.d.ts

@@ -0,0 +1,7 @@
+import { FileMap } from './types.ts';
+
+declare module '@travetto/web' {
+  interface WebRequestInternal {
+    uploads?: FileMap
+  }
+}

package/src/types.ts

@@ -0,0 +1,4 @@
+/**
+ * @concrete
+ */
+export interface FileMap extends Record<string, File> { }

package/src/util.ts

@@ -0,0 +1,191 @@
+import { createReadStream, createWriteStream } from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import fs from 'node:fs/promises';
+import { pipeline } from 'node:stream/promises';
+import { Readable, Transform } from 'node:stream';
+
+import busboy from '@fastify/busboy';
+
+import { WebRequest, MimeUtil, WebBodyUtil } from '@travetto/web';
+import { AsyncQueue, AppError, castTo, Util, BinaryUtil } from '@travetto/runtime';
+
+import { WebUploadConfig } from './config.ts';
+import { FileMap } from './types.ts';
+
+const MULTIPART = new Set(['application/x-www-form-urlencoded', 'multipart/form-data']);
+
+type UploadItem = { stream: Readable, filename?: string, field: string };
+type FileType = { ext: string, mime: string };
+const RawFileSymbol = Symbol();
+const WebUploadSymbol = Symbol();
+
+/**
+ * Web upload utilities
+ */
+export class WebUploadUtil {
+
+  /**
+   * Write limiter
+   * @returns
+   */
+  static limitWrite(maxSize: number, field?: string): Transform {
+    let read = 0;
+    return new Transform({
+      transform(chunk, encoding, callback): void {
+        read += (Buffer.isBuffer(chunk) || typeof chunk === 'string') ? chunk.length : (chunk instanceof Uint8Array ? chunk.byteLength : 0);
+        if (read > maxSize) {
+          callback(new AppError('File size exceeded', { category: 'data', details: { read, size: maxSize, field } }));
+        } else {
+          callback(null, chunk);
+        }
+      },
+    });
+  }
+
+  /**
+   * Get uploaded file path location
+   */
+  static getUploadLocation(file: File): string {
+    return castTo<{ [RawFileSymbol]: string }>(file)[RawFileSymbol];
+  }
+
+  /**
+   * Get all the uploads, separating multipart from direct
+   */
+  static async* getUploads(request: WebRequest, config: Partial<WebUploadConfig>): AsyncIterable<UploadItem> {
+    if (!WebBodyUtil.isRaw(request.body)) {
+      throw new AppError('No input stream provided for upload', { category: 'data' });
+    }
+
+    const bodyStream = Buffer.isBuffer(request.body) ? Readable.from(request.body) : request.body;
+    request.body = undefined;
+
+    if (MULTIPART.has(request.headers.getContentType()?.full!)) {
+      const fileMaxes = Object.values(config.uploads ?? {}).map(x => x.maxSize).filter(x => x !== undefined);
+      const largestMax = fileMaxes.length ? Math.max(...fileMaxes) : config.maxSize;
+      const itr = new AsyncQueue<UploadItem>();
+
+      // Upload
+      bodyStream.pipe(busboy({
+        headers: {
+          'content-type': request.headers.get('Content-Type')!,
+          'content-disposition': request.headers.get('Content-Disposition')!,
+          'content-length': request.headers.get('Content-Length')!,
+          'content-range': request.headers.get('Content-Range')!,
+          'content-encoding': request.headers.get('Content-Encoding')!,
+          'content-transfer-encoding': request.headers.get('Content-Transfer-Encoding')!,
+        },
+        limits: { fileSize: largestMax }
+      })
+        .on('file', (field, stream, filename) => itr.add({ stream, filename, field }))
+        .on('limit', field => itr.throw(new AppError(`File size exceeded for ${field}`, { category: 'data' })))
+        .on('finish', () => itr.close())
+        .on('error', (err) => itr.throw(err instanceof Error ? err : new Error(`${err}`))));
+
+      yield* itr;
+    } else {
+      yield { stream: bodyStream, filename: request.headers.getFilename(), field: 'file' };
+    }
+  }
+
+  /**
+   * Convert an UploadItem to a File
+   */
+  static async toFile({ stream, filename, field }: UploadItem, config: Partial<WebUploadConfig>): Promise<File> {
+    const uniqueDir = path.resolve(os.tmpdir(), `file_${Date.now()}_${Util.uuid(5)}`);
+    await fs.mkdir(uniqueDir, { recursive: true });
+
+    filename = filename ? path.basename(filename) : `unknown_${Date.now()}`;
+
+    const location = path.resolve(uniqueDir, filename);
+    const remove = (): Promise<void> => fs.rm(location).catch(() => { });
+    const mimeCheck = config.matcher ??= MimeUtil.matcher(config.types);
+
+    try {
+      const target = createWriteStream(location);
+
+      await (config.maxSize ?
+        pipeline(stream, this.limitWrite(config.maxSize, field), target) :
+        pipeline(stream, target));
+
+      const detected = await this.getFileType(location);
+
+      if (!mimeCheck(detected.mime)) {
+        throw new AppError(`Content type not allowed: ${detected.mime}`, { category: 'data' });
+      }
+
+      if (!path.extname(filename)) {
+        filename = `${filename}.${detected.ext}`;
+      }
+
+      const file = BinaryUtil.readableBlob(() => createReadStream(location), {
+        contentType: detected.mime,
+        filename,
+        hash: await BinaryUtil.hashInput(createReadStream(location)),
+        size: (await fs.stat(location)).size,
+      });
+
+      Object.assign(file, { [RawFileSymbol]: location });
+
+      return file;
+    } catch (err) {
+      await remove();
+      throw err;
+    }
+  }
+
+  /**
+   * Get file type
+   */
+  static async getFileType(input: string | Readable): Promise<FileType> {
+    const { FileTypeParser } = await import('file-type');
+    const { fromStream } = await import('strtok3');
+
+    const parser = new FileTypeParser();
+    let tok: ReturnType<typeof fromStream> | undefined;
+    let matched: FileType | undefined;
+
+    try {
+      tok = await fromStream(typeof input === 'string' ? createReadStream(input) : input);
+      matched = await parser.fromTokenizer(tok);
+    } finally {
+      await tok?.close();
+    }
+
+    if (!matched && typeof input === 'string') {
+      const { Mime } = (await import('mime'));
+      const otherTypes = (await import('mime/types/other.js')).default;
+      const standardTypes = (await import('mime/types/standard.js')).default;
+      const checker = new Mime(standardTypes, otherTypes);
+      const mime = checker.getType(input);
+      if (mime) {
+        return { ext: checker.getExtension(mime)!, mime };
+      }
+    }
+    return matched ?? { ext: 'bin', mime: 'application/octet-stream' };
+  }
+
+  /**
+   * Finish upload
+   */
+  static async finishUpload(upload: File, config: Partial<WebUploadConfig>): Promise<void> {
+    if (config.cleanupFiles !== false) {
+      await fs.rm(this.getUploadLocation(upload), { force: true });
+    }
+  }
+
+  /**
+   * Get Uploads
+   */
+  static getRequestUploads(request: WebRequest & { [WebUploadSymbol]?: FileMap }): FileMap {
+    return request[WebUploadSymbol] ?? {};
+  }
+
+  /**
+   * Set Uploads
+   */
+  static setRequestUploads(request: WebRequest & { [WebUploadSymbol]?: FileMap }, uploads: FileMap): void {
+    request[WebUploadSymbol] ??= uploads;
+  }
+}

package/support/fixtures/asset.yml

@@ -0,0 +1,2 @@
+---
+yml: text

package/support/test/server.ts

@@ -0,0 +1,181 @@
+import assert from 'node:assert';
+
+import { BinaryUtil, castTo } from '@travetto/runtime';
+import { Controller, Post } from '@travetto/web';
+import { BeforeAll, Suite, Test, TestFixtures } from '@travetto/test';
+import { RootRegistry } from '@travetto/registry';
+
+import { BaseWebSuite } from '@travetto/web/support/test/suite/base.ts';
+
+import { Upload } from '../../src/decorator.ts';
+import { FileMap } from '../../src/types.ts';
+
+const bHash = (blob: Blob) => BinaryUtil.getBlobMeta(blob)?.hash;
+
+@Controller('/test/upload')
+class TestUploadController {
+
+  @Post('/all')
+  async uploadAll(@Upload() uploads: FileMap): Promise<{ hash?: string } | undefined> {
+    for (const [, blob] of Object.entries(uploads)) {
+      return { hash: bHash(blob) };
+    }
+  }
+
+  @Post('/')
+  async upload(@Upload() file: File) {
+    return { hash: bHash(file) };
+  }
+
+  @Post('/all-named')
+  async uploads(@Upload('file1') file1: Blob, @Upload('file2') file2: Blob) {
+    return { hash1: bHash(file1), hash2: bHash(file2) };
+  }
+
+  @Post('/all-named-custom')
+  async uploadVariousLimits(@Upload({ name: 'file1', types: ['!image/png'] }) file1: Blob, @Upload('file2') file2: Blob) {
+    return { hash1: bHash(file1), hash2: bHash(file2) };
+  }
+
+  @Post('/all-named-size')
+  async uploadVariousSizeLimits(@Upload({ name: 'file1', maxSize: 100 }) file1: File, @Upload({ name: 'file2', maxSize: 8000 }) file2: File) {
+    return { hash1: bHash(file1), hash2: bHash(file2) };
+  }
+}
+
+@Suite()
+export abstract class WebUploadServerSuite extends BaseWebSuite {
+
+  fixture: TestFixtures;
+
+  async getUploads(...files: { name: string, resource: string, type?: string }[]): Promise<FormData> {
+    const data = new FormData();
+    await Promise.all(files.map(async ({ name, type, resource }) => {
+      const file = await this.fixture.readFile(resource);
+      if (type) {
+        Object.defineProperty(file, 'type', { get: () => type });
+      }
+      data.append(name, file);
+    }));
+    return data;
+  }
+
+  @BeforeAll()
+  async init() {
+    this.fixture = new TestFixtures(['@travetto/asset']);
+    await RootRegistry.init();
+  }
+
+  @Test()
+  async testUploadAll() {
+    const uploads = await this.getUploads({ name: 'random', resource: 'logo.png', type: 'image/png' });
+    const response = await this.request<{ hash: string }>({ body: uploads, context: { httpMethod: 'POST', path: '/test/upload/all' } });
+
+    const file = await this.fixture.readStream('/logo.png');
+    assert(response.body?.hash === await BinaryUtil.hashInput(file));
+  }
+
+  @Test()
+  async testUploadDirect() {
+    const uploads = await this.getUploads({ name: 'file', resource: 'logo.png', type: 'image/png' });
+    const sent = castTo<Blob>(uploads.get('file'));
+    const response = await this.request<{ hash: string }>({ context: { httpMethod: 'POST', path: '/test/upload' }, body: sent });
+
+    const file = await this.fixture.readStream('/logo.png');
+    assert(response.body?.hash === await BinaryUtil.hashInput(file));
+  }
+
+  @Test()
+  async testUpload() {
+    const uploads = await this.getUploads({ name: 'file', resource: 'logo.png', type: 'image/png' });
+    const response = await this.request<{ hash: string }>({ body: uploads, context: { httpMethod: 'POST', path: '/test/upload' } });
+
+    const file = await this.fixture.readStream('/logo.png');
+    assert(response.body?.hash === await BinaryUtil.hashInput(file));
+  }
+
+  @Test()
+  async testMultiUpload() {
+    const uploads = await this.getUploads(
+      { name: 'file1', resource: 'logo.png', type: 'image/png' },
+      { name: 'file2', resource: 'logo.png', type: 'image/png' }
+    );
+    const response = await this.request<{ hash1: string, hash2: string }>({
+      body: uploads,
+      context: {
+        httpMethod: 'POST', path: '/test/upload/all-named'
+      }
+    });
+    const file = await this.fixture.readStream('/logo.png');
+    const hash = await BinaryUtil.hashInput(file);
+
+    assert(response.body?.hash1 === hash);
+    assert(response.body?.hash2 === hash);
+  }
+
+  @Test()
+  async testMultiUploadCustom() {
+    const uploadBad = await this.getUploads(
+      { name: 'file1', resource: 'logo.png', type: 'image/png' },
+      { name: 'file2', resource: 'logo.png', type: 'image/png' }
+    );
+
+    const badResponse = await this.request<{ hash1: string, hash2: string }>({
+      body: uploadBad,
+      context: { httpMethod: 'POST', path: '/test/upload/all-named-custom' },
+    }, false);
+    assert(badResponse.context.httpStatusCode === 400);
+
+    const uploads = await this.getUploads(
+      { name: 'file1', resource: 'logo.gif', type: 'image/gif' },
+      { name: 'file2', resource: 'logo.png', type: 'image/png' }
+    );
+    const response = await this.request<{ hash1: string, hash2: string }>({
+      body: uploads,
+      context: { httpMethod: 'POST', path: '/test/upload/all-named-custom' },
+    }, false);
+    assert(response.context.httpStatusCode === 200);
+
+    const file1 = await this.fixture.readStream('/logo.gif');
+    const hash1 = await BinaryUtil.hashInput(file1);
+
+    const file2 = await this.fixture.readStream('/logo.png');
+    const hash2 = await BinaryUtil.hashInput(file2);
+
+    assert(response.body?.hash1 === hash1);
+    assert(response.body?.hash2 === hash2);
+  }
+
+  @Test()
+  async testMultiUploadSize() {
+    const uploadBad = await this.getUploads(
+      { name: 'file1', resource: 'logo.png', type: 'image/png' },
+      { name: 'file2', resource: 'logo.png', type: 'image/png' }
+    );
+
+    const badResponse = await this.request<{ hash1: string, hash2: string }>({
+      body: uploadBad,
+      context: { httpMethod: 'POST', path: '/test/upload/all-named-size' },
+    }, false);
+    assert(badResponse.context.httpStatusCode === 400);
+
+    const uploads = await this.getUploads(
+      { name: 'file1', resource: 'asset.yml', type: 'text/plain' },
+      { name: 'file2', resource: 'logo.png', type: 'image/png' }
+    );
+    const response = await this.request<{ hash1: string, hash2: string }>({
+      body: uploads,
+      context: { httpMethod: 'POST', path: '/test/upload/all-named-size' },
+    }, false);
+    assert(response.context.httpStatusCode === 200);
+
+    const file1 = await this.fixture.readStream('/asset.yml');
+    const hash1 = await BinaryUtil.hashInput(file1);
+
+    const file2 = await this.fixture.readStream('/logo.png');
+    const hash2 = await BinaryUtil.hashInput(file2);
+
+    assert(response.body?.hash1 === hash1);
+    assert(response.body?.hash2 === hash2);
+  }
+}