@travetto/web-upload 7.1.4 → 8.0.0-alpha.1

package/README.md

@@ -15,7 +15,7 @@ yarn add @travetto/web-upload
 
 This module provides a clean and direct mechanism for processing uploads, built upon [@fastify/busboy](https://github.com/fastify/busboy). The module also provides some best practices with respect to temporary file management.
 
-Once the files are uploaded, they are exposed via [Endpoint](https://github.com/travetto/travetto/tree/main/module/web/src/decorator/endpoint.ts#L14) parameters using the [@Upload](https://github.com/travetto/travetto/tree/main/module/web-upload/src/decorator.ts#L20) decorator.  This decorator requires the related field type to be a standard [Buffer](https://nodejs.org/api/buffer.html#class-file) object, or a [FileMap](https://github.com/travetto/travetto/tree/main/module/web-upload/src/types.ts#L4).
+Once the files are uploaded, they are exposed via [Endpoint](https://github.com/travetto/travetto/tree/main/module/web/src/decorator/endpoint.ts#L14) parameters using the [@Upload](https://github.com/travetto/travetto/tree/main/module/web-upload/src/decorator.ts#L20) decorator.  This decorator requires the related field type to be a standard [Buffer](https://nodejs.org/api/buffer.html#class-file) object, or a [FileMap](https://github.com/travetto/travetto/tree/main/module/web-upload/src/types.ts#L7).
 
 A simple example:
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/web-upload",
-  "version": "7.1.4",
+  "version": "8.0.0-alpha.1",
   "type": "module",
   "description": "Provides integration between the travetto asset and web module.",
   "keywords": [
@@ -27,13 +27,13 @@
   },
   "dependencies": {
     "@fastify/busboy": "^3.2.0",
-    "@travetto/config": "^7.1.4",
-    "@travetto/web": "^7.1.4",
+    "@travetto/config": "^8.0.0-alpha.1",
+    "@travetto/web": "^8.0.0-alpha.1",
     "file-type": "^21.3.0",
     "mime": "^4.1.0"
   },
   "peerDependencies": {
-    "@travetto/test": "^7.1.4"
+    "@travetto/test": "^8.0.0-alpha.1"
   },
   "peerDependenciesMeta": {
     "@travetto/test": {

package/src/decorator.ts

@@ -1,4 +1,4 @@
-import { AppError, toConcrete, type ClassInstance, getClass } from '@travetto/runtime';
+import { RuntimeError, toConcrete, type ClassInstance, getClass } from '@travetto/runtime';
 import { ControllerRegistryIndex, type EndpointParameterConfig, Param } from '@travetto/web';
 import { SchemaRegistryIndex } from '@travetto/schema';
 
@@ -55,11 +55,11 @@ export function Upload(
         const input = SchemaRegistryIndex.get(cls).getMethod(property).parameters[idx];
 
         if (!input) {
-          throw new AppError(`Unknown field type, ensure you are using ${Blob.name}, ${File.name} or ${FileMapContract.name}`);
+          throw new RuntimeError(`Unknown field type, ensure you are using ${Blob.name}, ${File.name} or ${FileMapContract.name}`);
         }
 
         if (!(input.type === Blob || input.type === File || input.type === FileMapContract)) {
-          throw new AppError(`Cannot use upload decorator with ${input.type.name}, but only an ${Blob.name}, ${File.name} or ${FileMapContract.name}`);
+          throw new RuntimeError(`Cannot use upload decorator with ${input.type.name}, but only an ${Blob.name}, ${File.name} or ${FileMapContract.name}`);
         }
 
         const isMap = input.type === FileMapContract;

package/src/interceptor.ts

@@ -6,7 +6,6 @@ import {
 
 import type { WebUploadConfig } from './config.ts';
 import { WebUploadUtil } from './util.ts';
-import type { FileMap } from './types.ts';
 
 @Injectable()
 export class WebUploadInterceptor implements WebInterceptor<WebUploadConfig> {
@@ -35,7 +34,7 @@ export class WebUploadInterceptor implements WebInterceptor<WebUploadConfig> {
   }
 
   async filter({ request, config, next }: WebChainedContext<WebUploadConfig>): Promise<WebResponse> {
-    const uploads: FileMap = {};
+    const uploads: Record<string, File & { cleanup?: () => Promise<void> }> = {};
 
     try {
       for await (const item of WebUploadUtil.getUploads(request, config)) {
@@ -46,8 +45,8 @@ export class WebUploadInterceptor implements WebInterceptor<WebUploadConfig> {
 
       return await next();
     } finally {
-      for (const [field, item] of Object.entries(uploads)) {
-        await WebUploadUtil.finishUpload(item, config.uploads?.[field] ?? config);
+      for (const item of Object.values(uploads)) {
+        await item.cleanup?.();
       }
     }
   }

package/src/types.ts

@@ -1,4 +1,10 @@
+import { toConcrete } from '@travetto/runtime';
+import { SchemaTypeUtil } from '@travetto/schema';
+
 /**
  * @concrete
  */
-export interface FileMap extends Record<string, File> { }
+export interface FileMap extends Record<string, File> { }
+
+SchemaTypeUtil.register(toConcrete<FileMap>(),
+  input => typeof input === 'object' && !!input && Object.values(input).every(v => v instanceof Blob));

package/src/util.ts

@@ -3,21 +3,20 @@ import path from 'node:path';
 import os from 'node:os';
 import fs from 'node:fs/promises';
 import { pipeline } from 'node:stream/promises';
-import { Readable, Transform } from 'node:stream';
+import { Transform } from 'node:stream';
 
 import busboy from '@fastify/busboy';
 
 import { type WebRequest, WebCommonUtil, WebBodyUtil, WebHeaderUtil } from '@travetto/web';
-import { AsyncQueue, AppError, castTo, Util, BinaryUtil } from '@travetto/runtime';
+import { AsyncQueue, RuntimeError, CodecUtil, Util, BinaryUtil, type BinaryType, type BinaryStream, BinaryMetadataUtil } from '@travetto/runtime';
 
 import type { WebUploadConfig } from './config.ts';
 import type { FileMap } from './types.ts';
 
 const MULTIPART = new Set(['application/x-www-form-urlencoded', 'multipart/form-data']);
 
-type UploadItem = { stream: Readable, filename?: string, field: string };
+type UploadItem = { stream: BinaryStream, filename?: string, field: string, contentType?: string };
 type FileType = { ext: string, mime: string };
-const RawFileSymbol = Symbol();
 const WebUploadSymbol = Symbol();
 
 /**
@@ -32,10 +31,11 @@ export class WebUploadUtil {
   static limitWrite(maxSize: number, field?: string): Transform {
     let read = 0;
     return new Transform({
-      transform(chunk, encoding, callback): void {
-        read += (Buffer.isBuffer(chunk) || typeof chunk === 'string') ? chunk.length : (chunk instanceof Uint8Array ? chunk.byteLength : 0);
+      transform(input, encoding, callback): void {
+        const chunk = CodecUtil.readChunk(input, encoding);
+        read += chunk.byteLength;
         if (read > maxSize) {
-          callback(new AppError('File size exceeded', { category: 'data', details: { read, size: maxSize, field } }));
+          callback(new RuntimeError('File size exceeded', { category: 'data', details: { read, size: maxSize, field } }));
         } else {
           callback(null, chunk);
         }
@@ -43,22 +43,15 @@ export class WebUploadUtil {
     });
   }
 
-  /**
-   * Get uploaded file path location
-   */
-  static getUploadLocation(file: File): string {
-    return castTo<{ [RawFileSymbol]: string }>(file)[RawFileSymbol];
-  }
-
   /**
    * Get all the uploads, separating multipart from direct
    */
-  static async* getUploads(request: WebRequest, config: Partial<WebUploadConfig>): AsyncIterable<UploadItem> {
-    if (!WebBodyUtil.isRaw(request.body)) {
-      throw new AppError('No input stream provided for upload', { category: 'data' });
+  static async * getUploads(request: WebRequest, config: Partial<WebUploadConfig>): AsyncIterable<UploadItem> {
+    if (!WebBodyUtil.isRawBinary(request.body)) {
+      throw new RuntimeError('No input stream provided for upload', { category: 'data' });
     }
 
-    const bodyStream = Buffer.isBuffer(request.body) ? Readable.from(request.body) : request.body;
+    const requestBody = request.body;
     request.body = undefined;
 
     const contentType = WebHeaderUtil.parseHeaderSegment(request.headers.get('Content-Type'));
@@ -70,8 +63,7 @@ export class WebUploadUtil {
       const largestMax = fileMaxes.length ? Math.max(...fileMaxes) : config.maxSize;
       const queue = new AsyncQueue<UploadItem>();
 
-      // Upload
-      bodyStream.pipe(busboy({
+      const uploadHandler = busboy({
         headers: {
           'content-type': request.headers.get('Content-Type')!,
           'content-disposition': request.headers.get('Content-Disposition')!,
@@ -82,22 +74,69 @@ export class WebUploadUtil {
         },
         limits: { fileSize: largestMax }
       })
-        .on('file', (field, stream, filename) => queue.add({ stream, filename, field }))
-        .on('limit', field => queue.throw(new AppError(`File size exceeded for ${field}`, { category: 'data' })))
+        .on('file', (field, stream, filename, _encoding, mimetype) => queue.add({ stream, filename, field, contentType: mimetype }))
+        .on('limit', field => queue.throw(new RuntimeError(`File size exceeded for ${field}`, { category: 'data' })))
         .on('finish', () => queue.close())
-        .on('error', (error) => queue.throw(error instanceof Error ? error : new Error(`${error}`))));
+        .on('error', (error) => queue.throw(error instanceof Error ? error : new Error(`${error}`)));
+
+      // Upload
+      void BinaryUtil.pipeline(requestBody, uploadHandler).catch(err => queue.throw(err));
 
       yield* queue;
     } else {
       const filename = WebHeaderUtil.parseHeaderSegment(request.headers.get('Content-Disposition')).parameters.filename;
-      yield { stream: bodyStream, filename, field: 'file' };
+      yield { stream: BinaryUtil.toBinaryStream(requestBody), filename, field: 'file', contentType: contentType.value };
+    }
+  }
+
+  /**
+   * Detect mime from request input, usually http headers
+   */
+  static async detectMimeTypeFromRequestInput(location?: string, contentType?: string): Promise<FileType | undefined> {
+    const { Mime } = (await import('mime'));
+    const otherTypes = (await import('mime/types/other.js')).default;
+    const standardTypes = (await import('mime/types/standard.js')).default;
+    const checker = new Mime(standardTypes, otherTypes);
+    if (contentType) {
+      return { ext: checker.getExtension(contentType)!, mime: contentType };
+    } else if (location) {
+      const mime = checker.getType(location);
+      if (mime) {
+        return { mime, ext: checker.getExtension(mime)! };
+      }
+    }
+  }
+
+  /**
+   * Detect mime from the binary source
+   */
+  static async detectMimeTypeFromBinary(input: BinaryType): Promise<FileType | undefined> {
+    let cleanup: (() => Promise<void>) | undefined;
+    try {
+      const { FileTypeParser } = await import('file-type');
+      const { fromWebStream } = await import('strtok3');
+      const parser = new FileTypeParser();
+      const token = fromWebStream(BinaryUtil.toReadableStream(input));
+      cleanup = (): Promise<void> => token.close();
+      return await parser.fromTokenizer(token);
+    } finally {
+      await cleanup?.();
     }
   }
 
+  /**
+   * Get file type
+   */
+  static async getFileType(input: BinaryType, filename?: string, contentType?: string): Promise<FileType> {
+    return (await this.detectMimeTypeFromBinary(input)) ??
+      (await this.detectMimeTypeFromRequestInput(filename, contentType)) ??
+      { ext: 'bin', mime: 'application/octet-stream' };
+  }
+
   /**
    * Convert an UploadItem to a File
    */
-  static async toFile({ stream, filename, field }: UploadItem, config: Partial<WebUploadConfig>): Promise<File> {
+  static async toFile({ stream, filename, field, contentType }: UploadItem, config: Partial<WebUploadConfig>): Promise<File> {
     const uniqueDirectory = path.resolve(os.tmpdir(), `file_${Date.now()}_${Util.uuid(5)}`);
     await fs.mkdir(uniqueDirectory, { recursive: true });
 
@@ -106,6 +145,7 @@ export class WebUploadUtil {
     const location = path.resolve(uniqueDirectory, filename);
     const remove = (): Promise<void> => fs.rm(location).catch(() => { });
     const mimeCheck = config.matcher ??= WebCommonUtil.mimeTypeMatcher(config.types);
+    const response = (): BinaryStream => createReadStream(location);
 
     try {
       const target = createWriteStream(location);
@@ -114,25 +154,21 @@ export class WebUploadUtil {
         pipeline(stream, this.limitWrite(config.maxSize, field), target) :
         pipeline(stream, target));
 
-      const detected = await this.getFileType(location);
+      const detected = await this.getFileType(response(), filename, contentType);
 
       if (!mimeCheck(detected.mime)) {
-        throw new AppError(`Content type not allowed: ${detected.mime}`, { category: 'data' });
+        throw new RuntimeError(`Content type not allowed: ${detected.mime}`, { category: 'data' });
       }
 
       if (!path.extname(filename)) {
         filename = `${filename}.${detected.ext}`;
       }
 
-      const file = BinaryUtil.readableBlob(() => createReadStream(location), {
-        contentType: detected.mime,
-        filename,
-        hash: await BinaryUtil.hashInput(createReadStream(location)),
-        size: (await fs.stat(location)).size,
+      const metadata = await BinaryMetadataUtil.compute(response, { contentType: detected.mime, filename, });
+      const file = BinaryMetadataUtil.defineBlob(new File([], ''), response, metadata);
+      Object.defineProperty(file, 'cleanup', {
+        value: () => config.cleanupFiles !== false && fs.rm(location).catch(() => { })
       });
-
-      Object.assign(file, { [RawFileSymbol]: location });
-
       return file;
     } catch (error) {
       await remove();
@@ -140,46 +176,6 @@ export class WebUploadUtil {
     }
   }
 
-  /**
-   * Get file type
-   */
-  static async getFileType(input: string | Readable): Promise<FileType> {
-    const { FileTypeParser } = await import('file-type');
-    const { fromStream } = await import('strtok3');
-
-    const parser = new FileTypeParser();
-    let token: ReturnType<typeof fromStream> | undefined;
-    let matched: FileType | undefined;
-
-    try {
-      token = await fromStream(typeof input === 'string' ? createReadStream(input) : input);
-      matched = await parser.fromTokenizer(token);
-    } finally {
-      await token?.close();
-    }
-
-    if (!matched && typeof input === 'string') {
-      const { Mime } = (await import('mime'));
-      const otherTypes = (await import('mime/types/other.js')).default;
-      const standardTypes = (await import('mime/types/standard.js')).default;
-      const checker = new Mime(standardTypes, otherTypes);
-      const mime = checker.getType(input);
-      if (mime) {
-        return { ext: checker.getExtension(mime)!, mime };
-      }
-    }
-    return matched ?? { ext: 'bin', mime: 'application/octet-stream' };
-  }
-
-  /**
-   * Finish upload
-   */
-  static async finishUpload(upload: File, config: Partial<WebUploadConfig>): Promise<void> {
-    if (config.cleanupFiles !== false) {
-      await fs.rm(this.getUploadLocation(upload), { force: true });
-    }
-  }
-
   /**
    * Get Uploads
    */

package/support/test/server.ts

@@ -1,6 +1,6 @@
 import assert from 'node:assert';
 
-import { BinaryUtil, castTo } from '@travetto/runtime';
+import { BinaryMetadataUtil, castTo } from '@travetto/runtime';
 import { Controller, Post } from '@travetto/web';
 import { BeforeAll, Suite, Test, TestFixtures } from '@travetto/test';
 import { Registry } from '@travetto/registry';
@@ -10,7 +10,7 @@ import { BaseWebSuite } from '@travetto/web/support/test/suite/base.ts';
 import { Upload } from '../../src/decorator.ts';
 import type { FileMap } from '../../src/types.ts';
 
-const bHash = (blob: Blob) => BinaryUtil.getBlobMeta(blob)?.hash;
+const getHash = (blob: Blob) => BinaryMetadataUtil.read(blob)?.hash;
 
 @Controller('/test/upload')
 class TestUploadController {
@@ -18,28 +18,28 @@ class TestUploadController {
   @Post('/all')
   async uploadAll(@Upload() uploads: FileMap): Promise<{ hash?: string } | undefined> {
     for (const [, blob] of Object.entries(uploads)) {
-      return { hash: bHash(blob) };
+      return { hash: getHash(blob) };
     }
   }
 
   @Post('/')
   async upload(@Upload() file: File) {
-    return { hash: bHash(file) };
+    return { hash: getHash(file) };
   }
 
   @Post('/all-named')
   async uploads(@Upload() file1: Blob, @Upload() file2: Blob) {
-    return { hash1: bHash(file1), hash2: bHash(file2) };
+    return { hash1: getHash(file1), hash2: getHash(file2) };
   }
 
   @Post('/all-named-custom')
   async uploadVariousLimits(@Upload({ types: ['!image/png'] }) file1: Blob, @Upload() file2: Blob) {
-    return { hash1: bHash(file1), hash2: bHash(file2) };
+    return { hash1: getHash(file1), hash2: getHash(file2) };
   }
 
   @Post('/all-named-size')
   async uploadVariousSizeLimits(@Upload({ maxSize: 100 }) file1: File, @Upload({ maxSize: 8000 }) file2: File) {
-    return { hash1: bHash(file1), hash2: bHash(file2) };
+    return { hash1: getHash(file1), hash2: getHash(file2) };
   }
 }
 
@@ -71,8 +71,8 @@ export abstract class WebUploadServerSuite extends BaseWebSuite {
     const uploads = await this.getUploads({ name: 'random', resource: 'logo.png', type: 'image/png' });
     const response = await this.request<{ hash: string }>({ body: uploads, context: { httpMethod: 'POST', path: '/test/upload/all' } });
 
-    const file = await this.fixture.readStream('/logo.png');
-    assert(response.body?.hash === await BinaryUtil.hashInput(file));
+    const file = await this.fixture.readBinaryStream('/logo.png');
+    assert(response.body?.hash === await BinaryMetadataUtil.hash(file, { hashAlgorithm: 'sha256' }));
   }
 
   @Test()
@@ -81,8 +81,8 @@ export abstract class WebUploadServerSuite extends BaseWebSuite {
     const sent = castTo<Blob>(uploads.get('file'));
     const response = await this.request<{ hash: string }>({ context: { httpMethod: 'POST', path: '/test/upload' }, body: sent });
 
-    const file = await this.fixture.readStream('/logo.png');
-    assert(response.body?.hash === await BinaryUtil.hashInput(file));
+    const file = await this.fixture.readBinaryStream('/logo.png');
+    assert(response.body?.hash === await BinaryMetadataUtil.hash(file, { hashAlgorithm: 'sha256' }));
   }
 
   @Test()
@@ -90,8 +90,8 @@ export abstract class WebUploadServerSuite extends BaseWebSuite {
     const uploads = await this.getUploads({ name: 'file', resource: 'logo.png', type: 'image/png' });
     const response = await this.request<{ hash: string }>({ body: uploads, context: { httpMethod: 'POST', path: '/test/upload' } });
 
-    const file = await this.fixture.readStream('/logo.png');
-    assert(response.body?.hash === await BinaryUtil.hashInput(file));
+    const file = await this.fixture.readBinaryStream('/logo.png');
+    assert(response.body?.hash === await BinaryMetadataUtil.hash(file, { hashAlgorithm: 'sha256' }));
   }
 
   @Test()
@@ -106,8 +106,8 @@ export abstract class WebUploadServerSuite extends BaseWebSuite {
         httpMethod: 'POST', path: '/test/upload/all-named'
       }
     });
-    const file = await this.fixture.readStream('/logo.png');
-    const hash = await BinaryUtil.hashInput(file);
+    const file = await this.fixture.readBinaryStream('/logo.png');
+    const hash = await BinaryMetadataUtil.hash(file, { hashAlgorithm: 'sha256' });
 
     assert(response.body?.hash1 === hash);
     assert(response.body?.hash2 === hash);
@@ -136,11 +136,11 @@ export abstract class WebUploadServerSuite extends BaseWebSuite {
     }, false);
     assert(response.context.httpStatusCode === 200);
 
-    const file1 = await this.fixture.readStream('/logo.gif');
-    const hash1 = await BinaryUtil.hashInput(file1);
+    const file1 = await this.fixture.readBinaryStream('/logo.gif');
+    const hash1 = await BinaryMetadataUtil.hash(file1, { hashAlgorithm: 'sha256' });
 
-    const file2 = await this.fixture.readStream('/logo.png');
-    const hash2 = await BinaryUtil.hashInput(file2);
+    const file2 = await this.fixture.readBinaryStream('/logo.png');
+    const hash2 = await BinaryMetadataUtil.hash(file2, { hashAlgorithm: 'sha256' });
 
     assert(response.body?.hash1 === hash1);
     assert(response.body?.hash2 === hash2);
@@ -169,11 +169,11 @@ export abstract class WebUploadServerSuite extends BaseWebSuite {
     }, false);
     assert(response.context.httpStatusCode === 200);
 
-    const file1 = await this.fixture.readStream('/asset.yml');
-    const hash1 = await BinaryUtil.hashInput(file1);
+    const file1 = await this.fixture.readBinaryStream('/asset.yml');
+    const hash1 = await BinaryMetadataUtil.hash(file1, { hashAlgorithm: 'sha256' });
 
-    const file2 = await this.fixture.readStream('/logo.png');
-    const hash2 = await BinaryUtil.hashInput(file2);
+    const file2 = await this.fixture.readBinaryStream('/logo.png');
+    const hash2 = await BinaryMetadataUtil.hash(file2, { hashAlgorithm: 'sha256' });
 
     assert(response.body?.hash1 === hash1);
     assert(response.body?.hash2 === hash2);