node-server-engine 1.2.4 → 1.2.6

package/src/middleware/fileUploader/fileUploader.ts

@@ -1,303 +1,103 @@
-import Busboy from 'busboy';
 import { Request, Response, NextFunction } from 'express';
 import { Schema } from 'express-validator';
-import { compressAvatar } from './compressAvatar';
+import { WebError } from 'entities/WebError';
+import { reportDebug } from 'utils/report';
+import multer from 'multer';
 import { FileUploaderConfig } from './fileUploader.types';
-import { isMultipart } from './isMultiPart';
 import { validate } from './validate';
-import { Storage } from 'entities/Storage';
-import { WebError } from 'entities/WebError';
-import { reportError, reportDebug } from 'utils/report';
-import { Readable } from 'stream';
 
 const namespace = 'engine:middleware:fileUploader';
 
 /**
- * Generate a custom fileUploader middleware
+ * Middleware for handling file uploads.
+ * @param filesOptions Configuration options for expected files.
+ * @param validator Validation schema for request fields.
  */
 export function fileUploader(
-  options: Array<FileUploaderConfig>,
+  filesOptions: Array<FileUploaderConfig>,
   validator: Schema
 ) {
-  return (
+  return async (
     request: Request,
     response: Response,
     next: NextFunction
-  ): Busboy.Busboy =>
-    fileUploaderMiddleware(request, response, next, options, validator);
-}
-
-/** Handle file upload and generic multi-part parsing */
-export function fileUploaderMiddleware(
-  request: Request,
-  response: Response,
-  next: NextFunction,
-  filesOptions: Array<FileUploaderConfig>,
-  validator: Schema
-): Busboy.Busboy {
-  reportDebug({
-    namespace,
-    message: 'Received request to upload file',
-    data: { filesOptions, validator }
-  });
-  // Check that the request is a proper multipart request
-  if (!isMultipart(request))
-    throw new WebError({
-      statusCode: 415,
-      errorCode: 'invalid-content-type',
-      message: 'Request is not of type multipart/form-data',
-      data: { contentType: request.get('Content-Type') }
+  ): Promise<void> => {
+    reportDebug({
+      namespace,
+      message: 'Received request to upload file',
+      data: { filesOptions, validator }
     });
-  // If no files to handle ignore
-  if (!filesOptions.length) next();
-  // Parse multipart data with Busboy
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  const busboy = new (Busboy as any)({
-    headers: {
-      ...request.headers,
-      'content-type': request.headers['content-type'] ?? ''
-    }
-  });
 
-  let hasError = false;
+    // Setup multer with memory storage
+    const storage = multer.memoryStorage();
+    const upload = multer({
+      storage,
+      limits: { fileSize: 10 * 1024 * 1024 } // Limit file size to 10MB
+    }).any();
 
-  /**
-   * Function called when an error happened in the process
-   * It immediately stops and drains all connection and deletes all files that have already been uploaded
-   */
-  function triggerError(error: unknown): void {
-    pendingWrites--;
-    if (hasError) return;
-    hasError = true;
-    request.unpipe(busboy);
-    request.on('readable', request.read.bind(request));
-    busboy.removeAllListeners();
-    next(error);
-    if (request.files) {
-      Object.values(request.files).forEach((files) => {
-        files.forEach((file) => {
-          Storage.delete(file.bucket, file.name).catch((error) => {
-            reportError(error);
-          });
+    upload(request, response, async (err) => {
+      if (err) {
+        reportDebug({
+          namespace,
+          message: 'Multer upload error',
+          data: { error: err.message }
         });
-      });
-    }
-  }
-
-  let readFinished = false;
-  let fieldsValidated = false;
-  let pendingWrites = 0;
-
-  /** Function called when all the body has been parsed */
-  function done(): void {
-    if (readFinished && pendingWrites === 0 && fieldsValidated && !hasError) {
-      const uploadedFiles = Object.keys(request.files ?? {});
-      if (
-        filesOptions.some(
-          (options) => options.required && !uploadedFiles.includes(options.key)
-        )
-      ) {
-        triggerError(
-          new WebError({
-            statusCode: 400,
-            errorCode: 'missing-field',
-            message: 'Some required files have not been uploaded',
-            data: { filesOptions, validator, uploadedFiles }
-          })
-        );
-        return;
+        return next(err);
       }
-      next();
-    }
-  }
 
-  /*
-   * On cancel request, while processing
-   */
-  request.connection.on('close', function () {
-    if (!response.writableEnded) triggerError(new Error('Request cancelled'));
-  });
+      // Ensure `request.files` is properly typed
+      let uploadedFiles: Express.Multer.File[] = [];
+      if (Array.isArray(request.files)) {
+        uploadedFiles = request.files;
+      } else if (typeof request.files === 'object') {
+        uploadedFiles = Object.values(request.files).flat();
+      }
 
-  /**
-   * Once all the fields have been read they are checked against the validation schema
-   */
-  async function validateFields(): Promise<void> {
-    try {
-      await validate(request, validator);
-    } catch (error) {
-      triggerError(error);
-    }
-    reportDebug({
-      namespace,
-      message: 'Validation processed',
-      data: { hasError, filesOptions, validator }
-    });
-    fieldsValidated = true;
-    done();
-  }
+      // Log uploaded files
+      reportDebug({
+        namespace,
+        message: 'Files Received',
+        data: { uploadedFiles: uploadedFiles.map((file) => file.originalname) }
+      });
 
-  busboy.on(
-    'file',
-    (
-      field: string,
-      file: Readable,
-      fileName: string,
-      encoding: unknown,
-      mimeType: string
-    ) => {
-      (async (): Promise<void> => {
-        reportDebug({
-          namespace,
-          message: 'Found file in request',
-          data: { hasError, field, fileName, encoding, mimeType, pendingWrites }
-        });
-        // Do not handle the stream if there has been an error previously
-        if (hasError) file.destroy();
-        pendingWrites++;
-        // Find related file options based on the field name
-        const options = filesOptions.find(
-          (fileOption) => fileOption.key === field
-        );
-        // Make sure that the field is in the config
-        if (!options) {
-          triggerError(
-            new WebError({
-              statusCode: 400,
-              errorCode: 'invalid-file-field',
-              message: 'Request contains a file at an invalid field',
-              hint: { field },
-              data: { field }
-            })
-          );
-          return;
-        }
+      // Validate request body fields
+      try {
+        await validate(request, validator);
+      } catch (error) {
         reportDebug({
           namespace,
-          message: 'Found options for file',
-          data: {
-            hasError,
-            field,
-            fileName,
-            encoding,
-            mimeType,
-            pendingWrites,
-            options
-          }
+          message: 'Validation failed',
+          data: { error }
         });
-        // If a specific MIME type is required check it
-        if (options.mimeTypes && !options.mimeTypes.includes(mimeType)) {
-          triggerError(
-            new WebError({
-              statusCode: 415,
-              errorCode: 'invalid-mime-type',
-              message: 'Request contains a file with an invalid MIME type',
-              hint: { field },
-              data: { field, mimeType, expected: options.mimeTypes }
-            })
-          );
-          return;
-        }
-        // Generate the destination path of the file and upload
+        return next(error);
+      }
+
+      // Extract uploaded field names
+      const uploadedFileKeys = uploadedFiles.map((file) => file.fieldname);
+
+      // Check for missing required files
+      const missingFiles = filesOptions.filter(
+        (option) => option.required && !uploadedFileKeys.includes(option.key)
+      );
+
+      if (missingFiles.length > 0) {
         reportDebug({
           namespace,
-          message: 'Starting file upload',
-          data: {
-            hasError,
-            field,
-            fileName,
-            encoding,
-            mimeType,
-            pendingWrites,
-            options
-          }
+          message: 'Missing required files',
+          data: { missingFiles, uploadedFileKeys }
         });
-        try {
-          const fileToUpload = options.image
-            ? compressAvatar(file, options.image)
-            : file;
-
-          const uploadedFile = await Storage.upload(
-            fileToUpload,
-            options.bucket,
-            {
-              directory: options.path,
-              mime: mimeType,
-              noExtension: options.noExtension
-            },
-            {
-              metadata: { contentType: mimeType }
-            },
-            { maxSize: options.maxSize }
-          );
 
-          reportDebug({
-            namespace,
-            message: 'File uploaded',
-            data: {
-              hasError,
-              field,
-              fileName,
-              encoding,
-              mimeType,
-              pendingWrites,
-              options,
-              uploadedFile
-            }
-          });
-          if (hasError) {
-            // If some other failed before this one terminated
-            // Delete file
-            await Storage.delete(uploadedFile.bucket, uploadedFile.name);
-          } else {
-            // Handle successful upload
-            // Add the file's data to the request
-            pendingWrites--;
-            if (!request.files) request.files = {};
-            if (!request.files[field]) request.files[field] = [];
-            request.files[field].push({
-              ...uploadedFile,
-              originalFileName: fileName
-            });
-            done();
-          }
-        } catch (error) {
-          triggerError(error);
-        }
-      })().catch((error) => {
-        triggerError(error);
-      });
-    }
-  );
-
-  /**
-   * Handle non file fields
-   * These fields will be pushed to the body as array of strings
-   */
-  busboy.on('field', (fieldName: string, value: unknown) => {
-    if (!request.body) request.body = {};
+        return next(
+          new WebError({
+            statusCode: 400,
+            errorCode: 'missing-field',
+            message: 'Some required files have not been uploaded',
+            data: { missingFiles, uploadedFileKeys }
+          })
+        );
+      }
 
-    if (request.body[fieldName]) {
-      request.body[fieldName].push(value);
-    } else {
-      request.body[fieldName] = [value];
-    }
-  });
-  busboy.on('error', function (error: unknown) {
-    triggerError(error);
-  });
-  busboy.on('finish', () => {
-    (async (): Promise<void> => {
-      reportDebug({
-        namespace,
-        message: 'Busboy finished reading',
-        data: { hasError, filesOptions, validator, pendingWrites }
-      });
-      readFinished = true;
-      await validateFields();
-      done();
-    })().catch((error) => {
-      triggerError(error);
+      next();
     });
-  });
-  return request.pipe(busboy);
+  };
 }

package/src/middleware/fileUploader/fileUploader.types.ts

@@ -1,52 +1,35 @@
-import { CompressionOptions } from './compressAvatar';
-import { StorageUploadedFile } from 'entities/Storage';
-
-// Override the Express request object to add the files
+// Extend Express request object to include `files`
 declare global {
   // eslint-disable-next-line @typescript-eslint/no-namespace
   namespace Express {
-    /** Express request object */
     interface Request {
-      /** Configuration for files that are uploaded with the request */
-      files?: { [key: string]: Array<UploadedFile> };
+      /** Uploaded files mapped by field name */
+      files?:
+        | { [key: string]: Express.Multer.File[] }
+        | Express.Multer.File[]
+        | undefined;
     }
   }
 }
 
 /** Custom uploaded file metadata */
-export interface UploadedFile extends StorageUploadedFile {
-  /** File name that was given to the file by the client */
+export interface UploadedFile extends Express.Multer.File {
+  /** Original file name (use `originalname` from `Express.Multer.File`) */
   originalFileName: string;
+  /** File data buffer (use `buffer` from `Express.Multer.File`) */
+  file: Buffer;
 }
 
-/** Outputs setting for the file upload. Defines where a file should be stored and how it should be processed. */
-interface FileUploaderOutputs {
-  /** Name of the output */
-  name: string;
-  /** Path to which this specific output is stored (defaults to default path if not set) */
-  path?: string;
-  /** Bucket to which this specific output is stored (defaults to default bucket if not set) */
-  bucket?: string;
-}
-
-/** Configuration for a file upload */
+/** Configuration for file upload */
 export interface FileUploaderConfig {
-  /** Field key for which this configuration applies */
+  /** Field name expected in the request */
   key: string;
-  /** A list of supported MIME Types for the file */
+  /** Allowed MIME types */
   mimeTypes?: Array<string>;
-  /** Base path where the file should be stored (default used when no output config is set, will be used as temporary storage when outputs are set) */
-  path?: string;
-  /** Bucket in which the file should be stored (will be used as temporary storage when outputs are set) */
-  bucket: string;
-  /** Maximum size of the file (ex: 2MB) */
+  /** Maximum file size (e.g., "2MB") */
   maxSize?: string;
-  /** Indicates that the file must be present */
+  /** Whether the file is required */
   required?: boolean;
-  /** Do not store with an extension (will simply be a uuid) */
+  /** Whether to store without an extension */
   noExtension?: boolean;
-  /** Specify multiple outputs with processing */
-  outputs?: Array<FileUploaderOutputs>;
-  /** @deprecated compression of images */
-  image?: CompressionOptions;
 }

package/src/middleware/geminiFileUploader/geminiFileUploader.ts

@@ -1,70 +1,12 @@
 import express from 'express';
 import multer from 'multer';
-import { GoogleAIFileManager } from '@google/generative-ai/server';
-import fs from 'fs';
-import path from 'path';
-import tmp from 'tmp';
-import { reportError, reportDebug } from 'utils/report';
+import { reportError, geminiFileUpload, reportDebug } from 'utils';
 import { checkEnvironment } from 'utils/checkEnvironment';
 import { envAssert } from 'utils/envAssert';
 
 const namespace = 'engine:middleware:geminiFileUploader';
-
-const apiKey = process.env.GOOGLE_AI_KEY ?? '';
-const fileManager = new GoogleAIFileManager(apiKey);
-
 const storage = multer.memoryStorage();
 
-async function uploadToGemini(
-  buffer: Buffer,
-  mimeType: string,
-  originalName: string
-): Promise<{ name: string; uri: string; mimeType: string }> {
-  // Add 'name' to the return type
-  const tempFile = tmp.fileSync({ postfix: path.extname(originalName) });
-
-  fs.writeFileSync(tempFile.name, buffer);
-
-  const uploadResult = await fileManager.uploadFile(tempFile.name, {
-    mimeType,
-    displayName: originalName
-  });
-
-  tempFile.removeCallback();
-
-  const file = uploadResult.file;
-  reportDebug({
-    namespace,
-    message: 'Uploaded file',
-    data: { displayName: file.displayName, name: file.name }
-  });
-  return { name: file.name, uri: file.uri, mimeType: file.mimeType }; // Include 'name' in the return object
-}
-
-async function waitForFilesActive(
-  files: { name: string; uri: string; mimeType: string }[]
-): Promise<void> {
-  reportDebug({
-    namespace,
-    message: 'Waiting for file processing...'
-  });
-  for (const { name } of files) {
-    let file = await fileManager.getFile(name);
-    while (file.state === 'PROCESSING') {
-      process.stdout.write('.');
-      await new Promise((resolve) => setTimeout(resolve, 10_000));
-      file = await fileManager.getFile(name);
-    }
-    if (file.state !== 'ACTIVE') {
-      reportError(`File ${file.name} failed to process`);
-    }
-  }
-  reportDebug({
-    namespace,
-    message: 'all files ready...\n'
-  });
-}
-
 export async function geminiFileUploader(
   req: express.Request,
   res: express.Response,
@@ -92,22 +34,31 @@ export async function geminiFileUploader(
       }
       const mimeType = file.mimetype;
 
-      const uploadedFile = await uploadToGemini(
+      reportDebug({
+        namespace,
+        message: 'Upload to Gemini AI',
+        data: {
+          hasBuffer: !!file.buffer,
+          mimeType,
+          originalFileName: file.originalname
+        }
+      });
+
+      const fileResponse = await geminiFileUpload(
         file.buffer,
         mimeType,
         file.originalname
       );
 
-      await waitForFilesActive([uploadedFile]);
+      reportDebug({
+        namespace,
+        message: 'File Uploaded to Gemini AI',
+        data: fileResponse
+      });
 
-      // Set the fileUri and mimeType in the request object for the next middleware/route handler
-      req.body.file = {
-        fileUri: uploadedFile.uri,
-        mimeType: uploadedFile.mimeType,
-        originalname: file.originalname
-      };
+      req.body.file = fileResponse;
 
-      next(); // Proceed to the next middleware or route handler
+      next();
     });
   } catch (error) {
     console.error(error);

package/src/middleware/index.ts

@@ -14,3 +14,4 @@ export * from './requestMetrics';
 export * from './version';
 export * from './geminiFileUploader';
 export * from './checkPermission';
+export * from './multiPartFileUploader';

package/src/middleware/multiPartFileUploader/index.ts

@@ -0,0 +1 @@
+export * from './multiPartFileUploader';