node-server-engine 1.0.5

package/src/middleware/fileUploader/fileUploader.ts

@@ -0,0 +1,303 @@
+import Busboy from 'busboy';
+import { Request, Response, NextFunction } from 'express';
+import { Schema } from 'express-validator';
+import { compressAvatar } from './compressAvatar';
+import { FileUploaderConfig } from './fileUploader.types';
+import { isMultipart } from './isMultiPart';
+import { validate } from './validate';
+import { Storage } from 'entities/Storage';
+import { WebError } from 'entities/WebError';
+import { reportError, reportDebug } from 'utils/report';
+import { Readable } from 'stream';
+
+const namespace = 'engine:middleware:fileUploader';
+
+/**
+ * Generate a custom fileUploader middleware
+ */
+export function fileUploader(
+  options: Array<FileUploaderConfig>,
+  validator: Schema
+) {
+  return (
+    request: Request,
+    response: Response,
+    next: NextFunction
+  ): Busboy.Busboy =>
+    fileUploaderMiddleware(request, response, next, options, validator);
+}
+
+/** Handle file upload and generic multi-part parsing */
+export function fileUploaderMiddleware(
+  request: Request,
+  response: Response,
+  next: NextFunction,
+  filesOptions: Array<FileUploaderConfig>,
+  validator: Schema
+): Busboy.Busboy {
+  reportDebug({
+    namespace,
+    message: 'Received request to upload file',
+    data: { filesOptions, validator }
+  });
+  // Check that the request is a proper multipart request
+  if (!isMultipart(request))
+    throw new WebError({
+      statusCode: 415,
+      errorCode: 'invalid-content-type',
+      message: 'Request is not of type multipart/form-data',
+      data: { contentType: request.get('Content-Type') }
+    });
+  // If no files to handle ignore
+  if (!filesOptions.length) next();
+  // Parse multipart data with Busboy
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const busboy = new (Busboy as any)({
+    headers: {
+      ...request.headers,
+      'content-type': request.headers['content-type'] ?? ''
+    }
+  });
+
+  let hasError = false;
+
+  /**
+   * Function called when an error happened in the process
+   * It immediately stops and drains all connection and deletes all files that have already been uploaded
+   */
+  function triggerError(error: unknown): void {
+    pendingWrites--;
+    if (hasError) return;
+    hasError = true;
+    request.unpipe(busboy);
+    request.on('readable', request.read.bind(request));
+    busboy.removeAllListeners();
+    next(error);
+    if (request.files) {
+      Object.values(request.files).forEach((files) => {
+        files.forEach((file) => {
+          Storage.delete(file.bucket, file.name).catch((error) => {
+            reportError(error);
+          });
+        });
+      });
+    }
+  }
+
+  let readFinished = false;
+  let fieldsValidated = false;
+  let pendingWrites = 0;
+
+  /** Function called when all the body has been parsed */
+  function done(): void {
+    if (readFinished && pendingWrites === 0 && fieldsValidated && !hasError) {
+      const uploadedFiles = Object.keys(request.files ?? {});
+      if (
+        filesOptions.some(
+          (options) => options.required && !uploadedFiles.includes(options.key)
+        )
+      ) {
+        triggerError(
+          new WebError({
+            statusCode: 400,
+            errorCode: 'missing-field',
+            message: 'Some required files have not been uploaded',
+            data: { filesOptions, validator, uploadedFiles }
+          })
+        );
+        return;
+      }
+      next();
+    }
+  }
+
+  /*
+   * On cancel request, while processing
+   */
+  request.connection.on('close', function () {
+    if (!response.writableEnded) triggerError(new Error('Request cancelled'));
+  });
+
+  /**
+   * Once all the fields have been read they are checked against the validation schema
+   */
+  async function validateFields(): Promise<void> {
+    try {
+      await validate(request, validator);
+    } catch (error) {
+      triggerError(error);
+    }
+    reportDebug({
+      namespace,
+      message: 'Validation processed',
+      data: { hasError, filesOptions, validator }
+    });
+    fieldsValidated = true;
+    done();
+  }
+
+  busboy.on(
+    'file',
+    (
+      field: string,
+      file: Readable,
+      fileName: string,
+      encoding: unknown,
+      mimeType: string
+    ) => {
+      (async (): Promise<void> => {
+        reportDebug({
+          namespace,
+          message: 'Found file in request',
+          data: { hasError, field, fileName, encoding, mimeType, pendingWrites }
+        });
+        // Do not handle the stream if there has been an error previously
+        if (hasError) file.destroy();
+        pendingWrites++;
+        // Find related file options based on the field name
+        const options = filesOptions.find(
+          (fileOption) => fileOption.key === field
+        );
+        // Make sure that the field is in the config
+        if (!options) {
+          triggerError(
+            new WebError({
+              statusCode: 400,
+              errorCode: 'invalid-file-field',
+              message: 'Request contains a file at an invalid field',
+              hint: { field },
+              data: { field }
+            })
+          );
+          return;
+        }
+        reportDebug({
+          namespace,
+          message: 'Found options for file',
+          data: {
+            hasError,
+            field,
+            fileName,
+            encoding,
+            mimeType,
+            pendingWrites,
+            options
+          }
+        });
+        // If a specific MIME type is required check it
+        if (options.mimeTypes && !options.mimeTypes.includes(mimeType)) {
+          triggerError(
+            new WebError({
+              statusCode: 415,
+              errorCode: 'invalid-mime-type',
+              message: 'Request contains a file with an invalid MIME type',
+              hint: { field },
+              data: { field, mimeType, expected: options.mimeTypes }
+            })
+          );
+          return;
+        }
+        // Generate the destination path of the file and upload
+        reportDebug({
+          namespace,
+          message: 'Starting file upload',
+          data: {
+            hasError,
+            field,
+            fileName,
+            encoding,
+            mimeType,
+            pendingWrites,
+            options
+          }
+        });
+        try {
+          const fileToUpload = options.image
+            ? compressAvatar(file, options.image)
+            : file;
+
+          const uploadedFile = await Storage.upload(
+            fileToUpload,
+            options.bucket,
+            {
+              directory: options.path,
+              mime: mimeType,
+              noExtension: options.noExtension
+            },
+            {
+              metadata: { contentType: mimeType }
+            },
+            { maxSize: options.maxSize }
+          );
+
+          reportDebug({
+            namespace,
+            message: 'File uploaded',
+            data: {
+              hasError,
+              field,
+              fileName,
+              encoding,
+              mimeType,
+              pendingWrites,
+              options,
+              uploadedFile
+            }
+          });
+          if (hasError) {
+            // If some other failed before this one terminated
+            // Delete file
+            await Storage.delete(uploadedFile.bucket, uploadedFile.name);
+          } else {
+            // Handle successful upload
+            // Add the file's data to the request
+            pendingWrites--;
+            if (!request.files) request.files = {};
+            if (!request.files[field]) request.files[field] = [];
+            request.files[field].push({
+              ...uploadedFile,
+              originalFileName: fileName
+            });
+            done();
+          }
+        } catch (error) {
+          triggerError(error);
+        }
+      })().catch((error) => {
+        triggerError(error);
+      });
+    }
+  );
+
+  /**
+   * Handle non file fields
+   * These fields will be pushed to the body as array of strings
+   */
+  busboy.on('field', (fieldName: string, value: unknown) => {
+    if (!request.body) request.body = {};
+
+    if (request.body[fieldName]) {
+      request.body[fieldName].push(value);
+    } else {
+      request.body[fieldName] = [value];
+    }
+  });
+  busboy.on('error', function (error: unknown) {
+    triggerError(error);
+  });
+  busboy.on('finish', () => {
+    (async (): Promise<void> => {
+      reportDebug({
+        namespace,
+        message: 'Busboy finished reading',
+        data: { hasError, filesOptions, validator, pendingWrites }
+      });
+      readFinished = true;
+      await validateFields();
+      done();
+    })().catch((error) => {
+      triggerError(error);
+    });
+  });
+  return request.pipe(busboy);
+}

package/src/middleware/fileUploader/fileUploader.types.ts

@@ -0,0 +1,52 @@
+import { CompressionOptions } from './compressAvatar';
+import { StorageUploadedFile } from 'entities/Storage';
+
+// Override the Express request object to add the files
+declare global {
+  // eslint-disable-next-line @typescript-eslint/no-namespace
+  namespace Express {
+    /** Express request object */
+    interface Request {
+      /** Configuration for files that are uploaded with the request */
+      files?: { [key: string]: Array<UploadedFile> };
+    }
+  }
+}
+
+/** Custom uploaded file metadata */
+export interface UploadedFile extends StorageUploadedFile {
+  /** File name that was given to the file by the client */
+  originalFileName: string;
+}
+
+/** Outputs setting for the file upload. Defines where a file should be stored and how it should be processed. */
+interface FileUploaderOutputs {
+  /** Name of the output */
+  name: string;
+  /** Path to which this specific output is stored (defaults to default path if not set) */
+  path?: string;
+  /** Bucket to which this specific output is stored (defaults to default bucket if not set) */
+  bucket?: string;
+}
+
+/** Configuration for a file upload */
+export interface FileUploaderConfig {
+  /** Field key for which this configuration applies */
+  key: string;
+  /** A list of supported MIME Types for the file */
+  mimeTypes?: Array<string>;
+  /** Base path where the file should be stored (default used when no output config is set, will be used as temporary storage when outputs are set) */
+  path?: string;
+  /** Bucket in which the file should be stored (will be used as temporary storage when outputs are set) */
+  bucket: string;
+  /** Maximum size of the file (ex: 2MB) */
+  maxSize?: string;
+  /** Indicates that the file must be present */
+  required?: boolean;
+  /** Do not store with an extension (will simply be a uuid) */
+  noExtension?: boolean;
+  /** Specify multiple outputs with processing */
+  outputs?: Array<FileUploaderOutputs>;
+  /** @deprecated compression of images */
+  image?: CompressionOptions;
+}

package/src/middleware/fileUploader/index.ts

@@ -0,0 +1,2 @@
+export * from './fileUploader';
+export * from './fileUploader.types';

package/src/middleware/fileUploader/isMultiPart/index.ts

@@ -0,0 +1 @@
+export * from './isMultipart';

package/src/middleware/fileUploader/isMultiPart/isMultipart.ts

@@ -0,0 +1,9 @@
+import { Request } from 'express';
+
+const multipartRegex = /^multipart\/form-data.+$/;
+
+/** Determines if the content type of a request is multipart/form-data */
+export function isMultipart(request: Request): boolean {
+  const type = request.get('Content-Type');
+  return !!type && multipartRegex.test(type.toLowerCase());
+}

package/src/middleware/fileUploader/validate/index.ts

@@ -0,0 +1 @@
+export * from './validate';

package/src/middleware/fileUploader/validate/validate.ts

@@ -0,0 +1,27 @@
+import { Request } from 'express';
+import { Schema } from 'express-validator';
+import { validate as validateMiddleware } from 'middleware/validate';
+
+/** Validate a schema against a request using our validation middleware */
+export async function validate(
+  request: Request,
+  schema: Schema
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    // Generate a validation chain using our middleware
+    const validationChain = validateMiddleware(schema).flat();
+    if (!validationChain.length) resolve();
+    // Recursively go through each middleware by imitating the next function from express
+    let position = -1;
+    const next = (error?: unknown): void => {
+      if (error) reject(error);
+      position++;
+      if (position === validationChain.length) {
+        resolve();
+        return;
+      }
+      validationChain[position](request, undefined, next);
+    };
+    next();
+  });
+}

package/src/middleware/fourOhFour/fourOhFour.test.ts

@@ -0,0 +1,14 @@
+import express from 'express';
+import supertest from 'supertest';
+import { fourOhFour } from './fourOhFour';
+
+describe('Middleware - fourOhFour', () => {
+  it('should 404 everything', async () => {
+    const app = express();
+    app.get('/example', (req: any, res: any) => res.sendStatus(200));
+    app.use(fourOhFour);
+
+    await supertest(app).get('/example').expect(200);
+    await supertest(app).get('/nope').expect(404);
+  });
+});

package/src/middleware/fourOhFour/fourOhFour.ts

@@ -0,0 +1,13 @@
+import { Request, Response } from 'express';
+import { reportDebug } from 'utils';
+
+const namespace = 'engine:middleware:fourOhFour';
+
+/** Unhandled path middleware */
+export function fourOhFour(request: Request, response: Response): void {
+  reportDebug({
+    namespace,
+    message: `No route found [${request.method} - ${request.path}]`
+  });
+  response.sendStatus(404);
+}

package/src/middleware/fourOhFour/index.ts

@@ -0,0 +1 @@
+export * from './fourOhFour';

package/src/middleware/healthCheck/healthCheck.test.ts

@@ -0,0 +1,12 @@
+import express from 'express';
+import request from 'supertest';
+import { healthCheck } from './healthCheck';
+
+describe('Middleware - healthCheck', () => {
+  it('default healtcheck route', async () => {
+    const app = express();
+    app.use(healthCheck());
+    await request(app).get('/').expect(200);
+    await request(app).get('/heartbeat').expect(200);
+  });
+});

package/src/middleware/healthCheck/healthCheck.ts

@@ -0,0 +1,14 @@
+import { Router, Request, Response } from 'express';
+
+/**
+ * respond to Kubernetes health-checks with 200
+ */
+export function healthCheck(path = '/heartbeat'): Router {
+  return Router()
+    .get('/', (req: Request, res: Response) => {
+      res.sendStatus(200);
+    })
+    .get(path, (req: Request, res: Response) => {
+      res.sendStatus(200);
+    });
+}

package/src/middleware/healthCheck/index.ts

@@ -0,0 +1 @@
+export * from './healthCheck';

package/src/middleware/index.ts

@@ -0,0 +1,15 @@
+export * from './error';
+export * from './healthCheck';
+export * from './validate';
+export * from './fourOhFour';
+export * from './authJwt';
+export * from './authHmac';
+export * from './authStatic';
+export * from './swaggerDocs';
+export * from './authTls';
+export * from './userResolver';
+export * from './fileUploader';
+export * from './metrics';
+export * from './requestMetrics';
+export * from './version';
+export * from './authAdmin';

package/src/middleware/metrics/index.ts

@@ -0,0 +1 @@
+export * from './metrics';

package/src/middleware/metrics/metrics.test.ts

@@ -0,0 +1,46 @@
+import { expect } from 'chai';
+import express from 'express';
+import { Counter } from 'prom-client';
+import request from 'supertest';
+import { metrics } from './metrics';
+import { serviceRegistry, instanceRegistry } from 'const/metrics';
+
+describe('Middleware - metrics', () => {
+  beforeEach(() => {
+    instanceRegistry.clear();
+    serviceRegistry.clear();
+  });
+
+  it('should export instance metrics', async () => {
+    new Counter({
+      name: 'instance_test',
+      help: 'instance_test',
+      registers: [instanceRegistry]
+    });
+
+    const app = express();
+    app.use(metrics());
+
+    const { text } = await request(app).get('/metrics/instance').expect(200);
+    expect(text).to.equal(
+      '# HELP instance_test instance_test\n# TYPE instance_test counter\ninstance_test 0\n'
+    );
+  });
+
+  it('should export service metrics', async () => {
+    new Counter({
+      name: 'service_test',
+      help: 'service_test',
+      registers: [serviceRegistry]
+    });
+
+    const app = express();
+    app.use(metrics());
+
+    const { text } = await request(app).get('/metrics/service').expect(200);
+
+    expect(text).to.equal(
+      '# HELP service_test service_test\n# TYPE service_test counter\nservice_test 0\n'
+    );
+  });
+});

package/src/middleware/metrics/metrics.ts

@@ -0,0 +1,24 @@
+import { Router, Request, Response } from 'express';
+import { instanceRegistry, serviceRegistry } from 'const/metrics';
+import { reportError } from 'utils/report';
+
+/** Middleware that calculates request time, and exposes metrics related to requests processed by this service */
+export function metrics(): Router {
+  return Router()
+    .use('/metrics/instance', (req: Request, res: Response) => {
+      (async (): Promise<void> => {
+        res.set('Content-Type', instanceRegistry.contentType);
+        res.end(await instanceRegistry.metrics());
+      })().catch((error: Error) => {
+        reportError(error);
+      });
+    })
+    .use('/metrics/service', (req: Request, res: Response) => {
+      (async (): Promise<void> => {
+        res.set('Content-Type', serviceRegistry.contentType);
+        res.end(await serviceRegistry.metrics());
+      })().catch((error: Error) => {
+        reportError(error);
+      });
+    });
+}

package/src/middleware/requestMetrics/index.ts

@@ -0,0 +1 @@
+export * from './requestMetrics';

package/src/middleware/requestMetrics/requestMetrics.test.ts

@@ -0,0 +1,77 @@
+import { expect } from 'chai';
+import express from 'express';
+import { faker } from '@faker-js/faker';
+import { stub, match, SinonStub } from 'sinon';
+import request from 'supertest';
+import { requestMetrics } from './requestMetrics';
+import { httpRequestCount, httpRequestDuration } from 'metrics/request';
+
+describe('Middleware - requestMetrics', function () {
+  let requestCountStub: SinonStub;
+  let requestDurationStub: SinonStub;
+
+  beforeEach(() => {
+    requestCountStub = stub(httpRequestCount, 'inc').returns();
+    requestDurationStub = stub(httpRequestDuration, 'observe').returns();
+  });
+
+  it('should handle metrics for the requests', async () => {
+    const service = faker.lorem.word();
+    process.env.CHART = service;
+
+    const app = express();
+    app.use(requestMetrics);
+    app.get('/test', function (req, res) {
+      res.sendStatus(200);
+    });
+
+    await request(app).get('/test').expect(200);
+
+    const labels = { service, path: '/test', method: 'GET', status: '2XX' };
+    expect(requestCountStub).to.have.been.calledOnceWithExactly(labels);
+    expect(requestDurationStub).to.have.been.calledOnceWithExactly(
+      labels,
+      match.number
+    );
+  });
+
+  it('should not keep query parameters in path', async () => {
+    const service = faker.lorem.word();
+    process.env.CHART = service;
+
+    const app = express();
+    app.use(requestMetrics);
+    app.get('/test', function (req, res) {
+      res.sendStatus(200);
+    });
+
+    await request(app).get('/test?param=value').expect(200);
+
+    const labels = { service, path: '/test', method: 'GET', status: '2XX' };
+    expect(requestCountStub).to.have.been.calledOnceWithExactly(labels);
+    expect(requestDurationStub).to.have.been.calledOnceWithExactly(
+      labels,
+      match.number
+    );
+  });
+
+  it('should not keep anchor in path', async () => {
+    const service = faker.lorem.word();
+    process.env.CHART = service;
+
+    const app = express();
+    app.use(requestMetrics);
+    app.get('/test', function (req, res) {
+      res.sendStatus(200);
+    });
+
+    await request(app).get('/test#place').expect(200);
+
+    const labels = { service, path: '/test', method: 'GET', status: '2XX' };
+    expect(requestCountStub).to.have.been.calledOnceWithExactly(labels);
+    expect(requestDurationStub).to.have.been.calledOnceWithExactly(
+      labels,
+      match.number
+    );
+  });
+});

package/src/middleware/requestMetrics/requestMetrics.ts

@@ -0,0 +1,20 @@
+import { Request, Response } from 'express';
+import responseTime from 'response-time';
+import { httpRequestCount, httpRequestDuration } from 'metrics';
+
+/** Middleware that gets metrics on each request */
+export const requestMetrics = responseTime(
+  (request: Request, response: Response, time: number) => {
+    const labels = {
+      path: (request.route?.path ?? request.path) as string,
+      method: request.method,
+      status: response.statusCode
+        ? `${response.statusCode.toString().substring(0, 1)}XX`
+        : '',
+      service: process.env.CHART
+    };
+
+    httpRequestCount.inc(labels);
+    httpRequestDuration.observe(labels, time / 1000);
+  }
+);

package/src/middleware/swaggerDocs/index.ts

@@ -0,0 +1,2 @@
+export * from './swaggerDocs';
+export * from './swaggerDocs.types';