node-server-engine 1.0.5

package/src/utils/dbMigration/dbMigration.test.ts

@@ -0,0 +1,209 @@
+// import path from 'path';
+// import { expect } from 'chai';
+// import mockFs, { restore } from 'mock-fs';
+// import mockRequire from 'mock-require';
+// import {
+//   runPendingMigrations,
+//   rollbackMigrations,
+//   testClearDB
+// } from './dbMigration';
+// import { SequelizeMeta } from 'db/models';
+// import { factory } from 'test';
+
+// describe('Utils - DB Migration', function () {
+//   beforeEach(async () => {
+//     await testClearDB({ includeMetadata: true });
+
+//     process.env.BUILD_ID = '1.5.0';
+
+//     const basePath = path.join(process.cwd(), './src/db/migrations');
+//     // Requires use the real filesystem and need to be mocked
+//     const requiredContent = {
+//       // eslint-disable-next-line @typescript-eslint/no-empty-function
+//       up: async (): Promise<void> => {},
+//       // eslint-disable-next-line @typescript-eslint/no-empty-function
+//       down: async (): Promise<void> => {}
+//     };
+//     mockRequire(`${basePath}/1-test.js`, requiredContent);
+//     mockRequire(`${basePath}/2-test.js`, requiredContent);
+//     mockRequire(`${basePath}/3-test.ts`, requiredContent);
+//     // Umzug checks that the files is on the FS
+//     mockFs({
+//       [basePath]: { '1-test.js': '', '2-test.js': '', '3-test.ts': '' }
+//     });
+//   });
+
+//   afterEach(() => {
+//     restore();
+//     mockRequire.stopAll();
+//   });
+
+//   describe('runPendingMigrations', function () {
+//     // it('should run the migrations when there are no history', async () => {
+//     //   await runPendingMigrations();
+
+//     //   const migrations = await SequelizeMeta.findAll();
+
+//     //   expect(migrations).to.have.length(3);
+//     //   migrations.forEach((migration) => {
+//     //     expect(['1-test.js', '2-test.js', '3-test.ts']).to.include(migration.name);
+//     //     expect(migration.version).to.equal(process.env.BUILD_ID);
+//     //   });
+//     // });
+
+//     it('should prefer migrations in the prod directory', async () => {
+//       const prodBasePath = path.join(process.cwd(), './dist/db/migrations');
+//       // Requires use the real filesystem and need to be mocked
+//       const requiredContent = {
+//         // eslint-disable-next-line @typescript-eslint/no-empty-function
+//         up: async (): Promise<void> => {},
+//         // eslint-disable-next-line @typescript-eslint/no-empty-function
+//         down: async (): Promise<void> => {}
+//       };
+//       mockRequire(`${prodBasePath}/1-prod-test.js`, requiredContent);
+//       mockRequire(`${prodBasePath}/2-prod-test.js`, requiredContent);
+//       mockRequire(`${prodBasePath}/3-prod-test.js`, requiredContent);
+//       // Umzug checks that the files is on the FS
+//       mockFs({
+//         [prodBasePath]: {
+//           '1-prod-test.js': '',
+//           '2-prod-test.js': '',
+//           '3-prod-test.js': ''
+//         }
+//       });
+
+//       await runPendingMigrations();
+
+//       const migrations = await SequelizeMeta.findAll();
+
+//       expect(migrations).to.have.length(3);
+//       migrations.forEach((migration) => {
+//         expect([
+//           '1-prod-test.js',
+//           '2-prod-test.js',
+//           '3-prod-test.js'
+//         ]).to.include(migration.name);
+//         expect(migration.version).to.equal(process.env.BUILD_ID);
+//       });
+
+//       it('should allow to specify a custom directory', async () => {
+//         const customBasePath = path.join(process.cwd(), './migrations');
+//         process.env.DB_MIGRATIONS_DIR = customBasePath;
+
+//         // Requires use the real filesystem and need to be mocked
+//         const requiredContent = {
+//           // eslint-disable-next-line @typescript-eslint/no-empty-function
+//           up: async (): Promise<void> => {},
+//           // eslint-disable-next-line @typescript-eslint/no-empty-function
+//           down: async (): Promise<void> => {}
+//         };
+//         mockRequire(`${customBasePath}/1-custom-test.js`, requiredContent);
+//         mockRequire(`${customBasePath}/2-custom-test.js`, requiredContent);
+//         mockRequire(`${customBasePath}/3-custom-test.js`, requiredContent);
+//         // Umzug checks that the files is on the FS
+//         mockFs({
+//           [customBasePath]: {
+//             '1-custom-test.js': '',
+//             '2-custom-test.js': '',
+//             '3-custom-test.js': ''
+//           }
+//         });
+
+//         await runPendingMigrations();
+
+//         const migrations = await SequelizeMeta.findAll();
+
+//         expect(migrations).to.have.length(3);
+//         migrations.forEach((migration) => {
+//           expect([
+//             '1-custom-test.js',
+//             '2-custom-test.js',
+//             '3-custom-test.js'
+//           ]).to.include(migration.name);
+//           expect(migration.version).to.equal(process.env.BUILD_ID);
+//         });
+//       });
+//     });
+//   });
+
+//   describe('rollbackMigrations', function () {
+//     it('should not rollback previous migrations', async () => {
+//       await factory.create('sequelizeMeta', {
+//         name: '1-test.js',
+//         version: '1.4.0'
+//       });
+//       await factory.create('sequelizeMeta', {
+//         name: '2-test.js',
+//         version: process.env.BUILD_ID
+//       });
+
+//       await rollbackMigrations();
+
+//       // expect(rolledBack).to.have.length(0);
+//     });
+
+//     it('should rollback the higher migrations', async () => {
+//       await factory.create('sequelizeMeta', {
+//         name: '1-test.js',
+//         version: '1.5.0'
+//       });
+//       await factory.create('sequelizeMeta', {
+//         name: '2-test.js',
+//         version: process.env.BUILD_ID
+//       });
+//       await factory.create('sequelizeMeta', {
+//         name: '3-test.ts',
+//         version: '1.5.1'
+//       });
+
+//       await rollbackMigrations();
+
+//       // expect(rolledBack).to.have.length(1);
+//       // expect(rolledBack[0].file).to.equal('3-test.ts');
+//     });
+
+//     it('should rollback multiple migrations with higher version', async () => {
+//       await factory.create('sequelizeMeta', {
+//         name: '1-test.js',
+//         version: process.env.BUILD_ID
+//       });
+//       await factory.create('sequelizeMeta', {
+//         name: '2-test.js',
+//         version: '1.5.2'
+//       });
+//       await factory.create('sequelizeMeta', {
+//         name: '3-test.ts',
+//         version: '1.6.0'
+//       });
+
+//       await rollbackMigrations();
+
+//       // expect(rolledBack).to.have.length(2);
+//       // rolledBack.forEach((migration) => {
+//       //   expect(['2-test.js', '3-test.ts']).to.include(migration.file);
+//       // });
+//     });
+
+//     it('should handle pre-release (alpha<beta)', async () => {
+//       process.env.BUILD_ID = '1.5.0-alpha.2';
+
+//       await factory.create('sequelizeMeta', {
+//         name: '1-test.js',
+//         version: '1.5.0-alpha.1'
+//       });
+//       await factory.create('sequelizeMeta', {
+//         name: '2-test.js',
+//         version: process.env.BUILD_ID
+//       });
+//       await factory.create('sequelizeMeta', {
+//         name: '3-test.ts',
+//         version: '1.5.0-beta.1'
+//       });
+
+//       await rollbackMigrations();
+
+//       // expect(rolledBack).to.have.length(1);
+//       // expect(rolledBack[0].file).to.equal('3-test.ts');
+//     });
+//   });
+// });

package/src/utils/dbMigration/dbMigration.ts

@@ -0,0 +1,196 @@
+/* eslint-disable @typescript-eslint/no-require-imports */
+// Register ts-node at the top of runMigrations.ts
+require('ts-node/register');
+
+import { existsSync, lstatSync } from 'fs';
+import { join } from 'path';
+import { Sequelize, Transaction } from 'sequelize';
+import { Umzug, SequelizeStorage } from 'umzug';
+import { ClearDbOptions, RecreateDbOptions } from './dbMigration.types';
+import { migrateMetadataTable } from './migrateMetadataTable';
+import { SequelizeMeta } from 'db/models';
+import { EngineError } from 'entities/EngineError';
+import { sequelize } from 'entities/Sequelize';
+import { reportDebug } from 'utils/report';
+import { semverCompare } from 'utils/semverCompare';
+
+const namespace = 'engine:utils:dbMigration';
+
+function getMigrationPaths(): Array<string> | undefined {
+  // We look between possibilities for the first directory that exists and matches our expectations
+  // First a user specified one, then the production default, then the development default
+  const prodDefaultPath = join(process.cwd(), './dist/db/migrations');
+  const devDefaultPath = join(process.cwd(), './src/db/migrations');
+  const path = [
+    process.env.DB_MIGRATIONS_DIR,
+    process.env.NODE_ENV === 'prod' ? prodDefaultPath : devDefaultPath
+  ].filter((path) => path && existsSync(path) && lstatSync(path).isDirectory());
+  reportDebug({
+    namespace,
+    message: 'Obtaining Umzug migration options',
+    data: {
+      path,
+      prodPath: prodDefaultPath,
+      devPath: devDefaultPath,
+      customPath: process.env.DB_MIGRATIONS_DIR
+    }
+  });
+  return path.map((p) =>
+    process.env.NODE_ENV === 'prod' ? `${p}/*.js` : `${p}/*.ts`
+  );
+}
+
+/** Run all the non-executed migrations one by one */
+export async function runPendingMigrations(): Promise<void> {
+  if (!sequelize)
+    throw new EngineError({ message: 'Sequelize is not initialized' });
+  // First make sure that the metadata table is up to date
+  await migrateMetadataTable(sequelize);
+  const migrationPaths = getMigrationPaths();
+
+  if (!migrationPaths || migrationPaths.length <= 0) {
+    reportDebug({
+      namespace,
+      message: 'No valid migration paths found.',
+      data: { migrationPaths }
+    });
+    return;
+  }
+
+  migrationPaths?.forEach(async (path) => {
+    const migrator = new Umzug({
+      migrations: {
+        glob: path,
+        resolve: ({ name, path, context }) => {
+          // Use require to import the migration file
+          const migration = require(path || '');
+          return {
+            name,
+            up: async () => migration.up(context),
+            down: async () => migration.down(context)
+          };
+        }
+      },
+      context: sequelize.getQueryInterface(),
+      storage: new SequelizeStorage({ sequelize }),
+      logger: console
+    });
+
+    reportDebug({
+      namespace,
+      message: 'Starting pending database migrations',
+      data: { path }
+    });
+    await migrator.up();
+    console.log('Db Migration Completed')
+  });
+}
+
+/** Rollback all migrations up to a specified target version */
+export async function rollbackMigrations(): Promise<void> {
+  if (!sequelize)
+    throw new EngineError({ message: 'Sequelize is not initialized' });
+  if (!process.env.BUILD_ID)
+    throw new EngineError({ message: 'BUILD_ID is not defined' });
+
+  // Fetch executed migrations in ascending order
+  const migrations = await SequelizeMeta.findAll({ order: [['name', 'ASC']] });
+  const rollbackUntil = migrations.find(
+    (migration) =>
+      migration.version &&
+      semverCompare(migration.version, process.env.BUILD_ID as string) === 1
+  );
+
+  if (!rollbackUntil) {
+    reportDebug({
+      namespace,
+      message: 'No rollback target found for the current BUILD_ID.',
+      data: { buildId: process.env.BUILD_ID }
+    });
+    return;
+  }
+
+  const migrationPaths = getMigrationPaths();
+
+  if (!migrationPaths || migrationPaths.length <= 0) {
+    reportDebug({
+      namespace,
+      message: 'No valid migration paths found for rollback.',
+      data: { migrationPaths }
+    });
+    return;
+  }
+
+  migrationPaths.forEach(async (path) => {
+    const migrator = new Umzug({
+      migrations: {
+        glob: path,
+        resolve: ({ name, path, context }) => {
+          const migration = require(path || '');
+          return {
+            name,
+            up: async () => migration.up(context),
+            down: async () => migration.down(context)
+          };
+        }
+      },
+      context: sequelize.getQueryInterface(),
+      storage: new SequelizeStorage({ sequelize }),
+      logger: console
+    });
+
+    reportDebug({
+      namespace,
+      message: `Rolling back migrations up to: ${rollbackUntil.name}`,
+      data: { path }
+    });
+
+    await migrator.down({ to: rollbackUntil.name });
+  });
+}
+
+/**
+ * Recreate the whole database
+ * Quick way to make sure that we are working on a clean database
+ * /!\ Should only be used when running tests
+ */
+export async function testRecreateDB(
+  options: RecreateDbOptions = {}
+): Promise<void> {
+  if (!sequelize)
+    throw new EngineError({ message: 'Sequelize is not initialized' });
+  await sequelize.query(
+    `DROP SCHEMA public${process.env.SQL_TYPE === 'postgres' ? ' CASCADE' : ''}`
+  );
+  await sequelize.query(`CREATE SCHEMA public`);
+  if (!options.clean) await runPendingMigrations();
+}
+
+/**
+ * Empty all the schemas in the database
+ * Quick way to flush the database data while keep the structure
+ * /!\ Should only be used when running tests
+ */
+export async function testClearDB(options: ClearDbOptions = {}): Promise<void> {
+  if (!sequelize)
+    throw new EngineError({ message: 'Sequelize is not initialized' });
+  await sequelize.transaction(clearDbTransaction(sequelize, options));
+}
+
+/** DB clearing transaction */
+function clearDbTransaction(sequelize: Sequelize, options: ClearDbOptions) {
+  return async (t: Transaction): Promise<void> => {
+    for (const model of Object.keys(sequelize.models)) {
+      // We ignore the migration metadata schema
+      if (!options.includeMetadata && model === 'sequelizeMeta') continue;
+      const table = sequelize.models[model].tableName;
+      await sequelize.query(
+        `TRUNCATE ${table}${process.env.SQL_TYPE === 'postgres' ? ' CASCADE' : ''};`,
+        {
+          raw: true,
+          transaction: t
+        }
+      );
+    }
+  };
+}

package/src/utils/dbMigration/dbMigration.types.ts

@@ -0,0 +1,11 @@
+/** Options to recreate the database */
+export interface RecreateDbOptions {
+  /** Do not run the migrations, leave the DB empty */
+  clean?: boolean;
+}
+
+/** Options to clear the database */
+export interface ClearDbOptions {
+  /** Also truncate the migration metadata table. This should only be used when testing on the metadata table itself */
+  includeMetadata?: boolean;
+}

package/src/utils/dbMigration/index.ts

@@ -0,0 +1,2 @@
+export * from './dbMigration';
+export * from './dbMigration.types';

package/src/utils/dbMigration/migrateMetadataTable/index.ts

@@ -0,0 +1 @@
+export * from './migrateMetadataTable';

package/src/utils/dbMigration/migrateMetadataTable/migrateMetadataTable.test.ts

@@ -0,0 +1,59 @@
+// import {expect} from 'chai';
+// import { faker } from '@faker-js/faker';
+// import {QueryTypes} from 'sequelize';
+// import {testRecreateDB} from '../dbMigration';
+// import {migrateMetadataTable} from './migrateMetadataTable';
+// import {sequelize} from 'entities/Sequelize';
+
+// describe('Utils - DB Migration - Migrate Metadata Table', function () {
+//   before(async () => {
+//     sequelize.init();
+//     await testRecreateDB({clean: true});
+//   });
+
+//   afterEach(async () => {
+//     await testRecreateDB({clean: true});
+//   });
+
+//   // it('should create the metadata table if it does not exist', async () => {
+//   //   await migrateMetadataTable(sequelize);
+
+//   //   const tables = await sequelize.query<[string]>(
+//   //     "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'",
+//   //     {type: QueryTypes.SELECT},
+//   //   );
+//   //   const hasTable = tables.some((table) => table[0] === 'sequelize_meta');
+//   //   expect(hasTable).to.be.true;
+//   // });
+
+//   it('should copy legacy table data to the new table', async () => {
+//     await sequelize.query('CREATE TABLE SequelizeMeta (name varchar(100) PRIMARY KEY)', {
+//       type: QueryTypes.RAW,
+//     });
+
+//     const migrations = [faker.lorem.word(), faker.lorem.word(), faker.lorem.word()];
+//     await sequelize.query(
+//       `INSERT INTO "SequelizeMeta" ("name") VALUES ('${migrations[0]}'), ('${migrations[1]}'), ('${migrations[2]}')`,
+//       {
+//         type: QueryTypes.RAW,
+//       },
+//     );
+
+//     await migrateMetadataTable(sequelize);
+
+//     const results = await sequelize.query<{
+//       /** Name of the migration */
+//       name: string;
+//       /** Version at which the migration was executed */
+//       version: string;
+//     }>('SELECT * FROM sequelize_meta', {
+//       type: QueryTypes.SELECT,
+//     });
+
+//     expect(results).to.have.length(3);
+//     results.forEach((result) => {
+//       expect(migrations).to.include(result.name);
+//       expect(result.version).to.be.null;
+//     });
+//   });
+// });

package/src/utils/dbMigration/migrateMetadataTable/migrateMetadataTable.ts

@@ -0,0 +1,70 @@
+import { Sequelize, QueryTypes } from 'sequelize';
+import { SequelizeMeta } from 'db/models';
+import { reportDebug } from 'utils/report';
+
+const namespace = 'engine:utils:dbMigration';
+
+/**
+ * Run migrations on the metadata table
+ * The table containing the migrations metadata can not be handled through migrations
+ * Therefore we use a more classic approach with this function and pure SQL queries
+ */
+export async function migrateMetadataTable(
+  sequelize: Sequelize
+): Promise<void> {
+  reportDebug({ namespace, message: 'Executing metadata migration routine' });
+
+  const query = `SELECT table_name FROM information_schema.tables WHERE table_schema = '${process.env.SQL_DB}'`;
+  // We check if we have a custom sequelize_meta table yet
+  const tables = await sequelize.query<Array<string>>(query, {
+    type: QueryTypes.SELECT
+  });
+
+  const hasCustomSequelizeMeta =
+    process.env.SQL_TYPE === 'mysql'
+      ? tables.some((table) => Object.values(table)[0] === 'sequelize_meta')
+      : tables.some((table) => table[0] === 'sequelize_meta');
+  const hasLegacySequelizeMeta =
+    process.env.SQL_TYPE === 'mysql'
+      ? tables.some((table) => Object.values(table)[0] === 'SequelizeMeta')
+      : tables.some((table) => table[0] === 'SequelizeMeta');
+
+  if (!hasCustomSequelizeMeta) {
+    reportDebug({
+      namespace,
+      message: 'No metadata table found, creating new one'
+    });
+    // Create the new sequelize_meta table if it does not exist
+    await sequelize.query(
+      `
+      CREATE TABLE sequelize_meta (
+        name      varchar(100) PRIMARY KEY,
+        version   varchar(25),
+        createdAt timestamp  NOT NULL,
+        updatedAt timestamp NOT NULL
+      )`,
+      { type: QueryTypes.RAW }
+    );
+
+    // Migrate the data from the old table to the new one if needed
+    if (hasLegacySequelizeMeta) {
+      reportDebug({
+        namespace,
+        message: 'Legacy metadata table found, migrating data'
+      });
+
+      const migrations = await sequelize.query<SequelizeMeta>(
+        'SELECT name FROM SequelizeMeta ORDER BY name',
+        {
+          type: QueryTypes.SELECT
+        }
+      );
+      const insertQuery = `
+      INSERT INTO sequelize_meta (name, version, createdAt, updatedAt) VALUES
+        ${migrations.map((migration) => `('${migration.name}', NULL, NOW(), NOW())`).join(',')}
+      `;
+
+      await sequelize.query(insertQuery, { type: QueryTypes.INSERT });
+    }
+  }
+}

package/src/utils/envAssert/envAssert.ts

@@ -0,0 +1,31 @@
+import { bool, num, host, email, port, url, json } from 'envalid';
+
+import {
+  envIsIpList,
+  envIsPath,
+  envIsHostList,
+  envIsStringList,
+  envIsString
+} from './validators';
+
+/**
+ * envalid validators to use with checkEnvironment made with evalid.makeValidator and validator.js
+ *  is* functions. + Also Added isString, isHost, and isIPList.
+ *
+ *
+ * envalid docs: https://github.com/af/envalid
+ */
+export const envAssert = {
+  isString: envIsString,
+  isBoolean: bool,
+  isNumber: num,
+  isEmail: email,
+  isHost: host,
+  isPort: port,
+  isURL: url,
+  isJSON: json,
+  isIPList: envIsIpList,
+  isPath: envIsPath,
+  isHostList: envIsHostList,
+  isStringList: envIsStringList
+};

package/src/utils/envAssert/index.ts

@@ -0,0 +1 @@
+export * from './envAssert';

package/src/utils/envAssert/validators/hostList.ts

@@ -0,0 +1,17 @@
+import { makeValidator } from 'envalid';
+import validator from 'validator';
+
+/**
+ * Validate that a string is a list of comma separated host names
+ */
+export const envIsHostList = makeValidator((value): boolean => {
+  if (!value) throw new Error('expected a host name');
+  const hosts = value.split(',');
+  const valid = hosts.every(
+    (host) =>
+      validator.isIP(host.trim()) ||
+      validator.isFQDN(host.trim(), { require_tld: false })
+  );
+  if (!valid) throw new Error('expected a host name');
+  return valid;
+});

package/src/utils/envAssert/validators/index.ts

@@ -0,0 +1,5 @@
+export * from './ipList';
+export * from './path';
+export * from './hostList';
+export * from './stringList';
+export * from './string';

package/src/utils/envAssert/validators/ipList.ts

@@ -0,0 +1,13 @@
+import { makeValidator } from 'envalid';
+import validator from 'validator';
+
+/**
+ * Validate that a string is a list of comma separated ip (V4/V6) addresses
+ */
+export const envIsIpList = makeValidator((value): boolean => {
+  if (!value) throw new Error('expected an ip address');
+  const ips = value.split(',');
+  const valid = ips.every((v) => validator.isIP(v));
+  if (!valid) throw new Error('expected an ip address');
+  return valid;
+});

package/src/utils/envAssert/validators/path.ts

@@ -0,0 +1,13 @@
+import { makeValidator } from 'envalid';
+
+const pathRegex = /^(\.\.\/(?:\.\.\/)*)?(?!.*?\/\/)(?!(?:.*\/)?\.+(?:\/|$)).+$/;
+
+/**
+ * Validate that a string is a system path (linux only)
+ */
+export const envIsPath = makeValidator((value): boolean => {
+  if (!value) throw new Error('expected a path');
+  const valid = pathRegex.test(value);
+  if (!valid) throw new Error('expected a path');
+  return valid;
+});

package/src/utils/envAssert/validators/string.ts

@@ -0,0 +1,9 @@
+import { makeValidator } from 'envalid';
+
+/**
+ * Validate that aa value is a string
+ */
+export const envIsString = makeValidator((value): boolean => {
+  if (!value || typeof value !== 'string') throw new Error('expected a string');
+  return value.length > 0;
+});

package/src/utils/envAssert/validators/stringList.ts

@@ -0,0 +1,13 @@
+import { makeValidator } from 'envalid';
+
+/**
+ * Validate that a string is a list of comma separated strings
+
+ */
+export const envIsStringList = makeValidator((value): boolean => {
+  if (!value) throw new Error('expected a string');
+  const strings = value.split(',');
+  const valid = strings.every((v) => typeof v === 'string');
+  if (!valid) throw new Error('expected a string');
+  return valid;
+});