@autofleet/sadot 1.0.0-beta.0 → 1.0.0

package/src/hooks/update.ts

@@ -1,7 +1,7 @@
 import logger from '../utils/logger';
-import * as ValueRepo from '../repository/value';
-import type { ModelOptions } from '../types';
+import type { CustomFieldOptions, ModelOptions } from '../types';
 import applyScopeToInstance from '../utils/scopeAttributes';
+import updateInstanceValues from './utils/updateInstanceValues';
 
 /**
  * A hook to update the custom fields when updating a model (more then one instance).
@@ -16,7 +16,11 @@ export const beforeBulkUpdate = (options): void => {
  * A hook to update the custom fields when updating a model instance.
  * TODO - cleanup if update fail
  */
-export const beforeUpdate = (scopeAttributes: string[], modelOptions: ModelOptions = {}) => async (
+export const beforeUpdate = (
+  scopeAttributes: string[],
+  modelOptions: ModelOptions = {},
+  sadotOptions: Pick<CustomFieldOptions, 'useCustomFieldsEntries'> = { useCustomFieldsEntries: false },
+) => async (
   instance,
   options,
 ): Promise<void> => {
@@ -28,14 +32,23 @@ export const beforeUpdate = (scopeAttributes: string[], modelOptions: ModelOptio
   const customFieldsIdx = fields.indexOf('customFields');
   if (customFieldsIdx > -1) {
     const { customFields } = instance;
-    await ValueRepo.updateValues(
+
+    if (!Object.keys(customFields).length) {
+      return;
+    }
+
+    await updateInstanceValues({
+      modelId: instance.id,
       modelType,
-      instance.id,
       identifiers,
       customFields,
-      { transaction: options.transaction, modelOptions },
+      options: {
+        useCustomFieldsEntries: sadotOptions.useCustomFieldsEntries,
+        transaction: options.transaction,
+        modelOptions,
+      },
+    });
 
-    );
     // eslint-disable-next-line no-param-reassign
     fields.splice(customFieldsIdx, 1);
   }

package/src/hooks/utils/updateInstanceValues.ts

@@ -0,0 +1,63 @@
+import type { Transaction } from 'sequelize';
+import * as ValueRepo from '../../repository/value';
+import * as EntriesRepo from '../../repository/entries';
+import type { ModelOptions } from '../../types';
+
+interface UpdateInstanceValuesParams {
+  modelId: string;
+  modelType: string;
+  identifiers: string[];
+  customFields: Record<string, any>;
+  options?: {
+    transaction?: Transaction;
+    modelOptions?: ModelOptions;
+    useCustomFieldsEntries?: boolean;
+  };
+}
+
+const updateInstanceValues = async ({
+  modelId,
+  modelType,
+  identifiers,
+  customFields,
+  options = {
+    modelOptions: {},
+    useCustomFieldsEntries: false,
+  },
+}: UpdateInstanceValuesParams): Promise<void> => {
+  await ValueRepo.updateValues(
+    modelType,
+    modelId,
+    identifiers,
+    customFields,
+    {
+      ...options,
+      modelOptions: options.modelOptions ?? {},
+    },
+  );
+
+  /*
+  T.Y TODO - Once we're ready to switch from custom_field_values to custom_field_entries, we should remove the ValueRepo.updateValues call.
+  Currently, We're updating both tables to keep the data in sync, but not all microservices are using the new table yet.
+  */
+
+  if (!options?.useCustomFieldsEntries) {
+    return;
+  }
+
+  const { dataValues: { customFields: oldCustomFields } } = await EntriesRepo.findEntriesByModelId(modelId, options) ?? { dataValues: {} };
+  const newCustomFields = { ...oldCustomFields, ...customFields };
+
+  await EntriesRepo.updateEntries(
+    modelId,
+    modelType,
+    newCustomFields,
+    identifiers,
+    {
+      ...options,
+      modelOptions: options.modelOptions ?? {},
+    },
+  );
+};
+
+export default updateInstanceValues;

package/src/index.ts

@@ -5,8 +5,8 @@ import {
 } from './models';
 import api from './api';
 import initDB from './utils/db';
-import logger from './utils/logger';
-import type { CustomFieldOptions, ModelFetcher } from './types';
+import logger, { tryAddingTraceIdMiddleware } from './utils/logger';
+import type { CustomFieldOptions, ModelFetcher, Models } from './types';
 import {
   addHooks, addScopes, applyCustomAssociation, removeHooks,
 } from './utils/init';
@@ -26,25 +26,27 @@ const useCustomFields = async (
   getModel: ModelFetcher,
   options: CustomFieldOptions,
 ): Promise<Sequelize> => {
-  const { models } = options;
+  tryAddingTraceIdMiddleware().catch(() => null);
+  const { models, useCustomFieldsEntries } = options;
   if (app) {
     app.use('/api', api);
   }
-  const sequelize = initDB(options.databaseConfig);
+  const sequelize = options.sequelize ?? initDB(options.databaseConfig);
   if (process.env.NODE_ENV === 'test') {
     await initTestModels(sequelize);
   }
   // The order is important
-  addHooks(models, getModel);
-  await initTables(sequelize, options.getUser);
-  addScopes(models, getModel);
+  addHooks(models, getModel, { useCustomFieldsEntries });
+  await initTables(sequelize, options.getUser, { useCustomFieldsEntries });
+  addScopes(models, getModel, { useCustomFieldsEntries });
   applyCustomAssociation(models);
+
   logger.debug('sadot - custom fields finished initializing with models', models);
   return sequelize;
 };
 
 export default useCustomFields;
 
-export const disableCustomFields = (models, getModel): void => {
+export const disableCustomFields = (models: Models[], getModel: ModelFetcher): void => {
   removeHooks(models, getModel);
 };

package/src/models/CustomFieldDefinition.ts

@@ -125,6 +125,13 @@ class CustomFieldDefinition extends Model {
   @Column
     deletedAt?: Date;
 
+    @Column({
+      type: DataType.BOOLEAN,
+      defaultValue: false,
+      allowNull: false,
+    })
+      blockEditingFromUI!: boolean;
+
   @HasMany(() => CustomFieldValue)
     values: CustomFieldValue[];
 
@@ -136,8 +143,8 @@ class CustomFieldDefinition extends Model {
     }
     if (![null, undefined].includes(instance.defaultValue)) {
       const isValid = validateValue(instance.defaultValue, instance.fieldType, instance.validation);
-      if (!isValid) {
-        throw new InvalidValueError(instance.defaultValue, instance.fieldType);
+      if (isValid.error) {
+        throw new InvalidValueError(instance.defaultValue, instance.name, isValid.error);
       }
     }
   }

package/src/models/CustomFieldEntries.ts

@@ -0,0 +1,81 @@
+import {
+  Table,
+  Column,
+  Model,
+  PrimaryKey,
+  DataType,
+  AfterUpsert,
+} from 'sequelize-typescript';
+import { sendDimEvent } from '../events';
+import * as CustomFieldDefinitionRepo from '../repository/definition';
+import { validateInstanceCustomFieldEntries } from '../utils/validations';
+
+@Table({
+  timestamps: true,
+  indexes: [
+    {
+      name: 'idx_cfe_custom_fields',
+      using: 'gin',
+      operator: 'jsonb_path_ops',
+      fields: ['custom_fields'],
+    },
+  ],
+  validate: {
+    async validationByType(this: CustomFieldEntries) {
+      if (!Object.keys(this.customFields ?? {}).length) {
+        return;
+      }
+
+      const definitionsByName = await CustomFieldDefinitionRepo.getCustomFieldDefinitionsDictionary([this]);
+      validateInstanceCustomFieldEntries(this, definitionsByName);
+    },
+  },
+})
+class CustomFieldEntries extends Model {
+  @PrimaryKey
+  @Column({
+    type: DataType.UUID,
+    allowNull: false,
+  })
+    /** The ID of the model of which this row hold the custom field entries of, e.g. vehicleId / stopPointId / etc.  */
+    modelId!: string;
+
+  @Column({
+    type: DataType.UUID,
+    allowNull: false,
+  })
+    /** The ID of the entity of which this row hold the custom field entries of, e.g. fleetId / etc.  */
+    entityId!: string;
+
+  @Column({
+    type: DataType.JSONB,
+    allowNull: false,
+    defaultValue: {},
+  })
+    /** A dictionary of customFields and values with the following structure: `{ customFieldName: 'CustomFieldValue' }` */
+    customFields!: Record<string, any>;
+
+  @Column({
+    type: DataType.STRING,
+    allowNull: false,
+  })
+    /** The type of model which this custom field entry represents. e.g. Vehicle / StopPoint / etc. */
+    modelType!: string;
+
+  @Column
+    createdAt?: Date;
+
+  @Column
+    updatedAt?: Date;
+
+  @AfterUpsert
+  static afterSaveHandler(instance: CustomFieldEntries, options): void {
+    if (options.transaction) {
+      options.transaction.afterCommit(() => sendDimEvent(instance[0]));
+    } else {
+      sendDimEvent(instance[0]);
+    }
+  }
+}
+
+export default CustomFieldEntries;

package/src/models/CustomFieldValue.ts

@@ -15,9 +15,9 @@ import {
 } from 'sequelize-typescript';
 import { sendDimEvent } from '../events';
 import { CustomFieldDefinition } from '.';
-import { validateValue } from '../utils/validations';
+import { validateFieldType, validateValue } from '../utils/validations';
 import * as CustomFieldDefinitionRepo from '../repository/definition';
-import { InvalidValueError } from '../errors';
+import { InvalidFieldTypeError, InvalidValueError } from '../errors';
 
 @Table({
   timestamps: true,
@@ -56,9 +56,26 @@ class CustomFieldValue extends Model {
   @BelongsTo(() => CustomFieldDefinition, { scope: { disabled: false } })
     customFieldDefinition: CustomFieldDefinition;
 
+  private static validateValueAgainstDefinition(
+    instance: CustomFieldValue,
+    definition: CustomFieldDefinition,
+  ): void {
+    const { validation, fieldType, name } = definition;
+    const isValidFieldType = validateFieldType(fieldType);
+    if (!isValidFieldType) {
+      throw new InvalidFieldTypeError(fieldType);
+    }
+    // Always allow null values
+    if (instance.value === null) return;
+    const validateValueResponse = validateValue(instance.value, fieldType, validation);
+    if (validateValueResponse.error) {
+      throw new InvalidValueError(instance.value, name, validateValueResponse.error);
+    }
+  }
+
   @BeforeBulkCreate
   @BeforeBulkUpdate
-  static async validateValues(instances: CustomFieldValue[]): Promise<void> {
+  static async validateCustomFieldValues(instances: CustomFieldValue[]): Promise<void> {
     const ids = instances.map((instance) => instance.customFieldDefinitionId);
     const uniqueIds = [...new Set(ids)];
     const definitions = await CustomFieldDefinitionRepo.findByIds(
@@ -71,14 +88,9 @@ class CustomFieldValue extends Model {
     }
 
     instances.forEach((instance) => {
-      const {
-        validation,
-        fieldType,
-      } = definitions
-        .find((definition) => definition.id === instance.customFieldDefinitionId);
-      const isValid = validateValue(instance.value, fieldType, validation);
-      if (!isValid) {
-        throw new InvalidValueError(instance.value, fieldType);
+      const definition = definitions.find((d) => d.id === instance.customFieldDefinitionId);
+      if (definition) {
+        this.validateValueAgainstDefinition(instance, definition);
       }
     });
   }
@@ -86,15 +98,11 @@ class CustomFieldValue extends Model {
   @BeforeUpdate
   @BeforeCreate
   @BeforeUpsert
-  static async validateValue(instance: CustomFieldValue): Promise<void> {
+  static async validateCustomFieldValue(instance: CustomFieldValue): Promise<void> {
     const { customFieldDefinitionId } = instance;
     // eslint-disable-next-line max-len
     const cfd = await CustomFieldDefinitionRepo.findById(customFieldDefinitionId, { withDisabled: true });
-    const { validation, fieldType } = cfd;
-    const isValid = validateValue(instance.value, fieldType, validation);
-    if (!isValid) {
-      throw new InvalidValueError(instance.value, fieldType);
-    }
+    this.validateValueAgainstDefinition(instance, cfd);
   }
 
   @AfterUpsert

package/src/models/CustomValidator.ts

@@ -0,0 +1,78 @@
+import {
+  Table,
+  Column,
+  Model,
+  PrimaryKey,
+  DataType,
+  Default,
+  AfterUpsert,
+  DefaultScope,
+} from 'sequelize-typescript';
+import { randomUUID } from 'node:crypto';
+import { sendDimEvent } from '../events';
+
+@DefaultScope(() => ({ where: { disabled: false } }))
+@Table({
+  timestamps: true,
+})
+class CustomValidator extends Model {
+  @PrimaryKey
+  @Default(() => randomUUID())
+  @Column({
+    type: DataType.UUID,
+    allowNull: false,
+  })
+    id!: string;
+
+  @Column({
+    type: DataType.UUID,
+    allowNull: false,
+  })
+    /** The ID of the entity for which this validator is defined, e.g. fleetId / etc. */
+    entityId!: string;
+
+  @Column({
+    type: DataType.STRING,
+    allowNull: false,
+  })
+    /** The type of entity (fleet, businessModel, etc). */
+    entityType!: string;
+
+  @Column({
+    type: DataType.STRING,
+    allowNull: false,
+  })
+    /** The type of model this validator applies to. e.g. Vehicle / StopPoint / etc. */
+    modelType!: string;
+
+  @Column({
+    type: DataType.JSONB,
+    allowNull: false,
+  })
+    /** JSON Schema to validate models against */
+    schema!: Record<string, unknown>;
+
+  @Column({
+    type: DataType.BOOLEAN,
+    allowNull: false,
+    defaultValue: false,
+  })
+    disabled!: boolean;
+
+  @Column
+    createdAt?: Date;
+
+  @Column
+    updatedAt?: Date;
+
+  @AfterUpsert
+  static afterSaveHandler(instance: CustomValidator, options): void {
+    if (options.transaction) {
+      options.transaction.afterCommit(() => sendDimEvent(instance));
+    } else {
+      sendDimEvent(instance);
+    }
+  }
+}
+
+export default CustomValidator;

package/src/models/index.ts

@@ -1,5 +1,5 @@
 /* eslint-disable no-param-reassign */
-import { DataTypes } from 'sequelize';
+import { DataTypes, Op } from 'sequelize';
 import type { Sequelize } from 'sequelize-typescript';
 import logger from '../utils/logger';
 import CustomFieldDefinition from './CustomFieldDefinition';
@@ -8,37 +8,76 @@ import TestModel from './tests/TestModel';
 import ContextAwareTestModel from './tests/contextAwareModels/ContextAwareTestModel';
 import ContextTestModel from './tests/contextAwareModels/ContextTestModel';
 import AssociatedTestModel from './tests/AssociatedTestModel';
+import type { CustomFieldOptions } from '../types';
+import CustomFieldEntries from './CustomFieldEntries';
+import CustomValidator from './CustomValidator';
 
-const productionModels = [CustomFieldDefinition, CustomFieldValue];
+type ProductionModel = typeof CustomFieldDefinition | typeof CustomFieldValue | typeof CustomFieldEntries | typeof CustomValidator
+interface InitTablesOptions {
+  schemaPrefix?: string
+  schemaVersion?: string
+  useCustomFieldsEntries?: boolean
+}
+
+const productionModels: ProductionModel[] = [CustomFieldDefinition, CustomFieldValue, CustomValidator];
 const testModels = [TestModel, AssociatedTestModel, ContextAwareTestModel, ContextTestModel];
 
 const SADOT_MIGRATION_PREFIX = 'sadot-migration';
-const SCHEMA_VERSION = 2;
-const CUSTOM_FIELDS_SCHEMA_VERSION = `${SADOT_MIGRATION_PREFIX}_${SCHEMA_VERSION}`;
+const SCHEMA_VERSION = '49c9dd1d-b1cc-445b-a911-fd349d783110';
 
-const initTables = async (sequelize: Sequelize, getUser): Promise<void> => {
+const initTables = async (
+  sequelize: Sequelize,
+  getUser: CustomFieldOptions['getUser'],
+  {
+    schemaPrefix = SADOT_MIGRATION_PREFIX,
+    schemaVersion = SCHEMA_VERSION,
+    useCustomFieldsEntries = false,
+  }: InitTablesOptions = {},
+): Promise<void> => {
+  const CUSTOM_FIELDS_SCHEMA_VERSION = `${schemaPrefix}_${schemaVersion}${useCustomFieldsEntries ? '_withEntries' : ''}`;
   logger.info('custom-fields: initialize custom-fields tables');
   // Detect models and import them to the orm
   // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
   if (!sequelize.addModels) {
     throw new Error('sequelize instance must have addModels function');
   }
+
+  if (useCustomFieldsEntries) {
+    productionModels.push(CustomFieldEntries);
+  }
+
   sequelize.addModels(productionModels);
 
   CustomFieldDefinition.addScope('userScope', () => {
     const user = getUser();
-    if (user?.permissions) {
-      return {
-        where: {
-          entityId: [
-            ...Object.keys(user.permissions.fleets),
-            ...Object.keys(user.permissions.businessModels),
-            ...Object.keys(user.permissions.demandSources),
-          ],
-        },
-      };
+    if (!user?.permissions) {
+      return {};
     }
-    return {};
+    return {
+      where: {
+        entityId: [
+          ...Object.keys(user.permissions.fleets),
+          ...Object.keys(user.permissions.businessModels),
+          ...Object.keys(user.permissions.demandSources),
+        ],
+      },
+    };
+  });
+
+  CustomValidator.addScope('userScope', () => {
+    const user = getUser();
+    if (!user?.permissions) {
+      return {};
+    }
+    return {
+      where: {
+        entityId: [
+          ...Object.keys(user.permissions.fleets),
+          ...Object.keys(user.permissions.businessModels),
+          ...Object.keys(user.permissions.demandSources),
+        ],
+      },
+    };
   });
 
   logger.info('custom-fields: models added');
@@ -60,18 +99,35 @@ const initTables = async (sequelize: Sequelize, getUser): Promise<void> => {
       schema: 'public',
     },
   );
-  const migrations = await SequelizeMeta.findAll({ raw: true });
-  const currentSadotSchemaVersion = migrations
-    .reverse().find((m: any) => m.name.includes(SADOT_MIGRATION_PREFIX));
-
-  if (
-    !currentSadotSchemaVersion
-    || (currentSadotSchemaVersion as any).name !== CUSTOM_FIELDS_SCHEMA_VERSION
-  ) {
+
+  logger.info('custom-fields: starting migrations');
+  const migrations = await SequelizeMeta.findAll({ where: { name: { [Op.like]: `${schemaPrefix}%` } }, raw: true });
+  const currentSadotSchemaVersion = migrations.at(-1);
+  const expectedSchemaVersionIndex = migrations.findIndex((m) => (m as any).name === CUSTOM_FIELDS_SCHEMA_VERSION);
+
+  logger.info('custom-fields: migrations', { migrations, currentSadotSchemaVersion, expectedSchemaVersionIndex });
+  if (!currentSadotSchemaVersion || (currentSadotSchemaVersion as any).name !== CUSTOM_FIELDS_SCHEMA_VERSION) {
+    logger.info('custom-fields: syncing models');
     await CustomFieldDefinition.sync({ alter: true });
     await CustomFieldValue.sync({ alter: true });
-    await SequelizeMeta.create({ name: CUSTOM_FIELDS_SCHEMA_VERSION });
+    // T.Y TODO: Remove the if statement once we're ready to add the new entries table for all MS
+    if (useCustomFieldsEntries) {
+      await CustomFieldEntries.sync({ alter: true });
+    }
+
+    // Always sync CustomValidator
+    await CustomValidator.sync({ alter: true });
+
+    if (expectedSchemaVersionIndex === -1) {
+      await SequelizeMeta.create({ name: CUSTOM_FIELDS_SCHEMA_VERSION });
+    }
     logger.info('custom-fields: models synced');
+    if (migrations.length && expectedSchemaVersionIndex !== -1 && expectedSchemaVersionIndex < migrations.length - 1) {
+      // We have existing migrations, and we are calling `sync`.
+      // This means we are in a `down` migration, and hence we should delete newer migrations to ensure we can reapply them.
+      const migrationsToDelete = migrations.slice(expectedSchemaVersionIndex + 1);
+      await SequelizeMeta.destroy({ where: { name: { [Op.in]: migrationsToDelete.map((m) => (m as any).name) } } });
+    }
   }
 };
 
@@ -98,6 +154,8 @@ const initTestModels = async (sequelize: Sequelize): Promise<void> => {
 export {
   CustomFieldValue,
   CustomFieldDefinition,
+  CustomFieldEntries,
+  CustomValidator,
   TestModel,
   AssociatedTestModel,
   ContextAwareTestModel,