@autofleet/sadot 0.0.1-beta

package/src/api/v1/definition/validations.ts

@@ -0,0 +1,35 @@
+import Joi from '@hapi/joi';
+import { CustomFieldDefinitionType } from '../../../utils/validations/type';
+
+const ValidationSchema = Joi.when('fieldType', {
+  is: CustomFieldDefinitionType.SELECT,
+  then: Joi.array().items(Joi.string()).min(1).unique(),
+  otherwise: Joi.any(),
+});
+
+const CustomFieldDefinitionCreationSchema = Joi.object({
+  name: Joi.string().required(),
+  displayName: Joi.string().required(),
+  validation: ValidationSchema,
+  fieldType: Joi.string().valid(...Object.values(CustomFieldDefinitionType)).required(),
+  entityId: Joi.string().guid().required(),
+  entityType: Joi.string().required(),
+  description: Joi.string(),
+  required: Joi.boolean(),
+  disabled: Joi.boolean(),
+});
+
+const CustomFieldDefinitionUpdateSchema = Joi.object({
+  displayName: Joi.string(),
+  validation: ValidationSchema,
+  fieldType: Joi.string().valid(...Object.values(CustomFieldDefinitionType)),
+  description: Joi.string(),
+  required: Joi.boolean(),
+  disabled: Joi.boolean(),
+});
+
+export const validateCustomFieldDefinitionCreation = (payload) =>
+  CustomFieldDefinitionCreationSchema.validateAsync(payload, { abortEarly: false });
+
+export const validateCustomFieldDefinitionUpdate = (payload) =>
+  CustomFieldDefinitionUpdateSchema.validateAsync(payload, { abortEarly: false });

package/src/api/v1/errors.ts

@@ -0,0 +1,16 @@
+import { handleError, BadRequest } from '@autofleet/errors';
+import { ValidationError as InputValidationError } from '@hapi/joi';
+import { ValidationError as DatabaseValidationError } from 'sequelize';
+
+export default (err, res, additionalData = {}) => {
+  let error = err;
+  if (err instanceof InputValidationError) {
+    error = new BadRequest([err], null);
+  }
+
+  if (err instanceof DatabaseValidationError) {
+    error = new BadRequest([err], null);
+  }
+
+  return handleError(error, res, additionalData);
+};

package/src/api/v1/index.ts

@@ -0,0 +1,9 @@
+import { Router } from 'express';
+
+import definitionRouter from './definition';
+
+const router = Router({ mergeParams: true });
+
+router.use('/custom-field-definitions/:modelName', definitionRouter);
+
+export default router;

package/src/errors/index.ts

@@ -0,0 +1,42 @@
+/* eslint-disable max-classes-per-file */
+import { BadRequest } from '@autofleet/errors';
+
+export class MissingRequiredCustomFieldError extends BadRequest {
+  constructor(missingFields: string[]) {
+    const err = new Error(`The following custom fields are required: ${missingFields.join(',')}`);
+    super([err], null, missingFields);
+    this.message = 'MISSING_REQUIRED_CUSTOM_FIELDS';
+  }
+}
+
+export class UnsupportedCustomFieldTypeError extends BadRequest {
+  constructor(fieldType: string) {
+    const err = new Error(`Type "${fieldType}" is not supported`);
+    super([err], null, null);
+    this.message = 'UNSUPPORTED_CUSTOM_FIELD_TYPE';
+  }
+}
+
+export class UnsupportedCustomValidationError extends BadRequest {
+  constructor(fieldType: string) {
+    const err = new Error(`Validation for "${fieldType}" is not supported`);
+    super([err], null, null);
+    this.message = 'UNSUPPORTED_CUSTOM_VALIDATION_TYPE';
+  }
+}
+
+export class InvalidValueError extends BadRequest {
+  constructor(value: any, fieldType: string) {
+    const err = new Error(`Invalid "${fieldType}" value ${JSON.stringify(value)}`);
+    super([err], null, null);
+    this.message = 'INVALID_VALUE';
+  }
+}
+
+export class MissingDefinitionError extends BadRequest {
+  constructor(fieldNames: string[]) {
+    const err = new Error(`Missing custom field definition for field ${fieldNames.join(',')}`);
+    super([err], null, null);
+    this.message = 'MISSING_DEFINITION';
+  }
+}

package/src/events/index.ts

@@ -0,0 +1,50 @@
+import Events from '@autofleet/events';
+import logger from '../utils/logger';
+
+const events = new Events({ logger });
+
+const KEYS_TO_CONVERT = ['value'];
+
+const stringifyBools = (savedObject: any, keysToConvert: Array<string>) => {
+  if (Object.keys(savedObject).some((key) => keysToConvert.includes(key))) {
+    const objectToReturn = { ...savedObject };
+    keysToConvert.forEach((key) => {
+      if (typeof savedObject[key] === 'boolean') {
+        objectToReturn[key] = savedObject[key].toString();
+      }
+    });
+    return objectToReturn;
+  }
+  return savedObject;
+};
+
+const modelTableMapping = {
+  CustomFieldDefinition: {
+    tableName: 'dim_custom_field_definition',
+    eventVersion: '1',
+  },
+  CustomFieldValue: {
+    tableName: 'dim_custom_field_value',
+    eventVersion: '1',
+  },
+};
+
+export const sendDimEvent = (instance): void => {
+  const mapping = modelTableMapping[instance.constructor.name];
+  if (mapping) {
+    let objectToSend = instance.get();
+    try {
+      objectToSend = stringifyBools(instance.get(), KEYS_TO_CONVERT);
+    } catch (err) {
+      logger.error('Failed to convert booleans in dim event payload', err);
+    }
+
+    events.sendObject(
+      mapping.tableName,
+      mapping.eventVersion,
+      objectToSend,
+    );
+  }
+};
+
+export default events;

package/src/hooks/create.ts

@@ -0,0 +1,51 @@
+import logger from '../utils/logger';
+import * as ValueRepo from '../repository/value';
+import * as DefinitionRepo from '../repository/definition';
+import { MissingRequiredCustomFieldError } from '../errors';
+
+/**
+ * A hook to create the custom fields when updating a model (more then one instance).
+ */
+export const beforeBulkCreate = (options): void => {
+  // This will activate the beforeCreate hook on each updating instance.
+  // eslint-disable-next-line no-param-reassign
+  options.individualHooks = true;
+};
+/**
+ * A hook to create the custom fields when updating a model instance.
+ * TODO - cleanup if update fail
+ */
+export const beforeCreate = (scopeAttributes: string[]) => async (
+  instance,
+  options,
+): Promise<void> => {
+  logger.debug('sadot - before create hook');
+  const { fields } = options;
+  const modelType = instance.constructor.name;
+  const identifiers = scopeAttributes.map((attribute) => instance[attribute]);
+  // get all model's required definitions
+  const requiredFieldsNames = await DefinitionRepo.getRequiredFields(modelType,
+    instance.id,
+    identifiers);
+  const customFieldsIdx = fields.indexOf('customFields');
+  const { customFields } = instance;
+  if (customFieldsIdx > -1 && customFields) {
+    const fieldsNames = Object.keys(customFields);
+    const missingFields = requiredFieldsNames.filter((name) => !fieldsNames.includes(name));
+    if (missingFields?.length > 0) {
+      throw new MissingRequiredCustomFieldError(missingFields);
+    }
+
+    await ValueRepo.updateValues(
+      modelType,
+      instance.id,
+      identifiers,
+      customFields,
+      { transaction: options.transaction },
+    );
+    // eslint-disable-next-line no-param-reassign
+    fields.splice(customFieldsIdx, 1);
+  } else if (requiredFieldsNames?.length > 0) {
+    throw new MissingRequiredCustomFieldError(requiredFieldsNames);
+  }
+};

package/src/hooks/enrich.ts

@@ -0,0 +1,125 @@
+/* eslint-disable no-param-reassign */
+import * as ValueRepo from '../repository/value';
+import * as DefinitionRepo from '../repository/definition';
+import CustomFieldValue from '../models/CustomFieldValue';
+import CustomFieldDefinition from '../models/CustomFieldDefinition';
+import { SerializedCustomFields } from '../types/definition';
+import logger from '../utils/logger';
+
+/**
+ * Serialize custom fields value into the format of {[name] -> [fieldData]}
+ */
+const serializeCustomFields = (
+  customFieldValues: CustomFieldValue[],
+  customFieldDefinitionsHash: Record<string, CustomFieldDefinition>,
+): SerializedCustomFields => {
+  const customFields = customFieldValues.reduce((acc, cfv) => ({
+    ...acc,
+    ...(
+      customFieldDefinitionsHash[cfv.customFieldDefinitionId]
+      && { [customFieldDefinitionsHash[cfv.customFieldDefinitionId].name]: cfv.value }
+    ),
+  }), {});
+  return customFields;
+};
+/**
+ * A hook to attach the custom fields when fetching a model instances.
+ */
+const enrichResults = (modelType: string, scopeAttributes: string[]) => async (
+  instancesOrInstance: any | any[],
+  options,
+): Promise<void> => {
+  if (
+    options.originalAttributes?.length > 0
+      && !options.originalAttributes?.includes?.('customFields')
+  ) {
+    return;
+  }
+
+  const primaryKey = 'id';
+  let instances = Array.isArray(instancesOrInstance)
+    ? instancesOrInstance
+    : [instancesOrInstance];
+
+  instances = instances.filter(Boolean);
+
+  const identifiers = instances.map((instance) =>
+    scopeAttributes
+      .map((attr) => instance[attr])).flat();
+
+  const uniqueIdentifiers = [...new Set(identifiers)].filter(Boolean);
+  const identifierCustomFieldDefinitionsMapping = uniqueIdentifiers.reduce((map, identifier) => ({
+    ...map,
+    [identifier]: [],
+  }), {});
+  const customFieldDefinitions = await DefinitionRepo.findByEntityIds(
+    modelType,
+    uniqueIdentifiers,
+    { transaction: options.transaction },
+  );
+
+  const definitionsMap = customFieldDefinitions.reduce((map, definition) => ({
+    ...map,
+    [definition.id]: definition,
+  }), {});
+
+  customFieldDefinitions.forEach((cfd) => {
+    identifierCustomFieldDefinitionsMapping[cfd.entityId].push(cfd);
+  });
+
+  // Get the values per instates ids:
+  const instancesIds = instances.map((i) => i[primaryKey]);
+
+  const customFieldValues = await ValueRepo.findValuesByModelIds(
+    instancesIds,
+    { transaction: options.transaction },
+  );
+
+  // Group fields by modelId
+  const valuesGroupByInstance: {
+      [modelId: string]: CustomFieldValue[];
+    } = customFieldValues.reduce((acc, v) => {
+      const { modelId } = v;
+      if (!acc[modelId]) {
+        acc[modelId] = [];
+      }
+      acc[modelId].push(v);
+      return acc;
+    }, {});
+
+  // Attach custom fields to the instances
+  instances.forEach((instance) => {
+    const customFields = {};
+    const { id } = instance;
+    const instanceValues = valuesGroupByInstance[id];
+    if (instanceValues) {
+      const serializedCustomFields = serializeCustomFields(instanceValues, definitionsMap);
+      Object.assign(customFields, serializedCustomFields);
+    }
+
+    scopeAttributes.forEach((attribute) => {
+      const identifier = instance[attribute];
+      const entityCustomFieldDefinitions = identifierCustomFieldDefinitionsMapping[identifier];
+      if (entityCustomFieldDefinitions?.length > 0) {
+        entityCustomFieldDefinitions.forEach((customFieldDefinition) => {
+          if (customFields[customFieldDefinition.name] === undefined) {
+            customFields[customFieldDefinition.name] = null;
+          }
+        });
+      }
+    });
+    if (customFields && Object.keys(customFields).length > 0) {
+      logger.info('sadot - enrichResults - customFields', customFields);
+      instance.customFields = customFields;
+    } else {
+      logger.info('sadot - enrichResults - no customFields');
+    }
+    options.attributesToRemove?.forEach?.((attribute) => {
+      delete instance.dataValues?.[attribute];
+      // if raw:
+      delete instance?.[attribute];
+    });
+  });
+};
+
+export default enrichResults;

package/src/hooks/find.ts

@@ -0,0 +1,27 @@
+/* eslint-disable no-param-reassign */
+import logger from '../utils/logger';
+
+const doScopeAttributesMissing = (
+  scopeAttributes: string[],
+  queryAttributes: (string | string[])[],
+): string[] => {
+  const attributes = scopeAttributes
+    .filter((attribute) => !queryAttributes.includes(attribute));
+  if (!queryAttributes.includes?.('id')) {
+    attributes.push('id');
+  }
+  return attributes;
+};
+
+// eslint-disable-next-line import/prefer-default-export
+export const beforeFind = (scopeAttributes: string[]) => (options): void => {
+  const { attributes: queryAttributes } = options;
+  if (queryAttributes?.includes('customFields')) {
+    const missingScopeAttributes = doScopeAttributesMissing(scopeAttributes, queryAttributes);
+    logger.debug('sadot - before find hook');
+    if (missingScopeAttributes?.length > 0) {
+      queryAttributes.push(...missingScopeAttributes);
+      options.attributesToRemove = missingScopeAttributes;
+    }
+  }
+};

package/src/hooks/index.ts

@@ -0,0 +1,15 @@
+import enrichResults from './enrich';
+import { beforeFind } from './find';
+import { beforeBulkUpdate, beforeUpdate } from './update';
+import { beforeBulkCreate, beforeCreate } from './create';
+import workaround from './workaround';
+
+export {
+  enrichResults,
+  beforeFind,
+  beforeBulkUpdate,
+  beforeUpdate,
+  beforeBulkCreate,
+  beforeCreate,
+  workaround,
+};

package/src/hooks/update.ts

@@ -0,0 +1,39 @@
+import logger from '../utils/logger';
+import * as ValueRepo from '../repository/value';
+
+/**
+ * A hook to update the custom fields when updating a model (more then one instance).
+ */
+export const beforeBulkUpdate = (options): void => {
+  // This will activate the beforeUpdate hook on each updating instance.
+  // eslint-disable-next-line no-param-reassign
+  options.individualHooks = true;
+};
+
+/**
+ * A hook to update the custom fields when updating a model instance.
+ * TODO - cleanup if update fail
+ */
+export const beforeUpdate = (scopeAttributes: string[]) => async (
+  instance,
+  options,
+): Promise<void> => {
+  logger.debug('sadot - before update hook');
+  const { fields } = options;
+  const modelType = instance.constructor.name;
+  const identifiers = scopeAttributes.map((attribute) => instance[attribute]);
+
+  const customFieldsIdx = fields.indexOf('customFields');
+  if (customFieldsIdx > -1) {
+    const { customFields } = instance;
+    await ValueRepo.updateValues(
+      modelType,
+      instance.id,
+      identifiers,
+      customFields,
+      { transaction: options.transaction },
+    );
+    // eslint-disable-next-line no-param-reassign
+    fields.splice(customFieldsIdx, 1);
+  }
+};

package/src/hooks/workaround.ts

@@ -0,0 +1,47 @@
+/**
+ * Workaround to a bug in sequelize.🐛
+ *
+ * **afterFind hook** isn't working on nested (included) models.
+ * The solution here is to add a global afterFind hook,
+ * which manually calls the afterFind hook of each model, recursively
+ * https://github.com/sequelize/sequelize/issues/4627
+ */
+
+const handleChildrenAfterFindHook = async (instances, options, level = 0) => {
+  if (!instances) return Promise.resolve();
+
+  if (Array.isArray(instances)) {
+    return Promise.all(instances.map((instance) => {
+      const { options: instanceOptions } = instance.constructor;
+      return handleChildrenAfterFindHook(
+        instance,
+        instanceOptions,
+        level,
+      );
+    }));
+  }
+
+  const instance = instances;
+  const { constructor } = instance;
+
+  /**
+       * Root model will have already run their "afterFind" hook.
+       * Only run children "afterFind" hooks.
+       */
+  if (level >= 1) {
+    await constructor.runHooks('afterFind', instance, options);
+  }
+
+  const { associations } = constructor;
+  const associatedNames = Object.keys(instance).filter((attribute) =>
+    Object.keys(associations).includes(attribute));
+
+  if (associatedNames.length) {
+    const childInstances = associatedNames.map((name) => instance[name]);
+    return handleChildrenAfterFindHook(childInstances, options, level + 1);
+  }
+
+  return Promise.resolve();
+};
+
+export default handleChildrenAfterFindHook;

package/src/index.ts

@@ -0,0 +1,101 @@
+import type { Application } from 'express';
+import { DataTypes } from 'sequelize';
+import { Sequelize } from 'sequelize-typescript';
+import { initTables, initTestModels } from './models';
+import api from './api';
+import {
+  beforeFind,
+  enrichResults,
+  beforeBulkUpdate,
+  beforeUpdate,
+  beforeCreate,
+  beforeBulkCreate,
+} from './hooks';
+import initDB from './utils/db';
+import logger from './utils/logger';
+import { CustomFieldOptions, ModelFetcher, ModelOptions } from './types';
+
+export * from './utils/validations/custom-fields';
+
+const addHooks = (models: ModelOptions[], getModel: ModelFetcher) => {
+  models.forEach(async ({ name, scopeAttributes }) => {
+    try {
+      const model = getModel(name);
+      if (!model) {
+        logger.warn('sadot - tried to addHooks to a model that does not exist yet', {
+          name,
+          scopeAttributes,
+        });
+        return;
+      }
+      model.rawAttributes.customFields = {
+        type: DataTypes.VIRTUAL,
+      };
+      model.refreshAttributes();
+      // TODO: Uncomment after tests are passed
+      // model.addHook('afterFind', workaround);
+      model.addHook('beforeFind', 'sadot-beforeFind', beforeFind(scopeAttributes));
+      model.addHook('beforeBulkCreate', 'sadot-beforeBulkCreate', beforeBulkCreate);
+      model.addHook('beforeBulkUpdate', 'sadot-beforeBulkUpdate', beforeBulkUpdate);
+      model.addHook('beforeCreate', 'sadot-beforeCreate', beforeCreate(scopeAttributes));
+      model.addHook('beforeUpdate', 'sadot-beforeUpdate', beforeUpdate(scopeAttributes));
+      model.addHook('afterFind', 'sadot-afterFind', enrichResults(name, scopeAttributes));
+      model.addHook('afterUpdate', 'sadot-afterUpdate', enrichResults(name, scopeAttributes));
+      model.addHook('afterCreate', 'sadot-afterCreate', enrichResults(name, scopeAttributes));
+    } catch (e) {
+      logger.error(`Could not add custom fields hook to model ${name}. `, e);
+    }
+  });
+};
+
+/**
+ * Adding custom fields enrichment to the models inside the MODELS_FILE_NAME json file
+ * @see {@link 'custom-fields/config'} for configurations
+ */
+const useCustomFields = async (
+  app: Application | null,
+  getModel: ModelFetcher,
+  options: CustomFieldOptions,
+): Promise<Sequelize> => {
+  const { models } = options;
+  if (app) {
+    app.use('/api', api);
+  }
+  const sequelize = initDB(options.databaseConfig);
+  if (process.env.NODE_ENV === 'test') {
+    await initTestModels(sequelize);
+  }
+  addHooks(models, getModel);
+  await initTables(sequelize, options.getUser);
+  logger.debug('sadot - custom fields finished initializing with models', models);
+  return sequelize;
+};
+
+export default useCustomFields;
+
+const removeHooks = (models: ModelOptions[], getModel: ModelFetcher) => {
+  models.forEach(async ({ name }) => {
+    try {
+      const model = getModel(name);
+      if (!model) return;
+      if (model.rawAttributes.customFields) {
+        delete model.rawAttributes.customFields;
+        model.refreshAttributes();
+      }
+      // model.removeHook('afterFind', 'sadot-workaround');
+      model.removeHook('beforeFind', 'sadot-beforeFind');
+      model.removeHook('beforeBulkCreate', 'sadot-beforeBulkCreate');
+      model.removeHook('beforeBulkUpdate', 'sadot-beforeBulkUpdate');
+      model.removeHook('beforeCreate', 'sadot-beforeCreate');
+      model.removeHook('beforeUpdate', 'sadot-beforeUpdate');
+      model.removeHook('afterFind', 'sadot-afterFind');
+      model.removeHook('afterUpdate', 'sadot-afterUpdate');
+    } catch (e) {
+      logger.error(`Could not add custom fields hook to model ${name}. `, e);
+    }
+  });
+};
+
+export const disableCustomFields = (models, getModel) => {
+  removeHooks(models, getModel);
+};