@autofleet/sadot 1.1.3 → 1.1.5-beta

package/src/hooks/hooks.ts

@@ -0,0 +1,482 @@
+import type { WhereOptions } from 'sequelize';
+import Ajv from 'ajv';
+import Joi from 'joi';
+import addFormats from 'ajv-formats';
+import { BadRequest } from '@autofleet/errors';
+import ajvErrors from 'ajv-errors';
+import logger from '../utils/logger';
+import * as ValidatorRepo from '../repository/validator';
+import * as DefinitionRepo from '../repository/definition';
+import { InvalidValueError, MissingRequiredCustomFieldError } from '../errors';
+import type { CustomFieldOptions, ModelOptions } from '../types';
+import applyScopeToInstance from '../utils/scopeAttributes';
+import updateInstanceValues from './utils/updateInstanceValues';
+import { CustomFieldDefinitionType } from '../utils/constants';
+import type { CustomFieldDefinition } from '../models';
+
+// Include all required fields for proper validation
+const CUSTOM_VALIDATOR_ATTRIBUTES_TO_PULL = ['id', 'schema', 'modelType', 'entityId', 'disabled'];
+
+// Initialize Ajv with relaxed settings to avoid warnings
+const ajv = new Ajv({
+  allErrors: true,
+  strict: false, // Disable strict mode to avoid warnings
+  strictTypes: false, // Disable strict type checking
+  $data: true, // Enable $data references
+});
+
+addFormats(ajv);
+ajvErrors(ajv);
+
+/**
+ * Helper function to manually copy object properties
+ * This is more efficient for large objects and avoids excessive object creation
+ */
+// eslint-disable-next-line prefer-object-spread
+const manualObjectCopy = (sourceObj: Record<string, any>, additionalProps?: Record<string, any>): Record<string, any> =>
+  ({ __proto__: null, ...sourceObj, ...additionalProps });
+
+/**
+ * Fetches complete custom fields for an instance by merging DB values with update values
+ * This is needed for partial updates to ensure all related fields are available for validation
+ */
+const getCompleteCustomFields = async (instance, options): Promise<Record<string, any>> => {
+  // If we don't have an instance id or no custom fields being updated, return original fields
+  if (!instance.id || !instance.customFields || Object.keys(instance.customFields).length === 0) {
+    return instance.customFields || {};
+  }
+
+  try {
+    const ModelClass = instance.constructor;
+    // Only select the customFields column to minimize data transfer
+    const currentCustomFields = await ModelClass.findOne({
+      where: { id: instance.id },
+      attributes: ['customFields'],
+      transaction: options.transaction,
+      raw: true, // Get plain object instead of model instance for better performance
+    });
+
+    if (currentCustomFields?.customFields) {
+      // Merge existing fields with update fields using our helper function
+      const completeFields = manualObjectCopy(
+        currentCustomFields.customFields,
+        instance.customFields,
+      );
+
+      logger.debug('sadot - fetched complete custom fields for validation', {
+        fieldsCount: Object.keys(completeFields).length,
+        updateFieldsCount: Object.keys(instance.customFields).length,
+      });
+
+      return completeFields;
+    }
+  } catch (error) {
+    logger.error('sadot - error fetching complete model for validation', { error });
+    // Continue with partial data if we can't fetch the complete model
+  }
+
+  return instance.customFields || {};
+};
+
+const buildPreChangeState = (instance: any) => {
+  const beforeFull: any = { ...instance.dataValues };
+
+  const changedKeys: string[] = instance.changed?.() || [];
+  changedKeys.forEach((key) => {
+    const prevVal = instance.previous?.(key);
+    if (prevVal !== undefined) {
+      beforeFull[key] = prevVal;
+    }
+  });
+
+  const prevCF = instance.previous?.('customFields');
+  if (prevCF !== undefined) {
+    beforeFull.customFields = prevCF;
+  }
+
+  return beforeFull;
+};
+
+const formatAjvErrors = (
+  errors: {
+  instancePath?: string;
+  keyword: string;
+  message?: string;
+  params?: Record<string, any>;
+}[],
+): Record<string, string> => errors.reduce((acc, err) => {
+  const basePath = (err.instancePath || '')
+    .split('/')
+    .filter(Boolean)
+    .join('.')
+    .replace(/^after\./, '');
+
+  const missingProp = err.keyword === 'required' ? `.${err.params?.missingProperty}` : '';
+  const key = (basePath + missingProp).replace(/^\./, '') || 'root';
+
+  const message = err.message || 'Invalid value';
+  acc[key] = message;
+
+  return acc;
+}, {} as Record<string, string>);
+
+/**
+ * Validates the model using custom validators
+ */
+const validateModel = async (
+  instance,
+  options,
+  scopeAttributes: string[],
+  modelOptions: ModelOptions = {},
+  isCreate = false,
+): Promise<void> => {
+  const modelType = instance.constructor.name;
+
+  logger.debug('sadot - validating model', { isCreate, modelType });
+  const identifiers = applyScopeToInstance(instance, scopeAttributes);
+
+  logger.debug('sadot - identifiers', { identifiers });
+
+  // Skip if no identifiers
+  if (!identifiers || Object.keys(identifiers).length === 0) {
+    logger.debug('sadot - skipping validation: no identifiers');
+    return;
+  }
+
+  // Find the entityId from identifiers (fleetId, businessModelId, etc.)
+  const entityId = Object.values(identifiers)[0]; // Get the first value as entityId
+
+  logger.debug('sadot - entityId', { entityId });
+
+  if (!entityId) {
+    logger.debug('sadot - skipping validation: no entityId');
+    return;
+  }
+
+  let validatorsPromise;
+  let cacheKey;
+  if (options.transaction) {
+    // eslint-disable-next-line no-param-reassign
+    options.transaction.validationsCache ||= new Map();
+    cacheKey = `${modelType}-${entityId}`;
+    validatorsPromise = options.transaction.validationsCache.get(cacheKey);
+  }
+
+  if (!validatorsPromise) {
+    validatorsPromise = ValidatorRepo.findAllByModelType(
+      modelType,
+      entityId,
+      {
+        transaction: options.transaction,
+        attributes: CUSTOM_VALIDATOR_ATTRIBUTES_TO_PULL,
+        ...(modelOptions.include && {
+          include: modelOptions.include?.(entityId),
+        }),
+        raw: true,
+      },
+    );
+    if (options.transaction) {
+      options?.transaction?.validationsCache.set(cacheKey, validatorsPromise);
+    }
+  }
+  const validators = await validatorsPromise;
+
+  logger.debug('sadot - validators found', { count: validators.length });
+
+  if (!validators.length) {
+    logger.debug('sadot - skipping validation: no validators found');
+    return;
+  }
+
+  // For updates, get the previous values
+  let originalValues = null;
+  if (!isCreate) {
+    originalValues = buildPreChangeState(instance);
+  }
+
+  // Get complete custom fields by merging DB values with update values
+  // This is especially important for partial updates to ensure all related fields are available
+  const completeCustomFields = !isCreate
+    ? await getCompleteCustomFields(instance, options)
+    : instance.customFields || {};
+
+  // eslint-disable-next-line no-restricted-syntax
+  for (const validator of validators) {
+    const { schema } = validator;
+    const typedSchema = schema as Record<string, any>;
+
+    logger.debug('sadot - validating with schema', {
+      schema,
+      hasAfterProps: !!typedSchema.properties?.after,
+      hasBeforeProps: !!typedSchema.properties?.before,
+    });
+
+    if (isCreate) {
+      // For create operations, we only need the 'after' state
+      if (typedSchema.properties?.after) {
+        const validateSchema = ajv.compile({
+          ...schema,
+          // Focus only on the 'after' validation part for create
+          properties: {
+            after: typedSchema.properties.after,
+          },
+        });
+
+        const isValid = validateSchema(JSON.parse(JSON.stringify({
+          after: {
+            ...instance.dataValues,
+            customFields: completeCustomFields,
+          },
+        })));
+
+        if (!isValid) {
+          const errorDetails = validateSchema.errors?.map((err) =>
+            `${(err as any).instancePath || ''} ${(err as any).message || 'Invalid value'}`).join(', ');
+
+          const formattedErrors = formatAjvErrors(validateSchema.errors);
+          throw new BadRequest(
+            [new Error(`Validation failed for ${modelType}: ${errorDetails}`)],
+            undefined,
+            {
+              customError: formattedErrors,
+            },
+          );
+        }
+      }
+    } else {
+      // For update operations, we need both before and after
+      const validateSchema = ajv.compile(typedSchema);
+
+      // Create after object with our helper function
+      const afterObj = manualObjectCopy(instance.dataValues);
+
+      // Add complete custom fields
+      afterObj.customFields = completeCustomFields;
+
+      // Create validation payload
+      const payload = {
+        before: originalValues,
+        after: afterObj,
+      };
+
+      // Validate
+      const isValid = validateSchema(JSON.parse(JSON.stringify(payload)));
+
+      logger.debug('sadot - validation result', {
+        isValid,
+        test: {
+          before: originalValues,
+          after: afterObj,
+        },
+      });
+
+      if (!isValid) {
+        const errorDetails = validateSchema
+          .errors
+          ?.map((err) => `${(err as any).instancePath || ''} ${(err as any).message || 'Invalid value'}`).join(', ');
+
+        const formattedErrors = formatAjvErrors(validateSchema.errors);
+        throw new BadRequest(
+          [new Error(`Validation failed for ${modelType}: ${errorDetails}`)],
+          undefined,
+          {
+            customError: formattedErrors,
+          },
+        );
+      }
+    }
+  }
+};
+
+const getFieldDefinitions = async ({
+  modelType,
+  modelOptions,
+  identifiers,
+  options,
+}: {
+  modelType: any,
+  modelOptions: ModelOptions,
+  identifiers: any[],
+  options: any
+}) => {
+  const { include, useEntityIdFromInclude } = modelOptions;
+  const where: WhereOptions = {
+    modelType,
+    disabled: false,
+    ...(!useEntityIdFromInclude && { entityId: identifiers }),
+  };
+
+  const fieldDefinitions = await DefinitionRepo.findAll(where, {
+    withDisabled: false,
+    transaction: options.transaction,
+    include: include?.(identifiers),
+  });
+  return fieldDefinitions;
+};
+
+const formatDates = (fieldDefinitions: CustomFieldDefinition[], instance: any) => {
+  (fieldDefinitions || []).forEach((fieldDefinition) => {
+    const { fieldType, name } = fieldDefinition;
+    if ([CustomFieldDefinitionType.DATE, CustomFieldDefinitionType.DATETIME].includes(fieldType)) {
+      const value = instance.customFields?.[name];
+      if (value) {
+        const { value: joiValue, error: validationError } = Joi.date().validate(value);
+        if (validationError) {
+          throw new InvalidValueError(value, name, validationError);
+        }
+        // eslint-disable-next-line no-param-reassign
+        instance.customFields[name] = joiValue.toISOString();
+      }
+    }
+  });
+};
+
+/**
+ * Hook to handle validation and custom fields during creation
+ */
+export const beforeCreate = (
+  scopeAttributes: string[],
+  modelOptions: ModelOptions = {},
+  sadotOptions: Pick<CustomFieldOptions, 'useCustomFieldsEntries'> = { useCustomFieldsEntries: false },
+) => async (
+  instance,
+  options,
+): Promise<void> => {
+  logger.debug('sadot - before create hook');
+  const { fields } = options;
+  const modelType = instance.constructor.name;
+
+  const identifiers = applyScopeToInstance(instance, scopeAttributes);
+
+  // Step 1: Handle custom fields default values and required fields
+
+  const fieldDefinitions = await getFieldDefinitions({
+    modelType, modelOptions, identifiers, options,
+  });
+
+  // Apply default values
+  const fieldsWithDefaultValue = fieldDefinitions.filter((def) => ![null, undefined].includes(def.defaultValue));
+  if (fieldsWithDefaultValue.length) {
+    // eslint-disable-next-line no-param-reassign
+    instance.customFields ||= {};
+    fieldsWithDefaultValue
+      .filter((def) => (instance.customFields?.[def.name] === undefined))
+      .forEach(({ name, defaultValue }) => {
+        // eslint-disable-next-line no-param-reassign
+        instance.customFields[name] = defaultValue;
+      });
+  }
+
+  // Check for required fields
+  const requiredFieldsNames = Array.from(
+    new Set(fieldDefinitions.filter(({ required }) => required).map(({ name }) => name)),
+  );
+  const { customFields } = instance;
+  const fieldsNames = Object.keys(customFields ?? {});
+  const missingFields = requiredFieldsNames.filter((name) => !fieldsNames.includes(name));
+  if (missingFields?.length) {
+    throw new MissingRequiredCustomFieldError(missingFields);
+  }
+
+  // Step 2: Validate the model data (including custom fields)
+  await validateModel(instance, options, scopeAttributes, modelOptions, true);
+
+  // format date and datetime fields
+  formatDates(fieldDefinitions, instance);
+
+  // Step 3: Save custom field values if they exist
+  const customFieldsIdx = fields.indexOf('customFields');
+  if (customFieldsIdx === -1 || !customFields || !Object.keys(customFields).length) {
+    // No custom fields to update
+    return;
+  }
+
+  // Save custom field values
+  await updateInstanceValues({
+    modelId: instance.id,
+    modelType,
+    identifiers,
+    customFields,
+    options: {
+      useCustomFieldsEntries: sadotOptions.useCustomFieldsEntries,
+      transaction: options.transaction,
+      modelOptions,
+    },
+  });
+
+  // Remove customFields from fields array after handling
+  // eslint-disable-next-line no-param-reassign
+  fields.splice(customFieldsIdx, 1);
+};
+
+/**
+ * Hook to handle validation and custom fields during update
+ */
+export const beforeUpdate = (
+  scopeAttributes: string[],
+  modelOptions: ModelOptions = {},
+  sadotOptions: Pick<CustomFieldOptions, 'useCustomFieldsEntries'> = { useCustomFieldsEntries: false },
+) => async (
+  instance,
+  options,
+): Promise<void> => {
+  logger.debug('sadot - before update hook');
+  const { fields } = options;
+  const modelType = instance.constructor.name;
+  const identifiers = applyScopeToInstance(instance, scopeAttributes);
+
+  const fieldDefinitions = await getFieldDefinitions({
+    modelType, modelOptions, identifiers, options,
+  });
+
+  // Step 1: Validate the model data (including custom fields)
+  await validateModel(instance, options, scopeAttributes, modelOptions, false);
+
+  // format date and datetime fields
+  formatDates(fieldDefinitions, instance);
+
+  // Step 2: Update custom field values if they exist
+  const customFieldsIdx = fields.indexOf('customFields');
+  if (customFieldsIdx > -1) {
+    const { customFields } = instance;
+
+    if (!Object.keys(customFields).length) {
+      return;
+    }
+
+    // Save custom field values
+    await updateInstanceValues({
+      modelId: instance.id,
+      modelType,
+      identifiers,
+      customFields,
+      options: {
+        useCustomFieldsEntries: sadotOptions.useCustomFieldsEntries,
+        transaction: options.transaction,
+        modelOptions,
+      },
+    });
+
+    // Remove customFields from fields array after handling
+    // eslint-disable-next-line no-param-reassign
+    fields.splice(customFieldsIdx, 1);
+  }
+};
+
+/**
+ * Hook to enable individual hooks for bulk create operations
+ */
+export const beforeBulkCreate = (options): void => {
+  // This will activate the beforeCreate hook on each instance
+  // eslint-disable-next-line no-param-reassign
+  options.individualHooks = true;
+};
+
+/**
+ * Hook to enable individual hooks for bulk update operations
+ */
+export const beforeBulkUpdate = (options): void => {
+  // This will activate the beforeUpdate hook on each instance
+  // eslint-disable-next-line no-param-reassign
+  options.individualHooks = true;
+};

package/src/hooks/index.ts

@@ -0,0 +1,20 @@
+import enrichResults from './enrich';
+import { beforeFind } from './find';
+import workaround from './workaround';
+import {
+  beforeCreate,
+  beforeUpdate,
+  beforeBulkCreate,
+  beforeBulkUpdate,
+} from './hooks';
+
+// Export the hooks
+export {
+  enrichResults,
+  beforeFind,
+  workaround,
+  beforeCreate,
+  beforeUpdate,
+  beforeBulkCreate,
+  beforeBulkUpdate,
+};

package/src/hooks/update.ts

@@ -0,0 +1,55 @@
+import logger from '../utils/logger';
+import type { CustomFieldOptions, ModelOptions } from '../types';
+import applyScopeToInstance from '../utils/scopeAttributes';
+import updateInstanceValues from './utils/updateInstanceValues';
+
+/**
+ * A hook to update the custom fields when updating a model (more then one instance).
+ */
+export const beforeBulkUpdate = (options): void => {
+  // This will activate the beforeUpdate hook on each updating instance.
+  // eslint-disable-next-line no-param-reassign
+  options.individualHooks = true;
+};
+
+/**
+ * A hook to update the custom fields when updating a model instance.
+ * TODO - cleanup if update fail
+ */
+export const beforeUpdate = (
+  scopeAttributes: string[],
+  modelOptions: ModelOptions = {},
+  sadotOptions: Pick<CustomFieldOptions, 'useCustomFieldsEntries'> = { useCustomFieldsEntries: false },
+) => async (
+  instance,
+  options,
+): Promise<void> => {
+  logger.debug('sadot - before update hook');
+  const { fields } = options;
+  const modelType = instance.constructor.name;
+  const identifiers = applyScopeToInstance(instance, scopeAttributes);
+
+  const customFieldsIdx = fields.indexOf('customFields');
+  if (customFieldsIdx > -1) {
+    const { customFields } = instance;
+
+    if (!Object.keys(customFields).length) {
+      return;
+    }
+
+    await updateInstanceValues({
+      modelId: instance.id,
+      modelType,
+      identifiers,
+      customFields,
+      options: {
+        useCustomFieldsEntries: sadotOptions.useCustomFieldsEntries,
+        transaction: options.transaction,
+        modelOptions,
+      },
+    });
+
+    // eslint-disable-next-line no-param-reassign
+    fields.splice(customFieldsIdx, 1);
+  }
+};

package/src/hooks/utils/updateInstanceValues.ts

@@ -0,0 +1,63 @@
+import type { Transaction } from 'sequelize';
+import * as ValueRepo from '../../repository/value';
+import * as EntriesRepo from '../../repository/entries';
+import type { ModelOptions } from '../../types';
+
+interface UpdateInstanceValuesParams {
+  modelId: string;
+  modelType: string;
+  identifiers: string[];
+  customFields: Record<string, any>;
+  options?: {
+    transaction?: Transaction;
+    modelOptions?: ModelOptions;
+    useCustomFieldsEntries?: boolean;
+  };
+}
+
+const updateInstanceValues = async ({
+  modelId,
+  modelType,
+  identifiers,
+  customFields,
+  options = {
+    modelOptions: {},
+    useCustomFieldsEntries: false,
+  },
+}: UpdateInstanceValuesParams): Promise<void> => {
+  await ValueRepo.updateValues(
+    modelType,
+    modelId,
+    identifiers,
+    customFields,
+    {
+      ...options,
+      modelOptions: options.modelOptions ?? {},
+    },
+  );
+
+  /*
+  T.Y TODO - Once we're ready to switch from custom_field_values to custom_field_entries, we should remove the ValueRepo.updateValues call.
+  Currently, We're updating both tables to keep the data in sync, but not all microservices are using the new table yet.
+  */
+
+  if (!options?.useCustomFieldsEntries) {
+    return;
+  }
+
+  const { dataValues: { customFields: oldCustomFields } } = await EntriesRepo.findEntriesByModelId(modelId, options) ?? { dataValues: {} };
+  const newCustomFields = { ...oldCustomFields, ...customFields };
+
+  await EntriesRepo.updateEntries(
+    modelId,
+    modelType,
+    newCustomFields,
+    identifiers,
+    {
+      ...options,
+      modelOptions: options.modelOptions ?? {},
+    },
+  );
+};
+
+export default updateInstanceValues;

package/src/hooks/workaround.ts

@@ -0,0 +1,47 @@
+/**
+ * Workaround to a bug in sequelize.🐛
+ *
+ * **afterFind hook** isn't working on nested (included) models.
+ * The solution here is to add a global afterFind hook,
+ * which manually calls the afterFind hook of each model, recursively
+ * https://github.com/sequelize/sequelize/issues/4627
+ */
+
+const handleChildrenAfterFindHook = async (instances, options, level = 0) => {
+  if (!instances) return Promise.resolve();
+
+  if (Array.isArray(instances)) {
+    return Promise.all(instances.map((instance) => {
+      const { options: instanceOptions } = instance.constructor;
+      return handleChildrenAfterFindHook(
+        instance,
+        instanceOptions,
+        level,
+      );
+    }));
+  }
+
+  const instance = instances;
+  const { constructor } = instance;
+
+  /**
+       * Root model will have already run their "afterFind" hook.
+       * Only run children "afterFind" hooks.
+       */
+  if (level >= 1) {
+    await constructor.runHooks('afterFind', instance, options);
+  }
+
+  const { associations } = constructor;
+  const associatedNames = Object.keys(instance).filter((attribute) =>
+    Object.keys(associations).includes(attribute));
+
+  if (associatedNames.length) {
+    const childInstances = associatedNames.map((name) => instance[name]);
+    return handleChildrenAfterFindHook(childInstances, options, level + 1);
+  }
+
+  return Promise.resolve();
+};
+
+export default handleChildrenAfterFindHook;