@autofleet/sadot 1.0.0-beta.1 → 1.0.0

package/src/hooks/enrich.ts

@@ -1,14 +1,100 @@
 /* eslint-disable no-param-reassign */
 import * as ValueRepo from '../repository/value';
 import * as DefinitionRepo from '../repository/definition';
+import * as EntriesRepo from '../repository/entries';
 import type CustomFieldValue from '../models/CustomFieldValue';
 import type CustomFieldDefinition from '../models/CustomFieldDefinition';
 import type { SerializedCustomFields } from '../types/definition';
-import type { ModelOptions } from '../types';
+import type { CustomFieldOptions, ModelOptions, TransactionOptions } from '../types';
 import applyScopeToInstance from '../utils/scopeAttributes';
 
+// Include all required fields for proper functioning
+const CUSTOM_FIELD_DEFINITION_ATTRIBUTES_TO_PULL = [
+  'id',
+  'name',
+  'entityId',
+  'fieldType',
+  'displayName',
+  'validation',
+  'entityType',
+  'modelType',
+  'required',
+  'disabled',
+  'defaultValue',
+];
+
 type SupportedHookTypes = 'afterFind' | 'afterCreate' | 'afterUpdate';
 
+type CustomFieldEntries = Record<string, any>;
+
+interface GetValuesGroupByInstanceResponse {
+  [modelId: string]: CustomFieldValue[];
+}
+
+interface GetCustomFieldEntriesByInstanceIdResponse {
+  [modelId: string]: CustomFieldEntries;
+}
+
+export const getCustomFieldEntriesByInstanceId = async ({
+  instancesIds,
+  options,
+  sadotOptions,
+}: {
+  instancesIds: string[],
+  options?: TransactionOptions,
+  sadotOptions: Pick<CustomFieldOptions, 'useCustomFieldsEntries'>,
+}): Promise<GetCustomFieldEntriesByInstanceIdResponse> => {
+  if (!sadotOptions.useCustomFieldsEntries) {
+    return {};
+  }
+
+  const customFieldEntries = await EntriesRepo.findEntriesByModelIds(
+    instancesIds,
+    options ?? {},
+  );
+
+  const customFieldEntriesByInstanceId = Object.fromEntries(customFieldEntries.map((instanceEntries) => {
+    const { modelId, customFields } = instanceEntries?.dataValues ?? {};
+    if (!modelId) {
+      return undefined;
+    }
+    return [modelId, customFields];
+  }).filter(Boolean));
+
+  instancesIds.forEach((instanceId) => {
+    customFieldEntriesByInstanceId[instanceId] ??= {};
+  });
+
+  return customFieldEntriesByInstanceId;
+};
+
+export const getValuesGroupByInstance = async ({
+  instancesIds,
+  options,
+  sadotOptions,
+}: {
+  instancesIds: string[],
+  options?: TransactionOptions,
+  sadotOptions: Pick<CustomFieldOptions, 'useCustomFieldsEntries'>,
+}): Promise<GetValuesGroupByInstanceResponse> => {
+  if (sadotOptions.useCustomFieldsEntries) {
+    return {};
+  }
+
+  const customFieldValues = await ValueRepo.findValuesByModelIds(
+    instancesIds,
+    options ?? {},
+  );
+
+  // Group fields by modelId
+  return customFieldValues.reduce((acc, v) => {
+    const { modelId } = v;
+    acc[modelId] ??= [];
+    acc[modelId].push(v);
+    return acc;
+  }, {});
+};
+
 /**
  * Serialize custom fields value into the format of {[name] -> [fieldData]}
  */
@@ -33,9 +119,10 @@ const enrichResults = (
   scopeAttributes: string[],
   hookType?: SupportedHookTypes,
   modelOptions: ModelOptions = {},
+  sadotOptions: Pick<CustomFieldOptions, 'useCustomFieldsEntries'> = { useCustomFieldsEntries: false },
 ) => async (
   instancesOrInstance: any | any[],
-  options,
+  options: TransactionOptions,
 ): Promise<void> => {
   if (
     options.originalAttributes?.length > 0
@@ -58,14 +145,38 @@ const enrichResults = (
   const identifierCustomFieldDefinitionsMapping = uniqueIdentifiers.reduce((map, identifier) => ({
     ...map,
     [identifier]: [],
-
   }), {});
 
-  const customFieldDefinitions = await DefinitionRepo.findByEntityIds(
-    modelType,
-    uniqueIdentifiers,
-    { transaction: options.transaction, modelOptions },
-  );
+  // Cache for definitions by model type and transaction to avoid redundant DB queries
+  let customFieldDefinitionsPromise;
+  let cacheKey;
+
+  if (options.transaction) {
+    // Initialize definition cache Map if not already present directly on the transaction object
+    options.transaction.definitionCache ||= new Map();
+    cacheKey = `${modelType}:${uniqueIdentifiers.slice().sort().join(',')}`;
+    customFieldDefinitionsPromise = options.transaction.definitionCache.get(cacheKey);
+  }
+
+  if (!customFieldDefinitionsPromise) {
+    // Fetch from database (either first time in this transaction or no transaction)
+    customFieldDefinitionsPromise = DefinitionRepo.findByEntityIds(
+      modelType,
+      uniqueIdentifiers,
+      { transaction: options.transaction, modelOptions, attributes: CUSTOM_FIELD_DEFINITION_ATTRIBUTES_TO_PULL },
+    );
+
+    options.transaction?.definitionCache?.set(cacheKey, customFieldDefinitionsPromise);
+  }
+  const customFieldDefinitions = await customFieldDefinitionsPromise;
+
+  if (customFieldDefinitions.length === 0) {
+    // if no custom fields, we can return
+    instances.forEach((instance) => {
+      instance.customFields = {};
+    });
+    return;
+  }
 
   if (modelOptions?.include && modelOptions.useEntityIdFromInclude) {
     // if we pass useEntityIdFromInclude,
@@ -90,32 +201,30 @@ const enrichResults = (
   // Get the values per instates ids:
   const instancesIds = instances.map((i) => i[primaryKey]);
 
-  const customFieldValues = await ValueRepo.findValuesByModelIds(
-    instancesIds,
-    { transaction: options.transaction },
-  );
-
   // Group fields by modelId
-  const valuesGroupByInstance: {
-      [modelId: string]: CustomFieldValue[];
-    } = customFieldValues.reduce((acc, v) => {
-      const { modelId } = v;
-      if (!acc[modelId]) {
-        acc[modelId] = [];
-      }
-      acc[modelId].push(v);
-      return acc;
-    }, {});
+  const [valuesGroupByInstance, customFieldEntriesByInstanceId] = await Promise.all([
+    getValuesGroupByInstance({
+      instancesIds,
+      options,
+      sadotOptions,
+    }),
+    getCustomFieldEntriesByInstanceId({
+      instancesIds,
+      options,
+      sadotOptions,
+    }),
+  ]);
 
   // Attach custom fields to the instances
   instances.forEach((instance) => {
-    const customFields = {};
     const { id } = instance;
+
     const instanceValues = valuesGroupByInstance[id];
-    if (instanceValues) {
-      const serializedCustomFields = serializeCustomFields(instanceValues, definitionsMap);
-      Object.assign(customFields, serializedCustomFields);
-    }
+    const serializedCustomFieldsValues = instanceValues ? serializeCustomFields(instanceValues, definitionsMap) : {};
+
+    const customFields = sadotOptions.useCustomFieldsEntries
+      ? customFieldEntriesByInstanceId[id]
+      : serializedCustomFieldsValues;
 
     scopeAttributes.forEach((attribute) => {
       const identifier = instance[attribute];

package/src/hooks/hooks.ts

@@ -0,0 +1,467 @@
+import type { WhereOptions } from 'sequelize';
+import Ajv from 'ajv';
+import Joi from 'joi';
+import addFormats from 'ajv-formats';
+import { BadRequest } from '@autofleet/errors';
+import ajvErrors from 'ajv-errors';
+import logger from '../utils/logger';
+import * as ValidatorRepo from '../repository/validator';
+import * as DefinitionRepo from '../repository/definition';
+import { InvalidValueError, MissingRequiredCustomFieldError } from '../errors';
+import type { CustomFieldOptions, ModelOptions } from '../types';
+import applyScopeToInstance from '../utils/scopeAttributes';
+import updateInstanceValues from './utils/updateInstanceValues';
+import { CustomFieldDefinitionType } from '../utils/constants';
+import type { CustomFieldDefinition } from '../models';
+
+// Include all required fields for proper validation
+const CUSTOM_VALIDATOR_ATTRIBUTES_TO_PULL = ['id', 'schema', 'modelType', 'entityId', 'disabled'];
+
+// Initialize Ajv with relaxed settings to avoid warnings
+const ajv = new Ajv({
+  allErrors: true,
+  strict: false, // Disable strict mode to avoid warnings
+  strictTypes: false, // Disable strict type checking
+  $data: true, // Enable $data references
+});
+
+addFormats(ajv);
+ajvErrors(ajv);
+
+/**
+ * Helper function to manually copy object properties
+ * This is more efficient for large objects and avoids excessive object creation
+ */
+// eslint-disable-next-line prefer-object-spread
+const manualObjectCopy = (sourceObj: Record<string, any>, additionalProps?: Record<string, any>): Record<string, any> =>
+  ({ __proto__: null, ...sourceObj, ...additionalProps });
+
+/**
+ * Fetches complete custom fields for an instance by merging DB values with update values
+ * This is needed for partial updates to ensure all related fields are available for validation
+ */
+const getCompleteCustomFields = async (instance, options): Promise<Record<string, any>> => {
+  // If we don't have an instance id or no custom fields being updated, return original fields
+  if (!instance.id || !instance.customFields || Object.keys(instance.customFields).length === 0) {
+    return instance.customFields || {};
+  }
+
+  try {
+    const ModelClass = instance.constructor;
+    // Only select the customFields column to minimize data transfer
+    const currentCustomFields = await ModelClass.findOne({
+      where: { id: instance.id },
+      attributes: ['customFields'],
+      transaction: options.transaction,
+      raw: true, // Get plain object instead of model instance for better performance
+    });
+
+    if (currentCustomFields?.customFields) {
+      // Merge existing fields with update fields using our helper function
+      const completeFields = manualObjectCopy(
+        currentCustomFields.customFields,
+        instance.customFields,
+      );
+
+      logger.debug('sadot - fetched complete custom fields for validation', {
+        fieldsCount: Object.keys(completeFields).length,
+        updateFieldsCount: Object.keys(instance.customFields).length,
+      });
+
+      return completeFields;
+    }
+  } catch (error) {
+    logger.error('sadot - error fetching complete model for validation', { error });
+    // Continue with partial data if we can't fetch the complete model
+  }
+
+  return instance.customFields || {};
+};
+
+const formatAjvErrors = (
+  errors: {
+  instancePath?: string;
+  keyword: string;
+  message?: string;
+  params?: Record<string, any>;
+}[],
+): Record<string, string> => errors.reduce((acc, err) => {
+  const basePath = (err.instancePath || '')
+    .split('/')
+    .filter(Boolean)
+    .join('.')
+    .replace(/^after\./, '');
+
+  const missingProp = err.keyword === 'required' ? `.${err.params?.missingProperty}` : '';
+  const key = (basePath + missingProp).replace(/^\./, '') || 'root';
+
+  const message = err.message || 'Invalid value';
+  acc[key] = message;
+
+  return acc;
+}, {} as Record<string, string>);
+
+/**
+ * Validates the model using custom validators
+ */
+const validateModel = async (
+  instance,
+  options,
+  scopeAttributes: string[],
+  modelOptions: ModelOptions = {},
+  isCreate = false,
+): Promise<void> => {
+  const modelType = instance.constructor.name;
+
+  logger.debug('sadot - validating model', { isCreate, modelType });
+  const identifiers = applyScopeToInstance(instance, scopeAttributes);
+
+  logger.debug('sadot - identifiers', { identifiers });
+
+  // Skip if no identifiers
+  if (!identifiers || Object.keys(identifiers).length === 0) {
+    logger.debug('sadot - skipping validation: no identifiers');
+    return;
+  }
+
+  // Find the entityId from identifiers (fleetId, businessModelId, etc.)
+  const entityId = Object.values(identifiers)[0]; // Get the first value as entityId
+
+  logger.debug('sadot - entityId', { entityId });
+
+  if (!entityId) {
+    logger.debug('sadot - skipping validation: no entityId');
+    return;
+  }
+
+  let validatorsPromise;
+  let cacheKey;
+  if (options.transaction) {
+    // eslint-disable-next-line no-param-reassign
+    options.transaction.validationsCache ||= new Map();
+    cacheKey = `${modelType}-${entityId}`;
+    validatorsPromise = options.transaction.validationsCache.get(cacheKey);
+  }
+
+  if (!validatorsPromise) {
+    validatorsPromise = ValidatorRepo.findAllByModelType(
+      modelType,
+      entityId,
+      {
+        transaction: options.transaction,
+        attributes: CUSTOM_VALIDATOR_ATTRIBUTES_TO_PULL,
+        ...(modelOptions.include && {
+          include: modelOptions.include?.(entityId),
+        }),
+        raw: true,
+      },
+    );
+    if (options.transaction) {
+      options?.transaction?.validationsCache.set(cacheKey, validatorsPromise);
+    }
+  }
+  const validators = await validatorsPromise;
+
+  logger.debug('sadot - validators found', { count: validators.length });
+
+  if (!validators.length) {
+    logger.debug('sadot - skipping validation: no validators found');
+    return;
+  }
+
+  // For updates, get the previous values
+  let originalValues = null;
+  if (!isCreate) {
+    // Create originalValues with our helper function
+    originalValues = manualObjectCopy(instance.previous());
+
+    // Add customFields separately
+    originalValues.customFields = instance.previous('customFields') || {};
+  }
+
+  // Get complete custom fields by merging DB values with update values
+  // This is especially important for partial updates to ensure all related fields are available
+  const completeCustomFields = !isCreate
+    ? await getCompleteCustomFields(instance, options)
+    : instance.customFields || {};
+
+  // eslint-disable-next-line no-restricted-syntax
+  for (const validator of validators) {
+    const { schema } = validator;
+    const typedSchema = schema as Record<string, any>;
+
+    logger.debug('sadot - validating with schema', {
+      schema,
+      hasAfterProps: !!typedSchema.properties?.after,
+      hasBeforeProps: !!typedSchema.properties?.before,
+    });
+
+    if (isCreate) {
+      // For create operations, we only need the 'after' state
+      if (typedSchema.properties?.after) {
+        const validateSchema = ajv.compile({
+          ...schema,
+          // Focus only on the 'after' validation part for create
+          properties: {
+            after: typedSchema.properties.after,
+          },
+        });
+
+        const isValid = validateSchema(JSON.parse(JSON.stringify({
+          after: {
+            ...instance.dataValues,
+            customFields: completeCustomFields,
+          },
+        })));
+
+        if (!isValid) {
+          const errorDetails = validateSchema.errors?.map((err) =>
+            `${(err as any).instancePath || ''} ${(err as any).message || 'Invalid value'}`).join(', ');
+
+          const formattedErrors = formatAjvErrors(validateSchema.errors);
+          throw new BadRequest(
+            [new Error(`Validation failed for ${modelType}: ${errorDetails}`)],
+            undefined,
+            {
+              customError: formattedErrors,
+            },
+          );
+        }
+      }
+    } else {
+      // For update operations, we need both before and after
+      const validateSchema = ajv.compile(typedSchema);
+
+      // Create after object with our helper function
+      const afterObj = manualObjectCopy(instance.dataValues);
+
+      // Add complete custom fields
+      afterObj.customFields = completeCustomFields;
+
+      // Create validation payload
+      const payload = {
+        before: originalValues,
+        after: afterObj,
+      };
+
+      // Validate
+      const isValid = validateSchema(JSON.parse(JSON.stringify(payload)));
+
+      logger.debug('sadot - validation result', {
+        isValid,
+        test: {
+          before: originalValues,
+          after: afterObj,
+        },
+      });
+
+      if (!isValid) {
+        const errorDetails = validateSchema
+          .errors
+          ?.map((err) => `${(err as any).instancePath || ''} ${(err as any).message || 'Invalid value'}`).join(', ');
+
+        const formattedErrors = formatAjvErrors(validateSchema.errors);
+        throw new BadRequest(
+          [new Error(`Validation failed for ${modelType}: ${errorDetails}`)],
+          undefined,
+          {
+            customError: formattedErrors,
+          },
+        );
+      }
+    }
+  }
+};
+
+const getFieldDefinitions = async ({
+  modelType,
+  modelOptions,
+  identifiers,
+  options,
+}: {
+  modelType: any,
+  modelOptions: ModelOptions,
+  identifiers: any[],
+  options: any
+}) => {
+  const { include, useEntityIdFromInclude } = modelOptions;
+  const where: WhereOptions = {
+    modelType,
+    disabled: false,
+    ...(!useEntityIdFromInclude && { entityId: identifiers }),
+  };
+
+  const fieldDefinitions = await DefinitionRepo.findAll(where, {
+    withDisabled: false,
+    transaction: options.transaction,
+    include: include?.(identifiers),
+  });
+  return fieldDefinitions;
+};
+
+const formatDates = (fieldDefinitions: CustomFieldDefinition[], instance: any) => {
+  (fieldDefinitions || []).forEach((fieldDefinition) => {
+    const { fieldType, name } = fieldDefinition;
+    if ([CustomFieldDefinitionType.DATE, CustomFieldDefinitionType.DATETIME].includes(fieldType)) {
+      const value = instance.customFields?.[name];
+      if (value) {
+        const { value: joiValue, error: validationError } = Joi.date().validate(value);
+        if (validationError) {
+          throw new InvalidValueError(value, name, validationError);
+        }
+        // eslint-disable-next-line no-param-reassign
+        instance.customFields[name] = joiValue.toISOString();
+      }
+    }
+  });
+};
+
+/**
+ * Hook to handle validation and custom fields during creation
+ */
+export const beforeCreate = (
+  scopeAttributes: string[],
+  modelOptions: ModelOptions = {},
+  sadotOptions: Pick<CustomFieldOptions, 'useCustomFieldsEntries'> = { useCustomFieldsEntries: false },
+) => async (
+  instance,
+  options,
+): Promise<void> => {
+  logger.debug('sadot - before create hook');
+  const { fields } = options;
+  const modelType = instance.constructor.name;
+
+  const identifiers = applyScopeToInstance(instance, scopeAttributes);
+
+  // Step 1: Handle custom fields default values and required fields
+
+  const fieldDefinitions = await getFieldDefinitions({
+    modelType, modelOptions, identifiers, options,
+  });
+
+  // Apply default values
+  const fieldsWithDefaultValue = fieldDefinitions.filter((def) => ![null, undefined].includes(def.defaultValue));
+  if (fieldsWithDefaultValue.length) {
+    // eslint-disable-next-line no-param-reassign
+    instance.customFields ||= {};
+    fieldsWithDefaultValue
+      .filter((def) => (instance.customFields?.[def.name] === undefined))
+      .forEach(({ name, defaultValue }) => {
+        // eslint-disable-next-line no-param-reassign
+        instance.customFields[name] = defaultValue;
+      });
+  }
+
+  // Check for required fields
+  const requiredFieldsNames = Array.from(
+    new Set(fieldDefinitions.filter(({ required }) => required).map(({ name }) => name)),
+  );
+  const { customFields } = instance;
+  const fieldsNames = Object.keys(customFields ?? {});
+  const missingFields = requiredFieldsNames.filter((name) => !fieldsNames.includes(name));
+  if (missingFields?.length) {
+    throw new MissingRequiredCustomFieldError(missingFields);
+  }
+
+  // Step 2: Validate the model data (including custom fields)
+  await validateModel(instance, options, scopeAttributes, modelOptions, true);
+
+  // format date and datetime fields
+  formatDates(fieldDefinitions, instance);
+
+  // Step 3: Save custom field values if they exist
+  const customFieldsIdx = fields.indexOf('customFields');
+  if (customFieldsIdx === -1 || !customFields || !Object.keys(customFields).length) {
+    // No custom fields to update
+    return;
+  }
+
+  // Save custom field values
+  await updateInstanceValues({
+    modelId: instance.id,
+    modelType,
+    identifiers,
+    customFields,
+    options: {
+      useCustomFieldsEntries: sadotOptions.useCustomFieldsEntries,
+      transaction: options.transaction,
+      modelOptions,
+    },
+  });
+
+  // Remove customFields from fields array after handling
+  // eslint-disable-next-line no-param-reassign
+  fields.splice(customFieldsIdx, 1);
+};
+
+/**
+ * Hook to handle validation and custom fields during update
+ */
+export const beforeUpdate = (
+  scopeAttributes: string[],
+  modelOptions: ModelOptions = {},
+  sadotOptions: Pick<CustomFieldOptions, 'useCustomFieldsEntries'> = { useCustomFieldsEntries: false },
+) => async (
+  instance,
+  options,
+): Promise<void> => {
+  logger.debug('sadot - before update hook');
+  const { fields } = options;
+  const modelType = instance.constructor.name;
+  const identifiers = applyScopeToInstance(instance, scopeAttributes);
+
+  const fieldDefinitions = await getFieldDefinitions({
+    modelType, modelOptions, identifiers, options,
+  });
+
+  // Step 1: Validate the model data (including custom fields)
+  await validateModel(instance, options, scopeAttributes, modelOptions, false);
+
+  // format date and datetime fields
+  formatDates(fieldDefinitions, instance);
+
+  // Step 2: Update custom field values if they exist
+  const customFieldsIdx = fields.indexOf('customFields');
+  if (customFieldsIdx > -1) {
+    const { customFields } = instance;
+
+    if (!Object.keys(customFields).length) {
+      return;
+    }
+
+    // Save custom field values
+    await updateInstanceValues({
+      modelId: instance.id,
+      modelType,
+      identifiers,
+      customFields,
+      options: {
+        useCustomFieldsEntries: sadotOptions.useCustomFieldsEntries,
+        transaction: options.transaction,
+        modelOptions,
+      },
+    });
+
+    // Remove customFields from fields array after handling
+    // eslint-disable-next-line no-param-reassign
+    fields.splice(customFieldsIdx, 1);
+  }
+};
+
+/**
+ * Hook to enable individual hooks for bulk create operations
+ */
+export const beforeBulkCreate = (options): void => {
+  // This will activate the beforeCreate hook on each instance
+  // eslint-disable-next-line no-param-reassign
+  options.individualHooks = true;
+};
+
+/**
+ * Hook to enable individual hooks for bulk update operations
+ */
+export const beforeBulkUpdate = (options): void => {
+  // This will activate the beforeUpdate hook on each instance
+  // eslint-disable-next-line no-param-reassign
+  options.individualHooks = true;
+};

package/src/hooks/index.ts

@@ -1,15 +1,20 @@
 import enrichResults from './enrich';
 import { beforeFind } from './find';
-import { beforeBulkUpdate, beforeUpdate } from './update';
-import { beforeBulkCreate, beforeCreate } from './create';
 import workaround from './workaround';
+import {
+  beforeCreate,
+  beforeUpdate,
+  beforeBulkCreate,
+  beforeBulkUpdate,
+} from './hooks';
 
+// Export the hooks
 export {
   enrichResults,
   beforeFind,
-  beforeBulkUpdate,
+  workaround,
+  beforeCreate,
   beforeUpdate,
   beforeBulkCreate,
-  beforeCreate,
-  workaround,
+  beforeBulkUpdate,
 };