@autofleet/sadot 1.0.0-beta.0 → 1.0.0

package/src/repository/definition.ts

@@ -1,14 +1,26 @@
-import { Op, type FindOptions, type WhereOptions } from 'sequelize';
-import { CustomFieldDefinition } from '../models';
+import {
+  Op,
+  type Includeable, type Transaction, type FindOptions, type WhereOptions,
+} from 'sequelize';
+import { CustomFieldDefinition, type CustomFieldEntries } from '../models';
 import type { CreateCustomFieldDefinition, UpdateCustomFieldDefinition } from '../types/definition';
 import type { ModelOptions } from '../types';
+import { MissingDefinitionError } from '../errors';
 
 export const create = (data: CreateCustomFieldDefinition): Promise<CustomFieldDefinition> =>
   CustomFieldDefinition.create(data);
 
+interface SadotFindOptions {
+  withDisabled?: boolean;
+  transaction?: Transaction;
+  include?: Includeable | Includeable[];
+}
+
+type SadotGetDefinitionsByEntityIdsOptions = FindOptions & { modelOptions?: ModelOptions } & Pick<SadotFindOptions, 'withDisabled'>;
+
 export const findAll = (
   where: WhereOptions,
-  options: any = { withDisabled: false },
+  options: SadotFindOptions = { withDisabled: false },
 ): Promise<CustomFieldDefinition[]> => {
   const queryModel = options.withDisabled
     ? CustomFieldDefinition.unscoped()
@@ -24,12 +36,12 @@ export const findAll = (
 
 export const findByIds = (
   ids: string[],
-  options: any = { withDisabled: false },
+  options: SadotFindOptions = { withDisabled: false },
 ): Promise<CustomFieldDefinition[]> => findAll({ id: { [Op.in]: ids } }, options);
 
 export const findById = (
   id: string,
-  options: any = { withDisabled: false },
+  options: Pick<SadotFindOptions, 'withDisabled'> = { withDisabled: false },
 ): Promise<CustomFieldDefinition | null> => {
   const { withDisabled } = options;
   if (withDisabled) {
@@ -46,11 +58,9 @@ export const findByEntityIds = async (
   const { include, useEntityIdFromInclude } = options.modelOptions;
   const where: WhereOptions = {
     modelType,
+    ...(!useEntityIdFromInclude && { entityId: { [Op.in]: entityIds } }),
   };
 
-  if (!useEntityIdFromInclude) {
-    where.entityId = entityIds;
-  }
   return CustomFieldDefinition.findAll({
     where,
     transaction: options.transaction,
@@ -87,13 +97,13 @@ export const update = async (
   return updatedDefinition;
 };
 
-export const disable = (id: string): Promise<any> =>
+export const disable = (id: string): Promise<[affectedCount: number]> =>
   CustomFieldDefinition.update(
     { disabled: true },
     { where: { id } },
   );
 
-export const destroy = (id: string): Promise<any> =>
+export const destroy = (id: string): Promise<number> =>
   CustomFieldDefinition.destroy({ where: { id } });
 
 /**
@@ -111,12 +121,9 @@ export const getRequiredFields = async (
   const where: WhereOptions = {
     modelType,
     required: true,
+    ...(!useEntityIdFromInclude && { entityId: { [Op.in]: entityIds } }),
   };
 
-  if (!useEntityIdFromInclude) {
-    where.entityId = entityIds;
-  }
-
   const requiredFields = await CustomFieldDefinition.findAll({
     where,
     include: include?.(entityIds),
@@ -125,3 +132,44 @@ export const getRequiredFields = async (
   const requiredFieldsNames = requiredFields.map((definition) => definition.name);
   return [...new Set(requiredFieldsNames)];
 };
+
+/**
+ * @returns A promise resolving with a dictionary of custom field definitions by name.
+ * @throws A {@link MissingDefinitionError} if any of the custom fields doesn't have a definition.
+ */
+export const getCustomFieldDefinitionsDictionary = async (
+  instances: CustomFieldEntries[],
+  options: SadotGetDefinitionsByEntityIdsOptions = { withDisabled: false, modelOptions: {} },
+): Promise<{ [definitionName: string]: CustomFieldDefinition }> => {
+  const { modelType } = instances[0]?.dataValues ?? {};
+  const customFields = new Set<string>();
+  const modelIds = [];
+  const entityIds = new Set<string>();
+  instances.forEach((instance) => {
+    const { dataValues: { modelId, entityId, customFields: instanceCustomFields } } = instance;
+    modelIds.push(modelId);
+    entityIds.add(entityId);
+
+    Object.keys(instanceCustomFields ?? {}).forEach((fieldName) => {
+      customFields.add(fieldName);
+    });
+  });
+
+  const where: WhereOptions = {
+    modelType,
+    entityId: { [Op.in]: Array.from(entityIds) },
+    name: { [Op.in]: Array.from(customFields) },
+  };
+
+  const definitions = await findAll(where, { ...options });
+
+  const matchedDefinitions = definitions.filter((def) => customFields.has(def.name));
+  const matchedDefinitionsByName = Object.fromEntries(matchedDefinitions.map((definition) => [definition.name, definition]));
+
+  if (!definitions?.length || matchedDefinitions.length !== customFields.size) {
+    const unmatchedCustomFields = Array.from(customFields).filter((customField) => !matchedDefinitionsByName[customField]);
+    throw new MissingDefinitionError(unmatchedCustomFields);
+  }
+
+  return matchedDefinitionsByName;
+};

package/src/repository/entries.ts

@@ -0,0 +1,88 @@
+/* eslint-disable no-param-reassign */
+import type {
+  FindOptions,
+  Includeable,
+  Transaction,
+  WhereOptions,
+} from 'sequelize';
+import { CustomFieldEntries } from '../models';
+import type { ModelOptions } from '../types';
+import logger from '../utils/logger';
+import { MissingDefinitionError } from '../errors';
+import * as DefinitionRepo from './definition';
+import { formatFunctions } from './utils/formatValues';
+
+type CustomFieldEntriesModelOptions = ModelOptions & { include?: Includeable, transaction?: Transaction };
+
+export const findEntriesByModelId = async (modelId: string, options: CustomFieldEntriesModelOptions = {}) => {
+  const { transaction } = options;
+  return CustomFieldEntries.findOne({
+    where: { modelId },
+    transaction,
+  });
+};
+
+export const findEntriesByModelIds = async (modelIds: string[], options: CustomFieldEntriesModelOptions = {}) => {
+  const { transaction } = options;
+  return CustomFieldEntries.findAll({
+    where: { modelId: modelIds },
+    transaction,
+  });
+};
+
+export const updateEntries = async (
+  modelId: string,
+  modelType: string,
+  customFields: Record<string, any>,
+  identifiers: string[],
+  options: FindOptions & { modelOptions?: ModelOptions } = {},
+) => {
+  const customFieldsNames = Object.keys(customFields);
+  logger.debug(`custom-fields: updating entries for ${modelType} ${modelId}`, {
+    customFieldsNames,
+    optionsKeys: options ? Object.keys(options) : null,
+    customFields,
+    identifiers,
+  });
+  const { modelOptions, transaction } = options;
+
+  const where: WhereOptions = {
+    modelType,
+    name: customFieldsNames,
+    ...(!options.modelOptions?.useEntityIdFromInclude && { entityId: identifiers }),
+  };
+
+  const fieldDefinitions = await DefinitionRepo.findAll(where, { withDisabled: true, transaction, include: modelOptions.include?.(identifiers) }) ?? [];
+
+  const disabledDefinitions = fieldDefinitions.filter((def) => def.disabled);
+  if (fieldDefinitions.length !== customFieldsNames.length) {
+    logger.warn(`custom-fields: missing definitions for ${modelType} ${modelId}`, { names: customFieldsNames, fieldDefinitions });
+    const missingDefinitions = customFieldsNames.filter((name) => !fieldDefinitions.some((def) => def.name === name));
+    throw new MissingDefinitionError(missingDefinitions);
+  }
+
+  const disabledNames = disabledDefinitions?.map((def) => def.name) || [];
+  const valuesWithDisabledDefinitions = customFieldsNames.filter((name) => disabledNames.includes(name));
+  if (valuesWithDisabledDefinitions?.length > 0) {
+    logger.warn(`custom-fields: trying to update disabled values: ${valuesWithDisabledDefinitions.join(', ')}`);
+  }
+
+  const definitionsByName = Object.fromEntries(fieldDefinitions.map((definition) => [definition.name, definition]));
+  // If we need to format the value before we save it
+  Object.entries(customFields)
+    .filter(([definitionName]) => formatFunctions[definitionsByName[definitionName].fieldType])
+    .forEach(([definitionName, value]) => {
+      const { fieldType } = definitionsByName[definitionName];
+      customFields[definitionName] = formatFunctions[fieldType](value);
+    });
+
+  return CustomFieldEntries.upsert(
+    {
+      modelId,
+      entityId: fieldDefinitions[0].entityId,
+      modelType,
+      customFields,
+    },
+    options,
+  );
+};

package/src/repository/utils/formatValues.ts

@@ -0,0 +1,14 @@
+import { CustomFieldDefinitionType } from '../../utils/constants';
+
+export const formatFunctions = {
+  [CustomFieldDefinitionType.DATE]: (value) => {
+    if (value) {
+      const date = new Date(value);
+      if (date.toString() === 'Invalid Date') {
+        throw new Error(`Invalid date value: ${value}`);
+      }
+      return date.toISOString();
+    }
+    return null;
+  },
+};

package/src/repository/validator.ts

@@ -0,0 +1,104 @@
+import type { IncludeOptions, Transactionable } from 'sequelize';
+import logger from '../utils/logger';
+import { CustomValidator } from '../models';
+
+export interface FindValidatorOptions extends Transactionable {
+  withDisabled?: boolean;
+  attributes?: string[];
+  raw?: boolean;
+  include?: IncludeOptions[];
+}
+
+// Make sure this interface is compatible with the Sequelize model
+export interface ValidatorAttributes {
+  entityId: string;
+  entityType: string;
+  modelType: string;
+  schema: CustomValidator['schema'];
+  disabled?: boolean;
+  [key: string]: unknown; // Add index signature for Sequelize compatibility
+}
+
+export const create = async (
+  validatorAttributes: ValidatorAttributes,
+  options: Transactionable = {},
+): Promise<CustomValidator> => {
+  logger.debug('custom-validator - create validator');
+
+  // Use unknown type to bypass TypeScript errors while maintaining compatibility
+  const validator = await CustomValidator.create(validatorAttributes as Record<string, unknown>, options);
+
+  return validator;
+};
+
+export const findAll = async (
+  where = {},
+  options: FindValidatorOptions = { withDisabled: false },
+): Promise<CustomValidator[]> => {
+  logger.debug('custom-validator - find all validators');
+
+  const { transaction, withDisabled } = options;
+
+  let validators;
+  if (withDisabled) {
+    // If withDisabled is true, use unscoped to ignore the default scope that filters disabled items
+    // Apply the userScope separately to maintain permission filtering
+    validators = await CustomValidator.unscoped().scope('userScope').findAll({
+      where,
+      transaction,
+    });
+  } else {
+    // Use defaultScope and userScope to filter both disabled and by permissions
+    // The defaultScope keeps only non-disabled validators
+    validators = await CustomValidator.scope(['defaultScope', 'userScope']).findAll({
+      where,
+      transaction,
+    });
+  }
+
+  return validators;
+};
+
+export const findAllByModelType = async (
+  modelType: string,
+  entityId: string,
+  options: FindValidatorOptions = { withDisabled: false },
+): Promise<CustomValidator[]> => {
+  logger.debug('custom-validator - find all validators by model type');
+
+  return findAll(
+    {
+      modelType,
+      ...(!options.include && {
+        entityId,
+      }),
+    },
+    options,
+  );
+};
+
+export const update = async (
+  id: string,
+  updates: Partial<ValidatorAttributes>,
+  options?: Transactionable,
+): Promise<[number, CustomValidator[]]> => {
+  logger.debug('custom-validator - update validator');
+
+  return CustomValidator.update(
+    updates,
+    {
+      where: { id },
+      returning: true,
+      ...options,
+    },
+  );
+};
+
+export const disable = async (
+  id: string,
+  options?: Transactionable,
+): Promise<[number, CustomValidator[]]> => {
+  logger.debug('custom-validator - disable validator');
+
+  return update(id, { disabled: true }, options);
+};

package/src/repository/value.ts

@@ -1,4 +1,3 @@
-/* eslint-disable max-len */
 import type { FindOptions, WhereOptions } from 'sequelize';
 import { CustomFieldValue, CustomFieldDefinition } from '../models';
 import * as DefinitionRepo from './definition';
@@ -6,7 +5,7 @@ import type { CreateCustomFieldValue, ValuesToUpdate } from '../types/value';
 import logger from '../utils/logger';
 import { MissingDefinitionError } from '../errors';
 import type { ModelOptions } from '../types';
-import { CustomFieldDefinitionType } from '../utils/constants';
+import { formatFunctions } from './utils/formatValues';
 
 export const findByModelIdAndDefinition = async (modelId: string, customFieldDefinitionId: string) =>
   CustomFieldValue.findAll({ where: { modelId, customFieldDefinitionId }, include: [CustomFieldDefinition] });
@@ -43,19 +42,6 @@ export const findValuesByModelIds = async (modelIds: string[], options?): Promis
   });
 };
 
-const formatFunctions = {
-  [CustomFieldDefinitionType.DATE]: (value) => {
-    if (value) {
-      const date = new Date(value);
-      if (date.toString() === 'Invalid Date') {
-        throw new Error(`Invalid date value: ${value}`);
-      }
-      return date.toISOString();
-    }
-    return null;
-  },
-};
-
 /**
  * Try to update custom field values for a model instance.
  * Create new value record if not exists, but fails if value's definition not exist.
@@ -67,7 +53,6 @@ export const updateValues = async (
   identifiers: string[],
   valuesToUpdate: ValuesToUpdate,
   options: FindOptions & { modelOptions?: ModelOptions } = {},
-  defineAllDefaults = false,
 ): Promise<CustomFieldValue[]> => {
   const names = Object.keys(valuesToUpdate);
   logger.debug(`custom-fields: updating values for ${modelType} ${modelId}`, {
@@ -81,12 +66,10 @@ export const updateValues = async (
   const where: WhereOptions = {
     modelType,
     name: names,
+    ...(!options.modelOptions?.useEntityIdFromInclude && { entityId: identifiers }),
   };
 
-  if (!options.modelOptions?.useEntityIdFromInclude) {
-    where.entityId = identifiers;
-  }
-  const fieldDefinitions = await DefinitionRepo.findAll(where, { withDisabled: true, transaction, include: modelOptions.include?.(identifiers) }) || [];
+  const fieldDefinitions = await DefinitionRepo.findAll(where, { withDisabled: true, transaction, include: modelOptions.include?.(identifiers) }) ?? [];
 
   const disabledDefinitions = fieldDefinitions.filter((def) => def.disabled);
   if (fieldDefinitions.length !== names.length) {
@@ -101,31 +84,18 @@ export const updateValues = async (
     logger.warn(`custom-fields: trying to update disabled values: ${valuesWithDisabledDefinitions.join(', ')}`);
   }
 
-  const visitedFields = new Set<CustomFieldDefinition>();
-
   const values: CreateCustomFieldValue[] = names.map((name) => {
     const fieldDefinition = fieldDefinitions.find((def) => def.name === name);
-    visitedFields.add(fieldDefinition);
     const formatFunction = formatFunctions[fieldDefinition.fieldType];
+    const value = formatFunction ? formatFunction(valuesToUpdate[name]) : valuesToUpdate[name];
     return {
       modelId,
-      value: (formatFunction ? formatFunction(valuesToUpdate[name]) : valuesToUpdate[name]) ?? fieldDefinition.defaultValue,
       updatedAt: new Date(),
       customFieldDefinitionId: fieldDefinition.id,
+      value: value !== undefined ? value : fieldDefinition.defaultValue,
     };
   });
 
-  if (defineAllDefaults) {
-    fieldDefinitions.filter((def) => !visitedFields.has(def) && ![null, undefined].includes(def.defaultValue)).forEach(({ id, defaultValue }) => {
-      values.push({
-        modelId,
-        value: defaultValue,
-        updatedAt: new Date(),
-        customFieldDefinitionId: id,
-      });
-    });
-  }
-
   return Promise.all(values.map(async (value) => {
     const [cfv] = await CustomFieldValue.upsert(value, {
       transaction: options.transaction,

package/src/scopes/filter.ts

@@ -1,50 +1,26 @@
 /* eslint-disable import/prefer-default-export */
-import { Op, type WhereOptions } from 'sequelize';
+import { Op } from 'sequelize';
 import { Sequelize } from 'sequelize-typescript';
 import { customFields } from '@autofleet/common-types';
 import { generateRandomString } from '../utils/helpers';
+import type { CustomFieldOptions } from '../types';
+import {
+  formatConditionsForEntries,
+  formatConditionsForValues,
+  getFilterCustomFieldsSubQuery,
+  getSortCustomFieldsSubQuery,
+  SubQueryType,
+  type ConditionValue,
+  type CustomFieldFilterOptions,
+} from './helpers/filter.helpers';
 
 const { CUSTOM_FIELDS_FILTER_SCOPE } = customFields;
 
-/**
- * Type representing possible condition values.
- * Currently supporting strings and arrays of strings.
- * More types to be added (TBA).
- */
-type ConditionWithOperator = {
-    operator: string;
-    value: string;
-};
-export type ConditionValue = ConditionWithOperator | ConditionWithOperator[] | string | string[];
-
-export type CustomFieldSort = {
-    field: string;
-    direction: 'ASC' | 'DESC';
-}
-
-export type CustomFieldFilterOptions = {
-    where?: WhereOptions;
-    replacements?: Record<string, string>;
-}
-
 type customFieldsFilterScopeParams = {
     replacementsMap: Record<string, string>;
     scopeValue: Record<string, ConditionValue>;
 }
 
-const isDate = (input: any): input is Date => input instanceof Date || Object.prototype.toString.call(input) === '[object Date]';
-
-const castIfNeeded = (conditionValue: string): string => {
-  if (isDate(conditionValue)) {
-    return '::timestamp';
-  }
-  if (!Number.isNaN(Number(conditionValue))) {
-    return '::numeric';
-  }
-  return '';
-};
-const AND_DELIMETER = ' AND ';
-
 /**
  * A Sequelize scope for filtering models by custom fields.
  * This scope builds a WHERE clause to be applied on the main query.
@@ -54,73 +30,30 @@ const AND_DELIMETER = ' AND ';
  */
 export const customFieldsFilterScope = (
   name: string,
-) => (
-  {
-    replacementsMap: replacements,
-    scopeValue: conditions,
-  }: customFieldsFilterScopeParams,
-): CustomFieldFilterOptions => {
+  options?: Pick<CustomFieldOptions, 'useCustomFieldsEntries'>,
+) => ({ replacementsMap: replacements, scopeValue: conditions }: customFieldsFilterScopeParams): CustomFieldFilterOptions => {
   if (!conditions || Object.keys(conditions).length === 0) {
     return {};
   }
-  // Build the WHERE clause for custom field filtering
-  const conditionsStrings = Object.entries(conditions)
-    .map(
-      ([key, condition]) => {
-        const replacemetKey = Object.keys(replacements).find(
-          (randomString) => replacements[randomString] === key,
-        );
-        if (!replacemetKey) return false;
 
-        if (Array.isArray(condition)) {
-          if (condition.length === 0) {
-            // if empty array, the condition is ignored
-            return false;
-          }
-          if (typeof condition[0] === 'string') {
-            const values = condition.map((v) => {
-              const valRandom = Object.keys(replacements).find(
-                (randomString) => replacements[randomString] === v,
-              );
-              return ` :${valRandom} `;
-            }).join(',');
-            return `(custom_fields->> :${replacemetKey} ) IN ( ${values} )`;
-          }
-          return condition
-            .map((c) => {
-              const valRep = Object.keys(replacements).find(
-                (replacementKey) => replacements[replacementKey] === c.value,
-              );
-              return `(custom_fields->> :${replacemetKey} )${castIfNeeded(c.value)} ${c.operator} :${valRep}`;
-            }).join(AND_DELIMETER);
-        }
-        if (typeof condition === 'string' || typeof condition === 'number') {
-          const conditionRep = Object.keys(replacements).find(
-            (replacementKey) => replacements[replacementKey] === condition,
-          );
-          return `(custom_fields->> :${replacemetKey} ) ${castIfNeeded(condition)} = :${conditionRep}`;
-        }
-        if (condition?.operator) {
-          const valueRep = Object.keys(replacements).find(
-            (replacementKey) => replacements[replacementKey] === condition.value,
-          );
-          return `(custom_fields->> :${replacemetKey} ) ${castIfNeeded(condition.value)} ${condition.operator} :${valueRep}`;
-        }
+  const queryType = options?.useCustomFieldsEntries ? SubQueryType.ENTRIES : SubQueryType.VALUES;
+  const reverseReplacementsMap = new Map(Object.entries(replacements).map(([key, value]) => [value, key]));
+  // Build the WHERE clause for custom field filtering
+  const conditionsStrings = Object.entries(conditions).map(([key, condition]) => {
+    switch (queryType) {
+      case SubQueryType.ENTRIES:
+        return formatConditionsForEntries(key, condition, reverseReplacementsMap);
+      case SubQueryType.VALUES:
+        return formatConditionsForValues(key, condition, reverseReplacementsMap);
+      default:
         return false;
-      },
-    )
-    .filter(Boolean);
+    }
+  }).filter(Boolean);
   if (conditionsStrings.length === 0) {
     return {};
   }
-  const customFieldConditions = conditionsStrings.join(AND_DELIMETER);
-  const subQuery = `${'SELECT model_id FROM ('
-    + 'SELECT cv.model_id, jsonb_object_agg(cd.name, cv.value) AS custom_fields '
-    + 'FROM custom_field_values AS cv '
-    + 'INNER JOIN custom_field_definitions AS cd ON cv.custom_field_definition_id = cd.id '
-    + `AND cd.model_type = '${name}'`
-    + 'GROUP BY cv.model_id'
-    + ') AS CustomFieldAggregation WHERE '} ${customFieldConditions}`;
+  const subQuery = getFilterCustomFieldsSubQuery(queryType, name, conditionsStrings);
+
   return {
     where: {
       id: {
@@ -135,27 +68,21 @@ export const scopeName = CUSTOM_FIELDS_FILTER_SCOPE;
 
 export const customFieldsSortScope = (
   name: string,
+  options?: Pick<CustomFieldOptions, 'useCustomFieldsEntries'>,
 ) => ({ replacementsMap, scopeValue: sort }) => {
   if (!sort || sort.length === 0) {
     return {};
   }
+
+  const queryType = options?.useCustomFieldsEntries ? SubQueryType.ENTRIES : SubQueryType.VALUES;
   const randomStr = generateRandomString();
   const includes = Object.entries(sort).map(([key]) => {
-    const replacemetKey = Object.keys(replacementsMap).find(
+    const replacementKey = Object.keys(replacementsMap).find(
       (randomString) => replacementsMap[randomString] === key,
     );
     return ([
-      Sequelize.literal(`(
-            SELECT value
-            FROM (SELECT cv.model_id, cv.value
-                  FROM custom_field_values AS cv INNER JOIN custom_field_definitions AS cd
-                    ON cv.custom_field_definition_id = cd.id
-                    AND cd.model_type = '${name}'
-                    WHERE cv.model_id = "${name}"."id"
-                          AND cd.name = :${replacemetKey}
-                          ) AS CustomFieldAggregation
-            )
-          `), randomStr,
+      Sequelize.literal(getSortCustomFieldsSubQuery(queryType, name, replacementKey)),
+      randomStr,
     ]);
   });