@autofleet/sadot 0.10.2 → 0.10.3

package/dist/utils/init.d.ts

@@ -1,7 +1,7 @@
-import type { ModelFetcher, Models } from '../types';
+import type { CustomFieldOptions, ModelFetcher, Models } from '../types';
 export declare const addHooks: (models: Models[], getModel: ModelFetcher, sadotOptions?: {
     useCustomFieldsEntries: boolean;
 }) => void;
 export declare const removeHooks: (models: Models[], getModel: ModelFetcher) => void;
-export declare const addScopes: (models: Models[], getModel: ModelFetcher) => void;
+export declare const addScopes: (models: Models[], getModel: ModelFetcher, options?: Pick<CustomFieldOptions, 'useCustomFieldsEntries'>) => void;
 export declare const applyCustomAssociation: (models: Models[]) => void;

package/dist/utils/init.js

@@ -35,9 +35,9 @@ const addHooks = (models, getModel, sadotOptions = { useCustomFieldsEntries: fal
             model.addHook('beforeBulkUpdate', 'sadot-beforeBulkUpdate', hooks_1.beforeBulkUpdate);
             model.addHook('beforeCreate', 'sadot-beforeCreate', (0, hooks_1.beforeCreate)(scopeAttributes, modelOptions, sadotOptions));
             model.addHook('beforeUpdate', 'sadot-beforeUpdate', (0, hooks_1.beforeUpdate)(scopeAttributes, modelOptions, sadotOptions));
-            model.addHook('afterFind', 'sadot-afterFind', (0, hooks_1.enrichResults)(name, scopeAttributes, 'afterFind', modelOptions));
-            model.addHook('afterUpdate', 'sadot-afterUpdate', (0, hooks_1.enrichResults)(name, scopeAttributes, null, modelOptions));
-            model.addHook('afterCreate', 'sadot-afterCreate', (0, hooks_1.enrichResults)(name, scopeAttributes, null, modelOptions));
+            model.addHook('afterFind', 'sadot-afterFind', (0, hooks_1.enrichResults)(name, scopeAttributes, 'afterFind', modelOptions, sadotOptions));
+            model.addHook('afterUpdate', 'sadot-afterUpdate', (0, hooks_1.enrichResults)(name, scopeAttributes, null, modelOptions, sadotOptions));
+            model.addHook('afterCreate', 'sadot-afterCreate', (0, hooks_1.enrichResults)(name, scopeAttributes, null, modelOptions, sadotOptions));
         }
         catch (e) {
             logger_1.default.error(`Could not add custom fields hook to model ${name}. `, e);
@@ -78,7 +78,7 @@ const addAssociations = (model, modelName) => {
     // TBC: maybe can be removed
     models_1.CustomFieldValue.belongsTo(model, { foreignKey: 'modelId', as: modelName });
 };
-const addScopes = (models, getModel) => {
+const addScopes = (models, getModel, options = { useCustomFieldsEntries: false }) => {
     models.forEach(async ({ name, scopeAttributes }) => {
         try {
             const model = getModel(name);
@@ -92,8 +92,8 @@ const addScopes = (models, getModel) => {
             // Necessary associations for the filtering scope
             addAssociations(model, name);
             // Add filter scope
-            model.addScope(CUSTOM_FIELDS_FILTER_SCOPE, (0, scopes_1.customFieldsFilterScope)(name));
-            model.addScope(custom_fields_1.CUSTOM_FIELDS_SORT_SCOPE, (0, filter_1.customFieldsSortScope)(name));
+            model.addScope(CUSTOM_FIELDS_FILTER_SCOPE, (0, scopes_1.customFieldsFilterScope)(name, options));
+            model.addScope(custom_fields_1.CUSTOM_FIELDS_SORT_SCOPE, (0, filter_1.customFieldsSortScope)(name, options));
         }
         catch (e) {
             logger_1.default.error(`Could not add custom fields scopes to model ${name}. `, e);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autofleet/sadot",
-  "version": "0.10.2",
+  "version": "0.10.3",
   "description": "",
   "main": "dist/index.js",
   "scripts": {

package/src/hooks/create.ts

@@ -45,7 +45,7 @@ export const beforeCreate = (
   if (fieldsWithDefaultValue.length) {
     // eslint-disable-next-line no-param-reassign
     instance.customFields ||= {};
-    fieldsWithDefaultValue.filter((def) => !(def.name in instance.customFields)).forEach(({ name, defaultValue }) => {
+    fieldsWithDefaultValue.filter((def) => (instance.customFields?.[def.name] === undefined)).forEach(({ name, defaultValue }) => {
       // eslint-disable-next-line no-param-reassign
       instance.customFields[name] = defaultValue;
     });

package/src/hooks/enrich.ts

@@ -1,14 +1,86 @@
 /* eslint-disable no-param-reassign */
+import type { Transaction } from 'sequelize';
 import * as ValueRepo from '../repository/value';
 import * as DefinitionRepo from '../repository/definition';
+import * as EntriesRepo from '../repository/entries';
 import type CustomFieldValue from '../models/CustomFieldValue';
 import type CustomFieldDefinition from '../models/CustomFieldDefinition';
 import type { SerializedCustomFields } from '../types/definition';
-import type { ModelOptions } from '../types';
+import type { CustomFieldOptions, ModelOptions } from '../types';
 import applyScopeToInstance from '../utils/scopeAttributes';
 
 type SupportedHookTypes = 'afterFind' | 'afterCreate' | 'afterUpdate';
 
+type CustomFieldEntries = Record<string, any>;
+
+interface GetValuesGroupByInstanceResponse {
+  [modelId: string]: CustomFieldValue[];
+}
+
+interface GetCustomFieldEntriesByInstanceIdResponse {
+  [modelId: string]: CustomFieldEntries;
+}
+
+export const getCustomFieldEntriesByInstanceId = async ({
+  instancesIds,
+  options,
+  sadotOptions,
+}: {
+  instancesIds: string[],
+  options?: { transaction: Transaction },
+  sadotOptions: Pick<CustomFieldOptions, 'useCustomFieldsEntries'>,
+}): Promise<GetCustomFieldEntriesByInstanceIdResponse> => {
+  if (!sadotOptions.useCustomFieldsEntries) {
+    return {};
+  }
+
+  const customFieldEntries = await EntriesRepo.findEntriesByModelIds(
+    instancesIds,
+    options ?? {},
+  );
+
+  const customFieldEntriesByInstanceId = Object.fromEntries(customFieldEntries.map((instanceEntries) => {
+    const { modelId, customFields } = instanceEntries?.dataValues ?? {};
+    if (!modelId) {
+      return undefined;
+    }
+    return [modelId, customFields];
+  }).filter(Boolean));
+
+  instancesIds.forEach((instanceId) => {
+    customFieldEntriesByInstanceId[instanceId] ??= {};
+  });
+
+  return customFieldEntriesByInstanceId;
+};
+
+export const getValuesGroupByInstance = async ({
+  instancesIds,
+  options,
+  sadotOptions,
+}: {
+  instancesIds: string[],
+  options?: { transaction: Transaction },
+  sadotOptions: Pick<CustomFieldOptions, 'useCustomFieldsEntries'>,
+}): Promise<GetValuesGroupByInstanceResponse> => {
+  if (sadotOptions.useCustomFieldsEntries) {
+    return {};
+  }
+
+  const customFieldValues = await ValueRepo.findValuesByModelIds(
+    instancesIds,
+    options ?? {},
+  );
+
+  // Group fields by modelId
+  return customFieldValues.reduce((acc, v) => {
+    const { modelId } = v;
+    acc[modelId] ??= [];
+    acc[modelId].push(v);
+    return acc;
+  }, {});
+};
+
 /**
  * Serialize custom fields value into the format of {[name] -> [fieldData]}
  */
@@ -33,6 +105,7 @@ const enrichResults = (
   scopeAttributes: string[],
   hookType?: SupportedHookTypes,
   modelOptions: ModelOptions = {},
+  sadotOptions: Pick<CustomFieldOptions, 'useCustomFieldsEntries'> = { useCustomFieldsEntries: false },
 ) => async (
   instancesOrInstance: any | any[],
   options,
@@ -90,32 +163,29 @@ const enrichResults = (
   // Get the values per instates ids:
   const instancesIds = instances.map((i) => i[primaryKey]);
 
-  const customFieldValues = await ValueRepo.findValuesByModelIds(
+  // Group fields by modelId
+  const valuesGroupByInstance = await getValuesGroupByInstance({
     instancesIds,
-    { transaction: options.transaction },
-  );
+    options,
+    sadotOptions,
+  });
 
-  // Group fields by modelId
-  const valuesGroupByInstance: {
-      [modelId: string]: CustomFieldValue[];
-    } = customFieldValues.reduce((acc, v) => {
-      const { modelId } = v;
-      if (!acc[modelId]) {
-        acc[modelId] = [];
-      }
-      acc[modelId].push(v);
-      return acc;
-    }, {});
+  const customFieldEntriesByInstanceId = await getCustomFieldEntriesByInstanceId({
+    instancesIds,
+    options,
+    sadotOptions,
+  });
 
   // Attach custom fields to the instances
   instances.forEach((instance) => {
-    const customFields = {};
     const { id } = instance;
+
     const instanceValues = valuesGroupByInstance[id];
-    if (instanceValues) {
-      const serializedCustomFields = serializeCustomFields(instanceValues, definitionsMap);
-      Object.assign(customFields, serializedCustomFields);
-    }
+    const serializedCustomFieldsValues = instanceValues ? serializeCustomFields(instanceValues, definitionsMap) : {};
+
+    const customFields = sadotOptions.useCustomFieldsEntries
+      ? customFieldEntriesByInstanceId[id]
+      : serializedCustomFieldsValues;
 
     scopeAttributes.forEach((attribute) => {
       const identifier = instance[attribute];

package/src/index.ts

@@ -37,7 +37,7 @@ const useCustomFields = async (
   // The order is important
   addHooks(models, getModel, { useCustomFieldsEntries });
   await initTables(sequelize, options.getUser, { useCustomFieldsEntries });
-  addScopes(models, getModel);
+  addScopes(models, getModel, { useCustomFieldsEntries });
   applyCustomAssociation(models);
   logger.debug('sadot - custom fields finished initializing with models', models);
   return sequelize;

package/src/repository/entries.ts

@@ -1,3 +1,4 @@
+/* eslint-disable no-param-reassign */
 import type {
   FindOptions,
   Includeable,
@@ -9,6 +10,7 @@ import type { ModelOptions } from '../types';
 import logger from '../utils/logger';
 import { MissingDefinitionError } from '../errors';
 import * as DefinitionRepo from './definition';
+import { formatFunctions } from './utils/formatValues';
 
 type CustomFieldEntriesModelOptions = ModelOptions & { include?: Includeable, transaction?: Transaction };
 
@@ -65,6 +67,15 @@ export const updateEntries = async (
     logger.warn(`custom-fields: trying to update disabled values: ${valuesWithDisabledDefinitions.join(', ')}`);
   }
 
+  const definitionsByName = Object.fromEntries(fieldDefinitions.map((definition) => [definition.name, definition]));
+  // If we need to format the value before we save it
+  Object.entries(customFields)
+    .filter(([definitionName]) => formatFunctions[definitionsByName[definitionName].fieldType])
+    .forEach(([definitionName, value]) => {
+      const { fieldType } = definitionsByName[definitionName];
+      customFields[definitionName] = formatFunctions[fieldType](value);
+    });
+
   return CustomFieldEntries.upsert(
     {
       modelId,

package/src/repository/utils/formatValues.ts

@@ -0,0 +1,14 @@
+import { CustomFieldDefinitionType } from '../../utils/constants';
+
+export const formatFunctions = {
+  [CustomFieldDefinitionType.DATE]: (value) => {
+    if (value) {
+      const date = new Date(value);
+      if (date.toString() === 'Invalid Date') {
+        throw new Error(`Invalid date value: ${value}`);
+      }
+      return date.toISOString();
+    }
+    return null;
+  },
+};

package/src/repository/value.ts

@@ -5,7 +5,7 @@ import type { CreateCustomFieldValue, ValuesToUpdate } from '../types/value';
 import logger from '../utils/logger';
 import { MissingDefinitionError } from '../errors';
 import type { ModelOptions } from '../types';
-import { CustomFieldDefinitionType } from '../utils/constants';
+import { formatFunctions } from './utils/formatValues';
 
 export const findByModelIdAndDefinition = async (modelId: string, customFieldDefinitionId: string) =>
   CustomFieldValue.findAll({ where: { modelId, customFieldDefinitionId }, include: [CustomFieldDefinition] });
@@ -42,19 +42,6 @@ export const findValuesByModelIds = async (modelIds: string[], options?): Promis
   });
 };
 
-const formatFunctions = {
-  [CustomFieldDefinitionType.DATE]: (value) => {
-    if (value) {
-      const date = new Date(value);
-      if (date.toString() === 'Invalid Date') {
-        throw new Error(`Invalid date value: ${value}`);
-      }
-      return date.toISOString();
-    }
-    return null;
-  },
-};
-
 /**
  * Try to update custom field values for a model instance.
  * Create new value record if not exists, but fails if value's definition not exist.

package/src/scopes/filter.ts

@@ -1,58 +1,26 @@
 /* eslint-disable import/prefer-default-export */
-import { Op, type WhereOptions } from 'sequelize';
+import { Op } from 'sequelize';
 import { Sequelize } from 'sequelize-typescript';
 import { customFields } from '@autofleet/common-types';
 import { generateRandomString } from '../utils/helpers';
+import type { CustomFieldOptions } from '../types';
+import {
+  formatConditionsForEntries,
+  formatConditionsForValues,
+  getFilterCustomFieldsSubQuery,
+  getSortCustomFieldsSubQuery,
+  SubQueryType,
+  type ConditionValue,
+  type CustomFieldFilterOptions,
+} from './helpers/filter.helpers';
 
 const { CUSTOM_FIELDS_FILTER_SCOPE } = customFields;
 
-/**
- * Type representing possible condition values.
- * Currently supporting strings and arrays of strings.
- * More types to be added (TBA).
- */
-type ConditionWithOperator = {
-    operator: string;
-    value: string;
-};
-export type ConditionValue = ConditionWithOperator | ConditionWithOperator[] | string | string[];
-
-export type CustomFieldSort = {
-    field: string;
-    direction: 'ASC' | 'DESC';
-}
-
-export type CustomFieldFilterOptions = {
-    where?: WhereOptions;
-    replacements?: Record<string, string>;
-}
-
 type customFieldsFilterScopeParams = {
     replacementsMap: Record<string, string>;
     scopeValue: Record<string, ConditionValue>;
 }
 
-const isConditionStringArray = (input: any): input is string[] => Array.isArray(input) && typeof input[0] === 'string';
-const isBooleanString = (input: string): boolean => ['true', 'false'].includes(input.toString());
-const isDate = (input: any): input is Date => input instanceof Date || Object.prototype.toString.call(input) === '[object Date]';
-
-const castValueToJsonb = (value: string, type: string) => `to_jsonb(${value}::${type})`;
-const castValueToJsonbText = (value: string) => castValueToJsonb(value, 'text');
-const castValueToJsonbBoolean = (value: string) => castValueToJsonb(value, 'boolean');
-const castValueToJsonbNumeric = (value: string) => castValueToJsonb(value, 'numeric');
-const castIfNeeded = (columnName: string, conditionValue: string): string => {
-  if (isDate(conditionValue)) {
-    return castValueToJsonb(columnName, 'timestamp');
-  }
-  return columnName;
-};
-const AND_DELIMITER = ' AND ';
-const OR_DELIMITER = ' OR ';
-const CD_TABLE_ALIAS = 'cd';
-const CD_NAME_COLUMN = `${CD_TABLE_ALIAS}.name`;
-const CV_TABLE_ALIAS = 'cv';
-const CV_VALUE_COLUMN = `${CV_TABLE_ALIAS}.value`;
-
 /**
  * A Sequelize scope for filtering models by custom fields.
  * This scope builds a WHERE clause to be applied on the main query.
@@ -62,68 +30,29 @@ const CV_VALUE_COLUMN = `${CV_TABLE_ALIAS}.value`;
  */
 export const customFieldsFilterScope = (
   name: string,
+  options?: Pick<CustomFieldOptions, 'useCustomFieldsEntries'>,
 ) => ({ replacementsMap: replacements, scopeValue: conditions }: customFieldsFilterScopeParams): CustomFieldFilterOptions => {
   if (!conditions || Object.keys(conditions).length === 0) {
     return {};
   }
+
+  const queryType = options?.useCustomFieldsEntries ? SubQueryType.ENTRIES : SubQueryType.VALUES;
   const reverseReplacementsMap = new Map(Object.entries(replacements).map(([key, value]) => [value, key]));
   // Build the WHERE clause for custom field filtering
   const conditionsStrings = Object.entries(conditions).map(([key, condition]) => {
-    const replacementKey = reverseReplacementsMap.get(key);
-    if (!replacementKey) return false;
-    const columnCondition = `(${CD_NAME_COLUMN} = :${replacementKey})`;
-
-    if (Array.isArray(condition)) {
-      if (condition.length === 0) {
-        // if empty array, the condition is ignored
+    switch (queryType) {
+      case SubQueryType.ENTRIES:
+        return formatConditionsForEntries(key, condition, reverseReplacementsMap);
+      case SubQueryType.VALUES:
+        return formatConditionsForValues(key, condition, reverseReplacementsMap);
+      default:
         return false;
-      }
-      if (isConditionStringArray(condition)) {
-        const values = condition.flatMap((v) => {
-          const valRandom = reverseReplacementsMap.get(v);
-          if (isBooleanString(v)) {
-            return [castValueToJsonbText(`:${valRandom}`), castValueToJsonbBoolean(`:${valRandom}`)];
-          }
-          if (!Number.isNaN(Number(v))) {
-            return castValueToJsonbNumeric(`:${valRandom}`);
-          }
-          return castValueToJsonbText(`:${valRandom}`);
-        }).join(',');
-        return `(${columnCondition}${AND_DELIMITER}${CV_VALUE_COLUMN} IN (${values}))`;
-      }
-      return condition.map((c) => {
-        const valRep = reverseReplacementsMap.get(c.value);
-        const valueAsJsonb = castValueToJsonbText(`:${valRep}`);
-        return `(${columnCondition}${AND_DELIMITER}${castIfNeeded(CV_VALUE_COLUMN, c.value)} ${c.operator} ${valueAsJsonb})`;
-      }).join(AND_DELIMITER);
-    }
-    if (typeof condition === 'string' || typeof condition === 'number') {
-      const conditionRep = reverseReplacementsMap.get(condition);
-      const valueAsJsonb = !Number.isNaN(Number(condition)) ? castValueToJsonbNumeric(`:${conditionRep}`) : castValueToJsonbText(`:${conditionRep}`);
-      const valueAsJsonbBoolean = isBooleanString(condition) ? `${OR_DELIMITER}${CV_VALUE_COLUMN} = ${castValueToJsonbBoolean(`:${conditionRep}`)}` : '';
-      return `(${columnCondition}${AND_DELIMITER}(${castIfNeeded(CV_VALUE_COLUMN, condition)} = ${valueAsJsonb}${valueAsJsonbBoolean}))`;
     }
-    if (condition?.operator) {
-      const valueRep = reverseReplacementsMap.get(condition.value);
-      const valueAsJsonb = castValueToJsonbText(`:${valueRep}`);
-      return `( ${columnCondition}${AND_DELIMITER}${castIfNeeded(CV_VALUE_COLUMN, condition.value)} ${condition.operator} ${valueAsJsonb})`;
-    }
-    return false;
   }).filter(Boolean);
   if (conditionsStrings.length === 0) {
     return {};
   }
-  const customFieldConditions = conditionsStrings.join(OR_DELIMITER);
-  const subQuery = `
-    SELECT cv.model_id
-    FROM custom_field_values AS cv
-    INNER JOIN custom_field_definitions AS cd ON cv.custom_field_definition_id = cd.id
-    ${AND_DELIMITER}cd.model_type = '${name}'
-    WHERE ${customFieldConditions}
-    ${AND_DELIMITER}cv.deleted_at IS NULL${AND_DELIMITER}cd.deleted_at IS NULL
-    GROUP BY cv.model_id
-    HAVING COUNT(DISTINCT cv.custom_field_definition_id) = ${conditionsStrings.length}
-  `.replace(/\n/g, '');
+  const subQuery = getFilterCustomFieldsSubQuery(queryType, name, conditionsStrings);
 
   return {
     where: {
@@ -139,27 +68,21 @@ export const scopeName = CUSTOM_FIELDS_FILTER_SCOPE;
 
 export const customFieldsSortScope = (
   name: string,
+  options?: Pick<CustomFieldOptions, 'useCustomFieldsEntries'>,
 ) => ({ replacementsMap, scopeValue: sort }) => {
   if (!sort || sort.length === 0) {
     return {};
   }
+
+  const queryType = options?.useCustomFieldsEntries ? SubQueryType.ENTRIES : SubQueryType.VALUES;
   const randomStr = generateRandomString();
   const includes = Object.entries(sort).map(([key]) => {
-    const replacemetKey = Object.keys(replacementsMap).find(
+    const replacementKey = Object.keys(replacementsMap).find(
       (randomString) => replacementsMap[randomString] === key,
     );
     return ([
-      Sequelize.literal(`(
-            SELECT value
-            FROM (SELECT cv.model_id, cv.value
-                  FROM custom_field_values AS cv INNER JOIN custom_field_definitions AS cd
-                    ON cv.custom_field_definition_id = cd.id
-                    ${AND_DELIMITER}cd.model_type = '${name}'
-                    WHERE cv.model_id = "${name}"."id"
-                          ${AND_DELIMITER}cd.name = :${replacemetKey}
-                          ) AS CustomFieldAggregation
-            )
-          `), randomStr,
+      Sequelize.literal(getSortCustomFieldsSubQuery(queryType, name, replacementKey)),
+      randomStr,
     ]);
   });