@autofleet/sadot 0.7.7-beta-0ecad376.0 → 0.7.7-beta.1

package/dist/hooks/create.js

@@ -48,27 +48,43 @@ exports.beforeBulkCreate = beforeBulkCreate;
 const beforeCreate = (scopeAttributes, modelOptions = {}) => async (instance, options) => {
     logger_1.default.debug('sadot - before create hook');
     const { fields } = options;
+    const { include, useEntityIdFromInclude } = modelOptions;
     const modelType = instance.constructor.name;
     const identifiers = (0, scopeAttributes_1.default)(instance, scopeAttributes);
-    // get all model's required definitions
-    const requiredFieldsNames = await DefinitionRepo.getRequiredFields(modelType, instance.id, identifiers, modelOptions);
+    const where = {
+        modelType,
+        disabled: false,
+        ...(!useEntityIdFromInclude && { entityId: identifiers }),
+    };
+    const fieldDefinitions = await DefinitionRepo.findAll(where, { withDisabled: false, transaction: options.transaction, include: include?.(identifiers) });
+    const requiredFieldsNames = Array.from(new Set(fieldDefinitions.filter(({ required }) => required).map(({ name }) => name)));
     const customFieldsIdx = fields.indexOf('customFields');
-    const { customFields } = instance;
-    if (customFieldsIdx > -1 && customFields) {
-        const fieldsNames = Object.keys(customFields);
-        const missingFields = requiredFieldsNames.filter((name) => !fieldsNames.includes(name));
-        if (missingFields?.length > 0) {
-            throw new errors_1.MissingRequiredCustomFieldError(missingFields);
-        }
-        await ValueRepo.updateValues(modelType, instance.id, identifiers, customFields, {
-            transaction: options.transaction,
-            modelOptions,
-        }, true);
+    if ((customFieldsIdx === -1 || !instance.customFields) && requiredFieldsNames?.length > 0) {
+        throw new errors_1.MissingRequiredCustomFieldError(requiredFieldsNames);
+    }
+    const fieldsWithDefaultValue = fieldDefinitions.filter((def) => ![null, undefined].includes(def.defaultValue));
+    if (fieldsWithDefaultValue.length) {
         // eslint-disable-next-line no-param-reassign
-        fields.splice(customFieldsIdx, 1);
+        instance.customFields || (instance.customFields = {});
+        fieldsWithDefaultValue.filter((def) => !(def.name in instance.customFields)).forEach(({ name, defaultValue }) => {
+            // eslint-disable-next-line no-param-reassign
+            instance.customFields[name] = defaultValue;
+        });
     }
-    else if (requiredFieldsNames?.length > 0) {
-        throw new errors_1.MissingRequiredCustomFieldError(requiredFieldsNames);
+    const { customFields } = instance;
+    if (customFieldsIdx === -1 || !customFields) {
+        return;
+    }
+    const fieldsNames = Object.keys(customFields);
+    const missingFields = requiredFieldsNames.filter((name) => !fieldsNames.includes(name));
+    if (missingFields?.length > 0) {
+        throw new errors_1.MissingRequiredCustomFieldError(missingFields);
     }
+    await ValueRepo.updateValues(modelType, instance.id, identifiers, customFields, {
+        transaction: options.transaction,
+        modelOptions,
+    });
+    // eslint-disable-next-line no-param-reassign
+    fields.splice(customFieldsIdx, 1);
 };
 exports.beforeCreate = beforeCreate;

package/dist/repository/value.d.ts

@@ -24,5 +24,5 @@ export declare const findValuesByModelIds: (modelIds: string[], options?: any) =
  */
 export declare const updateValues: (modelType: string, modelId: string, identifiers: string[], valuesToUpdate: ValuesToUpdate, options?: FindOptions & {
     modelOptions?: ModelOptions;
-}, defineAllDefaults?: boolean) => Promise<CustomFieldValue[]>;
+}) => Promise<CustomFieldValue[]>;
 export declare const deleteValue: (id: string, options?: any) => Promise<any>;

package/dist/repository/value.js

@@ -84,7 +84,7 @@ const formatFunctions = {
  * Create new value record if not exists, but fails if value's definition not exist.
  * Return the updated values
  */
-const updateValues = async (modelType, modelId, identifiers, valuesToUpdate, options = {}, defineAllDefaults = false) => {
+const updateValues = async (modelType, modelId, identifiers, valuesToUpdate, options = {}) => {
     const names = Object.keys(valuesToUpdate);
     logger_1.default.debug(`custom-fields: updating values for ${modelType} ${modelId}`, {
         names,
@@ -95,12 +95,10 @@ const updateValues = async (modelType, modelId, identifiers, valuesToUpdate, opt
     const { modelOptions, transaction } = options;
     const where = {
         modelType,
-        ...(!defineAllDefaults && { name: names }),
+        name: names,
         ...(!options.modelOptions?.useEntityIdFromInclude && { entityId: identifiers }),
     };
-    const allFieldDefinitions = await DefinitionRepo.findAll(where, { withDisabled: true, transaction, include: modelOptions.include?.(identifiers) }) || [];
-    const namesSet = new Set(defineAllDefaults ? names : undefined);
-    const fieldDefinitions = defineAllDefaults ? allFieldDefinitions.filter((cfd) => namesSet.has(cfd.name)) : allFieldDefinitions;
+    const fieldDefinitions = await DefinitionRepo.findAll(where, { withDisabled: true, transaction, include: modelOptions.include?.(identifiers) }) ?? [];
     const disabledDefinitions = fieldDefinitions.filter((def) => def.disabled);
     if (fieldDefinitions.length !== names.length) {
         logger_1.default.warn(`custom-fields: missing definitions for ${modelType} ${modelId}`, { names, fieldDefinitions });
@@ -112,28 +110,17 @@ const updateValues = async (modelType, modelId, identifiers, valuesToUpdate, opt
     if (valuesWithDisabledDefinitions?.length > 0) {
         logger_1.default.warn(`custom-fields: trying to update disabled values: ${valuesWithDisabledDefinitions.join(', ')}`);
     }
-    const visitedFields = new Set();
     const values = names.map((name) => {
         const fieldDefinition = fieldDefinitions.find((def) => def.name === name);
-        visitedFields.add(fieldDefinition);
         const formatFunction = formatFunctions[fieldDefinition.fieldType];
+        const value = formatFunction ? formatFunction(valuesToUpdate[name]) : valuesToUpdate[name];
         return {
             modelId,
-            value: (formatFunction ? formatFunction(valuesToUpdate[name]) : valuesToUpdate[name]) ?? fieldDefinition.defaultValue,
             updatedAt: new Date(),
             customFieldDefinitionId: fieldDefinition.id,
+            value: value !== undefined ? value : fieldDefinition.defaultValue,
         };
     });
-    if (defineAllDefaults) {
-        allFieldDefinitions.filter((def) => !visitedFields.has(def) && ![null, undefined].includes(def.defaultValue)).forEach(({ id, defaultValue }) => {
-            values.push({
-                modelId,
-                value: defaultValue,
-                updatedAt: new Date(),
-                customFieldDefinitionId: id,
-            });
-        });
-    }
     return Promise.all(values.map(async (value) => {
         const [cfv] = await models_1.CustomFieldValue.upsert(value, {
             transaction: options.transaction,

package/dist/scopes/filter.js

@@ -1,4 +1,7 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.customFieldsSortScope = exports.scopeName = exports.customFieldsFilterScope = void 0;
 /* eslint-disable import/prefer-default-export */
@@ -6,6 +9,7 @@ const sequelize_1 = require("sequelize");
 const sequelize_typescript_1 = require("sequelize-typescript");
 const common_types_1 = require("@autofleet/common-types");
 const helpers_1 = require("../utils/helpers");
+const logger_1 = __importDefault(require("../utils/logger"));
 const { CUSTOM_FIELDS_FILTER_SCOPE } = common_types_1.customFields;
 const isDate = (input) => input instanceof Date || Object.prototype.toString.call(input) === '[object Date]';
 const castIfNeeded = (conditionValue) => {
@@ -17,7 +21,18 @@ const castIfNeeded = (conditionValue) => {
     }
     return '';
 };
+const addQuotationIfNeeded = (conditionValue, valRep) => {
+    if (typeof conditionValue === 'string') {
+        return `'"' || :${valRep} || '"'`;
+    }
+    return `:${valRep}`;
+};
 const AND_DELIMETER = ' AND ';
+const OR_DELIMETER = ' OR ';
+const CD_TABLE_ALIAS = 'cd';
+const CD_NAME_COLUMN = `${CD_TABLE_ALIAS}.name`;
+const CV_TABLE_ALIAS = 'cv';
+const CV_VALUE_COLUMN = `(${CV_TABLE_ALIAS}.value::varchar)`;
 /**
  * A Sequelize scope for filtering models by custom fields.
  * This scope builds a WHERE clause to be applied on the main query.
@@ -35,6 +50,7 @@ const customFieldsFilterScope = (name) => ({ replacementsMap: replacements, scop
         const replacemetKey = Object.keys(replacements).find((randomString) => replacements[randomString] === key);
         if (!replacemetKey)
             return false;
+        const columnCondition = `(${CD_NAME_COLUMN} = :${replacemetKey})`;
         if (Array.isArray(condition)) {
             if (condition.length === 0) {
                 // if empty array, the condition is ignored
@@ -43,23 +59,23 @@ const customFieldsFilterScope = (name) => ({ replacementsMap: replacements, scop
             if (typeof condition[0] === 'string') {
                 const values = condition.map((v) => {
                     const valRandom = Object.keys(replacements).find((randomString) => replacements[randomString] === v);
-                    return ` :${valRandom} `;
+                    return addQuotationIfNeeded(v, valRandom);
                 }).join(',');
-                return `(custom_fields->> :${replacemetKey} ) IN ( ${values} )`;
+                return `(${columnCondition} AND ${CV_VALUE_COLUMN} IN ( ${values} ))`;
             }
             return condition
                 .map((c) => {
                 const valRep = Object.keys(replacements).find((replacementKey) => replacements[replacementKey] === c.value);
-                return `(custom_fields->> :${replacemetKey} )${castIfNeeded(c.value)} ${c.operator} :${valRep}`;
+                return `( ${columnCondition} AND ${CV_VALUE_COLUMN}${castIfNeeded(c.value)} ${c.operator} ${addQuotationIfNeeded(c.value, valRep)} )`;
             }).join(AND_DELIMETER);
         }
         if (typeof condition === 'string' || typeof condition === 'number') {
             const conditionRep = Object.keys(replacements).find((replacementKey) => replacements[replacementKey] === condition);
-            return `(custom_fields->> :${replacemetKey} ) ${castIfNeeded(condition)} = :${conditionRep}`;
+            return `(${columnCondition} AND ${CV_VALUE_COLUMN}${castIfNeeded(condition)} = ${addQuotationIfNeeded(condition, conditionRep)})`;
         }
         if (condition?.operator) {
             const valueRep = Object.keys(replacements).find((replacementKey) => replacements[replacementKey] === condition.value);
-            return `(custom_fields->> :${replacemetKey} ) ${castIfNeeded(condition.value)} ${condition.operator} :${valueRep}`;
+            return `( ${columnCondition} AND ${CV_VALUE_COLUMN}${castIfNeeded(condition.value)} ${condition.operator} ${addQuotationIfNeeded(condition.value, valueRep)})`;
         }
         return false;
     })
@@ -67,14 +83,17 @@ const customFieldsFilterScope = (name) => ({ replacementsMap: replacements, scop
     if (conditionsStrings.length === 0) {
         return {};
     }
-    const customFieldConditions = conditionsStrings.join(AND_DELIMETER);
-    const subQuery = `${'SELECT model_id FROM ('
-        + 'SELECT cv.model_id, jsonb_object_agg(cd.name, cv.value) AS custom_fields '
-        + 'FROM custom_field_values AS cv '
-        + 'INNER JOIN custom_field_definitions AS cd ON cv.custom_field_definition_id = cd.id '
-        + `AND cd.model_type = '${name}'`
-        + 'GROUP BY cv.model_id'
-        + ') AS CustomFieldAggregation WHERE '} ${customFieldConditions}`;
+    const customFieldConditions = conditionsStrings.join(OR_DELIMETER);
+    const subQuery = `
+  SELECT cv.model_id
+  FROM custom_field_values AS cv
+  INNER JOIN custom_field_definitions AS cd ON cv.custom_field_definition_id = cd.id
+  AND cd.model_type = '${name}'
+  WHERE ${customFieldConditions}
+  GROUP BY cv.model_id
+  HAVING COUNT(DISTINCT cv.custom_field_definition_id) = ${conditionsStrings.length}
+`.replace(/\n/g, '');
+    logger_1.default.info('custom fields filter scope', { subQuery, replacements });
     return {
         where: {
             id: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@autofleet/sadot",
-  "version": "0.7.7-beta-0ecad376.0",
+  "version": "0.7.7-beta.1",
   "description": "",
   "main": "dist/index.js",
   "scripts": {
@@ -8,6 +8,7 @@
     "build": "rm -rf dist && tsc",
     "linter": "eslint .",
     "test": "jest --forceExit --runInBand",
+    "test-debug": "node --inspect-brk node_modules/.bin/jest --runInBand --testTimeout=10000000",
     "coverage": "jest --coverage --forceExit --runInBand && rm -rf ./coverage",
     "build-to-local-repo": "node --run build && cp -r dist/* ../$REPO/node_modules/$npm_package_name/dist",
     "dev": "nodemon",

package/src/hooks/create.ts

@@ -1,3 +1,4 @@
+import type { WhereOptions } from 'sequelize';
 import logger from '../utils/logger';
 import * as ValueRepo from '../repository/value';
 import * as DefinitionRepo from '../repository/definition';
@@ -23,37 +24,54 @@ export const beforeCreate = (scopeAttributes: string[], modelOptions: ModelOptio
 ): Promise<void> => {
   logger.debug('sadot - before create hook');
   const { fields } = options;
+  const { include, useEntityIdFromInclude } = modelOptions;
   const modelType = instance.constructor.name;
 
   const identifiers = applyScopeToInstance(instance, scopeAttributes);
 
-  // get all model's required definitions
-  const requiredFieldsNames = await DefinitionRepo.getRequiredFields(modelType, instance.id, identifiers, modelOptions);
+  const where: WhereOptions = {
+    modelType,
+    disabled: false,
+    ...(!useEntityIdFromInclude && { entityId: identifiers }),
+  };
+  const fieldDefinitions = await DefinitionRepo.findAll(where, { withDisabled: false, transaction: options.transaction, include: include?.(identifiers) });
+  const requiredFieldsNames = Array.from(new Set(fieldDefinitions.filter(({ required }) => required).map(({ name }) => name)));
 
   const customFieldsIdx = fields.indexOf('customFields');
-  const { customFields } = instance;
-  if (customFieldsIdx > -1 && customFields) {
-    const fieldsNames = Object.keys(customFields);
-    const missingFields = requiredFieldsNames.filter((name) => !fieldsNames.includes(name));
-    if (missingFields?.length > 0) {
-      throw new MissingRequiredCustomFieldError(missingFields);
-    }
-
-    await ValueRepo.updateValues(
-      modelType,
-      instance.id,
-      identifiers,
-      customFields,
-      {
-        transaction: options.transaction,
-        modelOptions,
-      },
-      true,
-    );
 
-    // eslint-disable-next-line no-param-reassign
-    fields.splice(customFieldsIdx, 1);
-  } else if (requiredFieldsNames?.length > 0) {
+  if ((customFieldsIdx === -1 || !instance.customFields) && requiredFieldsNames?.length > 0) {
     throw new MissingRequiredCustomFieldError(requiredFieldsNames);
   }
+  const fieldsWithDefaultValue = fieldDefinitions.filter((def) => ![null, undefined].includes(def.defaultValue));
+  if (fieldsWithDefaultValue.length) {
+    // eslint-disable-next-line no-param-reassign
+    instance.customFields ||= {};
+    fieldsWithDefaultValue.filter((def) => !(def.name in instance.customFields)).forEach(({ name, defaultValue }) => {
+      // eslint-disable-next-line no-param-reassign
+      instance.customFields[name] = defaultValue;
+    });
+  }
+  const { customFields } = instance;
+  if (customFieldsIdx === -1 || !customFields) {
+    return;
+  }
+  const fieldsNames = Object.keys(customFields);
+  const missingFields = requiredFieldsNames.filter((name) => !fieldsNames.includes(name));
+  if (missingFields?.length > 0) {
+    throw new MissingRequiredCustomFieldError(missingFields);
+  }
+
+  await ValueRepo.updateValues(
+    modelType,
+    instance.id,
+    identifiers,
+    customFields,
+    {
+      transaction: options.transaction,
+      modelOptions,
+    },
+  );
+
+  // eslint-disable-next-line no-param-reassign
+  fields.splice(customFieldsIdx, 1);
 };

package/src/repository/value.ts

@@ -67,7 +67,6 @@ export const updateValues = async (
   identifiers: string[],
   valuesToUpdate: ValuesToUpdate,
   options: FindOptions & { modelOptions?: ModelOptions } = {},
-  defineAllDefaults = false,
 ): Promise<CustomFieldValue[]> => {
   const names = Object.keys(valuesToUpdate);
   logger.debug(`custom-fields: updating values for ${modelType} ${modelId}`, {
@@ -80,13 +79,11 @@ export const updateValues = async (
 
   const where: WhereOptions = {
     modelType,
-    ...(!defineAllDefaults && { name: names }),
+    name: names,
     ...(!options.modelOptions?.useEntityIdFromInclude && { entityId: identifiers }),
   };
 
-  const allFieldDefinitions = await DefinitionRepo.findAll(where, { withDisabled: true, transaction, include: modelOptions.include?.(identifiers) }) || [];
-  const namesSet = new Set(defineAllDefaults ? names : undefined);
-  const fieldDefinitions = defineAllDefaults ? allFieldDefinitions.filter((cfd) => namesSet.has(cfd.name)) : allFieldDefinitions;
+  const fieldDefinitions = await DefinitionRepo.findAll(where, { withDisabled: true, transaction, include: modelOptions.include?.(identifiers) }) ?? [];
 
   const disabledDefinitions = fieldDefinitions.filter((def) => def.disabled);
   if (fieldDefinitions.length !== names.length) {
@@ -101,31 +98,18 @@ export const updateValues = async (
     logger.warn(`custom-fields: trying to update disabled values: ${valuesWithDisabledDefinitions.join(', ')}`);
   }
 
-  const visitedFields = new Set<CustomFieldDefinition>();
-
   const values: CreateCustomFieldValue[] = names.map((name) => {
     const fieldDefinition = fieldDefinitions.find((def) => def.name === name);
-    visitedFields.add(fieldDefinition);
     const formatFunction = formatFunctions[fieldDefinition.fieldType];
+    const value = formatFunction ? formatFunction(valuesToUpdate[name]) : valuesToUpdate[name];
     return {
       modelId,
-      value: (formatFunction ? formatFunction(valuesToUpdate[name]) : valuesToUpdate[name]) ?? fieldDefinition.defaultValue,
       updatedAt: new Date(),
       customFieldDefinitionId: fieldDefinition.id,
+      value: value !== undefined ? value : fieldDefinition.defaultValue,
     };
   });
 
-  if (defineAllDefaults) {
-    allFieldDefinitions.filter((def) => !visitedFields.has(def) && ![null, undefined].includes(def.defaultValue)).forEach(({ id, defaultValue }) => {
-      values.push({
-        modelId,
-        value: defaultValue,
-        updatedAt: new Date(),
-        customFieldDefinitionId: id,
-      });
-    });
-  }
-
   return Promise.all(values.map(async (value) => {
     const [cfv] = await CustomFieldValue.upsert(value, {
       transaction: options.transaction,

package/src/scopes/filter.ts

@@ -3,6 +3,7 @@ import { Op, type WhereOptions } from 'sequelize';
 import { Sequelize } from 'sequelize-typescript';
 import { customFields } from '@autofleet/common-types';
 import { generateRandomString } from '../utils/helpers';
+import logger from '../utils/logger';
 
 const { CUSTOM_FIELDS_FILTER_SCOPE } = customFields;
 
@@ -43,7 +44,21 @@ const castIfNeeded = (conditionValue: string): string => {
   }
   return '';
 };
+
+const addQuotationIfNeeded = (conditionValue: string, valRep: string): string => {
+  if (typeof conditionValue === 'string') {
+    return `'"' || :${valRep} || '"'`;
+  }
+  return `:${valRep}`;
+};
+
 const AND_DELIMETER = ' AND ';
+const OR_DELIMETER = ' OR ';
+
+const CD_TABLE_ALIAS = 'cd';
+const CD_NAME_COLUMN = `${CD_TABLE_ALIAS}.name`;
+const CV_TABLE_ALIAS = 'cv';
+const CV_VALUE_COLUMN = `(${CV_TABLE_ALIAS}.value::varchar)`;
 
 /**
  * A Sequelize scope for filtering models by custom fields.
@@ -71,6 +86,7 @@ export const customFieldsFilterScope = (
           (randomString) => replacements[randomString] === key,
         );
         if (!replacemetKey) return false;
+        const columnCondition = `(${CD_NAME_COLUMN} = :${replacemetKey})`;
 
         if (Array.isArray(condition)) {
           if (condition.length === 0) {
@@ -82,29 +98,29 @@ export const customFieldsFilterScope = (
               const valRandom = Object.keys(replacements).find(
                 (randomString) => replacements[randomString] === v,
               );
-              return ` :${valRandom} `;
+              return addQuotationIfNeeded(v, valRandom);
             }).join(',');
-            return `(custom_fields->> :${replacemetKey} ) IN ( ${values} )`;
+            return `(${columnCondition} AND ${CV_VALUE_COLUMN} IN ( ${values} ))`;
           }
           return condition
             .map((c) => {
               const valRep = Object.keys(replacements).find(
                 (replacementKey) => replacements[replacementKey] === c.value,
               );
-              return `(custom_fields->> :${replacemetKey} )${castIfNeeded(c.value)} ${c.operator} :${valRep}`;
+              return `( ${columnCondition} AND ${CV_VALUE_COLUMN}${castIfNeeded(c.value)} ${c.operator} ${addQuotationIfNeeded(c.value, valRep)} )`;
             }).join(AND_DELIMETER);
         }
         if (typeof condition === 'string' || typeof condition === 'number') {
           const conditionRep = Object.keys(replacements).find(
             (replacementKey) => replacements[replacementKey] === condition,
           );
-          return `(custom_fields->> :${replacemetKey} ) ${castIfNeeded(condition)} = :${conditionRep}`;
+          return `(${columnCondition} AND ${CV_VALUE_COLUMN}${castIfNeeded(condition)} = ${addQuotationIfNeeded(condition, conditionRep)})`;
         }
         if (condition?.operator) {
           const valueRep = Object.keys(replacements).find(
             (replacementKey) => replacements[replacementKey] === condition.value,
           );
-          return `(custom_fields->> :${replacemetKey} ) ${castIfNeeded(condition.value)} ${condition.operator} :${valueRep}`;
+          return `( ${columnCondition} AND ${CV_VALUE_COLUMN}${castIfNeeded(condition.value)} ${condition.operator} ${addQuotationIfNeeded(condition.value, valueRep)})`;
         }
         return false;
       },
@@ -113,14 +129,17 @@ export const customFieldsFilterScope = (
   if (conditionsStrings.length === 0) {
     return {};
   }
-  const customFieldConditions = conditionsStrings.join(AND_DELIMETER);
-  const subQuery = `${'SELECT model_id FROM ('
-    + 'SELECT cv.model_id, jsonb_object_agg(cd.name, cv.value) AS custom_fields '
-    + 'FROM custom_field_values AS cv '
-    + 'INNER JOIN custom_field_definitions AS cd ON cv.custom_field_definition_id = cd.id '
-    + `AND cd.model_type = '${name}'`
-    + 'GROUP BY cv.model_id'
-    + ') AS CustomFieldAggregation WHERE '} ${customFieldConditions}`;
+  const customFieldConditions = conditionsStrings.join(OR_DELIMETER);
+  const subQuery = `
+  SELECT cv.model_id
+  FROM custom_field_values AS cv
+  INNER JOIN custom_field_definitions AS cd ON cv.custom_field_definition_id = cd.id
+  AND cd.model_type = '${name}'
+  WHERE ${customFieldConditions}
+  GROUP BY cv.model_id
+  HAVING COUNT(DISTINCT cv.custom_field_definition_id) = ${conditionsStrings.length}
+`.replace(/\n/g, '');
+  logger.info('custom fields filter scope', { subQuery, replacements });
   return {
     where: {
       id: {