@contrail/cli 3.4.5 → 3.4.6-alpha.cffb28e

package/node_modules/@contrail/loader/lib/entities-v2/entities-helper.spec.js

@@ -0,0 +1,137 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const entity_types_1 = require("@contrail/entity-types");
+const entity_builder_helpers_1 = require("../common/entity-builder-helpers");
+describe('addProperty', () => {
+    it('should set single-select value as null when value is ""', () => {
+        const obj = { testSingleSelect: '' };
+        const result = (0, entity_builder_helpers_1.addProperty)('testSingleSelect', ['testSingleSelect'], entity_types_1.PropertyType.SingleSelect, obj, obj.testSingleSelect);
+        expect(result.testSingleSelect).toEqual(null);
+    });
+    it('should set single-select value', () => {
+        const obj = { testSingleSelect: '1' };
+        const result = (0, entity_builder_helpers_1.addProperty)('testSingleSelect', ['testSingleSelect'], entity_types_1.PropertyType.SingleSelect, obj, obj.testSingleSelect);
+        expect(result.testSingleSelect).toEqual('1');
+    });
+    it('should set multi-select value as null when value is ""', () => {
+        const obj = { testMultiSelect: '' };
+        const result = (0, entity_builder_helpers_1.addProperty)('testMultiSelect', ['testMultiSelect'], entity_types_1.PropertyType.MultiSelect, obj, obj.testMultiSelect);
+        expect(result.testMultiSelect).toEqual(null);
+    });
+    it('should set multi-select value from a comma-delimited string', () => {
+        const obj = { testMultiSelect: '1,2' };
+        const result = (0, entity_builder_helpers_1.addProperty)('testMultiSelect', ['testMultiSelect'], entity_types_1.PropertyType.MultiSelect, obj, obj.testMultiSelect);
+        expect(result.testMultiSelect).toEqual(['1', '2']);
+    });
+    it('should set date value as null when value is ""', () => {
+        const obj = { testDate: '' };
+        const result = (0, entity_builder_helpers_1.addProperty)('testDate', ['testDate'], entity_types_1.PropertyType.Date, obj, obj.testDate);
+        expect(result.testDate).toEqual(null);
+    });
+    it('should set date value from a ISO-8601 formatted string', () => {
+        const obj = { testDate: '2024-01-16T13:25:02.619Z' };
+        const result = (0, entity_builder_helpers_1.addProperty)('testDate', ['testDate'], entity_types_1.PropertyType.Date, obj, obj.testDate);
+        expect(result.testDate).toBeInstanceOf(Date);
+        expect(result.testDate.toDateString()).toEqual('Tue Jan 16 2024');
+    });
+    it('should set number value', () => {
+        const obj = { testNumber: 0 };
+        const result = (0, entity_builder_helpers_1.addProperty)('testNumber', ['testNumber'], entity_types_1.PropertyType.Percent, obj, obj.testNumber);
+        expect(result.testNumber).toEqual(0);
+    });
+    it('should set percent value', () => {
+        const obj = { testPercent: 10 };
+        const result = (0, entity_builder_helpers_1.addProperty)('testPercent', ['testPercent'], entity_types_1.PropertyType.Percent, obj, obj.testPercent);
+        expect(result.testPercent).toEqual(10);
+    });
+    it('should set currency value', () => {
+        const obj = { testCurrency: 100 };
+        const result = (0, entity_builder_helpers_1.addProperty)('testCurrency', ['testCurrency'], entity_types_1.PropertyType.Currency, obj, obj.testCurrency);
+        expect(result.testCurrency).toEqual(100);
+    });
+    it('should set sequence value', () => {
+        const obj = { testSequence: 50 };
+        const result = (0, entity_builder_helpers_1.addProperty)('testSequence', ['testSequence'], entity_types_1.PropertyType.Sequence, obj, obj.testSequence);
+        expect(result.testSequence).toEqual(50);
+    });
+    it('should parse simple size range', () => {
+        const obj = { testSizeRange: 'B:05,40' };
+        const result = (0, entity_builder_helpers_1.addProperty)('testSizeRange', ['testSizeRange'], entity_types_1.PropertyType.SizeRange, obj, obj.testSizeRange);
+        expect(result.testSizeRange).toEqual({
+            sizes: {
+                B: ['05', '40'],
+            },
+        });
+    });
+    it('should parse size range with empty values', () => {
+        const obj = { testSizeRange: 'B:05,,,40,,,30' };
+        const result = (0, entity_builder_helpers_1.addProperty)('testSizeRange', ['testSizeRange'], entity_types_1.PropertyType.SizeRange, obj, obj.testSizeRange);
+        expect(result.testSizeRange).toEqual({
+            sizes: {
+                B: ['05', '40', '30'],
+            },
+        });
+    });
+    it('should parse size ranges', () => {
+        const obj = { testSizeRange: 'B:05,055,, D: 05,055,12,' };
+        const result = (0, entity_builder_helpers_1.addProperty)('testSizeRange', ['testSizeRange'], entity_types_1.PropertyType.SizeRange, obj, obj.testSizeRange);
+        expect(result.testSizeRange).toEqual({
+            sizes: {
+                B: ['05', '055'],
+                D: ['05', '055', '12'],
+            },
+        });
+    });
+    it('should parse size ranges if they are a comma separated one dimensional string', () => {
+        const obj = { testSizeRange: '05,055,12,' };
+        const result = (0, entity_builder_helpers_1.addProperty)('testSizeRange', ['testSizeRange'], entity_types_1.PropertyType.SizeRange, obj, obj.testSizeRange);
+        expect(result.testSizeRange).toEqual({
+            sizes: ['05', '055', '12'],
+        });
+    });
+});
+describe('shouldSkipProperty', () => {
+    it('should return true for valueType "formula"', () => {
+        const result = (0, entity_builder_helpers_1.shouldSkipProperty)(null, entity_types_1.PropertyType.Formula);
+        expect(result).toBe(true);
+    });
+    it('should return false for empty string or text value', () => {
+        const result = (0, entity_builder_helpers_1.shouldSkipProperty)('', entity_types_1.PropertyType.String);
+        expect(result).toBe(false);
+    });
+    it.each(Object.values(entity_types_1.PropertyType).filter((x) => x !== entity_types_1.PropertyType.Formula))('should not skip boolean value for  "%s" type', (valueType) => {
+        const result = (0, entity_builder_helpers_1.shouldSkipProperty)(false, valueType);
+        expect(result).toBe(false);
+    });
+    it('should return false for empty string value and valueType "Text"', () => {
+        const result = (0, entity_builder_helpers_1.shouldSkipProperty)('', entity_types_1.PropertyType.Text);
+        expect(result).toBe(false);
+    });
+    it('should return false for valueType "Boolean"', () => {
+        const result = (0, entity_builder_helpers_1.shouldSkipProperty)(false, entity_types_1.PropertyType.Boolean);
+        expect(result).toBe(false);
+    });
+    it('should return false for falsy number value', () => {
+        const result = (0, entity_builder_helpers_1.shouldSkipProperty)(0, entity_types_1.PropertyType.Number);
+        expect(result).toBe(false);
+    });
+    it('should return false for valueType "Single Select" and an empty string', () => {
+        const result = (0, entity_builder_helpers_1.shouldSkipProperty)('', entity_types_1.PropertyType.SingleSelect);
+        expect(result).toBe(false);
+    });
+    it('should handle empty values', () => {
+        const testCases = [
+            { value: '', expected: '' },
+            { value: null, expected: null },
+            { value: undefined, expected: undefined },
+            { value: '123', expected: 123 },
+            { value: '123.45', expected: 123.45 },
+            { value: '1,234.56', expected: 1234.56 },
+        ];
+        testCases.forEach(({ value, expected }) => {
+            const result = (0, entity_builder_helpers_1.addProperty)('test', ['test'], entity_types_1.PropertyType.Number, { test: value }, value);
+            console.log('result', result);
+            expect(result.test).toEqual(expected);
+        });
+    });
+});

package/node_modules/@contrail/loader/lib/entities-v2/entity-processor-v2.d.ts

@@ -0,0 +1,36 @@
+import { EntitiesClient, EntityType, GeneralIssueDetails, LoaderProcess, ProcessingIssues } from '@contrail/entity-types';
+import { BaseProcessorV2 } from './base-processor-v2';
+import { Type } from '@contrail/types';
+export interface ProcessedEntityV2 {
+    incomingEntities: string[];
+    badEntities: string[];
+    allCurrentEntitiesByFederatedId: Record<string, any>;
+    typesByPath: string[];
+    projectId: string;
+    generalWarnings?: GeneralIssueDetails[];
+}
+export interface UniqueAndDuplicateEntriesV2 {
+    uniqueFedIds: string[];
+    nonUniqueFedIds: string[];
+}
+export declare class EntityProcessorV2 {
+    static setupEntities(client: EntitiesClient, entityType: EntityType, rawData: any[], entitySpecificProperties?: any, loaderProcess?: LoaderProcess, itemsByFederatedId?: Record<string, any>, doConstructEntities?: boolean, typesByTypePath?: Record<string, Type>): Promise<ProcessedEntityV2 | any>;
+    private static constructEntities;
+    private static getTypePaths;
+    private static fetchTypesByPath;
+    private static buildIncomingEntities;
+    static extractFederatedIds(incomingEntities: any[], processor: typeof BaseProcessorV2, federatedIdkeys: string[]): string[];
+    private static fetchCurrentEntities;
+    private static getUniqueAndDuplicateEntries;
+    static validateLoadFile(data: any[], config: LoaderProcess, federatedIdKey?: string, nameKey?: string, returnImmediately?: boolean): {
+        errors: ProcessingIssues[];
+        warnings: ProcessingIssues[];
+        generalErrors: GeneralIssueDetails[];
+        generalWarnings: GeneralIssueDetails[];
+        csvHeaders: string[];
+        returnImmediately: boolean;
+    };
+    private static validateFederatedIdMapping;
+    private static validateNameMapping;
+    private static validateSkipStaleUpdatesUsing;
+}

package/node_modules/@contrail/loader/lib/entities-v2/entity-processor-v2.js

@@ -0,0 +1,316 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EntityProcessorV2 = void 0;
+const entities_helper_1 = require("../common/entities-helper");
+const entity_builder_helpers_1 = require("../common/entity-builder-helpers");
+const processor_registry_1 = require("../processor-registry");
+const lodash_1 = require("lodash");
+const entity_types_1 = require("@contrail/entity-types");
+const type_helper_1 = require("../common/type-helper");
+const constants_1 = require("../constants");
+const data_transformer_functions_1 = require("../common/data-transformer-functions");
+const pLimit = require('p-limit');
+const limit = pLimit(5);
+class EntityProcessorV2 {
+    static async setupEntities(client, entityType, rawData, entitySpecificProperties, loaderProcess, itemsByFederatedId = {}, doConstructEntities = true, typesByTypePath) {
+        try {
+            const processor = (0, processor_registry_1.getProcessorForEntityType)(entityType, 'v2');
+            rawData = processor.transformData(rawData);
+            const requiredProperties = processor.requiredProperties || [];
+            const result = doConstructEntities
+                ? await this.constructEntities({
+                    client,
+                    processor,
+                    entityType,
+                    requiredProperties,
+                    data: rawData,
+                    entitySpecificProperties,
+                    itemsByFederatedId,
+                    typesByTypePath,
+                })
+                :
+                    await processor.buildAssortmentEntities(client, rawData, loaderProcess, itemsByFederatedId, typesByTypePath);
+            return result;
+        }
+        catch (error) {
+            console.error(`Error processing and loading objects for ${entityType}:`, error);
+            return {
+                incomingEntities: rawData,
+                badEntities: [],
+                allCurrentEntitiesByFederatedId: {},
+                typesByPath: [],
+                projectId: null,
+            };
+        }
+    }
+    static async constructEntities(params) {
+        const { client, processor, entityType, requiredProperties = [], data = [], entitySpecificProperties, itemsByFederatedId = {}, typesByTypePath, } = params;
+        if (!data?.length) {
+            console.debug('No data to upload');
+            return {
+                incomingEntities: [],
+                badEntities: [],
+                allCurrentEntitiesByFederatedId: {},
+                typesByPath: [],
+                projectId: null,
+            };
+        }
+        const typePaths = this.getTypePaths(data, entityType, processor);
+        const typesByPath = await this.fetchTypesByPath({
+            client,
+            typePaths,
+            requiredProperties,
+            entityType,
+            typesByTypePath,
+        });
+        const error = [];
+        for (const typePath in typesByPath) {
+            if (typesByPath[typePath]?.error) {
+                error.push(typesByPath[typePath].error);
+            }
+        }
+        if (error.length) {
+            return {
+                incomingEntities: [],
+                badEntities: [],
+                allCurrentEntitiesByFederatedId: {},
+                typesByPath: [],
+                projectId: null,
+                error,
+            };
+        }
+        const { projectId } = await processor.processEntitiesProperties(client, entitySpecificProperties, data);
+        const { badEntities, incomingEntities } = this.buildIncomingEntities(processor, data, typesByPath, entityType);
+        if (incomingEntities.length === 0)
+            return {
+                incomingEntities,
+                badEntities,
+                allCurrentEntitiesByFederatedId: {},
+                typesByPath: [],
+                projectId,
+            };
+        const generalWarnings = await (0, entities_helper_1.assignObjectReferences)(client, typePaths, typesByPath, incomingEntities);
+        const result = await processor.assignExtraItems(client, typePaths, typesByPath, incomingEntities, projectId);
+        result.projectId = projectId;
+        const federatedIdKeys = processor.getFederatedIdKeys();
+        let federatedIds;
+        if (processor.canExtractFederatedIds(itemsByFederatedId, incomingEntities)) {
+            federatedIds = this.extractFederatedIds(incomingEntities, processor, federatedIdKeys);
+            console.debug(`Looking up ${federatedIds?.length} federated IDs to check for existing ${entityType} entities`);
+        }
+        const entityTypeForFederatedId = processor.getEntityTypeForFederatedId(entityType);
+        const allCurrentEntitiesByFederatedId = await this.fetchCurrentEntities(client, entityTypeForFederatedId, federatedIds);
+        processor.logIncommingEntities(incomingEntities, entityType, result);
+        return {
+            incomingEntities,
+            badEntities,
+            allCurrentEntitiesByFederatedId,
+            typesByPath,
+            ...result,
+            generalWarnings,
+        };
+    }
+    static getTypePaths(data, entityType, processor) {
+        return Array.from(new Set(data.map((row) => processor.getTypePath(row, entityType))));
+    }
+    static async fetchTypesByPath(params) {
+        const { client, typePaths, requiredProperties, entityType, typesByTypePath } = params;
+        return await (0, type_helper_1.buildTypesByPath)(client, typePaths, requiredProperties, entityType, typesByTypePath);
+    }
+    static buildIncomingEntities(processor, data, typesByPath, entityType) {
+        const incomingEntities = [];
+        const badEntities = [];
+        for (const row of data) {
+            if (!processor.validateRow(row)) {
+                badEntities.push(row);
+                console.error(`Invalid data found for this row ${entityType}:`, row);
+                continue;
+            }
+            const typePath = processor.getTypePath(row, entityType);
+            const type = typesByPath[typePath];
+            const entity = {};
+            for (const propertyName in row) {
+                if (processor.canAddProperty(type.allPropertiesSlugs, propertyName)) {
+                    const basePropertyName = (0, data_transformer_functions_1.removeOptionOrFamilySuffix)(propertyName);
+                    (0, entity_builder_helpers_1.addProperty)(propertyName, type.allPropertiesSlugs, type.typePropertyToValueType[basePropertyName], entity, row[propertyName]);
+                }
+            }
+            processor.setExtraProperties(entity, type, row);
+            incomingEntities.push(entity);
+        }
+        return { badEntities, incomingEntities };
+    }
+    static extractFederatedIds(incomingEntities, processor, federatedIdkeys) {
+        const allFederatedIds = [];
+        const re = /\//gi;
+        for (const incomingEntity of incomingEntities) {
+            for (const federatedIdkey of federatedIdkeys) {
+                if (!incomingEntity.hasOwnProperty(federatedIdkey))
+                    continue;
+                const processedFederatedId = processor.processFederatedId(incomingEntity[federatedIdkey], incomingEntity.typePath);
+                const federatedId = processedFederatedId.replace(re, '%2F');
+                allFederatedIds.push(federatedId);
+                incomingEntity[federatedIdkey] = processedFederatedId;
+            }
+        }
+        const { uniqueFedIds } = this.getUniqueAndDuplicateEntries(allFederatedIds);
+        return uniqueFedIds;
+    }
+    static async fetchCurrentEntities(client, entityType, federatedIds) {
+        const chunks = (0, lodash_1.chunk)(federatedIds, 200);
+        const allCurrentEntitiesByFederatedId = {};
+        const federatedPromises = [];
+        for (const chunk of chunks) {
+            if (!chunk?.length)
+                continue;
+            federatedPromises.push(limit(async () => {
+                try {
+                    const currentEntityBatch = await (0, entities_helper_1.retryOnTimeout)(client, entities_helper_1.EntitiesLambdaCallType.GET, {
+                        entityName: entityType,
+                        federatedIds: chunk,
+                        skipHydrateRelationships: true,
+                    });
+                    if (!currentEntityBatch) {
+                        console.warn(`No existing ${entityType} entities found for the provided federated IDs`);
+                    }
+                    if (currentEntityBatch?.length) {
+                        for (const currentEntity of currentEntityBatch) {
+                            allCurrentEntitiesByFederatedId[currentEntity.federatedId] = currentEntity;
+                        }
+                    }
+                }
+                catch (e) {
+                    console.error(`Failed to fetch ${entityType} by federatedIds:`, e);
+                    throw e;
+                }
+            }));
+        }
+        await Promise.all(federatedPromises);
+        console.info(`Existing ${entityType} lookup complete: ${Object.keys(allCurrentEntitiesByFederatedId)?.length} found by federatedId`);
+        return allCurrentEntitiesByFederatedId;
+    }
+    static getUniqueAndDuplicateEntries(strings) {
+        const uniqueFedIds = new Set();
+        const nonUniqueFedIds = new Set();
+        for (const string of strings) {
+            if (uniqueFedIds.has(string)) {
+                nonUniqueFedIds.add(string);
+            }
+            else {
+                uniqueFedIds.add(string);
+            }
+        }
+        console.debug(`Unique federatedId count: ${uniqueFedIds.size}`);
+        console.debug(`Non-Unique federatedId count: ${nonUniqueFedIds.size}`);
+        console.debug(`Non-Unique federatedIds: ${Array.from(uniqueFedIds)}`);
+        return {
+            uniqueFedIds: Array.from(uniqueFedIds),
+            nonUniqueFedIds: Array.from(nonUniqueFedIds),
+        };
+    }
+    static validateLoadFile(data, config, federatedIdKey = 'federatedId', nameKey = 'name', returnImmediately = false) {
+        const csvHeaders = Object.keys(data[0]);
+        const errors = [];
+        const warnings = [];
+        const generalErrors = [];
+        const generalWarnings = [];
+        if (!config) {
+            generalErrors.push({
+                message: constants_1.ENTITY_PROCESSOR_ERRORS.CONFIG_EMPTY,
+                issueType: entity_types_1.IssueType.CONFIG,
+            });
+            returnImmediately = true;
+            return {
+                errors,
+                warnings,
+                generalErrors,
+                generalWarnings,
+                csvHeaders,
+                returnImmediately,
+            };
+        }
+        if (!config?.loadType?.length) {
+            generalErrors.push({
+                message: constants_1.ENTITY_PROCESSOR_ERRORS.LOAD_TYPE_REQUIRED,
+                issueType: entity_types_1.IssueType.CONFIG,
+            });
+        }
+        this.validateFederatedIdMapping(config, csvHeaders, federatedIdKey, generalErrors);
+        this.validateNameMapping(config, csvHeaders, nameKey, generalErrors);
+        returnImmediately = this.validateSkipStaleUpdatesUsing(config, csvHeaders, generalErrors) || returnImmediately;
+        return {
+            errors,
+            warnings,
+            generalErrors,
+            generalWarnings,
+            csvHeaders,
+            returnImmediately,
+        };
+    }
+    static validateFederatedIdMapping(config, csvHeaders, federatedIdKey, generalErrors) {
+        const federatedIdInConditional = config?.conditionalColumns?.some((column) => column.toProperty === federatedIdKey);
+        const federatedIdInFederatedMapping = config?.federatedMappings?.[federatedIdKey];
+        const federatedIdInCSV = csvHeaders.includes(federatedIdKey);
+        if (!federatedIdInFederatedMapping && !federatedIdInCSV && !federatedIdInConditional) {
+            generalErrors.push({
+                message: constants_1.ENTITY_PROCESSOR_ERRORS.FEDERATED_ID_REQUIRED,
+                issueType: entity_types_1.IssueType.CONFIG,
+            });
+        }
+        else if (federatedIdInFederatedMapping && !csvHeaders.includes(config.federatedMappings[federatedIdKey])) {
+            generalErrors.push({
+                message: constants_1.ENTITY_PROCESSOR_ERRORS.FEDERATED_ID_REQUIRED,
+                issueType: entity_types_1.IssueType.CONFIG,
+            });
+        }
+    }
+    static validateNameMapping(config, csvHeaders, nameKey, generalErrors) {
+        const nameInConditional = config?.conditionalColumns?.some((column) => column.toProperty === nameKey);
+        const nameInFederatedMapping = config?.federatedMappings?.[nameKey];
+        const nameInCSV = csvHeaders.includes(nameKey);
+        if (!nameInFederatedMapping && !nameInCSV && !nameInConditional) {
+            generalErrors.push({
+                message: constants_1.ENTITY_PROCESSOR_ERRORS.NAME_REQUIRED,
+                issueType: entity_types_1.IssueType.CONFIG,
+            });
+        }
+        else if (nameInFederatedMapping && !csvHeaders.includes(config.federatedMappings[nameKey])) {
+            generalErrors.push({
+                message: constants_1.ENTITY_PROCESSOR_ERRORS.NAME_REQUIRED,
+                issueType: entity_types_1.IssueType.CONFIG,
+            });
+        }
+    }
+    static validateSkipStaleUpdatesUsing(config, csvHeaders, generalErrors) {
+        if (!config?.skipStaleUpdatesUsing) {
+            return false;
+        }
+        if (typeof config.skipStaleUpdatesUsing !== 'string') {
+            generalErrors.push({
+                message: `skipStaleUpdatesUsing must be a string column name, received: ${typeof config.skipStaleUpdatesUsing}`,
+                issueType: entity_types_1.IssueType.CONFIG,
+            });
+            return true;
+        }
+        const columnName = config.skipStaleUpdatesUsing;
+        const inCSV = csvHeaders.includes(columnName);
+        const inFederatedMapping = config?.federatedMappings?.[columnName];
+        const inConditional = config?.conditionalColumns?.some((column) => column.toProperty === columnName);
+        if (!inCSV && !inFederatedMapping && !inConditional) {
+            generalErrors.push({
+                message: `skipStaleUpdatesUsing column '${columnName}' not found in CSV headers, federated mappings, or conditional columns`,
+                issueType: entity_types_1.IssueType.CONFIG,
+            });
+            return true;
+        }
+        else if (inFederatedMapping && !csvHeaders.includes(config.federatedMappings[columnName])) {
+            generalErrors.push({
+                message: `skipStaleUpdatesUsing column '${columnName}' is mapped to '${config.federatedMappings[columnName]}' but that column is not in the CSV`,
+                issueType: entity_types_1.IssueType.CONFIG,
+            });
+            return true;
+        }
+        return false;
+    }
+}
+exports.EntityProcessorV2 = EntityProcessorV2;

package/node_modules/@contrail/loader/lib/entities-v2/entity-processor-v2.spec.d.ts

@@ -0,0 +1 @@
+export {};