@contrail/cli 3.4.5-alpha.18160f5 → 3.4.6-alpha.c203463

package/node_modules/@contrail/loader/lib/entities-v2/linesheet-v2/project-item-processor.spec.js

@@ -0,0 +1,172 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const project_item_processor_1 = require("./project-item-processor");
+const entities_helper_1 = require("../../common/entities-helper");
+const size_range_helpers_1 = require("../../common/size-range-helpers");
+jest.mock('../../common/entities-helper', () => ({
+    retryOnTimeout: jest.fn(),
+    assignUserLists: jest.fn(),
+    EntitiesLambdaCallType: {
+        GET: 'GET',
+    },
+}));
+jest.mock('../../common/size-range-helpers', () => ({
+    SizeRangeSanitizer: jest.fn().mockImplementation(() => ({
+        sanitizeSizeRanges: jest.fn(),
+        getWarningsAsStringList: jest.fn().mockReturnValue(['mockWarning']),
+    })),
+}));
+describe('ProjectItemProcessor', () => {
+    let client;
+    beforeEach(() => {
+        client = {};
+        jest.clearAllMocks();
+    });
+    describe('validateRow', () => {
+        it('should return true if itemFamilyFederatedId exists', () => {
+            const row = { itemFamilyFederatedId: 'mockId' };
+            expect(project_item_processor_1.ProjectItemProcessor.validateRow(row)).toBe(true);
+        });
+        it('should return false if itemFamilyFederatedId does not exist', () => {
+            const row = { itemFamilyFederatedId: '' };
+            expect(project_item_processor_1.ProjectItemProcessor.validateRow(row)).toBe(false);
+        });
+    });
+    describe('getFederatedIdKeys', () => {
+        it('should return the correct federated ID keys', () => {
+            expect(project_item_processor_1.ProjectItemProcessor.getFederatedIdKeys()).toEqual(['itemFamilyFederatedId', 'itemOptionFederatedId']);
+        });
+    });
+    describe('getTypePath', () => {
+        it('should return "project-item" as the type path', () => {
+            expect(project_item_processor_1.ProjectItemProcessor.getTypePath({}, 'mockEntityType')).toBe('project-item');
+        });
+    });
+    describe('processEntitiesProperties', () => {
+        it('should resolve projectId per-row from workspaceIdentifier column', async () => {
+            const mockWorkspaces = [{ id: 'mockWorkspaceId' }];
+            const mockWorkspace = { project: { id: 'mockProjectId' } };
+            entities_helper_1.retryOnTimeout
+                .mockResolvedValueOnce(mockWorkspaces)
+                .mockResolvedValueOnce(mockWorkspace);
+            const data = [
+                { itemFamilyFederatedId: 'fam1', workspaceIdentifier: 'mockWorkspaceIdentifier' },
+                { itemFamilyFederatedId: 'fam2', workspaceIdentifier: 'mockWorkspaceIdentifier' },
+            ];
+            const result = await project_item_processor_1.ProjectItemProcessor.processEntitiesProperties(client, {}, data);
+            expect(entities_helper_1.retryOnTimeout).toHaveBeenCalledTimes(2);
+            expect(result).toEqual({ projectId: 'mockProjectId' });
+            expect(data[0].projectId).toBe('mockProjectId');
+            expect(data[1].projectId).toBe('mockProjectId');
+            expect(data[0]).not.toHaveProperty('workspaceIdentifier');
+            expect(data[1]).not.toHaveProperty('workspaceIdentifier');
+        });
+        it('should use projectId column directly when provided', async () => {
+            const data = [
+                { itemFamilyFederatedId: 'fam1', projectId: 'directProjectId' },
+            ];
+            const result = await project_item_processor_1.ProjectItemProcessor.processEntitiesProperties(client, {}, data);
+            expect(entities_helper_1.retryOnTimeout).not.toHaveBeenCalled();
+            expect(result).toEqual({ projectId: 'directProjectId' });
+            expect(data[0].projectId).toBe('directProjectId');
+        });
+        it('should return null projectId when neither workspaceIdentifier nor projectId is present', async () => {
+            const data = [{ itemFamilyFederatedId: 'fam1' }];
+            const result = await project_item_processor_1.ProjectItemProcessor.processEntitiesProperties(client, {}, data);
+            expect(result).toEqual({ projectId: null });
+        });
+        it('should return null projectId when data is empty', async () => {
+            const result = await project_item_processor_1.ProjectItemProcessor.processEntitiesProperties(client, {}, []);
+            expect(result).toEqual({ projectId: null });
+        });
+        it('should return null projectId when both columns are present (mutual exclusivity)', async () => {
+            const data = [
+                { itemFamilyFederatedId: 'fam1', workspaceIdentifier: 'ws1', projectId: 'proj1' },
+            ];
+            const result = await project_item_processor_1.ProjectItemProcessor.processEntitiesProperties(client, {}, data);
+            expect(result).toEqual({ projectId: null });
+        });
+        it('should return null when workspace not found', async () => {
+            entities_helper_1.retryOnTimeout.mockResolvedValueOnce([]);
+            const data = [
+                { itemFamilyFederatedId: 'fam1', workspaceIdentifier: 'nonexistent' },
+            ];
+            const result = await project_item_processor_1.ProjectItemProcessor.processEntitiesProperties(client, {}, data);
+            expect(entities_helper_1.retryOnTimeout).toHaveBeenCalledTimes(1);
+            expect(result).toEqual({ projectId: null });
+            expect(data[0].projectId).toBeNull();
+        });
+        it('should cache workspace lookups for duplicate identifiers', async () => {
+            const mockWorkspaces = [{ id: 'ws1' }];
+            const mockWorkspace = { project: { id: 'proj1' } };
+            entities_helper_1.retryOnTimeout
+                .mockResolvedValueOnce(mockWorkspaces)
+                .mockResolvedValueOnce(mockWorkspace);
+            const data = [
+                { itemFamilyFederatedId: 'fam1', workspaceIdentifier: 'same-ws' },
+                { itemFamilyFederatedId: 'fam2', workspaceIdentifier: 'same-ws' },
+                { itemFamilyFederatedId: 'fam3', workspaceIdentifier: 'same-ws' },
+            ];
+            await project_item_processor_1.ProjectItemProcessor.processEntitiesProperties(client, {}, data);
+            expect(entities_helper_1.retryOnTimeout).toHaveBeenCalledTimes(2);
+            expect(data[0].projectId).toBe('proj1');
+            expect(data[1].projectId).toBe('proj1');
+            expect(data[2].projectId).toBe('proj1');
+        });
+    });
+    describe('setExtraProperties', () => {
+        it('should set typeId and typePath on the entity', () => {
+            const entity = {};
+            const type = { type: { id: 'mockTypeId', typePath: 'mockTypePath' } };
+            project_item_processor_1.ProjectItemProcessor.setExtraProperties(entity, type);
+            expect(entity).toEqual({
+                typeId: 'mockTypeId',
+                typePath: 'mockTypePath',
+            });
+        });
+        it('should copy projectId from row when provided', () => {
+            const entity = {};
+            const type = { type: { id: 'mockTypeId', typePath: 'mockTypePath' } };
+            const row = { projectId: 'rowProjectId' };
+            project_item_processor_1.ProjectItemProcessor.setExtraProperties(entity, type, row);
+            expect(entity.projectId).toBe('rowProjectId');
+        });
+    });
+    describe('assignExtraItems', () => {
+        it('should assign user lists, sanitize size ranges, and set projectId on entities without one', async () => {
+            const mockIncomingProjectItems = [{ id: 'mockItem1' }];
+            const mockTypesByPath = { 'project-item': {} };
+            const result = await project_item_processor_1.ProjectItemProcessor.assignExtraItems(client, ['project-item'], mockTypesByPath, mockIncomingProjectItems, 'mockProjectId');
+            expect(entities_helper_1.retryOnTimeout).toHaveBeenCalledTimes(0);
+            expect(size_range_helpers_1.SizeRangeSanitizer).toHaveBeenCalledTimes(1);
+            expect(result).toEqual({
+                sizeRangeWarningsAsStringList: ['mockWarning'],
+            });
+            expect(mockIncomingProjectItems[0].projectId).toBe('mockProjectId');
+        });
+        it('should not overwrite per-row projectId with legacy single projectId', async () => {
+            const mockIncomingProjectItems = [
+                { id: 'item1', projectId: 'perRowProjectId' },
+                { id: 'item2' },
+            ];
+            const mockTypesByPath = { 'project-item': {} };
+            await project_item_processor_1.ProjectItemProcessor.assignExtraItems(client, ['project-item'], mockTypesByPath, mockIncomingProjectItems, 'legacyProjectId');
+            expect(mockIncomingProjectItems[0].projectId).toBe('perRowProjectId');
+            expect(mockIncomingProjectItems[1].projectId).toBe('legacyProjectId');
+        });
+    });
+    describe('canExtractFederatedIds', () => {
+        it('should return true if itemsByFederatedId is empty and incomingEntities is not empty', () => {
+            const result = project_item_processor_1.ProjectItemProcessor.canExtractFederatedIds({}, [{ id: 'mockEntity' }]);
+            expect(result).toBe(true);
+        });
+        it('should return false if itemsByFederatedId is not empty', () => {
+            const result = project_item_processor_1.ProjectItemProcessor.canExtractFederatedIds({ mockId: {} }, [{ id: 'mockEntity' }]);
+            expect(result).toBe(false);
+        });
+        it('should return false if incomingEntities is empty', () => {
+            const result = project_item_processor_1.ProjectItemProcessor.canExtractFederatedIds({}, []);
+            expect(result).toBe(false);
+        });
+    });
+});

package/node_modules/@contrail/loader/lib/entity-processor.d.ts

@@ -0,0 +1,34 @@
+import { EntitiesClient, EntityType, GeneralIssueDetails, LoaderProcess, LoadType, ProcessingIssues } from '@contrail/entity-types';
+export interface ProcessedEntity {
+    incomingEntities: string[];
+    badEntities: string[];
+    allCurrentEntitiesByFederatedId: Record<string, any>;
+    generalWarnings?: GeneralIssueDetails[];
+}
+export interface UniqueAndDuplicateEntries {
+    uniqueStrings: string[];
+    duplicateStrings: string[];
+}
+export declare class EntityProcessor {
+    static setupEntities(client: EntitiesClient, entityType: EntityType, rawData: any[]): Promise<ProcessedEntity>;
+    private static constructEntities;
+    private static getTypePaths;
+    private static fetchTypesByPath;
+    private static buildIncomingEntities;
+    private static extractFederatedIds;
+    private static fetchCurrentEntities;
+    private static getUniqueAndDuplicateEntries;
+    static validateLoadFile(data: any[], config: LoaderProcess, loadType: LoadType): {
+        errors: ProcessingIssues[];
+        warnings: ProcessingIssues[];
+        generalErrors: GeneralIssueDetails[];
+        generalWarnings: GeneralIssueDetails[];
+        csvHeaders?: undefined;
+    } | {
+        errors: ProcessingIssues[];
+        warnings: ProcessingIssues[];
+        generalErrors: GeneralIssueDetails[];
+        generalWarnings: GeneralIssueDetails[];
+        csvHeaders: string[];
+    };
+}

package/node_modules/@contrail/loader/lib/entity-processor.js

@@ -0,0 +1,214 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EntityProcessor = void 0;
+const entities_helper_1 = require("./common/entities-helper");
+const entity_builder_helpers_1 = require("./common/entity-builder-helpers");
+const processor_registry_1 = require("./processor-registry");
+const lodash_1 = require("lodash");
+const type_helper_1 = require("./common/type-helper");
+const entity_types_1 = require("@contrail/entity-types");
+const util_1 = require("./common/util");
+const constants_1 = require("./constants");
+const pLimit = require('p-limit');
+const limit = pLimit(5);
+class EntityProcessor {
+    static async setupEntities(client, entityType, rawData) {
+        try {
+            const processor = (0, processor_registry_1.getProcessorForEntityType)(entityType);
+            rawData = processor.transformData(rawData);
+            const requiredProperties = processor.requiredProperties || [];
+            const result = await this.constructEntities(client, processor, entityType, requiredProperties, rawData);
+            return result;
+        }
+        catch (error) {
+            console.error(`Error processing and loading objects for ${entityType}:`, error);
+            return {
+                incomingEntities: rawData,
+                badEntities: [],
+                allCurrentEntitiesByFederatedId: {},
+            };
+        }
+    }
+    static async constructEntities(client, processor, entityType, requiredProperties = [], data = []) {
+        if (!data?.length) {
+            console.debug('No data to upload');
+            return {
+                incomingEntities: [],
+                badEntities: [],
+                allCurrentEntitiesByFederatedId: {},
+            };
+        }
+        const typePaths = this.getTypePaths(data, entityType);
+        const typesByPath = await this.fetchTypesByPath(client, typePaths, requiredProperties, entityType);
+        const { badEntities, incomingEntities } = this.buildIncomingEntities(processor, data, typesByPath, entityType);
+        if (incomingEntities.length === 0)
+            return {
+                incomingEntities,
+                badEntities,
+                allCurrentEntitiesByFederatedId: {},
+            };
+        const generalWarnings = await (0, entities_helper_1.assignObjectReferences)(client, typePaths, typesByPath, incomingEntities);
+        const federatedIds = this.extractFederatedIds(incomingEntities, processor);
+        const allCurrentEntitiesByFederatedId = await this.fetchCurrentEntities(client, entityType, federatedIds);
+        console.debug(`${entityType}s to upload: `, incomingEntities.length);
+        return {
+            incomingEntities,
+            badEntities,
+            allCurrentEntitiesByFederatedId,
+            generalWarnings,
+        };
+    }
+    static getTypePaths(data, entityType) {
+        return Array.from(new Set(data.map((row) => row.typePath || entityType)));
+    }
+    static async fetchTypesByPath(client, typePaths, requiredProperties, entityType) {
+        const typesByPath = await (0, type_helper_1.buildTypesByPath)(client, typePaths, requiredProperties, entityType);
+        console.debug(`Using ${entityType} types:`, JSON.stringify(typesByPath, null, 2));
+        return typesByPath;
+    }
+    static buildIncomingEntities(processor, data, typesByPath, entityType) {
+        const incomingEntities = [];
+        const badEntities = [];
+        for (const row of data) {
+            if (!processor.validateRow(row)) {
+                badEntities.push(row);
+                console.error(`Invalid data found for this row ${entityType}:`, row);
+                continue;
+            }
+            const typePath = row.typePath || entityType;
+            const type = typesByPath[typePath];
+            const entity = {};
+            for (const propertyName in row) {
+                if (type.allPropertiesSlugs.includes(propertyName)) {
+                    (0, entity_builder_helpers_1.addProperty)(propertyName, type.allPropertiesSlugs, type.typePropertyToValueType[propertyName], entity, row[propertyName]);
+                }
+            }
+            processor.setExtraProperties(entity, type);
+            incomingEntities.push(entity);
+        }
+        return { badEntities, incomingEntities };
+    }
+    static extractFederatedIds(incomingEntities, processor) {
+        const allFederatedIds = [];
+        const re = /\//gi;
+        for (const incomingEntity of incomingEntities) {
+            const processedFederatedId = processor.processFederatedId(incomingEntity.federatedId, incomingEntity.typePath);
+            const federatedId = processedFederatedId.replace(re, '%2F');
+            allFederatedIds.push(federatedId);
+            incomingEntity.federatedId = processedFederatedId;
+        }
+        const { uniqueStrings: uniqueFedIds, duplicateStrings: nonUniqueFedIds } = this.getUniqueAndDuplicateEntries(allFederatedIds);
+        console.debug(`Unique federatedId count: ${uniqueFedIds?.length}`);
+        console.debug(`Non-Unique federatedId count: ${nonUniqueFedIds?.length}`);
+        console.debug(`Non-Unique federatedIds: ${Array.from(nonUniqueFedIds)}`);
+        return [...uniqueFedIds, ...nonUniqueFedIds];
+    }
+    static async fetchCurrentEntities(client, entityType, federatedIds) {
+        const chunks = (0, lodash_1.chunk)(federatedIds, 200);
+        const allCurrentEntitiesByFederatedId = {};
+        const federatedPromises = [];
+        for (const chunk of chunks) {
+            if (!chunk?.length)
+                continue;
+            federatedPromises.push(limit(async () => {
+                try {
+                    const currentEntityBatch = await (0, entities_helper_1.retryOnTimeout)(client, entities_helper_1.EntitiesLambdaCallType.GET, {
+                        entityName: entityType,
+                        federatedIds: chunk,
+                        skipHydrateRelationships: true,
+                    });
+                    if (currentEntityBatch?.length) {
+                        for (const currentEntity of currentEntityBatch) {
+                            allCurrentEntitiesByFederatedId[currentEntity.federatedId] = currentEntity;
+                        }
+                    }
+                }
+                catch (e) {
+                    console.error(`Failed to fetch ${entityType} by federatedIds:`, e);
+                    throw e;
+                }
+            }));
+        }
+        await Promise.all(federatedPromises);
+        console.info(`${entityType} lookup complete. ${Object.keys(allCurrentEntitiesByFederatedId).length} ${entityType}s retrieved`);
+        return allCurrentEntitiesByFederatedId;
+    }
+    static getUniqueAndDuplicateEntries(strings) {
+        const uniqueStrings = new Set();
+        const duplicateStrings = new Set();
+        for (const string of strings) {
+            if (uniqueStrings.has(string)) {
+                duplicateStrings.add(string);
+            }
+            else {
+                uniqueStrings.add(string);
+            }
+        }
+        return {
+            uniqueStrings: Array.from(uniqueStrings),
+            duplicateStrings: Array.from(duplicateStrings),
+        };
+    }
+    static validateLoadFile(data, config, loadType) {
+        const csvHeaders = Object.keys(data[0]);
+        const errors = [];
+        const warnings = [];
+        const generalErrors = [];
+        const generalWarnings = [];
+        if (!config) {
+            generalErrors.push({
+                message: constants_1.ENTITY_PROCESSOR_ERRORS.CONFIG_EMPTY,
+                issueType: entity_types_1.IssueType.CONFIG,
+            });
+            return { errors, warnings, generalErrors, generalWarnings };
+        }
+        if (!config?.loadType?.length) {
+            generalErrors.push({
+                message: constants_1.ENTITY_PROCESSOR_ERRORS.LOAD_TYPE_REQUIRED,
+                issueType: entity_types_1.IssueType.CONFIG,
+            });
+        }
+        if (config?.loadType?.length && config?.loadType[0] !== loadType) {
+            generalErrors.push({
+                message: constants_1.ENTITY_PROCESSOR_ERRORS.LOAD_TYPE_INVALID(loadType),
+                issueType: entity_types_1.IssueType.CONFIG,
+            });
+        }
+        const federatedIdInConditional = config?.conditionalColumns?.some((column) => column.toProperty === 'federatedId');
+        const federatedIdInFederatedMapping = config?.federatedMappings?.federatedId;
+        const federatedIdInCSV = csvHeaders.includes('federatedId');
+        if (!federatedIdInFederatedMapping && !federatedIdInCSV && !federatedIdInConditional) {
+            generalErrors.push({
+                message: constants_1.ENTITY_PROCESSOR_ERRORS.FEDERATED_ID_REQUIRED,
+                issueType: entity_types_1.IssueType.CONFIG,
+            });
+        }
+        else if (federatedIdInFederatedMapping && !csvHeaders.includes(config.federatedMappings.federatedId)) {
+            generalErrors.push({
+                message: constants_1.ENTITY_PROCESSOR_ERRORS.FEDERATED_ID_REQUIRED,
+                issueType: entity_types_1.IssueType.CONFIG,
+            });
+        }
+        const nameInConditional = config?.conditionalColumns?.some((column) => column.toProperty === 'name');
+        const nameInFederatedMapping = config?.federatedMappings?.name;
+        const nameInCSV = csvHeaders.includes('name');
+        if (!nameInFederatedMapping && !nameInCSV && !nameInConditional) {
+            generalErrors.push({
+                message: constants_1.ENTITY_PROCESSOR_ERRORS.NAME_REQUIRED,
+                issueType: entity_types_1.IssueType.CONFIG,
+            });
+        }
+        else if (nameInFederatedMapping && !csvHeaders.includes(config.federatedMappings.name)) {
+            generalErrors.push({
+                message: constants_1.ENTITY_PROCESSOR_ERRORS.NAME_REQUIRED,
+                issueType: entity_types_1.IssueType.CONFIG,
+            });
+        }
+        const duplicateFederatedIdErrors = (0, util_1.getDuplicateFederatedIds)(data, config?.federatedMappings?.federatedId);
+        if (duplicateFederatedIdErrors.length) {
+            errors.push(...duplicateFederatedIdErrors);
+        }
+        return { errors, warnings, generalErrors, generalWarnings, csvHeaders };
+    }
+}
+exports.EntityProcessor = EntityProcessor;

package/node_modules/@contrail/loader/lib/entity-processor.spec.d.ts

@@ -0,0 +1 @@
+export {};