@contrail/cli 3.4.5 → 3.4.6-alpha.cffb28e

package/node_modules/@contrail/loader/lib/entities-v2/custom-entities-v2/custom-entity-processor-v2.spec.js

@@ -0,0 +1,211 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const constants_1 = require("../../constants");
+const custom_entity_processor_v2_1 = require("./custom-entity-processor-v2");
+const entity_types_1 = require("@contrail/entity-types");
+describe('Custom Entity Processor', () => {
+    const loaderProcess = {
+        id: 'mock-loader-process-id',
+        loadFileId: 'mock-load-file-id',
+        loadType: [entity_types_1.LoadType.CUSTOM_ENTITY],
+        status: entity_types_1.LoadStatus.COMPLETE,
+        federatedMappings: {
+            federatedId: 'name',
+            name: 'name',
+        },
+        conditionalColumns: [
+            {
+                toProperty: 'typePath',
+                default: 'custom-entity:material',
+            },
+        ],
+    };
+    describe('requiredProperties', () => {
+        it('should have the correct required properties', () => {
+            expect(custom_entity_processor_v2_1.CustomEntityProcessorV2.requiredProperties).toEqual(['typePath', 'typeId', 'federatedId', 'name']);
+        });
+    });
+    describe('validateLoadFile', () => {
+        it('should return duplicate federated ID errors', () => {
+            const data = [
+                { name: 'custom-entity1' },
+                { name: 'custom-entity2' },
+                { name: 'custom-entity1' },
+            ];
+            const result = custom_entity_processor_v2_1.CustomEntityProcessorV2.validateLoadFile(data, loaderProcess);
+            expect(result.errors).toEqual([
+                {
+                    rowNumber: 2,
+                    columnIssues: [
+                        {
+                            columnName: 'name',
+                            message: 'Duplicate federatedId found: custom-entity1',
+                            issueType: entity_types_1.IssueType.DUPLICATE_FEDERATED_ID,
+                            data: '{"name":"custom-entity1"}',
+                        },
+                    ],
+                },
+                {
+                    rowNumber: 4,
+                    columnIssues: [
+                        {
+                            columnName: 'name',
+                            message: 'Duplicate federatedId found: custom-entity1',
+                            issueType: entity_types_1.IssueType.DUPLICATE_FEDERATED_ID,
+                            data: '{"name":"custom-entity1"}',
+                        },
+                    ],
+                },
+            ]);
+        });
+        it('should handle no duplicate federated IDs', () => {
+            const data = [
+                { name: 'custom-entity1' },
+                { name: 'custom-entity2' },
+                { name: 'custom-entity3' },
+            ];
+            const result = custom_entity_processor_v2_1.CustomEntityProcessorV2.validateLoadFile(data, loaderProcess);
+            expect(result.errors).toEqual([]);
+        });
+        it('should throw general error if typePath is incorrect', () => {
+            const data = [{ name: 'custom-entity1' }];
+            const config = {
+                ...loaderProcess,
+                conditionalColumns: [
+                    {
+                        toProperty: 'typePath',
+                        default: 'custom-entity:',
+                    },
+                ],
+            };
+            const result = custom_entity_processor_v2_1.CustomEntityProcessorV2.validateLoadFile(data, config);
+            expect(result.generalErrors).toEqual([
+                {
+                    message: constants_1.CUSTOM_ENTITY_PROCESSOR_ERRORS.TYPEPATH_MISSING,
+                    issueType: entity_types_1.IssueType.CONFIG,
+                },
+            ]);
+        });
+        it('should throw general error if typePath is incorrect', () => {
+            const data = [{ name: 'custom-entity1' }];
+            const config = {
+                ...loaderProcess,
+                conditionalColumns: [
+                    {
+                        toProperty: 'typePath',
+                        default: 'color:material',
+                    },
+                ],
+            };
+            const result = custom_entity_processor_v2_1.CustomEntityProcessorV2.validateLoadFile(data, config);
+            expect(result.generalErrors).toEqual([
+                {
+                    message: constants_1.CUSTOM_ENTITY_PROCESSOR_ERRORS.TYPEPATH_MISSING,
+                    issueType: entity_types_1.IssueType.CONFIG,
+                },
+            ]);
+        });
+        it('should throw error if typepath is present in the config file but have incorrect typePath', () => {
+            const data = [{ name: 'custom-entity1' }];
+            const config = {
+                ...loaderProcess,
+                conditionalColumns: [
+                    {
+                        toProperty: 'typePath',
+                        default: 'custom-entity1',
+                    },
+                ],
+            };
+            const result = custom_entity_processor_v2_1.CustomEntityProcessorV2.validateLoadFile(data, config);
+            expect(result.generalErrors).toEqual([
+                {
+                    message: constants_1.CUSTOM_ENTITY_PROCESSOR_ERRORS.TYPEPATH_MISSING,
+                    issueType: entity_types_1.IssueType.CONFIG,
+                },
+            ]);
+        });
+        it('should throw error if typepath is present in the csv header but have incorrect typePath', () => {
+            const data = [{ name: 'custom-entity1', typePath: 'custom-entity1' }];
+            const config = {
+                ...loaderProcess,
+                conditionalColumns: [],
+            };
+            const result = custom_entity_processor_v2_1.CustomEntityProcessorV2.validateLoadFile(data, config);
+            expect(result.generalErrors).toEqual([
+                {
+                    message: constants_1.CUSTOM_ENTITY_PROCESSOR_ERRORS.TYPEPATH_MISSING,
+                    issueType: entity_types_1.IssueType.CONFIG,
+                },
+            ]);
+        });
+        it('should not throw error if typepath is present in the csv header and have correct typePath', () => {
+            const data = [{ name: 'custom-entity1', typePath: 'custom-entity:material' }];
+            const config = {
+                ...loaderProcess,
+                conditionalColumns: [],
+            };
+            const result = custom_entity_processor_v2_1.CustomEntityProcessorV2.validateLoadFile(data, config);
+            expect(result.generalErrors).toEqual([]);
+        });
+        it('should not throw error if typepath is present in the config file and have correct typePath', () => {
+            const data = [{ name: 'custom-entity1' }];
+            const config = {
+                ...loaderProcess,
+                conditionalColumns: [
+                    {
+                        toProperty: 'typePath',
+                        default: 'custom-entity:material',
+                    },
+                ],
+            };
+            const result = custom_entity_processor_v2_1.CustomEntityProcessorV2.validateLoadFile(data, config);
+            expect(result.generalErrors).toEqual([]);
+        });
+        it('should throw error if typepath is not present in the config file or csv header', () => {
+            const data = [{ name: 'custom-entity1' }];
+            const config = {
+                ...loaderProcess,
+                conditionalColumns: [],
+            };
+            const result = custom_entity_processor_v2_1.CustomEntityProcessorV2.validateLoadFile(data, config);
+            expect(result.generalErrors).toEqual([
+                {
+                    message: constants_1.CUSTOM_ENTITY_PROCESSOR_ERRORS.TYPEPATH_MISSING,
+                    issueType: entity_types_1.IssueType.CONFIG,
+                },
+            ]);
+        });
+        it('should return error for duplicate federated IDs', () => {
+            const data = [
+                { name: 'custom-entity1' },
+                { name: 'custom-entity1' },
+                { name: 'custom-entity3' },
+            ];
+            const result = custom_entity_processor_v2_1.CustomEntityProcessorV2.validateLoadFile(data, loaderProcess);
+            expect(result.errors).toEqual([
+                {
+                    rowNumber: 2,
+                    columnIssues: [
+                        {
+                            columnName: 'name',
+                            message: 'Duplicate federatedId found: custom-entity1',
+                            issueType: entity_types_1.IssueType.DUPLICATE_FEDERATED_ID,
+                            data: '{"name":"custom-entity1"}',
+                        },
+                    ],
+                },
+                {
+                    rowNumber: 3,
+                    columnIssues: [
+                        {
+                            columnName: 'name',
+                            message: 'Duplicate federatedId found: custom-entity1',
+                            issueType: entity_types_1.IssueType.DUPLICATE_FEDERATED_ID,
+                            data: '{"name":"custom-entity1"}',
+                        },
+                    ],
+                },
+            ]);
+        });
+    });
+});

package/node_modules/@contrail/loader/lib/entities-v2/entities-helper.d.ts

@@ -0,0 +1,33 @@
+import { Type, TypeProperty, PropertyType } from '@contrail/types';
+import { EntitiesClient, GeneralIssueDetails } from '@contrail/entity-types';
+export declare enum EntitiesLambdaCallType {
+    GET = "GET",
+    CREATE = "CREATE",
+    UPDATE = "UPDATE",
+    DELETE = "DELETE",
+    BATCHCREATE = "BATCHCREATE",
+    BATCHUPDATE = "BATCHUPDATE"
+}
+export interface TypesByPath {
+    [key: string]: {
+        type: Type;
+        allPropertiesSlugs: string[];
+        typePropertyToValueType: {
+            [key: string]: PropertyType;
+        };
+        objectReferenceProperties: TypeProperty[];
+        userListProperties: TypeProperty[];
+        error?: GeneralIssueDetails;
+    };
+}
+export declare function retryOnTimeout(client: EntitiesClient, callType: EntitiesLambdaCallType, callParams: any, maxRetries?: number): Promise<any>;
+export declare function UpdateNeeded(incoming: any, existing: any, fieldsToSkip?: any[], fieldsToSkipIfOnlyOnExisting?: any[], type?: Type): boolean;
+export declare function compareIncomingAndExisting(incoming: any, existing: any, fieldsToSkip?: any[], fieldsToSkipIfOnlyOnExisting?: any[], type?: Type): import("@contrail/util").ObjectDiff[];
+export declare function parseDateOrNumberKey(obj: any, key: any, propertyType?: any): void;
+export declare function assignObjectReferences(client: EntitiesClient, typePaths: string[], typesByPath: TypesByPath, incomingEntities: any[]): Promise<GeneralIssueDetails[]>;
+export declare function getUsersInOrg(client: EntitiesClient): Promise<any>;
+export declare function assignUserLists(client: EntitiesClient, typePaths: string[], typesByPath: TypesByPath, incomingEntities: any[]): Promise<void>;
+export declare function getPropertyVariants(entity: any, slug: string, returnType?: 'values' | 'keyValue'): Array<any> | Array<{
+    key: string;
+    value: any;
+}>;

package/node_modules/@contrail/loader/lib/entities-v2/entities-helper.js

@@ -0,0 +1,298 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getPropertyVariants = exports.assignUserLists = exports.getUsersInOrg = exports.assignObjectReferences = exports.parseDateOrNumberKey = exports.compareIncomingAndExisting = exports.UpdateNeeded = exports.retryOnTimeout = exports.EntitiesLambdaCallType = void 0;
+const util_1 = require("@contrail/util");
+const types_1 = require("@contrail/types");
+const lodash_1 = require("lodash");
+const entity_types_1 = require("@contrail/entity-types");
+const pLimit = require('p-limit');
+var EntitiesLambdaCallType;
+(function (EntitiesLambdaCallType) {
+    EntitiesLambdaCallType["GET"] = "GET";
+    EntitiesLambdaCallType["CREATE"] = "CREATE";
+    EntitiesLambdaCallType["UPDATE"] = "UPDATE";
+    EntitiesLambdaCallType["DELETE"] = "DELETE";
+    EntitiesLambdaCallType["BATCHCREATE"] = "BATCHCREATE";
+    EntitiesLambdaCallType["BATCHUPDATE"] = "BATCHUPDATE";
+})(EntitiesLambdaCallType = exports.EntitiesLambdaCallType || (exports.EntitiesLambdaCallType = {}));
+async function retryOnTimeout(client, callType, callParams, maxRetries = 3) {
+    let callCount = 0;
+    let done = false;
+    let response;
+    if (!client) {
+        throw new Error('No client to use.');
+    }
+    while (!done && callCount <= maxRetries) {
+        try {
+            if (callCount > 0) {
+                console.debug('Retrying call - Attempt: ', callCount);
+            }
+            switch (callType) {
+                case EntitiesLambdaCallType.GET:
+                    response = await client.get(callParams);
+                    break;
+                case EntitiesLambdaCallType.CREATE:
+                    response = await client.create(callParams);
+                    break;
+                case EntitiesLambdaCallType.UPDATE:
+                    response = await client.update(callParams);
+                    break;
+                case EntitiesLambdaCallType.DELETE:
+                    response = await client.delete(callParams);
+                    break;
+                case EntitiesLambdaCallType.BATCHCREATE:
+                    response = await client.batchCreate(callParams);
+                    break;
+                case EntitiesLambdaCallType.BATCHUPDATE:
+                    response = await client.batchUpdate(callParams);
+                    break;
+                default:
+                    throw Error('Invalid EntitiesLambdaCallType');
+            }
+            done = true;
+        }
+        catch (e) {
+            if (e instanceof entity_types_1.TimeoutError && callCount + 1 <= maxRetries) {
+                console.warn(`Lambda call failed due to timeout. callType: ${callType}, callParams: `, callParams);
+            }
+            else {
+                console.error('Failed Lambda call. ', e);
+                throw e;
+            }
+        }
+        callCount++;
+    }
+    return response;
+}
+exports.retryOnTimeout = retryOnTimeout;
+function UpdateNeeded(incoming, existing, fieldsToSkip = [], fieldsToSkipIfOnlyOnExisting = [], type) {
+    console.debug('Comparing existing and incoming objects...');
+    const diffs = compareIncomingAndExisting(incoming, existing, fieldsToSkip, fieldsToSkipIfOnlyOnExisting, type);
+    return diffs?.length > 0;
+}
+exports.UpdateNeeded = UpdateNeeded;
+function compareIncomingAndExisting(incoming, existing, fieldsToSkip = [], fieldsToSkipIfOnlyOnExisting = [], type) {
+    const slimExisting = {};
+    const keys = Object.keys(incoming);
+    const cloneIncoming = util_1.ObjectUtil.cloneDeep(incoming);
+    for (const key of keys) {
+        parseDateOrNumberKey(cloneIncoming, key);
+        slimExisting[key] = existing[key];
+        parseDateOrNumberKey(slimExisting, key);
+    }
+    for (const fieldToSkip of fieldsToSkip) {
+        delete slimExisting[fieldToSkip];
+        delete cloneIncoming[fieldToSkip];
+    }
+    if (type) {
+        const typePropertiesToSkip = type?.typeProperties
+            ?.filter((x) => [types_1.PropertyType.ObjectReference, types_1.PropertyType.TypeReference, types_1.PropertyType.UserList].includes(x.propertyType))
+            ?.map((x) => x.slug);
+        for (const typePropertyToSkip of typePropertiesToSkip) {
+            delete slimExisting[typePropertyToSkip];
+            delete cloneIncoming[typePropertyToSkip];
+        }
+    }
+    for (const fieldToSkipIfOnlyOnExisting of fieldsToSkipIfOnlyOnExisting) {
+        if (Object.prototype.hasOwnProperty.call(slimExisting, fieldToSkipIfOnlyOnExisting) &&
+            !Object.prototype.hasOwnProperty.call(cloneIncoming, fieldToSkipIfOnlyOnExisting)) {
+            delete slimExisting[fieldToSkipIfOnlyOnExisting];
+            delete cloneIncoming[fieldToSkipIfOnlyOnExisting];
+        }
+    }
+    const objectDiffs = util_1.ObjectUtil.compareDeep(slimExisting, cloneIncoming, '');
+    return objectDiffs;
+}
+exports.compareIncomingAndExisting = compareIncomingAndExisting;
+function parseDateOrNumberKey(obj, key, propertyType) {
+    const value = obj[key];
+    if (!propertyType) {
+        if (value != null && value instanceof Date) {
+            obj[key] = `${value.toISOString()}`;
+        }
+        else if (value != null && typeof value === 'string' && value !== '') {
+            const parse = Number(value);
+            if (!isNaN(parse)) {
+                obj[key] = parse;
+            }
+        }
+    }
+    else {
+        if (value != null && propertyType === types_1.PropertyType.Date) {
+            if (value instanceof Date) {
+                obj[key] = `${value.toISOString()}`;
+            }
+            else {
+                console.warn('PropertyType.Date is set but the value is not a Date object', value);
+                obj[key] = null;
+            }
+        }
+        else if (propertyType === types_1.PropertyType.Number) {
+            const parse = Number(value);
+            if (!isNaN(parse)) {
+                obj[key] = parse;
+            }
+        }
+    }
+}
+exports.parseDateOrNumberKey = parseDateOrNumberKey;
+async function assignObjectReferences(client, typePaths, typesByPath, incomingEntities) {
+    const limit = pLimit(5);
+    const allObjectReferenceColumnsWithRootType = [];
+    for (const typePath of typePaths ?? []) {
+        const typeObj = typesByPath[typePath];
+        if (!(typeObj?.objectReferenceProperties?.length > 0)) {
+            console.debug(`No ObjectReferences for ${typePath}, skipping.`);
+            continue;
+        }
+        for (const objectReferenceProperty of typeObj?.objectReferenceProperties ?? []) {
+            const slug = objectReferenceProperty.slug;
+            const rootType = objectReferenceProperty.referencedTypeRootSlug;
+            const referencedTypePath = objectReferenceProperty.referencedTypePath;
+            allObjectReferenceColumnsWithRootType.push({
+                slug,
+                rootType,
+                referencedTypePath,
+            });
+        }
+    }
+    const allORFedIdsByType = {};
+    for (const entity of incomingEntities) {
+        for (const allORColumnWithRootType of allObjectReferenceColumnsWithRootType) {
+            const slug = allORColumnWithRootType.slug;
+            const levelVariants = getPropertyVariants(entity, slug, 'values');
+            for (let fedId of levelVariants) {
+                if (!allORFedIdsByType[allORColumnWithRootType.rootType]) {
+                    allORFedIdsByType[allORColumnWithRootType.rootType] = [];
+                }
+                if (!fedId.startsWith('custom-entity:') &&
+                    allORColumnWithRootType.referencedTypePath.includes('custom-entity')) {
+                    fedId = `${allORColumnWithRootType.referencedTypePath}:${fedId}`;
+                }
+                allORFedIdsByType[allORColumnWithRootType.rootType].push(fedId);
+            }
+        }
+    }
+    const allORByFedId = {};
+    for (const rootType of Object.keys(allORFedIdsByType)) {
+        const fedIds = [...new Set(allORFedIdsByType[rootType])];
+        const chunks = (0, lodash_1.chunk)(fedIds, 200);
+        const federatedPromises = [];
+        for (const chunk of chunks) {
+            if (!chunk?.length) {
+                continue;
+            }
+            const federatedPromise = limit(async () => {
+                try {
+                    const currentORBatch = await retryOnTimeout(client, EntitiesLambdaCallType.GET, {
+                        entityName: rootType,
+                        federatedIds: (0, lodash_1.uniq)(chunk),
+                        skipHydrateRelationships: true,
+                    });
+                    if (!currentORBatch) {
+                    }
+                    if (currentORBatch?.length > 0) {
+                        for (const currentOR of currentORBatch) {
+                            allORByFedId[currentOR.federatedId] = currentOR;
+                        }
+                    }
+                }
+                catch (e) {
+                    const errorMessage = `'Failed to get ObjectReferences by FederatedIds: ${e}`;
+                    console.error(errorMessage);
+                    throw new Error(errorMessage);
+                }
+            });
+            federatedPromises.push(federatedPromise);
+        }
+        await Promise.all(federatedPromises);
+        console.info(`ObjectReferences lookup complete. ${Object.keys(allORByFedId)?.length} Object References retrieved`);
+    }
+    const objectReferenceWarnings = new Set();
+    for (const entity of incomingEntities) {
+        for (const allORColumnWithRootType of allObjectReferenceColumnsWithRootType) {
+            const slug = allORColumnWithRootType.slug;
+            const levelVariants = getPropertyVariants(entity, slug, 'keyValue');
+            for (const variant of levelVariants) {
+                let fedId = variant.value;
+                if (!fedId)
+                    continue;
+                if (!fedId.startsWith('custom-entity:') &&
+                    allORColumnWithRootType.referencedTypePath.includes('custom-entity')) {
+                    fedId = `${allORColumnWithRootType.referencedTypePath}:${fedId}`;
+                }
+                const objectReference = allORByFedId[fedId];
+                if (!objectReference) {
+                    const message = `ObjectReference not found for ${slug} with value ${fedId}`;
+                    objectReferenceWarnings.add(message);
+                    console.warn(message);
+                    continue;
+                }
+                entity[variant.key + 'Id'] = objectReference.id;
+                delete entity[variant.key];
+            }
+        }
+    }
+    return Array.from(objectReferenceWarnings).map((warning) => ({
+        message: warning,
+        issueType: entity_types_1.IssueType.INVALID_DATA,
+    }));
+}
+exports.assignObjectReferences = assignObjectReferences;
+async function getUsersInOrg(client) {
+    return await client.get({ entityName: 'user', relation: 'usersInOrg' });
+}
+exports.getUsersInOrg = getUsersInOrg;
+async function assignUserLists(client, typePaths, typesByPath, incomingEntities) {
+    const hasUserListProps = (typePaths ?? []).some((typePath) => {
+        const typeObj = typesByPath[typePath];
+        return typeObj?.userListProperties?.length > 0;
+    });
+    if (!hasUserListProps) {
+        console.debug(`No user list properties for this load, skipping.`);
+        return;
+    }
+    const users = await getUsersInOrg(client);
+    const userEmailToIdMap = [];
+    for (const user of users) {
+        userEmailToIdMap[user.email] = user.id;
+    }
+    for (const typePath of typePaths ?? []) {
+        const typeObj = typesByPath[typePath];
+        if (!(typeObj?.userListProperties?.length > 0)) {
+            console.debug(`No user lists for ${typePath}, skipping.`);
+            continue;
+        }
+        for (const entity of incomingEntities) {
+            for (const userListProperty of typeObj.userListProperties ?? []) {
+                const slug = userListProperty.slug;
+                const value = entity[slug];
+                if (!value) {
+                    continue;
+                }
+                let userId = userEmailToIdMap[value];
+                if (!userId) {
+                    userId = null;
+                }
+                entity[slug + 'Id'] = userId;
+                delete entity[slug];
+            }
+        }
+    }
+}
+exports.assignUserLists = assignUserLists;
+function getPropertyVariants(entity, slug, returnType = 'values') {
+    const hasFamilyOrOption = entity.hasOwnProperty(`${slug}Family`) || entity.hasOwnProperty(`${slug}Option`);
+    const variants = [];
+    if (entity.hasOwnProperty(`${slug}Family`)) {
+        variants.push(returnType === 'values' ? entity[`${slug}Family`] : { key: `${slug}Family`, value: entity[`${slug}Family`] });
+    }
+    if (entity.hasOwnProperty(`${slug}Option`)) {
+        variants.push(returnType === 'values' ? entity[`${slug}Option`] : { key: `${slug}Option`, value: entity[`${slug}Option`] });
+    }
+    if (!hasFamilyOrOption && entity.hasOwnProperty(slug)) {
+        variants.push(returnType === 'values' ? entity[slug] : { key: slug, value: entity[slug] });
+    }
+    return variants;
+}
+exports.getPropertyVariants = getPropertyVariants;

package/node_modules/@contrail/loader/lib/entities-v2/entities-helper.spec.d.ts

@@ -0,0 +1 @@
+export {};