@contrail/cli 2.4.0 → 3.0.1

package/dist/helpers/load/helpers.js

@@ -0,0 +1,141 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getConflictingFieldForFederatedId = exports.getConflictingFederatedIdsGroupedByField = exports.getDuplicateFederatedIds = exports.loadLocalFile = exports.uploadAndLoadLocalFile = exports.readFile = exports.FileExtension = void 0;
+const tslib_1 = require("tslib");
+const path = tslib_1.__importStar(require("path"));
+const sdk_1 = require("@contrail/sdk");
+const fs = tslib_1.__importStar(require("fs"));
+const js_yaml_1 = tslib_1.__importDefault(require("js-yaml"));
+const logger_1 = tslib_1.__importDefault(require("../../logger"));
+const file_helpers_1 = require("../file-helpers");
+const lodash_1 = require("lodash");
+var FileExtension;
+(function (FileExtension) {
+    FileExtension["JSON"] = "JSON";
+    FileExtension["YAML"] = "YAML";
+    FileExtension["CSV"] = "CSV";
+})(FileExtension = exports.FileExtension || (exports.FileExtension = {}));
+async function getLocalFile(incomingFileName) {
+    const ext = getFileExtension(incomingFileName);
+    if (ext !== '.csv') {
+        throw new Error('Upload file must be a .csv');
+    }
+    const TARGET_DIR = process.cwd();
+    try {
+        return fs.readFileSync(path.join(TARGET_DIR, incomingFileName));
+    }
+    catch (e) {
+        throw new Error(`ERROR LOADING CSV: ${e}`);
+    }
+}
+function readFile(configFileName, fileExtensionType = FileExtension.JSON) {
+    const CURRENT_DIR = process.cwd();
+    const fullPath = path.join(CURRENT_DIR, configFileName);
+    logger_1.default.info(`Getting FILE from path: ${fullPath}`);
+    if (!(fileExtensionType in FileExtension)) {
+        throw new Error(`Unsupported file extension type: ${fileExtensionType}`);
+    }
+    try {
+        if (fileExtensionType === FileExtension.YAML) {
+            return js_yaml_1.default.load(fs.readFileSync(fullPath, 'utf8'));
+        }
+        else if (fileExtensionType === FileExtension.JSON) {
+            return JSON.parse(fs.readFileSync(fullPath, 'utf8'));
+        }
+        else if (fileExtensionType === FileExtension.CSV) {
+            return (0, file_helpers_1.parseCSVData)(fs.readFileSync(fullPath, 'utf8'));
+        }
+    }
+    catch (e) {
+        throw new Error(`ERROR Reading file (${configFileName}): ${e}`);
+    }
+}
+exports.readFile = readFile;
+async function uploadFile(incomingFileName) {
+    const csvContent = await getLocalFile(incomingFileName);
+    const entities = new sdk_1.Entities();
+    const fileName = `${incomingFileName.slice(0, incomingFileName.length - 4)}-Upload-${new Date().getTime()}.csv`;
+    const file = await entities.create({
+        entityName: 'file',
+        object: { contentType: 'text/csv', fileName: fileName, ttl: 24 * 60 * 60 * 1000 },
+    });
+    logger_1.default.info(`VibeIQ file ID: ${file.id}`);
+    // Upload the CSV data to the newly created VibeIQ file
+    const fileDetails = {
+        contentType: 'text/csv',
+    };
+    await new sdk_1.Files().uploadFileAsStream(file.uploadPost, csvContent, fileDetails);
+    logger_1.default.info('File uploaded.');
+    return file;
+}
+async function loadFile(config) {
+    const entities = new sdk_1.Entities();
+    // start a load job referencing the newly created file
+    logger_1.default.info(`Loading with config: ${config}`);
+    const loadJob = await entities.create({
+        entityName: 'loader-process',
+        object: {
+            ...config,
+        },
+    });
+    logger_1.default.info(`Load has been kicked off. loader-process ID: ${loadJob.id}`);
+}
+async function uploadAndLoadLocalFile(incomingFileName, configFileName) {
+    const file = await uploadFile(incomingFileName);
+    const config = readFile(configFileName, FileExtension.YAML);
+    const fullConfig = {
+        fileLocation: file.id,
+        ...config,
+    };
+    await loadFile(fullConfig);
+}
+exports.uploadAndLoadLocalFile = uploadAndLoadLocalFile;
+async function loadLocalFile(configFileName) {
+    const config = readFile(configFileName, FileExtension.YAML);
+    await loadFile(config);
+}
+exports.loadLocalFile = loadLocalFile;
+function getFileExtension(fileName) {
+    let ext = '';
+    const parts = fileName?.split('.');
+    if (parts?.length > 1) {
+        ext = `.${parts[parts?.length - 1]}`;
+    }
+    return ext;
+}
+function getDuplicateFederatedIds(data, federatedIdField) {
+    if (!data || !federatedIdField)
+        return [];
+    const federatedIdCounts = (0, lodash_1.countBy)(data, federatedIdField);
+    const duplicateIds = Object.fromEntries(Object.entries(federatedIdCounts).filter(([, count]) => count > 1));
+    return Object.keys(duplicateIds);
+}
+exports.getDuplicateFederatedIds = getDuplicateFederatedIds;
+/**
+ * Returns federated IDs that have more than one unique mapping to the groupByField.
+ */
+function getConflictingFederatedIdsGroupedByField(data, federatedIdField, groupByField) {
+    if (!data || !federatedIdField || !groupByField)
+        return [];
+    const groupedByField = (0, lodash_1.groupBy)(data, groupByField);
+    const conflictingFedIds = Object.entries(groupedByField)
+        .map(([uniqueNames, rows]) => [uniqueNames, (0, lodash_1.uniq)(rows.map((x) => x[federatedIdField]))])
+        .filter(([uniqueNames, uniqueFedIds]) => uniqueFedIds.length > 1)
+        .map(([uniqueNames, conflictingFedIds]) => conflictingFedIds);
+    return (0, lodash_1.flatten)(conflictingFedIds);
+}
+exports.getConflictingFederatedIdsGroupedByField = getConflictingFederatedIdsGroupedByField;
+/**
+ * Returns all the conflicting (non-unique) field values when grouped by federated ID.
+ */
+function getConflictingFieldForFederatedId(data, federatedIdField, field) {
+    if (!data || !federatedIdField || !field)
+        return [];
+    const groupedByFedId = (0, lodash_1.groupBy)(data, federatedIdField);
+    const conflictingValues = Object.entries(groupedByFedId)
+        .map(([federatedId, rows]) => [federatedId, (0, lodash_1.uniq)(rows.map((x) => x[field]))])
+        .filter(([federatedId, uniqueNames]) => uniqueNames.length > 1)
+        .map(([federatedId, names]) => ({ federatedId, values: names }));
+    return conflictingValues;
+}
+exports.getConflictingFieldForFederatedId = getConflictingFieldForFederatedId;

package/dist/helpers/load/helpers.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/helpers/load/helpers.spec.js

@@ -0,0 +1,62 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const helpers_1 = require("./helpers");
+describe('getDuplicateFederatedIdsGroupedByField', () => {
+    it('should return an empty array if data is null', () => {
+        const data = null;
+        const federatedIdField = 'federatedId';
+        const groupByField = 'name';
+        const result = (0, helpers_1.getConflictingFederatedIdsGroupedByField)(data, federatedIdField, groupByField);
+        expect(result).toEqual([]);
+    });
+    it('should return an empty array if federatedIdField is null', () => {
+        const data = [{ federatedId: 1, name: 'Test' }];
+        const federatedIdField = null;
+        const groupByField = 'name';
+        const result = (0, helpers_1.getConflictingFederatedIdsGroupedByField)(data, federatedIdField, groupByField);
+        expect(result).toEqual([]);
+    });
+    it('should return an empty array if groupByField is null', () => {
+        const data = [{ federatedId: 1, name: 'Test' }];
+        const federatedIdField = 'fedId';
+        const groupByField = null;
+        const result = (0, helpers_1.getConflictingFederatedIdsGroupedByField)(data, federatedIdField, groupByField);
+        expect(result).toEqual([]);
+    });
+    it('should return an empty array if data is empty', () => {
+        const data = [];
+        const federatedIdField = 'federatedId';
+        const groupByField = 'name';
+        const result = (0, helpers_1.getConflictingFederatedIdsGroupedByField)(data, federatedIdField, groupByField);
+        expect(result).toEqual([]);
+    });
+    it('should return all the conflicting federated IDs when grouped by the same field', () => {
+        const data = [
+            { nameField: 'Test1', federatedId: 1 },
+            { nameField: 'Test1', federatedId: 999 },
+            { nameField: 'Test2', federatedId: 2 },
+            { nameField: 'Test3', federatedId: 3 },
+            { nameField: 'Test4', federatedId: 3 },
+        ];
+        const federatedIdField = 'federatedId';
+        const groupByField = 'nameField';
+        const result = (0, helpers_1.getConflictingFederatedIdsGroupedByField)(data, federatedIdField, groupByField);
+        expect(result).toEqual([1, 999]);
+    });
+    it('should return an array of conflicting fields when grouped by federated IDs', () => {
+        const data = [
+            { federatedId: 1, nameField: 'Test1' },
+            { federatedId: 2, nameField: 'Test2' },
+            { federatedId: 1, nameField: 'Test3' },
+            { federatedId: 3, nameField: 'Test4' },
+            { federatedId: 2, nameField: 'Test5' },
+        ];
+        const federatedIdField = 'federatedId';
+        const groupByField = 'nameField';
+        const result = (0, helpers_1.getConflictingFieldForFederatedId)(data, federatedIdField, groupByField);
+        expect(result).toEqual([
+            { federatedId: '1', values: ['Test1', 'Test3'] },
+            { federatedId: '2', values: ['Test2', 'Test5'] },
+        ]);
+    });
+});

package/dist/helpers/load/validators.d.ts

@@ -0,0 +1,14 @@
+export declare class LoadValidators {
+    static ValidateColorsLoadFile(colorsData: any, config: any): {
+        errors: string[];
+        warnings: string[];
+    };
+    static ValidateCustomEntityLoadFile(customEntityData: any, config: any): {
+        errors: string[];
+        warnings: string[];
+    };
+    static ValidateLineSheetLoadFile(lineSheetData: any[], config: any): {
+        errors: string[];
+        warnings: string[];
+    };
+}

package/dist/helpers/load/validators.js

@@ -0,0 +1,182 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LoadValidators = void 0;
+const helpers_1 = require("./helpers");
+class LoadValidators {
+    static ValidateColorsLoadFile(colorsData, config) {
+        const csvHeaders = Object.keys(colorsData[0]);
+        const errors = [];
+        const warnings = [];
+        if (!config) {
+            errors.push('Config file is empty');
+            return { errors, warnings };
+        }
+        if (!config?.loadType?.length) {
+            errors.push('"loadType" is required in the config file');
+        }
+        if (config?.loadType?.length && config?.loadType[0] !== 'COLOR') {
+            errors.push('"loadType" must be must be single value "COLOR" in the config file');
+        }
+        if (!config?.federatedMappings?.federatedId) {
+            errors.push('"federatedId" is required in the config file');
+        }
+        if (!config?.federatedMappings?.name) {
+            errors.push('"name" is required in the config file');
+        }
+        if (config?.conditionalColumns?.length &&
+            config?.conditionalColumns[0].toProperty === 'typePath' &&
+            config?.conditionalColumns[0].default !== 'color') {
+            errors.push('"typePath" must be "color" in the config file');
+        }
+        else if (csvHeaders.includes('typePath')) {
+            const hasCorrectTypePath = colorsData.every((row) => row.typePath === 'color');
+            if (!hasCorrectTypePath) {
+                errors.push('"typePath" must be "color" in each row in the csv file');
+            }
+        }
+        if (!csvHeaders.includes(config?.federatedMappings?.federatedId)) {
+            errors.push(`Federated ID column (${config?.federatedMappings?.federatedId}) not found in the CSV file`);
+        }
+        if (!csvHeaders.includes(config?.federatedMappings?.name)) {
+            errors.push(`Name column (${config.federatedMappings?.name}) not found in the CSV file`);
+        }
+        const duplicateFederatedIdErrors = (0, helpers_1.getDuplicateFederatedIds)(colorsData, config?.federatedMappings?.federatedId);
+        if (duplicateFederatedIdErrors.length) {
+            errors.push(`Duplicate FederatedIds found: ${duplicateFederatedIdErrors.join(', ')}`);
+        }
+        const redColumn = config?.federatedMappings?.red || 'red';
+        const blueColumn = config?.federatedMappings?.blue || 'blue';
+        const greenColumn = config?.federatedMappings?.green || 'green';
+        const hexColumn = config?.federatedMappings?.hex || 'hex';
+        const hasRedColumn = csvHeaders.includes(redColumn);
+        const hasBlueColumn = csvHeaders.includes(blueColumn);
+        const hasGreenColumn = csvHeaders.includes(greenColumn);
+        const hasHexColumn = csvHeaders.includes(hexColumn);
+        if (hasHexColumn && hasRedColumn && hasGreenColumn && hasBlueColumn) {
+            warnings.push('"Hex" is ignored when red, green, and blue columns are present');
+        }
+        return { errors, warnings };
+    }
+    static ValidateCustomEntityLoadFile(customEntityData, config) {
+        const csvHeaders = Object.keys(customEntityData[0]);
+        const errors = [];
+        const warnings = [];
+        if (!config) {
+            errors.push('Config file is empty');
+            return { errors, warnings };
+        }
+        if (!config?.loadType?.length) {
+            errors.push('"loadType" is required in the config file');
+        }
+        if (config?.loadType?.length && config?.loadType[0] !== 'CUSTOM_ENTITY') {
+            errors.push('"loadType" must be must be single value "CUSTOM_ENTITY" in the config file');
+        }
+        if (!config?.federatedMappings?.federatedId) {
+            errors.push('"federatedId" is required in the config file');
+        }
+        if (!config?.federatedMappings?.name) {
+            errors.push('"name" is required in the config file');
+        }
+        if (!csvHeaders.includes(config?.federatedMappings?.federatedId)) {
+            errors.push(`Federated ID column (${config?.federatedMappings?.federatedId}) not found in the CSV file`);
+        }
+        if (!csvHeaders.includes(config?.federatedMappings?.name)) {
+            errors.push('Name column not found in the CSV file');
+        }
+        const duplicateFederatedIdErrors = (0, helpers_1.getDuplicateFederatedIds)(customEntityData, config?.federatedMappings?.federatedId);
+        if (duplicateFederatedIdErrors.length) {
+            errors.push(`Duplicate FederatedIds found: ${duplicateFederatedIdErrors.join(', ')}`);
+        }
+        if (Array.isArray(config?.conditionalColumns) && config.conditionalColumns.length) {
+            const hasRequiredConditionalColumn = config.conditionalColumns.some((column) => column.toProperty === 'typePath' &&
+                column.default?.startsWith('custom-entity:') &&
+                column.default?.split(':')[1]?.length > 0);
+            if (!hasRequiredConditionalColumn) {
+                errors.push('"typePath" must be "custom-entity:<subtypepath>" in the config file');
+            }
+        }
+        else if (csvHeaders.includes('typePath')) {
+            const hasCorrectTypePath = customEntityData.every((row) => row.typePath?.startsWith('custom-entity:') && row.typePath?.split(':')[1]?.length > 0);
+            if (!hasCorrectTypePath) {
+                errors.push('"typePath" must be "custom-entity:<subtypepath>" in each row in the csv file');
+            }
+        }
+        else {
+            errors.push('Missing "typePath" column in the csv file or config file');
+        }
+        return { errors, warnings };
+    }
+    static ValidateLineSheetLoadFile(lineSheetData, config) {
+        const csvHeaders = Object.keys(lineSheetData[0]);
+        const errors = [];
+        const warnings = [];
+        if (!config) {
+            errors.push('Config file is empty');
+            return { errors, warnings };
+        }
+        if (!config?.loadType?.length) {
+            errors.push('"loadType" is required in the config file');
+        }
+        if (config?.loadType?.length &&
+            !config.loadType.every((type) => ['ITEM', 'PROJECT_ITEM', 'ASSORTMENT'].includes(type))) {
+            errors.push('"loadType" must be either of values ["ITEM", "PROJECT_ITEM", "ASSORTMENT"] in the config file');
+        }
+        if (!config.federatedMappings?.name) {
+            errors.push('"name" is required in the config file and must be mapped to Item Family Name');
+        }
+        if (!config.federatedMappings?.optionName) {
+            warnings.push('"optionName" is not mapped in the config file');
+        }
+        if (!config.federatedMappings?.itemFamilyFederatedId &&
+            config.federatedMappings?.itemFederatedId !== null) {
+            errors.push('"itemFamilyFederatedId" is required in the config file');
+        }
+        if (!config.federatedMappings?.itemOptionFederatedId) {
+            warnings.push('"itemOptionFederatedId" is not mapped in the config file');
+        }
+        if (!csvHeaders.includes(config.federatedMappings?.itemFamilyFederatedId)) {
+            errors.push(`itemFamilyFederatedId column (${config.federatedMappings?.itemFamilyFederatedId}) not found in the CSV file`);
+        }
+        const emptyItemFamilyFedIdRows = [];
+        for (const [index, row] of lineSheetData.entries()) {
+            const familyFedId = row[config.federatedMappings?.itemFamilyFederatedId];
+            if (!familyFedId || familyFedId === '') {
+                emptyItemFamilyFedIdRows.push(index);
+            }
+        }
+        if (emptyItemFamilyFedIdRows.length) {
+            errors.push(`some rows contain itemFamilyFederatedId is null or undefined or empty in the CSV file (rows ${emptyItemFamilyFedIdRows.join(',')})`);
+        }
+        // if the data file contains no names, then every row has to be unique federated ID
+        // otherwise, we need to check that each federated ID is only included with one name
+        const isFamilyNameMapped = Boolean(config?.federatedMappings?.name);
+        if (isFamilyNameMapped) {
+            const conflictingFamilyFedIds = (0, helpers_1.getConflictingFederatedIdsGroupedByField)(lineSheetData, config?.federatedMappings?.itemFamilyFederatedId, config?.federatedMappings?.name);
+            if (conflictingFamilyFedIds.length) {
+                errors.push(`Found conflicting family federated Ids: ${conflictingFamilyFedIds.join(', ')}`);
+            }
+            const conflictingFieldsByFedId = (0, helpers_1.getConflictingFieldForFederatedId)(lineSheetData, config?.federatedMappings?.itemFamilyFederatedId, config?.federatedMappings?.name);
+            if (conflictingFieldsByFedId.length) {
+                let prettyString = '';
+                for (const fedIdConflict of conflictingFieldsByFedId) {
+                    prettyString += `(${fedIdConflict.federatedId}: (${fedIdConflict.values.join(', ')}))`;
+                }
+                errors.push(`Found conflicting fields for family federated Ids: ${prettyString}`);
+            }
+        }
+        else {
+            const duplicateFamilyFedIds = (0, helpers_1.getDuplicateFederatedIds)(lineSheetData, config?.federatedMappings?.itemFamilyFederatedId);
+            if (duplicateFamilyFedIds.length) {
+                errors.push(`Duplicate itemFamilyFederatedId found: ${duplicateFamilyFedIds.join(', ')}`);
+            }
+        }
+        if (config?.federatedMappings?.itemOptionFederatedId) {
+            const duplicateOptionFedIds = (0, helpers_1.getDuplicateFederatedIds)(lineSheetData, config.federatedMappings.itemOptionFederatedId);
+            if (duplicateOptionFedIds.length) {
+                errors.push(`Duplicate itemOptionFederatedId found: ${duplicateOptionFedIds.join(', ')}`);
+            }
+        }
+        return { errors, warnings };
+    }
+}
+exports.LoadValidators = LoadValidators;

package/dist/helpers/load/validators.spec.d.ts

@@ -0,0 +1 @@
+export {};