@contrail/cli 3.4.5 → 3.4.6-alpha.cffb28e

package/node_modules/@contrail/loader/lib/common/csv-file-helper.js

@@ -0,0 +1,66 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InvalidCSVError = exports.parseCSVData = exports.getCSVFromFile = void 0;
+const sync_1 = require("csv-parse/sync");
+async function getCSVFromFile(loadFileId, client) {
+    const file = await client.get({ entityName: 'file', id: loadFileId }).catch((e) => {
+        console.error(e);
+        throw new Error(`Failed to fetch file with id ${loadFileId}: ${e.message}`);
+    });
+    if (!file) {
+        const errorMessage = `File: ${loadFileId} is not available`;
+        console.error(errorMessage);
+        throw new Error(errorMessage);
+    }
+    if (!file.downloadUrl) {
+        const errorMessage = `File: ${loadFileId} does not have a downloadUrl`;
+        console.error(errorMessage);
+        throw new Error(errorMessage);
+    }
+    let response;
+    try {
+        const res = await fetch(file.downloadUrl);
+        if (!res.ok) {
+            throw new Error(`Failed to fetch data from downloadUrl: ${file.downloadUrl}`);
+        }
+        response = await res.text();
+    }
+    catch (e) {
+        const errorMessage = `Failed to get data from downloadUrl: ${file.downloadUrl}`;
+        console.error(errorMessage, JSON.stringify(e));
+        throw new Error(`${errorMessage}: ${e.message}`);
+    }
+    if (!response) {
+        const errorMessage = 'Response is empty';
+        console.error(errorMessage);
+        throw new Error(errorMessage);
+    }
+    const csvJson = parseCSVData(response);
+    return csvJson;
+}
+exports.getCSVFromFile = getCSVFromFile;
+function parseCSVData(data) {
+    try {
+        return (0, sync_1.parse)(data, {
+            columns: true,
+            skip_empty_lines: true,
+            relax_quotes: true,
+            bom: true,
+            trim: true,
+        });
+    }
+    catch (e) {
+        if (e instanceof sync_1.CsvError) {
+            throw new InvalidCSVError(`Failed to parse csv data. ${e.message}`);
+        }
+        throw e;
+    }
+}
+exports.parseCSVData = parseCSVData;
+class InvalidCSVError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'InvalidCSVError';
+    }
+}
+exports.InvalidCSVError = InvalidCSVError;

package/node_modules/@contrail/loader/lib/common/csv-file-helper.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/node_modules/@contrail/loader/lib/common/csv-file-helper.spec.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const csv_file_helper_1 = require("./csv-file-helper");
+describe('CSV-Helper', () => {
+    describe('parseCSVData', () => {
+        const csvData = `id,btpString,bool
+fakeId,btp,True
+fakeId2,btp2,FALSE`;
+        const expectedParsedData = [
+            {
+                id: 'fakeId',
+                btpString: 'btp',
+                bool: 'True',
+            },
+            {
+                id: 'fakeId2',
+                btpString: 'btp2',
+                bool: 'FALSE',
+            },
+        ];
+        it('should parse CSV data correctly', () => {
+            const parsedData = (0, csv_file_helper_1.parseCSVData)(csvData);
+            expect(parsedData).toEqual(expectedParsedData);
+        });
+        it('should handle BOM marker in the beginning of the file', () => {
+            const csvDataWithoutBom = csvData;
+            const bom = '\uFEFF';
+            const csvDataWithBom = bom + csvData;
+            const parsedData = (0, csv_file_helper_1.parseCSVData)(csvDataWithBom);
+            expect(parsedData).toEqual(expectedParsedData);
+            expect((0, csv_file_helper_1.parseCSVData)(csvDataWithBom)).toEqual((0, csv_file_helper_1.parseCSVData)(csvDataWithoutBom));
+        });
+    });
+});

package/node_modules/@contrail/loader/lib/common/data-transformer-functions.d.ts

@@ -0,0 +1,32 @@
+import { Type, TypeProperty } from '@contrail/types';
+import { PreprocessingFunction } from '../file-preprocessor';
+import { EntitiesClient, ProcessingIssues } from '@contrail/entity-types';
+export declare type TypePropertyWithLookups = TypeProperty & {
+    optionLookupByValue?: Record<string, any>;
+    optionLookupByDisplay?: Record<string, any>;
+};
+export declare type TypePropertyMaps = {
+    typePropertiesBySlug: Record<string, TypeProperty>;
+    typePropertiesByLabel: Record<string, TypeProperty>;
+    optionSetUsingTypePropertiesBySlug: Record<string, TypePropertyWithLookups>;
+};
+export declare const applyFederatedMappings: PreprocessingFunction;
+export declare const removeUnwantedProperties: PreprocessingFunction;
+export declare const trimValues: PreprocessingFunction;
+export declare const transformDataBasedOnTypes: PreprocessingFunction;
+export declare const transformDataBasedOnTypesWithoutWarnings: PreprocessingFunction;
+export declare function getReducedTypes(typeRoots: string[], client: EntitiesClient): Promise<any[]>;
+export declare const applyConditionalColumns: PreprocessingFunction;
+export declare function getPropertiesBySlugAndLabel(reducedTypes: Partial<Type>[]): {
+    typePropertiesBySlug: Record<string, TypeProperty>;
+    typePropertiesByLabel: Record<string, TypeProperty>;
+};
+export declare function getOptionSetsUsingTypePropertiesBySlug(reducedTypes: Partial<Type>[]): Record<string, TypePropertyWithLookups>;
+export declare function convertFromLabelToSlug(csvJSON: any, typePropertiesBySlug: any, typePropertiesByLabel: any, convertedFields: any): ProcessingIssues[];
+export declare function convertFromDisplayToValue(csvJSON: any[], optionSetUsingTypePropertiesBySlug: any, typePropertySlugs: string[], typePropertyLabels: string[]): ProcessingIssues[];
+export declare function identifyKeysInProperties(csvJSON: any[], typePropertySlugs: string[], typePropertyLabels: string[]): {
+    keysInPropertySlugs: Map<string, boolean>;
+    keysInPropertyLabels: Map<string, boolean>;
+};
+export declare const convertToBoolean: PreprocessingFunction;
+export declare function removeOptionOrFamilySuffix(propertyName: string): string;

package/node_modules/@contrail/loader/lib/common/data-transformer-functions.js

@@ -0,0 +1,309 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.removeOptionOrFamilySuffix = exports.convertToBoolean = exports.identifyKeysInProperties = exports.convertFromDisplayToValue = exports.convertFromLabelToSlug = exports.getOptionSetsUsingTypePropertiesBySlug = exports.getPropertiesBySlugAndLabel = exports.applyConditionalColumns = exports.getReducedTypes = exports.transformDataBasedOnTypesWithoutWarnings = exports.transformDataBasedOnTypes = exports.trimValues = exports.removeUnwantedProperties = exports.applyFederatedMappings = void 0;
+const conditional_columns_1 = require("./conditional-columns");
+const entities_helper_1 = require("./entities-helper");
+const entity_types_1 = require("@contrail/entity-types");
+const pLimit = require('p-limit');
+const limit = pLimit(20);
+const applyFederatedMappings = function (csvJSON, config) {
+    const federatedMappings = config.federatedMappings || {};
+    const copy = createCopy(csvJSON);
+    const warnings = [];
+    if (copy.length > 0) {
+        const csvHeaders = Object.keys(copy[0]);
+        for (const [targetKey, sourceColumn] of Object.entries(federatedMappings)) {
+            if (sourceColumn && !csvHeaders.includes(sourceColumn)) {
+                warnings.push({
+                    rowNumber: 1,
+                    columnIssues: [{
+                            columnName: targetKey,
+                            message: `Federated mapping "${targetKey}" references source column "${sourceColumn}" which does not exist in the CSV headers. Check that the mapping value matches a CSV column name.`,
+                            issueType: entity_types_1.IssueType.CONFIG,
+                            data: '',
+                        }],
+                });
+            }
+        }
+    }
+    for (const row of copy) {
+        for (const [key, value] of Object.entries(federatedMappings)) {
+            if (!row[key]) {
+                row[key] = row[value];
+            }
+        }
+    }
+    return { rows: copy, errors: [], warnings, generalErrors: [] };
+};
+exports.applyFederatedMappings = applyFederatedMappings;
+const removeUnwantedProperties = function (csvJSON, config) {
+    const propertiesToRemove = config.propertiesToRemove || [];
+    const rows = createCopy(csvJSON);
+    if (propertiesToRemove?.length > 0) {
+        for (const row of rows) {
+            for (const prop of propertiesToRemove) {
+                delete row[prop];
+            }
+        }
+    }
+    return { rows, errors: [], warnings: [], generalErrors: [] };
+};
+exports.removeUnwantedProperties = removeUnwantedProperties;
+const trimValues = function (csvJSON) {
+    const rows = createCopy(csvJSON);
+    for (const row of rows) {
+        for (const key of Object.keys(row)) {
+            if (typeof row[key] === 'string') {
+                row[key] = row[key].trim();
+            }
+        }
+    }
+    return { rows, errors: [], warnings: [], generalErrors: [] };
+};
+exports.trimValues = trimValues;
+const transformDataBasedOnTypes = function (csvJSON, config, typePropertyMaps, addWarnings = true) {
+    const rows = createCopy(csvJSON);
+    const { typePropertiesBySlug, typePropertiesByLabel, optionSetUsingTypePropertiesBySlug } = typePropertyMaps;
+    const labelWarnings = convertFromLabelToSlug(rows, typePropertiesBySlug, typePropertiesByLabel, getConvertedFields(config.federatedMappings));
+    const displayValueWarnings = convertFromDisplayToValue(rows, optionSetUsingTypePropertiesBySlug, Object.keys(typePropertiesBySlug), Object.keys(typePropertiesByLabel));
+    if (!addWarnings) {
+        return { rows, errors: [], warnings: [], generalErrors: [] };
+    }
+    const warnings = [...labelWarnings, ...displayValueWarnings];
+    return { rows, errors: [], warnings, generalErrors: [] };
+};
+exports.transformDataBasedOnTypes = transformDataBasedOnTypes;
+const transformDataBasedOnTypesWithoutWarnings = function (csvJSON, config, typePropertyMaps) {
+    return (0, exports.transformDataBasedOnTypes)(csvJSON, config, typePropertyMaps, false);
+};
+exports.transformDataBasedOnTypesWithoutWarnings = transformDataBasedOnTypesWithoutWarnings;
+async function getReducedTypes(typeRoots, client) {
+    const allTypes = await (0, entities_helper_1.retryOnTimeout)(client, entities_helper_1.EntitiesLambdaCallType.GET, {
+        entityName: 'type',
+    });
+    const allSubtypesForRootTypes = allTypes.filter((type) => typeRoots.includes(type.typePath.split(':')[0]));
+    const reducedTypes = await Promise.all(allSubtypesForRootTypes.map((type) => limit(() => (0, entities_helper_1.retryOnTimeout)(client, entities_helper_1.EntitiesLambdaCallType.GET, {
+        entityName: 'type',
+        id: type.id,
+        relations: ['typeProperties'],
+        customParams: { includeArchivedTypeProperties: true },
+    }))));
+    return reducedTypes;
+}
+exports.getReducedTypes = getReducedTypes;
+const applyConditionalColumns = function (csvJSON, config) {
+    const rows = createCopy(csvJSON);
+    conditional_columns_1.ColumnDefinitionEvaluator.evaluateColumnDefinitionsOnBatch(config?.conditionalColumns, rows);
+    return { rows, errors: [], warnings: [], generalErrors: [] };
+};
+exports.applyConditionalColumns = applyConditionalColumns;
+function getPropertiesBySlugAndLabel(reducedTypes) {
+    const typePropertiesBySlug = {};
+    const typePropertiesByLabel = {};
+    for (const type of reducedTypes) {
+        if (type.typeProperties) {
+            for (const typeProperty of type.typeProperties) {
+                typePropertiesBySlug[typeProperty.slug.trim()] = typeProperty;
+                typePropertiesByLabel[typeProperty.label.trim()] = typeProperty;
+            }
+        }
+    }
+    return { typePropertiesBySlug, typePropertiesByLabel };
+}
+exports.getPropertiesBySlugAndLabel = getPropertiesBySlugAndLabel;
+function getOptionSetsUsingTypePropertiesBySlug(reducedTypes) {
+    const optionSetUsingTypePropertiesBySlug = {};
+    for (const type of reducedTypes) {
+        if (type.typeProperties) {
+            for (const typeProperty of type.typeProperties) {
+                if (['multi_select', 'choice'].includes(typeProperty.propertyType)) {
+                    optionSetUsingTypePropertiesBySlug[typeProperty.slug] = typeProperty;
+                    const optionLookupByValue = {};
+                    const optionLookupByDisplay = {};
+                    if (typeProperty.options) {
+                        for (const option of typeProperty.options) {
+                            optionLookupByValue[option.value] = option;
+                            optionLookupByDisplay[option.display] = option;
+                        }
+                    }
+                    const typePropertyWithLookups = { ...typeProperty };
+                    typePropertyWithLookups.optionLookupByValue = optionLookupByValue;
+                    typePropertyWithLookups.optionLookupByDisplay = optionLookupByDisplay;
+                    optionSetUsingTypePropertiesBySlug[typeProperty.slug] = typePropertyWithLookups;
+                }
+            }
+        }
+    }
+    return optionSetUsingTypePropertiesBySlug;
+}
+exports.getOptionSetsUsingTypePropertiesBySlug = getOptionSetsUsingTypePropertiesBySlug;
+function getConvertedFields(federatedMappings) {
+    const convertedFields = {};
+    if (federatedMappings && Object.values(federatedMappings).length > 0) {
+        for (const value of Object.values(federatedMappings)) {
+            convertedFields[value] = true;
+        }
+    }
+    return convertedFields;
+}
+function convertFromLabelToSlug(csvJSON, typePropertiesBySlug, typePropertiesByLabel, convertedFields) {
+    const keysToBeIgnored = ['federatedId', 'typePath', 'itemFamilyFederatedId', 'itemOptionFederatedId', 'optionGroup'];
+    const keysWithoutProperties = new Set();
+    const archivedKeys = new Set();
+    const warnings = [];
+    for (const row of csvJSON) {
+        for (const key of Object.keys(row)) {
+            const slugProperty = typePropertiesBySlug[key];
+            const labelProperty = typePropertiesByLabel[key];
+            if (slugProperty?.isArchived || labelProperty?.isArchived) {
+                archivedKeys.add(key);
+                continue;
+            }
+            if (slugProperty || keysToBeIgnored.includes(key)) {
+                continue;
+            }
+            if (labelProperty) {
+                if (!row[labelProperty.slug]) {
+                    Object.defineProperty(row, labelProperty.slug, Object.getOwnPropertyDescriptor(row, key));
+                }
+            }
+            else {
+                if (!slugProperty) {
+                    const formattedProperty = removeOptionOrFamilySuffix(key);
+                    const resolvedSlugProperty = typePropertiesBySlug[formattedProperty];
+                    if (!resolvedSlugProperty) {
+                        keysWithoutProperties.add(key);
+                    }
+                }
+                else if (!convertedFields[key]) {
+                    keysWithoutProperties.add(key);
+                }
+            }
+        }
+    }
+    if (archivedKeys.size > 0) {
+        warnings.push({
+            rowNumber: 1,
+            columnIssues: Array.from(archivedKeys).map((key) => ({
+                columnName: key,
+                message: `Property ${key} is disabled. This property's data will not be processed.`,
+                issueType: entity_types_1.IssueType.HEADER_MISMATCH,
+                data: '',
+            })),
+        });
+    }
+    if (keysWithoutProperties.size > 0) {
+        warnings.push({
+            rowNumber: 1,
+            columnIssues: Array.from(keysWithoutProperties).map((key) => ({
+                columnName: key,
+                message: `Type property ${key} does not have any mapping via CSV column or Config. This property will not be loaded into.`,
+                issueType: entity_types_1.IssueType.HEADER_MISMATCH,
+                data: '',
+            })),
+        });
+    }
+    return warnings;
+}
+exports.convertFromLabelToSlug = convertFromLabelToSlug;
+function convertFromDisplayToValue(csvJSON, optionSetUsingTypePropertiesBySlug, typePropertySlugs, typePropertyLabels) {
+    const warnings = [];
+    const { keysInPropertySlugs, keysInPropertyLabels } = identifyKeysInProperties(csvJSON, typePropertySlugs, typePropertyLabels);
+    csvJSON.forEach((row, index) => {
+        const columnIssues = [];
+        for (const [key, value] of Object.entries(row)) {
+            const isKeyAnExactPropertyMatch = keysInPropertySlugs.get(key) || keysInPropertyLabels.get(key);
+            const propertyName = isKeyAnExactPropertyMatch ? key : removeOptionOrFamilySuffix(key);
+            const property = optionSetUsingTypePropertiesBySlug[propertyName];
+            if (!property || property?.optionLookupByValue[value]) {
+                continue;
+            }
+            if (property.propertyType === 'choice') {
+                if (property?.optionLookupByDisplay[value]) {
+                    row[key] = property.optionLookupByDisplay[value].value;
+                }
+                else {
+                    if (value) {
+                        columnIssues.push({
+                            columnName: key,
+                            message: `Property: ${key} has unknown optionSet value: ${value}.`,
+                            issueType: entity_types_1.IssueType.VALIDATION_ERROR,
+                            data: JSON.stringify(row),
+                        });
+                    }
+                }
+            }
+            else if (property.propertyType === 'multi_select') {
+                if (value === null || value === undefined) {
+                    continue;
+                }
+                const values = value?.split(',');
+                for (let i = 0; i < values?.length; i++) {
+                    const innerValue = values[i].trim();
+                    if (!property || property?.optionLookupByValue[innerValue]) {
+                        continue;
+                    }
+                    if (property?.optionLookupByDisplay[innerValue]) {
+                        values[i] = property.optionLookupByDisplay[innerValue].value;
+                    }
+                    else {
+                        if (innerValue) {
+                            columnIssues.push({
+                                columnName: key,
+                                message: `Property: ${key} has unknown optionSet value: ${value}.`,
+                                issueType: entity_types_1.IssueType.VALIDATION_ERROR,
+                                data: JSON.stringify(row),
+                            });
+                        }
+                    }
+                }
+                row[key] = values.join(',');
+            }
+        }
+        if (columnIssues.length > 0) {
+            warnings.push({
+                rowNumber: index + 2,
+                columnIssues,
+            });
+        }
+    });
+    return warnings;
+}
+exports.convertFromDisplayToValue = convertFromDisplayToValue;
+function identifyKeysInProperties(csvJSON, typePropertySlugs, typePropertyLabels) {
+    const keysInPropertySlugs = new Map();
+    const keysInPropertyLabels = new Map();
+    if (csvJSON.length === 0)
+        return { keysInPropertySlugs, keysInPropertyLabels };
+    for (const key of Object.keys(csvJSON[0])) {
+        keysInPropertySlugs.set(key, typePropertySlugs.includes(key));
+        keysInPropertyLabels.set(key, typePropertyLabels.includes(key));
+    }
+    return { keysInPropertySlugs, keysInPropertyLabels };
+}
+exports.identifyKeysInProperties = identifyKeysInProperties;
+const convertToBoolean = function (csvJson) {
+    const rows = createCopy(csvJson);
+    for (const row of rows) {
+        for (const [key, value] of Object.entries(row)) {
+            if (typeof value === 'string' && value.toLocaleLowerCase() === 'false') {
+                row[key] = false;
+            }
+            else if (typeof value === 'string' && value.toLocaleLowerCase() === 'true') {
+                row[key] = true;
+            }
+        }
+    }
+    return { rows, errors: [], warnings: [], generalErrors: [] };
+};
+exports.convertToBoolean = convertToBoolean;
+function removeOptionOrFamilySuffix(propertyName) {
+    if (!propertyName.endsWith('Family') && !propertyName.endsWith('Option')) {
+        return propertyName;
+    }
+    return propertyName.replace(/(Family|Option)$/, '');
+}
+exports.removeOptionOrFamilySuffix = removeOptionOrFamilySuffix;
+function createCopy(csvJSON) {
+    return structuredClone(csvJSON);
+}

package/node_modules/@contrail/loader/lib/common/data-transformer.spec.d.ts

@@ -0,0 +1 @@
+export {};