@contrail/cli 3.4.5 → 3.4.6-alpha.cffb28e

package/node_modules/@contrail/loader/lib/file-preprocessor.d.ts

@@ -0,0 +1,31 @@
+import { LoaderProcess, EntitiesClient, GeneralIssueDetails, ProcessingIssues, EntityType, ProcessedData } from '@contrail/entity-types';
+import { TypePropertyMaps } from './common/data-transformer-functions';
+export interface PreProcessingStep {
+    name: string;
+    execute: PreprocessingFunction;
+}
+export interface StepResultOutput {
+    rows: any[];
+    errors: ProcessingIssues[];
+    warnings: ProcessingIssues[];
+    generalErrors?: GeneralIssueDetails[];
+    generalWarnings?: GeneralIssueDetails[];
+}
+export interface PreprocessingFunction {
+    (rows: any[], config?: {
+        federatedMappings?: any;
+        conditionalColumns?: any;
+        propertiesToRemove?: string[];
+    }, typePropertyMaps?: TypePropertyMaps, addWarnings?: boolean): StepResultOutput;
+}
+export declare class LoaderDataPreprocessor {
+    static getPreprocessingPipeline(): PreProcessingStep[];
+    static transformAndValidateData(client: EntitiesClient, loaderProcess: LoaderProcess, entityType: EntityType): Promise<ProcessedData>;
+    private static loadCSVDataWithErrorHandling;
+    private static validateData;
+    private static runPreprocessingPipeline;
+    private static loadCSVData;
+    private static logIssues;
+    private static updateLoaderProcess;
+    static uploadStepResult(client: EntitiesClient, loaderProcessId: string, stepName: string, stepResult: StepResultOutput): Promise<string>;
+}

package/node_modules/@contrail/loader/lib/file-preprocessor.js

@@ -0,0 +1,212 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LoaderDataPreprocessor = void 0;
+const entity_types_1 = require("@contrail/entity-types");
+const csv_file_helper_1 = require("./common/csv-file-helper");
+const data_transformer_functions_1 = require("./common/data-transformer-functions");
+const file_helper_1 = require("./common/file-helper");
+const processor_registry_1 = require("./processor-registry");
+class LoaderDataPreprocessor {
+    static getPreprocessingPipeline() {
+        return [
+            { name: 'Convert CSV Boolean Values', execute: data_transformer_functions_1.convertToBoolean },
+            {
+                name: 'Apply Federated Mapping',
+                execute: data_transformer_functions_1.applyFederatedMappings,
+            },
+            {
+                name: 'Remove Unwanted Properties',
+                execute: data_transformer_functions_1.removeUnwantedProperties,
+            },
+            {
+                name: 'Transform Data Based on Types',
+                execute: data_transformer_functions_1.transformDataBasedOnTypesWithoutWarnings,
+            },
+            {
+                name: 'Evaluate Conditional Columns',
+                execute: data_transformer_functions_1.applyConditionalColumns,
+            },
+            {
+                name: 'Transform Data Based on Types',
+                execute: data_transformer_functions_1.transformDataBasedOnTypes,
+            },
+        ];
+    }
+    static async transformAndValidateData(client, loaderProcess, entityType) {
+        let failureArea = entity_types_1.LoaderFailureArea.NONE;
+        const transformationErrors = [];
+        const transformationWarnings = [];
+        const generalErrors = [];
+        const generalWarnings = [];
+        const systemErrors = [];
+        const preprocessingStepResults = [];
+        let processedData = [];
+        try {
+            processedData = await this.loadCSVDataWithErrorHandling(client, loaderProcess, generalErrors);
+            if (generalErrors.length) {
+                failureArea = entity_types_1.LoaderFailureArea.PARSING_CSV;
+                return {
+                    csvRows: processedData,
+                    transformationErrors,
+                    transformationWarnings,
+                    generalErrors,
+                    generalWarnings,
+                    systemErrors,
+                    failureArea,
+                };
+            }
+            const processor = (0, processor_registry_1.getProcessorForEntityType)(entityType);
+            const validationIssues = this.validateData(processor, processedData, loaderProcess, transformationErrors, transformationWarnings, generalErrors, generalWarnings);
+            if (validationIssues.hasErrors) {
+                failureArea = entity_types_1.LoaderFailureArea.VALIDATING_DATA;
+                return {
+                    csvRows: processedData,
+                    transformationErrors,
+                    transformationWarnings,
+                    generalErrors,
+                    generalWarnings,
+                    systemErrors,
+                    failureArea,
+                };
+            }
+            const reducedTypes = await (0, data_transformer_functions_1.getReducedTypes)(processor.typesToTransformBy || [], client);
+            processedData = await this.runPreprocessingPipeline(client, loaderProcess, reducedTypes, processedData, preprocessingStepResults, transformationErrors, transformationWarnings, systemErrors);
+            await this.updateLoaderProcess(client, loaderProcess.id, preprocessingStepResults, systemErrors);
+            if (failureArea === entity_types_1.LoaderFailureArea.NONE &&
+                (transformationErrors.length || systemErrors.length || generalErrors.length)) {
+                failureArea = entity_types_1.LoaderFailureArea.TRANSFORMING_DATA;
+            }
+            return {
+                csvRows: processedData,
+                transformationErrors,
+                transformationWarnings,
+                generalErrors,
+                generalWarnings,
+                systemErrors,
+                failureArea,
+            };
+        }
+        catch (e) {
+            console.error('Error transforming data:', e.message);
+            failureArea = entity_types_1.LoaderFailureArea.TRANSFORMING_DATA;
+            systemErrors.push({
+                message: e.message,
+                issueType: entity_types_1.IssueType.SYSTEM_ERROR,
+            });
+            return {
+                csvRows: processedData,
+                failureArea,
+                systemErrors,
+                transformationErrors,
+                transformationWarnings,
+                generalErrors,
+                generalWarnings,
+            };
+        }
+    }
+    static async loadCSVDataWithErrorHandling(client, loaderProcess, generalErrors) {
+        try {
+            return await this.loadCSVData(client, loaderProcess);
+        }
+        catch (e) {
+            console.error('Error parsing CSV file:', e.message);
+            generalErrors.push({
+                message: e.message,
+                issueType: entity_types_1.IssueType.INVALID_DATA,
+            });
+            return [];
+        }
+    }
+    static validateData(processor, processedData, loaderProcess, transformationErrors, transformationWarnings, generalErrors, generalWarnings) {
+        const validationIssues = processor.validateLoadFile(processedData, loaderProcess);
+        generalErrors.push(...(validationIssues.generalErrors ?? []));
+        generalWarnings.push(...(validationIssues.generalWarnings ?? []));
+        transformationErrors.push(...(validationIssues.errors ?? []));
+        transformationWarnings.push(...(validationIssues.warnings ?? []));
+        return {
+            hasErrors: generalErrors.length > 0 || transformationErrors.length > 0,
+        };
+    }
+    static async runPreprocessingPipeline(client, loaderProcess, reducedTypes, processedData, preprocessingStepResults, transformationErrors, transformationWarnings, systemErrors) {
+        const { typePropertiesBySlug, typePropertiesByLabel } = (0, data_transformer_functions_1.getPropertiesBySlugAndLabel)(reducedTypes);
+        const optionSetUsingTypePropertiesBySlug = (0, data_transformer_functions_1.getOptionSetsUsingTypePropertiesBySlug)(reducedTypes);
+        const typePropertyMaps = {
+            typePropertiesBySlug,
+            typePropertiesByLabel,
+            optionSetUsingTypePropertiesBySlug,
+        };
+        for (const preprocessingStep of this.getPreprocessingPipeline()) {
+            try {
+                console.info(`Preprocessing step: ${preprocessingStep.name} started`);
+                const stepResult = preprocessingStep.execute(processedData, loaderProcess, typePropertyMaps);
+                processedData = stepResult.rows;
+                const fileId = await this.uploadStepResult(client, loaderProcess.id, preprocessingStep.name, stepResult);
+                preprocessingStepResults.push({
+                    preprocessingStepName: preprocessingStep.name,
+                    fileId,
+                    numberOfErrors: stepResult.errors?.length || 0,
+                    numberOfWarnings: stepResult.warnings?.length || 0,
+                });
+                transformationErrors.push(...(stepResult.errors ?? []));
+                transformationWarnings.push(...(stepResult.warnings ?? []));
+                this.logIssues(preprocessingStep.name, stepResult.errors, stepResult.warnings);
+                console.info(`Preprocessing step ${preprocessingStep.name} completed`);
+            }
+            catch (e) {
+                const errorMessage = `Error in preprocessing step ${preprocessingStep.name}: ${e.message}`;
+                console.error(errorMessage);
+                systemErrors.push({
+                    message: errorMessage,
+                    issueType: entity_types_1.IssueType.SYSTEM_ERROR,
+                });
+                break;
+            }
+        }
+        return processedData;
+    }
+    static async loadCSVData(client, loaderProcess) {
+        const csvJSON = await (0, csv_file_helper_1.getCSVFromFile)(loaderProcess.loadFileId, client);
+        return csvJSON;
+    }
+    static logIssues(stepName, errors, warnings) {
+        errors?.forEach((error) => console.error(`Error in preprocessing step ${stepName}`, error));
+        warnings?.forEach((warning) => console.warn(`Warning in preprocessing step ${stepName}`, warning));
+    }
+    static async updateLoaderProcess(client, loaderProcessId, preprocessingStepResults, systemErrors) {
+        try {
+            await client.update({
+                entityName: 'loader-process',
+                id: loaderProcessId,
+                object: { preprocessingStepResults },
+            });
+        }
+        catch (e) {
+            const errorMessage = `Error updating loader process with preprocessing results: ${e.message}`;
+            console.error(errorMessage);
+            systemErrors.push({
+                message: errorMessage,
+                issueType: entity_types_1.IssueType.SYSTEM_ERROR,
+            });
+        }
+    }
+    static async uploadStepResult(client, loaderProcessId, stepName, stepResult) {
+        const stringObject = JSON.stringify(stepResult);
+        const buffer = Buffer.from(stringObject);
+        const contentType = 'application/json';
+        const fileName = `loader-process-step-${stepName}.json`;
+        const fileEntity = await client.create({
+            entityName: 'file',
+            object: {
+                contentType,
+                fileName,
+                ownedByReference: `loader-process:${loaderProcessId}`,
+            },
+        });
+        await (0, file_helper_1.uploadFileAsStream)(fileEntity?.uploadPost, buffer, contentType);
+        if (!fileEntity?.id) {
+            console.warn(`Could not save process logs. loaderProcessId: ${loaderProcessId} stepName: ${stepName}`);
+        }
+        return fileEntity?.id;
+    }
+}
+exports.LoaderDataPreprocessor = LoaderDataPreprocessor;

package/node_modules/@contrail/loader/lib/file-preprocessor.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/node_modules/@contrail/loader/lib/file-preprocessor.spec.js

@@ -0,0 +1,151 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const csv_file_helpers = __importStar(require("./common/csv-file-helper"));
+const data_transformers = __importStar(require("./common/data-transformer-functions"));
+const file_helper = __importStar(require("./common/file-helper"));
+const file_preprocessor_1 = require("./file-preprocessor");
+const color_processor_1 = require("./color-processor");
+const entity_types_1 = require("@contrail/entity-types");
+describe('LoaderDataPreprocessor', () => {
+    const reducedTypes = [];
+    const mockClient = {
+        create: jest.fn().mockResolvedValue({ id: 'mock-file-id' }),
+        update: jest.fn().mockResolvedValue({}),
+        getAllReferences: jest.fn(),
+        get: jest.fn(),
+        delete: jest.fn(),
+        batchCreate: jest.fn(),
+        batchUpdate: jest.fn(),
+        batchDelete: jest.fn(),
+    };
+    const loaderProcess = {
+        id: 'mock-loader-process-id',
+        loadFileId: 'mock-load-file-id',
+        loadType: [entity_types_1.LoadType.COLOR],
+        status: entity_types_1.LoadStatus.COMPLETE,
+        federatedMappings: {
+            federatedId: 'name',
+            name: 'name',
+        },
+        conditionalColumns: [
+            {
+                toProperty: 'typePath',
+                default: 'color',
+            },
+        ],
+    };
+    const emptyFunction = (rows, config, types) => {
+        return { rows, errors: [], warnings: [] };
+    };
+    const setupMocks = (mockRows) => {
+        jest.spyOn(csv_file_helpers, 'getCSVFromFile').mockResolvedValue(mockRows);
+        jest.spyOn(file_helper, 'uploadFileAsStream').mockResolvedValue(true);
+        jest.spyOn(data_transformers, 'getReducedTypes').mockResolvedValue(reducedTypes);
+    };
+    afterEach(() => {
+        jest.resetAllMocks();
+    });
+    describe('transformAndValidateData', () => {
+        it('should handle CSV parsing errors', async () => {
+            jest.spyOn(csv_file_helpers, 'getCSVFromFile').mockRejectedValue(new Error('CSV parsing error'));
+            const result = await file_preprocessor_1.LoaderDataPreprocessor.transformAndValidateData(mockClient, loaderProcess, entity_types_1.EntityType.COLOR);
+            expect(result.failureArea).toBe(entity_types_1.LoaderFailureArea.PARSING_CSV);
+            expect(result.generalErrors.length).toBe(1);
+            expect(result.generalErrors[0].message).toBe('CSV parsing error');
+        });
+        it('should call validateLoadFile for valid CSV and Color entity type', async () => {
+            const mockRows = [{ name: 'id1' }, { name: 'id2' }];
+            setupMocks(mockRows);
+            jest.spyOn(color_processor_1.ColorProcessor, 'validateLoadFile').mockReturnValue({
+                errors: [],
+                warnings: [],
+                generalErrors: [],
+                generalWarnings: [],
+            });
+            const result = await file_preprocessor_1.LoaderDataPreprocessor.transformAndValidateData(mockClient, loaderProcess, entity_types_1.EntityType.COLOR);
+            console.log('result', JSON.stringify(result, null, 2));
+            expect(color_processor_1.ColorProcessor.validateLoadFile).toHaveBeenCalledWith(mockRows, loaderProcess);
+            expect(result.transformationErrors.length).toBe(0);
+            expect(result.generalErrors.length).toBe(0);
+            expect(result.failureArea).toBe(entity_types_1.LoaderFailureArea.NONE);
+        });
+        it('should handle generalErrors from validateLoadFile', async () => {
+            const mockRows = [{ name: 'id1' }, { name: 'id2' }];
+            setupMocks(mockRows);
+            jest.spyOn(color_processor_1.ColorProcessor, 'validateLoadFile').mockReturnValue({
+                generalErrors: [{ message: 'Validation error', issueType: entity_types_1.IssueType.CONFIG }],
+                errors: [],
+                warnings: [],
+                generalWarnings: [],
+            });
+            const result = await file_preprocessor_1.LoaderDataPreprocessor.transformAndValidateData(mockClient, loaderProcess, entity_types_1.EntityType.COLOR);
+            expect(result.failureArea).toBe(entity_types_1.LoaderFailureArea.VALIDATING_DATA);
+            expect(result.generalErrors.length).toBe(1);
+            expect(result.generalErrors[0].message).toBe('Validation error');
+        });
+        it('should run preprocessing pipeline if all validations pass', async () => {
+            const mockRows = [{ name: 'id1' }, { name: 'id2' }];
+            setupMocks(mockRows);
+            jest.spyOn(color_processor_1.ColorProcessor, 'validateLoadFile').mockReturnValue({
+                errors: [],
+                warnings: [],
+                generalErrors: [],
+                generalWarnings: [],
+            });
+            const preprocessingPipeline = [
+                { name: 'Convert CSV Boolean Values', execute: emptyFunction },
+                { name: 'Apply Federated Mapping', execute: emptyFunction },
+                { name: 'Remove Unwanted Properties', execute: emptyFunction },
+                { name: 'Transform Data Based on Types', execute: emptyFunction },
+                { name: 'Evaluate Conditional Columns', execute: emptyFunction },
+                { name: 'Transform Data Based on Types', execute: emptyFunction },
+            ];
+            jest.spyOn(file_preprocessor_1.LoaderDataPreprocessor, 'getPreprocessingPipeline').mockImplementation(() => preprocessingPipeline);
+            const result = await file_preprocessor_1.LoaderDataPreprocessor.transformAndValidateData(mockClient, loaderProcess, entity_types_1.EntityType.COLOR);
+            expect(result.transformationErrors.length).toBe(0);
+            expect(result.generalErrors.length).toBe(0);
+            expect(result.failureArea).toBe(entity_types_1.LoaderFailureArea.NONE);
+            expect(file_preprocessor_1.LoaderDataPreprocessor.getPreprocessingPipeline).toHaveBeenCalled();
+        });
+        it('should handle unexpected errors and set failureArea and systemErrors', async () => {
+            const mockRows = [{ name: 'id1' }, { name: 'id2' }];
+            setupMocks(mockRows);
+            jest.spyOn(color_processor_1.ColorProcessor, 'validateLoadFile').mockReturnValue({
+                errors: [],
+                warnings: [],
+                generalErrors: [],
+                generalWarnings: [],
+            });
+            jest.spyOn(file_preprocessor_1.LoaderDataPreprocessor, 'getPreprocessingPipeline').mockImplementation(() => {
+                throw new Error('Unexpected error');
+            });
+            const result = await file_preprocessor_1.LoaderDataPreprocessor.transformAndValidateData(mockClient, loaderProcess, entity_types_1.EntityType.COLOR);
+            expect(result.failureArea).toBe(entity_types_1.LoaderFailureArea.TRANSFORMING_DATA);
+            expect(result.systemErrors.length).toBe(1);
+            expect(result.systemErrors[0].message).toBe('Unexpected error');
+        });
+    });
+});