@contrail/cli 3.4.5-alpha.18160f5 → 3.4.6-alpha.c203463

package/node_modules/@contrail/loader/lib/entities-v2/linesheet-v2/assortment-processor.js

@@ -0,0 +1,388 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AssortmentProcessor = void 0;
+const entity_types_1 = require("@contrail/entity-types");
+const entity_builder_helpers_1 = require("../../common/entity-builder-helpers");
+const item_helper_1 = require("./item-helper");
+const entities_helper_1 = require("../../common/entities-helper");
+const base_processor_v2_1 = require("../base-processor-v2");
+const entity_compare_1 = require("@contrail/entity-compare");
+const partial_assortment_processor_1 = require("./partial-assortment-processor");
+const lodash_1 = require("lodash");
+const constants_1 = require("../../common/constants");
+class AssortmentProcessor extends base_processor_v2_1.BaseProcessorV2 {
+    static requiredProperties = ['itemOptionFederatedId', 'itemFamilyFederatedId', 'assortmentId'];
+    static validateRow(row) {
+        return !!row.itemFamilyFederatedId;
+    }
+    static getFederatedIdKeys() {
+        return ['itemFamilyFederatedId', 'itemOptionFederatedId'];
+    }
+    static async buildAssortmentEntities(client, rawData, loaderProcess, itemsByFederatedId, typesByTypePath) {
+        return await this.processAssortmentLoad(loaderProcess, rawData, itemsByFederatedId, client, typesByTypePath);
+    }
+    static async processAssortmentLoad(loaderProcess, rows, itemsByFederatedId, client, typesByTypePath) {
+        const errors = [];
+        let assortmentResult;
+        try {
+            const rowsHaveAssortmentId = rows.some((row) => row.assortmentId);
+            if (rowsHaveAssortmentId) {
+                assortmentResult = await this.processRowBasedAssortmentLoad(loaderProcess, rows, itemsByFederatedId, client, typesByTypePath);
+            }
+            else if (this.isUsingAssortmentSplit(loaderProcess)) {
+                assortmentResult = await this.processLegacySplitLoad(loaderProcess, rows, itemsByFederatedId, client, typesByTypePath);
+            }
+            else {
+                const result = await this.loadCSVJson(client, rows, itemsByFederatedId, null, null, typesByTypePath);
+                assortmentResult = {
+                    incomingAssortmentItems: result.incomingAssortmentItems,
+                    badAssortmentItems: result.badAssortmentItems,
+                    assortmentDetails: [result],
+                };
+            }
+        }
+        catch (error) {
+            console.error('Error processing assortment load:', error);
+            errors.push(error.message);
+        }
+        return {
+            incomingEntities: assortmentResult.incomingAssortmentItems,
+            badEntities: assortmentResult.badAssortmentItems,
+            allCurrentEntitiesByFederatedId: itemsByFederatedId,
+            typesByPath: [],
+            projectId: null,
+            ...assortmentResult,
+        };
+    }
+    static async processRowBasedAssortmentLoad(loaderProcess, rows, itemsByFederatedId, client, typesByTypePath) {
+        const isPartial = !!loaderProcess.partialAssortmentUpdate;
+        const dropField = loaderProcess.assortmentItemDropField;
+        const groupedByAssortment = {};
+        for (const row of rows) {
+            const id = row.assortmentId;
+            if (!id)
+                continue;
+            if (!groupedByAssortment[id])
+                groupedByAssortment[id] = [];
+            groupedByAssortment[id].push(row);
+        }
+        if (isPartial && dropField) {
+            const isFalsy = (val) => !val || constants_1.FALSY_VALUES.has(val.toLowerCase());
+            const operations = [];
+            for (const [assortmentId, groupRows] of Object.entries(groupedByAssortment)) {
+                const rowsToAdd = groupRows.filter((row) => isFalsy(row[dropField]));
+                const rowsToDrop = groupRows.filter((row) => !isFalsy(row[dropField]));
+                if (!rowsToAdd.length && !rowsToDrop.length)
+                    continue;
+                operations.push({
+                    rowsToAdd,
+                    rowsToDrop,
+                    rowsToUpdate: [],
+                    assortmentId,
+                });
+            }
+            return this.runPartialLoaderOperations(operations, client, itemsByFederatedId, typesByTypePath);
+        }
+        else {
+            const operations = [];
+            for (const [assortmentId, groupRows] of Object.entries(groupedByAssortment)) {
+                if (!groupRows.length)
+                    continue;
+                operations.push({ rowsToLoad: groupRows, assortmentId });
+            }
+            return this.runLoaderOperations(operations, itemsByFederatedId, client, typesByTypePath);
+        }
+    }
+    static async processLegacySplitLoad(loaderProcess, rows, itemsByFederatedId, client, typesByTypePath) {
+        const partialConfig = this.getPartialAssortmentLoadConfig(loaderProcess, this.getCSVHeaders(rows));
+        const rowsWithNoAssortmentSplit = rows.filter((row) => !loaderProcess.assortmentSplit.values
+            .map((value) => value.value)
+            .includes(row[loaderProcess.assortmentSplit.fieldToSplitOn]));
+        if (rowsWithNoAssortmentSplit.length) {
+            console.warn(`Warning: ${rowsWithNoAssortmentSplit.length} rows do not have a valid value for fieldToSplitOn`);
+        }
+        const groupedData = this.splitDataByAssortment(rows, loaderProcess);
+        if (partialConfig.isPartialAssortmentLoad) {
+            return this.handlePartialLoad(loaderProcess, groupedData, client, itemsByFederatedId, typesByTypePath);
+        }
+        else {
+            return this.handleFullLoad(loaderProcess, groupedData, client, itemsByFederatedId, typesByTypePath);
+        }
+    }
+    static getCSVHeaders(rows) {
+        return new Set((0, lodash_1.flatMap)(rows, Object.keys));
+    }
+    static getPartialAssortmentLoadConfig(loaderProcess, csvHeaders) {
+        let dropField;
+        let isPartialAssortmentLoad = false;
+        if (loaderProcess?.partialAssortmentUpdate) {
+            isPartialAssortmentLoad = true;
+            if (!loaderProcess?.assortmentItemDropField) {
+                console.error('Error: partialAssortmentUpdate requires assortmentItemDropField');
+                throw new Error('Error: partialAssortmentUpdate requires assortmentItemDropField in the main loader config');
+            }
+            if (!csvHeaders.has(loaderProcess.assortmentItemDropField)) {
+                console.error('Error: partialAssortmentUpdate requires assortmentItemDropField to be present in the data.');
+                throw new Error(`Error: partialAssortmentUpdate requires assortmentItemDropField (${loaderProcess.assortmentItemDropField}) to be present in the data.`);
+            }
+            dropField = loaderProcess.assortmentItemDropField;
+        }
+        else if (loaderProcess?.assortmentSplit?.partialAssortmentUpdate) {
+            isPartialAssortmentLoad = true;
+            if (!loaderProcess?.assortmentSplit?.dropField) {
+                console.error('Error: partialAssortmentUpdate requires dropField in the assortmentSplit config');
+                throw new Error('Error: partialAssortmentUpdate requires dropField in the assortmentSplit config');
+            }
+            if (!csvHeaders.has(loaderProcess?.assortmentSplit?.dropField)) {
+                console.error('Error: partialAssortmentUpdate requires dropField to be present in the data.');
+                throw new Error(`Error: partialAssortmentUpdate requires dropField (${loaderProcess?.assortmentSplit?.dropField}) to be present in the data.`);
+            }
+            dropField = loaderProcess?.assortmentSplit?.dropField;
+        }
+        return { isPartialAssortmentLoad, dropField };
+    }
+    static isUsingAssortmentSplit(loaderProcess) {
+        return !!(loaderProcess.assortmentSplit &&
+            loaderProcess.assortmentSplit.values?.length &&
+            loaderProcess.assortmentSplit.fieldToSplitOn);
+    }
+    static splitDataByAssortment(rows, loaderProcess) {
+        const fieldToSplitOn = loaderProcess.assortmentSplit.fieldToSplitOn;
+        const validValues = loaderProcess.assortmentSplit.values.map((v) => v.value);
+        const groups = {};
+        const rowsWithoutValidFieldToSplitOn = [];
+        for (const csvJSONrow of rows) {
+            if (!validValues.includes(csvJSONrow[fieldToSplitOn])) {
+                rowsWithoutValidFieldToSplitOn.push(csvJSONrow);
+                continue;
+            }
+            if (!groups[csvJSONrow[fieldToSplitOn]]) {
+                groups[csvJSONrow[fieldToSplitOn]] = [];
+            }
+            groups[csvJSONrow[fieldToSplitOn]].push(csvJSONrow);
+        }
+        console.debug('Row(s) do not have valid fieldToSplitOn.', rowsWithoutValidFieldToSplitOn.length);
+        console.debug('Row(s) do not have valid fieldToSplitOn.', rowsWithoutValidFieldToSplitOn);
+        console.debug('Group Data:');
+        const groupKeys = Object.keys(groups);
+        for (const groupKey of groupKeys) {
+            console.debug(`Group ${groupKey}:`, groups[groupKey].length);
+        }
+        return groups;
+    }
+    static async handlePartialLoad(loaderProcess, groupedData, client, itemsByFederatedId, typesByTypePath) {
+        const operations = this.createPartialLoadOperations(loaderProcess, groupedData);
+        return this.runPartialLoaderOperations(operations, client, itemsByFederatedId, typesByTypePath);
+    }
+    static async handleFullLoad(loaderProcess, groupedData, client, itemsByFederatedId, typesByTypePath) {
+        const operations = this.createFullLoadOperations(loaderProcess, groupedData);
+        return this.runLoaderOperations(operations, itemsByFederatedId, client, typesByTypePath);
+    }
+    static createPartialLoadOperations(loaderProcess, groupedData) {
+        const assortmentLoadOperations = [];
+        for (const valueToAssortment of loaderProcess.assortmentSplit.values) {
+            console.debug('Loading: ', valueToAssortment);
+            console.debug('Rows', groupedData[valueToAssortment.value]?.length);
+            const rowsToLoad = groupedData[valueToAssortment.value] || [];
+            const rowsToAdd = rowsToLoad.filter((row) => !row[loaderProcess.assortmentSplit.dropField]);
+            const rowsToDrop = rowsToLoad.filter((row) => row[loaderProcess.assortmentSplit.dropField]);
+            if (!rowsToAdd.length && !rowsToDrop.length) {
+                console.debug(`No rows to add, drop, or update for valueToAssortment ${valueToAssortment.value}, skipping...`);
+                continue;
+            }
+            assortmentLoadOperations.push({
+                rowsToAdd,
+                rowsToDrop,
+                rowsToUpdate: [],
+                assortmentId: valueToAssortment.assortmentId,
+                assortmentIdentifier: valueToAssortment.assortmentIdentifier,
+            });
+        }
+        return assortmentLoadOperations;
+    }
+    static createFullLoadOperations(loaderProcess, groupedData) {
+        const assortmentLoadOperations = [];
+        for (const valueToAssortment of loaderProcess.assortmentSplit.values) {
+            console.debug('Loading: ', valueToAssortment);
+            console.debug('Rows', groupedData[valueToAssortment.value]?.length);
+            const rowsToLoad = groupedData[valueToAssortment.value] || [];
+            if (!rowsToLoad.length) {
+                console.debug(`No rows to load for valueToAssortment ${valueToAssortment.value}, skipping...`);
+                continue;
+            }
+            assortmentLoadOperations.push({
+                rowsToLoad,
+                assortmentId: valueToAssortment.assortmentId,
+                assortmentIdentifier: valueToAssortment.assortmentIdentifier,
+            });
+        }
+        return assortmentLoadOperations;
+    }
+    static async runPartialLoaderOperations(operations, client, itemsByFederatedId, typesByTypePath) {
+        const results = await Promise.all(operations.map((operation) => partial_assortment_processor_1.PartialAssortmentProcessor.loadCSVJson(operation, client, itemsByFederatedId, typesByTypePath)));
+        const allBadRows = (0, lodash_1.flatMap)(results, (result) => result.badAssortmentItems || []);
+        const incomingAssortmentItems = (0, lodash_1.flatMap)(results, (result) => result.incomingAssortmentItems || []);
+        return {
+            badAssortmentItems: allBadRows,
+            assortmentDetails: results,
+            incomingAssortmentItems,
+        };
+    }
+    static async runLoaderOperations(operations, itemsByFederatedId, client, typesByTypePath) {
+        const results = await Promise.all(operations.map((operation) => this.loadCSVJson(client, operation.rowsToLoad, itemsByFederatedId, operation.assortmentId, operation.assortmentIdentifier, typesByTypePath)));
+        const allBadRows = (0, lodash_1.flatMap)(results, (result) => result.badAssortmentItems || []);
+        const incomingAssortmentItems = (0, lodash_1.flatMap)(results, (result) => result.incomingAssortmentItems || []);
+        return {
+            badAssortmentItems: allBadRows,
+            assortmentDetails: results,
+            incomingAssortmentItems,
+        };
+    }
+    static async loadCSVJson(client, data, itemsByFederatedId, assortmentId, assortmentIdentifier, typesByTypePath) {
+        try {
+            const result = await this.loadObjects(client, data, itemsByFederatedId, assortmentId, assortmentIdentifier, typesByTypePath);
+            if (result.badAssortmentItems) {
+                console.warn('Assortment-Items missing required properties: ', result.badAssortmentItems);
+            }
+            return result;
+        }
+        catch (error) {
+            console.error('Error loading CSV JSON:', error);
+            throw error;
+        }
+    }
+    static async loadObjects(client, data, passedItemsByFederatedId, assortmentId, assortmentIdentifier, typesByTypePath) {
+        const { allPropertiesSlugs, typePropertyToValueType } = await (0, item_helper_1.getAssortmentItemProperties)(client, this.requiredProperties, typesByTypePath);
+        console.debug('using assortment types:', JSON.stringify({ allPropertiesSlugs, typePropertyToValueType }, null, 2));
+        const { incomingAssortmentItems, badAssortmentItems } = this.processIncomingData(data, allPropertiesSlugs, typePropertyToValueType);
+        const targetAssortment = await (0, item_helper_1.getAssortment)(client, assortmentId, assortmentIdentifier, data);
+        if (!targetAssortment) {
+            const errorMessage = 'Target Assortment not found.';
+            console.error(errorMessage);
+            throw new Error(errorMessage);
+        }
+        const currentAssortmentItems = await (0, item_helper_1.getAssortmentItems)(targetAssortment);
+        console.debug('Got current assortment-items: ', currentAssortmentItems?.length);
+        targetAssortment.assortmentItems = currentAssortmentItems;
+        let itemsByFederatedId = passedItemsByFederatedId;
+        if (!(Object.keys(passedItemsByFederatedId)?.length > 0) && incomingAssortmentItems.length > 0) {
+            itemsByFederatedId = await (0, item_helper_1.getItemsFromFederatedIds)(client, incomingAssortmentItems);
+        }
+        const { completeIncomingAssortmentItems, unmappedAssortmentItems } = (0, item_helper_1.mapAssortmentItemsToItemsAndFillItemId)(incomingAssortmentItems, itemsByFederatedId);
+        console.debug(`unmappedAssortmentItems: ${unmappedAssortmentItems?.length}`);
+        const assortmentOptionItemsByFamilyFedId = {};
+        for (const completeIncomingAssortmentItem of completeIncomingAssortmentItems) {
+            const itemOptionFederatedId = completeIncomingAssortmentItem.itemOptionFederatedId;
+            const itemFamilyFederatedId = completeIncomingAssortmentItem.itemFamilyFederatedId;
+            if (!itemOptionFederatedId && itemFamilyFederatedId) {
+                delete completeIncomingAssortmentItem.itemFamilyFederatedId;
+            }
+            else if (itemOptionFederatedId && itemFamilyFederatedId) {
+                if (!assortmentOptionItemsByFamilyFedId[itemFamilyFederatedId]) {
+                    assortmentOptionItemsByFamilyFedId[itemFamilyFederatedId] = [];
+                }
+                delete completeIncomingAssortmentItem.itemOptionFederatedId;
+                delete completeIncomingAssortmentItem.itemFamilyFederatedId;
+                assortmentOptionItemsByFamilyFedId[itemFamilyFederatedId].push(completeIncomingAssortmentItem);
+            }
+        }
+        const familyFederatedIdsToRemove = Object.keys(assortmentOptionItemsByFamilyFedId);
+        console.debug(`familyFederatedIdsToRemove: ${familyFederatedIdsToRemove?.length}`);
+        const familyIdsToRemove = familyFederatedIdsToRemove.map((fedId) => itemsByFederatedId[fedId]?.id);
+        console.debug(`familyIdsToRemove: ${familyIdsToRemove?.length}`);
+        console.debug(`Assortment-Item hydration complete. ${completeIncomingAssortmentItems?.length} complete incomingAssortmentItems`);
+        const fullSummary = this.getSummary(completeIncomingAssortmentItems, currentAssortmentItems, allPropertiesSlugs, typePropertyToValueType, familyIdsToRemove);
+        this.logSummary(fullSummary);
+        return {
+            badAssortmentItems,
+            assortment: targetAssortment,
+            fullSummary,
+            incomingAssortmentItems,
+            unmappedAssortmentItems,
+        };
+    }
+    static logSummary(summary) {
+        console.info(`Total Adds: ${summary.adds?.length}`);
+        console.info(`Total Deletes: ${summary.deletes?.length}`);
+        console.info(`Total Updates: ${summary.updates?.length}`);
+        console.info(`Total Unchanged: ${summary.unchanged?.length}`);
+    }
+    static processIncomingData(data, allPropertiesSlugs, typePropertyToValueType) {
+        const incomingAssortmentItems = [];
+        const badAssortmentItems = [];
+        data.forEach((row) => {
+            if (!row.itemFamilyFederatedId) {
+                badAssortmentItems.push(row);
+                return;
+            }
+            const assortmentItem = {};
+            for (const propertyName in row) {
+                (0, entity_builder_helpers_1.addProperty)(propertyName, allPropertiesSlugs, typePropertyToValueType[propertyName], assortmentItem, row[propertyName]);
+            }
+            if (!row.loaderIsDropped)
+                incomingAssortmentItems.push(assortmentItem);
+        });
+        console.debug('Got incoming assortment-items: ', incomingAssortmentItems?.length);
+        return { incomingAssortmentItems, badAssortmentItems };
+    }
+    static getSummary(incomingItems, currentItems, allProperties, typePropertyToValueType, familyIdsToRemove) {
+        const incomingComparable = (0, item_helper_1.convertToComparableEntities)(incomingItems, allProperties, 'itemId');
+        const currentComparable = (0, item_helper_1.convertToComparableEntities)(currentItems, allProperties, 'itemId');
+        console.debug(`incomingAssortmentComparableEntities count: ${incomingComparable?.length}`);
+        console.debug(`targetAssortmentComparableEntities count: ${currentComparable?.length}`);
+        incomingComparable.forEach((entity) => Object.keys(entity).forEach((key) => (0, entities_helper_1.parseDateOrNumberKey)(entity, key, typePropertyToValueType[key])));
+        currentComparable.forEach((entity) => Object.keys(entity).forEach((key) => (0, entities_helper_1.parseDateOrNumberKey)(entity, key, typePropertyToValueType[key])));
+        const summary = (0, entity_compare_1.compareEntityCollections)(currentComparable, incomingComparable, 'id');
+        const addsMap = {};
+        const updatesMap = {};
+        const deletesMap = {};
+        const unchangedMap = {};
+        for (const add of summary.adds) {
+            addsMap[add.id] = add;
+        }
+        for (const update of summary.updates) {
+            updatesMap[update.id] = update;
+        }
+        for (const del of summary.deletes) {
+            deletesMap[del.id] = del;
+        }
+        for (const unchanged of summary.unchanged) {
+            unchangedMap[unchanged.id] = unchanged;
+        }
+        const removedFamilyAdds = [];
+        const removedFamilyUpdates = [];
+        const removedFamilyDeletes = [];
+        const removedFamilyUnchanged = [];
+        for (const familyIdToRemove of familyIdsToRemove) {
+            if (addsMap[familyIdToRemove]) {
+                removedFamilyAdds.push(addsMap[familyIdToRemove]);
+                delete addsMap[familyIdToRemove];
+            }
+            if (updatesMap[familyIdToRemove]) {
+                removedFamilyUpdates.push(updatesMap[familyIdToRemove]);
+                delete updatesMap[familyIdToRemove];
+            }
+            if (deletesMap[familyIdToRemove]) {
+                removedFamilyDeletes.push(deletesMap[familyIdToRemove]);
+                delete deletesMap[familyIdToRemove];
+            }
+            if (unchangedMap[familyIdToRemove]) {
+                removedFamilyUnchanged.push(unchangedMap[familyIdToRemove]);
+                delete unchangedMap[familyIdToRemove];
+            }
+        }
+        console.debug('Removed Family Adds: ', JSON.stringify(removedFamilyAdds));
+        console.debug('Removed Family Updates: ', JSON.stringify(removedFamilyUpdates));
+        console.debug('Removed Family Deletes: ', JSON.stringify(removedFamilyDeletes));
+        console.debug('Removed Family Unchanged: ', JSON.stringify(removedFamilyUnchanged));
+        summary.adds = Object.values(addsMap);
+        summary.updates = Object.values(updatesMap);
+        summary.deletes = Object.values(deletesMap);
+        summary.unchanged = Object.values(unchangedMap);
+        return summary;
+    }
+    static getEntityTypeForFederatedId(entityType) {
+        return entity_types_1.EntityType.ITEM;
+    }
+}
+exports.AssortmentProcessor = AssortmentProcessor;

package/node_modules/@contrail/loader/lib/entities-v2/linesheet-v2/assortment-processor.spec.d.ts

@@ -0,0 +1 @@
+export {};