@contrail/cli 3.4.5 → 3.4.6-alpha.cffb28e

package/node_modules/@contrail/loader/lib/pipeline/transform.js

@@ -0,0 +1,356 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.transform = void 0;
+const types_1 = require("./types");
+const data_transformer_functions_1 = require("../common/data-transformer-functions");
+const type_helper_1 = require("../common/type-helper");
+const types_2 = require("./types");
+const coerce_values_by_type_1 = require("./coerce-values-by-type");
+function runSteps(steps, rows, config, typePropertyMaps) {
+    const errors = [];
+    const warnings = [];
+    const generalErrors = [];
+    const generalWarnings = [];
+    const stepResults = [];
+    let processedData = rows;
+    for (const step of steps) {
+        try {
+            const stepResult = step.execute(processedData, config, typePropertyMaps);
+            processedData = stepResult.rows;
+            stepResults.push({
+                stepName: step.name,
+                rowCountAfter: processedData.length,
+                errorCount: stepResult.errors?.length || 0,
+                warningCount: stepResult.warnings?.length || 0,
+            });
+            if (stepResult.errors?.length)
+                errors.push(...stepResult.errors);
+            if (stepResult.warnings?.length)
+                warnings.push(...stepResult.warnings);
+            if (stepResult.generalErrors?.length)
+                generalErrors.push(...stepResult.generalErrors);
+            if (stepResult.generalWarnings?.length)
+                generalWarnings.push(...stepResult.generalWarnings);
+        }
+        catch (e) {
+            const errorMessage = `Error in transform step "${step.name}": ${e.message}`;
+            console.error(errorMessage);
+            generalErrors.push({ message: errorMessage, issueType: types_1.IssueType.SYSTEM_ERROR });
+            return { rows: processedData, errors, warnings, generalErrors, generalWarnings, stepResults, aborted: true };
+        }
+    }
+    return { rows: processedData, errors, warnings, generalErrors, generalWarnings, stepResults, aborted: false };
+}
+const EMPTY_TYPE_PROPERTY_MAPS = {
+    typePropertiesBySlug: {},
+    typePropertiesByLabel: {},
+    optionSetUsingTypePropertiesBySlug: {},
+};
+function buildTransformConfig(config) {
+    const conditionalColumns = [...(config.conditionalColumns || [])];
+    const identifierInjections = [
+        { field: 'workspaceIdentifier', value: config.workspaceIdentifier },
+        { field: 'projectId', value: config.projectId },
+        { field: 'assortmentId', value: config.assortmentId },
+        { field: 'assortmentIdentifier', value: config.assortmentIdentifier },
+    ];
+    for (const { field, value } of identifierInjections) {
+        if (value) {
+            conditionalColumns.push({ toProperty: field, default: value });
+        }
+    }
+    return {
+        federatedMappings: config.federatedMappings,
+        conditionalColumns,
+        propertiesToRemove: config.propertiesToRemove,
+    };
+}
+async function transform(client, rows, config) {
+    if (!rows?.length) {
+        return {
+            rows: [],
+            typePropertyMaps: EMPTY_TYPE_PROPERTY_MAPS,
+            typesByPath: {},
+            columns: [],
+            unmappedTypeProperties: [],
+            errors: [], warnings: [], generalErrors: [], generalWarnings: [], stepResults: [],
+        };
+    }
+    const transformConfig = buildTransformConfig(config);
+    const structural = runSteps([
+        { name: 'Evaluate Conditional Columns', execute: data_transformer_functions_1.applyConditionalColumns },
+        { name: 'Remove Unwanted Properties', execute: data_transformer_functions_1.removeUnwantedProperties },
+        { name: 'Trim Whitespace', execute: data_transformer_functions_1.trimValues },
+        { name: 'Apply Federated Mapping', execute: data_transformer_functions_1.applyFederatedMappings },
+    ], rows, transformConfig);
+    if (structural.aborted) {
+        return {
+            rows: structural.rows,
+            typePropertyMaps: EMPTY_TYPE_PROPERTY_MAPS,
+            typesByPath: {},
+            columns: [],
+            unmappedTypeProperties: [],
+            ...collectIssues(structural),
+        };
+    }
+    let typePropertyMaps = EMPTY_TYPE_PROPERTY_MAPS;
+    let typesByPath = {};
+    try {
+        const typeDefs = await fetchTypeDefinitions(client, structural.rows, config);
+        typePropertyMaps = typeDefs.typePropertyMaps;
+        typesByPath = typeDefs.typesByPath;
+        if (typeDefs.generalErrors.length) {
+            structural.generalErrors.push(...typeDefs.generalErrors);
+        }
+        structural.stepResults.push({
+            stepName: 'Fetch Type Definitions',
+            rowCountAfter: structural.rows.length,
+            errorCount: typeDefs.generalErrors.length,
+            warningCount: 0,
+        });
+    }
+    catch (e) {
+        const errorMessage = `Error fetching type definitions: ${e.message}`;
+        console.error(errorMessage);
+        structural.generalErrors.push({ message: errorMessage, issueType: types_1.IssueType.SYSTEM_ERROR });
+    }
+    const typeAware = runSteps([{ name: 'Transform Data Based on Types', execute: data_transformer_functions_1.transformDataBasedOnTypes }], structural.rows, transformConfig, typePropertyMaps);
+    let processedData = typeAware.rows;
+    try {
+        const coercionResult = (0, coerce_values_by_type_1.coerceValuesByType)(processedData, typePropertyMaps);
+        processedData = coercionResult.rows;
+        typeAware.stepResults.push(coercionResult.summary);
+    }
+    catch (e) {
+        const errorMessage = `Error in transform step "Coerce Values by Type": ${e.message}`;
+        console.error(errorMessage);
+        typeAware.generalErrors.push({ message: errorMessage, issueType: types_1.IssueType.SYSTEM_ERROR });
+    }
+    const { columns, unmappedTypeProperties } = analyzeColumns(processedData, typePropertyMaps, config);
+    const merged = mergeResults(structural, typeAware);
+    merged.stepResults.push({
+        stepName: 'Analyze Columns',
+        rowCountAfter: processedData.length,
+        errorCount: 0,
+        warningCount: 0,
+    });
+    return {
+        rows: processedData,
+        typePropertyMaps,
+        typesByPath,
+        columns,
+        unmappedTypeProperties,
+        ...collectIssues(merged),
+    };
+}
+exports.transform = transform;
+function collectIssues(result) {
+    return {
+        errors: result.errors,
+        warnings: result.warnings,
+        generalErrors: result.generalErrors,
+        generalWarnings: result.generalWarnings,
+        stepResults: result.stepResults,
+    };
+}
+function mergeResults(a, b) {
+    return {
+        rows: b.rows,
+        errors: [...a.errors, ...b.errors],
+        warnings: [...a.warnings, ...b.warnings],
+        generalErrors: [...a.generalErrors, ...b.generalErrors],
+        generalWarnings: [...a.generalWarnings, ...b.generalWarnings],
+        stepResults: [...a.stepResults, ...b.stepResults],
+        aborted: b.aborted,
+    };
+}
+const BASE_REQUIRED_PROPERTIES = new Set([
+    'federatedId',
+    'name',
+    'typePath',
+]);
+const ITEM_REQUIRED_PROPERTIES = new Set([
+    'itemFamilyFederatedId',
+    'name',
+]);
+const ITEM_PAIRED_PROPERTIES = new Set([
+    'itemOptionFederatedId',
+    'optionName',
+]);
+const SYSTEM_PROPERTIES = new Set([
+    ...BASE_REQUIRED_PROPERTIES,
+    'itemFamilyFederatedId',
+    'itemOptionFederatedId',
+    'optionGroup',
+    'optionName',
+]);
+const ENTITY_TYPE_SYSTEM_PROPERTIES = {
+    [types_1.EntityType.PROJECT_ITEM]: new Set(['workspaceIdentifier', 'projectId']),
+    [types_1.EntityType.ASSORTMENT]: new Set(['assortmentId']),
+};
+function analyzeColumns(rows, typePropertyMaps, config) {
+    if (!rows?.length)
+        return { columns: [], unmappedTypeProperties: [] };
+    const entityTypes = (0, types_2.loadTypeToEntityTypes)(config.loadType);
+    const { typePropertiesBySlug, typePropertiesByLabel } = typePropertyMaps;
+    const optionSetUsingTypePropertiesBySlug = typePropertyMaps.optionSetUsingTypePropertiesBySlug;
+    const csvHeaders = extractUniqueHeaders(rows);
+    const columns = csvHeaders.map((header) => classifyColumn(header, typePropertiesBySlug, typePropertiesByLabel, optionSetUsingTypePropertiesBySlug, entityTypes));
+    const coveredSlugs = new Set();
+    for (const column of columns) {
+        if (column.matchedSlug) {
+            coveredSlugs.add(column.matchedSlug.trim());
+        }
+        if (column.matchedLabel) {
+            const labelProperty = typePropertiesByLabel[column.matchedLabel];
+            if (labelProperty?.slug) {
+                coveredSlugs.add(labelProperty.slug.trim());
+            }
+        }
+    }
+    for (const systemProperty of SYSTEM_PROPERTIES) {
+        coveredSlugs.add(systemProperty.trim());
+    }
+    for (const entityType of entityTypes) {
+        const entityTypeSystemProperties = ENTITY_TYPE_SYSTEM_PROPERTIES[entityType];
+        if (entityTypeSystemProperties) {
+            for (const systemProperty of entityTypeSystemProperties) {
+                coveredSlugs.add(systemProperty.trim());
+            }
+        }
+    }
+    const unmappedTypeProperties = Object.entries(typePropertiesBySlug)
+        .filter(([slug, prop]) => !coveredSlugs.has(slug.trim()) && !prop.isArchived)
+        .map(([slug, prop]) => ({
+        slug: slug.trim(),
+        label: prop.label,
+        propertyType: prop.propertyType,
+        isRequired: prop.required === true,
+        hasOptionSet: !!optionSetUsingTypePropertiesBySlug[slug.trim()],
+    }));
+    return { columns, unmappedTypeProperties };
+}
+function extractUniqueHeaders(rows) {
+    const seen = new Set();
+    const headers = [];
+    for (const row of rows) {
+        for (const key of Object.keys(row)) {
+            if (!seen.has(key)) {
+                seen.add(key);
+                headers.push(key);
+            }
+        }
+    }
+    return headers;
+}
+function classifyColumn(header, typePropertiesBySlug, typePropertiesByLabel, optionSetProps, entityTypes) {
+    const isItemLoad = entityTypes.includes(types_1.EntityType.ITEM) || entityTypes.includes(types_1.EntityType.PROJECT_ITEM);
+    const isEntityTypeSystemProp = entityTypes.some((et) => ENTITY_TYPE_SYSTEM_PROPERTIES[et]?.has(header));
+    const isSystemProp = SYSTEM_PROPERTIES.has(header) || isEntityTypeSystemProp;
+    const slugMatch = typePropertiesBySlug[header];
+    const labelMatch = typePropertiesByLabel[header];
+    const baseName = stripLevelSuffix(header);
+    const baseSlugMatch = baseName !== header ? typePropertiesBySlug[baseName] : undefined;
+    const matchedProp = slugMatch || labelMatch || baseSlugMatch;
+    const isMapped = !!matchedProp || isSystemProp;
+    const isArchived = matchedProp?.isArchived === true;
+    const hasOptionSet = !!(optionSetProps[header] || optionSetProps[baseName]);
+    let isRequired;
+    let requiredNote;
+    if (isItemLoad) {
+        if (ITEM_REQUIRED_PROPERTIES.has(header)) {
+            isRequired = true;
+        }
+        else if (ITEM_PAIRED_PROPERTIES.has(header)) {
+            isRequired = true;
+            const pairedWith = header === 'itemOptionFederatedId' ? 'optionName' : 'itemOptionFederatedId';
+            requiredNote = `paired with ${pairedWith}`;
+        }
+        else {
+            isRequired = false;
+        }
+    }
+    else {
+        isRequired = BASE_REQUIRED_PROPERTIES.has(header);
+    }
+    return {
+        csvHeader: header,
+        matchedSlug: slugMatch?.slug || baseSlugMatch?.slug,
+        matchedLabel: labelMatch?.label,
+        propertyType: matchedProp?.propertyType,
+        isRequired,
+        requiredNote,
+        isMapped,
+        isArchived,
+        hasOptionSet,
+    };
+}
+function stripLevelSuffix(name) {
+    if (name.endsWith('Family'))
+        return name.slice(0, -6);
+    if (name.endsWith('Option'))
+        return name.slice(0, -6);
+    return name;
+}
+const SUBTYPE_ENTITY_TYPES = new Set([types_1.EntityType.ITEM, types_1.EntityType.CUSTOM_ENTITY]);
+const SUBTYPE_PREFIX = {
+    [types_1.EntityType.ITEM]: 'item',
+    [types_1.EntityType.CUSTOM_ENTITY]: 'custom-entity',
+};
+async function fetchTypeDefinitions(client, rows, config) {
+    const generalErrors = [];
+    const entityTypes = (0, types_2.loadTypeToEntityTypes)(config.loadType);
+    const subtypeEntityTypes = entityTypes.filter((et) => SUBTYPE_ENTITY_TYPES.has(et));
+    const nonSubtypeEntityTypes = entityTypes.filter((et) => !SUBTYPE_ENTITY_TYPES.has(et));
+    const requiresTypePath = subtypeEntityTypes.length > 0;
+    const typePaths = new Set();
+    if (requiresTypePath) {
+        const expectedPrefixes = subtypeEntityTypes.map((et) => SUBTYPE_PREFIX[et]).filter(Boolean);
+        const missingTypePathRows = [];
+        const invalidPrefixRows = [];
+        for (let i = 0; i < rows.length; i++) {
+            const val = rows[i].typePath;
+            if (val === undefined || val === null || val === '') {
+                missingTypePathRows.push(i + 2);
+            }
+            else {
+                const prefix = val.split(':')[0];
+                if (!expectedPrefixes.includes(prefix)) {
+                    invalidPrefixRows.push({ row: i + 2, typePath: val });
+                }
+                typePaths.add(val);
+            }
+        }
+        if (missingTypePathRows.length > 0) {
+            const rowList = missingTypePathRows.join(', ');
+            generalErrors.push({
+                message: `${missingTypePathRows.length} row(s) are missing a typePath value (rows: ${rowList}). Every row must have a typePath for ${subtypeEntityTypes.join(', ')} loads.`,
+                issueType: types_1.IssueType.MISSING_DATA,
+            });
+        }
+        if (invalidPrefixRows.length > 0) {
+            const details = invalidPrefixRows.map((r) => `row ${r.row}: "${r.typePath}"`).join(', ');
+            generalErrors.push({
+                message: `${invalidPrefixRows.length} row(s) have a typePath with an unexpected prefix (${details}). Expected prefix: ${expectedPrefixes.join(' or ')}.`,
+                issueType: types_1.IssueType.INVALID_DATA,
+            });
+        }
+    }
+    for (const et of nonSubtypeEntityTypes) {
+        const roots = (0, types_2.entityTypesToTypeRoots)([et]);
+        for (const root of roots) {
+            typePaths.add(root);
+        }
+    }
+    const typeRoots = (0, types_2.entityTypesToTypeRoots)(entityTypes);
+    const reducedTypes = await (0, data_transformer_functions_1.getReducedTypes)(typeRoots, client);
+    const { typePropertiesBySlug, typePropertiesByLabel } = (0, data_transformer_functions_1.getPropertiesBySlugAndLabel)(reducedTypes);
+    const optionSetUsingTypePropertiesBySlug = (0, data_transformer_functions_1.getOptionSetsUsingTypePropertiesBySlug)(reducedTypes);
+    const typePropertyMaps = {
+        typePropertiesBySlug,
+        typePropertiesByLabel,
+        optionSetUsingTypePropertiesBySlug,
+    };
+    const typesByPath = await (0, type_helper_1.buildTypesByPathForPipeline)(client, Array.from(typePaths));
+    return { typePropertyMaps, typesByPath, generalErrors };
+}

package/node_modules/@contrail/loader/lib/pipeline/transform.spec.d.ts

@@ -0,0 +1 @@
+export {};