@contrail/cli 3.4.5 → 3.4.6-alpha.cffb28e

package/node_modules/@contrail/loader/lib/pipeline/project-item-execute-helpers.js

@@ -0,0 +1,580 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.executeProjectItemEntities = exports.previewProjectItemEntities = void 0;
+const types_1 = require("./types");
+const util_1 = require("@contrail/util");
+const lodash_1 = require("lodash");
+const entities_helper_1 = require("../entities-v2/entities-helper");
+const type_helper_1 = require("../common/type-helper");
+const item_execute_helpers_1 = require("./item-execute-helpers");
+const pLimit = require('p-limit');
+const DEFAULT_BATCH_SIZE = 50;
+const DEFAULT_CONCURRENCY = 20;
+const DEFAULT_MAX_RETRIES = 3;
+async function previewProjectItemEntities(client, entitySet, config, projectNameById) {
+    const maxRetries = config.options?.maxRetries ?? DEFAULT_MAX_RETRIES;
+    const itemsByFederatedId = await resolveItemsByFederatedId(client, entitySet.incomingEntities, maxRetries);
+    const { families, options: itemOptions, optionToFamilyMap, } = splitForProjectItems(entitySet.incomingEntities, entitySet.typesByPath);
+    const allItemIds = resolveItemIds(families, itemOptions, itemsByFederatedId);
+    const uniqueProjectIds = new Set();
+    for (const entity of entitySet.incomingEntities) {
+        if (entity.projectId)
+            uniqueProjectIds.add(entity.projectId);
+    }
+    const currentProjectItemsByCompositeKey = {};
+    for (const projectId of uniqueProjectIds) {
+        const projectItems = await fetchCurrentProjectItemsByItemId(client, projectId, allItemIds, maxRetries);
+        for (const [itemId, pi] of Object.entries(projectItems)) {
+            currentProjectItemsByCompositeKey[`${projectId}:${itemId}`] = pi;
+        }
+    }
+    const badRows = [];
+    const staleRows = [];
+    const warnings = [];
+    const skippedFamilyKeys = new Set();
+    const familyToCreate = [];
+    const familyToUpdate = [];
+    const familyUnchanged = [];
+    for (const family of families) {
+        const item = itemsByFederatedId[family.federatedId];
+        if (!item) {
+            warnings.push(`Item "${family.federatedId}" does not exist in the database and will be skipped from the project-item load.`);
+            skippedFamilyKeys.add(`${family.federatedId}::${family.projectId || ''}`);
+            continue;
+        }
+        const projectId = family.projectId;
+        if (!projectId) {
+            badRows.push({
+                rowNumber: -1,
+                federatedId: family.federatedId,
+                reason: `No projectId resolved for family: ${family.federatedId}`,
+            });
+            skippedFamilyKeys.add(`${family.federatedId}::${family.projectId || ''}`);
+            continue;
+        }
+        const compositeKey = `${projectId}:${item.id}`;
+        const existing = currentProjectItemsByCompositeKey[compositeKey];
+        if (!existing) {
+            familyToCreate.push(family);
+        }
+        else {
+            const changes = { ...family };
+            delete changes.federatedId;
+            delete changes.projectId;
+            changes.projectId = projectId;
+            changes.itemId = item.id;
+            if ((0, entities_helper_1.UpdateNeeded)(changes, existing)) {
+                familyToUpdate.push(family);
+            }
+            else {
+                familyUnchanged.push(family);
+            }
+        }
+    }
+    const optionToCreate = [];
+    const optionToUpdate = [];
+    const optionUnchanged = [];
+    const skippedUpstream = [];
+    for (const opt of itemOptions) {
+        const famFedId = optionToFamilyMap.get(opt.federatedId);
+        const familyKey = `${famFedId}::${opt.projectId || ''}`;
+        if (skippedFamilyKeys.has(familyKey)) {
+            skippedUpstream.push({
+                rowNumber: -1,
+                federatedId: opt.federatedId,
+                reason: `Skipped: family item (${famFedId}) not found`,
+            });
+            continue;
+        }
+        const item = itemsByFederatedId[opt.federatedId];
+        if (!item) {
+            warnings.push(`Item "${opt.federatedId}" does not exist in the database and will be skipped from the project-item load.`);
+            continue;
+        }
+        const projectId = opt.projectId;
+        if (!projectId) {
+            badRows.push({
+                rowNumber: -1,
+                federatedId: opt.federatedId,
+                reason: `No projectId resolved for option: ${opt.federatedId}`,
+            });
+            continue;
+        }
+        const compositeKey = `${projectId}:${item.id}`;
+        const existing = currentProjectItemsByCompositeKey[compositeKey];
+        if (!existing) {
+            optionToCreate.push(opt);
+        }
+        else {
+            const changes = { ...opt };
+            delete changes.federatedId;
+            delete changes.projectId;
+            changes.projectId = projectId;
+            changes.itemId = item.id;
+            if ((0, entities_helper_1.UpdateNeeded)(changes, existing)) {
+                optionToUpdate.push(opt);
+            }
+            else {
+                optionUnchanged.push(opt);
+            }
+        }
+    }
+    const familyCountsByProject = {};
+    const optionCountsByProject = {};
+    const initFamilyCounts = () => ({ total: 0, creates: 0, updates: 0, unchanged: 0 });
+    const initOptionCounts = () => ({ total: 0, creates: 0, updates: 0, unchanged: 0, skipped: 0 });
+    for (const f of families) {
+        const pid = f.projectId || '_unknown';
+        if (!familyCountsByProject[pid])
+            familyCountsByProject[pid] = initFamilyCounts();
+        familyCountsByProject[pid].total++;
+    }
+    for (const f of familyToCreate) {
+        const pid = f.projectId || '_unknown';
+        familyCountsByProject[pid].creates++;
+    }
+    for (const f of familyToUpdate) {
+        const pid = f.projectId || '_unknown';
+        familyCountsByProject[pid].updates++;
+    }
+    for (const f of familyUnchanged) {
+        const pid = f.projectId || '_unknown';
+        familyCountsByProject[pid].unchanged++;
+    }
+    for (const o of itemOptions) {
+        const pid = o.projectId || '_unknown';
+        if (!optionCountsByProject[pid])
+            optionCountsByProject[pid] = initOptionCounts();
+        optionCountsByProject[pid].total++;
+    }
+    for (const o of optionToCreate) {
+        const pid = o.projectId || '_unknown';
+        optionCountsByProject[pid].creates++;
+    }
+    for (const o of optionToUpdate) {
+        const pid = o.projectId || '_unknown';
+        optionCountsByProject[pid].updates++;
+    }
+    for (const o of optionUnchanged) {
+        const pid = o.projectId || '_unknown';
+        optionCountsByProject[pid].unchanged++;
+    }
+    for (const s of skippedUpstream) {
+    }
+    const summaries = [];
+    const allProjectIds = new Set([...Object.keys(familyCountsByProject), ...Object.keys(optionCountsByProject)]);
+    for (const pid of allProjectIds) {
+        const fc = familyCountsByProject[pid] || initFamilyCounts();
+        const oc = optionCountsByProject[pid] || initOptionCounts();
+        const targetName = projectNameById?.[pid] || undefined;
+        const targetId = pid !== '_unknown' ? pid : undefined;
+        summaries.push({
+            entityType: 'project-item (families)',
+            total: fc.total,
+            creates: fc.creates,
+            updates: fc.updates,
+            drops: 0,
+            unchanged: fc.unchanged,
+            skippedStale: 0,
+            targetName,
+            targetId,
+        });
+        summaries.push({
+            entityType: 'project-item (options)',
+            total: oc.total,
+            creates: oc.creates,
+            updates: oc.updates,
+            drops: 0,
+            unchanged: oc.unchanged,
+            skippedStale: 0,
+            targetName,
+            targetId,
+        });
+    }
+    return {
+        summaries,
+        badRows,
+        staleRows,
+        warnings,
+        constructedFamilies: {
+            toCreate: familyToCreate,
+            toUpdate: familyToUpdate,
+            unchanged: familyUnchanged,
+            currentByFedId: {},
+        },
+        constructedOptions: {
+            toCreate: optionToCreate,
+            toUpdate: optionToUpdate,
+            unchanged: optionUnchanged,
+            skippedUpstream,
+        },
+    };
+}
+exports.previewProjectItemEntities = previewProjectItemEntities;
+async function executeProjectItemEntities(client, entitySet, config, options) {
+    const batchSize = options?.batchSize ?? config.options?.batchSize ?? DEFAULT_BATCH_SIZE;
+    const concurrency = options?.concurrency ?? config.options?.concurrency ?? DEFAULT_CONCURRENCY;
+    const maxRetries = options?.maxRetries ?? config.options?.maxRetries ?? DEFAULT_MAX_RETRIES;
+    const limit = pLimit(concurrency);
+    const entitiesWithProjectId = entitySet.incomingEntities.filter((e) => e.projectId);
+    if (entitiesWithProjectId.length === 0) {
+        return {
+            summaries: [{
+                    entityType: 'project-item',
+                    created: 0,
+                    updated: 0,
+                    dropped: 0,
+                    skippedStale: 0,
+                    failed: entitySet.incomingEntities.length,
+                    failures: [{
+                            federatedId: 'N/A',
+                            operation: 'create',
+                            error: 'Could not resolve a target Project\'s ID from the data or config provided. Include or map a workspaceIdentifier column if loading only ProjectItems, or a projectId column with the project\'s ID directly.',
+                        }],
+                }],
+            skippedRows: [],
+            warnings: ['Project ID resolution failed. No project-items were loaded because a Project could not be found to load into.'],
+            hasFailures: true,
+        };
+    }
+    const itemsByFederatedId = await resolveItemsByFederatedId(client, entitySet.incomingEntities, maxRetries);
+    const { families, options: itemOptions, optionToFamilyMap, } = splitForProjectItems(entitySet.incomingEntities, entitySet.typesByPath);
+    const allSkippedRows = [];
+    const allWarnings = [];
+    let hasFailures = false;
+    const allItemIds = resolveItemIds(families, itemOptions, itemsByFederatedId);
+    const uniqueProjectIds = new Set();
+    for (const entity of entitySet.incomingEntities) {
+        if (entity.projectId)
+            uniqueProjectIds.add(entity.projectId);
+    }
+    const currentProjectItemsByCompositeKey = {};
+    for (const projectId of uniqueProjectIds) {
+        const projectItems = await fetchCurrentProjectItemsByItemId(client, projectId, allItemIds, maxRetries);
+        for (const [itemId, pi] of Object.entries(projectItems)) {
+            currentProjectItemsByCompositeKey[`${projectId}:${itemId}`] = pi;
+        }
+    }
+    const familyFailures = [];
+    let familyCreatedCount = 0;
+    let familyUpdatedCount = 0;
+    let familyUnchangedCount = 0;
+    const familyUpserts = [];
+    const skippedFamilyFedIds = new Set();
+    for (const family of families) {
+        const item = itemsByFederatedId[family.federatedId];
+        if (!item) {
+            allSkippedRows.push({
+                rowNumber: -1,
+                federatedId: family.federatedId,
+                reason: `Item not found for family federatedId: ${family.federatedId}`,
+            });
+            skippedFamilyFedIds.add(family.federatedId);
+            continue;
+        }
+        const projectId = family.projectId;
+        if (!projectId) {
+            allSkippedRows.push({
+                rowNumber: -1,
+                federatedId: family.federatedId,
+                reason: `No projectId resolved for family: ${family.federatedId}`,
+            });
+            skippedFamilyFedIds.add(family.federatedId);
+            continue;
+        }
+        const compositeId = `${projectId}:${item.id}`;
+        const compositeKey = `${projectId}:${item.id}`;
+        const existingProjectItem = currentProjectItemsByCompositeKey[compositeKey];
+        const changes = { ...family };
+        delete changes.federatedId;
+        delete changes.projectId;
+        changes.projectId = projectId;
+        changes.itemId = item.id;
+        if (existingProjectItem) {
+            if (!(0, entities_helper_1.UpdateNeeded)(changes, existingProjectItem)) {
+                familyUnchangedCount++;
+                continue;
+            }
+            familyUpdatedCount++;
+        }
+        else {
+            familyCreatedCount++;
+        }
+        familyUpserts.push({ id: compositeId, changes });
+    }
+    const totalOps = familyUpserts.length;
+    let completedOps = 0;
+    const reportProgress = (count, stage) => {
+        completedOps += count;
+        options?.onProgress?.({ stage, current: completedOps, total: totalOps });
+    };
+    const familyUpsertResult = await batchUpsertProjectItems(client, familyUpserts, batchSize, limit, maxRetries, (n) => reportProgress(n, 'execute:project-item-families'));
+    familyFailures.push(...familyUpsertResult.failures);
+    if (familyUpsertResult.failures.length > 0)
+        hasFailures = true;
+    for (const upserted of familyUpsertResult.upsertedItems) {
+        if (upserted.itemId && upserted.projectId) {
+            currentProjectItemsByCompositeKey[`${upserted.projectId}:${upserted.itemId}`] = upserted;
+        }
+    }
+    const optionFailures = [];
+    let optionCreatedCount = 0;
+    let optionUpdatedCount = 0;
+    let optionUnchangedCount = 0;
+    const optionUpserts = [];
+    for (const opt of itemOptions) {
+        const famFedId = optionToFamilyMap.get(opt.federatedId);
+        if (skippedFamilyFedIds.has(famFedId)) {
+            allSkippedRows.push({
+                rowNumber: -1,
+                federatedId: opt.federatedId,
+                reason: `Skipped: family item (${famFedId}) not found`,
+            });
+            continue;
+        }
+        const item = itemsByFederatedId[opt.federatedId];
+        if (!item) {
+            allSkippedRows.push({
+                rowNumber: -1,
+                federatedId: opt.federatedId,
+                reason: `Item not found for option federatedId: ${opt.federatedId}`,
+            });
+            continue;
+        }
+        const projectId = opt.projectId;
+        if (!projectId) {
+            allSkippedRows.push({
+                rowNumber: -1,
+                federatedId: opt.federatedId,
+                reason: `No projectId resolved for option: ${opt.federatedId}`,
+            });
+            continue;
+        }
+        const compositeId = `${projectId}:${item.id}`;
+        const compositeKey = `${projectId}:${item.id}`;
+        const existingProjectItem = currentProjectItemsByCompositeKey[compositeKey];
+        const changes = { ...opt };
+        delete changes.federatedId;
+        delete changes.projectId;
+        changes.projectId = projectId;
+        changes.itemId = item.id;
+        if (existingProjectItem) {
+            if (!(0, entities_helper_1.UpdateNeeded)(changes, existingProjectItem)) {
+                optionUnchangedCount++;
+                continue;
+            }
+            optionUpdatedCount++;
+        }
+        else {
+            optionCreatedCount++;
+        }
+        optionUpserts.push({ id: compositeId, changes });
+    }
+    const optTotalOps = optionUpserts.length;
+    let optCompletedOps = 0;
+    const reportOptProgress = (count) => {
+        optCompletedOps += count;
+        options?.onProgress?.({ stage: 'execute:project-item-options', current: optCompletedOps, total: optTotalOps });
+    };
+    const optionUpsertResult = await batchUpsertProjectItems(client, optionUpserts, batchSize, limit, maxRetries, reportOptProgress);
+    optionFailures.push(...optionUpsertResult.failures);
+    if (optionUpsertResult.failures.length > 0)
+        hasFailures = true;
+    const projectNameById = {};
+    for (const entity of entitySet.incomingEntities) {
+        if (entity.projectId && entity.projectName && !projectNameById[entity.projectId]) {
+            projectNameById[entity.projectId] = entity.projectName;
+        }
+    }
+    const allProjectIds = new Set();
+    for (const entity of entitySet.incomingEntities) {
+        if (entity.projectId)
+            allProjectIds.add(entity.projectId);
+    }
+    const targetName = allProjectIds.size === 1 ? projectNameById[allProjectIds.values().next().value] : undefined;
+    const targetId = allProjectIds.size === 1 ? allProjectIds.values().next().value : undefined;
+    const summaries = [
+        {
+            entityType: 'project-item (families)',
+            created: Math.max(0, familyCreatedCount - familyUpsertResult.failures.filter(f => f.operation === 'create').length),
+            updated: Math.max(0, familyUpdatedCount - familyUpsertResult.failures.filter(f => f.operation === 'update').length),
+            dropped: 0,
+            skippedStale: 0,
+            failed: familyFailures.length,
+            failures: familyFailures,
+            targetName,
+            targetId,
+        },
+        {
+            entityType: 'project-item (options)',
+            created: Math.max(0, optionCreatedCount - optionUpsertResult.failures.filter(f => f.operation === 'create').length),
+            updated: Math.max(0, optionUpdatedCount - optionUpsertResult.failures.filter(f => f.operation === 'update').length),
+            dropped: 0,
+            skippedStale: 0,
+            failed: optionFailures.length,
+            failures: optionFailures,
+            targetName,
+            targetId,
+        },
+    ];
+    return { summaries, skippedRows: allSkippedRows, warnings: allWarnings, hasFailures };
+}
+exports.executeProjectItemEntities = executeProjectItemEntities;
+function splitForProjectItems(incomingEntities, typesByPath) {
+    const byCompositeKey = {};
+    for (const entity of incomingEntities) {
+        const famFedId = entity.itemFamilyFederatedId;
+        const projectId = entity.projectId;
+        if (!famFedId)
+            continue;
+        const key = `${famFedId}::${projectId || ''}`;
+        if (!byCompositeKey[key])
+            byCompositeKey[key] = [];
+        byCompositeKey[key].push(entity);
+    }
+    const families = [];
+    const options = [];
+    const optionToFamilyMap = new Map();
+    for (const [, rows] of Object.entries(byCompositeKey)) {
+        const familyRow = rows.find((r) => !r.itemOptionFederatedId) || rows[0];
+        const famFedId = familyRow.itemFamilyFederatedId;
+        const projectId = familyRow.projectId;
+        const typePath = familyRow.typePath || types_1.EntityType.ITEM;
+        const typeInfo = typesByPath[typePath];
+        const type = typeInfo?.type;
+        const familyClone = util_1.ObjectUtil.cloneDeep(familyRow);
+        const familyEntity = type ? (0, type_helper_1.reduceToFamilyProperties)(familyClone, type) : { ...familyClone };
+        familyEntity.federatedId = famFedId;
+        familyEntity.projectId = projectId;
+        delete familyEntity.optionGroup;
+        delete familyEntity.optionName;
+        delete familyEntity.itemFamilyFederatedId;
+        delete familyEntity.itemOptionFederatedId;
+        families.push(familyEntity);
+        for (const row of rows) {
+            if (!row.itemOptionFederatedId)
+                continue;
+            const optionFedId = row.itemOptionFederatedId;
+            const optClone = util_1.ObjectUtil.cloneDeep(row);
+            optClone.federatedId = optionFedId;
+            optClone.projectId = row.projectId;
+            optionToFamilyMap.set(optionFedId, famFedId);
+            delete optClone.itemFamilyFederatedId;
+            delete optClone.itemOptionFederatedId;
+            if (type) {
+                (0, type_helper_1.setOptionOverrides)(optClone, { typeProperties: type.typeProperties });
+                (0, type_helper_1.removeFamilyLevelPropsFromOption)(optClone, type);
+            }
+            options.push(optClone);
+        }
+    }
+    return { families, options, optionToFamilyMap };
+}
+async function resolveItemsByFederatedId(client, incomingEntities, maxRetries) {
+    const federatedIds = new Set();
+    for (const entity of incomingEntities) {
+        if (entity.itemFamilyFederatedId)
+            federatedIds.add(entity.itemFamilyFederatedId);
+        if (entity.itemOptionFederatedId)
+            federatedIds.add(entity.itemOptionFederatedId);
+    }
+    const result = {};
+    const chunks = (0, lodash_1.chunk)(Array.from(federatedIds), 200);
+    const limit = pLimit(5);
+    const promises = chunks.map((batch) => limit(async () => {
+        try {
+            const items = await (0, entities_helper_1.retryOnTimeout)(client, entities_helper_1.EntitiesLambdaCallType.GET, {
+                entityName: 'item',
+                federatedIds: batch,
+                skipHydrateRelationships: true,
+            }, maxRetries);
+            if (items?.length) {
+                for (const item of items) {
+                    result[item.federatedId] = item;
+                }
+            }
+        }
+        catch (e) {
+            console.warn('Failed to resolve items by federatedId:', e.message);
+        }
+    }));
+    await Promise.all(promises);
+    return result;
+}
+function resolveItemIds(families, options, itemsByFederatedId) {
+    const ids = new Set();
+    for (const entity of [...families, ...options]) {
+        const item = itemsByFederatedId[entity.federatedId];
+        if (item?.id)
+            ids.add(item.id);
+    }
+    return Array.from(ids);
+}
+async function fetchCurrentProjectItemsByItemId(client, projectId, itemIds, maxRetries) {
+    const result = {};
+    if (!itemIds.length)
+        return result;
+    const chunks = (0, lodash_1.chunk)(itemIds, 200);
+    const limit = pLimit(5);
+    const promises = chunks.map((batch) => limit(async () => {
+        try {
+            const projectItems = await (0, entities_helper_1.retryOnTimeout)(client, entities_helper_1.EntitiesLambdaCallType.GET, {
+                entityName: 'project-item',
+                criteria: { projectId, itemIds: batch },
+                skipHydrateRelationships: true,
+            }, maxRetries);
+            if (projectItems?.length) {
+                for (const pi of projectItems) {
+                    result[pi.itemId] = pi;
+                }
+            }
+        }
+        catch (e) {
+            console.warn('Failed to fetch current project-items:', e.message);
+        }
+    }));
+    await Promise.all(promises);
+    return result;
+}
+async function batchUpsertProjectItems(client, entities, batchSize, limit, maxRetries, onBatchComplete) {
+    let successCount = 0;
+    const failures = [];
+    const upsertedItems = [];
+    const chunks = (0, lodash_1.chunk)(entities, batchSize);
+    const promises = chunks.map((batch) => limit(async () => {
+        try {
+            const result = await (0, entities_helper_1.retryOnTimeout)(client, entities_helper_1.EntitiesLambdaCallType.BATCHUPDATE, {
+                entityName: 'project-item',
+                objects: batch,
+                apiVersion: '2',
+            }, maxRetries);
+            if (result?.errors?.length) {
+                for (const err of result.errors) {
+                    failures.push({
+                        federatedId: err?.entity?.id ?? (0, item_execute_helpers_1.extractFailureId)(err),
+                        operation: 'create',
+                        error: (0, item_execute_helpers_1.extractApiErrorMessage)(err),
+                    });
+                }
+            }
+            const updatedItems = result?.data || [];
+            upsertedItems.push(...updatedItems);
+            const errorCount = result?.errors?.length ?? 0;
+            const hasData = Array.isArray(result?.data) && result.data.length > 0;
+            const batchSuccessCount = hasData ? result.data.length : batch.length - errorCount;
+            successCount += Math.max(0, batchSuccessCount);
+            onBatchComplete(batch.length);
+        }
+        catch (e) {
+            for (const entity of batch) {
+                failures.push({
+                    federatedId: entity.id,
+                    operation: 'create',
+                    error: e.message,
+                });
+            }
+        }
+    }));
+    await Promise.all(promises);
+    return { successCount, failures, upsertedItems };
+}

package/node_modules/@contrail/loader/lib/pipeline/transform.d.ts

@@ -0,0 +1,3 @@
+import { EntitiesClient } from './types';
+import { LoaderConfig, TransformResult } from './types';
+export declare function transform(client: EntitiesClient, rows: Record<string, any>[], config: LoaderConfig): Promise<TransformResult>;