@contrail/cli 3.4.5 → 3.4.6-alpha.cffb28e

package/node_modules/@contrail/loader/lib/pipeline/execute.js

@@ -0,0 +1,271 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.execute = void 0;
+const types_1 = require("./types");
+const lodash_1 = require("lodash");
+const entities_helper_1 = require("../entities-v2/entities-helper");
+const entity_processor_v2_1 = require("../entities-v2/entity-processor-v2");
+const types_2 = require("./types");
+const item_execute_helpers_1 = require("./item-execute-helpers");
+const project_item_execute_helpers_1 = require("./project-item-execute-helpers");
+const assortment_execute_helpers_1 = require("./assortment-execute-helpers");
+const pLimit = require('p-limit');
+const DEFAULT_BATCH_SIZE = 25;
+const DEFAULT_CONCURRENCY = 5;
+const DEFAULT_MAX_RETRIES = 3;
+async function execute(client, rows, config, columnAnalysis, options, previewData) {
+    const startTime = Date.now();
+    const batchSize = options?.batchSize ?? config.options?.batchSize ?? DEFAULT_BATCH_SIZE;
+    const concurrency = options?.concurrency ?? config.options?.concurrency ?? DEFAULT_CONCURRENCY;
+    const maxRetries = options?.maxRetries ?? config.options?.maxRetries ?? DEFAULT_MAX_RETRIES;
+    const limit = pLimit(concurrency);
+    const entityTypes = (0, types_2.enforceItemFirstOrder)((0, types_2.loadTypeToEntityTypes)(config.loadType));
+    const loaderProcess = (0, types_2.toLoaderProcessCompat)(config);
+    const allEntitySummaries = [];
+    const allSkippedRows = [];
+    const allWarnings = [];
+    let hasFailures = false;
+    for (const entityType of entityTypes) {
+        let entitySet;
+        const isDependentType = entityType === types_1.EntityType.PROJECT_ITEM || entityType === types_1.EntityType.ASSORTMENT;
+        if (previewData?.constructedEntities?.[entityType] && !isDependentType) {
+            entitySet = previewData.constructedEntities[entityType];
+        }
+        else {
+            let clonedRows = rows.map(row => ({ ...row }));
+            if (entityType === types_1.EntityType.ASSORTMENT) {
+                clonedRows = clonedRows.filter(row => row.assortmentId);
+            }
+            const doConstructEntities = entityType !== types_1.EntityType.ASSORTMENT;
+            const result = await entity_processor_v2_1.EntityProcessorV2.setupEntities(client, entityType, clonedRows, undefined, loaderProcess, {}, doConstructEntities);
+            entitySet = {
+                incomingEntities: result.incomingEntities || [],
+                badEntities: result.badEntities || [],
+                allCurrentEntitiesByFederatedId: result.allCurrentEntitiesByFederatedId || {},
+                typesByPath: result.typesByPath || {},
+                projectId: result.projectId,
+                generalWarnings: result.generalWarnings,
+                assortmentDetails: result.assortmentDetails,
+            };
+        }
+        for (const bad of entitySet.badEntities) {
+            allSkippedRows.push({
+                rowNumber: -1,
+                federatedId: bad?.federatedId,
+                reason: 'Failed entity construction',
+            });
+        }
+        if (entitySet.generalWarnings?.length) {
+            for (const w of entitySet.generalWarnings) {
+                allWarnings.push(w.message);
+            }
+        }
+        if (entityType === types_1.EntityType.ITEM) {
+            const itemResult = await (0, item_execute_helpers_1.executeItemEntities)(client, entitySet, config, options);
+            allEntitySummaries.push(...itemResult.summaries);
+            allSkippedRows.push(...itemResult.skippedRows);
+            allWarnings.push(...itemResult.warnings);
+            if (itemResult.hasFailures)
+                hasFailures = true;
+            continue;
+        }
+        if (entityType === types_1.EntityType.PROJECT_ITEM) {
+            const projectItemResult = await (0, project_item_execute_helpers_1.executeProjectItemEntities)(client, entitySet, config, options);
+            allEntitySummaries.push(...projectItemResult.summaries);
+            allSkippedRows.push(...projectItemResult.skippedRows);
+            allWarnings.push(...projectItemResult.warnings);
+            if (projectItemResult.hasFailures)
+                hasFailures = true;
+            continue;
+        }
+        if (entityType === types_1.EntityType.ASSORTMENT) {
+            const assortmentResult = await (0, assortment_execute_helpers_1.executeAssortmentEntities)(client, entitySet, config, options);
+            allEntitySummaries.push(...assortmentResult.summaries);
+            allSkippedRows.push(...assortmentResult.skippedRows);
+            allWarnings.push(...assortmentResult.warnings);
+            if (assortmentResult.hasFailures)
+                hasFailures = true;
+            continue;
+        }
+        const { toCreate, toUpdate, stale, unchanged } = classifyForExecution(entitySet.incomingEntities, entitySet.allCurrentEntitiesByFederatedId, config);
+        for (const s of stale) {
+            allSkippedRows.push({
+                rowNumber: -1,
+                federatedId: s.entity?.federatedId,
+                reason: 'Stale update skipped',
+                details: {
+                    incomingTimestamp: s.incomingTimestamp,
+                    existingTimestamp: s.existingTimestamp,
+                },
+            });
+        }
+        if (config.options?.ignoreFamilyNameOnUpdate) {
+            for (const entity of toUpdate) {
+                delete entity.name;
+            }
+        }
+        if (config.options?.ignoreOptionNameOnUpdate) {
+            for (const entity of toUpdate) {
+                delete entity.optionName;
+            }
+        }
+        const sdkEntityName = getSdkEntityName(entityType);
+        const totalOperations = toCreate.length + toUpdate.length;
+        let completedOperations = 0;
+        const reportProgress = (count) => {
+            completedOperations += count;
+            options?.onProgress?.({
+                stage: `execute:${entityType}`,
+                current: completedOperations,
+                total: totalOperations,
+            });
+        };
+        const createFailures = [];
+        let createdCount = 0;
+        const createChunks = (0, lodash_1.chunk)(toCreate, batchSize);
+        const createPromises = createChunks.map((batch) => limit(async () => {
+            try {
+                const result = await (0, entities_helper_1.retryOnTimeout)(client, entities_helper_1.EntitiesLambdaCallType.BATCHCREATE, {
+                    entityName: sdkEntityName,
+                    objects: batch,
+                    apiVersion: '2',
+                }, maxRetries);
+                if (result?.errors?.length) {
+                    for (const err of result.errors) {
+                        createFailures.push({
+                            federatedId: (0, item_execute_helpers_1.extractFailureId)(err),
+                            operation: 'create',
+                            error: (0, item_execute_helpers_1.extractApiErrorMessage)(err),
+                        });
+                    }
+                }
+                const errorCount = result?.errors?.length ?? 0;
+                const hasData = Array.isArray(result?.data) && result.data.length > 0;
+                const successCount = hasData ? result.data.length : batch.length - errorCount;
+                createdCount += Math.max(0, successCount);
+                reportProgress(batch.length);
+            }
+            catch (e) {
+                for (const entity of batch) {
+                    createFailures.push({
+                        federatedId: entity.federatedId ?? 'unknown',
+                        operation: 'create',
+                        error: e.message,
+                    });
+                }
+                hasFailures = true;
+            }
+        }));
+        await Promise.all(createPromises);
+        const updateFailures = [];
+        let updatedCount = 0;
+        const updateEntities = toUpdate.map((incoming) => {
+            const existing = entitySet.allCurrentEntitiesByFederatedId[incoming.federatedId];
+            return (0, item_execute_helpers_1.buildDiffUpdatePayload)(incoming, existing);
+        });
+        const updateChunks = (0, lodash_1.chunk)(updateEntities, batchSize);
+        const updatePromises = updateChunks.map((batch) => limit(async () => {
+            try {
+                const result = await (0, entities_helper_1.retryOnTimeout)(client, entities_helper_1.EntitiesLambdaCallType.BATCHUPDATE, {
+                    entityName: sdkEntityName,
+                    objects: batch,
+                    apiVersion: '2',
+                }, maxRetries);
+                if (result?.errors?.length) {
+                    for (const err of result.errors) {
+                        updateFailures.push({
+                            federatedId: (0, item_execute_helpers_1.extractFailureId)(err),
+                            operation: 'update',
+                            error: (0, item_execute_helpers_1.extractApiErrorMessage)(err),
+                        });
+                    }
+                }
+                const errorCount = result?.errors?.length ?? 0;
+                const hasData = Array.isArray(result?.data) && result.data.length > 0;
+                const successCount = hasData ? result.data.length : batch.length - errorCount;
+                updatedCount += Math.max(0, successCount);
+                reportProgress(batch.length);
+            }
+            catch (e) {
+                for (const entity of batch) {
+                    updateFailures.push({
+                        federatedId: entity.federatedId ?? 'unknown',
+                        operation: 'update',
+                        error: e.message,
+                    });
+                }
+                hasFailures = true;
+            }
+        }));
+        await Promise.all(updatePromises);
+        const allFailures = [...createFailures, ...updateFailures];
+        if (allFailures.length > 0)
+            hasFailures = true;
+        allEntitySummaries.push({
+            entityType,
+            created: createdCount,
+            updated: updatedCount,
+            dropped: 0,
+            skippedStale: stale.length,
+            failed: allFailures.length,
+            failures: allFailures,
+        });
+    }
+    const duration = Date.now() - startTime;
+    let status;
+    const totalSuccessful = allEntitySummaries.reduce((sum, e) => sum + e.created + e.updated, 0);
+    const totalFailed = allEntitySummaries.reduce((sum, e) => sum + e.failed, 0);
+    if (totalFailed === 0) {
+        status = 'complete';
+    }
+    else if (totalSuccessful > 0) {
+        status = 'partial-failure';
+    }
+    else {
+        status = 'failed';
+    }
+    return {
+        status,
+        entities: allEntitySummaries,
+        skippedRows: allSkippedRows,
+        warnings: allWarnings,
+        duration,
+    };
+}
+exports.execute = execute;
+function classifyForExecution(incomingEntities, currentByFedId, config) {
+    const toCreate = [];
+    const toUpdate = [];
+    const unchanged = [];
+    const stale = [];
+    for (const incoming of incomingEntities) {
+        const fedId = incoming.federatedId;
+        const existing = fedId ? currentByFedId[fedId] : undefined;
+        if (!existing) {
+            toCreate.push(incoming);
+            continue;
+        }
+        if (config.skipStaleUpdatesUsing) {
+            const incomingTs = incoming[config.skipStaleUpdatesUsing];
+            const existingTs = existing[config.skipStaleUpdatesUsing];
+            if (incomingTs && existingTs && new Date(incomingTs) < new Date(existingTs)) {
+                stale.push({
+                    entity: incoming,
+                    incomingTimestamp: String(incomingTs),
+                    existingTimestamp: String(existingTs),
+                });
+                continue;
+            }
+        }
+        if ((0, entities_helper_1.UpdateNeeded)(incoming, existing)) {
+            toUpdate.push(incoming);
+        }
+        else {
+            unchanged.push(incoming);
+        }
+    }
+    return { toCreate, toUpdate, unchanged, stale };
+}
+function getSdkEntityName(entityType) {
+    return entityType;
+}

package/node_modules/@contrail/loader/lib/pipeline/execute.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/node_modules/@contrail/loader/lib/pipeline/execute.spec.js

@@ -0,0 +1,186 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const types_1 = require("./types");
+const execute_1 = require("./execute");
+const entity_processor_v2_1 = require("../entities-v2/entity-processor-v2");
+jest.mock('../entities-v2/entity-processor-v2');
+describe('execute (non-item entity types)', () => {
+    let mockClient;
+    const baseConfig = {
+        loadType: [types_1.LoadType.COLOR],
+    };
+    const baseAnalysis = {
+        columns: [],
+        unmappedTypeProperties: [],
+        typePropertyMaps: {
+            typePropertiesBySlug: {},
+            typePropertiesByLabel: {},
+            optionSetUsingTypePropertiesBySlug: {},
+        },
+        typesByPath: {},
+    };
+    beforeEach(() => {
+        mockClient = {
+            create: jest.fn(),
+            update: jest.fn(),
+            get: jest.fn(),
+            delete: jest.fn(),
+            batchCreate: jest.fn().mockResolvedValue({ data: [], errors: [] }),
+            batchUpdate: jest.fn().mockResolvedValue({ data: [], errors: [] }),
+            batchDelete: jest.fn(),
+            getAllReferences: jest.fn(),
+        };
+    });
+    afterEach(() => {
+        jest.resetAllMocks();
+    });
+    it('should batch create new entities', async () => {
+        const entitySet = {
+            incomingEntities: [
+                { federatedId: 'item-1', name: 'New Item 1' },
+                { federatedId: 'item-2', name: 'New Item 2' },
+            ],
+            badEntities: [],
+            allCurrentEntitiesByFederatedId: {},
+            typesByPath: {},
+        };
+        const previewData = {
+            entities: [{ entityType: types_1.EntityType.COLOR, total: 2, creates: 2, updates: 0, drops: 0, unchanged: 0, skippedStale: 0 }],
+            badRows: [],
+            staleRows: [],
+            warnings: [],
+            constructedEntities: { [types_1.EntityType.COLOR]: entitySet },
+        };
+        const result = await (0, execute_1.execute)(mockClient, [], baseConfig, baseAnalysis, undefined, previewData);
+        expect(mockClient.batchCreate).toHaveBeenCalled();
+        expect(result.status).toBe('complete');
+        expect(result.duration).toBeGreaterThanOrEqual(0);
+    });
+    it('should batch update existing entities', async () => {
+        const entitySet = {
+            incomingEntities: [
+                { federatedId: 'item-1', name: 'Updated Name', color: 'blue' },
+            ],
+            badEntities: [],
+            allCurrentEntitiesByFederatedId: {
+                'item-1': { id: 'id-1', federatedId: 'item-1', name: 'Old Name', color: 'red' },
+            },
+            typesByPath: {},
+        };
+        const previewData = {
+            entities: [{ entityType: types_1.EntityType.COLOR, total: 1, creates: 0, updates: 1, drops: 0, unchanged: 0, skippedStale: 0 }],
+            badRows: [],
+            staleRows: [],
+            warnings: [],
+            constructedEntities: { [types_1.EntityType.COLOR]: entitySet },
+        };
+        const result = await (0, execute_1.execute)(mockClient, [], baseConfig, baseAnalysis, undefined, previewData);
+        expect(mockClient.batchUpdate).toHaveBeenCalled();
+        const updateCall = mockClient.batchUpdate.mock.calls[0][0];
+        expect(updateCall.objects[0].id).toBe('id-1');
+        expect(updateCall.objects[0].changes).toBeDefined();
+    });
+    it('should skip stale entities', async () => {
+        const config = {
+            ...baseConfig,
+            skipStaleUpdatesUsing: 'updatedOn',
+        };
+        const entitySet = {
+            incomingEntities: [
+                { federatedId: 'item-1', name: 'Stale', updatedOn: '2024-01-01T00:00:00Z' },
+            ],
+            badEntities: [],
+            allCurrentEntitiesByFederatedId: {
+                'item-1': { id: 'id-1', federatedId: 'item-1', name: 'Current', updatedOn: '2024-06-01T00:00:00Z' },
+            },
+            typesByPath: {},
+        };
+        const previewData = {
+            entities: [],
+            badRows: [],
+            staleRows: [],
+            warnings: [],
+            constructedEntities: { [types_1.EntityType.COLOR]: entitySet },
+        };
+        const result = await (0, execute_1.execute)(mockClient, [], config, baseAnalysis, undefined, previewData);
+        expect(result.entities[0].skippedStale).toBe(1);
+        expect(result.entities[0].created).toBe(0);
+        expect(result.entities[0].updated).toBe(0);
+        expect(result.skippedRows).toHaveLength(1);
+    });
+    it('should report partial failure when some batches fail', async () => {
+        mockClient.batchCreate
+            .mockResolvedValueOnce({ data: [{ id: 'new-1' }], errors: [] })
+            .mockRejectedValueOnce(new Error('Batch failed'));
+        const entitySet = {
+            incomingEntities: Array.from({ length: 30 }, (_, i) => ({
+                federatedId: `item-${i}`,
+                name: `Item ${i}`,
+            })),
+            badEntities: [],
+            allCurrentEntitiesByFederatedId: {},
+            typesByPath: {},
+        };
+        const previewData = {
+            entities: [],
+            badRows: [],
+            staleRows: [],
+            warnings: [],
+            constructedEntities: { [types_1.EntityType.COLOR]: entitySet },
+        };
+        const result = await (0, execute_1.execute)(mockClient, [], baseConfig, baseAnalysis, undefined, previewData);
+        expect(result.status).toBe('partial-failure');
+        expect(result.entities[0].failures.length).toBeGreaterThan(0);
+    });
+    it('should call onProgress callback', async () => {
+        const progressEvents = [];
+        const entitySet = {
+            incomingEntities: [{ federatedId: 'item-1', name: 'New' }],
+            badEntities: [],
+            allCurrentEntitiesByFederatedId: {},
+            typesByPath: {},
+        };
+        const previewData = {
+            entities: [],
+            badRows: [],
+            staleRows: [],
+            warnings: [],
+            constructedEntities: { [types_1.EntityType.COLOR]: entitySet },
+        };
+        await (0, execute_1.execute)(mockClient, [], baseConfig, baseAnalysis, { onProgress: (p) => progressEvents.push(p) }, previewData);
+        expect(progressEvents.length).toBeGreaterThan(0);
+        expect(progressEvents[0].stage).toContain('execute');
+    });
+    it('should track duration', async () => {
+        const entitySet = {
+            incomingEntities: [],
+            badEntities: [],
+            allCurrentEntitiesByFederatedId: {},
+            typesByPath: {},
+        };
+        const previewData = {
+            entities: [],
+            badRows: [],
+            staleRows: [],
+            warnings: [],
+            constructedEntities: { [types_1.EntityType.COLOR]: entitySet },
+        };
+        const result = await (0, execute_1.execute)(mockClient, [], baseConfig, baseAnalysis, undefined, previewData);
+        expect(typeof result.duration).toBe('number');
+        expect(result.duration).toBeGreaterThanOrEqual(0);
+    });
+    it('should construct entities from scratch when no preview data provided', async () => {
+        entity_processor_v2_1.EntityProcessorV2.setupEntities.mockResolvedValue({
+            incomingEntities: [{ federatedId: 'item-1', name: 'Test' }],
+            badEntities: [],
+            allCurrentEntitiesByFederatedId: {},
+            typesByPath: {},
+            projectId: null,
+            generalWarnings: [],
+        });
+        const rows = [{ federatedId: 'item-1', name: 'Test' }];
+        const result = await (0, execute_1.execute)(mockClient, rows, baseConfig, baseAnalysis);
+        expect(entity_processor_v2_1.EntityProcessorV2.setupEntities).toHaveBeenCalled();
+        expect(result.status).toBe('complete');
+    });
+});

package/node_modules/@contrail/loader/lib/pipeline/index.d.ts

@@ -0,0 +1,7 @@
+export * from './types';
+export * from './loader-pipeline';
+export { transform } from './transform';
+export { validate } from './validate';
+export { preview } from './preview';
+export { execute } from './execute';
+export { executeProjectItemEntities } from './project-item-execute-helpers';

package/node_modules/@contrail/loader/lib/pipeline/index.js

@@ -0,0 +1,29 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.executeProjectItemEntities = exports.execute = exports.preview = exports.validate = exports.transform = void 0;
+__exportStar(require("./types"), exports);
+__exportStar(require("./loader-pipeline"), exports);
+var transform_1 = require("./transform");
+Object.defineProperty(exports, "transform", { enumerable: true, get: function () { return transform_1.transform; } });
+var validate_1 = require("./validate");
+Object.defineProperty(exports, "validate", { enumerable: true, get: function () { return validate_1.validate; } });
+var preview_1 = require("./preview");
+Object.defineProperty(exports, "preview", { enumerable: true, get: function () { return preview_1.preview; } });
+var execute_1 = require("./execute");
+Object.defineProperty(exports, "execute", { enumerable: true, get: function () { return execute_1.execute; } });
+var project_item_execute_helpers_1 = require("./project-item-execute-helpers");
+Object.defineProperty(exports, "executeProjectItemEntities", { enumerable: true, get: function () { return project_item_execute_helpers_1.executeProjectItemEntities; } });

package/node_modules/@contrail/loader/lib/pipeline/item-execute-helpers.d.ts

@@ -0,0 +1,38 @@
+import { EntitiesClient } from '@contrail/entity-types';
+import { TypesByPath } from '../common/entities-helper';
+import { LoaderConfig, EntityExecutionSummary, SkippedRow, ExecuteOptions, ConstructedEntitySet, EntityPreviewSummary } from './types';
+export interface ItemExecutionResult {
+    summaries: EntityExecutionSummary[];
+    skippedRows: SkippedRow[];
+    warnings: string[];
+    hasFailures: boolean;
+}
+export interface ItemPreviewResult {
+    summaries: EntityPreviewSummary[];
+    badRows: SkippedRow[];
+    staleRows: SkippedRow[];
+    warnings: string[];
+    constructedFamilies: {
+        toCreate: any[];
+        toUpdate: any[];
+        unchanged: any[];
+        currentByFedId: Record<string, any>;
+    };
+    constructedOptions: {
+        toCreate: any[];
+        toUpdate: any[];
+        unchanged: any[];
+        skippedUpstream: SkippedRow[];
+    };
+}
+export declare function extractApiErrorMessage(err: any): string;
+export declare function extractFailureId(err: any): string;
+export declare function buildDiffUpdatePayload(incoming: any, existing: any): any;
+export interface SplitItems {
+    families: any[];
+    options: any[];
+    optionToFamilyMap: Map<string, string>;
+}
+export declare function splitIntoFamiliesAndOptions(incomingEntities: any[], typesByPath: TypesByPath): SplitItems;
+export declare function previewItemEntities(incomingEntities: any[], currentByFedId: Record<string, any>, typesByPath: TypesByPath, config: LoaderConfig): ItemPreviewResult;
+export declare function executeItemEntities(client: EntitiesClient, entitySet: ConstructedEntitySet, config: LoaderConfig, options?: ExecuteOptions): Promise<ItemExecutionResult>;