@contrail/cli 3.4.10 → 3.4.11-alpha.11e1d43

package/dist/templates/typescript/package.json

@@ -23,7 +23,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@contrail/app-framework": "^1.2",
+    "@contrail/app-framework": "^2.1",
     "@contrail/sdk": "^1.3",
     "chalk": "^4.0.0",
     "js-yaml": "^4.1.0"

package/node_modules/@contrail/loader/lib/common/chunk-by-url-length.d.ts

@@ -0,0 +1,6 @@
+interface ChunkByUrlLengthOptions {
+    maxBytes?: number;
+    maxCount?: number;
+}
+export declare function chunkByUrlLength(ids: string[], options?: ChunkByUrlLengthOptions): string[][];
+export {};

package/node_modules/@contrail/loader/lib/common/chunk-by-url-length.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.chunkByUrlLength = void 0;
+const constants_1 = require("../constants");
+function wireEncodedLength(id) {
+    return encodeURIComponent(id.replace(/,/g, '_COMMA_')).length;
+}
+function chunkByUrlLength(ids, options = {}) {
+    const maxBytes = options.maxBytes ?? constants_1.FEDERATED_ID_FETCH_MAX_URL_BYTES;
+    const maxCount = options.maxCount ?? constants_1.FEDERATED_ID_FETCH_MAX_COUNT;
+    const result = [];
+    let current = [];
+    let currentBytes = 0;
+    for (const id of ids) {
+        const encodedLength = wireEncodedLength(id);
+        const additionalBytes = current.length === 0 ? encodedLength : encodedLength + 1;
+        const wouldExceedBytes = currentBytes + additionalBytes > maxBytes;
+        const wouldExceedCount = current.length >= maxCount;
+        if (current.length > 0 && (wouldExceedBytes || wouldExceedCount)) {
+            result.push(current);
+            current = [];
+            currentBytes = 0;
+        }
+        current.push(id);
+        currentBytes += current.length === 1 ? encodedLength : encodedLength + 1;
+    }
+    if (current.length > 0)
+        result.push(current);
+    return result;
+}
+exports.chunkByUrlLength = chunkByUrlLength;

package/node_modules/@contrail/loader/lib/common/chunk-by-url-length.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/node_modules/@contrail/loader/lib/common/chunk-by-url-length.spec.js

@@ -0,0 +1,70 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const chunk_by_url_length_1 = require("./chunk-by-url-length");
+describe('chunkByUrlLength', () => {
+    it('returns an empty array for empty input', () => {
+        expect((0, chunk_by_url_length_1.chunkByUrlLength)([])).toEqual([]);
+    });
+    it('packs short ids into a single chunk up to maxCount', () => {
+        const ids = Array.from({ length: 50 }, (_, i) => `id-${i}`);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 10000 });
+        expect(chunks).toHaveLength(1);
+        expect(chunks[0]).toHaveLength(50);
+    });
+    it('splits on the maxCount cap when bytes are not the constraint', () => {
+        const ids = Array.from({ length: 450 }, (_, i) => `id${i}`);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 10000 });
+        expect(chunks.map((c) => c.length)).toEqual([200, 200, 50]);
+    });
+    it('splits on the maxBytes cap when ids are long', () => {
+        const longId = 'A'.repeat(500);
+        const ids = Array.from({ length: 20 }, () => longId);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 2000 });
+        for (const chunk of chunks) {
+            const encoded = chunk.map((id) => encodeURIComponent(id)).join(',');
+            expect(encoded.length).toBeLessThanOrEqual(2000);
+        }
+        expect(chunks.flat()).toHaveLength(20);
+    });
+    it('accounts for URL-encoding expansion (spaces, percent signs, commas)', () => {
+        const id = 'FL63497 Carbon Brushed Cotton Part dyed 60% Cotton_COMMA_ 40% Polyester';
+        const ids = Array.from({ length: 200 }, () => id);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 6000 });
+        expect(chunks.length).toBeGreaterThan(1);
+        for (const chunk of chunks) {
+            const encoded = chunk.map((x) => encodeURIComponent(x)).join(',');
+            expect(encoded.length).toBeLessThanOrEqual(6000);
+        }
+        expect(chunks.flat()).toHaveLength(200);
+    });
+    it('isolates a single id that exceeds maxBytes into its own chunk', () => {
+        const oversized = 'X'.repeat(10000);
+        const ids = ['short-1', 'short-2', oversized, 'short-3'];
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 100 });
+        const oversizedChunk = chunks.find((c) => c.length === 1 && c[0] === oversized);
+        expect(oversizedChunk).toBeDefined();
+        expect(chunks.flat()).toEqual(ids);
+    });
+    it('preserves order across chunks', () => {
+        const ids = Array.from({ length: 123 }, (_, i) => `id-${i}`);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 25, maxBytes: 10000 });
+        expect(chunks.flat()).toEqual(ids);
+    });
+    it('accounts for the SDK\'s `,` → `_COMMA_` substitution', () => {
+        const idWithCommas = 'FOO Cotton, 60% Polyester, 40% Elastane';
+        const ids = Array.from({ length: 100 }, () => idWithCommas);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 1000 });
+        for (const chunk of chunks) {
+            const wireLength = chunk.map((id) => encodeURIComponent(id.replace(/,/g, '_COMMA_'))).join(',').length;
+            expect(wireLength).toBeLessThanOrEqual(1000);
+        }
+        expect(chunks.flat()).toHaveLength(100);
+    });
+    it('defaults to the module-level caps when no options are passed', () => {
+        const ids = Array.from({ length: 500 }, (_, i) => `id-${i}`);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids);
+        expect(chunks[0]).toHaveLength(200);
+        expect(chunks[1]).toHaveLength(200);
+        expect(chunks[2]).toHaveLength(100);
+    });
+});

package/node_modules/@contrail/loader/lib/common/data-transformer-functions.d.ts

@@ -9,6 +9,9 @@ export declare type TypePropertyMaps = {
     typePropertiesBySlug: Record<string, TypeProperty>;
     typePropertiesByLabel: Record<string, TypeProperty>;
     optionSetUsingTypePropertiesBySlug: Record<string, TypePropertyWithLookups>;
+    typePathStateBySlug?: Record<string, Map<string, {
+        isDisabled: boolean;
+    }>>;
 };
 export declare const applyFederatedMappings: PreprocessingFunction;
 export declare const removeUnwantedProperties: PreprocessingFunction;
@@ -20,9 +23,14 @@ export declare const applyConditionalColumns: PreprocessingFunction;
 export declare function getPropertiesBySlugAndLabel(reducedTypes: Partial<Type>[]): {
     typePropertiesBySlug: Record<string, TypeProperty>;
     typePropertiesByLabel: Record<string, TypeProperty>;
+    typePathStateBySlug: Record<string, Map<string, {
+        isDisabled: boolean;
+    }>>;
 };
 export declare function getOptionSetsUsingTypePropertiesBySlug(reducedTypes: Partial<Type>[]): Record<string, TypePropertyWithLookups>;
-export declare function convertFromLabelToSlug(csvJSON: any, typePropertiesBySlug: any, typePropertiesByLabel: any, convertedFields: any): ProcessingIssues[];
+export declare function convertFromLabelToSlug(csvJSON: any, typePropertiesBySlug: any, typePropertiesByLabel: any, convertedFields: any, typePathStateBySlug?: Record<string, Map<string, {
+    isDisabled: boolean;
+}>>): ProcessingIssues[];
 export declare function convertFromDisplayToValue(csvJSON: any[], optionSetUsingTypePropertiesBySlug: any, typePropertySlugs: string[], typePropertyLabels: string[]): ProcessingIssues[];
 export declare function identifyKeysInProperties(csvJSON: any[], typePropertySlugs: string[], typePropertyLabels: string[]): {
     keysInPropertySlugs: Map<string, boolean>;

package/node_modules/@contrail/loader/lib/common/data-transformer-functions.js

@@ -64,7 +64,7 @@ exports.trimValues = trimValues;
 const transformDataBasedOnTypes = function (csvJSON, config, typePropertyMaps, addWarnings = true) {
     const rows = createCopy(csvJSON);
     const { typePropertiesBySlug, typePropertiesByLabel, optionSetUsingTypePropertiesBySlug } = typePropertyMaps;
-    const labelWarnings = convertFromLabelToSlug(rows, typePropertiesBySlug, typePropertiesByLabel, getConvertedFields(config.federatedMappings));
+    const labelWarnings = convertFromLabelToSlug(rows, typePropertiesBySlug, typePropertiesByLabel, getConvertedFields(config.federatedMappings), typePropertyMaps.typePathStateBySlug);
     const displayValueWarnings = convertFromDisplayToValue(rows, optionSetUsingTypePropertiesBySlug, Object.keys(typePropertiesBySlug), Object.keys(typePropertiesByLabel));
     if (!addWarnings) {
         return { rows, errors: [], warnings: [], generalErrors: [] };
@@ -100,15 +100,25 @@ exports.applyConditionalColumns = applyConditionalColumns;
 function getPropertiesBySlugAndLabel(reducedTypes) {
     const typePropertiesBySlug = {};
     const typePropertiesByLabel = {};
+    const typePathStateBySlug = {};
     for (const type of reducedTypes) {
-        if (type.typeProperties) {
-            for (const typeProperty of type.typeProperties) {
-                typePropertiesBySlug[typeProperty.slug.trim()] = typeProperty;
+        if (!type.typeProperties)
+            continue;
+        const typePath = type.typePath;
+        for (const typeProperty of type.typeProperties) {
+            const slug = typeProperty.slug.trim();
+            typePropertiesBySlug[slug] = typeProperty;
+            if (typeProperty.label) {
                 typePropertiesByLabel[typeProperty.label.trim()] = typeProperty;
             }
+            if (typePath) {
+                if (!typePathStateBySlug[slug])
+                    typePathStateBySlug[slug] = new Map();
+                typePathStateBySlug[slug].set(typePath, { isDisabled: !!typeProperty.isArchived });
+            }
         }
     }
-    return { typePropertiesBySlug, typePropertiesByLabel };
+    return { typePropertiesBySlug, typePropertiesByLabel, typePathStateBySlug };
 }
 exports.getPropertiesBySlugAndLabel = getPropertiesBySlugAndLabel;
 function getOptionSetsUsingTypePropertiesBySlug(reducedTypes) {
@@ -146,16 +156,38 @@ function getConvertedFields(federatedMappings) {
     }
     return convertedFields;
 }
-function convertFromLabelToSlug(csvJSON, typePropertiesBySlug, typePropertiesByLabel, convertedFields) {
+function convertFromLabelToSlug(csvJSON, typePropertiesBySlug, typePropertiesByLabel, convertedFields, typePathStateBySlug) {
     const keysToBeIgnored = ['federatedId', 'typePath', 'itemFamilyFederatedId', 'itemOptionFederatedId', 'optionGroup'];
     const keysWithoutProperties = new Set();
     const archivedKeys = new Set();
+    const subtypeAwareArchivedSkips = new Map();
+    const useSubtypeAware = !!typePathStateBySlug;
     const warnings = [];
     for (const row of csvJSON) {
         for (const key of Object.keys(row)) {
             const slugProperty = typePropertiesBySlug[key];
             const labelProperty = typePropertiesByLabel[key];
-            if (slugProperty?.isArchived || labelProperty?.isArchived) {
+            if (useSubtypeAware) {
+                const resolvedSlug = slugProperty?.slug || labelProperty?.slug;
+                const rowTypePath = row.typePath;
+                if (resolvedSlug && rowTypePath && typePathStateBySlug[resolvedSlug]?.get(rowTypePath)?.isDisabled) {
+                    const aggregateKey = `${resolvedSlug}|${rowTypePath}`;
+                    const existing = subtypeAwareArchivedSkips.get(aggregateKey);
+                    if (existing) {
+                        existing.count++;
+                    }
+                    else {
+                        subtypeAwareArchivedSkips.set(aggregateKey, {
+                            slug: resolvedSlug,
+                            typePath: rowTypePath,
+                            columnName: key,
+                            count: 1,
+                        });
+                    }
+                    continue;
+                }
+            }
+            else if (slugProperty?.isArchived || labelProperty?.isArchived) {
                 archivedKeys.add(key);
                 continue;
             }
@@ -181,7 +213,20 @@ function convertFromLabelToSlug(csvJSON, typePropertiesBySlug, typePropertiesByL
             }
         }
     }
-    if (archivedKeys.size > 0) {
+    if (useSubtypeAware) {
+        if (subtypeAwareArchivedSkips.size > 0) {
+            warnings.push({
+                rowNumber: 1,
+                columnIssues: Array.from(subtypeAwareArchivedSkips.values()).map(({ slug, typePath, columnName, count }) => ({
+                    columnName,
+                    message: `Property "${slug}" is disabled for typePath "${typePath}". ${count} value(s) will not be processed.`,
+                    issueType: entity_types_1.IssueType.HEADER_MISMATCH,
+                    data: '',
+                })),
+            });
+        }
+    }
+    else if (archivedKeys.size > 0) {
         warnings.push({
             rowNumber: 1,
             columnIssues: Array.from(archivedKeys).map((key) => ({

package/node_modules/@contrail/loader/lib/common/data-transformer.spec.js

@@ -409,6 +409,110 @@ describe('Data Transformer Functions', () => {
                 expect(csvJSON[0]).toHaveProperty('size');
             });
         });
+        describe('subtype-aware archived handling', () => {
+            it('emits per-(slug, typePath) warnings for slug-headed columns archived on a subset of subtypes', () => {
+                const csvJSON = [
+                    { typePath: 'item:product', season: 'SS25' },
+                    { typePath: 'item:material', season: 'SS25' },
+                ];
+                const typePropertiesBySlug = {
+                    season: { slug: 'season', label: 'Season' },
+                };
+                const typePropertiesByLabel = {
+                    Season: { slug: 'season', label: 'Season' },
+                };
+                const typePathStateBySlug = {
+                    season: new Map([
+                        ['item:product', { isDisabled: false }],
+                        ['item:material', { isDisabled: true }],
+                    ]),
+                };
+                const warnings = (0, data_transformer_functions_1.convertFromLabelToSlug)(csvJSON, typePropertiesBySlug, typePropertiesByLabel, {}, typePathStateBySlug);
+                expect(csvJSON[0].season).toBe('SS25');
+                expect(warnings).toEqual([
+                    {
+                        rowNumber: 1,
+                        columnIssues: [
+                            {
+                                columnName: 'season',
+                                message: 'Property "season" is disabled for typePath "item:material". 1 value(s) will not be processed.',
+                                issueType: entity_types_1.IssueType.HEADER_MISMATCH,
+                                data: '',
+                            },
+                        ],
+                    },
+                ]);
+            });
+            it('renames label-headed columns to slug only on rows whose typePath does not archive the property', () => {
+                const csvJSON = [
+                    { typePath: 'item:product', Season: 'SS25' },
+                    { typePath: 'item:material', Season: 'SS25' },
+                ];
+                const typePropertiesBySlug = {
+                    season: { slug: 'season', label: 'Season' },
+                };
+                const typePropertiesByLabel = {
+                    Season: { slug: 'season', label: 'Season' },
+                };
+                const typePathStateBySlug = {
+                    season: new Map([
+                        ['item:product', { isDisabled: false }],
+                        ['item:material', { isDisabled: true }],
+                    ]),
+                };
+                (0, data_transformer_functions_1.convertFromLabelToSlug)(csvJSON, typePropertiesBySlug, typePropertiesByLabel, {}, typePathStateBySlug);
+                expect(csvJSON[0].season).toBe('SS25');
+                expect(csvJSON[1]).not.toHaveProperty('season');
+            });
+            it('emits a single aggregate warning grouping skips by (slug, typePath)', () => {
+                const csvJSON = [
+                    { typePath: 'item:material', season: 'SS25' },
+                    { typePath: 'item:material', season: 'FW25' },
+                    { typePath: 'item:cement', season: 'SS25' },
+                ];
+                const typePropertiesBySlug = {
+                    season: { slug: 'season', label: 'Season' },
+                };
+                const typePathStateBySlug = {
+                    season: new Map([
+                        ['item:material', { isDisabled: true }],
+                        ['item:cement', { isDisabled: true }],
+                    ]),
+                };
+                const warnings = (0, data_transformer_functions_1.convertFromLabelToSlug)(csvJSON, typePropertiesBySlug, {}, {}, typePathStateBySlug);
+                expect(warnings).toHaveLength(1);
+                expect(warnings[0].columnIssues).toHaveLength(2);
+                expect(warnings[0].columnIssues).toEqual(expect.arrayContaining([
+                    {
+                        columnName: 'season',
+                        message: 'Property "season" is disabled for typePath "item:material". 2 value(s) will not be processed.',
+                        issueType: entity_types_1.IssueType.HEADER_MISMATCH,
+                        data: '',
+                    },
+                    {
+                        columnName: 'season',
+                        message: 'Property "season" is disabled for typePath "item:cement". 1 value(s) will not be processed.',
+                        issueType: entity_types_1.IssueType.HEADER_MISMATCH,
+                        data: '',
+                    },
+                ]));
+            });
+            it('does not emit any archived warning when no row targets an archived subtype', () => {
+                const csvJSON = [{ typePath: 'item:product', season: 'SS25' }];
+                const typePropertiesBySlug = {
+                    season: { slug: 'season', label: 'Season' },
+                };
+                const typePathStateBySlug = {
+                    season: new Map([
+                        ['item:product', { isDisabled: false }],
+                        ['item:material', { isDisabled: true }],
+                    ]),
+                };
+                const warnings = (0, data_transformer_functions_1.convertFromLabelToSlug)(csvJSON, typePropertiesBySlug, {}, {}, typePathStateBySlug);
+                expect(warnings).toEqual([]);
+                expect(csvJSON[0].season).toBe('SS25');
+            });
+        });
     });
     describe('convertFromDisplayToValue', () => {
         it('should convert display values to optionSet values', () => {
@@ -633,6 +737,30 @@ describe('Data Transformer Functions', () => {
             ]);
         });
     });
+    describe('getPropertiesBySlugAndLabel archive tracking', () => {
+        it('records per-typePath archive state per slug', () => {
+            const reducedTypes = [
+                {
+                    typePath: 'item:product',
+                    typeProperties: [
+                        { slug: 'season', label: 'Season', isArchived: false },
+                        { slug: 'sku', label: 'SKU', isArchived: false },
+                    ],
+                },
+                {
+                    typePath: 'item:material',
+                    typeProperties: [
+                        { slug: 'season', label: 'Season', isArchived: true },
+                    ],
+                },
+            ];
+            const result = (0, data_transformer_functions_1.getPropertiesBySlugAndLabel)(reducedTypes);
+            expect(result.typePathStateBySlug.season.get('item:product')).toEqual({ isDisabled: false });
+            expect(result.typePathStateBySlug.season.get('item:material')).toEqual({ isDisabled: true });
+            expect(result.typePathStateBySlug.sku.get('item:product')).toEqual({ isDisabled: false });
+            expect(result.typePathStateBySlug.sku.has('item:material')).toBe(false);
+        });
+    });
     describe('getReducedTypes', () => {
         it('should get reduced types', async () => {
             mockClient.get.mockResolvedValueOnce([

package/node_modules/@contrail/loader/lib/common/entities-helper.js

@@ -5,6 +5,7 @@ const util_1 = require("@contrail/util");
 const types_1 = require("@contrail/types");
 const entity_types_1 = require("@contrail/entity-types");
 const lodash_1 = require("lodash");
+const chunk_by_url_length_1 = require("./chunk-by-url-length");
 const pLimit = require('p-limit');
 var EntitiesLambdaCallType;
 (function (EntitiesLambdaCallType) {
@@ -181,7 +182,7 @@ async function assignObjectReferences(client, typePaths, typesByPath, incomingEn
     const allORByFedId = {};
     for (const rootType of Object.keys(allORFedIdsByType)) {
         const fedIds = [...new Set(allORFedIdsByType[rootType])];
-        const chunks = (0, lodash_1.chunk)(fedIds, 200);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(fedIds);
         const federatedPromises = [];
         for (const chunk of chunks) {
             if (!chunk?.length) {

package/node_modules/@contrail/loader/lib/constants.d.ts

@@ -21,6 +21,8 @@ export declare const ITEM_PROCESSOR_ERRORS: {
     readonly FAMILY_ID_EMPTY: "itemFamilyFederatedId is null or undefined or empty in the CSV file";
     readonly CONFLICTING_IDS: (ids: any) => string;
 };
+export declare const FEDERATED_ID_FETCH_MAX_URL_BYTES = 4000;
+export declare const FEDERATED_ID_FETCH_MAX_COUNT = 200;
 export declare const ITEM_PROCESSOR_WARNINGS: {
     readonly OPTIONNAME_UNMAPPED: "\"optionName\" is not found in the federatedMapping config, conditionalColumns config, or as a header in the CSV file.";
     readonly OPTIONNAME_NOT_FOUND: (optionName: string) => string;

package/node_modules/@contrail/loader/lib/constants.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ITEM_PROCESSOR_WARNINGS = exports.ITEM_PROCESSOR_ERRORS = exports.ENTITY_PROCESSOR_ERRORS = exports.CUSTOM_ENTITY_PROCESSOR_ERRORS = exports.COLOR_PROCESSOR_WARNINGS = exports.COLOR_PROCESSOR_ERRORS = void 0;
+exports.ITEM_PROCESSOR_WARNINGS = exports.FEDERATED_ID_FETCH_MAX_COUNT = exports.FEDERATED_ID_FETCH_MAX_URL_BYTES = exports.ITEM_PROCESSOR_ERRORS = exports.ENTITY_PROCESSOR_ERRORS = exports.CUSTOM_ENTITY_PROCESSOR_ERRORS = exports.COLOR_PROCESSOR_WARNINGS = exports.COLOR_PROCESSOR_ERRORS = void 0;
 exports.COLOR_PROCESSOR_ERRORS = {
     TYPEPATH_MISSING: '"typePath" with value "color" is required property but is not found in the federatedMapping config, conditionalColumns config, or as a header in the CSV file. Please add a mapping "typePath" mapping with value "color" with one of these methods, then try the load again.',
 };
@@ -24,6 +24,8 @@ exports.ITEM_PROCESSOR_ERRORS = {
     FAMILY_ID_EMPTY: 'itemFamilyFederatedId is null or undefined or empty in the CSV file',
     CONFLICTING_IDS: (ids) => `Found conflicting family federated Ids: ${ids}`,
 };
+exports.FEDERATED_ID_FETCH_MAX_URL_BYTES = 4000;
+exports.FEDERATED_ID_FETCH_MAX_COUNT = 200;
 exports.ITEM_PROCESSOR_WARNINGS = {
     OPTIONNAME_UNMAPPED: '"optionName" is not found in the federatedMapping config, conditionalColumns config, or as a header in the CSV file.',
     OPTIONNAME_NOT_FOUND: (optionName) => `"${optionName}" column not found in the CSV file`,

package/node_modules/@contrail/loader/lib/entities-v2/entities-helper.js

@@ -4,6 +4,7 @@ exports.getPropertyVariants = exports.assignUserLists = exports.getUsersInOrg =
 const util_1 = require("@contrail/util");
 const types_1 = require("@contrail/types");
 const lodash_1 = require("lodash");
+const chunk_by_url_length_1 = require("../common/chunk-by-url-length");
 const entity_types_1 = require("@contrail/entity-types");
 const pLimit = require('p-limit');
 var EntitiesLambdaCallType;
@@ -176,7 +177,7 @@ async function assignObjectReferences(client, typePaths, typesByPath, incomingEn
     const allORByFedId = {};
     for (const rootType of Object.keys(allORFedIdsByType)) {
         const fedIds = [...new Set(allORFedIdsByType[rootType])];
-        const chunks = (0, lodash_1.chunk)(fedIds, 200);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(fedIds);
         const federatedPromises = [];
         for (const chunk of chunks) {
             if (!chunk?.length) {

package/node_modules/@contrail/loader/lib/entities-v2/entity-processor-v2.js

@@ -4,7 +4,7 @@ exports.EntityProcessorV2 = void 0;
 const entities_helper_1 = require("../common/entities-helper");
 const entity_builder_helpers_1 = require("../common/entity-builder-helpers");
 const processor_registry_1 = require("../processor-registry");
-const lodash_1 = require("lodash");
+const chunk_by_url_length_1 = require("../common/chunk-by-url-length");
 const entity_types_1 = require("@contrail/entity-types");
 const type_helper_1 = require("../common/type-helper");
 const constants_1 = require("../constants");
@@ -157,7 +157,7 @@ class EntityProcessorV2 {
         return uniqueFedIds;
     }
     static async fetchCurrentEntities(client, entityType, federatedIds) {
-        const chunks = (0, lodash_1.chunk)(federatedIds, 200);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(federatedIds);
         const allCurrentEntitiesByFederatedId = {};
         const federatedPromises = [];
         for (const chunk of chunks) {

package/node_modules/@contrail/loader/lib/entities-v2/entity-processor-v2.spec.js

@@ -6,7 +6,7 @@ const entity_builder_helpers_1 = require("../common/entity-builder-helpers");
 const types_1 = require("@contrail/types");
 const entity_types_1 = require("@contrail/entity-types");
 const processor_registry_1 = require("../processor-registry");
-const lodash_1 = require("lodash");
+const chunk_by_url_length_1 = require("../common/chunk-by-url-length");
 const util_1 = require("../common/util");
 const color_processor_v2_1 = require("./colors-v2/color-processor-v2");
 const constants_1 = require("../constants");
@@ -15,9 +15,7 @@ jest.mock('../processor-registry');
 jest.mock('@contrail/entity-types');
 jest.mock('../common/entities-helper');
 jest.mock('../common/type-helper');
-jest.mock('lodash', () => ({
-    chunk: jest.fn(),
-}));
+jest.mock('../common/chunk-by-url-length');
 describe('EntityProcessorV2', () => {
     const mockTypesByPath = {
         color: {
@@ -217,7 +215,7 @@ describe('EntityProcessorV2', () => {
                 ['1', '2'],
                 ['3', '4'],
             ];
-            lodash_1.chunk.mockReturnValue(chunks);
+            chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
             const mockEntities = [
                 { federatedId: '1', name: 'Entity 1' },
                 { federatedId: '2', name: 'Entity 2' },
@@ -228,12 +226,12 @@ describe('EntityProcessorV2', () => {
                 '1': { federatedId: '1', name: 'Entity 1' },
                 '2': { federatedId: '2', name: 'Entity 2' },
             });
-            expect(lodash_1.chunk).toHaveBeenCalledWith(federatedIds, 200);
+            expect(chunk_by_url_length_1.chunkByUrlLength).toHaveBeenCalledWith(federatedIds);
             expect(entities_helper_1.retryOnTimeout).toHaveBeenCalledTimes(2);
         });
         it('should handle empty federatedIds array', async () => {
             const federatedIds = [];
-            lodash_1.chunk.mockReturnValue([]);
+            chunk_by_url_length_1.chunkByUrlLength.mockReturnValue([]);
             const result = await entity_processor_v2_1.EntityProcessorV2.fetchCurrentEntities(client, entity_types_1.EntityType.COLOR, federatedIds);
             expect(result).toEqual({});
             expect(entities_helper_1.retryOnTimeout).not.toHaveBeenCalled();
@@ -241,7 +239,7 @@ describe('EntityProcessorV2', () => {
         it('should handle API errors and rethrow', async () => {
             const federatedIds = ['1'];
             const chunks = [['1']];
-            lodash_1.chunk.mockReturnValue(chunks);
+            chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
             const error = new Error('API Error');
             entities_helper_1.retryOnTimeout.mockRejectedValue(error);
             await expect(entity_processor_v2_1.EntityProcessorV2.fetchCurrentEntities(client, entity_types_1.EntityType.COLOR, federatedIds)).rejects.toThrow('API Error');
@@ -250,7 +248,7 @@ describe('EntityProcessorV2', () => {
         it('should handle empty response from API', async () => {
             const federatedIds = ['1'];
             const chunks = [['1']];
-            lodash_1.chunk.mockReturnValue(chunks);
+            chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
             entities_helper_1.retryOnTimeout.mockResolvedValue([]);
             const result = await entity_processor_v2_1.EntityProcessorV2.fetchCurrentEntities(client, entity_types_1.EntityType.COLOR, federatedIds);
             expect(result).toEqual({});
@@ -261,7 +259,7 @@ describe('EntityProcessorV2', () => {
                 ['1', '2'],
                 ['3', '4'],
             ];
-            lodash_1.chunk.mockReturnValue(chunks);
+            chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
             const mockEntities1 = [
                 { federatedId: '1', name: 'Entity 1' },
                 { federatedId: '2', name: 'Entity 2' },

package/node_modules/@contrail/loader/lib/entities-v2/file-preprocessor-v2.js

@@ -188,12 +188,13 @@ class LoaderDataPreprocessorV2 {
         };
     }
     static async runPreprocessingPipeline(client, loaderProcess, reducedTypes, processedData, preprocessingStepResults, transformationErrors, transformationWarnings, systemErrors) {
-        const { typePropertiesBySlug, typePropertiesByLabel } = (0, data_transformer_functions_1.getPropertiesBySlugAndLabel)(reducedTypes);
+        const { typePropertiesBySlug, typePropertiesByLabel, typePathStateBySlug } = (0, data_transformer_functions_1.getPropertiesBySlugAndLabel)(reducedTypes);
         const optionSetUsingTypePropertiesBySlug = (0, data_transformer_functions_1.getOptionSetsUsingTypePropertiesBySlug)(reducedTypes);
         const typePropertyMaps = {
             typePropertiesBySlug,
             typePropertiesByLabel,
             optionSetUsingTypePropertiesBySlug,
+            typePathStateBySlug,
         };
         for (const preprocessingStep of this.getPreprocessingPipeline()) {
             try {

package/node_modules/@contrail/loader/lib/entities-v2/linesheet-v2/item-helper.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getAssortment = exports.convertToComparableEntities = exports.mapAssortmentItemsToItemsAndFillItemId = exports.getItemsFromFederatedIds = exports.getAssortmentItemProperties = exports.getAssortmentItems = void 0;
 const lodash_1 = require("lodash");
+const chunk_by_url_length_1 = require("../../common/chunk-by-url-length");
 const entities_helper_1 = require("../../common/entities-helper");
 const pLimit = require('p-limit');
 const limit = pLimit(5);
@@ -58,7 +59,7 @@ function mapTypePropertiesToValueType(typeProperties) {
 }
 async function getItemsFromFederatedIds(client, incomingRows) {
     const federatedIds = extractUniqueFederatedIds(incomingRows);
-    const chunks = (0, lodash_1.chunk)(federatedIds, 200);
+    const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(federatedIds);
     const allCurrentItemsByFederatedId = await fetchItemsByFederatedIds(client, chunks);
     return allCurrentItemsByFederatedId;
 }

package/node_modules/@contrail/loader/lib/entity-processor.js

@@ -4,7 +4,7 @@ exports.EntityProcessor = void 0;
 const entities_helper_1 = require("./common/entities-helper");
 const entity_builder_helpers_1 = require("./common/entity-builder-helpers");
 const processor_registry_1 = require("./processor-registry");
-const lodash_1 = require("lodash");
+const chunk_by_url_length_1 = require("./common/chunk-by-url-length");
 const type_helper_1 = require("./common/type-helper");
 const entity_types_1 = require("@contrail/entity-types");
 const util_1 = require("./common/util");
@@ -104,7 +104,7 @@ class EntityProcessor {
         return [...uniqueFedIds, ...nonUniqueFedIds];
     }
     static async fetchCurrentEntities(client, entityType, federatedIds) {
-        const chunks = (0, lodash_1.chunk)(federatedIds, 200);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(federatedIds);
         const allCurrentEntitiesByFederatedId = {};
         const federatedPromises = [];
         for (const chunk of chunks) {