@contrail/cli 3.4.10 → 3.4.11-alpha.0f34b47

package/dist/commands/types/uploadTypePropertyPolicies.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.findAllOccurrences = exports.uploadTypePropertyPolicies = void 0;
+exports.uploadTypePropertyPolicies = void 0;
 const tslib_1 = require("tslib");
 const sdk_1 = require("@contrail/sdk");
 const fs_1 = tslib_1.__importDefault(require("fs"));
@@ -8,12 +8,10 @@ const path = tslib_1.__importStar(require("path"));
 const login_1 = require("../../auth/login");
 const chalk_1 = tslib_1.__importDefault(require("chalk"));
 const core_1 = require("@oclif/core");
-const type_extractor_1 = require("../../helpers/types/type-extractor");
 const users_1 = require("../../api/users");
 const basecommand_1 = require("../../basecommand");
 const logger_1 = tslib_1.__importDefault(require("../../logger"));
 const js_yaml_1 = tslib_1.__importDefault(require("js-yaml"));
-const _ = tslib_1.__importStar(require("lodash"));
 class UploadTypePropertyPolicies extends basecommand_1.BaseCommand {
     static description = 'Upload type property policies';
     static examples = [
@@ -38,14 +36,12 @@ async function uploadTypePropertyPolicies(dir) {
     try {
         await (0, login_1.authUser)();
         logger_1.default.info(`${chalk_1.default.yellow.bold('TYPE PROPERTY POLICY')} Uploading`);
-        const [reducedTypePropertyPolicies, reducedTypePropertyPolicyLinks] = await loadTPPolicies(dir);
+        const reducedTypePropertyPolicies = await loadTPPolicies(dir);
         await uploadTPPolicyObjects(reducedTypePropertyPolicies);
-        await uploadTypePropertyPolicyLinks(reducedTypePropertyPolicyLinks);
         logger_1.default.info(`${chalk_1.default.yellow.bold('TYPE PROPERTY POLICY')} Uploaded`);
     }
     catch (e) {
         logger_1.default.info(`${chalk_1.default.red.bold('FAIL')} Type Property Policy Upload Failed . ${e}`);
-        //TODO make this only print if a debug flag is set
         console.trace(e);
         return;
     }
@@ -54,15 +50,13 @@ async function uploadTypePropertyPolicies(dir) {
 exports.uploadTypePropertyPolicies = uploadTypePropertyPolicies;
 function loadTPPolicies(fromDir) {
     const tpps = [];
-    let tppLinks = [];
     const dir = path.join(fromDir, 'type-property-policies');
     try {
         const files = fs_1.default.readdirSync(dir);
         for (const file of files) {
             tpps.push(js_yaml_1.default.load(fs_1.default.readFileSync(path.join(dir, file), 'utf8')));
         }
-        tppLinks = js_yaml_1.default.load(fs_1.default.readFileSync(path.join(fromDir, 'type-property-policy-links.yml'), 'utf8'));
-        return [tpps, tppLinks];
+        return tpps;
     }
     catch (e) {
         throw new Error(`ERROR PARSING YAML: ${e}`);
@@ -155,133 +149,3 @@ async function uploadTPPolicyObjects(reducedTypePropertyPolicies) {
         }
     }
 }
-async function uploadTypePropertyPolicyLinks(reducedTypePropertyPolicyLinks) {
-    const typesWithProperties = await (0, type_extractor_1.getTypes)();
-    const typeProperties = [];
-    typesWithProperties.forEach((type) => {
-        if (type.typeProperties) {
-            for (const tp of type.typeProperties) {
-                tp.typeId = type.id;
-                tp.typeSlug = type.slug;
-                typeProperties.push(tp);
-            }
-        }
-    });
-    const currentTPPs = await new sdk_1.Entities().get({
-        entityName: 'type-property-policy',
-    });
-    const currentTPPLinks = await new sdk_1.Entities().get({
-        entityName: 'type-property-policy-link',
-    });
-    logger_1.default.info(`found ${currentTPPLinks.length} current type property policy links`);
-    logger_1.default.info(`found ${reducedTypePropertyPolicyLinks.length} type property policy links to load`);
-    const currentLinksHashes = {};
-    for (const currentLink of currentTPPLinks) {
-        const foundTPs = findAllOccurrences(typeProperties, (tp) => tp.id === currentLink.typePropertyId);
-        for (const foundTP of foundTPs) {
-            const foundTPP = currentTPPs.find((tpp) => tpp.id === currentLink.typePropertyPolicyId);
-            if (!foundTP || !foundTPP) {
-                logger_1.default.info(`skipping unhashable, invalid policy link: ${JSON.stringify(currentLink, null, 2)}`);
-                continue;
-            }
-            const hash = `${foundTP.typeSlug}|${foundTP.slug}|${foundTPP.identifier}`;
-            currentLinksHashes[hash] = currentLink;
-        }
-    }
-    logger_1.default.debug(`current links hashes: ${JSON.stringify(currentLinksHashes, null, 2)}`);
-    const reducedLinksByHashes = {};
-    for (const loadedLink of reducedTypePropertyPolicyLinks) {
-        const hash = `${loadedLink.typeSlug}|${loadedLink.typePropertySlug}|${loadedLink.typePropertyPolicyIdentifier}`;
-        reducedLinksByHashes[hash] = loadedLink;
-    }
-    const toCreate = Object.keys(reducedLinksByHashes)
-        .filter((x) => !currentLinksHashes[x])
-        .map((x) => {
-        return {
-            [x]: reducedLinksByHashes[x],
-        };
-    })
-        .reduce((a, b) => Object.assign(a, b), {});
-    const toDelete = _.uniqBy(Object.keys(currentLinksHashes)
-        .filter((x) => !reducedLinksByHashes[x])
-        .map((x) => {
-        return {
-            [x]: currentLinksHashes[x],
-        };
-    }), 'id').reduce((a, b) => Object.assign(a, b), {});
-    logger_1.default.debug(`reduced links hashes: ${JSON.stringify(reducedLinksByHashes, null, 2)}`);
-    logger_1.default.info(`found ${Object.keys(toCreate).length} type property policy links to create`);
-    logger_1.default.info(`found ${Object.keys(toDelete).length} type property policy links to delete`);
-    logger_1.default.debug(`to create: ${JSON.stringify(toCreate, null, 2)}`);
-    logger_1.default.debug(`to delete: ${JSON.stringify(toDelete, null, 2)}`);
-    logger_1.default.debug(`current links hashes to delete: ${JSON.stringify(Object.keys(currentLinksHashes).filter((x) => !reducedLinksByHashes[x]), null, 2)}`);
-    // fs.writeFileSync('toCreate.json', JSON.stringify(toCreate, null, 2));
-    // fs.writeFileSync('toDelete.json', JSON.stringify(toDelete, null, 2));
-    // fs.writeFileSync('currentLinksHashes.json', JSON.stringify(currentLinksHashes, null, 2));
-    // fs.writeFileSync('reducedLinksByHashes.json', JSON.stringify(reducedLinksByHashes, null, 2));
-    // fs.writeFileSync(
-    //   'current_link_hashes_to_delete.json',
-    //   JSON.stringify(
-    //     Object.keys(currentLinksHashes).filter((x) => !reducedLinksByHashes[x]),
-    //     null,
-    //     2,
-    //   ),
-    // );
-    // return;
-    for (const [hash, link] of Object.entries(toCreate)) {
-        try {
-            if (!transformReducedLink(link, typeProperties, currentTPPs)) {
-                continue;
-            }
-            logger_1.default.info(`Creating: ${hash}: ${JSON.stringify(link, null, 2)}`);
-            await new sdk_1.Entities().create({
-                entityName: 'type-property-policy-link',
-                object: {
-                    typePropertyPolicyId: link.typePropertyPolicyId,
-                    typePropertyId: link.typePropertyId,
-                },
-            });
-        }
-        catch (err) {
-            logger_1.default.error(`Error Uploading type property policy links: ${err}`);
-        }
-    }
-    for (const [hash, link] of Object.entries(toDelete)) {
-        try {
-            logger_1.default.info(`Deleting: ${hash}`);
-            await new sdk_1.Entities().delete({
-                entityName: 'type-property-policy-link',
-                id: link['id'],
-            });
-        }
-        catch (err) {
-            logger_1.default.error(`Error Deleting type property policy link: ${err}`);
-        }
-    }
-    function transformReducedLink(TPPLink, typeProperties, typePropertyPolicies) {
-        const foundTP = typeProperties.find((tp) => tp.slug === TPPLink.typePropertySlug && tp.typeSlug === TPPLink.typeSlug);
-        if (!foundTP) {
-            logger_1.default.error(`Could not find type property with slug ${TPPLink.typePropertySlug} `);
-            return false;
-        }
-        TPPLink.typePropertyId = foundTP.id;
-        const foundTPP = typePropertyPolicies.find((tpp) => tpp.identifier === TPPLink.typePropertyPolicyIdentifier);
-        if (!foundTPP) {
-            logger_1.default.error(`Could not find type property policy with identifier ${TPPLink.typePropertyPolicyIdentifier} `);
-            return false;
-        }
-        TPPLink.typePropertyPolicyId = foundTPP.id;
-        return true;
-    }
-}
-function findAllOccurrences(array, evaluator) {
-    const found = [];
-    for (let i = 0; i < array.length; i++) {
-        const element = array[i];
-        if (evaluator(element)) {
-            found.push(element);
-        }
-    }
-    return found;
-}
-exports.findAllOccurrences = findAllOccurrences;

package/dist/commands/types/uploadUsageInfo.d.ts

@@ -2,6 +2,9 @@ import { BaseCommand } from '../../basecommand';
 import { PersistReducedUsageInfo } from '../../interfaces';
 export default class UploadUsageInfo extends BaseCommand {
     static description: string;
+    static deprecationOptions: {
+        message: string;
+    };
     static examples: string[];
     static flags: {
         dir: import("@oclif/core/lib/interfaces").OptionFlag<string, import("@oclif/core/lib/interfaces/parser").CustomOptions>;

package/dist/commands/types/uploadUsageInfo.js

@@ -13,7 +13,10 @@ const basecommand_1 = require("../../basecommand");
 const logger_1 = tslib_1.__importDefault(require("../../logger"));
 const js_yaml_1 = tslib_1.__importDefault(require("js-yaml"));
 class UploadUsageInfo extends basecommand_1.BaseCommand {
-    static description = 'Upload extracted usage info';
+    static description = '[DEPRECATED] Upload extracted usage info. Use uploadTypeOverrides instead.';
+    static deprecationOptions = {
+        message: 'uploadUsageInfo is deprecated. Usage info is now managed as type overrides. Use `contrail types uploadTypeOverrides` instead.',
+    };
     static examples = [
         `$ contrail types uploadUsageInfo
 `,

package/node_modules/@contrail/loader/lib/common/chunk-by-url-length.d.ts

@@ -0,0 +1,6 @@
+interface ChunkByUrlLengthOptions {
+    maxBytes?: number;
+    maxCount?: number;
+}
+export declare function chunkByUrlLength(ids: string[], options?: ChunkByUrlLengthOptions): string[][];
+export {};

package/node_modules/@contrail/loader/lib/common/chunk-by-url-length.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.chunkByUrlLength = void 0;
+const constants_1 = require("../constants");
+function wireEncodedLength(id) {
+    return encodeURIComponent(id.replace(/,/g, '_COMMA_')).length;
+}
+function chunkByUrlLength(ids, options = {}) {
+    const maxBytes = options.maxBytes ?? constants_1.FEDERATED_ID_FETCH_MAX_URL_BYTES;
+    const maxCount = options.maxCount ?? constants_1.FEDERATED_ID_FETCH_MAX_COUNT;
+    const result = [];
+    let current = [];
+    let currentBytes = 0;
+    for (const id of ids) {
+        const encodedLength = wireEncodedLength(id);
+        const additionalBytes = current.length === 0 ? encodedLength : encodedLength + 1;
+        const wouldExceedBytes = currentBytes + additionalBytes > maxBytes;
+        const wouldExceedCount = current.length >= maxCount;
+        if (current.length > 0 && (wouldExceedBytes || wouldExceedCount)) {
+            result.push(current);
+            current = [];
+            currentBytes = 0;
+        }
+        current.push(id);
+        currentBytes += current.length === 1 ? encodedLength : encodedLength + 1;
+    }
+    if (current.length > 0)
+        result.push(current);
+    return result;
+}
+exports.chunkByUrlLength = chunkByUrlLength;

package/node_modules/@contrail/loader/lib/common/chunk-by-url-length.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/node_modules/@contrail/loader/lib/common/chunk-by-url-length.spec.js

@@ -0,0 +1,70 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const chunk_by_url_length_1 = require("./chunk-by-url-length");
+describe('chunkByUrlLength', () => {
+    it('returns an empty array for empty input', () => {
+        expect((0, chunk_by_url_length_1.chunkByUrlLength)([])).toEqual([]);
+    });
+    it('packs short ids into a single chunk up to maxCount', () => {
+        const ids = Array.from({ length: 50 }, (_, i) => `id-${i}`);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 10000 });
+        expect(chunks).toHaveLength(1);
+        expect(chunks[0]).toHaveLength(50);
+    });
+    it('splits on the maxCount cap when bytes are not the constraint', () => {
+        const ids = Array.from({ length: 450 }, (_, i) => `id${i}`);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 10000 });
+        expect(chunks.map((c) => c.length)).toEqual([200, 200, 50]);
+    });
+    it('splits on the maxBytes cap when ids are long', () => {
+        const longId = 'A'.repeat(500);
+        const ids = Array.from({ length: 20 }, () => longId);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 2000 });
+        for (const chunk of chunks) {
+            const encoded = chunk.map((id) => encodeURIComponent(id)).join(',');
+            expect(encoded.length).toBeLessThanOrEqual(2000);
+        }
+        expect(chunks.flat()).toHaveLength(20);
+    });
+    it('accounts for URL-encoding expansion (spaces, percent signs, commas)', () => {
+        const id = 'FL63497 Carbon Brushed Cotton Part dyed 60% Cotton_COMMA_ 40% Polyester';
+        const ids = Array.from({ length: 200 }, () => id);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 6000 });
+        expect(chunks.length).toBeGreaterThan(1);
+        for (const chunk of chunks) {
+            const encoded = chunk.map((x) => encodeURIComponent(x)).join(',');
+            expect(encoded.length).toBeLessThanOrEqual(6000);
+        }
+        expect(chunks.flat()).toHaveLength(200);
+    });
+    it('isolates a single id that exceeds maxBytes into its own chunk', () => {
+        const oversized = 'X'.repeat(10000);
+        const ids = ['short-1', 'short-2', oversized, 'short-3'];
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 100 });
+        const oversizedChunk = chunks.find((c) => c.length === 1 && c[0] === oversized);
+        expect(oversizedChunk).toBeDefined();
+        expect(chunks.flat()).toEqual(ids);
+    });
+    it('preserves order across chunks', () => {
+        const ids = Array.from({ length: 123 }, (_, i) => `id-${i}`);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 25, maxBytes: 10000 });
+        expect(chunks.flat()).toEqual(ids);
+    });
+    it('accounts for the SDK\'s `,` → `_COMMA_` substitution', () => {
+        const idWithCommas = 'FOO Cotton, 60% Polyester, 40% Elastane';
+        const ids = Array.from({ length: 100 }, () => idWithCommas);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 1000 });
+        for (const chunk of chunks) {
+            const wireLength = chunk.map((id) => encodeURIComponent(id.replace(/,/g, '_COMMA_'))).join(',').length;
+            expect(wireLength).toBeLessThanOrEqual(1000);
+        }
+        expect(chunks.flat()).toHaveLength(100);
+    });
+    it('defaults to the module-level caps when no options are passed', () => {
+        const ids = Array.from({ length: 500 }, (_, i) => `id-${i}`);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids);
+        expect(chunks[0]).toHaveLength(200);
+        expect(chunks[1]).toHaveLength(200);
+        expect(chunks[2]).toHaveLength(100);
+    });
+});

package/node_modules/@contrail/loader/lib/common/data-transformer-functions.d.ts

@@ -9,6 +9,9 @@ export declare type TypePropertyMaps = {
     typePropertiesBySlug: Record<string, TypeProperty>;
     typePropertiesByLabel: Record<string, TypeProperty>;
     optionSetUsingTypePropertiesBySlug: Record<string, TypePropertyWithLookups>;
+    typePathStateBySlug?: Record<string, Map<string, {
+        isDisabled: boolean;
+    }>>;
 };
 export declare const applyFederatedMappings: PreprocessingFunction;
 export declare const removeUnwantedProperties: PreprocessingFunction;
@@ -20,9 +23,14 @@ export declare const applyConditionalColumns: PreprocessingFunction;
 export declare function getPropertiesBySlugAndLabel(reducedTypes: Partial<Type>[]): {
     typePropertiesBySlug: Record<string, TypeProperty>;
     typePropertiesByLabel: Record<string, TypeProperty>;
+    typePathStateBySlug: Record<string, Map<string, {
+        isDisabled: boolean;
+    }>>;
 };
 export declare function getOptionSetsUsingTypePropertiesBySlug(reducedTypes: Partial<Type>[]): Record<string, TypePropertyWithLookups>;
-export declare function convertFromLabelToSlug(csvJSON: any, typePropertiesBySlug: any, typePropertiesByLabel: any, convertedFields: any): ProcessingIssues[];
+export declare function convertFromLabelToSlug(csvJSON: any, typePropertiesBySlug: any, typePropertiesByLabel: any, convertedFields: any, typePathStateBySlug?: Record<string, Map<string, {
+    isDisabled: boolean;
+}>>): ProcessingIssues[];
 export declare function convertFromDisplayToValue(csvJSON: any[], optionSetUsingTypePropertiesBySlug: any, typePropertySlugs: string[], typePropertyLabels: string[]): ProcessingIssues[];
 export declare function identifyKeysInProperties(csvJSON: any[], typePropertySlugs: string[], typePropertyLabels: string[]): {
     keysInPropertySlugs: Map<string, boolean>;

package/node_modules/@contrail/loader/lib/common/data-transformer-functions.js

@@ -64,7 +64,7 @@ exports.trimValues = trimValues;
 const transformDataBasedOnTypes = function (csvJSON, config, typePropertyMaps, addWarnings = true) {
     const rows = createCopy(csvJSON);
     const { typePropertiesBySlug, typePropertiesByLabel, optionSetUsingTypePropertiesBySlug } = typePropertyMaps;
-    const labelWarnings = convertFromLabelToSlug(rows, typePropertiesBySlug, typePropertiesByLabel, getConvertedFields(config.federatedMappings));
+    const labelWarnings = convertFromLabelToSlug(rows, typePropertiesBySlug, typePropertiesByLabel, getConvertedFields(config.federatedMappings), typePropertyMaps.typePathStateBySlug);
     const displayValueWarnings = convertFromDisplayToValue(rows, optionSetUsingTypePropertiesBySlug, Object.keys(typePropertiesBySlug), Object.keys(typePropertiesByLabel));
     if (!addWarnings) {
         return { rows, errors: [], warnings: [], generalErrors: [] };
@@ -100,15 +100,25 @@ exports.applyConditionalColumns = applyConditionalColumns;
 function getPropertiesBySlugAndLabel(reducedTypes) {
     const typePropertiesBySlug = {};
     const typePropertiesByLabel = {};
+    const typePathStateBySlug = {};
     for (const type of reducedTypes) {
-        if (type.typeProperties) {
-            for (const typeProperty of type.typeProperties) {
-                typePropertiesBySlug[typeProperty.slug.trim()] = typeProperty;
+        if (!type.typeProperties)
+            continue;
+        const typePath = type.typePath;
+        for (const typeProperty of type.typeProperties) {
+            const slug = typeProperty.slug.trim();
+            typePropertiesBySlug[slug] = typeProperty;
+            if (typeProperty.label) {
                 typePropertiesByLabel[typeProperty.label.trim()] = typeProperty;
             }
+            if (typePath) {
+                if (!typePathStateBySlug[slug])
+                    typePathStateBySlug[slug] = new Map();
+                typePathStateBySlug[slug].set(typePath, { isDisabled: !!typeProperty.isArchived });
+            }
         }
     }
-    return { typePropertiesBySlug, typePropertiesByLabel };
+    return { typePropertiesBySlug, typePropertiesByLabel, typePathStateBySlug };
 }
 exports.getPropertiesBySlugAndLabel = getPropertiesBySlugAndLabel;
 function getOptionSetsUsingTypePropertiesBySlug(reducedTypes) {
@@ -146,16 +156,38 @@ function getConvertedFields(federatedMappings) {
     }
     return convertedFields;
 }
-function convertFromLabelToSlug(csvJSON, typePropertiesBySlug, typePropertiesByLabel, convertedFields) {
+function convertFromLabelToSlug(csvJSON, typePropertiesBySlug, typePropertiesByLabel, convertedFields, typePathStateBySlug) {
     const keysToBeIgnored = ['federatedId', 'typePath', 'itemFamilyFederatedId', 'itemOptionFederatedId', 'optionGroup'];
     const keysWithoutProperties = new Set();
     const archivedKeys = new Set();
+    const subtypeAwareArchivedSkips = new Map();
+    const useSubtypeAware = !!typePathStateBySlug;
     const warnings = [];
     for (const row of csvJSON) {
         for (const key of Object.keys(row)) {
             const slugProperty = typePropertiesBySlug[key];
             const labelProperty = typePropertiesByLabel[key];
-            if (slugProperty?.isArchived || labelProperty?.isArchived) {
+            if (useSubtypeAware) {
+                const resolvedSlug = slugProperty?.slug || labelProperty?.slug;
+                const rowTypePath = row.typePath;
+                if (resolvedSlug && rowTypePath && typePathStateBySlug[resolvedSlug]?.get(rowTypePath)?.isDisabled) {
+                    const aggregateKey = `${resolvedSlug}|${rowTypePath}`;
+                    const existing = subtypeAwareArchivedSkips.get(aggregateKey);
+                    if (existing) {
+                        existing.count++;
+                    }
+                    else {
+                        subtypeAwareArchivedSkips.set(aggregateKey, {
+                            slug: resolvedSlug,
+                            typePath: rowTypePath,
+                            columnName: key,
+                            count: 1,
+                        });
+                    }
+                    continue;
+                }
+            }
+            else if (slugProperty?.isArchived || labelProperty?.isArchived) {
                 archivedKeys.add(key);
                 continue;
             }
@@ -181,7 +213,20 @@ function convertFromLabelToSlug(csvJSON, typePropertiesBySlug, typePropertiesByL
             }
         }
     }
-    if (archivedKeys.size > 0) {
+    if (useSubtypeAware) {
+        if (subtypeAwareArchivedSkips.size > 0) {
+            warnings.push({
+                rowNumber: 1,
+                columnIssues: Array.from(subtypeAwareArchivedSkips.values()).map(({ slug, typePath, columnName, count }) => ({
+                    columnName,
+                    message: `Property "${slug}" is disabled for typePath "${typePath}". ${count} value(s) will not be processed.`,
+                    issueType: entity_types_1.IssueType.HEADER_MISMATCH,
+                    data: '',
+                })),
+            });
+        }
+    }
+    else if (archivedKeys.size > 0) {
         warnings.push({
             rowNumber: 1,
             columnIssues: Array.from(archivedKeys).map((key) => ({

package/node_modules/@contrail/loader/lib/common/data-transformer.spec.js

@@ -409,6 +409,110 @@ describe('Data Transformer Functions', () => {
                 expect(csvJSON[0]).toHaveProperty('size');
             });
         });
+        describe('subtype-aware archived handling', () => {
+            it('emits per-(slug, typePath) warnings for slug-headed columns archived on a subset of subtypes', () => {
+                const csvJSON = [
+                    { typePath: 'item:product', season: 'SS25' },
+                    { typePath: 'item:material', season: 'SS25' },
+                ];
+                const typePropertiesBySlug = {
+                    season: { slug: 'season', label: 'Season' },
+                };
+                const typePropertiesByLabel = {
+                    Season: { slug: 'season', label: 'Season' },
+                };
+                const typePathStateBySlug = {
+                    season: new Map([
+                        ['item:product', { isDisabled: false }],
+                        ['item:material', { isDisabled: true }],
+                    ]),
+                };
+                const warnings = (0, data_transformer_functions_1.convertFromLabelToSlug)(csvJSON, typePropertiesBySlug, typePropertiesByLabel, {}, typePathStateBySlug);
+                expect(csvJSON[0].season).toBe('SS25');
+                expect(warnings).toEqual([
+                    {
+                        rowNumber: 1,
+                        columnIssues: [
+                            {
+                                columnName: 'season',
+                                message: 'Property "season" is disabled for typePath "item:material". 1 value(s) will not be processed.',
+                                issueType: entity_types_1.IssueType.HEADER_MISMATCH,
+                                data: '',
+                            },
+                        ],
+                    },
+                ]);
+            });
+            it('renames label-headed columns to slug only on rows whose typePath does not archive the property', () => {
+                const csvJSON = [
+                    { typePath: 'item:product', Season: 'SS25' },
+                    { typePath: 'item:material', Season: 'SS25' },
+                ];
+                const typePropertiesBySlug = {
+                    season: { slug: 'season', label: 'Season' },
+                };
+                const typePropertiesByLabel = {
+                    Season: { slug: 'season', label: 'Season' },
+                };
+                const typePathStateBySlug = {
+                    season: new Map([
+                        ['item:product', { isDisabled: false }],
+                        ['item:material', { isDisabled: true }],
+                    ]),
+                };
+                (0, data_transformer_functions_1.convertFromLabelToSlug)(csvJSON, typePropertiesBySlug, typePropertiesByLabel, {}, typePathStateBySlug);
+                expect(csvJSON[0].season).toBe('SS25');
+                expect(csvJSON[1]).not.toHaveProperty('season');
+            });
+            it('emits a single aggregate warning grouping skips by (slug, typePath)', () => {
+                const csvJSON = [
+                    { typePath: 'item:material', season: 'SS25' },
+                    { typePath: 'item:material', season: 'FW25' },
+                    { typePath: 'item:cement', season: 'SS25' },
+                ];
+                const typePropertiesBySlug = {
+                    season: { slug: 'season', label: 'Season' },
+                };
+                const typePathStateBySlug = {
+                    season: new Map([
+                        ['item:material', { isDisabled: true }],
+                        ['item:cement', { isDisabled: true }],
+                    ]),
+                };
+                const warnings = (0, data_transformer_functions_1.convertFromLabelToSlug)(csvJSON, typePropertiesBySlug, {}, {}, typePathStateBySlug);
+                expect(warnings).toHaveLength(1);
+                expect(warnings[0].columnIssues).toHaveLength(2);
+                expect(warnings[0].columnIssues).toEqual(expect.arrayContaining([
+                    {
+                        columnName: 'season',
+                        message: 'Property "season" is disabled for typePath "item:material". 2 value(s) will not be processed.',
+                        issueType: entity_types_1.IssueType.HEADER_MISMATCH,
+                        data: '',
+                    },
+                    {
+                        columnName: 'season',
+                        message: 'Property "season" is disabled for typePath "item:cement". 1 value(s) will not be processed.',
+                        issueType: entity_types_1.IssueType.HEADER_MISMATCH,
+                        data: '',
+                    },
+                ]));
+            });
+            it('does not emit any archived warning when no row targets an archived subtype', () => {
+                const csvJSON = [{ typePath: 'item:product', season: 'SS25' }];
+                const typePropertiesBySlug = {
+                    season: { slug: 'season', label: 'Season' },
+                };
+                const typePathStateBySlug = {
+                    season: new Map([
+                        ['item:product', { isDisabled: false }],
+                        ['item:material', { isDisabled: true }],
+                    ]),
+                };
+                const warnings = (0, data_transformer_functions_1.convertFromLabelToSlug)(csvJSON, typePropertiesBySlug, {}, {}, typePathStateBySlug);
+                expect(warnings).toEqual([]);
+                expect(csvJSON[0].season).toBe('SS25');
+            });
+        });
     });
     describe('convertFromDisplayToValue', () => {
         it('should convert display values to optionSet values', () => {
@@ -633,6 +737,30 @@ describe('Data Transformer Functions', () => {
             ]);
         });
     });
+    describe('getPropertiesBySlugAndLabel archive tracking', () => {
+        it('records per-typePath archive state per slug', () => {
+            const reducedTypes = [
+                {
+                    typePath: 'item:product',
+                    typeProperties: [
+                        { slug: 'season', label: 'Season', isArchived: false },
+                        { slug: 'sku', label: 'SKU', isArchived: false },
+                    ],
+                },
+                {
+                    typePath: 'item:material',
+                    typeProperties: [
+                        { slug: 'season', label: 'Season', isArchived: true },
+                    ],
+                },
+            ];
+            const result = (0, data_transformer_functions_1.getPropertiesBySlugAndLabel)(reducedTypes);
+            expect(result.typePathStateBySlug.season.get('item:product')).toEqual({ isDisabled: false });
+            expect(result.typePathStateBySlug.season.get('item:material')).toEqual({ isDisabled: true });
+            expect(result.typePathStateBySlug.sku.get('item:product')).toEqual({ isDisabled: false });
+            expect(result.typePathStateBySlug.sku.has('item:material')).toBe(false);
+        });
+    });
     describe('getReducedTypes', () => {
         it('should get reduced types', async () => {
             mockClient.get.mockResolvedValueOnce([

package/node_modules/@contrail/loader/lib/common/entities-helper.js

@@ -5,6 +5,7 @@ const util_1 = require("@contrail/util");
 const types_1 = require("@contrail/types");
 const entity_types_1 = require("@contrail/entity-types");
 const lodash_1 = require("lodash");
+const chunk_by_url_length_1 = require("./chunk-by-url-length");
 const pLimit = require('p-limit');
 var EntitiesLambdaCallType;
 (function (EntitiesLambdaCallType) {
@@ -181,7 +182,7 @@ async function assignObjectReferences(client, typePaths, typesByPath, incomingEn
     const allORByFedId = {};
     for (const rootType of Object.keys(allORFedIdsByType)) {
         const fedIds = [...new Set(allORFedIdsByType[rootType])];
-        const chunks = (0, lodash_1.chunk)(fedIds, 200);
+        const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(fedIds);
         const federatedPromises = [];
         for (const chunk of chunks) {
             if (!chunk?.length) {

package/node_modules/@contrail/loader/lib/constants.d.ts

@@ -21,6 +21,8 @@ export declare const ITEM_PROCESSOR_ERRORS: {
     readonly FAMILY_ID_EMPTY: "itemFamilyFederatedId is null or undefined or empty in the CSV file";
     readonly CONFLICTING_IDS: (ids: any) => string;
 };
+export declare const FEDERATED_ID_FETCH_MAX_URL_BYTES = 4000;
+export declare const FEDERATED_ID_FETCH_MAX_COUNT = 200;
 export declare const ITEM_PROCESSOR_WARNINGS: {
     readonly OPTIONNAME_UNMAPPED: "\"optionName\" is not found in the federatedMapping config, conditionalColumns config, or as a header in the CSV file.";
     readonly OPTIONNAME_NOT_FOUND: (optionName: string) => string;

package/node_modules/@contrail/loader/lib/constants.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ITEM_PROCESSOR_WARNINGS = exports.ITEM_PROCESSOR_ERRORS = exports.ENTITY_PROCESSOR_ERRORS = exports.CUSTOM_ENTITY_PROCESSOR_ERRORS = exports.COLOR_PROCESSOR_WARNINGS = exports.COLOR_PROCESSOR_ERRORS = void 0;
+exports.ITEM_PROCESSOR_WARNINGS = exports.FEDERATED_ID_FETCH_MAX_COUNT = exports.FEDERATED_ID_FETCH_MAX_URL_BYTES = exports.ITEM_PROCESSOR_ERRORS = exports.ENTITY_PROCESSOR_ERRORS = exports.CUSTOM_ENTITY_PROCESSOR_ERRORS = exports.COLOR_PROCESSOR_WARNINGS = exports.COLOR_PROCESSOR_ERRORS = void 0;
 exports.COLOR_PROCESSOR_ERRORS = {
     TYPEPATH_MISSING: '"typePath" with value "color" is required property but is not found in the federatedMapping config, conditionalColumns config, or as a header in the CSV file. Please add a mapping "typePath" mapping with value "color" with one of these methods, then try the load again.',
 };
@@ -24,6 +24,8 @@ exports.ITEM_PROCESSOR_ERRORS = {
     FAMILY_ID_EMPTY: 'itemFamilyFederatedId is null or undefined or empty in the CSV file',
     CONFLICTING_IDS: (ids) => `Found conflicting family federated Ids: ${ids}`,
 };
+exports.FEDERATED_ID_FETCH_MAX_URL_BYTES = 4000;
+exports.FEDERATED_ID_FETCH_MAX_COUNT = 200;
 exports.ITEM_PROCESSOR_WARNINGS = {
     OPTIONNAME_UNMAPPED: '"optionName" is not found in the federatedMapping config, conditionalColumns config, or as a header in the CSV file.',
     OPTIONNAME_NOT_FOUND: (optionName) => `"${optionName}" column not found in the CSV file`,