@contrail/cli 3.4.10 → 3.4.11-alpha.feeaf0b

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. package/README.md +1 -1
  2. package/node_modules/@contrail/loader/lib/common/chunk-by-url-length.d.ts +6 -0
  3. package/node_modules/@contrail/loader/lib/common/chunk-by-url-length.js +31 -0
  4. package/node_modules/@contrail/loader/lib/common/chunk-by-url-length.spec.d.ts +1 -0
  5. package/node_modules/@contrail/loader/lib/common/chunk-by-url-length.spec.js +70 -0
  6. package/node_modules/@contrail/loader/lib/common/data-transformer-functions.d.ts +9 -1
  7. package/node_modules/@contrail/loader/lib/common/data-transformer-functions.js +53 -8
  8. package/node_modules/@contrail/loader/lib/common/data-transformer.spec.js +128 -0
  9. package/node_modules/@contrail/loader/lib/common/entities-helper.js +2 -1
  10. package/node_modules/@contrail/loader/lib/constants.d.ts +2 -0
  11. package/node_modules/@contrail/loader/lib/constants.js +3 -1
  12. package/node_modules/@contrail/loader/lib/entities-v2/entities-helper.js +2 -1
  13. package/node_modules/@contrail/loader/lib/entities-v2/entity-processor-v2.js +2 -2
  14. package/node_modules/@contrail/loader/lib/entities-v2/entity-processor-v2.spec.js +8 -10
  15. package/node_modules/@contrail/loader/lib/entities-v2/file-preprocessor-v2.js +2 -1
  16. package/node_modules/@contrail/loader/lib/entities-v2/linesheet-v2/item-helper.js +2 -1
  17. package/node_modules/@contrail/loader/lib/entity-processor.js +2 -2
  18. package/node_modules/@contrail/loader/lib/entity-processor.spec.js +8 -10
  19. package/node_modules/@contrail/loader/lib/pipeline/item-execute-helpers.js +2 -1
  20. package/node_modules/@contrail/loader/lib/pipeline/project-item-execute-helpers.js +3 -2
  21. package/node_modules/@contrail/loader/lib/pipeline/transform.js +32 -6
  22. package/node_modules/@contrail/loader/lib/pipeline/transform.spec.js +64 -1
  23. package/node_modules/@contrail/loader/package.json +1 -1
  24. package/oclif.manifest.json +1 -1
  25. package/package.json +2 -2
package/README.md CHANGED
@@ -17,7 +17,7 @@ $ npm install -g @contrail/cli
17
17
  $ contrail COMMAND
18
18
  running command...
19
19
  $ contrail (--version)
20
- @contrail/cli/3.4.10 linux-x64 node-v24.14.1
20
+ @contrail/cli/3.4.11-alpha.feeaf0b linux-x64 node-v24.14.1
21
21
  $ contrail --help [COMMAND]
22
22
  USAGE
23
23
  $ contrail COMMAND
@@ -0,0 +1,6 @@
1
+ interface ChunkByUrlLengthOptions {
2
+ maxBytes?: number;
3
+ maxCount?: number;
4
+ }
5
+ export declare function chunkByUrlLength(ids: string[], options?: ChunkByUrlLengthOptions): string[][];
6
+ export {};
@@ -0,0 +1,31 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.chunkByUrlLength = void 0;
4
+ const constants_1 = require("../constants");
5
+ function wireEncodedLength(id) {
6
+ return encodeURIComponent(id.replace(/,/g, '_COMMA_')).length;
7
+ }
8
+ function chunkByUrlLength(ids, options = {}) {
9
+ const maxBytes = options.maxBytes ?? constants_1.FEDERATED_ID_FETCH_MAX_URL_BYTES;
10
+ const maxCount = options.maxCount ?? constants_1.FEDERATED_ID_FETCH_MAX_COUNT;
11
+ const result = [];
12
+ let current = [];
13
+ let currentBytes = 0;
14
+ for (const id of ids) {
15
+ const encodedLength = wireEncodedLength(id);
16
+ const additionalBytes = current.length === 0 ? encodedLength : encodedLength + 1;
17
+ const wouldExceedBytes = currentBytes + additionalBytes > maxBytes;
18
+ const wouldExceedCount = current.length >= maxCount;
19
+ if (current.length > 0 && (wouldExceedBytes || wouldExceedCount)) {
20
+ result.push(current);
21
+ current = [];
22
+ currentBytes = 0;
23
+ }
24
+ current.push(id);
25
+ currentBytes += current.length === 1 ? encodedLength : encodedLength + 1;
26
+ }
27
+ if (current.length > 0)
28
+ result.push(current);
29
+ return result;
30
+ }
31
+ exports.chunkByUrlLength = chunkByUrlLength;
@@ -0,0 +1,70 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const chunk_by_url_length_1 = require("./chunk-by-url-length");
4
+ describe('chunkByUrlLength', () => {
5
+ it('returns an empty array for empty input', () => {
6
+ expect((0, chunk_by_url_length_1.chunkByUrlLength)([])).toEqual([]);
7
+ });
8
+ it('packs short ids into a single chunk up to maxCount', () => {
9
+ const ids = Array.from({ length: 50 }, (_, i) => `id-${i}`);
10
+ const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 10000 });
11
+ expect(chunks).toHaveLength(1);
12
+ expect(chunks[0]).toHaveLength(50);
13
+ });
14
+ it('splits on the maxCount cap when bytes are not the constraint', () => {
15
+ const ids = Array.from({ length: 450 }, (_, i) => `id${i}`);
16
+ const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 10000 });
17
+ expect(chunks.map((c) => c.length)).toEqual([200, 200, 50]);
18
+ });
19
+ it('splits on the maxBytes cap when ids are long', () => {
20
+ const longId = 'A'.repeat(500);
21
+ const ids = Array.from({ length: 20 }, () => longId);
22
+ const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 2000 });
23
+ for (const chunk of chunks) {
24
+ const encoded = chunk.map((id) => encodeURIComponent(id)).join(',');
25
+ expect(encoded.length).toBeLessThanOrEqual(2000);
26
+ }
27
+ expect(chunks.flat()).toHaveLength(20);
28
+ });
29
+ it('accounts for URL-encoding expansion (spaces, percent signs, commas)', () => {
30
+ const id = 'FL63497 Carbon Brushed Cotton Part dyed 60% Cotton_COMMA_ 40% Polyester';
31
+ const ids = Array.from({ length: 200 }, () => id);
32
+ const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 6000 });
33
+ expect(chunks.length).toBeGreaterThan(1);
34
+ for (const chunk of chunks) {
35
+ const encoded = chunk.map((x) => encodeURIComponent(x)).join(',');
36
+ expect(encoded.length).toBeLessThanOrEqual(6000);
37
+ }
38
+ expect(chunks.flat()).toHaveLength(200);
39
+ });
40
+ it('isolates a single id that exceeds maxBytes into its own chunk', () => {
41
+ const oversized = 'X'.repeat(10000);
42
+ const ids = ['short-1', 'short-2', oversized, 'short-3'];
43
+ const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 100 });
44
+ const oversizedChunk = chunks.find((c) => c.length === 1 && c[0] === oversized);
45
+ expect(oversizedChunk).toBeDefined();
46
+ expect(chunks.flat()).toEqual(ids);
47
+ });
48
+ it('preserves order across chunks', () => {
49
+ const ids = Array.from({ length: 123 }, (_, i) => `id-${i}`);
50
+ const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 25, maxBytes: 10000 });
51
+ expect(chunks.flat()).toEqual(ids);
52
+ });
53
+ it('accounts for the SDK\'s `,` → `_COMMA_` substitution', () => {
54
+ const idWithCommas = 'FOO Cotton, 60% Polyester, 40% Elastane';
55
+ const ids = Array.from({ length: 100 }, () => idWithCommas);
56
+ const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 1000 });
57
+ for (const chunk of chunks) {
58
+ const wireLength = chunk.map((id) => encodeURIComponent(id.replace(/,/g, '_COMMA_'))).join(',').length;
59
+ expect(wireLength).toBeLessThanOrEqual(1000);
60
+ }
61
+ expect(chunks.flat()).toHaveLength(100);
62
+ });
63
+ it('defaults to the module-level caps when no options are passed', () => {
64
+ const ids = Array.from({ length: 500 }, (_, i) => `id-${i}`);
65
+ const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids);
66
+ expect(chunks[0]).toHaveLength(200);
67
+ expect(chunks[1]).toHaveLength(200);
68
+ expect(chunks[2]).toHaveLength(100);
69
+ });
70
+ });
@@ -9,6 +9,9 @@ export declare type TypePropertyMaps = {
9
9
  typePropertiesBySlug: Record<string, TypeProperty>;
10
10
  typePropertiesByLabel: Record<string, TypeProperty>;
11
11
  optionSetUsingTypePropertiesBySlug: Record<string, TypePropertyWithLookups>;
12
+ typePathStateBySlug?: Record<string, Map<string, {
13
+ isDisabled: boolean;
14
+ }>>;
12
15
  };
13
16
  export declare const applyFederatedMappings: PreprocessingFunction;
14
17
  export declare const removeUnwantedProperties: PreprocessingFunction;
@@ -20,9 +23,14 @@ export declare const applyConditionalColumns: PreprocessingFunction;
20
23
  export declare function getPropertiesBySlugAndLabel(reducedTypes: Partial<Type>[]): {
21
24
  typePropertiesBySlug: Record<string, TypeProperty>;
22
25
  typePropertiesByLabel: Record<string, TypeProperty>;
26
+ typePathStateBySlug: Record<string, Map<string, {
27
+ isDisabled: boolean;
28
+ }>>;
23
29
  };
24
30
  export declare function getOptionSetsUsingTypePropertiesBySlug(reducedTypes: Partial<Type>[]): Record<string, TypePropertyWithLookups>;
25
- export declare function convertFromLabelToSlug(csvJSON: any, typePropertiesBySlug: any, typePropertiesByLabel: any, convertedFields: any): ProcessingIssues[];
31
+ export declare function convertFromLabelToSlug(csvJSON: any, typePropertiesBySlug: any, typePropertiesByLabel: any, convertedFields: any, typePathStateBySlug?: Record<string, Map<string, {
32
+ isDisabled: boolean;
33
+ }>>): ProcessingIssues[];
26
34
  export declare function convertFromDisplayToValue(csvJSON: any[], optionSetUsingTypePropertiesBySlug: any, typePropertySlugs: string[], typePropertyLabels: string[]): ProcessingIssues[];
27
35
  export declare function identifyKeysInProperties(csvJSON: any[], typePropertySlugs: string[], typePropertyLabels: string[]): {
28
36
  keysInPropertySlugs: Map<string, boolean>;
@@ -64,7 +64,7 @@ exports.trimValues = trimValues;
64
64
  const transformDataBasedOnTypes = function (csvJSON, config, typePropertyMaps, addWarnings = true) {
65
65
  const rows = createCopy(csvJSON);
66
66
  const { typePropertiesBySlug, typePropertiesByLabel, optionSetUsingTypePropertiesBySlug } = typePropertyMaps;
67
- const labelWarnings = convertFromLabelToSlug(rows, typePropertiesBySlug, typePropertiesByLabel, getConvertedFields(config.federatedMappings));
67
+ const labelWarnings = convertFromLabelToSlug(rows, typePropertiesBySlug, typePropertiesByLabel, getConvertedFields(config.federatedMappings), typePropertyMaps.typePathStateBySlug);
68
68
  const displayValueWarnings = convertFromDisplayToValue(rows, optionSetUsingTypePropertiesBySlug, Object.keys(typePropertiesBySlug), Object.keys(typePropertiesByLabel));
69
69
  if (!addWarnings) {
70
70
  return { rows, errors: [], warnings: [], generalErrors: [] };
@@ -100,15 +100,25 @@ exports.applyConditionalColumns = applyConditionalColumns;
100
100
  function getPropertiesBySlugAndLabel(reducedTypes) {
101
101
  const typePropertiesBySlug = {};
102
102
  const typePropertiesByLabel = {};
103
+ const typePathStateBySlug = {};
103
104
  for (const type of reducedTypes) {
104
- if (type.typeProperties) {
105
- for (const typeProperty of type.typeProperties) {
106
- typePropertiesBySlug[typeProperty.slug.trim()] = typeProperty;
105
+ if (!type.typeProperties)
106
+ continue;
107
+ const typePath = type.typePath;
108
+ for (const typeProperty of type.typeProperties) {
109
+ const slug = typeProperty.slug.trim();
110
+ typePropertiesBySlug[slug] = typeProperty;
111
+ if (typeProperty.label) {
107
112
  typePropertiesByLabel[typeProperty.label.trim()] = typeProperty;
108
113
  }
114
+ if (typePath) {
115
+ if (!typePathStateBySlug[slug])
116
+ typePathStateBySlug[slug] = new Map();
117
+ typePathStateBySlug[slug].set(typePath, { isDisabled: !!typeProperty.isArchived });
118
+ }
109
119
  }
110
120
  }
111
- return { typePropertiesBySlug, typePropertiesByLabel };
121
+ return { typePropertiesBySlug, typePropertiesByLabel, typePathStateBySlug };
112
122
  }
113
123
  exports.getPropertiesBySlugAndLabel = getPropertiesBySlugAndLabel;
114
124
  function getOptionSetsUsingTypePropertiesBySlug(reducedTypes) {
@@ -146,16 +156,38 @@ function getConvertedFields(federatedMappings) {
146
156
  }
147
157
  return convertedFields;
148
158
  }
149
- function convertFromLabelToSlug(csvJSON, typePropertiesBySlug, typePropertiesByLabel, convertedFields) {
159
+ function convertFromLabelToSlug(csvJSON, typePropertiesBySlug, typePropertiesByLabel, convertedFields, typePathStateBySlug) {
150
160
  const keysToBeIgnored = ['federatedId', 'typePath', 'itemFamilyFederatedId', 'itemOptionFederatedId', 'optionGroup'];
151
161
  const keysWithoutProperties = new Set();
152
162
  const archivedKeys = new Set();
163
+ const subtypeAwareArchivedSkips = new Map();
164
+ const useSubtypeAware = !!typePathStateBySlug;
153
165
  const warnings = [];
154
166
  for (const row of csvJSON) {
155
167
  for (const key of Object.keys(row)) {
156
168
  const slugProperty = typePropertiesBySlug[key];
157
169
  const labelProperty = typePropertiesByLabel[key];
158
- if (slugProperty?.isArchived || labelProperty?.isArchived) {
170
+ if (useSubtypeAware) {
171
+ const resolvedSlug = slugProperty?.slug || labelProperty?.slug;
172
+ const rowTypePath = row.typePath;
173
+ if (resolvedSlug && rowTypePath && typePathStateBySlug[resolvedSlug]?.get(rowTypePath)?.isDisabled) {
174
+ const aggregateKey = `${resolvedSlug}|${rowTypePath}`;
175
+ const existing = subtypeAwareArchivedSkips.get(aggregateKey);
176
+ if (existing) {
177
+ existing.count++;
178
+ }
179
+ else {
180
+ subtypeAwareArchivedSkips.set(aggregateKey, {
181
+ slug: resolvedSlug,
182
+ typePath: rowTypePath,
183
+ columnName: key,
184
+ count: 1,
185
+ });
186
+ }
187
+ continue;
188
+ }
189
+ }
190
+ else if (slugProperty?.isArchived || labelProperty?.isArchived) {
159
191
  archivedKeys.add(key);
160
192
  continue;
161
193
  }
@@ -181,7 +213,20 @@ function convertFromLabelToSlug(csvJSON, typePropertiesBySlug, typePropertiesByL
181
213
  }
182
214
  }
183
215
  }
184
- if (archivedKeys.size > 0) {
216
+ if (useSubtypeAware) {
217
+ if (subtypeAwareArchivedSkips.size > 0) {
218
+ warnings.push({
219
+ rowNumber: 1,
220
+ columnIssues: Array.from(subtypeAwareArchivedSkips.values()).map(({ slug, typePath, columnName, count }) => ({
221
+ columnName,
222
+ message: `Property "${slug}" is disabled for typePath "${typePath}". ${count} value(s) will not be processed.`,
223
+ issueType: entity_types_1.IssueType.HEADER_MISMATCH,
224
+ data: '',
225
+ })),
226
+ });
227
+ }
228
+ }
229
+ else if (archivedKeys.size > 0) {
185
230
  warnings.push({
186
231
  rowNumber: 1,
187
232
  columnIssues: Array.from(archivedKeys).map((key) => ({
@@ -409,6 +409,110 @@ describe('Data Transformer Functions', () => {
409
409
  expect(csvJSON[0]).toHaveProperty('size');
410
410
  });
411
411
  });
412
+ describe('subtype-aware archived handling', () => {
413
+ it('emits per-(slug, typePath) warnings for slug-headed columns archived on a subset of subtypes', () => {
414
+ const csvJSON = [
415
+ { typePath: 'item:product', season: 'SS25' },
416
+ { typePath: 'item:material', season: 'SS25' },
417
+ ];
418
+ const typePropertiesBySlug = {
419
+ season: { slug: 'season', label: 'Season' },
420
+ };
421
+ const typePropertiesByLabel = {
422
+ Season: { slug: 'season', label: 'Season' },
423
+ };
424
+ const typePathStateBySlug = {
425
+ season: new Map([
426
+ ['item:product', { isDisabled: false }],
427
+ ['item:material', { isDisabled: true }],
428
+ ]),
429
+ };
430
+ const warnings = (0, data_transformer_functions_1.convertFromLabelToSlug)(csvJSON, typePropertiesBySlug, typePropertiesByLabel, {}, typePathStateBySlug);
431
+ expect(csvJSON[0].season).toBe('SS25');
432
+ expect(warnings).toEqual([
433
+ {
434
+ rowNumber: 1,
435
+ columnIssues: [
436
+ {
437
+ columnName: 'season',
438
+ message: 'Property "season" is disabled for typePath "item:material". 1 value(s) will not be processed.',
439
+ issueType: entity_types_1.IssueType.HEADER_MISMATCH,
440
+ data: '',
441
+ },
442
+ ],
443
+ },
444
+ ]);
445
+ });
446
+ it('renames label-headed columns to slug only on rows whose typePath does not archive the property', () => {
447
+ const csvJSON = [
448
+ { typePath: 'item:product', Season: 'SS25' },
449
+ { typePath: 'item:material', Season: 'SS25' },
450
+ ];
451
+ const typePropertiesBySlug = {
452
+ season: { slug: 'season', label: 'Season' },
453
+ };
454
+ const typePropertiesByLabel = {
455
+ Season: { slug: 'season', label: 'Season' },
456
+ };
457
+ const typePathStateBySlug = {
458
+ season: new Map([
459
+ ['item:product', { isDisabled: false }],
460
+ ['item:material', { isDisabled: true }],
461
+ ]),
462
+ };
463
+ (0, data_transformer_functions_1.convertFromLabelToSlug)(csvJSON, typePropertiesBySlug, typePropertiesByLabel, {}, typePathStateBySlug);
464
+ expect(csvJSON[0].season).toBe('SS25');
465
+ expect(csvJSON[1]).not.toHaveProperty('season');
466
+ });
467
+ it('emits a single aggregate warning grouping skips by (slug, typePath)', () => {
468
+ const csvJSON = [
469
+ { typePath: 'item:material', season: 'SS25' },
470
+ { typePath: 'item:material', season: 'FW25' },
471
+ { typePath: 'item:cement', season: 'SS25' },
472
+ ];
473
+ const typePropertiesBySlug = {
474
+ season: { slug: 'season', label: 'Season' },
475
+ };
476
+ const typePathStateBySlug = {
477
+ season: new Map([
478
+ ['item:material', { isDisabled: true }],
479
+ ['item:cement', { isDisabled: true }],
480
+ ]),
481
+ };
482
+ const warnings = (0, data_transformer_functions_1.convertFromLabelToSlug)(csvJSON, typePropertiesBySlug, {}, {}, typePathStateBySlug);
483
+ expect(warnings).toHaveLength(1);
484
+ expect(warnings[0].columnIssues).toHaveLength(2);
485
+ expect(warnings[0].columnIssues).toEqual(expect.arrayContaining([
486
+ {
487
+ columnName: 'season',
488
+ message: 'Property "season" is disabled for typePath "item:material". 2 value(s) will not be processed.',
489
+ issueType: entity_types_1.IssueType.HEADER_MISMATCH,
490
+ data: '',
491
+ },
492
+ {
493
+ columnName: 'season',
494
+ message: 'Property "season" is disabled for typePath "item:cement". 1 value(s) will not be processed.',
495
+ issueType: entity_types_1.IssueType.HEADER_MISMATCH,
496
+ data: '',
497
+ },
498
+ ]));
499
+ });
500
+ it('does not emit any archived warning when no row targets an archived subtype', () => {
501
+ const csvJSON = [{ typePath: 'item:product', season: 'SS25' }];
502
+ const typePropertiesBySlug = {
503
+ season: { slug: 'season', label: 'Season' },
504
+ };
505
+ const typePathStateBySlug = {
506
+ season: new Map([
507
+ ['item:product', { isDisabled: false }],
508
+ ['item:material', { isDisabled: true }],
509
+ ]),
510
+ };
511
+ const warnings = (0, data_transformer_functions_1.convertFromLabelToSlug)(csvJSON, typePropertiesBySlug, {}, {}, typePathStateBySlug);
512
+ expect(warnings).toEqual([]);
513
+ expect(csvJSON[0].season).toBe('SS25');
514
+ });
515
+ });
412
516
  });
413
517
  describe('convertFromDisplayToValue', () => {
414
518
  it('should convert display values to optionSet values', () => {
@@ -633,6 +737,30 @@ describe('Data Transformer Functions', () => {
633
737
  ]);
634
738
  });
635
739
  });
740
+ describe('getPropertiesBySlugAndLabel archive tracking', () => {
741
+ it('records per-typePath archive state per slug', () => {
742
+ const reducedTypes = [
743
+ {
744
+ typePath: 'item:product',
745
+ typeProperties: [
746
+ { slug: 'season', label: 'Season', isArchived: false },
747
+ { slug: 'sku', label: 'SKU', isArchived: false },
748
+ ],
749
+ },
750
+ {
751
+ typePath: 'item:material',
752
+ typeProperties: [
753
+ { slug: 'season', label: 'Season', isArchived: true },
754
+ ],
755
+ },
756
+ ];
757
+ const result = (0, data_transformer_functions_1.getPropertiesBySlugAndLabel)(reducedTypes);
758
+ expect(result.typePathStateBySlug.season.get('item:product')).toEqual({ isDisabled: false });
759
+ expect(result.typePathStateBySlug.season.get('item:material')).toEqual({ isDisabled: true });
760
+ expect(result.typePathStateBySlug.sku.get('item:product')).toEqual({ isDisabled: false });
761
+ expect(result.typePathStateBySlug.sku.has('item:material')).toBe(false);
762
+ });
763
+ });
636
764
  describe('getReducedTypes', () => {
637
765
  it('should get reduced types', async () => {
638
766
  mockClient.get.mockResolvedValueOnce([
@@ -5,6 +5,7 @@ const util_1 = require("@contrail/util");
5
5
  const types_1 = require("@contrail/types");
6
6
  const entity_types_1 = require("@contrail/entity-types");
7
7
  const lodash_1 = require("lodash");
8
+ const chunk_by_url_length_1 = require("./chunk-by-url-length");
8
9
  const pLimit = require('p-limit');
9
10
  var EntitiesLambdaCallType;
10
11
  (function (EntitiesLambdaCallType) {
@@ -181,7 +182,7 @@ async function assignObjectReferences(client, typePaths, typesByPath, incomingEn
181
182
  const allORByFedId = {};
182
183
  for (const rootType of Object.keys(allORFedIdsByType)) {
183
184
  const fedIds = [...new Set(allORFedIdsByType[rootType])];
184
- const chunks = (0, lodash_1.chunk)(fedIds, 200);
185
+ const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(fedIds);
185
186
  const federatedPromises = [];
186
187
  for (const chunk of chunks) {
187
188
  if (!chunk?.length) {
@@ -21,6 +21,8 @@ export declare const ITEM_PROCESSOR_ERRORS: {
21
21
  readonly FAMILY_ID_EMPTY: "itemFamilyFederatedId is null or undefined or empty in the CSV file";
22
22
  readonly CONFLICTING_IDS: (ids: any) => string;
23
23
  };
24
+ export declare const FEDERATED_ID_FETCH_MAX_URL_BYTES = 4000;
25
+ export declare const FEDERATED_ID_FETCH_MAX_COUNT = 200;
24
26
  export declare const ITEM_PROCESSOR_WARNINGS: {
25
27
  readonly OPTIONNAME_UNMAPPED: "\"optionName\" is not found in the federatedMapping config, conditionalColumns config, or as a header in the CSV file.";
26
28
  readonly OPTIONNAME_NOT_FOUND: (optionName: string) => string;
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ITEM_PROCESSOR_WARNINGS = exports.ITEM_PROCESSOR_ERRORS = exports.ENTITY_PROCESSOR_ERRORS = exports.CUSTOM_ENTITY_PROCESSOR_ERRORS = exports.COLOR_PROCESSOR_WARNINGS = exports.COLOR_PROCESSOR_ERRORS = void 0;
3
+ exports.ITEM_PROCESSOR_WARNINGS = exports.FEDERATED_ID_FETCH_MAX_COUNT = exports.FEDERATED_ID_FETCH_MAX_URL_BYTES = exports.ITEM_PROCESSOR_ERRORS = exports.ENTITY_PROCESSOR_ERRORS = exports.CUSTOM_ENTITY_PROCESSOR_ERRORS = exports.COLOR_PROCESSOR_WARNINGS = exports.COLOR_PROCESSOR_ERRORS = void 0;
4
4
  exports.COLOR_PROCESSOR_ERRORS = {
5
5
  TYPEPATH_MISSING: '"typePath" with value "color" is required property but is not found in the federatedMapping config, conditionalColumns config, or as a header in the CSV file. Please add a mapping "typePath" mapping with value "color" with one of these methods, then try the load again.',
6
6
  };
@@ -24,6 +24,8 @@ exports.ITEM_PROCESSOR_ERRORS = {
24
24
  FAMILY_ID_EMPTY: 'itemFamilyFederatedId is null or undefined or empty in the CSV file',
25
25
  CONFLICTING_IDS: (ids) => `Found conflicting family federated Ids: ${ids}`,
26
26
  };
27
+ exports.FEDERATED_ID_FETCH_MAX_URL_BYTES = 4000;
28
+ exports.FEDERATED_ID_FETCH_MAX_COUNT = 200;
27
29
  exports.ITEM_PROCESSOR_WARNINGS = {
28
30
  OPTIONNAME_UNMAPPED: '"optionName" is not found in the federatedMapping config, conditionalColumns config, or as a header in the CSV file.',
29
31
  OPTIONNAME_NOT_FOUND: (optionName) => `"${optionName}" column not found in the CSV file`,
@@ -4,6 +4,7 @@ exports.getPropertyVariants = exports.assignUserLists = exports.getUsersInOrg =
4
4
  const util_1 = require("@contrail/util");
5
5
  const types_1 = require("@contrail/types");
6
6
  const lodash_1 = require("lodash");
7
+ const chunk_by_url_length_1 = require("../common/chunk-by-url-length");
7
8
  const entity_types_1 = require("@contrail/entity-types");
8
9
  const pLimit = require('p-limit');
9
10
  var EntitiesLambdaCallType;
@@ -176,7 +177,7 @@ async function assignObjectReferences(client, typePaths, typesByPath, incomingEn
176
177
  const allORByFedId = {};
177
178
  for (const rootType of Object.keys(allORFedIdsByType)) {
178
179
  const fedIds = [...new Set(allORFedIdsByType[rootType])];
179
- const chunks = (0, lodash_1.chunk)(fedIds, 200);
180
+ const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(fedIds);
180
181
  const federatedPromises = [];
181
182
  for (const chunk of chunks) {
182
183
  if (!chunk?.length) {
@@ -4,7 +4,7 @@ exports.EntityProcessorV2 = void 0;
4
4
  const entities_helper_1 = require("../common/entities-helper");
5
5
  const entity_builder_helpers_1 = require("../common/entity-builder-helpers");
6
6
  const processor_registry_1 = require("../processor-registry");
7
- const lodash_1 = require("lodash");
7
+ const chunk_by_url_length_1 = require("../common/chunk-by-url-length");
8
8
  const entity_types_1 = require("@contrail/entity-types");
9
9
  const type_helper_1 = require("../common/type-helper");
10
10
  const constants_1 = require("../constants");
@@ -157,7 +157,7 @@ class EntityProcessorV2 {
157
157
  return uniqueFedIds;
158
158
  }
159
159
  static async fetchCurrentEntities(client, entityType, federatedIds) {
160
- const chunks = (0, lodash_1.chunk)(federatedIds, 200);
160
+ const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(federatedIds);
161
161
  const allCurrentEntitiesByFederatedId = {};
162
162
  const federatedPromises = [];
163
163
  for (const chunk of chunks) {
@@ -6,7 +6,7 @@ const entity_builder_helpers_1 = require("../common/entity-builder-helpers");
6
6
  const types_1 = require("@contrail/types");
7
7
  const entity_types_1 = require("@contrail/entity-types");
8
8
  const processor_registry_1 = require("../processor-registry");
9
- const lodash_1 = require("lodash");
9
+ const chunk_by_url_length_1 = require("../common/chunk-by-url-length");
10
10
  const util_1 = require("../common/util");
11
11
  const color_processor_v2_1 = require("./colors-v2/color-processor-v2");
12
12
  const constants_1 = require("../constants");
@@ -15,9 +15,7 @@ jest.mock('../processor-registry');
15
15
  jest.mock('@contrail/entity-types');
16
16
  jest.mock('../common/entities-helper');
17
17
  jest.mock('../common/type-helper');
18
- jest.mock('lodash', () => ({
19
- chunk: jest.fn(),
20
- }));
18
+ jest.mock('../common/chunk-by-url-length');
21
19
  describe('EntityProcessorV2', () => {
22
20
  const mockTypesByPath = {
23
21
  color: {
@@ -217,7 +215,7 @@ describe('EntityProcessorV2', () => {
217
215
  ['1', '2'],
218
216
  ['3', '4'],
219
217
  ];
220
- lodash_1.chunk.mockReturnValue(chunks);
218
+ chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
221
219
  const mockEntities = [
222
220
  { federatedId: '1', name: 'Entity 1' },
223
221
  { federatedId: '2', name: 'Entity 2' },
@@ -228,12 +226,12 @@ describe('EntityProcessorV2', () => {
228
226
  '1': { federatedId: '1', name: 'Entity 1' },
229
227
  '2': { federatedId: '2', name: 'Entity 2' },
230
228
  });
231
- expect(lodash_1.chunk).toHaveBeenCalledWith(federatedIds, 200);
229
+ expect(chunk_by_url_length_1.chunkByUrlLength).toHaveBeenCalledWith(federatedIds);
232
230
  expect(entities_helper_1.retryOnTimeout).toHaveBeenCalledTimes(2);
233
231
  });
234
232
  it('should handle empty federatedIds array', async () => {
235
233
  const federatedIds = [];
236
- lodash_1.chunk.mockReturnValue([]);
234
+ chunk_by_url_length_1.chunkByUrlLength.mockReturnValue([]);
237
235
  const result = await entity_processor_v2_1.EntityProcessorV2.fetchCurrentEntities(client, entity_types_1.EntityType.COLOR, federatedIds);
238
236
  expect(result).toEqual({});
239
237
  expect(entities_helper_1.retryOnTimeout).not.toHaveBeenCalled();
@@ -241,7 +239,7 @@ describe('EntityProcessorV2', () => {
241
239
  it('should handle API errors and rethrow', async () => {
242
240
  const federatedIds = ['1'];
243
241
  const chunks = [['1']];
244
- lodash_1.chunk.mockReturnValue(chunks);
242
+ chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
245
243
  const error = new Error('API Error');
246
244
  entities_helper_1.retryOnTimeout.mockRejectedValue(error);
247
245
  await expect(entity_processor_v2_1.EntityProcessorV2.fetchCurrentEntities(client, entity_types_1.EntityType.COLOR, federatedIds)).rejects.toThrow('API Error');
@@ -250,7 +248,7 @@ describe('EntityProcessorV2', () => {
250
248
  it('should handle empty response from API', async () => {
251
249
  const federatedIds = ['1'];
252
250
  const chunks = [['1']];
253
- lodash_1.chunk.mockReturnValue(chunks);
251
+ chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
254
252
  entities_helper_1.retryOnTimeout.mockResolvedValue([]);
255
253
  const result = await entity_processor_v2_1.EntityProcessorV2.fetchCurrentEntities(client, entity_types_1.EntityType.COLOR, federatedIds);
256
254
  expect(result).toEqual({});
@@ -261,7 +259,7 @@ describe('EntityProcessorV2', () => {
261
259
  ['1', '2'],
262
260
  ['3', '4'],
263
261
  ];
264
- lodash_1.chunk.mockReturnValue(chunks);
262
+ chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
265
263
  const mockEntities1 = [
266
264
  { federatedId: '1', name: 'Entity 1' },
267
265
  { federatedId: '2', name: 'Entity 2' },
@@ -188,12 +188,13 @@ class LoaderDataPreprocessorV2 {
188
188
  };
189
189
  }
190
190
  static async runPreprocessingPipeline(client, loaderProcess, reducedTypes, processedData, preprocessingStepResults, transformationErrors, transformationWarnings, systemErrors) {
191
- const { typePropertiesBySlug, typePropertiesByLabel } = (0, data_transformer_functions_1.getPropertiesBySlugAndLabel)(reducedTypes);
191
+ const { typePropertiesBySlug, typePropertiesByLabel, typePathStateBySlug } = (0, data_transformer_functions_1.getPropertiesBySlugAndLabel)(reducedTypes);
192
192
  const optionSetUsingTypePropertiesBySlug = (0, data_transformer_functions_1.getOptionSetsUsingTypePropertiesBySlug)(reducedTypes);
193
193
  const typePropertyMaps = {
194
194
  typePropertiesBySlug,
195
195
  typePropertiesByLabel,
196
196
  optionSetUsingTypePropertiesBySlug,
197
+ typePathStateBySlug,
197
198
  };
198
199
  for (const preprocessingStep of this.getPreprocessingPipeline()) {
199
200
  try {
@@ -2,6 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.getAssortment = exports.convertToComparableEntities = exports.mapAssortmentItemsToItemsAndFillItemId = exports.getItemsFromFederatedIds = exports.getAssortmentItemProperties = exports.getAssortmentItems = void 0;
4
4
  const lodash_1 = require("lodash");
5
+ const chunk_by_url_length_1 = require("../../common/chunk-by-url-length");
5
6
  const entities_helper_1 = require("../../common/entities-helper");
6
7
  const pLimit = require('p-limit');
7
8
  const limit = pLimit(5);
@@ -58,7 +59,7 @@ function mapTypePropertiesToValueType(typeProperties) {
58
59
  }
59
60
  async function getItemsFromFederatedIds(client, incomingRows) {
60
61
  const federatedIds = extractUniqueFederatedIds(incomingRows);
61
- const chunks = (0, lodash_1.chunk)(federatedIds, 200);
62
+ const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(federatedIds);
62
63
  const allCurrentItemsByFederatedId = await fetchItemsByFederatedIds(client, chunks);
63
64
  return allCurrentItemsByFederatedId;
64
65
  }
@@ -4,7 +4,7 @@ exports.EntityProcessor = void 0;
4
4
  const entities_helper_1 = require("./common/entities-helper");
5
5
  const entity_builder_helpers_1 = require("./common/entity-builder-helpers");
6
6
  const processor_registry_1 = require("./processor-registry");
7
- const lodash_1 = require("lodash");
7
+ const chunk_by_url_length_1 = require("./common/chunk-by-url-length");
8
8
  const type_helper_1 = require("./common/type-helper");
9
9
  const entity_types_1 = require("@contrail/entity-types");
10
10
  const util_1 = require("./common/util");
@@ -104,7 +104,7 @@ class EntityProcessor {
104
104
  return [...uniqueFedIds, ...nonUniqueFedIds];
105
105
  }
106
106
  static async fetchCurrentEntities(client, entityType, federatedIds) {
107
- const chunks = (0, lodash_1.chunk)(federatedIds, 200);
107
+ const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(federatedIds);
108
108
  const allCurrentEntitiesByFederatedId = {};
109
109
  const federatedPromises = [];
110
110
  for (const chunk of chunks) {
@@ -6,7 +6,7 @@ const entity_builder_helpers_1 = require("./common/entity-builder-helpers");
6
6
  const types_1 = require("@contrail/types");
7
7
  const entity_types_1 = require("@contrail/entity-types");
8
8
  const processor_registry_1 = require("./processor-registry");
9
- const lodash_1 = require("lodash");
9
+ const chunk_by_url_length_1 = require("./common/chunk-by-url-length");
10
10
  const util_1 = require("./common/util");
11
11
  const constants_1 = require("./constants");
12
12
  jest.mock('./common/entity-builder-helpers');
@@ -14,9 +14,7 @@ jest.mock('./processor-registry');
14
14
  jest.mock('@contrail/entity-types');
15
15
  jest.mock('./common/entities-helper');
16
16
  jest.mock('./common/type-helper');
17
- jest.mock('lodash', () => ({
18
- chunk: jest.fn(),
19
- }));
17
+ jest.mock('./common/chunk-by-url-length');
20
18
  describe('EntityProcessor', () => {
21
19
  const mockTypesByPath = {
22
20
  color: {
@@ -203,7 +201,7 @@ describe('EntityProcessor', () => {
203
201
  ['1', '2'],
204
202
  ['3', '4'],
205
203
  ];
206
- lodash_1.chunk.mockReturnValue(chunks);
204
+ chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
207
205
  const mockEntities = [
208
206
  { federatedId: '1', name: 'Entity 1' },
209
207
  { federatedId: '2', name: 'Entity 2' },
@@ -214,12 +212,12 @@ describe('EntityProcessor', () => {
214
212
  '1': { federatedId: '1', name: 'Entity 1' },
215
213
  '2': { federatedId: '2', name: 'Entity 2' },
216
214
  });
217
- expect(lodash_1.chunk).toHaveBeenCalledWith(federatedIds, 200);
215
+ expect(chunk_by_url_length_1.chunkByUrlLength).toHaveBeenCalledWith(federatedIds);
218
216
  expect(entities_helper_1.retryOnTimeout).toHaveBeenCalledTimes(2);
219
217
  });
220
218
  it('should handle empty federatedIds array', async () => {
221
219
  const federatedIds = [];
222
- lodash_1.chunk.mockReturnValue([]);
220
+ chunk_by_url_length_1.chunkByUrlLength.mockReturnValue([]);
223
221
  const result = await entity_processor_1.EntityProcessor.fetchCurrentEntities(client, entity_types_1.EntityType.COLOR, federatedIds);
224
222
  expect(result).toEqual({});
225
223
  expect(entities_helper_1.retryOnTimeout).not.toHaveBeenCalled();
@@ -227,7 +225,7 @@ describe('EntityProcessor', () => {
227
225
  it('should handle API errors and rethrow', async () => {
228
226
  const federatedIds = ['1'];
229
227
  const chunks = [['1']];
230
- lodash_1.chunk.mockReturnValue(chunks);
228
+ chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
231
229
  const error = new Error('API Error');
232
230
  entities_helper_1.retryOnTimeout.mockRejectedValue(error);
233
231
  await expect(entity_processor_1.EntityProcessor.fetchCurrentEntities(client, entity_types_1.EntityType.COLOR, federatedIds)).rejects.toThrow('API Error');
@@ -236,7 +234,7 @@ describe('EntityProcessor', () => {
236
234
  it('should handle empty response from API', async () => {
237
235
  const federatedIds = ['1'];
238
236
  const chunks = [['1']];
239
- lodash_1.chunk.mockReturnValue(chunks);
237
+ chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
240
238
  entities_helper_1.retryOnTimeout.mockResolvedValue([]);
241
239
  const result = await entity_processor_1.EntityProcessor.fetchCurrentEntities(client, entity_types_1.EntityType.COLOR, federatedIds);
242
240
  expect(result).toEqual({});
@@ -248,7 +246,7 @@ describe('EntityProcessor', () => {
248
246
  ['1', '2'],
249
247
  ['3', '4'],
250
248
  ];
251
- lodash_1.chunk.mockReturnValue(chunks);
249
+ chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
252
250
  const mockEntities1 = [
253
251
  { federatedId: '1', name: 'Entity 1' },
254
252
  { federatedId: '2', name: 'Entity 2' },
@@ -4,6 +4,7 @@ exports.executeItemEntities = exports.previewItemEntities = exports.splitIntoFam
4
4
  const types_1 = require("./types");
5
5
  const util_1 = require("@contrail/util");
6
6
  const lodash_1 = require("lodash");
7
+ const chunk_by_url_length_1 = require("../common/chunk-by-url-length");
7
8
  const entities_helper_1 = require("../entities-v2/entities-helper");
8
9
  const type_helper_1 = require("../common/type-helper");
9
10
  const pLimit = require('p-limit');
@@ -461,7 +462,7 @@ async function batchUpdateEntities(client, entities, entityName, batchSize, limi
461
462
  return { updated, failures };
462
463
  }
463
464
  async function refreshEntityCache(client, federatedIds, cache) {
464
- const chunks = (0, lodash_1.chunk)(federatedIds, 200);
465
+ const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(federatedIds);
465
466
  const pLimitFetch = pLimit(5);
466
467
  const promises = chunks.map((batch) => pLimitFetch(async () => {
467
468
  try {
@@ -4,6 +4,7 @@ exports.applyAddedFromSource = exports.executeProjectItemEntities = exports.prev
4
4
  const types_1 = require("./types");
5
5
  const util_1 = require("@contrail/util");
6
6
  const lodash_1 = require("lodash");
7
+ const chunk_by_url_length_1 = require("../common/chunk-by-url-length");
7
8
  const entities_helper_1 = require("../entities-v2/entities-helper");
8
9
  const type_helper_1 = require("../common/type-helper");
9
10
  const item_execute_helpers_1 = require("./item-execute-helpers");
@@ -492,7 +493,7 @@ async function resolveItemsByFederatedId(client, incomingEntities, maxRetries) {
492
493
  federatedIds.add(entity.itemOptionFederatedId);
493
494
  }
494
495
  const result = {};
495
- const chunks = (0, lodash_1.chunk)(Array.from(federatedIds), 200);
496
+ const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(Array.from(federatedIds));
496
497
  const limit = pLimit(5);
497
498
  const promises = chunks.map((batch) => limit(async () => {
498
499
  try {
@@ -527,7 +528,7 @@ async function fetchCurrentProjectItemsByItemId(client, projectId, itemIds, maxR
527
528
  const result = {};
528
529
  if (!itemIds.length)
529
530
  return result;
530
- const chunks = (0, lodash_1.chunk)(itemIds, 200);
531
+ const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(itemIds);
531
532
  const limit = pLimit(5);
532
533
  const promises = chunks.map((batch) => limit(async () => {
533
534
  try {
@@ -192,10 +192,10 @@ function analyzeColumns(rows, typePropertyMaps, config) {
192
192
  if (!rows?.length)
193
193
  return { columns: [], unmappedTypeProperties: [] };
194
194
  const entityTypes = (0, types_2.loadTypeToEntityTypes)(config.loadType);
195
- const { typePropertiesBySlug, typePropertiesByLabel } = typePropertyMaps;
195
+ const { typePropertiesBySlug, typePropertiesByLabel, typePathStateBySlug } = typePropertyMaps;
196
196
  const optionSetUsingTypePropertiesBySlug = typePropertyMaps.optionSetUsingTypePropertiesBySlug;
197
197
  const csvHeaders = extractUniqueHeaders(rows);
198
- const columns = csvHeaders.map((header) => classifyColumn(header, typePropertiesBySlug, typePropertiesByLabel, optionSetUsingTypePropertiesBySlug, entityTypes));
198
+ const columns = csvHeaders.map((header) => classifyColumn(header, typePropertiesBySlug, typePropertiesByLabel, optionSetUsingTypePropertiesBySlug, entityTypes, typePathStateBySlug));
199
199
  const coveredSlugs = new Set();
200
200
  for (const column of columns) {
201
201
  if (column.matchedSlug) {
@@ -220,7 +220,15 @@ function analyzeColumns(rows, typePropertyMaps, config) {
220
220
  }
221
221
  }
222
222
  const unmappedTypeProperties = Object.entries(typePropertiesBySlug)
223
- .filter(([slug, prop]) => !coveredSlugs.has(slug.trim()) && !prop.isArchived)
223
+ .filter(([slug, prop]) => {
224
+ const trimmedSlug = slug.trim();
225
+ if (coveredSlugs.has(trimmedSlug))
226
+ return false;
227
+ if (typePathStateBySlug) {
228
+ return !isArchivedAcrossAllSubtypes(trimmedSlug, typePathStateBySlug);
229
+ }
230
+ return !prop.isArchived;
231
+ })
224
232
  .map(([slug, prop]) => ({
225
233
  slug: slug.trim(),
226
234
  label: prop.label,
@@ -243,7 +251,7 @@ function extractUniqueHeaders(rows) {
243
251
  }
244
252
  return headers;
245
253
  }
246
- function classifyColumn(header, typePropertiesBySlug, typePropertiesByLabel, optionSetProps, entityTypes) {
254
+ function classifyColumn(header, typePropertiesBySlug, typePropertiesByLabel, optionSetProps, entityTypes, typePathStateBySlug) {
247
255
  const isItemLoad = entityTypes.includes(types_1.EntityType.ITEM) || entityTypes.includes(types_1.EntityType.PROJECT_ITEM);
248
256
  const isEntityTypeSystemProp = entityTypes.some((et) => ENTITY_TYPE_SYSTEM_PROPERTIES[et]?.has(header));
249
257
  const isSystemProp = SYSTEM_PROPERTIES.has(header) || isEntityTypeSystemProp;
@@ -253,7 +261,14 @@ function classifyColumn(header, typePropertiesBySlug, typePropertiesByLabel, opt
253
261
  const baseSlugMatch = baseName !== header ? typePropertiesBySlug[baseName] : undefined;
254
262
  const matchedProp = slugMatch || labelMatch || baseSlugMatch;
255
263
  const isMapped = !!matchedProp || isSystemProp;
256
- const isArchived = matchedProp?.isArchived === true;
264
+ let isArchived;
265
+ if (matchedProp && typePathStateBySlug) {
266
+ const matchedSlug = (matchedProp.slug || '').trim();
267
+ isArchived = isArchivedAcrossAllSubtypes(matchedSlug, typePathStateBySlug);
268
+ }
269
+ else {
270
+ isArchived = matchedProp?.isArchived === true;
271
+ }
257
272
  const hasOptionSet = !!(optionSetProps[header] || optionSetProps[baseName]);
258
273
  let isRequired;
259
274
  let requiredNote;
@@ -292,6 +307,16 @@ function stripLevelSuffix(name) {
292
307
  return name.slice(0, -6);
293
308
  return name;
294
309
  }
310
+ function isArchivedAcrossAllSubtypes(slug, typePathStateBySlug) {
311
+ const states = typePathStateBySlug[slug];
312
+ if (!states || states.size === 0)
313
+ return false;
314
+ for (const state of states.values()) {
315
+ if (!state.isDisabled)
316
+ return false;
317
+ }
318
+ return true;
319
+ }
295
320
  const SUBTYPE_ENTITY_TYPES = new Set([types_1.EntityType.ITEM, types_1.EntityType.CUSTOM_ENTITY]);
296
321
  const SUBTYPE_PREFIX = {
297
322
  [types_1.EntityType.ITEM]: 'item',
@@ -344,12 +369,13 @@ async function fetchTypeDefinitions(client, rows, config) {
344
369
  }
345
370
  const typeRoots = (0, types_2.entityTypesToTypeRoots)(entityTypes);
346
371
  const reducedTypes = await (0, data_transformer_functions_1.getReducedTypes)(typeRoots, client);
347
- const { typePropertiesBySlug, typePropertiesByLabel } = (0, data_transformer_functions_1.getPropertiesBySlugAndLabel)(reducedTypes);
372
+ const { typePropertiesBySlug, typePropertiesByLabel, typePathStateBySlug } = (0, data_transformer_functions_1.getPropertiesBySlugAndLabel)(reducedTypes);
348
373
  const optionSetUsingTypePropertiesBySlug = (0, data_transformer_functions_1.getOptionSetsUsingTypePropertiesBySlug)(reducedTypes);
349
374
  const typePropertyMaps = {
350
375
  typePropertiesBySlug,
351
376
  typePropertiesByLabel,
352
377
  optionSetUsingTypePropertiesBySlug,
378
+ typePathStateBySlug,
353
379
  };
354
380
  const typesByPath = await (0, type_helper_1.buildTypesByPathForPipeline)(client, Array.from(typePaths));
355
381
  return { typePropertyMaps, typesByPath, generalErrors };
@@ -49,7 +49,11 @@ describe('transform', () => {
49
49
  getReducedTypesSpy = jest.spyOn(dataTransformerFunctions, 'getReducedTypes')
50
50
  .mockResolvedValue([]);
51
51
  getPropertiesBySlugAndLabelSpy = jest.spyOn(dataTransformerFunctions, 'getPropertiesBySlugAndLabel')
52
- .mockReturnValue({ typePropertiesBySlug: {}, typePropertiesByLabel: {} });
52
+ .mockReturnValue({
53
+ typePropertiesBySlug: {},
54
+ typePropertiesByLabel: {},
55
+ typePathStateBySlug: {},
56
+ });
53
57
  getOptionSetsUsingTypePropertiesBySlugSpy = jest.spyOn(dataTransformerFunctions, 'getOptionSetsUsingTypePropertiesBySlug')
54
58
  .mockReturnValue({});
55
59
  });
@@ -170,6 +174,7 @@ describe('transform', () => {
170
174
  active: { slug: 'active', propertyType: 'boolean' },
171
175
  },
172
176
  typePropertiesByLabel: {},
177
+ typePathStateBySlug: {},
173
178
  });
174
179
  const rows = [{ active: 'true', unmappedField: 'true' }];
175
180
  const result = await (0, transform_1.transform)(mockClient, rows, baseConfig);
@@ -294,6 +299,11 @@ describe('transform', () => {
294
299
  getPropertiesBySlugAndLabelSpy.mockReturnValue({
295
300
  typePropertiesBySlug: columnAnalysisTypeMaps.typePropertiesBySlug,
296
301
  typePropertiesByLabel: columnAnalysisTypeMaps.typePropertiesByLabel,
302
+ typePathStateBySlug: {
303
+ oldProp: new Map([['color', { isDisabled: true }]]),
304
+ color: new Map([['color', { isDisabled: false }]]),
305
+ size: new Map([['color', { isDisabled: false }]]),
306
+ },
297
307
  });
298
308
  getOptionSetsUsingTypePropertiesBySlugSpy.mockReturnValue(columnAnalysisTypeMaps.optionSetUsingTypePropertiesBySlug);
299
309
  }
@@ -328,6 +338,59 @@ describe('transform', () => {
328
338
  expect(archivedCol.isArchived).toBe(true);
329
339
  expect(archivedCol.isMapped).toBe(true);
330
340
  });
341
+ it('should not mark a column as archived when the property is active in any subtype', async () => {
342
+ getPropertiesBySlugAndLabelSpy.mockReturnValue({
343
+ typePropertiesBySlug: {
344
+ season: { slug: 'season', label: 'Season', propertyType: 'string', isArchived: false },
345
+ },
346
+ typePropertiesByLabel: {
347
+ Season: { slug: 'season', label: 'Season', propertyType: 'string', isArchived: false },
348
+ },
349
+ typePathStateBySlug: {
350
+ season: new Map([
351
+ ['item:product', { isDisabled: false }],
352
+ ['item:material', { isDisabled: true }],
353
+ ]),
354
+ },
355
+ });
356
+ getOptionSetsUsingTypePropertiesBySlugSpy.mockReturnValue({});
357
+ const config = {
358
+ loadType: [types_1.LoadType.ITEM],
359
+ conditionalColumns: [{ toProperty: 'typePath', default: 'item:product' }],
360
+ };
361
+ const rows = [{ itemFamilyFederatedId: 'fam-1', name: 'Test', season: 'SS25' }];
362
+ const result = await (0, transform_1.transform)(mockClient, rows, config);
363
+ const seasonCol = result.columns.find((c) => c.csvHeader === 'season');
364
+ expect(seasonCol).toBeDefined();
365
+ expect(seasonCol.isArchived).toBe(false);
366
+ expect(seasonCol.isMapped).toBe(true);
367
+ });
368
+ it('should mark a column as archived only when archived in every subtype that defines it', async () => {
369
+ getPropertiesBySlugAndLabelSpy.mockReturnValue({
370
+ typePropertiesBySlug: {
371
+ legacyProp: { slug: 'legacyProp', label: 'Legacy Prop', propertyType: 'string', isArchived: true },
372
+ },
373
+ typePropertiesByLabel: {
374
+ 'Legacy Prop': { slug: 'legacyProp', label: 'Legacy Prop', propertyType: 'string', isArchived: true },
375
+ },
376
+ typePathStateBySlug: {
377
+ legacyProp: new Map([
378
+ ['item:product', { isDisabled: true }],
379
+ ['item:material', { isDisabled: true }],
380
+ ]),
381
+ },
382
+ });
383
+ getOptionSetsUsingTypePropertiesBySlugSpy.mockReturnValue({});
384
+ const config = {
385
+ loadType: [types_1.LoadType.ITEM],
386
+ conditionalColumns: [{ toProperty: 'typePath', default: 'item:product' }],
387
+ };
388
+ const rows = [{ itemFamilyFederatedId: 'fam-1', name: 'Test', legacyProp: 'value' }];
389
+ const result = await (0, transform_1.transform)(mockClient, rows, config);
390
+ const legacyCol = result.columns.find((c) => c.csvHeader === 'legacyProp');
391
+ expect(legacyCol).toBeDefined();
392
+ expect(legacyCol.isArchived).toBe(true);
393
+ });
331
394
  it('should mark system properties as required for non-item loads', async () => {
332
395
  setupColumnAnalysisMocks();
333
396
  const config = { loadType: [types_1.LoadType.COLOR] };
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@contrail/loader",
3
- "version": "2.0.2",
3
+ "version": "2.0.6",
4
4
  "description": "Client library for helping with loader jobs.",
5
5
  "homepage": "https://archergreysorg.github.io/contrail-loader/",
6
6
  "main": "lib/index.js",
@@ -1,5 +1,5 @@
1
1
  {
2
- "version": "3.4.10",
2
+ "version": "3.4.11-alpha.feeaf0b",
3
3
  "commands": {
4
4
  "app:create": {
5
5
  "id": "app:create",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@contrail/cli",
3
- "version": "3.4.10",
3
+ "version": "3.4.11-alpha.feeaf0b",
4
4
  "description": "VibeIQ's Contrail Platform CLI.",
5
5
  "bin": {
6
6
  "contrail-cli": "./bin/run",
@@ -31,7 +31,7 @@
31
31
  ],
32
32
  "dependencies": {
33
33
  "@contrail/entity-types": "^1.6.1",
34
- "@contrail/loader": "^2.0.2",
34
+ "@contrail/loader": "^2.0.6",
35
35
  "@contrail/org-config-migration": "^1.1.13",
36
36
  "@contrail/sdk": "^1.5.5",
37
37
  "@contrail/types": "^3.1.5",