@contrail/cli 3.4.10 → 3.4.11-alpha.feeaf0b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/node_modules/@contrail/loader/lib/common/chunk-by-url-length.d.ts +6 -0
- package/node_modules/@contrail/loader/lib/common/chunk-by-url-length.js +31 -0
- package/node_modules/@contrail/loader/lib/common/chunk-by-url-length.spec.d.ts +1 -0
- package/node_modules/@contrail/loader/lib/common/chunk-by-url-length.spec.js +70 -0
- package/node_modules/@contrail/loader/lib/common/data-transformer-functions.d.ts +9 -1
- package/node_modules/@contrail/loader/lib/common/data-transformer-functions.js +53 -8
- package/node_modules/@contrail/loader/lib/common/data-transformer.spec.js +128 -0
- package/node_modules/@contrail/loader/lib/common/entities-helper.js +2 -1
- package/node_modules/@contrail/loader/lib/constants.d.ts +2 -0
- package/node_modules/@contrail/loader/lib/constants.js +3 -1
- package/node_modules/@contrail/loader/lib/entities-v2/entities-helper.js +2 -1
- package/node_modules/@contrail/loader/lib/entities-v2/entity-processor-v2.js +2 -2
- package/node_modules/@contrail/loader/lib/entities-v2/entity-processor-v2.spec.js +8 -10
- package/node_modules/@contrail/loader/lib/entities-v2/file-preprocessor-v2.js +2 -1
- package/node_modules/@contrail/loader/lib/entities-v2/linesheet-v2/item-helper.js +2 -1
- package/node_modules/@contrail/loader/lib/entity-processor.js +2 -2
- package/node_modules/@contrail/loader/lib/entity-processor.spec.js +8 -10
- package/node_modules/@contrail/loader/lib/pipeline/item-execute-helpers.js +2 -1
- package/node_modules/@contrail/loader/lib/pipeline/project-item-execute-helpers.js +3 -2
- package/node_modules/@contrail/loader/lib/pipeline/transform.js +32 -6
- package/node_modules/@contrail/loader/lib/pipeline/transform.spec.js +64 -1
- package/node_modules/@contrail/loader/package.json +1 -1
- package/oclif.manifest.json +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -17,7 +17,7 @@ $ npm install -g @contrail/cli
|
|
|
17
17
|
$ contrail COMMAND
|
|
18
18
|
running command...
|
|
19
19
|
$ contrail (--version)
|
|
20
|
-
@contrail/cli/3.4.
|
|
20
|
+
@contrail/cli/3.4.11-alpha.feeaf0b linux-x64 node-v24.14.1
|
|
21
21
|
$ contrail --help [COMMAND]
|
|
22
22
|
USAGE
|
|
23
23
|
$ contrail COMMAND
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.chunkByUrlLength = void 0;
|
|
4
|
+
const constants_1 = require("../constants");
|
|
5
|
+
function wireEncodedLength(id) {
|
|
6
|
+
return encodeURIComponent(id.replace(/,/g, '_COMMA_')).length;
|
|
7
|
+
}
|
|
8
|
+
function chunkByUrlLength(ids, options = {}) {
|
|
9
|
+
const maxBytes = options.maxBytes ?? constants_1.FEDERATED_ID_FETCH_MAX_URL_BYTES;
|
|
10
|
+
const maxCount = options.maxCount ?? constants_1.FEDERATED_ID_FETCH_MAX_COUNT;
|
|
11
|
+
const result = [];
|
|
12
|
+
let current = [];
|
|
13
|
+
let currentBytes = 0;
|
|
14
|
+
for (const id of ids) {
|
|
15
|
+
const encodedLength = wireEncodedLength(id);
|
|
16
|
+
const additionalBytes = current.length === 0 ? encodedLength : encodedLength + 1;
|
|
17
|
+
const wouldExceedBytes = currentBytes + additionalBytes > maxBytes;
|
|
18
|
+
const wouldExceedCount = current.length >= maxCount;
|
|
19
|
+
if (current.length > 0 && (wouldExceedBytes || wouldExceedCount)) {
|
|
20
|
+
result.push(current);
|
|
21
|
+
current = [];
|
|
22
|
+
currentBytes = 0;
|
|
23
|
+
}
|
|
24
|
+
current.push(id);
|
|
25
|
+
currentBytes += current.length === 1 ? encodedLength : encodedLength + 1;
|
|
26
|
+
}
|
|
27
|
+
if (current.length > 0)
|
|
28
|
+
result.push(current);
|
|
29
|
+
return result;
|
|
30
|
+
}
|
|
31
|
+
exports.chunkByUrlLength = chunkByUrlLength;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const chunk_by_url_length_1 = require("./chunk-by-url-length");
|
|
4
|
+
describe('chunkByUrlLength', () => {
|
|
5
|
+
it('returns an empty array for empty input', () => {
|
|
6
|
+
expect((0, chunk_by_url_length_1.chunkByUrlLength)([])).toEqual([]);
|
|
7
|
+
});
|
|
8
|
+
it('packs short ids into a single chunk up to maxCount', () => {
|
|
9
|
+
const ids = Array.from({ length: 50 }, (_, i) => `id-${i}`);
|
|
10
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 10000 });
|
|
11
|
+
expect(chunks).toHaveLength(1);
|
|
12
|
+
expect(chunks[0]).toHaveLength(50);
|
|
13
|
+
});
|
|
14
|
+
it('splits on the maxCount cap when bytes are not the constraint', () => {
|
|
15
|
+
const ids = Array.from({ length: 450 }, (_, i) => `id${i}`);
|
|
16
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 10000 });
|
|
17
|
+
expect(chunks.map((c) => c.length)).toEqual([200, 200, 50]);
|
|
18
|
+
});
|
|
19
|
+
it('splits on the maxBytes cap when ids are long', () => {
|
|
20
|
+
const longId = 'A'.repeat(500);
|
|
21
|
+
const ids = Array.from({ length: 20 }, () => longId);
|
|
22
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 2000 });
|
|
23
|
+
for (const chunk of chunks) {
|
|
24
|
+
const encoded = chunk.map((id) => encodeURIComponent(id)).join(',');
|
|
25
|
+
expect(encoded.length).toBeLessThanOrEqual(2000);
|
|
26
|
+
}
|
|
27
|
+
expect(chunks.flat()).toHaveLength(20);
|
|
28
|
+
});
|
|
29
|
+
it('accounts for URL-encoding expansion (spaces, percent signs, commas)', () => {
|
|
30
|
+
const id = 'FL63497 Carbon Brushed Cotton Part dyed 60% Cotton_COMMA_ 40% Polyester';
|
|
31
|
+
const ids = Array.from({ length: 200 }, () => id);
|
|
32
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 6000 });
|
|
33
|
+
expect(chunks.length).toBeGreaterThan(1);
|
|
34
|
+
for (const chunk of chunks) {
|
|
35
|
+
const encoded = chunk.map((x) => encodeURIComponent(x)).join(',');
|
|
36
|
+
expect(encoded.length).toBeLessThanOrEqual(6000);
|
|
37
|
+
}
|
|
38
|
+
expect(chunks.flat()).toHaveLength(200);
|
|
39
|
+
});
|
|
40
|
+
it('isolates a single id that exceeds maxBytes into its own chunk', () => {
|
|
41
|
+
const oversized = 'X'.repeat(10000);
|
|
42
|
+
const ids = ['short-1', 'short-2', oversized, 'short-3'];
|
|
43
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 100 });
|
|
44
|
+
const oversizedChunk = chunks.find((c) => c.length === 1 && c[0] === oversized);
|
|
45
|
+
expect(oversizedChunk).toBeDefined();
|
|
46
|
+
expect(chunks.flat()).toEqual(ids);
|
|
47
|
+
});
|
|
48
|
+
it('preserves order across chunks', () => {
|
|
49
|
+
const ids = Array.from({ length: 123 }, (_, i) => `id-${i}`);
|
|
50
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 25, maxBytes: 10000 });
|
|
51
|
+
expect(chunks.flat()).toEqual(ids);
|
|
52
|
+
});
|
|
53
|
+
it('accounts for the SDK\'s `,` → `_COMMA_` substitution', () => {
|
|
54
|
+
const idWithCommas = 'FOO Cotton, 60% Polyester, 40% Elastane';
|
|
55
|
+
const ids = Array.from({ length: 100 }, () => idWithCommas);
|
|
56
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids, { maxCount: 200, maxBytes: 1000 });
|
|
57
|
+
for (const chunk of chunks) {
|
|
58
|
+
const wireLength = chunk.map((id) => encodeURIComponent(id.replace(/,/g, '_COMMA_'))).join(',').length;
|
|
59
|
+
expect(wireLength).toBeLessThanOrEqual(1000);
|
|
60
|
+
}
|
|
61
|
+
expect(chunks.flat()).toHaveLength(100);
|
|
62
|
+
});
|
|
63
|
+
it('defaults to the module-level caps when no options are passed', () => {
|
|
64
|
+
const ids = Array.from({ length: 500 }, (_, i) => `id-${i}`);
|
|
65
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(ids);
|
|
66
|
+
expect(chunks[0]).toHaveLength(200);
|
|
67
|
+
expect(chunks[1]).toHaveLength(200);
|
|
68
|
+
expect(chunks[2]).toHaveLength(100);
|
|
69
|
+
});
|
|
70
|
+
});
|
|
@@ -9,6 +9,9 @@ export declare type TypePropertyMaps = {
|
|
|
9
9
|
typePropertiesBySlug: Record<string, TypeProperty>;
|
|
10
10
|
typePropertiesByLabel: Record<string, TypeProperty>;
|
|
11
11
|
optionSetUsingTypePropertiesBySlug: Record<string, TypePropertyWithLookups>;
|
|
12
|
+
typePathStateBySlug?: Record<string, Map<string, {
|
|
13
|
+
isDisabled: boolean;
|
|
14
|
+
}>>;
|
|
12
15
|
};
|
|
13
16
|
export declare const applyFederatedMappings: PreprocessingFunction;
|
|
14
17
|
export declare const removeUnwantedProperties: PreprocessingFunction;
|
|
@@ -20,9 +23,14 @@ export declare const applyConditionalColumns: PreprocessingFunction;
|
|
|
20
23
|
export declare function getPropertiesBySlugAndLabel(reducedTypes: Partial<Type>[]): {
|
|
21
24
|
typePropertiesBySlug: Record<string, TypeProperty>;
|
|
22
25
|
typePropertiesByLabel: Record<string, TypeProperty>;
|
|
26
|
+
typePathStateBySlug: Record<string, Map<string, {
|
|
27
|
+
isDisabled: boolean;
|
|
28
|
+
}>>;
|
|
23
29
|
};
|
|
24
30
|
export declare function getOptionSetsUsingTypePropertiesBySlug(reducedTypes: Partial<Type>[]): Record<string, TypePropertyWithLookups>;
|
|
25
|
-
export declare function convertFromLabelToSlug(csvJSON: any, typePropertiesBySlug: any, typePropertiesByLabel: any, convertedFields: any
|
|
31
|
+
export declare function convertFromLabelToSlug(csvJSON: any, typePropertiesBySlug: any, typePropertiesByLabel: any, convertedFields: any, typePathStateBySlug?: Record<string, Map<string, {
|
|
32
|
+
isDisabled: boolean;
|
|
33
|
+
}>>): ProcessingIssues[];
|
|
26
34
|
export declare function convertFromDisplayToValue(csvJSON: any[], optionSetUsingTypePropertiesBySlug: any, typePropertySlugs: string[], typePropertyLabels: string[]): ProcessingIssues[];
|
|
27
35
|
export declare function identifyKeysInProperties(csvJSON: any[], typePropertySlugs: string[], typePropertyLabels: string[]): {
|
|
28
36
|
keysInPropertySlugs: Map<string, boolean>;
|
|
@@ -64,7 +64,7 @@ exports.trimValues = trimValues;
|
|
|
64
64
|
const transformDataBasedOnTypes = function (csvJSON, config, typePropertyMaps, addWarnings = true) {
|
|
65
65
|
const rows = createCopy(csvJSON);
|
|
66
66
|
const { typePropertiesBySlug, typePropertiesByLabel, optionSetUsingTypePropertiesBySlug } = typePropertyMaps;
|
|
67
|
-
const labelWarnings = convertFromLabelToSlug(rows, typePropertiesBySlug, typePropertiesByLabel, getConvertedFields(config.federatedMappings));
|
|
67
|
+
const labelWarnings = convertFromLabelToSlug(rows, typePropertiesBySlug, typePropertiesByLabel, getConvertedFields(config.federatedMappings), typePropertyMaps.typePathStateBySlug);
|
|
68
68
|
const displayValueWarnings = convertFromDisplayToValue(rows, optionSetUsingTypePropertiesBySlug, Object.keys(typePropertiesBySlug), Object.keys(typePropertiesByLabel));
|
|
69
69
|
if (!addWarnings) {
|
|
70
70
|
return { rows, errors: [], warnings: [], generalErrors: [] };
|
|
@@ -100,15 +100,25 @@ exports.applyConditionalColumns = applyConditionalColumns;
|
|
|
100
100
|
function getPropertiesBySlugAndLabel(reducedTypes) {
|
|
101
101
|
const typePropertiesBySlug = {};
|
|
102
102
|
const typePropertiesByLabel = {};
|
|
103
|
+
const typePathStateBySlug = {};
|
|
103
104
|
for (const type of reducedTypes) {
|
|
104
|
-
if (type.typeProperties)
|
|
105
|
-
|
|
106
|
-
|
|
105
|
+
if (!type.typeProperties)
|
|
106
|
+
continue;
|
|
107
|
+
const typePath = type.typePath;
|
|
108
|
+
for (const typeProperty of type.typeProperties) {
|
|
109
|
+
const slug = typeProperty.slug.trim();
|
|
110
|
+
typePropertiesBySlug[slug] = typeProperty;
|
|
111
|
+
if (typeProperty.label) {
|
|
107
112
|
typePropertiesByLabel[typeProperty.label.trim()] = typeProperty;
|
|
108
113
|
}
|
|
114
|
+
if (typePath) {
|
|
115
|
+
if (!typePathStateBySlug[slug])
|
|
116
|
+
typePathStateBySlug[slug] = new Map();
|
|
117
|
+
typePathStateBySlug[slug].set(typePath, { isDisabled: !!typeProperty.isArchived });
|
|
118
|
+
}
|
|
109
119
|
}
|
|
110
120
|
}
|
|
111
|
-
return { typePropertiesBySlug, typePropertiesByLabel };
|
|
121
|
+
return { typePropertiesBySlug, typePropertiesByLabel, typePathStateBySlug };
|
|
112
122
|
}
|
|
113
123
|
exports.getPropertiesBySlugAndLabel = getPropertiesBySlugAndLabel;
|
|
114
124
|
function getOptionSetsUsingTypePropertiesBySlug(reducedTypes) {
|
|
@@ -146,16 +156,38 @@ function getConvertedFields(federatedMappings) {
|
|
|
146
156
|
}
|
|
147
157
|
return convertedFields;
|
|
148
158
|
}
|
|
149
|
-
function convertFromLabelToSlug(csvJSON, typePropertiesBySlug, typePropertiesByLabel, convertedFields) {
|
|
159
|
+
function convertFromLabelToSlug(csvJSON, typePropertiesBySlug, typePropertiesByLabel, convertedFields, typePathStateBySlug) {
|
|
150
160
|
const keysToBeIgnored = ['federatedId', 'typePath', 'itemFamilyFederatedId', 'itemOptionFederatedId', 'optionGroup'];
|
|
151
161
|
const keysWithoutProperties = new Set();
|
|
152
162
|
const archivedKeys = new Set();
|
|
163
|
+
const subtypeAwareArchivedSkips = new Map();
|
|
164
|
+
const useSubtypeAware = !!typePathStateBySlug;
|
|
153
165
|
const warnings = [];
|
|
154
166
|
for (const row of csvJSON) {
|
|
155
167
|
for (const key of Object.keys(row)) {
|
|
156
168
|
const slugProperty = typePropertiesBySlug[key];
|
|
157
169
|
const labelProperty = typePropertiesByLabel[key];
|
|
158
|
-
if (
|
|
170
|
+
if (useSubtypeAware) {
|
|
171
|
+
const resolvedSlug = slugProperty?.slug || labelProperty?.slug;
|
|
172
|
+
const rowTypePath = row.typePath;
|
|
173
|
+
if (resolvedSlug && rowTypePath && typePathStateBySlug[resolvedSlug]?.get(rowTypePath)?.isDisabled) {
|
|
174
|
+
const aggregateKey = `${resolvedSlug}|${rowTypePath}`;
|
|
175
|
+
const existing = subtypeAwareArchivedSkips.get(aggregateKey);
|
|
176
|
+
if (existing) {
|
|
177
|
+
existing.count++;
|
|
178
|
+
}
|
|
179
|
+
else {
|
|
180
|
+
subtypeAwareArchivedSkips.set(aggregateKey, {
|
|
181
|
+
slug: resolvedSlug,
|
|
182
|
+
typePath: rowTypePath,
|
|
183
|
+
columnName: key,
|
|
184
|
+
count: 1,
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
else if (slugProperty?.isArchived || labelProperty?.isArchived) {
|
|
159
191
|
archivedKeys.add(key);
|
|
160
192
|
continue;
|
|
161
193
|
}
|
|
@@ -181,7 +213,20 @@ function convertFromLabelToSlug(csvJSON, typePropertiesBySlug, typePropertiesByL
|
|
|
181
213
|
}
|
|
182
214
|
}
|
|
183
215
|
}
|
|
184
|
-
if (
|
|
216
|
+
if (useSubtypeAware) {
|
|
217
|
+
if (subtypeAwareArchivedSkips.size > 0) {
|
|
218
|
+
warnings.push({
|
|
219
|
+
rowNumber: 1,
|
|
220
|
+
columnIssues: Array.from(subtypeAwareArchivedSkips.values()).map(({ slug, typePath, columnName, count }) => ({
|
|
221
|
+
columnName,
|
|
222
|
+
message: `Property "${slug}" is disabled for typePath "${typePath}". ${count} value(s) will not be processed.`,
|
|
223
|
+
issueType: entity_types_1.IssueType.HEADER_MISMATCH,
|
|
224
|
+
data: '',
|
|
225
|
+
})),
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
else if (archivedKeys.size > 0) {
|
|
185
230
|
warnings.push({
|
|
186
231
|
rowNumber: 1,
|
|
187
232
|
columnIssues: Array.from(archivedKeys).map((key) => ({
|
|
@@ -409,6 +409,110 @@ describe('Data Transformer Functions', () => {
|
|
|
409
409
|
expect(csvJSON[0]).toHaveProperty('size');
|
|
410
410
|
});
|
|
411
411
|
});
|
|
412
|
+
describe('subtype-aware archived handling', () => {
|
|
413
|
+
it('emits per-(slug, typePath) warnings for slug-headed columns archived on a subset of subtypes', () => {
|
|
414
|
+
const csvJSON = [
|
|
415
|
+
{ typePath: 'item:product', season: 'SS25' },
|
|
416
|
+
{ typePath: 'item:material', season: 'SS25' },
|
|
417
|
+
];
|
|
418
|
+
const typePropertiesBySlug = {
|
|
419
|
+
season: { slug: 'season', label: 'Season' },
|
|
420
|
+
};
|
|
421
|
+
const typePropertiesByLabel = {
|
|
422
|
+
Season: { slug: 'season', label: 'Season' },
|
|
423
|
+
};
|
|
424
|
+
const typePathStateBySlug = {
|
|
425
|
+
season: new Map([
|
|
426
|
+
['item:product', { isDisabled: false }],
|
|
427
|
+
['item:material', { isDisabled: true }],
|
|
428
|
+
]),
|
|
429
|
+
};
|
|
430
|
+
const warnings = (0, data_transformer_functions_1.convertFromLabelToSlug)(csvJSON, typePropertiesBySlug, typePropertiesByLabel, {}, typePathStateBySlug);
|
|
431
|
+
expect(csvJSON[0].season).toBe('SS25');
|
|
432
|
+
expect(warnings).toEqual([
|
|
433
|
+
{
|
|
434
|
+
rowNumber: 1,
|
|
435
|
+
columnIssues: [
|
|
436
|
+
{
|
|
437
|
+
columnName: 'season',
|
|
438
|
+
message: 'Property "season" is disabled for typePath "item:material". 1 value(s) will not be processed.',
|
|
439
|
+
issueType: entity_types_1.IssueType.HEADER_MISMATCH,
|
|
440
|
+
data: '',
|
|
441
|
+
},
|
|
442
|
+
],
|
|
443
|
+
},
|
|
444
|
+
]);
|
|
445
|
+
});
|
|
446
|
+
it('renames label-headed columns to slug only on rows whose typePath does not archive the property', () => {
|
|
447
|
+
const csvJSON = [
|
|
448
|
+
{ typePath: 'item:product', Season: 'SS25' },
|
|
449
|
+
{ typePath: 'item:material', Season: 'SS25' },
|
|
450
|
+
];
|
|
451
|
+
const typePropertiesBySlug = {
|
|
452
|
+
season: { slug: 'season', label: 'Season' },
|
|
453
|
+
};
|
|
454
|
+
const typePropertiesByLabel = {
|
|
455
|
+
Season: { slug: 'season', label: 'Season' },
|
|
456
|
+
};
|
|
457
|
+
const typePathStateBySlug = {
|
|
458
|
+
season: new Map([
|
|
459
|
+
['item:product', { isDisabled: false }],
|
|
460
|
+
['item:material', { isDisabled: true }],
|
|
461
|
+
]),
|
|
462
|
+
};
|
|
463
|
+
(0, data_transformer_functions_1.convertFromLabelToSlug)(csvJSON, typePropertiesBySlug, typePropertiesByLabel, {}, typePathStateBySlug);
|
|
464
|
+
expect(csvJSON[0].season).toBe('SS25');
|
|
465
|
+
expect(csvJSON[1]).not.toHaveProperty('season');
|
|
466
|
+
});
|
|
467
|
+
it('emits a single aggregate warning grouping skips by (slug, typePath)', () => {
|
|
468
|
+
const csvJSON = [
|
|
469
|
+
{ typePath: 'item:material', season: 'SS25' },
|
|
470
|
+
{ typePath: 'item:material', season: 'FW25' },
|
|
471
|
+
{ typePath: 'item:cement', season: 'SS25' },
|
|
472
|
+
];
|
|
473
|
+
const typePropertiesBySlug = {
|
|
474
|
+
season: { slug: 'season', label: 'Season' },
|
|
475
|
+
};
|
|
476
|
+
const typePathStateBySlug = {
|
|
477
|
+
season: new Map([
|
|
478
|
+
['item:material', { isDisabled: true }],
|
|
479
|
+
['item:cement', { isDisabled: true }],
|
|
480
|
+
]),
|
|
481
|
+
};
|
|
482
|
+
const warnings = (0, data_transformer_functions_1.convertFromLabelToSlug)(csvJSON, typePropertiesBySlug, {}, {}, typePathStateBySlug);
|
|
483
|
+
expect(warnings).toHaveLength(1);
|
|
484
|
+
expect(warnings[0].columnIssues).toHaveLength(2);
|
|
485
|
+
expect(warnings[0].columnIssues).toEqual(expect.arrayContaining([
|
|
486
|
+
{
|
|
487
|
+
columnName: 'season',
|
|
488
|
+
message: 'Property "season" is disabled for typePath "item:material". 2 value(s) will not be processed.',
|
|
489
|
+
issueType: entity_types_1.IssueType.HEADER_MISMATCH,
|
|
490
|
+
data: '',
|
|
491
|
+
},
|
|
492
|
+
{
|
|
493
|
+
columnName: 'season',
|
|
494
|
+
message: 'Property "season" is disabled for typePath "item:cement". 1 value(s) will not be processed.',
|
|
495
|
+
issueType: entity_types_1.IssueType.HEADER_MISMATCH,
|
|
496
|
+
data: '',
|
|
497
|
+
},
|
|
498
|
+
]));
|
|
499
|
+
});
|
|
500
|
+
it('does not emit any archived warning when no row targets an archived subtype', () => {
|
|
501
|
+
const csvJSON = [{ typePath: 'item:product', season: 'SS25' }];
|
|
502
|
+
const typePropertiesBySlug = {
|
|
503
|
+
season: { slug: 'season', label: 'Season' },
|
|
504
|
+
};
|
|
505
|
+
const typePathStateBySlug = {
|
|
506
|
+
season: new Map([
|
|
507
|
+
['item:product', { isDisabled: false }],
|
|
508
|
+
['item:material', { isDisabled: true }],
|
|
509
|
+
]),
|
|
510
|
+
};
|
|
511
|
+
const warnings = (0, data_transformer_functions_1.convertFromLabelToSlug)(csvJSON, typePropertiesBySlug, {}, {}, typePathStateBySlug);
|
|
512
|
+
expect(warnings).toEqual([]);
|
|
513
|
+
expect(csvJSON[0].season).toBe('SS25');
|
|
514
|
+
});
|
|
515
|
+
});
|
|
412
516
|
});
|
|
413
517
|
describe('convertFromDisplayToValue', () => {
|
|
414
518
|
it('should convert display values to optionSet values', () => {
|
|
@@ -633,6 +737,30 @@ describe('Data Transformer Functions', () => {
|
|
|
633
737
|
]);
|
|
634
738
|
});
|
|
635
739
|
});
|
|
740
|
+
describe('getPropertiesBySlugAndLabel archive tracking', () => {
|
|
741
|
+
it('records per-typePath archive state per slug', () => {
|
|
742
|
+
const reducedTypes = [
|
|
743
|
+
{
|
|
744
|
+
typePath: 'item:product',
|
|
745
|
+
typeProperties: [
|
|
746
|
+
{ slug: 'season', label: 'Season', isArchived: false },
|
|
747
|
+
{ slug: 'sku', label: 'SKU', isArchived: false },
|
|
748
|
+
],
|
|
749
|
+
},
|
|
750
|
+
{
|
|
751
|
+
typePath: 'item:material',
|
|
752
|
+
typeProperties: [
|
|
753
|
+
{ slug: 'season', label: 'Season', isArchived: true },
|
|
754
|
+
],
|
|
755
|
+
},
|
|
756
|
+
];
|
|
757
|
+
const result = (0, data_transformer_functions_1.getPropertiesBySlugAndLabel)(reducedTypes);
|
|
758
|
+
expect(result.typePathStateBySlug.season.get('item:product')).toEqual({ isDisabled: false });
|
|
759
|
+
expect(result.typePathStateBySlug.season.get('item:material')).toEqual({ isDisabled: true });
|
|
760
|
+
expect(result.typePathStateBySlug.sku.get('item:product')).toEqual({ isDisabled: false });
|
|
761
|
+
expect(result.typePathStateBySlug.sku.has('item:material')).toBe(false);
|
|
762
|
+
});
|
|
763
|
+
});
|
|
636
764
|
describe('getReducedTypes', () => {
|
|
637
765
|
it('should get reduced types', async () => {
|
|
638
766
|
mockClient.get.mockResolvedValueOnce([
|
|
@@ -5,6 +5,7 @@ const util_1 = require("@contrail/util");
|
|
|
5
5
|
const types_1 = require("@contrail/types");
|
|
6
6
|
const entity_types_1 = require("@contrail/entity-types");
|
|
7
7
|
const lodash_1 = require("lodash");
|
|
8
|
+
const chunk_by_url_length_1 = require("./chunk-by-url-length");
|
|
8
9
|
const pLimit = require('p-limit');
|
|
9
10
|
var EntitiesLambdaCallType;
|
|
10
11
|
(function (EntitiesLambdaCallType) {
|
|
@@ -181,7 +182,7 @@ async function assignObjectReferences(client, typePaths, typesByPath, incomingEn
|
|
|
181
182
|
const allORByFedId = {};
|
|
182
183
|
for (const rootType of Object.keys(allORFedIdsByType)) {
|
|
183
184
|
const fedIds = [...new Set(allORFedIdsByType[rootType])];
|
|
184
|
-
const chunks = (0,
|
|
185
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(fedIds);
|
|
185
186
|
const federatedPromises = [];
|
|
186
187
|
for (const chunk of chunks) {
|
|
187
188
|
if (!chunk?.length) {
|
|
@@ -21,6 +21,8 @@ export declare const ITEM_PROCESSOR_ERRORS: {
|
|
|
21
21
|
readonly FAMILY_ID_EMPTY: "itemFamilyFederatedId is null or undefined or empty in the CSV file";
|
|
22
22
|
readonly CONFLICTING_IDS: (ids: any) => string;
|
|
23
23
|
};
|
|
24
|
+
export declare const FEDERATED_ID_FETCH_MAX_URL_BYTES = 4000;
|
|
25
|
+
export declare const FEDERATED_ID_FETCH_MAX_COUNT = 200;
|
|
24
26
|
export declare const ITEM_PROCESSOR_WARNINGS: {
|
|
25
27
|
readonly OPTIONNAME_UNMAPPED: "\"optionName\" is not found in the federatedMapping config, conditionalColumns config, or as a header in the CSV file.";
|
|
26
28
|
readonly OPTIONNAME_NOT_FOUND: (optionName: string) => string;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.ITEM_PROCESSOR_WARNINGS = exports.ITEM_PROCESSOR_ERRORS = exports.ENTITY_PROCESSOR_ERRORS = exports.CUSTOM_ENTITY_PROCESSOR_ERRORS = exports.COLOR_PROCESSOR_WARNINGS = exports.COLOR_PROCESSOR_ERRORS = void 0;
|
|
3
|
+
exports.ITEM_PROCESSOR_WARNINGS = exports.FEDERATED_ID_FETCH_MAX_COUNT = exports.FEDERATED_ID_FETCH_MAX_URL_BYTES = exports.ITEM_PROCESSOR_ERRORS = exports.ENTITY_PROCESSOR_ERRORS = exports.CUSTOM_ENTITY_PROCESSOR_ERRORS = exports.COLOR_PROCESSOR_WARNINGS = exports.COLOR_PROCESSOR_ERRORS = void 0;
|
|
4
4
|
exports.COLOR_PROCESSOR_ERRORS = {
|
|
5
5
|
TYPEPATH_MISSING: '"typePath" with value "color" is required property but is not found in the federatedMapping config, conditionalColumns config, or as a header in the CSV file. Please add a mapping "typePath" mapping with value "color" with one of these methods, then try the load again.',
|
|
6
6
|
};
|
|
@@ -24,6 +24,8 @@ exports.ITEM_PROCESSOR_ERRORS = {
|
|
|
24
24
|
FAMILY_ID_EMPTY: 'itemFamilyFederatedId is null or undefined or empty in the CSV file',
|
|
25
25
|
CONFLICTING_IDS: (ids) => `Found conflicting family federated Ids: ${ids}`,
|
|
26
26
|
};
|
|
27
|
+
exports.FEDERATED_ID_FETCH_MAX_URL_BYTES = 4000;
|
|
28
|
+
exports.FEDERATED_ID_FETCH_MAX_COUNT = 200;
|
|
27
29
|
exports.ITEM_PROCESSOR_WARNINGS = {
|
|
28
30
|
OPTIONNAME_UNMAPPED: '"optionName" is not found in the federatedMapping config, conditionalColumns config, or as a header in the CSV file.',
|
|
29
31
|
OPTIONNAME_NOT_FOUND: (optionName) => `"${optionName}" column not found in the CSV file`,
|
|
@@ -4,6 +4,7 @@ exports.getPropertyVariants = exports.assignUserLists = exports.getUsersInOrg =
|
|
|
4
4
|
const util_1 = require("@contrail/util");
|
|
5
5
|
const types_1 = require("@contrail/types");
|
|
6
6
|
const lodash_1 = require("lodash");
|
|
7
|
+
const chunk_by_url_length_1 = require("../common/chunk-by-url-length");
|
|
7
8
|
const entity_types_1 = require("@contrail/entity-types");
|
|
8
9
|
const pLimit = require('p-limit');
|
|
9
10
|
var EntitiesLambdaCallType;
|
|
@@ -176,7 +177,7 @@ async function assignObjectReferences(client, typePaths, typesByPath, incomingEn
|
|
|
176
177
|
const allORByFedId = {};
|
|
177
178
|
for (const rootType of Object.keys(allORFedIdsByType)) {
|
|
178
179
|
const fedIds = [...new Set(allORFedIdsByType[rootType])];
|
|
179
|
-
const chunks = (0,
|
|
180
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(fedIds);
|
|
180
181
|
const federatedPromises = [];
|
|
181
182
|
for (const chunk of chunks) {
|
|
182
183
|
if (!chunk?.length) {
|
|
@@ -4,7 +4,7 @@ exports.EntityProcessorV2 = void 0;
|
|
|
4
4
|
const entities_helper_1 = require("../common/entities-helper");
|
|
5
5
|
const entity_builder_helpers_1 = require("../common/entity-builder-helpers");
|
|
6
6
|
const processor_registry_1 = require("../processor-registry");
|
|
7
|
-
const
|
|
7
|
+
const chunk_by_url_length_1 = require("../common/chunk-by-url-length");
|
|
8
8
|
const entity_types_1 = require("@contrail/entity-types");
|
|
9
9
|
const type_helper_1 = require("../common/type-helper");
|
|
10
10
|
const constants_1 = require("../constants");
|
|
@@ -157,7 +157,7 @@ class EntityProcessorV2 {
|
|
|
157
157
|
return uniqueFedIds;
|
|
158
158
|
}
|
|
159
159
|
static async fetchCurrentEntities(client, entityType, federatedIds) {
|
|
160
|
-
const chunks = (0,
|
|
160
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(federatedIds);
|
|
161
161
|
const allCurrentEntitiesByFederatedId = {};
|
|
162
162
|
const federatedPromises = [];
|
|
163
163
|
for (const chunk of chunks) {
|
|
@@ -6,7 +6,7 @@ const entity_builder_helpers_1 = require("../common/entity-builder-helpers");
|
|
|
6
6
|
const types_1 = require("@contrail/types");
|
|
7
7
|
const entity_types_1 = require("@contrail/entity-types");
|
|
8
8
|
const processor_registry_1 = require("../processor-registry");
|
|
9
|
-
const
|
|
9
|
+
const chunk_by_url_length_1 = require("../common/chunk-by-url-length");
|
|
10
10
|
const util_1 = require("../common/util");
|
|
11
11
|
const color_processor_v2_1 = require("./colors-v2/color-processor-v2");
|
|
12
12
|
const constants_1 = require("../constants");
|
|
@@ -15,9 +15,7 @@ jest.mock('../processor-registry');
|
|
|
15
15
|
jest.mock('@contrail/entity-types');
|
|
16
16
|
jest.mock('../common/entities-helper');
|
|
17
17
|
jest.mock('../common/type-helper');
|
|
18
|
-
jest.mock('
|
|
19
|
-
chunk: jest.fn(),
|
|
20
|
-
}));
|
|
18
|
+
jest.mock('../common/chunk-by-url-length');
|
|
21
19
|
describe('EntityProcessorV2', () => {
|
|
22
20
|
const mockTypesByPath = {
|
|
23
21
|
color: {
|
|
@@ -217,7 +215,7 @@ describe('EntityProcessorV2', () => {
|
|
|
217
215
|
['1', '2'],
|
|
218
216
|
['3', '4'],
|
|
219
217
|
];
|
|
220
|
-
|
|
218
|
+
chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
|
|
221
219
|
const mockEntities = [
|
|
222
220
|
{ federatedId: '1', name: 'Entity 1' },
|
|
223
221
|
{ federatedId: '2', name: 'Entity 2' },
|
|
@@ -228,12 +226,12 @@ describe('EntityProcessorV2', () => {
|
|
|
228
226
|
'1': { federatedId: '1', name: 'Entity 1' },
|
|
229
227
|
'2': { federatedId: '2', name: 'Entity 2' },
|
|
230
228
|
});
|
|
231
|
-
expect(
|
|
229
|
+
expect(chunk_by_url_length_1.chunkByUrlLength).toHaveBeenCalledWith(federatedIds);
|
|
232
230
|
expect(entities_helper_1.retryOnTimeout).toHaveBeenCalledTimes(2);
|
|
233
231
|
});
|
|
234
232
|
it('should handle empty federatedIds array', async () => {
|
|
235
233
|
const federatedIds = [];
|
|
236
|
-
|
|
234
|
+
chunk_by_url_length_1.chunkByUrlLength.mockReturnValue([]);
|
|
237
235
|
const result = await entity_processor_v2_1.EntityProcessorV2.fetchCurrentEntities(client, entity_types_1.EntityType.COLOR, federatedIds);
|
|
238
236
|
expect(result).toEqual({});
|
|
239
237
|
expect(entities_helper_1.retryOnTimeout).not.toHaveBeenCalled();
|
|
@@ -241,7 +239,7 @@ describe('EntityProcessorV2', () => {
|
|
|
241
239
|
it('should handle API errors and rethrow', async () => {
|
|
242
240
|
const federatedIds = ['1'];
|
|
243
241
|
const chunks = [['1']];
|
|
244
|
-
|
|
242
|
+
chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
|
|
245
243
|
const error = new Error('API Error');
|
|
246
244
|
entities_helper_1.retryOnTimeout.mockRejectedValue(error);
|
|
247
245
|
await expect(entity_processor_v2_1.EntityProcessorV2.fetchCurrentEntities(client, entity_types_1.EntityType.COLOR, federatedIds)).rejects.toThrow('API Error');
|
|
@@ -250,7 +248,7 @@ describe('EntityProcessorV2', () => {
|
|
|
250
248
|
it('should handle empty response from API', async () => {
|
|
251
249
|
const federatedIds = ['1'];
|
|
252
250
|
const chunks = [['1']];
|
|
253
|
-
|
|
251
|
+
chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
|
|
254
252
|
entities_helper_1.retryOnTimeout.mockResolvedValue([]);
|
|
255
253
|
const result = await entity_processor_v2_1.EntityProcessorV2.fetchCurrentEntities(client, entity_types_1.EntityType.COLOR, federatedIds);
|
|
256
254
|
expect(result).toEqual({});
|
|
@@ -261,7 +259,7 @@ describe('EntityProcessorV2', () => {
|
|
|
261
259
|
['1', '2'],
|
|
262
260
|
['3', '4'],
|
|
263
261
|
];
|
|
264
|
-
|
|
262
|
+
chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
|
|
265
263
|
const mockEntities1 = [
|
|
266
264
|
{ federatedId: '1', name: 'Entity 1' },
|
|
267
265
|
{ federatedId: '2', name: 'Entity 2' },
|
|
@@ -188,12 +188,13 @@ class LoaderDataPreprocessorV2 {
|
|
|
188
188
|
};
|
|
189
189
|
}
|
|
190
190
|
static async runPreprocessingPipeline(client, loaderProcess, reducedTypes, processedData, preprocessingStepResults, transformationErrors, transformationWarnings, systemErrors) {
|
|
191
|
-
const { typePropertiesBySlug, typePropertiesByLabel } = (0, data_transformer_functions_1.getPropertiesBySlugAndLabel)(reducedTypes);
|
|
191
|
+
const { typePropertiesBySlug, typePropertiesByLabel, typePathStateBySlug } = (0, data_transformer_functions_1.getPropertiesBySlugAndLabel)(reducedTypes);
|
|
192
192
|
const optionSetUsingTypePropertiesBySlug = (0, data_transformer_functions_1.getOptionSetsUsingTypePropertiesBySlug)(reducedTypes);
|
|
193
193
|
const typePropertyMaps = {
|
|
194
194
|
typePropertiesBySlug,
|
|
195
195
|
typePropertiesByLabel,
|
|
196
196
|
optionSetUsingTypePropertiesBySlug,
|
|
197
|
+
typePathStateBySlug,
|
|
197
198
|
};
|
|
198
199
|
for (const preprocessingStep of this.getPreprocessingPipeline()) {
|
|
199
200
|
try {
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.getAssortment = exports.convertToComparableEntities = exports.mapAssortmentItemsToItemsAndFillItemId = exports.getItemsFromFederatedIds = exports.getAssortmentItemProperties = exports.getAssortmentItems = void 0;
|
|
4
4
|
const lodash_1 = require("lodash");
|
|
5
|
+
const chunk_by_url_length_1 = require("../../common/chunk-by-url-length");
|
|
5
6
|
const entities_helper_1 = require("../../common/entities-helper");
|
|
6
7
|
const pLimit = require('p-limit');
|
|
7
8
|
const limit = pLimit(5);
|
|
@@ -58,7 +59,7 @@ function mapTypePropertiesToValueType(typeProperties) {
|
|
|
58
59
|
}
|
|
59
60
|
async function getItemsFromFederatedIds(client, incomingRows) {
|
|
60
61
|
const federatedIds = extractUniqueFederatedIds(incomingRows);
|
|
61
|
-
const chunks = (0,
|
|
62
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(federatedIds);
|
|
62
63
|
const allCurrentItemsByFederatedId = await fetchItemsByFederatedIds(client, chunks);
|
|
63
64
|
return allCurrentItemsByFederatedId;
|
|
64
65
|
}
|
|
@@ -4,7 +4,7 @@ exports.EntityProcessor = void 0;
|
|
|
4
4
|
const entities_helper_1 = require("./common/entities-helper");
|
|
5
5
|
const entity_builder_helpers_1 = require("./common/entity-builder-helpers");
|
|
6
6
|
const processor_registry_1 = require("./processor-registry");
|
|
7
|
-
const
|
|
7
|
+
const chunk_by_url_length_1 = require("./common/chunk-by-url-length");
|
|
8
8
|
const type_helper_1 = require("./common/type-helper");
|
|
9
9
|
const entity_types_1 = require("@contrail/entity-types");
|
|
10
10
|
const util_1 = require("./common/util");
|
|
@@ -104,7 +104,7 @@ class EntityProcessor {
|
|
|
104
104
|
return [...uniqueFedIds, ...nonUniqueFedIds];
|
|
105
105
|
}
|
|
106
106
|
static async fetchCurrentEntities(client, entityType, federatedIds) {
|
|
107
|
-
const chunks = (0,
|
|
107
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(federatedIds);
|
|
108
108
|
const allCurrentEntitiesByFederatedId = {};
|
|
109
109
|
const federatedPromises = [];
|
|
110
110
|
for (const chunk of chunks) {
|
|
@@ -6,7 +6,7 @@ const entity_builder_helpers_1 = require("./common/entity-builder-helpers");
|
|
|
6
6
|
const types_1 = require("@contrail/types");
|
|
7
7
|
const entity_types_1 = require("@contrail/entity-types");
|
|
8
8
|
const processor_registry_1 = require("./processor-registry");
|
|
9
|
-
const
|
|
9
|
+
const chunk_by_url_length_1 = require("./common/chunk-by-url-length");
|
|
10
10
|
const util_1 = require("./common/util");
|
|
11
11
|
const constants_1 = require("./constants");
|
|
12
12
|
jest.mock('./common/entity-builder-helpers');
|
|
@@ -14,9 +14,7 @@ jest.mock('./processor-registry');
|
|
|
14
14
|
jest.mock('@contrail/entity-types');
|
|
15
15
|
jest.mock('./common/entities-helper');
|
|
16
16
|
jest.mock('./common/type-helper');
|
|
17
|
-
jest.mock('
|
|
18
|
-
chunk: jest.fn(),
|
|
19
|
-
}));
|
|
17
|
+
jest.mock('./common/chunk-by-url-length');
|
|
20
18
|
describe('EntityProcessor', () => {
|
|
21
19
|
const mockTypesByPath = {
|
|
22
20
|
color: {
|
|
@@ -203,7 +201,7 @@ describe('EntityProcessor', () => {
|
|
|
203
201
|
['1', '2'],
|
|
204
202
|
['3', '4'],
|
|
205
203
|
];
|
|
206
|
-
|
|
204
|
+
chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
|
|
207
205
|
const mockEntities = [
|
|
208
206
|
{ federatedId: '1', name: 'Entity 1' },
|
|
209
207
|
{ federatedId: '2', name: 'Entity 2' },
|
|
@@ -214,12 +212,12 @@ describe('EntityProcessor', () => {
|
|
|
214
212
|
'1': { federatedId: '1', name: 'Entity 1' },
|
|
215
213
|
'2': { federatedId: '2', name: 'Entity 2' },
|
|
216
214
|
});
|
|
217
|
-
expect(
|
|
215
|
+
expect(chunk_by_url_length_1.chunkByUrlLength).toHaveBeenCalledWith(federatedIds);
|
|
218
216
|
expect(entities_helper_1.retryOnTimeout).toHaveBeenCalledTimes(2);
|
|
219
217
|
});
|
|
220
218
|
it('should handle empty federatedIds array', async () => {
|
|
221
219
|
const federatedIds = [];
|
|
222
|
-
|
|
220
|
+
chunk_by_url_length_1.chunkByUrlLength.mockReturnValue([]);
|
|
223
221
|
const result = await entity_processor_1.EntityProcessor.fetchCurrentEntities(client, entity_types_1.EntityType.COLOR, federatedIds);
|
|
224
222
|
expect(result).toEqual({});
|
|
225
223
|
expect(entities_helper_1.retryOnTimeout).not.toHaveBeenCalled();
|
|
@@ -227,7 +225,7 @@ describe('EntityProcessor', () => {
|
|
|
227
225
|
it('should handle API errors and rethrow', async () => {
|
|
228
226
|
const federatedIds = ['1'];
|
|
229
227
|
const chunks = [['1']];
|
|
230
|
-
|
|
228
|
+
chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
|
|
231
229
|
const error = new Error('API Error');
|
|
232
230
|
entities_helper_1.retryOnTimeout.mockRejectedValue(error);
|
|
233
231
|
await expect(entity_processor_1.EntityProcessor.fetchCurrentEntities(client, entity_types_1.EntityType.COLOR, federatedIds)).rejects.toThrow('API Error');
|
|
@@ -236,7 +234,7 @@ describe('EntityProcessor', () => {
|
|
|
236
234
|
it('should handle empty response from API', async () => {
|
|
237
235
|
const federatedIds = ['1'];
|
|
238
236
|
const chunks = [['1']];
|
|
239
|
-
|
|
237
|
+
chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
|
|
240
238
|
entities_helper_1.retryOnTimeout.mockResolvedValue([]);
|
|
241
239
|
const result = await entity_processor_1.EntityProcessor.fetchCurrentEntities(client, entity_types_1.EntityType.COLOR, federatedIds);
|
|
242
240
|
expect(result).toEqual({});
|
|
@@ -248,7 +246,7 @@ describe('EntityProcessor', () => {
|
|
|
248
246
|
['1', '2'],
|
|
249
247
|
['3', '4'],
|
|
250
248
|
];
|
|
251
|
-
|
|
249
|
+
chunk_by_url_length_1.chunkByUrlLength.mockReturnValue(chunks);
|
|
252
250
|
const mockEntities1 = [
|
|
253
251
|
{ federatedId: '1', name: 'Entity 1' },
|
|
254
252
|
{ federatedId: '2', name: 'Entity 2' },
|
|
@@ -4,6 +4,7 @@ exports.executeItemEntities = exports.previewItemEntities = exports.splitIntoFam
|
|
|
4
4
|
const types_1 = require("./types");
|
|
5
5
|
const util_1 = require("@contrail/util");
|
|
6
6
|
const lodash_1 = require("lodash");
|
|
7
|
+
const chunk_by_url_length_1 = require("../common/chunk-by-url-length");
|
|
7
8
|
const entities_helper_1 = require("../entities-v2/entities-helper");
|
|
8
9
|
const type_helper_1 = require("../common/type-helper");
|
|
9
10
|
const pLimit = require('p-limit');
|
|
@@ -461,7 +462,7 @@ async function batchUpdateEntities(client, entities, entityName, batchSize, limi
|
|
|
461
462
|
return { updated, failures };
|
|
462
463
|
}
|
|
463
464
|
async function refreshEntityCache(client, federatedIds, cache) {
|
|
464
|
-
const chunks = (0,
|
|
465
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(federatedIds);
|
|
465
466
|
const pLimitFetch = pLimit(5);
|
|
466
467
|
const promises = chunks.map((batch) => pLimitFetch(async () => {
|
|
467
468
|
try {
|
|
@@ -4,6 +4,7 @@ exports.applyAddedFromSource = exports.executeProjectItemEntities = exports.prev
|
|
|
4
4
|
const types_1 = require("./types");
|
|
5
5
|
const util_1 = require("@contrail/util");
|
|
6
6
|
const lodash_1 = require("lodash");
|
|
7
|
+
const chunk_by_url_length_1 = require("../common/chunk-by-url-length");
|
|
7
8
|
const entities_helper_1 = require("../entities-v2/entities-helper");
|
|
8
9
|
const type_helper_1 = require("../common/type-helper");
|
|
9
10
|
const item_execute_helpers_1 = require("./item-execute-helpers");
|
|
@@ -492,7 +493,7 @@ async function resolveItemsByFederatedId(client, incomingEntities, maxRetries) {
|
|
|
492
493
|
federatedIds.add(entity.itemOptionFederatedId);
|
|
493
494
|
}
|
|
494
495
|
const result = {};
|
|
495
|
-
const chunks = (0,
|
|
496
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(Array.from(federatedIds));
|
|
496
497
|
const limit = pLimit(5);
|
|
497
498
|
const promises = chunks.map((batch) => limit(async () => {
|
|
498
499
|
try {
|
|
@@ -527,7 +528,7 @@ async function fetchCurrentProjectItemsByItemId(client, projectId, itemIds, maxR
|
|
|
527
528
|
const result = {};
|
|
528
529
|
if (!itemIds.length)
|
|
529
530
|
return result;
|
|
530
|
-
const chunks = (0,
|
|
531
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(itemIds);
|
|
531
532
|
const limit = pLimit(5);
|
|
532
533
|
const promises = chunks.map((batch) => limit(async () => {
|
|
533
534
|
try {
|
|
@@ -192,10 +192,10 @@ function analyzeColumns(rows, typePropertyMaps, config) {
|
|
|
192
192
|
if (!rows?.length)
|
|
193
193
|
return { columns: [], unmappedTypeProperties: [] };
|
|
194
194
|
const entityTypes = (0, types_2.loadTypeToEntityTypes)(config.loadType);
|
|
195
|
-
const { typePropertiesBySlug, typePropertiesByLabel } = typePropertyMaps;
|
|
195
|
+
const { typePropertiesBySlug, typePropertiesByLabel, typePathStateBySlug } = typePropertyMaps;
|
|
196
196
|
const optionSetUsingTypePropertiesBySlug = typePropertyMaps.optionSetUsingTypePropertiesBySlug;
|
|
197
197
|
const csvHeaders = extractUniqueHeaders(rows);
|
|
198
|
-
const columns = csvHeaders.map((header) => classifyColumn(header, typePropertiesBySlug, typePropertiesByLabel, optionSetUsingTypePropertiesBySlug, entityTypes));
|
|
198
|
+
const columns = csvHeaders.map((header) => classifyColumn(header, typePropertiesBySlug, typePropertiesByLabel, optionSetUsingTypePropertiesBySlug, entityTypes, typePathStateBySlug));
|
|
199
199
|
const coveredSlugs = new Set();
|
|
200
200
|
for (const column of columns) {
|
|
201
201
|
if (column.matchedSlug) {
|
|
@@ -220,7 +220,15 @@ function analyzeColumns(rows, typePropertyMaps, config) {
|
|
|
220
220
|
}
|
|
221
221
|
}
|
|
222
222
|
const unmappedTypeProperties = Object.entries(typePropertiesBySlug)
|
|
223
|
-
.filter(([slug, prop]) =>
|
|
223
|
+
.filter(([slug, prop]) => {
|
|
224
|
+
const trimmedSlug = slug.trim();
|
|
225
|
+
if (coveredSlugs.has(trimmedSlug))
|
|
226
|
+
return false;
|
|
227
|
+
if (typePathStateBySlug) {
|
|
228
|
+
return !isArchivedAcrossAllSubtypes(trimmedSlug, typePathStateBySlug);
|
|
229
|
+
}
|
|
230
|
+
return !prop.isArchived;
|
|
231
|
+
})
|
|
224
232
|
.map(([slug, prop]) => ({
|
|
225
233
|
slug: slug.trim(),
|
|
226
234
|
label: prop.label,
|
|
@@ -243,7 +251,7 @@ function extractUniqueHeaders(rows) {
|
|
|
243
251
|
}
|
|
244
252
|
return headers;
|
|
245
253
|
}
|
|
246
|
-
function classifyColumn(header, typePropertiesBySlug, typePropertiesByLabel, optionSetProps, entityTypes) {
|
|
254
|
+
function classifyColumn(header, typePropertiesBySlug, typePropertiesByLabel, optionSetProps, entityTypes, typePathStateBySlug) {
|
|
247
255
|
const isItemLoad = entityTypes.includes(types_1.EntityType.ITEM) || entityTypes.includes(types_1.EntityType.PROJECT_ITEM);
|
|
248
256
|
const isEntityTypeSystemProp = entityTypes.some((et) => ENTITY_TYPE_SYSTEM_PROPERTIES[et]?.has(header));
|
|
249
257
|
const isSystemProp = SYSTEM_PROPERTIES.has(header) || isEntityTypeSystemProp;
|
|
@@ -253,7 +261,14 @@ function classifyColumn(header, typePropertiesBySlug, typePropertiesByLabel, opt
|
|
|
253
261
|
const baseSlugMatch = baseName !== header ? typePropertiesBySlug[baseName] : undefined;
|
|
254
262
|
const matchedProp = slugMatch || labelMatch || baseSlugMatch;
|
|
255
263
|
const isMapped = !!matchedProp || isSystemProp;
|
|
256
|
-
|
|
264
|
+
let isArchived;
|
|
265
|
+
if (matchedProp && typePathStateBySlug) {
|
|
266
|
+
const matchedSlug = (matchedProp.slug || '').trim();
|
|
267
|
+
isArchived = isArchivedAcrossAllSubtypes(matchedSlug, typePathStateBySlug);
|
|
268
|
+
}
|
|
269
|
+
else {
|
|
270
|
+
isArchived = matchedProp?.isArchived === true;
|
|
271
|
+
}
|
|
257
272
|
const hasOptionSet = !!(optionSetProps[header] || optionSetProps[baseName]);
|
|
258
273
|
let isRequired;
|
|
259
274
|
let requiredNote;
|
|
@@ -292,6 +307,16 @@ function stripLevelSuffix(name) {
|
|
|
292
307
|
return name.slice(0, -6);
|
|
293
308
|
return name;
|
|
294
309
|
}
|
|
310
|
+
function isArchivedAcrossAllSubtypes(slug, typePathStateBySlug) {
|
|
311
|
+
const states = typePathStateBySlug[slug];
|
|
312
|
+
if (!states || states.size === 0)
|
|
313
|
+
return false;
|
|
314
|
+
for (const state of states.values()) {
|
|
315
|
+
if (!state.isDisabled)
|
|
316
|
+
return false;
|
|
317
|
+
}
|
|
318
|
+
return true;
|
|
319
|
+
}
|
|
295
320
|
const SUBTYPE_ENTITY_TYPES = new Set([types_1.EntityType.ITEM, types_1.EntityType.CUSTOM_ENTITY]);
|
|
296
321
|
const SUBTYPE_PREFIX = {
|
|
297
322
|
[types_1.EntityType.ITEM]: 'item',
|
|
@@ -344,12 +369,13 @@ async function fetchTypeDefinitions(client, rows, config) {
|
|
|
344
369
|
}
|
|
345
370
|
const typeRoots = (0, types_2.entityTypesToTypeRoots)(entityTypes);
|
|
346
371
|
const reducedTypes = await (0, data_transformer_functions_1.getReducedTypes)(typeRoots, client);
|
|
347
|
-
const { typePropertiesBySlug, typePropertiesByLabel } = (0, data_transformer_functions_1.getPropertiesBySlugAndLabel)(reducedTypes);
|
|
372
|
+
const { typePropertiesBySlug, typePropertiesByLabel, typePathStateBySlug } = (0, data_transformer_functions_1.getPropertiesBySlugAndLabel)(reducedTypes);
|
|
348
373
|
const optionSetUsingTypePropertiesBySlug = (0, data_transformer_functions_1.getOptionSetsUsingTypePropertiesBySlug)(reducedTypes);
|
|
349
374
|
const typePropertyMaps = {
|
|
350
375
|
typePropertiesBySlug,
|
|
351
376
|
typePropertiesByLabel,
|
|
352
377
|
optionSetUsingTypePropertiesBySlug,
|
|
378
|
+
typePathStateBySlug,
|
|
353
379
|
};
|
|
354
380
|
const typesByPath = await (0, type_helper_1.buildTypesByPathForPipeline)(client, Array.from(typePaths));
|
|
355
381
|
return { typePropertyMaps, typesByPath, generalErrors };
|
|
@@ -49,7 +49,11 @@ describe('transform', () => {
|
|
|
49
49
|
getReducedTypesSpy = jest.spyOn(dataTransformerFunctions, 'getReducedTypes')
|
|
50
50
|
.mockResolvedValue([]);
|
|
51
51
|
getPropertiesBySlugAndLabelSpy = jest.spyOn(dataTransformerFunctions, 'getPropertiesBySlugAndLabel')
|
|
52
|
-
.mockReturnValue({
|
|
52
|
+
.mockReturnValue({
|
|
53
|
+
typePropertiesBySlug: {},
|
|
54
|
+
typePropertiesByLabel: {},
|
|
55
|
+
typePathStateBySlug: {},
|
|
56
|
+
});
|
|
53
57
|
getOptionSetsUsingTypePropertiesBySlugSpy = jest.spyOn(dataTransformerFunctions, 'getOptionSetsUsingTypePropertiesBySlug')
|
|
54
58
|
.mockReturnValue({});
|
|
55
59
|
});
|
|
@@ -170,6 +174,7 @@ describe('transform', () => {
|
|
|
170
174
|
active: { slug: 'active', propertyType: 'boolean' },
|
|
171
175
|
},
|
|
172
176
|
typePropertiesByLabel: {},
|
|
177
|
+
typePathStateBySlug: {},
|
|
173
178
|
});
|
|
174
179
|
const rows = [{ active: 'true', unmappedField: 'true' }];
|
|
175
180
|
const result = await (0, transform_1.transform)(mockClient, rows, baseConfig);
|
|
@@ -294,6 +299,11 @@ describe('transform', () => {
|
|
|
294
299
|
getPropertiesBySlugAndLabelSpy.mockReturnValue({
|
|
295
300
|
typePropertiesBySlug: columnAnalysisTypeMaps.typePropertiesBySlug,
|
|
296
301
|
typePropertiesByLabel: columnAnalysisTypeMaps.typePropertiesByLabel,
|
|
302
|
+
typePathStateBySlug: {
|
|
303
|
+
oldProp: new Map([['color', { isDisabled: true }]]),
|
|
304
|
+
color: new Map([['color', { isDisabled: false }]]),
|
|
305
|
+
size: new Map([['color', { isDisabled: false }]]),
|
|
306
|
+
},
|
|
297
307
|
});
|
|
298
308
|
getOptionSetsUsingTypePropertiesBySlugSpy.mockReturnValue(columnAnalysisTypeMaps.optionSetUsingTypePropertiesBySlug);
|
|
299
309
|
}
|
|
@@ -328,6 +338,59 @@ describe('transform', () => {
|
|
|
328
338
|
expect(archivedCol.isArchived).toBe(true);
|
|
329
339
|
expect(archivedCol.isMapped).toBe(true);
|
|
330
340
|
});
|
|
341
|
+
it('should not mark a column as archived when the property is active in any subtype', async () => {
|
|
342
|
+
getPropertiesBySlugAndLabelSpy.mockReturnValue({
|
|
343
|
+
typePropertiesBySlug: {
|
|
344
|
+
season: { slug: 'season', label: 'Season', propertyType: 'string', isArchived: false },
|
|
345
|
+
},
|
|
346
|
+
typePropertiesByLabel: {
|
|
347
|
+
Season: { slug: 'season', label: 'Season', propertyType: 'string', isArchived: false },
|
|
348
|
+
},
|
|
349
|
+
typePathStateBySlug: {
|
|
350
|
+
season: new Map([
|
|
351
|
+
['item:product', { isDisabled: false }],
|
|
352
|
+
['item:material', { isDisabled: true }],
|
|
353
|
+
]),
|
|
354
|
+
},
|
|
355
|
+
});
|
|
356
|
+
getOptionSetsUsingTypePropertiesBySlugSpy.mockReturnValue({});
|
|
357
|
+
const config = {
|
|
358
|
+
loadType: [types_1.LoadType.ITEM],
|
|
359
|
+
conditionalColumns: [{ toProperty: 'typePath', default: 'item:product' }],
|
|
360
|
+
};
|
|
361
|
+
const rows = [{ itemFamilyFederatedId: 'fam-1', name: 'Test', season: 'SS25' }];
|
|
362
|
+
const result = await (0, transform_1.transform)(mockClient, rows, config);
|
|
363
|
+
const seasonCol = result.columns.find((c) => c.csvHeader === 'season');
|
|
364
|
+
expect(seasonCol).toBeDefined();
|
|
365
|
+
expect(seasonCol.isArchived).toBe(false);
|
|
366
|
+
expect(seasonCol.isMapped).toBe(true);
|
|
367
|
+
});
|
|
368
|
+
it('should mark a column as archived only when archived in every subtype that defines it', async () => {
|
|
369
|
+
getPropertiesBySlugAndLabelSpy.mockReturnValue({
|
|
370
|
+
typePropertiesBySlug: {
|
|
371
|
+
legacyProp: { slug: 'legacyProp', label: 'Legacy Prop', propertyType: 'string', isArchived: true },
|
|
372
|
+
},
|
|
373
|
+
typePropertiesByLabel: {
|
|
374
|
+
'Legacy Prop': { slug: 'legacyProp', label: 'Legacy Prop', propertyType: 'string', isArchived: true },
|
|
375
|
+
},
|
|
376
|
+
typePathStateBySlug: {
|
|
377
|
+
legacyProp: new Map([
|
|
378
|
+
['item:product', { isDisabled: true }],
|
|
379
|
+
['item:material', { isDisabled: true }],
|
|
380
|
+
]),
|
|
381
|
+
},
|
|
382
|
+
});
|
|
383
|
+
getOptionSetsUsingTypePropertiesBySlugSpy.mockReturnValue({});
|
|
384
|
+
const config = {
|
|
385
|
+
loadType: [types_1.LoadType.ITEM],
|
|
386
|
+
conditionalColumns: [{ toProperty: 'typePath', default: 'item:product' }],
|
|
387
|
+
};
|
|
388
|
+
const rows = [{ itemFamilyFederatedId: 'fam-1', name: 'Test', legacyProp: 'value' }];
|
|
389
|
+
const result = await (0, transform_1.transform)(mockClient, rows, config);
|
|
390
|
+
const legacyCol = result.columns.find((c) => c.csvHeader === 'legacyProp');
|
|
391
|
+
expect(legacyCol).toBeDefined();
|
|
392
|
+
expect(legacyCol.isArchived).toBe(true);
|
|
393
|
+
});
|
|
331
394
|
it('should mark system properties as required for non-item loads', async () => {
|
|
332
395
|
setupColumnAnalysisMocks();
|
|
333
396
|
const config = { loadType: [types_1.LoadType.COLOR] };
|
package/oclif.manifest.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@contrail/cli",
|
|
3
|
-
"version": "3.4.
|
|
3
|
+
"version": "3.4.11-alpha.feeaf0b",
|
|
4
4
|
"description": "VibeIQ's Contrail Platform CLI.",
|
|
5
5
|
"bin": {
|
|
6
6
|
"contrail-cli": "./bin/run",
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
],
|
|
32
32
|
"dependencies": {
|
|
33
33
|
"@contrail/entity-types": "^1.6.1",
|
|
34
|
-
"@contrail/loader": "^2.0.
|
|
34
|
+
"@contrail/loader": "^2.0.6",
|
|
35
35
|
"@contrail/org-config-migration": "^1.1.13",
|
|
36
36
|
"@contrail/sdk": "^1.5.5",
|
|
37
37
|
"@contrail/types": "^3.1.5",
|