@contrail/cli 3.4.11-alpha.feeaf0b → 3.4.12-alpha.54199ed
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/node_modules/@contrail/loader/lib/color-processor.js +9 -23
- package/node_modules/@contrail/loader/lib/color-processor.spec.js +112 -62
- package/node_modules/@contrail/loader/lib/common/color-validation.d.ts +26 -0
- package/node_modules/@contrail/loader/lib/common/color-validation.js +163 -0
- package/node_modules/@contrail/loader/lib/common/color-validation.spec.d.ts +1 -0
- package/node_modules/@contrail/loader/lib/common/color-validation.spec.js +102 -0
- package/node_modules/@contrail/loader/lib/constants.d.ts +0 -3
- package/node_modules/@contrail/loader/lib/constants.js +1 -4
- package/node_modules/@contrail/loader/lib/entities-v2/colors-v2/color-processor-v2.js +9 -23
- package/node_modules/@contrail/loader/lib/entities-v2/colors-v2/color-processor-v2.spec.js +112 -62
- package/node_modules/@contrail/loader/lib/entities-v2/linesheet-v2/item-processor.js +8 -22
- package/node_modules/@contrail/loader/lib/entities-v2/linesheet-v2/item-processor.spec.js +43 -1
- package/node_modules/@contrail/loader/lib/pipeline/preview.js +0 -5
- package/node_modules/@contrail/loader/lib/pipeline/preview.spec.js +2 -2
- package/node_modules/@contrail/loader/lib/pipeline/types.d.ts +3 -1
- package/node_modules/@contrail/loader/lib/pipeline/types.js +2 -0
- package/node_modules/@contrail/loader/lib/pipeline/validate.js +124 -2
- package/node_modules/@contrail/loader/lib/pipeline/validate.spec.js +104 -0
- package/node_modules/@contrail/loader/package.json +3 -2
- package/node_modules/validate-color/LICENSE +21 -0
- package/node_modules/validate-color/README.md +265 -0
- package/node_modules/validate-color/lib/index.d.ts +10 -0
- package/node_modules/validate-color/lib/index.js +1 -0
- package/node_modules/validate-color/package.json +91 -0
- package/node_modules/validate-color/test/redos.js +29 -0
- package/oclif.manifest.json +1 -1
- package/package.json +2 -2
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
const constants_1 = require("../../constants");
|
|
4
4
|
const color_processor_v2_1 = require("./color-processor-v2");
|
|
5
5
|
const entity_types_1 = require("@contrail/entity-types");
|
|
6
|
+
const color_validation_1 = require("../../common/color-validation");
|
|
6
7
|
describe('ColorProcessorV2', () => {
|
|
7
8
|
const loaderProcess = {
|
|
8
9
|
id: 'mock-loader-process-id',
|
|
@@ -219,72 +220,38 @@ describe('ColorProcessorV2', () => {
|
|
|
219
220
|
});
|
|
220
221
|
});
|
|
221
222
|
describe('color column validation', () => {
|
|
222
|
-
it('
|
|
223
|
+
it('warns per-row when both hex and RGB are populated (hex prioritized)', () => {
|
|
223
224
|
const data = [
|
|
224
|
-
{
|
|
225
|
-
name: 'test-color',
|
|
226
|
-
red: '255',
|
|
227
|
-
green: '0',
|
|
228
|
-
blue: '0',
|
|
229
|
-
hex: '#ff0000',
|
|
230
|
-
},
|
|
225
|
+
{ name: 'test-color', red: '255', green: '0', blue: '0', hex: '#ff0000' },
|
|
231
226
|
];
|
|
232
|
-
const
|
|
233
|
-
|
|
234
|
-
federatedMappings: {
|
|
235
|
-
...loaderProcess.federatedMappings,
|
|
236
|
-
red: 'red',
|
|
237
|
-
green: 'green',
|
|
238
|
-
blue: 'blue',
|
|
239
|
-
hex: 'hex',
|
|
240
|
-
},
|
|
241
|
-
};
|
|
242
|
-
const result = color_processor_v2_1.ColorProcessorV2.validateLoadFile(data, config);
|
|
243
|
-
expect(result.generalWarnings).toContainEqual({
|
|
244
|
-
message: constants_1.COLOR_PROCESSOR_WARNINGS.HEX_IGNORED_WARNING,
|
|
245
|
-
issueType: entity_types_1.IssueType.CONFIG,
|
|
246
|
-
});
|
|
247
|
-
});
|
|
248
|
-
it('should not warn when only RGB columns are present', () => {
|
|
249
|
-
const data = [
|
|
227
|
+
const result = color_processor_v2_1.ColorProcessorV2.validateLoadFile(data, loaderProcess);
|
|
228
|
+
expect(result.warnings).toEqual([
|
|
250
229
|
{
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
...loaderProcess.federatedMappings,
|
|
261
|
-
red: 'red',
|
|
262
|
-
green: 'green',
|
|
263
|
-
blue: 'blue',
|
|
230
|
+
rowNumber: 2,
|
|
231
|
+
columnIssues: [
|
|
232
|
+
{
|
|
233
|
+
columnName: 'hex',
|
|
234
|
+
message: color_validation_1.COLOR_VALIDATION_MESSAGES.HEX_PRIORITIZED_OVER_RGB,
|
|
235
|
+
issueType: entity_types_1.IssueType.INVALID_DATA,
|
|
236
|
+
data: JSON.stringify(data[0]),
|
|
237
|
+
},
|
|
238
|
+
],
|
|
264
239
|
},
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
240
|
+
]);
|
|
241
|
+
});
|
|
242
|
+
it('does not warn when only RGB is populated and all components are valid', () => {
|
|
243
|
+
const data = [{ name: 'test-color', red: '255', green: '0', blue: '0' }];
|
|
244
|
+
const result = color_processor_v2_1.ColorProcessorV2.validateLoadFile(data, loaderProcess);
|
|
245
|
+
expect(result.warnings).toEqual([]);
|
|
246
|
+
expect(result.errors).toEqual([]);
|
|
271
247
|
});
|
|
272
|
-
it('
|
|
248
|
+
it('does not warn when only a valid hex is populated', () => {
|
|
273
249
|
const data = [{ name: 'test-color', hex: '#ff0000' }];
|
|
274
|
-
const
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
...loaderProcess.federatedMappings,
|
|
278
|
-
hex: 'hex',
|
|
279
|
-
},
|
|
280
|
-
};
|
|
281
|
-
const result = color_processor_v2_1.ColorProcessorV2.validateLoadFile(data, config);
|
|
282
|
-
expect(result.generalWarnings).not.toContainEqual({
|
|
283
|
-
message: constants_1.COLOR_PROCESSOR_WARNINGS.HEX_IGNORED_WARNING,
|
|
284
|
-
issueType: entity_types_1.IssueType.CONFIG,
|
|
285
|
-
});
|
|
250
|
+
const result = color_processor_v2_1.ColorProcessorV2.validateLoadFile(data, loaderProcess);
|
|
251
|
+
expect(result.warnings).toEqual([]);
|
|
252
|
+
expect(result.errors).toEqual([]);
|
|
286
253
|
});
|
|
287
|
-
it('
|
|
254
|
+
it('warns per-row when both hex and RGB are populated under federatedMappings', () => {
|
|
288
255
|
const data = [{ name: 'test-color', r: '255', g: '0', b: '0', hexcode: '#ff0000' }];
|
|
289
256
|
const config = {
|
|
290
257
|
...loaderProcess,
|
|
@@ -297,11 +264,94 @@ describe('ColorProcessorV2', () => {
|
|
|
297
264
|
},
|
|
298
265
|
};
|
|
299
266
|
const result = color_processor_v2_1.ColorProcessorV2.validateLoadFile(data, config);
|
|
300
|
-
expect(result.
|
|
301
|
-
|
|
302
|
-
|
|
267
|
+
expect(result.warnings).toEqual([
|
|
268
|
+
{
|
|
269
|
+
rowNumber: 2,
|
|
270
|
+
columnIssues: [
|
|
271
|
+
{
|
|
272
|
+
columnName: 'hex',
|
|
273
|
+
message: color_validation_1.COLOR_VALIDATION_MESSAGES.HEX_PRIORITIZED_OVER_RGB,
|
|
274
|
+
issueType: entity_types_1.IssueType.INVALID_DATA,
|
|
275
|
+
data: JSON.stringify(data[0]),
|
|
276
|
+
},
|
|
277
|
+
],
|
|
278
|
+
},
|
|
279
|
+
]);
|
|
280
|
+
});
|
|
281
|
+
it('errors when hex is invalid', () => {
|
|
282
|
+
const data = [{ name: 'bad', hex: 'notahex' }];
|
|
283
|
+
const result = color_processor_v2_1.ColorProcessorV2.validateLoadFile(data, loaderProcess);
|
|
284
|
+
expect(result.errors).toHaveLength(1);
|
|
285
|
+
expect(result.errors[0]).toMatchObject({
|
|
286
|
+
rowNumber: 2,
|
|
287
|
+
columnIssues: [
|
|
288
|
+
{
|
|
289
|
+
columnName: 'hex',
|
|
290
|
+
message: color_validation_1.COLOR_VALIDATION_MESSAGES.HEX_INVALID('notahex'),
|
|
291
|
+
issueType: entity_types_1.IssueType.INVALID_DATA,
|
|
292
|
+
},
|
|
293
|
+
],
|
|
303
294
|
});
|
|
304
295
|
});
|
|
296
|
+
it('errors when RGB is partially populated and hex is missing', () => {
|
|
297
|
+
const data = [{ name: 'partial', red: '255', green: '128' }];
|
|
298
|
+
const result = color_processor_v2_1.ColorProcessorV2.validateLoadFile(data, loaderProcess);
|
|
299
|
+
expect(result.errors).toHaveLength(1);
|
|
300
|
+
expect(result.errors[0].rowNumber).toBe(2);
|
|
301
|
+
expect(result.errors[0].columnIssues).toEqual([
|
|
302
|
+
{
|
|
303
|
+
columnName: 'blue',
|
|
304
|
+
message: color_validation_1.COLOR_VALIDATION_MESSAGES.RGB_INCOMPLETE(['blue']),
|
|
305
|
+
issueType: entity_types_1.IssueType.INVALID_DATA,
|
|
306
|
+
data: JSON.stringify(data[0]),
|
|
307
|
+
},
|
|
308
|
+
]);
|
|
309
|
+
});
|
|
310
|
+
it('errors when an RGB component is out of range or non-numeric', () => {
|
|
311
|
+
const data = [
|
|
312
|
+
{ name: 'bad-red', red: '300', green: '0', blue: '0' },
|
|
313
|
+
{ name: 'nan-green', red: '10', green: 'foo', blue: '0' },
|
|
314
|
+
];
|
|
315
|
+
const result = color_processor_v2_1.ColorProcessorV2.validateLoadFile(data, loaderProcess);
|
|
316
|
+
expect(result.errors).toHaveLength(2);
|
|
317
|
+
expect(result.errors[0]).toMatchObject({
|
|
318
|
+
rowNumber: 2,
|
|
319
|
+
columnIssues: [
|
|
320
|
+
{ columnName: 'red', message: color_validation_1.COLOR_VALIDATION_MESSAGES.RGB_OUT_OF_RANGE('red', '300') },
|
|
321
|
+
],
|
|
322
|
+
});
|
|
323
|
+
expect(result.errors[1]).toMatchObject({
|
|
324
|
+
rowNumber: 3,
|
|
325
|
+
columnIssues: [
|
|
326
|
+
{ columnName: 'green', message: color_validation_1.COLOR_VALIDATION_MESSAGES.RGB_OUT_OF_RANGE('green', 'foo') },
|
|
327
|
+
],
|
|
328
|
+
});
|
|
329
|
+
});
|
|
330
|
+
it('does not error when neither hex nor RGB are populated (defaults to black)', () => {
|
|
331
|
+
const data = [{ name: 'no-color' }];
|
|
332
|
+
const result = color_processor_v2_1.ColorProcessorV2.validateLoadFile(data, loaderProcess);
|
|
333
|
+
expect(result.errors).toEqual([]);
|
|
334
|
+
expect(result.warnings).toEqual([]);
|
|
335
|
+
});
|
|
336
|
+
});
|
|
337
|
+
describe('transformData (hex priority)', () => {
|
|
338
|
+
it('strips red/green/blue when hex is valid', () => {
|
|
339
|
+
const data = [
|
|
340
|
+
{ hex: '#ff0000', red: '255', green: '0', blue: '0', name: 'red' },
|
|
341
|
+
];
|
|
342
|
+
const result = color_processor_v2_1.ColorProcessorV2.transformData(data);
|
|
343
|
+
expect(result).toEqual([{ hex: 'ff0000', name: 'red' }]);
|
|
344
|
+
});
|
|
345
|
+
it('keeps red/green/blue and removes empty hex when hex is missing', () => {
|
|
346
|
+
const data = [{ red: '255', green: '0', blue: '0', name: 'red' }];
|
|
347
|
+
const result = color_processor_v2_1.ColorProcessorV2.transformData(data);
|
|
348
|
+
expect(result).toEqual([{ red: '255', green: '0', blue: '0', name: 'red' }]);
|
|
349
|
+
});
|
|
350
|
+
it('defaults to #000000 (without hash) when neither hex nor full RGB is provided', () => {
|
|
351
|
+
const data = [{ name: 'fallback' }];
|
|
352
|
+
const result = color_processor_v2_1.ColorProcessorV2.transformData(data);
|
|
353
|
+
expect(result).toEqual([{ name: 'fallback', hex: color_validation_1.DEFAULT_HEX }]);
|
|
354
|
+
});
|
|
305
355
|
});
|
|
306
356
|
it('should return error if conditionalColumns format is incorrect', () => {
|
|
307
357
|
const data = [{ name: 'color1' }];
|
|
@@ -60,7 +60,7 @@ class ItemProcessor extends base_processor_v2_1.BaseProcessorV2 {
|
|
|
60
60
|
return { errors, warnings, generalErrors, generalWarnings };
|
|
61
61
|
}
|
|
62
62
|
this.validateLoadType(config, generalErrors);
|
|
63
|
-
this.validateTypePathMapping(
|
|
63
|
+
this.validateTypePathMapping(data, generalErrors);
|
|
64
64
|
this.validateOptionNameMapping(config, csvHeaders, generalWarnings);
|
|
65
65
|
this.validateItemOptionFederatedIdMapping(config, csvHeaders, generalWarnings);
|
|
66
66
|
this.validateFederatedIds(data, config, errors, generalErrors, csvHeaders);
|
|
@@ -75,27 +75,13 @@ class ItemProcessor extends base_processor_v2_1.BaseProcessorV2 {
|
|
|
75
75
|
});
|
|
76
76
|
}
|
|
77
77
|
}
|
|
78
|
-
static validateTypePathMapping(
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
generalErrors.push({
|
|
86
|
-
message: constants_1.ITEM_PROCESSOR_ERRORS.TYPEPATH_INVALID,
|
|
87
|
-
issueType: entity_types_1.IssueType.CONFIG,
|
|
88
|
-
});
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
else if (csvHeaders.includes('typePath')) {
|
|
92
|
-
const hasCorrectTypePath = data.every((row) => row.typePath === 'item' || (row.typePath?.startsWith('item:') && row.typePath?.split(':')[1]?.length > 0));
|
|
93
|
-
if (!hasCorrectTypePath) {
|
|
94
|
-
generalErrors.push({
|
|
95
|
-
message: constants_1.ITEM_PROCESSOR_ERRORS.TYPEPATH_INVALID,
|
|
96
|
-
issueType: entity_types_1.IssueType.CONFIG,
|
|
97
|
-
});
|
|
98
|
-
}
|
|
78
|
+
static validateTypePathMapping(data, generalErrors) {
|
|
79
|
+
const hasCorrectTypePath = data.every((row) => row.typePath === 'item' || (row.typePath?.startsWith('item:') && row.typePath?.split(':')[1]?.length > 0));
|
|
80
|
+
if (!hasCorrectTypePath) {
|
|
81
|
+
generalErrors.push({
|
|
82
|
+
message: constants_1.ITEM_PROCESSOR_ERRORS.TYPEPATH_INVALID,
|
|
83
|
+
issueType: entity_types_1.IssueType.CONFIG,
|
|
84
|
+
});
|
|
99
85
|
}
|
|
100
86
|
}
|
|
101
87
|
static validateOptionNameMapping(config, csvHeaders, generalWarnings) {
|
|
@@ -28,18 +28,21 @@ describe('Item Processor', () => {
|
|
|
28
28
|
family_federated_id: 'family_1',
|
|
29
29
|
input_col_1: 'value_1',
|
|
30
30
|
input_col_2: 'value_2',
|
|
31
|
+
typePath: 'item',
|
|
31
32
|
},
|
|
32
33
|
{
|
|
33
34
|
name: 'item2',
|
|
34
35
|
family_federated_id: 'family_2',
|
|
35
36
|
input_col_1: 'value_3',
|
|
36
37
|
input_col_2: 'value_4',
|
|
38
|
+
typePath: 'item',
|
|
37
39
|
},
|
|
38
40
|
{
|
|
39
41
|
name: 'item3',
|
|
40
42
|
family_federated_id: 'family_3',
|
|
41
43
|
input_col_1: 'value_5',
|
|
42
44
|
input_col_2: 'value_6',
|
|
45
|
+
typePath: 'item',
|
|
43
46
|
},
|
|
44
47
|
];
|
|
45
48
|
const config = {
|
|
@@ -167,7 +170,7 @@ describe('Item Processor', () => {
|
|
|
167
170
|
},
|
|
168
171
|
]);
|
|
169
172
|
});
|
|
170
|
-
it('should
|
|
173
|
+
it('should throw error if typepath is not present in csv header', () => {
|
|
171
174
|
const data = [
|
|
172
175
|
{
|
|
173
176
|
name: 'item1',
|
|
@@ -193,6 +196,42 @@ describe('Item Processor', () => {
|
|
|
193
196
|
conditionalColumns: [],
|
|
194
197
|
};
|
|
195
198
|
const result = item_processor_1.ItemProcessor.validateLoadFile(data, config);
|
|
199
|
+
expect(result.generalErrors).toEqual([
|
|
200
|
+
{
|
|
201
|
+
message: constants_1.ITEM_PROCESSOR_ERRORS.TYPEPATH_INVALID,
|
|
202
|
+
issueType: entity_types_1.IssueType.CONFIG,
|
|
203
|
+
},
|
|
204
|
+
]);
|
|
205
|
+
});
|
|
206
|
+
it('should not throw error if every row has a valid typePath', () => {
|
|
207
|
+
const data = [
|
|
208
|
+
{
|
|
209
|
+
name: 'item1',
|
|
210
|
+
family_federated_id: 'family_1',
|
|
211
|
+
input_col_1: 'value_1',
|
|
212
|
+
input_col_2: 'value_2',
|
|
213
|
+
typePath: 'item',
|
|
214
|
+
},
|
|
215
|
+
{
|
|
216
|
+
name: 'item2',
|
|
217
|
+
family_federated_id: 'family_2',
|
|
218
|
+
input_col_1: 'value_3',
|
|
219
|
+
input_col_2: 'value_4',
|
|
220
|
+
typePath: 'item:material:fabric:yarn',
|
|
221
|
+
},
|
|
222
|
+
{
|
|
223
|
+
name: 'item3',
|
|
224
|
+
family_federated_id: 'family_3',
|
|
225
|
+
input_col_1: 'value_5',
|
|
226
|
+
input_col_2: 'value_6',
|
|
227
|
+
typePath: 'item:material:fabric:leather',
|
|
228
|
+
},
|
|
229
|
+
];
|
|
230
|
+
const config = {
|
|
231
|
+
...loaderProcess,
|
|
232
|
+
conditionalColumns: [],
|
|
233
|
+
};
|
|
234
|
+
const result = item_processor_1.ItemProcessor.validateLoadFile(data, config);
|
|
196
235
|
expect(result.generalErrors).toEqual([]);
|
|
197
236
|
});
|
|
198
237
|
});
|
|
@@ -600,14 +639,17 @@ describe('Item Processor', () => {
|
|
|
600
639
|
{
|
|
601
640
|
itemFamilyFederatedId: '1',
|
|
602
641
|
name: 'foo',
|
|
642
|
+
typePath: 'item',
|
|
603
643
|
},
|
|
604
644
|
{
|
|
605
645
|
itemFamilyFederatedId: '1',
|
|
606
646
|
name: 'bar',
|
|
647
|
+
typePath: 'item',
|
|
607
648
|
},
|
|
608
649
|
{
|
|
609
650
|
itemFamilyFederatedId: '3',
|
|
610
651
|
name: 'test',
|
|
652
|
+
typePath: 'item',
|
|
611
653
|
},
|
|
612
654
|
];
|
|
613
655
|
const config = {
|
|
@@ -21,11 +21,6 @@ async function preview(client, rows, config, columnAnalysis) {
|
|
|
21
21
|
const itemsByFederatedId = entityType !== types_1.EntityType.ITEM ? { ...sharedItemsByFederatedId } : {};
|
|
22
22
|
const clonedRows = rows.map(row => ({ ...row }));
|
|
23
23
|
const result = await entity_processor_v2_1.EntityProcessorV2.setupEntities(client, entityType, clonedRows, undefined, loaderProcess, itemsByFederatedId, doConstructEntities, columnAnalysis.typesByPath ? buildTypesByTypePathCache(columnAnalysis) : undefined);
|
|
24
|
-
if (result.generalWarnings?.length) {
|
|
25
|
-
for (const warning of result.generalWarnings) {
|
|
26
|
-
allWarnings.push(warning.message);
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
24
|
if (result.badEntities?.length) {
|
|
30
25
|
for (let i = 0; i < result.badEntities.length; i++) {
|
|
31
26
|
const badEntity = result.badEntities[i];
|
|
@@ -122,7 +122,7 @@ describe('preview (non-item entity types)', () => {
|
|
|
122
122
|
expect(result.staleRows[0].details?.incomingTimestamp).toBe('2024-01-01T00:00:00Z');
|
|
123
123
|
expect(result.staleRows[0].details?.existingTimestamp).toBe('2024-06-01T00:00:00Z');
|
|
124
124
|
});
|
|
125
|
-
it('should
|
|
125
|
+
it('should not duplicate object-reference resolution warnings (now surfaced by validate)', async () => {
|
|
126
126
|
entity_processor_v2_1.EntityProcessorV2.setupEntities.mockResolvedValue({
|
|
127
127
|
incomingEntities: [],
|
|
128
128
|
badEntities: [],
|
|
@@ -135,7 +135,7 @@ describe('preview (non-item entity types)', () => {
|
|
|
135
135
|
});
|
|
136
136
|
const rows = [{ federatedId: '1' }];
|
|
137
137
|
const result = await (0, preview_1.preview)(mockClient, rows, baseConfig, baseAnalysis);
|
|
138
|
-
expect(result.warnings).toContain('ObjectReference not found for vendor with value acme');
|
|
138
|
+
expect(result.warnings).not.toContain('ObjectReference not found for vendor with value acme');
|
|
139
139
|
});
|
|
140
140
|
it('should store constructed entities for execute stage', async () => {
|
|
141
141
|
entity_processor_v2_1.EntityProcessorV2.setupEntities.mockResolvedValue({
|
|
@@ -41,7 +41,9 @@ export declare enum PreValidationCategory {
|
|
|
41
41
|
SELF_REFERENCE = "SELF_REFERENCE",
|
|
42
42
|
UNMAPPED_COLUMN = "UNMAPPED_COLUMN",
|
|
43
43
|
UNMAPPED_TYPE_PROPERTY = "UNMAPPED_TYPE_PROPERTY",
|
|
44
|
-
FORMAT_NUMBER_PRECISION = "FORMAT_NUMBER_PRECISION"
|
|
44
|
+
FORMAT_NUMBER_PRECISION = "FORMAT_NUMBER_PRECISION",
|
|
45
|
+
INVALID_DATA = "INVALID_DATA",
|
|
46
|
+
DUPLICATE_FEDERATED_ID = "DUPLICATE_FEDERATED_ID"
|
|
45
47
|
}
|
|
46
48
|
export declare enum PreValidationSeverity {
|
|
47
49
|
ERROR = "error",
|
|
@@ -49,6 +49,8 @@ var PreValidationCategory;
|
|
|
49
49
|
PreValidationCategory["UNMAPPED_COLUMN"] = "UNMAPPED_COLUMN";
|
|
50
50
|
PreValidationCategory["UNMAPPED_TYPE_PROPERTY"] = "UNMAPPED_TYPE_PROPERTY";
|
|
51
51
|
PreValidationCategory["FORMAT_NUMBER_PRECISION"] = "FORMAT_NUMBER_PRECISION";
|
|
52
|
+
PreValidationCategory["INVALID_DATA"] = "INVALID_DATA";
|
|
53
|
+
PreValidationCategory["DUPLICATE_FEDERATED_ID"] = "DUPLICATE_FEDERATED_ID";
|
|
52
54
|
})(PreValidationCategory = exports.PreValidationCategory || (exports.PreValidationCategory = {}));
|
|
53
55
|
var PreValidationSeverity;
|
|
54
56
|
(function (PreValidationSeverity) {
|
|
@@ -2,15 +2,32 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.validate = void 0;
|
|
4
4
|
const types_1 = require("@contrail/types");
|
|
5
|
+
const lodash_1 = require("lodash");
|
|
5
6
|
const entity_processor_v2_1 = require("../entities-v2/entity-processor-v2");
|
|
6
7
|
const processor_registry_1 = require("../processor-registry");
|
|
7
8
|
const entities_helper_1 = require("../entities-v2/entities-helper");
|
|
9
|
+
const entities_helper_2 = require("../common/entities-helper");
|
|
10
|
+
const chunk_by_url_length_1 = require("../common/chunk-by-url-length");
|
|
11
|
+
const pLimit = require('p-limit');
|
|
8
12
|
const types_2 = require("./types");
|
|
9
13
|
const CONFIG_CATEGORY = types_2.PreValidationCategory.CONFIG;
|
|
10
14
|
const MISSING_REQUIRED_FIELD_CATEGORY = types_2.PreValidationCategory.MISSING_REQUIRED_FIELD;
|
|
11
15
|
const REFERENCE_NOT_FOUND_CATEGORY = types_2.PreValidationCategory.REFERENCE_DATA_NOT_FOUND;
|
|
12
16
|
const FAMILY_INCONSISTENT_CATEGORY = types_2.PreValidationCategory.FAMILY_PROPERTY_VALUES_INCONSISTENT;
|
|
13
17
|
const SELF_REFERENCE_CATEGORY = types_2.PreValidationCategory.SELF_REFERENCE;
|
|
18
|
+
function issueTypeToCategory(issueType) {
|
|
19
|
+
switch (issueType) {
|
|
20
|
+
case types_2.IssueType.INVALID_DATA:
|
|
21
|
+
return types_2.PreValidationCategory.INVALID_DATA;
|
|
22
|
+
case types_2.IssueType.MISSING_DATA:
|
|
23
|
+
return types_2.PreValidationCategory.MISSING_REQUIRED_FIELD;
|
|
24
|
+
case types_2.IssueType.DUPLICATE_FEDERATED_ID:
|
|
25
|
+
return types_2.PreValidationCategory.DUPLICATE_FEDERATED_ID;
|
|
26
|
+
case types_2.IssueType.CONFIG:
|
|
27
|
+
default:
|
|
28
|
+
return types_2.PreValidationCategory.CONFIG;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
14
31
|
function isEmpty(value) {
|
|
15
32
|
return value === null || value === undefined || value === '';
|
|
16
33
|
}
|
|
@@ -54,6 +71,10 @@ async function validate(rows, config, columnAnalysis, client) {
|
|
|
54
71
|
issues.push(...suffixIssues);
|
|
55
72
|
const selfRefIssues = validateSelfReferences(rows, config, columnAnalysis.typesByPath);
|
|
56
73
|
issues.push(...selfRefIssues);
|
|
74
|
+
if (client) {
|
|
75
|
+
const objectRefIssues = await validateObjectReferences(rows, columnAnalysis.typesByPath, client);
|
|
76
|
+
issues.push(...objectRefIssues);
|
|
77
|
+
}
|
|
57
78
|
const optionSetIssues = validateOptionSets(rows, columnAnalysis.typePropertyMaps);
|
|
58
79
|
issues.push(...optionSetIssues);
|
|
59
80
|
const booleanIssues = validateBooleanFormats(rows, columnAnalysis.typePropertyMaps);
|
|
@@ -143,7 +164,7 @@ function runProcessorValidation(rows, config) {
|
|
|
143
164
|
for (const colIssue of error.columnIssues ?? []) {
|
|
144
165
|
issues.push({
|
|
145
166
|
severity: types_2.PreValidationSeverity.ERROR,
|
|
146
|
-
category:
|
|
167
|
+
category: issueTypeToCategory(colIssue.issueType),
|
|
147
168
|
columnName: colIssue.columnName,
|
|
148
169
|
propertySlug: colIssue.columnName,
|
|
149
170
|
propertyType: '',
|
|
@@ -156,7 +177,7 @@ function runProcessorValidation(rows, config) {
|
|
|
156
177
|
for (const colIssue of warning.columnIssues ?? []) {
|
|
157
178
|
issues.push({
|
|
158
179
|
severity: types_2.PreValidationSeverity.WARNING,
|
|
159
|
-
category:
|
|
180
|
+
category: issueTypeToCategory(colIssue.issueType),
|
|
160
181
|
columnName: colIssue.columnName,
|
|
161
182
|
propertySlug: colIssue.columnName,
|
|
162
183
|
propertyType: '',
|
|
@@ -1130,6 +1151,107 @@ function buildObjectRefColumnChecks(typesByPath, isItemBased) {
|
|
|
1130
1151
|
}
|
|
1131
1152
|
return checks;
|
|
1132
1153
|
}
|
|
1154
|
+
async function validateObjectReferences(rows, typesByPath, client) {
|
|
1155
|
+
const objectRefColumns = [];
|
|
1156
|
+
for (const typePath of Object.keys(typesByPath ?? {})) {
|
|
1157
|
+
const typeObj = typesByPath[typePath];
|
|
1158
|
+
for (const prop of typeObj?.objectReferenceProperties ?? []) {
|
|
1159
|
+
objectRefColumns.push({
|
|
1160
|
+
slug: prop.slug,
|
|
1161
|
+
rootType: prop.referencedTypeRootSlug,
|
|
1162
|
+
referencedTypePath: prop.referencedTypePath,
|
|
1163
|
+
});
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
if (objectRefColumns.length === 0)
|
|
1167
|
+
return [];
|
|
1168
|
+
const fedIdsByRootType = {};
|
|
1169
|
+
const prefixFedId = (fedId, referencedTypePath) => {
|
|
1170
|
+
if (referencedTypePath?.startsWith('custom-entity') && !fedId.startsWith('custom-entity:')) {
|
|
1171
|
+
return `${referencedTypePath}:${fedId}`;
|
|
1172
|
+
}
|
|
1173
|
+
return fedId;
|
|
1174
|
+
};
|
|
1175
|
+
for (const row of rows) {
|
|
1176
|
+
for (const ref of objectRefColumns) {
|
|
1177
|
+
if (!ref.rootType)
|
|
1178
|
+
continue;
|
|
1179
|
+
const variants = (0, entities_helper_2.getPropertyVariants)(row, ref.slug, 'values');
|
|
1180
|
+
for (const rawValue of variants) {
|
|
1181
|
+
if (isEmpty(rawValue))
|
|
1182
|
+
continue;
|
|
1183
|
+
const fedId = prefixFedId(String(rawValue), ref.referencedTypePath);
|
|
1184
|
+
if (!fedIdsByRootType[ref.rootType])
|
|
1185
|
+
fedIdsByRootType[ref.rootType] = [];
|
|
1186
|
+
fedIdsByRootType[ref.rootType].push(fedId);
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
const limit = pLimit(5);
|
|
1191
|
+
const resolvedFedIds = new Set();
|
|
1192
|
+
for (const rootType of Object.keys(fedIdsByRootType)) {
|
|
1193
|
+
const uniqueFedIds = [...new Set(fedIdsByRootType[rootType])];
|
|
1194
|
+
if (uniqueFedIds.length === 0)
|
|
1195
|
+
continue;
|
|
1196
|
+
const chunks = (0, chunk_by_url_length_1.chunkByUrlLength)(uniqueFedIds);
|
|
1197
|
+
const promises = [];
|
|
1198
|
+
for (const chunk of chunks) {
|
|
1199
|
+
if (!chunk?.length)
|
|
1200
|
+
continue;
|
|
1201
|
+
promises.push(limit(async () => {
|
|
1202
|
+
const batch = await (0, entities_helper_1.retryOnTimeout)(client, entities_helper_1.EntitiesLambdaCallType.GET, {
|
|
1203
|
+
entityName: rootType,
|
|
1204
|
+
federatedIds: (0, lodash_1.uniq)(chunk),
|
|
1205
|
+
skipHydrateRelationships: true,
|
|
1206
|
+
});
|
|
1207
|
+
if (Array.isArray(batch)) {
|
|
1208
|
+
for (const entity of batch) {
|
|
1209
|
+
if (entity?.federatedId)
|
|
1210
|
+
resolvedFedIds.add(entity.federatedId);
|
|
1211
|
+
}
|
|
1212
|
+
}
|
|
1213
|
+
}));
|
|
1214
|
+
}
|
|
1215
|
+
await Promise.all(promises);
|
|
1216
|
+
}
|
|
1217
|
+
const missingByColumnAndValue = {};
|
|
1218
|
+
forEachRow(rows, (row, rowNumber) => {
|
|
1219
|
+
for (const ref of objectRefColumns) {
|
|
1220
|
+
if (!ref.rootType)
|
|
1221
|
+
continue;
|
|
1222
|
+
const variants = (0, entities_helper_2.getPropertyVariants)(row, ref.slug, 'keyValue');
|
|
1223
|
+
for (const variant of variants) {
|
|
1224
|
+
if (isEmpty(variant.value))
|
|
1225
|
+
continue;
|
|
1226
|
+
const rawValue = String(variant.value);
|
|
1227
|
+
const lookupKey = prefixFedId(rawValue, ref.referencedTypePath);
|
|
1228
|
+
if (resolvedFedIds.has(lookupKey))
|
|
1229
|
+
continue;
|
|
1230
|
+
if (!missingByColumnAndValue[variant.key])
|
|
1231
|
+
missingByColumnAndValue[variant.key] = new Map();
|
|
1232
|
+
const byValue = missingByColumnAndValue[variant.key];
|
|
1233
|
+
if (!byValue.has(rawValue))
|
|
1234
|
+
byValue.set(rawValue, []);
|
|
1235
|
+
byValue.get(rawValue).push({ row: rowNumber, value: rawValue });
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
});
|
|
1239
|
+
const issues = [];
|
|
1240
|
+
for (const [columnName, byValue] of Object.entries(missingByColumnAndValue)) {
|
|
1241
|
+
for (const [missingValue, affectedRows] of byValue) {
|
|
1242
|
+
issues.push({
|
|
1243
|
+
severity: types_2.PreValidationSeverity.WARNING,
|
|
1244
|
+
category: REFERENCE_NOT_FOUND_CATEGORY,
|
|
1245
|
+
columnName,
|
|
1246
|
+
propertySlug: columnName,
|
|
1247
|
+
propertyType: 'object_reference',
|
|
1248
|
+
message: `Value "${missingValue}" does not match any existing entity for object reference property "${columnName}".`,
|
|
1249
|
+
affectedRows,
|
|
1250
|
+
});
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
return issues;
|
|
1254
|
+
}
|
|
1133
1255
|
const NUMERIC_PROPERTY_TYPES = new Set(['number', 'currency', 'percent', 'sequence']);
|
|
1134
1256
|
function validateNumberPrecision(rows, typePropertyMaps) {
|
|
1135
1257
|
const { typePropertiesBySlug } = typePropertyMaps;
|