@mbc-cqrs-serverless/import 1.2.2 → 1.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -29,11 +29,11 @@ let ImportQueueEventHandler = ImportQueueEventHandler_1 = class ImportQueueEvent
|
|
|
29
29
|
}
|
|
30
30
|
else {
|
|
31
31
|
// Fallback to single import (or add more if/else for ZIP)
|
|
32
|
-
await this.singleImportProcessor.process(event.
|
|
32
|
+
await this.singleImportProcessor.process(event.importEvent);
|
|
33
33
|
}
|
|
34
34
|
}
|
|
35
35
|
catch (error) {
|
|
36
|
-
this.logger.error(
|
|
36
|
+
this.logger.error(`Failed to process message in ImportQueueEventHandler. Event Payload: ${JSON.stringify(event.payload)}`, error instanceof Error ? error.stack : String(error));
|
|
37
37
|
throw error; // Ensure SQS retries
|
|
38
38
|
}
|
|
39
39
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"import.queue.event.handler.js","sourceRoot":"","sources":["../../src/event/import.queue.event.handler.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,oDAAuE;AACvE,2CAAuC;AAEvC,6DAAuD;AACvD,yEAAmE;AACnE,iFAA2E;AAGpE,IAAM,uBAAuB,+BAA7B,MAAM,uBAAuB;IAKlC,YACmB,iBAAoC,EACpC,qBAA4C;QAD5C,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,0BAAqB,GAArB,qBAAqB,CAAuB;QAJ9C,WAAM,GAAG,IAAI,eAAM,CAAC,yBAAuB,CAAC,IAAI,CAAC,CAAA;IAK/D,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAC,KAAuB;QACnC,IAAI,CAAC;YACH,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;YACrD,CAAC;iBAAM,CAAC;gBACN,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC,
|
|
1
|
+
{"version":3,"file":"import.queue.event.handler.js","sourceRoot":"","sources":["../../src/event/import.queue.event.handler.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,oDAAuE;AACvE,2CAAuC;AAEvC,6DAAuD;AACvD,yEAAmE;AACnE,iFAA2E;AAGpE,IAAM,uBAAuB,+BAA7B,MAAM,uBAAuB;IAKlC,YACmB,iBAAoC,EACpC,qBAA4C;QAD5C,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,0BAAqB,GAArB,qBAAqB,CAAuB;QAJ9C,WAAM,GAAG,IAAI,eAAM,CAAC,yBAAuB,CAAC,IAAI,CAAC,CAAA;IAK/D,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAC,KAAuB;QACnC,IAAI,CAAC;YACH,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;YACrD,CAAC;iBAAM,CAAC;gBACN,0DAA0D;gBAC1D,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;YAC7D,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,wEAAwE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,EACvG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CACrD,CAAA;YACD,MAAM,KAAK,CAAA,CAAC,qBAAqB;QACnC,CAAC;IACH,CAAC;CACF,CAAA;AA1BY,0DAAuB;kCAAvB,uBAAuB;IADnC,IAAA,mBAAY,EAAC,qCAAgB,CAAC;qCAOS,uCAAiB;QACb,+CAAqB;GAPpD,uBAAuB,CA0BnC"}
|
|
@@ -37,22 +37,42 @@ let CsvBatchProcessor = CsvBatchProcessor_1 = class CsvBatchProcessor {
|
|
|
37
37
|
invokeContext: (0, core_1.extractInvokeContext)(),
|
|
38
38
|
source: sourceId,
|
|
39
39
|
};
|
|
40
|
+
const batchErrors = [];
|
|
40
41
|
for (const data of items) {
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
42
|
+
try {
|
|
43
|
+
const compareResult = await strategy.compare({ ...data, __s3Key: s3Key }, tenantCode);
|
|
44
|
+
if (compareResult.status === comparison_status_enum_1.ComparisonStatus.EQUAL)
|
|
45
|
+
continue;
|
|
46
|
+
const mappedData = await strategy.map(compareResult.status, data, tenantCode, compareResult.existingData);
|
|
47
|
+
if (compareResult.status === comparison_status_enum_1.ComparisonStatus.NOT_EXIST) {
|
|
48
|
+
publishMode === constant_1.ImportPublishMode.SYNC
|
|
49
|
+
? await commandService.publishSync(mappedData, options)
|
|
50
|
+
: await commandService.publishAsync(mappedData, options);
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
publishMode === constant_1.ImportPublishMode.SYNC
|
|
54
|
+
? await commandService.publishPartialUpdateSync(mappedData, options)
|
|
55
|
+
: await commandService.publishPartialUpdateAsync(mappedData, options);
|
|
56
|
+
}
|
|
49
57
|
}
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
:
|
|
58
|
+
catch (error) {
|
|
59
|
+
this.logger.error(`Row failed in batch for table ${tableName}`, {
|
|
60
|
+
data,
|
|
61
|
+
error: error instanceof Error ? error.message : String(error),
|
|
62
|
+
});
|
|
63
|
+
// Store BOTH the item data and the error
|
|
64
|
+
batchErrors.push({
|
|
65
|
+
item: data,
|
|
66
|
+
error: error instanceof Error ? error : new Error(String(error)),
|
|
67
|
+
});
|
|
54
68
|
}
|
|
55
69
|
}
|
|
70
|
+
if (batchErrors.length > 0) {
|
|
71
|
+
const allMessages = batchErrors
|
|
72
|
+
.map((err, idx) => `[Row ${idx + 1} - Data: ${JSON.stringify(err.item)} Error: ${err.error.message}]`)
|
|
73
|
+
.join(' | ');
|
|
74
|
+
throw new Error(`Batch processing completed with ${batchErrors.length} error(s). Failing batch to trigger SQS retry. Details: ${allMessages}`);
|
|
75
|
+
}
|
|
56
76
|
}
|
|
57
77
|
};
|
|
58
78
|
exports.CsvBatchProcessor = CsvBatchProcessor;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"csv-batch.processor.js","sourceRoot":"","sources":["../../../src/event/processor/csv-batch.processor.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,oDAKkC;AAClC,2CAA2D;AAE3D,6CAAkD;AAElD,8EAAoE;AACpE,6EAGuC;AAIhC,IAAM,iBAAiB,yBAAvB,MAAM,iBAAiB;IAG5B,YAEE,WAAqE,EAErE,cAA+D;QAF9C,gBAAW,GAAX,WAAW,CAAyC;QAEpD,mBAAc,GAAd,cAAc,CAAgC;QANhD,WAAM,GAAG,IAAI,eAAM,CAAC,mBAAiB,CAAC,IAAI,CAAC,CAAA;IAOzD,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAC,OAAwB;QACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAA;QACzE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,CAAA;QAEjE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAChD,IAAI,CAAC,QAAQ;YACX,MAAM,IAAI,KAAK,CAAC,6CAA6C,SAAS,EAAE,CAAC,CAAA;QAE3E,MAAM,WAAW,GACf,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,4BAAiB,CAAC,KAAK,CAAA;QAC/D,MAAM,cAAc,GAAG,QAAQ,CAAC,iBAAiB,EAAE,CAAA;QACnD,MAAM,OAAO,GAAoB;YAC/B,aAAa,EAAE,IAAA,2BAAoB,GAAE;YACrC,MAAM,EAAE,QAAQ;SACjB,CAAA;QAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC,OAAO,CAC1C,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAC3B,UAAU,CACX,CAAA;
|
|
1
|
+
{"version":3,"file":"csv-batch.processor.js","sourceRoot":"","sources":["../../../src/event/processor/csv-batch.processor.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,oDAKkC;AAClC,2CAA2D;AAE3D,6CAAkD;AAElD,8EAAoE;AACpE,6EAGuC;AAIhC,IAAM,iBAAiB,yBAAvB,MAAM,iBAAiB;IAG5B,YAEE,WAAqE,EAErE,cAA+D;QAF9C,gBAAW,GAAX,WAAW,CAAyC;QAEpD,mBAAc,GAAd,cAAc,CAAgC;QANhD,WAAM,GAAG,IAAI,eAAM,CAAC,mBAAiB,CAAC,IAAI,CAAC,CAAA;IAOzD,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAC,OAAwB;QACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAA;QACzE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,OAAO,CAAA;QAEjE,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAChD,IAAI,CAAC,QAAQ;YACX,MAAM,IAAI,KAAK,CAAC,6CAA6C,SAAS,EAAE,CAAC,CAAA;QAE3E,MAAM,WAAW,GACf,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,4BAAiB,CAAC,KAAK,CAAA;QAC/D,MAAM,cAAc,GAAG,QAAQ,CAAC,iBAAiB,EAAE,CAAA;QACnD,MAAM,OAAO,GAAoB;YAC/B,aAAa,EAAE,IAAA,2BAAoB,GAAE;YACrC,MAAM,EAAE,QAAQ;SACjB,CAAA;QAED,MAAM,WAAW,GAAkC,EAAE,CAAA;QAErD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC,OAAO,CAC1C,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAC3B,UAAU,CACX,CAAA;gBACD,IAAI,aAAa,CAAC,MAAM,KAAK,yCAAgB,CAAC,KAAK;oBAAE,SAAQ;gBAE7D,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,GAAG,CACnC,aAAa,CAAC,MAAM,EACpB,IAAI,EACJ,UAAU,EACV,aAAa,CAAC,YAAY,CAC3B,CAAA;gBAED,IAAI,aAAa,CAAC,MAAM,KAAK,yCAAgB,CAAC,SAAS,EAAE,CAAC;oBACxD,WAAW,KAAK,4BAAiB,CAAC,IAAI;wBACpC,CAAC,CAAC,MAAM,cAAc,CAAC,WAAW,CAC9B,UAA+B,EAC/B,OAAO,CACR;wBACH,CAAC,CAAC,MAAM,cAAc,CAAC,YAAY,CAC/B,UAA+B,EAC/B,OAAO,CACR,CAAA;gBACP,CAAC;qBAAM,CAAC;oBACN,WAAW,KAAK,4BAAiB,CAAC,IAAI;wBACpC,CAAC,CAAC,MAAM,cAAc,CAAC,wBAAwB,CAC3C,UAAsC,EACtC,OAAO,CACR;wBACH,CAAC,CAAC,MAAM,cAAc,CAAC,yBAAyB,CAC5C,UAAsC,EACtC,OAAO,CACR,CAAA;gBACP,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,SAAS,EAAE,EAAE;oBAC9D,IAAI;oBACJ,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC9D,CAAC,CAAA;gBAEF,yCAAyC;gBACzC,WAAW,CAAC,IAAI,CAAC;oBACf,IAAI,EAAE,IAAI;oBACV,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;iBACjE,CAAC,CAAA;YACJ,CAAC;QACH,CAAC;QAED,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,WAAW,GAAG,WAAW;iBAC5B,GAAG,CACF,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CACX,QAAQ,GAAG,GAAG,CAAC,YAAY,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,KAAK,CAAC,OAAO,GAAG,CACrF;iBACA,IAAI,CAAC,KAAK,CAAC,CAAA;YAEd,MAAM,IAAI,KAAK,CACb,mCAAmC,WAAW,CAAC,MAAM,2DAA2D,WAAW,EAAE,CAC9H,CAAA;QACH,CAAC;IACH,CAAC;CACF,CAAA;AA3FY,8CAAiB;4BAAjB,iBAAiB;IAD7B,IAAA,mBAAU,GAAE;IAKR,WAAA,IAAA,eAAM,EAAC,+CAAoB,CAAC,CAAA;IAE5B,WAAA,IAAA,eAAM,EAAC,2CAAgB,CAAC,CAAA;qCADK,GAAG;QAEA,GAAG;GAP3B,iBAAiB,CA2F7B"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mbc-cqrs-serverless/import",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.4",
|
|
4
4
|
"description": "Import module",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"mbc",
|
|
@@ -41,12 +41,12 @@
|
|
|
41
41
|
"access": "public"
|
|
42
42
|
},
|
|
43
43
|
"dependencies": {
|
|
44
|
-
"@mbc-cqrs-serverless/core": "1.2.
|
|
44
|
+
"@mbc-cqrs-serverless/core": "1.2.4",
|
|
45
45
|
"JSONStream": "^1.3.5",
|
|
46
46
|
"csv-parser": "^3.2.0"
|
|
47
47
|
},
|
|
48
48
|
"devDependencies": {
|
|
49
49
|
"@types/jsonstream": "^0.8.33"
|
|
50
50
|
},
|
|
51
|
-
"gitHead": "
|
|
51
|
+
"gitHead": "115721d0ef8c3332a29db37c31575e594163a865"
|
|
52
52
|
}
|